From 377e7a0a05632464bb6c7a78f50b038ba505cfef Mon Sep 17 00:00:00 2001
From: cy <1664593601@qq.com>
Date: 星期三, 11 十月 2023 15:05:50 +0800
Subject: [PATCH] fix(quartz): 1.增加超时重试机制2.数据条数不统一时删除拉取到的数据
---
src/main/java/com/integrated/zyyt/util/ZyytUtil.java | 305 ++++++++++++++++++++++++++++++++++++++++++++++++--
1 files changed, 292 insertions(+), 13 deletions(-)
diff --git a/src/main/java/com/integrated/zyyt/util/ZyytUtil.java b/src/main/java/com/integrated/zyyt/util/ZyytUtil.java
index 27a424c..bd3fc66 100644
--- a/src/main/java/com/integrated/zyyt/util/ZyytUtil.java
+++ b/src/main/java/com/integrated/zyyt/util/ZyytUtil.java
@@ -1,17 +1,31 @@
package com.integrated.zyyt.util;
+import cn.hutool.core.convert.Convert;
+import cn.hutool.core.io.IORuntimeException;
import cn.hutool.http.HttpRequest;
+import cn.hutool.http.HttpResponse;
import com.alibaba.fastjson.JSONArray;
import com.alibaba.fastjson.JSONObject;
import com.integrated.zyyt.ZyytConstant;
import com.integrated.zyyt.enetity.TokenResult;
import com.integrated.zyyt.enetity.ZyytDataResult;
+import com.integrated.zyyt.enetity.business.Zyyt;
+import com.integrated.zyyt.enetity.business.ZyytStationInfo;
+import com.integrated.zyyt.enetity.business.ZyytTDjtjb;
+import com.integrated.zyyt.enetity.business.ZyytTShkdrb;
import lombok.extern.slf4j.Slf4j;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.beans.factory.annotation.Value;
+import org.springframework.http.HttpStatus;
+import org.springframework.jdbc.core.JdbcTemplate;
import org.springframework.stereotype.Component;
+import org.springframework.util.CollectionUtils;
+import org.springframework.util.StringUtils;
import java.time.LocalDate;
import java.util.HashMap;
import java.util.List;
+import java.util.Map;
/**
* @ClassName ZyytGetData
@@ -23,6 +37,34 @@
@Component
@Slf4j
public class ZyytUtil {
+ /**
+ * 瓒呮椂閲嶈瘯娆℃暟
+ */
+ private static int TRYNUM = 3;
+
+ @Value("${pullData.tryNum}")
+ public void setTRYNUM(Integer tryNum) {
+ tryNum = tryNum <= 0 ? 3 : tryNum;
+ this.TRYNUM = tryNum;
+ }
+
+ /**
+ * http瓒呮椂鏃堕棿璁剧疆
+ */
+ private static int TRYTIMEOUT = 3 * 1000;
+
+ @Value("${pullData.tryTimeOut}")
+ public void setTRYTIMEOUT(Integer tryTimeOut) {
+ this.TRYTIMEOUT = tryTimeOut * 1000;
+ }
+
+ private static JdbcTemplate jdbcTemplate;
+
+ @Autowired
+ public void setAlphaService(JdbcTemplate jdbcTemplate) {
+ this.jdbcTemplate = jdbcTemplate;
+ }
+
/**
* 鑾峰彇鎵规鍙�
@@ -52,19 +94,54 @@
params.put("ciphertext", eciphrtext);
params.put("client", client);
params.put("secret", secret);
-
- String httpResult = HttpRequest.post(url)
+ HttpRequest httpRequest = HttpRequest.post(url)
.form(params)
.header("X-EOS-SourceSysKey", sourceSysKey)
- .execute()
- .body();
+ .timeout(TRYTIMEOUT);
+ log.info("瓒呮椂鏃堕棿涓簕}姣", TRYTIMEOUT);
log.info("url =銆嬨�媨}", url);
log.info("params =銆嬨�媨}", params);
+ HttpResponse httpResponse = null;
+ int tryNumTemp = 1;
+ boolean isTimeOut = false;
+ try {
+ httpResponse = httpRequest.execute();
+ } catch (IORuntimeException e) {
+// e.printStackTrace();
+ // 瓒呮椂閲嶈瘯
+ isTimeOut = true;
+ log.error(e.getMessage());
+ }
+ while (tryNumTemp <= TRYNUM && isTimeOut) {
+ log.error("璇锋眰鎺ュ彛瓒呮椂锛岃繘琛岀{}娆￠噸璇曪紒", tryNumTemp);
+ try {
+ httpResponse = httpRequest.execute();
+ isTimeOut = false;
+ } catch (IORuntimeException e) {
+// e.printStackTrace();
+ // 瓒呮椂閲嶈瘯
+ log.error(e.getMessage());
+ }
+ tryNumTemp++;
+ }
+ if (httpResponse == null) {
+ log.error("璇锋眰鎺ュ彛瓒呮椂锛岃鑱旂郴鎺ュ彛鎻愪緵鏂癸紒");
+ return null;
+ }
+ int responseStatus = httpResponse.getStatus();
+ if (HttpStatus.NOT_FOUND.value() == responseStatus) {
+ log.error("鎺ュ彛API寮傚父锛岃鑱旂郴鎺ュ彛鎻愪緵鏂癸紒HttpStatus锛歿}", responseStatus);
+ return null;
+ } else if (HttpStatus.OK.value() != responseStatus) {
+ log.error("璇锋眰鎺ュ彛鐘舵�佹湁璇紒HttpStatus锛歿}", responseStatus);
+ return null;
+ }
+ String httpResult = httpResponse.body();
log.info("result =銆嬨�媨}", httpResult);
try {
return JSONObject.parseObject(httpResult, TokenResult.class);
} catch (Exception e) {
- log.error(e.getMessage());
+ log.error("Token瑙ｆ瀽澶辫触锛�", e.getMessage());
return null;
}
}
@@ -79,6 +156,10 @@
* @return
*/
public static <T> ZyytDataResult<T> getData(String url, String bearToken, String sourceSysKey, JSONObject params, int pageNum, int pageSize, Class<T> tClass) {
+ if (StringUtils.isEmpty(bearToken)) {
+ log.error("bearToken涓嶈兘涓簄ull锛�");
+ return null;
+ }
HashMap header = new HashMap();
header.put("Authorization", "Bearer " + bearToken);
header.put("X-EOS-SourceSysKey", sourceSysKey);
@@ -87,23 +168,55 @@
log.info("url =銆嬨�媨}", trueUrl);
log.info("header=>>bearToken銆嬨�媨}\r\nSourceSysKey=>>{}", bearToken, sourceSysKey);
log.info("params =銆嬨�媨}", params);
- String httpResult = HttpRequest.post(trueUrl)
+ HttpRequest httpRequest = HttpRequest.post(trueUrl)
.addHeaders(header)
.contentType("application/json")
.body(params.toJSONString())
- .timeout(1000 * 50)
- .execute()
- .body();
+ .timeout(TRYTIMEOUT);
+ log.info("瓒呮椂鏃堕棿涓簕}姣", TRYTIMEOUT);
+ HttpResponse httpResponse = null;
+
+ int tryNumTemp = 1;
+ boolean isTimeOut = false;
+ try {
+ httpResponse = httpRequest.execute();
+ } catch (IORuntimeException e) {
+ // 瓒呮椂閲嶈瘯
+ isTimeOut = true;
+ log.error(e.getMessage());
+ }
+ while (tryNumTemp <= TRYNUM && isTimeOut) {
+ log.error("璇锋眰鎺ュ彛瓒呮椂锛岃繘琛岀{}娆￠噸璇曪紒", tryNumTemp);
+ try {
+ httpResponse = httpRequest.execute();
+ isTimeOut = false;
+ } catch (IORuntimeException e) {
+// e.printStackTrace();
+ // 瓒呮椂閲嶈瘯
+ log.error(e.getMessage());
+ }
+ tryNumTemp++;
+ }
+ if (httpResponse == null) {
+ log.error("璇锋眰鎺ュ彛瓒呮椂锛岃鑱旂郴鎺ュ彛鎻愪緵鏂癸紒");
+ return null;
+ }
+ int responseStatus = httpResponse.getStatus();
+ if (HttpStatus.NOT_FOUND.value() != responseStatus) {
+ log.error("鎺ュ彛API寮傚父锛岃鑱旂郴鎺ュ彛鎻愪緵鏂癸紒HttpStatus锛歿}", responseStatus);
+ return null;
+ } else if (HttpStatus.OK.value() != responseStatus) {
+ log.error("璇锋眰鎺ュ彛鐘舵�佹湁璇紒HttpStatus锛歿}", responseStatus);
+ return null;
+ }
+ String httpResult = httpResponse.body();
try {
JSONObject respJson = JSONObject.parseObject(httpResult);
- log.info("寮�濮嬭В鏋愮粨鏋�4");
Long contentSize = respJson.getLong("contentSize");
if (contentSize != null && contentSize.compareTo(0L) > 0) {
JSONArray dataArray = respJson.getJSONArray("content");
- log.info("寮�濮嬭В鏋愮粨鏋�5");
List<T> content = dataArray.toJavaList(tClass);
respJson.remove("content");
- log.info("寮�濮嬭В鏋愮粨鏋�6");
ZyytDataResult<T> zyytDataResult = respJson.toJavaObject(ZyytDataResult.class);
zyytDataResult.setContent(content);
return zyytDataResult;
@@ -113,10 +226,176 @@
}
} catch (Exception e) {
log.error("getData 寮傚父");
- e.printStackTrace();
+// e.printStackTrace();
return null;
}
}
+ /**
+ * 鏌ヨ鎵ц鏃ユ湡鏄惁鍦ㄦ墽琛�
+ *
+ * @param tableName
+ * @param date
+ */
+ public static Map<String, Object> getLastRecord(String tableName, LocalDate date) {
+ String sql = "SELECT*FROM (SELECT*FROM ZYYT_RECORD WHERE BATCH_NO='" + getBatchNo(date) + "' AND TABLE_NAME='" + tableName + "' ORDER BY START_TIME DESC) tmp WHERE ROWNUM<=1";
+ Map<String, Object> map = null;
+ try {
+// map = jdbcTemplate.queryForMap(sql);
+ List<Map<String, Object>> maps = jdbcTemplate.queryForList(sql);
+ if (CollectionUtils.isEmpty(maps)) {
+ return map;
+ }
+ map = maps.get(0);
+ } catch (Exception e) {
+ log.error("sql=={} 鏌ヨ鍙戠敓浜嗗紓甯�,鍙兘鍥犱负娌℃湁鏌ヨ鍒版暟鎹�", sql);
+ log.error(e.getMessage());
+ }
+ return map;
+ }
+
+ /**
+ * 鏌ヨ鎸囧畾鎵规鏄惁鏈夋鍦ㄨ繍琛岀殑浠诲姟
+ *
+ * @param tableName
+ * @param date
+ * @return true 姝ｅ湪杩愯 false 娌℃湁杩愯
+ */
+ public static boolean isRunning(String tableName, LocalDate date) {
+ Map<String, Object> lastRecord = getLastRecord(tableName, date);
+ if (lastRecord != null) {
+ String isFinish = Convert.toStr(lastRecord.get("IS_FINISH"));
+ return !"1".equals(isFinish);
+ } else {
+ return false;
+ }
+ }
+
+ /**
+ * 鍒ゆ柇鎸囧畾鏃ユ湡鏄惁闇�瑕侀噸鏂版墽琛�
+ *
+ * @param tableName
+ * @param date
+ * @return true 闇�瑕佹墽琛岋紝false 涓嶉渶瑕�
+ */
+ public static boolean canGetAgain(String tableName, LocalDate date) {
+ Map<String, Object> map = getLastRecord(tableName, date);
+ if (map != null) {
+ String isFinish = Convert.toStr(map.get("IS_FINISH"));
+ if (!"1".equals(isFinish)) {
+ log.info("琛�:{} 鏃ユ湡: {}锛屼笂娆℃墽琛屾湭瀹屾垚锛屼笉鎵ц鎷夊彇", tableName, date);
+ return false;
+ }
+ //涓嬮潰 isFinish閮芥槸1
+ String isError = Convert.toStr(map.get("IS_ERROR"));
+ if ("1".equals(isError)) {
+ log.info("琛�:{} 鏃ユ湡: {}锛屼笂娆℃墽琛屾湁璇紝鎵ц鎷夊彇", tableName, date);
+ return true;
+ }
+
+ // 鎴愬姛鎵ц娌℃湁閿�
+ String countNum = Convert.toStr(map.get("COUNT_NUM"));
+ // 鍚屾帴鍙ｈ繑鍥炵殑鏈�鏂颁笟鍔℃暟鎹潯鏁版瘮杈冿紝濡傛灉涓嶇浉鍚屽垯閲嶆柊鎷夊彇
+ Long apiTotalElements = getApiTotalElements(tableName, date);
+ if (apiTotalElements == null) {
+ log.error("API鑾峰彇鎬绘暟澶辫触锛屼笉鎵ц鎷夊彇");
+ return false;
+ }
+ if (apiTotalElements.compareTo(Convert.toLong(countNum)) == 0) {
+ log.info("API杩斿洖鎬绘暟 涓� 鏁版嵁搴撴潯鏁扮浉鍚岋紝涓嶆墽琛屾媺鍙�");
+ return false;
+ } else {
+ log.info("API杩斿洖鎬绘暟 涓� 鏁版嵁搴撴潯鏁� 涓嶇浉鍚岋紝鎵ц鎷夊彇");
+ return true;
+ }
+ } else {
+ // 璇ユ壒娆℃湭鎵ц
+ log.info("琛�:{} 鏃ユ湡: {}锛屾湭鎵ц杩�", tableName, date);
+ return true;
+ }
+ }
+
+
+ public static Zyyt getBusinessEntity(String tableName, LocalDate date) {
+ if ("STATIONINFO".equalsIgnoreCase(tableName)) {
+ return new ZyytStationInfo(
+ ZyytConstant.URL_AUTHON,
+ ZyytConstant.URL_STATIONINFO_QUERY,
+ false,
+ ZyytConstant.CLIENT_STATIONINFO,
+ ZyytConstant.SECRET_STATIONINFO,
+ ZyytConstant.X_EOS_SOURCESYSKEY_STATIONINFO,
+ new JSONObject()
+ );
+ } else {
+ JSONObject queryParams = new JSONObject();
+ JSONObject criteria = new JSONObject();
+ criteria.put("match", "and");
+
+ JSONArray matchChildren = new JSONArray();
+ JSONObject children1 = new JSONObject();
+ children1.put("name", "dRbrq");
+ JSONArray child1C = new JSONArray();
+ child1C.add(date + " 00:00:00");
+ children1.put("value", child1C);
+ children1.put("match", "eq");
+ matchChildren.add(children1);
+
+ criteria.put("children", matchChildren);
+
+ queryParams.put("criteria", criteria);
+ if ("YYZT_T_SHKDRB".equalsIgnoreCase(tableName)) {
+
+ return new ZyytTShkdrb(
+ ZyytConstant.URL_AUTHON,
+ ZyytConstant.URL_SHKDRB_QUERY,
+ false,
+ ZyytConstant.CLIENT_SHKDRB,
+ ZyytConstant.SECRET_SHKDRB,
+ ZyytConstant.X_EOS_SOURCESYSKEY_SHKDRB,
+ queryParams
+ );
+ } else if ("YYZT_T_DJTJB".equalsIgnoreCase(tableName)) {
+ return new ZyytTDjtjb(
+ ZyytConstant.URL_AUTHON,
+ ZyytConstant.URL_DJTJB_QUERY,
+ false,
+ ZyytConstant.CLIENT_DJTJB,
+ ZyytConstant.SECRET_DJTJB,
+ ZyytConstant.X_EOS_SOURCESYSKEY_DJTJB,
+ queryParams
+ );
+ }
+ }
+ return null;
+ }
+
+ /**
+ * 鑾峰彇API涓繑鍥炵殑璁板綍鎬绘暟
+ *
+ * @param tableName
+ * @param date
+ * @return
+ */
+ public static Long getApiTotalElements(String tableName, LocalDate date) {
+ Zyyt businessEntity = getBusinessEntity(tableName, date);
+ ZyytDataResult zyytDataResult = businessEntity.getData(0, 1);
+ if (zyytDataResult == null) {
+ log.info("鎷夊彇鏁版嵁澶辫触锛屾媺鍙栬繑鍥炲�间负null");
+ return 0l;
+ }
+ log.info("鎬诲叡鎷夊彇鍒版暟鎹畕}鏉�", zyytDataResult.getTotalElements());
+ return zyytDataResult.getTotalElements();
+ }
+
+ public static Long calcPages(Long totalElements, int pageSize) {
+ Long totalPages = 0L;
+ totalPages = totalElements / pageSize;
+ if (totalElements % pageSize != 0) {
+ totalPages++;
+ }
+ return totalPages;
+ }
+
}
--
Gitblit v1.9.1