From 52c8ee3120e4a44e48e217bced6f6c799ff7980c Mon Sep 17 00:00:00 2001
From: shikeying <shikeying@163.com>
Date: 星期日, 07 四月 2024 17:20:16 +0800
Subject: [PATCH] 添加milvus代码示例2
---
deploy-jar-template/src/main/java/com/iplatform/api/MilvusChatApi.java | 87 +++++++++++++++++
iplatform-base/src/main/java/com/iplatform/base/support/LogAspect.java | 30 +++++
deploy-jar-template/src/main/java/com/iplatform/milvus/SearchResult.java | 38 +++++++
deploy-jar-template/src/main/java/com/iplatform/milvus/ScoreText.java | 31 ++++++
deploy-jar-template/src/main/resources/application-dev.yml | 2
deploy-jar-template/src/main/java/com/iplatform/milvus/MilvusEngine.java | 30 +++++
deploy-jar-template/src/main/java/com/iplatform/milvus/service/EventServiceImpl.java | 15 +++
deploy-jar-template/src/main/java/com/iplatform/api/TestWebController.java | 17 +++
8 files changed, 241 insertions(+), 9 deletions(-)
diff --git a/deploy-jar-template/src/main/java/com/iplatform/api/MilvusChatApi.java b/deploy-jar-template/src/main/java/com/iplatform/api/MilvusChatApi.java
index 79483b2..ae37222 100644
--- a/deploy-jar-template/src/main/java/com/iplatform/api/MilvusChatApi.java
+++ b/deploy-jar-template/src/main/java/com/iplatform/api/MilvusChatApi.java
@@ -4,8 +4,13 @@
import com.iplatform.milvus.EventVo;
import com.iplatform.milvus.MilvusEngine;
import com.iplatform.milvus.ParamList;
+import com.iplatform.milvus.ScoreText;
+import com.iplatform.milvus.SearchResult;
import com.iplatform.milvus.service.EventServiceImpl;
+import com.walker.infrastructure.utils.FileUtils;
+import com.walker.infrastructure.utils.JsonUtils;
import com.walker.infrastructure.utils.StringUtils;
+import com.walker.support.milvus.OutData;
import com.walker.web.ResponseValue;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.http.HttpStatus;
@@ -18,6 +23,7 @@
import java.util.Collection;
import java.util.LinkedHashMap;
import java.util.List;
+import java.util.Map;
import java.util.concurrent.TimeUnit;
@RestController
@@ -32,7 +38,9 @@
private boolean isBreak = false;
private static final String URL_EMBEDDING = "http://120.26.128.84:7003/ai/text/embedding";
+ private static final String URL_SEARCH_SIMILAR = "http://120.26.128.84:7003/ai/text/search_similar";
private static final String URL_MILVUS = "120.26.128.84";
+ private static final double BEST_MATCH_SCORE = 0.75;
@Autowired
public MilvusChatApi(EventServiceImpl eventService, RestTemplate restTemplate){
@@ -41,6 +49,7 @@
if(this.milvusEngine == null){
MilvusEngine engine = new MilvusEngine(URL_MILVUS, 19530);
this.milvusEngine = engine;
+ this.milvusEngine.loadChatSimilar4Search();
logger.info("milvus engine ok!");
}
}
@@ -76,8 +85,84 @@
@RequestMapping("/query")
public ResponseValue testQueryMilvus(String text){
+ if(StringUtils.isEmpty(text)){
+ return ResponseValue.error("text is required!");
+ }
+ // 鐢ㄥ垎鍙峰垎闅斿涓鍙�
+ String[] array = StringUtils.toArray(text);
+ ParamList paramList = new ParamList();
+ for(String value : array){
+ paramList.add(value);
+ }
- return ResponseValue.success();
+ SearchResult searchResult = this.acquireSearchResult(paramList);
+ if(searchResult == null){
+ logger.warn("杩滅▼璋冪敤鏈�鍖归厤璇彞鍚戦噺澶辫触锛屾棤娉曠户缁悳绱㈢浉浼煎害");
+ return ResponseValue.error("杩滅▼璋冪敤鏈�鍖归厤璇彞鍚戦噺澶辫触锛屾棤娉曠户缁悳绱㈢浉浼煎害");
+ }
+
+ List<List<Float>> requestVectors = new ArrayList(4);
+
+ if(!StringUtils.isEmptyList(searchResult.getBest_list()) && searchResult.getBest_embedding() != null){
+ ScoreText bestScoreText = searchResult.getBest_list().get(0);
+ logger.debug("鍦ㄥ璇濋泦鍚堜腑锛屽瓨鍦ㄦ渶鍖归厤鐨勫彞瀛�: {}", bestScoreText);
+ if(bestScoreText.getScore() >= BEST_MATCH_SCORE){
+ logger.debug("鏈�鍖归厤鐨勫垎鍊�: {} 澶т簬璁剧疆闃堝�硷紝鍙互鐩存帴浣滀负鏌ヨ鍩哄噯", bestScoreText.getScore());
+ requestVectors.add(searchResult.getBest_embedding());
+ } else {
+ requestVectors.add(searchResult.getBest_embedding());
+ requestVectors.add(searchResult.getAll_embedding());
+ }
+ } else {
+ logger.debug("鍙娇鐢ㄥ叏閲忔煡璇㈠悜閲忥細{}", text);
+ requestVectors.add(searchResult.getAll_embedding());
+ }
+
+ OutData outData = this.milvusEngine.searchChatSimilar(requestVectors);
+ if(outData == null){
+ return ResponseValue.error("鏈绱㈠埌浠讳綍鍖归厤鐩镐技鎻愰棶");
+ }
+
+ List<OutData.Data> dataList = outData.getResultList();
+ for(OutData.Data d : dataList){
+ logger.info("data = {}", d);
+ }
+
+
+ return ResponseValue.success(dataList);
+ }
+
+ private SearchResult acquireSearchResult(ParamList paramList){
+ try {
+ ResponseEntity<ResponseValue> responseEntity = this.restTemplate.postForEntity(URL_SEARCH_SIMILAR, paramList, ResponseValue.class);
+// logger.debug(responseEntity.toString());
+ if(responseEntity.getStatusCode() == HttpStatus.OK){
+ ResponseValue<Map<String, Object>> responseValue = responseEntity.getBody();
+ if(responseValue.getCode() != 1){
+ logger.error("璋冪敤杩斿洖acquireSearchResult杩斿洖鐘舵�侀敊璇細{}", responseValue.getMsg());
+ return null;
+ }
+ Map<String, Object> map = responseValue.getData();
+ logger.debug("map = {}", map);
+// String bestListJson = JsonUtils.objectToJsonString(map.get("best_list"));
+// List<ScoreText> bestList = JsonUtils.jsonStringToList(bestListJson, ScoreText.class);
+// logger.debug("bestList = {}", bestList);
+// SearchResult searchResult = new SearchResult();
+// searchResult.setBest_embedding(this.transfer2FloatList((List<Float>)map.get("best_embedding")));
+// searchResult.setAll_embedding(this.transfer2FloatList((List<Float>)map.get("all_embedding")));
+ String json = JsonUtils.objectToJsonString(map);
+ SearchResult searchResult = JsonUtils.jsonStringToObject(json, SearchResult.class);
+ searchResult.setBest_embedding(this.transfer2FloatList(searchResult.getBest_embedding()));
+ searchResult.setAll_embedding(this.transfer2FloatList(searchResult.getAll_embedding()));
+ return searchResult;
+ } else {
+ logger.error("璋冪敤 {} 缁撴灉杩斿洖澶辫触锛歿}", URL_SEARCH_SIMILAR, responseEntity.getStatusCodeValue());
+ return null;
+ }
+ } catch (Exception ex){
+ logger.error("鑾峰彇鎼滅储璇彞鍚戦噺缁撴灉閿欒锛歿}" + ex.getMessage(), ex);
+ return null;
+ }
}
private boolean acquireEmbedding(List<EventVo> batchData){
diff --git a/deploy-jar-template/src/main/java/com/iplatform/api/TestWebController.java b/deploy-jar-template/src/main/java/com/iplatform/api/TestWebController.java
index 178804a..cdfa15e 100644
--- a/deploy-jar-template/src/main/java/com/iplatform/api/TestWebController.java
+++ b/deploy-jar-template/src/main/java/com/iplatform/api/TestWebController.java
@@ -2,6 +2,7 @@
import com.iplatform.base.PushController;
import com.iplatform.base.PushData;
+import com.iplatform.base.pojo.UserParam;
import com.iplatform.base.service.UserServiceImpl;
import com.iplatform.model.po.S_user_core;
import com.walker.db.page.GenericPager;
@@ -9,10 +10,13 @@
import com.walker.infrastructure.utils.JsonUtils;
import com.walker.web.ResponseValue;
import com.walker.web.WebRuntimeException;
+import com.walker.web.log.BusinessType;
+import com.walker.web.log.Log;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.jdbc.core.JdbcTemplate;
import org.springframework.security.core.parameters.P;
import org.springframework.transaction.annotation.Transactional;
+import org.springframework.web.bind.annotation.GetMapping;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RestController;
@@ -35,6 +39,19 @@
this.userService = userService;
}
+ @Log(title = "娴嬭瘯鍒嗛〉鍒楄〃", businessType = BusinessType.Delete, isSaveRequestData = true, isSaveResponseData = true)
+ @GetMapping("/list")
+ public ResponseValue list(UserParam userParam){
+ try {
+ logger.debug("userParam = {}", JsonUtils.objectToJsonString(userParam));
+ } catch (Exception e) {
+ throw new RuntimeException(e);
+ }
+ GenericPager<S_user_core> pager = this.userService.queryPageUserList(0
+ , 0, userParam.getUserName(), userParam.getPhonenumber(), userParam.getStatus());
+ return ResponseValue.success(pager);
+ }
+
@RequestMapping("/push_msg")
public ResponseValue testPushMessage() throws Exception{
diff --git a/deploy-jar-template/src/main/java/com/iplatform/milvus/MilvusEngine.java b/deploy-jar-template/src/main/java/com/iplatform/milvus/MilvusEngine.java
index 8cabab4..30605c6 100644
--- a/deploy-jar-template/src/main/java/com/iplatform/milvus/MilvusEngine.java
+++ b/deploy-jar-template/src/main/java/com/iplatform/milvus/MilvusEngine.java
@@ -6,6 +6,8 @@
import com.walker.support.milvus.FieldType;
import com.walker.support.milvus.MetricType;
import com.walker.support.milvus.OperateService;
+import com.walker.support.milvus.OutData;
+import com.walker.support.milvus.Query;
import com.walker.support.milvus.Table;
import com.walker.support.milvus.engine.DefaultOperateService;
import org.slf4j.Logger;
@@ -20,6 +22,8 @@
public class MilvusEngine {
protected final transient Logger logger = LoggerFactory.getLogger(this.getClass());
+
+ public static final String TABLE_CHAT_SIMILAR = "chat_similar";
public MilvusEngine(String ip, int port){
DefaultOperateService service = new DefaultOperateService();
@@ -44,7 +48,7 @@
*/
public void createChatSimilarTable(){
Table chatSimilarTable = new Table();
- chatSimilarTable.setCollectionName("chat_similar");
+ chatSimilarTable.setCollectionName(TABLE_CHAT_SIMILAR);
chatSimilarTable.setDescription("鑱婂ぉ鎻愬彇宸ュ崟鎽樿鍘嗗彶鏁版嵁");
chatSimilarTable.setShardsNum(1);
chatSimilarTable.setDimension(768); // 杩欎釜鏄牴鎹娇鐢ㄥ悜閲忔ā鍨嬬淮搴﹀畾鐨�
@@ -84,7 +88,7 @@
public void insertTestData(){
DataSet dataSet = new DataSet();
- dataSet.setTableName("chat_similar");
+ dataSet.setTableName(TABLE_CHAT_SIMILAR);
List<List<Float>> vectorList = new ArrayList<>();
vectorList.add(Arrays.asList(mockVector));
@@ -132,7 +136,7 @@
}
DataSet dataSet = new DataSet();
- dataSet.setTableName("chat_similar");
+ dataSet.setTableName(TABLE_CHAT_SIMILAR);
Map<String, List<?>> fieldMap = new HashMap();
fieldMap.put("id", ids);
@@ -145,6 +149,26 @@
logger.info("鍐欏叆浜�: {}", ids);
}
+ public OutData searchChatSimilar(List<List<Float>> vectors){
+ Query query = new Query();
+ query.setMetricType(MetricType.NLP.getIndex());
+ query.setTableName(TABLE_CHAT_SIMILAR);
+ query.setTopK(4);
+ query.setVectorName("embedding");
+ query.setOutFieldList(Arrays.asList(new String[]{"id","title","content"}));
+ query.setFieldPrimaryKey("id");
+ query.setSearchVectors(vectors);
+ return this.operateService.searchVector(query);
+ }
+
+ /**
+ * 蹇呴』鍦ㄦ煡璇箣鍓嶏紝鍔犺浇鏁版嵁鍒板唴瀛樹腑銆�
+ * @date 2024-03-31
+ */
+ public void loadChatSimilar4Search(){
+ this.operateService.prepareSearch(TABLE_CHAT_SIMILAR);
+ }
+
private OperateService operateService;
// private Double[] mockVector = new Double[]{-0.051114246249198914, 0.889954432};
diff --git a/deploy-jar-template/src/main/java/com/iplatform/milvus/ScoreText.java b/deploy-jar-template/src/main/java/com/iplatform/milvus/ScoreText.java
new file mode 100644
index 0000000..f65ae6b
--- /dev/null
+++ b/deploy-jar-template/src/main/java/com/iplatform/milvus/ScoreText.java
@@ -0,0 +1,31 @@
+package com.iplatform.milvus;
+
+public class ScoreText {
+
+ public double getScore() {
+ return score;
+ }
+
+ public void setScore(double score) {
+ this.score = score;
+ }
+
+ public String getText() {
+ return text;
+ }
+
+ public void setText(String text) {
+ this.text = text;
+ }
+
+ private double score;
+ private String text;
+
+ @Override
+ public String toString() {
+ return "ScoreText{" +
+ "score=" + score +
+ ", text='" + text + '\'' +
+ '}';
+ }
+}
diff --git a/deploy-jar-template/src/main/java/com/iplatform/milvus/SearchResult.java b/deploy-jar-template/src/main/java/com/iplatform/milvus/SearchResult.java
new file mode 100644
index 0000000..b4024c7
--- /dev/null
+++ b/deploy-jar-template/src/main/java/com/iplatform/milvus/SearchResult.java
@@ -0,0 +1,38 @@
+package com.iplatform.milvus;
+
+import java.util.List;
+
+/**
+ * 鏌ヨ缁欏畾璇彞闆嗗悎锛岃繑鍥炵殑鍚戦噺淇℃伅銆�
+ * @date 2024-03-31
+ */
+public class SearchResult {
+
+ public List<ScoreText> getBest_list() {
+ return best_list;
+ }
+
+ public void setBest_list(List<ScoreText> best_list) {
+ this.best_list = best_list;
+ }
+
+ public List<Float> getBest_embedding() {
+ return best_embedding;
+ }
+
+ public void setBest_embedding(List<Float> best_embedding) {
+ this.best_embedding = best_embedding;
+ }
+
+ public List<Float> getAll_embedding() {
+ return all_embedding;
+ }
+
+ public void setAll_embedding(List<Float> all_embedding) {
+ this.all_embedding = all_embedding;
+ }
+
+ private List<ScoreText> best_list;
+ private List<Float> best_embedding;
+ private List<Float> all_embedding;
+}
diff --git a/deploy-jar-template/src/main/java/com/iplatform/milvus/service/EventServiceImpl.java b/deploy-jar-template/src/main/java/com/iplatform/milvus/service/EventServiceImpl.java
index eb347ce..73a2ad1 100644
--- a/deploy-jar-template/src/main/java/com/iplatform/milvus/service/EventServiceImpl.java
+++ b/deploy-jar-template/src/main/java/com/iplatform/milvus/service/EventServiceImpl.java
@@ -3,17 +3,32 @@
import com.iplatform.milvus.EventVo;
import com.walker.jdbc.service.BaseServiceImpl;
import org.springframework.jdbc.core.RowMapper;
+import org.springframework.jdbc.core.namedparam.MapSqlParameterSource;
import org.springframework.stereotype.Service;
import java.sql.ResultSet;
import java.sql.SQLException;
+import java.util.HashMap;
import java.util.List;
+import java.util.Map;
@Service
public class EventServiceImpl extends BaseServiceImpl {
private final EventVoMapper eventVoMapper = new EventVoMapper();
+ /**
+ * 鏍规嵁宸ュ崟id闆嗗悎锛岃繑鍥炶繖浜涘伐鍗曞璞★紙鍘嗗彶宸ュ崟鍩烘湰淇℃伅锛�
+ * @param ids
+ * @return
+ * @date 2024-03-31
+ */
+ public List<EventVo> queryEventWhereIn(List<Long> ids){
+ MapSqlParameterSource parameterSource = new MapSqlParameterSource();
+ parameterSource.addValue("ids", ids);
+ return this.sqlListObjectWhereIn("select * from event_history where id in (:ids)", eventVoMapper, parameterSource);
+ }
+
public List<EventVo> queryEventAll(Long existId){
if(existId == null){
return this.select("select * from event_history order by id asc", new Object[]{}, this.eventVoMapper);
diff --git a/deploy-jar-template/src/main/resources/application-dev.yml b/deploy-jar-template/src/main/resources/application-dev.yml
index d0d3ca4..92d172f 100644
--- a/deploy-jar-template/src/main/resources/application-dev.yml
+++ b/deploy-jar-template/src/main/resources/application-dev.yml
@@ -288,7 +288,7 @@
# 濡傛灉涓嶆墦寮�锛屽垯璁惧鐧诲綍鐨剈uid鏇存柊鎿嶄綔涔熸棤娉曡幏寰楋紝鐢ㄤ簬璁板綍姣忎釜鐧诲綍鐢ㄦ埛鐨剈uid锛堢敤鎴风櫥褰曡鑹叉洿鏂帮級锛�2023-03-23
login-enabled: true
# 鏄惁鎵撳紑鎿嶄綔鏃ュ織锛�2023-01-05
- operate-enabled: false
+ operate-enabled: true
# 楠岃瘉鐮佺浉鍏抽厤缃紝2023-01-27
captcha:
diff --git a/iplatform-base/src/main/java/com/iplatform/base/support/LogAspect.java b/iplatform-base/src/main/java/com/iplatform/base/support/LogAspect.java
index 19bd1a0..91a2dd6 100644
--- a/iplatform-base/src/main/java/com/iplatform/base/support/LogAspect.java
+++ b/iplatform-base/src/main/java/com/iplatform/base/support/LogAspect.java
@@ -9,6 +9,7 @@
import com.walker.infrastructure.utils.StringUtils;
import com.walker.web.Constants;
import com.walker.web.ResponseCode;
+import com.walker.web.WebRuntimeException;
import com.walker.web.log.BusinessType;
import com.walker.web.log.Log;
import com.walker.web.util.IpUtils;
@@ -92,7 +93,13 @@
private void handleLog(final JoinPoint joinPoint, Log logAnnotation, final Exception e, Object jsonResult){
try{
S_oper_log s_oper_log = new S_oper_log();
- S_user_core user_core = this.securitySpi.getCurrentUser();
+// S_user_core user_core = this.securitySpi.getCurrentUser();
+ S_user_core user_core = null;
+ try {
+ user_core = this.securitySpi.getCurrentUser();
+ } catch (WebRuntimeException ex) {
+ logger.debug("璇ユ帴鍙ｆ棤闇�璁よ瘉锛屾棤娉曟壘鍒板綋鍓嶄汉淇℃伅");
+ }
if(user_core != null){
s_oper_log.setOper_name(user_core.getUser_name());
}
@@ -128,9 +135,24 @@
s_oper_log.setOper_param(params);
}
} else {
- Map<?, ?> paramsMap = (Map<?, ?>) ServletUtils.getRequest().getAttribute(HandlerMapping.URI_TEMPLATE_VARIABLES_ATTRIBUTE);
- if(paramsMap != null){
- s_oper_log.setOper_param(StringUtils.substring(paramsMap.toString(), 0, MAX_DATA_SIZE));
+ String queryString = ServletUtils.getRequest().getQueryString();
+// Object param2 = ServletUtils.getRequest().getAttribute(HandlerMapping.MATRIX_VARIABLES_ATTRIBUTE);
+// Object param3 = ServletUtils.getRequest().getAttribute(HandlerMapping.PRODUCIBLE_MEDIA_TYPES_ATTRIBUTE);
+// logger.debug(queryString);
+// logger.debug("param2 = {}", param2);
+// logger.debug("param3 = {}", param3);
+ if(StringUtils.isEmpty(queryString)){
+ Map<?, ?> paramsMap = (Map<?, ?>) ServletUtils.getRequest().getAttribute(HandlerMapping.URI_TEMPLATE_VARIABLES_ATTRIBUTE);
+ if(paramsMap != null){
+ queryString = paramsMap.toString();
+ }
+ }
+ if(StringUtils.isNotEmpty(queryString)){
+ if(queryString.length() > MAX_DATA_SIZE){
+ s_oper_log.setOper_param(StringUtils.substring(queryString, 0, MAX_DATA_SIZE));
+ } else {
+ s_oper_log.setOper_param(queryString);
+ }
}
}
}
--
Gitblit v1.9.1