//package com.iplatform.api;
|
//
|
//import com.iplatform.similarity.TextSimilarityEngine;
|
//import com.walker.infrastructure.utils.FileUtils;
|
//import com.walker.infrastructure.utils.StringUtils;
|
//import com.walker.support.milvus.DataSet;
|
//import com.walker.support.milvus.DataType;
|
//import com.walker.support.milvus.FieldType;
|
//import com.walker.support.milvus.OutData;
|
//import com.walker.support.milvus.Query;
|
//import com.walker.support.milvus.Table;
|
//import org.slf4j.Logger;
|
//import org.slf4j.LoggerFactory;
|
//import org.springframework.beans.factory.annotation.Autowired;
|
//import org.springframework.web.bind.annotation.RequestMapping;
|
//import org.springframework.web.bind.annotation.RestController;
|
//
|
//import java.util.ArrayList;
|
//import java.util.Arrays;
|
//import java.util.HashMap;
|
//import java.util.List;
|
//import java.util.Map;
|
//
|
//@RestController
|
//@RequestMapping("/test/api")
|
//public class TestTextSimilarityApi {
|
//
|
// private final transient Logger logger = LoggerFactory.getLogger(this.getClass());
|
//
|
// private static final int VECTOR_DIMENSION = 300;
|
// private static final String TABLE_NAME = "course";
|
// private static final float MIN_SCORE = 50F; // 匹配的最小分值（大于改分值）
|
//
|
// @Autowired(required = false)
|
// private TextSimilarityEngine textSimilarityEngine;
|
//
|
// @RequestMapping("/create_table")
|
// public String createTable(){
|
// Table table = new Table();
|
// table.setCollectionName(TABLE_NAME);
|
// table.setDescription("test");
|
// table.setDimension(VECTOR_DIMENSION);
|
// table.setShardsNum(2);
|
//
|
// // 字段信息
|
// FieldType fieldTypeKey = FieldType.newBuilder()
|
// .withName("id")
|
// .withDataType(DataType.Long)
|
// .withPrimaryKey(true).build();
|
// // 业务ID
|
// FieldType fieldTypeId = FieldType.newBuilder()
|
// .withName("course_id")
|
// .withDataType(DataType.Long)
|
// .withPrimaryKey(false).build();
|
// // 每个词的内容
|
//// FieldType fieldTypeText = FieldType.newBuilder()
|
//// .withName("text")
|
//// .withDataType(DataType.String)
|
//// .withPrimaryKey(false).build();
|
// FieldType fieldTypeVector = FieldType.newBuilder()
|
// .withName("course_vector")
|
// .withDataType(DataType.FloatVector)
|
// .withDescription("课程索引")
|
// .withDimension(VECTOR_DIMENSION).build();
|
// List<FieldType> fieldList = new ArrayList<>(8);
|
// fieldList.add(fieldTypeKey);
|
// fieldList.add(fieldTypeId);
|
//// fieldList.add(fieldTypeText);
|
// fieldList.add(fieldTypeVector);
|
//
|
// table.setFieldTypes(fieldList);
|
// boolean success = this.textSimilarityEngine.createTable(table);
|
// if(success){
|
// logger.info("创建表成功：" + TABLE_NAME);
|
// return "ok";
|
// } else {
|
// return "创建表失败：" + TABLE_NAME;
|
// }
|
// }
|
//
|
// @RequestMapping("/drop_table")
|
// public String dropTable(){
|
// this.textSimilarityEngine.dropTable(TABLE_NAME);
|
// return "ok";
|
// }
|
//
|
// @RequestMapping("/write_data")
|
// public String writeDataIntoMilvus(){
|
// List<String> textDataList = FileUtils.getFileLines("D:/dev_tools/ai/模拟课程信息.txt");
|
// if(StringUtils.isEmptyList(textDataList)){
|
// logger.warn("没有从文件中读出任何：课程记录，file = 模拟课程信息.txt");
|
// return "no file content";
|
// }
|
//
|
// List<Long> keyList = new ArrayList<>(128);
|
// List<Long> courseIdList = new ArrayList<>(128);
|
// List<List<Float>> vectorList = new ArrayList<>(128);
|
// List<List<Float>> vectorTemp = null;
|
// String oneText = null;
|
// List<Float> tempVector = null;
|
// long key = 0;
|
// for(int i=0; i<textDataList.size(); i++){
|
//// oneText = textDataList.get(i);
|
//// tempVector = this.textSimilarityEngine.textToVectorList(oneText);
|
//// vectorList.add(tempVector);
|
//// keyList.add((long)i);
|
// oneText = textDataList.get(i);
|
// // 把这句话的每个词作为一条记录，写入表中。
|
// vectorTemp = this.textSimilarityEngine.textToVectorSet(oneText);
|
// if(StringUtils.isEmptyList(vectorTemp)){
|
// logger.debug("vectorTemp 为空，oneText = " + oneText);
|
// continue;
|
// }
|
// for(int j=0; j<vectorTemp.size(); j++){
|
// keyList.add(key++);
|
// courseIdList.add((long)(i+1));
|
// }
|
// vectorList.addAll(vectorTemp);
|
// }
|
//
|
// Map<String, List<?>> fields = new HashMap<>();
|
//// fields.put("course_id", idList);
|
//// fields.put("course_vector", vectorList);
|
// fields.put("id", keyList);
|
// fields.put("course_id", courseIdList);
|
// fields.put("course_vector", vectorList);
|
//
|
// DataSet dataSet = new DataSet();
|
// dataSet.setTableName(TABLE_NAME);
|
// dataSet.setIndexFieldName("course_vector");
|
// dataSet.setFields(fields);
|
//// dataSet.setPartitionName("test_01");
|
// this.textSimilarityEngine.writeData(dataSet, "IVF_FLAT", null);
|
// return "写入数据成功";
|
// }
|
//
|
// @RequestMapping("/search")
|
// public String searchVector(String searchWord){
|
//
|
// List<List<Float>> searchVectors = this.textSimilarityEngine.textToVectorSet(searchWord);
|
// logger.info("返回搜索词向量集合几个？" + searchVectors.size());
|
// List<String> search_output_fields = Arrays.asList("course_id","id");
|
//
|
// Query query = new Query();
|
// query.setTableName(TABLE_NAME);
|
// query.setFieldPrimaryKey("id");
|
// query.setVectorName("course_vector");
|
// query.setOutFieldList(search_output_fields);
|
// query.setSearchVectors(searchVectors);
|
// query.setTopK(20);
|
//
|
// OutData datas = this.textSimilarityEngine.searchVector(query, MIN_SCORE);
|
// if(datas == null){
|
// return "没查出结果数据!";
|
// }
|
// List<OutData.Data> resultList = datas.getResultList();
|
// for(OutData.Data r : resultList){
|
// logger.info("一条数据：" + r);
|
// }
|
//
|
// // 按照规则排序
|
// List<OutData.Data> sortList = this.textSimilarityEngine.sortSearch(resultList);
|
// for(OutData.Data r : sortList){
|
// logger.info("排序后：" + r);
|
// }
|
// return "success";
|
// }
|
//}
|
