From b77abcbc0f17070a2a970e0c4aa5837e90f28e1f Mon Sep 17 00:00:00 2001
From: shikeying <pxzsky@163.com>
Date: 星期三, 03 四月 2024 14:02:49 +0800
Subject: [PATCH] 提供ip转地区离线查询（未完成）
---
walker-support-milvus/src/main/java/com/walker/support/milvus/engine/DefaultOperateService.java | 10 +
walker-web/src/main/java/com/walker/web/util/IpUtils.java | 72 ++++++--
walker-support-milvus/src/test/java/com/walker/support/milvus/MilvusEngine.java | 108 +++++++++++++
walker-support-milvus/pom.xml | 7
walker-support-milvus/src/main/java/com/walker/support/milvus/DataType.java | 48 +++++
walker-support-milvus/src/test/java/com/walker/support/milvus/TestMilvus.java | 45 +++++
walker-web/pom.xml | 14 +
walker-web/src/main/java/com/walker/web/agent/BrowserCapWebAgentService.java | 19 ++
walker-web/src/main/resources/ip2region.xdb | 0
walker-support-milvus/src/main/java/com/walker/support/milvus/FieldType.java | 124 ++++++++++++---
walker-support-milvus/src/main/java/com/walker/support/milvus/util/FieldTypeUtils.java | 5
walker-support-milvus/src/main/java/com/walker/support/milvus/OutData.java | 12
12 files changed, 409 insertions(+), 55 deletions(-)
diff --git a/walker-support-milvus/pom.xml b/walker-support-milvus/pom.xml
index 0caf03e..755c824 100644
--- a/walker-support-milvus/pom.xml
+++ b/walker-support-milvus/pom.xml
@@ -18,6 +18,13 @@
</properties>
<dependencies>
+
+ <dependency>
+ <groupId>junit</groupId>
+ <artifactId>junit</artifactId>
+ <scope>test</scope>
+ </dependency>
+
<dependency>
<groupId>io.milvus</groupId>
<artifactId>milvus-sdk-java</artifactId>
diff --git a/walker-support-milvus/src/main/java/com/walker/support/milvus/DataType.java b/walker-support-milvus/src/main/java/com/walker/support/milvus/DataType.java
index ef03486..f086b2c 100644
--- a/walker-support-milvus/src/main/java/com/walker/support/milvus/DataType.java
+++ b/walker-support-milvus/src/main/java/com/walker/support/milvus/DataType.java
@@ -21,9 +21,24 @@
return Int64_VALUE;
}
},
+ /**
+ * 搴熷純浜嗚瀛楃涓茬被鍨嬶紝浣跨敤锛歏arChar銆�
+ * @date 2024-03-28
+ */
+ @Deprecated
String{
public int getIndex(){
return String_VALUE;
+ }
+ },
+
+ /**
+ * 鏂扮増浣跨敤鐨勫瓧绗︿覆绫诲瀷
+ * @date 2024-03-28
+ */
+ VarChar {
+ public int getIndex(){
+ return VarChar_VALUE;
}
},
FloatVector{
@@ -31,6 +46,12 @@
return FloatVector_VALUE;
}
},
+ Float16Vector {
+ public int getIndex(){
+ return Float16Vector_VALUE;
+ }
+ },
+
BinaryVector{
public int getIndex(){
return BinaryVector_VALUE;
@@ -39,6 +60,18 @@
None{
public int getIndex(){
return None_VALUE;
+ }
+ },
+
+ Array {
+ public int getIndex(){
+ return Array_VALUE;
+ }
+ },
+
+ Json {
+ public int getIndex(){
+ return JSON_VALUE;
}
};
@@ -63,7 +96,15 @@
return BinaryVector;
} else if(index == None_VALUE){
return None;
- } else {
+ } else if(index == VarChar_VALUE){
+ return VarChar;
+ } else if(index == Array_VALUE){
+ return Array;
+ } else if(index == JSON_VALUE){
+ return Json;
+ } else if(index == Float16Vector_VALUE){
+ return Float16Vector;
+ } else {
throw new IllegalArgumentException("涓嶆敮鎸佺殑鏁版嵁鏍煎紡锛�" + index);
}
}
@@ -77,7 +118,10 @@
public static final int Float_VALUE = 10;
public static final int Double_VALUE = 11;
public static final int String_VALUE = 20;
-// public static final int VarChar_VALUE = 21;
+ public static final int VarChar_VALUE = 21;
+ public static final int Array_VALUE = 22;
+ public static final int JSON_VALUE = 23;
public static final int BinaryVector_VALUE = 100;
public static final int FloatVector_VALUE = 101;
+ public static final int Float16Vector_VALUE = 102;
}
diff --git a/walker-support-milvus/src/main/java/com/walker/support/milvus/FieldType.java b/walker-support-milvus/src/main/java/com/walker/support/milvus/FieldType.java
index 6c12313..794e996 100644
--- a/walker-support-milvus/src/main/java/com/walker/support/milvus/FieldType.java
+++ b/walker-support-milvus/src/main/java/com/walker/support/milvus/FieldType.java
@@ -2,7 +2,8 @@
//import io.milvus.param.ParamUtils;
-import org.apache.commons.lang3.StringUtils;
+import io.milvus.param.ParamUtils;
+import org.checkerframework.checker.nullness.qual.NonNull;
import java.util.HashMap;
import java.util.Map;
@@ -15,6 +16,9 @@
private String description;
private DataType dataType;
private Map<String, String> typeParams;
+ // 2024-03-28
+ private DataType elementType;
+ private boolean partitionKey;
private FieldType(FieldType.Builder builder) {
if (builder == null) {
@@ -26,6 +30,8 @@
this.dataType = builder.dataType;
this.typeParams = builder.typeParams;
// this.autoID = builder.autoID;
+ this.elementType = builder.elementType;
+ this.partitionKey = builder.partitionKey;
}
}
@@ -65,12 +71,16 @@
private DataType dataType;
private final Map<String, String> typeParams;
// private boolean autoID;
+ private DataType elementType;
+ private boolean partitionKey;
private Builder() {
this.primaryKey = false;
this.description = "";
this.typeParams = new HashMap();
// this.autoID = false;
+ this.elementType = DataType.None;
+ this.partitionKey = false;
}
public Builder withName(String name) {
@@ -101,6 +111,15 @@
throw new NullPointerException("dataType is marked non-null but is null");
} else {
this.dataType = dataType;
+ return this;
+ }
+ }
+
+ public Builder withElementType(@NonNull DataType elementType) {
+ if (elementType == null) {
+ throw new NullPointerException("elementType is marked non-null but is null");
+ } else {
+ this.elementType = elementType;
return this;
}
}
@@ -148,52 +167,109 @@
// this.autoID = autoID;
// return this;
// }
+ public Builder withPartitionKey(boolean partitionKey) {
+ this.partitionKey = partitionKey;
+ return this;
+ }
public FieldType build() throws ParamException {
-// ParamUtils.CheckNullEmptyString(this.name, "Field name");
- if (this.name == null || StringUtils.isBlank(this.name)) {
- throw new ParamException(name + " cannot be null or empty");
- }
+ ParamUtils.CheckNullEmptyString(this.name, "Field name");
if (this.dataType != null && this.dataType != DataType.None) {
if (this.dataType == DataType.String) {
- throw new ParamException("String type is not supported, use VarChar instead");
+ throw new io.milvus.exception.ParamException("String type is not supported, use Varchar instead");
} else {
int len;
if (this.dataType == DataType.FloatVector || this.dataType == DataType.BinaryVector) {
if (!this.typeParams.containsKey("dim")) {
- throw new ParamException("Vector field dimension must be specified");
+ throw new io.milvus.exception.ParamException("Vector field dimension must be specified");
}
try {
len = Integer.parseInt((String)this.typeParams.get("dim"));
if (len <= 0) {
- throw new ParamException("Vector field dimension must be larger than zero");
+ throw new io.milvus.exception.ParamException("Vector field dimension must be larger than zero");
}
} catch (NumberFormatException var3) {
- throw new ParamException("Vector field dimension must be an integer number");
+ throw new io.milvus.exception.ParamException("Vector field dimension must be an integer number");
}
}
-// if (this.dataType == io.milvus.grpc.DataType.VarChar) {
-// if (!this.typeParams.containsKey("max_length")) {
-// throw new ParamException("Varchar field max length must be specified");
-// }
-//
-// try {
-// len = Integer.parseInt((String)this.typeParams.get("max_length"));
-// if (len <= 0) {
-// throw new ParamException("Varchar field max length must be larger than zero");
-// }
-// } catch (NumberFormatException var2) {
-// throw new ParamException("Varchar field max length must be an integer number");
-// }
-// }
+ if (this.dataType == DataType.VarChar) {
+ if (!this.typeParams.containsKey("max_length")) {
+ throw new io.milvus.exception.ParamException("Varchar field max length must be specified");
+ }
+
+ try {
+ len = Integer.parseInt((String)this.typeParams.get("max_length"));
+ if (len <= 0) {
+ throw new io.milvus.exception.ParamException("Varchar field max length must be larger than zero");
+ }
+ } catch (NumberFormatException var2) {
+ throw new io.milvus.exception.ParamException("Varchar field max length must be an integer number");
+ }
+ }
+
+ if (this.partitionKey) {
+ if (this.primaryKey) {
+ throw new io.milvus.exception.ParamException("Primary key field can not be partition key");
+ }
+
+ if (this.dataType != DataType.Long && this.dataType != DataType.VarChar) {
+ throw new io.milvus.exception.ParamException("Only Int64 and Varchar type field can be partition key");
+ }
+ }
+
+ if (this.dataType == DataType.Array) {
+ if (this.elementType == DataType.String) {
+ throw new io.milvus.exception.ParamException("String type is not supported, use Varchar instead");
+ }
+
+ if (this.elementType == DataType.None || this.elementType == DataType.Array || this.elementType == DataType.Json || this.elementType == DataType.String || this.elementType == DataType.FloatVector || this.elementType == DataType.Float16Vector || this.elementType == DataType.BinaryVector) {
+ throw new io.milvus.exception.ParamException("Unsupported element type");
+ }
+
+ if (!this.typeParams.containsKey("max_capacity")) {
+ throw new io.milvus.exception.ParamException("Array field max capacity must be specified");
+ }
+
+ if (this.elementType == DataType.VarChar && !this.typeParams.containsKey("max_length")) {
+ throw new io.milvus.exception.ParamException("Varchar array field max length must be specified");
+ }
+ }
return new FieldType(this);
}
} else {
- throw new ParamException("Field data type is illegal");
+ throw new io.milvus.exception.ParamException("Field data type is illegal");
}
+// if (this.name == null || StringUtils.isBlank(this.name)) {
+// throw new ParamException(name + " cannot be null or empty");
+// }
+// if (this.dataType != null && this.dataType != DataType.None) {
+// if (this.dataType == DataType.String) {
+// throw new ParamException("String type is not supported, use VarChar instead");
+// } else {
+// int len;
+// if (this.dataType == DataType.FloatVector || this.dataType == DataType.BinaryVector) {
+// if (!this.typeParams.containsKey("dim")) {
+// throw new ParamException("Vector field dimension must be specified");
+// }
+//
+// try {
+// len = Integer.parseInt((String)this.typeParams.get("dim"));
+// if (len <= 0) {
+// throw new ParamException("Vector field dimension must be larger than zero");
+// }
+// } catch (NumberFormatException var3) {
+// throw new ParamException("Vector field dimension must be an integer number");
+// }
+// }
+// return new FieldType(this);
+// }
+// } else {
+// throw new ParamException("Field data type is illegal");
+// }
}
+
}
}
diff --git a/walker-support-milvus/src/main/java/com/walker/support/milvus/OutData.java b/walker-support-milvus/src/main/java/com/walker/support/milvus/OutData.java
index 2ce507e..6431417 100644
--- a/walker-support-milvus/src/main/java/com/walker/support/milvus/OutData.java
+++ b/walker-support-milvus/src/main/java/com/walker/support/milvus/OutData.java
@@ -23,7 +23,7 @@
private List<Long> keyList = new ArrayList<>();
- private List<Long> businessIdList = new ArrayList<>();
+ private List<String> businessIdList = new ArrayList<>();
/**
* 鏍规嵁milvus鏁版嵁搴撲富閿繑鍥炶褰曞搴旂殑棰勬祴鍒嗗�笺��
@@ -96,11 +96,11 @@
* 杩斿洖涓氬姟缂栧彿闆嗗悎銆�
* @return
*/
- public List<Long> getBusinessIdList() {
+ public List<String> getBusinessIdList() {
return businessIdList;
}
- public void setBusinessIdList(List<Long> businessIdList) {
+ public void setBusinessIdList(List<String> businessIdList) {
this.businessIdList = businessIdList;
}
@@ -111,7 +111,7 @@
public static class Data implements Serializable {
private long key;
- private long businessId;
+ private String businessId;
private float score = 0;
@@ -119,7 +119,7 @@
return key;
}
- public long getBusinessId() {
+ public String getBusinessId() {
return businessId;
}
@@ -127,7 +127,7 @@
return score;
}
- public Data(long key, long businessId, float score){
+ public Data(long key, String businessId, float score){
this.key = key;
this.businessId = businessId;
this.score = score;
diff --git a/walker-support-milvus/src/main/java/com/walker/support/milvus/engine/DefaultOperateService.java b/walker-support-milvus/src/main/java/com/walker/support/milvus/engine/DefaultOperateService.java
index 0247b14..6228326 100644
--- a/walker-support-milvus/src/main/java/com/walker/support/milvus/engine/DefaultOperateService.java
+++ b/walker-support-milvus/src/main/java/com/walker/support/milvus/engine/DefaultOperateService.java
@@ -147,7 +147,7 @@
return false;
}
if(statusR.getStatus().intValue() == R.Status.Success.getCode()){
- logger.error("insert 杩斿洖鍊硷細" + statusR.getStatus().intValue());
+ logger.debug("insert 杩斿洖鍊硷細" + statusR.getStatus().intValue());
return true;
}
return false;
@@ -254,6 +254,12 @@
MetricType metricType = null;
if(query.getMetricType() == null || query.getMetricType().equals("")){
metricType = MetricType.L2;
+ } else if(query.getMetricType().equals(com.walker.support.milvus.MetricType.INDEX_IMAGE)){
+ metricType = MetricType.L2;
+ } else if(query.getMetricType().equals(com.walker.support.milvus.MetricType.INDEX_NLP)){
+ metricType = MetricType.IP;
+ } else {
+ throw new UnsupportedOperationException("鏆傛湭鏀寔鐨勮窛绂荤被鍨嬶細" + query.getMetricType());
}
SearchParam searchParam = SearchParam.newBuilder()
@@ -287,7 +293,7 @@
if(outField.equals("id")){
outData.setKeyList((List<Long>)wrapperSearch.getFieldData("id", 0));
} else {
- outData.setBusinessIdList((List<Long>)wrapperSearch.getFieldData(outField, 0));
+ outData.setBusinessIdList((List<String>)wrapperSearch.getFieldData(outField, 0));
}
}
// System.out.println(wrapperSearch.getFieldData("book_id", 0));
diff --git a/walker-support-milvus/src/main/java/com/walker/support/milvus/util/FieldTypeUtils.java b/walker-support-milvus/src/main/java/com/walker/support/milvus/util/FieldTypeUtils.java
index ae4c399..050f10b 100644
--- a/walker-support-milvus/src/main/java/com/walker/support/milvus/util/FieldTypeUtils.java
+++ b/walker-support-milvus/src/main/java/com/walker/support/milvus/util/FieldTypeUtils.java
@@ -19,7 +19,10 @@
public static DataType toMilvusDataType(com.walker.support.milvus.DataType dt){
if(dt == com.walker.support.milvus.DataType.String){
- return DataType.String;
+// return DataType.String;
+ return DataType.VarChar;
+ } else if(dt == com.walker.support.milvus.DataType.VarChar){
+ return DataType.VarChar;
} else if(dt == com.walker.support.milvus.DataType.Float){
return DataType.Float;
} else if(dt == com.walker.support.milvus.DataType.Double){
diff --git a/walker-support-milvus/src/test/java/com/walker/support/milvus/MilvusEngine.java b/walker-support-milvus/src/test/java/com/walker/support/milvus/MilvusEngine.java
new file mode 100644
index 0000000..b6f2eb6
--- /dev/null
+++ b/walker-support-milvus/src/test/java/com/walker/support/milvus/MilvusEngine.java
@@ -0,0 +1,108 @@
+package com.walker.support.milvus;
+
+import com.walker.support.milvus.engine.DefaultOperateService;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+public class MilvusEngine {
+
+ protected final transient Logger logger = LoggerFactory.getLogger(this.getClass());
+
+ public MilvusEngine(String ip, int port){
+ DefaultOperateService service = new DefaultOperateService();
+ service.connect(ip, port);
+ this.operateService = service;
+ logger.info("connect milvus: {}:{}", ip, port);
+ }
+
+ public void close(){
+ if(this.operateService != null){
+ this.operateService.close();
+ }
+ }
+
+ /**
+ * 鍒涘缓琛細娴嬭瘯浠庤亰澶╀竴閿彁鍙栧伐鍗曞唴瀹逛娇鐢ㄣ��
+ * <pre>
+ * 1) 浠庡巻鍙插伐鍗曟暟鎹腑锛屾敹闆嗙敤鎴锋彁闂唴瀹癸紝鏁寸悊鍒拌〃涓�
+ * 2) 鎶婅繖浜涙暟鎹�氳繃鍚戦噺杞寲锛屽啓鍏ilvus鏁版嵁搴撱��
+ * </pre>
+ * @date 2024-03-28
+ */
+ public void createChatSimilarTable(){
+ Table chatSimilarTable = new Table();
+ chatSimilarTable.setCollectionName("chat_similar");
+ chatSimilarTable.setDescription("鑱婂ぉ鎻愬彇宸ュ崟鎽樿鍘嗗彶鏁版嵁");
+ chatSimilarTable.setShardsNum(1);
+ chatSimilarTable.setDimension(768); // 杩欎釜鏄牴鎹娇鐢ㄥ悜閲忔ā鍨嬬淮搴﹀畾鐨�
+
+ // 璁剧疆瀛楁
+ FieldType id = FieldType.newBuilder()
+ .withName("id").withPrimaryKey(true).withMaxLength(18).withDataType(DataType.Long).build();
+ FieldType title = FieldType.newBuilder()
+ .withName("title").withPrimaryKey(false).withMaxLength(180).withDataType(DataType.VarChar).build();
+ FieldType content = FieldType.newBuilder()
+ .withName("content").withPrimaryKey(false).withMaxLength(255).withDataType(DataType.VarChar).build();
+ FieldType answer = FieldType.newBuilder()
+ .withName("answer").withPrimaryKey(false).withMaxLength(255).withDataType(DataType.VarChar).build();
+ FieldType embedding = FieldType.newBuilder()
+ .withName("embedding").withPrimaryKey(false).withDataType(DataType.FloatVector).withDimension(768).build();
+
+ List<FieldType> fieldTypeList = new ArrayList<>(8);
+ fieldTypeList.add(id);
+ fieldTypeList.add(title);
+ fieldTypeList.add(content);
+ fieldTypeList.add(answer);
+ fieldTypeList.add(embedding);
+ chatSimilarTable.setFieldTypes(fieldTypeList);
+
+ this.operateService.createTable(chatSimilarTable);
+ logger.info("鍒涘缓浜� table = {}", chatSimilarTable.getCollectionName());
+
+ // 鍒涘缓绱㈠紩
+ this.operateService.createIndex(chatSimilarTable.getCollectionName()
+ , "embedding", "HNSW", "{\"nlist\":16384, \"efConstruction\":128, \"M\":8}", MetricType.NLP);
+ logger.info("鍒涘缓浜� index = {}", chatSimilarTable.getCollectionName() + "_index");
+ }
+
+ public void dropChatSimilarTable(){
+ this.operateService.dropTable("chat_similar");
+ }
+
+ public void insertTestData(){
+ DataSet dataSet = new DataSet();
+ dataSet.setTableName("chat_similar");
+
+ Map<String, List<?>> fieldMap = new HashMap();
+ fieldMap.put("id", Arrays.asList(new Long[]{100L}));
+ fieldMap.put("title", Arrays.asList(new String[]{"绗竴涓爣棰�"}));
+ fieldMap.put("content", Arrays.asList(new String[]{"绗竴涓唴瀹�"}));
+ fieldMap.put("answer", Arrays.asList(new String[]{"绗竴涓瓟妗�"}));
+ fieldMap.put("embedding", Arrays.asList(mockVector));
+ dataSet.setFields(fieldMap);
+ this.operateService.insertDataSet(dataSet);
+ logger.info("鍐欏叆浜嗘祴璇曟暟鎹�: {}", dataSet);
+ }
+
+ public OutData searchChatSimilar(List<List<Float>> vectors){
+ Query query = new Query();
+ query.setMetricType(MetricType.NLP.getIndex());
+ query.setTableName("chat_similar");
+ query.setTopK(4);
+ query.setVectorName("embedding");
+ query.setOutFieldList(Arrays.asList(new String[]{"id","title","content"}));
+ query.setFieldPrimaryKey("id");
+ query.setSearchVectors(vectors);
+ return this.operateService.searchVector(query);
+ }
+
+ private OperateService operateService;
+
+ private double[] mockVector = new double[]{-0.051114246249198914, 0.889954432};
+}
diff --git a/walker-support-milvus/src/test/java/com/walker/support/milvus/TestMilvus.java b/walker-support-milvus/src/test/java/com/walker/support/milvus/TestMilvus.java
new file mode 100644
index 0000000..fd5a4d6
--- /dev/null
+++ b/walker-support-milvus/src/test/java/com/walker/support/milvus/TestMilvus.java
@@ -0,0 +1,45 @@
+package com.walker.support.milvus;
+
+import org.junit.Test;
+
+public class TestMilvus {
+
+ public void testSearchMilvus(){
+
+ }
+
+// @Test
+ public void testFloatParse(){
+ double d = 0.013319502584636211;
+ float f = Float.parseFloat(String.valueOf(d));
+ System.out.println("f = " + f);
+ }
+
+ /**
+ * 鍒涘缓鑱婂ぉ涓彁鍙栧伐鍗曟憳瑕佸姛鑳斤紝milvus娴嬭瘯琛ㄥ垱寤恒��
+ * <p>绗竴姝ワ細鍒涘缓琛ㄥ拰绱㈠紩</p>
+ */
+// @Test
+ public void createTable(){
+ this.acquireMilvusEngine();
+ this.milvusEngine.createChatSimilarTable();
+ }
+
+ /**
+ * 娴嬭瘯锛氬啓鍏ヤ竴涓ā鎷熸暟鎹�
+ */
+// @Test
+ public void insertTestMockData(){
+ this.acquireMilvusEngine();
+ this.milvusEngine.insertTestData();
+ }
+
+ private void acquireMilvusEngine(){
+ if(this.milvusEngine == null){
+ MilvusEngine engine = new MilvusEngine("120.26.128.84", 19530);
+ this.milvusEngine = engine;
+ }
+ }
+
+ private MilvusEngine milvusEngine;
+}
diff --git a/walker-web/pom.xml b/walker-web/pom.xml
index 4927724..2d4a243 100644
--- a/walker-web/pom.xml
+++ b/walker-web/pom.xml
@@ -14,6 +14,7 @@
<packaging>jar</packaging>
<properties>
+ <ip2region.version>2.7.0</ip2region.version>
</properties>
<dependencies>
@@ -55,6 +56,19 @@
<artifactId>okhttp</artifactId>
</dependency>
+ <!-- IP瀹氫綅锛屼竴涓紑婧恑p涓庡湴鍧�瀵瑰簲鐨勬湰鍦版暟鎹簱xdb锛�2024-04-02 -->
+ <dependency>
+ <groupId>org.lionsoul</groupId>
+ <artifactId>ip2region</artifactId>
+ <version>${ip2region.version}</version>
+ <exclusions>
+ <exclusion>
+ <groupId>junit</groupId>
+ <artifactId>junit</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+
</dependencies>
</project>
diff --git a/walker-web/src/main/java/com/walker/web/agent/BrowserCapWebAgentService.java b/walker-web/src/main/java/com/walker/web/agent/BrowserCapWebAgentService.java
index b96e7e2..2511915 100644
--- a/walker-web/src/main/java/com/walker/web/agent/BrowserCapWebAgentService.java
+++ b/walker-web/src/main/java/com/walker/web/agent/BrowserCapWebAgentService.java
@@ -3,6 +3,7 @@
import com.walker.web.WebAgentService;
import com.walker.web.WebUserAgent;
import com.walker.web.util.IpUtils;
+import org.lionsoul.ip2region.xdb.Searcher;
import javax.servlet.http.HttpServletRequest;
import java.util.Arrays;
@@ -15,6 +16,9 @@
private Capabilities capabilities = null;
private UserAgentParser parser = null;
+
+ // 鎼滅储鍦板尯鐨勬湰鍦板璞★紙璇诲彇鏈湴xdb鏁版嵁搴擄級锛�2024-04-02
+ private Searcher searcher = null;
public BrowserCapWebAgentService(){
try {
@@ -30,6 +34,11 @@
} catch (Exception e) {
throw new RuntimeException("鍒涘缓娴忚鍣ㄨВ鏋愬璞￠敊璇�:" + e.getMessage(), e);
}
+
+// this.searcher = Searcher.newWithFileOnly();
+// ClassPathResource resource = new ClassPathResource();
+// resource.getFile();
+// RandomAccessFile randomAccessFile = new RandomAccessFile();
}
@Override
@@ -59,6 +68,7 @@
return webUserAgent;
}
+
/**
* 璁剧疆鏄惁鍔犺浇ip瀵瑰簲鐨勫尯鍩熷湴璐紝榛樿锛歠alse涓嶅姞杞斤紝鍥犱负闇�瑕佽繛澶栫綉鏌ユ壘銆�
* @param loadLocation
@@ -68,5 +78,14 @@
this.loadLocation = loadLocation;
}
+ /**
+ * 璁剧疆锛歩p2region鏁版嵁搴撴枃浠惰矾寰勶紝濡傦細
+ * @param xdbPath
+ */
+ public void setXdbPath(String xdbPath) {
+ this.xdbPath = xdbPath;
+ }
+
+ private String xdbPath;
private boolean loadLocation = false;
}
diff --git a/walker-web/src/main/java/com/walker/web/util/IpUtils.java b/walker-web/src/main/java/com/walker/web/util/IpUtils.java
index d438d55..122f43c 100644
--- a/walker-web/src/main/java/com/walker/web/util/IpUtils.java
+++ b/walker-web/src/main/java/com/walker/web/util/IpUtils.java
@@ -32,37 +32,69 @@
*
* @param request 璇锋眰瀵硅薄
* @return IP鍦板潃
+ * @date 2023-10-21
+ * @date 2024-04-02 澧炲姞鑾峰彇璇锋眰澶翠腑鍙傛暟锛堣ˉ鍏咃級
*/
- public static String getIpAddr(HttpServletRequest request)
- {
- if (request == null)
- {
+ public static String getIpAddr(HttpServletRequest request) {
+ if (request == null) {
return "unknown";
}
- String ip = request.getHeader("x-forwarded-for");
- if (ip == null || ip.length() == 0 || UNKNOWN_TEXT.equalsIgnoreCase(ip))
- {
- ip = request.getHeader("Proxy-Client-IP");
- }
- if (ip == null || ip.length() == 0 || UNKNOWN_TEXT.equalsIgnoreCase(ip))
- {
+
+ String ip = null;
+
+ // 浠ヤ笅涓や釜鑾峰彇鍦╧8s涓紝灏嗙湡瀹炵殑瀹㈡埛绔疘P锛屾斁鍒颁簡x-Original-Forwarded-For銆傝�屽皢WAF鐨勫洖婧愬湴鍧�鏀惧埌浜� x-Forwarded-For浜嗐��
+ ip = request.getHeader("X-Original-Forwarded-For");
+ if (ip == null || UNKNOWN_TEXT.equalsIgnoreCase(ip)) {
ip = request.getHeader("X-Forwarded-For");
}
- if (ip == null || ip.length() == 0 || UNKNOWN_TEXT.equalsIgnoreCase(ip))
- {
+
+ //鑾峰彇nginx绛変唬鐞嗙殑ip
+ if (ip == null || UNKNOWN_TEXT.equalsIgnoreCase(ip)) {
+ ip = request.getHeader("x-forwarded-for");
+ }
+
+ if (ip == null || ip.length() == 0 || UNKNOWN_TEXT.equalsIgnoreCase(ip)) {
+ ip = request.getHeader("Proxy-Client-IP");
+ }
+ if (ip == null || ip.length() == 0 || UNKNOWN_TEXT.equalsIgnoreCase(ip)) {
ip = request.getHeader("WL-Proxy-Client-IP");
}
- if (ip == null || ip.length() == 0 || UNKNOWN_TEXT.equalsIgnoreCase(ip))
- {
+ if (ip == null || UNKNOWN_TEXT.equalsIgnoreCase(ip)) {
+ ip = request.getHeader("HTTP_CLIENT_IP");
+ }
+ if (ip == null || ip.length() == 0 || UNKNOWN_TEXT.equalsIgnoreCase(ip)) {
ip = request.getHeader("X-Real-IP");
}
-
- if (ip == null || ip.length() == 0 || UNKNOWN_TEXT.equalsIgnoreCase(ip))
- {
- ip = request.getRemoteAddr();
+ if (ip == null || UNKNOWN_TEXT.equalsIgnoreCase(ip)) {
+ ip = request.getHeader("HTTP_X_FORWARDED_FOR");
}
- return "0:0:0:0:0:0:0:1".equals(ip) ? LOCALHOST : getMultistageReverseProxyIp(ip);
+ // 2.濡傛灉娌℃湁杞彂鐨刬p锛屽垯鍙栧綋鍓嶉�氫俊鐨勮姹傜鐨刬p(鍏煎k8s闆嗙兢鑾峰彇ip)
+ if (ip == null || ip.length() == 0 || UNKNOWN_TEXT.equalsIgnoreCase(ip)) {
+ ip = request.getRemoteAddr();
+ // 濡傛灉鏄�127.0.0.1锛屽垯鍙栨湰鍦扮湡瀹瀒p
+ if (LOCALHOST.equals(ip)) {
+ // 鏍规嵁缃戝崱鍙栨湰鏈洪厤缃殑IP
+ InetAddress inet = null;
+ try {
+ inet = InetAddress.getLocalHost();
+ ip = inet.getHostAddress();
+ } catch (UnknownHostException e) {
+ e.printStackTrace();
+ }
+ }
+ }
+
+ // 瀵逛簬閫氳繃澶氫釜浠ｇ悊鐨勬儏鍐碉紝绗竴涓狪P涓哄鎴风鐪熷疄IP,澶氫釜IP鎸夌収','鍒嗗壊
+ if (ip != null && ip.length() > 15) {
+ // = 15
+ if (ip.indexOf(StringUtils.DEFAULT_SPLIT_SEPARATOR) > 0) {
+ ip = ip.substring(0, ip.indexOf(StringUtils.DEFAULT_SPLIT_SEPARATOR));
+ }
+ }
+
+// return "0:0:0:0:0:0:0:1".equals(ip) ? LOCALHOST : getMultistageReverseProxyIp(ip);
+ return "0:0:0:0:0:0:0:1".equals(ip) ? LOCALHOST : ip;
}
/**
diff --git a/walker-web/src/main/resources/ip2region.xdb b/walker-web/src/main/resources/ip2region.xdb
new file mode 100644
index 0000000..7052c05
--- /dev/null
+++ b/walker-web/src/main/resources/ip2region.xdb
Binary files differ
--
Gitblit v1.9.1