运行测试

shikeying

2022-09-22 d87613ba3ee0652fe96285ac687e0b537b42df88

运行测试

 deploy-jar-template/pom.xml

@@ -17,6 +17,23 @@
    </properties>

    <dependencies>

        <dependency>
            <groupId>com.iplatform</groupId>
            <artifactId>recommend-text</artifactId>
        </dependency>

        <dependency>
            <groupId>com.iplatform</groupId>
            <artifactId>recommend-video</artifactId>
        </dependency>

        <!-- 为测试：启动web接口 -->
        <dependency>
            <groupId>org.springframework.boot</groupId>
            <artifactId>spring-boot-starter-web</artifactId>
        </dependency>

    </dependencies>

</project>

 deploy-jar-template/src/main/java/com/iplatform/RecommendApplication.java

New file
@@ -0,0 +1,12 @@
package com.iplatform;

import org.springframework.boot.SpringApplication;
import org.springframework.boot.autoconfigure.SpringBootApplication;

@SpringBootApplication(scanBasePackages = {"com.walker","com.iplatform"})
public class RecommendApplication {

    public static void main(String[] args) throws Exception {
        SpringApplication.run(RecommendApplication.class, args);
    }
}

 deploy-jar-template/src/main/resources/application-dev.yml

New file
@@ -0,0 +1,80 @@
spring:
  application:
    name: train_recommend
  datasource:
    # 是否显示dao中打印的SQL语句
    show-sql: true
    username: root
#    password: Bjjmy_63661766
    password: Bjjmy_2020
#    driver-class-name: sunje.goldilocks.jdbc.GoldilocksDriver
    url: jdbc:mysql://124.70.39.177:3306/train_recommend?useUnicode=true&characterEncoding=UTF-8&useSSL=false
    type: com.walker.jdbc.ds.DefaultDataSource
    hikari:
      minimum-idle: 5
      idle-timeout: 5000
      pool-name: databasePool_walker
      connection-timeout: 10000
      connection-test-query: select 1
#    username: root
#    password: SearunMaster@2019
#    driver-class-name: com.mysql.cj.jdbc.Driver
#    url: jdbc:mysql://193.193.193.236:3306/cpmsxc?serverTimezone=Asia/Shanghai&zeroDateTimeBehavior=convertToNull&autoReconnect=true&useSSL=false&failOverReadOnly=false

#    username: GQCG
#    password: GQCG
#    driver-class-name: oracle.jdbc.OracleDriver
#    url: jdbc:oracle:thin:@61.163.210.202:6021/orcl
server:
  port: 8082
  servlet:
    context-path: /
    session:
      timeout: 30m

logging:
  level:
    root: info
    org:
      springframework: error
    com:
      walker: info
#  file: # logging.file.path 和 logging.file.name，只会有一个生效，配了path不要配name，配了name不要配path，只配path时name默认为spring.log，想路径和文件名同时生效可配置logging.file.name=d:/logs/mylog.log
#    name: ${spring.application.name}.log #日志文件名
#    path: logs  #日志存储路径
#    max-history: 30 #保留多少天的日志
#    max-size: 10MB
#  pattern:
#    rolling-file-name: ${LOG_FILE}.%d{yyyy-MM-dd}.%i.gz
  logback:
    rollingpolicy:
      file-name-pattern: ${LOG_FILE}.%d{yyyy-MM-dd}.%i.gz
      max-history: 1
      max-file-size: 2

# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
# 平台配置支持功能
# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
iplatform:

  # 平台缓存，是否启用redis缓存，默认使用基于内存缓存.
  # 注意：如果开启redis缓存，业务工程必须依赖(walker-support-redis)
  cache:
    redis-enabled: false

  # 相似度分析模块使用
  similarity:
    # 是否启用相似度模块，如果禁用则要确保注入时支持为空
    enabled: true
    milvus-host: 172.16.60.65
    milvus-port: 19530
    word-file: d:/dev_tools/ai/w2v_sogou_dim300_vocab.txt
    embedding-file: d:/dev_tools/ai/w2v_sogou_dim300.npy

  # 调度器模块，是否启用
  scheduler:
    enabled: true

  # 数据采集模块
  gather:
    enabled: true

 deploy-jar-template/src/main/resources/application.yml

New file
@@ -0,0 +1,3 @@
spring:
  profiles:
    active: dev

 recommend-common/pom.xml

@@ -17,6 +17,34 @@
    </properties>

    <dependencies>

<!--        &lt;!&ndash; 平台基础模块，2022/09-22 &ndash;&gt;-->
<!--        <dependency>-->
<!--            <groupId>com.iplatform</groupId>-->
<!--            <artifactId>iplatform-base</artifactId>-->
<!--        </dependency>-->

<!--        &lt;!&ndash; 平台模块：调度任务，2022/09/08 &ndash;&gt;-->
<!--        <dependency>-->
<!--            <groupId>com.iplatform</groupId>-->
<!--            <artifactId>iplatform-scheduler</artifactId>-->
<!--            <exclusions>-->
<!--                <exclusion>-->
<!--                    <groupId>org.springframework.boot</groupId>-->
<!--                    <artifactId>spring-boot-autoconfigure</artifactId>-->
<!--                </exclusion>-->
<!--            </exclusions>-->
<!--        </dependency>-->
        <dependency>
            <groupId>com.iplatform</groupId>
            <artifactId>iplatform-base</artifactId>
        </dependency>

        <dependency>
            <groupId>com.iplatform</groupId>
            <artifactId>iplatform-gather</artifactId>
        </dependency>

    </dependencies>

</project>

 recommend-text/pom.xml

@@ -17,6 +17,28 @@
    </properties>

    <dependencies>

        <dependency>
            <groupId>com.iplatform</groupId>
            <artifactId>recommend-common</artifactId>
        </dependency>

        <!-- 平台模块：文本相似度相似度，2022/09/08 -->
        <dependency>
            <groupId>com.iplatform</groupId>
            <artifactId>iplatform-ml-similarity</artifactId>
            <exclusions>
                <exclusion>
                    <groupId>org.slf4j</groupId>
                    <artifactId>slf4j-api</artifactId>
                </exclusion>
                <exclusion>
                    <groupId>org.apache.logging.log4j</groupId>
                    <artifactId>log4j-slf4j-impl</artifactId>
                </exclusion>
            </exclusions>
        </dependency>

    </dependencies>

</project>

 recommend-text/src/main/java/com/iplatform/rectext/Test.java

New file
@@ -0,0 +1,4 @@
package com.iplatform.rectext;

public class Test {
}

 recommend-text/src/main/java/com/iplatform/rectext/scheduler/WriteCourseToMilvus.java

New file
@@ -0,0 +1,44 @@
package com.iplatform.rectext.scheduler;

import com.iplatform.gather.GatherScheduler;
import com.walker.store.AbstractStore;
import com.walker.store.support.EmptyDatabaseStore;
import com.walker.store.task.GatherTask;

/**
 * 持续写入课程文本描述到相似数据库中，在数据库中定义任务时间周期。
 * <pre>
 *     1）该任务不会停止，每间隔2小时执行一次;
 *     2）增量拉取数据，数据先组织好写入中间表，然后调用sdk写入milvus
 * </pre>
 */
public class WriteCourseToMilvus extends GatherScheduler {

    private GatherTask gatherTask;

    public WriteCourseToMilvus(int id, String name){
        super(id, name, new EmptyDatabaseStore());
    }

    @Override
    protected GatherTask providerTask(AbstractStore store) {
        return null;
    }

    @Override
    protected void terminateGatherScheduler(int schedulerId) {

    }

    @Override
    protected Object onProcess(Object[] inputParams) throws Exception {
        if(this.gatherTask == null){
            this.gatherTask = this.providerTask(this.getStore());
            if(this.gatherTask == null){
                throw new IllegalArgumentException("未提供任务对象，调度无法执行：" + this.getName());
            }
        }
        // srcName, createTableSQL, parameter, params[]
        return this.gatherTask.run((String)inputParams[0], (String)inputParams[1], inputParams[2], (Object[])inputParams[3]);
    }
}