package com.walker.di.univocity;
|
|
import com.univocity.parsers.common.processor.BatchedColumnProcessor;
|
import com.univocity.parsers.common.processor.BeanListProcessor;
|
import com.univocity.parsers.common.processor.ObjectRowListProcessor;
|
import com.univocity.parsers.common.processor.ObjectRowWriterProcessor;
|
import com.univocity.parsers.csv.CsvParser;
|
import com.univocity.parsers.csv.CsvParserSettings;
|
import com.univocity.parsers.csv.CsvWriter;
|
import com.univocity.parsers.csv.CsvWriterSettings;
|
import com.univocity.parsers.tsv.TsvWriter;
|
import com.univocity.parsers.tsv.TsvWriterSettings;
|
import com.walker.di.DataImportException;
|
import com.walker.di.DataImportor;
|
import com.walker.infrastructure.utils.StringUtils;
|
import org.junit.Before;
|
import org.junit.Test;
|
|
import java.io.BufferedInputStream;
|
import java.io.BufferedOutputStream;
|
import java.io.File;
|
import java.io.FileInputStream;
|
import java.io.FileOutputStream;
|
import java.io.InputStream;
|
import java.util.ArrayList;
|
import java.util.HashMap;
|
import java.util.List;
|
import java.util.Map;
|
|
public class TestImport {
|
|
private InputStream inputStream = null;
|
private InputStream inputStreamBatch = null;
|
private InputStream inputStreamExcel = null;
|
|
@Before
|
public void init() throws Exception{
|
inputStream = new BufferedInputStream(new FileInputStream(new File("d:/tmp/s_user_core.csv")));
|
inputStreamBatch = new BufferedInputStream(new FileInputStream(new File("d:/tmp/test_user_30.csv")));
|
inputStreamExcel = new BufferedInputStream(new FileInputStream(new File("d:/tmp/test_user_30.xlsx")));
|
}
|
|
// @Test
|
public void test(){
|
List<String> input = new ArrayList<>(8);
|
input.add("1");
|
input.add("2");
|
// String[] str = input.toArray(new String[0]);
|
String[] str = StringUtils.toStringArray(input);
|
for(String s : str){
|
System.out.println(s);
|
}
|
}
|
|
// @Test
|
public void testBatchExcel(){
|
// DataImportor dataImportor = new DefaultCsvDataImportor(this.inputStreamExcel);
|
DataImportor dataImportor = new DefaultCsvDataImportor(this.inputStreamBatch);
|
dataImportor.setBatchEnabled();
|
dataImportor.setIgnoreRows(1);
|
dataImportor.setBatchSleepMills(1000);
|
dataImportor.setId("123");
|
try {
|
dataImportor.load();
|
System.out.println("批量导入完成");
|
} catch (DataImportException e) {
|
System.out.println("导入异常:" + e.getMessage());
|
System.out.println(dataImportor.getBatchError());
|
e.printStackTrace();
|
}
|
}
|
|
// @Test
|
public void testBatchCsvImportor(){
|
DataImportor dataImportor = new DefaultCsvDataImportor(this.inputStreamBatch);
|
dataImportor.setBatchEnabled();
|
dataImportor.setIgnoreRows(1);
|
dataImportor.setBatchSleepMills(1000);
|
dataImportor.setSaveFileFolder("d:/tmp/");
|
dataImportor.setId("test_123");
|
try {
|
dataImportor.load();
|
System.out.println("批量导入完成");
|
} catch (DataImportException e) {
|
System.out.println("导入异常:" + e.getMessage());
|
System.out.println(dataImportor.getBatchError());
|
e.printStackTrace();
|
}
|
}
|
|
// @Test
|
public void testCsvDataImportor(){
|
DataImportor dataImportor = new DefaultCsvDataImportor(this.inputStream);
|
try {
|
dataImportor.load();
|
System.out.println("导入完成");
|
} catch (DataImportException e) {
|
System.out.println("导入异常:" + e.getMessage());
|
e.printStackTrace();
|
}
|
}
|
|
// @Test
|
public void testTsvWriter() throws Exception{
|
File errorFile = new File("d:/tmp/test_error.xlsx");
|
BufferedOutputStream outputStream = new BufferedOutputStream(new FileOutputStream(errorFile));
|
final TsvWriterSettings csvWriterSettings = new TsvWriterSettings();
|
csvWriterSettings.setHeaderWritingEnabled(Boolean.TRUE);
|
csvWriterSettings.setHeaders(new String[]{"id", "name", "card"});
|
csvWriterSettings.setRowWriterProcessor(new ObjectRowWriterProcessor());
|
TsvWriter writer = new TsvWriter(outputStream, csvWriterSettings);
|
|
Map<String, String> d1 = new HashMap<>();
|
d1.put("id", "123");
|
d1.put("name", "台上老君");
|
d1.put("card", "410987198011091235");
|
Map<String, String> d2 = new HashMap<>();
|
d2.put("id", "567");
|
d2.put("name", "丹尼斯超时百货");
|
d2.put("card", "99999999911111111110000000000");
|
writer.processRecord(d1);
|
writer.processRecord(d2);
|
|
writer.close();
|
}
|
|
// @Test
|
public void testWriter() throws Exception{
|
Student student = new Student();
|
|
// CsvFormat csvFormat = new CsvFormat();
|
// csvFormat.setQuote('@');
|
// csvFormat.setQuoteEscape('*');
|
// csvFormat.setDelimiter('|');
|
|
// ByteArrayOutputStream outputStream = new ByteArrayOutputStream();
|
// File errorFile = new File("d:/tmp/test_error.csv");
|
File errorFile = new File("d:/tmp/test_error.xlsx");
|
BufferedOutputStream outputStream = new BufferedOutputStream(new FileOutputStream(errorFile));
|
final CsvWriterSettings csvWriterSettings = new CsvWriterSettings();
|
csvWriterSettings.setHeaderWritingEnabled(Boolean.TRUE);
|
// csvWriterSettings.setQuoteAllFields(true);
|
// csvWriterSettings.setFormat(csvFormat);
|
// csvWriterSettings.setQuoteEscapingEnabled(true);
|
csvWriterSettings.setHeaders(new String[]{"id", "name", "card"});
|
// csvWriterSettings.setRowWriterProcessor(new BeanWriterProcessor<>(Student.class));
|
csvWriterSettings.setRowWriterProcessor(new ObjectRowWriterProcessor());
|
CsvWriter writer = new CsvWriter(outputStream, csvWriterSettings);
|
|
Map<String, String> d1 = new HashMap<>();
|
d1.put("id", "123");
|
d1.put("name", "台上老君");
|
d1.put("card", "410987198011091235");
|
Map<String, String> d2 = new HashMap<>();
|
d2.put("id", "567");
|
d2.put("name", "丹尼斯超时百货");
|
d2.put("card", "99999999911111111110000000000");
|
writer.processRecord(d1);
|
writer.processRecord(d2);
|
|
writer.close();
|
}
|
|
// @Test
|
public void testObjectProcessor(){
|
ObjectRowListProcessor rowListProcessor = new ObjectRowListProcessor();
|
CsvParserSettings csvParserSettings = new CsvParserSettings();
|
csvParserSettings.setHeaderExtractionEnabled(true);
|
csvParserSettings.setProcessor(rowListProcessor);
|
CsvParser csvParser = new CsvParser(csvParserSettings);
|
csvParser.parse(inputStream);
|
|
List<Object[]> rows = rowListProcessor.getRows();
|
for(Object[] data : rows){
|
for(Object obj : data){
|
System.out.println(obj);
|
}
|
System.out.println("----------------------");
|
}
|
}
|
|
// @Test
|
public void testColumnProcessor(){
|
CsvParserSettings csvParserSettings = new CsvParserSettings();
|
csvParserSettings.setHeaderExtractionEnabled(true);
|
BatchedColumnProcessor batchedColumnProcessor = new BatchedColumnProcessor(2) {
|
@Override
|
public void batchProcessed(int i) {
|
Map<String, List<String>> columnsByName = this.getColumnValuesAsMapOfNames();
|
for(Map.Entry<String, List<String>> entry : columnsByName.entrySet()){
|
System.out.println("column = " + entry.getKey());
|
System.out.println("values = " + entry.getValue());
|
System.out.println("------------------------------");
|
}
|
}
|
};
|
csvParserSettings.setProcessor(batchedColumnProcessor);
|
CsvParser csvParser = new CsvParser(csvParserSettings);
|
csvParser.parse(inputStream);
|
|
}
|
|
// @Test
|
public void testLoadFile() throws Exception{
|
CsvParserSettings csvParserSettings = new CsvParserSettings();
|
final BeanListProcessor beanListProcessor = new BeanListProcessor(Student.class);
|
csvParserSettings.setProcessor(beanListProcessor);
|
CsvParser csvParser = new CsvParser(csvParserSettings);
|
csvParser.parse(inputStream);
|
|
final List<Student> students = beanListProcessor.getBeans();
|
final String[] headers = beanListProcessor.getHeaders();
|
|
if(headers != null){
|
for(String header : headers){
|
System.out.println("header: " + header);
|
}
|
}
|
if(students != null){
|
for(Student s : students){
|
System.out.println(s);
|
}
|
}
|
}
|
}
|
