tzjy-code
/
DataRoom
5
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

添加批量导入csv接口

Browse Source
main
whb 6 months ago
parent 4a2b964c40
commit 3fb29b06ed
8 changed files with 501 additions and 27 deletions
Show Stats Download Patch File Download Diff File

5
DataRoom/dataroom-core/pom.xml
Unescape Escape View File

@ -95,6 +95,11 @@
<artifactId>easyexcel</artifactId>
<version>3.2.1</version>
</dependency>
<dependency>
<groupId>cn.hutool</groupId>
<artifactId>hutool-all</artifactId>
<version>5.8.27</version>
</dependency>
<dependency>
<groupId>com.github.stuxuhai</groupId>

41
DataRoom/dataroom-core/src/main/java/com/gccloud/dataroom/core/module/manage/controller/NeUploadExcelController.java
Unescape Escape View File

@ -4,12 +4,16 @@ import com.alibaba.excel.EasyExcel;
import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
import com.baomidou.mybatisplus.core.metadata.IPage;
import com.baomidou.mybatisplus.extension.plugins.pagination.Page;
import com.gccloud.common.exception.GlobalException;
import com.gccloud.common.permission.ApiPermission;
import com.gccloud.common.utils.JSON;
import com.gccloud.common.vo.R;
import com.gccloud.dataroom.core.module.manage.dao.NeUploadExcelDao;
import com.gccloud.dataroom.core.module.manage.dto.CvsDTO;
import com.gccloud.dataroom.core.module.manage.dto.DatasetEntityDTO;
import com.gccloud.dataroom.core.module.manage.entity.StuUploadExcelUser;
import com.gccloud.dataroom.core.module.manage.service.NewExcelService;
import com.gccloud.dataroom.core.utils.CsvImportUtil;
import com.gccloud.dataroom.core.utils.DemoDataListener;
import com.gccloud.dataset.constant.DatasetConstant;
import com.gccloud.dataset.dto.DatasetDTO;
@ -19,15 +23,19 @@ import com.gccloud.dataset.service.IBaseDataSetService;
import com.gccloud.dataset.service.IDatasetLabelService;
import io.swagger.annotations.Api;
import io.swagger.annotations.ApiOperation;
import net.sourceforge.pinyin4j.PinyinHelper;
import org.springframework.beans.BeanUtils;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.transaction.annotation.Transactional;
import org.springframework.util.CollectionUtils;
import org.springframework.web.bind.annotation.*;
import org.springframework.web.multipart.MultipartFile;
import javax.annotation.Resource;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.UUID;
/**
@ -39,7 +47,8 @@ import java.util.UUID;
@RequestMapping("/newDataset")
public class NeUploadExcelController {
@Autowired
private NewExcelService service;
@Resource
private IDatasetLabelService datasetLabelService;
@ -57,20 +66,33 @@ public class NeUploadExcelController {
@Transactional(rollbackFor = Exception.class)
public R uploadExcel(@RequestParam("file") @RequestPart MultipartFile file,String userId) throws IOException {
// file.getOriginalFilename().lastIndexOf(".")+1
String substring = file.getOriginalFilename().substring(file.getOriginalFilename().lastIndexOf(".") + 1);
if (substring.equals("csv")){
return service.uploadCVS(file,userId);
}else if ("xls".equals(substring)|| "xlsx".equals(substring)){
//存储excel数据 根据id关联
String s = UUID.randomUUID().toString().replaceAll("-", "");
DemoDataListener demoDataListener = new DemoDataListener(neUploadExcelDao, userId, s);
// 这里 需要指定读用哪个class去读，然后读取第一个sheet 文件流会自动关闭
EasyExcel.read(file.getInputStream(), StuUploadExcelUser.class,
new DemoDataListener(neUploadExcelDao, userId, s)).sheet().doRead();
//存储excel数据 根据id关联
String s = UUID.randomUUID().toString().replaceAll("-", "");
DemoDataListener demoDataListener = new DemoDataListener(neUploadExcelDao, userId, s);
// 这里 需要指定读用哪个class去读，然后读取第一个sheet 文件流会自动关闭
EasyExcel.read(file.getInputStream(), StuUploadExcelUser.class,
new DemoDataListener(neUploadExcelDao, userId, s)).sheet().doRead();
return R.success(s);
}else {
throw new GlobalException("请导入正确格式的文件");
}
return R.success(s);
}
@ApiOperation("分页查询excel")
@ -125,4 +147,7 @@ public class NeUploadExcelController {
}
}

2
DataRoom/dataroom-core/src/main/java/com/gccloud/dataroom/core/module/manage/controller/NewCategoryController.java
Unescape Escape View File

@ -94,4 +94,6 @@ public class NewCategoryController {
}

17
DataRoom/dataroom-core/src/main/java/com/gccloud/dataroom/core/module/manage/dto/CvsDTO.java
Unescape Escape View File

@ -0,0 +1,17 @@
package com.gccloud.dataroom.core.module.manage.dto;
import cn.hutool.core.text.csv.CsvRow;
import lombok.Data;
import java.util.List;
/**
* @author 17803
* @date 2024-09-05 17:11
*/
@Data
public class CvsDTO {
private List<String> header;
private List<CsvRow> rows ;
}

21
DataRoom/dataroom-core/src/main/java/com/gccloud/dataroom/core/module/manage/service/NewExcelService.java
Unescape Escape View File

@ -0,0 +1,21 @@
package com.gccloud.dataroom.core.module.manage.service;
import com.gccloud.common.vo.R;
import com.gccloud.dataroom.core.module.manage.dto.CategorySearchDTO;
import com.gccloud.dataroom.core.module.manage.entity.CategoryEntity;
import com.gccloud.dataset.vo.CategoryVO;
import org.springframework.web.multipart.MultipartFile;
import java.util.List;
/**
* @author 17803
* @date 2024-09-02 16:22
*/
public interface NewExcelService {
R uploadCVS(MultipartFile file, String userId);
}

165
DataRoom/dataroom-core/src/main/java/com/gccloud/dataroom/core/module/manage/service/impl/NewExcelServiceImpl.java
Unescape Escape View File

@ -0,0 +1,165 @@
package com.gccloud.dataroom.core.module.manage.service.impl;
import cn.hutool.core.text.csv.CsvRow;
import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
import com.baomidou.mybatisplus.extension.service.impl.ServiceImpl;
import com.gccloud.common.exception.GlobalException;
import com.gccloud.common.utils.BeanConvertUtils;
import com.gccloud.common.utils.JSON;
import com.gccloud.common.vo.R;
import com.gccloud.dataroom.core.module.basic.dao.DataRoomPagePreviewDao;
import com.gccloud.dataroom.core.module.basic.entity.PagePreviewEntity;
import com.gccloud.dataroom.core.module.manage.dao.NeUploadExcelDao;
import com.gccloud.dataroom.core.module.manage.dto.CvsDTO;
import com.gccloud.dataroom.core.module.manage.dto.DataRoomPageDTO;
import com.gccloud.dataroom.core.module.manage.entity.StuUploadExcelUser;
import com.gccloud.dataroom.core.module.manage.service.IDataRoomPagePreviewService;
import com.gccloud.dataroom.core.module.manage.service.NewExcelService;
import com.gccloud.dataroom.core.utils.CsvImportUtil;
import net.sourceforge.pinyin4j.PinyinHelper;
import org.springframework.stereotype.Service;
import org.springframework.util.CollectionUtils;
import org.springframework.web.multipart.MultipartFile;
import java.io.IOException;
import java.lang.reflect.Field;
import java.util.ArrayList;
import java.util.Date;
import java.util.List;
import java.util.UUID;
/**
* @author hongyang
* @version 1.0
* @date 2023/9/13 10:11
*/
@Service
public class NewExcelServiceImpl extends ServiceImpl<NeUploadExcelDao, StuUploadExcelUser> implements NewExcelService {
@Override
public R uploadCVS(MultipartFile file, String userId) {
int suff = file.getOriginalFilename().lastIndexOf(".");
String substring = file.getOriginalFilename().substring(suff+1);
//存储excel数据 根据id关联
String s = UUID.randomUUID().toString().replaceAll("-", "");
// 存储转换后的拼音列表
List<String> pinyinList = new ArrayList<>();
if ("csv".equals(substring))
{
List<StuUploadExcelUser> excelUsers=new ArrayList<>();
CvsDTO cvsDTO= null;
try {
cvsDTO = CsvImportUtil.csvImportsNew(file);
} catch (IOException e) {
throw new RuntimeException(e);
}
// 遍历list中的每个字符串
for (String chinese : cvsDTO.getHeader()) {
StringBuilder pinyin = new StringBuilder();
// 遍历字符串中的每个字符
for (char c : chinese.toCharArray()) {
if (Character.toString(c).matches("[\\u4E00-\\u9FA5]")) {
// 获取拼音
String[] pinyinArray = PinyinHelper.toHanyuPinyinStringArray(c);
if (pinyinArray != null) {
// 取第一个拼音
pinyin.append(pinyinArray[0].replaceAll("[1234]", ""));
}
} else {
// 非中文字符原样添加
pinyin.append(c);
}
}
// 将拼音结果添加到pinyinList
pinyinList.add(pinyin.toString().trim());
}
StuUploadExcelUser stuUploadExcelUsers = new StuUploadExcelUser();
stuUploadExcelUsers.setModule(s);
stuUploadExcelUsers.setUserId(userId);
if (CollectionUtils.isEmpty(pinyinList))
{
stuUploadExcelUsers.setStepOneA(JSON.toJSONString(cvsDTO.getHeader()));
}else {
stuUploadExcelUsers.setStepOneA(JSON.toJSONString(pinyinList));
}
// 获取 StuUploadExcelUser 的所有字段
Field[] fields = StuUploadExcelUser.class.getDeclaredFields();
int fieldIndex = 2;
int count = 0;
// 遍历 CsvRow
for (CsvRow row : cvsDTO.getRows()) {
StuUploadExcelUser stuUploadExcelUser = new StuUploadExcelUser();
fieldIndex = 2;
if (count >=1000)
{
break;
}
for (int j = 0; j < row.size(); j++) {
if (fieldIndex >= fields.length) {
// 如果字段超出范围，跳出循环
break;
}
try {
// 获取字段
Field field = fields[fieldIndex];
field.setAccessible(true); // 设置字段为可访问
// 获取 row 中的值
Object value = row.get(j);
// 根据字段类型进行赋值
if (field.getType().equals(String.class)) {
field.set(stuUploadExcelUser, value != null ? value.toString() : null);
}
else {
field.set(stuUploadExcelUser, value != null ? value.toString() : null);
}
if (row.size() == j)
{
fieldIndex = 2;
break;
}else {
fieldIndex++;
}
} catch (IllegalAccessException e) {
e.printStackTrace();
}
}
count++;
excelUsers.add(stuUploadExcelUser);
}
excelUsers.stream().forEach(item->{
item.setModule(s);
item.setStepOneA(JSON.toJSONString(pinyinList));
item.setUserId(userId);
});
this.saveBatch(excelUsers);
}
return R.success(s);
}
}

254
DataRoom/dataroom-core/src/main/java/com/gccloud/dataroom/core/utils/CsvImportUtil.java
Unescape Escape View File

@ -0,0 +1,254 @@
package com.gccloud.dataroom.core.utils;
import cn.hutool.core.io.FileUtil;
import cn.hutool.core.text.csv.*;
import cn.hutool.core.util.CharsetUtil;
import com.baomidou.mybatisplus.core.metadata.TableInfo;
import com.baomidou.mybatisplus.core.toolkit.ObjectUtils;
import com.gccloud.dataroom.core.module.manage.dto.CvsDTO;
import lombok.extern.slf4j.Slf4j;
import org.springframework.web.multipart.MultipartFile;
import org.apache.commons.csv.CSVFormat;
import org.apache.commons.csv.CSVParser;
import org.apache.commons.csv.CSVRecord;
import java.io.*;
import java.net.URL;
import java.nio.charset.Charset;
import java.util.*;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
@Slf4j
public class CsvImportUtil {
//上传文件的路径
private final static URL PATH = Thread.currentThread().getContextClassLoader().getResource("");
/**
* @return File 一般文件类型
* @Description 上传文件的文件类型
* @Param multipartFile
**/
public static File uploadFile(MultipartFile multipartFile) {
// 获 取上传 路径
String path = PATH.getPath() + multipartFile.getOriginalFilename();
try {
// 通过将给定的路径名字符串转换为抽象路径名来创建新的 File实例
File file = new File(path);
// 此抽象路径名表示的文件或目录是否存在
if (!file.getParentFile().exists()) {
// 创建由此抽象路径名命名的目录，包括任何必需但不存在的父目录
file.getParentFile().mkdirs();
}
// 转换为一般file 文件
multipartFile.transferTo(file);
return file;
} catch (IOException e) {
e.printStackTrace();
return null;
}
}
/**
* @return List<List < String>>
* @Description 读取CSV文件的内容（不含表头）
* @Param filePath 文件存储路径，colNum 列数
**/
public static List<List<String>> readCSV(String filePath, int colNum) {
BufferedReader bufferedReader = null;
InputStreamReader inputStreamReader = null;
FileInputStream fileInputStream = null;
CSVFormat csvFileFormat = CSVFormat.DEFAULT.withQuote(null);
try {
fileInputStream = new FileInputStream(filePath);
inputStreamReader = new InputStreamReader(fileInputStream, "utf-8");
bufferedReader = new BufferedReader(inputStreamReader);
// CSVParser parser = CSVFormat.DEFAULT.parse(bufferedReader);
CSVParser csvFileParser = new CSVParser(inputStreamReader, csvFileFormat);
// 表内容集合，外层 List为行的集合，内层 List为字段集合
List<List<String>> values = new ArrayList<>();
int rowIndex = 0;
// 读取文件每行内容
for (CSVRecord record : csvFileParser.getRecords()) {
// 跳过表头
if (rowIndex == 0) {
rowIndex++;
continue;
}
// 判断下角标是否越界
if (colNum > record.size()) {
// 返回空集合
return values;
}
// 每行的内容
List<String> value = new ArrayList<>();
for (int i = 0; i < colNum; i++) {
value.add(record.get(i));
}
values.add(value);
rowIndex++;
}
return values;
} catch (IOException e) {
e.printStackTrace();
} finally {
//关闭流
if (bufferedReader != null) {
try {
bufferedReader.close();
} catch (IOException e) {
e.printStackTrace();
}
}
if (inputStreamReader != null) {
try {
inputStreamReader.close();
} catch (IOException e) {
e.printStackTrace();
}
}
if (fileInputStream != null) {
try {
fileInputStream.close();
} catch (IOException e) {
e.printStackTrace();
}
}
}
return null;
}
// 读取csv中的数据
public static List<Map<String,Object>> csvImports(MultipartFile file) throws IOException {
//2. 进行配置
CsvReadConfig csvReadConfig=new CsvReadConfig();
// 是否跳过空白行
csvReadConfig.setSkipEmptyRows(true);
// 是否设置首行为标题行
csvReadConfig.setContainsHeader(true);
//构建 CsvReader 对象
CsvReader csvReader = CsvUtil.getReader(csvReadConfig);
// 这里转了下 可能会产生临时文件，临时文件目录可以设置，也可以立马删除
CsvData read = csvReader.read(multipartFile2File(file), CharsetUtil.CHARSET_UTF_8);
//CsvData read = csvReader.read(FileUtil.file(file.getOriginalFilename()), Charset.forName("utf-8"));
List<Map<String,Object>> mapList = new ArrayList<>();
List<String> header = read.getHeader(); // TODO: 2024/4/26 获取所有的标题头部信息
List<CsvRow> rows = read.getRows();// TODO: 2024/4/26 获取csv文件每行数据
for (CsvRow row : rows) {
Map<String,Object> map = new HashMap<>();
for (int i = 0; i < row.size(); i++) {
map.put(header.get(i),row.get(i));
}
mapList.add(map);
}
return mapList;
}
/**
* multipartFile转File
**/
public static File multipartFile2File(MultipartFile multipartFile){
File file = null;
if (multipartFile != null){
try {
file=File.createTempFile("tmp", null);
multipartFile.transferTo(file);
System.gc();
file.deleteOnExit();
}catch (Exception e){
e.printStackTrace();
log.warn("multipartFile转File发生异常："+e);
}
}
return file;
}
/**
* 读取CSV格式的文档数据
* @param filePath CSV格式的文件路劲
* @return dataList csv数据读取放入二维list中。
*/
public static List<List<String>> readCSVFileData(String filePath){
BufferedReader reader=null;
List<List<String>> dataList=new ArrayList<>();
try {
reader = new BufferedReader(new InputStreamReader(new FileInputStream(filePath), "UTF-8"));
}catch(FileNotFoundException | UnsupportedEncodingException e){
e.printStackTrace();
}
try{
String line=null;
int lineNum =0;
while ((line=reader.readLine())!=null){
if (lineNum != 0) {
//(1)内容不存在逗号
// String aa[]=line.split(",");
// List<String> cellList= Arrays.asList(aa);
// //System.out.println(cellList);
// dataList.add(cellList);
//(1)内容可能存在逗号，且存在“”英文双引号
String str;
line += ",";
Pattern pCells = Pattern.compile("(\"[^\"]*(\"{2})*[^\"]*\")*[^,]*,");
Matcher mCells = pCells.matcher(line);
List<String> cells = new LinkedList();//每行记录一个list
//读取每个单元格
while (mCells.find()) {
str = mCells.group();
str = str.replaceAll("(?sm)\"?([^\"]*(\"{2})*[^\"]*)\"?.*,", "$1");
str = str.replaceAll("(?sm)(\"(\"))", "$2");
cells.add(str);
}
dataList.add(cells);
}
lineNum++;
}
} catch (Exception e) {
e.printStackTrace();
}
finally {
try {
if (reader != null) {
//释放资源
reader.close();
}
} catch (IOException e) {
e.printStackTrace();
}
}
return dataList;
}
public static CvsDTO csvImportsNew(MultipartFile file) throws IOException {
//2. 进行配置
CsvReadConfig csvReadConfig=new CsvReadConfig();
// 是否跳过空白行
csvReadConfig.setSkipEmptyRows(true);
// 是否设置首行为标题行
csvReadConfig.setContainsHeader(true);
//构建 CsvReader 对象
CsvReader csvReader = CsvUtil.getReader(csvReadConfig);
// 这里转了下 可能会产生临时文件，临时文件目录可以设置，也可以立马删除
CsvData read = csvReader.read(multipartFile2File(file), CharsetUtil.CHARSET_UTF_8);
//CsvData read = csvReader.read(FileUtil.file(file.getOriginalFilename()), Charset.forName("utf-8"));
List<Map<String,Object>> mapList = new ArrayList<>();
List<String> header = read.getHeader(); // TODO: 2024/4/26 获取所有的标题头部信息
List<CsvRow> rows = read.getRows();// TODO: 2024/4/26 获取csv文件每行数据
CvsDTO cvsDTO = new CvsDTO();
cvsDTO.setHeader(header);
cvsDTO.setRows(rows);
return cvsDTO;
}
}

23
DataRoom/dataroom-core/src/main/resources/mapper/NeUploadExcelDao.xml
Unescape Escape View File

@ -9,14 +9,7 @@
step_two_a, step_two_b, step_two_c, step_three_a,
step_three_b, step_three_c, step_three_d, step_four_a,
step_four_b, step_four_c, step_four_d, step_five_a,
step_five_b, step_five_c, step_five_d, step_six_a,
step_six_b, step_six_c, step_six_d, step_seven_a,
step_seven_b, step_seven_c, step_seven_d, step_eight_a,
step_eight_b, step_eight_c, step_eight_d, step_nine_a,
step_nine_b, step_nine_c, success_number, user_id,
sub_state, module, step_nine_d, step_ten_a, step_ten_b,
step_ten_c, step_ten_d, step_eleven_a, step_eleven_b,
step_eleven_c, create_time, update_time, error_field)
step_five_b, success_number, user_id, module)
VALUES
<foreach collection="list" item="item" index="index" separator=",">
(#{item.id,jdbcType=INTEGER}, #{item.stepOneA,jdbcType=VARCHAR}, #{item.stepOneB,jdbcType=VARCHAR},
@ -25,17 +18,9 @@
#{item.stepThreeB,jdbcType=VARCHAR}, #{item.stepThreeC,jdbcType=VARCHAR}, #{item.stepThreeD,jdbcType=VARCHAR},
#{item.stepFourA,jdbcType=VARCHAR}, #{item.stepFourB,jdbcType=VARCHAR}, #{item.stepFourC,jdbcType=VARCHAR},
#{item.stepFourD,jdbcType=VARCHAR}, #{item.stepFiveA,jdbcType=VARCHAR}, #{item.stepFiveB,jdbcType=VARCHAR},
#{item.stepFiveC,jdbcType=VARCHAR}, #{item.stepFiveD,jdbcType=VARCHAR}, #{item.stepSixA,jdbcType=VARCHAR},
#{item.stepSixB,jdbcType=VARCHAR}, #{item.stepSixC,jdbcType=VARCHAR}, #{item.stepSixD,jdbcType=VARCHAR},
#{item.stepSevenA,jdbcType=VARCHAR}, #{item.stepSevenB,jdbcType=VARCHAR}, #{item.stepSevenC,jdbcType=VARCHAR},
#{item.stepSevenD,jdbcType=VARCHAR}, #{item.stepEightA,jdbcType=VARCHAR}, #{item.stepEightB,jdbcType=VARCHAR},
#{item.stepEightC,jdbcType=VARCHAR}, #{item.stepEightD,jdbcType=VARCHAR}, #{item.stepNineA,jdbcType=VARCHAR},
#{item.stepNineB,jdbcType=VARCHAR}, #{item.stepNineC,jdbcType=VARCHAR}, #{item.successNumber,jdbcType=INTEGER},
#{item.userId,jdbcType=VARCHAR}, #{item.subState,jdbcType=INTEGER}, #{item.module,jdbcType=VARCHAR},
#{item.stepNineD,jdbcType=VARCHAR}, #{item.stepTenA,jdbcType=VARCHAR}, #{item.stepTenB,jdbcType=VARCHAR},
#{item.stepTenC,jdbcType=VARCHAR}, #{item.stepTenD,jdbcType=VARCHAR}, #{item.stepElevenA,jdbcType=VARCHAR},
#{item.stepElevenB,jdbcType=VARCHAR}, #{item.stepElevenC,jdbcType=VARCHAR}, #{item.createTime,jdbcType=TIMESTAMP},
#{item.updateTime,jdbcType=TIMESTAMP}, #{item.errorField,jdbcType=VARCHAR})
#{item.successNumber,jdbcType=INTEGER}, #{item.userId,jdbcType=VARCHAR}, #{item.module,jdbcType=VARCHAR})
</foreach>
</insert>
</mapper>
Write Preview
Loading…
Cancel
Save