From 3fb29b06edafabd12527fa5ccb1ffdf03e9b678a Mon Sep 17 00:00:00 2001
From: whb <17803890193@163.com>
Date: Fri, 6 Sep 2024 09:23:15 +0800
Subject: [PATCH] =?UTF-8?q?=E6=B7=BB=E5=8A=A0=E6=89=B9=E9=87=8F=E5=AF=BC?=
=?UTF-8?q?=E5=85=A5csv=E6=8E=A5=E5=8F=A3?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
---
DataRoom/dataroom-core/pom.xml | 5 +
.../controller/NeUploadExcelController.java | 41 ++-
.../controller/NewCategoryController.java | 2 +
.../core/module/manage/dto/CvsDTO.java | 17 ++
.../manage/service/NewExcelService.java | 21 ++
.../service/impl/NewExcelServiceImpl.java | 165 ++++++++++++
.../dataroom/core/utils/CsvImportUtil.java | 254 ++++++++++++++++++
.../resources/mapper/NeUploadExcelDao.xml | 23 +-
8 files changed, 501 insertions(+), 27 deletions(-)
create mode 100644 DataRoom/dataroom-core/src/main/java/com/gccloud/dataroom/core/module/manage/dto/CvsDTO.java
create mode 100644 DataRoom/dataroom-core/src/main/java/com/gccloud/dataroom/core/module/manage/service/NewExcelService.java
create mode 100644 DataRoom/dataroom-core/src/main/java/com/gccloud/dataroom/core/module/manage/service/impl/NewExcelServiceImpl.java
create mode 100644 DataRoom/dataroom-core/src/main/java/com/gccloud/dataroom/core/utils/CsvImportUtil.java
diff --git a/DataRoom/dataroom-core/pom.xml b/DataRoom/dataroom-core/pom.xml
index 5d63b62c..447dc2d1 100644
--- a/DataRoom/dataroom-core/pom.xml
+++ b/DataRoom/dataroom-core/pom.xml
@@ -95,6 +95,11 @@
easyexcel
3.2.1
+
+ cn.hutool
+ hutool-all
+ 5.8.27
+
com.github.stuxuhai
diff --git a/DataRoom/dataroom-core/src/main/java/com/gccloud/dataroom/core/module/manage/controller/NeUploadExcelController.java b/DataRoom/dataroom-core/src/main/java/com/gccloud/dataroom/core/module/manage/controller/NeUploadExcelController.java
index f72c2d4b..122f5fe3 100644
--- a/DataRoom/dataroom-core/src/main/java/com/gccloud/dataroom/core/module/manage/controller/NeUploadExcelController.java
+++ b/DataRoom/dataroom-core/src/main/java/com/gccloud/dataroom/core/module/manage/controller/NeUploadExcelController.java
@@ -4,12 +4,16 @@ import com.alibaba.excel.EasyExcel;
import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
import com.baomidou.mybatisplus.core.metadata.IPage;
import com.baomidou.mybatisplus.extension.plugins.pagination.Page;
+import com.gccloud.common.exception.GlobalException;
import com.gccloud.common.permission.ApiPermission;
import com.gccloud.common.utils.JSON;
import com.gccloud.common.vo.R;
import com.gccloud.dataroom.core.module.manage.dao.NeUploadExcelDao;
+import com.gccloud.dataroom.core.module.manage.dto.CvsDTO;
import com.gccloud.dataroom.core.module.manage.dto.DatasetEntityDTO;
import com.gccloud.dataroom.core.module.manage.entity.StuUploadExcelUser;
+import com.gccloud.dataroom.core.module.manage.service.NewExcelService;
+import com.gccloud.dataroom.core.utils.CsvImportUtil;
import com.gccloud.dataroom.core.utils.DemoDataListener;
import com.gccloud.dataset.constant.DatasetConstant;
import com.gccloud.dataset.dto.DatasetDTO;
@@ -19,15 +23,19 @@ import com.gccloud.dataset.service.IBaseDataSetService;
import com.gccloud.dataset.service.IDatasetLabelService;
import io.swagger.annotations.Api;
import io.swagger.annotations.ApiOperation;
+import net.sourceforge.pinyin4j.PinyinHelper;
import org.springframework.beans.BeanUtils;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.transaction.annotation.Transactional;
+import org.springframework.util.CollectionUtils;
import org.springframework.web.bind.annotation.*;
import org.springframework.web.multipart.MultipartFile;
import javax.annotation.Resource;
import java.io.IOException;
+import java.util.ArrayList;
import java.util.List;
+import java.util.Map;
import java.util.UUID;
/**
@@ -39,7 +47,8 @@ import java.util.UUID;
@RequestMapping("/newDataset")
public class NeUploadExcelController {
-
+ @Autowired
+ private NewExcelService service;
@Resource
private IDatasetLabelService datasetLabelService;
@@ -57,20 +66,33 @@ public class NeUploadExcelController {
@Transactional(rollbackFor = Exception.class)
public R uploadExcel(@RequestParam("file") @RequestPart MultipartFile file,String userId) throws IOException {
+ // file.getOriginalFilename().lastIndexOf(".")+1
+ String substring = file.getOriginalFilename().substring(file.getOriginalFilename().lastIndexOf(".") + 1);
+ if (substring.equals("csv")){
+ return service.uploadCVS(file,userId);
+ }else if ("xls".equals(substring)|| "xlsx".equals(substring)){
+
+ //存储excel数据 根据id关联
+ String s = UUID.randomUUID().toString().replaceAll("-", "");
+
+ DemoDataListener demoDataListener = new DemoDataListener(neUploadExcelDao, userId, s);
+
+ // 这里 需要指定读用哪个class去读,然后读取第一个sheet 文件流会自动关闭
+ EasyExcel.read(file.getInputStream(), StuUploadExcelUser.class,
+ new DemoDataListener(neUploadExcelDao, userId, s)).sheet().doRead();
- //存储excel数据 根据id关联
- String s = UUID.randomUUID().toString().replaceAll("-", "");
- DemoDataListener demoDataListener = new DemoDataListener(neUploadExcelDao, userId, s);
- // 这里 需要指定读用哪个class去读,然后读取第一个sheet 文件流会自动关闭
- EasyExcel.read(file.getInputStream(), StuUploadExcelUser.class,
- new DemoDataListener(neUploadExcelDao, userId, s)).sheet().doRead();
+ return R.success(s);
+ }else {
+
+
+ throw new GlobalException("请导入正确格式的文件");
+ }
- return R.success(s);
}
@ApiOperation("分页查询excel")
@@ -125,4 +147,7 @@ public class NeUploadExcelController {
}
+
+
+
}
diff --git a/DataRoom/dataroom-core/src/main/java/com/gccloud/dataroom/core/module/manage/controller/NewCategoryController.java b/DataRoom/dataroom-core/src/main/java/com/gccloud/dataroom/core/module/manage/controller/NewCategoryController.java
index 08b26392..13951fb7 100644
--- a/DataRoom/dataroom-core/src/main/java/com/gccloud/dataroom/core/module/manage/controller/NewCategoryController.java
+++ b/DataRoom/dataroom-core/src/main/java/com/gccloud/dataroom/core/module/manage/controller/NewCategoryController.java
@@ -94,4 +94,6 @@ public class NewCategoryController {
+
+
}
diff --git a/DataRoom/dataroom-core/src/main/java/com/gccloud/dataroom/core/module/manage/dto/CvsDTO.java b/DataRoom/dataroom-core/src/main/java/com/gccloud/dataroom/core/module/manage/dto/CvsDTO.java
new file mode 100644
index 00000000..58367da9
--- /dev/null
+++ b/DataRoom/dataroom-core/src/main/java/com/gccloud/dataroom/core/module/manage/dto/CvsDTO.java
@@ -0,0 +1,17 @@
+package com.gccloud.dataroom.core.module.manage.dto;
+
+import cn.hutool.core.text.csv.CsvRow;
+import lombok.Data;
+
+import java.util.List;
+
+/**
+ * @author 17803
+ * @date 2024-09-05 17:11
+ */
+@Data
+public class CvsDTO {
+
+ private List header;
+ private List rows ;
+}
diff --git a/DataRoom/dataroom-core/src/main/java/com/gccloud/dataroom/core/module/manage/service/NewExcelService.java b/DataRoom/dataroom-core/src/main/java/com/gccloud/dataroom/core/module/manage/service/NewExcelService.java
new file mode 100644
index 00000000..92412fd3
--- /dev/null
+++ b/DataRoom/dataroom-core/src/main/java/com/gccloud/dataroom/core/module/manage/service/NewExcelService.java
@@ -0,0 +1,21 @@
+package com.gccloud.dataroom.core.module.manage.service;
+
+
+import com.gccloud.common.vo.R;
+import com.gccloud.dataroom.core.module.manage.dto.CategorySearchDTO;
+import com.gccloud.dataroom.core.module.manage.entity.CategoryEntity;
+import com.gccloud.dataset.vo.CategoryVO;
+import org.springframework.web.multipart.MultipartFile;
+
+import java.util.List;
+
+/**
+ * @author 17803
+ * @date 2024-09-02 16:22
+ */
+
+public interface NewExcelService {
+
+
+ R uploadCVS(MultipartFile file, String userId);
+}
diff --git a/DataRoom/dataroom-core/src/main/java/com/gccloud/dataroom/core/module/manage/service/impl/NewExcelServiceImpl.java b/DataRoom/dataroom-core/src/main/java/com/gccloud/dataroom/core/module/manage/service/impl/NewExcelServiceImpl.java
new file mode 100644
index 00000000..595b999e
--- /dev/null
+++ b/DataRoom/dataroom-core/src/main/java/com/gccloud/dataroom/core/module/manage/service/impl/NewExcelServiceImpl.java
@@ -0,0 +1,165 @@
+package com.gccloud.dataroom.core.module.manage.service.impl;
+
+import cn.hutool.core.text.csv.CsvRow;
+import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
+import com.baomidou.mybatisplus.extension.service.impl.ServiceImpl;
+import com.gccloud.common.exception.GlobalException;
+import com.gccloud.common.utils.BeanConvertUtils;
+import com.gccloud.common.utils.JSON;
+import com.gccloud.common.vo.R;
+import com.gccloud.dataroom.core.module.basic.dao.DataRoomPagePreviewDao;
+import com.gccloud.dataroom.core.module.basic.entity.PagePreviewEntity;
+import com.gccloud.dataroom.core.module.manage.dao.NeUploadExcelDao;
+import com.gccloud.dataroom.core.module.manage.dto.CvsDTO;
+import com.gccloud.dataroom.core.module.manage.dto.DataRoomPageDTO;
+import com.gccloud.dataroom.core.module.manage.entity.StuUploadExcelUser;
+import com.gccloud.dataroom.core.module.manage.service.IDataRoomPagePreviewService;
+import com.gccloud.dataroom.core.module.manage.service.NewExcelService;
+import com.gccloud.dataroom.core.utils.CsvImportUtil;
+import net.sourceforge.pinyin4j.PinyinHelper;
+import org.springframework.stereotype.Service;
+import org.springframework.util.CollectionUtils;
+import org.springframework.web.multipart.MultipartFile;
+
+import java.io.IOException;
+import java.lang.reflect.Field;
+import java.util.ArrayList;
+import java.util.Date;
+import java.util.List;
+import java.util.UUID;
+
+/**
+ * @author hongyang
+ * @version 1.0
+ * @date 2023/9/13 10:11
+ */
+@Service
+public class NewExcelServiceImpl extends ServiceImpl implements NewExcelService {
+
+
+ @Override
+ public R uploadCVS(MultipartFile file, String userId) {
+ int suff = file.getOriginalFilename().lastIndexOf(".");
+ String substring = file.getOriginalFilename().substring(suff+1);
+ //存储excel数据 根据id关联
+ String s = UUID.randomUUID().toString().replaceAll("-", "");
+
+ // 存储转换后的拼音列表
+ List pinyinList = new ArrayList<>();
+ if ("csv".equals(substring))
+ {
+ List excelUsers=new ArrayList<>();
+
+ CvsDTO cvsDTO= null;
+ try {
+ cvsDTO = CsvImportUtil.csvImportsNew(file);
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+
+ // 遍历list中的每个字符串
+ for (String chinese : cvsDTO.getHeader()) {
+ StringBuilder pinyin = new StringBuilder();
+
+ // 遍历字符串中的每个字符
+ for (char c : chinese.toCharArray()) {
+ if (Character.toString(c).matches("[\\u4E00-\\u9FA5]")) {
+ // 获取拼音
+ String[] pinyinArray = PinyinHelper.toHanyuPinyinStringArray(c);
+ if (pinyinArray != null) {
+ // 取第一个拼音
+ pinyin.append(pinyinArray[0].replaceAll("[1234]", ""));
+ }
+ } else {
+ // 非中文字符原样添加
+ pinyin.append(c);
+ }
+ }
+
+ // 将拼音结果添加到pinyinList
+ pinyinList.add(pinyin.toString().trim());
+ }
+
+
+
+ StuUploadExcelUser stuUploadExcelUsers = new StuUploadExcelUser();
+ stuUploadExcelUsers.setModule(s);
+ stuUploadExcelUsers.setUserId(userId);
+ if (CollectionUtils.isEmpty(pinyinList))
+ {
+ stuUploadExcelUsers.setStepOneA(JSON.toJSONString(cvsDTO.getHeader()));
+ }else {
+ stuUploadExcelUsers.setStepOneA(JSON.toJSONString(pinyinList));
+ }
+ // 获取 StuUploadExcelUser 的所有字段
+ Field[] fields = StuUploadExcelUser.class.getDeclaredFields();
+ int fieldIndex = 2;
+ int count = 0;
+ // 遍历 CsvRow
+ for (CsvRow row : cvsDTO.getRows()) {
+
+ StuUploadExcelUser stuUploadExcelUser = new StuUploadExcelUser();
+ fieldIndex = 2;
+ if (count >=1000)
+ {
+ break;
+ }
+ for (int j = 0; j < row.size(); j++) {
+
+ if (fieldIndex >= fields.length) {
+ // 如果字段超出范围,跳出循环
+ break;
+ }
+
+ try {
+ // 获取字段
+ Field field = fields[fieldIndex];
+ field.setAccessible(true); // 设置字段为可访问
+ // 获取 row 中的值
+ Object value = row.get(j);
+ // 根据字段类型进行赋值
+ if (field.getType().equals(String.class)) {
+ field.set(stuUploadExcelUser, value != null ? value.toString() : null);
+ }
+ else {
+ field.set(stuUploadExcelUser, value != null ? value.toString() : null);
+ }
+
+
+ if (row.size() == j)
+ {
+ fieldIndex = 2;
+ break;
+
+ }else {
+ fieldIndex++;
+ }
+
+
+ } catch (IllegalAccessException e) {
+ e.printStackTrace();
+ }
+
+ }
+ count++;
+ excelUsers.add(stuUploadExcelUser);
+
+ }
+
+
+
+ excelUsers.stream().forEach(item->{
+
+ item.setModule(s);
+ item.setStepOneA(JSON.toJSONString(pinyinList));
+ item.setUserId(userId);
+
+ });
+
+ this.saveBatch(excelUsers);
+ }
+
+
+ return R.success(s);
+ }
+}
diff --git a/DataRoom/dataroom-core/src/main/java/com/gccloud/dataroom/core/utils/CsvImportUtil.java b/DataRoom/dataroom-core/src/main/java/com/gccloud/dataroom/core/utils/CsvImportUtil.java
new file mode 100644
index 00000000..24cd823d
--- /dev/null
+++ b/DataRoom/dataroom-core/src/main/java/com/gccloud/dataroom/core/utils/CsvImportUtil.java
@@ -0,0 +1,254 @@
+package com.gccloud.dataroom.core.utils;
+
+import cn.hutool.core.io.FileUtil;
+import cn.hutool.core.text.csv.*;
+import cn.hutool.core.util.CharsetUtil;
+import com.baomidou.mybatisplus.core.metadata.TableInfo;
+import com.baomidou.mybatisplus.core.toolkit.ObjectUtils;
+import com.gccloud.dataroom.core.module.manage.dto.CvsDTO;
+import lombok.extern.slf4j.Slf4j;
+import org.springframework.web.multipart.MultipartFile;
+import org.apache.commons.csv.CSVFormat;
+import org.apache.commons.csv.CSVParser;
+import org.apache.commons.csv.CSVRecord;
+import java.io.*;
+import java.net.URL;
+import java.nio.charset.Charset;
+import java.util.*;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+@Slf4j
+public class CsvImportUtil {
+ //上传文件的路径
+ private final static URL PATH = Thread.currentThread().getContextClassLoader().getResource("");
+
+
+ /**
+ * @return File 一般文件类型
+ * @Description 上传文件的文件类型
+ * @Param multipartFile
+ **/
+ public static File uploadFile(MultipartFile multipartFile) {
+ // 获 取上传 路径
+ String path = PATH.getPath() + multipartFile.getOriginalFilename();
+ try {
+ // 通过将给定的路径名字符串转换为抽象路径名来创建新的 File实例
+ File file = new File(path);
+ // 此抽象路径名表示的文件或目录是否存在
+ if (!file.getParentFile().exists()) {
+ // 创建由此抽象路径名命名的目录,包括任何必需但不存在的父目录
+ file.getParentFile().mkdirs();
+ }
+ // 转换为一般file 文件
+ multipartFile.transferTo(file);
+
+ return file;
+ } catch (IOException e) {
+
+ e.printStackTrace();
+ return null;
+ }
+
+ }
+
+ /**
+ * @return List>
+ * @Description 读取CSV文件的内容(不含表头)
+ * @Param filePath 文件存储路径,colNum 列数
+ **/
+ public static List> readCSV(String filePath, int colNum) {
+ BufferedReader bufferedReader = null;
+ InputStreamReader inputStreamReader = null;
+ FileInputStream fileInputStream = null;
+ CSVFormat csvFileFormat = CSVFormat.DEFAULT.withQuote(null);
+ try {
+ fileInputStream = new FileInputStream(filePath);
+ inputStreamReader = new InputStreamReader(fileInputStream, "utf-8");
+ bufferedReader = new BufferedReader(inputStreamReader);
+ // CSVParser parser = CSVFormat.DEFAULT.parse(bufferedReader);
+ CSVParser csvFileParser = new CSVParser(inputStreamReader, csvFileFormat);
+
+ // 表内容集合,外层 List为行的集合,内层 List为字段集合
+ List> values = new ArrayList<>();
+
+ int rowIndex = 0;
+ // 读取文件每行内容
+
+ for (CSVRecord record : csvFileParser.getRecords()) {
+ // 跳过表头
+ if (rowIndex == 0) {
+ rowIndex++;
+ continue;
+ }
+ // 判断下角标是否越界
+ if (colNum > record.size()) {
+ // 返回空集合
+ return values;
+ }
+ // 每行的内容
+ List value = new ArrayList<>();
+ for (int i = 0; i < colNum; i++) {
+ value.add(record.get(i));
+ }
+ values.add(value);
+ rowIndex++;
+ }
+ return values;
+ } catch (IOException e) {
+ e.printStackTrace();
+
+ } finally {
+ //关闭流
+ if (bufferedReader != null) {
+ try {
+ bufferedReader.close();
+ } catch (IOException e) {
+ e.printStackTrace();
+ }
+ }
+ if (inputStreamReader != null) {
+ try {
+ inputStreamReader.close();
+ } catch (IOException e) {
+ e.printStackTrace();
+ }
+ }
+ if (fileInputStream != null) {
+ try {
+ fileInputStream.close();
+ } catch (IOException e) {
+ e.printStackTrace();
+ }
+ }
+ }
+ return null;
+ }
+
+ // 读取csv中的数据
+ public static List