diff --git a/DataRoom/dataroom-core/pom.xml b/DataRoom/dataroom-core/pom.xml index e46d289d..0f1a7db1 100644 --- a/DataRoom/dataroom-core/pom.xml +++ b/DataRoom/dataroom-core/pom.xml @@ -72,5 +72,24 @@ minio ${minio.version} + + + commons-net + commons-net + ${commons-net.version} + + + + org.apache.commons + commons-dbcp2 + ${commons-dbcp2.version} + + + + com.jcraft + jsch + ${jsch.version} + + diff --git a/DataRoom/dataroom-core/src/main/java/com/gccloud/dataroom/core/config/MinioConfig.java b/DataRoom/dataroom-core/src/main/java/com/gccloud/dataroom/core/config/MinioConfig.java deleted file mode 100644 index ec3ebd6d..00000000 --- a/DataRoom/dataroom-core/src/main/java/com/gccloud/dataroom/core/config/MinioConfig.java +++ /dev/null @@ -1,94 +0,0 @@ -package com.gccloud.dataroom.core.config; - -import io.minio.MinioClient; -import org.apache.commons.lang3.StringUtils; -import org.springframework.boot.context.properties.ConfigurationProperties; -import org.springframework.context.annotation.Bean; -import org.springframework.context.annotation.Configuration; - -/** - * Minio 配置信息 - * - * @author Acechengui - */ -@Configuration -@ConfigurationProperties(prefix = "minio") -public class MinioConfig -{ - /** - * 服务地址 - */ - private String url; - - /** - * 用户名 - */ - private String accessKey; - - /** - * 密码 - */ - private String secretKey; - - /** - * 存储桶名称 - */ - private String bucketName; - - public String getUrl() - { - return url; - } - - public void setUrl(String url) - { - this.url = url; - } - - public String getAccessKey() - { - return accessKey; - } - - public void setAccessKey(String accessKey) - { - this.accessKey = accessKey; - } - - public String getSecretKey() - { - return secretKey; - } - - public void setSecretKey(String secretKey) - { - this.secretKey = secretKey; - } - - public String getBucketName() - { - return bucketName; - } - - public void setBucketName(String bucketName) - { - this.bucketName = bucketName; - } - - @Bean - public MinioClient getMinioClient() { - if (StringUtils.isEmpty(url) || StringUtils.isEmpty(accessKey) || StringUtils.isEmpty(secretKey)) { - // 如果未配置Minio相关的配置项，则使用本地文件存储 - // 或者返回一个默认的MinioClient实例，用于本地文件存储 - return createDefaultMinioClient(); - } - return MinioClient.builder().endpoint(url).credentials(accessKey, secretKey).build(); - } - - private MinioClient createDefaultMinioClient() { - return MinioClient.builder() - .endpoint("http://minio.example.com") - .credentials("accessKey", "secretKey") - .build(); - } -} diff --git a/DataRoom/dataroom-core/src/main/java/com/gccloud/dataroom/core/config/bean/FileConfig.java b/DataRoom/dataroom-core/src/main/java/com/gccloud/dataroom/core/config/bean/FileConfig.java index 1ab44833..b4fa1a54 100644 --- a/DataRoom/dataroom-core/src/main/java/com/gccloud/dataroom/core/config/bean/FileConfig.java +++ b/DataRoom/dataroom-core/src/main/java/com/gccloud/dataroom/core/config/bean/FileConfig.java @@ -30,4 +30,19 @@ public class FileConfig { "mp4", "mov", "mp3", "rar", "zip" ); + + /** + * ftp配置 + */ + private FtpConfig ftp; + + /** + * sftp配置 + */ + private SftpConfig sftp; + + /** + * minio配置 + */ + private MinioConfig minio; } diff --git a/DataRoom/dataroom-core/src/main/java/com/gccloud/dataroom/core/config/bean/FtpConfig.java b/DataRoom/dataroom-core/src/main/java/com/gccloud/dataroom/core/config/bean/FtpConfig.java new file mode 100644 index 00000000..c4af9c55 --- /dev/null +++ b/DataRoom/dataroom-core/src/main/java/com/gccloud/dataroom/core/config/bean/FtpConfig.java @@ -0,0 +1,108 @@ +package com.gccloud.dataroom.core.config.bean; + +import lombok.Data; +import org.apache.commons.pool2.impl.GenericObjectPoolConfig; +import org.springframework.boot.context.properties.ConfigurationProperties; +import org.springframework.context.annotation.Configuration; + +/** + * @author hongyang + * @version 1.0 + * @date 2023/10/17 15:18 + */ +@Data +@Configuration +@ConfigurationProperties(prefix = "gc.starter.file.ftp") +public class FtpConfig extends GenericObjectPoolConfig { + + /** + * ftp服务器地址 + */ + private String host; + + /** + * ftp服务器端口 + */ + private Integer port; + + /** + * ftp服务器用户名 + */ + private String username; + + /** + * ftp服务器密码 + */ + private String password; + + /** + * 传输编码 + */ + String encoding = "utf-8"; + /** + * 被动模式：在这种模式下，数据连接是由客户程序发起的 + */ + boolean passiveMode = true; + /** + * 连接超时时间 + */ + int clientTimeout = 30000; + + /** + * 0=ASCII_FILE_TYPE(ASCII格式)，1=EBCDIC_FILE_TYPE，2=LOCAL_FILE_TYPE(二进制文件) + */ + int transferFileType = 2; + /** + * 重新连接时间 + */ + int retryTimes; + /** + * 缓存大小 + */ + int bufferSize = 1024; + + /* 连接池配置 */ + + /** + * 最大连接数 + */ + int maxTotal = DEFAULT_MAX_TOTAL; + /** + * 最小空闲 + */ + int minIdle = DEFAULT_MIN_IDLE; + /** + * 最大空闲 + */ + int maxIdle = DEFAULT_MAX_IDLE; + /** + * 最大等待时间 10s + */ + int maxWait = 10000; + /** + * 池对象耗尽之后是否阻塞，maxWait < 0 时一直等待 + */ + boolean blockWhenExhausted = DEFAULT_BLOCK_WHEN_EXHAUSTED; + /** + * 取对象时验证 + */ + boolean testOnBorrow = true ; + /** + * 回收验证 + */ + boolean testOnReturn = true; + /** + * 创建时验证 + */ + boolean testOnCreate = true; + /** + * 空闲验证 + */ + boolean testWhileIdle = true; + /** + * 后进先出 + */ + boolean lifo = DEFAULT_LIFO; + + +} diff --git a/DataRoom/dataroom-core/src/main/java/com/gccloud/dataroom/core/config/bean/MinioConfig.java b/DataRoom/dataroom-core/src/main/java/com/gccloud/dataroom/core/config/bean/MinioConfig.java new file mode 100644 index 00000000..bebb10eb --- /dev/null +++ b/DataRoom/dataroom-core/src/main/java/com/gccloud/dataroom/core/config/bean/MinioConfig.java @@ -0,0 +1,50 @@ +package com.gccloud.dataroom.core.config.bean; + +import com.gccloud.common.exception.GlobalException; +import io.minio.MinioClient; +import lombok.Data; +import org.apache.commons.lang3.StringUtils; +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.boot.context.properties.ConfigurationProperties; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; + +/** + * Minio 配置信息 + * + * @author Acechengui + */ +@Data +@Configuration +@ConfigurationProperties(prefix = "gc.starter.file.minio") +public class MinioConfig +{ + /** + * 服务地址 + */ + private String url; + + /** + * 用户名 + */ + private String accessKey; + + /** + * 密码 + */ + private String secretKey; + + /** + * 存储桶名称 + */ + private String bucketName; + + @Bean + @ConditionalOnProperty(prefix = "gc.starter.file", name = "type", havingValue = "minio") + public MinioClient getMinioClient() { + if (StringUtils.isBlank(bucketName)) { + throw new GlobalException("Minio bucketName 不能为空"); + } + return MinioClient.builder().endpoint(url).credentials(accessKey, secretKey).build(); + } +} diff --git a/DataRoom/dataroom-core/src/main/java/com/gccloud/dataroom/core/config/bean/SftpConfig.java b/DataRoom/dataroom-core/src/main/java/com/gccloud/dataroom/core/config/bean/SftpConfig.java new file mode 100644 index 00000000..6f0220e9 --- /dev/null +++ b/DataRoom/dataroom-core/src/main/java/com/gccloud/dataroom/core/config/bean/SftpConfig.java @@ -0,0 +1,116 @@ +package com.gccloud.dataroom.core.config.bean; + +import lombok.Data; +import org.apache.commons.pool2.impl.GenericObjectPoolConfig; +import org.springframework.boot.context.properties.ConfigurationProperties; +import org.springframework.context.annotation.Configuration; + +/** + * @author hongyang + * @version 1.0 + * @date 2023/10/17 15:18 + */ +@Configuration +@ConfigurationProperties(prefix = "gc.starter.file.sftp") +@Data +public class SftpConfig extends GenericObjectPoolConfig { + + /** + * ftp服务器地址 + */ + private String host; + + /** + * ftp服务器端口 + */ + private Integer port; + + /** + * ftp服务器用户名 + */ + private String username; + + /** + * ftp服务器密码 + */ + private String password; + + /** + * 私钥 + */ + private String privateKey; + + /** + * 传输编码 + */ + String encoding = "utf-8"; + /** + * 被动模式：在这种模式下，数据连接是由客户程序发起的 + */ + boolean passiveMode = true; + /** + * 连接超时时间 + */ + int clientTimeout = 30000; + + /** + * 0=ASCII_FILE_TYPE(ASCII格式)，1=EBCDIC_FILE_TYPE，2=LOCAL_FILE_TYPE(二进制文件) + */ + int transferFileType = 2; + /** + * 重新连接时间 + */ + int retryTimes; + + /** + * 缓存大小 + */ + int bufferSize = 1024; + + + /* 连接池配置 */ + + + /** + * 最大连接数 + */ + int maxTotal = DEFAULT_MAX_TOTAL; + /** + * 最小空闲 + */ + int minIdle = DEFAULT_MIN_IDLE; + /** + * 最大空闲 + */ + int maxIdle = DEFAULT_MAX_IDLE; + /** + * 最大等待时间 10s + */ + int maxWait = 10000; + /** + * 池对象耗尽之后是否阻塞，maxWait < 0 时一直等待 + */ + boolean blockWhenExhausted = DEFAULT_BLOCK_WHEN_EXHAUSTED; + /** + * 取对象时验证 + */ + boolean testOnBorrow = true ; + /** + * 回收验证 + */ + boolean testOnReturn = true; + /** + * 创建时验证 + */ + boolean testOnCreate = true; + /** + * 空闲验证 + */ + boolean testWhileIdle = true; + /** + * 后进先出 + */ + boolean lifo = DEFAULT_LIFO; + + +} diff --git a/DataRoom/dataroom-core/src/main/java/com/gccloud/dataroom/core/module/biz/component/service/impl/BizComponentServiceImpl.java b/DataRoom/dataroom-core/src/main/java/com/gccloud/dataroom/core/module/biz/component/service/impl/BizComponentServiceImpl.java index 4d856111..4fd0b0e7 100644 --- a/DataRoom/dataroom-core/src/main/java/com/gccloud/dataroom/core/module/biz/component/service/impl/BizComponentServiceImpl.java +++ b/DataRoom/dataroom-core/src/main/java/com/gccloud/dataroom/core/module/biz/component/service/impl/BizComponentServiceImpl.java @@ -7,9 +7,12 @@ import com.gccloud.dataroom.core.module.biz.component.dao.DataRoomBizComponentDa import com.gccloud.dataroom.core.module.biz.component.dto.BizComponentSearchDTO; import com.gccloud.dataroom.core.module.biz.component.entity.BizComponentEntity; import com.gccloud.dataroom.core.module.biz.component.service.IBizComponentService; +import com.gccloud.dataroom.core.module.file.entity.DataRoomFileEntity; +import com.gccloud.dataroom.core.module.file.service.IDataRoomOssService; import com.gccloud.dataroom.core.utils.CodeGenerateUtils; import com.gccloud.common.exception.GlobalException; import com.gccloud.common.vo.PageVO; +import com.gccloud.dataroom.core.utils.PathUtils; import lombok.extern.slf4j.Slf4j; import org.apache.commons.io.FileUtils; import org.apache.commons.lang3.StringUtils; @@ -17,9 +20,7 @@ import org.apache.commons.lang3.exception.ExceptionUtils; import org.springframework.stereotype.Service; import javax.annotation.Resource; -import java.io.File; -import java.io.FileOutputStream; -import java.io.IOException; +import java.io.*; import java.util.Base64; import java.util.List; @@ -35,6 +36,9 @@ public class BizComponentServiceImpl extends ServiceImpl getPage(BizComponentSearchDTO searchDTO) { LambdaQueryWrapper queryWrapper = new LambdaQueryWrapper<>(); @@ -93,10 +97,14 @@ public class BizComponentServiceImpl extends ServiceImpl 0) { + outputStream.write(buffer, 0, length); + } + outputStream.close(); + inputStream.close(); + } catch (Exception e) { + log.error(ExceptionUtils.getStackTrace(e)); + log.error(String.format("文件 %s 存储到 %s 失败", fileName, destPath)); + throw new GlobalException("文件上传失败"); + } + fileEntity.setOriginalName(fileName); + fileEntity.setNewName(fileName); + fileEntity.setPath(basePath); + fileEntity.setSize(size); + fileEntity.setExtension(extension); + fileEntity.setUrl("/" + fileName); + return fileEntity; + } + @Override public void download(String fileId, HttpServletResponse response, HttpServletRequest request) { DataRoomFileEntity fileEntity = sysFileService.getById(fileId); @@ -131,4 +164,29 @@ public class DataRoomLocalFileServiceImpl implements IDataRoomOssService { } file.delete(); } + + @Override + public String copy(String sourcePath, String targetPath) { + String basePath = bigScreenConfig.getFile().getBasePath() + File.separator; + File sourceFile = new File(basePath + sourcePath); + File targetFile = new File(basePath + targetPath); + // 检查源文件是否存在 + if (!sourceFile.exists()) { + log.error("复制源文件不存在:{}", sourcePath); + return ""; + } + // 检查源文件是否是文件夹 + if (sourceFile.isDirectory()) { + log.error("源文件为文件夹:{}，无法复制", sourcePath); + return ""; + } + try { + FileUtils.copyFile(sourceFile, targetFile); + } catch (IOException e) { + log.error(String.format("文件 %s 复制到 %s 失败", sourcePath, targetPath)); + log.error(ExceptionUtils.getStackTrace(e)); + return ""; + } + return targetPath; + } } diff --git a/DataRoom/dataroom-core/src/main/java/com/gccloud/dataroom/core/module/file/service/impl/DataRoomMinioServiceImpl.java b/DataRoom/dataroom-core/src/main/java/com/gccloud/dataroom/core/module/file/service/impl/DataRoomMinioServiceImpl.java index a3f4200d..6dd9185e 100644 --- a/DataRoom/dataroom-core/src/main/java/com/gccloud/dataroom/core/module/file/service/impl/DataRoomMinioServiceImpl.java +++ b/DataRoom/dataroom-core/src/main/java/com/gccloud/dataroom/core/module/file/service/impl/DataRoomMinioServiceImpl.java @@ -3,21 +3,20 @@ package com.gccloud.dataroom.core.module.file.service.impl; import com.baomidou.mybatisplus.core.toolkit.IdWorker; import com.gccloud.common.exception.GlobalException; import com.gccloud.dataroom.core.config.DataRoomConfig; -import com.gccloud.dataroom.core.config.MinioConfig; import com.gccloud.dataroom.core.config.bean.FileConfig; +import com.gccloud.dataroom.core.config.bean.MinioConfig; import com.gccloud.dataroom.core.module.file.entity.DataRoomFileEntity; -import com.gccloud.dataroom.core.module.file.service.FileOperationStrategy; import com.gccloud.dataroom.core.module.file.service.IDataRoomFileService; import com.gccloud.dataroom.core.module.file.service.IDataRoomOssService; import com.gccloud.dataroom.core.utils.MinioFileInterface; -import io.minio.MinioClient; -import io.minio.PutObjectArgs; -import lombok.SneakyThrows; +import com.gccloud.dataroom.core.utils.PathUtils; +import io.minio.*; import lombok.extern.slf4j.Slf4j; import org.apache.commons.io.FilenameUtils; import org.apache.commons.io.IOUtils; import org.apache.commons.lang3.StringUtils; -import org.springframework.context.annotation.Conditional; +import org.apache.commons.lang3.exception.ExceptionUtils; +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; import org.springframework.http.HttpStatus; import org.springframework.stereotype.Service; import org.springframework.web.multipart.MultipartFile; @@ -26,6 +25,7 @@ import javax.annotation.Resource; import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletResponse; import java.io.InputStream; +import java.io.UnsupportedEncodingException; import java.net.URLEncoder; import java.time.LocalDate; import java.time.format.DateTimeFormatter; @@ -35,14 +35,11 @@ import java.time.format.DateTimeFormatter; * * @author Acechengui */ -@Service("minioFileService") -@Conditional(FileOperationStrategy.MinioFileCondition.class) @Slf4j +@Service +@ConditionalOnProperty(prefix = "gc.starter.file", name = "type", havingValue = "minio") public class DataRoomMinioServiceImpl implements IDataRoomOssService { - @Resource - private MinioConfig minioConfig; - @Resource private MinioClient minioclient; @@ -51,14 +48,13 @@ public class DataRoomMinioServiceImpl implements IDataRoomOssService { @Resource private IDataRoomFileService sysFileService; + @Resource private MinioFileInterface minioFileInterface; /** * 上传文件 - * */ - @SneakyThrows @Override public DataRoomFileEntity upload(MultipartFile file, DataRoomFileEntity fileEntity, HttpServletResponse response, HttpServletRequest request) { String originalFilename = file.getOriginalFilename(); @@ -66,7 +62,7 @@ public class DataRoomMinioServiceImpl implements IDataRoomOssService { String extension = FilenameUtils.getExtension(originalFilename); FileConfig fileConfig = bigScreenConfig.getFile(); if (!fileConfig.getAllowedFileExtensionName().contains("*") && !fileConfig.getAllowedFileExtensionName().contains(extension)) { - log.error("不支持 {} 文件类型",extension); + log.error("不支持 {} 文件类型", extension); throw new GlobalException("不支持的文件类型"); } String module = request.getParameter("module"); @@ -77,31 +73,74 @@ public class DataRoomMinioServiceImpl implements IDataRoomOssService { // 重命名 String newFileName = IdWorker.getIdStr() + "." + extension; // 组装路径:获取当前日期并格式化为"yyyy/mm/dd"格式的字符串 - String filePath = LocalDate.now().format(DateTimeFormatter.ofPattern("yyyy/MM/dd")) + "/" + newFileName; - InputStream inputStream = file.getInputStream(); - PutObjectArgs args = PutObjectArgs.builder() + String basePath = LocalDate.now().format(DateTimeFormatter.ofPattern("yyyy/MM/dd")); + String filePath = basePath + "/" + newFileName; + MinioConfig minioConfig = bigScreenConfig.getFile().getMinio(); + try (InputStream inputStream = file.getInputStream()) { + PutObjectArgs args = PutObjectArgs.builder() .bucket(minioConfig.getBucketName()) .object(filePath) .stream(inputStream, file.getSize(), -1) .contentType(file.getContentType()) .build(); - minioclient.putObject(args); - - String url = minioConfig.getUrl() + "/" + minioConfig.getBucketName() + "/" + filePath; + minioclient.putObject(args); + } catch (Exception e) { + log.error("上传文件到Minio失败"); + log.error(ExceptionUtils.getStackTrace(e)); + throw new GlobalException("上传文件失败"); + } + // 现在url存储文件的相对路径，对于minio来说，就是bucketName/文件名 + String url = "/" + minioConfig.getBucketName() + "/" + filePath; fileEntity.setOriginalName(originalFilename); fileEntity.setNewName(newFileName); fileEntity.setPath(filePath); fileEntity.setSize(file.getSize()); fileEntity.setExtension(extension); fileEntity.setUrl(url); + fileEntity.setModule(module); + fileEntity.setBucket(minioConfig.getBucketName()); return fileEntity; } + @Override + public DataRoomFileEntity upload(InputStream inputStream, String fileName, long size, DataRoomFileEntity entity) { + fileName = PathUtils.normalizePath(fileName); + String extension = FilenameUtils.getExtension(fileName); + MinioConfig minioConfig = bigScreenConfig.getFile().getMinio(); + long fileSize = size == 0 ? -1 : size; + // 使用minio的最小分片大小 + long partSize = fileSize == -1 ? ObjectWriteArgs.MIN_MULTIPART_SIZE : -1; + try { + PutObjectArgs args = PutObjectArgs.builder() + .bucket(minioConfig.getBucketName()) + .object(fileName) + .stream(inputStream, fileSize, partSize) + .contentType("image/png") + .build(); + minioclient.putObject(args); + } catch (Exception e) { + log.error("上传文件到Minio失败"); + log.error(ExceptionUtils.getStackTrace(e)); + throw new GlobalException("上传文件失败"); + } finally { + IOUtils.closeQuietly(inputStream); + } + // 现在url存储文件的相对路径，对于minio来说，就是bucketName/文件名 + String url = "/" + minioConfig.getBucketName() + "/" + fileName; + entity.setOriginalName(fileName); + entity.setNewName(fileName); + entity.setPath(fileName); + entity.setSize(fileSize); + entity.setExtension(extension); + entity.setUrl(url); + entity.setBucket(minioConfig.getBucketName()); + return entity; + } + + /** * 下载文件 - * */ - @SneakyThrows @Override public void download(String fileId, HttpServletResponse response, HttpServletRequest request) { DataRoomFileEntity fileEntity = sysFileService.getById(fileId); @@ -114,9 +153,14 @@ public class DataRoomMinioServiceImpl implements IDataRoomOssService { response.setContentType("multipart/form-data"); // 不设置前端无法从header获取文件名 response.setHeader("Access-Control-Expose-Headers", "filename"); - response.setHeader("filename", URLEncoder.encode(fileEntity.getOriginalName(), "UTF-8")); - // 解决下载的文件不携带后缀 - response.setHeader("Content-Disposition", "attachment;fileName="+URLEncoder.encode(fileEntity.getOriginalName(), "UTF-8")); + try { + response.setHeader("filename", URLEncoder.encode(fileEntity.getOriginalName(), "UTF-8")); + // 解决下载的文件不携带后缀 + response.setHeader("Content-Disposition", "attachment;fileName=" + URLEncoder.encode(fileEntity.getOriginalName(), "UTF-8")); + } catch (UnsupportedEncodingException e) { + log.error("文件名编码失败"); + log.error(ExceptionUtils.getStackTrace(e)); + } try { InputStream is = minioFileInterface.download(fileEntity.getPath()); IOUtils.copy(is, response.getOutputStream()); @@ -124,7 +168,8 @@ public class DataRoomMinioServiceImpl implements IDataRoomOssService { response.getOutputStream().close(); } catch (Exception e) { response.setStatus(HttpStatus.INTERNAL_SERVER_ERROR.value()); - log.error(String.format("下载文件%s失败", fileEntity.getOriginalName())); + log.error("下载文件失败: {}", fileEntity.getPath()); + log.error(ExceptionUtils.getStackTrace(e)); } finally { sysFileService.updateDownloadCount(1, fileId); } @@ -133,10 +178,39 @@ public class DataRoomMinioServiceImpl implements IDataRoomOssService { /** * 删除文件 */ - @SneakyThrows @Override public void delete(String fileId) { - String path = sysFileService.getById(fileId).getPath(); - minioFileInterface.deleteObject(path.substring(path.indexOf(minioConfig.getBucketName())+minioConfig.getBucketName().length()+1)); + DataRoomFileEntity fileEntity = sysFileService.getById(fileId); + if (fileEntity == null) { + log.error("删除的文件不存在"); + return; + } + sysFileService.removeById(fileId); + String path = fileEntity.getPath(); + try { + minioFileInterface.deleteObject(path); + } catch (Exception e) { + log.error("删除Minio文件失败: {}", path); + log.error(ExceptionUtils.getStackTrace(e)); + } + } + + @Override + public String copy(String sourcePath, String targetPath) { + MinioConfig minioConfig = bigScreenConfig.getFile().getMinio(); + CopySource source = CopySource.builder().bucket(minioConfig.getBucketName()).object(sourcePath).build(); + CopyObjectArgs args = CopyObjectArgs.builder() + .bucket(minioConfig.getBucketName()) + .object(PathUtils.normalizePath(targetPath)) + .source(source) + .build(); + try { + minioclient.copyObject(args); + } catch (Exception e) { + log.error("复制Minio文件失败: {}", sourcePath); + log.error(ExceptionUtils.getStackTrace(e)); + return ""; + } + return minioConfig.getBucketName() + "/" + targetPath; } } diff --git a/DataRoom/dataroom-core/src/main/java/com/gccloud/dataroom/core/module/file/service/impl/DataRoomSftpFileServiceImpl.java b/DataRoom/dataroom-core/src/main/java/com/gccloud/dataroom/core/module/file/service/impl/DataRoomSftpFileServiceImpl.java new file mode 100644 index 00000000..1b3a0158 --- /dev/null +++ b/DataRoom/dataroom-core/src/main/java/com/gccloud/dataroom/core/module/file/service/impl/DataRoomSftpFileServiceImpl.java @@ -0,0 +1,157 @@ +package com.gccloud.dataroom.core.module.file.service.impl; + +import com.baomidou.mybatisplus.core.toolkit.IdWorker; +import com.gccloud.common.exception.GlobalException; +import com.gccloud.dataroom.core.config.DataRoomConfig; +import com.gccloud.dataroom.core.config.bean.FileConfig; +import com.gccloud.dataroom.core.module.file.entity.DataRoomFileEntity; +import com.gccloud.dataroom.core.module.file.service.IDataRoomFileService; +import com.gccloud.dataroom.core.module.file.service.IDataRoomOssService; +import com.gccloud.dataroom.core.utils.SftpClientUtils; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.io.FilenameUtils; +import org.apache.commons.lang3.exception.ExceptionUtils; +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.http.HttpStatus; +import org.springframework.stereotype.Service; +import org.springframework.web.multipart.MultipartFile; + +import javax.annotation.Resource; +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; +import java.io.*; +import java.net.URLEncoder; + +/** + * sftp文件管理实现类 + * 将文件上传至sftp服务器，需要配置sftp服务器相关信息 + * 由于该方案无法直接通过url访问文件，所以需要手动在对应的服务器上部署nginx等服务，将sftp服务器上的文件开放访问，然后将该服务地址配置到gc.starter.file.urlPrefix中 + * @author hongyang + * @version 1.0 + * @date 2023/10/17 15:12 + */ +@Slf4j +@Service +@ConditionalOnProperty(prefix = "gc.starter.file", name = "type", havingValue = "sftp") +public class DataRoomSftpFileServiceImpl implements IDataRoomOssService { + + @Resource + private SftpClientUtils sftpUtil; + @Resource + private DataRoomConfig bigScreenConfig; + @Resource + private IDataRoomFileService sysFileService; + + + @Override + public DataRoomFileEntity upload(MultipartFile file, DataRoomFileEntity fileEntity, HttpServletResponse response, HttpServletRequest request) { + String originalFilename = file.getOriginalFilename(); + // 提取文件后缀名 + String extension = FilenameUtils.getExtension(originalFilename); + FileConfig fileConfig = bigScreenConfig.getFile(); + if (!fileConfig.getAllowedFileExtensionName().contains("*") && !fileConfig.getAllowedFileExtensionName().contains(extension)) { + log.error("不支持 {} 文件类型",extension); + throw new GlobalException("不支持的文件类型"); + } + long size = file.getSize(); + // 重命名 + String id = IdWorker.getIdStr(); + String newFileName = id + "." + extension; + InputStream inputStream; + try { + inputStream = file.getInputStream(); + } catch (IOException e) { + log.error("上传文件到SFTP服务失败：获取文件流失败"); + log.error(ExceptionUtils.getStackTrace(e)); + throw new GlobalException("获取文件流失败"); + } + this.upload(inputStream, newFileName, size, fileEntity); + return fileEntity; + } + + + @Override + public DataRoomFileEntity upload(InputStream inputStream, String fileName, long size, DataRoomFileEntity fileEntity) { + // 提取文件后缀名 + String extension = FilenameUtils.getExtension(fileName); + // 上传的目标路径 + String basePath = bigScreenConfig.getFile().getBasePath(); + // 上传文件到sftp + boolean upload = sftpUtil.upload(basePath, fileName, inputStream); + if (!upload) { + log.error("上传文件到sftp失败"); + throw new GlobalException("上传文件到sftp失败"); + } + fileEntity.setOriginalName(fileName); + fileEntity.setNewName(fileName); + fileEntity.setPath(basePath); + fileEntity.setSize(size); + fileEntity.setExtension(extension); + fileEntity.setUrl("/" + fileName); + return fileEntity; + } + + @Override + public void download(String fileId, HttpServletResponse response, HttpServletRequest request) { + DataRoomFileEntity fileEntity = sysFileService.getById(fileId); + if (fileEntity == null) { + response.setStatus(HttpStatus.NOT_FOUND.value()); + log.error("下载的文件不存在"); + return; + } + response.setContentType("application/x-msdownload"); + response.setContentType("multipart/form-data"); + // 不设置前端无法从header获取文件名 + response.setHeader("Access-Control-Expose-Headers", "filename"); + try { + response.setHeader("filename", URLEncoder.encode(fileEntity.getOriginalName(), "UTF-8")); + // 解决下载的文件不携带后缀 + response.setHeader("Content-Disposition", "attachment;fileName="+URLEncoder.encode(fileEntity.getOriginalName(),"UTF-8")); + } catch (UnsupportedEncodingException e) { + log.error(ExceptionUtils.getStackTrace(e)); + } + OutputStream outputStream; + try { + outputStream = response.getOutputStream(); + } catch (IOException e) { + log.error(ExceptionUtils.getStackTrace(e)); + response.setStatus(HttpStatus.INTERNAL_SERVER_ERROR.value()); + return; + } + boolean download = sftpUtil.download(fileEntity.getPath(), fileEntity.getNewName(), outputStream); + if (!download) { + log.error("下载文件失败"); + response.setStatus(HttpStatus.INTERNAL_SERVER_ERROR.value()); + } + try { + outputStream.close(); + } catch (IOException e) { + log.error(ExceptionUtils.getStackTrace(e)); + } + sysFileService.updateDownloadCount(1, fileId); + } + + @Override + public void delete(String fileId) { + DataRoomFileEntity fileEntity = sysFileService.getById(fileId); + if (fileEntity == null) { + log.error("删除的文件不存在"); + return; + } + sysFileService.removeById(fileId); + // 删除sftp上的文件 + sftpUtil.delete(fileEntity.getPath(), fileEntity.getNewName()); + } + + + @Override + public String copy(String sourcePath, String targetPath) { + String basePath = bigScreenConfig.getFile().getBasePath() + File.separator; + + boolean copySuccess = sftpUtil.copy(basePath + sourcePath, basePath + targetPath); + if (!copySuccess) { + return ""; + } + return targetPath; + } +} diff --git a/DataRoom/dataroom-core/src/main/java/com/gccloud/dataroom/core/module/file/service/pool/ftp/FtpClientFactory.java b/DataRoom/dataroom-core/src/main/java/com/gccloud/dataroom/core/module/file/service/pool/ftp/FtpClientFactory.java new file mode 100644 index 00000000..19dfd2c0 --- /dev/null +++ b/DataRoom/dataroom-core/src/main/java/com/gccloud/dataroom/core/module/file/service/pool/ftp/FtpClientFactory.java @@ -0,0 +1,153 @@ +package com.gccloud.dataroom.core.module.file.service.pool.ftp; + +import com.gccloud.dataroom.core.config.DataRoomConfig; +import com.gccloud.dataroom.core.config.bean.FtpConfig; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.StringUtils; +import org.apache.commons.lang3.exception.ExceptionUtils; +import org.apache.commons.net.ftp.FTP; +import org.apache.commons.net.ftp.FTPClient; +import org.apache.commons.net.ftp.FTPReply; +import org.apache.commons.pool2.PooledObject; +import org.apache.commons.pool2.PooledObjectFactory; +import org.apache.commons.pool2.impl.DefaultPooledObject; +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.stereotype.Component; + +import javax.annotation.Resource; + +/** + * FtpClient 工厂连接对象 + * @author hongyang + * @version 1.0 + * @date 2023/10/18 10:17 + */ +@Slf4j +@Component +@ConditionalOnProperty(prefix = "gc.starter.file", name = "type", havingValue = "ftp") +public class FtpClientFactory implements PooledObjectFactory { + /** + * 注入 ftp 连接配置 + */ + @Resource + private DataRoomConfig config; + + /** + * 创建连接到池中 + * + * @return + * @throws Exception + */ + @Override + public PooledObject makeObject() throws Exception { + log.info("创建ftp连接"); + FtpConfig ftp = config.getFile().getFtp(); + FTPClient ftpClient = new FTPClient(); + ftpClient.setConnectTimeout(ftp.getClientTimeout()); + ftpClient.connect(ftp.getHost(), ftp.getPort()); + int reply = ftpClient.getReplyCode(); + if (!FTPReply.isPositiveCompletion(reply)) { + ftpClient.disconnect(); + return null; + } + boolean success; + if (StringUtils.isBlank(ftp.getUsername())) { + success = ftpClient.login("anonymous", "anonymous"); + } else { + success = ftpClient.login(ftp.getUsername(), ftp.getPassword()); + } + if (!success) { + return null; + } + ftpClient.setFileType(ftp.getTransferFileType()); + ftpClient.setBufferSize(1024); + ftpClient.setControlEncoding(ftp.getEncoding()); + if (ftp.isPassiveMode()) { + ftpClient.enterLocalPassiveMode(); + } + log.debug("创建ftp连接"); + return new DefaultPooledObject<>(ftpClient); + } + + /** + * 链接状态检查 + * + * @param pool + * @return + */ + @Override + public boolean validateObject(PooledObject pool) { + FTPClient ftpClient = pool.getObject(); + try { + return ftpClient != null && ftpClient.sendNoOp(); + } catch (Exception e) { + return false; + } + } + + /** + * 销毁连接，当连接池空闲数量达到上限时，调用此方法销毁连接 + * + * @param pool + * @throws Exception + */ + @Override + public void destroyObject(PooledObject pool) throws Exception { + FTPClient ftpClient = pool.getObject(); + if (ftpClient != null) { + try { + ftpClient.disconnect(); + log.debug("销毁ftp连接"); + } catch (Exception e) { + log.error("销毁FtpClient异常"); + log.error(ExceptionUtils.getStackTrace(e)); + } + } + } + + /** + * 钝化连接 + * 在连接被归还到连接池时，调用此方法 + * @param p + * @throws Exception + */ + @Override + public void passivateObject(PooledObject p) throws Exception{ + FTPClient ftpClient = p.getObject(); + try { + ftpClient.changeWorkingDirectory(config.getFile().getBasePath()); + // 钝化链接时，如果logout，下次再使用重新连接时间会长，所以先不做logout操作 + /* + * ftpClient.logout(); + * if (ftpClient.isConnected()) { + * ftpClient.disconnect(); + * } + */ + } catch (Exception e) { + log.error("钝化FtpClient异常"); + log.error(ExceptionUtils.getStackTrace(e)); + } + } + + /** + * 激活连接 + * 在连接从连接池中取出时，调用此方法 + * @param pool + * @throws Exception + */ + @Override + public void activateObject(PooledObject pool) throws Exception { + FtpConfig ftp = config.getFile().getFtp(); + FTPClient ftpClient = pool.getObject(); + if (!ftpClient.isConnected()) { + log.info("ftp连接已关闭，重新连接"); + ftpClient.connect(ftp.getHost(),ftp.getPort()); + ftpClient.login(ftp.getUsername(), ftp.getPassword()); + } + ftpClient.setControlEncoding(ftp.getEncoding()); + ftpClient.changeWorkingDirectory(config.getFile().getBasePath()); + // 设置上传文件类型为二进制，否则将无法打开文件 + ftpClient.setFileType(FTP.BINARY_FILE_TYPE); + } + +} diff --git a/DataRoom/dataroom-core/src/main/java/com/gccloud/dataroom/core/module/file/service/pool/ftp/FtpPoolServiceImpl.java b/DataRoom/dataroom-core/src/main/java/com/gccloud/dataroom/core/module/file/service/pool/ftp/FtpPoolServiceImpl.java new file mode 100644 index 00000000..cf1a3647 --- /dev/null +++ b/DataRoom/dataroom-core/src/main/java/com/gccloud/dataroom/core/module/file/service/pool/ftp/FtpPoolServiceImpl.java @@ -0,0 +1,79 @@ +package com.gccloud.dataroom.core.module.file.service.pool.ftp; + +import com.gccloud.dataroom.core.config.bean.FtpConfig; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.net.ftp.FTPClient; +import org.apache.commons.pool2.impl.GenericObjectPool; +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.stereotype.Component; + +import javax.annotation.PostConstruct; +import javax.annotation.Resource; + +/** + * @author hongyang + * @version 1.0 + * @date 2023/10/18 10:20 + */ +@Slf4j +@Component +@ConditionalOnProperty(prefix = "gc.starter.file", name = "type", havingValue = "ftp") +public class FtpPoolServiceImpl { + + /** + * ftp 连接池生成 + */ + private GenericObjectPool pool; + + /** + * ftp 客户端配置文件 + */ + @Resource + private FtpConfig config; + + /** + * ftp 客户端工厂 + */ + @Resource + private FtpClientFactory factory; + + /** + * 初始化pool + */ + @PostConstruct + private void initPool() { + log.info("初始化FTP连接池"); + this.pool = new GenericObjectPool(this.factory, this.config); + } + + /** + * 获取ftpClient + */ + public FTPClient borrowObject() { + log.info("获取 FTPClient"); + if (this.pool != null) { + try { + return this.pool.borrowObject(); + } catch (Exception e) { + log.error("获取 FTPClient 失败 ", e); + } + } + return null; + } + + /** + * 归还 ftpClient + */ + public void returnObject(FTPClient ftpClient) { + if (this.pool == null || ftpClient == null) { + return; + } + try { + ftpClient.changeWorkingDirectory("/"); + } catch (Exception e) { + log.error("FTPClient 重置目录失败 ", e); + } + this.pool.returnObject(ftpClient); + } + +} diff --git a/DataRoom/dataroom-core/src/main/java/com/gccloud/dataroom/core/module/file/service/pool/sftp/SftpClientFactory.java b/DataRoom/dataroom-core/src/main/java/com/gccloud/dataroom/core/module/file/service/pool/sftp/SftpClientFactory.java new file mode 100644 index 00000000..c90ab162 --- /dev/null +++ b/DataRoom/dataroom-core/src/main/java/com/gccloud/dataroom/core/module/file/service/pool/sftp/SftpClientFactory.java @@ -0,0 +1,105 @@ +package com.gccloud.dataroom.core.module.file.service.pool.sftp; + +import com.gccloud.dataroom.core.config.bean.SftpConfig; +import com.jcraft.jsch.ChannelSftp; +import com.jcraft.jsch.JSch; +import com.jcraft.jsch.Session; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.StringUtils; +import org.apache.commons.lang3.exception.ExceptionUtils; +import org.apache.commons.pool2.BasePooledObjectFactory; +import org.apache.commons.pool2.PooledObject; +import org.apache.commons.pool2.impl.DefaultPooledObject; +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.stereotype.Component; + +import javax.annotation.Resource; +import java.util.Properties; + +/** + * SFTP 工厂连接对象 + * @author hongyang + * @version 1.0 + * @date 2023/10/18 15:17 + */ +@Slf4j +@Component +@ConditionalOnProperty(prefix = "gc.starter.file", name = "type", havingValue = "sftp") +public class SftpClientFactory extends BasePooledObjectFactory{ + + @Resource + private SftpConfig config; + + /** + * 新建对象 + */ + @Override + public ChannelSftp create() { + ChannelSftp channel = null; + try { + // 用户名密码不能为空 + if (StringUtils.isBlank(config.getUsername()) || StringUtils.isBlank(config.getPassword())) { + log.error("SFTP用户名密码不能为空"); + return null; + } + JSch jsch = new JSch(); + // 设置私钥 + if (StringUtils.isNotBlank(config.getPrivateKey())) { + jsch.addIdentity(config.getPrivateKey()); + } + // jsch的session需要补充设置sshConfig.put("PreferredAuthentications", "publickey,keyboard-interactive,password")来跳过Kerberos认证，同样的HutoolSFTPUtil工具类里面也有这个问题 + Session sshSession = jsch.getSession(config.getUsername(), config.getHost(), config.getPort()); + sshSession.setPassword(config.getPassword()); + Properties sshConfig = new Properties(); + // “StrictHostKeyChecking”如果设置成“yes”，ssh就不会自动把计算机的密匙加入“$HOME/.ssh/known_hosts”文件，并且一旦计算机的密匙发生了变化，就拒绝连接。 + sshConfig.put("StrictHostKeyChecking", "no"); + sshSession.setConfig(sshConfig); + sshSession.connect(); + channel = (ChannelSftp) sshSession.openChannel("sftp"); + channel.connect(); + } catch (Exception e) { + log.error("连接 sftp 失败，请检查配置"); + log.error(ExceptionUtils.getStackTrace(e)); + } + return channel; + } + + /** + * 创建一个连接 + * + * @param channelSftp + * @return + */ + @Override + public PooledObject wrap(ChannelSftp channelSftp) { + return new DefaultPooledObject<>(channelSftp); + } + + /** + * 销毁一个连接 + * + * @param p + */ + @Override + public void destroyObject(PooledObject p) { + ChannelSftp channelSftp = p.getObject(); + channelSftp.disconnect(); + } + + @Override + public boolean validateObject(final PooledObject p) { + final ChannelSftp channelSftp = p.getObject(); + try { + if (channelSftp.isClosed()) { + return false; + } + channelSftp.cd("/"); + } catch (Exception e) { + log.error("ChannelSftp 不可用"); + log.error(ExceptionUtils.getStackTrace(e)); + return false; + } + return true; + } + +} diff --git a/DataRoom/dataroom-core/src/main/java/com/gccloud/dataroom/core/module/file/service/pool/sftp/SftpPoolService.java b/DataRoom/dataroom-core/src/main/java/com/gccloud/dataroom/core/module/file/service/pool/sftp/SftpPoolService.java new file mode 100644 index 00000000..929a995e --- /dev/null +++ b/DataRoom/dataroom-core/src/main/java/com/gccloud/dataroom/core/module/file/service/pool/sftp/SftpPoolService.java @@ -0,0 +1,74 @@ +package com.gccloud.dataroom.core.module.file.service.pool.sftp; + +import com.gccloud.dataroom.core.config.bean.SftpConfig; +import com.jcraft.jsch.ChannelSftp; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.pool2.impl.GenericObjectPool; +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.stereotype.Component; + +import javax.annotation.PostConstruct; +import javax.annotation.Resource; + +/** + * Sftp 连接池服务类 + * @author hongyang + * @version 1.0 + * @date 2023/10/18 15:21 + */ +@Slf4j +@Component +@ConditionalOnProperty(prefix = "gc.starter.file", name = "type", havingValue = "sftp") +public class SftpPoolService { + + /** + * ftp 连接池生成 + */ + private GenericObjectPool pool; + + /** + * ftp 客户端配置文件 + */ + @Resource + private SftpConfig config; + + /** + * ftp 客户端工厂 + */ + @Resource + private SftpClientFactory factory; + + /** + * 初始化pool + */ + @PostConstruct + private void initPool() { + log.info("初始化SFTP连接池"); + this.pool = new GenericObjectPool(this.factory, this.config); + } + + /** + * 获取sftp + */ + public ChannelSftp borrowObject() { + if (this.pool != null) { + try { + return this.pool.borrowObject(); + } catch (Exception e) { + log.error("获取 ChannelSftp 失败", e); + e.printStackTrace(); + } + } + return null; + } + + /** + * 归还 sftp + */ + public void returnObject(ChannelSftp channelSftp) { + if (this.pool != null && channelSftp != null) { + this.pool.returnObject(channelSftp); + } + } + +} diff --git a/DataRoom/dataroom-core/src/main/java/com/gccloud/dataroom/core/module/manage/service/impl/DataRoomPageServiceImpl.java b/DataRoom/dataroom-core/src/main/java/com/gccloud/dataroom/core/module/manage/service/impl/DataRoomPageServiceImpl.java index 3e1c3d07..7fe48eee 100644 --- a/DataRoom/dataroom-core/src/main/java/com/gccloud/dataroom/core/module/manage/service/impl/DataRoomPageServiceImpl.java +++ b/DataRoom/dataroom-core/src/main/java/com/gccloud/dataroom/core/module/manage/service/impl/DataRoomPageServiceImpl.java @@ -10,6 +10,9 @@ import com.gccloud.dataroom.core.module.basic.entity.PagePreviewEntity; import com.gccloud.dataroom.core.module.chart.bean.Chart; import com.gccloud.dataroom.core.module.chart.bean.Linkage; import com.gccloud.dataroom.core.module.chart.components.datasource.DataSetDataSource; +import com.gccloud.dataroom.core.module.file.entity.DataRoomFileEntity; +import com.gccloud.dataroom.core.module.file.service.IDataRoomFileService; +import com.gccloud.dataroom.core.module.file.service.IDataRoomOssService; import com.gccloud.dataroom.core.module.manage.dto.DataRoomPageDTO; import com.gccloud.dataroom.core.module.manage.dto.DataRoomSearchDTO; import com.gccloud.dataroom.core.module.manage.extend.DataRoomExtendClient; @@ -23,6 +26,7 @@ import com.gccloud.common.exception.GlobalException; import com.gccloud.common.utils.AssertUtils; import com.gccloud.common.utils.BeanConvertUtils; import com.gccloud.common.vo.PageVO; +import com.gccloud.dataroom.core.utils.PathUtils; import com.google.common.collect.Lists; import com.google.common.collect.Maps; import lombok.extern.slf4j.Slf4j; @@ -30,12 +34,13 @@ import org.apache.commons.collections4.CollectionUtils; import org.apache.commons.io.FileUtils; import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.exception.ExceptionUtils; +import org.apache.tomcat.util.http.fileupload.FileItem; import org.springframework.stereotype.Service; +import org.springframework.web.multipart.MultipartFile; +import org.springframework.web.multipart.commons.CommonsMultipartFile; import javax.annotation.Resource; -import java.io.File; -import java.io.FileOutputStream; -import java.io.IOException; +import java.io.*; import java.util.Base64; import java.util.List; import java.util.Map; @@ -67,6 +72,9 @@ public class DataRoomPageServiceImpl extends ServiceImpl file.isFile() && file.getName().equals(finalFtpFileName)); + if (ftpFiles == null || ftpFiles.length == 0) { + log.info("FTP服务器文件不存在:{}", ftpPath + ftpFileName); + return false; + } + ftpClient.setControlEncoding("UTF-8"); + ftpClient.setFileType(FTPClient.BINARY_FILE_TYPE); + ftpClient.enterLocalPassiveMode(); + ftpClient.changeWorkingDirectory(ftpPath); + boolean success = ftpClient.retrieveFile(finalFtpFileName, outputStream); + if (!success) { + log.info("FTP文件下载失败:{}", ftpPath + ftpFileName); + return false; + } + log.info("FTP文件下载成功:{}", ftpPath + ftpFileName); + return true; + } catch (Exception e) { + log.info("FTP文件下载失败:{}", ftpPath + ftpFileName); + log.error(ExceptionUtils.getStackTrace(e)); + } finally { + try { + if (outputStream != null) { + outputStream.close(); + } + } catch (IOException e) { + log.error(ExceptionUtils.getStackTrace(e)); + } + ftpPoolService.returnObject(ftpClient); + } + return false; + } + + + + /** + * 从FTP服务器删除文件 + * 存在文件的目录无法删除 + * + * @param ftpPath 服务器文件存储路径 + * @param fileName 服务器文件存储名称 + * @return 删除结果 + */ + public boolean delete(String ftpPath, String fileName) { + String[] paths = PathUtils.handlePath(ftpPath, fileName); + ftpPath = paths[0] + "/"; + fileName = paths[1]; + FTPClient ftpClient = ftpPoolService.borrowObject(); + try { + // 在 ftp 目录下获取文件名与 fileName 匹配的文件信息 + String finalFileName = fileName; + FTPFile[] ftpFiles = ftpClient.listFiles(ftpPath, file -> file.isFile() && file.getName().equals(finalFileName)); + if (ftpFiles == null || ftpFiles.length == 0) { + log.error("FTP服务器文件不存在:{},", ftpPath + fileName); + return false; + } + // 删除文件 + boolean del; + ftpClient.changeWorkingDirectory(ftpPath); + del = ftpClient.deleteFile(finalFileName); + log.info(del ? "文件:{}删除成功" : "文件:{}删除失败", ftpPath + fileName); + return del; + } catch (IOException e) { + log.error(ExceptionUtils.getStackTrace(e)); + } finally { + ftpPoolService.returnObject(ftpClient); + } + return false; + } + + /** + * 复制文件，目前仅支持文件复制 + * @param sourcePath + * @param targetPath + */ + public boolean copy(String sourcePath, String targetPath) { + sourcePath = PathUtils.normalizePath(sourcePath); + targetPath = PathUtils.normalizePath(targetPath); + // 分割路径和文件名 + String[] sourceSplit = sourcePath.split("/"); + String[] targetSplit = targetPath.split("/"); + sourcePath = sourcePath.substring(0, sourcePath.length() - sourceSplit[sourceSplit.length - 1].length()); + targetPath = targetPath.substring(0, targetPath.length() - targetSplit[targetSplit.length - 1].length()); + String sourceFileName = sourceSplit[sourceSplit.length - 1]; + String targetFileName = targetSplit[targetSplit.length - 1]; + FTPClient ftpClient = ftpPoolService.borrowObject(); + try { + // 检查目标文件是否存在 + FTPFile[] ftpFiles = ftpClient.listFiles(sourcePath, file -> file.isFile() && file.getName().equals(sourceFileName)); + if (ftpFiles == null || ftpFiles.length == 0) { + log.error("FTP服务器文件不存在:{}", sourcePath + sourceFileName); + return false; + } + ftpClient.setControlEncoding("UTF-8"); + ftpClient.setFileType(FTPClient.BINARY_FILE_TYPE); + ftpClient.enterLocalPassiveMode(); + ftpClient.changeWorkingDirectory(sourcePath); + boolean success = ftpClient.retrieveFile(sourceFileName, new FileOutputStream(targetPath + targetFileName)); + if (!success) { + log.error("FTP文件复制失败:{}", sourcePath + sourceFileName); + return false; + } + log.info("FTP文件复制成功:{}", targetPath + targetFileName); + return true; + } catch (Exception e) { + log.info("FTP文件复制失败:{}", sourcePath + sourceFileName); + log.error(ExceptionUtils.getStackTrace(e)); + } finally { + ftpPoolService.returnObject(ftpClient); + } + return false; + } + + /** + * 改变当前目录 + * + * @param workPath 新的当前工作目录 + * @return + */ + private boolean changeWorkingDirectory(String workPath, FTPClient ftpClient) throws IOException { + boolean success = ftpClient.changeWorkingDirectory(workPath); + if (!success) { + String[] dirs = workPath.split("/"); + for (String str : dirs) { + if(StringUtils.isBlank(str)) { + continue; + } + if (!ftpClient.changeWorkingDirectory(str)) { + boolean makeDirectory = ftpClient.makeDirectory(str); + log.info("创建目录:{}, 结果: {}", str, makeDirectory ? "成功" : "失败"); + success = ftpClient.changeWorkingDirectory(str); + } + } + } + return success; + } + +} + diff --git a/DataRoom/dataroom-core/src/main/java/com/gccloud/dataroom/core/utils/MinioFileInterface.java b/DataRoom/dataroom-core/src/main/java/com/gccloud/dataroom/core/utils/MinioFileInterface.java index ae2d3fc9..c11553c1 100644 --- a/DataRoom/dataroom-core/src/main/java/com/gccloud/dataroom/core/utils/MinioFileInterface.java +++ b/DataRoom/dataroom-core/src/main/java/com/gccloud/dataroom/core/utils/MinioFileInterface.java @@ -1,6 +1,6 @@ package com.gccloud.dataroom.core.utils; -import com.gccloud.dataroom.core.config.MinioConfig; +import com.gccloud.dataroom.core.config.bean.MinioConfig; import io.minio.BucketExistsArgs; import io.minio.GetObjectArgs; import io.minio.GetPresignedObjectUrlArgs; diff --git a/DataRoom/dataroom-core/src/main/java/com/gccloud/dataroom/core/utils/PathUtils.java b/DataRoom/dataroom-core/src/main/java/com/gccloud/dataroom/core/utils/PathUtils.java new file mode 100644 index 00000000..4a1b6102 --- /dev/null +++ b/DataRoom/dataroom-core/src/main/java/com/gccloud/dataroom/core/utils/PathUtils.java @@ -0,0 +1,56 @@ +package com.gccloud.dataroom.core.utils; + +import org.apache.commons.lang3.StringUtils; + +/** + * @author hongyang + * @version 1.0 + * @date 2023/10/18 17:29 + */ +public class PathUtils { + + /** + * 处理路径，如果路径中包含\,则替换为/,再检查替换后路径，是否包含连续的/，如果包含，则替换为单个/ + * @param path + * @return + */ + public static String normalizePath(String path) { + if (StringUtils.isBlank(path)) { + return path; + } + path = path.replace("\\", "/"); + while (path.contains("//")) { + path = path.replace("//", "/"); + } + return path; + } + + /** + * 处理文件路径和文件名 + * 文件名可能包含路径，需要将路径分离出来，转移到filePath下 + * @param filePath + * @param fileName + * @return + */ + public static String[] handlePath(String filePath, String fileName) { + filePath = normalizePath(filePath); + // 去除路径最后的/ + if (filePath.endsWith("/")) { + filePath = filePath.substring(0, filePath.length() - 1); + } + fileName = normalizePath(fileName); + // 去除文件名前的/ + if (fileName.startsWith("/")) { + fileName = fileName.substring(1); + } + // fileName可能包含路径，需要将路径分离出来，转移到filePath下 + String[] split = fileName.split("/"); + if (split.length > 1) { + String fileNameTemp = split[split.length - 1]; + String filePathTemp = fileName.substring(0, fileName.length() - fileNameTemp.length()); + filePath = filePath + "/" + filePathTemp; + fileName = fileNameTemp; + } + return new String[]{filePath, fileName}; + } +} diff --git a/DataRoom/dataroom-core/src/main/java/com/gccloud/dataroom/core/utils/SftpClientUtils.java b/DataRoom/dataroom-core/src/main/java/com/gccloud/dataroom/core/utils/SftpClientUtils.java new file mode 100644 index 00000000..af8d26b9 --- /dev/null +++ b/DataRoom/dataroom-core/src/main/java/com/gccloud/dataroom/core/utils/SftpClientUtils.java @@ -0,0 +1,242 @@ +package com.gccloud.dataroom.core.utils; + +import com.gccloud.dataroom.core.config.bean.SftpConfig; +import com.gccloud.dataroom.core.module.file.service.pool.sftp.SftpPoolService; +import com.jcraft.jsch.ChannelSftp; +import com.jcraft.jsch.SftpException; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.exception.ExceptionUtils; +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.stereotype.Component; + +import javax.annotation.Resource; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; + +/** + * Sftp 客户端工具类 + * @author hongyang + * @version 1.0 + * @date 2023/10/18 15:28 + */ +@Slf4j +@Component +@ConditionalOnProperty(prefix = "gc.starter.file", name = "type", havingValue = "sftp") +public class SftpClientUtils { + + @Resource + private SftpConfig config; + + @Resource + private SftpPoolService sFtpPoolService; + + + /** + * 上传文件 + * @param uploadPath + * @param fileName + * @param inputStream + * @return + */ + public boolean upload(String uploadPath, String fileName, InputStream inputStream) { + ChannelSftp sftp = sFtpPoolService.borrowObject(); + + String[] paths = PathUtils.handlePath(uploadPath, fileName); + uploadPath = paths[0]; + // TODO 检查目录是否存在，不存在则创建 + if (!this.exist(uploadPath)) { + try { + sftp.mkdir(uploadPath); + } catch (SftpException e) { + log.error(ExceptionUtils.getStackTrace(e)); + return false; + } + } + fileName = "/" + paths[1]; + String filePath = uploadPath.concat(fileName); + try { + sftp.put(inputStream, filePath); + /** + * 权限 + */ + String permission = "755"; + sftp.chmod(Integer.parseInt(permission, 8), filePath); + log.info("文件上传成功:{}", filePath); + return true; + } catch (SftpException e) { + log.error("文件上传失败:{}", filePath); + log.error(ExceptionUtils.getStackTrace(e)); + } finally { + if (null != inputStream){ + try { + inputStream.close(); + } catch (IOException e) { + log.error(ExceptionUtils.getStackTrace(e)); + } + } + sFtpPoolService.returnObject(sftp); + } + return false; + } + + /** + * 下载文件 + * @param sftpPath + * @param fileName + * @param outputStream + * @return + */ + public boolean download(String sftpPath, String fileName, OutputStream outputStream) { + ChannelSftp sftp = sFtpPoolService.borrowObject(); + String[] paths = PathUtils.handlePath(sftpPath, fileName); + sftpPath = paths[0]; + fileName = "/" + paths[1]; + String filePath = sftpPath.concat(fileName); + try (InputStream inputStream = sftp.get(filePath)){ + // 将输入流的数据复制到输出流中 + byte[] bytes = new byte[config.getBufferSize()]; + int len; + while ((len = inputStream.read(bytes)) != -1) { + outputStream.write(bytes, 0, len); + } + log.info("文件下载成功:{}", filePath); + return true; + } catch (Exception e ) { + log.info("文件下载失败:{}", filePath); + log.error(ExceptionUtils.getStackTrace(e)); + } finally { + if (null != outputStream){ + try { + outputStream.close(); + } catch (IOException e) { + log.error(ExceptionUtils.getStackTrace(e)); + } + } + sFtpPoolService.returnObject(sftp); + } + return false; + } + + /** + * 从SFTP服务器删除文件 + * 存在文件的目录无法删除 + * + * @param sftpPath 服务器文件存储路径 + * @param fileName 服务器文件存储名称 + * @return 删除结果 + */ + public boolean delete(String sftpPath, String fileName) { + ChannelSftp sftp = sFtpPoolService.borrowObject(); + String[] paths = PathUtils.handlePath(sftpPath, fileName); + sftpPath = paths[0]; + fileName = "/" + paths[1]; + String filePath = sftpPath.concat(fileName); + // 检查文件是否存在 + if (!this.exist(filePath)) { + log.info("文件不存在:{}", filePath); + return true; + } + // 检查是否为文件夹 + if (this.isDirectory(filePath)) { + log.info("该路径为文件夹:{}，无法删除", filePath); + return false; + } + try { + sftp.rm(filePath); + log.info("文件删除成功:{}", filePath); + return true; + } catch (SftpException e) { + log.info("文件删除失败:{}", filePath); + log.error(ExceptionUtils.getStackTrace(e)); + } finally { + sFtpPoolService.returnObject(sftp); + } + return false; + } + + /** + * 文件复制，目前只支持文件复制 + * @param sourcePath + * @param targetPath + */ + public boolean copy(String sourcePath, String targetPath) { + ChannelSftp sftp1 = sFtpPoolService.borrowObject(); + ChannelSftp sftp2 = sFtpPoolService.borrowObject(); + sourcePath = PathUtils.normalizePath(sourcePath); + targetPath = PathUtils.normalizePath(targetPath); + // 检查源文件是否存在 + if (!this.exist(sourcePath)) { + log.error("复制源文件不存在:{}", sourcePath); + return false; + } + // 检查源文件是否为文件夹 + if (this.isDirectory(sourcePath)) { + log.error("源文件为文件夹:{}，无法复制", sourcePath); + return false; + } + // 复制 + InputStream inputStream = null; + try { + inputStream = sftp1.get(sourcePath); + sftp2.put(inputStream, targetPath); + log.info("文件复制成功:{}", sourcePath); + return true; + } catch (SftpException e) { + log.error("文件复制失败:{}", sourcePath); + log.error(ExceptionUtils.getStackTrace(e)); + } finally { + if (null != inputStream){ + try { + inputStream.close(); + } catch (IOException e) { + log.error(ExceptionUtils.getStackTrace(e)); + } + } + sFtpPoolService.returnObject(sftp1); + sFtpPoolService.returnObject(sftp2); + } + return false; + } + + /** + * 判断SFTP上的path是否为文件夹 + * 注:如果该路径不存在，那么会返回false + * + * @param path SFTP上的路径 + * @return 判断结果 + */ + public boolean isDirectory(String path) { + ChannelSftp sftp = sFtpPoolService.borrowObject(); + // 合法的错误id + // int legalErrorId = 4; + try { + sftp.cd(path); + return true; + } catch (SftpException e) { + // 如果 path不存在，那么报错信息为【No such file】，错误id为【2】 + // 如果 path存在，但是不能cd进去，那么报错信息形如【Can't change directory: /files/sqljdbc4-3.0.jar】，错误id为【4】 + return false; + } finally { + sFtpPoolService.returnObject(sftp); + } + } + + /** + * 检查文件是否存在 + * @param filePath + * @return + */ + private boolean exist(String filePath) { + ChannelSftp sftp = sFtpPoolService.borrowObject(); + try { + sftp.lstat(filePath); + return true; + } catch (SftpException e) { + return false; + } finally { + sFtpPoolService.returnObject(sftp); + } + } + +} diff --git a/DataRoom/dataroom-server/src/main/resources/application.yml b/DataRoom/dataroom-server/src/main/resources/application.yml index 7283cd91..8819574b 100644 --- a/DataRoom/dataroom-server/src/main/resources/application.yml +++ b/DataRoom/dataroom-server/src/main/resources/application.yml @@ -30,18 +30,6 @@ spring: resources: static-locations: classpath:/static/,classpath:/META-INF/resources/,classpath:/META-INF/resources/webjars/,file:${gc.starter.file.basePath} -gc: - starter: - file: - # minio | local - uploadType: local - -# Minio配置 -#minio: -# url: http://192.168.20.98:9000 -# accessKey: admin -# secretKey: 123456 -# bucketName: test mybatis-plus: # mybatis plus xml配置文件扫描，多个通过分号隔开 diff --git a/DataRoom/pom.xml b/DataRoom/pom.xml index 266bd267..bc1d548a 100644 --- a/DataRoom/pom.xml +++ b/DataRoom/pom.xml @@ -47,6 +47,9 @@ 8.2.2 2.0.0.RELEASE 2.22.2 + 3.6 + 2.10.0 + 0.1.55