文件打包下载迁移到file

This commit is contained in:
方亮 2025-12-31 14:47:25 +08:00
parent 16293941c2
commit f966a1588f
13 changed files with 181 additions and 50 deletions

View File

@ -3,8 +3,9 @@ package com.bonus.system.api;
import com.bonus.common.core.constant.SecurityConstants; import com.bonus.common.core.constant.SecurityConstants;
import com.bonus.common.core.constant.ServiceNameConstants; import com.bonus.common.core.constant.ServiceNameConstants;
import com.bonus.common.core.domain.R; import com.bonus.common.core.domain.R;
import com.bonus.system.api.domain.DownloadTask;
import com.bonus.system.api.domain.FileVo; import com.bonus.system.api.domain.FileVo;
import com.bonus.system.api.domain.ZipFileMapping;
import com.bonus.system.api.factory.RemoteUploadUtilsFallbackFactory; import com.bonus.system.api.factory.RemoteUploadUtilsFallbackFactory;
import com.bonus.system.api.model.UploadFileVo; import com.bonus.system.api.model.UploadFileVo;
import org.springframework.cloud.openfeign.FeignClient; import org.springframework.cloud.openfeign.FeignClient;
@ -142,4 +143,7 @@ public interface RemoteUploadUtilsService {
@PostMapping(value = "/uploadFile/getFileUrl") @PostMapping(value = "/uploadFile/getFileUrl")
public R<String> getFileUrl(@RequestParam(value = "filePath") String filePath, @RequestParam(value = "bucketName")String bucketName, public R<String> getFileUrl(@RequestParam(value = "filePath") String filePath, @RequestParam(value = "bucketName")String bucketName,
@RequestHeader(SecurityConstants.FROM_SOURCE) String source); @RequestHeader(SecurityConstants.FROM_SOURCE) String source);
@PostMapping(value = "/uploadFile/zipFile")
R<DownloadTask> zipFile(@RequestBody List<ZipFileMapping> objectNames, @RequestHeader(SecurityConstants.FROM_SOURCE) String source);
} }

View File

@ -1,4 +1,4 @@
package com.bonus.bmw.domain.po; package com.bonus.system.api.domain;
import lombok.AllArgsConstructor; import lombok.AllArgsConstructor;
import lombok.Data; import lombok.Data;

View File

@ -1,4 +1,4 @@
package com.bonus.bmw.domain.po; package com.bonus.system.api.domain;
import lombok.AllArgsConstructor; import lombok.AllArgsConstructor;
import lombok.Data; import lombok.Data;

View File

@ -1,10 +1,10 @@
package com.bonus.system.api.factory; package com.bonus.system.api.factory;
import com.bonus.common.core.domain.R; import com.bonus.common.core.domain.R;
import com.bonus.common.core.web.domain.AjaxResult;
import com.bonus.system.api.RemoteUploadUtilsService; import com.bonus.system.api.RemoteUploadUtilsService;
import com.bonus.system.api.domain.DownloadTask;
import com.bonus.system.api.domain.FileVo; import com.bonus.system.api.domain.FileVo;
import com.bonus.system.api.domain.SysNotice; import com.bonus.system.api.domain.ZipFileMapping;
import com.bonus.system.api.model.UploadFileVo; import com.bonus.system.api.model.UploadFileVo;
import org.slf4j.Logger; import org.slf4j.Logger;
import org.slf4j.LoggerFactory; import org.slf4j.LoggerFactory;
@ -77,6 +77,11 @@ public class RemoteUploadUtilsFallbackFactory implements FallbackFactory<RemoteU
return R.fail("文件bast64获取失败:" + throwable.getMessage()); return R.fail("文件bast64获取失败:" + throwable.getMessage());
} }
@Override
public R<DownloadTask> zipFile(List<ZipFileMapping> objectNames, String source) {
return R.fail("文件打包异常:" + throwable.getMessage());
}
}; };

View File

@ -1,15 +1,17 @@
package com.bonus.bmw.controller; package com.bonus.bmw.controller;
import com.bonus.bmw.domain.po.DownloadTask;
import com.bonus.bmw.domain.po.ZipFileMapping;
import com.bonus.bmw.domain.vo.BmWorkerAtt; import com.bonus.bmw.domain.vo.BmWorkerAtt;
import com.bonus.bmw.service.ZipDownloadService; import com.bonus.bmw.service.ZipDownloadService;
import com.bonus.bmw.service.impl.FileUploadUtils;
import com.bonus.system.api.domain.DownloadTask;
import com.bonus.system.api.domain.ZipFileMapping;
import io.minio.GetObjectArgs; import io.minio.GetObjectArgs;
import io.minio.MinioClient; import io.minio.MinioClient;
import lombok.extern.slf4j.Slf4j; import lombok.extern.slf4j.Slf4j;
import org.apache.commons.compress.archivers.zip.Zip64Mode; import org.apache.commons.compress.archivers.zip.Zip64Mode;
import org.apache.commons.compress.archivers.zip.ZipArchiveEntry; import org.apache.commons.compress.archivers.zip.ZipArchiveEntry;
import org.apache.commons.compress.archivers.zip.ZipArchiveOutputStream; import org.apache.commons.compress.archivers.zip.ZipArchiveOutputStream;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.data.redis.core.RedisTemplate; import org.springframework.data.redis.core.RedisTemplate;
import org.springframework.http.ResponseEntity; import org.springframework.http.ResponseEntity;
import org.springframework.scheduling.annotation.Scheduled; import org.springframework.scheduling.annotation.Scheduled;
@ -22,10 +24,8 @@ import java.nio.file.Files;
import java.nio.file.Path; import java.nio.file.Path;
import java.nio.file.Paths; import java.nio.file.Paths;
import java.util.List; import java.util.List;
import java.util.UUID;
import java.util.concurrent.ExecutorService; import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors; import java.util.concurrent.Executors;
import java.util.concurrent.TimeUnit;
import java.util.stream.Stream; import java.util.stream.Stream;
@RestController @RestController
@ -39,6 +39,9 @@ public class ZipDownloadController {
@Resource @Resource
private RedisTemplate<String, DownloadTask> redisTemplate; private RedisTemplate<String, DownloadTask> redisTemplate;
@Autowired
private FileUploadUtils fileUploadUtils;
private static final String TASK_PREFIX = "zip_task:"; // key 前缀 private static final String TASK_PREFIX = "zip_task:"; // key 前缀
private static final long TASK_EXPIRE_SECONDS = 2 * 3600; // 2小时过期 private static final long TASK_EXPIRE_SECONDS = 2 * 3600; // 2小时过期
@ -52,36 +55,31 @@ public class ZipDownloadController {
@PostMapping("/createZipTask") @PostMapping("/createZipTask")
public ResponseEntity<DownloadTask> createZipTask(@RequestBody BmWorkerAtt o) { public ResponseEntity<DownloadTask> createZipTask(@RequestBody BmWorkerAtt o) {
String taskId = UUID.randomUUID().toString();
String redisKey = TASK_PREFIX + taskId;
DownloadTask task = new DownloadTask(taskId);
// 保存到 Redis设置过期时间
redisTemplate.opsForValue().set(redisKey, task, TASK_EXPIRE_SECONDS, TimeUnit.SECONDS);
//查询文件地址 //查询文件地址
List<ZipFileMapping> objectNames = service.getFileList(o); List<ZipFileMapping> objectNames = service.getFileList(o);
// List<ZipFileMapping> objectNames = new ArrayList<>(); DownloadTask task = fileUploadUtils.zipFile(objectNames);
// objectNames.add(new ZipFileMapping("relname/2025/11/11/1f18f8b3c0b14143be1f30db6b078e2d.jpg","工程1/分包1/班组1/张三/人脸/user123_face.jpg")); // String taskId = UUID.randomUUID().toString();
// objectNames.add(new ZipFileMapping("relname/2025/11/11/ae3044c218714985ba8977479f712278.jpg","工程1/分包1/班组1/张三/合同/con.jpg")); // String redisKey = TASK_PREFIX + taskId;
// 异步执行打包 //
executor.submit(() -> { // DownloadTask task = new DownloadTask(taskId);
try { // // 保存到 Redis设置过期时间
String zipName = taskId + ".zip"; // redisTemplate.opsForValue().set(redisKey, task, TASK_EXPIRE_SECONDS, TimeUnit.SECONDS);
String zipPath = "/tmp/downloads/" + zipName; // // 异步执行打包
// generateZip(request.getFilePaths(), zipPath); // 你的打包逻辑 // executor.submit(() -> {
generateZipFromMinIO(minioClient, bucket, objectNames, zipPath); // try {
// 更新任务状态 // String zipName = taskId + ".zip";
task.setStatus("completed"); // String zipPath = "/tmp/downloads/" + zipName;
// task.setDownloadUrl("/downloadFile/" + zipName); // generateZipFromMinIO(minioClient, bucket, objectNames, zipPath);
task.setDownloadUrl("/" + zipName); // // 更新任务状态
redisTemplate.opsForValue().set(redisKey, task, TASK_EXPIRE_SECONDS, TimeUnit.SECONDS); // task.setStatus("completed");
} catch (Exception e) { // task.setDownloadUrl("/" + zipName);
task.setStatus("failed"); // redisTemplate.opsForValue().set(redisKey, task, TASK_EXPIRE_SECONDS, TimeUnit.SECONDS);
task.setErrorMessage(e.getMessage()); // } catch (Exception e) {
redisTemplate.opsForValue().set(redisKey, task, TASK_EXPIRE_SECONDS, TimeUnit.SECONDS); // task.setStatus("failed");
} // task.setErrorMessage(e.getMessage());
}); // redisTemplate.opsForValue().set(redisKey, task, TASK_EXPIRE_SECONDS, TimeUnit.SECONDS);
// }
// });
return ResponseEntity.accepted().body(task); return ResponseEntity.accepted().body(task);
} }

View File

@ -1,7 +1,7 @@
package com.bonus.bmw.mapper; package com.bonus.bmw.mapper;
import com.bonus.bmw.domain.po.ZipFileMapping;
import com.bonus.bmw.domain.vo.BmWorkerAtt; import com.bonus.bmw.domain.vo.BmWorkerAtt;
import com.bonus.system.api.domain.ZipFileMapping;
import java.util.List; import java.util.List;

View File

@ -1,7 +1,7 @@
package com.bonus.bmw.service; package com.bonus.bmw.service;
import com.bonus.bmw.domain.po.ZipFileMapping;
import com.bonus.bmw.domain.vo.BmWorkerAtt; import com.bonus.bmw.domain.vo.BmWorkerAtt;
import com.bonus.system.api.domain.ZipFileMapping;
import java.util.List; import java.util.List;

View File

@ -1,21 +1,17 @@
package com.bonus.bmw.service.impl; package com.bonus.bmw.service.impl;
import com.bonus.bmw.domain.dto.WebFileDto;
import com.bonus.common.core.constant.SecurityConstants; import com.bonus.common.core.constant.SecurityConstants;
import com.bonus.common.core.domain.R; import com.bonus.common.core.domain.R;
import com.bonus.system.api.RemoteUploadUtilsService; import com.bonus.system.api.RemoteUploadUtilsService;
import com.bonus.system.api.domain.DownloadTask;
import com.bonus.system.api.domain.FileVo; import com.bonus.system.api.domain.FileVo;
import com.bonus.system.api.domain.ZipFileMapping;
import com.bonus.system.api.model.UploadFileVo; import com.bonus.system.api.model.UploadFileVo;
import lombok.extern.slf4j.Slf4j; import lombok.extern.slf4j.Slf4j;
import org.springframework.stereotype.Service; import org.springframework.stereotype.Service;
import org.springframework.web.bind.annotation.PostMapping;
import org.springframework.web.bind.annotation.RequestParam;
import org.springframework.web.bind.annotation.RequestPart;
import org.springframework.web.bind.annotation.ResponseBody;
import org.springframework.web.multipart.MultipartFile; import org.springframework.web.multipart.MultipartFile;
import javax.annotation.Resource; import javax.annotation.Resource;
import javax.servlet.http.HttpServletRequest;
import java.util.List; import java.util.List;
/** /**
@ -169,4 +165,15 @@ public class FileUploadUtils {
return null; return null;
} }
/**
* 打包文件
*/
public DownloadTask zipFile(List<ZipFileMapping> objectNames){
R<DownloadTask> res = service.zipFile(objectNames, SecurityConstants.INNER);
if(res.getCode()==R.SUCCESS){
return res.getData();
}
return null;
}
} }

View File

@ -1,9 +1,9 @@
package com.bonus.bmw.service.impl; package com.bonus.bmw.service.impl;
import com.bonus.bmw.domain.po.ZipFileMapping;
import com.bonus.bmw.domain.vo.BmWorkerAtt; import com.bonus.bmw.domain.vo.BmWorkerAtt;
import com.bonus.bmw.mapper.ZipDownloadMapper; import com.bonus.bmw.mapper.ZipDownloadMapper;
import com.bonus.bmw.service.ZipDownloadService; import com.bonus.bmw.service.ZipDownloadService;
import com.bonus.system.api.domain.ZipFileMapping;
import lombok.extern.slf4j.Slf4j; import lombok.extern.slf4j.Slf4j;
import org.springframework.stereotype.Service; import org.springframework.stereotype.Service;

View File

@ -392,7 +392,7 @@
</select> </select>
<update id="updateWorkerPost"> <update id="updateWorkerPost">
update bm_worker_ein_msg set post_id = #{postId},post_name = #{postName} where worker_id = #{id}; update bm_worker_ein_msg set post_id = #{postId},post_name = #{postName} where worker_id = #{id} and pro_id = #{proId};
update bm_worker_ein_pro_record set post_id = #{postId},post_name = #{postName} where worker_id = #{id}; update bm_worker_ein_pro_record set post_id = #{postId},post_name = #{postName} where worker_id = #{id} and pro_id = #{proId};
</update> </update>
</mapper> </mapper>

View File

@ -29,7 +29,7 @@
</where> </where>
</select> </select>
<select id="getFileList" resultType="com.bonus.bmw.domain.po.ZipFileMapping"> <select id="getFileList" resultType="com.bonus.system.api.domain.ZipFileMapping">
SELECT SELECT
bf.file_key as objectName, bf.file_key as objectName,
CONCAT(bwem.pro_name,'/',bwem.sub_name,'/',bwem.team_name,'/',pw.`name`,'/','人脸/',pw.`name`,bf.file_name) as targetPath CONCAT(bwem.pro_name,'/',bwem.sub_name,'/',bwem.team_name,'/',pw.`name`,'/','人脸/',pw.`name`,bf.file_name) as targetPath

View File

@ -3,7 +3,9 @@ package com.bonus.file.controller;
import cn.hutool.core.util.ObjectUtil; import cn.hutool.core.util.ObjectUtil;
import com.bonus.common.core.domain.R; import com.bonus.common.core.domain.R;
import com.bonus.file.service.impl.FileUtilsServiceImpl; import com.bonus.file.service.impl.FileUtilsServiceImpl;
import com.bonus.system.api.domain.DownloadTask;
import com.bonus.system.api.domain.FileVo; import com.bonus.system.api.domain.FileVo;
import com.bonus.system.api.domain.ZipFileMapping;
import com.bonus.system.api.model.UploadFileVo; import com.bonus.system.api.model.UploadFileVo;
import lombok.extern.slf4j.Slf4j; import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Autowired;
@ -231,4 +233,9 @@ public class FileUtilController {
return R.ok(service.getFileUrl(filePath,bucketName)); return R.ok(service.getFileUrl(filePath,bucketName));
} }
@PostMapping("/zipFile")
public R<DownloadTask> zipFile(@RequestBody List<ZipFileMapping> objectNames) {
return R.ok(service.zipFile(objectNames));
}
} }

View File

@ -6,23 +6,44 @@ import com.bonus.common.security.utils.SecurityUtils;
import com.bonus.file.config.MinioConfig; import com.bonus.file.config.MinioConfig;
import com.bonus.file.mapper.FileUtilMapper; import com.bonus.file.mapper.FileUtilMapper;
import com.bonus.file.utils.MinioUtil; import com.bonus.file.utils.MinioUtil;
import com.bonus.system.api.domain.DownloadTask;
import com.bonus.system.api.domain.ZipFileMapping;
import com.bonus.system.api.model.UploadFileVo; import com.bonus.system.api.model.UploadFileVo;
import io.minio.GetObjectArgs;
import io.minio.MinioClient;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.compress.archivers.zip.Zip64Mode;
import org.apache.commons.compress.archivers.zip.ZipArchiveEntry;
import org.apache.commons.compress.archivers.zip.ZipArchiveOutputStream;
import org.hibernate.validator.internal.util.StringHelper; import org.hibernate.validator.internal.util.StringHelper;
import org.slf4j.Logger; import org.slf4j.Logger;
import org.slf4j.LoggerFactory; import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.data.redis.core.RedisTemplate;
import org.springframework.stereotype.Service; import org.springframework.stereotype.Service;
import org.springframework.web.multipart.MultipartFile; import org.springframework.web.multipart.MultipartFile;
import javax.annotation.Resource;
import java.io.BufferedOutputStream;
import java.io.FileOutputStream;
import java.io.InputStream;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.Base64; import java.util.Base64;
import java.util.List; import java.util.List;
import java.util.UUID;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.TimeUnit;
/** /**
* 通用文件上传 * 通用文件上传
* @author 黑子 * @author 黑子
*/ */
@Service @Service
@Slf4j
public class FileUtilsServiceImpl { public class FileUtilsServiceImpl {
public static String floor="relname"; public static String floor="relname";
@ -32,12 +53,30 @@ public class FileUtilsServiceImpl {
@Autowired @Autowired
private MinioUtil minioUtils; private MinioUtil minioUtils;
@Resource
private MinioClient minioClient;
@Autowired @Autowired
private MinioConfig minioConfig; private MinioConfig minioConfig;
@Autowired @Autowired
private FileUtilMapper mapper; private FileUtilMapper mapper;
@Resource
private RedisTemplate<String, DownloadTask> redisTemplate;
/**
* 压缩任务缓存key前缀
*/
private static final String TASK_PREFIX = "zip_task:";
/**
* 压缩任务缓存时间
*/
private static final long TASK_EXPIRE_SECONDS = 2 * 3600; // 2小时过期
/**
* 压缩任务线程池
*/
private final ExecutorService executor = Executors.newFixedThreadPool(4);
public UploadFileVo upload(String sourceTable, String sourceId, String sourceType,String prefix,String bast64,String bucketName) { public UploadFileVo upload(String sourceTable, String sourceId, String sourceType,String prefix,String bast64,String bucketName) {
String suffix=null; String suffix=null;
@ -472,4 +511,75 @@ public class FileUtilsServiceImpl {
} }
return null; return null;
} }
public DownloadTask zipFile(List<ZipFileMapping> objectNames) {
String taskId = UUID.randomUUID().toString();
String redisKey = TASK_PREFIX + taskId;
DownloadTask task = new DownloadTask(taskId);
// 保存到 Redis设置过期时间
redisTemplate.opsForValue().set(redisKey, task, TASK_EXPIRE_SECONDS, TimeUnit.SECONDS);
// 异步执行打包
executor.submit(() -> {
try {
String zipName = taskId + ".zip";
String zipPath = "/tmp/downloads/" + zipName;
generateZipFromMinIO(objectNames, zipPath);
// 更新任务状态
task.setStatus("completed");
task.setDownloadUrl("/" + zipName);
redisTemplate.opsForValue().set(redisKey, task, TASK_EXPIRE_SECONDS, TimeUnit.SECONDS);
} catch (Exception e) {
task.setStatus("failed");
task.setErrorMessage(e.getMessage());
redisTemplate.opsForValue().set(redisKey, task, TASK_EXPIRE_SECONDS, TimeUnit.SECONDS);
}
});
return task;
}
/**
* 使用 Apache Commons Compress MinIO 打包 ZIP支持 UTF-8 中文
*/
public void generateZipFromMinIO( List<ZipFileMapping> fileMappings, String zipPath) throws Exception {
Path outputPath = Paths.get(zipPath);
Files.createDirectories(outputPath.getParent());
try (FileOutputStream fos = new FileOutputStream(outputPath.toFile());
BufferedOutputStream bos = new BufferedOutputStream(fos, 64 * 1024);
ZipArchiveOutputStream zipOut = new ZipArchiveOutputStream(bos)) {
zipOut.setEncoding("UTF-8");
zipOut.setUseZip64(Zip64Mode.AsNeeded);
for (ZipFileMapping mapping : fileMappings) {
String objectName = mapping.getObjectName();
String targetPath = mapping.getTargetPath();
// 安全校验
if (objectName == null || targetPath == null ||
objectName.contains("..") || targetPath.contains("..") ||
objectName.startsWith("/") || targetPath.startsWith("/")) {
continue; // 跳过非法项
}
// 统一使用正斜杠兼容 Windows
String entryName = targetPath.replace('\\', '/');
try (InputStream objectStream = minioClient.getObject(
GetObjectArgs.builder()
.bucket(minioConfig.getBucketName())
.object(objectName)
.build())) {
ZipArchiveEntry entry = new ZipArchiveEntry(entryName);
zipOut.putArchiveEntry(entry);
byte[] buffer = new byte[8192];
int len;
while ((len = objectStream.read(buffer)) > 0) {
zipOut.write(buffer, 0, len);
}
zipOut.closeArchiveEntry();
} catch (Exception e) {
log.error(e.toString(), e);
System.err.println("Failed to process object: " + objectName + " -> " + targetPath);
// 可选择继续或中断
}
}
zipOut.finish();
}
}
} }