diff --git a/bonus-modules/bonus-mongodb/src/main/resources/bootstrap.yml b/bonus-modules/bonus-mongodb/src/main/resources/bootstrap.yml index 60389c4..b6dab15 100644 --- a/bonus-modules/bonus-mongodb/src/main/resources/bootstrap.yml +++ b/bonus-modules/bonus-mongodb/src/main/resources/bootstrap.yml @@ -3,7 +3,11 @@ server: port: 9206 # Spring -spring: +spring: + servlet: + multipart: + max-file-size: 5GB + max-request-size: 5GB application: # 应用名称 name: bonus-mongodb diff --git a/bonus-modules/bonus-obs/src/main/java/com/bonus/obs/controller/ObsController.java b/bonus-modules/bonus-obs/src/main/java/com/bonus/obs/controller/ObsController.java index f972de7..756be18 100644 --- a/bonus-modules/bonus-obs/src/main/java/com/bonus/obs/controller/ObsController.java +++ b/bonus-modules/bonus-obs/src/main/java/com/bonus/obs/controller/ObsController.java @@ -54,8 +54,8 @@ public class ObsController { * * @param objectKey obs文件存储地址 */ - @GetMapping("/download") - public void download(HttpServletResponse response, @RequestParam String objectKey) { + @GetMapping("/downloadFile") + public void downloadFile(HttpServletResponse response, @RequestParam String objectKey) { R obsObjectR = service.downloadFile(objectKey); try { if (R.isError(obsObjectR)) { diff --git a/bonus-modules/bonus-obs/src/main/java/com/bonus/obs/domain/ObsInfo.java b/bonus-modules/bonus-obs/src/main/java/com/bonus/obs/domain/ObsInfo.java index 4253fb8..dd9bcf6 100644 --- a/bonus-modules/bonus-obs/src/main/java/com/bonus/obs/domain/ObsInfo.java +++ b/bonus-modules/bonus-obs/src/main/java/com/bonus/obs/domain/ObsInfo.java @@ -26,7 +26,7 @@ public class ObsInfo { * 对象的大小。 * 以字符串形式表示，可能包含单位（如字节、KB、MB等）。 */ - private String length; + private Long length; /** * 对象的文件类型。 diff --git a/bonus-modules/bonus-obs/src/main/java/com/bonus/obs/service/impl/ObsServiceImpl.java b/bonus-modules/bonus-obs/src/main/java/com/bonus/obs/service/impl/ObsServiceImpl.java index 229791a..e83fd99 100644 --- a/bonus-modules/bonus-obs/src/main/java/com/bonus/obs/service/impl/ObsServiceImpl.java +++ b/bonus-modules/bonus-obs/src/main/java/com/bonus/obs/service/impl/ObsServiceImpl.java @@ -8,15 +8,17 @@ import com.bonus.obs.service.ObsService; import com.bonus.obs.utils.ObsUtils; import com.obs.services.model.DeleteObjectResult; import com.obs.services.model.ObsObject; -import com.obs.services.model.PutObjectResult; import org.springframework.stereotype.Service; import org.springframework.web.multipart.MultipartFile; import javax.annotation.Resource; -import java.io.File; import java.util.ArrayList; import java.util.List; import java.util.Objects; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.stream.Collectors; @Service public class ObsServiceImpl implements ObsService { @@ -36,8 +38,9 @@ public class ObsServiceImpl implements ObsService { String extension = originalFilename.substring(originalFilename.lastIndexOf('.')); String objectKey = UuidUtils.generateUuid() + extension; objectKey = FileUtils.generateObjectName(objectKey); + ObsInfo obsInfo = obsUtils.uploadFile(objectKey, FileUtils.multipartFileToFile(file)); - return R.ok(obsInfo); + return R.ok(obsInfo, "文件上传成功"); } catch (Exception e) { return R.fail(e.getMessage()); } @@ -71,15 +74,27 @@ public class ObsServiceImpl implements ObsService { */ @Override public R> uploadFiles(MultipartFile[] files) { + // 调整线程池大小 + int threadPoolSize = 10; + ExecutorService executorService = Executors.newFixedThreadPool(threadPoolSize); try { - List obsInfos = new ArrayList<>(); + List> futures = new ArrayList<>(); for (MultipartFile multipartFile : files) { - String originalFilename = Objects.requireNonNull(multipartFile.getOriginalFilename(), "文件名不能为空"); - String extension = originalFilename.substring(originalFilename.lastIndexOf('.')); - String objectKey = UuidUtils.generateUuid() + extension; - objectKey = FileUtils.generateObjectName(objectKey); - obsInfos.add(obsUtils.uploadFile(objectKey, FileUtils.multipartFileToFile(multipartFile))); + futures.add(executorService.submit(() -> { + String originalFilename = Objects.requireNonNull(multipartFile.getOriginalFilename(), "文件名不能为空"); + String extension = originalFilename.substring(originalFilename.lastIndexOf('.')); + String objectKey = UuidUtils.generateUuid() + extension; + objectKey = FileUtils.generateObjectName(objectKey); + return obsUtils.uploadFile(objectKey, FileUtils.multipartFileToFile(multipartFile)); + })); } + List obsInfos = futures.stream().map(future -> { + try { + return future.get(); + } catch (Exception e) { + throw new RuntimeException(e); + } + }).collect(Collectors.toList()); return R.ok(obsInfos, "文件上传成功"); } catch (Exception e) { return R.fail("File upload failed."); diff --git a/bonus-modules/bonus-obs/src/main/java/com/bonus/obs/utils/ObsUtils.java b/bonus-modules/bonus-obs/src/main/java/com/bonus/obs/utils/ObsUtils.java index 92b788f..6af1304 100644 --- a/bonus-modules/bonus-obs/src/main/java/com/bonus/obs/utils/ObsUtils.java +++ b/bonus-modules/bonus-obs/src/main/java/com/bonus/obs/utils/ObsUtils.java @@ -17,6 +17,7 @@ import java.io.File; import java.io.FileOutputStream; import java.io.IOException; import java.io.InputStream; +import java.net.URLConnection; @Service public class ObsUtils { @@ -46,12 +47,7 @@ public class ObsUtils { */ public ObsInfo uploadFile(String objectKey, File file) { PutObjectResult putObjectResult = obsClient.putObject(obsConfig.getBucket(), objectKey, file); - - return ObsInfo.builder() - .bucketName(obsConfig.getBucket()) - .name(FileUtils.getName(objectKey)) - .path(objectKey) - .build(); + return ObsInfo.builder().bucketName(obsConfig.getBucket()).name(FileUtils.getName(objectKey)).fileType(FileUtils.getName(objectKey).substring(FileUtils.getName(objectKey).lastIndexOf('.')).toLowerCase()).length(file.length()).path(objectKey).build(); } /** @@ -75,7 +71,7 @@ public class ObsUtils { if (!doesObjectExist(objectKey)) { return R.fail("文件不存在"); } - return R.ok(obsClient.deleteObject(obsConfig.getBucket(), objectKey)); + return R.ok(null, "文件删除成功"); } /** diff --git a/bonus-modules/bonus-obs/src/main/resources/bootstrap.yml b/bonus-modules/bonus-obs/src/main/resources/bootstrap.yml index 9046f6f..fc67aa2 100644 --- a/bonus-modules/bonus-obs/src/main/resources/bootstrap.yml +++ b/bonus-modules/bonus-obs/src/main/resources/bootstrap.yml @@ -3,7 +3,11 @@ server: port: 9205 # Spring -spring: +spring: + servlet: + multipart: + max-file-size: 5GB + max-request-size: 5GB application: # 应用名称 name: bonus-obs diff --git a/bonus-modules/bonus-oss/src/main/java/com/bonus/oss/service/impl/OssServiceImpl.java b/bonus-modules/bonus-oss/src/main/java/com/bonus/oss/service/impl/OssServiceImpl.java index e28b04e..89ab81a 100644 --- a/bonus-modules/bonus-oss/src/main/java/com/bonus/oss/service/impl/OssServiceImpl.java +++ b/bonus-modules/bonus-oss/src/main/java/com/bonus/oss/service/impl/OssServiceImpl.java @@ -3,7 +3,6 @@ package com.bonus.oss.service.impl; import com.alibaba.nacos.common.utils.UuidUtils; import com.aliyun.oss.model.OSSObject; import com.bonus.common.core.domain.R; -import com.bonus.common.core.utils.StringUtils; import com.bonus.common.core.utils.file.FileUtils; import com.bonus.oss.domain.OssInfo; import com.bonus.oss.service.OssService; @@ -14,12 +13,13 @@ import org.springframework.stereotype.Service; import org.springframework.web.multipart.MultipartFile; import javax.annotation.Resource; -import java.io.File; import java.util.ArrayList; import java.util.List; import java.util.Objects; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.stream.Collectors; @Service public class OssServiceImpl implements OssService { @@ -27,8 +27,6 @@ public class OssServiceImpl implements OssService { @Resource private OssUtils ossUtils; - private ExecutorService executor = Executors.newFixedThreadPool(5); // 限制并发线程数为5 - /** * 文件上传 * @@ -90,17 +88,28 @@ public class OssServiceImpl implements OssService { */ @Override public R> uploadFiles(MultipartFile[] files) { + // 调整线程池大小 + int threadPoolSize = 10; + ExecutorService executorService = Executors.newFixedThreadPool(threadPoolSize); try { - List ossInfos = new ArrayList<>(); + List> futures = new ArrayList<>(); for (MultipartFile multipartFile : files) { - String originalFilename = Objects.requireNonNull(multipartFile.getOriginalFilename(), "文件名不能为空"); - String extension = originalFilename.substring(originalFilename.lastIndexOf('.')); - String objectKey = UuidUtils.generateUuid() + extension; - File file = FileUtils.multipartFileToFile(multipartFile); - objectKey = FileUtils.generateObjectName(objectKey); - ossInfos.add(ossUtils.upload(objectKey, file).getData()); + futures.add(executorService.submit(() -> { + String originalFilename = Objects.requireNonNull(multipartFile.getOriginalFilename(), "文件名不能为空"); + String extension = originalFilename.substring(originalFilename.lastIndexOf('.')); + String objectKey = UuidUtils.generateUuid() + extension; + objectKey = FileUtils.generateObjectName(objectKey); + return ossUtils.upload(objectKey, FileUtils.multipartFileToFile(multipartFile)).getData(); + })); } - return R.ok(ossInfos, "文件上传成功"); + List obsInfos = futures.stream().map(future -> { + try { + return future.get(); + } catch (Exception e) { + throw new RuntimeException(e); + } + }).collect(Collectors.toList()); + return R.ok(obsInfos, "文件上传成功"); } catch (Exception e) { return R.fail("File upload failed."); } diff --git a/bonus-modules/bonus-oss/src/main/java/com/bonus/oss/utils/OssUtils.java b/bonus-modules/bonus-oss/src/main/java/com/bonus/oss/utils/OssUtils.java index 0d33f2b..21d8d2f 100644 --- a/bonus-modules/bonus-oss/src/main/java/com/bonus/oss/utils/OssUtils.java +++ b/bonus-modules/bonus-oss/src/main/java/com/bonus/oss/utils/OssUtils.java @@ -2,8 +2,7 @@ package com.bonus.oss.utils; import com.aliyun.oss.OSS; import com.aliyun.oss.OSSClientBuilder; -import com.aliyun.oss.model.OSSObject; -import com.aliyun.oss.model.ObjectMetadata; +import com.aliyun.oss.model.*; import com.bonus.common.core.domain.R; import com.bonus.common.core.text.Convert; import com.bonus.common.core.utils.file.FileUtils; @@ -17,6 +16,13 @@ import org.springframework.stereotype.Service; import javax.annotation.PostConstruct; import javax.annotation.Resource; import java.io.File; +import java.io.FileInputStream; +import java.io.InputStream; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; /** * OSS存储服务类 @@ -51,7 +57,12 @@ public class OssUtils { */ public R upload(String objectKey, File file) { try { - ossClient.putObject(ossConfig.getBucket(), objectKey, file); + if (file.length() < 10 * 1024 * 1024L) { + ossClient.putObject(ossConfig.getBucket(), objectKey, file); + } else { + ossMultipartParallelUpload(objectKey, file); + } + return R.ok(getInfo(objectKey), "文件上传成功"); } catch (Exception e) { LOGGER.error("文件上传失败", e); @@ -63,6 +74,76 @@ public class OssUtils { } } + /** + * 并行方式实现OSS的大文件分片上传。 + * 通过多线程并行上传文件的各个分片，提高上传效率。 + * + * @param objectKey OSS对象的关键字，即文件名。 + * @param file 需要上传到OSS的本地文件。 + */ + public void ossMultipartParallelUpload(String objectKey, File file) { + // 定义线程池大小，用于并发上传分片 + // 调整线程池大小 + int threadPoolSize = 10; + ExecutorService executorService = Executors.newFixedThreadPool(threadPoolSize); + try { + // 初始化分片上传，获取上传ID + // 1. 初始化分段上传 + InitiateMultipartUploadRequest request = new InitiateMultipartUploadRequest(ossConfig.getBucket(), objectKey); + InitiateMultipartUploadResult result = ossClient.initiateMultipartUpload(request); + String uploadId = result.getUploadId(); + + // 计算分片大小和分片数量 + // 2. 上传分段 + // 调整分段大小，例如10MB + final long partSize = 10 * 1024 * 1024L; + long fileLength = file.length(); + int partCount = (int) (fileLength / partSize); + if (fileLength % partSize != 0) { + partCount++; + } + + // 使用Future列表保存每个分片上传的结果 + List> futures = new ArrayList<>(); + for (int i = 0; i < partCount; i++) { + final int partNumber = i + 1; + final long startPos = i * partSize; + final long curPartSize = (i + 1 == partCount) ? (fileLength - startPos) : partSize; + futures.add(executorService.submit(() -> { + try (InputStream instream = new FileInputStream(file)) { + instream.skip(startPos); + UploadPartRequest uploadPartRequest = new UploadPartRequest(); + uploadPartRequest.setBucketName(ossConfig.getBucket()); + uploadPartRequest.setKey(objectKey); + uploadPartRequest.setUploadId(uploadId); + uploadPartRequest.setInputStream(instream); + uploadPartRequest.setPartSize(curPartSize); + uploadPartRequest.setPartNumber(partNumber); + UploadPartResult uploadPartResult = ossClient.uploadPart(uploadPartRequest); + return uploadPartResult.getPartETag(); + } + })); + } + + // 收集所有分片的上传结果 + List partETags = new ArrayList<>(); + for (Future future : futures) { + partETags.add(future.get()); + } + + // 完成分片上传 + // 4. 完成分段上传 + CompleteMultipartUploadRequest completeMultipartUploadRequest = new CompleteMultipartUploadRequest(ossConfig.getBucket(), objectKey, uploadId, partETags); + ossClient.completeMultipartUpload(completeMultipartUploadRequest); + } catch (Exception e) { + e.printStackTrace(); + } finally { + // 关闭线程池 + executorService.shutdown(); + } + } + + /** * 获取文件在OSS中的信息 * @@ -75,7 +156,7 @@ public class OssUtils { return OssInfo.builder() .name(FileUtils.getName(objectKey)) .bucketName(ossConfig.getBucket()) - .fileType(objectMetadata.getContentType()) + .fileType(FileUtils.getName(objectKey).substring(FileUtils.getName(objectKey).lastIndexOf('.')).toLowerCase()) .length(Convert.toStr(objectMetadata.getContentLength())) .path(objectKey).build(); } catch (Exception e) { @@ -114,7 +195,7 @@ public class OssUtils { return R.fail("文件不存在"); } ossClient.deleteObject(ossConfig.getBucket(), objectKey); - return R.ok(null,"文件删除成功"); + return R.ok(null, "文件删除成功"); } catch (Exception e) { LOGGER.error("文件删除失败", e); return R.fail("文件删除失败"); diff --git a/bonus-modules/bonus-oss/src/main/resources/bootstrap.yml b/bonus-modules/bonus-oss/src/main/resources/bootstrap.yml index 192147a..ba57d35 100644 --- a/bonus-modules/bonus-oss/src/main/resources/bootstrap.yml +++ b/bonus-modules/bonus-oss/src/main/resources/bootstrap.yml @@ -1,9 +1,12 @@ # Tomcat server: port: 9204 - # Spring -spring: +spring: + servlet: + multipart: + max-file-size: 5GB + max-request-size: 5GB application: # 应用名称 name: bonus-oss