文件存储服务

bonus-modules/bonus-mongodb/src/main/resources/bootstrap.yml

@@ -3,7 +3,11 @@ server:
   port: 9206
 
 # Spring
-spring: 
+spring:
+  servlet:
+    multipart:
+      max-file-size: 5GB
+      max-request-size: 5GB
   application:
     # 应用名称
     name: bonus-mongodb

bonus-modules/bonus-obs/src/main/java/com/bonus/obs/controller/ObsController.java

@@ -54,8 +54,8 @@ public class ObsController {
      *
      * @param objectKey obs文件存储地址
      */
-    @GetMapping("/download")
+    @GetMapping("/downloadFile")
-    public void download(HttpServletResponse response, @RequestParam String objectKey) {
+    public void downloadFile(HttpServletResponse response, @RequestParam String objectKey) {
         R<ObsObject> obsObjectR = service.downloadFile(objectKey);
         try {
             if (R.isError(obsObjectR)) {

bonus-modules/bonus-obs/src/main/java/com/bonus/obs/domain/ObsInfo.java

@@ -26,7 +26,7 @@ public class ObsInfo {
      * 对象的大小。
      * 以字符串形式表示，可能包含单位（如字节、KB、MB等）。
      */
-    private String length;
+    private Long length;
 
     /**
      * 对象的文件类型。

bonus-modules/bonus-obs/src/main/java/com/bonus/obs/service/impl/ObsServiceImpl.java

@@ -8,15 +8,17 @@ import com.bonus.obs.service.ObsService;
 import com.bonus.obs.utils.ObsUtils;
 import com.obs.services.model.DeleteObjectResult;
 import com.obs.services.model.ObsObject;
-import com.obs.services.model.PutObjectResult;
 import org.springframework.stereotype.Service;
 import org.springframework.web.multipart.MultipartFile;
 
 import javax.annotation.Resource;
-import java.io.File;
 import java.util.ArrayList;
 import java.util.List;
 import java.util.Objects;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
+import java.util.stream.Collectors;
 
 @Service
 public class ObsServiceImpl implements ObsService {
@@ -36,8 +38,9 @@ public class ObsServiceImpl implements ObsService {
             String extension = originalFilename.substring(originalFilename.lastIndexOf('.'));
             String objectKey = UuidUtils.generateUuid() + extension;
             objectKey = FileUtils.generateObjectName(objectKey);
+
             ObsInfo obsInfo = obsUtils.uploadFile(objectKey, FileUtils.multipartFileToFile(file));
-            return R.ok(obsInfo);
+            return R.ok(obsInfo, "文件上传成功");
         } catch (Exception e) {
             return R.fail(e.getMessage());
         }
@@ -71,15 +74,27 @@ public class ObsServiceImpl implements ObsService {
      */
     @Override
     public R<List<ObsInfo>> uploadFiles(MultipartFile[] files) {
+        // 调整线程池大小
+        int threadPoolSize = 10;
+        ExecutorService executorService = Executors.newFixedThreadPool(threadPoolSize);
         try {
-            List<ObsInfo> obsInfos = new ArrayList<>();
+            List<Future<ObsInfo>> futures = new ArrayList<>();
             for (MultipartFile multipartFile : files) {
-                String originalFilename = Objects.requireNonNull(multipartFile.getOriginalFilename(), "文件名不能为空");
+                futures.add(executorService.submit(() -> {
-                String extension = originalFilename.substring(originalFilename.lastIndexOf('.'));
+                    String originalFilename = Objects.requireNonNull(multipartFile.getOriginalFilename(), "文件名不能为空");
-                String objectKey = UuidUtils.generateUuid() + extension;
+                    String extension = originalFilename.substring(originalFilename.lastIndexOf('.'));
-                objectKey = FileUtils.generateObjectName(objectKey);
+                    String objectKey = UuidUtils.generateUuid() + extension;
-                obsInfos.add(obsUtils.uploadFile(objectKey, FileUtils.multipartFileToFile(multipartFile)));
+                    objectKey = FileUtils.generateObjectName(objectKey);
+                    return obsUtils.uploadFile(objectKey, FileUtils.multipartFileToFile(multipartFile));
+                }));
             }
+            List<ObsInfo> obsInfos = futures.stream().map(future -> {
+                try {
+                    return future.get();
+                } catch (Exception e) {
+                    throw new RuntimeException(e);
+                }
+            }).collect(Collectors.toList());
             return R.ok(obsInfos, "文件上传成功");
         } catch (Exception e) {
             return R.fail("File upload failed.");

bonus-modules/bonus-obs/src/main/java/com/bonus/obs/utils/ObsUtils.java

@@ -17,6 +17,7 @@ import java.io.File;
 import java.io.FileOutputStream;
 import java.io.IOException;
 import java.io.InputStream;
+import java.net.URLConnection;
 
 @Service
 public class ObsUtils {
@@ -46,12 +47,7 @@ public class ObsUtils {
      */
     public ObsInfo uploadFile(String objectKey, File file) {
         PutObjectResult putObjectResult = obsClient.putObject(obsConfig.getBucket(), objectKey, file);
-
+        return ObsInfo.builder().bucketName(obsConfig.getBucket()).name(FileUtils.getName(objectKey)).fileType(FileUtils.getName(objectKey).substring(FileUtils.getName(objectKey).lastIndexOf('.')).toLowerCase()).length(file.length()).path(objectKey).build();
-        return ObsInfo.builder()
-                .bucketName(obsConfig.getBucket())
-                .name(FileUtils.getName(objectKey))
-                .path(objectKey)
-                .build();
     }
 
     /**
@@ -75,7 +71,7 @@ public class ObsUtils {
         if (!doesObjectExist(objectKey)) {
             return R.fail("文件不存在");
         }
-        return R.ok(obsClient.deleteObject(obsConfig.getBucket(), objectKey));
+        return R.ok(null, "文件删除成功");
     }
 
     /**

bonus-modules/bonus-obs/src/main/resources/bootstrap.yml

@@ -3,7 +3,11 @@ server:
   port: 9205
 
 # Spring
-spring: 
+spring:
+  servlet:
+    multipart:
+      max-file-size: 5GB
+      max-request-size: 5GB
   application:
     # 应用名称
     name: bonus-obs

bonus-modules/bonus-oss/src/main/java/com/bonus/oss/service/impl/OssServiceImpl.java

@@ -3,7 +3,6 @@ package com.bonus.oss.service.impl;
 import com.alibaba.nacos.common.utils.UuidUtils;
 import com.aliyun.oss.model.OSSObject;
 import com.bonus.common.core.domain.R;
-import com.bonus.common.core.utils.StringUtils;
 import com.bonus.common.core.utils.file.FileUtils;
 import com.bonus.oss.domain.OssInfo;
 import com.bonus.oss.service.OssService;
@@ -14,12 +13,13 @@ import org.springframework.stereotype.Service;
 import org.springframework.web.multipart.MultipartFile;
 
 import javax.annotation.Resource;
-import java.io.File;
 import java.util.ArrayList;
 import java.util.List;
 import java.util.Objects;
 import java.util.concurrent.ExecutorService;
 import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
+import java.util.stream.Collectors;
 
 @Service
 public class OssServiceImpl implements OssService {
@@ -27,8 +27,6 @@ public class OssServiceImpl implements OssService {
     @Resource
     private OssUtils ossUtils;
 
-    private ExecutorService executor = Executors.newFixedThreadPool(5); // 限制并发线程数为5
-
     /**
      * 文件上传
      *
@@ -90,17 +88,28 @@ public class OssServiceImpl implements OssService {
      */
     @Override
     public R<List<OssInfo>> uploadFiles(MultipartFile[] files) {
+        // 调整线程池大小
+        int threadPoolSize = 10;
+        ExecutorService executorService = Executors.newFixedThreadPool(threadPoolSize);
         try {
-            List<OssInfo> ossInfos = new ArrayList<>();
+            List<Future<OssInfo>> futures = new ArrayList<>();
             for (MultipartFile multipartFile : files) {
-                String originalFilename = Objects.requireNonNull(multipartFile.getOriginalFilename(), "文件名不能为空");
+                futures.add(executorService.submit(() -> {
-                String extension = originalFilename.substring(originalFilename.lastIndexOf('.'));
+                    String originalFilename = Objects.requireNonNull(multipartFile.getOriginalFilename(), "文件名不能为空");
-                String objectKey = UuidUtils.generateUuid() + extension;
+                    String extension = originalFilename.substring(originalFilename.lastIndexOf('.'));
-                File file = FileUtils.multipartFileToFile(multipartFile);
+                    String objectKey = UuidUtils.generateUuid() + extension;
-                objectKey = FileUtils.generateObjectName(objectKey);
+                    objectKey = FileUtils.generateObjectName(objectKey);
-                ossInfos.add(ossUtils.upload(objectKey, file).getData());
+                    return ossUtils.upload(objectKey, FileUtils.multipartFileToFile(multipartFile)).getData();
+                }));
             }
-            return R.ok(ossInfos, "文件上传成功");
+            List<OssInfo> obsInfos = futures.stream().map(future -> {
+                try {
+                    return future.get();
+                } catch (Exception e) {
+                    throw new RuntimeException(e);
+                }
+            }).collect(Collectors.toList());
+            return R.ok(obsInfos, "文件上传成功");
         } catch (Exception e) {
             return R.fail("File upload failed.");
         }

bonus-modules/bonus-oss/src/main/java/com/bonus/oss/utils/OssUtils.java

@@ -2,8 +2,7 @@ package com.bonus.oss.utils;
 
 import com.aliyun.oss.OSS;
 import com.aliyun.oss.OSSClientBuilder;
-import com.aliyun.oss.model.OSSObject;
+import com.aliyun.oss.model.*;
-import com.aliyun.oss.model.ObjectMetadata;
 import com.bonus.common.core.domain.R;
 import com.bonus.common.core.text.Convert;
 import com.bonus.common.core.utils.file.FileUtils;
@@ -17,6 +16,13 @@ import org.springframework.stereotype.Service;
 import javax.annotation.PostConstruct;
 import javax.annotation.Resource;
 import java.io.File;
+import java.io.FileInputStream;
+import java.io.InputStream;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
 
 /**
  * OSS存储服务类
@@ -51,7 +57,12 @@ public class OssUtils {
      */
     public R<OssInfo> upload(String objectKey, File file) {
         try {
-            ossClient.putObject(ossConfig.getBucket(), objectKey, file);
+            if (file.length() < 10 * 1024 * 1024L) {
+                ossClient.putObject(ossConfig.getBucket(), objectKey, file);
+            } else {
+                ossMultipartParallelUpload(objectKey, file);
+            }
+
             return R.ok(getInfo(objectKey), "文件上传成功");
         } catch (Exception e) {
             LOGGER.error("文件上传失败", e);
@@ -63,6 +74,76 @@ public class OssUtils {
         }
     }
 
+    /**
+     * 并行方式实现OSS的大文件分片上传。
+     * 通过多线程并行上传文件的各个分片，提高上传效率。
+     *
+     * @param objectKey OSS对象的关键字，即文件名。
+     * @param file      需要上传到OSS的本地文件。
+     */
+    public void ossMultipartParallelUpload(String objectKey, File file) {
+        // 定义线程池大小，用于并发上传分片
+        // 调整线程池大小
+        int threadPoolSize = 10;
+        ExecutorService executorService = Executors.newFixedThreadPool(threadPoolSize);
+        try {
+            // 初始化分片上传，获取上传ID
+            // 1. 初始化分段上传
+            InitiateMultipartUploadRequest request = new InitiateMultipartUploadRequest(ossConfig.getBucket(), objectKey);
+            InitiateMultipartUploadResult result = ossClient.initiateMultipartUpload(request);
+            String uploadId = result.getUploadId();
+
+            // 计算分片大小和分片数量
+            // 2. 上传分段
+            // 调整分段大小，例如10MB
+            final long partSize = 10 * 1024 * 1024L;
+            long fileLength = file.length();
+            int partCount = (int) (fileLength / partSize);
+            if (fileLength % partSize != 0) {
+                partCount++;
+            }
+
+            // 使用Future列表保存每个分片上传的结果
+            List<Future<PartETag>> futures = new ArrayList<>();
+            for (int i = 0; i < partCount; i++) {
+                final int partNumber = i + 1;
+                final long startPos = i * partSize;
+                final long curPartSize = (i + 1 == partCount) ? (fileLength - startPos) : partSize;
+                futures.add(executorService.submit(() -> {
+                    try (InputStream instream = new FileInputStream(file)) {
+                        instream.skip(startPos);
+                        UploadPartRequest uploadPartRequest = new UploadPartRequest();
+                        uploadPartRequest.setBucketName(ossConfig.getBucket());
+                        uploadPartRequest.setKey(objectKey);
+                        uploadPartRequest.setUploadId(uploadId);
+                        uploadPartRequest.setInputStream(instream);
+                        uploadPartRequest.setPartSize(curPartSize);
+                        uploadPartRequest.setPartNumber(partNumber);
+                        UploadPartResult uploadPartResult = ossClient.uploadPart(uploadPartRequest);
+                        return uploadPartResult.getPartETag();
+                    }
+                }));
+            }
+
+            // 收集所有分片的上传结果
+            List<PartETag> partETags = new ArrayList<>();
+            for (Future<PartETag> future : futures) {
+                partETags.add(future.get());
+            }
+
+            // 完成分片上传
+            // 4. 完成分段上传
+            CompleteMultipartUploadRequest completeMultipartUploadRequest = new CompleteMultipartUploadRequest(ossConfig.getBucket(), objectKey, uploadId, partETags);
+            ossClient.completeMultipartUpload(completeMultipartUploadRequest);
+        } catch (Exception e) {
+            e.printStackTrace();
+        } finally {
+            // 关闭线程池
+            executorService.shutdown();
+        }
+    }
+
+
     /**
      * 获取文件在OSS中的信息
      *
@@ -75,7 +156,7 @@ public class OssUtils {
             return OssInfo.builder()
                     .name(FileUtils.getName(objectKey))
                     .bucketName(ossConfig.getBucket())
-                    .fileType(objectMetadata.getContentType())
+                    .fileType(FileUtils.getName(objectKey).substring(FileUtils.getName(objectKey).lastIndexOf('.')).toLowerCase())
                     .length(Convert.toStr(objectMetadata.getContentLength()))
                     .path(objectKey).build();
         } catch (Exception e) {
@@ -114,7 +195,7 @@ public class OssUtils {
                 return R.fail("文件不存在");
             }
             ossClient.deleteObject(ossConfig.getBucket(), objectKey);
-            return R.ok(null,"文件删除成功");
+            return R.ok(null, "文件删除成功");
         } catch (Exception e) {
             LOGGER.error("文件删除失败", e);
             return R.fail("文件删除失败");

bonus-modules/bonus-oss/src/main/resources/bootstrap.yml

@@ -1,9 +1,12 @@
 # Tomcat
 server:
   port: 9204
-
 # Spring
-spring: 
+spring:
+  servlet:
+    multipart:
+      max-file-size: 5GB
+      max-request-size: 5GB
   application:
     # 应用名称
     name: bonus-oss