bonus
/
Bonus-Cloud
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases 2 Wiki Activity

文件存储服务

This commit is contained in:
jiang 2024-07-05 10:29:09 +08:00
parent f3f36e4be7
commit d5acac245a
9 changed files with 152 additions and 40 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
bonus-modules/bonus-mongodb/src/main/resources/bootstrap.yml
View File

@ -4,6 +4,10 @@ server:
# Spring
spring:
servlet:
multipart:
max-file-size: 5GB
max-request-size: 5GB
application:
# 应用名称
name: bonus-mongodb

4
bonus-modules/bonus-obs/src/main/java/com/bonus/obs/controller/ObsController.java
View File

@ -54,8 +54,8 @@ public class ObsController {
*
* @param objectKey obs文件存储地址
*/
@GetMapping("/download")
public void download(HttpServletResponse response, @RequestParam String objectKey) {
@GetMapping("/downloadFile")
public void downloadFile(HttpServletResponse response, @RequestParam String objectKey) {
R<ObsObject> obsObjectR = service.downloadFile(objectKey);
try {
if (R.isError(obsObjectR)) {

2
bonus-modules/bonus-obs/src/main/java/com/bonus/obs/domain/ObsInfo.java
View File

@ -26,7 +26,7 @@ public class ObsInfo {
* 对象的大小。
* 以字符串形式表示，可能包含单位（如字节、KB、MB等）。
*/
private String length;
private Long length;
/**
* 对象的文件类型。

25
bonus-modules/bonus-obs/src/main/java/com/bonus/obs/service/impl/ObsServiceImpl.java
View File

@ -8,15 +8,17 @@ import com.bonus.obs.service.ObsService;
import com.bonus.obs.utils.ObsUtils;
import com.obs.services.model.DeleteObjectResult;
import com.obs.services.model.ObsObject;
import com.obs.services.model.PutObjectResult;
import org.springframework.stereotype.Service;
import org.springframework.web.multipart.MultipartFile;
import javax.annotation.Resource;
import java.io.File;
import java.util.ArrayList;
import java.util.List;
import java.util.Objects;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.Future;
import java.util.stream.Collectors;
@Service
public class ObsServiceImpl implements ObsService {
@ -36,8 +38,9 @@ public class ObsServiceImpl implements ObsService {
String extension = originalFilename.substring(originalFilename.lastIndexOf('.'));
String objectKey = UuidUtils.generateUuid() + extension;
objectKey = FileUtils.generateObjectName(objectKey);
ObsInfo obsInfo = obsUtils.uploadFile(objectKey, FileUtils.multipartFileToFile(file));
return R.ok(obsInfo);
return R.ok(obsInfo, "文件上传成功");
} catch (Exception e) {
return R.fail(e.getMessage());
}
@ -71,15 +74,27 @@ public class ObsServiceImpl implements ObsService {
*/
@Override
public R<List<ObsInfo>> uploadFiles(MultipartFile[] files) {
// 调整线程池大小
int threadPoolSize = 10;
ExecutorService executorService = Executors.newFixedThreadPool(threadPoolSize);
try {
List<ObsInfo> obsInfos = new ArrayList<>();
List<Future<ObsInfo>> futures = new ArrayList<>();
for (MultipartFile multipartFile : files) {
futures.add(executorService.submit(() -> {
String originalFilename = Objects.requireNonNull(multipartFile.getOriginalFilename(), "文件名不能为空");
String extension = originalFilename.substring(originalFilename.lastIndexOf('.'));
String objectKey = UuidUtils.generateUuid() + extension;
objectKey = FileUtils.generateObjectName(objectKey);
obsInfos.add(obsUtils.uploadFile(objectKey, FileUtils.multipartFileToFile(multipartFile)));
return obsUtils.uploadFile(objectKey, FileUtils.multipartFileToFile(multipartFile));
}));
}
List<ObsInfo> obsInfos = futures.stream().map(future -> {
try {
return future.get();
} catch (Exception e) {
throw new RuntimeException(e);
}
}).collect(Collectors.toList());
return R.ok(obsInfos, "文件上传成功");
} catch (Exception e) {
return R.fail("File upload failed.");

10
bonus-modules/bonus-obs/src/main/java/com/bonus/obs/utils/ObsUtils.java
View File

@ -17,6 +17,7 @@ import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.net.URLConnection;
@Service
public class ObsUtils {
@ -46,12 +47,7 @@ public class ObsUtils {
*/
public ObsInfo uploadFile(String objectKey, File file) {
PutObjectResult putObjectResult = obsClient.putObject(obsConfig.getBucket(), objectKey, file);
return ObsInfo.builder()
.bucketName(obsConfig.getBucket())
.name(FileUtils.getName(objectKey))
.path(objectKey)
.build();
return ObsInfo.builder().bucketName(obsConfig.getBucket()).name(FileUtils.getName(objectKey)).fileType(FileUtils.getName(objectKey).substring(FileUtils.getName(objectKey).lastIndexOf('.')).toLowerCase()).length(file.length()).path(objectKey).build();
}
/**
@ -75,7 +71,7 @@ public class ObsUtils {
if (!doesObjectExist(objectKey)) {
return R.fail("文件不存在");
}
return R.ok(obsClient.deleteObject(obsConfig.getBucket(), objectKey));
return R.ok(null, "文件删除成功");
}
/**

4
bonus-modules/bonus-obs/src/main/resources/bootstrap.yml
View File

@ -4,6 +4,10 @@ server:
# Spring
spring:
servlet:
multipart:
max-file-size: 5GB
max-request-size: 5GB
application:
# 应用名称
name: bonus-obs

25
bonus-modules/bonus-oss/src/main/java/com/bonus/oss/service/impl/OssServiceImpl.java
View File

@ -3,7 +3,6 @@ package com.bonus.oss.service.impl;
import com.alibaba.nacos.common.utils.UuidUtils;
import com.aliyun.oss.model.OSSObject;
import com.bonus.common.core.domain.R;
import com.bonus.common.core.utils.StringUtils;
import com.bonus.common.core.utils.file.FileUtils;
import com.bonus.oss.domain.OssInfo;
import com.bonus.oss.service.OssService;
@ -14,12 +13,13 @@ import org.springframework.stereotype.Service;
import org.springframework.web.multipart.MultipartFile;
import javax.annotation.Resource;
import java.io.File;
import java.util.ArrayList;
import java.util.List;
import java.util.Objects;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.Future;
import java.util.stream.Collectors;
@Service
public class OssServiceImpl implements OssService {
@ -27,8 +27,6 @@ public class OssServiceImpl implements OssService {
@Resource
private OssUtils ossUtils;
private ExecutorService executor = Executors.newFixedThreadPool(5); // 限制并发线程数为5
/**
* 文件上传
*
@ -90,17 +88,28 @@ public class OssServiceImpl implements OssService {
*/
@Override
public R<List<OssInfo>> uploadFiles(MultipartFile[] files) {
// 调整线程池大小
int threadPoolSize = 10;
ExecutorService executorService = Executors.newFixedThreadPool(threadPoolSize);
try {
List<OssInfo> ossInfos = new ArrayList<>();
List<Future<OssInfo>> futures = new ArrayList<>();
for (MultipartFile multipartFile : files) {
futures.add(executorService.submit(() -> {
String originalFilename = Objects.requireNonNull(multipartFile.getOriginalFilename(), "文件名不能为空");
String extension = originalFilename.substring(originalFilename.lastIndexOf('.'));
String objectKey = UuidUtils.generateUuid() + extension;
File file = FileUtils.multipartFileToFile(multipartFile);
objectKey = FileUtils.generateObjectName(objectKey);
ossInfos.add(ossUtils.upload(objectKey, file).getData());
return ossUtils.upload(objectKey, FileUtils.multipartFileToFile(multipartFile)).getData();
}));
}
return R.ok(ossInfos, "文件上传成功");
List<OssInfo> obsInfos = futures.stream().map(future -> {
try {
return future.get();
} catch (Exception e) {
throw new RuntimeException(e);
}
}).collect(Collectors.toList());
return R.ok(obsInfos, "文件上传成功");
} catch (Exception e) {
return R.fail("File upload failed.");
}

87
bonus-modules/bonus-oss/src/main/java/com/bonus/oss/utils/OssUtils.java
View File

@ -2,8 +2,7 @@ package com.bonus.oss.utils;
import com.aliyun.oss.OSS;
import com.aliyun.oss.OSSClientBuilder;
import com.aliyun.oss.model.OSSObject;
import com.aliyun.oss.model.ObjectMetadata;
import com.aliyun.oss.model.*;
import com.bonus.common.core.domain.R;
import com.bonus.common.core.text.Convert;
import com.bonus.common.core.utils.file.FileUtils;
@ -17,6 +16,13 @@ import org.springframework.stereotype.Service;
import javax.annotation.PostConstruct;
import javax.annotation.Resource;
import java.io.File;
import java.io.FileInputStream;
import java.io.InputStream;
import java.util.ArrayList;
import java.util.List;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.Future;
/**
* OSS存储服务类
@ -51,7 +57,12 @@ public class OssUtils {
*/
public R<OssInfo> upload(String objectKey, File file) {
try {
if (file.length() < 10 * 1024 * 1024L) {
ossClient.putObject(ossConfig.getBucket(), objectKey, file);
} else {
ossMultipartParallelUpload(objectKey, file);
}
return R.ok(getInfo(objectKey), "文件上传成功");
} catch (Exception e) {
LOGGER.error("文件上传失败", e);
@ -63,6 +74,76 @@ public class OssUtils {
}
}
/**
* 并行方式实现OSS的大文件分片上传。
* 通过多线程并行上传文件的各个分片，提高上传效率。
*
* @param objectKey OSS对象的关键字，即文件名。
* @param file 需要上传到OSS的本地文件。
*/
public void ossMultipartParallelUpload(String objectKey, File file) {
// 定义线程池大小，用于并发上传分片
// 调整线程池大小
int threadPoolSize = 10;
ExecutorService executorService = Executors.newFixedThreadPool(threadPoolSize);
try {
// 初始化分片上传，获取上传ID
// 1. 初始化分段上传
InitiateMultipartUploadRequest request = new InitiateMultipartUploadRequest(ossConfig.getBucket(), objectKey);
InitiateMultipartUploadResult result = ossClient.initiateMultipartUpload(request);
String uploadId = result.getUploadId();
// 计算分片大小和分片数量
// 2. 上传分段
// 调整分段大小，例如10MB
final long partSize = 10 * 1024 * 1024L;
long fileLength = file.length();
int partCount = (int) (fileLength / partSize);
if (fileLength % partSize != 0) {
partCount++;
}
// 使用Future列表保存每个分片上传的结果
List<Future<PartETag>> futures = new ArrayList<>();
for (int i = 0; i < partCount; i++) {
final int partNumber = i + 1;
final long startPos = i * partSize;
final long curPartSize = (i + 1 == partCount) ? (fileLength - startPos) : partSize;
futures.add(executorService.submit(() -> {
try (InputStream instream = new FileInputStream(file)) {
instream.skip(startPos);
UploadPartRequest uploadPartRequest = new UploadPartRequest();
uploadPartRequest.setBucketName(ossConfig.getBucket());
uploadPartRequest.setKey(objectKey);
uploadPartRequest.setUploadId(uploadId);
uploadPartRequest.setInputStream(instream);
uploadPartRequest.setPartSize(curPartSize);
uploadPartRequest.setPartNumber(partNumber);
UploadPartResult uploadPartResult = ossClient.uploadPart(uploadPartRequest);
return uploadPartResult.getPartETag();
}
}));
}
// 收集所有分片的上传结果
List<PartETag> partETags = new ArrayList<>();
for (Future<PartETag> future : futures) {
partETags.add(future.get());
}
// 完成分片上传
// 4. 完成分段上传
CompleteMultipartUploadRequest completeMultipartUploadRequest = new CompleteMultipartUploadRequest(ossConfig.getBucket(), objectKey, uploadId, partETags);
ossClient.completeMultipartUpload(completeMultipartUploadRequest);
} catch (Exception e) {
e.printStackTrace();
} finally {
// 关闭线程池
executorService.shutdown();
}
}
/**
* 获取文件在OSS中的信息
*
@ -75,7 +156,7 @@ public class OssUtils {
return OssInfo.builder()
.name(FileUtils.getName(objectKey))
.bucketName(ossConfig.getBucket())
.fileType(objectMetadata.getContentType())
.fileType(FileUtils.getName(objectKey).substring(FileUtils.getName(objectKey).lastIndexOf('.')).toLowerCase())
.length(Convert.toStr(objectMetadata.getContentLength()))
.path(objectKey).build();
} catch (Exception e) {

5
bonus-modules/bonus-oss/src/main/resources/bootstrap.yml
View File

@ -1,9 +1,12 @@
# Tomcat
server:
port: 9204
# Spring
spring:
servlet:
multipart:
max-file-size: 5GB
max-request-size: 5GB
application:
# 应用名称
name: bonus-oss