bonus
/
Bonus-Cloud
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases 2 Wiki Activity

Merge remote-tracking branch 'origin/main'

This commit is contained in:
weiweiw 2024-07-05 10:54:32 +08:00
parent 256899fa06 bbf47b7043
commit be4cf7ca69
12 changed files with 198 additions and 119 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

121
bonus-common/bonus-common-core/src/main/java/com/bonus/common/core/utils/file/FileUtils.java
View File

@ -12,8 +12,10 @@ import java.nio.file.Path;
import java.nio.file.Paths;
import java.text.SimpleDateFormat;
import java.util.Date;
import java.util.UUID;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
import org.apache.commons.lang3.ArrayUtils;
import com.bonus.common.core.utils.StringUtils;
import org.springframework.web.multipart.MultipartFile;
@ -24,12 +26,15 @@ import org.springframework.web.multipart.MultipartFile;
* @author bonus
*/
public class FileUtils
{
/** 字符常量：斜杠 {@code '/'} */
public class FileUtils {
/**
* 字符常量：斜杠 {@code '/'}
*/
public static final char SLASH = '/';
/** 字符常量：反斜杠 {@code '\\'} */
/**
* 字符常量：反斜杠 {@code '\\'}
*/
public static final char BACKSLASH = '\\';
public static String FILENAME_PATTERN = "[a-zA-Z0-9_\\-\\|\\.\\u4e00-\\u9fa5]+";
@ -38,51 +43,35 @@ public class FileUtils
* 输出指定文件的byte数组
*
* @param filePath 文件路径
* @param os 输出流
* @param os 输出流
*/
public static void writeBytes(String filePath, OutputStream os) throws IOException
{
public static void writeBytes(String filePath, OutputStream os) throws IOException {
FileInputStream fis = null;
try
{
try {
File file = new File(filePath);
if (!file.exists())
{
if (!file.exists()) {
throw new FileNotFoundException(filePath);
}
fis = new FileInputStream(file);
byte[] b = new byte[1024];
int length;
while ((length = fis.read(b)) > 0)
{
while ((length = fis.read(b)) > 0) {
os.write(b, 0, length);
}
}
catch (IOException e)
{
} catch (IOException e) {
throw e;
}
finally
{
if (os != null)
{
try
{
} finally {
if (os != null) {
try {
os.close();
}
catch (IOException e1)
{
} catch (IOException e1) {
e1.printStackTrace();
}
}
if (fis != null)
{
try
{
if (fis != null) {
try {
fis.close();
}
catch (IOException e1)
{
} catch (IOException e1) {
e1.printStackTrace();
}
}
@ -95,13 +84,11 @@ public class FileUtils
* @param filePath 文件
* @return
*/
public static boolean deleteFile(String filePath)
{
public static boolean deleteFile(String filePath) {
boolean flag = false;
File file = new File(filePath);
// 路径为文件且不为空则进行删除
if (file.isFile() && file.exists())
{
if (file.isFile() && file.exists()) {
flag = file.delete();
}
return flag;
@ -113,8 +100,7 @@ public class FileUtils
* @param filename 文件名称
* @return true 正常 false 非法
*/
public static boolean isValidFilename(String filename)
{
public static boolean isValidFilename(String filename) {
return filename.matches(FILENAME_PATTERN);
}
@ -124,11 +110,9 @@ public class FileUtils
* @param resource 需要下载的文件
* @return true 正常 false 非法
*/
public static boolean checkAllowDownload(String resource)
{
public static boolean checkAllowDownload(String resource) {
// 禁止目录上跳级别
if (StringUtils.contains(resource, ".."))
{
if (StringUtils.contains(resource, "..")) {
return false;
}
// 判断是否在允许下载的文件规则内
@ -138,32 +122,24 @@ public class FileUtils
/**
* 下载文件名重新编码
*
* @param request 请求对象
* @param request 请求对象
* @param fileName 文件名
* @return 编码后的文件名
*/
public static String setFileDownloadHeader(HttpServletRequest request, String fileName) throws UnsupportedEncodingException
{
public static String setFileDownloadHeader(HttpServletRequest request, String fileName) throws UnsupportedEncodingException {
final String agent = request.getHeader("USER-AGENT");
String filename = fileName;
if (agent.contains("MSIE"))
{
if (agent.contains("MSIE")) {
// IE浏览器
filename = URLEncoder.encode(filename, "utf-8");
filename = filename.replace("+", " ");
}
else if (agent.contains("Firefox"))
{
} else if (agent.contains("Firefox")) {
// 火狐浏览器
filename = new String(fileName.getBytes(), "ISO8859-1");
}
else if (agent.contains("Chrome"))
{
} else if (agent.contains("Chrome")) {
// google浏览器
filename = URLEncoder.encode(filename, "utf-8");
}
else
{
} else {
// 其它浏览器
filename = URLEncoder.encode(filename, "utf-8");
}
@ -176,30 +152,24 @@ public class FileUtils
* @param filePath 文件
* @return 文件名
*/
public static String getName(String filePath)
{
if (null == filePath)
{
public static String getName(String filePath) {
if (null == filePath) {
return null;
}
int len = filePath.length();
if (0 == len)
{
if (0 == len) {
return filePath;
}
if (isFileSeparator(filePath.charAt(len - 1)))
{
if (isFileSeparator(filePath.charAt(len - 1))) {
// 以分隔符结尾的去掉结尾分隔符
len--;
}
int begin = 0;
char c;
for (int i = len - 1; i > -1; i--)
{
for (int i = len - 1; i > -1; i--) {
c = filePath.charAt(i);
if (isFileSeparator(c))
{
if (isFileSeparator(c)) {
// 查找最后一个路径分隔符（/或者\）
begin = i + 1;
break;
@ -216,19 +186,17 @@ public class FileUtils
* @param c 字符
* @return 是否为Windows或者Linux（Unix）文件分隔符
*/
public static boolean isFileSeparator(char c)
{
public static boolean isFileSeparator(char c) {
return SLASH == c || BACKSLASH == c;
}
/**
* 下载文件名重新编码
*
* @param response 响应对象
* @param response 响应对象
* @param realFileName 真实文件名
*/
public static void setAttachmentResponseHeader(HttpServletResponse response, String realFileName) throws UnsupportedEncodingException
{
public static void setAttachmentResponseHeader(HttpServletResponse response, String realFileName) throws UnsupportedEncodingException {
String percentEncodedFileName = percentEncode(realFileName);
StringBuilder contentDispositionValue = new StringBuilder();
@ -249,8 +217,7 @@ public class FileUtils
* @param s 需要百分号编码的字符串
* @return 百分号编码后的字符串
*/
public static String percentEncode(String s) throws UnsupportedEncodingException
{
public static String percentEncode(String s) throws UnsupportedEncodingException {
String encode = URLEncoder.encode(s, StandardCharsets.UTF_8.toString());
return encode.replaceAll("\\+", "%20");
}
@ -294,7 +261,7 @@ public class FileUtils
if (fileName == null) {
return null;
}
String prefix = fileName.substring(0, fileName.lastIndexOf("."));
String prefix = UUID.randomUUID().toString().replace("-", "");
String suffix = fileName.substring(fileName.lastIndexOf("."));
File file = File.createTempFile(prefix, suffix);
multiFile.transferTo(file);

1
bonus-modules/bonus-file/src/main/java/com/bonus/file/controller/SysFileController.java
View File

@ -1,7 +1,6 @@
package com.bonus.file.controller;
import com.bonus.common.core.utils.Base64Utils;
import com.bonus.file.utils.FileDownloadUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;

6
bonus-modules/bonus-mongodb/src/main/resources/bootstrap.yml
View File

@ -3,7 +3,11 @@ server:
port: 9206
# Spring
spring:
spring:
servlet:
multipart:
max-file-size: 5GB
max-request-size: 5GB
application:
# 应用名称
name: bonus-mongodb

4
bonus-modules/bonus-obs/src/main/java/com/bonus/obs/controller/ObsController.java
View File

@ -54,8 +54,8 @@ public class ObsController {
*
* @param objectKey obs文件存储地址
*/
@GetMapping("/download")
public void download(HttpServletResponse response, @RequestParam String objectKey) {
@GetMapping("/downloadFile")
public void downloadFile(HttpServletResponse response, @RequestParam String objectKey) {
R<ObsObject> obsObjectR = service.downloadFile(objectKey);
try {
if (R.isError(obsObjectR)) {

2
bonus-modules/bonus-obs/src/main/java/com/bonus/obs/domain/ObsInfo.java
View File

@ -26,7 +26,7 @@ public class ObsInfo {
* 对象的大小。
* 以字符串形式表示，可能包含单位（如字节、KB、MB等）。
*/
private String length;
private Long length;
/**
* 对象的文件类型。

33
bonus-modules/bonus-obs/src/main/java/com/bonus/obs/service/impl/ObsServiceImpl.java
View File

@ -8,15 +8,17 @@ import com.bonus.obs.service.ObsService;
import com.bonus.obs.utils.ObsUtils;
import com.obs.services.model.DeleteObjectResult;
import com.obs.services.model.ObsObject;
import com.obs.services.model.PutObjectResult;
import org.springframework.stereotype.Service;
import org.springframework.web.multipart.MultipartFile;
import javax.annotation.Resource;
import java.io.File;
import java.util.ArrayList;
import java.util.List;
import java.util.Objects;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.Future;
import java.util.stream.Collectors;
@Service
public class ObsServiceImpl implements ObsService {
@ -36,8 +38,9 @@ public class ObsServiceImpl implements ObsService {
String extension = originalFilename.substring(originalFilename.lastIndexOf('.'));
String objectKey = UuidUtils.generateUuid() + extension;
objectKey = FileUtils.generateObjectName(objectKey);
ObsInfo obsInfo = obsUtils.uploadFile(objectKey, FileUtils.multipartFileToFile(file));
return R.ok(obsInfo);
return R.ok(obsInfo, "文件上传成功");
} catch (Exception e) {
return R.fail(e.getMessage());
}
@ -71,15 +74,27 @@ public class ObsServiceImpl implements ObsService {
*/
@Override
public R<List<ObsInfo>> uploadFiles(MultipartFile[] files) {
// 调整线程池大小
int threadPoolSize = 10;
ExecutorService executorService = Executors.newFixedThreadPool(threadPoolSize);
try {
List<ObsInfo> obsInfos = new ArrayList<>();
List<Future<ObsInfo>> futures = new ArrayList<>();
for (MultipartFile multipartFile : files) {
String originalFilename = Objects.requireNonNull(multipartFile.getOriginalFilename(), "文件名不能为空");
String extension = originalFilename.substring(originalFilename.lastIndexOf('.'));
String objectKey = UuidUtils.generateUuid() + extension;
objectKey = FileUtils.generateObjectName(objectKey);
obsInfos.add(obsUtils.uploadFile(objectKey, FileUtils.multipartFileToFile(multipartFile)));
futures.add(executorService.submit(() -> {
String originalFilename = Objects.requireNonNull(multipartFile.getOriginalFilename(), "文件名不能为空");
String extension = originalFilename.substring(originalFilename.lastIndexOf('.'));
String objectKey = UuidUtils.generateUuid() + extension;
objectKey = FileUtils.generateObjectName(objectKey);
return obsUtils.uploadFile(objectKey, FileUtils.multipartFileToFile(multipartFile));
}));
}
List<ObsInfo> obsInfos = futures.stream().map(future -> {
try {
return future.get();
} catch (Exception e) {
throw new RuntimeException(e);
}
}).collect(Collectors.toList());
return R.ok(obsInfos, "文件上传成功");
} catch (Exception e) {
return R.fail("File upload failed.");

10
bonus-modules/bonus-obs/src/main/java/com/bonus/obs/utils/ObsUtils.java
View File

@ -17,6 +17,7 @@ import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.net.URLConnection;
@Service
public class ObsUtils {
@ -46,12 +47,7 @@ public class ObsUtils {
*/
public ObsInfo uploadFile(String objectKey, File file) {
PutObjectResult putObjectResult = obsClient.putObject(obsConfig.getBucket(), objectKey, file);
return ObsInfo.builder()
.bucketName(obsConfig.getBucket())
.name(FileUtils.getName(objectKey))
.path(objectKey)
.build();
return ObsInfo.builder().bucketName(obsConfig.getBucket()).name(FileUtils.getName(objectKey)).fileType(FileUtils.getName(objectKey).substring(FileUtils.getName(objectKey).lastIndexOf('.')).toLowerCase()).length(file.length()).path(objectKey).build();
}
/**
@ -75,7 +71,7 @@ public class ObsUtils {
if (!doesObjectExist(objectKey)) {
return R.fail("文件不存在");
}
return R.ok(obsClient.deleteObject(obsConfig.getBucket(), objectKey));
return R.ok(null, "文件删除成功");
}
/**

6
bonus-modules/bonus-obs/src/main/resources/bootstrap.yml
View File

@ -3,7 +3,11 @@ server:
port: 9205
# Spring
spring:
spring:
servlet:
multipart:
max-file-size: 5GB
max-request-size: 5GB
application:
# 应用名称
name: bonus-obs

33
bonus-modules/bonus-oss/src/main/java/com/bonus/oss/service/impl/OssServiceImpl.java
View File

@ -3,7 +3,6 @@ package com.bonus.oss.service.impl;
import com.alibaba.nacos.common.utils.UuidUtils;
import com.aliyun.oss.model.OSSObject;
import com.bonus.common.core.domain.R;
import com.bonus.common.core.utils.StringUtils;
import com.bonus.common.core.utils.file.FileUtils;
import com.bonus.oss.domain.OssInfo;
import com.bonus.oss.service.OssService;
@ -14,12 +13,13 @@ import org.springframework.stereotype.Service;
import org.springframework.web.multipart.MultipartFile;
import javax.annotation.Resource;
import java.io.File;
import java.util.ArrayList;
import java.util.List;
import java.util.Objects;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.Future;
import java.util.stream.Collectors;
@Service
public class OssServiceImpl implements OssService {
@ -27,8 +27,6 @@ public class OssServiceImpl implements OssService {
@Resource
private OssUtils ossUtils;
private ExecutorService executor = Executors.newFixedThreadPool(5); // 限制并发线程数为5
/**
* 文件上传
*
@ -90,17 +88,28 @@ public class OssServiceImpl implements OssService {
*/
@Override
public R<List<OssInfo>> uploadFiles(MultipartFile[] files) {
// 调整线程池大小
int threadPoolSize = 10;
ExecutorService executorService = Executors.newFixedThreadPool(threadPoolSize);
try {
List<OssInfo> ossInfos = new ArrayList<>();
List<Future<OssInfo>> futures = new ArrayList<>();
for (MultipartFile multipartFile : files) {
String originalFilename = Objects.requireNonNull(multipartFile.getOriginalFilename(), "文件名不能为空");
String extension = originalFilename.substring(originalFilename.lastIndexOf('.'));
String objectKey = UuidUtils.generateUuid() + extension;
File file = FileUtils.multipartFileToFile(multipartFile);
objectKey = FileUtils.generateObjectName(objectKey);
ossInfos.add(ossUtils.upload(objectKey, file).getData());
futures.add(executorService.submit(() -> {
String originalFilename = Objects.requireNonNull(multipartFile.getOriginalFilename(), "文件名不能为空");
String extension = originalFilename.substring(originalFilename.lastIndexOf('.'));
String objectKey = UuidUtils.generateUuid() + extension;
objectKey = FileUtils.generateObjectName(objectKey);
return ossUtils.upload(objectKey, FileUtils.multipartFileToFile(multipartFile)).getData();
}));
}
return R.ok(ossInfos, "文件上传成功");
List<OssInfo> obsInfos = futures.stream().map(future -> {
try {
return future.get();
} catch (Exception e) {
throw new RuntimeException(e);
}
}).collect(Collectors.toList());
return R.ok(obsInfos, "文件上传成功");
} catch (Exception e) {
return R.fail("File upload failed.");
}

91
bonus-modules/bonus-oss/src/main/java/com/bonus/oss/utils/OssUtils.java
View File

@ -2,8 +2,7 @@ package com.bonus.oss.utils;
import com.aliyun.oss.OSS;
import com.aliyun.oss.OSSClientBuilder;
import com.aliyun.oss.model.OSSObject;
import com.aliyun.oss.model.ObjectMetadata;
import com.aliyun.oss.model.*;
import com.bonus.common.core.domain.R;
import com.bonus.common.core.text.Convert;
import com.bonus.common.core.utils.file.FileUtils;
@ -17,6 +16,13 @@ import org.springframework.stereotype.Service;
import javax.annotation.PostConstruct;
import javax.annotation.Resource;
import java.io.File;
import java.io.FileInputStream;
import java.io.InputStream;
import java.util.ArrayList;
import java.util.List;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.Future;
/**
* OSS存储服务类
@ -51,7 +57,12 @@ public class OssUtils {
*/
public R<OssInfo> upload(String objectKey, File file) {
try {
ossClient.putObject(ossConfig.getBucket(), objectKey, file);
if (file.length() < 10 * 1024 * 1024L) {
ossClient.putObject(ossConfig.getBucket(), objectKey, file);
} else {
ossMultipartParallelUpload(objectKey, file);
}
return R.ok(getInfo(objectKey), "文件上传成功");
} catch (Exception e) {
LOGGER.error("文件上传失败", e);
@ -63,6 +74,76 @@ public class OssUtils {
}
}
/**
* 并行方式实现OSS的大文件分片上传。
* 通过多线程并行上传文件的各个分片，提高上传效率。
*
* @param objectKey OSS对象的关键字，即文件名。
* @param file 需要上传到OSS的本地文件。
*/
public void ossMultipartParallelUpload(String objectKey, File file) {
// 定义线程池大小，用于并发上传分片
// 调整线程池大小
int threadPoolSize = 10;
ExecutorService executorService = Executors.newFixedThreadPool(threadPoolSize);
try {
// 初始化分片上传，获取上传ID
// 1. 初始化分段上传
InitiateMultipartUploadRequest request = new InitiateMultipartUploadRequest(ossConfig.getBucket(), objectKey);
InitiateMultipartUploadResult result = ossClient.initiateMultipartUpload(request);
String uploadId = result.getUploadId();
// 计算分片大小和分片数量
// 2. 上传分段
// 调整分段大小，例如10MB
final long partSize = 10 * 1024 * 1024L;
long fileLength = file.length();
int partCount = (int) (fileLength / partSize);
if (fileLength % partSize != 0) {
partCount++;
}
// 使用Future列表保存每个分片上传的结果
List<Future<PartETag>> futures = new ArrayList<>();
for (int i = 0; i < partCount; i++) {
final int partNumber = i + 1;
final long startPos = i * partSize;
final long curPartSize = (i + 1 == partCount) ? (fileLength - startPos) : partSize;
futures.add(executorService.submit(() -> {
try (InputStream instream = new FileInputStream(file)) {
instream.skip(startPos);
UploadPartRequest uploadPartRequest = new UploadPartRequest();
uploadPartRequest.setBucketName(ossConfig.getBucket());
uploadPartRequest.setKey(objectKey);
uploadPartRequest.setUploadId(uploadId);
uploadPartRequest.setInputStream(instream);
uploadPartRequest.setPartSize(curPartSize);
uploadPartRequest.setPartNumber(partNumber);
UploadPartResult uploadPartResult = ossClient.uploadPart(uploadPartRequest);
return uploadPartResult.getPartETag();
}
}));
}
// 收集所有分片的上传结果
List<PartETag> partETags = new ArrayList<>();
for (Future<PartETag> future : futures) {
partETags.add(future.get());
}
// 完成分片上传
// 4. 完成分段上传
CompleteMultipartUploadRequest completeMultipartUploadRequest = new CompleteMultipartUploadRequest(ossConfig.getBucket(), objectKey, uploadId, partETags);
ossClient.completeMultipartUpload(completeMultipartUploadRequest);
} catch (Exception e) {
e.printStackTrace();
} finally {
// 关闭线程池
executorService.shutdown();
}
}
/**
* 获取文件在OSS中的信息
*
@ -75,7 +156,7 @@ public class OssUtils {
return OssInfo.builder()
.name(FileUtils.getName(objectKey))
.bucketName(ossConfig.getBucket())
.fileType(objectMetadata.getContentType())
.fileType(FileUtils.getName(objectKey).substring(FileUtils.getName(objectKey).lastIndexOf('.')).toLowerCase())
.length(Convert.toStr(objectMetadata.getContentLength()))
.path(objectKey).build();
} catch (Exception e) {
@ -114,7 +195,7 @@ public class OssUtils {
return R.fail("文件不存在");
}
ossClient.deleteObject(ossConfig.getBucket(), objectKey);
return R.ok(null,"文件删除成功");
return R.ok(null, "文件删除成功");
} catch (Exception e) {
LOGGER.error("文件删除失败", e);
return R.fail("文件删除失败");

7
bonus-modules/bonus-oss/src/main/resources/bootstrap.yml
View File

@ -1,9 +1,12 @@
# Tomcat
server:
port: 9204
# Spring
spring:
spring:
servlet:
multipart:
max-file-size: 5GB
max-request-size: 5GB
application:
# 应用名称
name: bonus-oss

3
bonus-modules/pom.xml
View File

@ -14,6 +14,8 @@
<module>bonus-job</module>
<module>bonus-file</module>
<module>bonus-oss</module>
<module>bonus-obs</module>
<module>bonus-mongodb</module>
</modules>
<artifactId>bonus-modules</artifactId>
@ -22,5 +24,4 @@
<description>
bonus-modules业务模块
</description>
</project>