bonus
/
Bonus-Cloud
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases 2 Wiki Activity

重构文件存储服务

This commit is contained in:
weiweiw 2024-09-19 16:54:13 +08:00
parent 2bcedea253
commit 7adc846c3a
9 changed files with 521 additions and 31 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
bonus-common/bonus-common-core/src/test/java/com/bonus/common/core/utils/test.properties
View File

@ -1,4 +1,4 @@
#Sat Aug 24 09:01:48 CST 2024
#Thu Sep 19 15:42:27 CST 2024
anotherKey=anotherValue
key=value
anotherKey1=anotherValue1

132
bonus-modules/bonus-file/src/main/java/com/bonus/file/service/impl/MinioSysFileServiceImpl.java Normal file
View File

@ -0,0 +1,132 @@
package com.bonus.file.service.impl;
import java.io.InputStream;
import java.util.ArrayList;
import java.util.List;
import com.bonus.file.service.ISysFileService;
import io.minio.RemoveObjectArgs;
import org.apache.commons.io.IOUtils;
import com.bonus.file.utils.FileDownloadUtils;
import com.bonus.system.api.domain.SysFile;
import io.minio.GetObjectArgs;
import io.minio.errors.MinioException;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
import org.springframework.stereotype.Service;
import org.springframework.web.multipart.MultipartFile;
import com.alibaba.nacos.common.utils.IoUtils;
import com.bonus.file.config.MinioConfig;
import com.bonus.file.utils.FileUploadUtils;
import io.minio.MinioClient;
import io.minio.PutObjectArgs;
import javax.servlet.http.HttpServletResponse;
/**
* Minio 文件存储
*
* @author bonus
*/
@Service
@ConditionalOnProperty(name = "storage.type", havingValue = "minio")
public class MinioSysFileServiceImpl implements ISysFileService
{
@Autowired
private MinioConfig minioConfig;
@Autowired
private MinioClient client;
/**
* Minio文件上传接口
*
* @param file 上传的文件
* @return 访问地址
* @throws Exception
*/
@Override
public SysFile uploadFile(MultipartFile file) throws Exception
{
String fileName = FileUploadUtils.extractFilename(file);
InputStream inputStream = file.getInputStream();
PutObjectArgs args = PutObjectArgs.builder()
.bucket(minioConfig.getBucketName())
.object(fileName)
.stream(inputStream, file.getSize(), -1)
.contentType(file.getContentType())
.build();
client.putObject(args);
IoUtils.closeQuietly(inputStream);
return SysFile.builder().url(minioConfig.getUrl() + "/" + minioConfig.getBucketName() + "/" + fileName).name(fileName).build();
}
/**
* Minio文件上传接口
*
* @param files 上传的文件
* @return 访问地址
*/
@Override
public List<SysFile> uploadFiles(MultipartFile[] files) throws Exception {
List<SysFile> sysFiles = new ArrayList<>();
for (MultipartFile file : files) {
SysFile sysFile = uploadFile(file);
sysFiles.add(sysFile);
}
return sysFiles;
}
/**
* Minio文件下载接口，待测试
*
* @param response 响应对象
* @param urlStr 文件URL地址
* @return 是否成功
*/
@Override
public void downloadFile(HttpServletResponse response, String urlStr) throws Exception
{
String fileName = com.bonus.file.utils.FileUtils.setResponseHeaderByUrl(response, urlStr);
if (fileName == null){
throw new Exception("Can't get fileName" + urlStr);
}
try {
// 获取文件的输入流
GetObjectArgs args = GetObjectArgs.builder()
.bucket(minioConfig.getBucketName())
.object(fileName)
.build();
InputStream inputStream = client.getObject(args);
// 将文件流写入响应
IOUtils.copy(inputStream, response.getOutputStream());
response.flushBuffer();
inputStream.close();
} catch (MinioException e) {
throw new Exception("Error occurred while downloading file from Minio" + urlStr, e);
}
}
/**
* Minio文件删除接口，待测试
*
* @param urlStr 文件URL地址
* @return 是否删除成功
*/
@Override
public void deleteFile(String urlStr) throws Exception
{
String fileName = urlStr.substring(urlStr.lastIndexOf("/") + 1);
try {
// 删除文件
RemoveObjectArgs args = RemoveObjectArgs.builder()
.bucket(minioConfig.getBucketName())
.object(fileName)
.build();
client.removeObject(args);
} catch (MinioException e) {
throw new Exception("Error occurred while deleting file from Minio", e);
}
}
}

158
bonus-modules/bonus-file/src/main/java/com/bonus/file/service/impl/MongodbServiceImpl.java Normal file
View File

@ -0,0 +1,158 @@
package com.bonus.file.service.impl;
import com.bonus.file.service.ISysFileService;
import com.bonus.system.api.domain.SysFile;
import com.mongodb.client.gridfs.model.GridFSFile;
import lombok.RequiredArgsConstructor;
import org.apache.commons.io.IOUtils;
import org.bson.types.Binary;
import org.bson.types.ObjectId;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
import org.springframework.data.mongodb.core.MongoTemplate;
import org.springframework.data.mongodb.gridfs.GridFsTemplate;
import org.springframework.http.HttpHeaders;
import org.springframework.http.MediaType;
import org.springframework.stereotype.Service;
import org.springframework.web.multipart.MultipartFile;
import javax.servlet.http.HttpServletResponse;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.net.URLEncoder;
import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import org.bson.Document;
/**
* @author wangvivi
*/
@Service
@ConditionalOnProperty(name = "storage.type", havingValue = "mongodb")
@RequiredArgsConstructor(onConstructor = @__(@Autowired))
public class MongodbServiceImpl implements ISysFileService {
@Value("${spring.data.mongodb.gridfs-size-threshold}")
private long gridFsSizeThreshold;
final static String COLLECTION_NAME = "smallFiles";
/**
* GridFsTemplate 用于大文件存储
*/
private final GridFsTemplate gridFsTemplate;
/**
* MongoTemplate 用于小文件存储
*/
private final MongoTemplate mongoTemplate;
@Override
public SysFile uploadFile(MultipartFile file) throws Exception {
String fileName = file.getOriginalFilename();
long fileSize = file.getSize();
// 判断是否使用 GridFS 存储
if (fileSize >= gridFsSizeThreshold) {
// 使用 GridFS 存储大文件
ObjectId fileId = gridFsTemplate.store(file.getInputStream(), fileName, file.getContentType());
return SysFile.builder().name(fileName).url(fileId.toHexString()).build();
} else {
// 小文件直接存储为二进制数据
// 创建一个Map存储文件信息
Map<String, Object> fileData = new HashMap<>();
fileData.put("fileName", file.getOriginalFilename());
fileData.put("fileSize", file.getSize());
fileData.put("contentType", file.getContentType());
// 将文件内容以byte[]存储
fileData.put("fileData", file.getBytes());
// 插入文件信息到MongoDB的集合中（可以使用指定的集合名）
Document insertedFile = mongoTemplate.insert(new Document(fileData), COLLECTION_NAME);
// 返回文件的唯一标识符（MongoDB的ObjectId）
return SysFile.builder().name(fileName).url(insertedFile.getObjectId("_id").toString()).build();
}
}
@Override
public List<SysFile> uploadFiles(MultipartFile[] files) throws Exception {
List<SysFile> uploadedFiles = new ArrayList<>();
for (MultipartFile file : files) {
uploadedFiles.add(uploadFile(file));
}
return uploadedFiles;
}
@Override
public void downloadFile(HttpServletResponse response, String fileId) throws Exception {
// 先尝试从 GridFS 中读取
GridFSFile gridFSFile = gridFsTemplate.findOne(new org.springframework.data.mongodb.core.query.Query()
.addCriteria(org.springframework.data.mongodb.core.query.Criteria.where("_id").is(new ObjectId(fileId))));
if (gridFSFile != null) {
// 设置响应头，告知客户端文件下载信息
String encodedFileName = URLEncoder.encode(gridFSFile.getFilename(), StandardCharsets.UTF_8.toString());
response.setContentType(gridFSFile.getMetadata().getString("_contentType"));
response.setHeader("Content-Disposition", "attachment; filename=\"" + encodedFileName + "\"");
// GridFS 文件下载
InputStream inputStream = gridFsTemplate.getResource(gridFSFile).getInputStream();
IOUtils.copy(inputStream, response.getOutputStream());
response.flushBuffer();
}
else {
downloadFromCollection(response, fileId);
}
}
@Override
public void deleteFile(String fileId) throws Exception {
try {
// 尝试从 GridFS 中删除
gridFsTemplate.delete(new org.springframework.data.mongodb.core.query.Query()
.addCriteria(org.springframework.data.mongodb.core.query.Criteria.where("_id").is(new ObjectId(fileId))));
// 尝试删除普通集合中的文件
mongoTemplate.findAllAndRemove(new org.springframework.data.mongodb.core.query.Query().addCriteria(org.springframework.data.mongodb.core.query.Criteria.where("_id").is(fileId)), Document.class, COLLECTION_NAME);
} catch (Exception e) {
throw new Exception("删除文件失败" );
}
}
private void downloadFromCollection(HttpServletResponse response, String fileId) throws IOException {
// 尝试从普通集合中获取二进制文件
Document fileDocument = mongoTemplate.findById(fileId, Document.class, COLLECTION_NAME);
if (fileDocument == null) {
// 如果文件不存在，返回错误信息
throw new IOException("文件不存在");
}
// 获取 Binary 对象并转换为 byte[]
Binary binaryData = fileDocument.get("fileData", Binary.class);
// 从 Binary 中提取 byte[]
byte[] fileData = binaryData.getData();
// 假设文件名保存在 fileName 字段
String fileName = fileDocument.getString("fileName");
// 假设文件类型保存在 contentType 字段
String contentType = fileDocument.getString("contentType");
// 如果 contentType 为空，默认设置为 application/octet-stream
if (contentType == null || contentType.isEmpty()) {
contentType = MediaType.APPLICATION_OCTET_STREAM_VALUE;
}
// 设置响应头，包含文件名和内容类型
String encodedFileName = URLEncoder.encode(fileName, StandardCharsets.UTF_8.toString());
response.setHeader(HttpHeaders.CONTENT_DISPOSITION, "attachment; filename=" + encodedFileName);
response.setContentType(contentType);
response.setContentLength(fileData.length);
// 将文件的二进制数据写入到响应输出流中
try (OutputStream os = response.getOutputStream()) {
os.write(fileData);
os.flush();
} catch (IOException e) {
// 如果写入过程出现异常，返回错误信息
throw new IOException("Failed to download file");
}
}
}

111
bonus-modules/bonus-file/src/main/java/com/bonus/file/service/impl/ObsServiceImpl.java Normal file
View File

@ -0,0 +1,111 @@
package com.bonus.file.service.impl;
import com.bonus.common.core.domain.R;
import com.bonus.file.service.ISysFileService;
import com.bonus.file.utils.ObsUtils;
import com.alibaba.nacos.common.utils.UuidUtils;
import com.bonus.common.core.utils.file.FileUtils;
import com.bonus.common.core.web.domain.AjaxResult;
import com.bonus.system.api.domain.SysFile;
import com.obs.services.model.ObsObject;
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
import org.springframework.http.HttpHeaders;
import org.springframework.http.HttpStatus;
import org.springframework.http.MediaType;
import org.springframework.stereotype.Service;
import org.springframework.web.multipart.MultipartFile;
import javax.annotation.Resource;
import javax.servlet.http.HttpServletResponse;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.net.URLEncoder;
import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import java.util.Objects;
/**
* Minio 文件存储
*
* @author bonus
*/
@Service
@ConditionalOnProperty(name = "storage.type", havingValue = "obs")
public class ObsServiceImpl implements ISysFileService {
@Resource
private ObsUtils obsUtils;
/**
* 文件上传
*
* @param file 文件流
* @return 文件信息
*/
@Override
public SysFile uploadFile(MultipartFile file) {
try {
String originalFilename = Objects.requireNonNull(file.getOriginalFilename(), "文件名不能为空");
String extension = originalFilename.substring(originalFilename.lastIndexOf('.'));
String objectKey = UuidUtils.generateUuid() + extension;
objectKey = FileUtils.generateObjectName(objectKey);
SysFile sysFile = obsUtils.uploadFile(objectKey, FileUtils.multipartFileToFile(file));
sysFile.setName(originalFilename);
return sysFile;
} catch (Exception e) {
return null;
}
}
@Override
public List<SysFile> uploadFiles(MultipartFile[] files) throws Exception {
try {
List<SysFile> sysFiles = new ArrayList<>();
for (MultipartFile file : files) {
SysFile sysFile = uploadFile(file);
sysFiles.add(sysFile);
}
return sysFiles;
} catch (Exception e) {
throw new Exception(e);
}
}
@Override
public void downloadFile(HttpServletResponse response, String urlStr) throws Exception {
R<ObsObject> obsObjectR = obsUtils.downloadFile(urlStr);
if (R.isError(obsObjectR)) {
response.setStatus(HttpStatus.BAD_REQUEST.value());
throw new Exception("文件不存在");
}
if (obsObjectR.getData() == null) {
response.setStatus(HttpStatus.NOT_FOUND.value());
throw new Exception("文件不存在");
}
InputStream inputStream = obsObjectR.getData().getObjectContent();
com.bonus.file.utils.FileUtils.setResponseHeaderByUrl(response, urlStr);
try (OutputStream outputStream = response.getOutputStream()) {
byte[] buffer = new byte[8192];
int bytesRead;
while ((bytesRead = inputStream.read(buffer)) != -1) {
outputStream.write(buffer, 0, bytesRead);
}
outputStream.flush();
} catch (IOException e) {
response.setStatus(HttpStatus.INTERNAL_SERVER_ERROR.value());
throw new Exception("下载失败");
}
}
@Override
public void deleteFile(String urlStr) throws Exception {
obsUtils.deleteFile(urlStr);
}
}

111
bonus-modules/bonus-file/src/main/java/com/bonus/file/service/impl/OssServiceImpl.java Normal file
View File

@ -0,0 +1,111 @@
package com.bonus.file.service.impl;
import com.alibaba.nacos.common.utils.UuidUtils;
import com.aliyun.oss.model.OSSObject;
import com.bonus.common.core.domain.R;
import com.bonus.common.core.utils.StringUtils;
import com.bonus.common.core.utils.file.FileUtils;
import com.bonus.file.controller.SysFileController;
import com.bonus.file.service.ISysFileService;
import com.bonus.file.utils.OssUtils;
import com.bonus.system.api.domain.SysFile;
import org.apache.commons.lang3.ObjectUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
import org.springframework.http.HttpHeaders;
import org.springframework.http.HttpStatus;
import org.springframework.http.MediaType;
import org.springframework.stereotype.Service;
import org.springframework.web.multipart.MultipartFile;
import javax.annotation.Resource;
import javax.servlet.http.HttpServletResponse;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.net.URLEncoder;
import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import java.util.Objects;
/**
* @author jiang
*/
@Service
@ConditionalOnProperty(name = "storage.type", havingValue = "oss")
public class OssServiceImpl implements ISysFileService {
private static final Logger log = LoggerFactory.getLogger(OssServiceImpl.class);
@Resource
private OssUtils ossUtils;
/**
* 文件上传
*
* @param file 文件流
* @return 文件信息
*/
@Override
public SysFile uploadFile(MultipartFile file) throws Exception {
if (ObjectUtils.isEmpty(file)) {
throw new Exception("文件名为空");
}
try {
String originalFilename = Objects.requireNonNull(file.getOriginalFilename(), "文件名不能为空");
String extension = originalFilename.substring(originalFilename.lastIndexOf('.'));
String objectKey = UuidUtils.generateUuid() + extension;
objectKey = FileUtils.generateObjectName(objectKey);
return ossUtils.upload(objectKey, FileUtils.multipartFileToFile(file));
} catch (Exception e) {
throw new Exception("上传文件异常:"+ e.getMessage());
}
}
@Override
public List<SysFile> uploadFiles(MultipartFile[] files) throws Exception {
try {
List<SysFile> sysFiles = new ArrayList<>();
for (MultipartFile file : files) {
SysFile sysFile = uploadFile(file);
sysFiles.add(sysFile);
}
return sysFiles;
} catch (Exception e) {
throw new Exception(e);
}
}
@Override
public void downloadFile(HttpServletResponse response, String urlStr) throws Exception {
R<OSSObject> ossObjectR = ossUtils.download(urlStr);
if (ossObjectR.getData() == null) {
response.setStatus(HttpStatus.NOT_FOUND.value());
throw new Exception("未发现文件");
}
OSSObject ossObject = ossObjectR.getData();
com.bonus.file.utils.FileUtils.setResponseHeaderByUrl(response, urlStr);
try (InputStream inputStream = ossObject.getObjectContent();
OutputStream outputStream = response.getOutputStream()) {
byte[] buffer = new byte[8192];
int bytesRead;
while ((bytesRead = inputStream.read(buffer)) != -1) {
outputStream.write(buffer, 0, bytesRead);
}
outputStream.flush();
} catch (Exception e) {
log.error("文件下载过程中出现未知异常，原因是：", e);
throw new Exception("文件下载过程中出现未知异常" );
}
}
@Override
public void deleteFile(String urlStr) throws Exception {
ossUtils.delete(urlStr);
}
}

12
bonus-modules/bonus-file/src/main/resources/bootstrap.yml
View File

@ -7,6 +7,10 @@ spring:
application:
# 应用名称
name: bonus-file
# servlet:
# multipart:
# max-file-size: 5GB
# max-request-size: 5GB
profiles:
# 环境配置
active: dev
@ -14,12 +18,12 @@ spring:
nacos:
discovery:
# 服务注册地址
server-addr: 192.168.0.56:8848
namespace: 9cde1ce1-98bc-4b9c-9213-f1fbf8a5b3cc
server-addr: 192.168.0.14:8848
namespace: f648524d-0a7b-449e-8f92-64e05236fd51
config:
# 配置中心地址
server-addr: 192.168.0.56:8848
namespace: 9cde1ce1-98bc-4b9c-9213-f1fbf8a5b3cc
server-addr: 192.168.0.14:8848
namespace: f648524d-0a7b-449e-8f92-64e05236fd51
# 配置文件格式
file-extension: yml
# 共享配置

9
bonus-modules/bonus-mongodb/src/main/resources/banner.txt
View File

@ -1,9 +0,0 @@
Spring Boot Version: ${spring-boot.version}
Spring Application Name: ${spring.application.name}
_ _
| | | |
| |__ ___ _ __ _ _ ___ ______ ___ | |__ ___
| '_ \ / _ \ | '_ \ | | | | / __| |______| / _ \ | '_ \ / __|
| |_) | | (_) | | | | | | |_| | \__ \ | (_) | | |_) | \__ \
|_.__/ \___/ |_| |_| \__,_| |___/ \___/ |_.__/ |___/

9
bonus-modules/bonus-obs/src/main/resources/banner.txt
View File

@ -1,9 +0,0 @@
Spring Boot Version: ${spring-boot.version}
Spring Application Name: ${spring.application.name}
_ _
| | | |
| |__ ___ _ __ _ _ ___ ______ ___ | |__ ___
| '_ \ / _ \ | '_ \ | | | | / __| |______| / _ \ | '_ \ / __|
| |_) | | (_) | | | | | | |_| | \__ \ | (_) | | |_) | \__ \
|_.__/ \___/ |_| |_| \__,_| |___/ \___/ |_.__/ |___/

8
bonus-modules/bonus-oss/src/main/resources/banner.txt
View File

@ -1,8 +0,0 @@
Spring Boot Version: ${spring-boot.version}
Spring Application Name: ${spring.application.name}
_
| |
| |__ ___ _ __ _ _ ___ ______ ___ ___ ___
| '_ \ / _ \ | '_ \ | | | | / __| |______| / _ \ / __| / __|
| |_) | | (_) | | | | | | |_| | \__ \ | (_) | \__ \ \__ \
|_.__/ \___/ |_| |_| \__,_| |___/ \___/ |___/ |___/