weiweiw
/
Bonus-Cloud-AI-V2
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

提交代码

This commit is contained in:
jiang 2024-12-01 18:43:04 +08:00
parent e15b42c84c
commit a83bb8d0f5
4 changed files with 170 additions and 64 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

102
bonus-modules/bonus-ai/src/main/java/com/bonus/ai/service/Impl/DataSetBasicFileServiceImpl.java
View File

@ -145,6 +145,7 @@ public class DataSetBasicFileServiceImpl implements DataSetBasicFileService {
@Override
public AjaxResult deleteDataSetBasicFileByFileIds(Long[] fileIds) {
try {
int rows = dataSetBasicFileMapper.deleteDataSetBasicFileByFileIds(fileIds);
return rows > 0 ? AjaxResult.success() : AjaxResult.error();
} catch (Exception e) {
@ -171,10 +172,10 @@ public class DataSetBasicFileServiceImpl implements DataSetBasicFileService {
// 如果存在，则修改文件名并递增 num
if (ObjectUtils.isNotEmpty(basicFile)) {
num++; // 递增 num
entity.setFileName(changeNumberInName(entity.getFileName(),num ));
entity.setFileName(changeNumberInName(entity.getFileName(), num));
}
} while (ObjectUtils.isNotEmpty(basicFile));
} while (ObjectUtils.isNotEmpty(basicFile));
entity.setDelFlag("0");
entity.setCreateBy(SecurityUtils.getUserId().toString());
rows += dataSetBasicFileMapper.updateDataSetBasicFile(entity);
@ -185,6 +186,7 @@ public class DataSetBasicFileServiceImpl implements DataSetBasicFileService {
return AjaxResult.error();
}
}
// 改变文件夹或文件名最后()里的数字，同时保留原始后缀
public static String changeNumberInName(String name, int newNumber) {
// 提取文件扩展名
@ -201,7 +203,7 @@ public class DataSetBasicFileServiceImpl implements DataSetBasicFileService {
if (matcher.matches()) {
// 替换括号中的数字为新的数字
String newName = name.substring(0, matcher.start(1)) +newNumber + name.substring(matcher.end(1));
String newName = name.substring(0, matcher.start(1)) + newNumber + name.substring(matcher.end(1));
return newName + extension; // 加上原始文件扩展名
} else {
// 如果没有找到括号中的数字，返回原文件夹/文件名和扩展名
@ -223,6 +225,7 @@ public class DataSetBasicFileServiceImpl implements DataSetBasicFileService {
return 0;
}
}
/**
* 文件分片上传
*
@ -383,7 +386,7 @@ public class DataSetBasicFileServiceImpl implements DataSetBasicFileService {
@Override
public AjaxResult emptyRecycleBin() {
dataSetBasicFileMapper.emptyRecycleBin();
return AjaxResult.success();
return AjaxResult.success();
}
/**
@ -598,45 +601,72 @@ public class DataSetBasicFileServiceImpl implements DataSetBasicFileService {
// 下载多个文件（可以选择压缩成 ZIP）
private void downloadMultipleFiles(HttpServletResponse response, List<DataSetBasicFileEntity> list) throws IOException, MinioException {
// 示例：将多个文件打包成一个 ZIP 文件
response.setContentType("application/zip");
response.setHeader("Content-Disposition", "attachment; filename=files.zip");
try (ZipOutputStream zos = new ZipOutputStream(response.getOutputStream())) {
// 遍历每个文件实体
for (DataSetBasicFileEntity entity : list) {
if ("1".equals(entity.getIsDirectory())) {
Iterable<Result<Item>> objectList = minioUtil.listObjects(entity.getFileUrl());
// 遍历每个对象
for (Result<Item> itemResult : objectList) {
Item item = itemResult.get();
// 跳过目录（如果 MinIO 中有虚拟目录）
if (item.isDir()) {
continue;
}
// 获取文件路径和文件流
String objectName = item.objectName();
InputStream fileStream = minioUtil.downloadFile(item.objectName());
// 在 ZIP 中创建对应的条目，保持目录结构
String result = objectName.replace(entity.getFileUrl(), "");
ZipEntry zipEntry = new ZipEntry(entity.getFileName() + File.separator + result);
zos.putNextEntry(zipEntry);
IOUtils.copy(fileStream, zos);
zos.closeEntry();
fileStream.close();
}
} else {
InputStream fileStream = minioUtil.downloadFile(entity.getFileUrl());
// 创建 ZIP 条目
ZipEntry zipEntry = new ZipEntry(entity.getFileName());
zos.putNextEntry(zipEntry);
// 使用 IOUtils.copy 直接复制文件流
IOUtils.copy(fileStream, zos);
zos.closeEntry();
if ("1".equals(entity.getIsDirectory())) { // 如果是目录，处理文件夹中的文件
handleDirectoryFiles(zos, entity);
} else { // 普通文件直接打包
handleSingleFile(zos, entity);
}
}
} catch (Exception e) {
e.printStackTrace();
throw new IOException("Error occurred while downloading files", e);
}
}
// 处理目录文件
private void handleDirectoryFiles(ZipOutputStream zos, DataSetBasicFileEntity directory) throws IOException, MinioException {
Set<Long> fileWithChildren = getFileWithChildren(directory.getFileId());
for (Long fileId : fileWithChildren) {
DataSetBasicFileEntity fileEntity = dataSetBasicFileMapper.selectDataSetBasicFileByFileId(fileId);
if (!"1".equals(fileEntity.getIsDirectory())) {
String fullDirectoryPath = getFullDirectoryPath(fileEntity, directory.getParentId());
System.err.println(fullDirectoryPath);
addFileToZip(zos, fullDirectoryPath, fileEntity);
}
}
}
// 递归获取文件的完整路径（从文件到根目录）
private String getFullDirectoryPath(DataSetBasicFileEntity fileEntity,Long directoryId) {
StringBuilder directoryPath = new StringBuilder(fileEntity.getFileName());
Long parentId = fileEntity.getParentId();
while (parentId != null && !parentId.equals(directoryId)) { // 判断是否有父目录
DataSetBasicFileEntity parentEntity = dataSetBasicFileMapper.selectDataSetBasicFileByFileId(parentId);
if (parentEntity != null) {
directoryPath.insert(0, parentEntity.getFileName() + File.separator); // 将父目录名加到路径前
parentId = parentEntity.getParentId(); // 更新父目录
} else {
break; // 如果找不到父目录，则退出
}
}
return directoryPath.toString();
}
// 处理单个文件
private void handleSingleFile(ZipOutputStream zos, DataSetBasicFileEntity fileEntity) throws IOException, MinioException {
addFileToZip(zos, "", fileEntity);
}
// 将文件添加到 ZIP 文件中
private void addFileToZip(ZipOutputStream zos, String parentDirectory, DataSetBasicFileEntity fileEntity) throws IOException, MinioException {
InputStream fileStream = null;
try {
fileStream = minioUtil.downloadFile(fileEntity.getFileUrl());
String zipEntryName = parentDirectory.isEmpty() ? fileEntity.getFileName() : parentDirectory + File.separator + fileEntity.getFileName();
ZipEntry zipEntry = new ZipEntry(parentDirectory);
zos.putNextEntry(zipEntry);
IOUtils.copy(fileStream, zos);
zos.closeEntry();
} finally {
if (fileStream != null) {
fileStream.close();
}
}
}
}

116
bonus-modules/bonus-ai/src/main/java/com/bonus/ai/service/Impl/dataset/DatasetServiceImpl.java
View File

@ -16,6 +16,7 @@ import com.bonus.common.security.utils.SecurityUtils;
import io.minio.Result;
import io.minio.errors.MinioException;
import io.minio.messages.Item;
import javafx.util.Pair;
import org.apache.commons.io.IOUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@ -27,13 +28,12 @@ import com.bonus.common.security.utils.SecurityUtils;
import javax.annotation.Resource;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
import java.io.File;
import java.io.IOException;
import java.io.InputStream;
import java.io.*;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.List;
import java.util.concurrent.*;
import java.util.stream.Collectors;
import java.util.zip.ZipEntry;
import java.util.zip.ZipOutputStream;
@ -208,25 +208,106 @@ public class DatasetServiceImpl implements DatasetService {
}
// 下载多个文件（可以选择压缩成 ZIP）
private void downloadMultipleFiles(HttpServletResponse response, List<DataSetBasicFileEntity> list) throws IOException, MinioException {
// 示例：将多个文件打包成一个 ZIP 文件
private void downloadMultipleFiles(HttpServletResponse response, List<DataSetBasicFileEntity> list) throws IOException {
int batchSize = 500; // 每批次的文件数量
response.setContentType("application/zip");
response.setHeader("Content-Disposition", "attachment; filename=files.zip");
try (ZipOutputStream zos = new ZipOutputStream(response.getOutputStream())) {
for (DataSetBasicFileEntity entity : list) {
InputStream fileStream = minioUtil.downloadFile(entity.getFileUrl());
// 创建 ZIP 条目
ZipEntry zipEntry = new ZipEntry(entity.getFileName());
zos.putNextEntry(zipEntry);
// 使用 IOUtils.copy 直接复制文件流
IOUtils.copy(fileStream, zos);
zos.closeEntry();
response.setHeader("Content-Disposition", "attachment; filename=all_files.zip");
ExecutorService executorService = Executors.newFixedThreadPool(Math.min(16, Runtime.getRuntime().availableProcessors()));
BlockingQueue<ZipEntryData> zipQueue = new LinkedBlockingQueue<>();
try (ZipOutputStream zos = new ZipOutputStream(new BufferedOutputStream(response.getOutputStream()))) {
// 异步压缩线程
Thread zipThread = new Thread(() -> {
try {
while (true) {
ZipEntryData zipData = zipQueue.take();
if (zipData.isPoisonPill()) {
break; // 检测结束标志
}
zos.putNextEntry(new ZipEntry(zipData.getFileName()));
zos.write(zipData.getData());
zos.closeEntry();
}
} catch (Exception e) {
e.printStackTrace();
}
});
zipThread.start();
// 并行下载与数据入队
List<CompletableFuture<Void>> futures = new ArrayList<>();
for (int i = 0; i < list.size(); i += batchSize) {
int start = i;
int end = Math.min(i + batchSize, list.size());
List<DataSetBasicFileEntity> batch = list.subList(start, end);
futures.add(CompletableFuture.runAsync(() -> {
for (DataSetBasicFileEntity entity : batch) {
try (InputStream fileStream = minioUtil.downloadFile(entity.getFileUrl());
ByteArrayOutputStream byteStream = new ByteArrayOutputStream()) {
byte[] buffer = new byte[16384];
int len;
while ((len = fileStream.read(buffer)) != -1) {
byteStream.write(buffer, 0, len);
}
zipQueue.put(new ZipEntryData(entity.getFileName(), byteStream.toByteArray()));
} catch (Exception e) {
e.printStackTrace();
}
}
}, executorService));
}
// 等待所有下载完成
CompletableFuture.allOf(futures.toArray(new CompletableFuture[0])).join();
zipQueue.put(ZipEntryData.poisonPill()); // 放入结束标志
zipThread.join();
} catch (Exception e) {
e.printStackTrace();
} finally {
executorService.shutdown();
}
}
private static class ZipEntryData {
private final String fileName;
private final byte[] data;
private final boolean poisonPill;
// 常规构造函数
public ZipEntryData(String fileName, byte[] data, boolean poisonPill) {
this.fileName = fileName;
this.data = data;
this.poisonPill = poisonPill;
}
// 重载构造函数，默认 poisonPill 为 false
public ZipEntryData(String fileName, byte[] data) {
this(fileName, data, false);
}
public static ZipEntryData poisonPill() {
return new ZipEntryData(null, null, true);
}
public String getFileName() {
return fileName;
}
public byte[] getData() {
return data;
}
public boolean isPoisonPill() {
return poisonPill;
}
}
/**
* 插入数据集文件映射关系
@ -274,6 +355,11 @@ public class DatasetServiceImpl implements DatasetService {
* @return 如果文件符合条件则返回 true，否则返回 false
*/
private boolean isValidFile(DataSetBasicFileEntity file, List<String> supportedFormats) {
// 如果 supportedFormats 为空，不校验后缀，直接返回文件不是目录的结果
if (supportedFormats == null || supportedFormats.isEmpty()) {
return "0".equals(file.getIsDirectory());
}
// 校验文件后缀
return "0".equals(file.getIsDirectory()) && // 确保文件不是目录
supportedFormats.stream() // 遍历支持的文件后缀
.anyMatch(format -> file.getFileName().toLowerCase().endsWith(format)); // 文件名后缀匹配

5
bonus-modules/bonus-ai/src/main/resources/mapper/DataSetBasicFileMapper.xml
View File

@ -158,9 +158,8 @@
</foreach>
</update>
<update id="emptyRecycleBin">
update ai_basic_file
set del_flag='2'
where del_flag = '1'
delete from ai_basic_file where del_flag = '1'
</update>
<delete id="deleteDataSetBasicFileByFileIds" parameterType="String">

11
bonus-modules/bonus-ai/src/main/resources/mapper/DatasetMapper.xml
View File

@ -48,18 +48,9 @@
WHERE adfm.dataset_id = ad.dataset_id) AS annotatedCount,
(SELECT COUNT(*)
FROM ai_dataset_file_map adfm
WHERE adfm.dataset_id = ad.dataset_id AND adfm.is_annotated = '0') AS notAnnotatedCount,
adv.version_name AS latestVersionName
WHERE adfm.dataset_id = ad.dataset_id AND adfm.is_annotated = '0') AS notAnnotatedCount
FROM
ai_dataset ad
LEFT JOIN (
SELECT task_id, dataset_id, version_name
FROM ai_dataset_version
WHERE (task_id, dataset_id, create_time) IN (
SELECT task_id, dataset_id, MAX(create_time)
FROM ai_dataset_version
GROUP BY task_id, dataset_id
)) adv on adv.dataset_id = ad.dataset_id
</sql>
<insert id="insert" parameterType="com.bonus.ai.domain.dataset.DataSetEntity" useGeneratedKeys="true" keyProperty="datasetId">