From a83bb8d0f55994839d43c5108a8b23b0bf1ae026 Mon Sep 17 00:00:00 2001 From: jiang Date: Sun, 1 Dec 2024 18:43:04 +0800 Subject: [PATCH] =?UTF-8?q?=E6=8F=90=E4=BA=A4=E4=BB=A3=E7=A0=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../Impl/DataSetBasicFileServiceImpl.java | 102 +++++++++------ .../Impl/dataset/DatasetServiceImpl.java | 116 +++++++++++++++--- .../mapper/DataSetBasicFileMapper.xml | 5 +- .../main/resources/mapper/DatasetMapper.xml | 11 +- 4 files changed, 170 insertions(+), 64 deletions(-) diff --git a/bonus-modules/bonus-ai/src/main/java/com/bonus/ai/service/Impl/DataSetBasicFileServiceImpl.java b/bonus-modules/bonus-ai/src/main/java/com/bonus/ai/service/Impl/DataSetBasicFileServiceImpl.java index 34ec287..39fb114 100644 --- a/bonus-modules/bonus-ai/src/main/java/com/bonus/ai/service/Impl/DataSetBasicFileServiceImpl.java +++ b/bonus-modules/bonus-ai/src/main/java/com/bonus/ai/service/Impl/DataSetBasicFileServiceImpl.java @@ -145,6 +145,7 @@ public class DataSetBasicFileServiceImpl implements DataSetBasicFileService { @Override public AjaxResult deleteDataSetBasicFileByFileIds(Long[] fileIds) { try { + int rows = dataSetBasicFileMapper.deleteDataSetBasicFileByFileIds(fileIds); return rows > 0 ? AjaxResult.success() : AjaxResult.error(); } catch (Exception e) { @@ -171,10 +172,10 @@ public class DataSetBasicFileServiceImpl implements DataSetBasicFileService { // 如果存在，则修改文件名并递增 num if (ObjectUtils.isNotEmpty(basicFile)) { num++; // 递增 num - entity.setFileName(changeNumberInName(entity.getFileName(),num )); + entity.setFileName(changeNumberInName(entity.getFileName(), num)); } - } while (ObjectUtils.isNotEmpty(basicFile)); + } while (ObjectUtils.isNotEmpty(basicFile)); entity.setDelFlag("0"); entity.setCreateBy(SecurityUtils.getUserId().toString()); rows += dataSetBasicFileMapper.updateDataSetBasicFile(entity); @@ -185,6 +186,7 @@ public class DataSetBasicFileServiceImpl implements DataSetBasicFileService { return AjaxResult.error(); } } + // 改变文件夹或文件名最后()里的数字，同时保留原始后缀 public static String changeNumberInName(String name, int newNumber) { // 提取文件扩展名 @@ -201,7 +203,7 @@ public class DataSetBasicFileServiceImpl implements DataSetBasicFileService { if (matcher.matches()) { // 替换括号中的数字为新的数字 - String newName = name.substring(0, matcher.start(1)) +newNumber + name.substring(matcher.end(1)); + String newName = name.substring(0, matcher.start(1)) + newNumber + name.substring(matcher.end(1)); return newName + extension; // 加上原始文件扩展名 } else { // 如果没有找到括号中的数字，返回原文件夹/文件名和扩展名 @@ -223,6 +225,7 @@ public class DataSetBasicFileServiceImpl implements DataSetBasicFileService { return 0; } } + /** * 文件分片上传 * @@ -383,7 +386,7 @@ public class DataSetBasicFileServiceImpl implements DataSetBasicFileService { @Override public AjaxResult emptyRecycleBin() { dataSetBasicFileMapper.emptyRecycleBin(); - return AjaxResult.success(); + return AjaxResult.success(); } /** @@ -598,45 +601,72 @@ public class DataSetBasicFileServiceImpl implements DataSetBasicFileService { // 下载多个文件（可以选择压缩成 ZIP） private void downloadMultipleFiles(HttpServletResponse response, List list) throws IOException, MinioException { - // 示例：将多个文件打包成一个 ZIP 文件 response.setContentType("application/zip"); response.setHeader("Content-Disposition", "attachment; filename=files.zip"); + try (ZipOutputStream zos = new ZipOutputStream(response.getOutputStream())) { + // 遍历每个文件实体 for (DataSetBasicFileEntity entity : list) { - if ("1".equals(entity.getIsDirectory())) { - Iterable> objectList = minioUtil.listObjects(entity.getFileUrl()); - // 遍历每个对象 - for (Result itemResult : objectList) { - Item item = itemResult.get(); - // 跳过目录（如果 MinIO 中有虚拟目录） - if (item.isDir()) { - continue; - } - // 获取文件路径和文件流 - String objectName = item.objectName(); - InputStream fileStream = minioUtil.downloadFile(item.objectName()); - // 在 ZIP 中创建对应的条目，保持目录结构 - String result = objectName.replace(entity.getFileUrl(), ""); - ZipEntry zipEntry = new ZipEntry(entity.getFileName() + File.separator + result); - zos.putNextEntry(zipEntry); - IOUtils.copy(fileStream, zos); - zos.closeEntry(); - fileStream.close(); - - } - } else { - InputStream fileStream = minioUtil.downloadFile(entity.getFileUrl()); - // 创建 ZIP 条目 - ZipEntry zipEntry = new ZipEntry(entity.getFileName()); - zos.putNextEntry(zipEntry); - // 使用 IOUtils.copy 直接复制文件流 - IOUtils.copy(fileStream, zos); - zos.closeEntry(); + if ("1".equals(entity.getIsDirectory())) { // 如果是目录，处理文件夹中的文件 + handleDirectoryFiles(zos, entity); + } else { // 普通文件直接打包 + handleSingleFile(zos, entity); } - } } catch (Exception e) { - e.printStackTrace(); + throw new IOException("Error occurred while downloading files", e); + } + } + + // 处理目录文件 + private void handleDirectoryFiles(ZipOutputStream zos, DataSetBasicFileEntity directory) throws IOException, MinioException { + Set fileWithChildren = getFileWithChildren(directory.getFileId()); + for (Long fileId : fileWithChildren) { + DataSetBasicFileEntity fileEntity = dataSetBasicFileMapper.selectDataSetBasicFileByFileId(fileId); + if (!"1".equals(fileEntity.getIsDirectory())) { + String fullDirectoryPath = getFullDirectoryPath(fileEntity, directory.getParentId()); + System.err.println(fullDirectoryPath); + addFileToZip(zos, fullDirectoryPath, fileEntity); + } + } + } + + // 递归获取文件的完整路径（从文件到根目录） + private String getFullDirectoryPath(DataSetBasicFileEntity fileEntity,Long directoryId) { + StringBuilder directoryPath = new StringBuilder(fileEntity.getFileName()); + Long parentId = fileEntity.getParentId(); + while (parentId != null && !parentId.equals(directoryId)) { // 判断是否有父目录 + DataSetBasicFileEntity parentEntity = dataSetBasicFileMapper.selectDataSetBasicFileByFileId(parentId); + if (parentEntity != null) { + directoryPath.insert(0, parentEntity.getFileName() + File.separator); // 将父目录名加到路径前 + parentId = parentEntity.getParentId(); // 更新父目录 + } else { + break; // 如果找不到父目录，则退出 + } + } + + return directoryPath.toString(); + } + + // 处理单个文件 + private void handleSingleFile(ZipOutputStream zos, DataSetBasicFileEntity fileEntity) throws IOException, MinioException { + addFileToZip(zos, "", fileEntity); + } + + // 将文件添加到 ZIP 文件中 + private void addFileToZip(ZipOutputStream zos, String parentDirectory, DataSetBasicFileEntity fileEntity) throws IOException, MinioException { + InputStream fileStream = null; + try { + fileStream = minioUtil.downloadFile(fileEntity.getFileUrl()); + String zipEntryName = parentDirectory.isEmpty() ? fileEntity.getFileName() : parentDirectory + File.separator + fileEntity.getFileName(); + ZipEntry zipEntry = new ZipEntry(parentDirectory); + zos.putNextEntry(zipEntry); + IOUtils.copy(fileStream, zos); + zos.closeEntry(); + } finally { + if (fileStream != null) { + fileStream.close(); + } } } } diff --git a/bonus-modules/bonus-ai/src/main/java/com/bonus/ai/service/Impl/dataset/DatasetServiceImpl.java b/bonus-modules/bonus-ai/src/main/java/com/bonus/ai/service/Impl/dataset/DatasetServiceImpl.java index ef29e84..5a08652 100644 --- a/bonus-modules/bonus-ai/src/main/java/com/bonus/ai/service/Impl/dataset/DatasetServiceImpl.java +++ b/bonus-modules/bonus-ai/src/main/java/com/bonus/ai/service/Impl/dataset/DatasetServiceImpl.java @@ -16,6 +16,7 @@ import com.bonus.common.security.utils.SecurityUtils; import io.minio.Result; import io.minio.errors.MinioException; import io.minio.messages.Item; +import javafx.util.Pair; import org.apache.commons.io.IOUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -27,13 +28,12 @@ import com.bonus.common.security.utils.SecurityUtils; import javax.annotation.Resource; import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletResponse; -import java.io.File; -import java.io.IOException; -import java.io.InputStream; +import java.io.*; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; import java.util.List; +import java.util.concurrent.*; import java.util.stream.Collectors; import java.util.zip.ZipEntry; import java.util.zip.ZipOutputStream; @@ -208,25 +208,106 @@ public class DatasetServiceImpl implements DatasetService { } // 下载多个文件（可以选择压缩成 ZIP） - private void downloadMultipleFiles(HttpServletResponse response, List list) throws IOException, MinioException { - // 示例：将多个文件打包成一个 ZIP 文件 + private void downloadMultipleFiles(HttpServletResponse response, List list) throws IOException { + int batchSize = 500; // 每批次的文件数量 response.setContentType("application/zip"); - response.setHeader("Content-Disposition", "attachment; filename=files.zip"); - try (ZipOutputStream zos = new ZipOutputStream(response.getOutputStream())) { - for (DataSetBasicFileEntity entity : list) { - InputStream fileStream = minioUtil.downloadFile(entity.getFileUrl()); - // 创建 ZIP 条目 - ZipEntry zipEntry = new ZipEntry(entity.getFileName()); - zos.putNextEntry(zipEntry); - // 使用 IOUtils.copy 直接复制文件流 - IOUtils.copy(fileStream, zos); - zos.closeEntry(); + response.setHeader("Content-Disposition", "attachment; filename=all_files.zip"); + + ExecutorService executorService = Executors.newFixedThreadPool(Math.min(16, Runtime.getRuntime().availableProcessors())); + BlockingQueue zipQueue = new LinkedBlockingQueue<>(); + + try (ZipOutputStream zos = new ZipOutputStream(new BufferedOutputStream(response.getOutputStream()))) { + // 异步压缩线程 + Thread zipThread = new Thread(() -> { + try { + while (true) { + ZipEntryData zipData = zipQueue.take(); + if (zipData.isPoisonPill()) { + break; // 检测结束标志 + } + zos.putNextEntry(new ZipEntry(zipData.getFileName())); + zos.write(zipData.getData()); + zos.closeEntry(); + } + } catch (Exception e) { + e.printStackTrace(); + } + }); + zipThread.start(); + + // 并行下载与数据入队 + List> futures = new ArrayList<>(); + for (int i = 0; i < list.size(); i += batchSize) { + int start = i; + int end = Math.min(i + batchSize, list.size()); + List batch = list.subList(start, end); + + futures.add(CompletableFuture.runAsync(() -> { + for (DataSetBasicFileEntity entity : batch) { + try (InputStream fileStream = minioUtil.downloadFile(entity.getFileUrl()); + ByteArrayOutputStream byteStream = new ByteArrayOutputStream()) { + byte[] buffer = new byte[16384]; + int len; + while ((len = fileStream.read(buffer)) != -1) { + byteStream.write(buffer, 0, len); + } + zipQueue.put(new ZipEntryData(entity.getFileName(), byteStream.toByteArray())); + } catch (Exception e) { + e.printStackTrace(); + } + } + }, executorService)); } + + // 等待所有下载完成 + CompletableFuture.allOf(futures.toArray(new CompletableFuture[0])).join(); + zipQueue.put(ZipEntryData.poisonPill()); // 放入结束标志 + zipThread.join(); } catch (Exception e) { e.printStackTrace(); + } finally { + executorService.shutdown(); } } + private static class ZipEntryData { + private final String fileName; + private final byte[] data; + private final boolean poisonPill; + + // 常规构造函数 + public ZipEntryData(String fileName, byte[] data, boolean poisonPill) { + this.fileName = fileName; + this.data = data; + this.poisonPill = poisonPill; + } + + // 重载构造函数，默认 poisonPill 为 false + public ZipEntryData(String fileName, byte[] data) { + this(fileName, data, false); + } + + public static ZipEntryData poisonPill() { + return new ZipEntryData(null, null, true); + } + + public String getFileName() { + return fileName; + } + + public byte[] getData() { + return data; + } + + public boolean isPoisonPill() { + return poisonPill; + } + } + + + + + /** * 插入数据集文件映射关系 @@ -274,6 +355,11 @@ public class DatasetServiceImpl implements DatasetService { * @return 如果文件符合条件则返回 true，否则返回 false */ private boolean isValidFile(DataSetBasicFileEntity file, List supportedFormats) { + // 如果 supportedFormats 为空，不校验后缀，直接返回文件不是目录的结果 + if (supportedFormats == null || supportedFormats.isEmpty()) { + return "0".equals(file.getIsDirectory()); + } + // 校验文件后缀 return "0".equals(file.getIsDirectory()) && // 确保文件不是目录 supportedFormats.stream() // 遍历支持的文件后缀 .anyMatch(format -> file.getFileName().toLowerCase().endsWith(format)); // 文件名后缀匹配 diff --git a/bonus-modules/bonus-ai/src/main/resources/mapper/DataSetBasicFileMapper.xml b/bonus-modules/bonus-ai/src/main/resources/mapper/DataSetBasicFileMapper.xml index 26c5dc9..fe1430d 100644 --- a/bonus-modules/bonus-ai/src/main/resources/mapper/DataSetBasicFileMapper.xml +++ b/bonus-modules/bonus-ai/src/main/resources/mapper/DataSetBasicFileMapper.xml @@ -158,9 +158,8 @@ - update ai_basic_file - set del_flag='2' - where del_flag = '1' + delete from ai_basic_file where del_flag = '1' + diff --git a/bonus-modules/bonus-ai/src/main/resources/mapper/DatasetMapper.xml b/bonus-modules/bonus-ai/src/main/resources/mapper/DatasetMapper.xml index 70d3348..8969e5d 100644 --- a/bonus-modules/bonus-ai/src/main/resources/mapper/DatasetMapper.xml +++ b/bonus-modules/bonus-ai/src/main/resources/mapper/DatasetMapper.xml @@ -48,18 +48,9 @@ WHERE adfm.dataset_id = ad.dataset_id) AS annotatedCount, (SELECT COUNT(*) FROM ai_dataset_file_map adfm - WHERE adfm.dataset_id = ad.dataset_id AND adfm.is_annotated = '0') AS notAnnotatedCount, - adv.version_name AS latestVersionName + WHERE adfm.dataset_id = ad.dataset_id AND adfm.is_annotated = '0') AS notAnnotatedCount FROM ai_dataset ad - LEFT JOIN ( - SELECT task_id, dataset_id, version_name - FROM ai_dataset_version - WHERE (task_id, dataset_id, create_time) IN ( - SELECT task_id, dataset_id, MAX(create_time) - FROM ai_dataset_version - GROUP BY task_id, dataset_id - )) adv on adv.dataset_id = ad.dataset_id