提交代码

bonus-modules/bonus-ai/src/main/java/com/bonus/ai/service/Impl/DataSetBasicFileServiceImpl.java

@@ -145,6 +145,7 @@ public class DataSetBasicFileServiceImpl implements DataSetBasicFileService {
     @Override
     public AjaxResult deleteDataSetBasicFileByFileIds(Long[] fileIds) {
         try {
+
             int rows = dataSetBasicFileMapper.deleteDataSetBasicFileByFileIds(fileIds);
             return rows > 0 ? AjaxResult.success() : AjaxResult.error();
         } catch (Exception e) {
@@ -171,10 +172,10 @@ public class DataSetBasicFileServiceImpl implements DataSetBasicFileService {
                     // 如果存在，则修改文件名并递增 num
                     if (ObjectUtils.isNotEmpty(basicFile)) {
                         num++;  // 递增 num
-                        entity.setFileName(changeNumberInName(entity.getFileName(),num ));
+                        entity.setFileName(changeNumberInName(entity.getFileName(), num));
 
                     }
-                 } while (ObjectUtils.isNotEmpty(basicFile));
+                } while (ObjectUtils.isNotEmpty(basicFile));
                 entity.setDelFlag("0");
                 entity.setCreateBy(SecurityUtils.getUserId().toString());
                 rows += dataSetBasicFileMapper.updateDataSetBasicFile(entity);
@@ -185,6 +186,7 @@ public class DataSetBasicFileServiceImpl implements DataSetBasicFileService {
             return AjaxResult.error();
         }
     }
+
     // 改变文件夹或文件名最后()里的数字，同时保留原始后缀
     public static String changeNumberInName(String name, int newNumber) {
         // 提取文件扩展名
@@ -201,7 +203,7 @@ public class DataSetBasicFileServiceImpl implements DataSetBasicFileService {
 
         if (matcher.matches()) {
             // 替换括号中的数字为新的数字
-            String newName = name.substring(0, matcher.start(1)) +newNumber + name.substring(matcher.end(1));
+            String newName = name.substring(0, matcher.start(1)) + newNumber + name.substring(matcher.end(1));
             return newName + extension; // 加上原始文件扩展名
         } else {
             // 如果没有找到括号中的数字，返回原文件夹/文件名和扩展名
@@ -223,6 +225,7 @@ public class DataSetBasicFileServiceImpl implements DataSetBasicFileService {
             return 0;
         }
     }
+
     /**
      * 文件分片上传
      *
@@ -383,7 +386,7 @@ public class DataSetBasicFileServiceImpl implements DataSetBasicFileService {
     @Override
     public AjaxResult emptyRecycleBin() {
         dataSetBasicFileMapper.emptyRecycleBin();
-        return  AjaxResult.success();
+        return AjaxResult.success();
     }
 
     /**
@@ -598,45 +601,72 @@ public class DataSetBasicFileServiceImpl implements DataSetBasicFileService {
 
     // 下载多个文件（可以选择压缩成 ZIP）
     private void downloadMultipleFiles(HttpServletResponse response, List<DataSetBasicFileEntity> list) throws IOException, MinioException {
-        // 示例：将多个文件打包成一个 ZIP 文件
         response.setContentType("application/zip");
         response.setHeader("Content-Disposition", "attachment; filename=files.zip");
+
         try (ZipOutputStream zos = new ZipOutputStream(response.getOutputStream())) {
+            // 遍历每个文件实体
             for (DataSetBasicFileEntity entity : list) {
-                if ("1".equals(entity.getIsDirectory())) {
+                if ("1".equals(entity.getIsDirectory())) {  // 如果是目录，处理文件夹中的文件
-                    Iterable<Result<Item>> objectList = minioUtil.listObjects(entity.getFileUrl());
+                    handleDirectoryFiles(zos, entity);
-                    // 遍历每个对象
+                } else {  // 普通文件直接打包
-                    for (Result<Item> itemResult : objectList) {
+                    handleSingleFile(zos, entity);
-                        Item item = itemResult.get();
-                        // 跳过目录（如果 MinIO 中有虚拟目录）
-                        if (item.isDir()) {
-                            continue;
-                        }
-                        // 获取文件路径和文件流
-                        String objectName = item.objectName();
-                        InputStream fileStream = minioUtil.downloadFile(item.objectName());
-                        // 在 ZIP 中创建对应的条目，保持目录结构
-                        String result = objectName.replace(entity.getFileUrl(), "");
-                        ZipEntry zipEntry = new ZipEntry(entity.getFileName() + File.separator + result);
-                        zos.putNextEntry(zipEntry);
-                        IOUtils.copy(fileStream, zos);
-                        zos.closeEntry();
-                        fileStream.close();
-
-                    }
-                } else {
-                    InputStream fileStream = minioUtil.downloadFile(entity.getFileUrl());
-                    // 创建 ZIP 条目
-                    ZipEntry zipEntry = new ZipEntry(entity.getFileName());
-                    zos.putNextEntry(zipEntry);
-                    // 使用 IOUtils.copy 直接复制文件流
-                    IOUtils.copy(fileStream, zos);
-                    zos.closeEntry();
                 }
-
             }
         } catch (Exception e) {
-            e.printStackTrace();
+            throw new IOException("Error occurred while downloading files", e);
+        }
+    }
+
+    // 处理目录文件
+    private void handleDirectoryFiles(ZipOutputStream zos, DataSetBasicFileEntity directory) throws IOException, MinioException {
+        Set<Long> fileWithChildren = getFileWithChildren(directory.getFileId());
+        for (Long fileId : fileWithChildren) {
+            DataSetBasicFileEntity fileEntity = dataSetBasicFileMapper.selectDataSetBasicFileByFileId(fileId);
+            if (!"1".equals(fileEntity.getIsDirectory())) {
+                String fullDirectoryPath = getFullDirectoryPath(fileEntity, directory.getParentId());
+                System.err.println(fullDirectoryPath);
+                addFileToZip(zos, fullDirectoryPath, fileEntity);
+            }
+        }
+    }
+
+    // 递归获取文件的完整路径（从文件到根目录）
+    private String getFullDirectoryPath(DataSetBasicFileEntity fileEntity,Long directoryId) {
+        StringBuilder directoryPath = new StringBuilder(fileEntity.getFileName());
+        Long parentId = fileEntity.getParentId();
+        while (parentId != null && !parentId.equals(directoryId)) {  // 判断是否有父目录
+            DataSetBasicFileEntity parentEntity = dataSetBasicFileMapper.selectDataSetBasicFileByFileId(parentId);
+            if (parentEntity != null) {
+                directoryPath.insert(0, parentEntity.getFileName() + File.separator);  // 将父目录名加到路径前
+                parentId = parentEntity.getParentId();  // 更新父目录
+            } else {
+                break;  // 如果找不到父目录，则退出
+            }
+        }
+
+        return directoryPath.toString();
+    }
+
+    // 处理单个文件
+    private void handleSingleFile(ZipOutputStream zos, DataSetBasicFileEntity fileEntity) throws IOException, MinioException {
+        addFileToZip(zos, "", fileEntity);
+    }
+
+    // 将文件添加到 ZIP 文件中
+    private void addFileToZip(ZipOutputStream zos, String parentDirectory, DataSetBasicFileEntity fileEntity) throws IOException, MinioException {
+        InputStream fileStream = null;
+        try {
+            fileStream = minioUtil.downloadFile(fileEntity.getFileUrl());
+            String zipEntryName = parentDirectory.isEmpty() ? fileEntity.getFileName() : parentDirectory + File.separator + fileEntity.getFileName();
+            ZipEntry zipEntry = new ZipEntry(parentDirectory);
+            zos.putNextEntry(zipEntry);
+            IOUtils.copy(fileStream, zos);
+            zos.closeEntry();
+        } finally {
+            if (fileStream != null) {
+                fileStream.close();
+            }
         }
     }
 }

bonus-modules/bonus-ai/src/main/java/com/bonus/ai/service/Impl/dataset/DatasetServiceImpl.java

@@ -16,6 +16,7 @@ import com.bonus.common.security.utils.SecurityUtils;
 import io.minio.Result;
 import io.minio.errors.MinioException;
 import io.minio.messages.Item;
+import javafx.util.Pair;
 import org.apache.commons.io.IOUtils;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -27,13 +28,12 @@ import com.bonus.common.security.utils.SecurityUtils;
 import javax.annotation.Resource;
 import javax.servlet.http.HttpServletRequest;
 import javax.servlet.http.HttpServletResponse;
-import java.io.File;
+import java.io.*;
-import java.io.IOException;
-import java.io.InputStream;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collections;
 import java.util.List;
+import java.util.concurrent.*;
 import java.util.stream.Collectors;
 import java.util.zip.ZipEntry;
 import java.util.zip.ZipOutputStream;
@@ -208,25 +208,106 @@ public class DatasetServiceImpl implements DatasetService {
     }
 
     // 下载多个文件（可以选择压缩成 ZIP）
-    private void downloadMultipleFiles(HttpServletResponse response, List<DataSetBasicFileEntity> list) throws IOException, MinioException {
+    private void downloadMultipleFiles(HttpServletResponse response, List<DataSetBasicFileEntity> list) throws IOException {
-        // 示例：将多个文件打包成一个 ZIP 文件
+        int batchSize = 500; // 每批次的文件数量
         response.setContentType("application/zip");
-        response.setHeader("Content-Disposition", "attachment; filename=files.zip");
+        response.setHeader("Content-Disposition", "attachment; filename=all_files.zip");
-        try (ZipOutputStream zos = new ZipOutputStream(response.getOutputStream())) {
+
-            for (DataSetBasicFileEntity entity : list) {
+        ExecutorService executorService = Executors.newFixedThreadPool(Math.min(16, Runtime.getRuntime().availableProcessors()));
-                InputStream fileStream = minioUtil.downloadFile(entity.getFileUrl());
+        BlockingQueue<ZipEntryData> zipQueue = new LinkedBlockingQueue<>();
-                // 创建 ZIP 条目
+
-                ZipEntry zipEntry = new ZipEntry(entity.getFileName());
+        try (ZipOutputStream zos = new ZipOutputStream(new BufferedOutputStream(response.getOutputStream()))) {
-                zos.putNextEntry(zipEntry);
+            // 异步压缩线程
-                // 使用 IOUtils.copy 直接复制文件流
+            Thread zipThread = new Thread(() -> {
-                IOUtils.copy(fileStream, zos);
+                try {
-                zos.closeEntry();
+                    while (true) {
+                        ZipEntryData zipData = zipQueue.take();
+                        if (zipData.isPoisonPill()) {
+                            break; // 检测结束标志
+                        }
+                        zos.putNextEntry(new ZipEntry(zipData.getFileName()));
+                        zos.write(zipData.getData());
+                        zos.closeEntry();
+                    }
+                } catch (Exception e) {
+                    e.printStackTrace();
+                }
+            });
+            zipThread.start();
+
+            // 并行下载与数据入队
+            List<CompletableFuture<Void>> futures = new ArrayList<>();
+            for (int i = 0; i < list.size(); i += batchSize) {
+                int start = i;
+                int end = Math.min(i + batchSize, list.size());
+                List<DataSetBasicFileEntity> batch = list.subList(start, end);
+
+                futures.add(CompletableFuture.runAsync(() -> {
+                    for (DataSetBasicFileEntity entity : batch) {
+                        try (InputStream fileStream = minioUtil.downloadFile(entity.getFileUrl());
+                             ByteArrayOutputStream byteStream = new ByteArrayOutputStream()) {
+                            byte[] buffer = new byte[16384];
+                            int len;
+                            while ((len = fileStream.read(buffer)) != -1) {
+                                byteStream.write(buffer, 0, len);
+                            }
+                            zipQueue.put(new ZipEntryData(entity.getFileName(), byteStream.toByteArray()));
+                        } catch (Exception e) {
+                            e.printStackTrace();
+                        }
+                    }
+                }, executorService));
             }
+
+            // 等待所有下载完成
+            CompletableFuture.allOf(futures.toArray(new CompletableFuture[0])).join();
+            zipQueue.put(ZipEntryData.poisonPill()); // 放入结束标志
+            zipThread.join();
         } catch (Exception e) {
             e.printStackTrace();
+        } finally {
+            executorService.shutdown();
         }
     }
 
+    private static class ZipEntryData {
+        private final String fileName;
+        private final byte[] data;
+        private final boolean poisonPill;
+
+        // 常规构造函数
+        public ZipEntryData(String fileName, byte[] data, boolean poisonPill) {
+            this.fileName = fileName;
+            this.data = data;
+            this.poisonPill = poisonPill;
+        }
+
+        // 重载构造函数，默认 poisonPill 为 false
+        public ZipEntryData(String fileName, byte[] data) {
+            this(fileName, data, false);
+        }
+
+        public static ZipEntryData poisonPill() {
+            return new ZipEntryData(null, null, true);
+        }
+
+        public String getFileName() {
+            return fileName;
+        }
+
+        public byte[] getData() {
+            return data;
+        }
+
+        public boolean isPoisonPill() {
+            return poisonPill;
+        }
+    }
+
+
+
+
+
 
     /**
      * 插入数据集文件映射关系
@@ -274,6 +355,11 @@ public class DatasetServiceImpl implements DatasetService {
      * @return 如果文件符合条件则返回 true，否则返回 false
      */
     private boolean isValidFile(DataSetBasicFileEntity file, List<String> supportedFormats) {
+        // 如果 supportedFormats 为空，不校验后缀，直接返回文件不是目录的结果
+        if (supportedFormats == null || supportedFormats.isEmpty()) {
+            return "0".equals(file.getIsDirectory());
+        }
+        // 校验文件后缀
         return "0".equals(file.getIsDirectory()) && // 确保文件不是目录
                 supportedFormats.stream() // 遍历支持的文件后缀
                         .anyMatch(format -> file.getFileName().toLowerCase().endsWith(format)); // 文件名后缀匹配

bonus-modules/bonus-ai/src/main/resources/mapper/DataSetBasicFileMapper.xml

@@ -158,9 +158,8 @@
         </foreach>
     </update>
     <update id="emptyRecycleBin">
-        update ai_basic_file
+        delete from ai_basic_file  where del_flag = '1'
-        set del_flag='2'
+
-        where del_flag = '1'
     </update>
 
     <delete id="deleteDataSetBasicFileByFileIds" parameterType="String">

bonus-modules/bonus-ai/src/main/resources/mapper/DatasetMapper.xml

@@ -48,18 +48,9 @@
              WHERE adfm.dataset_id = ad.dataset_id) AS annotatedCount,
             (SELECT COUNT(*)
              FROM ai_dataset_file_map adfm
-             WHERE adfm.dataset_id = ad.dataset_id AND adfm.is_annotated = '0') AS notAnnotatedCount,
+             WHERE adfm.dataset_id = ad.dataset_id AND adfm.is_annotated = '0') AS notAnnotatedCount
-            adv.version_name AS latestVersionName
         FROM
             ai_dataset ad
-        LEFT JOIN (
-        SELECT task_id, dataset_id, version_name
-        FROM ai_dataset_version
-        WHERE (task_id, dataset_id, create_time) IN (
-        SELECT task_id, dataset_id, MAX(create_time)
-        FROM ai_dataset_version
-        GROUP BY task_id, dataset_id
-        )) adv on adv.dataset_id  = ad.dataset_id
     </sql>
 
     <insert id="insert" parameterType="com.bonus.ai.domain.dataset.DataSetEntity" useGeneratedKeys="true" keyProperty="datasetId">