数据识别导出下载

bonus-admin/src/main/java/com/bonus/web/service/data/ExportExcelService.java

@@ -1,6 +1,9 @@
 package com.bonus.web.service.data;
 
+import com.alibaba.excel.EasyExcel;
+import com.alibaba.excel.ExcelWriter;
 import com.alibaba.excel.write.handler.SheetWriteHandler;
+import com.alibaba.excel.write.metadata.WriteSheet;
 import com.alibaba.excel.write.metadata.holder.WriteSheetHolder;
 import com.alibaba.excel.write.metadata.holder.WriteWorkbookHolder;
 import com.bonus.common.domain.data.dto.ParamsDto;
@@ -11,19 +14,17 @@ import org.springframework.stereotype.Service;
 import cn.hutool.core.date.DateUtil;
 import cn.hutool.core.date.DateUnit;
 import cn.hutool.core.util.ZipUtil;
-import com.alibaba.excel.EasyExcel;
-import com.alibaba.excel.ExcelWriter;
-import com.alibaba.excel.write.metadata.WriteSheet;
 import org.springframework.data.redis.core.RedisTemplate;
 import org.springframework.scheduling.annotation.Async;
 
 import javax.annotation.Resource;
 import java.io.File;
 import java.util.*;
-import java.util.concurrent.TimeUnit;
+import java.util.concurrent.*;
+import java.util.concurrent.atomic.AtomicInteger;
 
 /**
- * 识别数据导出-业务逻辑层 (已修复 IO 关闭异常)
+ * 识别数据导出-业务逻辑层 (多线程+防OOM版本)
  */
 @Service(value = "ExportExcelService")
 @Slf4j
@@ -32,6 +33,13 @@ public class ExportExcelService {
     @Resource
     private RedisTemplate<String, Object> redisTemplate;
 
+    // 自定义线程池：限制核心并发数为2，最大为3，防止大量图片IO导致内存溢出
+    private static final ExecutorService exportThreadPool = new ThreadPoolExecutor(
+            2, 3, 60L, TimeUnit.SECONDS,
+            new LinkedBlockingQueue<>(100),
+            new ThreadPoolExecutor.CallerRunsPolicy() // 队列满时由主线程执行，防止任务堆积
+    );
+
     private static final String REDIS_KEY_PREFIX = "export:task:";
     private static final String TEMP_PATH = System.getProperty("java.io.tmpdir") + File.separator + "export_task" + File.separator;
 
@@ -46,21 +54,13 @@ public class ExportExcelService {
         return taskId;
     }
 
-    public ExportTask getTask(String taskId) {
-        return (ExportTask) redisTemplate.opsForValue().get(REDIS_KEY_PREFIX + taskId);
-    }
-
-    private void updateTaskInRedis(ExportTask task) {
-        redisTemplate.opsForValue().set(REDIS_KEY_PREFIX + task.getTaskId(), task, 24, TimeUnit.HOURS);
-    }
-
     @Async("taskExecutor")
     public void runAsyncExport(String taskId, ParamsDto dto) {
         try {
             Date start = dto.getStartTime();
             Date end = dto.getEndTime();
             long dayDiff = DateUtil.between(start, end, DateUnit.DAY);
-            log.info("临时文件地址：{}", TEMP_PATH);
+
             File tempDir = new File(TEMP_PATH);
             if (!tempDir.exists()) tempDir.mkdirs();
 
@@ -68,51 +68,58 @@ public class ExportExcelService {
             String downloadFileName;
 
             if (dayDiff <= 0) {
+                // 单天任务：直接处理
                 downloadFileName = "数据识别_" + DateUtil.formatDate(start) + ".xlsx";
                 finalResultFile = new File(TEMP_PATH + taskId + "_" + downloadFileName);
-                generateExcel(finalResultFile, start, taskId, 0, 100);
+                generateExcel(finalResultFile, start, taskId, 0, 100, true);
             } else {
-                List<File> subFiles = new ArrayList<>();
+                // 多天任务：多线程处理
                 List<Date> dateRange = getDatesBetween(start, end);
                 int totalDays = dateRange.size();
+                List<File> subFiles = Collections.synchronizedList(new ArrayList<>());
+                CountDownLatch latch = new CountDownLatch(totalDays);
+                AtomicInteger finishedCount = new AtomicInteger(0);
 
-                for (int i = 0; i < totalDays; i++) {
+                for (Date currentDate : dateRange) {
-                    Date currentDate = dateRange.get(i);
+                    exportThreadPool.execute(() -> {
-                    File subFile = new File(TEMP_PATH + taskId + "_" + i + ".xlsx");
+                        try {
-                    int startPct = (int) ((double) i / totalDays * 95);
+                            String subFileName = taskId + "_数据识别_" + DateUtil.formatDate(currentDate) + ".xlsx";
-                    int endPct = (int) ((double) (i + 1) / totalDays * 95);
+                            File subFile = new File(TEMP_PATH + subFileName);
-                    generateExcel(subFile, currentDate, taskId, startPct, endPct);
+                            // 传入 false，表示子线程不直接更新 Redis 整体进度，由主线程负责更新
-                    subFiles.add(subFile);
+                            generateExcel(subFile, currentDate, taskId, -1, -1, false);
+                            subFiles.add(subFile);
+                        } catch (Exception e) {
+                            log.error("子任务执行失败: {}", currentDate, e);
+                        } finally {
+                            // 更新整体进度：0-95%
+                            int finished = finishedCount.incrementAndGet();
+                            int currentProgress = (int) ((double) finished / totalDays * 95);
+                            updateStatusProgress(taskId, currentProgress, "running");
+                            latch.countDown();
+                        }
+                    });
                 }
 
+                // 最多等待子任务处理 30 分钟
+                latch.await(30, TimeUnit.MINUTES);
+
                 updateStatusProgress(taskId, 98, "running");
-                downloadFileName = "批量数据识别_" + DateUtil.formatDate(new Date()) + ".zip";
+                String dateRangeStr = DateUtil.format(start, "yyyyMMdd") + "-" + DateUtil.format(end, "yyyyMMdd");
+                downloadFileName = "批量数据识别_" + dateRangeStr + ".zip";
                 finalResultFile = new File(TEMP_PATH + taskId + ".zip");
+
                 ZipUtil.zip(finalResultFile, false, subFiles.toArray(new File[0]));
                 subFiles.forEach(File::delete);
             }
 
-            ExportTask task = getTask(taskId);
+            // 完成任务更新
-            if (task != null) {
+            completeTask(taskId, finalResultFile, downloadFileName);
-                task.setFilePath(finalResultFile.getAbsolutePath());
-                task.setFileName(downloadFileName);
-                task.setProgress(100);
-                task.setStatus("completed");
-                updateTaskInRedis(task);
-            }
         } catch (Exception e) {
-            log.error("导出异步任务失败", e);
+            handleError(taskId, e);
-            ExportTask task = getTask(taskId);
-            if (task != null) {
-                task.setStatus("failed");
-                task.setMessage("导出异常: " + e.getMessage());
-                updateTaskInRedis(task);
-            }
         }
     }
 
-    private void generateExcel(File file, Date date, String taskId, int baseProgress, int maxProgress) {
+    private void generateExcel(File file, Date date, String taskId, int basePct, int maxPct, boolean updateRedis) {
-        // 定义冻结表头的拦截器
         SheetWriteHandler freezePaneHandler = new SheetWriteHandler() {
             @Override
             public void afterSheetCreate(WriteWorkbookHolder writeWorkbookHolder, WriteSheetHolder writeSheetHolder) {
@@ -121,14 +128,16 @@ public class ExportExcelService {
                 writeSheetHolder.getSheet().createFreezePane(0, 1, 0, 1);
             }
         };
+
         ExcelWriter excelWriter = null;
         try {
-            // 修复点1: 显式构建并管理 ExcelWriter，不完全依赖外部 try-with-resources
+            excelWriter = EasyExcel.write(file, DataRecognitionExportVO.class)
-            excelWriter = EasyExcel.write(file, DataRecognitionExportVO.class).registerWriteHandler(freezePaneHandler).build();
+                    .registerWriteHandler(freezePaneHandler)
+                    .build();
             WriteSheet writeSheet = EasyExcel.writerSheet("识别数据").build();
 
             int pageNo = 1;
-            int pageSize = 200; // 修复点2: 有图片时减小 pageSize (建议200)，降低 IO 负载
+            int pageSize = 150; // 多线程下进一步减小 pageSize 降低内存压力
             long totalCount = mockDbCount(date);
             int totalPages = (int) Math.ceil((double) totalCount / pageSize);
 
@@ -144,56 +153,71 @@ public class ExportExcelService {
                 for (int i = 0; i < dbDataList.size(); i++) {
                     DataRecognitionExportVO entity = dbDataList.get(i);
                     DataRecognitionExportVO vo = new DataRecognitionExportVO();
-
+                    // 属性映射...
-                    // 属性映射
                     vo.setIndex((pageNo - 1) * pageSize + i + 1);
                     vo.setLocation(entity.getLocation());
                     vo.setRecognitionTime(entity.getRecognitionTime());
                     vo.setPlateColor(entity.getPlateColor());
                     vo.setVehicleType(entity.getVehicleType());
 
-                    // 修复点3: 图片路径防御性处理，防止读取流失败中断整个写入
                     try {
-                        String absolutePath = entity.getPhotoPath();
+                        if (entity.getPhotoPath() != null) {
-                        if (absolutePath != null && !absolutePath.isEmpty()) {
+                            File img = new File(entity.getPhotoPath());
-                            File imageFile = new File(absolutePath);
+                            if (img.exists() && img.canRead()) vo.setPhotoFile(img);
-                            if (imageFile.exists() && imageFile.canRead() && imageFile.isFile()) {
-                                vo.setPhotoFile(imageFile);
-                            }
                         }
-                    } catch (Exception imgEx) {
+                    } catch (Exception ignored) {}
-                        log.warn("行 {} 图片读取失败，跳过图片插入: {}", vo.getIndex(), imgEx.getMessage());
-                    }
-
                     voList.add(vo);
                 }
 
                 excelWriter.write(voList, writeSheet);
 
-                // 更新进度
+                // 单天模式下更新详细进度，多天模式由主任务更新，此处置空
-                double fileRatio = (double) pageNo / totalPages;
+                if (updateRedis) {
-                int currentProgress = baseProgress + (int) (fileRatio * (maxProgress - baseProgress));
+                    double fileRatio = (double) pageNo / totalPages;
-                if (currentProgress >= 100) currentProgress = 99;
+                    int progress = basePct + (int) (fileRatio * (maxPct - basePct));
-                updateStatusProgress(taskId, currentProgress, "running");
+                    updateStatusProgress(taskId, Math.min(progress, 99), "running");
+                }
 
-                voList.clear();
+                voList.clear(); // 显式清理辅助 GC
                 pageNo++;
             }
-        } catch (Exception e) {
-            log.error("Excel 生成核心逻辑失败: ", e);
-            throw e;
         } finally {
-            // 修复点4: 确保在 finally 中安全关闭，并捕获关闭时的异常
             if (excelWriter != null) {
-                try {
+                try { excelWriter.finish(); } catch (Exception e) { log.error("IO Close Error", e); }
-                    excelWriter.finish(); // finish 内部会调用 close() 并清理 IO
-                } catch (Exception closeEx) {
-                    log.error("关闭 ExcelWriter IO 失败: ", closeEx);
-                }
             }
         }
     }
 
+    private void completeTask(String taskId, File file, String name) {
+        ExportTask task = getTask(taskId);
+        if (task != null) {
+            task.setFilePath(file.getAbsolutePath());
+            task.setFileName(name);
+            task.setProgress(100);
+            task.setStatus("completed");
+            updateTaskInRedis(task);
+        }
+    }
+
+    private void handleError(String taskId, Exception e) {
+        log.error("Export Error", e);
+        ExportTask task = getTask(taskId);
+        if (task != null) {
+            task.setStatus("failed");
+            task.setMessage(e.getMessage());
+            updateTaskInRedis(task);
+        }
+    }
+
+    // ... getTask, updateTaskInRedis, getDatesBetween, updateStatusProgress 保持不变 ...
+    public ExportTask getTask(String taskId) {
+        return (ExportTask) redisTemplate.opsForValue().get(REDIS_KEY_PREFIX + taskId);
+    }
+
+    private void updateTaskInRedis(ExportTask task) {
+        redisTemplate.opsForValue().set(REDIS_KEY_PREFIX + task.getTaskId(), task, 24, TimeUnit.HOURS);
+    }
+
     private void updateStatusProgress(String taskId, int progress, String status) {
         ExportTask task = getTask(taskId);
         if (task != null) {
@@ -208,12 +232,16 @@ public class ExportExcelService {
         Calendar cal = Calendar.getInstance();
         cal.setTime(start);
         while (cal.getTime().getTime() <= end.getTime()) {
-            list.add(cal.getTime());
+            list.add(new Date(cal.getTime().getTime()));
             cal.add(Calendar.DATE, 1);
         }
         return list;
     }
 
+    /*private long mockDbCount(Date date) { return 500; }
+    private List<DataRecognitionExportVO> mockDbQuery(Date date, int p, int s) {
+        return new ArrayList<>(); // 替换为实际查询
+    }*/
     private long mockDbCount(Date date) {
         return 1000; // 模拟少量数据测试
     }