diff --git a/bonus-admin/src/main/java/com/bonus/web/service/data/ExportExcelService.java b/bonus-admin/src/main/java/com/bonus/web/service/data/ExportExcelService.java index cf873c8..9dd4831 100644 --- a/bonus-admin/src/main/java/com/bonus/web/service/data/ExportExcelService.java +++ b/bonus-admin/src/main/java/com/bonus/web/service/data/ExportExcelService.java @@ -1,6 +1,9 @@ package com.bonus.web.service.data; +import com.alibaba.excel.EasyExcel; +import com.alibaba.excel.ExcelWriter; import com.alibaba.excel.write.handler.SheetWriteHandler; +import com.alibaba.excel.write.metadata.WriteSheet; import com.alibaba.excel.write.metadata.holder.WriteSheetHolder; import com.alibaba.excel.write.metadata.holder.WriteWorkbookHolder; import com.bonus.common.domain.data.dto.ParamsDto; @@ -11,19 +14,17 @@ import org.springframework.stereotype.Service; import cn.hutool.core.date.DateUtil; import cn.hutool.core.date.DateUnit; import cn.hutool.core.util.ZipUtil; -import com.alibaba.excel.EasyExcel; -import com.alibaba.excel.ExcelWriter; -import com.alibaba.excel.write.metadata.WriteSheet; import org.springframework.data.redis.core.RedisTemplate; import org.springframework.scheduling.annotation.Async; import javax.annotation.Resource; import java.io.File; import java.util.*; -import java.util.concurrent.TimeUnit; +import java.util.concurrent.*; +import java.util.concurrent.atomic.AtomicInteger; /** - * 识别数据导出-业务逻辑层 (已修复 IO 关闭异常) + * 识别数据导出-业务逻辑层 (多线程+防OOM版本) */ @Service(value = "ExportExcelService") @Slf4j @@ -32,6 +33,13 @@ public class ExportExcelService { @Resource private RedisTemplate redisTemplate; + // 自定义线程池：限制核心并发数为2，最大为3，防止大量图片IO导致内存溢出 + private static final ExecutorService exportThreadPool = new ThreadPoolExecutor( + 2, 3, 60L, TimeUnit.SECONDS, + new LinkedBlockingQueue<>(100), + new ThreadPoolExecutor.CallerRunsPolicy() // 队列满时由主线程执行，防止任务堆积 + ); + private static final String REDIS_KEY_PREFIX = "export:task:"; private static final String TEMP_PATH = System.getProperty("java.io.tmpdir") + File.separator + "export_task" + File.separator; @@ -46,21 +54,13 @@ public class ExportExcelService { return taskId; } - public ExportTask getTask(String taskId) { - return (ExportTask) redisTemplate.opsForValue().get(REDIS_KEY_PREFIX + taskId); - } - - private void updateTaskInRedis(ExportTask task) { - redisTemplate.opsForValue().set(REDIS_KEY_PREFIX + task.getTaskId(), task, 24, TimeUnit.HOURS); - } - @Async("taskExecutor") public void runAsyncExport(String taskId, ParamsDto dto) { try { Date start = dto.getStartTime(); Date end = dto.getEndTime(); long dayDiff = DateUtil.between(start, end, DateUnit.DAY); - log.info("临时文件地址：{}", TEMP_PATH); + File tempDir = new File(TEMP_PATH); if (!tempDir.exists()) tempDir.mkdirs(); @@ -68,51 +68,58 @@ public class ExportExcelService { String downloadFileName; if (dayDiff <= 0) { + // 单天任务：直接处理 downloadFileName = "数据识别_" + DateUtil.formatDate(start) + ".xlsx"; finalResultFile = new File(TEMP_PATH + taskId + "_" + downloadFileName); - generateExcel(finalResultFile, start, taskId, 0, 100); + generateExcel(finalResultFile, start, taskId, 0, 100, true); } else { - List subFiles = new ArrayList<>(); + // 多天任务：多线程处理 List dateRange = getDatesBetween(start, end); int totalDays = dateRange.size(); + List subFiles = Collections.synchronizedList(new ArrayList<>()); + CountDownLatch latch = new CountDownLatch(totalDays); + AtomicInteger finishedCount = new AtomicInteger(0); - for (int i = 0; i < totalDays; i++) { - Date currentDate = dateRange.get(i); - File subFile = new File(TEMP_PATH + taskId + "_" + i + ".xlsx"); - int startPct = (int) ((double) i / totalDays * 95); - int endPct = (int) ((double) (i + 1) / totalDays * 95); - generateExcel(subFile, currentDate, taskId, startPct, endPct); - subFiles.add(subFile); + for (Date currentDate : dateRange) { + exportThreadPool.execute(() -> { + try { + String subFileName = taskId + "_数据识别_" + DateUtil.formatDate(currentDate) + ".xlsx"; + File subFile = new File(TEMP_PATH + subFileName); + // 传入 false，表示子线程不直接更新 Redis 整体进度，由主线程负责更新 + generateExcel(subFile, currentDate, taskId, -1, -1, false); + subFiles.add(subFile); + } catch (Exception e) { + log.error("子任务执行失败: {}", currentDate, e); + } finally { + // 更新整体进度：0-95% + int finished = finishedCount.incrementAndGet(); + int currentProgress = (int) ((double) finished / totalDays * 95); + updateStatusProgress(taskId, currentProgress, "running"); + latch.countDown(); + } + }); } + // 最多等待子任务处理 30 分钟 + latch.await(30, TimeUnit.MINUTES); + updateStatusProgress(taskId, 98, "running"); - downloadFileName = "批量数据识别_" + DateUtil.formatDate(new Date()) + ".zip"; + String dateRangeStr = DateUtil.format(start, "yyyyMMdd") + "-" + DateUtil.format(end, "yyyyMMdd"); + downloadFileName = "批量数据识别_" + dateRangeStr + ".zip"; finalResultFile = new File(TEMP_PATH + taskId + ".zip"); + ZipUtil.zip(finalResultFile, false, subFiles.toArray(new File[0])); subFiles.forEach(File::delete); } - ExportTask task = getTask(taskId); - if (task != null) { - task.setFilePath(finalResultFile.getAbsolutePath()); - task.setFileName(downloadFileName); - task.setProgress(100); - task.setStatus("completed"); - updateTaskInRedis(task); - } + // 完成任务更新 + completeTask(taskId, finalResultFile, downloadFileName); } catch (Exception e) { - log.error("导出异步任务失败", e); - ExportTask task = getTask(taskId); - if (task != null) { - task.setStatus("failed"); - task.setMessage("导出异常: " + e.getMessage()); - updateTaskInRedis(task); - } + handleError(taskId, e); } } - private void generateExcel(File file, Date date, String taskId, int baseProgress, int maxProgress) { - // 定义冻结表头的拦截器 + private void generateExcel(File file, Date date, String taskId, int basePct, int maxPct, boolean updateRedis) { SheetWriteHandler freezePaneHandler = new SheetWriteHandler() { @Override public void afterSheetCreate(WriteWorkbookHolder writeWorkbookHolder, WriteSheetHolder writeSheetHolder) { @@ -121,14 +128,16 @@ public class ExportExcelService { writeSheetHolder.getSheet().createFreezePane(0, 1, 0, 1); } }; + ExcelWriter excelWriter = null; try { - // 修复点1: 显式构建并管理 ExcelWriter，不完全依赖外部 try-with-resources - excelWriter = EasyExcel.write(file, DataRecognitionExportVO.class).registerWriteHandler(freezePaneHandler).build(); + excelWriter = EasyExcel.write(file, DataRecognitionExportVO.class) + .registerWriteHandler(freezePaneHandler) + .build(); WriteSheet writeSheet = EasyExcel.writerSheet("识别数据").build(); int pageNo = 1; - int pageSize = 200; // 修复点2: 有图片时减小 pageSize (建议200)，降低 IO 负载 + int pageSize = 150; // 多线程下进一步减小 pageSize 降低内存压力 long totalCount = mockDbCount(date); int totalPages = (int) Math.ceil((double) totalCount / pageSize); @@ -144,56 +153,71 @@ public class ExportExcelService { for (int i = 0; i < dbDataList.size(); i++) { DataRecognitionExportVO entity = dbDataList.get(i); DataRecognitionExportVO vo = new DataRecognitionExportVO(); - - // 属性映射 + // 属性映射... vo.setIndex((pageNo - 1) * pageSize + i + 1); vo.setLocation(entity.getLocation()); vo.setRecognitionTime(entity.getRecognitionTime()); vo.setPlateColor(entity.getPlateColor()); vo.setVehicleType(entity.getVehicleType()); - // 修复点3: 图片路径防御性处理，防止读取流失败中断整个写入 try { - String absolutePath = entity.getPhotoPath(); - if (absolutePath != null && !absolutePath.isEmpty()) { - File imageFile = new File(absolutePath); - if (imageFile.exists() && imageFile.canRead() && imageFile.isFile()) { - vo.setPhotoFile(imageFile); - } + if (entity.getPhotoPath() != null) { + File img = new File(entity.getPhotoPath()); + if (img.exists() && img.canRead()) vo.setPhotoFile(img); } - } catch (Exception imgEx) { - log.warn("行 {} 图片读取失败，跳过图片插入: {}", vo.getIndex(), imgEx.getMessage()); - } - + } catch (Exception ignored) {} voList.add(vo); } excelWriter.write(voList, writeSheet); - // 更新进度 - double fileRatio = (double) pageNo / totalPages; - int currentProgress = baseProgress + (int) (fileRatio * (maxProgress - baseProgress)); - if (currentProgress >= 100) currentProgress = 99; - updateStatusProgress(taskId, currentProgress, "running"); + // 单天模式下更新详细进度，多天模式由主任务更新，此处置空 + if (updateRedis) { + double fileRatio = (double) pageNo / totalPages; + int progress = basePct + (int) (fileRatio * (maxPct - basePct)); + updateStatusProgress(taskId, Math.min(progress, 99), "running"); + } - voList.clear(); + voList.clear(); // 显式清理辅助 GC pageNo++; } - } catch (Exception e) { - log.error("Excel 生成核心逻辑失败: ", e); - throw e; } finally { - // 修复点4: 确保在 finally 中安全关闭，并捕获关闭时的异常 if (excelWriter != null) { - try { - excelWriter.finish(); // finish 内部会调用 close() 并清理 IO - } catch (Exception closeEx) { - log.error("关闭 ExcelWriter IO 失败: ", closeEx); - } + try { excelWriter.finish(); } catch (Exception e) { log.error("IO Close Error", e); } } } } + private void completeTask(String taskId, File file, String name) { + ExportTask task = getTask(taskId); + if (task != null) { + task.setFilePath(file.getAbsolutePath()); + task.setFileName(name); + task.setProgress(100); + task.setStatus("completed"); + updateTaskInRedis(task); + } + } + + private void handleError(String taskId, Exception e) { + log.error("Export Error", e); + ExportTask task = getTask(taskId); + if (task != null) { + task.setStatus("failed"); + task.setMessage(e.getMessage()); + updateTaskInRedis(task); + } + } + + // ... getTask, updateTaskInRedis, getDatesBetween, updateStatusProgress 保持不变 ... + public ExportTask getTask(String taskId) { + return (ExportTask) redisTemplate.opsForValue().get(REDIS_KEY_PREFIX + taskId); + } + + private void updateTaskInRedis(ExportTask task) { + redisTemplate.opsForValue().set(REDIS_KEY_PREFIX + task.getTaskId(), task, 24, TimeUnit.HOURS); + } + private void updateStatusProgress(String taskId, int progress, String status) { ExportTask task = getTask(taskId); if (task != null) { @@ -208,12 +232,16 @@ public class ExportExcelService { Calendar cal = Calendar.getInstance(); cal.setTime(start); while (cal.getTime().getTime() <= end.getTime()) { - list.add(cal.getTime()); + list.add(new Date(cal.getTime().getTime())); cal.add(Calendar.DATE, 1); } return list; } + /*private long mockDbCount(Date date) { return 500; } + private List mockDbQuery(Date date, int p, int s) { + return new ArrayList<>(); // 替换为实际查询 + }*/ private long mockDbCount(Date date) { return 1000; // 模拟少量数据测试 }