bonus
/
smart-car-service
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

数据识别导出下载

This commit is contained in:
cwchen 2025-12-30 18:36:14 +08:00
parent a579dd6418
commit 94436bd6d9
1 changed files with 101 additions and 73 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

174
bonus-admin/src/main/java/com/bonus/web/service/data/ExportExcelService.java
View File

@ -1,6 +1,9 @@
package com.bonus.web.service.data;
import com.alibaba.excel.EasyExcel;
import com.alibaba.excel.ExcelWriter;
import com.alibaba.excel.write.handler.SheetWriteHandler;
import com.alibaba.excel.write.metadata.WriteSheet;
import com.alibaba.excel.write.metadata.holder.WriteSheetHolder;
import com.alibaba.excel.write.metadata.holder.WriteWorkbookHolder;
import com.bonus.common.domain.data.dto.ParamsDto;
@ -11,19 +14,17 @@ import org.springframework.stereotype.Service;
import cn.hutool.core.date.DateUtil;
import cn.hutool.core.date.DateUnit;
import cn.hutool.core.util.ZipUtil;
import com.alibaba.excel.EasyExcel;
import com.alibaba.excel.ExcelWriter;
import com.alibaba.excel.write.metadata.WriteSheet;
import org.springframework.data.redis.core.RedisTemplate;
import org.springframework.scheduling.annotation.Async;
import javax.annotation.Resource;
import java.io.File;
import java.util.*;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.*;
import java.util.concurrent.atomic.AtomicInteger;
/**
* 识别数据导出-业务逻辑层 (已修复 IO 关闭异常)
* 识别数据导出-业务逻辑层 (多线程+防OOM版本)
*/
@Service(value = "ExportExcelService")
@Slf4j
@ -32,6 +33,13 @@ public class ExportExcelService {
@Resource
private RedisTemplate<String, Object> redisTemplate;
// 自定义线程池：限制核心并发数为2，最大为3，防止大量图片IO导致内存溢出
private static final ExecutorService exportThreadPool = new ThreadPoolExecutor(
2, 3, 60L, TimeUnit.SECONDS,
new LinkedBlockingQueue<>(100),
new ThreadPoolExecutor.CallerRunsPolicy() // 队列满时由主线程执行，防止任务堆积
);
private static final String REDIS_KEY_PREFIX = "export:task:";
private static final String TEMP_PATH = System.getProperty("java.io.tmpdir") + File.separator + "export_task" + File.separator;
@ -46,21 +54,13 @@ public class ExportExcelService {
return taskId;
}
public ExportTask getTask(String taskId) {
return (ExportTask) redisTemplate.opsForValue().get(REDIS_KEY_PREFIX + taskId);
}
private void updateTaskInRedis(ExportTask task) {
redisTemplate.opsForValue().set(REDIS_KEY_PREFIX + task.getTaskId(), task, 24, TimeUnit.HOURS);
}
@Async("taskExecutor")
public void runAsyncExport(String taskId, ParamsDto dto) {
try {
Date start = dto.getStartTime();
Date end = dto.getEndTime();
long dayDiff = DateUtil.between(start, end, DateUnit.DAY);
log.info("临时文件地址：{}", TEMP_PATH);
File tempDir = new File(TEMP_PATH);
if (!tempDir.exists()) tempDir.mkdirs();
@ -68,51 +68,58 @@ public class ExportExcelService {
String downloadFileName;
if (dayDiff <= 0) {
// 单天任务：直接处理
downloadFileName = "数据识别_" + DateUtil.formatDate(start) + ".xlsx";
finalResultFile = new File(TEMP_PATH + taskId + "_" + downloadFileName);
generateExcel(finalResultFile, start, taskId, 0, 100);
generateExcel(finalResultFile, start, taskId, 0, 100, true);
} else {
List<File> subFiles = new ArrayList<>();
// 多天任务：多线程处理
List<Date> dateRange = getDatesBetween(start, end);
int totalDays = dateRange.size();
List<File> subFiles = Collections.synchronizedList(new ArrayList<>());
CountDownLatch latch = new CountDownLatch(totalDays);
AtomicInteger finishedCount = new AtomicInteger(0);
for (int i = 0; i < totalDays; i++) {
Date currentDate = dateRange.get(i);
File subFile = new File(TEMP_PATH + taskId + "_" + i + ".xlsx");
int startPct = (int) ((double) i / totalDays * 95);
int endPct = (int) ((double) (i + 1) / totalDays * 95);
generateExcel(subFile, currentDate, taskId, startPct, endPct);
subFiles.add(subFile);
for (Date currentDate : dateRange) {
exportThreadPool.execute(() -> {
try {
String subFileName = taskId + "_数据识别_" + DateUtil.formatDate(currentDate) + ".xlsx";
File subFile = new File(TEMP_PATH + subFileName);
// 传入 false，表示子线程不直接更新 Redis 整体进度，由主线程负责更新
generateExcel(subFile, currentDate, taskId, -1, -1, false);
subFiles.add(subFile);
} catch (Exception e) {
log.error("子任务执行失败: {}", currentDate, e);
} finally {
// 更新整体进度：0-95%
int finished = finishedCount.incrementAndGet();
int currentProgress = (int) ((double) finished / totalDays * 95);
updateStatusProgress(taskId, currentProgress, "running");
latch.countDown();
}
});
}
// 最多等待子任务处理 30 分钟
latch.await(30, TimeUnit.MINUTES);
updateStatusProgress(taskId, 98, "running");
downloadFileName = "批量数据识别_" + DateUtil.formatDate(new Date()) + ".zip";
String dateRangeStr = DateUtil.format(start, "yyyyMMdd") + "-" + DateUtil.format(end, "yyyyMMdd");
downloadFileName = "批量数据识别_" + dateRangeStr + ".zip";
finalResultFile = new File(TEMP_PATH + taskId + ".zip");
ZipUtil.zip(finalResultFile, false, subFiles.toArray(new File[0]));
subFiles.forEach(File::delete);
}
ExportTask task = getTask(taskId);
if (task != null) {
task.setFilePath(finalResultFile.getAbsolutePath());
task.setFileName(downloadFileName);
task.setProgress(100);
task.setStatus("completed");
updateTaskInRedis(task);
}
// 完成任务更新
completeTask(taskId, finalResultFile, downloadFileName);
} catch (Exception e) {
log.error("导出异步任务失败", e);
ExportTask task = getTask(taskId);
if (task != null) {
task.setStatus("failed");
task.setMessage("导出异常: " + e.getMessage());
updateTaskInRedis(task);
}
handleError(taskId, e);
}
}
private void generateExcel(File file, Date date, String taskId, int baseProgress, int maxProgress) {
// 定义冻结表头的拦截器
private void generateExcel(File file, Date date, String taskId, int basePct, int maxPct, boolean updateRedis) {
SheetWriteHandler freezePaneHandler = new SheetWriteHandler() {
@Override
public void afterSheetCreate(WriteWorkbookHolder writeWorkbookHolder, WriteSheetHolder writeSheetHolder) {
@ -121,14 +128,16 @@ public class ExportExcelService {
writeSheetHolder.getSheet().createFreezePane(0, 1, 0, 1);
}
};
ExcelWriter excelWriter = null;
try {
// 修复点1: 显式构建并管理 ExcelWriter，不完全依赖外部 try-with-resources
excelWriter = EasyExcel.write(file, DataRecognitionExportVO.class).registerWriteHandler(freezePaneHandler).build();
excelWriter = EasyExcel.write(file, DataRecognitionExportVO.class)
.registerWriteHandler(freezePaneHandler)
.build();
WriteSheet writeSheet = EasyExcel.writerSheet("识别数据").build();
int pageNo = 1;
int pageSize = 200; // 修复点2: 有图片时减小 pageSize (建议200)，降低 IO 负载
int pageSize = 150; // 多线程下进一步减小 pageSize 降低内存压力
long totalCount = mockDbCount(date);
int totalPages = (int) Math.ceil((double) totalCount / pageSize);
@ -144,56 +153,71 @@ public class ExportExcelService {
for (int i = 0; i < dbDataList.size(); i++) {
DataRecognitionExportVO entity = dbDataList.get(i);
DataRecognitionExportVO vo = new DataRecognitionExportVO();
// 属性映射
// 属性映射...
vo.setIndex((pageNo - 1) * pageSize + i + 1);
vo.setLocation(entity.getLocation());
vo.setRecognitionTime(entity.getRecognitionTime());
vo.setPlateColor(entity.getPlateColor());
vo.setVehicleType(entity.getVehicleType());
// 修复点3: 图片路径防御性处理，防止读取流失败中断整个写入
try {
String absolutePath = entity.getPhotoPath();
if (absolutePath != null && !absolutePath.isEmpty()) {
File imageFile = new File(absolutePath);
if (imageFile.exists() && imageFile.canRead() && imageFile.isFile()) {
vo.setPhotoFile(imageFile);
}
if (entity.getPhotoPath() != null) {
File img = new File(entity.getPhotoPath());
if (img.exists() && img.canRead()) vo.setPhotoFile(img);
}
} catch (Exception imgEx) {
log.warn("行 {} 图片读取失败，跳过图片插入: {}", vo.getIndex(), imgEx.getMessage());
}
} catch (Exception ignored) {}
voList.add(vo);
}
excelWriter.write(voList, writeSheet);
// 更新进度
double fileRatio = (double) pageNo / totalPages;
int currentProgress = baseProgress + (int) (fileRatio * (maxProgress - baseProgress));
if (currentProgress >= 100) currentProgress = 99;
updateStatusProgress(taskId, currentProgress, "running");
// 单天模式下更新详细进度，多天模式由主任务更新，此处置空
if (updateRedis) {
double fileRatio = (double) pageNo / totalPages;
int progress = basePct + (int) (fileRatio * (maxPct - basePct));
updateStatusProgress(taskId, Math.min(progress, 99), "running");
}
voList.clear();
voList.clear(); // 显式清理辅助 GC
pageNo++;
}
} catch (Exception e) {
log.error("Excel 生成核心逻辑失败: ", e);
throw e;
} finally {
// 修复点4: 确保在 finally 中安全关闭，并捕获关闭时的异常
if (excelWriter != null) {
try {
excelWriter.finish(); // finish 内部会调用 close() 并清理 IO
} catch (Exception closeEx) {
log.error("关闭 ExcelWriter IO 失败: ", closeEx);
}
try { excelWriter.finish(); } catch (Exception e) { log.error("IO Close Error", e); }
}
}
}
private void completeTask(String taskId, File file, String name) {
ExportTask task = getTask(taskId);
if (task != null) {
task.setFilePath(file.getAbsolutePath());
task.setFileName(name);
task.setProgress(100);
task.setStatus("completed");
updateTaskInRedis(task);
}
}
private void handleError(String taskId, Exception e) {
log.error("Export Error", e);
ExportTask task = getTask(taskId);
if (task != null) {
task.setStatus("failed");
task.setMessage(e.getMessage());
updateTaskInRedis(task);
}
}
// ... getTask, updateTaskInRedis, getDatesBetween, updateStatusProgress 保持不变 ...
public ExportTask getTask(String taskId) {
return (ExportTask) redisTemplate.opsForValue().get(REDIS_KEY_PREFIX + taskId);
}
private void updateTaskInRedis(ExportTask task) {
redisTemplate.opsForValue().set(REDIS_KEY_PREFIX + task.getTaskId(), task, 24, TimeUnit.HOURS);
}
private void updateStatusProgress(String taskId, int progress, String status) {
ExportTask task = getTask(taskId);
if (task != null) {
@ -208,12 +232,16 @@ public class ExportExcelService {
Calendar cal = Calendar.getInstance();
cal.setTime(start);
while (cal.getTime().getTime() <= end.getTime()) {
list.add(cal.getTime());
list.add(new Date(cal.getTime().getTime()));
cal.add(Calendar.DATE, 1);
}
return list;
}
/*private long mockDbCount(Date date) { return 500; }
private List<DataRecognitionExportVO> mockDbQuery(Date date, int p, int s) {
return new ArrayList<>(); // 替换为实际查询
}*/
private long mockDbCount(Date date) {
return 1000; // 模拟少量数据测试
}