| | |
| | | package com.trafficaudit.reportexport.service; |
| | | |
| | | import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper; |
| | | import com.trafficaudit.dataimport.entity.ImportBatch; |
| | | import com.trafficaudit.dataimport.mapper.ImportBatchMapper; |
| | | import lombok.extern.slf4j.Slf4j; |
| | | import org.springframework.stereotype.Service; |
| | | |
| | | import javax.annotation.PreDestroy; |
| | | import javax.annotation.Resource; |
| | | import javax.sql.DataSource; |
| | | import java.io.File; |
| | | import java.nio.charset.StandardCharsets; |
| | | import java.security.MessageDigest; |
| | | import java.sql.Connection; |
| | | import java.sql.ResultSet; |
| | | import java.sql.Statement; |
| | | import java.util.ArrayList; |
| | | import java.util.Arrays; |
| | | import java.util.Comparator; |
| | | import java.util.Date; |
| | | import java.util.HashSet; |
| | | import java.util.Iterator; |
| | | import java.util.LinkedHashMap; |
| | | import java.util.List; |
| | | import java.util.Map; |
| | | import java.util.Set; |
| | | import java.util.UUID; |
| | | import java.util.concurrent.ConcurrentHashMap; |
| | | import java.util.concurrent.ExecutorService; |
| | |
| | | * 2) çæè¿ç¨ä¸å ç¨ HTTP 请æ±çº¿ç¨ï¼é¿å
å¤äººåæ¶ç¹æå®¹å¨ææ»ï¼å·¥ä½çº¿ç¨åºå® 1 个ï¼å¤ä½è¯·æ±æ¾ç¤ºãæéä¸ãï¼ï¼ |
| | | * 3) å䏿¥è¡¨æçç»æå¯å¤ç¨ï¼**è½çä¿åï¼è¿ç¨éå¯åä¾ç¶ææ**ï¼ï¼é¿å
æ¯æ¬¡éç®æ´æ¬çº¦ 4 䏿 ¼å
¬å¼ã |
| | | * |
| | | * ç¼åæ£ç¡®æ§å®å«ï¼åªæå½ãæé´æ²¡æä»»ä½æ°çæ°æ®å¯¼å
¥ï¼import_batch æå¤§ id æªåï¼ãä¸ãç»æä¸è¶
è¿ 7 å¤©ãæ¶æå¤ç¨ï¼ |
| | | * 导å
¥å³å¤±æï¼é¿å
æè¿ææ¥è¡¨ç»ç¨æ·ãåç«¯ä»æä¾ãéæ°çæã强å¶éç®ï¼è·³è¿ç¼åï¼ã |
| | | * è½çä½ç½®ï¼{summary.template-dir}/_cache/<key>_v<æ°æ®çæ¬>.xlsx |
| | | * ç¼åæ£ç¡®æ§å®å«ï¼**æ°æ®æçº¹**ï¼æçº¹ç±ãæ±æ»å¤§è¡¨ä¾èµçæºè¡¨å
å®¹æ ¡éªåã+ãæ¯ççæä»¶èº«ä»½ï¼å¤§å°/ä¿®æ¹æ¶é´ï¼ãç®åºï¼ |
| | | * åªææçº¹ä¸è´ï¼ä¸ç»æä¸è¶
è¿ 7 å¤©ï¼æå¤ç¨ãå æ¤ï¼ |
| | | * - æèµ/è½è/å®¡æ ¸ç**䏿±æ»å¤§è¡¨æ å
³**ç导å
¥ä¸å让ç¼å失æï¼ |
| | | * - æ¢æ¯çãæ¹åç
§ä»¶ãå¨ç¸å
³æºè¡¨ï¼å«åå° UPDATEï¼é½ä¼è®©ç¼å失æï¼ä¸ä¼ææ§è¡¨å½æ°è¡¨ååºã |
| | | * åç«¯ä»æä¾ãéæ°çæã强å¶éç®ï¼è·³è¿ç¼åï¼ãè½çä½ç½®ï¼{summary.template-dir}/_cache/<key>_v<æçº¹>.xlsx |
| | | */ |
| | | @Slf4j |
| | | @Service |
| | |
| | | @Resource |
| | | private ReportExportService reportService; |
| | | @Resource |
| | | private ImportBatchMapper importBatchMapper; |
| | | private DataSource dataSource; |
| | | |
| | | /** æ±æ»å·¥ä½ç°¿ç®å½ï¼ä¸ ReportExportService åé
ç½®ï¼ï¼è½çç¼åæ¾å¨å
¶ _cache åç®å½ */ |
| | | @org.springframework.beans.factory.annotation.Value("${summary.template-dir:docs/çææ±æ»å¤§è¡¨}") |
| | | private String summaryTemplateDir; |
| | | |
| | | /** åå¸å®¢è¿ç®å½ï¼ç½çº¦è½¦è®¢ååå
¨çæ»é.xlsxï¼åºéç¼ºè¯¥ææ¶çå
åºè¾å
¥ï¼ */ |
| | | @org.springframework.beans.factory.annotation.Value("${city-passenger.template-dir:docs/åå¸å®¢è¿}") |
| | | private String cityPassengerTemplateDir; |
| | | |
| | | /** è®¡ç®æçº¹æ¶**æé¤**çè¡¨ï¼æèµçº¿ãè½è线ãå®¡æ ¸çº¿ãç³»ç»è¡¨ã导å
¥æ¥å¿ã |
| | | * è¿äºè¡¨ååä¸å½±åæ±æ»å¤§è¡¨å
å®¹ï¼æ²¡å¿
è¦è®©ç¼å失æï¼å
¶ä½è¡¨ä¸å¾çº³å
¥ï¼å®å¯å¤ç®ä¸å¯åºéï¼ã */ |
| | | private static final Set<String> FINGERPRINT_EXCLUDE = new HashSet<>(Arrays.asList( |
| | | "investment_project", "investment_monthly", "investment_system", |
| | | "h204_vehicle_quarterly", "h204_auth_vehicle", |
| | | "audit_result", "audit_run", "audit_rule", "audit_explanation", |
| | | "sys_user", "sys_role", "sys_user_role", "sys_dept", "sys_operation_log", |
| | | "llm_desensitize_map", "import_batch")); |
| | | |
| | | /** è½çç¼åç®å½ï¼å®ä½çç¥ä¸æ¨¡æ¿ä¸è´ï¼é
ç½®ç®å½ / user.dir ç¸å¯¹ / ä¸çº§ç®å½ï¼ */ |
| | | private File cacheDir() { |
| | |
| | | private final String mode; |
| | | private final Integer toMonth; |
| | | private volatile long createdAt = System.currentTimeMillis(); |
| | | /** çææ¶çæ°æ®çæ¬ï¼import_batch æå¤§ idï¼ï¼ç¨äºå¤æç¼åæ¯å¦è¿æ */ |
| | | private final long importVersion; |
| | | /** çææ¶çæ°æ®æçº¹ï¼æºè¡¨æ ¡éªå + æ¯çæä»¶èº«ä»½ï¼ï¼ç¨äºå¤æç¼åæ¯å¦è¿æ */ |
| | | private final String fingerprint; |
| | | private volatile String status = "PENDING"; // PENDING/RUNNING/DONE/FAILED |
| | | private volatile int percent = 0; |
| | | private volatile String step = "æéä¸"; |
| | |
| | | private volatile File cacheFile; |
| | | private volatile String fileName; |
| | | private volatile boolean fromCache; |
| | | /** æ¬æ¬¡ã没å½ä¸ç¼åãæ¯å 为å卿§ç»æä½æ°æ®/æ¯çå·²åï¼ç¨äºç»ç¨æ·ä¸å¥è§£éï¼ */ |
| | | private volatile boolean staleCache; |
| | | private volatile long finishedAt; |
| | | |
| | | Task(String type, String period, String mode, Integer toMonth, long importVersion) { |
| | | Task(String type, String period, String mode, Integer toMonth, String fingerprint) { |
| | | this.type = type; |
| | | this.period = period; |
| | | this.mode = mode; |
| | | this.toMonth = toMonth; |
| | | this.importVersion = importVersion; |
| | | this.fingerprint = fingerprint; |
| | | } |
| | | |
| | | void progress(int pct, String stepText) { |
| | |
| | | m.put("percent", "DONE".equals(status) ? 100 : percent); |
| | | m.put("step", step); |
| | | m.put("fromCache", fromCache); |
| | | m.put("staleCache", staleCache); |
| | | m.put("fileName", fileName); |
| | | m.put("error", error); |
| | | long end = finishedAt > 0 ? finishedAt : System.currentTimeMillis(); |
| | |
| | | |
| | | /** |
| | | * æäº¤å¯¼åºä»»å¡ãforceRefresh=true æ¶å¿½ç¥ç¼åéæ°çæï¼ |
| | | * å¦åå¨ãå䏿¥è¡¨æã10 åéå
ãæé´æ æ°å¯¼å
¥ãçåæä¸ç´æ¥å¤ç¨ä¸æ¬¡ç»æã |
| | | * å¦åå¨ãå䏿¥è¡¨æ + æ°æ®æçº¹ä¸è´ + 7 天å
ãçåæä¸ç´æ¥å¤ç¨ä¸æ¬¡ç»æã |
| | | * å䏿¥è¡¨æå·²æä»»å¡å¨è·æ¶ç´æ¥è¿å该任å¡ï¼é¿å
é夿éã |
| | | */ |
| | | public Task submit(String type, String period, String mode, Integer toMonth, boolean forceRefresh) { |
| | | cleanupExpired(); |
| | | String key = cacheKey(type, period, mode, toMonth); |
| | | String fingerprint = dataFingerprint(); |
| | | if (!forceRefresh) { |
| | | Task hit = cache.get(key); |
| | | if (cacheUsable(hit)) { |
| | | if (cacheUsable(hit, fingerprint)) { |
| | | hit.fromCache = true; |
| | | hit.staleCache = false; // å½ä¸ç¼åæ¶ä¸è½æ®çä¸ä¸æ¬¡ãç»æå·²å¤±æãçæç¤º |
| | | log.info("æ¥è¡¨å¯¼åºä»»å¡å½ä¸å
åç¼åï¼{} {}ï¼çæäº {}ï¼", type, period, new Date(hit.finishedAt)); |
| | | return hit; |
| | | } |
| | | Task disk = loadFromDisk(key, type, period, mode, toMonth); |
| | | Task disk = loadFromDisk(key, type, period, mode, toMonth, fingerprint); |
| | | if (disk != null) { |
| | | tasks.put(disk.id, disk); |
| | | cache.put(key, disk); |
| | |
| | | log.info("æ¥è¡¨å¯¼åºä»»å¡å·²å¨æ§è¡ï¼å¤ç¨è¿è¡ä¸çä»»å¡ï¼{} {}ï¼{}ï¼", type, period, running.id); |
| | | return running; |
| | | } |
| | | Task task = new Task(type, period, mode, toMonth, importVersion()); |
| | | Task task = new Task(type, period, mode, toMonth, fingerprint); |
| | | // ææ§ç»æä½æ²¡å½ä¸ï¼æ°æ®ææ¯çåäºï¼â å端æç¤ºä¸å¥ï¼é¿å
ç¨æ·ä»¥ä¸º"ç¼ååäº" |
| | | task.staleCache = !forceRefresh && hasStaleCache(type, period, mode, toMonth, fingerprint); |
| | | tasks.put(task.id, task); |
| | | pool.submit(() -> run(task)); |
| | | return task; |
| | |
| | | return null; |
| | | } |
| | | |
| | | private boolean cacheUsable(Task t) { |
| | | if (t == null || !"DONE".equals(t.status) || t.data == null) return false; |
| | | private boolean cacheUsable(Task t, String fingerprint) { |
| | | if (t == null || !"DONE".equals(t.status)) return false; |
| | | if (t.data == null && (t.cacheFile == null || !t.cacheFile.isFile())) return false; |
| | | if (System.currentTimeMillis() - t.finishedAt > CACHE_MAX_AGE_MS) return false; |
| | | long now = importVersion(); |
| | | if (now < 0 || t.importVersion < 0) return false; // åä¸å°æ°æ®çæ¬æ¶ä¸åé£é©å¤ç¨ |
| | | return now == t.importVersion; |
| | | if (fingerprint == null || t.fingerprint == null) return false; // ç®ä¸åºæçº¹æ¶ä¸åé£é©å¤ç¨ |
| | | return fingerprint.equals(t.fingerprint); |
| | | } |
| | | |
| | | /** æ°æ®çæ¬ï¼import_batch æå¤§ idï¼ä»»ä½ä¸æ¬¡å¯¼å
¥é½ä¼è®©å®ååï¼åä¸å°è¿å -1ï¼æ¤æ¶ç¦ç¨ç¼åï¼ */ |
| | | private long importVersion() { |
| | | /** |
| | | * æ±æ»å¤§è¡¨æ°æ®æçº¹ï¼ãä¾èµçæºè¡¨å
å®¹æ ¡éªåã+ãæ¯ççæä»¶èº«ä»½ãã |
| | | * - æºè¡¨ï¼åºå
é¤ FINGERPRINT_EXCLUDEï¼æèµ/è½è/å®¡æ ¸/ç³»ç»/导å
¥æ¥å¿ï¼ä»¥å¤çå
¨é¨è¡¨ï¼ç¨ CHECKSUM TABLE åæ ¡éªåï¼ |
| | | * è½ææè¡æ°ååãæ°å¢å é¤ã**åå° UPDATE**ï¼ |
| | | * - æä»¶ï¼æ¯çç®å½ä¸å
¨é¨æä»¶ï¼æ¯ç / åç
§ä»¶ / _å¤ä»½_ åå¡«ä»¶ï¼+ãç½çº¦è½¦è®¢ååå
¨çæ»é.xlsxãï¼åãåç§°+大å°+ä¿®æ¹æ¶é´ãã |
| | | * ä»»ä¸é¡¹ååå³æçº¹åå â ç¼å失æã计ç®å¤±è´¥è¿å nullï¼æ¤æ¶ç¦ç¨ç¼åï¼åªéç®ä¸å¤ç¨ï¼ã |
| | | */ |
| | | private String dataFingerprint() { |
| | | try { |
| | | List<Object> v = importBatchMapper.selectObjs(new QueryWrapper<ImportBatch>().select("IFNULL(MAX(id),0)")); |
| | | if (v == null || v.isEmpty() || v.get(0) == null) return 0L; |
| | | return ((Number) v.get(0)).longValue(); |
| | | List<String> parts = new ArrayList<>(); |
| | | try (Connection conn = dataSource.getConnection()) { |
| | | List<String> tables = new ArrayList<>(); |
| | | try (Statement st = conn.createStatement(); |
| | | ResultSet rs = st.executeQuery("SELECT table_name FROM information_schema.tables " |
| | | + "WHERE table_schema = DATABASE() AND table_type = 'BASE TABLE' ORDER BY table_name")) { |
| | | while (rs.next()) tables.add(rs.getString(1)); |
| | | } |
| | | for (String t : tables) { |
| | | if (FINGERPRINT_EXCLUDE.contains(t)) continue; |
| | | try (Statement st = conn.createStatement(); |
| | | ResultSet rs = st.executeQuery("CHECKSUM TABLE `" + t + "`")) { |
| | | if (rs.next()) parts.add("T:" + t + ":" + rs.getString(2)); |
| | | } |
| | | } |
| | | } |
| | | for (File f : listIdentityFiles(resolveBaseDir(summaryTemplateDir))) { |
| | | parts.add("F:" + f.getName() + ":" + f.length() + ":" + f.lastModified()); |
| | | } |
| | | File wyc = new File(resolveBaseDir(cityPassengerTemplateDir), "ç½çº¦è½¦è®¢ååå
¨çæ»é.xlsx"); |
| | | if (wyc.isFile()) parts.add("F:" + wyc.getName() + ":" + wyc.length() + ":" + wyc.lastModified()); |
| | | return sha256Hex(String.join("|", parts)).substring(0, 16); |
| | | } catch (Exception e) { |
| | | log.warn("读å导å
¥æ°æ®çæ¬å¤±è´¥ï¼æ¬æ¬¡ç¦ç¨å¯¼åºç¼åï¼{}", e.getMessage()); |
| | | return -1L; |
| | | log.warn("è®¡ç®æ°æ®æçº¹å¤±è´¥ï¼æ¬æ¬¡ç¦ç¨å¯¼åºç¼åï¼{}", e.getMessage()); |
| | | return null; |
| | | } |
| | | } |
| | | |
| | | /** æ¯çç®å½çåºç¡ç®å½å®ä½ï¼ä¸æ¨¡æ¿è§£æåçç¥ï¼é
ç½®ç®å½ / user.dir ç¸å¯¹ / ä¸çº§ç®å½ï¼ */ |
| | | private File resolveBaseDir(String dir) { |
| | | if (dir == null) return null; |
| | | String rel = dir; |
| | | while (rel.startsWith("./")) rel = rel.substring(2); |
| | | String[] roots = {rel, System.getProperty("user.dir") + "/" + rel, System.getProperty("user.dir") + "/../" + rel}; |
| | | for (String root : roots) { |
| | | if (root == null || root.trim().isEmpty()) continue; |
| | | File f = new File(root); |
| | | if (f.isDirectory()) return f; |
| | | } |
| | | return null; |
| | | } |
| | | |
| | | /** ç®å½ä¸çæä»¶ï¼æååæåºï¼æé¤åç®å½ï¼ä¿è¯æçº¹ç¨³å®ï¼ */ |
| | | private List<File> listIdentityFiles(File dir) { |
| | | List<File> out = new ArrayList<>(); |
| | | if (dir == null) return out; |
| | | File[] fs = dir.listFiles(File::isFile); |
| | | if (fs == null) return out; |
| | | Arrays.sort(fs, Comparator.comparing(File::getName)); |
| | | out.addAll(Arrays.asList(fs)); |
| | | return out; |
| | | } |
| | | |
| | | private String sha256Hex(String text) throws Exception { |
| | | MessageDigest md = MessageDigest.getInstance("SHA-256"); |
| | | byte[] d = md.digest(text.getBytes(StandardCharsets.UTF_8)); |
| | | StringBuilder sb = new StringBuilder(); |
| | | for (int i = 0; i < 8; i++) sb.append(String.format("%02x", d[i])); // åå 8 åèï¼16 ä½åå
è¿å¶ï¼å¤ç¨ä¸æä»¶åç |
| | | return sb.toString(); |
| | | } |
| | | |
| | | /** 该æ¥è¡¨ææ¯å¦åå¨ãä¸å½åæçº¹ä¸ä¸è´ãçæ§ç»æï¼ç¨äºæç¤ºã䏿¬¡ç»æå·²å¤±æï¼æ£å¨æææ°æ°æ®éç®ãï¼ */ |
| | | private boolean hasStaleCache(String type, String period, String mode, Integer toMonth, String fingerprint) { |
| | | try { |
| | | File dir = cacheDir(); |
| | | if (dir == null) return false; |
| | | String prefix = cacheFilePrefix(type, period, mode, toMonth); |
| | | File[] fs = dir.listFiles((d, n) -> n.startsWith(prefix + "_v") && n.endsWith(".xlsx")); |
| | | if (fs == null || fs.length == 0) return false; |
| | | String current = prefix + "_v" + fingerprint + ".xlsx"; |
| | | for (File f : fs) { |
| | | if (!f.getName().equals(current)) return true; |
| | | } |
| | | return false; |
| | | } catch (Exception e) { |
| | | return false; |
| | | } |
| | | } |
| | | |
| | |
| | | try { |
| | | File dir = cacheDir(); |
| | | if (dir == null) return; |
| | | long version = t.importVersion; |
| | | if (version < 0) return; |
| | | String fingerprint = t.fingerprint; |
| | | if (fingerprint == null) return; |
| | | String prefix = cacheFilePrefix(t.type, t.period, t.mode, t.toMonth); |
| | | File target = new File(dir, prefix + "_v" + version + ".xlsx"); |
| | | File target = new File(dir, prefix + "_v" + fingerprint + ".xlsx"); |
| | | java.nio.file.Files.write(target.toPath(), data); |
| | | File[] siblings = dir.listFiles((d, n) -> n.startsWith(prefix + "_v") && n.endsWith(".xlsx")); |
| | | if (siblings != null) { |
| | |
| | | } |
| | | } |
| | | |
| | | /** ä»è½çç¼åéæ¾ãå䏿¥è¡¨æ + å䏿°æ®çæ¬ + æªè¶
æãçç»æï¼æ²¡æè¿å null */ |
| | | private Task loadFromDisk(String key, String type, String period, String mode, Integer toMonth) { |
| | | /** ä»è½çç¼åéæ¾ãå䏿¥è¡¨æ + å䏿°æ®æçº¹ + æªè¶
æãçç»æï¼æ²¡æè¿å null */ |
| | | private Task loadFromDisk(String key, String type, String period, String mode, Integer toMonth, String fingerprint) { |
| | | try { |
| | | if (fingerprint == null) return null; // ç®ä¸åºæçº¹æ¶ä¸åé£é©å¤ç¨ |
| | | File dir = cacheDir(); |
| | | if (dir == null) return null; |
| | | long version = importVersion(); |
| | | if (version < 0) return null; // åä¸å°æ°æ®çæ¬æ¶ä¸åé£é©å¤ç¨ |
| | | File f = new File(dir, cacheFilePrefix(type, period, mode, toMonth) + "_v" + version + ".xlsx"); |
| | | File f = new File(dir, cacheFilePrefix(type, period, mode, toMonth) + "_v" + fingerprint + ".xlsx"); |
| | | if (!f.isFile()) return null; |
| | | long finished = f.lastModified(); |
| | | if (finished <= 0 || System.currentTimeMillis() - finished > CACHE_MAX_AGE_MS) return null; |
| | | Task t = new Task(type, period, mode, toMonth, version); |
| | | Task t = new Task(type, period, mode, toMonth, fingerprint); |
| | | t.cacheFile = f; |
| | | t.createdAt = finished; // å¤ç¨åå²ç»æï¼ä»¥çææ¶é´ä½ä¸ºèµ·ç¹ï¼é¿å
elapsedMs åºç°è´æ° |
| | | t.fileName = "çæ_éè·¯è¿è¾éæ±æ»è¡¨_" + period + ".xlsx"; |