tt-help-cli-ycl 1.4.23 → 1.4.25

package/src/watch/server.js

@@ -27,21 +27,87 @@ const publicDir = join(__dirname, "public");
 // Stats 缓存：避免每次请求都扫描 100 万+ 条 jobs 数据
 let statsCache = null;
 let statsCacheTime = 0;
-const STATS_CACHE_TTL = 10000; // 10 秒缓存
+const STATS_CACHE_TTL = 60000; // 60 秒缓存
+
+// 慢计数缓存：每张大表独立缓存，错峰刷新，避免集中阻塞
+const SLOW_CACHE_TTL = 600000; // 10 分钟
+const slowCaches = {
+  rawJobs: { value: 0, time: 0, sql: "SELECT COUNT(*) as c FROM raw_jobs" },
+  commentTaskCount: {
+    value: 0,
+    time: 0,
+    sql: "SELECT COUNT(*) as c FROM videos WHERE user_update_count IS NULL OR user_update_count <= 0",
+  },
+  dbTotalUsers: { value: 0, time: 0, sql: "SELECT COUNT(*) as c FROM users" },
+  tagCount: { value: 0, time: 0, sql: "SELECT COUNT(*) as c FROM tags" },
+};
+
+function refreshOneSlowCount(st, key) {
+  const entry = slowCaches[key];
+  if (!entry) return;
+  try {
+    const result = st.rawQuery(entry.sql);
+    entry.value = result?.rows?.[0]?.c || 0;
+    entry.time = Date.now();
+  } catch (e) {
+    console.error(`[stats] 慢计数 ${key} 更新失败: ${e.message}`);
+  }
+}
 
-function invalidateStatsCache() {
-  statsCache = null;
-  statsCacheTime = 0;
+function getSlowCounts(st) {
+  const result = {};
+  for (const [key, entry] of Object.entries(slowCaches)) {
+    if (Date.now() - entry.time > SLOW_CACHE_TTL) {
+      refreshOneSlowCount(st, key);
+    }
+    result[key] = entry.value;
+  }
+  return result;
 }
 
+let statsRefreshing = false; // 防止并发刷新
+
 function computeStatsIncremental(st) {
   const now = Date.now();
+  // 缓存有效 → 直接返回
   if (statsCache && now - statsCacheTime < STATS_CACHE_TTL) {
     return statsCache;
   }
-  statsCache = st.getDashboardStats(DEFAULT_TARGET_LOCATIONS);
+  // 缓存过期但存在 → 先返回旧数据，后台异步刷新
+  if (statsCache && !statsRefreshing) {
+    statsRefreshing = true;
+    setImmediate(async () => {
+      try {
+        const slowCounts = getSlowCounts(st);
+        const stats = st.getDashboardStats(DEFAULT_TARGET_LOCATIONS, {
+          skipSlowCounts: true,
+        });
+        stats.rawJobs = slowCounts.rawJobs;
+        stats.commentTaskCount = slowCounts.commentTaskCount;
+        stats.dbTotalUsers = slowCounts.dbTotalUsers;
+        stats.tagCount = slowCounts.tagCount;
+        statsCache = stats;
+        statsCacheTime = Date.now();
+      } catch (e) {
+        console.error(`[stats] 后台刷新失败: ${e.message}`);
+      } finally {
+        statsRefreshing = false;
+      }
+    });
+    return statsCache;
+  }
+  // 首次加载 → 同步计算
+  const slowCounts = getSlowCounts(st);
+  const stats = st.getDashboardStats(DEFAULT_TARGET_LOCATIONS, {
+    skipSlowCounts: true,
+  });
+  stats.rawJobs = slowCounts.rawJobs;
+  stats.commentTaskCount = slowCounts.commentTaskCount;
+  stats.dbTotalUsers = slowCounts.dbTotalUsers;
+  stats.tagCount = slowCounts.tagCount;
+  statsCache = stats;
   statsCacheTime = now;
-  return statsCache;
+  return stats;
 }
 
 function readBody(req) {
@@ -329,8 +395,24 @@ export function startWatchServer(
     }
 
     const server = http.createServer(async (req, res) => {
+      const startTime = Date.now();
       const { path: routePath, params } = parseQuery(req.url);
 
+      // 请求耗时日志：仅慢请求打印（>1s）
+      res.on("finish", () => {
+        const elapsed = Date.now() - startTime;
+        if (elapsed <= 1000) return;
+        const method = req.method;
+        const shortPath =
+          routePath.length > 80
+            ? routePath.substring(0, 80) + "..."
+            : routePath;
+        const suffix = elapsed > 5000 ? "[SLOW]" : "[WARN]";
+        console.error(
+          `${method} ${shortPath} → ${res.statusCode} (${elapsed}ms)${suffix}`,
+        );
+      });
+
       // 客户端追踪：从 header 提取 clientId 和元数据，根据路由推断类型
       const clientId = req.headers["x-client-id"] || "";
       if (clientId) {
@@ -500,7 +582,7 @@ export function startWatchServer(
       ) {
         try {
           const result = store.moveNoCountryJobsToBase();
-          const ts = new Date().toISOString().slice(11, 19);
+          const ts = new Date().toLocaleTimeString("zh-CN", { hour12: false });
           if (result.ok) {
             console.error(
               `[JOB ${ts}] MOVE-NO-COUNTRY-TO-BASE: moved=${result.moved}, total=${result.total}`,
@@ -524,7 +606,7 @@ export function startWatchServer(
       ) {
         try {
           const result = store.moveSellerJobsToBase();
-          const ts = new Date().toISOString().slice(11, 19);
+          const ts = new Date().toLocaleTimeString("zh-CN", { hour12: false });
           if (result.ok) {
             console.error(
               `[JOB ${ts}] MOVE-SELLER-JOBS: jobs=${result.fromJobs}, raw_jobs=${result.fromRawJobs}, reset=${result.resetCount}, available=${result.availableInBase}`,
@@ -734,7 +816,7 @@ export function startWatchServer(
               .filter(Boolean)
           : [];
         const tasks = store.getPendingUserUpdateTasks(limit, countries);
-        const ts = new Date().toISOString().slice(11, 19);
+        const ts = new Date().toLocaleTimeString("zh-CN", { hour12: false });
         console.error(
           `[JOB ${ts}] USER-UPDATE-TASKS: ${tasks.length} tasks${countries.length ? ` (countries: ${countries.join(",")})` : ""}`,
         );
@@ -749,7 +831,7 @@ export function startWatchServer(
           const results = store.batchUpdateUserInfo(updates);
           const okCount = results.filter((r) => r.ok).length;
           const errCount = results.filter((r) => r.error).length;
-          const ts = new Date().toISOString().slice(11, 19);
+          const ts = new Date().toLocaleTimeString("zh-CN", { hour12: false });
           console.error(
             `[JOB ${ts}] USER-INFO-BATCH: ${okCount} ok, ${errCount} error (total=${updates.length})`,
           );
@@ -775,7 +857,7 @@ export function startWatchServer(
               .filter(Boolean)
           : [];
         const tasks = store.getRecheckJobs(limit, countries);
-        const ts = new Date().toISOString().slice(11, 19);
+        const ts = new Date().toLocaleTimeString("zh-CN", { hour12: false });
         console.error(
           `[JOB ${ts}] RECHECK-JOBS: ${tasks.length} tasks${countries.length ? ` (countries: ${countries.join(",")})` : ""}`,
         );
@@ -790,7 +872,7 @@ export function startWatchServer(
           const { results } = store.batchUpdateRecheckJobs(updates);
           const okCount = results.filter((r) => r.ok).length;
           const errCount = results.filter((r) => r.error).length;
-          const ts = new Date().toISOString().slice(11, 19);
+          const ts = new Date().toLocaleTimeString("zh-CN", { hour12: false });
           console.error(
             `[JOB ${ts}] RECHECK-JOBS-BATCH: ${okCount} ok, ${errCount} error (total=${updates.length})`,
           );
@@ -818,7 +900,7 @@ export function startWatchServer(
             sendJSON(res, 404, { error: ret.error });
             return;
           }
-          const ts = new Date().toISOString().slice(11, 19);
+          const ts = new Date().toLocaleTimeString("zh-CN", { hour12: false });
           console.error(
             `[JOB ${ts}] USER-INFO-UPDATE: ${uniqueId} (userUpdateCount=${ret.userUpdateCount})`,
           );
@@ -849,7 +931,7 @@ export function startWatchServer(
             sendJSON(res, 404, { error: ret.error });
             return;
           }
-          const ts = new Date().toISOString().slice(11, 19);
+          const ts = new Date().toLocaleTimeString("zh-CN", { hour12: false });
           console.error(
             `[JOB ${ts}] USER-LOCATION: ${uniqueId} → ${body.location} (modifiedAt=${ret.modifiedAt})`,
           );
@@ -871,7 +953,7 @@ export function startWatchServer(
             sendJSON(res, 404, { error: ret.error });
             return;
           }
-          const ts = new Date().toISOString().slice(11, 19);
+          const ts = new Date().toLocaleTimeString("zh-CN", { hour12: false });
           console.error(`[JOB ${ts}] NON-SELLER: ${uniqueId} → ttSeller=false`);
           sendJSON(res, 200, ret);
         } catch (e) {
@@ -892,7 +974,7 @@ export function startWatchServer(
             sendJSON(res, 404, { error: ret.error });
             return;
           }
-          const ts = new Date().toISOString().slice(11, 19);
+          const ts = new Date().toLocaleTimeString("zh-CN", { hour12: false });
           console.error(`[JOB ${ts}] MOVE-TO-BASE: ${uniqueId} → jobs_base`);
           sendJSON(res, 200, ret);
         } catch (e) {
@@ -911,7 +993,7 @@ export function startWatchServer(
               .filter(Boolean)
           : null;
         const tasks = store.getPendingCommentTasks(limit, locations);
-        const ts = new Date().toISOString().slice(11, 19);
+        const ts = new Date().toLocaleTimeString("zh-CN", { hour12: false });
         console.error(
           `[JOB ${ts}] COMMENT-TASKS: ${tasks.length} tasks${locations ? " (locations: " + locations.join(",") + ")" : ""}`,
         );
@@ -930,7 +1012,7 @@ export function startWatchServer(
             sendJSON(res, 404, { error: ret.error });
             return;
           }
-          const ts = new Date().toISOString().slice(11, 19);
+          const ts = new Date().toLocaleTimeString("zh-CN", { hour12: false });
           console.error(
             `[JOB ${ts}] COMMENT-TASK-COMMIT: ${videoId} (userUpdateCount=${ret.userUpdateCount})`,
           );
@@ -2365,6 +2447,22 @@ export function startWatchServer(
       console.error(`  本地访问: http://127.0.0.1:${port}`);
       console.error(`  局域网访问: http://${localIP}:${port}`);
 
+      // 预热 stats 缓存 + 错峰后台更新慢计数（每张表独立，间隔 2.5 分钟）
+      setTimeout(() => {
+        for (const key of Object.keys(slowCaches))
+          refreshOneSlowCount(store, key);
+        computeStatsIncremental(store);
+      }, 3000);
+      const slowKeys = Object.keys(slowCaches);
+      slowKeys.forEach((key, i) => {
+        setInterval(() => refreshOneSlowCount(store, key), SLOW_CACHE_TTL);
+        // 首次错峰：各表间隔 2.5 分钟启动
+        setTimeout(
+          () => refreshOneSlowCount(store, key),
+          3000 + (i + 1) * 150000,
+        );
+      });
+
       // 启动时清理超时的 scoring 标签
       try {
         const { resetStaleScoringTags } = store;
@@ -2401,7 +2499,7 @@ export function startWatchServer(
       setInterval(() => {
         try {
           const result = store.moveNoCountryJobsToBase();
-          const ts = new Date().toISOString().slice(11, 19);
+          const ts = new Date().toLocaleTimeString("zh-CN", { hour12: false });
           if (result.moved > 0) {
             console.error(
               `[JOB ${ts}] AUTO-MOVE-NO-COUNTRY: moved=${result.moved}, total=${result.total}`,