tt-help-cli-ycl 1.3.62 → 1.3.64

package/src/watch/server.js

@@ -356,7 +356,8 @@ export function startWatchServer(dataAnchor, port = 3000, existingStore) {
 
       if (req.method === "GET" && routePath === "/api/redo-job") {
         const userId = params.userId || "";
-        const job = store.getNextRedoJob(userId);
+        const maxAge = parseInt(params.maxAge) || 43200;
+        const job = store.getNextRedoJob(userId, maxAge);
         if (job) {
           logJob("REDO-CLAIM", { user: job.uniqueId, clientId: userId });
           sendJSON(res, 200, { hasJob: true, user: job });
@@ -525,6 +526,66 @@ export function startWatchServer(dataAnchor, port = 3000, existingStore) {
         return;
       }
 
+      if (
+        req.method === "GET" &&
+        routePath === "/api/target-users-by-country"
+      ) {
+        const result = store.getTargetUsersByCountry(DEFAULT_TARGET_LOCATIONS);
+        if (req.headers["accept"]?.includes("text/csv")) {
+          const columns = [
+            "uniqueId",
+            "nickname",
+            "followerCount",
+            "videoCount",
+            "locationCreated",
+            "latestVideoTime",
+            "refreshTime",
+            "status",
+            "sources",
+          ];
+          const allUsers = [];
+          for (const country of result.countries) {
+            for (const u of country.users) {
+              allUsers.push({
+                uniqueId: u.uniqueId,
+                nickname: u.nickname || "",
+                followerCount: u.followerCount ?? 0,
+                videoCount: u.videoCount ?? 0,
+                locationCreated: u.locationCreated || "",
+                latestVideoTime: u.latestVideoTime
+                  ? new Date(u.latestVideoTime * 1000)
+                      .toISOString()
+                      .slice(0, 19)
+                      .replace("T", " ")
+                  : "",
+                refreshTime: u.refreshTime
+                  ? new Date(u.refreshTime)
+                      .toISOString()
+                      .slice(0, 19)
+                      .replace("T", " ")
+                  : "",
+                status: u.status || "",
+                sources: (u.sources || []).join(";"),
+              });
+            }
+          }
+          res.writeHead(200, {
+            "Content-Type": "text/csv; charset=utf-8",
+            "Content-Disposition":
+              'attachment; filename="target-users-by-country.csv"',
+          });
+          const BOM = "\uFEFF";
+          const header = columns.join(",");
+          const lines = allUsers.map((r) =>
+            columns.map((c) => csvEscape(r[c])).join(","),
+          );
+          res.end(BOM + [header, ...lines].join("\r\n"));
+        } else {
+          sendJSON(res, 200, result);
+        }
+        return;
+      }
+
       if (req.method === "GET" && routePath === "/api/client-errors") {
         sendJSON(res, 200, { clients: store.getClientErrors() });
         return;
@@ -563,6 +624,8 @@ export function startWatchServer(dataAnchor, port = 3000, existingStore) {
           location: params.location,
           limit: params.limit,
           offset: params.offset,
+          hasVideo: params.hasVideo,
+          hasFollower: params.hasFollower,
         });
         sendJSON(res, 200, result);
         return;
@@ -589,10 +652,17 @@ export function startWatchServer(dataAnchor, port = 3000, existingStore) {
           let result;
           if (body.uniqueId) {
             result = store.restoreRawJobById(body.uniqueId);
-          } else if (body.search || body.location) {
+          } else if (
+            body.search ||
+            body.location ||
+            body.hasVideo ||
+            body.hasFollower
+          ) {
             result = store.restoreRawJobsByFilter({
               search: body.search || "",
               location: body.location || "",
+              hasVideo: body.hasVideo,
+              hasFollower: body.hasFollower,
             });
           } else if (body.country) {
             result = store.restoreRawJobsByCountry(body.country);
@@ -628,6 +698,24 @@ export function startWatchServer(dataAnchor, port = 3000, existingStore) {
         return;
       }
 
+      if (
+        req.method === "POST" &&
+        routePath === "/api/pending-by-country/reset"
+      ) {
+        try {
+          const body = await readBody(req);
+          const result = store.resetPendingByCountry(body.country);
+          if (result.error) {
+            sendJSON(res, 400, result);
+            return;
+          }
+          sendJSON(res, 200, result);
+        } catch (e) {
+          sendJSON(res, 400, { error: e.message });
+        }
+        return;
+      }
+
       if (
         req.method === "DELETE" &&
         routePath.startsWith("/api/client-error/")
@@ -683,6 +771,7 @@ export function startWatchServer(dataAnchor, port = 3000, existingStore) {
             followerCount: u.followerCount,
             locationCreated: u.locationCreated,
             latestVideoTime: u.latestVideoTime,
+            refreshTime: u.refreshTime,
             guessedLocation: u.guessedLocation,
             pinned: u.pinned,
             processedAt: u.processedAt,

package/src/scraper/refresh-core.js

@@ -1,213 +0,0 @@
-import {
-  delay,
-  retryWithBackoff,
-  detectPageError,
-  assertPageUrl,
-} from "./modules/page-helpers.js";
-import { detectCaptcha } from "./modules/captcha-handler.js";
-import { getUserInfo, collectVideos } from "../videos/core.js";
-import { extractFollowAndFollowers } from "./modules/follow-extractor.js";
-import { processExplore } from "./explore-core.js";
-import { DEFAULT_TARGET_LOCATIONS_CSV } from "../lib/target-locations.js";
-
-export async function processRefresh(page, username, serverUrl, options, log) {
-  const { maxFollowing = 100, maxFollowers = 100, maxVideos = 100 } = options;
-
-  const result = {
-    userInfo: null,
-    discoveredVideoAuthors: [],
-    discoveredFollowing: [],
-    discoveredFollowers: [],
-    newUsersAdded: 0,
-    collectedVideos: 0,
-    error: null,
-  };
-
-  try {
-    log(`  访问 @${username} 主页...`);
-    const homeUrl = `https://www.tiktok.com/@${username}`;
-    await retryWithBackoff(
-      async () => {
-        await page.goto(homeUrl, {
-          waitUntil: "domcontentloaded",
-          timeout: 30000,
-        });
-        assertPageUrl(page, `@${username}`);
-      },
-      { log },
-    );
-    await page
-      .waitForSelector('[class*="DivVideoList"]', { timeout: 10000 })
-      .catch(() => {});
-    await delay(1000, 2000);
-
-    log("  获取用户信息...");
-    const info = await getUserInfo(page);
-    if (info) {
-      result.userInfo = info;
-      log(
-        `  用户: ${info.nickname || username} | 粉丝: ${info.followerCount || "-"} | 视频: ${info.videoCount || "-"}`,
-      );
-    }
-
-    const captcha = await detectCaptcha(page);
-    if (captcha && captcha.visible) {
-      log(`[验证码] @${username} 页面出现验证码`);
-      result.captchaDetected = true;
-      result.captchaStage = result.captchaStage || "video-page";
-      result.captchaMessage = result.captchaMessage || "视频页出现验证码";
-    }
-
-    // 采集视频
-    log(`  采集视频 (最多 ${maxVideos} 个)...`);
-    const videoList = await collectVideos(page, username, maxVideos, log);
-    const videoArray = videoList ? [...videoList.values()] : [];
-    result.collectedVideos = videoArray.length;
-    result.discoveredVideoAuthors = videoArray.map((v) => v.author);
-
-    if (videoArray.length <= 0) {
-      result.noVideo = true;
-      const pageError = await detectPageError(page);
-      if (pageError) {
-        result.restricted = true;
-        log(`  @${username} 页面受限（${pageError}），标记跳过`);
-      }
-      return result;
-    }
-
-    // 采集关注和粉丝
-    log(`  采集关注 (最多 ${maxFollowing}) + 粉丝 (最多 ${maxFollowers})...`);
-    try {
-      const followResult = await extractFollowAndFollowers(page, {
-        maxFollowing,
-        maxFollowers,
-      });
-      result.discoveredFollowing = followResult.following || [];
-      result.discoveredFollowers = followResult.followers || [];
-      log(
-        `  关注: ${result.discoveredFollowing.length}, 粉丝: ${result.discoveredFollowers.length}`,
-      );
-    } catch (e) {
-      log(`  [关注/粉丝采集失败] ${e.message}`);
-      result.discoveredFollowing = [];
-      result.discoveredFollowers = [];
-    }
-
-    // 处理新发现的用户（关注 + 粉丝），循环执行完整 explore
-    // follow-extractor 返回 [handle, displayName] 数组
-    const allDiscovered = [
-      ...result.discoveredFollowing.map((h) => ({
-        handle: Array.isArray(h) ? h[0] : h,
-        source: "refresh-following",
-      })),
-      ...result.discoveredFollowers.map((h) => ({
-        handle: Array.isArray(h) ? h[0] : h,
-        source: "refresh-follower",
-      })),
-    ];
-
-    for (const { handle, source } of allDiscovered) {
-      const uniqueId = handle.replace("@", "");
-
-      // 检查用户是否已存在
-      const existsResp = await fetch(
-        `${serverUrl}/api/user-exists/${encodeURIComponent(uniqueId)}`,
-      );
-      const existsData = await existsResp.json();
-
-      if (existsData.exists) {
-        continue;
-      }
-
-      log(`  [新用户] @${uniqueId} 不存在，开始探索 (来源: ${source})...`);
-      await delay(1000, 2000);
-
-      // 对新用户做完整 explore（与 explore 命令逻辑一致）
-      const exploreResult = await processExplore(
-        page,
-        uniqueId,
-        {
-          maxComments: 10,
-          maxGuess: 0,
-          enableFollow: true,
-          maxFollowing: 5,
-          maxFollowers: 5,
-          location: DEFAULT_TARGET_LOCATIONS_CSV,
-        },
-        log,
-      );
-
-      // 提交 explore 结果到服务端（和 explore 命令的 commitJob 一致）
-      if (exploreResult.userInfo) {
-        const guessedLocation = exploreResult.locationCreated || null;
-
-        const payload = {
-          userInfo: exploreResult.userInfo || {},
-          discoveredVideoAuthors: (
-            exploreResult.discoveredVideoAuthors || []
-          ).map((item) =>
-            typeof item === "object" ? { ...item, guessedLocation } : item,
-          ),
-          discoveredCommentAuthors: (
-            exploreResult.discoveredCommentAuthors || []
-          ).map((author) => ({ author, guessedLocation })),
-          discoveredGuessAuthors: (
-            exploreResult.discoveredGuessAuthors || []
-          ).map((author) => ({ author, guessedLocation })),
-          discoveredFollowing: (exploreResult.discoveredFollowing || []).map(
-            (f) => ({
-              handle: Array.isArray(f) ? f[0] : f,
-              displayName: Array.isArray(f) ? f[1] : null,
-              guessedLocation,
-            }),
-          ),
-          discoveredFollowers: (exploreResult.discoveredFollowers || []).map(
-            (f) => ({
-              handle: Array.isArray(f) ? f[0] : f,
-              displayName: Array.isArray(f) ? f[1] : null,
-              guessedLocation,
-            }),
-          ),
-          processed: exploreResult.processed,
-          hasFollowData: exploreResult.hasFollowData,
-          keepFollow: exploreResult.keepFollow,
-          locationCreated: exploreResult.locationCreated,
-          noVideo: exploreResult.noVideo,
-          restricted: exploreResult.restricted,
-          error: exploreResult.error,
-        };
-
-        const addResp = await fetch(
-          `${serverUrl}/api/explore-new/${uniqueId}`,
-          {
-            method: "POST",
-            headers: { "Content-Type": "application/json" },
-            body: JSON.stringify(payload),
-          },
-        );
-        const addResult = await addResp.json();
-
-        if (!addResult.saved) {
-          log(`  [跳过] @${uniqueId} 提交失败`);
-          continue;
-        }
-
-        result.newUsersAdded++;
-        if (exploreResult.captchaDetected) {
-          result.captchaDetected = true;
-        }
-        log(
-          `  [已提交] @${uniqueId} ${addResult.created ? "(新用户)" : "(已存在)"} | 发现: ${addResult.newUsers?.length || 0} 个`,
-        );
-      }
-
-      await delay(2000, 4000);
-    }
-  } catch (e) {
-    log(`  [错误] ${e.message}`);
-    result.error = e.message;
-    result.errorStack = e.stack || "";
-  }
-
-  return result;
-}