tt-help-cli-ycl 1.4.172 → 1.4.176

package/src/lib/linux-env-init.js

@@ -0,0 +1,258 @@
+/**
+ * Linux 节点环境自举（watchdog 启动时调用一次；幂等；失败不阻断心跳）
+ *
+ * 仅 Linux 加载（agent.js 有 platform guard）——Windows/macOS 稳定运行路径零改动。
+ * 目标：新装 Linux 节点无需任何手工环境步骤，watchdog 启动即自举完毕：
+ *
+ *   1. DISPLAY 检测：headless（systemd/ssh 无显示）时自动 spawn Xvfb 并注入
+ *      process.env.DISPLAY —— 服务端下发的业务进程（explore 等）继承后可直接起浏览器；
+ *      已有可用显示（桌面会话/外部 Xvfb）则直接复用
+ *   2. Xvfb 生命周期：写 pid 文件（~/.tt-help/xvfb.pid），watchdog 重启时复用存活实例；
+ *      watchdog 退出（exit handler）时清理自己起的实例
+ *   3. Edge 浏览器探测：msedge / microsoft-edge / microsoft-edge-stable 依次探测
+ *      （官方 deb 命令非 msedge，F-11 实测踩坑）
+ *   4. 能力汇总（返回值）→ agent 随心跳上报 env 字段，服务端可见节点就绪度
+ *      （旧版服务端忽略未知字段，向后兼容）
+ *
+ * 设计约束：任何一步失败只记日志、不抛出、不阻断心跳——问题经 env 上报在服务端可见。
+ */
+
+import { spawn, execSync } from "child_process";
+import { existsSync, readFileSync, writeFileSync, unlinkSync, readdirSync } from "fs";
+import { join } from "path";
+import { homedir } from "os";
+
+const X11_SOCKET_DIR = "/tmp/.X11-unix";
+const XVFB_PID_FILE = join(homedir(), ".tt-help", "xvfb.pid");
+const XVFB_SCREEN = "1600x900x24";
+
+function log(m) {
+  console.error(`[linux-env] ${m}`);
+}
+
+/** 探测命令是否存在（command -v，PATH 语义） */
+function which(cmd) {
+  try {
+    const p = execSync(`command -v ${cmd} 2>/dev/null`, { encoding: "utf-8" }).trim();
+    return p || null;
+  } catch {
+    return null;
+  }
+}
+
+/** display 号对应的 X socket 是否存在（= 该号被占用/可用） */
+function xSocketExists(n) {
+  return existsSync(join(X11_SOCKET_DIR, `X${n}`));
+}
+
+/** 当前 DISPLAY 是否真的可用（变量存在且对应 socket 在） */
+function displayUsable() {
+  const d = process.env.DISPLAY || "";
+  if (!d) return false;
+  const m = /^:(\d+)$/.exec(d.trim());
+  return m ? xSocketExists(parseInt(m[1], 10)) : true; // 非标准格式（如 host:N）放行交给 X 自己报错
+}
+
+/** 读取 pid 文件并校验进程存活，返回可复用的 Xvfb pid 或 0 */
+function findReusableXvfb() {
+  try {
+    const pid = parseInt(readFileSync(XVFB_PID_FILE, "utf-8").trim(), 10);
+    if (!pid) return 0;
+    process.kill(pid, 0); // 探测存活
+    return pid;
+  } catch {
+    try {
+      unlinkSync(XVFB_PID_FILE);
+    } catch {}
+    return 0;
+  }
+}
+
+/**
+ * 确保可用显示：复用外部显示 > 复用历史 Xvfb > spawn 新 Xvfb。
+ * 成功后注入 process.env.DISPLAY（业务子进程继承）。
+ * @returns {{display: string, xvfbPid: number, source: string, error: string|null}}
+ */
+function ensureDisplay() {
+  if (displayUsable()) {
+    return { display: process.env.DISPLAY, xvfbPid: 0, source: "external", error: null };
+  }
+  delete process.env.DISPLAY;
+
+  // 复用上次自托管且仍存活的 Xvfb（watchdog 重启场景，display 号可能已变）
+  const reusePid = findReusableXvfb();
+  if (reusePid) {
+    try {
+      const cmdline = readFileSync(`/proc/${reusePid}/cmdline`, "utf-8");
+      const m = /:(\d+)/.exec(cmdline);
+      if (m && xSocketExists(parseInt(m[1], 10))) {
+        const display = `:${m[1]}`;
+        process.env.DISPLAY = display;
+        log(`复用历史 Xvfb (pid=${reusePid}, display=${display})`);
+        return { display, xvfbPid: reusePid, source: "reuse", error: null };
+      }
+    } catch {}
+    try {
+      unlinkSync(XVFB_PID_FILE);
+    } catch {}
+  }
+
+  const xvfbBin = which("Xvfb");
+  if (!xvfbBin) {
+    return {
+      display: "",
+      xvfbPid: 0,
+      source: "none",
+      error: "无可用显示且未安装 Xvfb（sudo apt install -y xvfb 后重启 watchdog 自动补齐）",
+    };
+  }
+
+  // 找空闲 display 号：99 起向上，socket 文件不存在即视为空闲
+  let n = 99;
+  while (xSocketExists(n) && n < 120) n++;
+  const display = `:${n}`;
+
+  const child = spawn(xvfbBin, [display, "-screen", "0", XVFB_SCREEN, "-nolisten", "tcp"], {
+    stdio: "ignore",
+    detached: false,
+  });
+  // 等 X socket 就绪（正常 <500ms；2s 未起视为失败）。同步休眠：本模块在启动路径同步执行
+  const sleepSync = (ms) => {
+    Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, ms);
+  };
+  const deadline = Date.now() + 2000;
+  while (Date.now() < deadline && !xSocketExists(n)) {
+    sleepSync(100);
+  }
+  if (!xSocketExists(n)) {
+    try {
+      child.kill("SIGKILL");
+    } catch {}
+    return { display: "", xvfbPid: 0, source: "none", error: `Xvfb 启动超时 (display=${display})` };
+  }
+
+  process.env.DISPLAY = display;
+  try {
+    writeFileSync(XVFB_PID_FILE, String(child.pid));
+  } catch {}
+  // watchdog 退出时清理自托管实例（pid 文件先删，防误判复用）
+  process.on("exit", () => {
+    try {
+      unlinkSync(XVFB_PID_FILE);
+    } catch {}
+    try {
+      process.kill(child.pid, "SIGTERM");
+    } catch {}
+  });
+  log(`已自托管 Xvfb (pid=${child.pid}, display=${display})`);
+  return { display, xvfbPid: child.pid, source: "self-hosted", error: null };
+}
+
+/** Edge 探测（与 cdp.js getEdgePath 同候选序，此处独立实现避免循环依赖） */
+function detectEdge() {
+  for (const c of ["msedge", "microsoft-edge", "microsoft-edge-stable"]) {
+    const p = which(c);
+    if (p) {
+      let version = "";
+      try {
+        version = execSync(`${c} --version 2>/dev/null`, { encoding: "utf-8", timeout: 5000 }).trim();
+      } catch {}
+      return { path: p, version };
+    }
+  }
+  return { path: "", version: "" };
+}
+
+/**
+ * Linux 环境自举入口（幂等，可重复调用；每次启动全量探测，缺什么补什么）
+ * @returns {Promise<object>} envCapability（随心跳上报；旧版服务端忽略）
+ */
+export async function ensureLinuxEnv() {
+  const cap = {
+    platform: "linux",
+    display: "",
+    xvfbPid: 0,
+    displaySource: "none",
+    edgePath: "",
+    edgeVersion: "",
+    errors: [],
+  };
+
+  try {
+    const d = ensureDisplay();
+    cap.display = d.display;
+    cap.xvfbPid = d.xvfbPid;
+    cap.displaySource = d.source;
+    if (d.error) cap.errors.push(d.error);
+  } catch (e) {
+    cap.errors.push(`display: ${e.message}`);
+  }
+
+  try {
+    const e = detectEdge();
+    cap.edgePath = e.path;
+    cap.edgeVersion = e.version;
+    if (!e.path) cap.errors.push("未检测到 Edge 浏览器（业务不可用；微软源 apt install microsoft-edge-stable）");
+  } catch (e) {
+    cap.errors.push(`edge: ${e.message}`);
+  }
+
+  if (cap.errors.length) {
+    for (const m of cap.errors) log(`⚠ ${m}`);
+  } else {
+    log(`✓ 环境 OK (display=${cap.display} 来源=${cap.displaySource}, Edge=${cap.edgeVersion || "?"})`);
+  }
+  return cap;
+}
+
+/**
+ * 生成并启用 tt-watchdog systemd 系统级服务（每台机器一次性 root 动作）。
+ * 由 `tt-help watchdog install-service` 调用（需 root）。
+ * sudo 场景：unit 必须以真实用户运行（User= + HOME 取 SUDO_USER），
+ * 否则 watchdog 以 root 跑、读 /root/.tt-help.json → 配置/心跳/命令匹配全错（实机踩坑）。
+ * @param {{clientId: string, serverUrl: string, intervalSec: number}} opts
+ */
+export function installWatchdogService({ clientId, serverUrl, intervalSec = 15 }) {
+  const unitPath = "/etc/systemd/system/tt-watchdog.service";
+  // sudo 时还原真实用户与 HOME（直接跑 root 场景两值为空 → 退回当前环境）
+  let runUser = "";
+  let runHome = homedir();
+  if (process.env.SUDO_USER) {
+    try {
+      runHome = execSync(
+        `getent passwd ${process.env.SUDO_USER} | cut -d: -f6`,
+        { encoding: "utf-8" },
+      ).trim();
+      runUser = process.env.SUDO_USER;
+    } catch {}
+  }
+  const execLine = [
+    "/usr/local/bin/tt-help",
+    "watchdog",
+    serverUrl ? `--server ${serverUrl}` : "",
+    clientId ? `--client-id ${clientId}` : "",
+    `--interval ${intervalSec}`,
+  ]
+    .filter(Boolean)
+    .join(" ");
+  const userLines = runUser ? `User=${runUser}\nEnvironment=HOME=${runHome}` : `Environment=HOME=${runHome}`;
+  const unit = `# Generated by tt-help watchdog install-service (${new Date().toISOString()})
+[Unit]
+Description=tt-help watchdog (${clientId || "linux node"})
+After=network-online.target
+
+[Service]
+Type=simple
+${userLines}
+ExecStart=${execLine}
+Restart=always
+RestartSec=10
+
+[Install]
+WantedBy=multi-user.target
+`;
+  writeFileSync(unitPath, unit, "utf-8"); // root 直写；非 root 时 writeFileSync 抛错由调用方提示
+  execSync("systemctl daemon-reload");
+  execSync("systemctl enable --now tt-watchdog.service");
+  return unitPath;
+}

package/src/lib/login-pool-client.js

@@ -0,0 +1,156 @@
+/**
+ * 登录池客户端（Phase 2）— 业务进程直接向账号中心取号/还号 + 浏览器原地注入
+ *
+ * 与心跳命令通道无关：acquire 响应直接携带明文 cookies（POST 专属通道，与心跳下发同级），
+ * 注入由业务进程在自己持有的浏览器实例上执行（POC 实证 SOP，见知识库 edge-login-transfer.md）:
+ *   context.clearCookies() + addCookies → 导航 tiktok.com → passport/web/account/info 验证身份
+ *
+ * 语义约定（2026-10-26 定稿：以号为主键，无幂等/无端口键）:
+ *   - acquire: 挑最久未用的空闲号 → 租约 → 响应带明文 cookies；池空/服务器不可达抛错
+ *   - 换号 = 先取新号、验证通过后再 release 旧号（无空窗，池空保旧重试）
+ *   - release: 按账号 id 还号（谁持账号谁还）；崩溃未还的租约由 TTL 兜底回收
+ *   - port/appName 仅作租约展示元数据，不参与任何键/约束
+ *   - 无续租心跳: 48min 换号让租约永远年轻，服务端 TTL 12h 仅为崩溃兜底
+ */
+
+const REQUEST_TIMEOUT_MS = 30 * 1000;
+
+/** 账号中心 POST（fail-fast：业务错误如「池空」直接抛错，不无限重试——重试节奏由调用方掌控） */
+async function poolPost(url, body) {
+  const res = await fetch(url, {
+    method: "POST",
+    headers: { "Content-Type": "application/json" },
+    body: JSON.stringify(body),
+    signal: AbortSignal.timeout(REQUEST_TIMEOUT_MS),
+  });
+  const data = await res.json().catch(() => null);
+  if (!res.ok || !data || data.error) {
+    throw new Error((data && data.error) || `HTTP ${res.status}`);
+  }
+  return data;
+}
+
+/**
+ * 创建登录池客户端
+ * @param {object} opts { serverUrl, userId, port?, appName? }
+ */
+export function createLoginPoolClient({ serverUrl, userId, port, appName = "explore" }) {
+  const base = `${String(serverUrl).replace(/\/+$/, "")}/api/login-pool`;
+  return {
+    /** 取号：挑最久未用的空闲号，响应带明文 cookies。失败抛错（池空/服务器不可达） */
+    async acquire() {
+      const r = await poolPost(`${base}/acquire`, { userId, port, appName });
+      return r.account;
+    },
+    /** 还号：以号为主键，按 id 释放（谁持账号谁还） */
+    async release(account) {
+      if (!account || !account.id) return null;
+      const r = await poolPost(`${base}/release`, { id: account.id });
+      return r.account;
+    },
+  };
+}
+
+/**
+ * TikTok 登录身份判定（POC 实证）：必须在 tiktok.com 页面上下文执行
+ * （about:blank 等跨源上下文 fetch account/info 会因 CORS 带不上凭证）
+ */
+export async function whoAmI(page) {
+  return page.evaluate(async () => {
+    try {
+      const controller = new AbortController();
+      const timer = setTimeout(() => controller.abort(), 10000);
+      const res = await fetch("https://www.tiktok.com/passport/web/account/info/", {
+        signal: controller.signal,
+        credentials: "include",
+        headers: { accept: "application/json" },
+      });
+      clearTimeout(timer);
+      const j = await res.json().catch(() => null);
+      const d = (j && j.data) || null;
+      return {
+        httpStatus: res.status,
+        username: (d && d.username) || null,
+        hasAccount: !!d && !!(d.username || d.user_id || d.sec_user_id),
+        expired: (d && d.name === "session_expired") || (d && d.error_code === 13),
+      };
+    } catch (e) {
+      return { httpStatus: 0, username: null, hasAccount: false, expired: false, error: String(e).slice(0, 120) };
+    }
+  });
+}
+
+/** 导航到 tiktok.com 首页（身份验证/注入前置） */
+export async function gotoTiktok(page, timeout = 45000) {
+  await page.goto("https://www.tiktok.com/", {
+    waitUntil: "domcontentloaded",
+    timeout,
+  });
+}
+
+function toPlaywrightCookies(cookies) {
+  return (cookies || []).map((c) => ({
+    name: c.name,
+    value: c.value,
+    domain: c.domain || ".tiktok.com",
+    path: c.path || "/",
+    secure: c.secure !== false,
+    httpOnly: c.httpOnly !== false,
+    ...(typeof c.expires === "number" && c.expires > 0 ? { expires: c.expires } : {}),
+  }));
+}
+
+/**
+ * 原地注入账号（同一 Edge 运行实例，不重启浏览器；POC T2 实证）
+ * 清旧 cookie → 注入 → 导航 → passport 验证身份。
+ * @returns {Promise<{ok: boolean, username?: string, reason?: string}>}
+ */
+export async function injectPoolAccount(page, account, { verifyUsername } = {}) {
+  if (!account || !Array.isArray(account.cookies) || account.cookies.length === 0) {
+    return { ok: false, reason: "账号 cookies 为空" };
+  }
+  const context = page.context();
+  await context.clearCookies();
+  await context.addCookies(toPlaywrightCookies(account.cookies));
+  await gotoTiktok(page);
+  const id = await whoAmI(page);
+  if (!id.hasAccount) {
+    return {
+      ok: false,
+      reason: id.expired ? "会话过期" : id.error || "无账号数据",
+      identity: id,
+    };
+  }
+  if (verifyUsername && id.username !== verifyUsername) {
+    return {
+      ok: false,
+      reason: `身份不符（期望 @${verifyUsername}，实际 @${id.username}）`,
+      identity: id,
+    };
+  }
+  return { ok: true, username: id.username, identity: id };
+}
+
+/**
+ * 浏览器重建/重启后恢复在手账号：磁盘态一致则免注入，不一致用内存 cookies 补注入。
+ * （以号为主键模型下仅用于「浏览器崩溃/内存重启」场景；进程启动一律直接注入新取的号）
+ * @returns {Promise<{ok: boolean, injected: boolean, username?: string, reason?: string}>}
+ */
+export async function ensurePoolLogin(page, account) {
+  if (!account) return { ok: false, injected: false, reason: "无在手账号" };
+  let id = null;
+  try {
+    await gotoTiktok(page);
+    id = await whoAmI(page);
+  } catch (e) {
+    return { ok: false, injected: false, reason: `导航失败: ${e.message}` };
+  }
+  if (id.hasAccount && id.username === account.username) {
+    return { ok: true, injected: false, username: id.username };
+  }
+  const r = await injectPoolAccount(page, account, { verifyUsername: account.username }).catch((e) => ({
+    ok: false,
+    reason: e.message,
+  }));
+  return { ...r, injected: true };
+}

package/src/scraper/modules/follow-extractor.js

@@ -2,7 +2,7 @@ import { delay, getDelayConfig } from "./page-helpers.js";
 import { scrollAndCollect } from "./scroll-collector.js";
 import { extractUniqueId, toProfileUrl } from "../../lib/url.js";
 
-const FILTER_WORDS = ["主页", "已关注", "粉丝"];
+const FILTER_WORDS = ["主页", "已关注", "粉丝", "Home", "Following", "Followers"];
 
 const FOLLOW_TRIGGER_SELECTORS = [
   "[data-e2e=following]",
@@ -164,9 +164,20 @@ async function openFollowModal(page, log = () => {}) {
 
 async function switchToFollowersTab(page) {
   await page.evaluate(() => {
-    const tabs = document.querySelectorAll("[class*=DivTabItem]");
-    for (const tab of tabs) {
-      if (tab.textContent?.includes("粉丝")) {
+    // 结构化定位（语言无关）：弹窗 Tab 产品序固定 Following/Followers/Suggested，
+    // DivTabs 容器内第 2 个即粉丝 Tab。实测英文 profile 文本为 "Followers134"，
+    // 旧版单中文 includes("粉丝") 必败（ubuntu26 实机踩坑；勿改回文字匹配）
+    const tabs = [
+      ...document.querySelectorAll("[class*=DivTabs] [class*=DivTabItem]"),
+    ];
+    if (tabs.length >= 2) {
+      tabs[1].click();
+      return;
+    }
+    // 兑底：任意位置的双语文本匹配（结构变化时降级）
+    for (const tab of document.querySelectorAll("[class*=DivTabItem]")) {
+      const text = tab.textContent || "";
+      if (text.includes("粉丝") || /followers/i.test(text)) {
         tab.click();
         return;
       }
@@ -189,7 +200,14 @@ async function closeFollowModal(page) {
 
 function createUserCollectFn() {
   return (container) => {
-    const FILTER_WORDS = ["主页", "已关注", "粉丝"];
+    const FILTER_WORDS = [
+      "主页",
+      "已关注",
+      "粉丝",
+      "Home",
+      "Following",
+      "Followers",
+    ];
     const modal = document.querySelector("[class*=eyhy6180]");
     const root = modal || document;
     const users = [];

package/src/scraper/modules/guess-extractor.js

@@ -3,7 +3,10 @@ import { scrollAndCollect } from './scroll-collector.js';
 
 async function openGuessTab(page) {
   const tabs = page.locator('[class*="tabbar-item"]');
-  const guessTab = tabs.filter({ hasText: /猜你喜欢/i }).first();
+  // 双语：中文页「猜你喜欢」；英文页 "Suggested for you"（Linux 英文 profile 踩坑）
+  const guessTab = tabs
+    .filter({ hasText: /猜你喜欢|Suggested|You may like/i })
+    .first();
   await guessTab.click();
   const config = getDelayConfig();
   await delay(Math.round(config.commentMax * 0.5), config.commentMax);