ppxc-leads-mcp 0.1.0

package/dist/browser/kernel/runner-page-session.search-parser.js

@@ -0,0 +1,187 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.isLikelyDouyinSearchJsonResponse = isLikelyDouyinSearchJsonResponse;
+exports.parseSearchItemPayload = parseSearchItemPayload;
+exports.sourceUrlPattern = sourceUrlPattern;
+exports.redactSearchUrl = redactSearchUrl;
+const DOUYIN_JSON_HOST_RE = /(^|\.)douyin\.com$/i;
+const STATIC_RESOURCE_EXT_RE = /\.(?:js|mjs|css|png|jpe?g|webp|gif|svg|ico|woff2?|ttf|mp4|webm|m4a|mp3)(?:$|\?)/i;
+function isLikelyDouyinSearchJsonResponse(url, mimeType) {
+    const parsed = parseHttpUrl(url);
+    if (!parsed)
+        return false;
+    if (!DOUYIN_JSON_HOST_RE.test(parsed.hostname))
+        return false;
+    const lowerUrl = parsed.pathname.toLowerCase() + parsed.search.toLowerCase();
+    if (STATIC_RESOURCE_EXT_RE.test(lowerUrl))
+        return false;
+    const lowerMime = String(mimeType || "").toLowerCase();
+    if (lowerMime && !/(^|[/+])json\b|text\/plain|application\/octet-stream/.test(lowerMime)) {
+        return false;
+    }
+    if (parsed.pathname.includes("/aweme/v1/web/search/item/"))
+        return true;
+    return (lowerUrl.includes("search") &&
+        lowerUrl.includes("aweme") &&
+        (lowerUrl.includes("item") || lowerUrl.includes("general")));
+}
+function parseSearchItemPayload(payload, meta) {
+    if (!payload || typeof payload !== "object")
+        return null;
+    const awemes = extractAwemeObjects(payload);
+    const items = [];
+    const seen = new Set();
+    for (const aw of awemes) {
+        const awemeId = asNonEmptyString(aw.aweme_id);
+        if (!awemeId || seen.has(awemeId))
+            continue;
+        seen.add(awemeId);
+        const author = asRecord(aw.author);
+        const stats = asRecord(aw.statistics);
+        const video = asRecord(aw.video);
+        const avatarObj = asRecord(author?.avatar_thumb) ?? asRecord(author?.avatar_medium);
+        const avatarList = asArray(avatarObj?.url_list);
+        const authorAvatarUrl = typeof avatarList?.[0] === "string"
+            ? avatarList[0]
+            : typeof author?.avatar_url === "string"
+                ? author.avatar_url
+                : undefined;
+        const cover = asRecord(video?.cover);
+        const urlList = asArray(cover?.url_list);
+        const coverUrl = typeof urlList?.[0] === "string" ? urlList[0] : undefined;
+        const duration = asNumber(video?.duration);
+        items.push({
+            awemeId,
+            title: typeof aw.desc === "string" ? aw.desc : undefined,
+            authorName: typeof author?.nickname === "string" ? author.nickname : undefined,
+            authorSecUid: typeof author?.sec_uid === "string" ? author.sec_uid : undefined,
+            authorAvatarUrl,
+            coverUrl,
+            commentCount: asNumber(stats?.comment_count),
+            createTime: asNumber(aw.create_time),
+            durationSec: typeof duration === "number" ? Math.round((duration || 0) / 1000) : undefined,
+        });
+    }
+    const hasBatchSignal = items.length > 0 ||
+        "verify_check" in payload ||
+        "has_more" in payload ||
+        "cursor" in payload ||
+        "status_code" in payload ||
+        "data" in payload ||
+        "business_data" in payload ||
+        "aweme_list" in payload;
+    if (!hasBatchSignal)
+        return null;
+    const verifyCheck = Boolean(payload.verify_check);
+    return {
+        arrivedAt: Date.now(),
+        offset: meta.offset,
+        count: meta.count,
+        cursor: Number(payload.cursor ?? meta.offset + items.length),
+        hasMore: Number(payload.has_more ?? 0),
+        statusCode: Number(payload.status_code ?? 0),
+        verifyCheck,
+        sourceUrlPattern: sourceUrlPattern(meta.url),
+        parsedVideoCount: items.length,
+        items,
+        rawUrl: redactSearchUrl(meta.url),
+    };
+}
+function sourceUrlPattern(url) {
+    const parsed = parseHttpUrl(url);
+    if (!parsed)
+        return "[unparseable]";
+    const path = parsed.pathname.endsWith("/") ? parsed.pathname : `${parsed.pathname}/`;
+    if (path.includes("/aweme/v1/web/search/item/")) {
+        return "/aweme/v1/web/search/item/";
+    }
+    return path;
+}
+function redactSearchUrl(url) {
+    const parsed = parseHttpUrl(url);
+    if (!parsed)
+        return "[unparseable]";
+    const offset = parsed.searchParams.get("offset") ?? "";
+    const count = parsed.searchParams.get("count") ?? "";
+    const cursor = parsed.searchParams.get("cursor") ?? "";
+    const kw = parsed.searchParams.get("keyword") ?? parsed.searchParams.get("query") ?? "";
+    return `${parsed.pathname}?offset=${offset}&count=${count}&cursor=${cursor}&keyword.len=${kw.length}`;
+}
+function extractAwemeObjects(root) {
+    const out = [];
+    const seenObjects = new WeakSet();
+    const seenAwemeIds = new Set();
+    let visited = 0;
+    const pushAweme = (value) => {
+        const aw = asRecord(value);
+        if (!aw)
+            return;
+        const awemeId = asNonEmptyString(aw.aweme_id);
+        if (!awemeId || seenAwemeIds.has(awemeId))
+            return;
+        seenAwemeIds.add(awemeId);
+        out.push(aw);
+    };
+    const visit = (value, depth) => {
+        if (value === null || typeof value !== "object")
+            return;
+        if (depth > 8 || visited > 1500 || out.length >= 300)
+            return;
+        visited++;
+        const obj = value;
+        if (seenObjects.has(obj))
+            return;
+        seenObjects.add(obj);
+        if (Array.isArray(value)) {
+            for (const item of value)
+                visit(item, depth + 1);
+            return;
+        }
+        const rec = value;
+        pushAweme(rec);
+        const directAweme = asRecord(rec.aweme_info);
+        if (directAweme)
+            pushAweme(directAweme);
+        for (const child of Object.values(rec)) {
+            visit(child, depth + 1);
+        }
+    };
+    visit(root, 0);
+    return out;
+}
+function parseHttpUrl(url) {
+    try {
+        const parsed = new URL(url);
+        if (!/^https?:$/i.test(parsed.protocol))
+            return null;
+        return parsed;
+    }
+    catch {
+        return null;
+    }
+}
+function asRecord(value) {
+    return value && typeof value === "object" && !Array.isArray(value)
+        ? value
+        : undefined;
+}
+function asArray(value) {
+    return Array.isArray(value) ? value : undefined;
+}
+function asNonEmptyString(value) {
+    if (typeof value === "string")
+        return value.trim();
+    if (typeof value === "number" && Number.isFinite(value))
+        return String(Math.trunc(value));
+    return "";
+}
+function asNumber(value) {
+    if (typeof value === "number" && Number.isFinite(value))
+        return value;
+    if (typeof value === "string" && value.trim()) {
+        const parsed = Number(value);
+        return Number.isFinite(parsed) ? parsed : undefined;
+    }
+    return undefined;
+}
+//# sourceMappingURL=runner-page-session.search-parser.js.map

package/dist/browser/kernel/runner-page-session.user-agent.js

@@ -0,0 +1,32 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.resolveDouyinUserAgent = resolveDouyinUserAgent;
+const DEFAULT_CHROME_VERSION = "148.0.0.0";
+function pickChromeVersion(versions) {
+    const raw = String(versions?.chrome ?? "").trim();
+    if (!/^\d+(?:\.\d+){0,3}$/.test(raw)) {
+        return DEFAULT_CHROME_VERSION;
+    }
+    const parts = raw.split(".");
+    while (parts.length < 4)
+        parts.push("0");
+    return parts.join(".");
+}
+function buildUserAgent(platformLabel, chromeVersion) {
+    return (`Mozilla/5.0 (${platformLabel}) ` +
+        `AppleWebKit/537.36 (KHTML, like Gecko) ` +
+        `Chrome/${chromeVersion} Safari/537.36`);
+}
+function resolveDouyinUserAgent(platform = process.platform, electronVersions = process.versions) {
+    const chromeVersion = pickChromeVersion(electronVersions);
+    switch (platform) {
+        case "win32":
+            return buildUserAgent("Windows NT 10.0; Win64; x64", chromeVersion);
+        case "linux":
+            return buildUserAgent("X11; Linux x86_64", chromeVersion);
+        case "darwin":
+        default:
+            return buildUserAgent("Macintosh; Intel Mac OS X 10_15_7", chromeVersion);
+    }
+}
+//# sourceMappingURL=runner-page-session.user-agent.js.map

package/dist/browser/platform-runner.js

@@ -0,0 +1,312 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.startDouyinLogin = exports.fetchVideoComments = exports.RunnerError = void 0;
+exports.getLoginStatus = getLoginStatus;
+exports.startPlatformLogin = startPlatformLogin;
+exports.fetchContentComments = fetchContentComments;
+exports.searchKeywordForLeads = searchKeywordForLeads;
+exports.searchKeywordsBatch = searchKeywordsBatch;
+exports.shutdownRunner = shutdownRunner;
+exports.mapSearchResultForDouyin = mapSearchResultForDouyin;
+const logger_1 = require("./kernel/logger");
+const registry_1 = require("./platforms/registry");
+const usage_throttle_1 = require("./usage-throttle");
+const log = logger_1.logger.scope("platform-runner");
+const SLOT_ID = 0;
+const MAX_SLOTS = 2;
+const DEFAULT_FETCH_WAIT_MS = 25000;
+class RunnerError extends Error {
+    constructor(code, message) {
+        super(message);
+        this.code = code;
+        this.name = "RunnerError";
+    }
+}
+exports.RunnerError = RunnerError;
+let runnerBusyWith = null;
+const managers = new Map();
+function sleep(ms) {
+    return new Promise((resolve) => setTimeout(resolve, ms));
+}
+async function withRunnerLock(operation, fn) {
+    if (runnerBusyWith) {
+        throw new RunnerError("BUSY", `正在执行「${runnerBusyWith}」，请等它结束后再试`);
+    }
+    runnerBusyWith = operation;
+    try {
+        return await fn();
+    }
+    finally {
+        runnerBusyWith = null;
+    }
+}
+function assertCrawlQuota(platform, units) {
+    const adapter = (0, registry_1.getPlatformAdapter)(platform);
+    const verdict = (0, usage_throttle_1.consumeCrawlQuota)(platform, units, adapter.risk.dailyQuotaUnits);
+    if (verdict.ok)
+        return;
+    if (verdict.reason === "interval") {
+        throw new RunnerError("RATE_LIMITED", `两次抓取之间需要间隔 30 秒（还差 ${verdict.waitSec} 秒）`);
+    }
+    throw new RunnerError("DAILY_LIMITED", `今天 ${adapter.displayName} 的安全抓取额度已用完（${verdict.usedToday}/${verdict.limit}），明天再继续`);
+}
+function getManager(platform) {
+    let mgr = managers.get(platform);
+    if (!mgr) {
+        const adapter = (0, registry_1.getPlatformAdapter)(platform);
+        mgr = adapter.createManager(process.env.PPXC_MCP_PAGE_VISIBLE === "1");
+        managers.set(platform, mgr);
+    }
+    return mgr;
+}
+function randomDelay(adapter) {
+    const { perItemMinDelayMs, perItemMaxDelayMs } = adapter.risk;
+    return perItemMinDelayMs + Math.floor(Math.random() * (perItemMaxDelayMs - perItemMinDelayMs));
+}
+async function ensureOnContentPage(session, contentId, contentUrl) {
+    const current = session.currentUrl();
+    if (current.includes(contentId)) {
+        await sleep(1000);
+        return;
+    }
+    try {
+        await session.loadUrl(contentUrl);
+    }
+    catch {
+    }
+    await sleep(3000);
+}
+async function throwIfVerification(session, adapter) {
+    if (await adapter.probeVerification(session)) {
+        session.reveal();
+        throw new RunnerError("VERIFICATION_REQUIRED", `${adapter.displayName}要求验证，已弹出窗口请人工处理`);
+    }
+}
+async function getLoginStatus(platform) {
+    const adapter = (0, registry_1.getPlatformAdapter)(platform);
+    const session = getManager(platform).acquire(SLOT_ID, { visible: false });
+    const loggedIn = await adapter.isLoggedIn(session);
+    if (!loggedIn)
+        return { loggedIn: false };
+    const displayName = await adapter.getProfileDisplayName(session);
+    return { loggedIn: true, displayName };
+}
+async function startPlatformLogin(platform, timeoutMs = 180000) {
+    const adapter = (0, registry_1.getPlatformAdapter)(platform);
+    return withRunnerLock(`${adapter.displayName}扫码登录`, async () => {
+        const result = await adapter.startLogin(getManager(platform), SLOT_ID, timeoutMs);
+        return { loggedIn: result.loggedIn, displayName: result.displayName };
+    });
+}
+async function fetchContentComments(platform, contentUrlOrId, maxComments = 30) {
+    const adapter = (0, registry_1.getPlatformAdapter)(platform);
+    return withRunnerLock(`${adapter.displayName}分析评论`, () => fetchContentCommentsInner(adapter, contentUrlOrId, maxComments));
+}
+async function fetchContentCommentsInner(adapter, contentUrlOrId, maxComments) {
+    const count = Math.max(1, Math.min(50, Math.floor(maxComments) || 30));
+    const manager = getManager(adapter.id);
+    const session = manager.acquire(SLOT_ID, { visible: process.env.PPXC_MCP_PAGE_VISIBLE === "1" });
+    await sleep(500);
+    if (!(await adapter.isLoggedIn(session))) {
+        throw new RunnerError("LOGIN_REQUIRED", `${adapter.displayName}未登录，需要先扫码登录`);
+    }
+    assertCrawlQuota(adapter.id, 1);
+    let contentId;
+    try {
+        contentId = await adapter.resolveContentId(contentUrlOrId, session);
+    }
+    catch {
+        throw new RunnerError("BAD_VIDEO_LINK", `无法从链接里识别出${adapter.displayName}内容：${contentUrlOrId}`);
+    }
+    const contentUrl = adapter.preferredContentUrl?.(contentUrlOrId, contentId, session) ??
+        adapter.buildContentUrl(contentId);
+    await ensureOnContentPage(session, contentId, contentUrl);
+    const fetched = await adapter.fetchComments(session, contentId, contentUrl, count, DEFAULT_FETCH_WAIT_MS);
+    if (!fetched.json) {
+        await throwIfVerification(session, adapter);
+        throw new RunnerError("FETCH_TIMEOUT", `${adapter.displayName}评论没有返回数据`);
+    }
+    const json = fetched.json;
+    if (adapter.id === "douyin" && Number(json.status_code ?? 0) !== 0) {
+        await throwIfVerification(session, adapter);
+        throw new RunnerError("FETCH_TIMEOUT", `${adapter.displayName}返回非成功状态`);
+    }
+    const comments = adapter.parseComments(contentId, fetched.json);
+    return {
+        contentId,
+        contentUrl,
+        awemeId: contentId,
+        videoUrl: contentUrl,
+        comments,
+        rawCount: fetched.rawCount,
+        hasMore: fetched.hasMore,
+    };
+}
+async function searchKeywordForLeads(platform, keyword, maxItems = 5, commentsPerItem = 30, slotId = SLOT_ID, shouldAbort) {
+    const adapter = (0, registry_1.getPlatformAdapter)(platform);
+    const kw = String(keyword ?? "").trim();
+    if (!kw)
+        throw new RunnerError("NO_SEARCH_RESULT", "搜索词不能为空");
+    const itemLimit = Math.max(1, Math.min(adapter.risk.maxVideosPerKeyword, Math.floor(maxItems) || 5));
+    const commentCount = Math.max(1, Math.min(50, Math.floor(commentsPerItem) || 30));
+    const manager = getManager(platform);
+    const session = manager.acquire(slotId, { visible: process.env.PPXC_MCP_PAGE_VISIBLE === "1" });
+    await sleep(500);
+    if (!(await adapter.isLoggedIn(session))) {
+        throw new RunnerError("LOGIN_REQUIRED", `${adapter.displayName}未登录，需要先扫码登录`);
+    }
+    const batch = await adapter.awaitSearchBatch(session, kw, DEFAULT_FETCH_WAIT_MS);
+    if (!batch) {
+        await throwIfVerification(session, adapter);
+        throw new RunnerError("NO_SEARCH_RESULT", `没搜到「${kw}」的内容，换个词或稍后再试`);
+    }
+    if (batch.verifyCheck) {
+        await throwIfVerification(session, adapter);
+    }
+    let ranked = batch.items
+        .filter((it) => it.contentId && (it.commentCount ?? 0) >= adapter.risk.minCommentCount)
+        .sort((a, b) => (b.commentCount ?? 0) - (a.commentCount ?? 0));
+    if (ranked.length === 0) {
+        ranked = batch.items.filter((it) => it.contentId);
+    }
+    const perAuthor = new Map();
+    const selected = [];
+    for (const it of ranked) {
+        if (selected.length >= itemLimit)
+            break;
+        const authorKey = String(it.authorSecUid ?? it.authorName ?? `__anon_${it.contentId}`);
+        const seen = perAuthor.get(authorKey) ?? 0;
+        if (seen >= adapter.risk.maxPerAuthor)
+            continue;
+        perAuthor.set(authorKey, seen + 1);
+        selected.push(it);
+    }
+    if (selected.length === 0) {
+        throw new RunnerError("NO_SEARCH_RESULT", `「${kw}」搜到的内容评论都太少，换个词试试`);
+    }
+    const items = [];
+    let totalComments = 0;
+    for (const it of selected) {
+        if (shouldAbort?.()) {
+            log.warn("search aborted between items", { platform: adapter.id, keyword: kw });
+            break;
+        }
+        const contentId = String(it.contentId);
+        const contentUrl = it.contentUrl || adapter.buildContentUrl(contentId);
+        try {
+            await ensureOnContentPage(session, contentId, contentUrl);
+            const fetched = await adapter.fetchComments(session, contentId, contentUrl, commentCount, DEFAULT_FETCH_WAIT_MS);
+            if (fetched.json) {
+                const comments = adapter.parseComments(contentId, fetched.json);
+                totalComments += comments.length;
+                items.push({
+                    contentId,
+                    contentUrl,
+                    awemeId: contentId,
+                    videoUrl: contentUrl,
+                    title: it.title ?? "",
+                    authorName: it.authorName ?? "",
+                    commentCount: it.commentCount ?? 0,
+                    comments,
+                });
+            }
+        }
+        catch (err) {
+            log.warn("search item comment fetch failed", {
+                contentId,
+                msg: err instanceof Error ? err.message : String(err),
+            });
+        }
+        await sleep(randomDelay(adapter));
+    }
+    return {
+        keyword: kw,
+        itemsFound: ranked.length,
+        itemsRead: items.length,
+        items,
+        videos: items,
+        totalComments,
+    };
+}
+async function searchKeywordsBatch(platform, keywords, maxItemsPerKeyword = 5, commentsPerItem = 30) {
+    const adapter = (0, registry_1.getPlatformAdapter)(platform);
+    const cleaned = [];
+    const seen = new Set();
+    for (const raw of keywords) {
+        const kw = String(raw ?? "").trim();
+        if (kw && !seen.has(kw)) {
+            seen.add(kw);
+            cleaned.push(kw);
+        }
+    }
+    if (cleaned.length === 0)
+        throw new RunnerError("NO_SEARCH_RESULT", "搜索词不能为空");
+    return withRunnerLock(`${adapter.displayName}关键词搜索`, async () => {
+        const probeSession = getManager(platform).acquire(SLOT_ID, {
+            visible: process.env.PPXC_MCP_PAGE_VISIBLE === "1",
+        });
+        if (!(await adapter.isLoggedIn(probeSession))) {
+            throw new RunnerError("LOGIN_REQUIRED", `${adapter.displayName}未登录，需要先扫码登录`);
+        }
+        assertCrawlQuota(platform, cleaned.length);
+        const perKeywordCap = Math.max(1, Math.min(Math.max(1, Math.min(adapter.risk.maxVideosPerKeyword, Math.floor(maxItemsPerKeyword) || 5)), Math.floor(adapter.risk.totalContentBudget / cleaned.length)));
+        const slotCount = Math.min(MAX_SLOTS, cleaned.length);
+        const outcomes = [];
+        let verificationHit = false;
+        const runSlot = async (slotId) => {
+            const myKeywords = cleaned.filter((_kw, idx) => idx % slotCount === slotId);
+            if (slotId > 0)
+                await sleep(adapter.risk.slotStaggerMs);
+            for (const kw of myKeywords) {
+                if (verificationHit) {
+                    outcomes.push({ keyword: kw, ok: false, errorCode: "VERIFICATION_REQUIRED" });
+                    continue;
+                }
+                try {
+                    const result = await searchKeywordForLeads(platform, kw, perKeywordCap, commentsPerItem, slotId, () => verificationHit);
+                    outcomes.push({ keyword: kw, ok: true, result });
+                }
+                catch (err) {
+                    const code = err instanceof RunnerError ? err.code : "INTERNAL";
+                    log.warn("batch keyword failed", { platform, keyword: kw, code });
+                    outcomes.push({ keyword: kw, ok: false, errorCode: code });
+                    if (code === "VERIFICATION_REQUIRED")
+                        verificationHit = true;
+                }
+            }
+        };
+        await Promise.all(Array.from({ length: slotCount }, (_v, slotId) => runSlot(slotId)));
+        return outcomes;
+    });
+}
+async function shutdownRunner() {
+    for (const mgr of managers.values()) {
+        try {
+            await mgr.destroyAll();
+        }
+        catch {
+        }
+    }
+    managers.clear();
+}
+const fetchVideoComments = (videoUrlOrId, maxComments) => fetchContentComments("douyin", videoUrlOrId, maxComments);
+exports.fetchVideoComments = fetchVideoComments;
+const startDouyinLogin = (timeoutMs) => startPlatformLogin("douyin", timeoutMs);
+exports.startDouyinLogin = startDouyinLogin;
+function mapSearchResultForDouyin(result) {
+    return {
+        keyword: result.keyword,
+        videosFound: result.itemsFound,
+        videosRead: result.itemsRead,
+        videos: result.items.map((it) => ({
+            awemeId: it.contentId,
+            videoUrl: it.contentUrl,
+            title: it.title,
+            authorName: it.authorName,
+            commentCount: it.commentCount,
+            comments: it.comments,
+        })),
+        totalComments: result.totalComments,
+    };
+}
+//# sourceMappingURL=platform-runner.js.map

package/dist/browser/platforms/detect-platform.js

@@ -0,0 +1,33 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.detectPlatformFromUrl = detectPlatformFromUrl;
+exports.normalizePlatformId = normalizePlatformId;
+function detectPlatformFromUrl(input) {
+    const raw = String(input ?? "").trim().toLowerCase();
+    if (!raw)
+        return null;
+    if (/douyin\.com|v\.douyin\.com|iesdouyin\.com/.test(raw) ||
+        /^\d{15,25}$/.test(raw)) {
+        return "douyin";
+    }
+    if (/xiaohongshu\.com|xhslink\.com|xhs\.cn/.test(raw)) {
+        return "xiaohongshu";
+    }
+    if (/kuaishou\.com|gifshow\.com|chenzhongtech\.com|ksurl\.cn/.test(raw)) {
+        return "kuaishou";
+    }
+    return null;
+}
+function normalizePlatformId(explicit, contentUrl) {
+    const fromArg = String(explicit ?? "").trim().toLowerCase();
+    if (fromArg === "douyin" || fromArg === "xiaohongshu" || fromArg === "kuaishou") {
+        return fromArg;
+    }
+    if (contentUrl) {
+        const detected = detectPlatformFromUrl(contentUrl);
+        if (detected)
+            return detected;
+    }
+    return "douyin";
+}
+//# sourceMappingURL=detect-platform.js.map