tianshu-harness 3.23.0

package/dist/chunk-2HKD2UFM.js

@@ -0,0 +1,1000 @@
+#!/usr/bin/env -S node --expose-gc --max-old-space-size=4096
+import { createRequire as __rivetCreateRequire } from 'node:module'; const require = __rivetCreateRequire(import.meta.url);
+import {
+  SSRFError,
+  httpFetchGuarded,
+  resolveAndAssertPublic
+} from "./chunk-5JRZPLI6.js";
+import {
+  resolveProxyForUrl
+} from "./chunk-2HDP7RMR.js";
+import {
+  launchHeadlessChromium
+} from "./chunk-Z3QE7TZM.js";
+import {
+  fetchCauseDetail
+} from "./chunk-BKTQAYO2.js";
+import {
+  init_esm_shims
+} from "./chunk-5GGOGLSN.js";
+
+// src/tools/web-fetch/fetch-core.ts
+init_esm_shims();
+
+// src/tools/web-fetch/extract.ts
+init_esm_shims();
+var _TurndownCtor = null;
+async function loadTurndownCtor() {
+  if (!_TurndownCtor) {
+    const { default: TurndownService } = await import("./turndown.es-OI55K6C3.js");
+    _TurndownCtor = TurndownService;
+  }
+  return _TurndownCtor;
+}
+var MIN_SUBSTANTIAL_LENGTH = 200;
+var EXCLUDE_NON_MAIN_SELECTORS = [
+  "header",
+  "footer",
+  "nav",
+  "aside",
+  ".header",
+  ".top",
+  ".navbar",
+  "#header",
+  ".footer",
+  ".bottom",
+  "#footer",
+  ".sidebar",
+  ".side",
+  ".aside",
+  "#sidebar",
+  ".modal",
+  ".popup",
+  "#modal",
+  ".overlay",
+  ".ad",
+  ".ads",
+  ".advert",
+  "#ad",
+  ".lang-selector",
+  ".language",
+  "#language-selector",
+  ".social",
+  ".social-media",
+  ".social-links",
+  "#social",
+  ".menu",
+  ".navigation",
+  "#nav",
+  ".breadcrumbs",
+  "#breadcrumbs",
+  ".share",
+  "#share",
+  ".widget",
+  "#widget",
+  ".cookie",
+  "#cookie"
+];
+var FORCE_INCLUDE_SELECTORS = ["#main", "#content"];
+function safeMatches(node, selector) {
+  try {
+    return node.matches(selector);
+  } catch {
+    return false;
+  }
+}
+function safeClosest(node, selector) {
+  try {
+    return node.closest(selector) !== null;
+  } catch {
+    return false;
+  }
+}
+function pickBestSrcsetCandidate(srcset, src) {
+  const candidates = [];
+  let hasLowRes = false;
+  for (const part of srcset.split(",")) {
+    const bits = part.trim().split(/\s+/);
+    const url = bits[0];
+    if (!url) continue;
+    const desc = bits[1] ?? "";
+    let score = 1;
+    if (desc.endsWith("w")) score = parseFloat(desc) || 1;
+    else if (desc.endsWith("x")) score = (parseFloat(desc) || 1) * 1e3;
+    else hasLowRes = true;
+    if (score <= 1e3) hasLowRes = true;
+    candidates.push({ url, score });
+  }
+  if (src && hasLowRes) candidates.push({ url: src, score: 1e3 });
+  candidates.sort((a, b) => b.score - a.score);
+  return candidates[0]?.url;
+}
+function pickImageSource(el) {
+  const srcset = el.getAttribute("srcset") || el.getAttribute("data-srcset");
+  if (srcset) {
+    const best = pickBestSrcsetCandidate(srcset, el.getAttribute("src"));
+    if (best) return best;
+  }
+  return el.getAttribute("data-src") || el.getAttribute("src") || void 0;
+}
+function absolutizeUrl(url, base) {
+  if (!base || /^(data|blob|about):/i.test(url)) return void 0;
+  try {
+    return new URL(url, base).href;
+  } catch {
+    return void 0;
+  }
+}
+function extractLinks(html, pageUrl) {
+  const base = resolveBaseUrl(html, pageUrl);
+  const links = /* @__PURE__ */ new Set();
+  const re = /<a\b[^>]*?href=["']([^"']+)["']/gi;
+  let m;
+  while ((m = re.exec(html)) !== null) {
+    const href = m[1].trim();
+    if (!href || href.startsWith("#") || /^(mailto|tel|javascript|data):/i.test(href)) continue;
+    const abs = absolutizeUrl(href, base);
+    if (abs && (abs.startsWith("http:") || abs.startsWith("https:"))) links.add(abs);
+  }
+  return [...links];
+}
+function extractLinksFromMarkdown(markdown) {
+  const links = /* @__PURE__ */ new Set();
+  const re = /\]\((https?:\/\/[^)\s]+?)(?:\s+"[^"]*")?\)/g;
+  let m;
+  while ((m = re.exec(markdown)) !== null) {
+    links.add(m[1]);
+  }
+  return [...links];
+}
+function resolveBaseUrl(html, pageUrl) {
+  const headWindow = html.slice(0, 16384);
+  const baseHref = headWindow.match(/<base\b[^>]*?href=["']([^"']+)["']/i)?.[1];
+  if (baseHref) {
+    try {
+      return pageUrl ? new URL(baseHref, pageUrl).href : baseHref;
+    } catch {
+      return baseHref;
+    }
+  }
+  return pageUrl;
+}
+function postProcessMarkdown(md) {
+  let out = "";
+  let depth = 0;
+  let inFence = false;
+  for (let i = 0; i < md.length; i++) {
+    const ch = md[i];
+    if (ch === "`" && md.startsWith("```", i)) {
+      inFence = !inFence;
+      out += "```";
+      i += 2;
+      continue;
+    }
+    if (!inFence) {
+      const prev = i > 0 ? md[i - 1] : "";
+      if (ch === "[" && prev !== "\\") depth++;
+      else if (ch === "]" && prev !== "\\" && depth > 0) depth--;
+      else if (ch === "\n" && depth > 0) {
+        out += "\\\n";
+        continue;
+      }
+    }
+    out += ch;
+  }
+  out = out.replace(/\[ *skip to (?:main )?content *\]\([^)]*\) *\n?/gi, "");
+  out = out.replace(/!\[[^\]]*\]\(data:[^)]+\)/g, "<Base64-Image-Removed>");
+  out = out.replace(/\n{3,}/g, "\n\n");
+  return out;
+}
+async function createTurndown(baseUrl) {
+  const TurndownService = await loadTurndownCtor();
+  const td = new TurndownService({
+    headingStyle: "atx",
+    codeBlockStyle: "fenced"
+  });
+  td.remove(["script", "style", "noscript", "svg", "iframe", "template", "head"]);
+  td.addRule("excludeNonMain", {
+    filter: (node) => node.nodeType === 1 && EXCLUDE_NON_MAIN_SELECTORS.some((sel) => safeMatches(node, sel)) && !FORCE_INCLUDE_SELECTORS.some((sel) => safeClosest(node, sel)),
+    replacement: () => ""
+  });
+  td.addRule("imageFix", {
+    filter: "img",
+    replacement: (_content, node) => {
+      const src = pickImageSource(node);
+      if (!src) return "";
+      if (src.startsWith("data:")) return "<Base64-Image-Removed>";
+      const abs = absolutizeUrl(src, baseUrl) ?? src;
+      const alt = (node.getAttribute("alt") || "").replace(/[[\]]/g, "").trim();
+      return `![${alt}](${abs})`;
+    }
+  });
+  td.addRule("absolutizeLink", {
+    filter: "a",
+    replacement: (content, node) => {
+      const href = node.getAttribute("href");
+      const text = content.trim();
+      if (!href || href.startsWith("#") || /^(mailto|tel|javascript):/i.test(href)) {
+        if (href?.startsWith("#") && /skip to/i.test(text)) return "";
+        return text || "";
+      }
+      const final = absolutizeUrl(href, baseUrl) ?? href;
+      if (!text) return final;
+      const title = node.getAttribute("title");
+      return title ? `[${text}](${final} "${title}")` : `[${text}](${final})`;
+    }
+  });
+  return td;
+}
+async function htmlToMarkdown(html, opts = {}) {
+  const td = await createTurndown(resolveBaseUrl(html, opts.pageUrl));
+  return postProcessMarkdown(td.turndown(html));
+}
+async function htmlToMarkdownSmart(html, opts = {}) {
+  if (opts.onlyMainContent !== false) {
+    const md = await htmlToMarkdown(extractMainContent(html), opts);
+    if (md.trim().length >= MIN_SUBSTANTIAL_LENGTH) return md;
+  }
+  return htmlToMarkdown(html, opts);
+}
+function decodeBody(bytes, contentType) {
+  const charset = detectCharset(bytes, contentType);
+  try {
+    return new TextDecoder(charset, { fatal: false }).decode(bytes);
+  } catch {
+    return new TextDecoder("utf-8", { fatal: false }).decode(bytes);
+  }
+}
+function detectCharset(bytes, contentType) {
+  const header = contentType.match(/charset=([^;]+)/i)?.[1]?.trim().replace(/^["']|["']$/g, "");
+  if (header) return header.toLowerCase();
+  if (contentType.includes("text/html")) {
+    const head = new TextDecoder("utf-8", { fatal: false }).decode(bytes.slice(0, 1024));
+    const meta = head.match(/<meta[^>]+charset=["']?([^"';>\s]+)/i)?.[1];
+    if (meta) return meta.toLowerCase();
+  }
+  return "utf-8";
+}
+var NOISE_TAGS = ["script", "style", "noscript", "svg", "nav", "header", "footer", "aside"];
+function extractMainContent(html) {
+  const region = extractRegion(html, "main") ?? extractRegion(html, "article");
+  const source = region ?? html;
+  const noisePattern = new RegExp(`<(${NOISE_TAGS.join("|")})\\b[^>]*>[\\s\\S]*?</\\1>`, "gi");
+  const stripped = source.replace(noisePattern, "");
+  return stripped.trim();
+}
+function extractRegion(html, tag) {
+  const openRegex = new RegExp(`<${tag}\\b[^>]*>`, "i");
+  const openMatch = html.match(openRegex);
+  if (!openMatch) return void 0;
+  const start = openMatch.index + openMatch[0].length;
+  const openScanner = new RegExp(`<${tag}\\b[^>]*>`, "gi");
+  const closeScanner = new RegExp(`</${tag}>`, "gi");
+  let depth = 1;
+  let idx = start;
+  while (depth > 0) {
+    openScanner.lastIndex = idx;
+    closeScanner.lastIndex = idx;
+    const nextOpen = openScanner.exec(html);
+    const nextClose = closeScanner.exec(html);
+    if (!nextClose) return void 0;
+    if (nextOpen && nextOpen.index < nextClose.index) {
+      depth++;
+      idx = nextOpen.index + nextOpen[0].length;
+    } else {
+      depth--;
+      if (depth === 0) return html.slice(start, nextClose.index);
+      idx = nextClose.index + nextClose[0].length;
+    }
+  }
+  return void 0;
+}
+
+// src/tools/web-fetch/jina-fetch.ts
+init_esm_shims();
+var DEFAULT_JINA_BASE = "https://r.jina.ai";
+function isJinaQualityHeuristic(md) {
+  const trimmed = md.trim();
+  if (trimmed.length < 200) return true;
+  const jsPageSignals = [
+    "Please enable JavaScript",
+    "Enable JavaScript",
+    "This page requires JavaScript",
+    "noscript",
+    "Checking your browser",
+    "Just a moment",
+    "DDOS protection",
+    'id="challenge-form"',
+    // GitHub and similar SPA error shells — the page "loaded" but the app
+    // failed to hydrate, leaving a stub error banner as the only content.
+    "error while loading",
+    "please reload this page",
+    "uh oh"
+  ];
+  const lower = trimmed.toLowerCase();
+  return jsPageSignals.some((s) => lower.includes(s.toLowerCase()));
+}
+async function fetchViaJina(url, deps = {}, options = {}) {
+  const base = (options.jinaBaseUrl ?? DEFAULT_JINA_BASE).replace(/\/+$/, "");
+  const jinaUrl = `${base}/${url}`;
+  const { status, contentType, bytes } = await httpFetchGuarded(jinaUrl, deps, {
+    ...options
+    // Jina's text/plain responses are usually much smaller than raw HTML.
+    // Keep the same cap for consistency; if Jina returns oversized content
+    // the caller can re-fetch the raw URL with a higher cap.
+  });
+  const body = new TextDecoder("utf-8", { fatal: false }).decode(bytes);
+  const sourceStatus = status;
+  const ok = status >= 200 && status < 400 && body.trim().length > 0;
+  return { sourceStatus, markdown: body, ok };
+}
+
+// src/tools/web-fetch/render-fetch.ts
+init_esm_shims();
+import { lookup as dnsLookup } from "dns/promises";
+
+// src/tools/web-fetch/render-actions.ts
+init_esm_shims();
+var MAX_ACTIONS = 50;
+var MAX_TOTAL_WAIT_MS = 6e4;
+var ACTION_STEP_TIMEOUT_MS = 1e4;
+var EXECUTE_JS_RESULT_LIMIT = 2048;
+function isNonEmptyString(v) {
+  return typeof v === "string" && v.trim().length > 0;
+}
+function parseRenderActions(raw) {
+  if (!Array.isArray(raw)) return { error: "actions \u5FC5\u987B\u662F\u6570\u7EC4" };
+  if (raw.length === 0) return { error: "actions \u4E0D\u80FD\u4E3A\u7A7A\u6570\u7EC4" };
+  if (raw.length > MAX_ACTIONS) return { error: `actions \u6B65\u6570\u8D85\u9650\uFF08${raw.length} > ${MAX_ACTIONS}\uFF09` };
+  let totalWaitMs = 0;
+  const actions = [];
+  for (let i = 0; i < raw.length; i++) {
+    const a = raw[i];
+    if (!a || typeof a !== "object") return { error: `actions[${i}] \u4E0D\u662F\u5BF9\u8C61` };
+    switch (a.type) {
+      case "wait":
+        if (typeof a.ms === "number") {
+          if (!Number.isFinite(a.ms) || a.ms <= 0 || a.ms > MAX_TOTAL_WAIT_MS) {
+            return { error: `actions[${i}].ms \u975E\u6CD5\uFF08\u987B\u4E3A 1..${MAX_TOTAL_WAIT_MS}\uFF09` };
+          }
+          totalWaitMs += a.ms;
+          actions.push({ type: "wait", ms: a.ms });
+        } else if (isNonEmptyString(a.selector)) {
+          actions.push({ type: "wait", selector: a.selector.trim() });
+        } else {
+          return { error: `actions[${i}] wait \u9700\u8981 ms \u6216 selector` };
+        }
+        break;
+      case "click":
+        if (!isNonEmptyString(a.selector)) return { error: `actions[${i}].selector \u4E0D\u80FD\u4E3A\u7A7A` };
+        actions.push({ type: "click", selector: a.selector.trim(), all: a.all === true });
+        break;
+      case "write":
+        if (!isNonEmptyString(a.selector)) return { error: `actions[${i}].selector \u4E0D\u80FD\u4E3A\u7A7A` };
+        if (typeof a.text !== "string") return { error: `actions[${i}].text \u5FC5\u987B\u662F\u5B57\u7B26\u4E32` };
+        actions.push({ type: "write", selector: a.selector.trim(), text: a.text });
+        break;
+      case "press":
+        if (!isNonEmptyString(a.key)) return { error: `actions[${i}].key \u4E0D\u80FD\u4E3A\u7A7A` };
+        actions.push({
+          type: "press",
+          key: a.key.trim(),
+          ...isNonEmptyString(a.selector) ? { selector: a.selector.trim() } : {}
+        });
+        break;
+      case "scroll": {
+        const dir = a.direction;
+        actions.push({
+          type: "scroll",
+          direction: dir === "up" || dir === "top" || dir === "bottom" ? dir : "down",
+          ...isNonEmptyString(a.selector) ? { selector: a.selector.trim() } : {}
+        });
+        break;
+      }
+      case "execute_js":
+        if (!isNonEmptyString(a.script)) return { error: `actions[${i}].script \u4E0D\u80FD\u4E3A\u7A7A` };
+        actions.push({ type: "execute_js", script: a.script });
+        break;
+      default:
+        return { error: `actions[${i}] \u672A\u77E5\u7C7B\u578B\uFF1A${String(a.type)}\uFF08\u652F\u6301 wait/click/write/press/scroll/execute_js\uFF09` };
+    }
+  }
+  if (totalWaitMs > MAX_TOTAL_WAIT_MS) {
+    return { error: `wait \u603B\u65F6\u957F\u8D85\u9650\uFF08${totalWaitMs}ms > ${MAX_TOTAL_WAIT_MS}ms\uFF09` };
+  }
+  return { actions };
+}
+async function scrollPage(page, action) {
+  if (action.selector) {
+    await page.evaluate(
+      `document.querySelector(${JSON.stringify(action.selector)})?.scrollIntoView({ block: 'center', inline: 'nearest' })`
+    );
+    return;
+  }
+  switch (action.direction) {
+    case "top":
+      await page.evaluate("window.scrollTo(0, 0)");
+      break;
+    case "bottom":
+      await page.evaluate("window.scrollTo(0, document.body.scrollHeight)");
+      break;
+    case "up":
+      await page.evaluate("window.scrollBy(0, -window.innerHeight * 0.8)");
+      break;
+    default:
+      await page.evaluate("window.scrollBy(0, window.innerHeight * 0.8)");
+  }
+}
+async function executeRenderActions(page, actions) {
+  const results = [];
+  for (const action of actions) {
+    try {
+      switch (action.type) {
+        case "wait":
+          if ("ms" in action) await new Promise((r) => setTimeout(r, action.ms));
+          else await page.waitForSelector(action.selector, { timeout: ACTION_STEP_TIMEOUT_MS });
+          results.push({ type: "wait", ok: true });
+          break;
+        case "click":
+          if (action.all) {
+            await page.evaluate(
+              `document.querySelectorAll(${JSON.stringify(action.selector)}).forEach((el) => el.click())`
+            );
+          } else {
+            await page.click(action.selector, { timeout: ACTION_STEP_TIMEOUT_MS });
+          }
+          results.push({ type: "click", ok: true });
+          break;
+        case "write":
+          await page.fill(action.selector, action.text, { timeout: ACTION_STEP_TIMEOUT_MS });
+          results.push({ type: "write", ok: true });
+          break;
+        case "press":
+          if (action.selector) await page.press(action.selector, action.key, { timeout: ACTION_STEP_TIMEOUT_MS });
+          else if (page.keyboard) await page.keyboard.press(action.key);
+          results.push({ type: "press", ok: true });
+          break;
+        case "scroll":
+          await scrollPage(page, action);
+          results.push({ type: "scroll", ok: true });
+          break;
+        case "execute_js": {
+          const value = await page.evaluate(action.script);
+          let detail;
+          try {
+            detail = JSON.stringify(value) ?? String(value);
+          } catch {
+            detail = String(value);
+          }
+          if (detail.length > EXECUTE_JS_RESULT_LIMIT) {
+            detail = `${detail.slice(0, EXECUTE_JS_RESULT_LIMIT)}\u2026\uFF08\u622A\u65AD\uFF09`;
+          }
+          results.push({ type: "execute_js", ok: true, detail });
+          break;
+        }
+      }
+    } catch (err) {
+      const msg = err instanceof Error ? err.message.split("\n")[0] ?? err.message : String(err);
+      results.push({ type: action.type, ok: false, detail: msg });
+      break;
+    }
+  }
+  return results;
+}
+
+// src/tools/web-fetch/render-pool.ts
+init_esm_shims();
+var DEFAULT_IDLE_TIMEOUT_MS = 10 * 6e4;
+var DEFAULT_MAX_PAGES = 8;
+var RENDER_USER_AGENTS = [
+  "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/151.0.0.0 Safari/537.36",
+  "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/151.0.0.0 Safari/537.36",
+  "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/151.0.0.0 Safari/537.36"
+];
+function pickUserAgent() {
+  return RENDER_USER_AGENTS[Math.floor(Math.random() * RENDER_USER_AGENTS.length)];
+}
+var RenderPool = class {
+  browser = null;
+  launching = null;
+  activePages = 0;
+  idleTimer = null;
+  /** page → 其所属 context（每页独立 context：serviceWorkers 阻断 + UA/viewport 隔离） */
+  pageContexts = /* @__PURE__ */ new WeakMap();
+  idleTimeoutMs;
+  maxPages;
+  launch;
+  constructor(opts = {}) {
+    this.idleTimeoutMs = opts.idleTimeoutMs ?? DEFAULT_IDLE_TIMEOUT_MS;
+    this.maxPages = opts.maxPages ?? DEFAULT_MAX_PAGES;
+    this.launch = opts.launchBrowser ?? (() => launchHeadlessChromium({ proxy: opts.proxy }));
+  }
+  /** 当前活跃 Page 数（诊断与测试用）。 */
+  get pageCount() {
+    return this.activePages;
+  }
+  /** Browser 是否在跑（诊断与测试用）。 */
+  get isRunning() {
+    return this.browser !== null;
+  }
+  async acquirePage() {
+    if (this.activePages >= this.maxPages) {
+      throw new Error(`\u6E32\u67D3\u6C60 Page \u6570\u8D85\u9650\uFF08${this.maxPages}\uFF09`);
+    }
+    const browser = await this.ensureBrowser();
+    this.activePages += 1;
+    try {
+      const context = await browser.newContext({
+        serviceWorkers: "block",
+        viewport: { width: 1280, height: 800 },
+        userAgent: pickUserAgent()
+      });
+      const page = await context.newPage();
+      this.pageContexts.set(page, context);
+      this.disarmIdleTimer();
+      return page;
+    } catch (err) {
+      this.activePages -= 1;
+      throw err;
+    }
+  }
+  async releasePage(page) {
+    this.activePages = Math.max(0, this.activePages - 1);
+    const context = this.pageContexts.get(page);
+    try {
+      await (context ?? page).close();
+    } catch {
+    }
+    this.pageContexts.delete(page);
+    if (this.activePages === 0) this.armIdleTimer();
+  }
+  /** 立即关闭 Browser（空闲回收与进程退出共用）。 */
+  async closeBrowser() {
+    this.disarmIdleTimer();
+    const browser = this.browser;
+    this.browser = null;
+    if (browser) {
+      try {
+        await browser.close();
+      } catch {
+      }
+    }
+  }
+  async ensureBrowser() {
+    if (this.browser) {
+      if (this.browser.isConnected && !this.browser.isConnected()) {
+        this.browser = null;
+      } else {
+        return this.browser;
+      }
+    }
+    if (this.launching) return this.launching;
+    this.launching = this.launch().then((browser) => {
+      this.browser = browser;
+      this.launching = null;
+      browser.on("disconnected", (() => {
+        this.browser = null;
+      }));
+      return browser;
+    }).catch((err) => {
+      this.launching = null;
+      throw err;
+    });
+    return this.launching;
+  }
+  armIdleTimer() {
+    this.disarmIdleTimer();
+    this.idleTimer = setTimeout(() => {
+      void this.closeBrowser();
+    }, this.idleTimeoutMs);
+    this.idleTimer.unref?.();
+  }
+  disarmIdleTimer() {
+    if (this.idleTimer) {
+      clearTimeout(this.idleTimer);
+      this.idleTimer = null;
+    }
+  }
+};
+var defaultPool = null;
+var exitHookInstalled = false;
+function installExitHook() {
+  if (exitHookInstalled) return;
+  exitHookInstalled = true;
+  const cleanup = () => {
+    if (defaultPool) void defaultPool.closeBrowser().catch(() => {
+    });
+  };
+  process.once("exit", cleanup);
+  process.once("SIGINT", cleanup);
+  process.once("SIGTERM", cleanup);
+}
+function getDefaultRenderPool(opts = {}) {
+  if (!defaultPool) {
+    defaultPool = new RenderPool(opts);
+    installExitHook();
+  }
+  return defaultPool;
+}
+
+// src/tools/web-fetch/render-fetch.ts
+var DEFAULT_RENDER_TIMEOUT_MS = 3e4;
+var AD_SERVING_DOMAINS = [
+  "doubleclick.net",
+  "adservice.google.com",
+  "googlesyndication.com",
+  "googletagservices.com",
+  "googletagmanager.com",
+  "google-analytics.com",
+  "adsystem.com",
+  "adservice.com",
+  "adnxs.com",
+  "ads-twitter.com",
+  "facebook.net",
+  "fbcdn.net",
+  "amazon-adsystem.com"
+];
+function isAdHost(hostname) {
+  return AD_SERVING_DOMAINS.some((d) => hostname === d || hostname.endsWith("." + d));
+}
+function isLocalScheme(protocol) {
+  return protocol === "data:" || protocol === "blob:" || protocol === "about:";
+}
+async function fetchViaPlaywright(rawUrl, opts = {}) {
+  let target;
+  try {
+    target = new URL(rawUrl);
+  } catch {
+    return void 0;
+  }
+  const lookup = opts.lookup ?? ((hostname) => dnsLookup(hostname));
+  const timeoutMs = opts.timeoutMs ?? DEFAULT_RENDER_TIMEOUT_MS;
+  try {
+    await resolveAndAssertPublic(target.hostname, lookup);
+  } catch (err) {
+    if (err instanceof SSRFError) throw err;
+    return void 0;
+  }
+  const proxyServer = resolveProxyForUrl(rawUrl, opts.proxy);
+  const pool = opts.pool ?? getDefaultRenderPool(proxyServer ? { proxy: { server: proxyServer } } : {});
+  let page;
+  try {
+    page = await pool.acquirePage();
+  } catch {
+    return void 0;
+  }
+  let blockedRequests = 0;
+  let blockedAds = 0;
+  try {
+    await page.route("**/*", async (route, request) => {
+      const reqUrl = request.url();
+      let parsed;
+      try {
+        parsed = new URL(reqUrl);
+      } catch {
+        blockedRequests += 1;
+        await route.abort();
+        return;
+      }
+      if (parsed.protocol !== "http:" && parsed.protocol !== "https:") {
+        if (isLocalScheme(parsed.protocol)) {
+          await route.continue();
+          return;
+        }
+        blockedRequests += 1;
+        await route.abort();
+        return;
+      }
+      if (isAdHost(parsed.hostname)) {
+        blockedAds += 1;
+        await route.abort();
+        return;
+      }
+      try {
+        await resolveAndAssertPublic(parsed.hostname, lookup);
+        await route.continue();
+      } catch {
+        blockedRequests += 1;
+        await route.abort();
+      }
+    });
+    await page.goto(rawUrl, { waitUntil: "domcontentloaded", timeout: timeoutMs });
+    if (opts.waitMs && opts.waitMs > 0) {
+      await new Promise((r) => setTimeout(r, opts.waitMs));
+    }
+    let actionResults;
+    if (opts.actions && opts.actions.length > 0) {
+      actionResults = await executeRenderActions(page, opts.actions);
+    }
+    const finalUrl = page.url();
+    if (finalUrl.startsWith("http:") || finalUrl.startsWith("https:")) {
+      const finalHost = new URL(finalUrl).hostname;
+      if (finalHost !== target.hostname) {
+        await resolveAndAssertPublic(finalHost, lookup);
+      }
+    }
+    const html = await page.content();
+    const pageUrl = finalUrl.startsWith("http") ? finalUrl : rawUrl;
+    const markdown = await htmlToMarkdownSmart(html, {
+      pageUrl,
+      onlyMainContent: opts.extractMainContent !== false
+    });
+    return {
+      markdown,
+      blockedRequests,
+      blockedAds,
+      ...actionResults ? { actionResults } : {},
+      links: extractLinks(html, pageUrl)
+    };
+  } catch (err) {
+    if (err instanceof SSRFError) throw err;
+    return void 0;
+  } finally {
+    await pool.releasePage(page);
+  }
+}
+
+// src/tools/web-fetch/fetch-cache.ts
+init_esm_shims();
+import { createHash } from "crypto";
+import { mkdir, readFile, readdir, rm, writeFile } from "fs/promises";
+import { join } from "path";
+var DEFAULT_CACHE_MAX_AGE_MS = 2 * 24 * 60 * 6e4;
+var SWEEP_EVERY_N_WRITES = 20;
+function normalizeCacheUrl(rawUrl) {
+  try {
+    const u = new URL(rawUrl);
+    u.hostname = u.hostname.toLowerCase();
+    if (u.protocol === "https:" && u.port === "443" || u.protocol === "http:" && u.port === "80") {
+      u.port = "";
+    }
+    if (u.hash && !u.hash.startsWith("#/") && !u.hash.startsWith("#!/")) u.hash = "";
+    return u.toString();
+  } catch {
+    return rawUrl;
+  }
+}
+var FetchCache = class {
+  constructor(dir, opts = {}) {
+    this.dir = dir;
+    this.maxAgeMs = opts.maxAgeMs ?? DEFAULT_CACHE_MAX_AGE_MS;
+    this.now = opts.now ?? Date.now;
+  }
+  dir;
+  writesSinceSweep = 0;
+  maxAgeMs;
+  now;
+  key(url, variant) {
+    return createHash("sha256").update(`${normalizeCacheUrl(url)}
+${variant}`).digest("hex");
+  }
+  async read(url, variant) {
+    if (this.maxAgeMs === 0) return void 0;
+    try {
+      const raw = await readFile(join(this.dir, `${this.key(url, variant)}.json`), "utf8");
+      const entry = JSON.parse(raw);
+      if (typeof entry.fetchedAt !== "number" || typeof entry.markdown !== "string") return void 0;
+      if (entry.fetchedAt + this.maxAgeMs < this.now()) return void 0;
+      return entry;
+    } catch {
+      return void 0;
+    }
+  }
+  async write(url, variant, entry) {
+    try {
+      await mkdir(this.dir, { recursive: true });
+      const full = { ...entry, fetchedAt: this.now() };
+      await writeFile(join(this.dir, `${this.key(url, variant)}.json`), JSON.stringify(full));
+      this.writesSinceSweep += 1;
+      if (this.writesSinceSweep >= SWEEP_EVERY_N_WRITES) {
+        this.writesSinceSweep = 0;
+        void this.sweep().catch(() => {
+        });
+      }
+    } catch {
+    }
+  }
+  /** 清理过期条目（按 entry.fetchedAt 判定，与 read 的 TTL 语义严格一致；损坏条目一并清理）。 */
+  async sweep() {
+    const now = this.now();
+    let names;
+    try {
+      names = await readdir(this.dir);
+    } catch {
+      return;
+    }
+    for (const name of names) {
+      if (!name.endsWith(".json")) continue;
+      const path = join(this.dir, name);
+      try {
+        const raw = await readFile(path, "utf8");
+        const entry = JSON.parse(raw);
+        if (typeof entry.fetchedAt !== "number" || entry.fetchedAt + this.maxAgeMs < now) {
+          await rm(path, { force: true });
+        }
+      } catch {
+        await rm(path, { force: true }).catch(() => {
+        });
+      }
+    }
+  }
+};
+function fetchCacheDir(cwd) {
+  return join(cwd, ".rivet", "cache", "web-fetch");
+}
+var caches = /* @__PURE__ */ new Map();
+function getFetchCache(cwd, opts = {}) {
+  const dir = fetchCacheDir(cwd);
+  let cache = caches.get(dir);
+  if (!cache) {
+    cache = new FetchCache(dir, opts);
+    caches.set(dir, cache);
+  }
+  return cache;
+}
+function formatCacheAge(fetchedAt, now = Date.now()) {
+  const minutes = Math.max(1, Math.round((now - fetchedAt) / 6e4));
+  if (minutes < 60) return `${minutes} \u5206\u949F`;
+  const hours = Math.round(minutes / 60);
+  if (hours < 24) return `${hours} \u5C0F\u65F6`;
+  return `${Math.round(hours / 24)} \u5929`;
+}
+
+// src/tools/web-fetch/fetch-core.ts
+var DEFAULT_TIMEOUT_MS = 15e3;
+var DEFAULT_MAX_BYTES = 10485760;
+var DEFAULT_MAX_REDIRECTS = 5;
+var DEFAULT_USER_AGENT = "Tianshu/1.0 (terminal coding agent)";
+var BINARY_CONTENT_TYPE_PREFIXES = [
+  "image/",
+  "application/pdf",
+  "application/octet-stream",
+  "video/",
+  "audio/",
+  "font/"
+];
+function httpApiErrorKind(status) {
+  if (status === 429 || status === 500 || status === 502 || status === 503) return "api_error";
+  return void 0;
+}
+async function fetchMarkdown(rawUrl, deps, opts) {
+  let url;
+  try {
+    url = new URL(rawUrl);
+  } catch {
+    return { ok: false, error: `\u65E0\u6548 URL\uFF1A${rawUrl}` };
+  }
+  if (url.protocol !== "http:" && url.protocol !== "https:") {
+    return { ok: false, error: `\u4E0D\u652F\u6301\u7684\u534F\u8BAE\uFF1A${url.protocol}\u3002\u4EC5\u5141\u8BB8 http \u548C https\u3002` };
+  }
+  const options = {
+    timeoutMs: opts.timeoutMs ?? DEFAULT_TIMEOUT_MS,
+    maxResponseBytes: opts.maxResponseBytes ?? DEFAULT_MAX_BYTES,
+    maxRedirects: opts.maxRedirects ?? DEFAULT_MAX_REDIRECTS,
+    userAgent: opts.userAgent ?? DEFAULT_USER_AGENT
+  };
+  const extractMainContentEnabled = opts.extractMainContent ?? true;
+  const renderFetch = deps.renderFetch ?? (opts.enablePlaywright ? (u) => fetchViaPlaywright(u, {
+    timeoutMs: opts.renderTimeoutMs,
+    waitMs: opts.renderWaitMs,
+    lookup: deps.lookup,
+    extractMainContent: extractMainContentEnabled
+  }) : void 0);
+  const cache = deps.cache ?? getFetchCache(opts.cwd, { maxAgeMs: opts.cacheMaxAgeMs });
+  const cacheVariant = `e${extractMainContentEnabled ? 1 : 0}`;
+  const cached = await cache.read(rawUrl, cacheVariant);
+  if (cached) {
+    return {
+      ok: true,
+      status: cached.status,
+      markdown: cached.markdown,
+      via: cached.via,
+      links: extractLinksFromMarkdown(cached.markdown),
+      rawBytes: cached.markdown.length,
+      fromCache: true,
+      fetchedAt: cached.fetchedAt
+    };
+  }
+  const writeCache = async (status, markdown, via) => {
+    if (markdown.trim().length < MIN_SUBSTANTIAL_LENGTH) return;
+    await cache.write(rawUrl, cacheVariant, { url: rawUrl, markdown, via, status });
+  };
+  try {
+    const { status, contentType, bytes } = await httpFetchGuarded(rawUrl, deps, options);
+    const contentTypeLower = contentType.toLowerCase();
+    if (status >= 400) {
+      if (contentTypeLower.includes("text/html")) {
+        const body2 = decodeBody(bytes, contentType);
+        const md = await htmlToMarkdownSmart(body2, {
+          pageUrl: rawUrl,
+          onlyMainContent: extractMainContentEnabled
+        });
+        if (md.trim().length >= MIN_SUBSTANTIAL_LENGTH) {
+          await writeCache(status, md, "");
+          return {
+            ok: true,
+            status,
+            markdown: md,
+            via: "",
+            links: extractLinks(body2, rawUrl),
+            rawBytes: bytes.length,
+            fromCache: false
+          };
+        }
+      }
+      return {
+        ok: false,
+        error: `HTTP ${status}\uFF1A${rawUrl}`,
+        ...httpApiErrorKind(status) ? { errorKind: "api_error" } : {}
+      };
+    }
+    if (BINARY_CONTENT_TYPE_PREFIXES.some((prefix) => contentTypeLower.includes(prefix))) {
+      return {
+        ok: false,
+        error: `\u4E8C\u8FDB\u5236\u5185\u5BB9\uFF08${contentType}\uFF09\u4E0D\u4F1A\u4EE5\u6587\u672C\u8FD4\u56DE\u3002\u8BF7\u4F7F\u7528 import_resource \u4E0B\u8F7D\u6B64 URL\u3002`
+      };
+    }
+    const body = decodeBody(bytes, contentType);
+    let content;
+    let via = "";
+    let links = [];
+    if (contentTypeLower.includes("text/html")) {
+      links = extractLinks(body, rawUrl);
+      content = await htmlToMarkdownSmart(body, {
+        pageUrl: rawUrl,
+        onlyMainContent: extractMainContentEnabled
+      });
+      if (isJinaQualityHeuristic(content)) {
+        if (renderFetch) {
+          const rendered = await renderFetch(rawUrl);
+          if (rendered && rendered.markdown.trim().length >= MIN_SUBSTANTIAL_LENGTH) {
+            content = rendered.markdown;
+            via = "\uFF08\u7ECF Playwright \u6E32\u67D3\uFF09";
+            links = rendered.links ?? extractLinksFromMarkdown(content);
+          }
+        }
+        if (!via) {
+          const jinaResult = await fetchViaJina(rawUrl, deps, {
+            ...options,
+            ...opts.jinaBaseUrl ? { jinaBaseUrl: opts.jinaBaseUrl } : {}
+          });
+          if (jinaResult) {
+            content = jinaResult.markdown;
+            via = "\uFF08\u7ECF Jina Reader\uFF09";
+            links = extractLinksFromMarkdown(content);
+          }
+        }
+      }
+    } else {
+      content = body;
+    }
+    await writeCache(status, content, via);
+    return {
+      ok: true,
+      status,
+      markdown: content,
+      via,
+      links,
+      rawBytes: bytes.length,
+      fromCache: false
+    };
+  } catch (err) {
+    if (err instanceof SSRFError) {
+      return { ok: false, error: err.message };
+    }
+    const message = err instanceof Error ? err.message : String(err);
+    const detail = fetchCauseDetail(err);
+    const full = detail ? `${message}: ${detail}` : message;
+    return { ok: false, error: `\u6293\u53D6\u5931\u8D25 ${rawUrl}\uFF1A${full}` };
+  }
+}
+
+export {
+  MIN_SUBSTANTIAL_LENGTH,
+  decodeBody,
+  parseRenderActions,
+  fetchViaPlaywright,
+  normalizeCacheUrl,
+  formatCacheAge,
+  fetchMarkdown
+};