npm - pi-smart-fetch - Versions diffs - 0.1.7 → 0.1.8 - Mend

pi-smart-fetch 0.1.7 → 0.1.8

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Files changed (4) hide show

package/README.md CHANGED Viewed

@@ -9,7 +9,7 @@ Compared with naive Node.js `fetch()`, this package gives you:
 - **clean readable extraction** via `Defuddle`, so agents get article content instead of raw noisy HTML
 - **better success on bot-defended pages** where plain server-side requests are blocked, challenged, or degraded
 - **useful metadata** like title, author, published date, site, and language when available
-- **multiple output formats**: `markdown`, `html`, or `text`
+- **multiple output formats**: `markdown`, `html`, `text`, or `json`
 - **the familiar pi tool name**: it registers `web_fetch`
 - **pi-specific behavior** including an optional `verbose` flag and defaults from pi settings
 - **lower overhead than browser automation** when you do not need JS execution, login, scrolling, or clicks
@@ -103,7 +103,7 @@ Error: Invalid URL: not-a-url
 | `os`              | string                        | `windows`       | OS profile: `windows`, `macos`, `linux`, `android`, `ios`                   |
 | `headers`         | object                        | auto            | Extra request headers                                                        |
 | `maxChars`        | number                        | `50000`         | Maximum returned characters. Can be overridden by pi settings                |
-| `format`          | `markdown` \| `html` \| `text` | `markdown`      | Output format                                                                |
+| `format`          | `markdown` \| `html` \| `text` \| `json` | `markdown`      | Output format                                                                |
 | `removeImages`    | boolean                       | `false`         | Strip image references from output                                           |
 | `includeReplies`  | boolean \| `extractors`       | `extractors`    | Include replies/comments                                                     |
 | `proxy`           | string                        | none            | Proxy URL                                                                    |

package/dist/index.js CHANGED Viewed

@@ -15,6 +15,7 @@ var DEFAULT_MAX_CHARS = 5e4;
 var DEFAULT_TIMEOUT_MS = 15e3;
 var DEFAULT_INCLUDE_REPLIES = "extractors";
 var DEFAULT_ACCEPT_HEADER = "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8";
+var DEFAULT_JSON_ACCEPT_HEADER = "application/json,text/json,application/ld+json;q=0.9,text/plain;q=0.8,*/*;q=0.7";
 var DEFAULT_ACCEPT_LANGUAGE_HEADER = "en-US,en;q=0.9";
 var runtimeDependencies = {
   fetch: fetch,
@@ -78,6 +79,41 @@ function buildFetchResponseText(result, options = {}) {
 ${result.content}` : result.content;
 }
+function estimateWordCount(content) {
+  const words = content.trim().match(/\S+/g);
+  return words?.length ?? 0;
+}
+function escapeHtml(value) {
+  return value.replace(/&/g, "&amp;").replace(/</g, "&lt;").replace(/>/g, "&gt;").replace(/"/g, "&quot;").replace(/'/g, "&#39;");
+}
+function parseAndFormatJson(raw) {
+  try {
+    return {
+      formatted: JSON.stringify(JSON.parse(raw), null, 2)
+    };
+  } catch {
+    return { error: "Invalid JSON response" };
+  }
+}
+function renderJsonContent(formattedJson, format) {
+  switch (format) {
+    case "json":
+    case "text":
+      return formattedJson;
+    case "html":
+      return `<pre><code class="language-json">${escapeHtml(formattedJson)}</code></pre>`;
+    default:
+      return `\`\`\`json
+${formattedJson}
+\`\`\``;
+  }
+}
+function stripExtractorComments(content, format) {
+  if (format === "html") {
+    return content.replace(/\s*<hr>\s*<div class="[^"]* comments">[\s\S]*$/i, "").trimEnd();
+  }
+  return content.replace(/\n---\n+## Comments\n[\s\S]*$/i, "").trimEnd();
+}
 // ../core/src/extract.ts
 var HTML_CONTENT_TYPES = [
@@ -86,6 +122,46 @@ var HTML_CONTENT_TYPES = [
   "text/plain",
   "text/markdown"
 ];
+function resolveAcceptHeader(format) {
+  return format === "json" ? DEFAULT_JSON_ACCEPT_HEADER : DEFAULT_ACCEPT_HEADER;
+}
+function isJsonContentType(contentType) {
+  const normalized = contentType.split(";")[0]?.trim().toLowerCase() ?? "";
+  return normalized === "application/json" || normalized === "text/json" || normalized.endsWith("+json");
+}
+function isLikelyJsonBody(body) {
+  const trimmed = body.trim();
+  return trimmed.startsWith("{") || trimmed.startsWith("[");
+}
+function isJsonResponse(contentType, body) {
+  return isJsonContentType(contentType) || isLikelyJsonBody(body);
+}
+function buildJsonResult(opts, finalUrl, rawBody, format, maxChars, browser, os) {
+  const parsedJson = parseAndFormatJson(rawBody);
+  if ("error" in parsedJson) {
+    return parsedJson;
+  }
+  const content = truncateContent(
+    renderJsonContent(parsedJson.formatted, format),
+    maxChars
+  );
+  return {
+    url: opts.url,
+    finalUrl,
+    title: "",
+    author: "",
+    published: "",
+    site: new URL(finalUrl).hostname,
+    language: "",
+    wordCount: estimateWordCount(parsedJson.formatted),
+    content,
+    browser,
+    os
+  };
+}
+function shouldStripReplies(site) {
+  return site === "Hacker News" || site.startsWith("r/") || site.startsWith("GitHub - ");
+}
 function createDefuddleFetch(dependencies = runtimeDependencies) {
   return async function defuddleFetch2(opts) {
     const browser = opts.browser ?? DEFAULT_BROWSER;
@@ -110,7 +186,7 @@ function createDefuddleFetch(dependencies = runtimeDependencies) {
       browser,
       os,
       headers: {
-        Accept: DEFAULT_ACCEPT_HEADER,
+        Accept: resolveAcceptHeader(format),
         "Accept-Language": DEFAULT_ACCEPT_LANGUAGE_HEADER,
         ...opts.headers
       },
@@ -128,11 +204,37 @@ function createDefuddleFetch(dependencies = runtimeDependencies) {
     }
     const finalUrl = response.url ?? opts.url;
     const contentType = response.headers.get("content-type") ?? "";
+    const rawBody = await response.text();
+    const jsonResponse = isJsonResponse(contentType, rawBody);
+    if (format === "json") {
+      if (!jsonResponse) {
+        return { error: `Not a JSON response (content-type: ${contentType})` };
+      }
+      return buildJsonResult(
+        opts,
+        finalUrl,
+        rawBody,
+        format,
+        maxChars,
+        browser,
+        os
+      );
+    }
+    if (jsonResponse) {
+      return buildJsonResult(
+        opts,
+        finalUrl,
+        rawBody,
+        format,
+        maxChars,
+        browser,
+        os
+      );
+    }
     if (!HTML_CONTENT_TYPES.some((value) => contentType.includes(value))) {
       return { error: `Not an HTML page (content-type: ${contentType})` };
     }
-    const html = await response.text();
-    const document = parseLinkedomHTML(html, finalUrl);
+    const document = parseLinkedomHTML(rawBody, finalUrl);
     const extracted = await dependencies.defuddle(document, finalUrl, {
       markdown: format !== "html",
       removeImages,
@@ -143,7 +245,18 @@ function createDefuddleFetch(dependencies = runtimeDependencies) {
         error: `No content extracted from ${opts.url}. May need JS rendering or is blocked.`
       };
     }
-    const normalizedContent = format === "text" ? markdownToText(extracted.content) : extracted.content;
+    let extractedContent = extracted.content;
+    let wordCount = extracted.wordCount;
+    if (includeReplies === false && shouldStripReplies(extracted.site ?? "")) {
+      const strippedContent = stripExtractorComments(extractedContent, format);
+      if (strippedContent !== extractedContent) {
+        extractedContent = strippedContent;
+        wordCount = estimateWordCount(
+          format === "text" ? markdownToText(extractedContent) : extractedContent
+        );
+      }
+    }
+    const normalizedContent = format === "text" ? markdownToText(extractedContent) : extractedContent;
     return {
       url: opts.url,
       finalUrl,
@@ -152,7 +265,7 @@ function createDefuddleFetch(dependencies = runtimeDependencies) {
       published: extracted.published ?? "",
       site: extracted.site ?? "",
       language: extracted.language ?? "",
-      wordCount: extracted.wordCount,
+      wordCount,
       content: truncateContent(normalizedContent, maxChars),
       browser,
       os
@@ -198,9 +311,14 @@ function createBaseFetchToolParameterProperties(defaults) {
     ),
     format: Type.Optional(
       Type.Union(
-        [Type.Literal("markdown"), Type.Literal("html"), Type.Literal("text")],
+        [
+          Type.Literal("markdown"),
+          Type.Literal("html"),
+          Type.Literal("text"),
+          Type.Literal("json")
+        ],
         {
-          description: 'Output format. "markdown" (default), "html" (cleaned HTML), or "text" (plain text, no formatting)'
+          description: 'Output format. "markdown" (default), "html" (cleaned HTML), "text" (plain text, no formatting), or "json" (pretty-printed JSON)'
         }
       )
     ),

package/dist/index.js.map CHANGED Viewed

	@@ -1 +1 @@
1	- {"version":3,"sources":["../../core/src/constants.ts","../../core/src/dependencies.ts","../../core/src/dom.ts","../../core/src/format.ts","../../core/src/extract.ts","../../core/src/tool.ts","../src/settings.ts","../src/index.ts"],"names":["wreqFetch","defuddleFetch","Type","getAgentDir"],"mappings":";;;;;;;;;;;AAEO,IAAM,eAAA,GAAkB,YAAA;AACxB,IAAM,UAAA,GAA4B,SAAA;AAClC,IAAM,iBAAA,GAAoB,GAAA;AAC1B,IAAM,kBAAA,GAAqB,IAAA;AAC3B,IAAM,uBAAA,GAA0B,YAAA;AAChC,IAAM,qBAAA,GACX,iEAAA;AACK,IAAM,8BAAA,GAAiC,gBAAA;ACLvC,IAAM,mBAAA,GAAyC;AAAA,EACpD,KAAA,EAAOA,KAAA;AAAA,EACP,QAAA,EAAU,QAAA;AAAA,EACV;AACF,CAAA;ACLO,SAAS,iBAAA,CAAkB,MAAc,GAAA,EAAwB;AACtE,EAAA,MAAM,EAAE,QAAA,EAAS,GAAI,SAAA,CAAU,IAAI,CAAA;AACnC,EAAA,MAAM,GAAA,GAAM,QAAA;AACZ,EAAA,MAAM,cAAc,GAAA,CAAI,WAAA;AASxB,EAAA,IAAI,CAAE,IAAkC,WAAA,EAAa;AACnD,IAAC,GAAA,CAAkC,cACjC,EAAC;AAAA,EACL;AAEA,EAAA,IAAI,WAAA,IAAe,CAAC,WAAA,CAAY,gBAAA,EAAkB;AAChD,IAAA,WAAA,CAAY,oBAAoB,OAAO;AAAA,MACrC,OAAA,EAAS;AAAA,KACX,CAAA,CAAA;AAAA,EACF;AAEA,EAAA,IAAI,GAAA,EAAK;AACP,IAAC,IAAyB,GAAA,GAAM,GAAA;AAAA,EAClC;AAEA,EAAA,OAAO,QAAA;AACT;;;AC7BA,SAAS,YACP,KAAA,EACA;AACA,EAAA,OAAO,KAAA,CACJ,MAAA,CAAO,CAAC,GAAG,KAAK,CAAA,KAAM,KAAA,KAAU,MAAA,IAAa,KAAA,KAAU,EAAE,CAAA,CACzD,IAAI,CAAC,CAAC,KAAA,EAAO,KAAK,CAAA,KAAM,CAAA,EAAA,EAAK,KAAK,CAAA,EAAA,EAAK,KAAK,CAAA,CAAE,CAAA,CAC9C,IAAA,CAAK,IAAI,CAAA;AACd;AAEO,SAAS,eAAe,QAAA,EAA0B;AACvD,EAAA,OAAO,QAAA,CACJ,OAAA,CAAQ,cAAA,EAAgB,EAAE,CAAA,CAC1B,OAAA,CAAQ,kBAAA,EAAoB,IAAI,CAAA,CAChC,OAAA,CAAQ,cAAA,EAAgB,IAAI,EAC5B,OAAA,CAAQ,wBAAA,EAA0B,IAAI,CAAA,CACtC,OAAA,CAAQ,uBAAA,EAAyB,EAAE,CAAA,CACnC,QAAQ,SAAA,EAAW,EAAE,CAAA,CACrB,OAAA,CAAQ,aAAA,EAAe,SAAI,CAAA,CAC3B,OAAA,CAAQ,cAAc,IAAI,CAAA;AAC/B;AAEO,SAAS,eAAA,CAAgB,SAAiB,QAAA,EAA0B;AACzE,EAAA,IAAI,OAAA,CAAQ,MAAA,IAAU,QAAA,EAAU,OAAO,OAAA;AACvC,EAAA,OAAO,CAAA,EAAG,OAAA,CAAQ,KAAA,CAAM,CAAA,EAAG,QAAQ,CAAC;;AAAA,eAAA,CAAA;AACtC;AAEO,SAAS,2BAA2B,MAAA,EAA6B;AACtE,EAAA,OAAO,WAAA,CAAY;AAAA,IACjB,CAAC,KAAA,EAAO,MAAA,CAAO,QAAQ,CAAA;AAAA,IACvB,CAAC,OAAA,EAAS,MAAA,CAAO,KAAK,CAAA;AAAA,IACtB,CAAC,QAAA,EAAU,MAAA,CAAO,MAAM,CAAA;AAAA,IACxB,CAAC,WAAA,EAAa,MAAA,CAAO,SAAS;AAAA,GAC/B,CAAA;AACH;AAEO,SAAS,oBAAoB,MAAA,EAA6B;AAC/D,EAAA,OAAO,WAAA,CAAY;AAAA,IACjB,CAAC,KAAA,EAAO,MAAA,CAAO,QAAQ,CAAA;AAAA,IACvB,CAAC,OAAA,EAAS,MAAA,CAAO,KAAK,CAAA;AAAA,IACtB,CAAC,QAAA,EAAU,MAAA,CAAO,MAAM,CAAA;AAAA,IACxB,CAAC,WAAA,EAAa,MAAA,CAAO,SAAS,CAAA;AAAA,IAC9B,CAAC,MAAA,EAAQ,MAAA,CAAO,IAAI,CAAA;AAAA,IACpB,CAAC,UAAA,EAAY,MAAA,CAAO,QAAQ,CAAA;AAAA,IAC5B,CAAC,OAAA,EAAS,MAAA,CAAO,SAAS,CAAA;AAAA,IAC1B,CAAC,WAAW,CAAA,EAAG,MAAA,CAAO,OAAO,CAAA,CAAA,EAAI,MAAA,CAAO,EAAE,CAAA,CAAE;AAAA,GAC7C,CAAA;AACH;AAEO,SAAS,sBAAA,CACd,MAAA,EACA,OAAA,GAAiC,EAAC,EAC1B;AACR,EAAA,MAAM,SAAS,OAAA,CAAQ,OAAA,GACnB,oBAAoB,MAAM,CAAA,GAC1B,2BAA2B,MAAM,CAAA;AAErC,EAAA,OAAO,MAAA,GAAS,GAAG,MAAM;;AAAA,EAAO,MAAA,CAAO,OAAO,CAAA,CAAA,GAAK,MAAA,CAAO,OAAA;AAC5D;;;ACnBA,IAAM,kBAAA,GAAqB;AAAA,EACzB,WAAA;AAAA,EACA,uBAAA;AAAA,EACA,YAAA;AAAA,EACA;AACF,CAAA;AAMO,SAAS,mBAAA,CACd,eAAkC,mBAAA,EAClC;AACA,EAAA,OAAO,eAAeC,eACpB,IAAA,EACmC;AACnC,IAAA,MAAM,OAAA,GAAU,KAAK,OAAA,IAAW,eAAA;AAChC,IAAA,MAAM,EAAA,GAAK,KAAK,EAAA,IAAM,UAAA;AACtB,IAAA,MAAM,MAAA,GAAuB,KAAK,MAAA,IAAU,UAAA;AAC5C,IAAA,MAAM,QAAA,GAAW,KAAK,QAAA,IAAY,iBAAA;AAClC,IAAA,MAAM,YAAA,GAAe,KAAK,YAAA,IAAgB,KAAA;AAC1C,IAAA,MAAM,cAAA,GAAiB,KAAK,cAAA,IAAkB,uBAAA;AAC9C,IAAA,MAAM,SAAA,GAAY,KAAK,SAAA,IAAa,kBAAA;AAEpC,IAAA,IAAI,MAAA;AACJ,IAAA,IAAI;AACF,MAAA,MAAA,GAAS,IAAI,GAAA,CAAI,IAAA,CAAK,GAAG,CAAA;AAAA,IAC3B,CAAA,CAAA,MAAQ;AACN,MAAA,OAAO,EAAE,KAAA,EAAO,CAAA,aAAA,EAAgB,IAAA,CAAK,GAAG,CAAA,CAAA,EAAG;AAAA,IAC7C;AAEA,IAAA,IAAI,CAAC,CAAC,OAAA,EAAS,QAAQ,EAAE,QAAA,CAAS,MAAA,CAAO,QAAQ,CAAA,EAAG;AAClD,MAAA,OAAO;AAAA,QACL,KAAA,EAAO,CAAA,oCAAA,EAAuC,MAAA,CAAO,QAAQ,CAAA;AAAA,OAC/D;AAAA,IACF;AAEA,IAAA,MAAM,YAAA,GAAwC;AAAA,MAC5C,OAAA;AAAA,MACA,EAAA;AAAA,MACA,OAAA,EAAS;AAAA,QACP,MAAA,EAAQ,qBAAA;AAAA,QACR,iBAAA,EAAmB,8BAAA;AAAA,QACnB,GAAG,IAAA,CAAK;AAAA,OACV;AAAA,MACA,QAAA,EAAU,QAAA;AAAA,MACV,OAAA,EAAS;AAAA,KACX;AAEA,IAAA,IAAI,KAAK,KAAA,EAAO;AACd,MAAA,YAAA,CAAa,QAAQ,IAAA,CAAK,KAAA;AAAA,IAC5B;AAEA,IAAA,MAAM,WAAW,MAAM,YAAA,CAAa,KAAA,CAAM,IAAA,CAAK,KAAK,YAAY,CAAA;AAEhE,IAAA,IAAI,CAAC,SAAS,EAAA,EAAI;AAChB,MAAA,OAAO;AAAA,QACL,KAAA,EAAO,QAAQ,QAAA,CAAS,MAAM,IAAI,QAAA,CAAS,UAAU,CAAA,KAAA,EAAQ,IAAA,CAAK,GAAG,CAAA;AAAA,OACvE;AAAA,IACF;AAEA,IAAA,MAAM,QAAA,GAAW,QAAA,CAAS,GAAA,IAAO,IAAA,CAAK,GAAA;AACtC,IAAA,MAAM,WAAA,GAAc,QAAA,CAAS,OAAA,CAAQ,GAAA,CAAI,cAAc,CAAA,IAAK,EAAA;AAE5D,IAAA,IAAI,CAAC,mBAAmB,IAAA,CAAK,CAAC,UAAU,WAAA,CAAY,QAAA,CAAS,KAAK,CAAC,CAAA,EAAG;AACpE,MAAA,OAAO,EAAE,KAAA,EAAO,CAAA,gCAAA,EAAmC,WAAW,CAAA,CAAA,CAAA,EAAI;AAAA,IACpE;AAEA,IAAA,MAAM,IAAA,GAAO,MAAM,QAAA,CAAS,IAAA,EAAK;AACjC,IAAA,MAAM,QAAA,GAAW,iBAAA,CAAkB,IAAA,EAAM,QAAQ,CAAA;AACjD,IAAA,MAAM,SAAA,GAAY,MAAM,YAAA,CAAa,QAAA,CAAS,UAAU,QAAA,EAAU;AAAA,MAChE,UAAU,MAAA,KAAW,MAAA;AAAA,MACrB,YAAA;AAAA,MACA;AAAA,KACD,CAAA;AAED,IAAA,IAAI,CAAC,SAAA,CAAU,OAAA,IAAW,SAAA,CAAU,cAAc,CAAA,EAAG;AACnD,MAAA,OAAO;AAAA,QACL,KAAA,EAAO,CAAA,0BAAA,EAA6B,IAAA,CAAK,GAAG,CAAA,sCAAA;AAAA,OAC9C;AAAA,IACF;AAEA,IAAA,MAAM,oBACJ,MAAA,KAAW,MAAA,GAAS,eAAe,SAAA,CAAU,OAAO,IAAI,SAAA,CAAU,OAAA;AAEpE,IAAA,OAAO;AAAA,MACL,KAAK,IAAA,CAAK,GAAA;AAAA,MACV,QAAA;AAAA,MACA,KAAA,EAAO,UAAU,KAAA,IAAS,EAAA;AAAA,MAC1B,MAAA,EAAQ,UAAU,MAAA,IAAU,EAAA;AAAA,MAC5B,SAAA,EAAW,UAAU,SAAA,IAAa,EAAA;AAAA,MAClC,IAAA,EAAM,UAAU,IAAA,IAAQ,EAAA;AAAA,MACxB,QAAA,EAAU,UAAU,QAAA,IAAY,EAAA;AAAA,MAChC,WAAW,SAAA,CAAU,SAAA;AAAA,MACrB,OAAA,EAAS,eAAA,CAAgB,iBAAA,EAAmB,QAAQ,CAAA;AAAA,MACpD,OAAA;AAAA,MACA;AAAA,KACF;AAAA,EACF,CAAA;AACF;AAEO,IAAM,gBAAgB,mBAAA,EAAoB;AAG1C,SAAS,QACd,MAAA,EACsB;AACtB,EAAA,OAAO,OAAA,IAAW,MAAA;AACpB;ACrIO,SAAS,wBAAA,CACd,MAAA,GAA0B,EAAC,EACR;AACnB,EAAA,OAAO;AAAA,IACL,QAAA,EAAU,OAAO,QAAA,IAAY,iBAAA;AAAA,IAC7B,SAAA,EAAW,OAAO,SAAA,IAAa,kBAAA;AAAA,IAC/B,OAAA,EAAS,OAAO,OAAA,IAAW,eAAA;AAAA,IAC3B,EAAA,EAAI,OAAO,EAAA,IAAM,UAAA;AAAA,IACjB,YAAA,EAAc,OAAO,YAAA,IAAgB,KAAA;AAAA,IACrC,cAAA,EAAgB,OAAO,cAAA,IAAkB;AAAA,GAC3C;AACF;AAEO,SAAS,uCACd,QAAA,EACyB;AACzB,EAAA,OAAO;AAAA,IACL,KAAK,IAAA,CAAK,MAAA,CAAO,EAAE,WAAA,EAAa,kCAAkC,CAAA;AAAA,IAClE,SAAS,IAAA,CAAK,QAAA;AAAA,MACZ,KAAK,MAAA,CAAO;AAAA,QACV,WAAA,EAAa,CAAA,kDAAA,EAAqD,QAAA,CAAS,OAAO,CAAA,oEAAA;AAAA,OACnF;AAAA,KACH;AAAA,IACA,IAAI,IAAA,CAAK,QAAA;AAAA,MACP,KAAK,MAAA,CAAO;AAAA,QACV,WAAA,EAAa,CAAA,yCAAA,EAA4C,QAAA,CAAS,EAAE,CAAA,+CAAA;AAAA,OACrE;AAAA,KACH;AAAA,IACA,SAAS,IAAA,CAAK,QAAA;AAAA,MACZ,KAAK,MAAA,CAAO,IAAA,CAAK,QAAO,EAAG,IAAA,CAAK,QAAO,EAAG;AAAA,QACxC,WAAA,EACE;AAAA,OACH;AAAA,KACH;AAAA,IACA,UAAU,IAAA,CAAK,QAAA;AAAA,MACb,KAAK,MAAA,CAAO;AAAA,QACV,WAAA,EAAa,CAAA,uCAAA,EAA0C,QAAA,CAAS,QAAQ,CAAA;AAAA,OACzE;AAAA,KACH;AAAA,IACA,QAAQ,IAAA,CAAK,QAAA;AAAA,MACX,IAAA,CAAK,KAAA;AAAA,QACH,CAAC,IAAA,CAAK,OAAA,CAAQ,UAAU,CAAA,EAAG,IAAA,CAAK,OAAA,CAAQ,MAAM,CAAA,EAAG,IAAA,CAAK,OAAA,CAAQ,MAAM,CAAC,CAAA;AAAA,QACrE;AAAA,UACE,WAAA,EACE;AAAA;AACJ;AACF,KACF;AAAA,IACA,cAAc,IAAA,CAAK,QAAA;AAAA,MACjB,KAAK,OAAA,CAAQ;AAAA,QACX,WAAA,EAAa;AAAA,OACd;AAAA,KACH;AAAA,IACA,gBAAgB,IAAA,CAAK,QAAA;AAAA,MACnB,IAAA,CAAK,KAAA,CAAM,CAAC,IAAA,CAAK,OAAA,IAAW,IAAA,CAAK,OAAA,CAAQ,YAAY,CAAC,CAAA,EAAG;AAAA,QACvD,WAAA,EACE;AAAA,OACH;AAAA,KACH;AAAA,IACA,OAAO,IAAA,CAAK,QAAA;AAAA,MACV,KAAK,MAAA,CAAO;AAAA,QACV,WAAA,EACE;AAAA,OACH;AAAA;AACH,GACF;AACF;AAEA,eAAsB,oBAAA,CACpB,QACA,QAAA,EACmC;AACnC,EAAA,OAAO,aAAA,CAAc;AAAA,IACnB,KAAK,MAAA,CAAO,GAAA;AAAA,IACZ,OAAA,EAAU,MAAA,CAAO,OAAA,IAAsB,QAAA,CAAS,OAAA;AAAA,IAChD,EAAA,EAAK,MAAA,CAAO,EAAA,IAAiB,QAAA,CAAS,EAAA;AAAA,IACtC,SAAS,MAAA,CAAO,OAAA;AAAA,IAChB,QAAA,EAAW,MAAA,CAAO,QAAA,IAAuB,QAAA,CAAS,QAAA;AAAA,IAClD,MAAA,EAAS,OAAO,MAAA,IAA2C,UAAA;AAAA,IAC3D,YAAA,EAAe,MAAA,CAAO,YAAA,IAA4B,QAAA,CAAS,YAAA;AAAA,IAC3D,cAAA,EACG,MAAA,CAAO,cAAA,IACR,QAAA,CAAS,cAAA;AAAA,IACX,OAAO,MAAA,CAAO,KAAA;AAAA,IACd,WAAW,QAAA,CAAS;AAAA,GACrB,CAAA;AACH;AC7FA,IAAM,eAAA,uBAAsB,GAAA,CAAmB;AAAA,EAC7C,SAAA;AAAA,EACA,OAAA;AAAA,EACA,OAAA;AAAA,EACA,SAAA;AAAA,EACA;AACF,CAAC,CAAA;AAgBD,SAAS,WAAA,CACP,QACA,IAAA,EACqB;AACrB,EAAA,KAAA,MAAW,OAAO,IAAA,EAAM;AACtB,IAAA,IAAI,OAAO,MAAA,CAAO,GAAG,CAAA,KAAM,SAAA,EAAW;AACpC,MAAA,OAAO,OAAO,GAAG,CAAA;AAAA,IACnB;AAAA,EACF;AAEA,EAAA,OAAO,MAAA;AACT;AAEA,SAAS,kBAAA,CACP,QACA,IAAA,EACoB;AACpB,EAAA,KAAA,MAAW,OAAO,IAAA,EAAM;AACtB,IAAA,MAAM,KAAA,GAAQ,OAAO,GAAG,CAAA;AACxB,IAAA,IAAI,OAAO,UAAU,QAAA,IAAY,MAAA,CAAO,SAAS,KAAK,CAAA,IAAK,QAAQ,CAAA,EAAG;AACpE,MAAA,OAAO,KAAA;AAAA,IACT;AAAA,EACF;AAEA,EAAA,OAAO,MAAA;AACT;AAEA,SAAS,kBAAA,CACP,QACA,IAAA,EACoB;AACpB,EAAA,KAAA,MAAW,OAAO,IAAA,EAAM;AACtB,IAAA,MAAM,KAAA,GAAQ,OAAO,GAAG,CAAA;AACxB,IAAA,IAAI,OAAO,KAAA,KAAU,QAAA,IAAY,KAAA,CAAM,IAAA,OAAW,EAAA,EAAI;AACpD,MAAA,OAAO,KAAA;AAAA,IACT;AAAA,EACF;AAEA,EAAA,OAAO,MAAA;AACT;AAEA,SAAS,MAAA,CACP,QACA,IAAA,EAC2B;AAC3B,EAAA,KAAA,MAAW,OAAO,IAAA,EAAM;AACtB,IAAA,MAAM,KAAA,GAAQ,OAAO,GAAG,CAAA;AACxB,IAAA,IACE,OAAO,KAAA,KAAU,QAAA,IACjB,eAAA,CAAgB,GAAA,CAAI,KAAsB,CAAA,EAC1C;AACA,MAAA,OAAO,KAAA;AAAA,IACT;AAAA,EACF;AAEA,EAAA,OAAO,MAAA;AACT;AAEA,SAAS,kBAAA,CACP,QACA,IAAA,EACkC;AAClC,EAAA,KAAA,MAAW,OAAO,IAAA,EAAM;AACtB,IAAA,MAAM,KAAA,GAAQ,OAAO,GAAG,CAAA;AACxB,IAAA,IAAI,OAAO,KAAA,KAAU,SAAA,IAAa,KAAA,KAAU,YAAA,EAAc;AACxD,MAAA,OAAO,KAAA;AAAA,IACT;AAAA,EACF;AAEA,EAAA,OAAO,MAAA;AACT;AAEA,SAAS,8BAA8B,KAAA,EAAsC;AAC3E,EAAA,IAAI,CAAC,KAAA,IAAS,OAAO,KAAA,KAAU,QAAA,SAAiB,EAAC;AAEjD,EAAA,MAAM,MAAA,GAAS,KAAA;AAEf,EAAA,OAAO;AAAA,IACL,0BAAA,EAA4B,YAAY,MAAA,EAAQ;AAAA,MAC9C,4BAAA;AAAA,MACA;AAAA,KACD,CAAA;AAAA,IACD,yBAAA,EAA2B,mBAAmB,MAAA,EAAQ;AAAA,MACpD,2BAAA;AAAA,MACA;AAAA,KACD,CAAA;AAAA,IACD,0BAAA,EAA4B,mBAAmB,MAAA,EAAQ;AAAA,MACrD;AAAA,KACD,CAAA;AAAA,IACD,wBAAA,EAA0B,mBAAmB,MAAA,EAAQ;AAAA,MACnD;AAAA,KACD,CAAA;AAAA,IACD,mBAAA,EAAqB,MAAA,CAAO,MAAA,EAAQ,CAAC,qBAAqB,CAAC,CAAA;AAAA,IAC3D,6BAAA,EAA+B,YAAY,MAAA,EAAQ;AAAA,MACjD;AAAA,KACD,CAAA;AAAA,IACD,+BAAA,EAAiC,mBAAmB,MAAA,EAAQ;AAAA,MAC1D;AAAA,KACD;AAAA,GACH;AACF;AAEO,SAAS,2BAAA,CACd,gBACA,eAAA,EAC8B;AAC9B,EAAA,MAAM,MAAA,GAAS,8BAA8B,cAAc,CAAA;AAC3D,EAAA,MAAM,OAAA,GAAU,8BAA8B,eAAe,CAAA;AAE7D,EAAA,OAAO;AAAA,IACL,gBAAA,EACE,OAAA,CAAQ,0BAAA,IACR,MAAA,CAAO,0BAAA,IACP,KAAA;AAAA,IACF,QAAA,EACE,OAAA,CAAQ,yBAAA,IAA6B,MAAA,CAAO,yBAAA;AAAA,IAC9C,SAAA,EACE,OAAA,CAAQ,0BAAA,IAA8B,MAAA,CAAO,0BAAA;AAAA,IAC/C,OAAA,EACE,OAAA,CAAQ,wBAAA,IAA4B,MAAA,CAAO,wBAAA;AAAA,IAC7C,EAAA,EAAI,OAAA,CAAQ,mBAAA,IAAuB,MAAA,CAAO,mBAAA;AAAA,IAC1C,YAAA,EACE,OAAA,CAAQ,6BAAA,IACR,MAAA,CAAO,6BAAA;AAAA,IACT,cAAA,EACE,OAAA,CAAQ,+BAAA,IACR,MAAA,CAAO;AAAA,GACX;AACF;AAEA,eAAe,iBAAiB,IAAA,EAAgC;AAC9D,EAAA,IAAI;AACF,IAAA,OAAO,KAAK,KAAA,CAAM,MAAM,QAAA,CAAS,IAAA,EAAM,OAAO,CAAC,CAAA;AAAA,EACjD,CAAA,CAAA,MAAQ;AACN,IAAA,OAAO,EAAC;AAAA,EACV;AACF;AAEA,eAAsB,wBAAA,CACpB,GAAA,EACA,QAAA,GAAW,WAAA,EAAY,EACgB;AACvC,EAAA,MAAM,iBAAiB,MAAM,gBAAA;AAAA,IAC3B,IAAA,CAAK,UAAU,eAAe;AAAA,GAChC;AACA,EAAA,MAAM,kBAAkB,MAAM,gBAAA;AAAA,IAC5B,IAAA,CAAK,GAAA,EAAK,KAAA,EAAO,eAAe;AAAA,GAClC;AAEA,EAAA,OAAO,2BAAA,CAA4B,gBAAgB,eAAe,CAAA;AACpE;;;AC1KA,IAAM,eAAA,GAAkB;AAAA,EACtB,wFAAA;AAAA,EACA,4FAAA;AAAA,EACA,yFAAA;AAAA,EACA;AACF,CAAA,CAAE,KAAK,GAAG,CAAA;AAEK,SAAR,sBAAuC,EAAA,EAAkB;AAC9D,EAAA,MAAM,WAAW,wBAAA,EAAyB;AAE1C,EAAA,EAAA,CAAG,YAAA,CAAa;AAAA,IACd,IAAA,EAAM,WAAA;AAAA,IACN,KAAA,EAAO,WAAA;AAAA,IACP,WAAA,EAAa,eAAA;AAAA,IACb,aAAA,EACE,iKAAA;AAAA,IACF,UAAA,EAAYC,KAAK,MAAA,CAAO;AAAA,MACtB,GAAG,uCAAuC,QAAQ,CAAA;AAAA,MAClD,SAASA,IAAAA,CAAK,QAAA;AAAA,QACZA,KAAK,OAAA,CAAQ;AAAA,UACX,WAAA,EACE;AAAA,SACH;AAAA;AACH,KACD,CAAA;AAAA,IAED,MAAM,OAAA,CAAQ,WAAA,EAAa,MAAA,EAAQ,OAAA,EAAS,WAAW,GAAA,EAAK;AAC1D,MAAA,MAAM,WAAW,MAAM,wBAAA,CAAyB,GAAA,CAAI,GAAA,EAAKC,aAAa,CAAA;AACtE,MAAA,MAAM,eAAA,GAAkB,yBAAyB,QAAQ,CAAA;AACzD,MAAA,MAAM,OAAA,GACH,MAAA,CAAO,OAAA,IAAmC,QAAA,CAAS,gBAAA;AACtD,MAAA,MAAM,MAAA,GAAS,MAAM,oBAAA,CAAqB,MAAA,EAAQ,eAAe,CAAA;AAEjE,MAAA,IAAI,OAAA,CAAQ,MAAM,CAAA,EAAG;AACnB,QAAA,OAAO;AAAA,UACL,OAAA,EAAS,CAAC,EAAE,IAAA,EAAM,MAAA,EAAQ,MAAM,CAAA,OAAA,EAAU,MAAA,CAAO,KAAK,CAAA,CAAA,EAAI,CAAA;AAAA,UAC1D,OAAA,EAAS,EAAE,KAAA,EAAO,IAAA,EAAM,OAAA;AAAQ,SAClC;AAAA,MACF;AAEA,MAAA,OAAO;AAAA,QACL,OAAA,EAAS;AAAA,UACP,EAAE,MAAM,MAAA,EAAQ,IAAA,EAAM,uBAAuB,MAAA,EAAQ,EAAE,OAAA,EAAS,CAAA;AAAE,SACpE;AAAA,QACA,OAAA,EAAS,EAAE,OAAA,EAAS,QAAA,EAAU,gBAAgB,QAAA;AAAS,OACzD;AAAA,IACF;AAAA,GACD,CAAA;AACH","file":"index.js","sourcesContent":["import type { FingerprintOs } from \"./types\";\n\nexport const DEFAULT_BROWSER = \"chrome_145\";\nexport const DEFAULT_OS: FingerprintOs = \"windows\";\nexport const DEFAULT_MAX_CHARS = 50_000;\nexport const DEFAULT_TIMEOUT_MS = 15_000;\nexport const DEFAULT_INCLUDE_REPLIES = \"extractors\" as const;\nexport const DEFAULT_ACCEPT_HEADER =\n \"text/html,application/xhtml+xml,application/xml;q=0.9,/;q=0.8\";\nexport const DEFAULT_ACCEPT_LANGUAGE_HEADER = \"en-US,en;q=0.9\";\n","import { Defuddle } from \"defuddle/node\";\nimport { getProfiles, fetch as wreqFetch } from \"wreq-js\";\nimport type { FetchDependencies } from \"./types\";\n\nexport const runtimeDependencies: FetchDependencies = {\n fetch: wreqFetch,\n defuddle: Defuddle,\n getProfiles,\n};\n","import { parseHTML } from \"linkedom\";\n\n/** Apply linkedom polyfills that Defuddle expects (getComputedStyle, styleSheets). /\nexport function parseLinkedomHTML(html: string, url?: string): Document {\n const { document } = parseHTML(html);\n const doc = document as Document & Record<string, unknown>;\n const defaultView = doc.defaultView as\n \| (Window & {\n getComputedStyle?: (\n elt: Element,\n pseudoElt?: string \| null,\n ) => CSSStyleDeclaration;\n })\n \| undefined;\n\n if (!(doc as { styleSheets?: unknown }).styleSheets) {\n (doc as { styleSheets?: unknown }).styleSheets =\n [] as unknown as StyleSheetList;\n }\n\n if (defaultView && !defaultView.getComputedStyle) {\n defaultView.getComputedStyle = (() => ({\n display: \"\",\n })) as unknown as typeof defaultView.getComputedStyle;\n }\n\n if (url) {\n (doc as { URL?: string }).URL = url;\n }\n\n return document;\n}\n","import type { FetchResult } from \"./types\";\n\nfunction buildHeader(\n parts: Array<[label: string, value: string \| number \| undefined]>,\n) {\n return parts\n .filter(([, value]) => value !== undefined && value !== \"\")\n .map(([label, value]) => `> ${label}: ${value}`)\n .join(\"\\n\");\n}\n\nexport function markdownToText(markdown: string): string {\n return markdown\n .replace(/^#{1,6}\\s+/gm, \"\")\n .replace(/\\\\([^]+)\\\\/g, \"$1\")\n .replace(/\\([^]+)\\/g, \"$1\")\n .replace(/\\[([^\\]]+)\\]\$[^)]+\$/g, \"$1\")\n .replace(/!\\[[^\\]]\\]\$[^)]+\$/g, \"\")\n .replace(/^>\\s+/gm, \"\")\n .replace(/^[-+]\\s+/gm, \"• \")\n .replace(/`([^`]+)`/g, \"$1\");\n}\n\nexport function truncateContent(content: string, maxChars: number): string {\n if (content.length <= maxChars) return content;\n return `${content.slice(0, maxChars)}\\n\\n[... truncated]`;\n}\n\nexport function buildCompactMetadataHeader(result: FetchResult): string {\n return buildHeader([\n [\"URL\", result.finalUrl],\n [\"Title\", result.title],\n [\"Author\", result.author],\n [\"Published\", result.published],\n ]);\n}\n\nexport function buildMetadataHeader(result: FetchResult): string {\n return buildHeader([\n [\"URL\", result.finalUrl],\n [\"Title\", result.title],\n [\"Author\", result.author],\n [\"Published\", result.published],\n [\"Site\", result.site],\n [\"Language\", result.language],\n [\"Words\", result.wordCount],\n [\"Browser\", `${result.browser}/${result.os}`],\n ]);\n}\n\nexport function buildFetchResponseText(\n result: FetchResult,\n options: { verbose?: boolean } = {},\n): string {\n const header = options.verbose\n ? buildMetadataHeader(result)\n : buildCompactMetadataHeader(result);\n\n return header ? `${header}\\n\\n${result.content}` : result.content;\n}\n","/\n Core extraction pipeline: fetch with TLS fingerprinting → parse → Defuddle extract.\n * Separated from the plugin entry so it can be tested independently.\n /\n\nimport {\n DEFAULT_ACCEPT_HEADER,\n DEFAULT_ACCEPT_LANGUAGE_HEADER,\n DEFAULT_BROWSER,\n DEFAULT_INCLUDE_REPLIES,\n DEFAULT_MAX_CHARS,\n DEFAULT_OS,\n DEFAULT_TIMEOUT_MS,\n} from \"./constants\";\nimport { runtimeDependencies } from \"./dependencies\";\nimport { parseLinkedomHTML } from \"./dom\";\nimport { markdownToText, truncateContent } from \"./format\";\nimport { getLatestChromeProfile as getLatestChromeProfileFrom } from \"./profiles\";\nimport type {\n FetchDependencies,\n FetchError,\n FetchOptions,\n FetchResult,\n OutputFormat,\n} from \"./types\";\n\nexport {\n DEFAULT_BROWSER,\n DEFAULT_INCLUDE_REPLIES,\n DEFAULT_MAX_CHARS,\n DEFAULT_OS,\n DEFAULT_TIMEOUT_MS,\n} from \"./constants\";\nexport type {\n FetchError,\n FetchOptions,\n FetchResult,\n OutputFormat,\n} from \"./types\";\n\nconst HTML_CONTENT_TYPES = [\n \"text/html\",\n \"application/xhtml+xml\",\n \"text/plain\",\n \"text/markdown\",\n];\n\nexport function getLatestChromeProfile(): string {\n return getLatestChromeProfileFrom(runtimeDependencies.getProfiles);\n}\n\nexport function createDefuddleFetch(\n dependencies: FetchDependencies = runtimeDependencies,\n) {\n return async function defuddleFetch(\n opts: FetchOptions,\n ): Promise<FetchResult \| FetchError> {\n const browser = opts.browser ?? DEFAULT_BROWSER;\n const os = opts.os ?? DEFAULT_OS;\n const format: OutputFormat = opts.format ?? \"markdown\";\n const maxChars = opts.maxChars ?? DEFAULT_MAX_CHARS;\n const removeImages = opts.removeImages ?? false;\n const includeReplies = opts.includeReplies ?? DEFAULT_INCLUDE_REPLIES;\n const timeoutMs = opts.timeoutMs ?? DEFAULT_TIMEOUT_MS;\n\n let parsed: URL;\n try {\n parsed = new URL(opts.url);\n } catch {\n return { error: `Invalid URL: ${opts.url}` };\n }\n\n if (![\"http:\", \"https:\"].includes(parsed.protocol)) {\n return {\n error: `Only http/https URLs supported, got ${parsed.protocol}`,\n };\n }\n\n const fetchOptions: Record<string, unknown> = {\n browser,\n os,\n headers: {\n Accept: DEFAULT_ACCEPT_HEADER,\n \"Accept-Language\": DEFAULT_ACCEPT_LANGUAGE_HEADER,\n ...opts.headers,\n },\n redirect: \"follow\",\n timeout: timeoutMs,\n };\n\n if (opts.proxy) {\n fetchOptions.proxy = opts.proxy;\n }\n\n const response = await dependencies.fetch(opts.url, fetchOptions);\n\n if (!response.ok) {\n return {\n error: `HTTP ${response.status} ${response.statusText} for ${opts.url}`,\n };\n }\n\n const finalUrl = response.url ?? opts.url;\n const contentType = response.headers.get(\"content-type\") ?? \"\";\n\n if (!HTML_CONTENT_TYPES.some((value) => contentType.includes(value))) {\n return { error: `Not an HTML page (content-type: ${contentType})` };\n }\n\n const html = await response.text();\n const document = parseLinkedomHTML(html, finalUrl);\n const extracted = await dependencies.defuddle(document, finalUrl, {\n markdown: format !== \"html\",\n removeImages,\n includeReplies,\n });\n\n if (!extracted.content \|\| extracted.wordCount === 0) {\n return {\n error: `No content extracted from ${opts.url}. May need JS rendering or is blocked.`,\n };\n }\n\n const normalizedContent =\n format === \"text\" ? markdownToText(extracted.content) : extracted.content;\n\n return {\n url: opts.url,\n finalUrl,\n title: extracted.title ?? \"\",\n author: extracted.author ?? \"\",\n published: extracted.published ?? \"\",\n site: extracted.site ?? \"\",\n language: extracted.language ?? \"\",\n wordCount: extracted.wordCount,\n content: truncateContent(normalizedContent, maxChars),\n browser,\n os,\n };\n };\n}\n\nexport const defuddleFetch = createDefuddleFetch();\n\n/* Type guard: check if result is an error. */\nexport function isError(\n result: FetchResult \| FetchError,\n): result is FetchError {\n return \"error\" in result;\n}\n","import { type TSchema, Type } from \"@sinclair/typebox\";\nimport {\n DEFAULT_BROWSER,\n DEFAULT_INCLUDE_REPLIES,\n DEFAULT_MAX_CHARS,\n DEFAULT_OS,\n DEFAULT_TIMEOUT_MS,\n} from \"./constants\";\nimport { defuddleFetch } from \"./extract\";\nimport type {\n FetchError,\n FetchResult,\n FetchToolConfig,\n FetchToolDefaults,\n} from \"./types\";\n\nexport function resolveFetchToolDefaults(\n config: FetchToolConfig = {},\n): FetchToolDefaults {\n return {\n maxChars: config.maxChars ?? DEFAULT_MAX_CHARS,\n timeoutMs: config.timeoutMs ?? DEFAULT_TIMEOUT_MS,\n browser: config.browser ?? DEFAULT_BROWSER,\n os: config.os ?? DEFAULT_OS,\n removeImages: config.removeImages ?? false,\n includeReplies: config.includeReplies ?? DEFAULT_INCLUDE_REPLIES,\n };\n}\n\nexport function createBaseFetchToolParameterProperties(\n defaults: FetchToolDefaults,\n): Record<string, TSchema> {\n return {\n url: Type.String({ description: \"URL to fetch (http/https only)\" }),\n browser: Type.Optional(\n Type.String({\n description: `Browser profile for TLS fingerprinting. Default: \"${defaults.browser}\". Examples: chrome_145, firefox_147, safari_26, edge_145, opera_127`,\n }),\n ),\n os: Type.Optional(\n Type.String({\n description: `OS profile for fingerprinting. Default: \"${defaults.os}\". Options: windows, macos, linux, android, ios`,\n }),\n ),\n headers: Type.Optional(\n Type.Record(Type.String(), Type.String(), {\n description:\n \"Custom HTTP headers to send. By default, Accept and Accept-Language are set automatically.\",\n }),\n ),\n maxChars: Type.Optional(\n Type.Number({\n description: `Maximum characters to return. Default: ${defaults.maxChars}`,\n }),\n ),\n format: Type.Optional(\n Type.Union(\n [Type.Literal(\"markdown\"), Type.Literal(\"html\"), Type.Literal(\"text\")],\n {\n description:\n 'Output format. \"markdown\" (default), \"html\" (cleaned HTML), or \"text\" (plain text, no formatting)',\n },\n ),\n ),\n removeImages: Type.Optional(\n Type.Boolean({\n description: \"Strip image references from output. Default: false\",\n }),\n ),\n includeReplies: Type.Optional(\n Type.Union([Type.Boolean(), Type.Literal(\"extractors\")], {\n description:\n \"Include replies/comments: 'extractors' for site-specific only (default), true for all, false for none\",\n }),\n ),\n proxy: Type.Optional(\n Type.String({\n description:\n \"Proxy URL (http://user:pass@host:port or socks5://host:port)\",\n }),\n ),\n };\n}\n\nexport async function executeFetchToolCall(\n params: Record<string, unknown>,\n defaults: FetchToolDefaults,\n): Promise<FetchResult \| FetchError> {\n return defuddleFetch({\n url: params.url as string,\n browser: (params.browser as string) ?? defaults.browser,\n os: (params.os as string) ?? defaults.os,\n headers: params.headers as Record<string, string> \| undefined,\n maxChars: (params.maxChars as number) ?? defaults.maxChars,\n format: (params.format as \"markdown\" \| \"html\" \| \"text\") ?? \"markdown\",\n removeImages: (params.removeImages as boolean) ?? defaults.removeImages,\n includeReplies:\n (params.includeReplies as boolean \| \"extractors\") ??\n defaults.includeReplies,\n proxy: params.proxy as string \| undefined,\n timeoutMs: defaults.timeoutMs,\n });\n}\n","import { readFile } from \"node:fs/promises\";\nimport { join } from \"node:path\";\nimport { getAgentDir } from \"@mariozechner/pi-coding-agent\";\nimport type {\n FetchToolConfig,\n FingerprintOs,\n IncludeRepliesOption,\n} from \"smart-fetch-core\";\n\nconst VALID_OS_VALUES = new Set<FingerprintOs>([\n \"windows\",\n \"macos\",\n \"linux\",\n \"android\",\n \"ios\",\n]);\n\ninterface PiSmartFetchSettings {\n smartFetchVerboseByDefault?: boolean;\n smartFetchDefaultMaxChars?: number;\n smartFetchDefaultTimeoutMs?: number;\n smartFetchDefaultBrowser?: string;\n smartFetchDefaultOs?: FingerprintOs;\n smartFetchDefaultRemoveImages?: boolean;\n smartFetchDefaultIncludeReplies?: IncludeRepliesOption;\n}\n\nexport interface ResolvedPiSmartFetchSettings extends FetchToolConfig {\n verboseByDefault: boolean;\n}\n\nfunction readBoolean(\n source: Record<string, unknown>,\n keys: string[],\n): boolean \| undefined {\n for (const key of keys) {\n if (typeof source[key] === \"boolean\") {\n return source[key] as boolean;\n }\n }\n\n return undefined;\n}\n\nfunction readPositiveNumber(\n source: Record<string, unknown>,\n keys: string[],\n): number \| undefined {\n for (const key of keys) {\n const value = source[key];\n if (typeof value === \"number\" && Number.isFinite(value) && value > 0) {\n return value;\n }\n }\n\n return undefined;\n}\n\nfunction readNonEmptyString(\n source: Record<string, unknown>,\n keys: string[],\n): string \| undefined {\n for (const key of keys) {\n const value = source[key];\n if (typeof value === \"string\" && value.trim() !== \"\") {\n return value;\n }\n }\n\n return undefined;\n}\n\nfunction readOs(\n source: Record<string, unknown>,\n keys: string[],\n): FingerprintOs \| undefined {\n for (const key of keys) {\n const value = source[key];\n if (\n typeof value === \"string\" &&\n VALID_OS_VALUES.has(value as FingerprintOs)\n ) {\n return value as FingerprintOs;\n }\n }\n\n return undefined;\n}\n\nfunction readIncludeReplies(\n source: Record<string, unknown>,\n keys: string[],\n): IncludeRepliesOption \| undefined {\n for (const key of keys) {\n const value = source[key];\n if (typeof value === \"boolean\" \|\| value === \"extractors\") {\n return value;\n }\n }\n\n return undefined;\n}\n\nfunction normalizePiSmartFetchSettings(input: unknown): PiSmartFetchSettings {\n if (!input \|\| typeof input !== \"object\") return {};\n\n const source = input as Record<string, unknown>;\n\n return {\n smartFetchVerboseByDefault: readBoolean(source, [\n \"smartFetchVerboseByDefault\",\n \"webFetchVerboseByDefault\",\n ]),\n smartFetchDefaultMaxChars: readPositiveNumber(source, [\n \"smartFetchDefaultMaxChars\",\n \"webFetchDefaultMaxChars\",\n ]),\n smartFetchDefaultTimeoutMs: readPositiveNumber(source, [\n \"smartFetchDefaultTimeoutMs\",\n ]),\n smartFetchDefaultBrowser: readNonEmptyString(source, [\n \"smartFetchDefaultBrowser\",\n ]),\n smartFetchDefaultOs: readOs(source, [\"smartFetchDefaultOs\"]),\n smartFetchDefaultRemoveImages: readBoolean(source, [\n \"smartFetchDefaultRemoveImages\",\n ]),\n smartFetchDefaultIncludeReplies: readIncludeReplies(source, [\n \"smartFetchDefaultIncludeReplies\",\n ]),\n };\n}\n\nexport function resolvePiSmartFetchSettings(\n globalSettings: unknown,\n projectSettings: unknown,\n): ResolvedPiSmartFetchSettings {\n const global = normalizePiSmartFetchSettings(globalSettings);\n const project = normalizePiSmartFetchSettings(projectSettings);\n\n return {\n verboseByDefault:\n project.smartFetchVerboseByDefault ??\n global.smartFetchVerboseByDefault ??\n false,\n maxChars:\n project.smartFetchDefaultMaxChars ?? global.smartFetchDefaultMaxChars,\n timeoutMs:\n project.smartFetchDefaultTimeoutMs ?? global.smartFetchDefaultTimeoutMs,\n browser:\n project.smartFetchDefaultBrowser ?? global.smartFetchDefaultBrowser,\n os: project.smartFetchDefaultOs ?? global.smartFetchDefaultOs,\n removeImages:\n project.smartFetchDefaultRemoveImages ??\n global.smartFetchDefaultRemoveImages,\n includeReplies:\n project.smartFetchDefaultIncludeReplies ??\n global.smartFetchDefaultIncludeReplies,\n };\n}\n\nasync function readSettingsFile(path: string): Promise<unknown> {\n try {\n return JSON.parse(await readFile(path, \"utf-8\"));\n } catch {\n return {};\n }\n}\n\nexport async function loadPiSmartFetchSettings(\n cwd: string,\n agentDir = getAgentDir(),\n): Promise<ResolvedPiSmartFetchSettings> {\n const globalSettings = await readSettingsFile(\n join(agentDir, \"settings.json\"),\n );\n const projectSettings = await readSettingsFile(\n join(cwd, \".pi\", \"settings.json\"),\n );\n\n return resolvePiSmartFetchSettings(globalSettings, projectSettings);\n}\n","import { type ExtensionAPI, getAgentDir } from \"@mariozechner/pi-coding-agent\";\nimport { Type } from \"@sinclair/typebox\";\nimport {\n buildFetchResponseText,\n createBaseFetchToolParameterProperties,\n executeFetchToolCall,\n isError,\n resolveFetchToolDefaults,\n} from \"smart-fetch-core\";\nimport { loadPiSmartFetchSettings } from \"./settings\";\n\nconst toolDescription = [\n \"Fetch a URL with browser-grade TLS fingerprinting and extract clean, readable content.\",\n \"Uses wreq-js for browser-like TLS/HTTP2 impersonation and Defuddle for article extraction.\",\n \"Supports the same fetch parameters as the OpenClaw tool, plus an optional verbose flag.\",\n \"Does NOT execute JavaScript — use a browser automation tool for JS-heavy pages.\",\n].join(\" \");\n\nexport default function piSmartFetchExtension(pi: ExtensionAPI) {\n const defaults = resolveFetchToolDefaults();\n\n pi.registerTool({\n name: \"web_fetch\",\n label: \"web_fetch\",\n description: toolDescription,\n promptSnippet:\n \"web_fetch(url, browser?, os?, headers?, maxChars?, format?, removeImages?, includeReplies?, proxy?, verbose?): fetch browser-fingerprinted readable web content\",\n parameters: Type.Object({\n ...createBaseFetchToolParameterProperties(defaults),\n verbose: Type.Optional(\n Type.Boolean({\n description:\n \"Include the full metadata header (site, language, word count, browser fingerprint info). Default: false, or smartFetchVerboseByDefault from pi settings.\",\n }),\n ),\n }),\n\n async execute(_toolCallId, params, _signal, _onUpdate, ctx) {\n const settings = await loadPiSmartFetchSettings(ctx.cwd, getAgentDir());\n const runtimeDefaults = resolveFetchToolDefaults(settings);\n const verbose =\n (params.verbose as boolean \| undefined) ?? settings.verboseByDefault;\n const result = await executeFetchToolCall(params, runtimeDefaults);\n\n if (isError(result)) {\n return {\n content: [{ type: \"text\", text: `Error: ${result.error}` }],\n details: { error: true, verbose },\n };\n }\n\n return {\n content: [\n { type: \"text\", text: buildFetchResponseText(result, { verbose }) },\n ],\n details: { verbose, maxChars: runtimeDefaults.maxChars },\n };\n },\n });\n}\n"]}
1	+ {"version":3,"sources":["../../core/src/constants.ts","../../core/src/dependencies.ts","../../core/src/dom.ts","../../core/src/format.ts","../../core/src/extract.ts","../../core/src/tool.ts","../src/settings.ts","../src/index.ts"],"names":["wreqFetch","defuddleFetch","Type","getAgentDir"],"mappings":";;;;;;;;;;;AAEO,IAAM,eAAA,GAAkB,YAAA;AACxB,IAAM,UAAA,GAA4B,SAAA;AAClC,IAAM,iBAAA,GAAoB,GAAA;AAC1B,IAAM,kBAAA,GAAqB,IAAA;AAC3B,IAAM,uBAAA,GAA0B,YAAA;AAChC,IAAM,qBAAA,GACX,iEAAA;AACK,IAAM,0BAAA,GACX,iFAAA;AACK,IAAM,8BAAA,GAAiC,gBAAA;ACPvC,IAAM,mBAAA,GAAyC;AAAA,EACpD,KAAA,EAAOA,KAAA;AAAA,EACP,QAAA,EAAU,QAAA;AAAA,EACV;AACF,CAAA;ACLO,SAAS,iBAAA,CAAkB,MAAc,GAAA,EAAwB;AACtE,EAAA,MAAM,EAAE,QAAA,EAAS,GAAI,SAAA,CAAU,IAAI,CAAA;AACnC,EAAA,MAAM,GAAA,GAAM,QAAA;AACZ,EAAA,MAAM,cAAc,GAAA,CAAI,WAAA;AASxB,EAAA,IAAI,CAAE,IAAkC,WAAA,EAAa;AACnD,IAAC,GAAA,CAAkC,cACjC,EAAC;AAAA,EACL;AAEA,EAAA,IAAI,WAAA,IAAe,CAAC,WAAA,CAAY,gBAAA,EAAkB;AAChD,IAAA,WAAA,CAAY,oBAAoB,OAAO;AAAA,MACrC,OAAA,EAAS;AAAA,KACX,CAAA,CAAA;AAAA,EACF;AAEA,EAAA,IAAI,GAAA,EAAK;AACP,IAAC,IAAyB,GAAA,GAAM,GAAA;AAAA,EAClC;AAEA,EAAA,OAAO,QAAA;AACT;;;AC7BA,SAAS,YACP,KAAA,EACA;AACA,EAAA,OAAO,KAAA,CACJ,MAAA,CAAO,CAAC,GAAG,KAAK,CAAA,KAAM,KAAA,KAAU,MAAA,IAAa,KAAA,KAAU,EAAE,CAAA,CACzD,IAAI,CAAC,CAAC,KAAA,EAAO,KAAK,CAAA,KAAM,CAAA,EAAA,EAAK,KAAK,CAAA,EAAA,EAAK,KAAK,CAAA,CAAE,CAAA,CAC9C,IAAA,CAAK,IAAI,CAAA;AACd;AAEO,SAAS,eAAe,QAAA,EAA0B;AACvD,EAAA,OAAO,QAAA,CACJ,OAAA,CAAQ,cAAA,EAAgB,EAAE,CAAA,CAC1B,OAAA,CAAQ,kBAAA,EAAoB,IAAI,CAAA,CAChC,OAAA,CAAQ,cAAA,EAAgB,IAAI,EAC5B,OAAA,CAAQ,wBAAA,EAA0B,IAAI,CAAA,CACtC,OAAA,CAAQ,uBAAA,EAAyB,EAAE,CAAA,CACnC,QAAQ,SAAA,EAAW,EAAE,CAAA,CACrB,OAAA,CAAQ,aAAA,EAAe,SAAI,CAAA,CAC3B,OAAA,CAAQ,cAAc,IAAI,CAAA;AAC/B;AAEO,SAAS,eAAA,CAAgB,SAAiB,QAAA,EAA0B;AACzE,EAAA,IAAI,OAAA,CAAQ,MAAA,IAAU,QAAA,EAAU,OAAO,OAAA;AACvC,EAAA,OAAO,CAAA,EAAG,OAAA,CAAQ,KAAA,CAAM,CAAA,EAAG,QAAQ,CAAC;;AAAA,eAAA,CAAA;AACtC;AAEO,SAAS,2BAA2B,MAAA,EAA6B;AACtE,EAAA,OAAO,WAAA,CAAY;AAAA,IACjB,CAAC,KAAA,EAAO,MAAA,CAAO,QAAQ,CAAA;AAAA,IACvB,CAAC,OAAA,EAAS,MAAA,CAAO,KAAK,CAAA;AAAA,IACtB,CAAC,QAAA,EAAU,MAAA,CAAO,MAAM,CAAA;AAAA,IACxB,CAAC,WAAA,EAAa,MAAA,CAAO,SAAS;AAAA,GAC/B,CAAA;AACH;AAEO,SAAS,oBAAoB,MAAA,EAA6B;AAC/D,EAAA,OAAO,WAAA,CAAY;AAAA,IACjB,CAAC,KAAA,EAAO,MAAA,CAAO,QAAQ,CAAA;AAAA,IACvB,CAAC,OAAA,EAAS,MAAA,CAAO,KAAK,CAAA;AAAA,IACtB,CAAC,QAAA,EAAU,MAAA,CAAO,MAAM,CAAA;AAAA,IACxB,CAAC,WAAA,EAAa,MAAA,CAAO,SAAS,CAAA;AAAA,IAC9B,CAAC,MAAA,EAAQ,MAAA,CAAO,IAAI,CAAA;AAAA,IACpB,CAAC,UAAA,EAAY,MAAA,CAAO,QAAQ,CAAA;AAAA,IAC5B,CAAC,OAAA,EAAS,MAAA,CAAO,SAAS,CAAA;AAAA,IAC1B,CAAC,WAAW,CAAA,EAAG,MAAA,CAAO,OAAO,CAAA,CAAA,EAAI,MAAA,CAAO,EAAE,CAAA,CAAE;AAAA,GAC7C,CAAA;AACH;AAEO,SAAS,sBAAA,CACd,MAAA,EACA,OAAA,GAAiC,EAAC,EAC1B;AACR,EAAA,MAAM,SAAS,OAAA,CAAQ,OAAA,GACnB,oBAAoB,MAAM,CAAA,GAC1B,2BAA2B,MAAM,CAAA;AAErC,EAAA,OAAO,MAAA,GAAS,GAAG,MAAM;;AAAA,EAAO,MAAA,CAAO,OAAO,CAAA,CAAA,GAAK,MAAA,CAAO,OAAA;AAC5D;AAEO,SAAS,kBAAkB,OAAA,EAAyB;AACzD,EAAA,MAAM,KAAA,GAAQ,OAAA,CAAQ,IAAA,EAAK,CAAE,MAAM,MAAM,CAAA;AACzC,EAAA,OAAO,OAAO,MAAA,IAAU,CAAA;AAC1B;AAEO,SAAS,WAAW,KAAA,EAAuB;AAChD,EAAA,OAAO,MACJ,OAAA,CAAQ,IAAA,EAAM,OAAO,CAAA,CACrB,OAAA,CAAQ,MAAM,MAAM,CAAA,CACpB,QAAQ,IAAA,EAAM,MAAM,EACpB,OAAA,CAAQ,IAAA,EAAM,QAAQ,CAAA,CACtB,OAAA,CAAQ,MAAM,OAAO,CAAA;AAC1B;AAEO,SAAS,mBACd,GAAA,EACoC;AACpC,EAAA,IAAI;AACF,IAAA,OAAO;AAAA,MACL,SAAA,EAAW,KAAK,SAAA,CAAU,IAAA,CAAK,MAAM,GAAG,CAAA,EAAG,MAAM,CAAC;AAAA,KACpD;AAAA,EACF,CAAA,CAAA,MAAQ;AACN,IAAA,OAAO,EAAE,OAAO,uBAAA,EAAwB;AAAA,EAC1C;AACF;AAEO,SAAS,iBAAA,CACd,eACA,MAAA,EACQ;AACR,EAAA,QAAQ,MAAA;AAAQ,IACd,KAAK,MAAA;AAAA,IACL,KAAK,MAAA;AACH,MAAA,OAAO,aAAA;AAAA,IACT,KAAK,MAAA;AACH,MAAA,OAAO,CAAA,iCAAA,EAAoC,UAAA,CAAW,aAAa,CAAC,CAAA,aAAA,CAAA;AAAA,IACtE;AACE,MAAA,OAAO,CAAA;AAAA,EAAe,aAAa;AAAA,MAAA,CAAA;AAAA;AAEzC;AAEO,SAAS,sBAAA,CACd,SACA,MAAA,EACQ;AACR,EAAA,IAAI,WAAW,MAAA,EAAQ;AACrB,IAAA,OAAO,OAAA,CACJ,OAAA,CAAQ,iDAAA,EAAmD,EAAE,EAC7D,OAAA,EAAQ;AAAA,EACb;AAEA,EAAA,OAAO,OAAA,CAAQ,OAAA,CAAQ,gCAAA,EAAkC,EAAE,EAAE,OAAA,EAAQ;AACvE;;;ACjEA,IAAM,kBAAA,GAAqB;AAAA,EACzB,WAAA;AAAA,EACA,uBAAA;AAAA,EACA,YAAA;AAAA,EACA;AACF,CAAA;AAEA,SAAS,oBAAoB,MAAA,EAA8B;AACzD,EAAA,OAAO,MAAA,KAAW,SAAS,0BAAA,GAA6B,qBAAA;AAC1D;AAEA,SAAS,kBAAkB,WAAA,EAA8B;AACvD,EAAA,MAAM,UAAA,GAAa,WAAA,CAAY,KAAA,CAAM,GAAG,CAAA,CAAE,CAAC,CAAA,EAAG,IAAA,EAAK,CAAE,WAAA,EAAY,IAAK,EAAA;AACtE,EAAA,OACE,eAAe,kBAAA,IACf,UAAA,KAAe,WAAA,IACf,UAAA,CAAW,SAAS,OAAO,CAAA;AAE/B;AAEA,SAAS,iBAAiB,IAAA,EAAuB;AAC/C,EAAA,MAAM,OAAA,GAAU,KAAK,IAAA,EAAK;AAC1B,EAAA,OAAO,QAAQ,UAAA,CAAW,GAAG,CAAA,IAAK,OAAA,CAAQ,WAAW,GAAG,CAAA;AAC1D;AAEA,SAAS,cAAA,CAAe,aAAqB,IAAA,EAAuB;AAClE,EAAA,OAAO,iBAAA,CAAkB,WAAW,CAAA,IAAK,gBAAA,CAAiB,IAAI,CAAA;AAChE;AAEA,SAAS,gBACP,IAAA,EACA,QAAA,EACA,SACA,MAAA,EACA,QAAA,EACA,SACA,EAAA,EAC0B;AAC1B,EAAA,MAAM,UAAA,GAAa,mBAAmB,OAAO,CAAA;AAE7C,EAAA,IAAI,WAAW,UAAA,EAAY;AACzB,IAAA,OAAO,UAAA;AAAA,EACT;AAEA,EAAA,MAAM,OAAA,GAAU,eAAA;AAAA,IACd,iBAAA,CAAkB,UAAA,CAAW,SAAA,EAAW,MAAM,CAAA;AAAA,IAC9C;AAAA,GACF;AAEA,EAAA,OAAO;AAAA,IACL,KAAK,IAAA,CAAK,GAAA;AAAA,IACV,QAAA;AAAA,IACA,KAAA,EAAO,EAAA;AAAA,IACP,MAAA,EAAQ,EAAA;AAAA,IACR,SAAA,EAAW,EAAA;AAAA,IACX,IAAA,EAAM,IAAI,GAAA,CAAI,QAAQ,CAAA,CAAE,QAAA;AAAA,IACxB,QAAA,EAAU,EAAA;AAAA,IACV,SAAA,EAAW,iBAAA,CAAkB,UAAA,CAAW,SAAS,CAAA;AAAA,IACjD,OAAA;AAAA,IACA,OAAA;AAAA,IACA;AAAA,GACF;AACF;AAEA,SAAS,mBAAmB,IAAA,EAAuB;AACjD,EAAA,OACE,IAAA,KAAS,iBACT,IAAA,CAAK,UAAA,CAAW,IAAI,CAAA,IACpB,IAAA,CAAK,WAAW,WAAW,CAAA;AAE/B;AAMO,SAAS,mBAAA,CACd,eAAkC,mBAAA,EAClC;AACA,EAAA,OAAO,eAAeC,eACpB,IAAA,EACmC;AACnC,IAAA,MAAM,OAAA,GAAU,KAAK,OAAA,IAAW,eAAA;AAChC,IAAA,MAAM,EAAA,GAAK,KAAK,EAAA,IAAM,UAAA;AACtB,IAAA,MAAM,MAAA,GAAuB,KAAK,MAAA,IAAU,UAAA;AAC5C,IAAA,MAAM,QAAA,GAAW,KAAK,QAAA,IAAY,iBAAA;AAClC,IAAA,MAAM,YAAA,GAAe,KAAK,YAAA,IAAgB,KAAA;AAC1C,IAAA,MAAM,cAAA,GAAiB,KAAK,cAAA,IAAkB,uBAAA;AAC9C,IAAA,MAAM,SAAA,GAAY,KAAK,SAAA,IAAa,kBAAA;AAEpC,IAAA,IAAI,MAAA;AACJ,IAAA,IAAI;AACF,MAAA,MAAA,GAAS,IAAI,GAAA,CAAI,IAAA,CAAK,GAAG,CAAA;AAAA,IAC3B,CAAA,CAAA,MAAQ;AACN,MAAA,OAAO,EAAE,KAAA,EAAO,CAAA,aAAA,EAAgB,IAAA,CAAK,GAAG,CAAA,CAAA,EAAG;AAAA,IAC7C;AAEA,IAAA,IAAI,CAAC,CAAC,OAAA,EAAS,QAAQ,EAAE,QAAA,CAAS,MAAA,CAAO,QAAQ,CAAA,EAAG;AAClD,MAAA,OAAO;AAAA,QACL,KAAA,EAAO,CAAA,oCAAA,EAAuC,MAAA,CAAO,QAAQ,CAAA;AAAA,OAC/D;AAAA,IACF;AAEA,IAAA,MAAM,YAAA,GAAwC;AAAA,MAC5C,OAAA;AAAA,MACA,EAAA;AAAA,MACA,OAAA,EAAS;AAAA,QACP,MAAA,EAAQ,oBAAoB,MAAM,CAAA;AAAA,QAClC,iBAAA,EAAmB,8BAAA;AAAA,QACnB,GAAG,IAAA,CAAK;AAAA,OACV;AAAA,MACA,QAAA,EAAU,QAAA;AAAA,MACV,OAAA,EAAS;AAAA,KACX;AAEA,IAAA,IAAI,KAAK,KAAA,EAAO;AACd,MAAA,YAAA,CAAa,QAAQ,IAAA,CAAK,KAAA;AAAA,IAC5B;AAEA,IAAA,MAAM,WAAW,MAAM,YAAA,CAAa,KAAA,CAAM,IAAA,CAAK,KAAK,YAAY,CAAA;AAEhE,IAAA,IAAI,CAAC,SAAS,EAAA,EAAI;AAChB,MAAA,OAAO;AAAA,QACL,KAAA,EAAO,QAAQ,QAAA,CAAS,MAAM,IAAI,QAAA,CAAS,UAAU,CAAA,KAAA,EAAQ,IAAA,CAAK,GAAG,CAAA;AAAA,OACvE;AAAA,IACF;AAEA,IAAA,MAAM,QAAA,GAAW,QAAA,CAAS,GAAA,IAAO,IAAA,CAAK,GAAA;AACtC,IAAA,MAAM,WAAA,GAAc,QAAA,CAAS,OAAA,CAAQ,GAAA,CAAI,cAAc,CAAA,IAAK,EAAA;AAC5D,IAAA,MAAM,OAAA,GAAU,MAAM,QAAA,CAAS,IAAA,EAAK;AACpC,IAAA,MAAM,YAAA,GAAe,cAAA,CAAe,WAAA,EAAa,OAAO,CAAA;AAExD,IAAA,IAAI,WAAW,MAAA,EAAQ;AACrB,MAAA,IAAI,CAAC,YAAA,EAAc;AACjB,QAAA,OAAO,EAAE,KAAA,EAAO,CAAA,mCAAA,EAAsC,WAAW,CAAA,CAAA,CAAA,EAAI;AAAA,MACvE;AAEA,MAAA,OAAO,eAAA;AAAA,QACL,IAAA;AAAA,QACA,QAAA;AAAA,QACA,OAAA;AAAA,QACA,MAAA;AAAA,QACA,QAAA;AAAA,QACA,OAAA;AAAA,QACA;AAAA,OACF;AAAA,IACF;AAEA,IAAA,IAAI,YAAA,EAAc;AAChB,MAAA,OAAO,eAAA;AAAA,QACL,IAAA;AAAA,QACA,QAAA;AAAA,QACA,OAAA;AAAA,QACA,MAAA;AAAA,QACA,QAAA;AAAA,QACA,OAAA;AAAA,QACA;AAAA,OACF;AAAA,IACF;AAEA,IAAA,IAAI,CAAC,mBAAmB,IAAA,CAAK,CAAC,UAAU,WAAA,CAAY,QAAA,CAAS,KAAK,CAAC,CAAA,EAAG;AACpE,MAAA,OAAO,EAAE,KAAA,EAAO,CAAA,gCAAA,EAAmC,WAAW,CAAA,CAAA,CAAA,EAAI;AAAA,IACpE;AAEA,IAAA,MAAM,QAAA,GAAW,iBAAA,CAAkB,OAAA,EAAS,QAAQ,CAAA;AACpD,IAAA,MAAM,SAAA,GAAY,MAAM,YAAA,CAAa,QAAA,CAAS,UAAU,QAAA,EAAU;AAAA,MAChE,UAAU,MAAA,KAAW,MAAA;AAAA,MACrB,YAAA;AAAA,MACA;AAAA,KACD,CAAA;AAED,IAAA,IAAI,CAAC,SAAA,CAAU,OAAA,IAAW,SAAA,CAAU,cAAc,CAAA,EAAG;AACnD,MAAA,OAAO;AAAA,QACL,KAAA,EAAO,CAAA,0BAAA,EAA6B,IAAA,CAAK,GAAG,CAAA,sCAAA;AAAA,OAC9C;AAAA,IACF;AAEA,IAAA,IAAI,mBAAmB,SAAA,CAAU,OAAA;AACjC,IAAA,IAAI,YAAY,SAAA,CAAU,SAAA;AAE1B,IAAA,IAAI,mBAAmB,KAAA,IAAS,kBAAA,CAAmB,SAAA,CAAU,IAAA,IAAQ,EAAE,CAAA,EAAG;AACxE,MAAA,MAAM,eAAA,GAAkB,sBAAA,CAAuB,gBAAA,EAAkB,MAAM,CAAA;AACvE,MAAA,IAAI,oBAAoB,gBAAA,EAAkB;AACxC,QAAA,gBAAA,GAAmB,eAAA;AACnB,QAAA,SAAA,GAAY,iBAAA;AAAA,UACV,MAAA,KAAW,MAAA,GACP,cAAA,CAAe,gBAAgB,CAAA,GAC/B;AAAA,SACN;AAAA,MACF;AAAA,IACF;AAEA,IAAA,MAAM,iBAAA,GACJ,MAAA,KAAW,MAAA,GAAS,cAAA,CAAe,gBAAgB,CAAA,GAAI,gBAAA;AAEzD,IAAA,OAAO;AAAA,MACL,KAAK,IAAA,CAAK,GAAA;AAAA,MACV,QAAA;AAAA,MACA,KAAA,EAAO,UAAU,KAAA,IAAS,EAAA;AAAA,MAC1B,MAAA,EAAQ,UAAU,MAAA,IAAU,EAAA;AAAA,MAC5B,SAAA,EAAW,UAAU,SAAA,IAAa,EAAA;AAAA,MAClC,IAAA,EAAM,UAAU,IAAA,IAAQ,EAAA;AAAA,MACxB,QAAA,EAAU,UAAU,QAAA,IAAY,EAAA;AAAA,MAChC,SAAA;AAAA,MACA,OAAA,EAAS,eAAA,CAAgB,iBAAA,EAAmB,QAAQ,CAAA;AAAA,MACpD,OAAA;AAAA,MACA;AAAA,KACF;AAAA,EACF,CAAA;AACF;AAEO,IAAM,gBAAgB,mBAAA,EAAoB;AAG1C,SAAS,QACd,MAAA,EACsB;AACtB,EAAA,OAAO,OAAA,IAAW,MAAA;AACpB;AC1PO,SAAS,wBAAA,CACd,MAAA,GAA0B,EAAC,EACR;AACnB,EAAA,OAAO;AAAA,IACL,QAAA,EAAU,OAAO,QAAA,IAAY,iBAAA;AAAA,IAC7B,SAAA,EAAW,OAAO,SAAA,IAAa,kBAAA;AAAA,IAC/B,OAAA,EAAS,OAAO,OAAA,IAAW,eAAA;AAAA,IAC3B,EAAA,EAAI,OAAO,EAAA,IAAM,UAAA;AAAA,IACjB,YAAA,EAAc,OAAO,YAAA,IAAgB,KAAA;AAAA,IACrC,cAAA,EAAgB,OAAO,cAAA,IAAkB;AAAA,GAC3C;AACF;AAEO,SAAS,uCACd,QAAA,EACyB;AACzB,EAAA,OAAO;AAAA,IACL,KAAK,IAAA,CAAK,MAAA,CAAO,EAAE,WAAA,EAAa,kCAAkC,CAAA;AAAA,IAClE,SAAS,IAAA,CAAK,QAAA;AAAA,MACZ,KAAK,MAAA,CAAO;AAAA,QACV,WAAA,EAAa,CAAA,kDAAA,EAAqD,QAAA,CAAS,OAAO,CAAA,oEAAA;AAAA,OACnF;AAAA,KACH;AAAA,IACA,IAAI,IAAA,CAAK,QAAA;AAAA,MACP,KAAK,MAAA,CAAO;AAAA,QACV,WAAA,EAAa,CAAA,yCAAA,EAA4C,QAAA,CAAS,EAAE,CAAA,+CAAA;AAAA,OACrE;AAAA,KACH;AAAA,IACA,SAAS,IAAA,CAAK,QAAA;AAAA,MACZ,KAAK,MAAA,CAAO,IAAA,CAAK,QAAO,EAAG,IAAA,CAAK,QAAO,EAAG;AAAA,QACxC,WAAA,EACE;AAAA,OACH;AAAA,KACH;AAAA,IACA,UAAU,IAAA,CAAK,QAAA;AAAA,MACb,KAAK,MAAA,CAAO;AAAA,QACV,WAAA,EAAa,CAAA,uCAAA,EAA0C,QAAA,CAAS,QAAQ,CAAA;AAAA,OACzE;AAAA,KACH;AAAA,IACA,QAAQ,IAAA,CAAK,QAAA;AAAA,MACX,IAAA,CAAK,KAAA;AAAA,QACH;AAAA,UACE,IAAA,CAAK,QAAQ,UAAU,CAAA;AAAA,UACvB,IAAA,CAAK,QAAQ,MAAM,CAAA;AAAA,UACnB,IAAA,CAAK,QAAQ,MAAM,CAAA;AAAA,UACnB,IAAA,CAAK,QAAQ,MAAM;AAAA,SACrB;AAAA,QACA;AAAA,UACE,WAAA,EACE;AAAA;AACJ;AACF,KACF;AAAA,IACA,cAAc,IAAA,CAAK,QAAA;AAAA,MACjB,KAAK,OAAA,CAAQ;AAAA,QACX,WAAA,EAAa;AAAA,OACd;AAAA,KACH;AAAA,IACA,gBAAgB,IAAA,CAAK,QAAA;AAAA,MACnB,IAAA,CAAK,KAAA,CAAM,CAAC,IAAA,CAAK,OAAA,IAAW,IAAA,CAAK,OAAA,CAAQ,YAAY,CAAC,CAAA,EAAG;AAAA,QACvD,WAAA,EACE;AAAA,OACH;AAAA,KACH;AAAA,IACA,OAAO,IAAA,CAAK,QAAA;AAAA,MACV,KAAK,MAAA,CAAO;AAAA,QACV,WAAA,EACE;AAAA,OACH;AAAA;AACH,GACF;AACF;AAEA,eAAsB,oBAAA,CACpB,QACA,QAAA,EACmC;AACnC,EAAA,OAAO,aAAA,CAAc;AAAA,IACnB,KAAK,MAAA,CAAO,GAAA;AAAA,IACZ,OAAA,EAAU,MAAA,CAAO,OAAA,IAAsB,QAAA,CAAS,OAAA;AAAA,IAChD,EAAA,EAAK,MAAA,CAAO,EAAA,IAAiB,QAAA,CAAS,EAAA;AAAA,IACtC,SAAS,MAAA,CAAO,OAAA;AAAA,IAChB,QAAA,EAAW,MAAA,CAAO,QAAA,IAAuB,QAAA,CAAS,QAAA;AAAA,IAClD,MAAA,EACG,OAAO,MAAA,IAAoD,UAAA;AAAA,IAC9D,YAAA,EAAe,MAAA,CAAO,YAAA,IAA4B,QAAA,CAAS,YAAA;AAAA,IAC3D,cAAA,EACG,MAAA,CAAO,cAAA,IACR,QAAA,CAAS,cAAA;AAAA,IACX,OAAO,MAAA,CAAO,KAAA;AAAA,IACd,WAAW,QAAA,CAAS;AAAA,GACrB,CAAA;AACH;ACnGA,IAAM,eAAA,uBAAsB,GAAA,CAAmB;AAAA,EAC7C,SAAA;AAAA,EACA,OAAA;AAAA,EACA,OAAA;AAAA,EACA,SAAA;AAAA,EACA;AACF,CAAC,CAAA;AAgBD,SAAS,WAAA,CACP,QACA,IAAA,EACqB;AACrB,EAAA,KAAA,MAAW,OAAO,IAAA,EAAM;AACtB,IAAA,IAAI,OAAO,MAAA,CAAO,GAAG,CAAA,KAAM,SAAA,EAAW;AACpC,MAAA,OAAO,OAAO,GAAG,CAAA;AAAA,IACnB;AAAA,EACF;AAEA,EAAA,OAAO,MAAA;AACT;AAEA,SAAS,kBAAA,CACP,QACA,IAAA,EACoB;AACpB,EAAA,KAAA,MAAW,OAAO,IAAA,EAAM;AACtB,IAAA,MAAM,KAAA,GAAQ,OAAO,GAAG,CAAA;AACxB,IAAA,IAAI,OAAO,UAAU,QAAA,IAAY,MAAA,CAAO,SAAS,KAAK,CAAA,IAAK,QAAQ,CAAA,EAAG;AACpE,MAAA,OAAO,KAAA;AAAA,IACT;AAAA,EACF;AAEA,EAAA,OAAO,MAAA;AACT;AAEA,SAAS,kBAAA,CACP,QACA,IAAA,EACoB;AACpB,EAAA,KAAA,MAAW,OAAO,IAAA,EAAM;AACtB,IAAA,MAAM,KAAA,GAAQ,OAAO,GAAG,CAAA;AACxB,IAAA,IAAI,OAAO,KAAA,KAAU,QAAA,IAAY,KAAA,CAAM,IAAA,OAAW,EAAA,EAAI;AACpD,MAAA,OAAO,KAAA;AAAA,IACT;AAAA,EACF;AAEA,EAAA,OAAO,MAAA;AACT;AAEA,SAAS,MAAA,CACP,QACA,IAAA,EAC2B;AAC3B,EAAA,KAAA,MAAW,OAAO,IAAA,EAAM;AACtB,IAAA,MAAM,KAAA,GAAQ,OAAO,GAAG,CAAA;AACxB,IAAA,IACE,OAAO,KAAA,KAAU,QAAA,IACjB,eAAA,CAAgB,GAAA,CAAI,KAAsB,CAAA,EAC1C;AACA,MAAA,OAAO,KAAA;AAAA,IACT;AAAA,EACF;AAEA,EAAA,OAAO,MAAA;AACT;AAEA,SAAS,kBAAA,CACP,QACA,IAAA,EACkC;AAClC,EAAA,KAAA,MAAW,OAAO,IAAA,EAAM;AACtB,IAAA,MAAM,KAAA,GAAQ,OAAO,GAAG,CAAA;AACxB,IAAA,IAAI,OAAO,KAAA,KAAU,SAAA,IAAa,KAAA,KAAU,YAAA,EAAc;AACxD,MAAA,OAAO,KAAA;AAAA,IACT;AAAA,EACF;AAEA,EAAA,OAAO,MAAA;AACT;AAEA,SAAS,8BAA8B,KAAA,EAAsC;AAC3E,EAAA,IAAI,CAAC,KAAA,IAAS,OAAO,KAAA,KAAU,QAAA,SAAiB,EAAC;AAEjD,EAAA,MAAM,MAAA,GAAS,KAAA;AAEf,EAAA,OAAO;AAAA,IACL,0BAAA,EAA4B,YAAY,MAAA,EAAQ;AAAA,MAC9C,4BAAA;AAAA,MACA;AAAA,KACD,CAAA;AAAA,IACD,yBAAA,EAA2B,mBAAmB,MAAA,EAAQ;AAAA,MACpD,2BAAA;AAAA,MACA;AAAA,KACD,CAAA;AAAA,IACD,0BAAA,EAA4B,mBAAmB,MAAA,EAAQ;AAAA,MACrD;AAAA,KACD,CAAA;AAAA,IACD,wBAAA,EAA0B,mBAAmB,MAAA,EAAQ;AAAA,MACnD;AAAA,KACD,CAAA;AAAA,IACD,mBAAA,EAAqB,MAAA,CAAO,MAAA,EAAQ,CAAC,qBAAqB,CAAC,CAAA;AAAA,IAC3D,6BAAA,EAA+B,YAAY,MAAA,EAAQ;AAAA,MACjD;AAAA,KACD,CAAA;AAAA,IACD,+BAAA,EAAiC,mBAAmB,MAAA,EAAQ;AAAA,MAC1D;AAAA,KACD;AAAA,GACH;AACF;AAEO,SAAS,2BAAA,CACd,gBACA,eAAA,EAC8B;AAC9B,EAAA,MAAM,MAAA,GAAS,8BAA8B,cAAc,CAAA;AAC3D,EAAA,MAAM,OAAA,GAAU,8BAA8B,eAAe,CAAA;AAE7D,EAAA,OAAO;AAAA,IACL,gBAAA,EACE,OAAA,CAAQ,0BAAA,IACR,MAAA,CAAO,0BAAA,IACP,KAAA;AAAA,IACF,QAAA,EACE,OAAA,CAAQ,yBAAA,IAA6B,MAAA,CAAO,yBAAA;AAAA,IAC9C,SAAA,EACE,OAAA,CAAQ,0BAAA,IAA8B,MAAA,CAAO,0BAAA;AAAA,IAC/C,OAAA,EACE,OAAA,CAAQ,wBAAA,IAA4B,MAAA,CAAO,wBAAA;AAAA,IAC7C,EAAA,EAAI,OAAA,CAAQ,mBAAA,IAAuB,MAAA,CAAO,mBAAA;AAAA,IAC1C,YAAA,EACE,OAAA,CAAQ,6BAAA,IACR,MAAA,CAAO,6BAAA;AAAA,IACT,cAAA,EACE,OAAA,CAAQ,+BAAA,IACR,MAAA,CAAO;AAAA,GACX;AACF;AAEA,eAAe,iBAAiB,IAAA,EAAgC;AAC9D,EAAA,IAAI;AACF,IAAA,OAAO,KAAK,KAAA,CAAM,MAAM,QAAA,CAAS,IAAA,EAAM,OAAO,CAAC,CAAA;AAAA,EACjD,CAAA,CAAA,MAAQ;AACN,IAAA,OAAO,EAAC;AAAA,EACV;AACF;AAEA,eAAsB,wBAAA,CACpB,GAAA,EACA,QAAA,GAAW,WAAA,EAAY,EACgB;AACvC,EAAA,MAAM,iBAAiB,MAAM,gBAAA;AAAA,IAC3B,IAAA,CAAK,UAAU,eAAe;AAAA,GAChC;AACA,EAAA,MAAM,kBAAkB,MAAM,gBAAA;AAAA,IAC5B,IAAA,CAAK,GAAA,EAAK,KAAA,EAAO,eAAe;AAAA,GAClC;AAEA,EAAA,OAAO,2BAAA,CAA4B,gBAAgB,eAAe,CAAA;AACpE;;;AC1KA,IAAM,eAAA,GAAkB;AAAA,EACtB,wFAAA;AAAA,EACA,4FAAA;AAAA,EACA,yFAAA;AAAA,EACA;AACF,CAAA,CAAE,KAAK,GAAG,CAAA;AAEK,SAAR,sBAAuC,EAAA,EAAkB;AAC9D,EAAA,MAAM,WAAW,wBAAA,EAAyB;AAE1C,EAAA,EAAA,CAAG,YAAA,CAAa;AAAA,IACd,IAAA,EAAM,WAAA;AAAA,IACN,KAAA,EAAO,WAAA;AAAA,IACP,WAAA,EAAa,eAAA;AAAA,IACb,aAAA,EACE,iKAAA;AAAA,IACF,UAAA,EAAYC,KAAK,MAAA,CAAO;AAAA,MACtB,GAAG,uCAAuC,QAAQ,CAAA;AAAA,MAClD,SAASA,IAAAA,CAAK,QAAA;AAAA,QACZA,KAAK,OAAA,CAAQ;AAAA,UACX,WAAA,EACE;AAAA,SACH;AAAA;AACH,KACD,CAAA;AAAA,IAED,MAAM,OAAA,CAAQ,WAAA,EAAa,MAAA,EAAQ,OAAA,EAAS,WAAW,GAAA,EAAK;AAC1D,MAAA,MAAM,WAAW,MAAM,wBAAA,CAAyB,GAAA,CAAI,GAAA,EAAKC,aAAa,CAAA;AACtE,MAAA,MAAM,eAAA,GAAkB,yBAAyB,QAAQ,CAAA;AACzD,MAAA,MAAM,OAAA,GACH,MAAA,CAAO,OAAA,IAAmC,QAAA,CAAS,gBAAA;AACtD,MAAA,MAAM,MAAA,GAAS,MAAM,oBAAA,CAAqB,MAAA,EAAQ,eAAe,CAAA;AAEjE,MAAA,IAAI,OAAA,CAAQ,MAAM,CAAA,EAAG;AACnB,QAAA,OAAO;AAAA,UACL,OAAA,EAAS,CAAC,EAAE,IAAA,EAAM,MAAA,EAAQ,MAAM,CAAA,OAAA,EAAU,MAAA,CAAO,KAAK,CAAA,CAAA,EAAI,CAAA;AAAA,UAC1D,OAAA,EAAS,EAAE,KAAA,EAAO,IAAA,EAAM,OAAA;AAAQ,SAClC;AAAA,MACF;AAEA,MAAA,OAAO;AAAA,QACL,OAAA,EAAS;AAAA,UACP,EAAE,MAAM,MAAA,EAAQ,IAAA,EAAM,uBAAuB,MAAA,EAAQ,EAAE,OAAA,EAAS,CAAA;AAAE,SACpE;AAAA,QACA,OAAA,EAAS,EAAE,OAAA,EAAS,QAAA,EAAU,gBAAgB,QAAA;AAAS,OACzD;AAAA,IACF;AAAA,GACD,CAAA;AACH","file":"index.js","sourcesContent":["import type { FingerprintOs } from \"./types\";\n\nexport const DEFAULT_BROWSER = \"chrome_145\";\nexport const DEFAULT_OS: FingerprintOs = \"windows\";\nexport const DEFAULT_MAX_CHARS = 50_000;\nexport const DEFAULT_TIMEOUT_MS = 15_000;\nexport const DEFAULT_INCLUDE_REPLIES = \"extractors\" as const;\nexport const DEFAULT_ACCEPT_HEADER =\n \"text/html,application/xhtml+xml,application/xml;q=0.9,/;q=0.8\";\nexport const DEFAULT_JSON_ACCEPT_HEADER =\n \"application/json,text/json,application/ld+json;q=0.9,text/plain;q=0.8,/;q=0.7\";\nexport const DEFAULT_ACCEPT_LANGUAGE_HEADER = \"en-US,en;q=0.9\";\n","import { Defuddle } from \"defuddle/node\";\nimport { getProfiles, fetch as wreqFetch } from \"wreq-js\";\nimport type { FetchDependencies } from \"./types\";\n\nexport const runtimeDependencies: FetchDependencies = {\n fetch: wreqFetch,\n defuddle: Defuddle,\n getProfiles,\n};\n","import { parseHTML } from \"linkedom\";\n\n/** Apply linkedom polyfills that Defuddle expects (getComputedStyle, styleSheets). /\nexport function parseLinkedomHTML(html: string, url?: string): Document {\n const { document } = parseHTML(html);\n const doc = document as Document & Record<string, unknown>;\n const defaultView = doc.defaultView as\n \| (Window & {\n getComputedStyle?: (\n elt: Element,\n pseudoElt?: string \| null,\n ) => CSSStyleDeclaration;\n })\n \| undefined;\n\n if (!(doc as { styleSheets?: unknown }).styleSheets) {\n (doc as { styleSheets?: unknown }).styleSheets =\n [] as unknown as StyleSheetList;\n }\n\n if (defaultView && !defaultView.getComputedStyle) {\n defaultView.getComputedStyle = (() => ({\n display: \"\",\n })) as unknown as typeof defaultView.getComputedStyle;\n }\n\n if (url) {\n (doc as { URL?: string }).URL = url;\n }\n\n return document;\n}\n","import type { FetchError, FetchResult, OutputFormat } from \"./types\";\n\nfunction buildHeader(\n parts: Array<[label: string, value: string \| number \| undefined]>,\n) {\n return parts\n .filter(([, value]) => value !== undefined && value !== \"\")\n .map(([label, value]) => `> ${label}: ${value}`)\n .join(\"\\n\");\n}\n\nexport function markdownToText(markdown: string): string {\n return markdown\n .replace(/^#{1,6}\\s+/gm, \"\")\n .replace(/\\\\([^]+)\\\\/g, \"$1\")\n .replace(/\\([^]+)\\/g, \"$1\")\n .replace(/\\[([^\\]]+)\\]\$[^)]+\$/g, \"$1\")\n .replace(/!\\[[^\\]]\\]\$[^)]+\$/g, \"\")\n .replace(/^>\\s+/gm, \"\")\n .replace(/^[-+]\\s+/gm, \"• \")\n .replace(/`([^`]+)`/g, \"$1\");\n}\n\nexport function truncateContent(content: string, maxChars: number): string {\n if (content.length <= maxChars) return content;\n return `${content.slice(0, maxChars)}\\n\\n[... truncated]`;\n}\n\nexport function buildCompactMetadataHeader(result: FetchResult): string {\n return buildHeader([\n [\"URL\", result.finalUrl],\n [\"Title\", result.title],\n [\"Author\", result.author],\n [\"Published\", result.published],\n ]);\n}\n\nexport function buildMetadataHeader(result: FetchResult): string {\n return buildHeader([\n [\"URL\", result.finalUrl],\n [\"Title\", result.title],\n [\"Author\", result.author],\n [\"Published\", result.published],\n [\"Site\", result.site],\n [\"Language\", result.language],\n [\"Words\", result.wordCount],\n [\"Browser\", `${result.browser}/${result.os}`],\n ]);\n}\n\nexport function buildFetchResponseText(\n result: FetchResult,\n options: { verbose?: boolean } = {},\n): string {\n const header = options.verbose\n ? buildMetadataHeader(result)\n : buildCompactMetadataHeader(result);\n\n return header ? `${header}\\n\\n${result.content}` : result.content;\n}\n\nexport function estimateWordCount(content: string): number {\n const words = content.trim().match(/\\S+/g);\n return words?.length ?? 0;\n}\n\nexport function escapeHtml(value: string): string {\n return value\n .replace(/&/g, \"&\")\n .replace(/</g, \"<\")\n .replace(/>/g, \">\")\n .replace(/\"/g, \""\")\n .replace(/'/g, \"'\");\n}\n\nexport function parseAndFormatJson(\n raw: string,\n): { formatted: string } \| FetchError {\n try {\n return {\n formatted: JSON.stringify(JSON.parse(raw), null, 2),\n };\n } catch {\n return { error: \"Invalid JSON response\" };\n }\n}\n\nexport function renderJsonContent(\n formattedJson: string,\n format: OutputFormat,\n): string {\n switch (format) {\n case \"json\":\n case \"text\":\n return formattedJson;\n case \"html\":\n return `<pre><code class=\"language-json\">${escapeHtml(formattedJson)}</code></pre>`;\n default:\n return `\\`\\`\\`json\\n${formattedJson}\\n\\`\\`\\``;\n }\n}\n\nexport function stripExtractorComments(\n content: string,\n format: OutputFormat,\n): string {\n if (format === \"html\") {\n return content\n .replace(/\\s<hr>\\s<div class=\"[^\"] comments\">[\\s\\S]$/i, \"\")\n .trimEnd();\n }\n\n return content.replace(/\\n---\\n+## Comments\\n[\\s\\S]$/i, \"\").trimEnd();\n}\n","/*\n Core extraction pipeline: fetch with TLS fingerprinting → parse → Defuddle extract.\n * Separated from the plugin entry so it can be tested independently.\n /\n\nimport {\n DEFAULT_ACCEPT_HEADER,\n DEFAULT_ACCEPT_LANGUAGE_HEADER,\n DEFAULT_BROWSER,\n DEFAULT_INCLUDE_REPLIES,\n DEFAULT_JSON_ACCEPT_HEADER,\n DEFAULT_MAX_CHARS,\n DEFAULT_OS,\n DEFAULT_TIMEOUT_MS,\n} from \"./constants\";\nimport { runtimeDependencies } from \"./dependencies\";\nimport { parseLinkedomHTML } from \"./dom\";\nimport {\n estimateWordCount,\n markdownToText,\n parseAndFormatJson,\n renderJsonContent,\n stripExtractorComments,\n truncateContent,\n} from \"./format\";\nimport { getLatestChromeProfile as getLatestChromeProfileFrom } from \"./profiles\";\nimport type {\n FetchDependencies,\n FetchError,\n FetchOptions,\n FetchResult,\n OutputFormat,\n} from \"./types\";\n\nexport {\n DEFAULT_BROWSER,\n DEFAULT_INCLUDE_REPLIES,\n DEFAULT_MAX_CHARS,\n DEFAULT_OS,\n DEFAULT_TIMEOUT_MS,\n} from \"./constants\";\nexport type {\n FetchError,\n FetchOptions,\n FetchResult,\n OutputFormat,\n} from \"./types\";\n\nconst HTML_CONTENT_TYPES = [\n \"text/html\",\n \"application/xhtml+xml\",\n \"text/plain\",\n \"text/markdown\",\n];\n\nfunction resolveAcceptHeader(format: OutputFormat): string {\n return format === \"json\" ? DEFAULT_JSON_ACCEPT_HEADER : DEFAULT_ACCEPT_HEADER;\n}\n\nfunction isJsonContentType(contentType: string): boolean {\n const normalized = contentType.split(\";\")[0]?.trim().toLowerCase() ?? \"\";\n return (\n normalized === \"application/json\" \|\|\n normalized === \"text/json\" \|\|\n normalized.endsWith(\"+json\")\n );\n}\n\nfunction isLikelyJsonBody(body: string): boolean {\n const trimmed = body.trim();\n return trimmed.startsWith(\"{\") \|\| trimmed.startsWith(\"[\");\n}\n\nfunction isJsonResponse(contentType: string, body: string): boolean {\n return isJsonContentType(contentType) \|\| isLikelyJsonBody(body);\n}\n\nfunction buildJsonResult(\n opts: FetchOptions,\n finalUrl: string,\n rawBody: string,\n format: OutputFormat,\n maxChars: number,\n browser: string,\n os: string,\n): FetchResult \| FetchError {\n const parsedJson = parseAndFormatJson(rawBody);\n\n if (\"error\" in parsedJson) {\n return parsedJson;\n }\n\n const content = truncateContent(\n renderJsonContent(parsedJson.formatted, format),\n maxChars,\n );\n\n return {\n url: opts.url,\n finalUrl,\n title: \"\",\n author: \"\",\n published: \"\",\n site: new URL(finalUrl).hostname,\n language: \"\",\n wordCount: estimateWordCount(parsedJson.formatted),\n content,\n browser,\n os,\n };\n}\n\nfunction shouldStripReplies(site: string): boolean {\n return (\n site === \"Hacker News\" \|\|\n site.startsWith(\"r/\") \|\|\n site.startsWith(\"GitHub - \")\n );\n}\n\nexport function getLatestChromeProfile(): string {\n return getLatestChromeProfileFrom(runtimeDependencies.getProfiles);\n}\n\nexport function createDefuddleFetch(\n dependencies: FetchDependencies = runtimeDependencies,\n) {\n return async function defuddleFetch(\n opts: FetchOptions,\n ): Promise<FetchResult \| FetchError> {\n const browser = opts.browser ?? DEFAULT_BROWSER;\n const os = opts.os ?? DEFAULT_OS;\n const format: OutputFormat = opts.format ?? \"markdown\";\n const maxChars = opts.maxChars ?? DEFAULT_MAX_CHARS;\n const removeImages = opts.removeImages ?? false;\n const includeReplies = opts.includeReplies ?? DEFAULT_INCLUDE_REPLIES;\n const timeoutMs = opts.timeoutMs ?? DEFAULT_TIMEOUT_MS;\n\n let parsed: URL;\n try {\n parsed = new URL(opts.url);\n } catch {\n return { error: `Invalid URL: ${opts.url}` };\n }\n\n if (![\"http:\", \"https:\"].includes(parsed.protocol)) {\n return {\n error: `Only http/https URLs supported, got ${parsed.protocol}`,\n };\n }\n\n const fetchOptions: Record<string, unknown> = {\n browser,\n os,\n headers: {\n Accept: resolveAcceptHeader(format),\n \"Accept-Language\": DEFAULT_ACCEPT_LANGUAGE_HEADER,\n ...opts.headers,\n },\n redirect: \"follow\",\n timeout: timeoutMs,\n };\n\n if (opts.proxy) {\n fetchOptions.proxy = opts.proxy;\n }\n\n const response = await dependencies.fetch(opts.url, fetchOptions);\n\n if (!response.ok) {\n return {\n error: `HTTP ${response.status} ${response.statusText} for ${opts.url}`,\n };\n }\n\n const finalUrl = response.url ?? opts.url;\n const contentType = response.headers.get(\"content-type\") ?? \"\";\n const rawBody = await response.text();\n const jsonResponse = isJsonResponse(contentType, rawBody);\n\n if (format === \"json\") {\n if (!jsonResponse) {\n return { error: `Not a JSON response (content-type: ${contentType})` };\n }\n\n return buildJsonResult(\n opts,\n finalUrl,\n rawBody,\n format,\n maxChars,\n browser,\n os,\n );\n }\n\n if (jsonResponse) {\n return buildJsonResult(\n opts,\n finalUrl,\n rawBody,\n format,\n maxChars,\n browser,\n os,\n );\n }\n\n if (!HTML_CONTENT_TYPES.some((value) => contentType.includes(value))) {\n return { error: `Not an HTML page (content-type: ${contentType})` };\n }\n\n const document = parseLinkedomHTML(rawBody, finalUrl);\n const extracted = await dependencies.defuddle(document, finalUrl, {\n markdown: format !== \"html\",\n removeImages,\n includeReplies,\n });\n\n if (!extracted.content \|\| extracted.wordCount === 0) {\n return {\n error: `No content extracted from ${opts.url}. May need JS rendering or is blocked.`,\n };\n }\n\n let extractedContent = extracted.content;\n let wordCount = extracted.wordCount;\n\n if (includeReplies === false && shouldStripReplies(extracted.site ?? \"\")) {\n const strippedContent = stripExtractorComments(extractedContent, format);\n if (strippedContent !== extractedContent) {\n extractedContent = strippedContent;\n wordCount = estimateWordCount(\n format === \"text\"\n ? markdownToText(extractedContent)\n : extractedContent,\n );\n }\n }\n\n const normalizedContent =\n format === \"text\" ? markdownToText(extractedContent) : extractedContent;\n\n return {\n url: opts.url,\n finalUrl,\n title: extracted.title ?? \"\",\n author: extracted.author ?? \"\",\n published: extracted.published ?? \"\",\n site: extracted.site ?? \"\",\n language: extracted.language ?? \"\",\n wordCount,\n content: truncateContent(normalizedContent, maxChars),\n browser,\n os,\n };\n };\n}\n\nexport const defuddleFetch = createDefuddleFetch();\n\n/* Type guard: check if result is an error. */\nexport function isError(\n result: FetchResult \| FetchError,\n): result is FetchError {\n return \"error\" in result;\n}\n","import { type TSchema, Type } from \"@sinclair/typebox\";\nimport {\n DEFAULT_BROWSER,\n DEFAULT_INCLUDE_REPLIES,\n DEFAULT_MAX_CHARS,\n DEFAULT_OS,\n DEFAULT_TIMEOUT_MS,\n} from \"./constants\";\nimport { defuddleFetch } from \"./extract\";\nimport type {\n FetchError,\n FetchResult,\n FetchToolConfig,\n FetchToolDefaults,\n} from \"./types\";\n\nexport function resolveFetchToolDefaults(\n config: FetchToolConfig = {},\n): FetchToolDefaults {\n return {\n maxChars: config.maxChars ?? DEFAULT_MAX_CHARS,\n timeoutMs: config.timeoutMs ?? DEFAULT_TIMEOUT_MS,\n browser: config.browser ?? DEFAULT_BROWSER,\n os: config.os ?? DEFAULT_OS,\n removeImages: config.removeImages ?? false,\n includeReplies: config.includeReplies ?? DEFAULT_INCLUDE_REPLIES,\n };\n}\n\nexport function createBaseFetchToolParameterProperties(\n defaults: FetchToolDefaults,\n): Record<string, TSchema> {\n return {\n url: Type.String({ description: \"URL to fetch (http/https only)\" }),\n browser: Type.Optional(\n Type.String({\n description: `Browser profile for TLS fingerprinting. Default: \"${defaults.browser}\". Examples: chrome_145, firefox_147, safari_26, edge_145, opera_127`,\n }),\n ),\n os: Type.Optional(\n Type.String({\n description: `OS profile for fingerprinting. Default: \"${defaults.os}\". Options: windows, macos, linux, android, ios`,\n }),\n ),\n headers: Type.Optional(\n Type.Record(Type.String(), Type.String(), {\n description:\n \"Custom HTTP headers to send. By default, Accept and Accept-Language are set automatically.\",\n }),\n ),\n maxChars: Type.Optional(\n Type.Number({\n description: `Maximum characters to return. Default: ${defaults.maxChars}`,\n }),\n ),\n format: Type.Optional(\n Type.Union(\n [\n Type.Literal(\"markdown\"),\n Type.Literal(\"html\"),\n Type.Literal(\"text\"),\n Type.Literal(\"json\"),\n ],\n {\n description:\n 'Output format. \"markdown\" (default), \"html\" (cleaned HTML), \"text\" (plain text, no formatting), or \"json\" (pretty-printed JSON)',\n },\n ),\n ),\n removeImages: Type.Optional(\n Type.Boolean({\n description: \"Strip image references from output. Default: false\",\n }),\n ),\n includeReplies: Type.Optional(\n Type.Union([Type.Boolean(), Type.Literal(\"extractors\")], {\n description:\n \"Include replies/comments: 'extractors' for site-specific only (default), true for all, false for none\",\n }),\n ),\n proxy: Type.Optional(\n Type.String({\n description:\n \"Proxy URL (http://user:pass@host:port or socks5://host:port)\",\n }),\n ),\n };\n}\n\nexport async function executeFetchToolCall(\n params: Record<string, unknown>,\n defaults: FetchToolDefaults,\n): Promise<FetchResult \| FetchError> {\n return defuddleFetch({\n url: params.url as string,\n browser: (params.browser as string) ?? defaults.browser,\n os: (params.os as string) ?? defaults.os,\n headers: params.headers as Record<string, string> \| undefined,\n maxChars: (params.maxChars as number) ?? defaults.maxChars,\n format:\n (params.format as \"markdown\" \| \"html\" \| \"text\" \| \"json\") ?? \"markdown\",\n removeImages: (params.removeImages as boolean) ?? defaults.removeImages,\n includeReplies:\n (params.includeReplies as boolean \| \"extractors\") ??\n defaults.includeReplies,\n proxy: params.proxy as string \| undefined,\n timeoutMs: defaults.timeoutMs,\n });\n}\n","import { readFile } from \"node:fs/promises\";\nimport { join } from \"node:path\";\nimport { getAgentDir } from \"@mariozechner/pi-coding-agent\";\nimport type {\n FetchToolConfig,\n FingerprintOs,\n IncludeRepliesOption,\n} from \"smart-fetch-core\";\n\nconst VALID_OS_VALUES = new Set<FingerprintOs>([\n \"windows\",\n \"macos\",\n \"linux\",\n \"android\",\n \"ios\",\n]);\n\ninterface PiSmartFetchSettings {\n smartFetchVerboseByDefault?: boolean;\n smartFetchDefaultMaxChars?: number;\n smartFetchDefaultTimeoutMs?: number;\n smartFetchDefaultBrowser?: string;\n smartFetchDefaultOs?: FingerprintOs;\n smartFetchDefaultRemoveImages?: boolean;\n smartFetchDefaultIncludeReplies?: IncludeRepliesOption;\n}\n\nexport interface ResolvedPiSmartFetchSettings extends FetchToolConfig {\n verboseByDefault: boolean;\n}\n\nfunction readBoolean(\n source: Record<string, unknown>,\n keys: string[],\n): boolean \| undefined {\n for (const key of keys) {\n if (typeof source[key] === \"boolean\") {\n return source[key] as boolean;\n }\n }\n\n return undefined;\n}\n\nfunction readPositiveNumber(\n source: Record<string, unknown>,\n keys: string[],\n): number \| undefined {\n for (const key of keys) {\n const value = source[key];\n if (typeof value === \"number\" && Number.isFinite(value) && value > 0) {\n return value;\n }\n }\n\n return undefined;\n}\n\nfunction readNonEmptyString(\n source: Record<string, unknown>,\n keys: string[],\n): string \| undefined {\n for (const key of keys) {\n const value = source[key];\n if (typeof value === \"string\" && value.trim() !== \"\") {\n return value;\n }\n }\n\n return undefined;\n}\n\nfunction readOs(\n source: Record<string, unknown>,\n keys: string[],\n): FingerprintOs \| undefined {\n for (const key of keys) {\n const value = source[key];\n if (\n typeof value === \"string\" &&\n VALID_OS_VALUES.has(value as FingerprintOs)\n ) {\n return value as FingerprintOs;\n }\n }\n\n return undefined;\n}\n\nfunction readIncludeReplies(\n source: Record<string, unknown>,\n keys: string[],\n): IncludeRepliesOption \| undefined {\n for (const key of keys) {\n const value = source[key];\n if (typeof value === \"boolean\" \|\| value === \"extractors\") {\n return value;\n }\n }\n\n return undefined;\n}\n\nfunction normalizePiSmartFetchSettings(input: unknown): PiSmartFetchSettings {\n if (!input \|\| typeof input !== \"object\") return {};\n\n const source = input as Record<string, unknown>;\n\n return {\n smartFetchVerboseByDefault: readBoolean(source, [\n \"smartFetchVerboseByDefault\",\n \"webFetchVerboseByDefault\",\n ]),\n smartFetchDefaultMaxChars: readPositiveNumber(source, [\n \"smartFetchDefaultMaxChars\",\n \"webFetchDefaultMaxChars\",\n ]),\n smartFetchDefaultTimeoutMs: readPositiveNumber(source, [\n \"smartFetchDefaultTimeoutMs\",\n ]),\n smartFetchDefaultBrowser: readNonEmptyString(source, [\n \"smartFetchDefaultBrowser\",\n ]),\n smartFetchDefaultOs: readOs(source, [\"smartFetchDefaultOs\"]),\n smartFetchDefaultRemoveImages: readBoolean(source, [\n \"smartFetchDefaultRemoveImages\",\n ]),\n smartFetchDefaultIncludeReplies: readIncludeReplies(source, [\n \"smartFetchDefaultIncludeReplies\",\n ]),\n };\n}\n\nexport function resolvePiSmartFetchSettings(\n globalSettings: unknown,\n projectSettings: unknown,\n): ResolvedPiSmartFetchSettings {\n const global = normalizePiSmartFetchSettings(globalSettings);\n const project = normalizePiSmartFetchSettings(projectSettings);\n\n return {\n verboseByDefault:\n project.smartFetchVerboseByDefault ??\n global.smartFetchVerboseByDefault ??\n false,\n maxChars:\n project.smartFetchDefaultMaxChars ?? global.smartFetchDefaultMaxChars,\n timeoutMs:\n project.smartFetchDefaultTimeoutMs ?? global.smartFetchDefaultTimeoutMs,\n browser:\n project.smartFetchDefaultBrowser ?? global.smartFetchDefaultBrowser,\n os: project.smartFetchDefaultOs ?? global.smartFetchDefaultOs,\n removeImages:\n project.smartFetchDefaultRemoveImages ??\n global.smartFetchDefaultRemoveImages,\n includeReplies:\n project.smartFetchDefaultIncludeReplies ??\n global.smartFetchDefaultIncludeReplies,\n };\n}\n\nasync function readSettingsFile(path: string): Promise<unknown> {\n try {\n return JSON.parse(await readFile(path, \"utf-8\"));\n } catch {\n return {};\n }\n}\n\nexport async function loadPiSmartFetchSettings(\n cwd: string,\n agentDir = getAgentDir(),\n): Promise<ResolvedPiSmartFetchSettings> {\n const globalSettings = await readSettingsFile(\n join(agentDir, \"settings.json\"),\n );\n const projectSettings = await readSettingsFile(\n join(cwd, \".pi\", \"settings.json\"),\n );\n\n return resolvePiSmartFetchSettings(globalSettings, projectSettings);\n}\n","import { type ExtensionAPI, getAgentDir } from \"@mariozechner/pi-coding-agent\";\nimport { Type } from \"@sinclair/typebox\";\nimport {\n buildFetchResponseText,\n createBaseFetchToolParameterProperties,\n executeFetchToolCall,\n isError,\n resolveFetchToolDefaults,\n} from \"smart-fetch-core\";\nimport { loadPiSmartFetchSettings } from \"./settings\";\n\nconst toolDescription = [\n \"Fetch a URL with browser-grade TLS fingerprinting and extract clean, readable content.\",\n \"Uses wreq-js for browser-like TLS/HTTP2 impersonation and Defuddle for article extraction.\",\n \"Supports the same fetch parameters as the OpenClaw tool, plus an optional verbose flag.\",\n \"Does NOT execute JavaScript — use a browser automation tool for JS-heavy pages.\",\n].join(\" \");\n\nexport default function piSmartFetchExtension(pi: ExtensionAPI) {\n const defaults = resolveFetchToolDefaults();\n\n pi.registerTool({\n name: \"web_fetch\",\n label: \"web_fetch\",\n description: toolDescription,\n promptSnippet:\n \"web_fetch(url, browser?, os?, headers?, maxChars?, format?, removeImages?, includeReplies?, proxy?, verbose?): fetch browser-fingerprinted readable web content\",\n parameters: Type.Object({\n ...createBaseFetchToolParameterProperties(defaults),\n verbose: Type.Optional(\n Type.Boolean({\n description:\n \"Include the full metadata header (site, language, word count, browser fingerprint info). Default: false, or smartFetchVerboseByDefault from pi settings.\",\n }),\n ),\n }),\n\n async execute(_toolCallId, params, _signal, _onUpdate, ctx) {\n const settings = await loadPiSmartFetchSettings(ctx.cwd, getAgentDir());\n const runtimeDefaults = resolveFetchToolDefaults(settings);\n const verbose =\n (params.verbose as boolean \| undefined) ?? settings.verboseByDefault;\n const result = await executeFetchToolCall(params, runtimeDefaults);\n\n if (isError(result)) {\n return {\n content: [{ type: \"text\", text: `Error: ${result.error}` }],\n details: { error: true, verbose },\n };\n }\n\n return {\n content: [\n { type: \"text\", text: buildFetchResponseText(result, { verbose }) },\n ],\n details: { verbose, maxChars: runtimeDefaults.maxChars },\n };\n },\n });\n}\n"]}

package/package.json CHANGED Viewed

@@ -1,6 +1,6 @@
 {
   "name": "pi-smart-fetch",
-  "version": "0.1.7",
+  "version": "0.1.8",
   "type": "module",
   "description": "pi.dev smart fetch extension with browser-grade TLS fingerprinting and Defuddle extraction.",
   "license": "MIT",
@@ -38,7 +38,7 @@
   ],
   "engines": {
     "bun": ">=1.3.0",
-    "node": ">=22"
+    "node": ">=24"
   },
   "publishConfig": {
     "access": "public"