smart-web-mcp 0.5.0 → 0.5.2

package/dist/smartfetch/providers/hackernews.js

@@ -0,0 +1,154 @@
+import { asNumber, asString, dedupeUrls, extractAnchorHrefs, extractUrls, stripTags } from "../../shared.js";
+import { fetchProviderJson } from "../provider-policy.js";
+const USER_AGENT = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/134.0.0.0 Safari/537.36";
+function extractItemId(url) {
+    try {
+        const parsed = new URL(url);
+        const id = parsed.searchParams.get("id");
+        return id ? String(id).trim() : "";
+    }
+    catch {
+        return "";
+    }
+}
+function mapComment(node, outbound) {
+    if (!node || typeof node !== "object")
+        return null;
+    if (String(node.type || "") !== "comment")
+        return null;
+    const bodyHtml = asString(node.text);
+    for (const item of extractUrls(bodyHtml))
+        outbound.add(item);
+    for (const item of extractAnchorHrefs(bodyHtml))
+        outbound.add(item);
+    const children = Array.isArray(node.children)
+        ? node.children.map((item) => mapComment(item, outbound)).filter(Boolean)
+        : [];
+    return {
+        id: asString(node.id),
+        author: asString(node.author),
+        body: stripTags(bodyHtml),
+        created_at: asString(node.created_at),
+        children,
+    };
+}
+function countComments(node) {
+    if (!node || typeof node !== "object")
+        return 0;
+    const children = Array.isArray(node.children) ? node.children : [];
+    return (String(node.type || "") === "comment" ? 1 : 0)
+        + children.reduce((sum, item) => sum + countComments(item), 0);
+}
+export const hackernewsProvider = {
+    id: "hackernews",
+    matches: (_url, target) => target === "hackernews_post",
+    async normalize(context) {
+        const itemId = extractItemId(context.url);
+        if (!itemId) {
+            return {
+                post: null,
+                thread: [],
+                comments: [],
+                outbound_links: [],
+                partial: true,
+                errors: [
+                    ...context.errors,
+                    { category: "parse_error", code: "invalid_hackernews_url", message: "URL is not a Hacker News item URL" },
+                ],
+                method: context.active.method,
+            };
+        }
+        const result = await fetchProviderJson(`https://hn.algolia.com/api/v1/items/${encodeURIComponent(itemId)}`, context.timeoutMs, {
+            headers: {
+                accept: "application/json,text/plain,*/*",
+                "user-agent": USER_AGENT,
+            },
+        }, {
+            mode: "relay",
+            sourceUrl: context.url,
+        });
+        if (!result.ok || !result.data || typeof result.data !== "object") {
+            const providerError = !result.ok ? result.error : null;
+            return {
+                post: {
+                    url: context.url,
+                    text: stripTags(context.active.content),
+                    status: "partial_text_only",
+                },
+                thread: [],
+                comments: [],
+                outbound_links: dedupeUrls([...context.active.links, ...extractUrls(context.active.content)]),
+                partial: true,
+                errors: [
+                    ...context.errors,
+                    {
+                        category: providerError?.category || "unavailable",
+                        code: providerError?.code || "hackernews_item_failed",
+                        message: providerError?.message || `status=${result.status}`,
+                    },
+                ],
+                method: context.active.method,
+            };
+        }
+        const item = result.data;
+        const itemType = asString(item.type);
+        const selftext = asString(item.text);
+        const outbound = new Set();
+        for (const link of extractUrls(selftext))
+            outbound.add(link);
+        for (const link of extractAnchorHrefs(selftext))
+            outbound.add(link);
+        if (item.url)
+            outbound.add(String(item.url));
+        if (item.story_url)
+            outbound.add(String(item.story_url));
+        const comments = Array.isArray(item.children)
+            ? item.children.map((node) => mapComment(node, outbound)).filter(Boolean)
+            : [];
+        if (itemType === "comment") {
+            return {
+                post: {
+                    id: asString(item.id),
+                    title: item.story_title ? `Comment on: ${asString(item.story_title)}` : "Hacker News comment",
+                    author: asString(item.author),
+                    url: "",
+                    selftext: stripTags(selftext),
+                    score: asNumber(item.points),
+                    num_comments: Array.isArray(item.children)
+                        ? item.children.reduce((sum, node) => sum + countComments(node), 0)
+                        : asNumber(item.children_count),
+                    created_at: asString(item.created_at),
+                    discussion_url: context.url,
+                    story_title: asString(item.story_title),
+                    story_url: asString(item.story_url),
+                },
+                thread: [],
+                comments,
+                outbound_links: dedupeUrls([...outbound]),
+                partial: false,
+                errors: context.errors,
+                method: "hackernews_algolia_public",
+            };
+        }
+        return {
+            post: {
+                id: asString(item.id),
+                title: asString(item.title),
+                author: asString(item.author),
+                url: item.url ? asString(item.url) : "",
+                selftext: stripTags(selftext),
+                score: asNumber(item.points),
+                num_comments: Array.isArray(item.children) ? item.children.reduce((sum, node) => sum + countComments(node), 0) : asNumber(item.children_count),
+                created_at: asString(item.created_at),
+                discussion_url: context.url,
+            },
+            thread: [],
+            comments,
+            outbound_links: dedupeUrls([...outbound]),
+            partial: false,
+            errors: context.errors,
+            method: "hackernews_algolia_public",
+        };
+    },
+};
+//# sourceMappingURL=hackernews.js.map

package/dist/smartfetch/providers/hackernews.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"hackernews.js","sourceRoot":"","sources":["../../../src/smartfetch/providers/hackernews.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,QAAQ,EAAE,UAAU,EAAE,kBAAkB,EAAE,WAAW,EAAE,SAAS,EAAE,MAAM,iBAAiB,CAAA;AAC5G,OAAO,EAAE,iBAAiB,EAAE,MAAM,uBAAuB,CAAA;AAGzD,MAAM,UAAU,GAAG,iHAAiH,CAAA;AAEpI,SAAS,aAAa,CAAC,GAAW;IAChC,IAAI,CAAC;QACH,MAAM,MAAM,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,CAAA;QAC3B,MAAM,EAAE,GAAG,MAAM,CAAC,YAAY,CAAC,GAAG,CAAC,IAAI,CAAC,CAAA;QACxC,OAAO,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,EAAE,CAAA;IACpC,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,EAAE,CAAA;IACX,CAAC;AACH,CAAC;AAED,SAAS,UAAU,CAAC,IAAS,EAAE,QAAqB;IAClD,IAAI,CAAC,IAAI,IAAI,OAAO,IAAI,KAAK,QAAQ;QAAE,OAAO,IAAI,CAAA;IAClD,IAAI,MAAM,CAAC,IAAI,CAAC,IAAI,IAAI,EAAE,CAAC,KAAK,SAAS;QAAE,OAAO,IAAI,CAAA;IACtD,MAAM,QAAQ,GAAG,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;IACpC,KAAK,MAAM,IAAI,IAAI,WAAW,CAAC,QAAQ,CAAC;QAAE,QAAQ,CAAC,GAAG,CAAC,IAAI,CAAC,CAAA;IAC5D,KAAK,MAAM,IAAI,IAAI,kBAAkB,CAAC,QAAQ,CAAC;QAAE,QAAQ,CAAC,GAAG,CAAC,IAAI,CAAC,CAAA;IACnE,MAAM,QAAQ,GAAG,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,QAAQ,CAAC;QAC3C,CAAC,CAAC,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,IAAS,EAAE,EAAE,CAAC,UAAU,CAAC,IAAI,EAAE,QAAQ,CAAC,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC;QAC9E,CAAC,CAAC,EAAE,CAAA;IACN,OAAO;QACL,EAAE,EAAE,QAAQ,CAAC,IAAI,CAAC,EAAE,CAAC;QACrB,MAAM,EAAE,QAAQ,CAAC,IAAI,CAAC,MAAM,CAAC;QAC7B,IAAI,EAAE,SAAS,CAAC,QAAQ,CAAC;QACzB,UAAU,EAAE,QAAQ,CAAC,IAAI,CAAC,UAAU,CAAC;QACrC,QAAQ;KACT,CAAA;AACH,CAAC;AAED,SAAS,aAAa,CAAC,IAAS;IAC9B,IAAI,CAAC,IAAI,IAAI,OAAO,IAAI,KAAK,QAAQ;QAAE,OAAO,CAAC,CAAA;IAC/C,MAAM,QAAQ,GAAG,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC,EAAE,CAAA;IAClE,OAAO,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,IAAI,EAAE,CAAC,KAAK,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;UAClD,QAAQ,CAAC,MAAM,CAAC,CAAC,GAAW,EAAE,IAAS,EAAE,EAAE,CAAC,GAAG,GAAG,aAAa,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,CAAA;AAC/E,CAAC;AAED,MAAM,CAAC,MAAM,kBAAkB,GAAuB;IACpD,EAAE,EAAE,YAAY;IAChB,OAAO,EAAE,CAAC,IAAI,EAAE,MAAM,EAAE,EAAE,CAAC,MAAM,KAAK,iBAAiB;IACvD,KAAK,CAAC,SAAS,CAAC,OAA0B;QACxC,MAAM,MAAM,GAAG,aAAa,CAAC,OAAO,CAAC,GAAG,CAAC,CAAA;QACzC,IAAI,CAAC,MAAM,EAAE,CAAC;YACZ,OAAO;gBACL,IAAI,EAAE,IAAI;gBACV,MAAM,EAAE,EAAE;gBACV,QAAQ,EAAE,EAAE;gBACZ,cAAc,EAAE,EAAE;gBAClB,OAAO,EAAE,IAAI;gBACb,MAAM,EAAE;oBACN,GAAG,OAAO,CAAC,MAAM;oBACjB,EAAE,QAAQ,EAAE,aAAa,EAAE,IAAI,EAAE,wBAAwB,EAAE,OAAO,EAAE,mCAAmC,EAAE;iBAC1G;gBACD,MAAM,EAAE,OAAO,CAAC,MAAM,CAAC,MAAM;aAC9B,CAAA;QACH,CAAC;QAED,MAAM,MAAM,GAAG,MAAM,iBAAiB,CACpC,uCAAuC,kBAAkB,CAAC,MAAM,CAAC,EAAE,EACnE,OAAO,CAAC,SAAS,EACjB;YACE,OAAO,EAAE;gBACP,MAAM,EAAE,iCAAiC;gBACzC,YAAY,EAAE,UAAU;aACzB;SACF,EACD;YACE,IAAI,EAAE,OAAO;YACb,SAAS,EAAE,OAAO,CAAC,GAAG;SACvB,CACF,CAAA;QAED,IAAI,CAAC,MAAM,CAAC,EAAE,IAAI,CAAC,MAAM,CAAC,IAAI,IAAI,OAAO,MAAM,CAAC,IAAI,KAAK,QAAQ,EAAE,CAAC;YAClE,MAAM,aAAa,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,CAAA;YACtD,OAAO;gBACL,IAAI,EAAE;oBACJ,GAAG,EAAE,OAAO,CAAC,GAAG;oBAChB,IAAI,EAAE,SAAS,CAAC,OAAO,CAAC,MAAM,CAAC,OAAO,CAAC;oBACvC,MAAM,EAAE,mBAAmB;iBAC5B;gBACD,MAAM,EAAE,EAAE;gBACV,QAAQ,EAAE,EAAE;gBACZ,cAAc,EAAE,UAAU,CAAC,CAAC,GAAG,OAAO,CAAC,MAAM,CAAC,KAAK,EAAE,GAAG,WAAW,CAAC,OAAO,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC;gBAC7F,OAAO,EAAE,IAAI;gBACb,MAAM,EAAE;oBACN,GAAG,OAAO,CAAC,MAAM;oBACjB;wBACE,QAAQ,EAAE,aAAa,EAAE,QAAQ,IAAI,aAAa;wBAClD,IAAI,EAAE,aAAa,EAAE,IAAI,IAAI,wBAAwB;wBACrD,OAAO,EAAE,aAAa,EAAE,OAAO,IAAI,UAAU,MAAM,CAAC,MAAM,EAAE;qBAC7D;iBACF;gBACD,MAAM,EAAE,OAAO,CAAC,MAAM,CAAC,MAAM;aAC9B,CAAA;QACH,CAAC;QAED,MAAM,IAAI,GAAG,MAAM,CAAC,IAAW,CAAA;QAC/B,MAAM,QAAQ,GAAG,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;QACpC,MAAM,QAAQ,GAAG,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;QACpC,MAAM,QAAQ,GAAG,IAAI,GAAG,EAAU,CAAA;QAClC,KAAK,MAAM,IAAI,IAAI,WAAW,CAAC,QAAQ,CAAC;YAAE,QAAQ,CAAC,GAAG,CAAC,IAAI,CAAC,CAAA;QAC5D,KAAK,MAAM,IAAI,IAAI,kBAAkB,CAAC,QAAQ,CAAC;YAAE,QAAQ,CAAC,GAAG,CAAC,IAAI,CAAC,CAAA;QACnE,IAAI,IAAI,CAAC,GAAG;YAAE,QAAQ,CAAC,GAAG,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAA;QAC5C,IAAI,IAAI,CAAC,SAAS;YAAE,QAAQ,CAAC,GAAG,CAAC,MAAM,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,CAAA;QAExD,MAAM,QAAQ,GAAG,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,QAAQ,CAAC;YAC3C,CAAC,CAAC,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,IAAS,EAAE,EAAE,CAAC,UAAU,CAAC,IAAI,EAAE,QAAQ,CAAC,CAAC,CAAC,MAAM,CAAC,OAAO,CAA8B;YAC3G,CAAC,CAAC,EAAE,CAAA;QAEN,IAAI,QAAQ,KAAK,SAAS,EAAE,CAAC;YAC3B,OAAO;gBACL,IAAI,EAAE;oBACJ,EAAE,EAAE,QAAQ,CAAC,IAAI,CAAC,EAAE,CAAC;oBACrB,KAAK,EAAE,IAAI,CAAC,WAAW,CAAC,CAAC,CAAC,eAAe,QAAQ,CAAC,IAAI,CAAC,WAAW,CAAC,EAAE,CAAC,CAAC,CAAC,qBAAqB;oBAC7F,MAAM,EAAE,QAAQ,CAAC,IAAI,CAAC,MAAM,CAAC;oBAC7B,GAAG,EAAE,EAAE;oBACP,QAAQ,EAAE,SAAS,CAAC,QAAQ,CAAC;oBAC7B,KAAK,EAAE,QAAQ,CAAC,IAAI,CAAC,MAAM,CAAC;oBAC5B,YAAY,EAAE,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,QAAQ,CAAC;wBACxC,CAAC,CAAC,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,GAAW,EAAE,IAAS,EAAE,EAAE,CAAC,GAAG,GAAG,aAAa,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;wBAChF,CAAC,CAAC,QAAQ,CAAC,IAAI,CAAC,cAAc,CAAC;oBACjC,UAAU,EAAE,QAAQ,CAAC,IAAI,CAAC,UAAU,CAAC;oBACrC,cAAc,EAAE,OAAO,CAAC,GAAG;oBAC3B,WAAW,EAAE,QAAQ,CAAC,IAAI,CAAC,WAAW,CAAC;oBACvC,SAAS,EAAE,QAAQ,CAAC,IAAI,CAAC,SAAS,CAAC;iBACpC;gBACD,MAAM,EAAE,EAAE;gBACV,QAAQ;gBACR,cAAc,EAAE,UAAU,CAAC,CAAC,GAAG,QAAQ,CAAC,CAAC;gBACzC,OAAO,EAAE,KAAK;gBACd,MAAM,EAAE,OAAO,CAAC,MAAM;gBACtB,MAAM,EAAE,2BAA2B;aACpC,CAAA;QACH,CAAC;QAED,OAAO;YACL,IAAI,EAAE;gBACJ,EAAE,EAAE,QAAQ,CAAC,IAAI,CAAC,EAAE,CAAC;gBACrB,KAAK,EAAE,QAAQ,CAAC,IAAI,CAAC,KAAK,CAAC;gBAC3B,MAAM,EAAE,QAAQ,CAAC,IAAI,CAAC,MAAM,CAAC;gBAC7B,GAAG,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,QAAQ,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE;gBACvC,QAAQ,EAAE,SAAS,CAAC,QAAQ,CAAC;gBAC7B,KAAK,EAAE,QAAQ,CAAC,IAAI,CAAC,MAAM,CAAC;gBAC5B,YAAY,EAAE,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,GAAW,EAAE,IAAS,EAAE,EAAE,CAAC,GAAG,GAAG,aAAa,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,IAAI,CAAC,cAAc,CAAC;gBAC3J,UAAU,EAAE,QAAQ,CAAC,IAAI,CAAC,UAAU,CAAC;gBACrC,cAAc,EAAE,OAAO,CAAC,GAAG;aAC5B;YACD,MAAM,EAAE,EAAE;YACV,QAAQ;YACR,cAAc,EAAE,UAAU,CAAC,CAAC,GAAG,QAAQ,CAAC,CAAC;YACzC,OAAO,EAAE,KAAK;YACd,MAAM,EAAE,OAAO,CAAC,MAAM;YACtB,MAAM,EAAE,2BAA2B;SACpC,CAAA;IACH,CAAC;CACF,CAAA"}

package/dist/smartfetch/providers/index.js

@@ -1,12 +1,14 @@
 import { dcinsideProvider } from "./dcinside.js";
 import { genericProvider } from "./generic.js";
+import { hackernewsProvider } from "./hackernews.js";
 import { naverBlogProvider } from "./naver-blog.js";
+import { naverCafeProvider } from "./naver-cafe.js";
 import { redditProvider } from "./reddit.js";
 import { tistoryProvider } from "./tistory.js";
 import { velogProvider } from "./velog.js";
 import { xProvider } from "./x.js";
 import { youtubeProvider } from "./youtube.js";
-const providers = [redditProvider, xProvider, dcinsideProvider, youtubeProvider, naverBlogProvider, tistoryProvider, velogProvider, genericProvider];
+const providers = [hackernewsProvider, redditProvider, xProvider, dcinsideProvider, youtubeProvider, naverBlogProvider, naverCafeProvider, tistoryProvider, velogProvider, genericProvider];
 export function resolveSmartfetchProvider(url, target) {
     return providers.find((provider) => provider.matches(url, target)) || genericProvider;
 }

package/dist/smartfetch/providers/index.js.map

@@ -1 +1 @@
-{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../src/smartfetch/providers/index.ts"],"names":[],"mappings":"AAGA,OAAO,EAAE,gBAAgB,EAAE,MAAM,eAAe,CAAA;AAChD,OAAO,EAAE,eAAe,EAAE,MAAM,cAAc,CAAA;AAC9C,OAAO,EAAE,iBAAiB,EAAE,MAAM,iBAAiB,CAAA;AACnD,OAAO,EAAE,cAAc,EAAE,MAAM,aAAa,CAAA;AAC5C,OAAO,EAAE,eAAe,EAAE,MAAM,cAAc,CAAA;AAC9C,OAAO,EAAE,aAAa,EAAE,MAAM,YAAY,CAAA;AAC1C,OAAO,EAAE,SAAS,EAAE,MAAM,QAAQ,CAAA;AAClC,OAAO,EAAE,eAAe,EAAE,MAAM,cAAc,CAAA;AAE9C,MAAM,SAAS,GAAyB,CAAC,cAAc,EAAE,SAAS,EAAE,gBAAgB,EAAE,eAAe,EAAE,iBAAiB,EAAE,eAAe,EAAE,aAAa,EAAE,eAAe,CAAC,CAAA;AAE1K,MAAM,UAAU,yBAAyB,CAAC,GAAW,EAAE,MAAsB;IAC3E,OAAO,SAAS,CAAC,IAAI,CAAC,CAAC,QAAQ,EAAE,EAAE,CAAC,QAAQ,CAAC,OAAO,CAAC,GAAG,EAAE,MAAM,CAAC,CAAC,IAAI,eAAe,CAAA;AACvF,CAAC"}
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../src/smartfetch/providers/index.ts"],"names":[],"mappings":"AAGA,OAAO,EAAE,gBAAgB,EAAE,MAAM,eAAe,CAAA;AAChD,OAAO,EAAE,eAAe,EAAE,MAAM,cAAc,CAAA;AAC9C,OAAO,EAAE,kBAAkB,EAAE,MAAM,iBAAiB,CAAA;AACpD,OAAO,EAAE,iBAAiB,EAAE,MAAM,iBAAiB,CAAA;AACnD,OAAO,EAAE,iBAAiB,EAAE,MAAM,iBAAiB,CAAA;AACnD,OAAO,EAAE,cAAc,EAAE,MAAM,aAAa,CAAA;AAC5C,OAAO,EAAE,eAAe,EAAE,MAAM,cAAc,CAAA;AAC9C,OAAO,EAAE,aAAa,EAAE,MAAM,YAAY,CAAA;AAC1C,OAAO,EAAE,SAAS,EAAE,MAAM,QAAQ,CAAA;AAClC,OAAO,EAAE,eAAe,EAAE,MAAM,cAAc,CAAA;AAE9C,MAAM,SAAS,GAAyB,CAAC,kBAAkB,EAAE,cAAc,EAAE,SAAS,EAAE,gBAAgB,EAAE,eAAe,EAAE,iBAAiB,EAAE,iBAAiB,EAAE,eAAe,EAAE,aAAa,EAAE,eAAe,CAAC,CAAA;AAEjN,MAAM,UAAU,yBAAyB,CAAC,GAAW,EAAE,MAAsB;IAC3E,OAAO,SAAS,CAAC,IAAI,CAAC,CAAC,QAAQ,EAAE,EAAE,CAAC,QAAQ,CAAC,OAAO,CAAC,GAAG,EAAE,MAAM,CAAC,CAAC,IAAI,eAAe,CAAA;AACvF,CAAC"}

package/dist/smartfetch/providers/naver-cafe.d.ts

@@ -0,0 +1,2 @@
+import type { SmartfetchProvider } from "../provider-types.js";
+export declare const naverCafeProvider: SmartfetchProvider;

package/dist/smartfetch/providers/naver-cafe.js

@@ -0,0 +1,163 @@
+import { asNumber, asString, dedupeUrls, extractMetaDescription, extractMetaProperty, extractTitleFromHtml, extractUrls, fetchJson, isNaverCafeUrl, stripTags } from "../../shared.js";
+const USER_AGENT = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/134.0.0.0 Safari/537.36";
+function decodeJwtPayload(value) {
+    const parts = String(value || "").split(".");
+    if (parts.length < 2)
+        return null;
+    try {
+        const normalized = (parts[1] || "").replace(/-/g, "+").replace(/_/g, "/");
+        const padded = normalized + "=".repeat((4 - normalized.length % 4) % 4);
+        return JSON.parse(Buffer.from(padded, "base64").toString("utf-8"));
+    }
+    catch {
+        return null;
+    }
+}
+function parseNaverCafeIdentifiers(url, html) {
+    let art = "";
+    let cafeId = 0;
+    let articleId = 0;
+    try {
+        const parsed = new URL(url);
+        art = parsed.searchParams.get("art") || "";
+        cafeId = asNumber(parsed.searchParams.get("clubid"));
+        articleId = asNumber(parsed.searchParams.get("articleid"));
+        if (!articleId) {
+            const mobilePath = parsed.pathname.match(/\/cafes\/(\d+)\/articles\/(\d+)/i);
+            if (mobilePath) {
+                cafeId ||= asNumber(mobilePath[1]);
+                articleId ||= asNumber(mobilePath[2]);
+            }
+        }
+        if (!articleId) {
+            const parts = parsed.pathname.split("/").filter(Boolean);
+            const last = parts.at(-1) || "";
+            if (/^\d+$/.test(last))
+                articleId = asNumber(last);
+        }
+    }
+    catch {
+        // Keep the regex fallbacks below.
+    }
+    if (!art)
+        art = html.match(/[?&]art=([^"'&\s]+)/i)?.[1] || "";
+    if (!cafeId)
+        cafeId = asNumber(html.match(/g_sClubId\s*=\s*"(\d+)"/i)?.[1] || html.match(/[?&]clubid=(\d+)/i)?.[1]);
+    if (!articleId)
+        articleId = asNumber(html.match(/[?&]articleid=(\d+)/i)?.[1]);
+    const tokenPayload = art ? decodeJwtPayload(art) : null;
+    cafeId ||= asNumber(tokenPayload?.cafeId);
+    articleId ||= asNumber(tokenPayload?.articleId);
+    return {
+        art,
+        cafeId,
+        articleId,
+    };
+}
+function buildArticleApiUrl(cafeId, articleId, art) {
+    const apiUrl = new URL(`https://article.cafe.naver.com/gw/v4/cafes/${cafeId}/articles/${articleId}`);
+    if (art)
+        apiUrl.searchParams.set("art", art);
+    return apiUrl.toString();
+}
+function looksLikeNaverCafeArticle(url) {
+    try {
+        const parsed = new URL(url);
+        const path = parsed.pathname;
+        return /\/ArticleRead\.nhn$/i.test(path) || /\/ca-fe\/web\/cafes\/\d+\/articles\/\d+/i.test(path) || /\/[^/]+\/\d+$/i.test(path);
+    }
+    catch {
+        return false;
+    }
+}
+export const naverCafeProvider = {
+    id: "naver-cafe",
+    matches: (url, target) => target === "naver_cafe_post" || (target === "generic" && isNaverCafeUrl(url) && looksLikeNaverCafeArticle(url)),
+    async normalize(context) {
+        const fallbackTitle = extractMetaProperty(context.active.content, "og:title") || extractTitleFromHtml(context.active.content);
+        const fallbackDescription = extractMetaProperty(context.active.content, "og:description") || extractMetaDescription(context.active.content);
+        const fallbackText = stripTags(context.active.content);
+        const identifiers = parseNaverCafeIdentifiers(context.resolvedUrl || context.url, context.active.content);
+        if (!identifiers.cafeId || !identifiers.articleId || !identifiers.art) {
+            return {
+                post: {
+                    url: context.url,
+                    title: fallbackTitle,
+                    description: fallbackDescription,
+                    text: fallbackText.slice(0, 30000),
+                    extractor: "fallback",
+                    status: fallbackText ? "partial_text_only" : "blocked_or_unavailable",
+                },
+                thread: [],
+                comments: [],
+                outbound_links: dedupeUrls([...context.active.links, ...extractUrls(context.active.content)]),
+                partial: true,
+                errors: [
+                    ...context.errors,
+                    { category: "parse_error", code: "naver_cafe_public_token_missing", message: "Could not resolve the public Naver Cafe article token and identifiers" },
+                ],
+                method: context.active.method,
+            };
+        }
+        const apiUrl = buildArticleApiUrl(identifiers.cafeId, identifiers.articleId, identifiers.art);
+        const result = await fetchJson(apiUrl, context.timeoutMs, {
+            headers: {
+                accept: "application/json,text/plain,*/*",
+                "user-agent": USER_AGENT,
+                referer: context.resolvedUrl || context.url,
+            },
+        });
+        const article = result.ok ? result.data?.result?.article || {} : {};
+        const writer = article?.writer || {};
+        const menu = article?.menu || {};
+        const cafe = result.ok ? result.data?.result?.cafe || {} : {};
+        const contentHtml = asString(article?.contentHtml);
+        const body = stripTags(contentHtml);
+        if (!result.ok || !article?.subject || !body) {
+            return {
+                post: {
+                    url: context.url,
+                    canonical_url: apiUrl,
+                    title: asString(article?.subject) || fallbackTitle,
+                    description: fallbackDescription,
+                    text: (body || fallbackText).slice(0, 30000),
+                    author: asString(writer?.nick),
+                    extractor: body ? "naver_cafe_article_json" : "fallback",
+                    status: body || fallbackText ? "partial_text_only" : "blocked_or_unavailable",
+                },
+                thread: [],
+                comments: [],
+                outbound_links: dedupeUrls([...context.active.links, ...extractUrls(contentHtml || context.active.content)]),
+                partial: true,
+                errors: result.ok
+                    ? [...context.errors, { category: "parse_error", code: "naver_cafe_article_missing_body", message: "Naver Cafe article JSON did not include readable body content" }]
+                    : [...context.errors, { category: "unavailable", code: "naver_cafe_article_fetch_failed", message: result.error || `status=${result.status}` }],
+                method: result.ok ? "naver_cafe_article_json" : context.active.method,
+            };
+        }
+        return {
+            post: {
+                url: context.url,
+                canonical_url: apiUrl,
+                title: asString(article?.subject),
+                description: fallbackDescription,
+                text: body.slice(0, 50000),
+                author: asString(writer?.nick),
+                menu_name: asString(menu?.name),
+                cafe_name: asString(cafe?.name || cafe?.cafeName),
+                write_date: asString(article?.writeDate),
+                read_count: asNumber(article?.readCount),
+                comment_count: asNumber(article?.commentCount),
+                extractor: "naver_cafe_article_json",
+                status: "ok",
+            },
+            thread: [],
+            comments: [],
+            outbound_links: dedupeUrls([...context.active.links, ...extractUrls(contentHtml)]),
+            partial: false,
+            errors: context.errors,
+            method: "naver_cafe_article_json",
+        };
+    },
+};
+//# sourceMappingURL=naver-cafe.js.map

package/dist/smartfetch/providers/naver-cafe.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"naver-cafe.js","sourceRoot":"","sources":["../../../src/smartfetch/providers/naver-cafe.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,QAAQ,EAAE,UAAU,EAAE,sBAAsB,EAAE,mBAAmB,EAAE,oBAAoB,EAAE,WAAW,EAAE,SAAS,EAAE,cAAc,EAAE,SAAS,EAAE,MAAM,iBAAiB,CAAA;AAGtL,MAAM,UAAU,GAAG,iHAAiH,CAAA;AAEpI,SAAS,gBAAgB,CAAC,KAAa;IACrC,MAAM,KAAK,GAAG,MAAM,CAAC,KAAK,IAAI,EAAE,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC,CAAA;IAC5C,IAAI,KAAK,CAAC,MAAM,GAAG,CAAC;QAAE,OAAO,IAAI,CAAA;IACjC,IAAI,CAAC;QACH,MAAM,UAAU,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,OAAO,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC,OAAO,CAAC,IAAI,EAAE,GAAG,CAAC,CAAA;QACzE,MAAM,MAAM,GAAG,UAAU,GAAG,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,GAAG,UAAU,CAAC,MAAM,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,CAAA;QACvE,OAAO,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,IAAI,CAAC,MAAM,EAAE,QAAQ,CAAC,CAAC,QAAQ,CAAC,OAAO,CAAC,CAA4B,CAAA;IAC/F,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,IAAI,CAAA;IACb,CAAC;AACH,CAAC;AAED,SAAS,yBAAyB,CAAC,GAAW,EAAE,IAAY;IAC1D,IAAI,GAAG,GAAG,EAAE,CAAA;IACZ,IAAI,MAAM,GAAG,CAAC,CAAA;IACd,IAAI,SAAS,GAAG,CAAC,CAAA;IAEjB,IAAI,CAAC;QACH,MAAM,MAAM,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,CAAA;QAC3B,GAAG,GAAG,MAAM,CAAC,YAAY,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,EAAE,CAAA;QAC1C,MAAM,GAAG,QAAQ,CAAC,MAAM,CAAC,YAAY,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC,CAAA;QACpD,SAAS,GAAG,QAAQ,CAAC,MAAM,CAAC,YAAY,CAAC,GAAG,CAAC,WAAW,CAAC,CAAC,CAAA;QAE1D,IAAI,CAAC,SAAS,EAAE,CAAC;YACf,MAAM,UAAU,GAAG,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,kCAAkC,CAAC,CAAA;YAC5E,IAAI,UAAU,EAAE,CAAC;gBACf,MAAM,KAAK,QAAQ,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,CAAA;gBAClC,SAAS,KAAK,QAAQ,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,CAAA;YACvC,CAAC;QACH,CAAC;QAED,IAAI,CAAC,SAAS,EAAE,CAAC;YACf,MAAM,KAAK,GAAG,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAA;YACxD,MAAM,IAAI,GAAG,KAAK,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAA;YAC/B,IAAI,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC;gBAAE,SAAS,GAAG,QAAQ,CAAC,IAAI,CAAC,CAAA;QACpD,CAAC;IACH,CAAC;IAAC,MAAM,CAAC;QACP,kCAAkC;IACpC,CAAC;IAED,IAAI,CAAC,GAAG;QAAE,GAAG,GAAG,IAAI,CAAC,KAAK,CAAC,sBAAsB,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAA;IAC7D,IAAI,CAAC,MAAM;QAAE,MAAM,GAAG,QAAQ,CAAC,IAAI,CAAC,KAAK,CAAC,0BAA0B,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC,KAAK,CAAC,mBAAmB,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAA;IACnH,IAAI,CAAC,SAAS;QAAE,SAAS,GAAG,QAAQ,CAAC,IAAI,CAAC,KAAK,CAAC,sBAAsB,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAA;IAE7E,MAAM,YAAY,GAAG,GAAG,CAAC,CAAC,CAAC,gBAAgB,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,CAAA;IACvD,MAAM,KAAK,QAAQ,CAAC,YAAY,EAAE,MAAM,CAAC,CAAA;IACzC,SAAS,KAAK,QAAQ,CAAC,YAAY,EAAE,SAAS,CAAC,CAAA;IAE/C,OAAO;QACL,GAAG;QACH,MAAM;QACN,SAAS;KACV,CAAA;AACH,CAAC;AAED,SAAS,kBAAkB,CAAC,MAAc,EAAE,SAAiB,EAAE,GAAW;IACxE,MAAM,MAAM,GAAG,IAAI,GAAG,CAAC,8CAA8C,MAAM,aAAa,SAAS,EAAE,CAAC,CAAA;IACpG,IAAI,GAAG;QAAE,MAAM,CAAC,YAAY,CAAC,GAAG,CAAC,KAAK,EAAE,GAAG,CAAC,CAAA;IAC5C,OAAO,MAAM,CAAC,QAAQ,EAAE,CAAA;AAC1B,CAAC;AAED,SAAS,yBAAyB,CAAC,GAAW;IAC5C,IAAI,CAAC;QACH,MAAM,MAAM,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,CAAA;QAC3B,MAAM,IAAI,GAAG,MAAM,CAAC,QAAQ,CAAA;QAC5B,OAAO,sBAAsB,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,0CAA0C,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,gBAAgB,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;IAClI,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,KAAK,CAAA;IACd,CAAC;AACH,CAAC;AAED,MAAM,CAAC,MAAM,iBAAiB,GAAuB;IACnD,EAAE,EAAE,YAAY;IAChB,OAAO,EAAE,CAAC,GAAG,EAAE,MAAM,EAAE,EAAE,CAAC,MAAM,KAAK,iBAAiB,IAAI,CAAC,MAAM,KAAK,SAAS,IAAI,cAAc,CAAC,GAAG,CAAC,IAAI,yBAAyB,CAAC,GAAG,CAAC,CAAC;IACzI,KAAK,CAAC,SAAS,CAAC,OAA0B;QACxC,MAAM,aAAa,GAAG,mBAAmB,CAAC,OAAO,CAAC,MAAM,CAAC,OAAO,EAAE,UAAU,CAAC,IAAI,oBAAoB,CAAC,OAAO,CAAC,MAAM,CAAC,OAAO,CAAC,CAAA;QAC7H,MAAM,mBAAmB,GAAG,mBAAmB,CAAC,OAAO,CAAC,MAAM,CAAC,OAAO,EAAE,gBAAgB,CAAC,IAAI,sBAAsB,CAAC,OAAO,CAAC,MAAM,CAAC,OAAO,CAAC,CAAA;QAC3I,MAAM,YAAY,GAAG,SAAS,CAAC,OAAO,CAAC,MAAM,CAAC,OAAO,CAAC,CAAA;QACtD,MAAM,WAAW,GAAG,yBAAyB,CAAC,OAAO,CAAC,WAAW,IAAI,OAAO,CAAC,GAAG,EAAE,OAAO,CAAC,MAAM,CAAC,OAAO,CAAC,CAAA;QAEzG,IAAI,CAAC,WAAW,CAAC,MAAM,IAAI,CAAC,WAAW,CAAC,SAAS,IAAI,CAAC,WAAW,CAAC,GAAG,EAAE,CAAC;YACtE,OAAO;gBACL,IAAI,EAAE;oBACJ,GAAG,EAAE,OAAO,CAAC,GAAG;oBAChB,KAAK,EAAE,aAAa;oBACpB,WAAW,EAAE,mBAAmB;oBAChC,IAAI,EAAE,YAAY,CAAC,KAAK,CAAC,CAAC,EAAE,KAAK,CAAC;oBAClC,SAAS,EAAE,UAAU;oBACrB,MAAM,EAAE,YAAY,CAAC,CAAC,CAAC,mBAAmB,CAAC,CAAC,CAAC,wBAAwB;iBACtE;gBACD,MAAM,EAAE,EAAE;gBACV,QAAQ,EAAE,EAAE;gBACZ,cAAc,EAAE,UAAU,CAAC,CAAC,GAAG,OAAO,CAAC,MAAM,CAAC,KAAK,EAAE,GAAG,WAAW,CAAC,OAAO,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC;gBAC7F,OAAO,EAAE,IAAI;gBACb,MAAM,EAAE;oBACN,GAAG,OAAO,CAAC,MAAM;oBACjB,EAAE,QAAQ,EAAE,aAAa,EAAE,IAAI,EAAE,iCAAiC,EAAE,OAAO,EAAE,uEAAuE,EAAE;iBACvJ;gBACD,MAAM,EAAE,OAAO,CAAC,MAAM,CAAC,MAAM;aAC9B,CAAA;QACH,CAAC;QAED,MAAM,MAAM,GAAG,kBAAkB,CAAC,WAAW,CAAC,MAAM,EAAE,WAAW,CAAC,SAAS,EAAE,WAAW,CAAC,GAAG,CAAC,CAAA;QAC7F,MAAM,MAAM,GAAG,MAAM,SAAS,CAAC,MAAM,EAAE,OAAO,CAAC,SAAS,EAAE;YACxD,OAAO,EAAE;gBACP,MAAM,EAAE,iCAAiC;gBACzC,YAAY,EAAE,UAAU;gBACxB,OAAO,EAAE,OAAO,CAAC,WAAW,IAAI,OAAO,CAAC,GAAG;aAC5C;SACF,CAAC,CAAA;QAEF,MAAM,OAAO,GAAG,MAAM,CAAC,EAAE,CAAC,CAAC,CAAE,MAAM,CAAC,IAAY,EAAE,MAAM,EAAE,OAAO,IAAI,EAAE,CAAC,CAAC,CAAC,EAAE,CAAA;QAC5E,MAAM,MAAM,GAAG,OAAO,EAAE,MAAM,IAAI,EAAE,CAAA;QACpC,MAAM,IAAI,GAAG,OAAO,EAAE,IAAI,IAAI,EAAE,CAAA;QAChC,MAAM,IAAI,GAAG,MAAM,CAAC,EAAE,CAAC,CAAC,CAAE,MAAM,CAAC,IAAY,EAAE,MAAM,EAAE,IAAI,IAAI,EAAE,CAAC,CAAC,CAAC,EAAE,CAAA;QACtE,MAAM,WAAW,GAAG,QAAQ,CAAC,OAAO,EAAE,WAAW,CAAC,CAAA;QAClD,MAAM,IAAI,GAAG,SAAS,CAAC,WAAW,CAAC,CAAA;QAEnC,IAAI,CAAC,MAAM,CAAC,EAAE,IAAI,CAAC,OAAO,EAAE,OAAO,IAAI,CAAC,IAAI,EAAE,CAAC;YAC7C,OAAO;gBACL,IAAI,EAAE;oBACJ,GAAG,EAAE,OAAO,CAAC,GAAG;oBAChB,aAAa,EAAE,MAAM;oBACrB,KAAK,EAAE,QAAQ,CAAC,OAAO,EAAE,OAAO,CAAC,IAAI,aAAa;oBAClD,WAAW,EAAE,mBAAmB;oBAChC,IAAI,EAAE,CAAC,IAAI,IAAI,YAAY,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,KAAK,CAAC;oBAC5C,MAAM,EAAE,QAAQ,CAAC,MAAM,EAAE,IAAI,CAAC;oBAC9B,SAAS,EAAE,IAAI,CAAC,CAAC,CAAC,yBAAyB,CAAC,CAAC,CAAC,UAAU;oBACxD,MAAM,EAAE,IAAI,IAAI,YAAY,CAAC,CAAC,CAAC,mBAAmB,CAAC,CAAC,CAAC,wBAAwB;iBAC9E;gBACD,MAAM,EAAE,EAAE;gBACV,QAAQ,EAAE,EAAE;gBACZ,cAAc,EAAE,UAAU,CAAC,CAAC,GAAG,OAAO,CAAC,MAAM,CAAC,KAAK,EAAE,GAAG,WAAW,CAAC,WAAW,IAAI,OAAO,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC;gBAC5G,OAAO,EAAE,IAAI;gBACb,MAAM,EAAE,MAAM,CAAC,EAAE;oBACf,CAAC,CAAC,CAAC,GAAG,OAAO,CAAC,MAAM,EAAE,EAAE,QAAQ,EAAE,aAAa,EAAE,IAAI,EAAE,iCAAiC,EAAE,OAAO,EAAE,+DAA+D,EAAE,CAAC;oBACrK,CAAC,CAAC,CAAC,GAAG,OAAO,CAAC,MAAM,EAAE,EAAE,QAAQ,EAAE,aAAa,EAAE,IAAI,EAAE,iCAAiC,EAAE,OAAO,EAAE,MAAM,CAAC,KAAK,IAAI,UAAU,MAAM,CAAC,MAAM,EAAE,EAAE,CAAC;gBACjJ,MAAM,EAAE,MAAM,CAAC,EAAE,CAAC,CAAC,CAAC,yBAAyB,CAAC,CAAC,CAAC,OAAO,CAAC,MAAM,CAAC,MAAM;aACtE,CAAA;QACH,CAAC;QAED,OAAO;YACL,IAAI,EAAE;gBACJ,GAAG,EAAE,OAAO,CAAC,GAAG;gBAChB,aAAa,EAAE,MAAM;gBACrB,KAAK,EAAE,QAAQ,CAAC,OAAO,EAAE,OAAO,CAAC;gBACjC,WAAW,EAAE,mBAAmB;gBAChC,IAAI,EAAE,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,KAAK,CAAC;gBAC1B,MAAM,EAAE,QAAQ,CAAC,MAAM,EAAE,IAAI,CAAC;gBAC9B,SAAS,EAAE,QAAQ,CAAC,IAAI,EAAE,IAAI,CAAC;gBAC/B,SAAS,EAAE,QAAQ,CAAC,IAAI,EAAE,IAAI,IAAI,IAAI,EAAE,QAAQ,CAAC;gBACjD,UAAU,EAAE,QAAQ,CAAC,OAAO,EAAE,SAAS,CAAC;gBACxC,UAAU,EAAE,QAAQ,CAAC,OAAO,EAAE,SAAS,CAAC;gBACxC,aAAa,EAAE,QAAQ,CAAC,OAAO,EAAE,YAAY,CAAC;gBAC9C,SAAS,EAAE,yBAAyB;gBACpC,MAAM,EAAE,IAAI;aACb;YACD,MAAM,EAAE,EAAE;YACV,QAAQ,EAAE,EAAE;YACZ,cAAc,EAAE,UAAU,CAAC,CAAC,GAAG,OAAO,CAAC,MAAM,CAAC,KAAK,EAAE,GAAG,WAAW,CAAC,WAAW,CAAC,CAAC,CAAC;YAClF,OAAO,EAAE,KAAK;YACd,MAAM,EAAE,OAAO,CAAC,MAAM;YACtB,MAAM,EAAE,yBAAyB;SAClC,CAAA;IACH,CAAC;CACF,CAAA"}

package/dist/smartfetch/providers/youtube.d.ts

@@ -1,2 +1,16 @@
 import type { SmartfetchProvider } from "../provider-types.js";
+type TranscriptChunk = {
+    text: string;
+    duration: number;
+    offset: number;
+    lang?: string;
+};
+type YoutubeTranscriptModule = {
+    fetchTranscript: (url: string, options?: {
+        lang?: string;
+        fetch?: typeof fetch;
+    }) => Promise<TranscriptChunk[]>;
+};
+export declare function TEST_setYoutubeTranscriptLoader(loader?: () => Promise<YoutubeTranscriptModule>): void;
 export declare const youtubeProvider: SmartfetchProvider;
+export {};

package/dist/smartfetch/providers/youtube.js

@@ -1,6 +1,10 @@
-import { execFileSync } from "node:child_process";
 import { dedupeUrls, envEnabled, extractMetaDescription, extractMetaProperty, extractTitleFromHtml, extractUrls, stripTags } from "../../shared.js";
 import { fetchProviderJson } from "../provider-policy.js";
+const defaultYoutubeTranscriptLoader = () => import("youtube-transcript/dist/youtube-transcript.esm.js");
+let youtubeTranscriptLoader = defaultYoutubeTranscriptLoader;
+export function TEST_setYoutubeTranscriptLoader(loader) {
+    youtubeTranscriptLoader = loader || defaultYoutubeTranscriptLoader;
+}
 function allowYoutubeTranscript() {
     return envEnabled("SMARTFETCH_ENABLE_YOUTUBE_TRANSCRIPT", true);
 }
@@ -18,22 +22,22 @@ async function fetchYoutubeMetadata(url, timeoutMs) {
     return result.data;
 }
 async function fetchYoutubeTranscript(url, timeoutMs) {
+    const waitMs = Math.max(1000, timeoutMs);
+    const controller = new AbortController();
+    const signal = controller.signal;
+    const timer = setTimeout(() => controller.abort(new Error(`Transcript fetch timed out after ${waitMs}ms`)), waitMs);
     try {
-        const script = `
-      const mod = await import("youtube-transcript/dist/youtube-transcript.esm.js")
-      const transcript = await mod.fetchTranscript(${JSON.stringify(url)})
-      console.log(JSON.stringify(transcript))
-    `;
-        const output = execFileSync(process.execPath, ["--input-type=module", "-e", script], {
-            timeout: Math.max(1000, timeoutMs),
-            encoding: "utf-8",
-            maxBuffer: 5 * 1024 * 1024,
-        }).trim();
-        return (output ? JSON.parse(output) : []);
+        const mod = await youtubeTranscriptLoader();
+        return await mod.fetchTranscript(url, {
+            fetch: (input, init) => fetch(input, { ...init, signal }),
+        });
     }
     catch (error) {
         const message = error instanceof Error ? error.message : String(error);
-        throw new Error(message.toLowerCase().includes("timed out") ? `Transcript fetch timed out after ${timeoutMs}ms` : message);
+        throw new Error(signal.aborted || message.toLowerCase().includes("abort") || message.toLowerCase().includes("timed out") ? `Transcript fetch timed out after ${waitMs}ms` : message);
+    }
+    finally {
+        clearTimeout(timer);
     }
 }
 export const youtubeProvider = {

package/dist/smartfetch/providers/youtube.js.map

@@ -1 +1 @@
-{"version":3,"file":"youtube.js","sourceRoot":"","sources":["../../../src/smartfetch/providers/youtube.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAA;AAEjD,OAAO,EAAE,UAAU,EAAE,UAAU,EAAE,sBAAsB,EAAE,mBAAmB,EAAE,oBAAoB,EAAE,WAAW,EAAE,SAAS,EAAE,MAAM,iBAAiB,CAAA;AACnJ,OAAO,EAAE,iBAAiB,EAAE,MAAM,uBAAuB,CAAA;AAGzD,SAAS,sBAAsB;IAC7B,OAAO,UAAU,CAAC,sCAAsC,EAAE,IAAI,CAAC,CAAA;AACjE,CAAC;AAED,KAAK,UAAU,oBAAoB,CAAC,GAAW,EAAE,SAAiB;IAChE,MAAM,MAAM,GAAG,MAAM,iBAAiB,CACpC,sCAAsC,kBAAkB,CAAC,GAAG,CAAC,cAAc,EAC3E,SAAS,EACT;QACE,OAAO,EAAE;YACP,MAAM,EAAE,iCAAiC;SAC1C;KACF,EACD;QACE,IAAI,EAAE,OAAO;QACb,SAAS,EAAE,GAAG;KACf,CACF,CAAA;IACD,IAAI,CAAC,MAAM,CAAC,EAAE,IAAI,CAAC,MAAM,CAAC,IAAI,IAAI,OAAO,MAAM,CAAC,IAAI,KAAK,QAAQ;QAAE,OAAO,IAAI,CAAA;IAC9E,OAAO,MAAM,CAAC,IAA+B,CAAA;AAC/C,CAAC;AAED,KAAK,UAAU,sBAAsB,CAAC,GAAW,EAAE,SAAiB;IAClE,IAAI,CAAC;QACH,MAAM,MAAM,GAAG;;qDAEkC,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC;;KAEnE,CAAA;QACD,MAAM,MAAM,GAAG,YAAY,CAAC,OAAO,CAAC,QAAQ,EAAE,CAAC,qBAAqB,EAAE,IAAI,EAAE,MAAM,CAAC,EAAE;YACnF,OAAO,EAAE,IAAI,CAAC,GAAG,CAAC,IAAI,EAAE,SAAS,CAAC;YAClC,QAAQ,EAAE,OAAO;YACjB,SAAS,EAAE,CAAC,GAAG,IAAI,GAAG,IAAI;SAC3B,CAAC,CAAC,IAAI,EAAE,CAAA;QACT,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,EAAE,CAAsB,CAAA;IAChE,CAAC;IAAC,OAAO,KAAc,EAAE,CAAC;QACxB,MAAM,OAAO,GAAG,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAA;QACtE,MAAM,IAAI,KAAK,CAAC,OAAO,CAAC,WAAW,EAAE,CAAC,QAAQ,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC,oCAAoC,SAAS,IAAI,CAAC,CAAC,CAAC,OAAO,CAAC,CAAA;IAC5H,CAAC;AACH,CAAC;AAED,MAAM,CAAC,MAAM,eAAe,GAAuB;IACjD,EAAE,EAAE,SAAS;IACb,OAAO,EAAE,CAAC,IAAI,EAAE,MAAM,EAAE,EAAE,CAAC,MAAM,KAAK,eAAe;IACrD,KAAK,CAAC,SAAS,CAAC,OAA0B;QACxC,MAAM,MAAM,GAAG,MAAM,oBAAoB,CAAC,OAAO,CAAC,GAAG,EAAE,OAAO,CAAC,SAAS,CAAC,CAAA;QACzE,MAAM,KAAK,GAAG,mBAAmB,CAAC,OAAO,CAAC,MAAM,CAAC,OAAO,EAAE,UAAU,CAAC,IAAI,MAAM,CAAC,MAAM,EAAE,KAAK,IAAI,EAAE,CAAC,IAAI,oBAAoB,CAAC,OAAO,CAAC,MAAM,CAAC,OAAO,CAAC,CAAA;QACpJ,MAAM,WAAW,GAAG,mBAAmB,CAAC,OAAO,CAAC,MAAM,CAAC,OAAO,EAAE,gBAAgB,CAAC,IAAI,sBAAsB,CAAC,OAAO,CAAC,MAAM,CAAC,OAAO,CAAC,CAAA;QACnI,MAAM,YAAY,GAAG,SAAS,CAAC,OAAO,CAAC,MAAM,CAAC,OAAO,CAAC,CAAA;QACtD,MAAM,MAAM,GAAG,MAAM,CAAC,MAAM,EAAE,WAAW,IAAI,EAAE,CAAC,CAAA;QAChD,MAAM,SAAS,GAAG,MAAM,CAAC,MAAM,EAAE,aAAa,IAAI,EAAE,CAAC,CAAA;QAErD,IAAI,CAAC,sBAAsB,EAAE,EAAE,CAAC;YAC9B,OAAO;gBACL,IAAI,EAAE;oBACJ,GAAG,EAAE,OAAO,CAAC,GAAG;oBAChB,KAAK;oBACL,WAAW;oBACX,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;oBAC7B,GAAG,CAAC,SAAS,CAAC,CAAC,CAAC,EAAE,SAAS,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;oBACnC,IAAI,EAAE,YAAY,CAAC,KAAK,CAAC,CAAC,EAAE,KAAK,CAAC;oBAClC,oBAAoB,EAAE,KAAK;oBAC3B,MAAM,EAAE,YAAY,CAAC,CAAC,CAAC,mBAAmB,CAAC,CAAC,CAAC,wBAAwB;iBACtE;gBACD,MAAM,EAAE,EAAE;gBACV,QAAQ,EAAE,EAAE;gBACZ,cAAc,EAAE,UAAU,CAAC,CAAC,GAAG,OAAO,CAAC,MAAM,CAAC,KAAK,EAAE,GAAG,WAAW,CAAC,OAAO,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC;gBAC7F,OAAO,EAAE,IAAI;gBACb,MAAM,EAAE;oBACN,GAAG,OAAO,CAAC,MAAM;oBACjB,EAAE,QAAQ,EAAE,aAAa,EAAE,IAAI,EAAE,6BAA6B,EAAE,OAAO,EAAE,yDAAyD,EAAE;iBACrI;gBACD,MAAM,EAAE,OAAO,CAAC,MAAM,CAAC,MAAM;aAC9B,CAAA;QACH,CAAC;QAED,IAAI,CAAC;YACH,MAAM,UAAU,GAAG,MAAM,sBAAsB,CAAC,OAAO,CAAC,GAAG,EAAE,OAAO,CAAC,SAAS,CAAC,CAAA;YAC/E,MAAM,cAAc,GAAG,UAAU,CAAC,GAAG,CAAC,CAAC,IAAqB,EAAE,EAAE,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,IAAI,EAAE,CAAC,CAAC,OAAO,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAA;YAC/I,OAAO;gBACL,IAAI,EAAE;oBACJ,GAAG,EAAE,OAAO,CAAC,GAAG;oBAChB,KAAK;oBACL,WAAW;oBACX,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;oBAC7B,GAAG,CAAC,SAAS,CAAC,CAAC,CAAC,EAAE,SAAS,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;oBACnC,IAAI,EAAE,cAAc,CAAC,KAAK,CAAC,CAAC,EAAE,KAAK,CAAC;oBACpC,oBAAoB,EAAE,OAAO,CAAC,cAAc,CAAC;oBAC7C,wBAAwB,EAAE,UAAU,CAAC,MAAM;oBAC3C,MAAM,EAAE,cAAc,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,wBAAwB;iBACzD;gBACD,MAAM,EAAE,EAAE;gBACV,QAAQ,EAAE,EAAE;gBACZ,cAAc,EAAE,UAAU,CAAC,CAAC,GAAG,OAAO,CAAC,MAAM,CAAC,KAAK,EAAE,GAAG,WAAW,CAAC,OAAO,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC;gBAC7F,OAAO,EAAE,CAAC,cAAc;gBACxB,MAAM,EAAE,OAAO,CAAC,MAAM;gBACtB,MAAM,EAAE,oBAAoB;aAC7B,CAAA;QACH,CAAC;QAAC,OAAO,KAAc,EAAE,CAAC;YACxB,MAAM,OAAO,GAAG,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAA;YACtE,OAAO;gBACL,IAAI,EAAE;oBACJ,GAAG,EAAE,OAAO,CAAC,GAAG;oBAChB,KAAK;oBACL,WAAW;oBACX,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;oBAC7B,GAAG,CAAC,SAAS,CAAC,CAAC,CAAC,EAAE,SAAS,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;oBACnC,IAAI,EAAE,YAAY,CAAC,KAAK,CAAC,CAAC,EAAE,KAAK,CAAC;oBAClC,oBAAoB,EAAE,KAAK;oBAC3B,MAAM,EAAE,YAAY,CAAC,CAAC,CAAC,mBAAmB,CAAC,CAAC,CAAC,wBAAwB;iBACtE;gBACD,MAAM,EAAE,EAAE;gBACV,QAAQ,EAAE,EAAE;gBACZ,cAAc,EAAE,UAAU,CAAC,CAAC,GAAG,OAAO,CAAC,MAAM,CAAC,KAAK,EAAE,GAAG,WAAW,CAAC,OAAO,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC;gBAC7F,OAAO,EAAE,IAAI;gBACb,MAAM,EAAE;oBACN,GAAG,OAAO,CAAC,MAAM;oBACjB,EAAE,QAAQ,EAAE,aAAa,EAAE,IAAI,EAAE,gCAAgC,EAAE,OAAO,EAAE;iBAC7E;gBACD,MAAM,EAAE,OAAO,CAAC,MAAM,CAAC,MAAM;aAC9B,CAAA;QACH,CAAC;IACH,CAAC;CACF,CAAA"}
+{"version":3,"file":"youtube.js","sourceRoot":"","sources":["../../../src/smartfetch/providers/youtube.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,UAAU,EAAE,sBAAsB,EAAE,mBAAmB,EAAE,oBAAoB,EAAE,WAAW,EAAE,SAAS,EAAE,MAAM,iBAAiB,CAAA;AACnJ,OAAO,EAAE,iBAAiB,EAAE,MAAM,uBAAuB,CAAA;AAczD,MAAM,8BAA8B,GAAG,GAAG,EAAE,CAAC,MAAM,CAAC,mDAAmD,CAAqC,CAAA;AAE5I,IAAI,uBAAuB,GAAG,8BAA8B,CAAA;AAE5D,MAAM,UAAU,+BAA+B,CAAC,MAA+C;IAC7F,uBAAuB,GAAG,MAAM,IAAI,8BAA8B,CAAA;AACpE,CAAC;AAED,SAAS,sBAAsB;IAC7B,OAAO,UAAU,CAAC,sCAAsC,EAAE,IAAI,CAAC,CAAA;AACjE,CAAC;AAED,KAAK,UAAU,oBAAoB,CAAC,GAAW,EAAE,SAAiB;IAChE,MAAM,MAAM,GAAG,MAAM,iBAAiB,CACpC,sCAAsC,kBAAkB,CAAC,GAAG,CAAC,cAAc,EAC3E,SAAS,EACT;QACE,OAAO,EAAE;YACP,MAAM,EAAE,iCAAiC;SAC1C;KACF,EACD;QACE,IAAI,EAAE,OAAO;QACb,SAAS,EAAE,GAAG;KACf,CACF,CAAA;IACD,IAAI,CAAC,MAAM,CAAC,EAAE,IAAI,CAAC,MAAM,CAAC,IAAI,IAAI,OAAO,MAAM,CAAC,IAAI,KAAK,QAAQ;QAAE,OAAO,IAAI,CAAA;IAC9E,OAAO,MAAM,CAAC,IAA+B,CAAA;AAC/C,CAAC;AAED,KAAK,UAAU,sBAAsB,CAAC,GAAW,EAAE,SAAiB;IAClE,MAAM,MAAM,GAAG,IAAI,CAAC,GAAG,CAAC,IAAI,EAAE,SAAS,CAAC,CAAA;IACxC,MAAM,UAAU,GAAG,IAAI,eAAe,EAAE,CAAA;IACxC,MAAM,MAAM,GAAG,UAAU,CAAC,MAAM,CAAA;IAChC,MAAM,KAAK,GAAG,UAAU,CAAC,GAAG,EAAE,CAAC,UAAU,CAAC,KAAK,CAAC,IAAI,KAAK,CAAC,oCAAoC,MAAM,IAAI,CAAC,CAAC,EAAE,MAAM,CAAC,CAAA;IACnH,IAAI,CAAC;QACH,MAAM,GAAG,GAAG,MAAM,uBAAuB,EAAE,CAAA;QAC3C,OAAO,MAAM,GAAG,CAAC,eAAe,CAAC,GAAG,EAAE;YACpC,KAAK,EAAE,CAAC,KAA6B,EAAE,IAAkB,EAAE,EAAE,CAAC,KAAK,CAAC,KAAK,EAAE,EAAE,GAAG,IAAI,EAAE,MAAM,EAAE,CAAC;SACzF,CAAsB,CAAA;IAChC,CAAC;IAAC,OAAO,KAAc,EAAE,CAAC;QACxB,MAAM,OAAO,GAAG,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAA;QACtE,MAAM,IAAI,KAAK,CAAC,MAAM,CAAC,OAAO,IAAI,OAAO,CAAC,WAAW,EAAE,CAAC,QAAQ,CAAC,OAAO,CAAC,IAAI,OAAO,CAAC,WAAW,EAAE,CAAC,QAAQ,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC,oCAAoC,MAAM,IAAI,CAAC,CAAC,CAAC,OAAO,CAAC,CAAA;IACtL,CAAC;YAAS,CAAC;QACT,YAAY,CAAC,KAAK,CAAC,CAAA;IACrB,CAAC;AACH,CAAC;AAED,MAAM,CAAC,MAAM,eAAe,GAAuB;IACjD,EAAE,EAAE,SAAS;IACb,OAAO,EAAE,CAAC,IAAI,EAAE,MAAM,EAAE,EAAE,CAAC,MAAM,KAAK,eAAe;IACrD,KAAK,CAAC,SAAS,CAAC,OAA0B;QACxC,MAAM,MAAM,GAAG,MAAM,oBAAoB,CAAC,OAAO,CAAC,GAAG,EAAE,OAAO,CAAC,SAAS,CAAC,CAAA;QACzE,MAAM,KAAK,GAAG,mBAAmB,CAAC,OAAO,CAAC,MAAM,CAAC,OAAO,EAAE,UAAU,CAAC,IAAI,MAAM,CAAC,MAAM,EAAE,KAAK,IAAI,EAAE,CAAC,IAAI,oBAAoB,CAAC,OAAO,CAAC,MAAM,CAAC,OAAO,CAAC,CAAA;QACpJ,MAAM,WAAW,GAAG,mBAAmB,CAAC,OAAO,CAAC,MAAM,CAAC,OAAO,EAAE,gBAAgB,CAAC,IAAI,sBAAsB,CAAC,OAAO,CAAC,MAAM,CAAC,OAAO,CAAC,CAAA;QACnI,MAAM,YAAY,GAAG,SAAS,CAAC,OAAO,CAAC,MAAM,CAAC,OAAO,CAAC,CAAA;QACtD,MAAM,MAAM,GAAG,MAAM,CAAC,MAAM,EAAE,WAAW,IAAI,EAAE,CAAC,CAAA;QAChD,MAAM,SAAS,GAAG,MAAM,CAAC,MAAM,EAAE,aAAa,IAAI,EAAE,CAAC,CAAA;QAErD,IAAI,CAAC,sBAAsB,EAAE,EAAE,CAAC;YAC9B,OAAO;gBACL,IAAI,EAAE;oBACJ,GAAG,EAAE,OAAO,CAAC,GAAG;oBAChB,KAAK;oBACL,WAAW;oBACX,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;oBAC7B,GAAG,CAAC,SAAS,CAAC,CAAC,CAAC,EAAE,SAAS,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;oBACnC,IAAI,EAAE,YAAY,CAAC,KAAK,CAAC,CAAC,EAAE,KAAK,CAAC;oBAClC,oBAAoB,EAAE,KAAK;oBAC3B,MAAM,EAAE,YAAY,CAAC,CAAC,CAAC,mBAAmB,CAAC,CAAC,CAAC,wBAAwB;iBACtE;gBACD,MAAM,EAAE,EAAE;gBACV,QAAQ,EAAE,EAAE;gBACZ,cAAc,EAAE,UAAU,CAAC,CAAC,GAAG,OAAO,CAAC,MAAM,CAAC,KAAK,EAAE,GAAG,WAAW,CAAC,OAAO,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC;gBAC7F,OAAO,EAAE,IAAI;gBACb,MAAM,EAAE;oBACN,GAAG,OAAO,CAAC,MAAM;oBACjB,EAAE,QAAQ,EAAE,aAAa,EAAE,IAAI,EAAE,6BAA6B,EAAE,OAAO,EAAE,yDAAyD,EAAE;iBACrI;gBACD,MAAM,EAAE,OAAO,CAAC,MAAM,CAAC,MAAM;aAC9B,CAAA;QACH,CAAC;QAED,IAAI,CAAC;YACH,MAAM,UAAU,GAAG,MAAM,sBAAsB,CAAC,OAAO,CAAC,GAAG,EAAE,OAAO,CAAC,SAAS,CAAC,CAAA;YAC/E,MAAM,cAAc,GAAG,UAAU,CAAC,GAAG,CAAC,CAAC,IAAqB,EAAE,EAAE,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,IAAI,EAAE,CAAC,CAAC,OAAO,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAA;YAC/I,OAAO;gBACL,IAAI,EAAE;oBACJ,GAAG,EAAE,OAAO,CAAC,GAAG;oBAChB,KAAK;oBACL,WAAW;oBACX,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;oBAC7B,GAAG,CAAC,SAAS,CAAC,CAAC,CAAC,EAAE,SAAS,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;oBACnC,IAAI,EAAE,cAAc,CAAC,KAAK,CAAC,CAAC,EAAE,KAAK,CAAC;oBACpC,oBAAoB,EAAE,OAAO,CAAC,cAAc,CAAC;oBAC7C,wBAAwB,EAAE,UAAU,CAAC,MAAM;oBAC3C,MAAM,EAAE,cAAc,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,wBAAwB;iBACzD;gBACD,MAAM,EAAE,EAAE;gBACV,QAAQ,EAAE,EAAE;gBACZ,cAAc,EAAE,UAAU,CAAC,CAAC,GAAG,OAAO,CAAC,MAAM,CAAC,KAAK,EAAE,GAAG,WAAW,CAAC,OAAO,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC;gBAC7F,OAAO,EAAE,CAAC,cAAc;gBACxB,MAAM,EAAE,OAAO,CAAC,MAAM;gBACtB,MAAM,EAAE,oBAAoB;aAC7B,CAAA;QACH,CAAC;QAAC,OAAO,KAAc,EAAE,CAAC;YACxB,MAAM,OAAO,GAAG,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAA;YACtE,OAAO;gBACL,IAAI,EAAE;oBACJ,GAAG,EAAE,OAAO,CAAC,GAAG;oBAChB,KAAK;oBACL,WAAW;oBACX,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;oBAC7B,GAAG,CAAC,SAAS,CAAC,CAAC,CAAC,EAAE,SAAS,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;oBACnC,IAAI,EAAE,YAAY,CAAC,KAAK,CAAC,CAAC,EAAE,KAAK,CAAC;oBAClC,oBAAoB,EAAE,KAAK;oBAC3B,MAAM,EAAE,YAAY,CAAC,CAAC,CAAC,mBAAmB,CAAC,CAAC,CAAC,wBAAwB;iBACtE;gBACD,MAAM,EAAE,EAAE;gBACV,QAAQ,EAAE,EAAE;gBACZ,cAAc,EAAE,UAAU,CAAC,CAAC,GAAG,OAAO,CAAC,MAAM,CAAC,KAAK,EAAE,GAAG,WAAW,CAAC,OAAO,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC;gBAC7F,OAAO,EAAE,IAAI;gBACb,MAAM,EAAE;oBACN,GAAG,OAAO,CAAC,MAAM;oBACjB,EAAE,QAAQ,EAAE,aAAa,EAAE,IAAI,EAAE,gCAAgC,EAAE,OAAO,EAAE;iBAC7E;gBACD,MAAM,EAAE,OAAO,CAAC,MAAM,CAAC,MAAM;aAC9B,CAAA;QACH,CAAC;IACH,CAAC;CACF,CAAA"}

package/dist/smartfetch-internals.d.ts

@@ -0,0 +1,13 @@
+export type TimedCache = {
+    value: unknown;
+    expiresAt: number;
+};
+export declare function getImpitCache(): TimedCache | undefined;
+export declare function setImpitCache(value: TimedCache | undefined): void;
+export declare function getImpitLoader(): () => Promise<{
+    Impit: unknown;
+}>;
+export declare function TEST_setImpitLoader(loader: () => Promise<{
+    Impit: unknown;
+}>): void;
+export declare function TEST_resetImpitInternals(): void;

package/dist/smartfetch-internals.js

@@ -0,0 +1,19 @@
+let impitCache = undefined;
+let impitLoader = () => import("impit");
+export function getImpitCache() {
+    return impitCache;
+}
+export function setImpitCache(value) {
+    impitCache = value;
+}
+export function getImpitLoader() {
+    return impitLoader;
+}
+export function TEST_setImpitLoader(loader) {
+    impitLoader = loader;
+}
+export function TEST_resetImpitInternals() {
+    impitCache = undefined;
+    impitLoader = () => import("impit");
+}
+//# sourceMappingURL=smartfetch-internals.js.map

package/dist/smartfetch-internals.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"smartfetch-internals.js","sourceRoot":"","sources":["../src/smartfetch-internals.ts"],"names":[],"mappings":"AAKA,IAAI,UAAU,GAA2B,SAAS,CAAA;AAClD,IAAI,WAAW,GAAsC,GAAG,EAAE,CAAC,MAAM,CAAC,OAAO,CAAC,CAAA;AAE1E,MAAM,UAAU,aAAa;IAC3B,OAAO,UAAU,CAAA;AACnB,CAAC;AAED,MAAM,UAAU,aAAa,CAAC,KAA6B;IACzD,UAAU,GAAG,KAAK,CAAA;AACpB,CAAC;AAED,MAAM,UAAU,cAAc;IAC5B,OAAO,WAAW,CAAA;AACpB,CAAC;AAED,MAAM,UAAU,mBAAmB,CAAC,MAAyC;IAC3E,WAAW,GAAG,MAAM,CAAA;AACtB,CAAC;AAED,MAAM,UAAU,wBAAwB;IACtC,UAAU,GAAG,SAAS,CAAA;IACtB,WAAW,GAAG,GAAG,EAAE,CAAC,MAAM,CAAC,OAAO,CAAC,CAAA;AACrC,CAAC"}

package/dist/smartfetch.js

@@ -1,23 +1,25 @@
 import { asString, cleanLinks, dedupeUrls, envFlag, extractAnchorHrefs, extractUrls, inferTarget, isDcinsideUrl, needsDynamicCrawl, resolveValidatedUrl, validateOutboundUrl, } from "./shared.js";
 import { runBrowserSession } from "./browser-session.js";
+import { getImpitCache, getImpitLoader, setImpitCache } from "./smartfetch-internals.js";
 import { discoverAssets } from "./smartfetch/assets.js";
 import { maybeUseArchiveFallback } from "./smartfetch/archive-fallback.js";
 import { resolveSmartfetchProvider } from "./smartfetch/providers/index.js";
 const DEFAULT_USER_AGENT = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/134.0.0.0 Safari/537.36";
-let impitCache = undefined;
+const FAILURE_CACHE_TTL_MS = 30_000;
 function allowPrivateHosts() {
     return envFlag("SMART_WEB_ALLOW_PRIVATE_HOSTS");
 }
 async function getImpit() {
-    if (impitCache !== undefined)
-        return impitCache;
+    const impitCache = getImpitCache();
+    if (impitCache !== undefined && Date.now() < impitCache.expiresAt)
+        return impitCache.value;
     try {
-        impitCache = (await import("impit")).Impit;
+        setImpitCache({ value: (await getImpitLoader()()).Impit, expiresAt: Number.POSITIVE_INFINITY });
     }
     catch {
-        impitCache = false;
+        setImpitCache({ value: false, expiresAt: Date.now() + FAILURE_CACHE_TTL_MS });
     }
-    return impitCache;
+    return getImpitCache()?.value;
 }
 async function runNativeFetch(url, timeoutMs) {
     const headers = {
@@ -65,7 +67,7 @@ async function runNativeFetch(url, timeoutMs) {
             };
         }
         const links = [...extractAnchorHrefs(body), ...extractUrls(body)];
-        return { ok: true, method: "plain_fetch", content: body, links: dedupeUrls(links) };
+        return { ok: true, method: "plain_fetch", content: body, links: dedupeUrls(links), final_url: resolved.url };
     }
     catch (error) {
         const message = error instanceof Error ? error.message : String(error);
@@ -130,7 +132,7 @@ async function runImpitFetch(url, timeoutMs) {
             };
         }
         const links = [...extractAnchorHrefs(body), ...extractUrls(body)];
-        return { ok: true, method: "impit_fetch", content: body, links: dedupeUrls(links) };
+        return { ok: true, method: "impit_fetch", content: body, links: dedupeUrls(links), final_url: resolved.url };
     }
     catch (error) {
         const message = error instanceof Error ? error.message : String(error);
@@ -184,8 +186,12 @@ async function runPlaywrightFetch(url, timeoutMs, target) {
         method: result.method,
         content: choosePlaywrightContent(result),
         links: dedupeUrls(result.links.map((item) => asString(item))),
+        final_url: result.final_url,
     };
 }
+function riskLevelForTarget(target) {
+    return target === "x_post" ? "high" : target === "generic" ? "low" : "medium";
+}
 function baseOutput(url, target) {
     return {
         source: "smartfetch",
@@ -264,6 +270,7 @@ export async function runSmartfetch(options, runtime = {}) {
             method: target === "youtube_video" ? "youtube_metadata_seed" : "x_public_seed",
             content: "",
             links: [],
+            final_url: options.url,
         };
         output.retrieval_method.push(active.method);
     }
@@ -294,10 +301,21 @@ export async function runSmartfetch(options, runtime = {}) {
             output.retrieval_method.push(active.method);
         }
     }
-    const provider = resolveSmartfetchProvider(options.url, target);
+    let effectiveTarget = target;
+    if ((options.target || "auto") === "auto") {
+        const inferred = inferTarget(active.final_url || options.url, "auto");
+        if (inferred !== effectiveTarget) {
+            effectiveTarget = inferred;
+            output.target = inferred;
+            output.risk_level = riskLevelForTarget(inferred);
+        }
+    }
+    const providerUrl = active.final_url || options.url;
+    const provider = resolveSmartfetchProvider(providerUrl, effectiveTarget);
     const normalized = await provider.normalize({
         url: options.url,
-        target,
+        resolvedUrl: providerUrl,
+        target: effectiveTarget,
         timeoutMs,
         active,
         errors: output.errors,
@@ -428,6 +446,7 @@ async function runPlaywrightFetchWithRuntime(url, timeoutMs, target, runtime) {
         method: result.method,
         content: choosePlaywrightContent(result),
         links: dedupeUrls(result.links.map((item) => asString(item))),
+        final_url: result.final_url,
     };
 }
 //# sourceMappingURL=smartfetch.js.map