smart-web-mcp 0.8.8 → 0.9.0

package/dist/smartfetch/generic-fallbacks.js

@@ -0,0 +1,616 @@
+import { absolutizeUrl, asString, decodeHtml, dedupeUrls, extractMetaDescription, extractMetaName, extractMetaProperty, extractTitleFromHtml, extractUrls, stripTags, } from "../shared.js";
+import { fetchProviderText } from "./provider-policy.js";
+function isRecord(value) {
+    return Boolean(value) && typeof value === "object" && !Array.isArray(value);
+}
+function cleanText(value) {
+    return String(value || "").replace(/\s+/g, " ").trim();
+}
+function truncate(value, maxLength) {
+    const text = cleanText(value);
+    if (text.length <= maxLength)
+        return text;
+    return `${text.slice(0, maxLength - 1).trimEnd()}…`;
+}
+function tokenize(value) {
+    return Array.from(new Set(cleanText(value).toLowerCase().split(/[^a-z0-9가-힣]+/).filter((item) => item.length >= 3)));
+}
+function overlapCount(left, right) {
+    const rightTokens = new Set(tokenize(right));
+    return tokenize(left).reduce((count, token) => count + (rightTokens.has(token) ? 1 : 0), 0);
+}
+function slugWords(url) {
+    try {
+        return decodeURIComponent(new URL(url).pathname)
+            .replace(/[-_/]+/g, " ")
+            .replace(/\b\d+\b/g, " ")
+            .replace(/\s+/g, " ")
+            .trim();
+    }
+    catch {
+        return "";
+    }
+}
+function looksProductLikeUrl(url) {
+    try {
+        const parsed = new URL(url);
+        const path = parsed.pathname.toLowerCase();
+        if (/\/(?:product|products|item|shop|store|sku|dp|goods)\//.test(path))
+            return true;
+        if (/(^|\/)(?:p|pd)\/[a-z0-9-]+/i.test(path))
+            return true;
+        return ["id", "item", "sku", "product", "productid"].some((key) => parsed.searchParams.has(key));
+    }
+    catch {
+        return false;
+    }
+}
+function looksCollectionLikeUrl(url) {
+    try {
+        const parsed = new URL(url);
+        const path = parsed.pathname.toLowerCase();
+        if (/\/(?:search|shop|store|catalog|collection|collections|products|items|category|tag)s?(?:\/|$)/.test(path))
+            return true;
+        return ["q", "query", "keyword", "keywords", "search", "searchtext"].some((key) => parsed.searchParams.has(key));
+    }
+    catch {
+        return false;
+    }
+}
+function looksProfileLikeUrl(url) {
+    try {
+        const parsed = new URL(url);
+        const path = parsed.pathname.toLowerCase();
+        if (path === "/about" || path === "/team")
+            return true;
+        return /\/(?:about|profile|profiles|people|team|authors?|members?|u|user)\//.test(path)
+            || /\/@[a-z0-9._-]+/i.test(path);
+    }
+    catch {
+        return false;
+    }
+}
+function looksFeedDiscoveryPage(url) {
+    try {
+        const parsed = new URL(url);
+        const path = parsed.pathname.toLowerCase().replace(/\/+$/, "") || "/";
+        if (path === "/")
+            return true;
+        if (/\/(?:blog|blogs|news|updates|articles|posts|journal|category|categories|tags?)$/.test(path))
+            return true;
+        return ["page", "q", "query", "tag", "category", "search"].some((key) => parsed.searchParams.has(key));
+    }
+    catch {
+        return false;
+    }
+}
+function textFromUnknown(value) {
+    if (typeof value === "string" || typeof value === "number" || typeof value === "boolean")
+        return cleanText(value);
+    if (Array.isArray(value))
+        return value.map((item) => textFromUnknown(item)).filter(Boolean).join(", ");
+    if (!isRecord(value))
+        return "";
+    return cleanText(textFromUnknown(value.name)
+        || textFromUnknown(value.text)
+        || textFromUnknown(value.description)
+        || textFromUnknown(value.headline));
+}
+function firstUrl(value, baseUrl) {
+    if (typeof value === "string")
+        return absolutizeUrl(value, baseUrl);
+    if (Array.isArray(value)) {
+        for (const item of value) {
+            const candidate = firstUrl(item, baseUrl);
+            if (candidate)
+                return candidate;
+        }
+        return "";
+    }
+    if (!isRecord(value))
+        return "";
+    for (const key of ["url", "@id", "contentUrl", "thumbnailUrl"]) {
+        const candidate = firstUrl(value[key], baseUrl);
+        if (candidate)
+            return candidate;
+    }
+    return "";
+}
+function typeNames(record) {
+    const raw = record["@type"];
+    if (Array.isArray(raw))
+        return raw.map((item) => cleanText(item)).filter(Boolean);
+    const single = cleanText(raw);
+    return single ? [single] : [];
+}
+function collectJsonRecords(value, out) {
+    if (Array.isArray(value)) {
+        for (const item of value)
+            collectJsonRecords(item, out);
+        return;
+    }
+    if (!isRecord(value))
+        return;
+    out.push(value);
+    if (Array.isArray(value["@graph"]))
+        collectJsonRecords(value["@graph"], out);
+    if (Array.isArray(value.itemListElement))
+        collectJsonRecords(value.itemListElement, out);
+    if (isRecord(value.item))
+        collectJsonRecords(value.item, out);
+    if (isRecord(value.offers))
+        collectJsonRecords(value.offers, out);
+}
+function extractJsonLdRecords(html) {
+    const out = [];
+    for (const match of html.matchAll(/<script[^>]*type=["']application\/ld\+json["'][^>]*>([\s\S]*?)<\/script>/gi)) {
+        const raw = asString(match[1]).trim();
+        if (!raw)
+            continue;
+        try {
+            collectJsonRecords(JSON.parse(raw), out);
+        }
+        catch {
+            // Ignore malformed JSON-LD blocks.
+        }
+    }
+    return out;
+}
+function interestingText(value, out, depth = 0) {
+    if (depth > 6)
+        return;
+    if (typeof value === "string" || typeof value === "number" || typeof value === "boolean") {
+        const text = cleanText(stripTags(decodeHtml(String(value))));
+        if (text.length >= 24 && !/^https?:\/\//i.test(text))
+            out.push(text);
+        return;
+    }
+    if (Array.isArray(value)) {
+        value.forEach((item) => {
+            interestingText(item, out, depth + 1);
+        });
+        return;
+    }
+    if (!isRecord(value))
+        return;
+    Object.values(value).forEach((item) => {
+        interestingText(item, out, depth + 1);
+    });
+}
+function decodeJsonStringLiteral(value) {
+    try {
+        return JSON.parse(`"${value}"`);
+    }
+    catch {
+        return "";
+    }
+}
+function extractNextPayloadText(html) {
+    const out = [];
+    const nextDataMatch = html.match(/<script[^>]*id=["']__NEXT_DATA__["'][^>]*>([\s\S]*?)<\/script>/i);
+    if (nextDataMatch?.[1]) {
+        try {
+            interestingText(JSON.parse(nextDataMatch[1]), out);
+        }
+        catch {
+            // Ignore invalid payloads.
+        }
+    }
+    for (const match of html.matchAll(/self\.__next_f\.push\(\[1,\s*(?:\\?")([\s\S]*?)(?:\\?")\]\)/g)) {
+        const decoded = decodeJsonStringLiteral(asString(match[1]));
+        if (!decoded)
+            continue;
+        interestingText(decoded, out);
+    }
+    return Array.from(new Set(out)).join(" ").trim();
+}
+function ratingText(record) {
+    const aggregate = isRecord(record.aggregateRating) ? record.aggregateRating : null;
+    if (!aggregate)
+        return "";
+    const ratingValue = cleanText(aggregate.ratingValue);
+    const reviewCount = cleanText(aggregate.reviewCount || aggregate.ratingCount);
+    return ratingValue ? `${ratingValue}${reviewCount ? ` (${reviewCount})` : ""}` : "";
+}
+function productFallback(context, records) {
+    const pageTitle = cleanText(extractMetaProperty(context.active.content, "og:title") || extractTitleFromHtml(context.active.content));
+    const productRecords = records.filter((record) => typeNames(record).some((type) => /Product/i.test(type)));
+    const product = productRecords
+        .map((record) => ({ record, score: overlapCount(pageTitle, cleanText(textFromUnknown(record.name))) }))
+        .sort((left, right) => right.score - left.score)[0]?.record;
+    if (!product)
+        return null;
+    const offers = Array.isArray(product.offers) ? product.offers[0] : product.offers;
+    const offer = isRecord(offers) ? offers : {};
+    const title = cleanText(textFromUnknown(product.name) || extractMetaProperty(context.active.content, "og:title") || extractTitleFromHtml(context.active.content));
+    const description = cleanText(textFromUnknown(product.description) || extractMetaProperty(context.active.content, "og:description") || extractMetaDescription(context.active.content));
+    const price = cleanText(textFromUnknown(offer.price) || textFromUnknown(product.price));
+    const availability = cleanText(textFromUnknown(offer.availability).split("/").at(-1) || "");
+    const image = firstUrl(product.image, context.resolvedUrl || context.url)
+        || absolutizeUrl(extractMetaProperty(context.active.content, "og:image") || extractMetaName(context.active.content, "twitter:image"), context.resolvedUrl || context.url);
+    const url = firstUrl(product.url, context.resolvedUrl || context.url) || context.url;
+    const strongSignal = looksProductLikeUrl(context.resolvedUrl || context.url)
+        || overlapCount(pageTitle, title) >= 2
+        || Boolean(extractMetaProperty(context.active.content, "product:price:amount"));
+    if (!title || (!price && !description && !ratingText(product)) || !strongSignal || productRecords.length > 4)
+        return null;
+    return {
+        post: {
+            url: context.url,
+            canonical_url: url,
+            title,
+            description,
+            text: [description, price ? `Price: ${price}` : "", ratingText(product) ? `Rating: ${ratingText(product)}` : "", availability ? `Availability: ${availability}` : ""].filter(Boolean).join("\n") || description,
+            ...(price ? { price } : {}),
+            ...(availability ? { availability } : {}),
+            ...(ratingText(product) ? { rating: ratingText(product) } : {}),
+            ...(image ? { image } : {}),
+            kind: "structured_product",
+            extractor: "jsonld_product",
+            status: "ok",
+        },
+        thread: [],
+        comments: [],
+        outbound_links: dedupeUrls([url, image, ...extractUrls(description)].filter(Boolean)),
+        partial: false,
+        errors: context.errors,
+        method: "generic_jsonld_product",
+    };
+}
+function collectionItems(record, baseUrl) {
+    const elements = Array.isArray(record.itemListElement) ? record.itemListElement : [];
+    return elements.map((element, index) => {
+        const node = isRecord(element) && isRecord(element.item) ? element.item : isRecord(element) ? element : null;
+        if (!node)
+            return null;
+        const title = cleanText(textFromUnknown(node.name) || textFromUnknown(node.headline));
+        const url = firstUrl(node.url, baseUrl);
+        const offers = Array.isArray(node.offers) ? node.offers[0] : node.offers;
+        const offer = isRecord(offers) ? offers : {};
+        const price = cleanText(textFromUnknown(offer.price) || textFromUnknown(node.price));
+        const snippet = truncate(cleanText(textFromUnknown(node.description)), 220);
+        if (!title || !url)
+            return null;
+        return {
+            index: index + 1,
+            title,
+            url,
+            ...(price ? { price } : {}),
+            ...(snippet ? { snippet } : {}),
+            source: "jsonld_itemlist",
+        };
+    }).filter(Boolean);
+}
+function collectionFallback(context, records) {
+    const collection = records.find((record) => typeNames(record).some((type) => /(?:CollectionPage|ItemList)/i.test(type)) && Array.isArray(record.itemListElement));
+    if (!collection)
+        return null;
+    const thread = collectionItems(collection, context.resolvedUrl || context.url).slice(0, 12);
+    if (thread.length < 2)
+        return null;
+    const title = cleanText(textFromUnknown(collection.name) || extractMetaProperty(context.active.content, "og:title") || extractTitleFromHtml(context.active.content) || `Items from ${(new URL(context.url)).hostname}`);
+    const description = cleanText(textFromUnknown(collection.description) || extractMetaProperty(context.active.content, "og:description") || extractMetaDescription(context.active.content));
+    const strongSignal = looksCollectionLikeUrl(context.resolvedUrl || context.url)
+        || /results|search|catalog|collection|products|items|shop/i.test(title)
+        || thread.length >= 4;
+    if (!strongSignal)
+        return null;
+    return {
+        post: {
+            url: context.url,
+            title,
+            description,
+            text: thread.slice(0, 8).map((item, index) => `${index + 1}. ${cleanText(item.title)}${cleanText(item.price) ? ` — ${cleanText(item.price)}` : ""}`).join("\n"),
+            item_count: thread.length,
+            kind: "structured_collection",
+            extractor: "jsonld_itemlist",
+            status: "ok",
+        },
+        thread,
+        comments: [],
+        outbound_links: dedupeUrls(thread.map((item) => cleanText(item.url))),
+        partial: false,
+        errors: context.errors,
+        method: "generic_jsonld_itemlist",
+    };
+}
+function articleFallback(context, article, records, nextPayloadText) {
+    const pageTitle = cleanText(article.title || extractMetaProperty(context.active.content, "og:title") || extractTitleFromHtml(context.active.content));
+    const pageSlug = slugWords(context.url);
+    const ranked = records
+        .map((record) => {
+        const title = cleanText(textFromUnknown(record.headline) || textFromUnknown(record.name));
+        const recordUrl = firstUrl(record.url || record["@id"], context.resolvedUrl || context.url);
+        const titleOverlap = overlapCount(pageTitle, title);
+        const slugOverlap = overlapCount(pageSlug, `${title} ${recordUrl}`);
+        const exactUrlMatch = cleanText(recordUrl) === cleanText(context.url) || cleanText(recordUrl) === cleanText(context.resolvedUrl);
+        const score = typeNames(record).reduce((sum, type) => sum + (/Article|NewsArticle|BlogPosting|DiscussionForumPosting|SocialMediaPosting/i.test(type) ? 5 : 0), 0)
+            + (cleanText(record.articleBody).length > 0 ? 4 : 0)
+            + (cleanText(record.headline).length > 0 ? 2 : 0)
+            + (cleanText(record.description).length > 0 ? 1 : 0)
+            + (exactUrlMatch ? 6 : 0)
+            + (titleOverlap >= 2 ? 4 : 0)
+            + (slugOverlap >= 2 ? 2 : 0);
+        return { record, score, exactUrlMatch, titleOverlap, slugOverlap };
+    })
+        .filter((item) => item.score > 0 && (item.exactUrlMatch || item.titleOverlap >= 2 || item.slugOverlap >= 2))
+        .sort((left, right) => right.score - left.score);
+    const best = ranked[0]?.record;
+    const structuredTitle = best ? cleanText(textFromUnknown(best.headline) || textFromUnknown(best.name)) : "";
+    const structuredDescription = best ? cleanText(textFromUnknown(best.description)) : "";
+    const structuredText = best ? cleanText(textFromUnknown(best.articleBody) || textFromUnknown(best.text)) : "";
+    const candidateText = structuredText || nextPayloadText;
+    const currentTextLength = cleanText(article.text).length;
+    if (!candidateText)
+        return null;
+    if (!structuredText) {
+        const nextShell = /<script[^>]*id=["']__NEXT_DATA__["']|self\.__next_f\.push\(/i.test(context.active.content);
+        if (!nextShell || candidateText.length < 60)
+            return null;
+    }
+    else if (candidateText.length < Math.max(currentTextLength + 120, 280)) {
+        return null;
+    }
+    const image = best ? firstUrl(best.image, context.resolvedUrl || context.url) : "";
+    return {
+        post: {
+            url: context.url,
+            title: structuredTitle || article.title || extractTitleFromHtml(context.active.content),
+            description: structuredDescription || article.description || extractMetaProperty(context.active.content, "og:description") || extractMetaDescription(context.active.content),
+            text: candidateText.slice(0, 50000),
+            author: article.author,
+            image: image || article.image,
+            published: article.published,
+            extractor: structuredText ? "jsonld_article" : "nextjs_payload",
+            status: "ok",
+        },
+        thread: [],
+        comments: [],
+        outbound_links: dedupeUrls([...(context.active.links || []), ...extractUrls(candidateText), image].filter(Boolean)),
+        partial: false,
+        errors: context.errors,
+        method: structuredText ? "generic_jsonld_article" : "generic_nextjs_payload",
+    };
+}
+function profileFallback(context, article, records) {
+    const profile = records.find((record) => typeNames(record).some((type) => /Person/i.test(type)));
+    if (!profile)
+        return null;
+    if (!looksProfileLikeUrl(context.resolvedUrl || context.url))
+        return null;
+    const name = cleanText(textFromUnknown(profile.name));
+    const jobTitle = cleanText(textFromUnknown(profile.jobTitle));
+    const description = cleanText(textFromUnknown(profile.description) || extractMetaProperty(context.active.content, "og:description") || extractMetaDescription(context.active.content));
+    const affiliation = cleanText(textFromUnknown(profile.worksFor) || textFromUnknown(profile.alumniOf));
+    if (!name || (cleanText(article.text).length >= 400 && !jobTitle && !affiliation))
+        return null;
+    const image = firstUrl(profile.image, context.resolvedUrl || context.url);
+    return {
+        post: {
+            url: context.url,
+            title: name,
+            description: [jobTitle, affiliation].filter(Boolean).join(" · ") || description,
+            text: [description, jobTitle ? `Role: ${jobTitle}` : "", affiliation ? `Affiliation: ${affiliation}` : ""].filter(Boolean).join("\n") || description,
+            ...(image ? { image } : {}),
+            kind: "structured_profile",
+            extractor: "jsonld_person",
+            status: "ok",
+        },
+        thread: [],
+        comments: [],
+        outbound_links: dedupeUrls([firstUrl(profile.url, context.resolvedUrl || context.url), image].filter(Boolean)),
+        partial: false,
+        errors: context.errors,
+        method: "generic_jsonld_profile",
+    };
+}
+export function tryStructuredGenericFallback(context, article) {
+    const html = context.active.content;
+    if (!/<(?:html|body|script|main|article|meta)\b/i.test(html))
+        return null;
+    const records = extractJsonLdRecords(html);
+    const nextPayloadText = extractNextPayloadText(html);
+    const collection = collectionFallback(context, records);
+    if (collection)
+        return collection;
+    const product = productFallback(context, records);
+    if (product)
+        return product;
+    const structuredArticle = articleFallback(context, article, records, nextPayloadText);
+    if (structuredArticle)
+        return structuredArticle;
+    return profileFallback(context, article, records);
+}
+function parseLinkAttributes(tag) {
+    const attributes = {};
+    const regex = /([^\s=/>]+)(?:\s*=\s*(?:"([^"]*)"|'([^']*)'|([^\s"'=<>`]+)))?/g;
+    for (const match of tag.matchAll(regex)) {
+        const key = cleanText(match[1]).toLowerCase();
+        if (!key || key === "link")
+            continue;
+        attributes[key] = decodeHtml(String(match[2] ?? match[3] ?? match[4] ?? "")).trim();
+    }
+    return attributes;
+}
+function sameOrigin(left, right) {
+    try {
+        return new URL(left).origin === new URL(right).origin;
+    }
+    catch {
+        return false;
+    }
+}
+function originCandidates(url) {
+    try {
+        const parsed = new URL(url);
+        return ["/rss", "/feed", "/rss.xml", "/atom.xml", "/index.xml"].map((path) => `${parsed.origin}${path}`);
+    }
+    catch {
+        return [];
+    }
+}
+function htmlFeedCandidates(html, pageUrl) {
+    const out = [];
+    for (const match of html.matchAll(/<link\b[^>]*>/gi)) {
+        const attributes = parseLinkAttributes(asString(match[0]));
+        const rel = cleanText(attributes.rel).toLowerCase();
+        const type = cleanText(attributes.type).toLowerCase();
+        if (!rel.includes("alternate"))
+            continue;
+        if (!/(rss|atom|xml)/i.test(type))
+            continue;
+        const href = absolutizeUrl(attributes.href || "", pageUrl);
+        if (href && sameOrigin(href, pageUrl))
+            out.push(href);
+    }
+    return dedupeUrls(out);
+}
+function extractXmlTag(block, tagNames) {
+    for (const tag of tagNames) {
+        const match = block.match(new RegExp(`<${tag}[^>]*>([\\s\\S]*?)<\/${tag}>`, "i"));
+        if (match?.[1])
+            return cleanText(stripTags(decodeHtml(match[1])));
+    }
+    return "";
+}
+function extractLinkHref(block, preferredRels) {
+    const relWanted = preferredRels.map((value) => value.toLowerCase());
+    const matches = Array.from(block.matchAll(/<link\b[^>]+href=["']([^"']+)["'][^>]*>/gi));
+    for (const match of matches) {
+        const attributes = parseLinkAttributes(asString(match[0]));
+        const rel = cleanText(attributes.rel).toLowerCase();
+        if (relWanted.length > 0 && !relWanted.includes(rel))
+            continue;
+        if (attributes.href)
+            return attributes.href;
+    }
+    if (relWanted.length > 0)
+        return "";
+    return cleanText(matches[0]?.[1] || "");
+}
+function extractXmlLink(block, tagNames, baseUrl, preferredRels = []) {
+    for (const tag of tagNames) {
+        if (tag === "link") {
+            const href = extractLinkHref(block, preferredRels);
+            const absolute = absolutizeUrl(href, baseUrl);
+            if (absolute)
+                return absolute;
+        }
+        const withHref = block.match(new RegExp(`<${tag}[^>]+href=["']([^"']+)["'][^>]*>`, "i"));
+        if (withHref?.[1]) {
+            const absolute = absolutizeUrl(withHref[1], baseUrl);
+            if (absolute)
+                return absolute;
+        }
+        const inline = block.match(new RegExp(`<${tag}[^>]*>([\\s\\S]*?)<\/${tag}>`, "i"));
+        if (inline?.[1]) {
+            const absolute = absolutizeUrl(stripTags(decodeHtml(inline[1])), baseUrl);
+            if (absolute)
+                return absolute;
+        }
+    }
+    return "";
+}
+function parseFeed(xml, baseUrl) {
+    const trimmed = xml.trim();
+    if (!trimmed)
+        return null;
+    if (/<rss\b|<channel\b/i.test(trimmed)) {
+        const channelMatch = trimmed.match(/<channel\b[^>]*>([\s\S]*?)<\/channel>/i);
+        const channel = channelMatch?.[1] || trimmed;
+        const title = extractXmlTag(channel, ["title"]);
+        const description = extractXmlTag(channel, ["description"]);
+        const url = extractXmlLink(channel, ["link"], baseUrl) || baseUrl;
+        const entries = Array.from(channel.matchAll(/<item\b[^>]*>([\s\S]*?)<\/item>/gi)).map((match) => {
+            const block = match[1] || "";
+            return {
+                title: extractXmlTag(block, ["title"]),
+                url: extractXmlLink(block, ["link", "guid"], url || baseUrl),
+                snippet: truncate(extractXmlTag(block, ["description", "content:encoded"]), 220),
+                published: extractXmlTag(block, ["pubDate", "dc:date"]),
+            };
+        }).filter((entry) => entry.title && entry.url);
+        return entries.length > 0 ? { title, description, url, format: "rss", entries } : null;
+    }
+    if (/<feed\b/i.test(trimmed)) {
+        const title = extractXmlTag(trimmed, ["title"]);
+        const description = extractXmlTag(trimmed, ["subtitle"]);
+        const url = extractXmlLink(trimmed, ["link"], baseUrl, ["alternate"]) || extractXmlLink(trimmed, ["link"], baseUrl) || baseUrl;
+        const entries = Array.from(trimmed.matchAll(/<entry\b[^>]*>([\s\S]*?)<\/entry>/gi)).map((match) => {
+            const block = match[1] || "";
+            return {
+                title: extractXmlTag(block, ["title"]),
+                url: extractXmlLink(block, ["link"], url || baseUrl, ["alternate"]) || extractXmlLink(block, ["link", "id"], url || baseUrl),
+                snippet: truncate(extractXmlTag(block, ["summary", "content"]), 220),
+                published: extractXmlTag(block, ["updated", "published"]),
+            };
+        }).filter((entry) => entry.title && entry.url);
+        return entries.length > 0 ? { title, description, url, format: "atom", entries } : null;
+    }
+    return null;
+}
+function feedNormalization(context, article, feed, method, sourceUrl) {
+    const title = feed.title || article.title || extractTitleFromHtml(context.active.content) || sourceUrl;
+    const description = feed.description || article.description;
+    return {
+        post: {
+            url: context.url,
+            canonical_url: feed.url || sourceUrl,
+            title,
+            description,
+            text: feed.entries.slice(0, 8).map((entry, index) => `${index + 1}. ${entry.title}${entry.published ? ` — ${entry.published}` : ""}`).join("\n"),
+            feed_url: sourceUrl,
+            feed_format: feed.format,
+            item_count: feed.entries.length,
+            kind: "feed_index",
+            extractor: "feed_discovery",
+            status: "ok",
+        },
+        thread: feed.entries.slice(0, 12).map((entry, index) => ({
+            index: index + 1,
+            title: entry.title,
+            url: entry.url,
+            ...(entry.snippet ? { snippet: entry.snippet } : {}),
+            ...(entry.published ? { published: entry.published } : {}),
+            source: "feed",
+        })),
+        comments: [],
+        outbound_links: dedupeUrls([sourceUrl, ...feed.entries.map((entry) => entry.url)]),
+        partial: false,
+        errors: context.errors,
+        method,
+    };
+}
+export async function tryFeedFallback(context, article) {
+    const direct = parseFeed(context.active.content, context.resolvedUrl || context.url);
+    if (direct)
+        return feedNormalization(context, article, direct, "rss_feed_direct", context.resolvedUrl || context.url);
+    const currentTextLength = cleanText(article.text).length;
+    if (currentTextLength >= 320)
+        return null;
+    if (!looksFeedDiscoveryPage(context.resolvedUrl || context.url))
+        return null;
+    const candidates = dedupeUrls([
+        ...htmlFeedCandidates(context.active.content, context.resolvedUrl || context.url),
+        ...originCandidates(context.resolvedUrl || context.url),
+    ]).slice(0, 6);
+    for (const candidate of candidates) {
+        if (!sameOrigin(candidate, context.resolvedUrl || context.url))
+            continue;
+        const fetched = await fetchProviderText(candidate, Math.min(context.timeoutMs, 12000), {
+            headers: {
+                accept: "application/rss+xml,application/atom+xml,application/xml,text/xml,text/plain,*/*",
+            },
+        }, {
+            mode: "same-origin",
+            sourceUrl: context.resolvedUrl || context.url,
+        });
+        if (!fetched.ok)
+            continue;
+        const parsed = parseFeed(fetched.text, candidate);
+        if (parsed)
+            return feedNormalization(context, article, parsed, "rss_feed_discovery", candidate);
+    }
+    return null;
+}
+//# sourceMappingURL=generic-fallbacks.js.map