spiderlint 0.0.0 → 0.143.0

package/dist/plugins/feed-content.js

@@ -0,0 +1,132 @@
+// SPDX-FileCopyrightText: 2026 Damián Búho <damian.buho@proton.me>
+//
+// SPDX-License-Identifier: AGPL-3.0-only
+import { Parser } from "htmlparser2";
+const SAMPLE = 60;
+const LITERAL = new Set(["pre", "code", "kbd", "samp", "script", "style"]);
+const UNSAFE = new Set(["script", "iframe", "object", "embed", "form", "frame", "frameset", "applet", "base", "meta", "link"]);
+const URL_ATTRIBUTES = { a: ["href"], img: ["src", "srcset"], source: ["src", "srcset"], video: ["src", "poster"], audio: ["src"], iframe: ["src"], link: ["href"] };
+// Markdown signs, each counted once per item; two different signs make it Markdown.
+const MARKDOWN = [
+    ["heading", /^#{1,6}[ \t]+\S/m],
+    ["bold", /\*\*[^*\n]+\*\*/],
+    ["fence", /^```/m],
+    ["link", /\[[^\]\n]+\]\((?:https?:|\/|\.\/|#)[^)\s]*\)/],
+    ["list", /^[ \t]*[-*+][ \t]+\S.*\n[ \t]*[-*+][ \t]+\S/m],
+    ["front matter", /^---\n(?:[\w-]+:.*\n)+---/],
+];
+const MDX_STATEMENT = /^[ \t]*(?:import\s+[\w{}\s,*]+\s+from\s+["'][^"']+["']|export\s+(?:const|default|function)\s)/m;
+const MDX_EXPRESSION = /(?<![{$])\{(?!\{)[ \t]*[A-Za-z_$][\w$.]*(?:\([^)]*\))?[ \t]*\}(?!\})/;
+const TEMPLATE = /\{\{[^}]*\}\}|\{%[^%]*%\}|\$\{[^}]*\}|\[object Object\]/;
+const NULLISH = /^(?:undefined|null|NaN)$/;
+const ESCAPED_TAG = /<\/?(?:p|a|div|span|br|img|em|strong|ul|ol|li|h[1-6]|blockquote|figure|code|pre)\b[^>]*>/i;
+const ESCAPED_ENTITY = /&(?:amp|lt|gt|quot|#\d+|#x[\da-f]+|[a-z]+);/i;
+const MOJIBAKE = /Ã[\u{80}-\u{BF}]|â€[\u{80}-\u{BF}™œ“”˜]/u;
+// The line of `text` holding the match of `pattern`.
+const lineOf = (text, pattern) => {
+    const index = pattern.exec(text)?.index ?? 0;
+    const end = text.indexOf("\n", index);
+    return text.slice(text.lastIndexOf("\n", index) + 1, end === -1 ? undefined : end);
+};
+const clip = (text) => {
+    const flat = text.replaceAll(/\s+/g, " ").trim();
+    return flat.length > SAMPLE ? `${flat.slice(0, SAMPLE - 1)}…` : flat;
+};
+// Whether a URL attribute value resolves on its own, without the feed’s or the page’s address.
+const isAbsolute = (value) => /^(?:[a-z][a-z\d+.-]*:|\/\/|#)/i.test(value.trim()) || value.trim() === "";
+// Problems of one HTML or text body, each a short sample; HTML is parsed, never matched as a whole.
+export function judgeContent(content) {
+    const found = {};
+    const add = (problem, sample) => {
+        const list = (found[problem] ??= []);
+        if (!list.includes(sample))
+            list.push(sample);
+    };
+    const prose = [];
+    if (content.type === "text") {
+        prose.push(content.value);
+        if (/<\/?[a-z][a-z\d]*(?:\s[^<>]*)?>/i.test(content.value))
+            add("content-type", `${content.field} is plain text but holds tags`);
+    }
+    else {
+        const literal = [];
+        let element = "";
+        let buffer = "";
+        // Judges the text gathered since the last tag, since the parser hands each decoded entity over on its own.
+        const flush = () => {
+            const text = buffer;
+            buffer = "";
+            if (!text)
+                return;
+            prose.push(text);
+            if (element && NULLISH.test(text.trim()))
+                add("template-leak", `<${element}>${text.trim()}</${element}>`);
+            if (ESCAPED_TAG.test(text))
+                add("double-escaped", clip(ESCAPED_TAG.exec(text)?.[0] ?? text));
+            if (ESCAPED_ENTITY.test(text))
+                add("double-escaped", ESCAPED_ENTITY.exec(text)?.[0] ?? text);
+        };
+        const parser = new Parser({
+            onopentag(name, attributes) {
+                flush();
+                const lower = name.toLowerCase();
+                element = lower;
+                if (LITERAL.has(lower))
+                    literal.push(lower);
+                if (/^[A-Z]/.test(name) && literal.length === 0)
+                    add("raw-markup", `<${name}>`);
+                if (UNSAFE.has(lower))
+                    add("unsafe-html", `<${lower}>`);
+                for (const [attribute, value] of Object.entries(attributes)) {
+                    if (/^on[a-z]+$/i.test(attribute))
+                        add("unsafe-html", `${attribute} on <${lower}>`);
+                    if (attribute === "style" && /position\s*:\s*(?:fixed|absolute)/i.test(value))
+                        add("unsafe-html", `positioned style on <${lower}>`);
+                    if (URL_ATTRIBUTES[lower]?.includes(attribute.toLowerCase()) && !content.base && value.split(",").some((part) => !isAbsolute(part.trim().split(/\s+/, 1)[0] ?? "")))
+                        add("relative-url", `${attribute}="${clip(value)}"`);
+                }
+            },
+            ontext(text) {
+                if (literal.length === 0)
+                    buffer += text;
+            },
+            onclosetag(name) {
+                flush();
+                if (literal.at(-1) === name.toLowerCase())
+                    literal.pop();
+                element = "";
+            },
+        }, { lowerCaseTags: false, lowerCaseAttributeNames: false, decodeEntities: true, recognizeSelfClosing: true });
+        parser.write(content.value);
+        parser.end();
+        flush();
+    }
+    const text = prose.join("");
+    if (MDX_STATEMENT.test(text))
+        add("raw-markup", clip(MDX_STATEMENT.exec(text)?.[0] ?? ""));
+    if (MDX_EXPRESSION.test(text))
+        add("raw-markup", clip(MDX_EXPRESSION.exec(text)?.[0] ?? ""));
+    const signs = MARKDOWN.filter(([, pattern]) => pattern.test(text));
+    if (signs.length >= 2)
+        add("raw-markup", `Markdown ${signs.map(([name]) => name).join(", ")}: ${clip(lineOf(text, signs[0]?.[1] ?? /^/))}`);
+    if (TEMPLATE.test(text))
+        add("template-leak", clip(TEMPLATE.exec(text)?.[0] ?? ""));
+    if (MOJIBAKE.test(text))
+        add("mojibake", clip(MOJIBAKE.exec(text)?.[0] ?? ""));
+    return found;
+}
+// Problems of an item title, which every format holds as plain text unless Atom says otherwise.
+export function judgeTitle(title, type) {
+    if (!title.trim())
+        return ["empty title"];
+    if (type !== "text")
+        return [];
+    const tag = /<\/?[a-z][a-z\d]*(?:\s[^<>]*)?>/i.exec(title)?.[0];
+    const entity = ESCAPED_ENTITY.exec(title)?.[0];
+    return [...(tag ? [`tag ${tag}`] : []), ...(entity ? [`entity ${entity}`] : []), ...(TEMPLATE.test(title) || NULLISH.test(title.trim()) ? [`template ${clip(title)}`] : [])];
+}
+// Visible words of a body, to compare an item with the page it links.
+export function wordsOf(content) {
+    const text = content.type === "text" ? content.value : content.value.replaceAll(/<[^>]*>/g, " ");
+    return text.split(/\s+/).filter(Boolean).length;
+}

package/dist/plugins/feed-model.js

@@ -0,0 +1,282 @@
+// SPDX-FileCopyrightText: 2026 Damián Búho <damian.buho@proton.me>
+//
+// SPDX-License-Identifier: AGPL-3.0-only
+import { SaxesParser } from "saxes";
+import { resolve } from "../rules/builtin.js";
+/* eslint-disable unicorn/prefer-https -- namespace names compare as exact strings, scheme included */
+export const NS = {
+    atom: "http://www.w3.org/2005/Atom",
+    rdf: "http://www.w3.org/1999/02/22-rdf-syntax-ns#",
+    rss1: "http://purl.org/rss/1.0/",
+    content: "http://purl.org/rss/1.0/modules/content/",
+    dc: "http://purl.org/dc/elements/1.1/",
+    itunes: "http://www.itunes.com/dtds/podcast-1.0.dtd",
+    podcast: "https://podcastindex.org/namespace/1.0",
+    history: "http://purl.org/syndication/history/1.0",
+    xml: "http://www.w3.org/XML/1998/namespace",
+};
+// Elements RSS 2.0 defines on a channel and on an item.
+const RSS_CHANNEL = new Set(["title", "link", "description", "language", "copyright", "managingEditor", "webMaster", "pubDate", "lastBuildDate", "category", "generator", "docs", "cloud", "ttl", "image", "rating", "textInput", "skipHours", "skipDays", "item"]);
+const RSS_ITEM = new Set(["title", "link", "description", "author", "category", "comments", "enclosure", "guid", "pubDate", "source"]);
+const VOID = new Set(["area", "base", "br", "col", "embed", "hr", "img", "input", "link", "meta", "source", "track", "wbr"]);
+const escapeText = (text) => text.replaceAll("&", "&amp;").replaceAll("<", "&lt;").replaceAll(">", "&gt;");
+// The markup of a parsed element, as an `xhtml` construct’s content is read.
+function serialize(node) {
+    const attributes = Object.entries(node.attributes)
+        .map(([name, value]) => ` ${name}="${escapeText(value).replaceAll('"', "&quot;")}"`)
+        .join("");
+    const inner = node.markup ?? "";
+    return !inner && VOID.has(node.local) ? `<${node.local}${attributes}/>` : `<${node.local}${attributes}>${inner}</${node.local}>`;
+}
+// The element tree of an XML document, `xml:base` resolved down it; throws on the first well-formedness error.
+export function parseXml(body, base) {
+    const parser = new SaxesParser({ xmlns: true });
+    const stack = [];
+    const namespaces = new Set();
+    let root;
+    let stylesheet;
+    parser.on("processinginstruction", (instruction) => {
+        if (!root && instruction.target === "xml-stylesheet")
+            stylesheet ??= /\bhref\s*=\s*["']([^"']*)["']/.exec(instruction.body)?.[1];
+    });
+    parser.on("opentag", (tag) => {
+        const parent = stack.at(-1);
+        const attributes = Object.fromEntries(Object.values(tag.attributes).map((attribute) => [attribute.name, attribute.value]));
+        const bindings = Object.entries(tag.ns ?? {});
+        for (const [prefix, uri] of bindings)
+            if (prefix)
+                namespaces.add(uri);
+        const declared = attributes["xml:base"] ? resolve(attributes["xml:base"], parent?.base ?? base) : parent?.base;
+        const node = { local: tag.local, uri: tag.uri, attributes, children: [], text: "", ...(declared && { base: declared }) };
+        if (parent?.markup !== undefined)
+            node.markup = "";
+        if (parent && parent.uri === NS.atom && parent.attributes.type === "xhtml" && ["content", "summary", "title", "subtitle", "rights"].includes(parent.local))
+            node.markup = "";
+        parent?.children.push(node);
+        root ??= node;
+        stack.push(node);
+    });
+    const append = (chunk) => {
+        const node = stack.at(-1);
+        if (!node)
+            return;
+        node.text += chunk;
+        if (node.markup !== undefined)
+            node.markup += escapeText(chunk);
+    };
+    parser.on("text", append);
+    parser.on("cdata", append);
+    parser.on("closetag", () => {
+        const node = stack.pop();
+        const parent = stack.at(-1);
+        if (node && parent?.markup !== undefined)
+            parent.markup += serialize(node);
+    });
+    parser.write(body).close();
+    if (!root)
+        throw new Error("document has no root element");
+    return { root, ...(stylesheet && { stylesheet }), namespaces };
+}
+const childrenOf = (node, local, uri = "") => (node?.children ?? []).filter((child) => child.local === local && child.uri === uri);
+const childOf = (node, local, uri = "") => childrenOf(node, local, uri)[0];
+const textOf = (node) => node?.text.trim() || undefined;
+const typeOf = (node) => (node?.attributes.type === "html" || node?.attributes.type === "xhtml" ? node.attributes.type : "text");
+const dated = (node, field) => (textOf(node) ? { field, raw: textOf(node) } : undefined);
+const emailsOf = (node, names) => names.flatMap((name) => childrenOf(node, name).flatMap((child) => (textOf(child) ? [{ field: name, raw: textOf(child) }] : [])));
+// Un-namespaced children `known` does not name, once each.
+const unknownIn = (node, known) => [...new Set((node?.children ?? []).filter((child) => child.uri === "" && !known.has(child.local)).map((child) => child.local))];
+// Atom links of `node` by relation, the default relation `alternate`, resolved against the element’s base.
+function atomLinks(node, base) {
+    const links = {};
+    for (const link of childrenOf(node, "link", NS.atom)) {
+        const relation = link.attributes.rel ?? "alternate";
+        (links[relation] ??= []).push(resolve(link.attributes.href ?? "", link.base ?? base));
+    }
+    return links;
+}
+// RFC 5005 archive relations a feed names.
+function archivesOf(node, base) {
+    const links = atomLinks(node, base);
+    return Object.fromEntries(["current", "prev-archive", "next-archive"].flatMap((relation) => (links[relation]?.[0] ? [[relation, links[relation][0]]] : [])));
+}
+const itunesOf = (node) => Object.fromEntries((node?.children ?? []).filter((child) => child.uri === NS.itunes).map((child) => [child.local, child.attributes.href ?? child.attributes.text ?? child.text.trim()]));
+// An RSS 2.0 or RSS 1.0 item.
+function rssItem(node, position, isRdf) {
+    const ns = isRdf ? NS.rss1 : "";
+    const guid = childOf(node, "guid");
+    const id = textOf(guid) ?? node.attributes["rdf:about"];
+    const contents = [childOf(node, "description", ns), childOf(node, "encoded", NS.content)].flatMap((child) => (child && child.text.trim() ? [{ field: child.uri === NS.content ? "content:encoded" : "description", type: "html", value: child.text, ...(child.base && { base: child.base }) }] : []));
+    const title = childOf(node, "title", ns);
+    const link = textOf(childOf(node, "link", ns));
+    const missing = [...(!title && !childOf(node, "description", ns) ? ["title or description"] : []), ...(isRdf && !link ? ["link"] : [])];
+    const enclosures = childrenOf(node, "enclosure").map((child) => ({ url: child.attributes.url, length: child.attributes.length, type: child.attributes.type }));
+    return {
+        position,
+        ...(id && { id }),
+        isPermalink: !isRdf && guid !== undefined && guid.attributes.isPermaLink !== "false",
+        ...(link && { link }),
+        ...(title && { title: title.text.trim() }),
+        titleType: "text",
+        ...((dated(childOf(node, "pubDate"), "pubDate") ?? dated(childOf(node, "date", NS.dc), "dc:date")) && { published: dated(childOf(node, "pubDate"), "pubDate") ?? dated(childOf(node, "date", NS.dc), "dc:date") }),
+        hasAuthor: childOf(node, "author") !== undefined || childOf(node, "creator", NS.dc) !== undefined,
+        emails: emailsOf(node, ["author"]),
+        contents,
+        enclosures,
+        unknown: isRdf ? [] : unknownIn(node, RSS_ITEM),
+        missing,
+        durations: childrenOf(node, "duration", NS.itunes).map((child) => child.text.trim()),
+    };
+}
+// An RSS 2.0, 0.9x or 1.0 document.
+function readRss(root, namespaces, base) {
+    const isRdf = root.uri === NS.rdf;
+    const ns = isRdf ? NS.rss1 : "";
+    const channel = childOf(root, "channel", ns);
+    const items = isRdf ? childrenOf(root, "item", NS.rss1) : childrenOf(channel, "item");
+    const links = atomLinks(channel, base);
+    const missing = [...(channel ? ["title", "link", "description"].filter((name) => !childOf(channel, name, ns)) : ["channel"]), ...(!isRdf && !root.attributes.version ? ["version attribute"] : [])];
+    const podcastGuid = textOf(childOf(channel, "guid", NS.podcast));
+    const locked = textOf(childOf(channel, "locked", NS.podcast));
+    // RSS 2.0 names the channel date `lastBuildDate`, and `pubDate` where it means the content date.
+    const updated = dated(childOf(channel, "lastBuildDate"), "lastBuildDate") ?? dated(childOf(channel, "pubDate"), "pubDate");
+    return {
+        format: "rss",
+        ...(root.attributes.version && { version: root.attributes.version }),
+        ...(textOf(childOf(channel, "title", ns)) && { title: textOf(childOf(channel, "title", ns)) }),
+        ...(textOf(childOf(channel, "link", ns)) && { link: textOf(childOf(channel, "link", ns)) }),
+        ...((textOf(childOf(channel, "language")) ?? textOf(childOf(channel, "language", NS.dc))) && { language: textOf(childOf(channel, "language")) ?? textOf(childOf(channel, "language", NS.dc)) }),
+        ...(updated && { updated }),
+        ...(textOf(childOf(channel, "ttl")) && { ttl: textOf(childOf(channel, "ttl")) }),
+        ...(links.self?.[0] && { self: links.self[0] }),
+        hubs: links.hub ?? [],
+        archives: archivesOf(channel, base),
+        isComplete: childOf(channel, "complete", NS.history) !== undefined,
+        hasAuthor: true,
+        emails: emailsOf(channel, ["managingEditor", "webMaster"]),
+        unknown: isRdf ? [] : unknownIn(channel, RSS_CHANNEL),
+        missing,
+        namespaces,
+        itunes: itunesOf(channel),
+        ...(podcastGuid && { podcastGuid }),
+        ...(locked !== undefined && { locked }),
+        items: items.map((node, index) => rssItem(node, index + 1, isRdf)),
+    };
+}
+// An Atom text construct as content.
+function atomContent(node, field) {
+    if (!node || node.attributes.src)
+        return [];
+    const type = typeOf(node);
+    const value = type === "xhtml" ? (node.children.at(0)?.markup ?? node.markup ?? "") : node.text;
+    return value.trim() ? [{ field, type, value, ...(node.base && { base: node.base }) }] : [];
+}
+// An Atom entry.
+function atomEntry(node, position, base) {
+    const title = childOf(node, "title", NS.atom);
+    const links = atomLinks(node, base);
+    const id = textOf(childOf(node, "id", NS.atom));
+    const hasAuthor = childOf(node, "author", NS.atom) !== undefined || childOf(childOf(node, "source", NS.atom), "author", NS.atom) !== undefined;
+    return {
+        position,
+        ...(id && { id }),
+        isPermalink: false,
+        ...(links.alternate?.[0] && { link: links.alternate[0] }),
+        ...(title && { title: title.text.trim() }),
+        titleType: typeOf(title),
+        ...(dated(childOf(node, "published", NS.atom), "published") && { published: dated(childOf(node, "published", NS.atom), "published") }),
+        ...(dated(childOf(node, "updated", NS.atom), "updated") && { updated: dated(childOf(node, "updated", NS.atom), "updated") }),
+        hasAuthor,
+        emails: [],
+        contents: [...atomContent(childOf(node, "content", NS.atom), "content"), ...atomContent(childOf(node, "summary", NS.atom), "summary")],
+        enclosures: childrenOf(node, "link", NS.atom)
+            .filter((link) => link.attributes.rel === "enclosure")
+            .map((link) => ({ url: link.attributes.href, length: link.attributes.length, type: link.attributes.type })),
+        unknown: [],
+        missing: ["id", "title", "updated"].filter((name) => !childOf(node, name, NS.atom)),
+        durations: [],
+    };
+}
+// An Atom document.
+function readAtom(root, namespaces, base) {
+    const links = atomLinks(root, base);
+    const language = root.attributes["xml:lang"];
+    return {
+        format: "atom",
+        ...(textOf(childOf(root, "title", NS.atom)) && { title: textOf(childOf(root, "title", NS.atom)) }),
+        ...(links.alternate?.[0] && { link: links.alternate[0] }),
+        ...(language && { language }),
+        ...(dated(childOf(root, "updated", NS.atom), "updated") && { updated: dated(childOf(root, "updated", NS.atom), "updated") }),
+        ...(links.self?.[0] && { self: links.self[0] }),
+        hubs: links.hub ?? [],
+        archives: archivesOf(root, base),
+        isComplete: childOf(root, "complete", NS.history) !== undefined,
+        hasAuthor: childOf(root, "author", NS.atom) !== undefined,
+        emails: [],
+        unknown: [],
+        missing: ["id", "title", "updated"].filter((name) => !childOf(root, name, NS.atom)),
+        namespaces,
+        itunes: {},
+        items: childrenOf(root, "entry", NS.atom).map((node, index) => atomEntry(node, index + 1, base)),
+    };
+}
+// The XML model of an RSS or Atom document; throws on the first well-formedness error.
+export function readXml(body, base) {
+    const { root, stylesheet, namespaces } = parseXml(body, base);
+    const model = root.uri === NS.atom && root.local === "feed" ? readAtom(root, namespaces, base) : readRss(root, namespaces, base);
+    return { ...model, ...(stylesheet && { stylesheet: resolve(stylesheet, base) }) };
+}
+const isObject = (value) => typeof value === "object" && value !== null && !Array.isArray(value);
+const string = (value) => (typeof value === "string" && value.trim() ? value : undefined);
+const hasAuthors = (value) => (Array.isArray(value.authors) && value.authors.length > 0) || isObject(value.author);
+// A JSON Feed item.
+function jsonItem(value, position) {
+    const item = isObject(value) ? value : {};
+    const id = typeof item.id === "number" ? String(item.id) : string(item.id);
+    const contents = [
+        ...(string(item.content_html) ? [{ field: "content_html", type: "html", value: item.content_html }] : []),
+        ...(string(item.content_text) ? [{ field: "content_text", type: "text", value: item.content_text }] : []),
+        ...(string(item.summary) ? [{ field: "summary", type: "text", value: item.summary }] : []),
+    ];
+    return {
+        position,
+        ...(id && { id }),
+        isPermalink: false,
+        ...(string(item.url) && { link: item.url }),
+        ...(string(item.title) && { title: item.title }),
+        titleType: "text",
+        ...(string(item.date_published) && { published: { field: "date_published", raw: item.date_published } }),
+        ...(string(item.date_modified) && { updated: { field: "date_modified", raw: item.date_modified } }),
+        hasAuthor: hasAuthors(item),
+        emails: [],
+        contents,
+        enclosures: (Array.isArray(item.attachments) ? item.attachments.filter(isObject) : []).map((attachment) => ({ url: string(attachment.url), length: attachment.size_in_bytes === undefined ? undefined : String(attachment.size_in_bytes), type: string(attachment.mime_type) })),
+        unknown: isObject(item.author) ? ["author"] : [],
+        missing: [...(id ? [] : ["id"]), ...(contents.some((content) => content.field.startsWith("content_")) ? [] : ["content_html or content_text"])],
+        durations: [],
+    };
+}
+// The model of a JSON Feed; undefined when the document is JSON but no JSON Feed.
+export function readJson(body, base) {
+    const feed = JSON.parse(body);
+    if (!isObject(feed) || typeof feed.version !== "string" || !/^https:\/\/jsonfeed\.org\/version\//.test(feed.version))
+        return undefined;
+    const hubs = (Array.isArray(feed.hubs) ? feed.hubs.filter(isObject) : []).flatMap((hub) => (string(hub.url) ? [resolve(hub.url, base)] : []));
+    return {
+        format: "json",
+        version: feed.version,
+        ...(string(feed.title) && { title: feed.title }),
+        ...(string(feed.home_page_url) && { link: feed.home_page_url }),
+        ...(string(feed.language) && { language: feed.language }),
+        ...(string(feed.feed_url) && { self: resolve(feed.feed_url, base) }),
+        hubs,
+        archives: {},
+        isComplete: false,
+        hasAuthor: hasAuthors(feed),
+        emails: [],
+        unknown: isObject(feed.author) ? ["author"] : [],
+        missing: [...(string(feed.title) ? [] : ["title"]), ...(Array.isArray(feed.items) ? [] : ["items"])],
+        namespaces: new Set(),
+        itunes: {},
+        items: (Array.isArray(feed.items) ? feed.items : []).map((item, index) => jsonItem(item, index + 1)),
+    };
+}