websnap-reader 1.0.0

package/dist/formatter.js

@@ -0,0 +1,116 @@
+"use strict";
+/**
+ * formatter.ts - Convert parsed articles to markdown or JSON output
+ */
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.formatMarkdown = formatMarkdown;
+exports.formatJSON = formatJSON;
+exports.formatSummaryPrompt = formatSummaryPrompt;
+const node_html_markdown_1 = require("node-html-markdown");
+// Singleton converter with sensible defaults
+const nhm = new node_html_markdown_1.NodeHtmlMarkdown({
+    preferNativeParser: false,
+    codeBlockStyle: "fenced",
+    bulletMarker: "-",
+    strongDelimiter: "**",
+    emDelimiter: "*",
+    maxConsecutiveNewlines: 2,
+}, 
+// Custom translators for better output
+undefined, undefined);
+/**
+ * Format article as clean markdown
+ */
+function formatMarkdown(article, url) {
+    const sections = [];
+    // Title
+    sections.push(`# ${article.title}`);
+    // Metadata line
+    const metaParts = [];
+    if (article.author)
+        metaParts.push(`By ${article.author}`);
+    if (article.date)
+        metaParts.push(article.date);
+    if (article.siteName)
+        metaParts.push(article.siteName);
+    if (metaParts.length > 0) {
+        sections.push(`*${metaParts.join(" | ")}*`);
+    }
+    // Reading stats
+    sections.push(`> ${article.wordCount.toLocaleString()} words | ${article.readingTime}`);
+    // Description / lead
+    if (article.description) {
+        sections.push(`**${article.description}**`);
+    }
+    // Separator
+    sections.push("---");
+    // Main content converted to markdown
+    const markdown = nhm.translate(article.content);
+    sections.push(cleanMarkdown(markdown));
+    // Footer
+    sections.push("---");
+    sections.push(`*Source: [${article.title}](${url})*`);
+    return sections.join("\n\n");
+}
+/**
+ * Format article as structured JSON
+ */
+function formatJSON(article, url) {
+    const markdown = nhm.translate(article.content);
+    const output = {
+        url,
+        title: article.title,
+        author: article.author,
+        date: article.date,
+        siteName: article.siteName,
+        description: article.description,
+        content: cleanMarkdown(markdown),
+        wordCount: article.wordCount,
+        readingTime: article.readingTime,
+        extractedAt: new Date().toISOString(),
+    };
+    return JSON.stringify(output, null, 2);
+}
+/**
+ * Format a prompt for AI summarization
+ */
+function formatSummaryPrompt(article) {
+    // Truncate to ~3000 words to keep within context limits
+    const words = article.textContent.split(/\s+/);
+    const truncated = words.length > 3000 ? words.slice(0, 3000).join(" ") + "..." : article.textContent;
+    return [
+        "Summarize the following article in exactly 3 concise sentences.",
+        "Focus on the key points, findings, or arguments.",
+        "Write in a neutral, informative tone.",
+        "",
+        `Title: ${article.title}`,
+        article.author ? `Author: ${article.author}` : "",
+        "",
+        "Article text:",
+        truncated,
+        "",
+        "Your 3-sentence summary:",
+    ]
+        .filter((line) => line !== undefined)
+        .join("\n");
+}
+/**
+ * Clean up markdown output
+ */
+function cleanMarkdown(md) {
+    return (md
+        // Remove excessive blank lines
+        .replace(/\n{4,}/g, "\n\n\n")
+        // Remove trailing whitespace on lines
+        .replace(/[ \t]+$/gm, "")
+        // Clean up link references that have no href
+        .replace(/\[([^\]]+)\]\(\s*\)/g, "$1")
+        // Remove empty headings
+        .replace(/^#{1,6}\s*$/gm, "")
+        // Remove image alt text that's just whitespace
+        .replace(/!\[\s*\]\([^)]+\)/g, "")
+        // Normalize bullet lists
+        .replace(/^\*\s/gm, "- ")
+        .trim());
+}
+//# sourceMappingURL=formatter.js.map

package/dist/formatter.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"formatter.js","sourceRoot":"","sources":["../src/formatter.ts"],"names":[],"mappings":";AAAA;;GAEG;;AAuBH,wCAqCC;AAKD,gCAiBC;AAKD,kDAqBC;AA1GD,2DAAsD;AAGtD,6CAA6C;AAC7C,MAAM,GAAG,GAAG,IAAI,qCAAgB,CAC9B;IACE,kBAAkB,EAAE,KAAK;IACzB,cAAc,EAAE,QAAQ;IACxB,YAAY,EAAE,GAAG;IACjB,eAAe,EAAE,IAAI;IACrB,WAAW,EAAE,GAAG;IAChB,sBAAsB,EAAE,CAAC;CAC1B;AACD,uCAAuC;AACvC,SAAS,EACT,SAAS,CACV,CAAC;AAEF;;GAEG;AACH,SAAgB,cAAc,CAAC,OAAsB,EAAE,GAAW;IAChE,MAAM,QAAQ,GAAa,EAAE,CAAC;IAE9B,QAAQ;IACR,QAAQ,CAAC,IAAI,CAAC,KAAK,OAAO,CAAC,KAAK,EAAE,CAAC,CAAC;IAEpC,gBAAgB;IAChB,MAAM,SAAS,GAAa,EAAE,CAAC;IAC/B,IAAI,OAAO,CAAC,MAAM;QAAE,SAAS,CAAC,IAAI,CAAC,MAAM,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC;IAC3D,IAAI,OAAO,CAAC,IAAI;QAAE,SAAS,CAAC,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;IAC/C,IAAI,OAAO,CAAC,QAAQ;QAAE,SAAS,CAAC,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC;IACvD,IAAI,SAAS,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACzB,QAAQ,CAAC,IAAI,CAAC,IAAI,SAAS,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;IAC9C,CAAC;IAED,gBAAgB;IAChB,QAAQ,CAAC,IAAI,CACX,KAAK,OAAO,CAAC,SAAS,CAAC,cAAc,EAAE,YAAY,OAAO,CAAC,WAAW,EAAE,CACzE,CAAC;IAEF,qBAAqB;IACrB,IAAI,OAAO,CAAC,WAAW,EAAE,CAAC;QACxB,QAAQ,CAAC,IAAI,CAAC,KAAK,OAAO,CAAC,WAAW,IAAI,CAAC,CAAC;IAC9C,CAAC;IAED,YAAY;IACZ,QAAQ,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;IAErB,qCAAqC;IACrC,MAAM,QAAQ,GAAG,GAAG,CAAC,SAAS,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;IAChD,QAAQ,CAAC,IAAI,CAAC,aAAa,CAAC,QAAQ,CAAC,CAAC,CAAC;IAEvC,SAAS;IACT,QAAQ,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;IACrB,QAAQ,CAAC,IAAI,CAAC,aAAa,OAAO,CAAC,KAAK,KAAK,GAAG,IAAI,CAAC,CAAC;IAEtD,OAAO,QAAQ,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;AAC/B,CAAC;AAED;;GAEG;AACH,SAAgB,UAAU,CAAC,OAAsB,EAAE,GAAW;IAC5D,MAAM,QAAQ,GAAG,GAAG,CAAC,SAAS,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;IAEhD,MAAM,MAAM,GAAG;QACb,GAAG;QACH,KAAK,EAAE,OAAO,CAAC,KAAK;QACpB,MAAM,EAAE,OAAO,CAAC,MAAM;QACtB,IAAI,EAAE,OAAO,CAAC,IAAI;QAClB,QAAQ,EAAE,OAAO,CAAC,QAAQ;QAC1B,WAAW,EAAE,OAAO,CAAC,WAAW;QAChC,OAAO,EAAE,aAAa,CAAC,QAAQ,CAAC;QAChC,SAAS,EAAE,OAAO,CAAC,SAAS;QAC5B,WAAW,EAAE,OAAO,CAAC,WAAW;QAChC,WAAW,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;KACtC,CAAC;IAEF,OAAO,IAAI,CAAC,SAAS,CAAC,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC;AACzC,CAAC;AAED;;GAEG;AACH,SAAgB,mBAAmB,CAAC,OAAsB;IACxD,wDAAwD;IACxD,MAAM,KAAK,GAAG,OAAO,CAAC,WAAW,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;IAC/C,MAAM,SAAS,GACb,KAAK,CAAC,MAAM,GAAG,IAAI,CAAC,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,IAAI,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,KAAK,CAAC,CAAC,CAAC,OAAO,CAAC,WAAW,CAAC;IAErF,OAAO;QACL,iEAAiE;QACjE,kDAAkD;QAClD,uCAAuC;QACvC,EAAE;QACF,UAAU,OAAO,CAAC,KAAK,EAAE;QACzB,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,WAAW,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,EAAE;QACjD,EAAE;QACF,eAAe;QACf,SAAS;QACT,EAAE;QACF,0BAA0B;KAC3B;SACE,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,KAAK,SAAS,CAAC;SACpC,IAAI,CAAC,IAAI,CAAC,CAAC;AAChB,CAAC;AAED;;GAEG;AACH,SAAS,aAAa,CAAC,EAAU;IAC/B,OAAO,CACL,EAAE;QACA,+BAA+B;SAC9B,OAAO,CAAC,SAAS,EAAE,QAAQ,CAAC;QAC7B,sCAAsC;SACrC,OAAO,CAAC,WAAW,EAAE,EAAE,CAAC;QACzB,6CAA6C;SAC5C,OAAO,CAAC,sBAAsB,EAAE,IAAI,CAAC;QACtC,wBAAwB;SACvB,OAAO,CAAC,eAAe,EAAE,EAAE,CAAC;QAC7B,+CAA+C;SAC9C,OAAO,CAAC,oBAAoB,EAAE,EAAE,CAAC;QAClC,yBAAyB;SACxB,OAAO,CAAC,SAAS,EAAE,IAAI,CAAC;SACxB,IAAI,EAAE,CACV,CAAC;AACJ,CAAC"}

package/dist/index.d.ts

@@ -0,0 +1,3 @@
+#!/usr/bin/env node
+export {};
+//# sourceMappingURL=index.d.ts.map

package/dist/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":""}

package/dist/index.js

@@ -0,0 +1,248 @@
+#!/usr/bin/env node
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || (function () {
+    var ownKeys = function(o) {
+        ownKeys = Object.getOwnPropertyNames || function (o) {
+            var ar = [];
+            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
+            return ar;
+        };
+        return ownKeys(o);
+    };
+    return function (mod) {
+        if (mod && mod.__esModule) return mod;
+        var result = {};
+        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
+        __setModuleDefault(result, mod);
+        return result;
+    };
+})();
+Object.defineProperty(exports, "__esModule", { value: true });
+const commander_1 = require("commander");
+const fetcher_1 = require("./fetcher");
+const parser_1 = require("./parser");
+const formatter_1 = require("./formatter");
+const summarizer_1 = require("./summarizer");
+const fs = __importStar(require("fs"));
+const path = __importStar(require("path"));
+const VERSION = "1.0.0";
+const program = new commander_1.Command();
+program
+    .name("websnap")
+    .description("Turn any URL into clean markdown. A better reader mode for your terminal.")
+    .version(VERSION);
+// Default command: snap a single URL
+program
+    .argument("[url]", "URL to snap")
+    .option("--json", "Output structured JSON instead of markdown")
+    .option("--summary", "Generate an AI-powered 3-sentence summary")
+    .option("--raw", "Output raw extracted HTML before markdown conversion")
+    .option("-o, --output <file>", "Write output to a file instead of stdout")
+    .option("--cdp <endpoint>", "Chrome DevTools Protocol endpoint", "http://127.0.0.1:9222")
+    .option("--timeout <ms>", "Page load timeout in milliseconds", "15000")
+    .option("--no-headless", "Use visible browser (requires CDP connection)")
+    .option("--user-agent <string>", "Custom User-Agent header")
+    .action(async (url, options) => {
+    if (!url) {
+        program.help();
+        return;
+    }
+    await snapURL(url, options);
+});
+// Batch command
+program
+    .command("batch <file>")
+    .description("Batch process URLs from a file (one URL per line)")
+    .option("--json", "Output structured JSON for each URL")
+    .option("--summary", "Generate AI summaries for each URL")
+    .option("--outdir <dir>", "Write each result to a separate file in this directory")
+    .option("--cdp <endpoint>", "Chrome DevTools Protocol endpoint", "http://127.0.0.1:9222")
+    .option("--timeout <ms>", "Page load timeout in milliseconds", "15000")
+    .option("--delay <ms>", "Delay between requests in milliseconds", "1000")
+    .action(async (file, options) => {
+    await batchProcess(file, options);
+});
+async function snapURL(url, options) {
+    const timeout = parseInt(options.timeout, 10) || 15000;
+    try {
+        // Validate URL
+        const parsedUrl = new URL(url);
+        if (!["http:", "https:"].includes(parsedUrl.protocol)) {
+            console.error(`\x1b[31mError:\x1b[0m Invalid protocol "${parsedUrl.protocol}". Use http: or https:`);
+            process.exit(1);
+        }
+    }
+    catch {
+        console.error(`\x1b[31mError:\x1b[0m Invalid URL "${url}"`);
+        process.exit(1);
+    }
+    let html;
+    try {
+        process.stderr.write(`\x1b[90mFetching ${url}...\x1b[0m\n`);
+        html = await (0, fetcher_1.fetchPage)(url, {
+            cdpEndpoint: options.cdp,
+            timeout,
+            userAgent: options.userAgent,
+        });
+    }
+    catch (err) {
+        console.error(`\x1b[31mError fetching page:\x1b[0m ${err.message || err}`);
+        process.exit(1);
+    }
+    if (options.raw) {
+        output(html, options.output);
+        return null;
+    }
+    const article = (0, parser_1.parseContent)(html, url);
+    if (options.summary) {
+        process.stderr.write(`\x1b[90mGenerating summary...\x1b[0m\n`);
+        const summaryText = await (0, summarizer_1.summarize)(article);
+        if (options.json) {
+            const jsonOut = {
+                url,
+                title: article.title,
+                author: article.author,
+                date: article.date,
+                summary: summaryText,
+                wordCount: article.wordCount,
+                readingTime: article.readingTime,
+            };
+            output(JSON.stringify(jsonOut, null, 2), options.output);
+        }
+        else {
+            const lines = [
+                `# ${article.title}`,
+                "",
+                article.author ? `*By ${article.author}*` : "",
+                article.date ? `*${article.date}*` : "",
+                "",
+                "## Summary",
+                "",
+                summaryText,
+                "",
+                `---`,
+                `*Source: ${url}*`,
+            ]
+                .filter(Boolean)
+                .join("\n");
+            output(lines, options.output);
+        }
+        return article;
+    }
+    if (options.json) {
+        output((0, formatter_1.formatJSON)(article, url), options.output);
+    }
+    else {
+        output((0, formatter_1.formatMarkdown)(article, url), options.output);
+    }
+    return article;
+}
+async function batchProcess(file, options) {
+    const filePath = path.resolve(file);
+    if (!fs.existsSync(filePath)) {
+        console.error(`\x1b[31mError:\x1b[0m File not found: ${filePath}`);
+        process.exit(1);
+    }
+    const content = fs.readFileSync(filePath, "utf-8");
+    const urls = content
+        .split("\n")
+        .map((line) => line.trim())
+        .filter((line) => line && !line.startsWith("#"));
+    if (urls.length === 0) {
+        console.error(`\x1b[31mError:\x1b[0m No URLs found in ${filePath}`);
+        process.exit(1);
+    }
+    process.stderr.write(`\x1b[90mProcessing ${urls.length} URL(s)...\x1b[0m\n`);
+    if (options.outdir) {
+        fs.mkdirSync(options.outdir, { recursive: true });
+    }
+    const delay = parseInt(options.delay, 10) || 1000;
+    const results = [];
+    let successCount = 0;
+    let failCount = 0;
+    for (let i = 0; i < urls.length; i++) {
+        const url = urls[i];
+        process.stderr.write(`\x1b[90m[${i + 1}/${urls.length}] ${url}\x1b[0m\n`);
+        try {
+            const html = await (0, fetcher_1.fetchPage)(url, {
+                cdpEndpoint: options.cdp,
+                timeout: parseInt(options.timeout, 10) || 15000,
+                userAgent: options.userAgent,
+            });
+            const article = (0, parser_1.parseContent)(html, url);
+            if (options.outdir) {
+                const slug = slugify(article.title || `page-${i + 1}`);
+                const ext = options.json ? ".json" : ".md";
+                const outPath = path.join(options.outdir, slug + ext);
+                const content = options.json
+                    ? (0, formatter_1.formatJSON)(article, url)
+                    : (0, formatter_1.formatMarkdown)(article, url);
+                fs.writeFileSync(outPath, content, "utf-8");
+                process.stderr.write(`  \x1b[32m->\x1b[0m ${outPath}\n`);
+            }
+            if (options.json) {
+                results.push({
+                    url,
+                    title: article.title,
+                    author: article.author,
+                    date: article.date,
+                    wordCount: article.wordCount,
+                    readingTime: article.readingTime,
+                    status: "ok",
+                });
+            }
+            successCount++;
+        }
+        catch (err) {
+            process.stderr.write(`  \x1b[31mFailed:\x1b[0m ${err.message || err}\n`);
+            if (options.json) {
+                results.push({ url, status: "error", error: err.message || String(err) });
+            }
+            failCount++;
+        }
+        // Delay between requests (skip after last)
+        if (i < urls.length - 1 && delay > 0) {
+            await new Promise((r) => setTimeout(r, delay));
+        }
+    }
+    process.stderr.write(`\n\x1b[90mDone: ${successCount} succeeded, ${failCount} failed\x1b[0m\n`);
+    if (options.json && !options.outdir) {
+        console.log(JSON.stringify(results, null, 2));
+    }
+}
+function output(content, filePath) {
+    if (filePath) {
+        const resolved = path.resolve(filePath);
+        fs.mkdirSync(path.dirname(resolved), { recursive: true });
+        fs.writeFileSync(resolved, content, "utf-8");
+        process.stderr.write(`\x1b[32mWritten to ${resolved}\x1b[0m\n`);
+    }
+    else {
+        console.log(content);
+    }
+}
+function slugify(text) {
+    return text
+        .toLowerCase()
+        .replace(/[^a-z0-9]+/g, "-")
+        .replace(/^-+|-+$/g, "")
+        .substring(0, 80);
+}
+program.parse(process.argv);
+//# sourceMappingURL=index.js.map

package/dist/index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAEA,yCAAoC;AACpC,uCAAsC;AACtC,qCAAuD;AACvD,2CAA8E;AAC9E,6CAAyC;AACzC,uCAAyB;AACzB,2CAA6B;AAG7B,MAAM,OAAO,GAAG,OAAO,CAAC;AAExB,MAAM,OAAO,GAAG,IAAI,mBAAO,EAAE,CAAC;AAE9B,OAAO;KACJ,IAAI,CAAC,SAAS,CAAC;KACf,WAAW,CACV,2EAA2E,CAC5E;KACA,OAAO,CAAC,OAAO,CAAC,CAAC;AAEpB,qCAAqC;AACrC,OAAO;KACJ,QAAQ,CAAC,OAAO,EAAE,aAAa,CAAC;KAChC,MAAM,CAAC,QAAQ,EAAE,4CAA4C,CAAC;KAC9D,MAAM,CAAC,WAAW,EAAE,2CAA2C,CAAC;KAChE,MAAM,CAAC,OAAO,EAAE,sDAAsD,CAAC;KACvE,MAAM,CAAC,qBAAqB,EAAE,0CAA0C,CAAC;KACzE,MAAM,CACL,kBAAkB,EAClB,mCAAmC,EACnC,uBAAuB,CACxB;KACA,MAAM,CAAC,gBAAgB,EAAE,mCAAmC,EAAE,OAAO,CAAC;KACtE,MAAM,CAAC,eAAe,EAAE,+CAA+C,CAAC;KACxE,MAAM,CACL,uBAAuB,EACvB,0BAA0B,CAC3B;KACA,MAAM,CAAC,KAAK,EAAE,GAAuB,EAAE,OAA4B,EAAE,EAAE;IACtE,IAAI,CAAC,GAAG,EAAE,CAAC;QACT,OAAO,CAAC,IAAI,EAAE,CAAC;QACf,OAAO;IACT,CAAC;IACD,MAAM,OAAO,CAAC,GAAG,EAAE,OAAO,CAAC,CAAC;AAC9B,CAAC,CAAC,CAAC;AAEL,gBAAgB;AAChB,OAAO;KACJ,OAAO,CAAC,cAAc,CAAC;KACvB,WAAW,CAAC,mDAAmD,CAAC;KAChE,MAAM,CAAC,QAAQ,EAAE,qCAAqC,CAAC;KACvD,MAAM,CAAC,WAAW,EAAE,oCAAoC,CAAC;KACzD,MAAM,CACL,gBAAgB,EAChB,wDAAwD,CACzD;KACA,MAAM,CACL,kBAAkB,EAClB,mCAAmC,EACnC,uBAAuB,CACxB;KACA,MAAM,CAAC,gBAAgB,EAAE,mCAAmC,EAAE,OAAO,CAAC;KACtE,MAAM,CAAC,cAAc,EAAE,wCAAwC,EAAE,MAAM,CAAC;KACxE,MAAM,CAAC,KAAK,EAAE,IAAY,EAAE,OAA4B,EAAE,EAAE;IAC3D,MAAM,YAAY,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;AACpC,CAAC,CAAC,CAAC;AAEL,KAAK,UAAU,OAAO,CACpB,GAAW,EACX,OAA4B;IAE5B,MAAM,OAAO,GAAG,QAAQ,CAAC,OAAO,CAAC,OAAO,EAAE,EAAE,CAAC,IAAI,KAAK,CAAC;IAEvD,IAAI,CAAC;QACH,eAAe;QACf,MAAM,SAAS,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC;QAC/B,IAAI,CAAC,CAAC,OAAO,EAAE,QAAQ,CAAC,CAAC,QAAQ,CAAC,SAAS,CAAC,QAAQ,CAAC,EAAE,CAAC;YACtD,OAAO,CAAC,KAAK,CAAC,2CAA2C,SAAS,CAAC,QAAQ,wBAAwB,CAAC,CAAC;YACrG,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QAClB,CAAC;IACH,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,CAAC,KAAK,CAAC,sCAAsC,GAAG,GAAG,CAAC,CAAC;QAC5D,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC;IAED,IAAI,IAAY,CAAC;IAEjB,IAAI,CAAC;QACH,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,oBAAoB,GAAG,cAAc,CAAC,CAAC;QAC5D,IAAI,GAAG,MAAM,IAAA,mBAAS,EAAC,GAAG,EAAE;YAC1B,WAAW,EAAE,OAAO,CAAC,GAAG;YACxB,OAAO;YACP,SAAS,EAAE,OAAO,CAAC,SAAS;SAC7B,CAAC,CAAC;IACL,CAAC;IAAC,OAAO,GAAQ,EAAE,CAAC;QAClB,OAAO,CAAC,KAAK,CACX,uCAAuC,GAAG,CAAC,OAAO,IAAI,GAAG,EAAE,CAC5D,CAAC;QACF,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC;IAED,IAAI,OAAO,CAAC,GAAG,EAAE,CAAC;QAChB,MAAM,CAAC,IAAI,EAAE,OAAO,CAAC,MAAM,CAAC,CAAC;QAC7B,OAAO,IAAI,CAAC;IACd,CAAC;IAED,MAAM,OAAO,GAAG,IAAA,qBAAY,EAAC,IAAI,EAAE,GAAG,CAAC,CAAC;IAExC,IAAI,OAAO,CAAC,OAAO,EAAE,CAAC;QACpB,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,wCAAwC,CAAC,CAAC;QAC/D,MAAM,WAAW,GAAG,MAAM,IAAA,sBAAS,EAAC,OAAO,CAAC,CAAC;QAC7C,IAAI,OAAO,CAAC,IAAI,EAAE,CAAC;YACjB,MAAM,OAAO,GAAG;gBACd,GAAG;gBACH,KAAK,EAAE,OAAO,CAAC,KAAK;gBACpB,MAAM,EAAE,OAAO,CAAC,MAAM;gBACtB,IAAI,EAAE,OAAO,CAAC,IAAI;gBAClB,OAAO,EAAE,WAAW;gBACpB,SAAS,EAAE,OAAO,CAAC,SAAS;gBAC5B,WAAW,EAAE,OAAO,CAAC,WAAW;aACjC,CAAC;YACF,MAAM,CAAC,IAAI,CAAC,SAAS,CAAC,OAAO,EAAE,IAAI,EAAE,CAAC,CAAC,EAAE,OAAO,CAAC,MAAM,CAAC,CAAC;QAC3D,CAAC;aAAM,CAAC;YACN,MAAM,KAAK,GAAG;gBACZ,KAAK,OAAO,CAAC,KAAK,EAAE;gBACpB,EAAE;gBACF,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,OAAO,OAAO,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,EAAE;gBAC9C,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,OAAO,CAAC,IAAI,GAAG,CAAC,CAAC,CAAC,EAAE;gBACvC,EAAE;gBACF,YAAY;gBACZ,EAAE;gBACF,WAAW;gBACX,EAAE;gBACF,KAAK;gBACL,YAAY,GAAG,GAAG;aACnB;iBACE,MAAM,CAAC,OAAO,CAAC;iBACf,IAAI,CAAC,IAAI,CAAC,CAAC;YACd,MAAM,CAAC,KAAK,EAAE,OAAO,CAAC,MAAM,CAAC,CAAC;QAChC,CAAC;QACD,OAAO,OAAO,CAAC;IACjB,CAAC;IAED,IAAI,OAAO,CAAC,IAAI,EAAE,CAAC;QACjB,MAAM,CAAC,IAAA,sBAAU,EAAC,OAAO,EAAE,GAAG,CAAC,EAAE,OAAO,CAAC,MAAM,CAAC,CAAC;IACnD,CAAC;SAAM,CAAC;QACN,MAAM,CAAC,IAAA,0BAAc,EAAC,OAAO,EAAE,GAAG,CAAC,EAAE,OAAO,CAAC,MAAM,CAAC,CAAC;IACvD,CAAC;IAED,OAAO,OAAO,CAAC;AACjB,CAAC;AAED,KAAK,UAAU,YAAY,CACzB,IAAY,EACZ,OAA4B;IAE5B,MAAM,QAAQ,GAAG,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;IAEpC,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,QAAQ,CAAC,EAAE,CAAC;QAC7B,OAAO,CAAC,KAAK,CAAC,yCAAyC,QAAQ,EAAE,CAAC,CAAC;QACnE,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC;IAED,MAAM,OAAO,GAAG,EAAE,CAAC,YAAY,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;IACnD,MAAM,IAAI,GAAG,OAAO;SACjB,KAAK,CAAC,IAAI,CAAC;SACX,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC;SAC1B,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,IAAI,CAAC,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC;IAEnD,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACtB,OAAO,CAAC,KAAK,CAAC,0CAA0C,QAAQ,EAAE,CAAC,CAAC;QACpE,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC;IAED,OAAO,CAAC,MAAM,CAAC,KAAK,CAClB,sBAAsB,IAAI,CAAC,MAAM,qBAAqB,CACvD,CAAC;IAEF,IAAI,OAAO,CAAC,MAAM,EAAE,CAAC;QACnB,EAAE,CAAC,SAAS,CAAC,OAAO,CAAC,MAAM,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IACpD,CAAC;IAED,MAAM,KAAK,GAAG,QAAQ,CAAC,OAAO,CAAC,KAAK,EAAE,EAAE,CAAC,IAAI,IAAI,CAAC;IAClD,MAAM,OAAO,GAAU,EAAE,CAAC;IAC1B,IAAI,YAAY,GAAG,CAAC,CAAC;IACrB,IAAI,SAAS,GAAG,CAAC,CAAC;IAElB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QACrC,MAAM,GAAG,GAAG,IAAI,CAAC,CAAC,CAAC,CAAC;QACpB,OAAO,CAAC,MAAM,CAAC,KAAK,CAClB,YAAY,CAAC,GAAG,CAAC,IAAI,IAAI,CAAC,MAAM,KAAK,GAAG,WAAW,CACpD,CAAC;QAEF,IAAI,CAAC;YACH,MAAM,IAAI,GAAG,MAAM,IAAA,mBAAS,EAAC,GAAG,EAAE;gBAChC,WAAW,EAAE,OAAO,CAAC,GAAG;gBACxB,OAAO,EAAE,QAAQ,CAAC,OAAO,CAAC,OAAO,EAAE,EAAE,CAAC,IAAI,KAAK;gBAC/C,SAAS,EAAE,OAAO,CAAC,SAAS;aAC7B,CAAC,CAAC;YAEH,MAAM,OAAO,GAAG,IAAA,qBAAY,EAAC,IAAI,EAAE,GAAG,CAAC,CAAC;YAExC,IAAI,OAAO,CAAC,MAAM,EAAE,CAAC;gBACnB,MAAM,IAAI,GAAG,OAAO,CAAC,OAAO,CAAC,KAAK,IAAI,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;gBACvD,MAAM,GAAG,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,KAAK,CAAC;gBAC3C,MAAM,OAAO,GAAG,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE,IAAI,GAAG,GAAG,CAAC,CAAC;gBACtD,MAAM,OAAO,GAAG,OAAO,CAAC,IAAI;oBAC1B,CAAC,CAAC,IAAA,sBAAU,EAAC,OAAO,EAAE,GAAG,CAAC;oBAC1B,CAAC,CAAC,IAAA,0BAAc,EAAC,OAAO,EAAE,GAAG,CAAC,CAAC;gBACjC,EAAE,CAAC,aAAa,CAAC,OAAO,EAAE,OAAO,EAAE,OAAO,CAAC,CAAC;gBAC5C,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,uBAAuB,OAAO,IAAI,CAAC,CAAC;YAC3D,CAAC;YAED,IAAI,OAAO,CAAC,IAAI,EAAE,CAAC;gBACjB,OAAO,CAAC,IAAI,CAAC;oBACX,GAAG;oBACH,KAAK,EAAE,OAAO,CAAC,KAAK;oBACpB,MAAM,EAAE,OAAO,CAAC,MAAM;oBACtB,IAAI,EAAE,OAAO,CAAC,IAAI;oBAClB,SAAS,EAAE,OAAO,CAAC,SAAS;oBAC5B,WAAW,EAAE,OAAO,CAAC,WAAW;oBAChC,MAAM,EAAE,IAAI;iBACb,CAAC,CAAC;YACL,CAAC;YAED,YAAY,EAAE,CAAC;QACjB,CAAC;QAAC,OAAO,GAAQ,EAAE,CAAC;YAClB,OAAO,CAAC,MAAM,CAAC,KAAK,CAClB,4BAA4B,GAAG,CAAC,OAAO,IAAI,GAAG,IAAI,CACnD,CAAC;YACF,IAAI,OAAO,CAAC,IAAI,EAAE,CAAC;gBACjB,OAAO,CAAC,IAAI,CAAC,EAAE,GAAG,EAAE,MAAM,EAAE,OAAO,EAAE,KAAK,EAAE,GAAG,CAAC,OAAO,IAAI,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;YAC5E,CAAC;YACD,SAAS,EAAE,CAAC;QACd,CAAC;QAED,2CAA2C;QAC3C,IAAI,CAAC,GAAG,IAAI,CAAC,MAAM,GAAG,CAAC,IAAI,KAAK,GAAG,CAAC,EAAE,CAAC;YACrC,MAAM,IAAI,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,UAAU,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC,CAAC;QACjD,CAAC;IACH,CAAC;IAED,OAAO,CAAC,MAAM,CAAC,KAAK,CAClB,mBAAmB,YAAY,eAAe,SAAS,kBAAkB,CAC1E,CAAC;IAEF,IAAI,OAAO,CAAC,IAAI,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE,CAAC;QACpC,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,SAAS,CAAC,OAAO,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC;IAChD,CAAC;AACH,CAAC;AAED,SAAS,MAAM,CAAC,OAAe,EAAE,QAAiB;IAChD,IAAI,QAAQ,EAAE,CAAC;QACb,MAAM,QAAQ,GAAG,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC;QACxC,EAAE,CAAC,SAAS,CAAC,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;QAC1D,EAAE,CAAC,aAAa,CAAC,QAAQ,EAAE,OAAO,EAAE,OAAO,CAAC,CAAC;QAC7C,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,sBAAsB,QAAQ,WAAW,CAAC,CAAC;IAClE,CAAC;SAAM,CAAC;QACN,OAAO,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;IACvB,CAAC;AACH,CAAC;AAED,SAAS,OAAO,CAAC,IAAY;IAC3B,OAAO,IAAI;SACR,WAAW,EAAE;SACb,OAAO,CAAC,aAAa,EAAE,GAAG,CAAC;SAC3B,OAAO,CAAC,UAAU,EAAE,EAAE,CAAC;SACvB,SAAS,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;AACtB,CAAC;AAED,OAAO,CAAC,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC"}

package/dist/parser.d.ts

@@ -0,0 +1,25 @@
+/**
+ * parser.ts - Extract clean article content from raw HTML
+ *
+ * Implements a simplified readability algorithm:
+ * 1. Strip non-content elements (nav, ads, scripts, styles, etc.)
+ * 2. Identify the main content container
+ * 3. Extract metadata (title, author, date)
+ * 4. Calculate reading statistics
+ */
+export interface ParsedArticle {
+    title: string;
+    author: string | null;
+    date: string | null;
+    siteName: string | null;
+    description: string | null;
+    content: string;
+    textContent: string;
+    wordCount: number;
+    readingTime: string;
+}
+/**
+ * Parse HTML and extract clean article content
+ */
+export declare function parseContent(html: string, url: string): ParsedArticle;
+//# sourceMappingURL=parser.d.ts.map

package/dist/parser.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"parser.d.ts","sourceRoot":"","sources":["../src/parser.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AAEH,MAAM,WAAW,aAAa;IAC5B,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,MAAM,GAAG,IAAI,CAAC;IACtB,IAAI,EAAE,MAAM,GAAG,IAAI,CAAC;IACpB,QAAQ,EAAE,MAAM,GAAG,IAAI,CAAC;IACxB,WAAW,EAAE,MAAM,GAAG,IAAI,CAAC;IAC3B,OAAO,EAAE,MAAM,CAAC;IAChB,WAAW,EAAE,MAAM,CAAC;IACpB,SAAS,EAAE,MAAM,CAAC;IAClB,WAAW,EAAE,MAAM,CAAC;CACrB;AAoFD;;GAEG;AACH,wBAAgB,YAAY,CAAC,IAAI,EAAE,MAAM,EAAE,GAAG,EAAE,MAAM,GAAG,aAAa,CAiDrE"}