websnap-reader 1.0.0

package/README.md

@@ -0,0 +1,187 @@
+# websnap
+
+Turn any URL into clean markdown. A better "reader mode" for your terminal.
+
+## Install
+
+```bash
+npm install -g @wilsonxu/websnap
+```
+
+Or install locally from source:
+
+```bash
+git clone <repo-url>
+cd websnap
+npm install && npm run build
+npm link
+```
+
+## Usage
+
+### Basic: URL to Markdown
+
+```bash
+websnap https://example.com
+```
+
+Outputs clean, readable markdown to stdout. Pipe it anywhere:
+
+```bash
+websnap https://blog.example.com/post > article.md
+```
+
+### Structured JSON Output
+
+```bash
+websnap https://example.com --json
+```
+
+Returns:
+
+```json
+{
+  "url": "https://example.com",
+  "title": "Example Article",
+  "author": "John Doe",
+  "date": "March 15, 2024",
+  "content": "# Example Article\n\nArticle content in markdown...",
+  "wordCount": 1234,
+  "readingTime": "6 min read",
+  "extractedAt": "2024-03-15T10:30:00.000Z"
+}
+```
+
+### AI-Powered Summary
+
+```bash
+websnap https://example.com --summary
+```
+
+Generates a concise 3-sentence summary. Supports multiple AI backends:
+
+| Backend   | Setup                              | Model Default       |
+| --------- | ---------------------------------- | ------------------- |
+| OpenAI    | `export OPENAI_API_KEY=sk-...`     | gpt-4o-mini         |
+| Anthropic | `export ANTHROPIC_API_KEY=sk-...`  | claude-sonnet-4-20250514 |
+| Ollama    | Run `ollama serve` locally         | llama3.2            |
+| Fallback  | No setup needed                    | Extractive summary  |
+
+Combine with JSON:
+
+```bash
+websnap https://example.com --summary --json
+```
+
+### Batch Processing
+
+Create a file with one URL per line:
+
+```
+# urls.txt
+https://example.com/article-1
+https://example.com/article-2
+https://example.com/article-3
+```
+
+Process all at once:
+
+```bash
+websnap batch urls.txt --outdir ./articles
+websnap batch urls.txt --json
+websnap batch urls.txt --outdir ./summaries --summary
+```
+
+### Save to File
+
+```bash
+websnap https://example.com -o article.md
+websnap https://example.com --json -o article.json
+```
+
+## Chrome CDP Integration
+
+For JavaScript-heavy sites or login-required pages, websnap can connect to your running Chrome browser via the Chrome DevTools Protocol.
+
+### Setup
+
+Start Chrome with remote debugging enabled:
+
+```bash
+# macOS
+/Applications/Google\ Chrome.app/Contents/MacOS/Google\ Chrome --remote-debugging-port=9222
+
+# Linux
+google-chrome --remote-debugging-port=9222
+```
+
+Now websnap will automatically use CDP to fetch pages, reusing your existing cookies and sessions. This means **login-required pages just work** if you're already logged in.
+
+### Custom CDP Endpoint
+
+```bash
+websnap https://example.com --cdp http://localhost:9333
+```
+
+### Fallback
+
+If Chrome CDP is not available, websnap automatically falls back to plain HTTP fetch. Static pages work without any Chrome setup.
+
+## Options
+
+| Flag                   | Description                                |
+| ---------------------- | ------------------------------------------ |
+| `--json`               | Output structured JSON                     |
+| `--summary`            | Generate AI-powered 3-sentence summary     |
+| `--raw`                | Output raw extracted HTML                  |
+| `-o, --output <file>`  | Write output to file                       |
+| `--cdp <endpoint>`     | Chrome CDP endpoint (default: `http://127.0.0.1:9222`) |
+| `--timeout <ms>`       | Page load timeout (default: 15000)         |
+| `--user-agent <str>`   | Custom User-Agent string                   |
+| `-V, --version`        | Show version number                        |
+| `-h, --help`           | Show help                                  |
+
+### Batch Options
+
+| Flag               | Description                                    |
+| ------------------ | ---------------------------------------------- |
+| `--outdir <dir>`   | Write each result as a separate file           |
+| `--delay <ms>`     | Delay between requests (default: 1000)         |
+
+## Environment Variables
+
+| Variable           | Description                        |
+| ------------------ | ---------------------------------- |
+| `OPENAI_API_KEY`   | OpenAI API key for summaries       |
+| `OPENAI_MODEL`     | OpenAI model (default: gpt-4o-mini)|
+| `ANTHROPIC_API_KEY`| Anthropic API key for summaries    |
+| `ANTHROPIC_MODEL`  | Anthropic model (default: claude-sonnet-4-20250514) |
+| `OLLAMA_URL`       | Ollama server URL (default: http://127.0.0.1:11434) |
+| `OLLAMA_MODEL`     | Ollama model (default: llama3.2)   |
+
+## Examples
+
+```bash
+# Quick read of a blog post
+websnap https://paulgraham.com/greatwork.html
+
+# Save an article as JSON for processing
+websnap https://arxiv.org/abs/2301.00001 --json -o paper.json
+
+# Get a quick summary
+websnap https://news.ycombinator.com/item?id=12345 --summary
+
+# Batch scrape a list of articles
+websnap batch research-urls.txt --outdir ./research --json
+
+# Use with jq for data extraction
+websnap https://example.com --json | jq '.title, .wordCount'
+
+# Pipe to other tools
+websnap https://example.com | glow -    # render with glow
+websnap https://example.com | pbcopy    # copy to clipboard (macOS)
+```
+
+## License
+
+MIT

package/dist/fetcher.d.ts

@@ -0,0 +1,20 @@
+/**
+ * fetcher.ts - Page fetching via Chrome CDP or plain HTTP
+ *
+ * Strategy:
+ * 1. Try Chrome CDP connection (port 9222 by default) for JS-rendered pages
+ * 2. Fall back to plain HTTP fetch for static pages
+ *
+ * CDP mode reuses existing Chrome cookies, so login-required pages work
+ * if the user has already logged in via their Chrome browser.
+ */
+export interface FetchOptions {
+    cdpEndpoint?: string;
+    timeout?: number;
+    userAgent?: string;
+}
+/**
+ * Fetch a page's HTML content. Tries CDP first, then falls back to HTTP.
+ */
+export declare function fetchPage(url: string, options?: FetchOptions): Promise<string>;
+//# sourceMappingURL=fetcher.d.ts.map

package/dist/fetcher.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"fetcher.d.ts","sourceRoot":"","sources":["../src/fetcher.ts"],"names":[],"mappings":"AAAA;;;;;;;;;GASG;AAEH,MAAM,WAAW,YAAY;IAC3B,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB;AASD;;GAEG;AACH,wBAAsB,SAAS,CAC7B,GAAG,EAAE,MAAM,EACX,OAAO,GAAE,YAAiB,GACzB,OAAO,CAAC,MAAM,CAAC,CAejB"}

package/dist/fetcher.js

@@ -0,0 +1,350 @@
+"use strict";
+/**
+ * fetcher.ts - Page fetching via Chrome CDP or plain HTTP
+ *
+ * Strategy:
+ * 1. Try Chrome CDP connection (port 9222 by default) for JS-rendered pages
+ * 2. Fall back to plain HTTP fetch for static pages
+ *
+ * CDP mode reuses existing Chrome cookies, so login-required pages work
+ * if the user has already logged in via their Chrome browser.
+ */
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || (function () {
+    var ownKeys = function(o) {
+        ownKeys = Object.getOwnPropertyNames || function (o) {
+            var ar = [];
+            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
+            return ar;
+        };
+        return ownKeys(o);
+    };
+    return function (mod) {
+        if (mod && mod.__esModule) return mod;
+        var result = {};
+        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
+        __setModuleDefault(result, mod);
+        return result;
+    };
+})();
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.fetchPage = fetchPage;
+/**
+ * Fetch a page's HTML content. Tries CDP first, then falls back to HTTP.
+ */
+async function fetchPage(url, options = {}) {
+    const { cdpEndpoint = "http://127.0.0.1:9222", timeout = 15000 } = options;
+    // Try CDP first
+    try {
+        const html = await fetchViaCDP(url, cdpEndpoint, timeout, options.userAgent);
+        return html;
+    }
+    catch (cdpErr) {
+        // CDP not available; fall back to HTTP
+        process.stderr.write(`\x1b[90mCDP unavailable (${cdpErr.message?.substring(0, 60) || "connection failed"}), using HTTP fetch...\x1b[0m\n`);
+    }
+    return fetchViaHTTP(url, timeout, options.userAgent);
+}
+/**
+ * Fetch via Chrome DevTools Protocol
+ */
+async function fetchViaCDP(url, cdpEndpoint, timeout, userAgent) {
+    // Get browser websocket URL
+    const versionUrl = `${cdpEndpoint}/json/version`;
+    const versionRes = await fetchWithTimeout(versionUrl, 3000);
+    if (!versionRes.ok)
+        throw new Error(`CDP version endpoint returned ${versionRes.status}`);
+    const versionInfo = await versionRes.json();
+    // Create a new target (tab)
+    const newTabUrl = `${cdpEndpoint}/json/new?${encodeURIComponent("about:blank")}`;
+    const newTabRes = await fetchWithTimeout(newTabUrl, 3000);
+    if (!newTabRes.ok)
+        throw new Error(`CDP new tab returned ${newTabRes.status}`);
+    const tabInfo = (await newTabRes.json());
+    const wsUrl = tabInfo.webSocketDebuggerUrl;
+    if (!wsUrl)
+        throw new Error("No WebSocket URL for new tab");
+    // Connect via WebSocket
+    const session = await connectCDP(wsUrl);
+    try {
+        // Set user agent if provided
+        if (userAgent) {
+            await session.send("Network.setUserAgentOverride", {
+                userAgent,
+            });
+        }
+        // Enable page events
+        await session.send("Page.enable");
+        await session.send("Network.enable");
+        // Navigate
+        const navResult = await session.send("Page.navigate", { url });
+        // Wait for load
+        await waitForLoad(session, timeout);
+        // Small extra delay for JS rendering
+        await new Promise((r) => setTimeout(r, 1500));
+        // Get document HTML
+        const result = await session.send("Runtime.evaluate", {
+            expression: "document.documentElement.outerHTML",
+            returnByValue: true,
+        });
+        const html = result?.result?.value;
+        if (!html || typeof html !== "string") {
+            throw new Error("Failed to extract HTML from page");
+        }
+        return html;
+    }
+    finally {
+        // Close the tab
+        try {
+            const closeUrl = `${cdpEndpoint}/json/close/${tabInfo.id}`;
+            await fetchWithTimeout(closeUrl, 2000).catch(() => { });
+        }
+        catch { }
+        session.close();
+    }
+}
+/**
+ * Minimal CDP WebSocket connection using Node.js built-ins
+ */
+async function connectCDP(wsUrl) {
+    // Dynamic import for WebSocket (available in Node 18+)
+    const WebSocket = globalThis.WebSocket || (await getWebSocket());
+    return new Promise((resolve, reject) => {
+        const ws = new WebSocket(wsUrl);
+        let msgId = 0;
+        const pending = new Map();
+        const eventHandlers = new Map();
+        ws.onopen = () => {
+            const session = {
+                ws,
+                id: 0,
+                send(method, params = {}) {
+                    return new Promise((res, rej) => {
+                        const id = ++msgId;
+                        pending.set(id, { resolve: res, reject: rej });
+                        ws.send(JSON.stringify({ id, method, params }));
+                        // Timeout for individual commands
+                        setTimeout(() => {
+                            if (pending.has(id)) {
+                                pending.delete(id);
+                                rej(new Error(`CDP command "${method}" timed out`));
+                            }
+                        }, 30000);
+                    });
+                },
+                close() {
+                    try {
+                        ws.close();
+                    }
+                    catch { }
+                },
+            };
+            session._eventHandlers = eventHandlers;
+            resolve(session);
+        };
+        ws.onmessage = (event) => {
+            try {
+                const data = typeof event.data === "string" ? event.data : event.data.toString();
+                const msg = JSON.parse(data);
+                if (msg.id && pending.has(msg.id)) {
+                    const handler = pending.get(msg.id);
+                    pending.delete(msg.id);
+                    if (msg.error) {
+                        handler.reject(new Error(msg.error.message || JSON.stringify(msg.error)));
+                    }
+                    else {
+                        handler.resolve(msg.result);
+                    }
+                }
+                if (msg.method && eventHandlers.has(msg.method)) {
+                    for (const h of eventHandlers.get(msg.method)) {
+                        h(msg.params);
+                    }
+                }
+            }
+            catch { }
+        };
+        ws.onerror = (err) => {
+            reject(new Error(`WebSocket error: ${err.message || "connection failed"}`));
+        };
+        // Connection timeout
+        setTimeout(() => {
+            reject(new Error("CDP WebSocket connection timed out"));
+        }, 5000);
+    });
+}
+/**
+ * Try to get WebSocket from 'ws' package or undici
+ */
+async function getWebSocket() {
+    try {
+        // Node 18+ has WebSocket behind a flag, Node 21+ has it globally
+        const { WebSocket } = await Promise.resolve(`${"undici"}`).then(s => __importStar(require(s)));
+        return WebSocket;
+    }
+    catch {
+        try {
+            const ws = await Promise.resolve(`${"ws"}`).then(s => __importStar(require(s)));
+            return ws.default || ws.WebSocket || ws;
+        }
+        catch {
+            throw new Error("No WebSocket implementation available. Use Node.js 21+ or install 'ws' package.");
+        }
+    }
+}
+/**
+ * Wait for page load event via CDP
+ */
+function waitForLoad(session, timeout) {
+    return new Promise((resolve, reject) => {
+        const eventHandlers = session._eventHandlers;
+        const handlers = eventHandlers.get("Page.loadEventFired") || [];
+        handlers.push(() => {
+            clearTimeout(timer);
+            resolve();
+        });
+        eventHandlers.set("Page.loadEventFired", handlers);
+        const timer = setTimeout(() => {
+            // Resolve anyway after timeout - page may have partially loaded
+            resolve();
+        }, timeout);
+    });
+}
+/**
+ * Simple HTTP fetch fallback for static pages.
+ * Tries native fetch first, then falls back to node:https for TLS issues.
+ */
+async function fetchViaHTTP(url, timeout, userAgent) {
+    const ua = userAgent ||
+        "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36";
+    // Try native fetch first
+    try {
+        return await fetchViaHTTPNative(url, timeout, ua);
+    }
+    catch (nativeErr) {
+        // If it's a TLS certificate error, fall back to node:https
+        const msg = String(nativeErr?.cause || nativeErr?.message || "");
+        if (msg.includes("UNABLE_TO_GET_ISSUER_CERT") ||
+            msg.includes("CERT_HAS_EXPIRED") ||
+            msg.includes("DEPTH_ZERO_SELF_SIGNED") ||
+            msg.includes("certificate") ||
+            msg.includes("fetch failed")) {
+            process.stderr.write(`\x1b[90mNative fetch failed, using node:https fallback...\x1b[0m\n`);
+            return fetchViaNodeHTTPS(url, timeout, ua);
+        }
+        throw nativeErr;
+    }
+}
+async function fetchViaHTTPNative(url, timeout, userAgent) {
+    const headers = {
+        Accept: "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
+        "Accept-Language": "en-US,en;q=0.9",
+        "User-Agent": userAgent,
+    };
+    const controller = new AbortController();
+    const timer = setTimeout(() => controller.abort(), timeout);
+    try {
+        const response = await fetch(url, {
+            headers,
+            signal: controller.signal,
+            redirect: "follow",
+        });
+        if (!response.ok) {
+            throw new Error(`HTTP ${response.status} ${response.statusText}`);
+        }
+        const contentType = response.headers.get("content-type") || "";
+        if (!contentType.includes("text/html") &&
+            !contentType.includes("application/xhtml")) {
+            process.stderr.write(`\x1b[33mWarning:\x1b[0m Content-Type is "${contentType}", not HTML\n`);
+        }
+        return await response.text();
+    }
+    finally {
+        clearTimeout(timer);
+    }
+}
+/**
+ * Fallback HTTP fetcher using node:https with relaxed TLS for environments
+ * that have certificate issues (corporate proxies, outdated cert stores, etc.)
+ */
+async function fetchViaNodeHTTPS(url, timeout, userAgent) {
+    const https = await Promise.resolve().then(() => __importStar(require("node:https")));
+    const http = await Promise.resolve().then(() => __importStar(require("node:http")));
+    const { URL } = await Promise.resolve().then(() => __importStar(require("node:url")));
+    return new Promise((resolve, reject) => {
+        const parsedUrl = new URL(url);
+        const isHTTPS = parsedUrl.protocol === "https:";
+        const mod = isHTTPS ? https : http;
+        const options = {
+            hostname: parsedUrl.hostname,
+            port: parsedUrl.port || (isHTTPS ? 443 : 80),
+            path: parsedUrl.pathname + parsedUrl.search,
+            method: "GET",
+            headers: {
+                Accept: "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
+                "Accept-Language": "en-US,en;q=0.9",
+                "User-Agent": userAgent,
+            },
+            rejectUnauthorized: false, // Handle self-signed/corporate certs
+            timeout,
+        };
+        const req = mod.request(options, (res) => {
+            // Follow redirects (up to 5)
+            if (res.statusCode >= 300 &&
+                res.statusCode < 400 &&
+                res.headers.location) {
+                const redirectUrl = new URL(res.headers.location, url).toString();
+                fetchViaNodeHTTPS(redirectUrl, timeout, userAgent)
+                    .then(resolve)
+                    .catch(reject);
+                return;
+            }
+            if (res.statusCode < 200 || res.statusCode >= 400) {
+                reject(new Error(`HTTP ${res.statusCode}`));
+                return;
+            }
+            const chunks = [];
+            res.on("data", (chunk) => chunks.push(chunk));
+            res.on("end", () => {
+                const body = Buffer.concat(chunks).toString("utf-8");
+                resolve(body);
+            });
+            res.on("error", reject);
+        });
+        req.on("error", reject);
+        req.on("timeout", () => {
+            req.destroy();
+            reject(new Error("Request timed out"));
+        });
+        req.end();
+    });
+}
+/**
+ * Fetch helper with timeout
+ */
+async function fetchWithTimeout(url, timeout) {
+    const controller = new AbortController();
+    const timer = setTimeout(() => controller.abort(), timeout);
+    try {
+        return await fetch(url, { signal: controller.signal });
+    }
+    finally {
+        clearTimeout(timer);
+    }
+}
+//# sourceMappingURL=fetcher.js.map

package/dist/fetcher.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"fetcher.js","sourceRoot":"","sources":["../src/fetcher.ts"],"names":[],"mappings":";AAAA;;;;;;;;;GASG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAkBH,8BAkBC;AArBD;;GAEG;AACI,KAAK,UAAU,SAAS,CAC7B,GAAW,EACX,UAAwB,EAAE;IAE1B,MAAM,EAAE,WAAW,GAAG,uBAAuB,EAAE,OAAO,GAAG,KAAK,EAAE,GAAG,OAAO,CAAC;IAE3E,gBAAgB;IAChB,IAAI,CAAC;QACH,MAAM,IAAI,GAAG,MAAM,WAAW,CAAC,GAAG,EAAE,WAAW,EAAE,OAAO,EAAE,OAAO,CAAC,SAAS,CAAC,CAAC;QAC7E,OAAO,IAAI,CAAC;IACd,CAAC;IAAC,OAAO,MAAW,EAAE,CAAC;QACrB,uCAAuC;QACvC,OAAO,CAAC,MAAM,CAAC,KAAK,CAClB,4BAA4B,MAAM,CAAC,OAAO,EAAE,SAAS,CAAC,CAAC,EAAE,EAAE,CAAC,IAAI,mBAAmB,iCAAiC,CACrH,CAAC;IACJ,CAAC;IAED,OAAO,YAAY,CAAC,GAAG,EAAE,OAAO,EAAE,OAAO,CAAC,SAAS,CAAC,CAAC;AACvD,CAAC;AAED;;GAEG;AACH,KAAK,UAAU,WAAW,CACxB,GAAW,EACX,WAAmB,EACnB,OAAe,EACf,SAAkB;IAElB,4BAA4B;IAC5B,MAAM,UAAU,GAAG,GAAG,WAAW,eAAe,CAAC;IACjD,MAAM,UAAU,GAAG,MAAM,gBAAgB,CAAC,UAAU,EAAE,IAAI,CAAC,CAAC;IAC5D,IAAI,CAAC,UAAU,CAAC,EAAE;QAAE,MAAM,IAAI,KAAK,CAAC,iCAAiC,UAAU,CAAC,MAAM,EAAE,CAAC,CAAC;IAC1F,MAAM,WAAW,GAAG,MAAM,UAAU,CAAC,IAAI,EAAE,CAAC;IAE5C,4BAA4B;IAC5B,MAAM,SAAS,GAAG,GAAG,WAAW,aAAa,kBAAkB,CAAC,aAAa,CAAC,EAAE,CAAC;IACjF,MAAM,SAAS,GAAG,MAAM,gBAAgB,CAAC,SAAS,EAAE,IAAI,CAAC,CAAC;IAC1D,IAAI,CAAC,SAAS,CAAC,EAAE;QAAE,MAAM,IAAI,KAAK,CAAC,wBAAwB,SAAS,CAAC,MAAM,EAAE,CAAC,CAAC;IAC/E,MAAM,OAAO,GAAG,CAAC,MAAM,SAAS,CAAC,IAAI,EAAE,CAAwB,CAAC;IAChE,MAAM,KAAK,GAAG,OAAO,CAAC,oBAAoB,CAAC;IAE3C,IAAI,CAAC,KAAK;QAAE,MAAM,IAAI,KAAK,CAAC,8BAA8B,CAAC,CAAC;IAE5D,wBAAwB;IACxB,MAAM,OAAO,GAAG,MAAM,UAAU,CAAC,KAAK,CAAC,CAAC;IAExC,IAAI,CAAC;QACH,6BAA6B;QAC7B,IAAI,SAAS,EAAE,CAAC;YACd,MAAM,OAAO,CAAC,IAAI,CAAC,8BAA8B,EAAE;gBACjD,SAAS;aACV,CAAC,CAAC;QACL,CAAC;QAED,qBAAqB;QACrB,MAAM,OAAO,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC;QAClC,MAAM,OAAO,CAAC,IAAI,CAAC,gBAAgB,CAAC,CAAC;QAErC,WAAW;QACX,MAAM,SAAS,GAAG,MAAM,OAAO,CAAC,IAAI,CAAC,eAAe,EAAE,EAAE,GAAG,EAAE,CAAC,CAAC;QAE/D,gBAAgB;QAChB,MAAM,WAAW,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC;QAEpC,qCAAqC;QACrC,MAAM,IAAI,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,UAAU,CAAC,CAAC,EAAE,IAAI,CAAC,CAAC,CAAC;QAE9C,oBAAoB;QACpB,MAAM,MAAM,GAAG,MAAM,OAAO,CAAC,IAAI,CAAC,kBAAkB,EAAE;YACpD,UAAU,EAAE,oCAAoC;YAChD,aAAa,EAAE,IAAI;SACpB,CAAC,CAAC;QAEH,MAAM,IAAI,GAAG,MAAM,EAAE,MAAM,EAAE,KAAK,CAAC;QACnC,IAAI,CAAC,IAAI,IAAI,OAAO,IAAI,KAAK,QAAQ,EAAE,CAAC;YACtC,MAAM,IAAI,KAAK,CAAC,kCAAkC,CAAC,CAAC;QACtD,CAAC;QAED,OAAO,IAAI,CAAC;IACd,CAAC;YAAS,CAAC;QACT,gBAAgB;QAChB,IAAI,CAAC;YACH,MAAM,QAAQ,GAAG,GAAG,WAAW,eAAe,OAAO,CAAC,EAAE,EAAE,CAAC;YAC3D,MAAM,gBAAgB,CAAC,QAAQ,EAAE,IAAI,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,GAAE,CAAC,CAAC,CAAC;QACzD,CAAC;QAAC,MAAM,CAAC,CAAA,CAAC;QACV,OAAO,CAAC,KAAK,EAAE,CAAC;IAClB,CAAC;AACH,CAAC;AAED;;GAEG;AACH,KAAK,UAAU,UAAU,CAAC,KAAa;IACrC,uDAAuD;IACvD,MAAM,SAAS,GAAI,UAAkB,CAAC,SAAS,IAAI,CAAC,MAAM,YAAY,EAAE,CAAC,CAAC;IAE1E,OAAO,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;QACrC,MAAM,EAAE,GAAG,IAAI,SAAS,CAAC,KAAK,CAAC,CAAC;QAChC,IAAI,KAAK,GAAG,CAAC,CAAC;QACd,MAAM,OAAO,GAAG,IAAI,GAAG,EAGpB,CAAC;QACJ,MAAM,aAAa,GAAG,IAAI,GAAG,EAAqC,CAAC;QAEnE,EAAE,CAAC,MAAM,GAAG,GAAG,EAAE;YACf,MAAM,OAAO,GAAe;gBAC1B,EAAE;gBACF,EAAE,EAAE,CAAC;gBACL,IAAI,CAAC,MAAc,EAAE,SAA8B,EAAE;oBACnD,OAAO,IAAI,OAAO,CAAC,CAAC,GAAG,EAAE,GAAG,EAAE,EAAE;wBAC9B,MAAM,EAAE,GAAG,EAAE,KAAK,CAAC;wBACnB,OAAO,CAAC,GAAG,CAAC,EAAE,EAAE,EAAE,OAAO,EAAE,GAAG,EAAE,MAAM,EAAE,GAAG,EAAE,CAAC,CAAC;wBAC/C,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,EAAE,EAAE,EAAE,MAAM,EAAE,MAAM,EAAE,CAAC,CAAC,CAAC;wBAEhD,kCAAkC;wBAClC,UAAU,CAAC,GAAG,EAAE;4BACd,IAAI,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC,EAAE,CAAC;gCACpB,OAAO,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC;gCACnB,GAAG,CAAC,IAAI,KAAK,CAAC,gBAAgB,MAAM,aAAa,CAAC,CAAC,CAAC;4BACtD,CAAC;wBACH,CAAC,EAAE,KAAK,CAAC,CAAC;oBACZ,CAAC,CAAC,CAAC;gBACL,CAAC;gBACD,KAAK;oBACH,IAAI,CAAC;wBACH,EAAE,CAAC,KAAK,EAAE,CAAC;oBACb,CAAC;oBAAC,MAAM,CAAC,CAAA,CAAC;gBACZ,CAAC;aACF,CAAC;YAED,OAAe,CAAC,cAAc,GAAG,aAAa,CAAC;YAChD,OAAO,CAAC,OAAO,CAAC,CAAC;QACnB,CAAC,CAAC;QAEF,EAAE,CAAC,SAAS,GAAG,CAAC,KAAU,EAAE,EAAE;YAC5B,IAAI,CAAC;gBACH,MAAM,IAAI,GACR,OAAO,KAAK,CAAC,IAAI,KAAK,QAAQ,CAAC,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,QAAQ,EAAE,CAAC;gBACtE,MAAM,GAAG,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;gBAE7B,IAAI,GAAG,CAAC,EAAE,IAAI,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC,EAAE,CAAC;oBAClC,MAAM,OAAO,GAAG,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAE,CAAC;oBACrC,OAAO,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;oBACvB,IAAI,GAAG,CAAC,KAAK,EAAE,CAAC;wBACd,OAAO,CAAC,MAAM,CAAC,IAAI,KAAK,CAAC,GAAG,CAAC,KAAK,CAAC,OAAO,IAAI,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;oBAC5E,CAAC;yBAAM,CAAC;wBACN,OAAO,CAAC,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;oBAC9B,CAAC;gBACH,CAAC;gBAED,IAAI,GAAG,CAAC,MAAM,IAAI,aAAa,CAAC,GAAG,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC;oBAChD,KAAK,MAAM,CAAC,IAAI,aAAa,CAAC,GAAG,CAAC,GAAG,CAAC,MAAM,CAAE,EAAE,CAAC;wBAC/C,CAAC,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;oBAChB,CAAC;gBACH,CAAC;YACH,CAAC;YAAC,MAAM,CAAC,CAAA,CAAC;QACZ,CAAC,CAAC;QAEF,EAAE,CAAC,OAAO,GAAG,CAAC,GAAQ,EAAE,EAAE;YACxB,MAAM,CAAC,IAAI,KAAK,CAAC,oBAAoB,GAAG,CAAC,OAAO,IAAI,mBAAmB,EAAE,CAAC,CAAC,CAAC;QAC9E,CAAC,CAAC;QAEF,qBAAqB;QACrB,UAAU,CAAC,GAAG,EAAE;YACd,MAAM,CAAC,IAAI,KAAK,CAAC,oCAAoC,CAAC,CAAC,CAAC;QAC1D,CAAC,EAAE,IAAI,CAAC,CAAC;IACX,CAAC,CAAC,CAAC;AACL,CAAC;AAED;;GAEG;AACH,KAAK,UAAU,YAAY;IACzB,IAAI,CAAC;QACH,iEAAiE;QACjE,MAAM,EAAE,SAAS,EAAE,GAAG,yBAAa,QAAe,uCAAC,CAAC;QACpD,OAAO,SAAS,CAAC;IACnB,CAAC;IAAC,MAAM,CAAC;QACP,IAAI,CAAC;YACH,MAAM,EAAE,GAAG,yBAAa,IAAW,uCAAC,CAAC;YACrC,OAAO,EAAE,CAAC,OAAO,IAAI,EAAE,CAAC,SAAS,IAAI,EAAE,CAAC;QAC1C,CAAC;QAAC,MAAM,CAAC;YACP,MAAM,IAAI,KAAK,CACb,iFAAiF,CAClF,CAAC;QACJ,CAAC;IACH,CAAC;AACH,CAAC;AAED;;GAEG;AACH,SAAS,WAAW,CAAC,OAAmB,EAAE,OAAe;IACvD,OAAO,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;QACrC,MAAM,aAAa,GAChB,OAAe,CAAC,cAAc,CAAC;QAElC,MAAM,QAAQ,GAAG,aAAa,CAAC,GAAG,CAAC,qBAAqB,CAAC,IAAI,EAAE,CAAC;QAChE,QAAQ,CAAC,IAAI,CAAC,GAAG,EAAE;YACjB,YAAY,CAAC,KAAK,CAAC,CAAC;YACpB,OAAO,EAAE,CAAC;QACZ,CAAC,CAAC,CAAC;QACH,aAAa,CAAC,GAAG,CAAC,qBAAqB,EAAE,QAAQ,CAAC,CAAC;QAEnD,MAAM,KAAK,GAAG,UAAU,CAAC,GAAG,EAAE;YAC5B,gEAAgE;YAChE,OAAO,EAAE,CAAC;QACZ,CAAC,EAAE,OAAO,CAAC,CAAC;IACd,CAAC,CAAC,CAAC;AACL,CAAC;AAED;;;GAGG;AACH,KAAK,UAAU,YAAY,CACzB,GAAW,EACX,OAAe,EACf,SAAkB;IAElB,MAAM,EAAE,GACN,SAAS;QACT,uHAAuH,CAAC;IAE1H,yBAAyB;IACzB,IAAI,CAAC;QACH,OAAO,MAAM,kBAAkB,CAAC,GAAG,EAAE,OAAO,EAAE,EAAE,CAAC,CAAC;IACpD,CAAC;IAAC,OAAO,SAAc,EAAE,CAAC;QACxB,2DAA2D;QAC3D,MAAM,GAAG,GAAG,MAAM,CAAC,SAAS,EAAE,KAAK,IAAI,SAAS,EAAE,OAAO,IAAI,EAAE,CAAC,CAAC;QACjE,IACE,GAAG,CAAC,QAAQ,CAAC,2BAA2B,CAAC;YACzC,GAAG,CAAC,QAAQ,CAAC,kBAAkB,CAAC;YAChC,GAAG,CAAC,QAAQ,CAAC,wBAAwB,CAAC;YACtC,GAAG,CAAC,QAAQ,CAAC,aAAa,CAAC;YAC3B,GAAG,CAAC,QAAQ,CAAC,cAAc,CAAC,EAC5B,CAAC;YACD,OAAO,CAAC,MAAM,CAAC,KAAK,CAClB,oEAAoE,CACrE,CAAC;YACF,OAAO,iBAAiB,CAAC,GAAG,EAAE,OAAO,EAAE,EAAE,CAAC,CAAC;QAC7C,CAAC;QACD,MAAM,SAAS,CAAC;IAClB,CAAC;AACH,CAAC;AAED,KAAK,UAAU,kBAAkB,CAC/B,GAAW,EACX,OAAe,EACf,SAAiB;IAEjB,MAAM,OAAO,GAA2B;QACtC,MAAM,EACJ,iEAAiE;QACnE,iBAAiB,EAAE,gBAAgB;QACnC,YAAY,EAAE,SAAS;KACxB,CAAC;IAEF,MAAM,UAAU,GAAG,IAAI,eAAe,EAAE,CAAC;IACzC,MAAM,KAAK,GAAG,UAAU,CAAC,GAAG,EAAE,CAAC,UAAU,CAAC,KAAK,EAAE,EAAE,OAAO,CAAC,CAAC;IAE5D,IAAI,CAAC;QACH,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,GAAG,EAAE;YAChC,OAAO;YACP,MAAM,EAAE,UAAU,CAAC,MAAM;YACzB,QAAQ,EAAE,QAAQ;SACnB,CAAC,CAAC;QAEH,IAAI,CAAC,QAAQ,CAAC,EAAE,EAAE,CAAC;YACjB,MAAM,IAAI,KAAK,CAAC,QAAQ,QAAQ,CAAC,MAAM,IAAI,QAAQ,CAAC,UAAU,EAAE,CAAC,CAAC;QACpE,CAAC;QAED,MAAM,WAAW,GAAG,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,cAAc,CAAC,IAAI,EAAE,CAAC;QAC/D,IACE,CAAC,WAAW,CAAC,QAAQ,CAAC,WAAW,CAAC;YAClC,CAAC,WAAW,CAAC,QAAQ,CAAC,mBAAmB,CAAC,EAC1C,CAAC;YACD,OAAO,CAAC,MAAM,CAAC,KAAK,CAClB,4CAA4C,WAAW,eAAe,CACvE,CAAC;QACJ,CAAC;QAED,OAAO,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAC;IAC/B,CAAC;YAAS,CAAC;QACT,YAAY,CAAC,KAAK,CAAC,CAAC;IACtB,CAAC;AACH,CAAC;AAED;;;GAGG;AACH,KAAK,UAAU,iBAAiB,CAC9B,GAAW,EACX,OAAe,EACf,SAAiB;IAEjB,MAAM,KAAK,GAAG,wDAAa,YAAY,GAAC,CAAC;IACzC,MAAM,IAAI,GAAG,wDAAa,WAAW,GAAC,CAAC;IACvC,MAAM,EAAE,GAAG,EAAE,GAAG,wDAAa,UAAU,GAAC,CAAC;IAEzC,OAAO,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;QACrC,MAAM,SAAS,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC;QAC/B,MAAM,OAAO,GAAG,SAAS,CAAC,QAAQ,KAAK,QAAQ,CAAC;QAChD,MAAM,GAAG,GAAG,OAAO,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC;QAEnC,MAAM,OAAO,GAAG;YACd,QAAQ,EAAE,SAAS,CAAC,QAAQ;YAC5B,IAAI,EAAE,SAAS,CAAC,IAAI,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC;YAC5C,IAAI,EAAE,SAAS,CAAC,QAAQ,GAAG,SAAS,CAAC,MAAM;YAC3C,MAAM,EAAE,KAAK;YACb,OAAO,EAAE;gBACP,MAAM,EACJ,iEAAiE;gBACnE,iBAAiB,EAAE,gBAAgB;gBACnC,YAAY,EAAE,SAAS;aACxB;YACD,kBAAkB,EAAE,KAAK,EAAE,qCAAqC;YAChE,OAAO;SACR,CAAC;QAEF,MAAM,GAAG,GAAG,GAAG,CAAC,OAAO,CAAC,OAAO,EAAE,CAAC,GAAQ,EAAE,EAAE;YAC5C,6BAA6B;YAC7B,IACE,GAAG,CAAC,UAAU,IAAI,GAAG;gBACrB,GAAG,CAAC,UAAU,GAAG,GAAG;gBACpB,GAAG,CAAC,OAAO,CAAC,QAAQ,EACpB,CAAC;gBACD,MAAM,WAAW,GAAG,IAAI,GAAG,CACzB,GAAG,CAAC,OAAO,CAAC,QAAQ,EACpB,GAAG,CACJ,CAAC,QAAQ,EAAE,CAAC;gBACb,iBAAiB,CAAC,WAAW,EAAE,OAAO,EAAE,SAAS,CAAC;qBAC/C,IAAI,CAAC,OAAO,CAAC;qBACb,KAAK,CAAC,MAAM,CAAC,CAAC;gBACjB,OAAO;YACT,CAAC;YAED,IAAI,GAAG,CAAC,UAAU,GAAG,GAAG,IAAI,GAAG,CAAC,UAAU,IAAI,GAAG,EAAE,CAAC;gBAClD,MAAM,CAAC,IAAI,KAAK,CAAC,QAAQ,GAAG,CAAC,UAAU,EAAE,CAAC,CAAC,CAAC;gBAC5C,OAAO;YACT,CAAC;YAED,MAAM,MAAM,GAAa,EAAE,CAAC;YAC5B,GAAG,CAAC,EAAE,CAAC,MAAM,EAAE,CAAC,KAAa,EAAE,EAAE,CAAC,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC;YACtD,GAAG,CAAC,EAAE,CAAC,KAAK,EAAE,GAAG,EAAE;gBACjB,MAAM,IAAI,GAAG,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC;gBACrD,OAAO,CAAC,IAAI,CAAC,CAAC;YAChB,CAAC,CAAC,CAAC;YACH,GAAG,CAAC,EAAE,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC;QAC1B,CAAC,CAAC,CAAC;QAEH,GAAG,CAAC,EAAE,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC;QACxB,GAAG,CAAC,EAAE,CAAC,SAAS,EAAE,GAAG,EAAE;YACrB,GAAG,CAAC,OAAO,EAAE,CAAC;YACd,MAAM,CAAC,IAAI,KAAK,CAAC,mBAAmB,CAAC,CAAC,CAAC;QACzC,CAAC,CAAC,CAAC;QACH,GAAG,CAAC,GAAG,EAAE,CAAC;IACZ,CAAC,CAAC,CAAC;AACL,CAAC;AAED;;GAEG;AACH,KAAK,UAAU,gBAAgB,CAC7B,GAAW,EACX,OAAe;IAEf,MAAM,UAAU,GAAG,IAAI,eAAe,EAAE,CAAC;IACzC,MAAM,KAAK,GAAG,UAAU,CAAC,GAAG,EAAE,CAAC,UAAU,CAAC,KAAK,EAAE,EAAE,OAAO,CAAC,CAAC;IAC5D,IAAI,CAAC;QACH,OAAO,MAAM,KAAK,CAAC,GAAG,EAAE,EAAE,MAAM,EAAE,UAAU,CAAC,MAAM,EAAE,CAAC,CAAC;IACzD,CAAC;YAAS,CAAC;QACT,YAAY,CAAC,KAAK,CAAC,CAAC;IACtB,CAAC;AACH,CAAC"}

package/dist/formatter.d.ts

@@ -0,0 +1,17 @@
+/**
+ * formatter.ts - Convert parsed articles to markdown or JSON output
+ */
+import { ParsedArticle } from "./parser";
+/**
+ * Format article as clean markdown
+ */
+export declare function formatMarkdown(article: ParsedArticle, url: string): string;
+/**
+ * Format article as structured JSON
+ */
+export declare function formatJSON(article: ParsedArticle, url: string): string;
+/**
+ * Format a prompt for AI summarization
+ */
+export declare function formatSummaryPrompt(article: ParsedArticle): string;
+//# sourceMappingURL=formatter.d.ts.map

package/dist/formatter.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"formatter.d.ts","sourceRoot":"","sources":["../src/formatter.ts"],"names":[],"mappings":"AAAA;;GAEG;AAGH,OAAO,EAAE,aAAa,EAAE,MAAM,UAAU,CAAC;AAiBzC;;GAEG;AACH,wBAAgB,cAAc,CAAC,OAAO,EAAE,aAAa,EAAE,GAAG,EAAE,MAAM,GAAG,MAAM,CAqC1E;AAED;;GAEG;AACH,wBAAgB,UAAU,CAAC,OAAO,EAAE,aAAa,EAAE,GAAG,EAAE,MAAM,GAAG,MAAM,CAiBtE;AAED;;GAEG;AACH,wBAAgB,mBAAmB,CAAC,OAAO,EAAE,aAAa,GAAG,MAAM,CAqBlE"}