webpeel 0.13.4 → 0.14.0

package/dist/core/watch-manager.js

@@ -0,0 +1,348 @@
+/**
+ * WebPeel WatchManager — Database-backed persistent URL monitoring
+ *
+ * Stores watch entries in PostgreSQL, periodically fetches watched URLs,
+ * compares content fingerprints to detect changes, and fires webhook
+ * notifications when a page is updated.
+ *
+ * This module is complementary to the in-process `watch.ts` poller:
+ *  - `watch.ts`        → ephemeral, CLI/in-process, no DB
+ *  - `watch-manager.ts` → persistent, server-side, PostgreSQL-backed
+ */
+import { createHash } from 'crypto';
+import { fetch as undiciFetch } from 'undici';
+// ─── Internal helpers ──────────────────────────────────────────────────────────
+/**
+ * Compute a stable SHA-256 fingerprint of page content.
+ * Normalises whitespace so cosmetic-only reformatting doesn't trigger alerts.
+ */
+export function computeFingerprint(content) {
+    const normalized = content.trim().replace(/\s+/g, ' ');
+    return createHash('sha256').update(normalized).digest('hex');
+}
+/**
+ * Paragraph-level diff — splits both versions of a page into paragraph blocks
+ * (separated by blank lines), then finds paragraphs that appear exclusively in
+ * each version.  Only blocks longer than 10 characters are considered to avoid
+ * noise from short punctuation-only lines.
+ */
+export function computeParagraphDiff(oldContent, newContent) {
+    const toSet = (text) => new Set(text
+        .split(/\n{2,}/)
+        .map(p => p.trim())
+        .filter(p => p.length > 10));
+    const oldSet = toSet(oldContent);
+    const newSet = toSet(newContent);
+    const addedText = [];
+    const removedText = [];
+    for (const p of newSet) {
+        if (!oldSet.has(p))
+            addedText.push(p.slice(0, 500));
+    }
+    for (const p of oldSet) {
+        if (!newSet.has(p))
+            removedText.push(p.slice(0, 500));
+    }
+    return { addedText, removedText };
+}
+/** Post a JSON payload to a webhook URL, silently swallowing delivery errors. */
+async function sendWatchWebhook(webhookUrl, payload) {
+    try {
+        await undiciFetch(webhookUrl, {
+            method: 'POST',
+            headers: {
+                'Content-Type': 'application/json',
+                'User-Agent': 'WebPeel-Watch/1.0 (+https://webpeel.dev)',
+            },
+            body: JSON.stringify(payload),
+            signal: AbortSignal.timeout(10_000),
+        });
+    }
+    catch (err) {
+        process.stderr.write(`[watch-manager] Webhook delivery failed to ${webhookUrl}: ${err instanceof Error ? err.message : String(err)}\n`);
+    }
+}
+/** Map a raw database row to a typed {@link WatchEntry}. */
+function rowToEntry(row) {
+    return {
+        id: row.id,
+        accountId: row.account_id,
+        url: row.url,
+        webhookUrl: row.webhook_url ?? undefined,
+        checkIntervalMinutes: row.check_interval_minutes ?? 60,
+        selector: row.selector ?? undefined,
+        lastFingerprint: row.last_fingerprint ?? undefined,
+        lastCheckedAt: row.last_checked_at ? new Date(row.last_checked_at) : undefined,
+        lastChangedAt: row.last_changed_at ? new Date(row.last_changed_at) : undefined,
+        changeCount: row.change_count ?? 0,
+        status: row.status ?? 'active',
+        errorMessage: row.error_message ?? undefined,
+        createdAt: new Date(row.created_at),
+        updatedAt: new Date(row.updated_at),
+    };
+}
+// ─── WatchManager ──────────────────────────────────────────────────────────────
+/**
+ * Database-backed URL watch manager.
+ *
+ * Stores watch entries in a PostgreSQL `watches` table (see
+ * `migrations/007_watch.sql`) and handles periodic checks, change detection,
+ * and webhook delivery.
+ *
+ * @example
+ * ```typescript
+ * const manager = new WatchManager(pool);
+ * const watch = await manager.create('acct-uuid', 'https://example.com/pricing', {
+ *   webhookUrl: 'https://hooks.example.com/alert',
+ *   checkIntervalMinutes: 30,
+ * });
+ * const diff = await manager.check(watch.id);
+ * console.log(diff.summary);
+ * ```
+ */
+export class WatchManager {
+    db;
+    constructor(db) {
+        this.db = db;
+    }
+    // ─── CRUD ────────────────────────────────────────────────────────────────────
+    /**
+     * Create a new watch entry for the given URL.
+     * The watch is immediately active; the first check will establish the baseline.
+     */
+    async create(accountId, url, options = {}) {
+        const { webhookUrl, checkIntervalMinutes = 60, selector } = options;
+        const result = await this.db.query(`INSERT INTO watches (account_id, url, webhook_url, check_interval_minutes, selector)
+       VALUES ($1, $2, $3, $4, $5)
+       RETURNING *`, [accountId, url, webhookUrl ?? null, checkIntervalMinutes, selector ?? null]);
+        return rowToEntry(result.rows[0]);
+    }
+    /** List all watches owned by the given account, most recent first. */
+    async list(accountId) {
+        const result = await this.db.query(`SELECT * FROM watches WHERE account_id = $1 ORDER BY created_at DESC`, [accountId]);
+        return result.rows.map(rowToEntry);
+    }
+    /** Get a single watch by ID, or null if not found. */
+    async get(watchId) {
+        const result = await this.db.query(`SELECT * FROM watches WHERE id = $1`, [watchId]);
+        if (result.rows.length === 0)
+            return null;
+        return rowToEntry(result.rows[0]);
+    }
+    /** Pause a watch — it will not be included in {@link checkDue} runs. */
+    async pause(watchId) {
+        await this.db.query(`UPDATE watches SET status = 'paused', updated_at = NOW() WHERE id = $1`, [watchId]);
+    }
+    /** Resume a previously paused (or errored) watch. */
+    async resume(watchId) {
+        await this.db.query(`UPDATE watches
+       SET status = 'active', error_message = NULL, updated_at = NOW()
+       WHERE id = $1`, [watchId]);
+    }
+    /** Permanently delete a watch. */
+    async delete(watchId) {
+        await this.db.query(`DELETE FROM watches WHERE id = $1`, [watchId]);
+    }
+    /**
+     * Update mutable properties of a watch.
+     * Only the fields present in `updates` are changed.
+     */
+    async update(watchId, updates) {
+        const setClauses = ['updated_at = NOW()'];
+        const values = [];
+        let idx = 1;
+        if ('webhookUrl' in updates) {
+            setClauses.push(`webhook_url = $${idx++}`);
+            values.push(updates.webhookUrl ?? null);
+        }
+        if ('checkIntervalMinutes' in updates) {
+            setClauses.push(`check_interval_minutes = $${idx++}`);
+            values.push(updates.checkIntervalMinutes);
+        }
+        if ('selector' in updates) {
+            setClauses.push(`selector = $${idx++}`);
+            values.push(updates.selector ?? null);
+        }
+        if ('status' in updates) {
+            setClauses.push(`status = $${idx++}`);
+            values.push(updates.status);
+        }
+        values.push(watchId);
+        const result = await this.db.query(`UPDATE watches SET ${setClauses.join(', ')} WHERE id = $${idx} RETURNING *`, values);
+        if (result.rows.length === 0)
+            return null;
+        return rowToEntry(result.rows[0]);
+    }
+    // ─── Checking ────────────────────────────────────────────────────────────────
+    /**
+     * Perform an immediate content check for the given watch ID.
+     *
+     * Steps:
+     * 1. Load watch entry and previous content snapshot from disk (if any).
+     * 2. Fetch the current page via `peel()`.
+     * 3. Compute a SHA-256 fingerprint of the normalised content.
+     * 4. If the fingerprint changed, compute a paragraph-level diff and fire the webhook.
+     * 5. Persist `last_fingerprint`, `last_checked_at`, and `change_count` to the DB.
+     * 6. Return a {@link WatchDiff} describing what changed.
+     */
+    async check(watchId) {
+        const watch = await this.get(watchId);
+        if (!watch)
+            throw new Error(`Watch not found: ${watchId}`);
+        const now = new Date();
+        try {
+            // Load previous content snapshot for text-diff computation.
+            const { getSnapshot } = await import('./change-tracking.js');
+            const prevSnapshot = await getSnapshot(watch.url);
+            // Fetch current content.
+            const { peel } = await import('../index.js');
+            const peelResult = await peel(watch.url, {
+                format: 'markdown',
+                selector: watch.selector,
+                timeout: 30_000,
+                // Enable change tracking so the snapshot is persisted for future diffs.
+                changeTracking: true,
+            });
+            const currentContent = peelResult.content;
+            const currentFingerprint = computeFingerprint(currentContent);
+            const previousFingerprint = watch.lastFingerprint ?? '';
+            // Determine whether content actually changed relative to our DB record.
+            const isFirstCheck = !previousFingerprint;
+            const changed = !isFirstCheck && currentFingerprint !== previousFingerprint;
+            // Compute text diff when changed and we have old content to compare against.
+            let addedText = [];
+            let removedText = [];
+            let summary;
+            if (isFirstCheck) {
+                summary = 'Baseline fingerprint established — monitoring active.';
+            }
+            else if (changed) {
+                const oldContent = prevSnapshot?.content ?? '';
+                if (oldContent) {
+                    const diff = computeParagraphDiff(oldContent, currentContent);
+                    addedText = diff.addedText;
+                    removedText = diff.removedText;
+                    summary =
+                        addedText.length > 0 || removedText.length > 0
+                            ? `Page updated: ${addedText.length} block${addedText.length !== 1 ? 's' : ''} added, ` +
+                                `${removedText.length} block${removedText.length !== 1 ? 's' : ''} removed.`
+                            : 'Page content changed (fingerprint mismatch — no paragraph-level diff available).';
+                }
+                else {
+                    summary = 'Page content changed (no previous snapshot available for text diff).';
+                }
+            }
+            else {
+                summary = 'No changes detected.';
+            }
+            // Update DB.
+            if (changed) {
+                await this.db.query(`UPDATE watches
+           SET last_fingerprint    = $1,
+               last_checked_at     = $2,
+               last_changed_at     = $2,
+               change_count        = change_count + 1,
+               status              = 'active',
+               error_message       = NULL,
+               updated_at          = $2
+           WHERE id = $3`, [currentFingerprint, now, watchId]);
+                // Reload to get the latest change_count for the webhook payload.
+                const updated = await this.get(watchId);
+                // Fire webhook.
+                if (watch.webhookUrl && updated) {
+                    const payload = {
+                        event: 'watch.changed',
+                        watchId: watch.id,
+                        url: watch.url,
+                        changedAt: now.toISOString(),
+                        changeCount: updated.changeCount,
+                        diff: { addedText, removedText, summary },
+                    };
+                    await sendWatchWebhook(watch.webhookUrl, payload);
+                }
+            }
+            else {
+                await this.db.query(`UPDATE watches
+           SET last_fingerprint = COALESCE($1, last_fingerprint),
+               last_checked_at  = $2,
+               status           = 'active',
+               error_message    = NULL,
+               updated_at       = $2
+           WHERE id = $3`, [currentFingerprint || null, now, watchId]);
+            }
+            return {
+                changed,
+                previousFingerprint,
+                currentFingerprint,
+                summary,
+                addedText,
+                removedText,
+            };
+        }
+        catch (error) {
+            const errMsg = (error instanceof Error ? error.message : String(error)).slice(0, 500);
+            // Mark the watch as errored so operators can investigate.
+            await this.db.query(`UPDATE watches
+         SET status        = 'error',
+             error_message = $1,
+             last_checked_at = $2,
+             updated_at    = $2
+         WHERE id = $3`, [errMsg, now, watchId]);
+            throw error;
+        }
+    }
+    /**
+     * Scan the database for watches that are due for a check and run them.
+     *
+     * A watch is "due" when:
+     *  - `status = 'active'`
+     *  - `last_checked_at` is NULL (never checked) OR older than `check_interval_minutes`
+     *
+     * Processes up to 50 watches per invocation to avoid long-running cycles.
+     */
+    async checkDue() {
+        const result = await this.db.query(`SELECT * FROM watches
+       WHERE status = 'active'
+         AND (
+           last_checked_at IS NULL
+           OR last_checked_at < NOW() - (check_interval_minutes * INTERVAL '1 minute')
+         )
+       ORDER BY last_checked_at ASC NULLS FIRST
+       LIMIT 50`);
+        for (const row of result.rows) {
+            const watch = rowToEntry(row);
+            try {
+                await this.check(watch.id);
+            }
+            catch (err) {
+                process.stderr.write(`[watch-manager] Error checking watch ${watch.id} (${watch.url}): ` +
+                    `${err instanceof Error ? err.message : String(err)}\n`);
+            }
+        }
+    }
+}
+// ─── Background checker ────────────────────────────────────────────────────────
+/**
+ * Start a background interval that calls {@link WatchManager.checkDue} every
+ * minute.  Wire this up in `app.ts` after the server starts.
+ *
+ * @returns The interval handle (pass to `clearInterval` for clean shutdown).
+ *
+ * @example
+ * ```typescript
+ * const handle = startWatchChecker(pool);
+ * process.on('SIGTERM', () => clearInterval(handle));
+ * ```
+ */
+export function startWatchChecker(db) {
+    const manager = new WatchManager(db);
+    return setInterval(async () => {
+        try {
+            await manager.checkDue();
+        }
+        catch (err) {
+            process.stderr.write(`[watch-manager] Background checker error: ${err instanceof Error ? err.message : String(err)}\n`);
+        }
+    }, 60_000); // Every 1 minute
+}
+//# sourceMappingURL=watch-manager.js.map

package/dist/core/watch-manager.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"watch-manager.js","sourceRoot":"","sources":["../../src/core/watch-manager.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;GAUG;AAEH,OAAO,EAAE,UAAU,EAAE,MAAM,QAAQ,CAAC;AACpC,OAAO,EAAE,KAAK,IAAI,WAAW,EAAE,MAAM,QAAQ,CAAC;AA2C9C,kFAAkF;AAElF;;;GAGG;AACH,MAAM,UAAU,kBAAkB,CAAC,OAAe;IAChD,MAAM,UAAU,GAAG,OAAO,CAAC,IAAI,EAAE,CAAC,OAAO,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;IACvD,OAAO,UAAU,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;AAC/D,CAAC;AAED;;;;;GAKG;AACH,MAAM,UAAU,oBAAoB,CAClC,UAAkB,EAClB,UAAkB;IAElB,MAAM,KAAK,GAAG,CAAC,IAAY,EAAe,EAAE,CAC1C,IAAI,GAAG,CACL,IAAI;SACD,KAAK,CAAC,QAAQ,CAAC;SACf,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;SAClB,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,MAAM,GAAG,EAAE,CAAC,CAC9B,CAAC;IAEJ,MAAM,MAAM,GAAG,KAAK,CAAC,UAAU,CAAC,CAAC;IACjC,MAAM,MAAM,GAAG,KAAK,CAAC,UAAU,CAAC,CAAC;IAEjC,MAAM,SAAS,GAAa,EAAE,CAAC;IAC/B,MAAM,WAAW,GAAa,EAAE,CAAC;IAEjC,KAAK,MAAM,CAAC,IAAI,MAAM,EAAE,CAAC;QACvB,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC;YAAE,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC,CAAC;IACtD,CAAC;IACD,KAAK,MAAM,CAAC,IAAI,MAAM,EAAE,CAAC;QACvB,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC;YAAE,WAAW,CAAC,IAAI,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC,CAAC;IACxD,CAAC;IAED,OAAO,EAAE,SAAS,EAAE,WAAW,EAAE,CAAC;AACpC,CAAC;AAED,iFAAiF;AACjF,KAAK,UAAU,gBAAgB,CAAC,UAAkB,EAAE,OAAgB;IAClE,IAAI,CAAC;QACH,MAAM,WAAW,CAAC,UAAU,EAAE;YAC5B,MAAM,EAAE,MAAM;YACd,OAAO,EAAE;gBACP,cAAc,EAAE,kBAAkB;gBAClC,YAAY,EAAE,0CAA0C;aACzD;YACD,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC;YAC7B,MAAM,EAAE,WAAW,CAAC,OAAO,CAAC,MAAM,CAAC;SACpC,CAAC,CAAC;IACL,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,OAAO,CAAC,MAAM,CAAC,KAAK,CAClB,8CAA8C,UAAU,KAAK,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,IAAI,CAClH,CAAC;IACJ,CAAC;AACH,CAAC;AAED,4DAA4D;AAC5D,SAAS,UAAU,CAAC,GAA4B;IAC9C,OAAO;QACL,EAAE,EAAE,GAAG,CAAC,EAAY;QACpB,SAAS,EAAE,GAAG,CAAC,UAAoB;QACnC,GAAG,EAAE,GAAG,CAAC,GAAa;QACtB,UAAU,EAAG,GAAG,CAAC,WAA6B,IAAI,SAAS;QAC3D,oBAAoB,EAAG,GAAG,CAAC,sBAAiC,IAAI,EAAE;QAClE,QAAQ,EAAG,GAAG,CAAC,QAA0B,IAAI,SAAS;QACtD,eAAe,EAAG,GAAG,CAAC,gBAAkC,IAAI,SAAS;QACrE,aAAa,EAAE,GAAG,CAAC,eAAe,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC,GAAG,CAAC,eAAyB,CAAC,CAAC,CAAC,CAAC,SAAS;QACxF,aAAa,EAAE,GAAG,CAAC,eAAe,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC,GAAG,CAAC,eAAyB,CAAC,CAAC,CAAC,CAAC,SAAS;QACxF,WAAW,EAAG,GAAG,CAAC,YAAuB,IAAI,CAAC;QAC9C,MAAM,EAAG,GAAG,CAAC,MAAwC,IAAI,QAAQ;QACjE,YAAY,EAAG,GAAG,CAAC,aAA+B,IAAI,SAAS;QAC/D,SAAS,EAAE,IAAI,IAAI,CAAC,GAAG,CAAC,UAAoB,CAAC;QAC7C,SAAS,EAAE,IAAI,IAAI,CAAC,GAAG,CAAC,UAAoB,CAAC;KAC9C,CAAC;AACJ,CAAC;AAED,kFAAkF;AAElF;;;;;;;;;;;;;;;;;GAiBG;AACH,MAAM,OAAO,YAAY;IACM;IAA7B,YAA6B,EAAW;QAAX,OAAE,GAAF,EAAE,CAAS;IAAG,CAAC;IAE5C,gFAAgF;IAEhF;;;OAGG;IACH,KAAK,CAAC,MAAM,CACV,SAAiB,EACjB,GAAW,EACX,UAA8B,EAAE;QAEhC,MAAM,EAAE,UAAU,EAAE,oBAAoB,GAAG,EAAE,EAAE,QAAQ,EAAE,GAAG,OAAO,CAAC;QAEpE,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,EAAE,CAAC,KAAK,CAChC;;mBAEa,EACb,CAAC,SAAS,EAAE,GAAG,EAAE,UAAU,IAAI,IAAI,EAAE,oBAAoB,EAAE,QAAQ,IAAI,IAAI,CAAC,CAC7E,CAAC;QAEF,OAAO,UAAU,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC;IACpC,CAAC;IAED,sEAAsE;IACtE,KAAK,CAAC,IAAI,CAAC,SAAiB;QAC1B,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,EAAE,CAAC,KAAK,CAChC,sEAAsE,EACtE,CAAC,SAAS,CAAC,CACZ,CAAC;QACF,OAAO,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,UAAU,CAAC,CAAC;IACrC,CAAC;IAED,sDAAsD;IACtD,KAAK,CAAC,GAAG,CAAC,OAAe;QACvB,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,EAAE,CAAC,KAAK,CAChC,qCAAqC,EACrC,CAAC,OAAO,CAAC,CACV,CAAC;QACF,IAAI,MAAM,CAAC,IAAI,CAAC,MAAM,KAAK,CAAC;YAAE,OAAO,IAAI,CAAC;QAC1C,OAAO,UAAU,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC;IACpC,CAAC;IAED,wEAAwE;IACxE,KAAK,CAAC,KAAK,CAAC,OAAe;QACzB,MAAM,IAAI,CAAC,EAAE,CAAC,KAAK,CACjB,wEAAwE,EACxE,CAAC,OAAO,CAAC,CACV,CAAC;IACJ,CAAC;IAED,qDAAqD;IACrD,KAAK,CAAC,MAAM,CAAC,OAAe;QAC1B,MAAM,IAAI,CAAC,EAAE,CAAC,KAAK,CACjB;;qBAEe,EACf,CAAC,OAAO,CAAC,CACV,CAAC;IACJ,CAAC;IAED,kCAAkC;IAClC,KAAK,CAAC,MAAM,CAAC,OAAe;QAC1B,MAAM,IAAI,CAAC,EAAE,CAAC,KAAK,CAAC,mCAAmC,EAAE,CAAC,OAAO,CAAC,CAAC,CAAC;IACtE,CAAC;IAED;;;OAGG;IACH,KAAK,CAAC,MAAM,CACV,OAAe,EACf,OAAiG;QAEjG,MAAM,UAAU,GAAa,CAAC,oBAAoB,CAAC,CAAC;QACpD,MAAM,MAAM,GAAc,EAAE,CAAC;QAC7B,IAAI,GAAG,GAAG,CAAC,CAAC;QAEZ,IAAI,YAAY,IAAI,OAAO,EAAE,CAAC;YAC5B,UAAU,CAAC,IAAI,CAAC,kBAAkB,GAAG,EAAE,EAAE,CAAC,CAAC;YAC3C,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,UAAU,IAAI,IAAI,CAAC,CAAC;QAC1C,CAAC;QACD,IAAI,sBAAsB,IAAI,OAAO,EAAE,CAAC;YACtC,UAAU,CAAC,IAAI,CAAC,6BAA6B,GAAG,EAAE,EAAE,CAAC,CAAC;YACtD,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,oBAAoB,CAAC,CAAC;QAC5C,CAAC;QACD,IAAI,UAAU,IAAI,OAAO,EAAE,CAAC;YAC1B,UAAU,CAAC,IAAI,CAAC,eAAe,GAAG,EAAE,EAAE,CAAC,CAAC;YACxC,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,QAAQ,IAAI,IAAI,CAAC,CAAC;QACxC,CAAC;QACD,IAAI,QAAQ,IAAI,OAAO,EAAE,CAAC;YACxB,UAAU,CAAC,IAAI,CAAC,aAAa,GAAG,EAAE,EAAE,CAAC,CAAC;YACtC,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC;QAC9B,CAAC;QAED,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;QAErB,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,EAAE,CAAC,KAAK,CAChC,sBAAsB,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,gBAAgB,GAAG,cAAc,EAC5E,MAAM,CACP,CAAC;QAEF,IAAI,MAAM,CAAC,IAAI,CAAC,MAAM,KAAK,CAAC;YAAE,OAAO,IAAI,CAAC;QAC1C,OAAO,UAAU,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC;IACpC,CAAC;IAED,gFAAgF;IAEhF;;;;;;;;;;OAUG;IACH,KAAK,CAAC,KAAK,CAAC,OAAe;QACzB,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;QACtC,IAAI,CAAC,KAAK;YAAE,MAAM,IAAI,KAAK,CAAC,oBAAoB,OAAO,EAAE,CAAC,CAAC;QAE3D,MAAM,GAAG,GAAG,IAAI,IAAI,EAAE,CAAC;QAEvB,IAAI,CAAC;YACH,4DAA4D;YAC5D,MAAM,EAAE,WAAW,EAAE,GAAG,MAAM,MAAM,CAAC,sBAAsB,CAAC,CAAC;YAC7D,MAAM,YAAY,GAAG,MAAM,WAAW,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;YAElD,yBAAyB;YACzB,MAAM,EAAE,IAAI,EAAE,GAAG,MAAM,MAAM,CAAC,aAAa,CAAC,CAAC;YAC7C,MAAM,UAAU,GAAG,MAAM,IAAI,CAAC,KAAK,CAAC,GAAG,EAAE;gBACvC,MAAM,EAAE,UAAU;gBAClB,QAAQ,EAAE,KAAK,CAAC,QAAQ;gBACxB,OAAO,EAAE,MAAM;gBACf,wEAAwE;gBACxE,cAAc,EAAE,IAAI;aACrB,CAAC,CAAC;YAEH,MAAM,cAAc,GAAG,UAAU,CAAC,OAAO,CAAC;YAC1C,MAAM,kBAAkB,GAAG,kBAAkB,CAAC,cAAc,CAAC,CAAC;YAC9D,MAAM,mBAAmB,GAAG,KAAK,CAAC,eAAe,IAAI,EAAE,CAAC;YAExD,wEAAwE;YACxE,MAAM,YAAY,GAAG,CAAC,mBAAmB,CAAC;YAC1C,MAAM,OAAO,GAAG,CAAC,YAAY,IAAI,kBAAkB,KAAK,mBAAmB,CAAC;YAE5E,6EAA6E;YAC7E,IAAI,SAAS,GAAa,EAAE,CAAC;YAC7B,IAAI,WAAW,GAAa,EAAE,CAAC;YAC/B,IAAI,OAAe,CAAC;YAEpB,IAAI,YAAY,EAAE,CAAC;gBACjB,OAAO,GAAG,uDAAuD,CAAC;YACpE,CAAC;iBAAM,IAAI,OAAO,EAAE,CAAC;gBACnB,MAAM,UAAU,GAAG,YAAY,EAAE,OAAO,IAAI,EAAE,CAAC;gBAC/C,IAAI,UAAU,EAAE,CAAC;oBACf,MAAM,IAAI,GAAG,oBAAoB,CAAC,UAAU,EAAE,cAAc,CAAC,CAAC;oBAC9D,SAAS,GAAG,IAAI,CAAC,SAAS,CAAC;oBAC3B,WAAW,GAAG,IAAI,CAAC,WAAW,CAAC;oBAC/B,OAAO;wBACL,SAAS,CAAC,MAAM,GAAG,CAAC,IAAI,WAAW,CAAC,MAAM,GAAG,CAAC;4BAC5C,CAAC,CAAC,iBAAiB,SAAS,CAAC,MAAM,SAAS,SAAS,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,UAAU;gCACrF,GAAG,WAAW,CAAC,MAAM,SAAS,WAAW,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,WAAW;4BAC9E,CAAC,CAAC,kFAAkF,CAAC;gBAC3F,CAAC;qBAAM,CAAC;oBACN,OAAO,GAAG,sEAAsE,CAAC;gBACnF,CAAC;YACH,CAAC;iBAAM,CAAC;gBACN,OAAO,GAAG,sBAAsB,CAAC;YACnC,CAAC;YAED,aAAa;YACb,IAAI,OAAO,EAAE,CAAC;gBACZ,MAAM,IAAI,CAAC,EAAE,CAAC,KAAK,CACjB;;;;;;;;yBAQe,EACf,CAAC,kBAAkB,EAAE,GAAG,EAAE,OAAO,CAAC,CACnC,CAAC;gBAEF,iEAAiE;gBACjE,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;gBAExC,gBAAgB;gBAChB,IAAI,KAAK,CAAC,UAAU,IAAI,OAAO,EAAE,CAAC;oBAChC,MAAM,OAAO,GAAG;wBACd,KAAK,EAAE,eAAe;wBACtB,OAAO,EAAE,KAAK,CAAC,EAAE;wBACjB,GAAG,EAAE,KAAK,CAAC,GAAG;wBACd,SAAS,EAAE,GAAG,CAAC,WAAW,EAAE;wBAC5B,WAAW,EAAE,OAAO,CAAC,WAAW;wBAChC,IAAI,EAAE,EAAE,SAAS,EAAE,WAAW,EAAE,OAAO,EAAE;qBAC1C,CAAC;oBACF,MAAM,gBAAgB,CAAC,KAAK,CAAC,UAAU,EAAE,OAAO,CAAC,CAAC;gBACpD,CAAC;YACH,CAAC;iBAAM,CAAC;gBACN,MAAM,IAAI,CAAC,EAAE,CAAC,KAAK,CACjB;;;;;;yBAMe,EACf,CAAC,kBAAkB,IAAI,IAAI,EAAE,GAAG,EAAE,OAAO,CAAC,CAC3C,CAAC;YACJ,CAAC;YAED,OAAO;gBACL,OAAO;gBACP,mBAAmB;gBACnB,kBAAkB;gBAClB,OAAO;gBACP,SAAS;gBACT,WAAW;aACZ,CAAC;QACJ,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,MAAM,MAAM,GAAG,CAAC,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC;YAEtF,0DAA0D;YAC1D,MAAM,IAAI,CAAC,EAAE,CAAC,KAAK,CACjB;;;;;uBAKe,EACf,CAAC,MAAM,EAAE,GAAG,EAAE,OAAO,CAAC,CACvB,CAAC;YAEF,MAAM,KAAK,CAAC;QACd,CAAC;IACH,CAAC;IAED;;;;;;;;OAQG;IACH,KAAK,CAAC,QAAQ;QACZ,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,EAAE,CAAC,KAAK,CAChC;;;;;;;gBAOU,CACX,CAAC;QAEF,KAAK,MAAM,GAAG,IAAI,MAAM,CAAC,IAAI,EAAE,CAAC;YAC9B,MAAM,KAAK,GAAG,UAAU,CAAC,GAAG,CAAC,CAAC;YAC9B,IAAI,CAAC;gBACH,MAAM,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;YAC7B,CAAC;YAAC,OAAO,GAAG,EAAE,CAAC;gBACb,OAAO,CAAC,MAAM,CAAC,KAAK,CAClB,wCAAwC,KAAK,CAAC,EAAE,KAAK,KAAK,CAAC,GAAG,KAAK;oBACjE,GAAG,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,IAAI,CAC1D,CAAC;YACJ,CAAC;QACH,CAAC;IACH,CAAC;CACF;AAED,kFAAkF;AAElF;;;;;;;;;;;GAWG;AACH,MAAM,UAAU,iBAAiB,CAAC,EAAW;IAC3C,MAAM,OAAO,GAAG,IAAI,YAAY,CAAC,EAAE,CAAC,CAAC;IAErC,OAAO,WAAW,CAAC,KAAK,IAAI,EAAE;QAC5B,IAAI,CAAC;YACH,MAAM,OAAO,CAAC,QAAQ,EAAE,CAAC;QAC3B,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,OAAO,CAAC,MAAM,CAAC,KAAK,CAClB,6CAA6C,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,IAAI,CAClG,CAAC;QACJ,CAAC;IACH,CAAC,EAAE,MAAM,CAAC,CAAC,CAAC,iBAAiB;AAC/B,CAAC"}

package/dist/core/youtube.d.ts

@@ -0,0 +1,91 @@
+/**
+ * YouTube transcript extraction — no API key required.
+ *
+ * YouTube embeds caption/transcript data directly in the page HTML as JSON
+ * (inside ytInitialPlayerResponse). We parse that JSON, extract caption
+ * track URLs, fetch the timedtext XML, and return structured transcript data.
+ */
+export interface TranscriptSegment {
+    /** Caption text (HTML entities decoded) */
+    text: string;
+    /** Start time in seconds */
+    start: number;
+    /** Duration in seconds */
+    duration: number;
+}
+export interface YouTubeTranscript {
+    videoId: string;
+    title: string;
+    channel: string;
+    /** Duration formatted as "MM:SS" or "HH:MM:SS" */
+    duration: string;
+    /** BCP-47 language code, e.g. "en" */
+    language: string;
+    /** Timestamped caption segments */
+    segments: TranscriptSegment[];
+    /** All segments joined as plain text */
+    fullText: string;
+    /** Language codes available for this video */
+    availableLanguages: string[];
+}
+export interface YouTubeVideoInfo {
+    videoId: string;
+    title: string;
+    channel: string;
+    description: string;
+    /** Duration formatted as "MM:SS" or "HH:MM:SS" */
+    duration: string;
+    publishDate: string;
+    viewCount: string;
+    likeCount: string;
+    thumbnail: string;
+}
+/**
+ * Extract the video ID from any common YouTube URL format.
+ * Returns null if the URL is not a recognisable YouTube URL.
+ *
+ * Supported formats:
+ *   https://www.youtube.com/watch?v=VIDEO_ID
+ *   https://youtu.be/VIDEO_ID
+ *   https://www.youtube.com/embed/VIDEO_ID
+ *   https://m.youtube.com/watch?v=VIDEO_ID
+ *   URLs with extra params (&t=120, &list=PLxxx, etc.)
+ */
+export declare function parseYouTubeUrl(url: string): string | null;
+/**
+ * Extract video metadata from YouTube page HTML.
+ * Parses ytInitialPlayerResponse JSON embedded in the page.
+ */
+export declare function extractVideoInfo(html: string): YouTubeVideoInfo;
+/**
+ * Fetch and return the transcript for a YouTube video.
+ *
+ * @param url - Any YouTube URL format
+ * @param options.language - Preferred language code (default: "en")
+ */
+export declare function getYouTubeTranscript(url: string, options?: {
+    language?: string;
+}): Promise<YouTubeTranscript>;
+/**
+ * Extract the ytInitialPlayerResponse JSON object from page HTML.
+ */
+export declare function extractPlayerResponse(html: string): Record<string, any> | null;
+/**
+ * Parse YouTube caption XML into transcript segments.
+ *
+ * Format: <transcript><text start="0.5" dur="2.1">Hello &amp; world</text>...</transcript>
+ */
+export declare function parseCaptionXml(xml: string): TranscriptSegment[];
+/**
+ * Decode common HTML entities found in YouTube caption XML.
+ *
+ * Order of operations:
+ * 1. Strip real HTML tags (e.g. <font color="...">) — these appear literally in the XML
+ * 2. Decode all HTML entities (including &lt; → < which represents literal angle brackets)
+ */
+export declare function decodeHtmlEntities(text: string): string;
+/**
+ * Format seconds into MM:SS or HH:MM:SS.
+ */
+export declare function formatDuration(seconds: number): string;
+//# sourceMappingURL=youtube.d.ts.map

package/dist/core/youtube.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"youtube.d.ts","sourceRoot":"","sources":["../../src/core/youtube.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AAQH,MAAM,WAAW,iBAAiB;IAChC,2CAA2C;IAC3C,IAAI,EAAE,MAAM,CAAC;IACb,4BAA4B;IAC5B,KAAK,EAAE,MAAM,CAAC;IACd,0BAA0B;IAC1B,QAAQ,EAAE,MAAM,CAAC;CAClB;AAED,MAAM,WAAW,iBAAiB;IAChC,OAAO,EAAE,MAAM,CAAC;IAChB,KAAK,EAAE,MAAM,CAAC;IACd,OAAO,EAAE,MAAM,CAAC;IAChB,kDAAkD;IAClD,QAAQ,EAAE,MAAM,CAAC;IACjB,sCAAsC;IACtC,QAAQ,EAAE,MAAM,CAAC;IACjB,mCAAmC;IACnC,QAAQ,EAAE,iBAAiB,EAAE,CAAC;IAC9B,wCAAwC;IACxC,QAAQ,EAAE,MAAM,CAAC;IACjB,8CAA8C;IAC9C,kBAAkB,EAAE,MAAM,EAAE,CAAC;CAC9B;AAED,MAAM,WAAW,gBAAgB;IAC/B,OAAO,EAAE,MAAM,CAAC;IAChB,KAAK,EAAE,MAAM,CAAC;IACd,OAAO,EAAE,MAAM,CAAC;IAChB,WAAW,EAAE,MAAM,CAAC;IACpB,kDAAkD;IAClD,QAAQ,EAAE,MAAM,CAAC;IACjB,WAAW,EAAE,MAAM,CAAC;IACpB,SAAS,EAAE,MAAM,CAAC;IAClB,SAAS,EAAE,MAAM,CAAC;IAClB,SAAS,EAAE,MAAM,CAAC;CACnB;AAMD;;;;;;;;;;GAUG;AACH,wBAAgB,eAAe,CAAC,GAAG,EAAE,MAAM,GAAG,MAAM,GAAG,IAAI,CA6C1D;AAUD;;;GAGG;AACH,wBAAgB,gBAAgB,CAAC,IAAI,EAAE,MAAM,GAAG,gBAAgB,CAwC/D;AAMD;;;;;GAKG;AACH,wBAAsB,oBAAoB,CACxC,GAAG,EAAE,MAAM,EACX,OAAO,GAAE;IAAE,QAAQ,CAAC,EAAE,MAAM,CAAA;CAAO,GAClC,OAAO,CAAC,iBAAiB,CAAC,CAsD5B;AAaD;;GAEG;AACH,wBAAgB,qBAAqB,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,GAAG,IAAI,CAgD9E;AAwGD;;;;GAIG;AACH,wBAAgB,eAAe,CAAC,GAAG,EAAE,MAAM,GAAG,iBAAiB,EAAE,CAqBhE;AAWD;;;;;;GAMG;AACH,wBAAgB,kBAAkB,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,CAgBvD;AAED;;GAEG;AACH,wBAAgB,cAAc,CAAC,OAAO,EAAE,MAAM,GAAG,MAAM,CAUtD"}