webpeel 0.8.1 → 0.9.0

package/dist/core/diff.d.ts

@@ -0,0 +1,62 @@
+/**
+ * WebPeel Diff - Semantic content diff against stored snapshots
+ *
+ * Fetches the current content of a URL, loads the previous snapshot from the
+ * change-tracking store, computes a structured diff (field-by-field for JSON,
+ * section-by-section for text/markdown), saves the new snapshot, and returns
+ * a structured {@link DiffResult}.
+ */
+export interface DiffOptions {
+    /** Compare against the last tracked snapshot (default behaviour). */
+    last?: boolean;
+    /** Only compare these dot-notation fields (for JSON responses only). */
+    fields?: string[];
+    /** Use browser rendering for the fetch. */
+    render?: boolean;
+    /** Per-request timeout in milliseconds (default: 30 000). */
+    timeout?: number;
+}
+export interface DiffResult {
+    url: string;
+    /** Whether any changes were detected. */
+    changed: boolean;
+    /** ISO-8601 timestamp of the current fetch. */
+    timestamp: string;
+    /** ISO-8601 timestamp of the previous snapshot (undefined if none). */
+    previousTimestamp?: string;
+    changes: DiffChange[];
+    /** Human-readable summary sentence. */
+    summary: string;
+}
+export interface DiffChange {
+    type: 'added' | 'removed' | 'modified';
+    /** For JSON diffs: dot-notation path to the changed field. */
+    field?: string;
+    /** For text diffs: nearest section heading or line reference. */
+    path?: string;
+    before?: string;
+    after?: string;
+}
+/**
+ * Fetch a URL and compute a semantic diff against the last tracked snapshot.
+ *
+ * The function:
+ * 1. Loads the previous snapshot (if any) from the change-tracking store.
+ * 2. Fetches the current content via {@link peel}.
+ * 3. Saves the current content as the new snapshot (auto-tracking).
+ * 4. Computes a structured diff — field-by-field for JSON, section-by-section
+ *    for text/markdown.
+ * 5. Returns a {@link DiffResult}.
+ *
+ * @example
+ * ```typescript
+ * const result = await diffUrl('https://api.example.com/health', { last: true });
+ * console.log(result.summary);
+ * result.changes.forEach(c => {
+ *   if (c.field) console.log(`${c.type}: ${c.field}  ${c.before} → ${c.after}`);
+ * });
+ * ```
+ */
+export declare function diffUrl(url: string, options?: DiffOptions): Promise<DiffResult>;
+export { trackChange, getSnapshot } from './change-tracking.js';
+//# sourceMappingURL=diff.d.ts.map

package/dist/core/diff.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"diff.d.ts","sourceRoot":"","sources":["../../src/core/diff.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AAOH,MAAM,WAAW,WAAW;IAC1B,qEAAqE;IACrE,IAAI,CAAC,EAAE,OAAO,CAAC;IACf,wEAAwE;IACxE,MAAM,CAAC,EAAE,MAAM,EAAE,CAAC;IAClB,2CAA2C;IAC3C,MAAM,CAAC,EAAE,OAAO,CAAC;IACjB,6DAA6D;IAC7D,OAAO,CAAC,EAAE,MAAM,CAAC;CAClB;AAED,MAAM,WAAW,UAAU;IACzB,GAAG,EAAE,MAAM,CAAC;IACZ,yCAAyC;IACzC,OAAO,EAAE,OAAO,CAAC;IACjB,+CAA+C;IAC/C,SAAS,EAAE,MAAM,CAAC;IAClB,uEAAuE;IACvE,iBAAiB,CAAC,EAAE,MAAM,CAAC;IAC3B,OAAO,EAAE,UAAU,EAAE,CAAC;IACtB,uCAAuC;IACvC,OAAO,EAAE,MAAM,CAAC;CACjB;AAED,MAAM,WAAW,UAAU;IACzB,IAAI,EAAE,OAAO,GAAG,SAAS,GAAG,UAAU,CAAC;IACvC,8DAA8D;IAC9D,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,iEAAiE;IACjE,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB;AA0OD;;;;;;;;;;;;;;;;;;;GAmBG;AACH,wBAAsB,OAAO,CAAC,GAAG,EAAE,MAAM,EAAE,OAAO,GAAE,WAAgB,GAAG,OAAO,CAAC,UAAU,CAAC,CAqEzF;AAID,OAAO,EAAE,WAAW,EAAE,WAAW,EAAE,MAAM,sBAAsB,CAAC"}

package/dist/core/diff.js

@@ -0,0 +1,289 @@
+/**
+ * WebPeel Diff - Semantic content diff against stored snapshots
+ *
+ * Fetches the current content of a URL, loads the previous snapshot from the
+ * change-tracking store, computes a structured diff (field-by-field for JSON,
+ * section-by-section for text/markdown), saves the new snapshot, and returns
+ * a structured {@link DiffResult}.
+ */
+import { peel } from '../index.js';
+import { getSnapshot } from './change-tracking.js';
+// ─── JSON diffing ──────────────────────────────────────────────────────────────
+/**
+ * Compare two JSON values recursively, returning structured {@link DiffChange}
+ * objects.  Non-object values (numbers, strings, arrays) are reported as atomic
+ * modifications.
+ *
+ * @param before   - Previous JSON value
+ * @param after    - Current JSON value
+ * @param path     - Current dot-notation path (for recursion; start with "")
+ * @param fields   - Optional allowlist of dot-notation paths to compare
+ */
+function diffJson(before, after, path = '', fields) {
+    const changes = [];
+    const isPlainObject = (v) => typeof v === 'object' && v !== null && !Array.isArray(v);
+    if (!isPlainObject(before) || !isPlainObject(after)) {
+        // Atomic comparison.
+        if (JSON.stringify(before) !== JSON.stringify(after)) {
+            if (!fields || fields.length === 0 || fields.some(f => f === path || path.startsWith(f + '.'))) {
+                changes.push({
+                    type: 'modified',
+                    field: path || '(root)',
+                    before: stringify(before),
+                    after: stringify(after),
+                });
+            }
+        }
+        return changes;
+    }
+    const allKeys = new Set([...Object.keys(before), ...Object.keys(after)]);
+    for (const key of allKeys) {
+        const fullPath = path ? `${path}.${key}` : key;
+        // Apply field filter when specified.
+        if (fields && fields.length > 0) {
+            const inScope = fields.some(f => fullPath === f || fullPath.startsWith(f + '.') || f.startsWith(fullPath + '.'));
+            if (!inScope)
+                continue;
+        }
+        const bVal = before[key];
+        const aVal = after[key];
+        if (bVal === undefined && aVal !== undefined) {
+            changes.push({ type: 'added', field: fullPath, after: stringify(aVal) });
+        }
+        else if (bVal !== undefined && aVal === undefined) {
+            changes.push({ type: 'removed', field: fullPath, before: stringify(bVal) });
+        }
+        else if (JSON.stringify(bVal) !== JSON.stringify(aVal)) {
+            // Recurse into nested objects.
+            if (isPlainObject(bVal) && isPlainObject(aVal)) {
+                changes.push(...diffJson(bVal, aVal, fullPath, fields));
+            }
+            else {
+                changes.push({
+                    type: 'modified',
+                    field: fullPath,
+                    before: stringify(bVal),
+                    after: stringify(aVal),
+                });
+            }
+        }
+    }
+    return changes;
+}
+/** Serialize a JSON value concisely for display. */
+function stringify(v) {
+    if (typeof v === 'string')
+        return v;
+    return JSON.stringify(v) ?? 'undefined';
+}
+/**
+ * Diff two text blobs line-by-line using LCS (Longest Common Subsequence).
+ * Groups consecutive additions/deletions into sections keyed by the nearest
+ * Markdown heading.
+ */
+function diffText(beforeText, afterText) {
+    const MAX_LINES = 5_000;
+    const bLines = beforeText.split('\n').slice(0, MAX_LINES);
+    const aLines = afterText.split('\n').slice(0, MAX_LINES);
+    // Build LCS table.
+    const m = bLines.length;
+    const n = aLines.length;
+    const lcs = Array.from({ length: m + 1 }, () => new Array(n + 1).fill(0));
+    for (let i = 1; i <= m; i++) {
+        for (let j = 1; j <= n; j++) {
+            if (bLines[i - 1] === aLines[j - 1]) {
+                lcs[i][j] = lcs[i - 1][j - 1] + 1;
+            }
+            else {
+                lcs[i][j] = Math.max(lcs[i - 1][j], lcs[i][j - 1]);
+            }
+        }
+    }
+    const ops = [];
+    let bi = m;
+    let ai = n;
+    while (bi > 0 || ai > 0) {
+        if (bi > 0 && ai > 0 && bLines[bi - 1] === aLines[ai - 1]) {
+            ops.unshift({ op: 'same', line: aLines[ai - 1] });
+            bi--;
+            ai--;
+        }
+        else if (ai > 0 && (bi === 0 || lcs[bi][ai - 1] >= lcs[bi - 1][ai])) {
+            ops.unshift({ op: 'add', line: aLines[ai - 1] });
+            ai--;
+        }
+        else {
+            ops.unshift({ op: 'del', line: bLines[bi - 1] });
+            bi--;
+        }
+    }
+    // Group consecutive non-same ops into sections.
+    const changes = [];
+    let linesAdded = 0;
+    let linesRemoved = 0;
+    let i = 0;
+    while (i < ops.length) {
+        if (ops[i].op === 'same') {
+            i++;
+            continue;
+        }
+        // Collect the run of changes.
+        const added = [];
+        const removed = [];
+        while (i < ops.length && ops[i].op !== 'same') {
+            if (ops[i].op === 'add')
+                added.push(ops[i].line);
+            if (ops[i].op === 'del')
+                removed.push(ops[i].line);
+            i++;
+        }
+        linesAdded += added.length;
+        linesRemoved += removed.length;
+        // Find nearest Markdown heading in the 'same' lines before this block.
+        let sectionLabel = `line ~${i}`;
+        for (let k = i - added.length - removed.length - 1; k >= 0; k--) {
+            const prevOp = ops[k];
+            if (prevOp && prevOp.op === 'same' && /^#{1,6}\s/.test(prevOp.line)) {
+                sectionLabel = prevOp.line.trim();
+                break;
+            }
+        }
+        if (removed.length > 0 && added.length > 0) {
+            changes.push({ type: 'modified', path: sectionLabel, before: removed.join('\n'), after: added.join('\n') });
+        }
+        else if (added.length > 0) {
+            changes.push({ type: 'added', path: sectionLabel, after: added.join('\n') });
+        }
+        else if (removed.length > 0) {
+            changes.push({ type: 'removed', path: sectionLabel, before: removed.join('\n') });
+        }
+    }
+    return { changes, stats: { linesAdded, linesRemoved } };
+}
+// ─── Summary generation ────────────────────────────────────────────────────────
+function buildSummary(changes, mode, jsonTotalFields, textStats) {
+    if (changes.length === 0)
+        return 'No changes detected.';
+    if (mode === 'json') {
+        const unchanged = Math.max(0, (jsonTotalFields ?? 0) - changes.length);
+        const parts = [
+            `${changes.length} field${changes.length === 1 ? '' : 's'} changed`,
+        ];
+        if (unchanged > 0)
+            parts.push(`${unchanged} unchanged`);
+        return parts.join(', ') + '.';
+    }
+    // Text mode.
+    const sections = changes.length;
+    const added = textStats?.linesAdded ?? 0;
+    const removed = textStats?.linesRemoved ?? 0;
+    return (`${sections} section${sections === 1 ? '' : 's'} changed` +
+        (added > 0 ? `, ${added} lines added` : '') +
+        (removed > 0 ? `, ${removed} removed` : '') +
+        '.');
+}
+// ─── Key counting helpers ──────────────────────────────────────────────────────
+/** Count the total number of leaf-level keys (dot-notation) in two JSON objects combined. */
+function countTotalFields(a, b) {
+    const keys = new Set();
+    collectKeys(a, '', keys);
+    collectKeys(b, '', keys);
+    return keys.size;
+}
+function collectKeys(obj, prefix, acc) {
+    if (typeof obj !== 'object' || obj === null || Array.isArray(obj)) {
+        if (prefix)
+            acc.add(prefix);
+        return;
+    }
+    for (const key of Object.keys(obj)) {
+        const path = prefix ? `${prefix}.${key}` : key;
+        acc.add(path);
+        collectKeys(obj[key], path, acc);
+    }
+}
+// ─── Public API ────────────────────────────────────────────────────────────────
+/**
+ * Fetch a URL and compute a semantic diff against the last tracked snapshot.
+ *
+ * The function:
+ * 1. Loads the previous snapshot (if any) from the change-tracking store.
+ * 2. Fetches the current content via {@link peel}.
+ * 3. Saves the current content as the new snapshot (auto-tracking).
+ * 4. Computes a structured diff — field-by-field for JSON, section-by-section
+ *    for text/markdown.
+ * 5. Returns a {@link DiffResult}.
+ *
+ * @example
+ * ```typescript
+ * const result = await diffUrl('https://api.example.com/health', { last: true });
+ * console.log(result.summary);
+ * result.changes.forEach(c => {
+ *   if (c.field) console.log(`${c.type}: ${c.field}  ${c.before} → ${c.after}`);
+ * });
+ * ```
+ */
+export async function diffUrl(url, options = {}) {
+    const { fields, render = false, timeout = 30_000 } = options;
+    // 1. Load previous snapshot before fetching (fetch overwrites it).
+    const prevSnapshot = await getSnapshot(url);
+    // 2. Fetch current content.  changeTracking: true auto-saves the new snapshot.
+    const result = await peel(url, {
+        render,
+        timeout,
+        format: 'markdown',
+        changeTracking: true,
+    });
+    const now = new Date().toISOString();
+    // 3. No baseline → return an informational result.
+    if (!prevSnapshot) {
+        return {
+            url: result.url,
+            changed: false,
+            timestamp: now,
+            changes: [],
+            summary: 'No previous snapshot found. Current content saved as baseline.',
+        };
+    }
+    const previousTimestamp = new Date(prevSnapshot.timestamp).toISOString();
+    const previousContent = prevSnapshot.content;
+    const currentContent = result.content;
+    // 4. Detect content type and compute appropriate diff.
+    let mode = 'text';
+    let changes = [];
+    let jsonTotalFields = 0;
+    let textStats;
+    let prevJson = null;
+    let currJson = null;
+    try {
+        prevJson = JSON.parse(previousContent);
+        currJson = JSON.parse(currentContent);
+        mode = 'json';
+    }
+    catch {
+        /* Not JSON — fall through to text diffing */
+    }
+    if (mode === 'json' && prevJson !== null && currJson !== null) {
+        changes = diffJson(prevJson, currJson, '', fields);
+        jsonTotalFields = countTotalFields(prevJson, currJson);
+    }
+    else {
+        const { changes: textChanges, stats } = diffText(previousContent, currentContent);
+        changes = textChanges;
+        textStats = stats;
+    }
+    // 5. Handle edge case: content changed but we couldn't detect it (e.g. fingerprint
+    //    mismatch recorded by peel, but diff shows no changes at field level).
+    const changed = changes.length > 0;
+    return {
+        url: result.url,
+        changed,
+        timestamp: now,
+        previousTimestamp,
+        changes,
+        summary: buildSummary(changes, mode, jsonTotalFields, textStats),
+    };
+}
+// ─── Re-export trackChange for CLI convenience ─────────────────────────────────
+export { trackChange, getSnapshot } from './change-tracking.js';
+//# sourceMappingURL=diff.js.map

package/dist/core/diff.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"diff.js","sourceRoot":"","sources":["../../src/core/diff.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AAEH,OAAO,EAAE,IAAI,EAAE,MAAM,aAAa,CAAC;AACnC,OAAO,EAAE,WAAW,EAAE,MAAM,sBAAsB,CAAC;AAsCnD,kFAAkF;AAElF;;;;;;;;;GASG;AACH,SAAS,QAAQ,CACf,MAAe,EACf,KAAc,EACd,OAAe,EAAE,EACjB,MAAiB;IAEjB,MAAM,OAAO,GAAiB,EAAE,CAAC;IAEjC,MAAM,aAAa,GAAG,CAAC,CAAU,EAAgC,EAAE,CACjE,OAAO,CAAC,KAAK,QAAQ,IAAI,CAAC,KAAK,IAAI,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC;IAE3D,IAAI,CAAC,aAAa,CAAC,MAAM,CAAC,IAAI,CAAC,aAAa,CAAC,KAAK,CAAC,EAAE,CAAC;QACpD,qBAAqB;QACrB,IAAI,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,KAAK,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,EAAE,CAAC;YACrD,IAAI,CAAC,MAAM,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC,IAAI,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,KAAK,IAAI,IAAI,IAAI,CAAC,UAAU,CAAC,CAAC,GAAG,GAAG,CAAC,CAAC,EAAE,CAAC;gBAC/F,OAAO,CAAC,IAAI,CAAC;oBACX,IAAI,EAAE,UAAU;oBAChB,KAAK,EAAE,IAAI,IAAI,QAAQ;oBACvB,MAAM,EAAE,SAAS,CAAC,MAAM,CAAC;oBACzB,KAAK,EAAE,SAAS,CAAC,KAAK,CAAC;iBACxB,CAAC,CAAC;YACL,CAAC;QACH,CAAC;QACD,OAAO,OAAO,CAAC;IACjB,CAAC;IAED,MAAM,OAAO,GAAG,IAAI,GAAG,CAAC,CAAC,GAAG,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,EAAE,GAAG,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;IAEzE,KAAK,MAAM,GAAG,IAAI,OAAO,EAAE,CAAC;QAC1B,MAAM,QAAQ,GAAG,IAAI,CAAC,CAAC,CAAC,GAAG,IAAI,IAAI,GAAG,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC;QAE/C,qCAAqC;QACrC,IAAI,MAAM,IAAI,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAChC,MAAM,OAAO,GAAG,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAC9B,QAAQ,KAAK,CAAC,IAAI,QAAQ,CAAC,UAAU,CAAC,CAAC,GAAG,GAAG,CAAC,IAAI,CAAC,CAAC,UAAU,CAAC,QAAQ,GAAG,GAAG,CAAC,CAC/E,CAAC;YACF,IAAI,CAAC,OAAO;gBAAE,SAAS;QACzB,CAAC;QAED,MAAM,IAAI,GAAG,MAAM,CAAC,GAAG,CAAC,CAAC;QACzB,MAAM,IAAI,GAAG,KAAK,CAAC,GAAG,CAAC,CAAC;QAExB,IAAI,IAAI,KAAK,SAAS,IAAI,IAAI,KAAK,SAAS,EAAE,CAAC;YAC7C,OAAO,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,OAAO,EAAE,KAAK,EAAE,QAAQ,EAAE,KAAK,EAAE,SAAS,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QAC3E,CAAC;aAAM,IAAI,IAAI,KAAK,SAAS,IAAI,IAAI,KAAK,SAAS,EAAE,CAAC;YACpD,OAAO,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,SAAS,EAAE,KAAK,EAAE,QAAQ,EAAE,MAAM,EAAE,SAAS,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QAC9E,CAAC;aAAM,IAAI,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,KAAK,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,EAAE,CAAC;YACzD,+BAA+B;YAC/B,IAAI,aAAa,CAAC,IAAI,CAAC,IAAI,aAAa,CAAC,IAAI,CAAC,EAAE,CAAC;gBAC/C,OAAO,CAAC,IAAI,CAAC,GAAG,QAAQ,CAAC,IAAI,EAAE,IAAI,EAAE,QAAQ,EAAE,MAAM,CAAC,CAAC,CAAC;YAC1D,CAAC;iBAAM,CAAC;gBACN,OAAO,CAAC,IAAI,CAAC;oBACX,IAAI,EAAE,UAAU;oBAChB,KAAK,EAAE,QAAQ;oBACf,MAAM,EAAE,SAAS,CAAC,IAAI,CAAC;oBACvB,KAAK,EAAE,SAAS,CAAC,IAAI,CAAC;iBACvB,CAAC,CAAC;YACL,CAAC;QACH,CAAC;IACH,CAAC;IAED,OAAO,OAAO,CAAC;AACjB,CAAC;AAED,oDAAoD;AACpD,SAAS,SAAS,CAAC,CAAU;IAC3B,IAAI,OAAO,CAAC,KAAK,QAAQ;QAAE,OAAO,CAAC,CAAC;IACpC,OAAO,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI,WAAW,CAAC;AAC1C,CAAC;AASD;;;;GAIG;AACH,SAAS,QAAQ,CAAC,UAAkB,EAAE,SAAiB;IACrD,MAAM,SAAS,GAAG,KAAK,CAAC;IACxB,MAAM,MAAM,GAAG,UAAU,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,SAAS,CAAC,CAAC;IAC1D,MAAM,MAAM,GAAG,SAAS,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,SAAS,CAAC,CAAC;IAEzD,mBAAmB;IACnB,MAAM,CAAC,GAAG,MAAM,CAAC,MAAM,CAAC;IACxB,MAAM,CAAC,GAAG,MAAM,CAAC,MAAM,CAAC;IACxB,MAAM,GAAG,GAAe,KAAK,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,CAAC,GAAG,CAAC,EAAE,EAAE,GAAG,EAAE,CAAC,IAAI,KAAK,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAa,CAAC,CAAC;IAElG,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,IAAI,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;QAC5B,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,IAAI,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;YAC5B,IAAI,MAAM,CAAC,CAAC,GAAG,CAAC,CAAC,KAAK,MAAM,CAAC,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC;gBACpC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC;YACpC,CAAC;iBAAM,CAAC;gBACN,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAE,EAAE,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAE,CAAC,CAAC;YACvD,CAAC;QACH,CAAC;IACH,CAAC;IAID,MAAM,GAAG,GAAS,EAAE,CAAC;IACrB,IAAI,EAAE,GAAG,CAAC,CAAC;IACX,IAAI,EAAE,GAAG,CAAC,CAAC;IAEX,OAAO,EAAE,GAAG,CAAC,IAAI,EAAE,GAAG,CAAC,EAAE,CAAC;QACxB,IAAI,EAAE,GAAG,CAAC,IAAI,EAAE,GAAG,CAAC,IAAI,MAAM,CAAC,EAAE,GAAG,CAAC,CAAC,KAAK,MAAM,CAAC,EAAE,GAAG,CAAC,CAAC,EAAE,CAAC;YAC1D,GAAG,CAAC,OAAO,CAAC,EAAE,EAAE,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,CAAC,EAAE,GAAG,CAAC,CAAE,EAAE,CAAC,CAAC;YACnD,EAAE,EAAE,CAAC;YAAC,EAAE,EAAE,CAAC;QACb,CAAC;aAAM,IAAI,EAAE,GAAG,CAAC,IAAI,CAAC,EAAE,KAAK,CAAC,IAAI,GAAG,CAAC,EAAE,CAAE,CAAC,EAAE,GAAG,CAAC,CAAE,IAAI,GAAG,CAAC,EAAE,GAAG,CAAC,CAAE,CAAC,EAAE,CAAE,CAAC,EAAE,CAAC;YAC1E,GAAG,CAAC,OAAO,CAAC,EAAE,EAAE,EAAE,KAAK,EAAE,IAAI,EAAE,MAAM,CAAC,EAAE,GAAG,CAAC,CAAE,EAAE,CAAC,CAAC;YAClD,EAAE,EAAE,CAAC;QACP,CAAC;aAAM,CAAC;YACN,GAAG,CAAC,OAAO,CAAC,EAAE,EAAE,EAAE,KAAK,EAAE,IAAI,EAAE,MAAM,CAAC,EAAE,GAAG,CAAC,CAAE,EAAE,CAAC,CAAC;YAClD,EAAE,EAAE,CAAC;QACP,CAAC;IACH,CAAC;IAED,gDAAgD;IAChD,MAAM,OAAO,GAAiB,EAAE,CAAC;IACjC,IAAI,UAAU,GAAG,CAAC,CAAC;IACnB,IAAI,YAAY,GAAG,CAAC,CAAC;IACrB,IAAI,CAAC,GAAG,CAAC,CAAC;IAEV,OAAO,CAAC,GAAG,GAAG,CAAC,MAAM,EAAE,CAAC;QACtB,IAAI,GAAG,CAAC,CAAC,CAAE,CAAC,EAAE,KAAK,MAAM,EAAE,CAAC;YAAC,CAAC,EAAE,CAAC;YAAC,SAAS;QAAC,CAAC;QAE7C,8BAA8B;QAC9B,MAAM,KAAK,GAAa,EAAE,CAAC;QAC3B,MAAM,OAAO,GAAa,EAAE,CAAC;QAE7B,OAAO,CAAC,GAAG,GAAG,CAAC,MAAM,IAAI,GAAG,CAAC,CAAC,CAAE,CAAC,EAAE,KAAK,MAAM,EAAE,CAAC;YAC/C,IAAI,GAAG,CAAC,CAAC,CAAE,CAAC,EAAE,KAAK,KAAK;gBAAE,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAE,CAAC,IAAI,CAAC,CAAC;YACnD,IAAI,GAAG,CAAC,CAAC,CAAE,CAAC,EAAE,KAAK,KAAK;gBAAE,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAE,CAAC,IAAI,CAAC,CAAC;YACrD,CAAC,EAAE,CAAC;QACN,CAAC;QAED,UAAU,IAAI,KAAK,CAAC,MAAM,CAAC;QAC3B,YAAY,IAAI,OAAO,CAAC,MAAM,CAAC;QAE/B,uEAAuE;QACvE,IAAI,YAAY,GAAG,SAAS,CAAC,EAAE,CAAC;QAChC,KAAK,IAAI,CAAC,GAAG,CAAC,GAAG,KAAK,CAAC,MAAM,GAAG,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC,IAAI,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;YAChE,MAAM,MAAM,GAAG,GAAG,CAAC,CAAC,CAAC,CAAC;YACtB,IAAI,MAAM,IAAI,MAAM,CAAC,EAAE,KAAK,MAAM,IAAI,WAAW,CAAC,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC;gBACpE,YAAY,GAAG,MAAM,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC;gBAClC,MAAM;YACR,CAAC;QACH,CAAC;QAED,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC,IAAI,KAAK,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAC3C,OAAO,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,UAAU,EAAE,IAAI,EAAE,YAAY,EAAE,MAAM,EAAE,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QAC9G,CAAC;aAAM,IAAI,KAAK,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAC5B,OAAO,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,OAAO,EAAE,IAAI,EAAE,YAAY,EAAE,KAAK,EAAE,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QAC/E,CAAC;aAAM,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAC9B,OAAO,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,SAAS,EAAE,IAAI,EAAE,YAAY,EAAE,MAAM,EAAE,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QACpF,CAAC;IACH,CAAC;IAED,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,EAAE,UAAU,EAAE,YAAY,EAAE,EAAE,CAAC;AAC1D,CAAC;AAED,kFAAkF;AAElF,SAAS,YAAY,CACnB,OAAqB,EACrB,IAAqB,EACrB,eAAwB,EACxB,SAAyB;IAEzB,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,sBAAsB,CAAC;IAExD,IAAI,IAAI,KAAK,MAAM,EAAE,CAAC;QACpB,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC,eAAe,IAAI,CAAC,CAAC,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC;QACvE,MAAM,KAAK,GAAa;YACtB,GAAG,OAAO,CAAC,MAAM,SAAS,OAAO,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,GAAG,UAAU;SACpE,CAAC;QACF,IAAI,SAAS,GAAG,CAAC;YAAE,KAAK,CAAC,IAAI,CAAC,GAAG,SAAS,YAAY,CAAC,CAAC;QACxD,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,GAAG,CAAC;IAChC,CAAC;IAED,aAAa;IACb,MAAM,QAAQ,GAAG,OAAO,CAAC,MAAM,CAAC;IAChC,MAAM,KAAK,GAAG,SAAS,EAAE,UAAU,IAAI,CAAC,CAAC;IACzC,MAAM,OAAO,GAAG,SAAS,EAAE,YAAY,IAAI,CAAC,CAAC;IAC7C,OAAO,CACL,GAAG,QAAQ,WAAW,QAAQ,KAAK,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,GAAG,UAAU;QACzD,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC,CAAC,KAAK,KAAK,cAAc,CAAC,CAAC,CAAC,EAAE,CAAC;QAC3C,CAAC,OAAO,GAAG,CAAC,CAAC,CAAC,CAAC,KAAK,OAAO,UAAU,CAAC,CAAC,CAAC,EAAE,CAAC;QAC3C,GAAG,CACJ,CAAC;AACJ,CAAC;AAED,kFAAkF;AAElF,6FAA6F;AAC7F,SAAS,gBAAgB,CAAC,CAAU,EAAE,CAAU;IAC9C,MAAM,IAAI,GAAG,IAAI,GAAG,EAAU,CAAC;IAC/B,WAAW,CAAC,CAAC,EAAE,EAAE,EAAE,IAAI,CAAC,CAAC;IACzB,WAAW,CAAC,CAAC,EAAE,EAAE,EAAE,IAAI,CAAC,CAAC;IACzB,OAAO,IAAI,CAAC,IAAI,CAAC;AACnB,CAAC;AAED,SAAS,WAAW,CAAC,GAAY,EAAE,MAAc,EAAE,GAAgB;IACjE,IAAI,OAAO,GAAG,KAAK,QAAQ,IAAI,GAAG,KAAK,IAAI,IAAI,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC;QAClE,IAAI,MAAM;YAAE,GAAG,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;QAC5B,OAAO;IACT,CAAC;IACD,KAAK,MAAM,GAAG,IAAI,MAAM,CAAC,IAAI,CAAC,GAA8B,CAAC,EAAE,CAAC;QAC9D,MAAM,IAAI,GAAG,MAAM,CAAC,CAAC,CAAC,GAAG,MAAM,IAAI,GAAG,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC;QAC/C,GAAG,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;QACd,WAAW,CAAE,GAA+B,CAAC,GAAG,CAAC,EAAE,IAAI,EAAE,GAAG,CAAC,CAAC;IAChE,CAAC;AACH,CAAC;AAED,kFAAkF;AAElF;;;;;;;;;;;;;;;;;;;GAmBG;AACH,MAAM,CAAC,KAAK,UAAU,OAAO,CAAC,GAAW,EAAE,UAAuB,EAAE;IAClE,MAAM,EAAE,MAAM,EAAE,MAAM,GAAG,KAAK,EAAE,OAAO,GAAG,MAAM,EAAE,GAAG,OAAO,CAAC;IAE7D,mEAAmE;IACnE,MAAM,YAAY,GAAG,MAAM,WAAW,CAAC,GAAG,CAAC,CAAC;IAE5C,+EAA+E;IAC/E,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,GAAG,EAAE;QAC7B,MAAM;QACN,OAAO;QACP,MAAM,EAAE,UAAU;QAClB,cAAc,EAAE,IAAI;KACrB,CAAC,CAAC;IAEH,MAAM,GAAG,GAAG,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;IAErC,mDAAmD;IACnD,IAAI,CAAC,YAAY,EAAE,CAAC;QAClB,OAAO;YACL,GAAG,EAAE,MAAM,CAAC,GAAG;YACf,OAAO,EAAE,KAAK;YACd,SAAS,EAAE,GAAG;YACd,OAAO,EAAE,EAAE;YACX,OAAO,EAAE,gEAAgE;SAC1E,CAAC;IACJ,CAAC;IAED,MAAM,iBAAiB,GAAG,IAAI,IAAI,CAAC,YAAY,CAAC,SAAS,CAAC,CAAC,WAAW,EAAE,CAAC;IACzE,MAAM,eAAe,GAAG,YAAY,CAAC,OAAO,CAAC;IAC7C,MAAM,cAAc,GAAG,MAAM,CAAC,OAAO,CAAC;IAEtC,uDAAuD;IACvD,IAAI,IAAI,GAAoB,MAAM,CAAC;IACnC,IAAI,OAAO,GAAiB,EAAE,CAAC;IAC/B,IAAI,eAAe,GAAG,CAAC,CAAC;IACxB,IAAI,SAAoC,CAAC;IAEzC,IAAI,QAAQ,GAAY,IAAI,CAAC;IAC7B,IAAI,QAAQ,GAAY,IAAI,CAAC;IAE7B,IAAI,CAAC;QACH,QAAQ,GAAG,IAAI,CAAC,KAAK,CAAC,eAAe,CAAC,CAAC;QACvC,QAAQ,GAAG,IAAI,CAAC,KAAK,CAAC,cAAc,CAAC,CAAC;QACtC,IAAI,GAAG,MAAM,CAAC;IAChB,CAAC;IAAC,MAAM,CAAC;QACP,6CAA6C;IAC/C,CAAC;IAED,IAAI,IAAI,KAAK,MAAM,IAAI,QAAQ,KAAK,IAAI,IAAI,QAAQ,KAAK,IAAI,EAAE,CAAC;QAC9D,OAAO,GAAG,QAAQ,CAAC,QAAQ,EAAE,QAAQ,EAAE,EAAE,EAAE,MAAM,CAAC,CAAC;QACnD,eAAe,GAAG,gBAAgB,CAAC,QAAQ,EAAE,QAAQ,CAAC,CAAC;IACzD,CAAC;SAAM,CAAC;QACN,MAAM,EAAE,OAAO,EAAE,WAAW,EAAE,KAAK,EAAE,GAAG,QAAQ,CAAC,eAAe,EAAE,cAAc,CAAC,CAAC;QAClF,OAAO,GAAG,WAAW,CAAC;QACtB,SAAS,GAAG,KAAK,CAAC;IACpB,CAAC;IAED,mFAAmF;IACnF,2EAA2E;IAC3E,MAAM,OAAO,GAAG,OAAO,CAAC,MAAM,GAAG,CAAC,CAAC;IAEnC,OAAO;QACL,GAAG,EAAE,MAAM,CAAC,GAAG;QACf,OAAO;QACP,SAAS,EAAE,GAAG;QACd,iBAAiB;QACjB,OAAO;QACP,OAAO,EAAE,YAAY,CAAC,OAAO,EAAE,IAAI,EAAE,eAAe,EAAE,SAAS,CAAC;KACjE,CAAC;AACJ,CAAC;AAED,kFAAkF;AAElF,OAAO,EAAE,WAAW,EAAE,WAAW,EAAE,MAAM,sBAAsB,CAAC"}

package/dist/core/extract-listings.d.ts

@@ -0,0 +1,39 @@
+/**
+ * Auto-extract repeated listing patterns from HTML pages.
+ *
+ * Given raw HTML (e.g. an eBay search results page), this module detects the
+ * largest group of sibling elements with a consistent internal structure and
+ * extracts structured fields (title, price, image, link, description, rating)
+ * from each item.
+ *
+ * @module extract-listings
+ */
+/** A single extracted listing item. */
+export interface ListingItem {
+    title?: string;
+    price?: string;
+    image?: string;
+    link?: string;
+    description?: string;
+    rating?: string;
+    [key: string]: string | undefined;
+}
+/**
+ * Automatically detect repeated listing patterns in raw HTML and extract
+ * structured items.
+ *
+ * @param html - Raw HTML string to parse.
+ * @param url  - Optional base URL for resolving relative links and images.
+ * @returns    Array of extracted listing items (may be empty).
+ *
+ * @example
+ * ```typescript
+ * import { extractListings } from 'webpeel';
+ *
+ * const items = extractListings(ebayHtml, 'https://ebay.com/sch?q=card');
+ * console.log(items[0].title);  // "Charizard VMAX 020/189"
+ * console.log(items[0].price);  // "$24.99"
+ * ```
+ */
+export declare function extractListings(html: string, url?: string): ListingItem[];
+//# sourceMappingURL=extract-listings.d.ts.map

package/dist/core/extract-listings.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"extract-listings.d.ts","sourceRoot":"","sources":["../../src/core/extract-listings.ts"],"names":[],"mappings":"AAAA;;;;;;;;;GASG;AASH,uCAAuC;AACvC,MAAM,WAAW,WAAW;IAC1B,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,CAAC,GAAG,EAAE,MAAM,GAAG,MAAM,GAAG,SAAS,CAAC;CACnC;AAyTD;;;;;;;;;;;;;;;;GAgBG;AACH,wBAAgB,eAAe,CAAC,IAAI,EAAE,MAAM,EAAE,GAAG,CAAC,EAAE,MAAM,GAAG,WAAW,EAAE,CAgBzE"}