webpeel 0.9.0 → 0.10.0

package/dist/core/profiles.js

@@ -0,0 +1,211 @@
+/**
+ * WebPeel Profile Management
+ *
+ * Manages named browser profiles stored in ~/.webpeel/profiles/<name>/
+ * Each profile contains:
+ *   - storage-state.json  (Playwright storage state: cookies, localStorage, origins)
+ *   - metadata.json       (name, created, lastUsed, domains, description)
+ */
+import { chromium } from 'playwright';
+import { homedir } from 'os';
+import { existsSync, mkdirSync, readFileSync, writeFileSync, rmSync, readdirSync, } from 'fs';
+import path from 'path';
+// ─── Paths ───────────────────────────────────────────────────────────────────
+const PROFILES_DIR = path.join(homedir(), '.webpeel', 'profiles');
+function ensureProfilesDir() {
+    if (!existsSync(PROFILES_DIR)) {
+        mkdirSync(PROFILES_DIR, { recursive: true });
+    }
+}
+// ─── Name validation ─────────────────────────────────────────────────────────
+/**
+ * Valid profile names: letters, digits, hyphens only. No spaces or special chars.
+ */
+export function isValidProfileName(name) {
+    return /^[a-zA-Z0-9-]+$/.test(name) && name.length > 0 && name.length <= 64;
+}
+// ─── Core helpers ─────────────────────────────────────────────────────────────
+/**
+ * Get the directory path for a named profile, or null if it doesn't exist.
+ */
+export function getProfilePath(name) {
+    const dir = path.join(PROFILES_DIR, name);
+    if (existsSync(dir) && existsSync(path.join(dir, 'metadata.json'))) {
+        return dir;
+    }
+    return null;
+}
+/**
+ * Load the Playwright storage state (cookies + localStorage) for a named profile.
+ * Returns null if the profile or storage-state.json doesn't exist.
+ */
+export function loadStorageState(name) {
+    const statePath = path.join(PROFILES_DIR, name, 'storage-state.json');
+    if (!existsSync(statePath))
+        return null;
+    try {
+        return JSON.parse(readFileSync(statePath, 'utf-8'));
+    }
+    catch {
+        return null;
+    }
+}
+/**
+ * Update the lastUsed timestamp for a profile.
+ */
+export function touchProfile(name) {
+    const metaPath = path.join(PROFILES_DIR, name, 'metadata.json');
+    if (!existsSync(metaPath))
+        return;
+    try {
+        const meta = JSON.parse(readFileSync(metaPath, 'utf-8'));
+        meta.lastUsed = new Date().toISOString();
+        writeFileSync(metaPath, JSON.stringify(meta, null, 2));
+    }
+    catch {
+        /* ignore */
+    }
+}
+/**
+ * List all profiles, sorted by lastUsed descending.
+ */
+export function listProfiles() {
+    ensureProfilesDir();
+    const profiles = [];
+    try {
+        const entries = readdirSync(PROFILES_DIR, { withFileTypes: true });
+        for (const entry of entries) {
+            if (!entry.isDirectory())
+                continue;
+            const metaPath = path.join(PROFILES_DIR, entry.name, 'metadata.json');
+            if (!existsSync(metaPath))
+                continue;
+            try {
+                const meta = JSON.parse(readFileSync(metaPath, 'utf-8'));
+                profiles.push(meta);
+            }
+            catch {
+                /* skip corrupt profile */
+            }
+        }
+    }
+    catch {
+        /* ignore read errors */
+    }
+    // Sort: most recently used first
+    profiles.sort((a, b) => b.lastUsed.localeCompare(a.lastUsed));
+    return profiles;
+}
+/**
+ * Delete a named profile. Returns true if deleted, false if not found.
+ */
+export function deleteProfile(name) {
+    const dir = path.join(PROFILES_DIR, name);
+    if (!existsSync(dir))
+        return false;
+    try {
+        rmSync(dir, { recursive: true, force: true });
+        return true;
+    }
+    catch {
+        return false;
+    }
+}
+// ─── Interactive profile creation ─────────────────────────────────────────────
+/**
+ * Interactively create a new profile:
+ * 1. Launches a VISIBLE (headed) Chromium browser
+ * 2. User navigates and logs into sites
+ * 3. On browser close or Ctrl+C, captures storage state and saves the profile
+ */
+export async function createProfile(name, description) {
+    if (!isValidProfileName(name)) {
+        throw new Error(`Invalid profile name "${name}". Use only letters, numbers, and hyphens (no spaces or special characters).`);
+    }
+    ensureProfilesDir();
+    const profileDir = path.join(PROFILES_DIR, name);
+    if (existsSync(profileDir)) {
+        throw new Error(`Profile "${name}" already exists. Delete it first with:\n  webpeel profile delete ${name}`);
+    }
+    mkdirSync(profileDir, { recursive: true });
+    // Launch headed (visible) Chromium — no user-data-dir so we start fresh
+    const browser = await chromium.launch({ headless: false });
+    const context = await browser.newContext();
+    const page = await context.newPage();
+    await page.goto('about:blank').catch(() => { });
+    console.log('');
+    console.log('╔══════════════════════════════════════════════════════╗');
+    console.log(`║  WebPeel Profile Setup: "${name}"`);
+    console.log('║                                                      ║');
+    console.log('║  Navigate to websites and log in.                   ║');
+    console.log('║  When done, press Ctrl+C or close this window.      ║');
+    console.log('╚══════════════════════════════════════════════════════╝');
+    console.log('');
+    let saved = false;
+    const saveAndClose = async () => {
+        if (saved)
+            return;
+        saved = true;
+        console.log('\nCapturing browser session...');
+        try {
+            const storageState = await context.storageState();
+            writeFileSync(path.join(profileDir, 'storage-state.json'), JSON.stringify(storageState, null, 2));
+            // Extract unique domains from cookies (strip leading dot)
+            const domains = [
+                ...new Set((storageState.cookies ?? [])
+                    .map((c) => (c.domain ?? '').replace(/^\./, ''))
+                    .filter(Boolean)),
+            ];
+            const now = new Date().toISOString();
+            const meta = {
+                name,
+                created: now,
+                lastUsed: now,
+                domains,
+                ...(description ? { description } : {}),
+            };
+            writeFileSync(path.join(profileDir, 'metadata.json'), JSON.stringify(meta, null, 2));
+            console.log(`✓ Profile "${name}" saved to ${profileDir}`);
+            if (domains.length > 0) {
+                console.log(`  Domains: ${domains.join(', ')}`);
+            }
+            else {
+                console.log('  No login sessions detected (no cookies).');
+            }
+        }
+        catch (e) {
+            console.error('Warning: Failed to save storage state:', e instanceof Error ? e.message : String(e));
+            // Clean up partial directory
+            try {
+                rmSync(profileDir, { recursive: true, force: true });
+            }
+            catch {
+                /* ignore */
+            }
+        }
+        try {
+            await browser.close();
+        }
+        catch {
+            /* ignore — browser may already be closed */
+        }
+    };
+    // Wait for the browser to disconnect (user closed the window) OR SIGINT (Ctrl+C)
+    await new Promise((resolve) => {
+        browser.on('disconnected', async () => {
+            await saveAndClose();
+            resolve();
+        });
+        // Handle Ctrl+C gracefully
+        const sigintHandler = async () => {
+            await saveAndClose();
+            resolve();
+        };
+        process.once('SIGINT', sigintHandler);
+        // Clean up the SIGINT handler if browser closes first
+        browser.on('disconnected', () => {
+            process.removeListener('SIGINT', sigintHandler);
+        });
+    });
+}
+//# sourceMappingURL=profiles.js.map

package/dist/core/profiles.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"profiles.js","sourceRoot":"","sources":["../../src/core/profiles.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AAEH,OAAO,EAAE,QAAQ,EAAE,MAAM,YAAY,CAAC;AACtC,OAAO,EAAE,OAAO,EAAE,MAAM,IAAI,CAAC;AAC7B,OAAO,EACL,UAAU,EACV,SAAS,EACT,YAAY,EACZ,aAAa,EACb,MAAM,EACN,WAAW,GACZ,MAAM,IAAI,CAAC;AACZ,OAAO,IAAI,MAAM,MAAM,CAAC;AAYxB,gFAAgF;AAEhF,MAAM,YAAY,GAAG,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,EAAE,UAAU,EAAE,UAAU,CAAC,CAAC;AAElE,SAAS,iBAAiB;IACxB,IAAI,CAAC,UAAU,CAAC,YAAY,CAAC,EAAE,CAAC;QAC9B,SAAS,CAAC,YAAY,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IAC/C,CAAC;AACH,CAAC;AAED,gFAAgF;AAEhF;;GAEG;AACH,MAAM,UAAU,kBAAkB,CAAC,IAAY;IAC7C,OAAO,iBAAiB,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,IAAI,CAAC,MAAM,GAAG,CAAC,IAAI,IAAI,CAAC,MAAM,IAAI,EAAE,CAAC;AAC9E,CAAC;AAED,iFAAiF;AAEjF;;GAEG;AACH,MAAM,UAAU,cAAc,CAAC,IAAY;IACzC,MAAM,GAAG,GAAG,IAAI,CAAC,IAAI,CAAC,YAAY,EAAE,IAAI,CAAC,CAAC;IAC1C,IAAI,UAAU,CAAC,GAAG,CAAC,IAAI,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,eAAe,CAAC,CAAC,EAAE,CAAC;QACnE,OAAO,GAAG,CAAC;IACb,CAAC;IACD,OAAO,IAAI,CAAC;AACd,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,gBAAgB,CAAC,IAAY;IAC3C,MAAM,SAAS,GAAG,IAAI,CAAC,IAAI,CAAC,YAAY,EAAE,IAAI,EAAE,oBAAoB,CAAC,CAAC;IACtE,IAAI,CAAC,UAAU,CAAC,SAAS,CAAC;QAAE,OAAO,IAAI,CAAC;IACxC,IAAI,CAAC;QACH,OAAO,IAAI,CAAC,KAAK,CAAC,YAAY,CAAC,SAAS,EAAE,OAAO,CAAC,CAAC,CAAC;IACtD,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,IAAI,CAAC;IACd,CAAC;AACH,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,YAAY,CAAC,IAAY;IACvC,MAAM,QAAQ,GAAG,IAAI,CAAC,IAAI,CAAC,YAAY,EAAE,IAAI,EAAE,eAAe,CAAC,CAAC;IAChE,IAAI,CAAC,UAAU,CAAC,QAAQ,CAAC;QAAE,OAAO;IAClC,IAAI,CAAC;QACH,MAAM,IAAI,GAAoB,IAAI,CAAC,KAAK,CAAC,YAAY,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC,CAAC;QAC1E,IAAI,CAAC,QAAQ,GAAG,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;QACzC,aAAa,CAAC,QAAQ,EAAE,IAAI,CAAC,SAAS,CAAC,IAAI,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC;IACzD,CAAC;IAAC,MAAM,CAAC;QACP,YAAY;IACd,CAAC;AACH,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,YAAY;IAC1B,iBAAiB,EAAE,CAAC;IACpB,MAAM,QAAQ,GAAsB,EAAE,CAAC;IACvC,IAAI,CAAC;QACH,MAAM,OAAO,GAAG,WAAW,CAAC,YAAY,EAAE,EAAE,aAAa,EAAE,IAAI,EAAE,CAAC,CAAC;QACnE,KAAK,MAAM,KAAK,IAAI,OAAO,EAAE,CAAC;YAC5B,IAAI,CAAC,KAAK,CAAC,WAAW,EAAE;gBAAE,SAAS;YACnC,MAAM,QAAQ,GAAG,IAAI,CAAC,IAAI,CAAC,YAAY,EAAE,KAAK,CAAC,IAAI,EAAE,eAAe,CAAC,CAAC;YACtE,IAAI,CAAC,UAAU,CAAC,QAAQ,CAAC;gBAAE,SAAS;YACpC,IAAI,CAAC;gBACH,MAAM,IAAI,GAAoB,IAAI,CAAC,KAAK,CAAC,YAAY,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC,CAAC;gBAC1E,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YACtB,CAAC;YAAC,MAAM,CAAC;gBACP,0BAA0B;YAC5B,CAAC;QACH,CAAC;IACH,CAAC;IAAC,MAAM,CAAC;QACP,wBAAwB;IAC1B,CAAC;IACD,iCAAiC;IACjC,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,aAAa,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC;IAC9D,OAAO,QAAQ,CAAC;AAClB,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,aAAa,CAAC,IAAY;IACxC,MAAM,GAAG,GAAG,IAAI,CAAC,IAAI,CAAC,YAAY,EAAE,IAAI,CAAC,CAAC;IAC1C,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC;QAAE,OAAO,KAAK,CAAC;IACnC,IAAI,CAAC;QACH,MAAM,CAAC,GAAG,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;QAC9C,OAAO,IAAI,CAAC;IACd,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,KAAK,CAAC;IACf,CAAC;AACH,CAAC;AAED,iFAAiF;AAEjF;;;;;GAKG;AACH,MAAM,CAAC,KAAK,UAAU,aAAa,CAAC,IAAY,EAAE,WAAoB;IACpE,IAAI,CAAC,kBAAkB,CAAC,IAAI,CAAC,EAAE,CAAC;QAC9B,MAAM,IAAI,KAAK,CACb,yBAAyB,IAAI,8EAA8E,CAC5G,CAAC;IACJ,CAAC;IAED,iBAAiB,EAAE,CAAC;IAEpB,MAAM,UAAU,GAAG,IAAI,CAAC,IAAI,CAAC,YAAY,EAAE,IAAI,CAAC,CAAC;IACjD,IAAI,UAAU,CAAC,UAAU,CAAC,EAAE,CAAC;QAC3B,MAAM,IAAI,KAAK,CACb,YAAY,IAAI,qEAAqE,IAAI,EAAE,CAC5F,CAAC;IACJ,CAAC;IAED,SAAS,CAAC,UAAU,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IAE3C,wEAAwE;IACxE,MAAM,OAAO,GAAG,MAAM,QAAQ,CAAC,MAAM,CAAC,EAAE,QAAQ,EAAE,KAAK,EAAE,CAAC,CAAC;IAC3D,MAAM,OAAO,GAAG,MAAM,OAAO,CAAC,UAAU,EAAE,CAAC;IAC3C,MAAM,IAAI,GAAG,MAAM,OAAO,CAAC,OAAO,EAAE,CAAC;IACrC,MAAM,IAAI,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,GAAE,CAAC,CAAC,CAAC;IAE/C,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;IAChB,OAAO,CAAC,GAAG,CAAC,0DAA0D,CAAC,CAAC;IACxE,OAAO,CAAC,GAAG,CAAC,8BAA8B,IAAI,GAAG,CAAC,CAAC;IACnD,OAAO,CAAC,GAAG,CAAC,0DAA0D,CAAC,CAAC;IACxE,OAAO,CAAC,GAAG,CAAC,yDAAyD,CAAC,CAAC;IACvE,OAAO,CAAC,GAAG,CAAC,yDAAyD,CAAC,CAAC;IACvE,OAAO,CAAC,GAAG,CAAC,0DAA0D,CAAC,CAAC;IACxE,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;IAEhB,IAAI,KAAK,GAAG,KAAK,CAAC;IAElB,MAAM,YAAY,GAAG,KAAK,IAAmB,EAAE;QAC7C,IAAI,KAAK;YAAE,OAAO;QAClB,KAAK,GAAG,IAAI,CAAC;QAEb,OAAO,CAAC,GAAG,CAAC,gCAAgC,CAAC,CAAC;QAE9C,IAAI,CAAC;YACH,MAAM,YAAY,GAAG,MAAM,OAAO,CAAC,YAAY,EAAE,CAAC;YAElD,aAAa,CACX,IAAI,CAAC,IAAI,CAAC,UAAU,EAAE,oBAAoB,CAAC,EAC3C,IAAI,CAAC,SAAS,CAAC,YAAY,EAAE,IAAI,EAAE,CAAC,CAAC,CACtC,CAAC;YAEF,0DAA0D;YAC1D,MAAM,OAAO,GAAa;gBACxB,GAAG,IAAI,GAAG,CACR,CAAC,YAAY,CAAC,OAAO,IAAI,EAAE,CAAC;qBACzB,GAAG,CAAC,CAAC,CAAM,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,MAAM,IAAI,EAAE,CAAC,CAAC,OAAO,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC;qBACpD,MAAM,CAAC,OAAO,CAAC,CACnB;aACF,CAAC;YAEF,MAAM,GAAG,GAAG,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;YACrC,MAAM,IAAI,GAAoB;gBAC5B,IAAI;gBACJ,OAAO,EAAE,GAAG;gBACZ,QAAQ,EAAE,GAAG;gBACb,OAAO;gBACP,GAAG,CAAC,WAAW,CAAC,CAAC,CAAC,EAAE,WAAW,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;aACxC,CAAC;YAEF,aAAa,CACX,IAAI,CAAC,IAAI,CAAC,UAAU,EAAE,eAAe,CAAC,EACtC,IAAI,CAAC,SAAS,CAAC,IAAI,EAAE,IAAI,EAAE,CAAC,CAAC,CAC9B,CAAC;YAEF,OAAO,CAAC,GAAG,CAAC,cAAc,IAAI,cAAc,UAAU,EAAE,CAAC,CAAC;YAC1D,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBACvB,OAAO,CAAC,GAAG,CAAC,cAAc,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;YAClD,CAAC;iBAAM,CAAC;gBACN,OAAO,CAAC,GAAG,CAAC,4CAA4C,CAAC,CAAC;YAC5D,CAAC;QACH,CAAC;QAAC,OAAO,CAAC,EAAE,CAAC;YACX,OAAO,CAAC,KAAK,CACX,wCAAwC,EACxC,CAAC,YAAY,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAC3C,CAAC;YACF,6BAA6B;YAC7B,IAAI,CAAC;gBACH,MAAM,CAAC,UAAU,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;YACvD,CAAC;YAAC,MAAM,CAAC;gBACP,YAAY;YACd,CAAC;QACH,CAAC;QAED,IAAI,CAAC;YACH,MAAM,OAAO,CAAC,KAAK,EAAE,CAAC;QACxB,CAAC;QAAC,MAAM,CAAC;YACP,4CAA4C;QAC9C,CAAC;IACH,CAAC,CAAC;IAEF,iFAAiF;IACjF,MAAM,IAAI,OAAO,CAAO,CAAC,OAAO,EAAE,EAAE;QAClC,OAAO,CAAC,EAAE,CAAC,cAAc,EAAE,KAAK,IAAI,EAAE;YACpC,MAAM,YAAY,EAAE,CAAC;YACrB,OAAO,EAAE,CAAC;QACZ,CAAC,CAAC,CAAC;QAEH,2BAA2B;QAC3B,MAAM,aAAa,GAAG,KAAK,IAAI,EAAE;YAC/B,MAAM,YAAY,EAAE,CAAC;YACrB,OAAO,EAAE,CAAC;QACZ,CAAC,CAAC;QACF,OAAO,CAAC,IAAI,CAAC,QAAQ,EAAE,aAAa,CAAC,CAAC;QAEtC,sDAAsD;QACtD,OAAO,CAAC,EAAE,CAAC,cAAc,EAAE,GAAG,EAAE;YAC9B,OAAO,CAAC,cAAc,CAAC,QAAQ,EAAE,aAAa,CAAC,CAAC;QAClD,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;AACL,CAAC"}

package/dist/core/schema-extraction.d.ts

@@ -0,0 +1,67 @@
+/**
+ * Schema-based extraction using CSS selectors.
+ *
+ * Each schema defines how to extract listings from a specific domain,
+ * inspired by Crawl4AI's JsonCssExtractionStrategy. Unlike generic
+ * auto-detection, schemas provide exact selectors for each site's DOM.
+ *
+ * @module schema-extraction
+ */
+export interface SchemaField {
+    /** Field name in output (e.g., "title", "price", "rating") */
+    name: string;
+    /** CSS selector relative to baseSelector. Empty string selects the base element itself. */
+    selector: string;
+    /** What to extract */
+    type: 'text' | 'attribute' | 'html' | 'exists';
+    /** For type='attribute', which attribute to read */
+    attribute?: string;
+    /** Extract all matches (returns array instead of first match) */
+    multiple?: boolean;
+    /** Optional transform to apply after extraction */
+    transform?: 'trim' | 'number' | 'stripCurrency';
+}
+export interface ExtractionSchema {
+    /** Human-readable schema name (e.g., "Booking.com Hotel Search") */
+    name: string;
+    /** Schema version string */
+    version: string;
+    /** Matching domains (e.g., ["booking.com", "www.booking.com"]) */
+    domains: string[];
+    /** Optional URL path patterns (regex strings) for more specific matching */
+    urlPatterns?: string[];
+    /** CSS selector for each listing item */
+    baseSelector: string;
+    /** Fields to extract from each item */
+    fields: SchemaField[];
+    /** Optional pagination config */
+    pagination?: {
+        nextSelector?: string;
+        pageParam?: string;
+    };
+}
+/** A single extracted item — field names map to extracted values */
+export interface ExtractedItem {
+    [key: string]: string | string[] | boolean | number | undefined;
+}
+/**
+ * Load all bundled schemas.
+ */
+export declare function loadBundledSchemas(): ExtractionSchema[];
+/**
+ * Find a matching schema for a given URL.
+ *
+ * Matches by domain first, then optionally by URL patterns (regex).
+ * Returns the first matching schema or null.
+ */
+export declare function findSchemaForUrl(url: string): ExtractionSchema | null;
+/**
+ * Extract listings from HTML using a schema's CSS selectors.
+ *
+ * @param html    - Raw HTML string to parse
+ * @param schema  - Extraction schema to use
+ * @param baseUrl - Optional base URL for resolving relative links
+ * @returns Array of extracted items (may be empty)
+ */
+export declare function extractWithSchema(html: string, schema: ExtractionSchema, baseUrl?: string): ExtractedItem[];
+//# sourceMappingURL=schema-extraction.d.ts.map

package/dist/core/schema-extraction.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"schema-extraction.d.ts","sourceRoot":"","sources":["../../src/core/schema-extraction.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AASH,MAAM,WAAW,WAAW;IAC1B,8DAA8D;IAC9D,IAAI,EAAE,MAAM,CAAC;IACb,2FAA2F;IAC3F,QAAQ,EAAE,MAAM,CAAC;IACjB,sBAAsB;IACtB,IAAI,EAAE,MAAM,GAAG,WAAW,GAAG,MAAM,GAAG,QAAQ,CAAC;IAC/C,oDAAoD;IACpD,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,iEAAiE;IACjE,QAAQ,CAAC,EAAE,OAAO,CAAC;IACnB,mDAAmD;IACnD,SAAS,CAAC,EAAE,MAAM,GAAG,QAAQ,GAAG,eAAe,CAAC;CACjD;AAED,MAAM,WAAW,gBAAgB;IAC/B,oEAAoE;IACpE,IAAI,EAAE,MAAM,CAAC;IACb,4BAA4B;IAC5B,OAAO,EAAE,MAAM,CAAC;IAChB,kEAAkE;IAClE,OAAO,EAAE,MAAM,EAAE,CAAC;IAClB,4EAA4E;IAC5E,WAAW,CAAC,EAAE,MAAM,EAAE,CAAC;IACvB,yCAAyC;IACzC,YAAY,EAAE,MAAM,CAAC;IACrB,uCAAuC;IACvC,MAAM,EAAE,WAAW,EAAE,CAAC;IACtB,iCAAiC;IACjC,UAAU,CAAC,EAAE;QACX,YAAY,CAAC,EAAE,MAAM,CAAC;QACtB,SAAS,CAAC,EAAE,MAAM,CAAC;KACpB,CAAC;CACH;AAED,oEAAoE;AACpE,MAAM,WAAW,aAAa;IAC5B,CAAC,GAAG,EAAE,MAAM,GAAG,MAAM,GAAG,MAAM,EAAE,GAAG,OAAO,GAAG,MAAM,GAAG,SAAS,CAAC;CACjE;AA4PD;;GAEG;AACH,wBAAgB,kBAAkB,IAAI,gBAAgB,EAAE,CAEvD;AAED;;;;;GAKG;AACH,wBAAgB,gBAAgB,CAAC,GAAG,EAAE,MAAM,GAAG,gBAAgB,GAAG,IAAI,CAsCrE;AAED;;;;;;;GAOG;AACH,wBAAgB,iBAAiB,CAC/B,IAAI,EAAE,MAAM,EACZ,MAAM,EAAE,gBAAgB,EACxB,OAAO,CAAC,EAAE,MAAM,GACf,aAAa,EAAE,CA0DjB"}

package/dist/core/schema-extraction.js

@@ -0,0 +1,353 @@
+/**
+ * Schema-based extraction using CSS selectors.
+ *
+ * Each schema defines how to extract listings from a specific domain,
+ * inspired by Crawl4AI's JsonCssExtractionStrategy. Unlike generic
+ * auto-detection, schemas provide exact selectors for each site's DOM.
+ *
+ * @module schema-extraction
+ */
+import { load } from 'cheerio';
+/* ------------------------------------------------------------------ */
+/*  Bundled schemas (hardcoded to avoid JSON import complications)     */
+/* ------------------------------------------------------------------ */
+const BOOKING_COM_SCHEMA = {
+    name: 'Booking.com Hotel Search',
+    version: '1.0',
+    domains: ['booking.com', 'www.booking.com'],
+    urlPatterns: ['searchresults'],
+    baseSelector: "[data-testid='property-card']",
+    fields: [
+        { name: 'title', selector: "[data-testid='title'], .sr-hotel__name, h3 a", type: 'text' },
+        { name: 'price', selector: "[data-testid='price-and-discounted-price'], .bui-price-display__value, [data-testid='price-for-x-nights']", type: 'text', transform: 'trim' },
+        { name: 'rating', selector: "[data-testid='review-score'] div:first-child, .bui-review-score__badge", type: 'text' },
+        { name: 'reviewCount', selector: "[data-testid='review-score'] div:nth-child(2) div:nth-child(2), .bui-review-score__text", type: 'text' },
+        { name: 'location', selector: "[data-testid='address'], .sr_card_address_line", type: 'text' },
+        { name: 'link', selector: "a[data-testid='title-link'], h3 a, a.hotel_name_link", type: 'attribute', attribute: 'href' },
+        { name: 'image', selector: "img[data-testid='image'], img.hotel_image", type: 'attribute', attribute: 'src' },
+        { name: 'stars', selector: "[data-testid='rating-stars'] span, .bui-star-rating .bui-star-rating__star", type: 'text' },
+    ],
+};
+const AMAZON_COM_SCHEMA = {
+    name: 'Amazon Product Search',
+    version: '1.0',
+    domains: ['amazon.com', 'www.amazon.com', 'amazon.co.uk', 'amazon.de', 'amazon.fr', 'amazon.ca'],
+    urlPatterns: ['/s\\?', '/s/'],
+    baseSelector: "[data-component-type='s-search-result']",
+    fields: [
+        { name: 'title', selector: 'h2 a span, h2 span a span', type: 'text' },
+        { name: 'price', selector: '.a-price .a-offscreen', type: 'text' },
+        { name: 'originalPrice', selector: '.a-price.a-text-price .a-offscreen', type: 'text' },
+        { name: 'rating', selector: '.a-icon-star-small .a-icon-alt, .a-icon-star-mini .a-icon-alt', type: 'text' },
+        { name: 'reviewCount', selector: "[data-csa-c-func-deps='aui-da-a-popover'] ~ span span, .a-size-base.s-underline-text", type: 'text' },
+        { name: 'link', selector: 'h2 a', type: 'attribute', attribute: 'href' },
+        { name: 'image', selector: '.s-image', type: 'attribute', attribute: 'src' },
+        { name: 'sponsored', selector: '.puis-sponsored-label-text', type: 'exists' },
+        { name: 'asin', selector: '', type: 'attribute', attribute: 'data-asin' },
+    ],
+};
+const EBAY_COM_SCHEMA = {
+    name: 'eBay Search Results',
+    version: '1.0',
+    domains: ['ebay.com', 'www.ebay.com'],
+    urlPatterns: ['/sch/'],
+    baseSelector: '.s-item, [data-viewport]',
+    fields: [
+        { name: 'title', selector: '.s-item__title span, .s-item__title', type: 'text' },
+        { name: 'price', selector: '.s-item__price', type: 'text' },
+        { name: 'link', selector: '.s-item__link, a.s-item__link', type: 'attribute', attribute: 'href' },
+        { name: 'image', selector: '.s-item__image-wrapper img, .s-item__image img', type: 'attribute', attribute: 'src' },
+        { name: 'condition', selector: '.SECONDARY_INFO', type: 'text' },
+        { name: 'shipping', selector: '.s-item__shipping, .s-item__freeXDays', type: 'text' },
+        { name: 'seller', selector: '.s-item__seller-info-text', type: 'text' },
+    ],
+};
+const YELP_COM_SCHEMA = {
+    name: 'Yelp Business Search',
+    version: '1.0',
+    domains: ['yelp.com', 'www.yelp.com'],
+    urlPatterns: ['/search'],
+    baseSelector: "[data-testid='serp-ia-card'], li.border-color--default",
+    fields: [
+        { name: 'title', selector: "a[href*='/biz/'] span, h3 a span", type: 'text' },
+        { name: 'rating', selector: "[aria-label*='star rating'], .i-stars", type: 'attribute', attribute: 'aria-label' },
+        { name: 'reviewCount', selector: ".reviewCount, span[class*='css-']", type: 'text' },
+        { name: 'price', selector: '.priceRange, span.priceRange', type: 'text' },
+        { name: 'category', selector: ".priceCategory span, p[class*='css-'] a", type: 'text' },
+        { name: 'link', selector: "a[href*='/biz/']", type: 'attribute', attribute: 'href' },
+        { name: 'address', selector: "address, span[class*='css-']", type: 'text' },
+    ],
+};
+const WALMART_COM_SCHEMA = {
+    name: 'Walmart Product Search',
+    version: '1.0',
+    domains: ['walmart.com', 'www.walmart.com'],
+    urlPatterns: ['/search'],
+    baseSelector: "[data-testid='list-view'] > div, [data-item-id]",
+    fields: [
+        { name: 'title', selector: "a[link-identifier] span, [data-automation-id='product-title']", type: 'text' },
+        { name: 'price', selector: "[data-automation-id='product-price'] .f2, [itemprop='price']", type: 'text' },
+        { name: 'rating', selector: "[data-testid='product-ratings'] .w_iUH7, .stars-reviews-count", type: 'text' },
+        { name: 'link', selector: "a[link-identifier], a[href*='/ip/']", type: 'attribute', attribute: 'href' },
+        { name: 'image', selector: "img[data-testid='productTileImage'], img[loading]", type: 'attribute', attribute: 'src' },
+        { name: 'seller', selector: "[data-automation-id='fulfillment-badge']", type: 'text' },
+    ],
+};
+const HACKERNEWS_SCHEMA = {
+    name: 'Hacker News',
+    version: '1.0',
+    domains: ['news.ycombinator.com'],
+    baseSelector: 'tr.athing',
+    fields: [
+        { name: 'title', selector: '.titleline a', type: 'text' },
+        { name: 'link', selector: '.titleline a', type: 'attribute', attribute: 'href' },
+        { name: 'rank', selector: '.rank', type: 'text' },
+        { name: 'site', selector: '.sitestr', type: 'text' },
+    ],
+};
+const EXPEDIA_COM_SCHEMA = {
+    name: 'Expedia Hotel Search',
+    version: '1.0',
+    domains: ['expedia.com', 'www.expedia.com'],
+    urlPatterns: ['Hotel-Search', 'hotel-search'],
+    baseSelector: "[data-stid='property-listing'], li.uitk-spacing[class*='uitk-spacing'], [data-stid='lodging-card-responsive']",
+    fields: [
+        { name: 'title', selector: "[data-stid='content-hotel-title'], .uitk-heading-5, .uitk-heading-6, h3[class*='uitk-heading']", type: 'text' },
+        { name: 'price', selector: "[data-stid='price-summary'] .uitk-type-500, [data-stid='price-summary-message-total'], .uitk-type-500", type: 'text', transform: 'trim' },
+        { name: 'rating', selector: "[data-stid='star-rating-msg'], .uitk-badge-base, [aria-label*='out of']", type: 'text' },
+        { name: 'reviewCount', selector: "[data-stid='review-info-text'], .uitk-type-200", type: 'text' },
+        { name: 'location', selector: "[data-stid='location-info'], [data-stid='neighborhood-name']", type: 'text' },
+        { name: 'link', selector: "a[data-stid='open-hotel-information'], a[href*='/h/'], a.uitk-card-link", type: 'attribute', attribute: 'href' },
+        { name: 'image', selector: "img[data-stid='image'], .uitk-image-media img", type: 'attribute', attribute: 'src' },
+    ],
+};
+/** All bundled schemas in priority order */
+const BUNDLED_SCHEMAS = [
+    BOOKING_COM_SCHEMA,
+    AMAZON_COM_SCHEMA,
+    EBAY_COM_SCHEMA,
+    YELP_COM_SCHEMA,
+    WALMART_COM_SCHEMA,
+    HACKERNEWS_SCHEMA,
+    EXPEDIA_COM_SCHEMA,
+];
+/* ------------------------------------------------------------------ */
+/*  Helpers                                                             */
+/* ------------------------------------------------------------------ */
+/**
+ * Apply a transform to an extracted string value.
+ */
+function applyTransform(value, transform) {
+    if (!transform)
+        return value;
+    switch (transform) {
+        case 'trim':
+            return value.trim();
+        case 'number': {
+            const num = parseFloat(value.replace(/[^\d.]/g, ''));
+            return isNaN(num) ? value : num;
+        }
+        case 'stripCurrency':
+            return value.replace(/[^\d.,]/g, '').trim();
+        default:
+            return value;
+    }
+}
+/**
+ * Resolve a potentially relative URL against a base URL.
+ */
+function resolveUrl(href, baseUrl) {
+    if (!href)
+        return undefined;
+    if (href.startsWith('data:') || href.startsWith('javascript:'))
+        return undefined;
+    if (!baseUrl)
+        return href;
+    try {
+        return new URL(href, baseUrl).href;
+    }
+    catch {
+        return href;
+    }
+}
+/**
+ * Extract a single field value from a cheerio element.
+ */
+function extractFieldValue($, $el, field, baseUrl) {
+    // For empty selector on attribute type, read from the base element itself
+    const useBaseEl = field.selector === '' || field.selector.trim() === '';
+    if (field.multiple && !useBaseEl) {
+        // Collect all matches
+        const results = [];
+        $el.find(field.selector).each((_, el) => {
+            const $match = $(el);
+            let val;
+            switch (field.type) {
+                case 'text':
+                    val = $match.text().trim();
+                    break;
+                case 'attribute':
+                    val = field.attribute ? ($match.attr(field.attribute) ?? undefined) : undefined;
+                    if (field.attribute === 'href' || field.attribute === 'src') {
+                        val = resolveUrl(val, baseUrl);
+                    }
+                    break;
+                case 'html':
+                    val = $match.html() ?? undefined;
+                    break;
+                case 'exists':
+                    // not meaningful for multiple
+                    break;
+            }
+            if (val !== undefined && val !== '')
+                results.push(val);
+        });
+        return results.length > 0 ? results : undefined;
+    }
+    // Single match mode
+    const $target = useBaseEl ? $el : $el.find(field.selector).first();
+    switch (field.type) {
+        case 'exists':
+            return useBaseEl ? true : $el.find(field.selector).length > 0;
+        case 'text': {
+            if (!useBaseEl && $target.length === 0)
+                return undefined;
+            const text = $target.text().trim();
+            if (text === '')
+                return undefined;
+            const transformed = applyTransform(text, field.transform);
+            return transformed;
+        }
+        case 'attribute': {
+            if (!field.attribute)
+                return undefined;
+            const attrVal = $target.attr(field.attribute) ?? undefined;
+            if (attrVal === undefined)
+                return undefined;
+            if (field.attribute === 'href' || field.attribute === 'src') {
+                const resolved = resolveUrl(attrVal, baseUrl);
+                if (!resolved)
+                    return undefined;
+                return applyTransform(resolved, field.transform);
+            }
+            return applyTransform(attrVal, field.transform);
+        }
+        case 'html': {
+            if (!useBaseEl && $target.length === 0)
+                return undefined;
+            return $target.html() ?? undefined;
+        }
+        default:
+            return undefined;
+    }
+}
+/* ------------------------------------------------------------------ */
+/*  Public API                                                          */
+/* ------------------------------------------------------------------ */
+/**
+ * Load all bundled schemas.
+ */
+export function loadBundledSchemas() {
+    return [...BUNDLED_SCHEMAS];
+}
+/**
+ * Find a matching schema for a given URL.
+ *
+ * Matches by domain first, then optionally by URL patterns (regex).
+ * Returns the first matching schema or null.
+ */
+export function findSchemaForUrl(url) {
+    let parsed;
+    try {
+        parsed = new URL(url);
+    }
+    catch {
+        return null;
+    }
+    const hostname = parsed.hostname.toLowerCase();
+    const fullUrl = url;
+    for (const schema of BUNDLED_SCHEMAS) {
+        // Check domain match
+        const domainMatch = schema.domains.some(domain => {
+            const d = domain.toLowerCase();
+            return hostname === d || hostname.endsWith('.' + d) || d.endsWith('.' + hostname);
+        });
+        if (!domainMatch)
+            continue;
+        // If no urlPatterns, domain match is enough
+        if (!schema.urlPatterns || schema.urlPatterns.length === 0) {
+            return schema;
+        }
+        // Check URL patterns against the full URL
+        const patternMatch = schema.urlPatterns.some(pattern => {
+            try {
+                return new RegExp(pattern).test(fullUrl);
+            }
+            catch {
+                return false;
+            }
+        });
+        if (patternMatch)
+            return schema;
+    }
+    return null;
+}
+/**
+ * Extract listings from HTML using a schema's CSS selectors.
+ *
+ * @param html    - Raw HTML string to parse
+ * @param schema  - Extraction schema to use
+ * @param baseUrl - Optional base URL for resolving relative links
+ * @returns Array of extracted items (may be empty)
+ */
+export function extractWithSchema(html, schema, baseUrl) {
+    if (!html || html.trim().length === 0)
+        return [];
+    const $ = load(html);
+    const items = [];
+    // Find the title/name field to use for filtering empty items
+    const titleFieldName = schema.fields.find(f => f.name === 'title' || f.name === 'name')?.name;
+    $(schema.baseSelector).each((_, el) => {
+        const $el = $(el);
+        const item = {};
+        for (const field of schema.fields) {
+            const value = extractFieldValue($, $el, field, baseUrl);
+            if (value !== undefined) {
+                item[field.name] = value;
+            }
+        }
+        // Clean title/name field: strip common junk suffixes (e.g., "Opens in new window")
+        if (titleFieldName !== undefined && typeof item[titleFieldName] === 'string') {
+            let title = item[titleFieldName];
+            // Strip "Opens in (a) new window/tab" variants
+            title = title.replace(/\s*Opens?\s+in\s+(?:a\s+)?new\s+(?:window|tab)(?:\s+or\s+(?:window|tab))?/gi, '');
+            // Strip "New Listing", "Sponsored", "Ad" prefixes
+            title = title.replace(/^(?:New\s+Listing|Sponsored|Ad)\s*[-–—:·]?\s*/i, '');
+            item[titleFieldName] = title.trim();
+        }
+        // Skip items with no title/name (likely empty/phantom elements)
+        if (titleFieldName !== undefined) {
+            const titleVal = item[titleFieldName];
+            if (!titleVal || (typeof titleVal === 'string' && titleVal.trim() === '')) {
+                return; // skip
+            }
+        }
+        // Skip completely empty items
+        if (Object.keys(item).length === 0)
+            return;
+        items.push(item);
+    });
+    // Deduplicate: remove items with identical title + price (common with nested selectors)
+    if (titleFieldName) {
+        const seen = new Set();
+        return items.filter(item => {
+            const key = `${String(item[titleFieldName] ?? '')}|${String(item.price ?? '')}`;
+            if (seen.has(key))
+                return false;
+            seen.add(key);
+            return true;
+        });
+    }
+    return items;
+}
+//# sourceMappingURL=schema-extraction.js.map