webpeel 0.9.0 → 0.11.0

package/dist/core/llm-extract.js

@@ -0,0 +1,264 @@
+/**
+ * LLM-based extraction: sends markdown/text content to an LLM
+ * with instructions to extract structured data.
+ *
+ * Supports OpenAI-compatible APIs (OpenAI, Anthropic via proxy, local models).
+ */
+// Cost per 1M tokens (input, output) for known models
+const MODEL_COSTS = {
+    'gpt-4o-mini': [0.15, 0.60],
+    'gpt-4o': [2.50, 10.0],
+};
+const GENERIC_SYSTEM_PROMPT = `You are a data extraction assistant. Extract structured data from the provided web content.
+Return a JSON array of objects. Each object represents one item/listing found on the page.
+Always include these fields when available: title, price, link, rating, description, image.
+If the user provides additional instructions, follow them.
+Return ONLY valid JSON — no markdown, no explanation, just the array.`;
+const SCHEMA_SYSTEM_PROMPT = `You are a data extraction assistant. Extract structured data from the web content below.
+Return a JSON object that EXACTLY matches the provided schema structure.
+Fill in the values from the page content. Use null for fields you can't find.
+Return ONLY valid JSON matching the schema — no markdown, no explanation.`;
+/**
+ * Detect if schema is a "full" JSON Schema (has type:"object" and properties).
+ */
+export function isFullJsonSchema(schema) {
+    const s = schema;
+    return s['type'] === 'object' && typeof s['properties'] === 'object';
+}
+/**
+ * Convert a simple example object to a proper JSON Schema.
+ *
+ * Supports:
+ *   - Primitive values: "" → { type: "string" }, 0 → { type: "number" }
+ *   - Arrays of objects: [{name:"", price:""}] → { type: "array", items: { type: "object", properties: {...} } }
+ *   - Nested objects
+ */
+export function convertSimpleToJsonSchema(example) {
+    return buildSchemaFromValue(example);
+}
+function buildSchemaFromValue(value) {
+    if (value === null || value === undefined) {
+        return { type: 'string' };
+    }
+    if (typeof value === 'string') {
+        return { type: 'string' };
+    }
+    if (typeof value === 'number') {
+        return Number.isInteger(value) ? { type: 'integer' } : { type: 'number' };
+    }
+    if (typeof value === 'boolean') {
+        return { type: 'boolean' };
+    }
+    if (Array.isArray(value)) {
+        if (value.length === 0) {
+            return { type: 'array', items: {} };
+        }
+        // Use the first element as the template for item schema
+        const itemSchema = buildSchemaFromValue(value[0]);
+        return { type: 'array', items: itemSchema };
+    }
+    if (typeof value === 'object') {
+        const obj = value;
+        const properties = {};
+        for (const [key, val] of Object.entries(obj)) {
+            properties[key] = buildSchemaFromValue(val);
+        }
+        return {
+            type: 'object',
+            properties,
+        };
+    }
+    return { type: 'string' };
+}
+/**
+ * Build the user message from content + optional instruction + optional schema.
+ */
+export function buildUserMessage(content, instruction, schema) {
+    // Truncate content if over 100K chars
+    const truncated = content.length > 100_000 ? content.slice(0, 50_000) : content;
+    let msg = `Here is the web content to extract data from:\n\n${truncated}`;
+    if (schema) {
+        msg += `\n\nExtract data matching this schema: ${JSON.stringify(schema, null, 2)}`;
+    }
+    if (instruction) {
+        msg += `\n\nAdditional instruction: ${instruction}`;
+    }
+    return msg;
+}
+/**
+ * Calculate estimated cost in USD for a given model and token counts.
+ */
+export function estimateCost(model, inputTokens, outputTokens) {
+    // Normalize model key (strip version suffixes like -2024-11-20 for matching)
+    const key = Object.keys(MODEL_COSTS).find(k => model.startsWith(k) || model === k);
+    if (!key)
+        return undefined;
+    const [inputRate, outputRate] = MODEL_COSTS[key];
+    return (inputTokens / 1_000_000) * inputRate + (outputTokens / 1_000_000) * outputRate;
+}
+/**
+ * Parse the LLM response text into an items array.
+ * Handles both `{ "items": [...] }` and `[...]` formats.
+ * When a schema is provided, also handles single-object responses.
+ */
+export function parseItems(text, _schema) {
+    const trimmed = text.trim();
+    // Try to parse as-is first
+    let parsed;
+    try {
+        parsed = JSON.parse(trimmed);
+    }
+    catch {
+        // Try to extract JSON from the text (sometimes LLMs add preamble despite instructions)
+        const arrayMatch = trimmed.match(/\[[\s\S]*\]/);
+        const objMatch = trimmed.match(/\{[\s\S]*\}/);
+        if (arrayMatch) {
+            try {
+                parsed = JSON.parse(arrayMatch[0]);
+            }
+            catch { /* fall through */ }
+        }
+        else if (objMatch) {
+            try {
+                parsed = JSON.parse(objMatch[0]);
+            }
+            catch { /* fall through */ }
+        }
+        if (parsed === undefined) {
+            throw new Error(`Failed to parse LLM response as JSON: ${trimmed.slice(0, 200)}`);
+        }
+    }
+    // Handle { items: [...] } or { data: [...] } or { results: [...] }
+    if (parsed && typeof parsed === 'object' && !Array.isArray(parsed)) {
+        const obj = parsed;
+        if (Array.isArray(obj['items']))
+            return obj['items'];
+        if (Array.isArray(obj['data']))
+            return obj['data'];
+        if (Array.isArray(obj['results']))
+            return obj['results'];
+        // Single object — wrap in array
+        return [obj];
+    }
+    // Handle bare array
+    if (Array.isArray(parsed)) {
+        return parsed;
+    }
+    return [];
+}
+/**
+ * Validate that a parsed result roughly matches the expected schema shape.
+ * Logs a warning if the top-level keys don't match, but returns the result anyway.
+ */
+function validateSchemaShape(result, schema) {
+    if (result.length === 0)
+        return;
+    const schemaObj = schema;
+    // For full JSON Schema: check that the object has the expected top-level properties
+    if (isFullJsonSchema(schema)) {
+        const expectedKeys = Object.keys(schemaObj['properties'] || {});
+        if (expectedKeys.length > 0 && result[0]) {
+            const actualKeys = Object.keys(result[0]);
+            const missingKeys = expectedKeys.filter(k => !actualKeys.includes(k));
+            if (missingKeys.length > 0) {
+                console.warn(`[webpeel] Schema validation warning: response missing expected keys: ${missingKeys.join(', ')}`);
+            }
+        }
+        return;
+    }
+    // For simple example schema: check top-level keys exist
+    const expectedTopLevelKeys = Object.keys(schemaObj);
+    if (expectedTopLevelKeys.length > 0 && result[0]) {
+        const actualKeys = Object.keys(result[0]);
+        const missingKeys = expectedTopLevelKeys.filter(k => !actualKeys.includes(k));
+        if (missingKeys.length > 0) {
+            console.warn(`[webpeel] Schema validation warning: response missing expected keys: ${missingKeys.join(', ')}`);
+        }
+    }
+}
+/**
+ * Build the response_format parameter for the OpenAI API call.
+ */
+function buildResponseFormat(schema) {
+    if (!schema) {
+        return { type: 'json_object' };
+    }
+    // Use structured output only for full JSON Schema (has type:"object" and properties)
+    if (isFullJsonSchema(schema)) {
+        return {
+            type: 'json_schema',
+            json_schema: {
+                name: 'extraction',
+                strict: true,
+                schema,
+            },
+        };
+    }
+    // For simple example schemas, fall back to json_object
+    return { type: 'json_object' };
+}
+/**
+ * Extract structured data from content using an LLM.
+ */
+export async function extractWithLLM(options) {
+    const { content, instruction, baseUrl = 'https://api.openai.com/v1', model = 'gpt-4o-mini', maxTokens = 4000, } = options;
+    const apiKey = options.apiKey || process.env.OPENAI_API_KEY;
+    if (!apiKey) {
+        throw new Error('LLM extraction requires an API key.\n' +
+            'Set OPENAI_API_KEY environment variable or use --llm-key <key>');
+    }
+    // Resolve schema: convert simple schemas to full JSON Schema if needed
+    let resolvedSchema = options.schema;
+    if (resolvedSchema && !isFullJsonSchema(resolvedSchema)) {
+        resolvedSchema = convertSimpleToJsonSchema(resolvedSchema);
+    }
+    // Choose system prompt based on whether a schema is provided
+    const systemPrompt = resolvedSchema ? SCHEMA_SYSTEM_PROMPT : GENERIC_SYSTEM_PROMPT;
+    const userMessage = buildUserMessage(content, instruction, resolvedSchema ?? options.schema);
+    const responseFormat = buildResponseFormat(resolvedSchema);
+    const response = await fetch(`${baseUrl}/chat/completions`, {
+        method: 'POST',
+        headers: {
+            'Content-Type': 'application/json',
+            'Authorization': `Bearer ${apiKey}`,
+        },
+        body: JSON.stringify({
+            model,
+            messages: [
+                { role: 'system', content: systemPrompt },
+                { role: 'user', content: userMessage },
+            ],
+            temperature: 0,
+            max_tokens: maxTokens,
+            response_format: responseFormat,
+        }),
+    });
+    if (!response.ok) {
+        const body = await response.text().catch(() => '');
+        if (response.status === 401) {
+            throw new Error(`LLM API authentication failed (401). Check your API key.`);
+        }
+        if (response.status === 429) {
+            throw new Error(`LLM API rate limit exceeded (429). Please wait and retry.`);
+        }
+        throw new Error(`LLM API error: HTTP ${response.status}${body ? ` — ${body.slice(0, 200)}` : ''}`);
+    }
+    const data = await response.json();
+    const rawText = data.choices?.[0]?.message?.content ?? '';
+    const items = parseItems(rawText, resolvedSchema);
+    // Validate schema shape and warn if mismatch
+    if (resolvedSchema) {
+        validateSchemaShape(items, resolvedSchema);
+    }
+    const inputTokens = data.usage?.prompt_tokens ?? 0;
+    const outputTokens = data.usage?.completion_tokens ?? 0;
+    const resolvedModel = data.model ?? model;
+    const cost = estimateCost(resolvedModel, inputTokens, outputTokens);
+    return {
+        items,
+        tokensUsed: { input: inputTokens, output: outputTokens },
+        model: resolvedModel,
+        cost,
+    };
+}
+//# sourceMappingURL=llm-extract.js.map

package/dist/core/llm-extract.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"llm-extract.js","sourceRoot":"","sources":["../../src/core/llm-extract.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAmBH,sDAAsD;AACtD,MAAM,WAAW,GAAqC;IACpD,aAAa,EAAE,CAAC,IAAI,EAAE,IAAI,CAAC;IAC3B,QAAQ,EAAE,CAAC,IAAI,EAAE,IAAI,CAAC;CACvB,CAAC;AAEF,MAAM,qBAAqB,GAAG;;;;sEAIwC,CAAC;AAEvE,MAAM,oBAAoB,GAAG;;;0EAG6C,CAAC;AAE3E;;GAEG;AACH,MAAM,UAAU,gBAAgB,CAAC,MAAc;IAC7C,MAAM,CAAC,GAAG,MAA6B,CAAC;IACxC,OAAO,CAAC,CAAC,MAAM,CAAC,KAAK,QAAQ,IAAI,OAAO,CAAC,CAAC,YAAY,CAAC,KAAK,QAAQ,CAAC;AACvE,CAAC;AAED;;;;;;;GAOG;AACH,MAAM,UAAU,yBAAyB,CAAC,OAAe;IACvD,OAAO,oBAAoB,CAAC,OAAO,CAAC,CAAC;AACvC,CAAC;AAED,SAAS,oBAAoB,CAAC,KAAc;IAC1C,IAAI,KAAK,KAAK,IAAI,IAAI,KAAK,KAAK,SAAS,EAAE,CAAC;QAC1C,OAAO,EAAE,IAAI,EAAE,QAAQ,EAAE,CAAC;IAC5B,CAAC;IAED,IAAI,OAAO,KAAK,KAAK,QAAQ,EAAE,CAAC;QAC9B,OAAO,EAAE,IAAI,EAAE,QAAQ,EAAE,CAAC;IAC5B,CAAC;IAED,IAAI,OAAO,KAAK,KAAK,QAAQ,EAAE,CAAC;QAC9B,OAAO,MAAM,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,SAAS,EAAE,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,QAAQ,EAAE,CAAC;IAC5E,CAAC;IAED,IAAI,OAAO,KAAK,KAAK,SAAS,EAAE,CAAC;QAC/B,OAAO,EAAE,IAAI,EAAE,SAAS,EAAE,CAAC;IAC7B,CAAC;IAED,IAAI,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE,CAAC;QACzB,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACvB,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,KAAK,EAAE,EAAE,EAAE,CAAC;QACtC,CAAC;QACD,wDAAwD;QACxD,MAAM,UAAU,GAAG,oBAAoB,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC;QAClD,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,KAAK,EAAE,UAAU,EAAE,CAAC;IAC9C,CAAC;IAED,IAAI,OAAO,KAAK,KAAK,QAAQ,EAAE,CAAC;QAC9B,MAAM,GAAG,GAAG,KAAgC,CAAC;QAC7C,MAAM,UAAU,GAA2B,EAAE,CAAC;QAC9C,KAAK,MAAM,CAAC,GAAG,EAAE,GAAG,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC;YAC7C,UAAU,CAAC,GAAG,CAAC,GAAG,oBAAoB,CAAC,GAAG,CAAC,CAAC;QAC9C,CAAC;QACD,OAAO;YACL,IAAI,EAAE,QAAQ;YACd,UAAU;SACX,CAAC;IACJ,CAAC;IAED,OAAO,EAAE,IAAI,EAAE,QAAQ,EAAE,CAAC;AAC5B,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,gBAAgB,CAAC,OAAe,EAAE,WAAoB,EAAE,MAAe;IACrF,sCAAsC;IACtC,MAAM,SAAS,GAAG,OAAO,CAAC,MAAM,GAAG,OAAO,CAAC,CAAC,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,MAAM,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC;IAEhF,IAAI,GAAG,GAAG,oDAAoD,SAAS,EAAE,CAAC;IAE1E,IAAI,MAAM,EAAE,CAAC;QACX,GAAG,IAAI,0CAA0C,IAAI,CAAC,SAAS,CAAC,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC,EAAE,CAAC;IACrF,CAAC;IAED,IAAI,WAAW,EAAE,CAAC;QAChB,GAAG,IAAI,+BAA+B,WAAW,EAAE,CAAC;IACtD,CAAC;IAED,OAAO,GAAG,CAAC;AACb,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,YAAY,CAAC,KAAa,EAAE,WAAmB,EAAE,YAAoB;IACnF,6EAA6E;IAC7E,MAAM,GAAG,GAAG,MAAM,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC,CAAC,IAAI,KAAK,KAAK,CAAC,CAAC,CAAC;IACnF,IAAI,CAAC,GAAG;QAAE,OAAO,SAAS,CAAC;IAC3B,MAAM,CAAC,SAAS,EAAE,UAAU,CAAC,GAAG,WAAW,CAAC,GAAG,CAAE,CAAC;IAClD,OAAO,CAAC,WAAW,GAAG,SAAS,CAAC,GAAG,SAAS,GAAG,CAAC,YAAY,GAAG,SAAS,CAAC,GAAG,UAAU,CAAC;AACzF,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,UAAU,CAAC,IAAY,EAAE,OAAgB;IACvD,MAAM,OAAO,GAAG,IAAI,CAAC,IAAI,EAAE,CAAC;IAE5B,2BAA2B;IAC3B,IAAI,MAAe,CAAC;IACpB,IAAI,CAAC;QACH,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;IAC/B,CAAC;IAAC,MAAM,CAAC;QACP,uFAAuF;QACvF,MAAM,UAAU,GAAG,OAAO,CAAC,KAAK,CAAC,aAAa,CAAC,CAAC;QAChD,MAAM,QAAQ,GAAG,OAAO,CAAC,KAAK,CAAC,aAAa,CAAC,CAAC;QAC9C,IAAI,UAAU,EAAE,CAAC;YACf,IAAI,CAAC;gBAAC,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,CAAC;YAAC,CAAC;YAAC,MAAM,CAAC,CAAC,kBAAkB,CAAC,CAAC;QAC1E,CAAC;aAAM,IAAI,QAAQ,EAAE,CAAC;YACpB,IAAI,CAAC;gBAAC,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC;YAAC,CAAC;YAAC,MAAM,CAAC,CAAC,kBAAkB,CAAC,CAAC;QACxE,CAAC;QACD,IAAI,MAAM,KAAK,SAAS,EAAE,CAAC;YACzB,MAAM,IAAI,KAAK,CAAC,yCAAyC,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,CAAC,CAAC;QACpF,CAAC;IACH,CAAC;IAED,mEAAmE;IACnE,IAAI,MAAM,IAAI,OAAO,MAAM,KAAK,QAAQ,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,CAAC;QACnE,MAAM,GAAG,GAAG,MAA6B,CAAC;QAC1C,IAAI,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;YAAE,OAAO,GAAG,CAAC,OAAO,CAAC,CAAC;QACrD,IAAI,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;YAAE,OAAO,GAAG,CAAC,MAAM,CAAC,CAAC;QACnD,IAAI,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC;YAAE,OAAO,GAAG,CAAC,SAAS,CAAC,CAAC;QACzD,gCAAgC;QAChC,OAAO,CAAC,GAAG,CAAC,CAAC;IACf,CAAC;IAED,oBAAoB;IACpB,IAAI,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,CAAC;QAC1B,OAAO,MAAM,CAAC;IAChB,CAAC;IAED,OAAO,EAAE,CAAC;AACZ,CAAC;AAED;;;GAGG;AACH,SAAS,mBAAmB,CAAC,MAAkC,EAAE,MAAc;IAC7E,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO;IAEhC,MAAM,SAAS,GAAG,MAA6B,CAAC;IAEhD,oFAAoF;IACpF,IAAI,gBAAgB,CAAC,MAAM,CAAC,EAAE,CAAC;QAC7B,MAAM,YAAY,GAAG,MAAM,CAAC,IAAI,CAAC,SAAS,CAAC,YAAY,CAAC,IAAI,EAAE,CAAC,CAAC;QAChE,IAAI,YAAY,CAAC,MAAM,GAAG,CAAC,IAAI,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC;YACzC,MAAM,UAAU,GAAG,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC;YAC1C,MAAM,WAAW,GAAG,YAAY,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,UAAU,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC;YACtE,IAAI,WAAW,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBAC3B,OAAO,CAAC,IAAI,CAAC,wEAAwE,WAAW,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;YACjH,CAAC;QACH,CAAC;QACD,OAAO;IACT,CAAC;IAED,wDAAwD;IACxD,MAAM,oBAAoB,GAAG,MAAM,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;IACpD,IAAI,oBAAoB,CAAC,MAAM,GAAG,CAAC,IAAI,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC;QACjD,MAAM,UAAU,GAAG,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC;QAC1C,MAAM,WAAW,GAAG,oBAAoB,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,UAAU,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC;QAC9E,IAAI,WAAW,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAC3B,OAAO,CAAC,IAAI,CAAC,wEAAwE,WAAW,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QACjH,CAAC;IACH,CAAC;AACH,CAAC;AAED;;GAEG;AACH,SAAS,mBAAmB,CAAC,MAAe;IAC1C,IAAI,CAAC,MAAM,EAAE,CAAC;QACZ,OAAO,EAAE,IAAI,EAAE,aAAa,EAAE,CAAC;IACjC,CAAC;IAED,qFAAqF;IACrF,IAAI,gBAAgB,CAAC,MAAM,CAAC,EAAE,CAAC;QAC7B,OAAO;YACL,IAAI,EAAE,aAAa;YACnB,WAAW,EAAE;gBACX,IAAI,EAAE,YAAY;gBAClB,MAAM,EAAE,IAAI;gBACZ,MAAM;aACP;SACF,CAAC;IACJ,CAAC;IAED,uDAAuD;IACvD,OAAO,EAAE,IAAI,EAAE,aAAa,EAAE,CAAC;AACjC,CAAC;AAED;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,cAAc,CAAC,OAA6B;IAChE,MAAM,EACJ,OAAO,EACP,WAAW,EACX,OAAO,GAAG,2BAA2B,EACrC,KAAK,GAAG,aAAa,EACrB,SAAS,GAAG,IAAI,GACjB,GAAG,OAAO,CAAC;IAEZ,MAAM,MAAM,GAAG,OAAO,CAAC,MAAM,IAAI,OAAO,CAAC,GAAG,CAAC,cAAc,CAAC;IAE5D,IAAI,CAAC,MAAM,EAAE,CAAC;QACZ,MAAM,IAAI,KAAK,CACb,uCAAuC;YACvC,gEAAgE,CACjE,CAAC;IACJ,CAAC;IAED,uEAAuE;IACvE,IAAI,cAAc,GAAG,OAAO,CAAC,MAAM,CAAC;IACpC,IAAI,cAAc,IAAI,CAAC,gBAAgB,CAAC,cAAc,CAAC,EAAE,CAAC;QACxD,cAAc,GAAG,yBAAyB,CAAC,cAAc,CAAC,CAAC;IAC7D,CAAC;IAED,6DAA6D;IAC7D,MAAM,YAAY,GAAG,cAAc,CAAC,CAAC,CAAC,oBAAoB,CAAC,CAAC,CAAC,qBAAqB,CAAC;IAEnF,MAAM,WAAW,GAAG,gBAAgB,CAAC,OAAO,EAAE,WAAW,EAAE,cAAc,IAAI,OAAO,CAAC,MAAM,CAAC,CAAC;IAE7F,MAAM,cAAc,GAAG,mBAAmB,CAAC,cAAc,CAAC,CAAC;IAE3D,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,GAAG,OAAO,mBAAmB,EAAE;QAC1D,MAAM,EAAE,MAAM;QACd,OAAO,EAAE;YACP,cAAc,EAAE,kBAAkB;YAClC,eAAe,EAAE,UAAU,MAAM,EAAE;SACpC;QACD,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC;YACnB,KAAK;YACL,QAAQ,EAAE;gBACR,EAAE,IAAI,EAAE,QAAQ,EAAE,OAAO,EAAE,YAAY,EAAE;gBACzC,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,WAAW,EAAE;aACvC;YACD,WAAW,EAAE,CAAC;YACd,UAAU,EAAE,SAAS;YACrB,eAAe,EAAE,cAAc;SAChC,CAAC;KACH,CAAC,CAAC;IAEH,IAAI,CAAC,QAAQ,CAAC,EAAE,EAAE,CAAC;QACjB,MAAM,IAAI,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,EAAE,CAAC,CAAC;QACnD,IAAI,QAAQ,CAAC,MAAM,KAAK,GAAG,EAAE,CAAC;YAC5B,MAAM,IAAI,KAAK,CAAC,0DAA0D,CAAC,CAAC;QAC9E,CAAC;QACD,IAAI,QAAQ,CAAC,MAAM,KAAK,GAAG,EAAE,CAAC;YAC5B,MAAM,IAAI,KAAK,CAAC,2DAA2D,CAAC,CAAC;QAC/E,CAAC;QACD,MAAM,IAAI,KAAK,CAAC,uBAAuB,QAAQ,CAAC,MAAM,GAAG,IAAI,CAAC,CAAC,CAAC,MAAM,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;IACrG,CAAC;IAED,MAAM,IAAI,GAAG,MAAM,QAAQ,CAAC,IAAI,EAI/B,CAAC;IAEF,MAAM,OAAO,GAAG,IAAI,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,OAAO,IAAI,EAAE,CAAC;IAC1D,MAAM,KAAK,GAAG,UAAU,CAAC,OAAO,EAAE,cAAc,CAAC,CAAC;IAElD,6CAA6C;IAC7C,IAAI,cAAc,EAAE,CAAC;QACnB,mBAAmB,CAAC,KAAK,EAAE,cAAc,CAAC,CAAC;IAC7C,CAAC;IAED,MAAM,WAAW,GAAG,IAAI,CAAC,KAAK,EAAE,aAAa,IAAI,CAAC,CAAC;IACnD,MAAM,YAAY,GAAG,IAAI,CAAC,KAAK,EAAE,iBAAiB,IAAI,CAAC,CAAC;IACxD,MAAM,aAAa,GAAG,IAAI,CAAC,KAAK,IAAI,KAAK,CAAC;IAC1C,MAAM,IAAI,GAAG,YAAY,CAAC,aAAa,EAAE,WAAW,EAAE,YAAY,CAAC,CAAC;IAEpE,OAAO;QACL,KAAK;QACL,UAAU,EAAE,EAAE,KAAK,EAAE,WAAW,EAAE,MAAM,EAAE,YAAY,EAAE;QACxD,KAAK,EAAE,aAAa;QACpB,IAAI;KACL,CAAC;AACJ,CAAC"}

package/dist/core/profiles.d.ts

@@ -0,0 +1,48 @@
+/**
+ * WebPeel Profile Management
+ *
+ * Manages named browser profiles stored in ~/.webpeel/profiles/<name>/
+ * Each profile contains:
+ *   - storage-state.json  (Playwright storage state: cookies, localStorage, origins)
+ *   - metadata.json       (name, created, lastUsed, domains, description)
+ */
+export interface ProfileMetadata {
+    name: string;
+    created: string;
+    lastUsed: string;
+    domains: string[];
+    description?: string;
+}
+/**
+ * Valid profile names: letters, digits, hyphens only. No spaces or special chars.
+ */
+export declare function isValidProfileName(name: string): boolean;
+/**
+ * Get the directory path for a named profile, or null if it doesn't exist.
+ */
+export declare function getProfilePath(name: string): string | null;
+/**
+ * Load the Playwright storage state (cookies + localStorage) for a named profile.
+ * Returns null if the profile or storage-state.json doesn't exist.
+ */
+export declare function loadStorageState(name: string): any | null;
+/**
+ * Update the lastUsed timestamp for a profile.
+ */
+export declare function touchProfile(name: string): void;
+/**
+ * List all profiles, sorted by lastUsed descending.
+ */
+export declare function listProfiles(): ProfileMetadata[];
+/**
+ * Delete a named profile. Returns true if deleted, false if not found.
+ */
+export declare function deleteProfile(name: string): boolean;
+/**
+ * Interactively create a new profile:
+ * 1. Launches a VISIBLE (headed) Chromium browser
+ * 2. User navigates and logs into sites
+ * 3. On browser close or Ctrl+C, captures storage state and saves the profile
+ */
+export declare function createProfile(name: string, description?: string): Promise<void>;
+//# sourceMappingURL=profiles.d.ts.map

package/dist/core/profiles.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"profiles.d.ts","sourceRoot":"","sources":["../../src/core/profiles.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AAgBH,MAAM,WAAW,eAAe;IAC9B,IAAI,EAAE,MAAM,CAAC;IACb,OAAO,EAAE,MAAM,CAAC;IAChB,QAAQ,EAAE,MAAM,CAAC;IACjB,OAAO,EAAE,MAAM,EAAE,CAAC;IAClB,WAAW,CAAC,EAAE,MAAM,CAAC;CACtB;AAcD;;GAEG;AACH,wBAAgB,kBAAkB,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAExD;AAID;;GAEG;AACH,wBAAgB,cAAc,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,GAAG,IAAI,CAM1D;AAED;;;GAGG;AACH,wBAAgB,gBAAgB,CAAC,IAAI,EAAE,MAAM,GAAG,GAAG,GAAG,IAAI,CAQzD;AAED;;GAEG;AACH,wBAAgB,YAAY,CAAC,IAAI,EAAE,MAAM,GAAG,IAAI,CAU/C;AAED;;GAEG;AACH,wBAAgB,YAAY,IAAI,eAAe,EAAE,CAsBhD;AAED;;GAEG;AACH,wBAAgB,aAAa,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CASnD;AAID;;;;;GAKG;AACH,wBAAsB,aAAa,CAAC,IAAI,EAAE,MAAM,EAAE,WAAW,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAqHrF"}

package/dist/core/profiles.js

@@ -0,0 +1,211 @@
+/**
+ * WebPeel Profile Management
+ *
+ * Manages named browser profiles stored in ~/.webpeel/profiles/<name>/
+ * Each profile contains:
+ *   - storage-state.json  (Playwright storage state: cookies, localStorage, origins)
+ *   - metadata.json       (name, created, lastUsed, domains, description)
+ */
+import { chromium } from 'playwright';
+import { homedir } from 'os';
+import { existsSync, mkdirSync, readFileSync, writeFileSync, rmSync, readdirSync, } from 'fs';
+import path from 'path';
+// ─── Paths ───────────────────────────────────────────────────────────────────
+const PROFILES_DIR = path.join(homedir(), '.webpeel', 'profiles');
+function ensureProfilesDir() {
+    if (!existsSync(PROFILES_DIR)) {
+        mkdirSync(PROFILES_DIR, { recursive: true });
+    }
+}
+// ─── Name validation ─────────────────────────────────────────────────────────
+/**
+ * Valid profile names: letters, digits, hyphens only. No spaces or special chars.
+ */
+export function isValidProfileName(name) {
+    return /^[a-zA-Z0-9-]+$/.test(name) && name.length > 0 && name.length <= 64;
+}
+// ─── Core helpers ─────────────────────────────────────────────────────────────
+/**
+ * Get the directory path for a named profile, or null if it doesn't exist.
+ */
+export function getProfilePath(name) {
+    const dir = path.join(PROFILES_DIR, name);
+    if (existsSync(dir) && existsSync(path.join(dir, 'metadata.json'))) {
+        return dir;
+    }
+    return null;
+}
+/**
+ * Load the Playwright storage state (cookies + localStorage) for a named profile.
+ * Returns null if the profile or storage-state.json doesn't exist.
+ */
+export function loadStorageState(name) {
+    const statePath = path.join(PROFILES_DIR, name, 'storage-state.json');
+    if (!existsSync(statePath))
+        return null;
+    try {
+        return JSON.parse(readFileSync(statePath, 'utf-8'));
+    }
+    catch {
+        return null;
+    }
+}
+/**
+ * Update the lastUsed timestamp for a profile.
+ */
+export function touchProfile(name) {
+    const metaPath = path.join(PROFILES_DIR, name, 'metadata.json');
+    if (!existsSync(metaPath))
+        return;
+    try {
+        const meta = JSON.parse(readFileSync(metaPath, 'utf-8'));
+        meta.lastUsed = new Date().toISOString();
+        writeFileSync(metaPath, JSON.stringify(meta, null, 2));
+    }
+    catch {
+        /* ignore */
+    }
+}
+/**
+ * List all profiles, sorted by lastUsed descending.
+ */
+export function listProfiles() {
+    ensureProfilesDir();
+    const profiles = [];
+    try {
+        const entries = readdirSync(PROFILES_DIR, { withFileTypes: true });
+        for (const entry of entries) {
+            if (!entry.isDirectory())
+                continue;
+            const metaPath = path.join(PROFILES_DIR, entry.name, 'metadata.json');
+            if (!existsSync(metaPath))
+                continue;
+            try {
+                const meta = JSON.parse(readFileSync(metaPath, 'utf-8'));
+                profiles.push(meta);
+            }
+            catch {
+                /* skip corrupt profile */
+            }
+        }
+    }
+    catch {
+        /* ignore read errors */
+    }
+    // Sort: most recently used first
+    profiles.sort((a, b) => b.lastUsed.localeCompare(a.lastUsed));
+    return profiles;
+}
+/**
+ * Delete a named profile. Returns true if deleted, false if not found.
+ */
+export function deleteProfile(name) {
+    const dir = path.join(PROFILES_DIR, name);
+    if (!existsSync(dir))
+        return false;
+    try {
+        rmSync(dir, { recursive: true, force: true });
+        return true;
+    }
+    catch {
+        return false;
+    }
+}
+// ─── Interactive profile creation ─────────────────────────────────────────────
+/**
+ * Interactively create a new profile:
+ * 1. Launches a VISIBLE (headed) Chromium browser
+ * 2. User navigates and logs into sites
+ * 3. On browser close or Ctrl+C, captures storage state and saves the profile
+ */
+export async function createProfile(name, description) {
+    if (!isValidProfileName(name)) {
+        throw new Error(`Invalid profile name "${name}". Use only letters, numbers, and hyphens (no spaces or special characters).`);
+    }
+    ensureProfilesDir();
+    const profileDir = path.join(PROFILES_DIR, name);
+    if (existsSync(profileDir)) {
+        throw new Error(`Profile "${name}" already exists. Delete it first with:\n  webpeel profile delete ${name}`);
+    }
+    mkdirSync(profileDir, { recursive: true });
+    // Launch headed (visible) Chromium — no user-data-dir so we start fresh
+    const browser = await chromium.launch({ headless: false });
+    const context = await browser.newContext();
+    const page = await context.newPage();
+    await page.goto('about:blank').catch(() => { });
+    console.log('');
+    console.log('╔══════════════════════════════════════════════════════╗');
+    console.log(`║  WebPeel Profile Setup: "${name}"`);
+    console.log('║                                                      ║');
+    console.log('║  Navigate to websites and log in.                   ║');
+    console.log('║  When done, press Ctrl+C or close this window.      ║');
+    console.log('╚══════════════════════════════════════════════════════╝');
+    console.log('');
+    let saved = false;
+    const saveAndClose = async () => {
+        if (saved)
+            return;
+        saved = true;
+        console.log('\nCapturing browser session...');
+        try {
+            const storageState = await context.storageState();
+            writeFileSync(path.join(profileDir, 'storage-state.json'), JSON.stringify(storageState, null, 2));
+            // Extract unique domains from cookies (strip leading dot)
+            const domains = [
+                ...new Set((storageState.cookies ?? [])
+                    .map((c) => (c.domain ?? '').replace(/^\./, ''))
+                    .filter(Boolean)),
+            ];
+            const now = new Date().toISOString();
+            const meta = {
+                name,
+                created: now,
+                lastUsed: now,
+                domains,
+                ...(description ? { description } : {}),
+            };
+            writeFileSync(path.join(profileDir, 'metadata.json'), JSON.stringify(meta, null, 2));
+            console.log(`✓ Profile "${name}" saved to ${profileDir}`);
+            if (domains.length > 0) {
+                console.log(`  Domains: ${domains.join(', ')}`);
+            }
+            else {
+                console.log('  No login sessions detected (no cookies).');
+            }
+        }
+        catch (e) {
+            console.error('Warning: Failed to save storage state:', e instanceof Error ? e.message : String(e));
+            // Clean up partial directory
+            try {
+                rmSync(profileDir, { recursive: true, force: true });
+            }
+            catch {
+                /* ignore */
+            }
+        }
+        try {
+            await browser.close();
+        }
+        catch {
+            /* ignore — browser may already be closed */
+        }
+    };
+    // Wait for the browser to disconnect (user closed the window) OR SIGINT (Ctrl+C)
+    await new Promise((resolve) => {
+        browser.on('disconnected', async () => {
+            await saveAndClose();
+            resolve();
+        });
+        // Handle Ctrl+C gracefully
+        const sigintHandler = async () => {
+            await saveAndClose();
+            resolve();
+        };
+        process.once('SIGINT', sigintHandler);
+        // Clean up the SIGINT handler if browser closes first
+        browser.on('disconnected', () => {
+            process.removeListener('SIGINT', sigintHandler);
+        });
+    });
+}
+//# sourceMappingURL=profiles.js.map

package/dist/core/profiles.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"profiles.js","sourceRoot":"","sources":["../../src/core/profiles.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AAEH,OAAO,EAAE,QAAQ,EAAE,MAAM,YAAY,CAAC;AACtC,OAAO,EAAE,OAAO,EAAE,MAAM,IAAI,CAAC;AAC7B,OAAO,EACL,UAAU,EACV,SAAS,EACT,YAAY,EACZ,aAAa,EACb,MAAM,EACN,WAAW,GACZ,MAAM,IAAI,CAAC;AACZ,OAAO,IAAI,MAAM,MAAM,CAAC;AAYxB,gFAAgF;AAEhF,MAAM,YAAY,GAAG,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,EAAE,UAAU,EAAE,UAAU,CAAC,CAAC;AAElE,SAAS,iBAAiB;IACxB,IAAI,CAAC,UAAU,CAAC,YAAY,CAAC,EAAE,CAAC;QAC9B,SAAS,CAAC,YAAY,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IAC/C,CAAC;AACH,CAAC;AAED,gFAAgF;AAEhF;;GAEG;AACH,MAAM,UAAU,kBAAkB,CAAC,IAAY;IAC7C,OAAO,iBAAiB,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,IAAI,CAAC,MAAM,GAAG,CAAC,IAAI,IAAI,CAAC,MAAM,IAAI,EAAE,CAAC;AAC9E,CAAC;AAED,iFAAiF;AAEjF;;GAEG;AACH,MAAM,UAAU,cAAc,CAAC,IAAY;IACzC,MAAM,GAAG,GAAG,IAAI,CAAC,IAAI,CAAC,YAAY,EAAE,IAAI,CAAC,CAAC;IAC1C,IAAI,UAAU,CAAC,GAAG,CAAC,IAAI,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,eAAe,CAAC,CAAC,EAAE,CAAC;QACnE,OAAO,GAAG,CAAC;IACb,CAAC;IACD,OAAO,IAAI,CAAC;AACd,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,gBAAgB,CAAC,IAAY;IAC3C,MAAM,SAAS,GAAG,IAAI,CAAC,IAAI,CAAC,YAAY,EAAE,IAAI,EAAE,oBAAoB,CAAC,CAAC;IACtE,IAAI,CAAC,UAAU,CAAC,SAAS,CAAC;QAAE,OAAO,IAAI,CAAC;IACxC,IAAI,CAAC;QACH,OAAO,IAAI,CAAC,KAAK,CAAC,YAAY,CAAC,SAAS,EAAE,OAAO,CAAC,CAAC,CAAC;IACtD,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,IAAI,CAAC;IACd,CAAC;AACH,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,YAAY,CAAC,IAAY;IACvC,MAAM,QAAQ,GAAG,IAAI,CAAC,IAAI,CAAC,YAAY,EAAE,IAAI,EAAE,eAAe,CAAC,CAAC;IAChE,IAAI,CAAC,UAAU,CAAC,QAAQ,CAAC;QAAE,OAAO;IAClC,IAAI,CAAC;QACH,MAAM,IAAI,GAAoB,IAAI,CAAC,KAAK,CAAC,YAAY,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC,CAAC;QAC1E,IAAI,CAAC,QAAQ,GAAG,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;QACzC,aAAa,CAAC,QAAQ,EAAE,IAAI,CAAC,SAAS,CAAC,IAAI,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC;IACzD,CAAC;IAAC,MAAM,CAAC;QACP,YAAY;IACd,CAAC;AACH,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,YAAY;IAC1B,iBAAiB,EAAE,CAAC;IACpB,MAAM,QAAQ,GAAsB,EAAE,CAAC;IACvC,IAAI,CAAC;QACH,MAAM,OAAO,GAAG,WAAW,CAAC,YAAY,EAAE,EAAE,aAAa,EAAE,IAAI,EAAE,CAAC,CAAC;QACnE,KAAK,MAAM,KAAK,IAAI,OAAO,EAAE,CAAC;YAC5B,IAAI,CAAC,KAAK,CAAC,WAAW,EAAE;gBAAE,SAAS;YACnC,MAAM,QAAQ,GAAG,IAAI,CAAC,IAAI,CAAC,YAAY,EAAE,KAAK,CAAC,IAAI,EAAE,eAAe,CAAC,CAAC;YACtE,IAAI,CAAC,UAAU,CAAC,QAAQ,CAAC;gBAAE,SAAS;YACpC,IAAI,CAAC;gBACH,MAAM,IAAI,GAAoB,IAAI,CAAC,KAAK,CAAC,YAAY,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC,CAAC;gBAC1E,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YACtB,CAAC;YAAC,MAAM,CAAC;gBACP,0BAA0B;YAC5B,CAAC;QACH,CAAC;IACH,CAAC;IAAC,MAAM,CAAC;QACP,wBAAwB;IAC1B,CAAC;IACD,iCAAiC;IACjC,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,aAAa,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC;IAC9D,OAAO,QAAQ,CAAC;AAClB,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,aAAa,CAAC,IAAY;IACxC,MAAM,GAAG,GAAG,IAAI,CAAC,IAAI,CAAC,YAAY,EAAE,IAAI,CAAC,CAAC;IAC1C,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC;QAAE,OAAO,KAAK,CAAC;IACnC,IAAI,CAAC;QACH,MAAM,CAAC,GAAG,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;QAC9C,OAAO,IAAI,CAAC;IACd,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,KAAK,CAAC;IACf,CAAC;AACH,CAAC;AAED,iFAAiF;AAEjF;;;;;GAKG;AACH,MAAM,CAAC,KAAK,UAAU,aAAa,CAAC,IAAY,EAAE,WAAoB;IACpE,IAAI,CAAC,kBAAkB,CAAC,IAAI,CAAC,EAAE,CAAC;QAC9B,MAAM,IAAI,KAAK,CACb,yBAAyB,IAAI,8EAA8E,CAC5G,CAAC;IACJ,CAAC;IAED,iBAAiB,EAAE,CAAC;IAEpB,MAAM,UAAU,GAAG,IAAI,CAAC,IAAI,CAAC,YAAY,EAAE,IAAI,CAAC,CAAC;IACjD,IAAI,UAAU,CAAC,UAAU,CAAC,EAAE,CAAC;QAC3B,MAAM,IAAI,KAAK,CACb,YAAY,IAAI,qEAAqE,IAAI,EAAE,CAC5F,CAAC;IACJ,CAAC;IAED,SAAS,CAAC,UAAU,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IAE3C,wEAAwE;IACxE,MAAM,OAAO,GAAG,MAAM,QAAQ,CAAC,MAAM,CAAC,EAAE,QAAQ,EAAE,KAAK,EAAE,CAAC,CAAC;IAC3D,MAAM,OAAO,GAAG,MAAM,OAAO,CAAC,UAAU,EAAE,CAAC;IAC3C,MAAM,IAAI,GAAG,MAAM,OAAO,CAAC,OAAO,EAAE,CAAC;IACrC,MAAM,IAAI,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,GAAE,CAAC,CAAC,CAAC;IAE/C,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;IAChB,OAAO,CAAC,GAAG,CAAC,0DAA0D,CAAC,CAAC;IACxE,OAAO,CAAC,GAAG,CAAC,8BAA8B,IAAI,GAAG,CAAC,CAAC;IACnD,OAAO,CAAC,GAAG,CAAC,0DAA0D,CAAC,CAAC;IACxE,OAAO,CAAC,GAAG,CAAC,yDAAyD,CAAC,CAAC;IACvE,OAAO,CAAC,GAAG,CAAC,yDAAyD,CAAC,CAAC;IACvE,OAAO,CAAC,GAAG,CAAC,0DAA0D,CAAC,CAAC;IACxE,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;IAEhB,IAAI,KAAK,GAAG,KAAK,CAAC;IAElB,MAAM,YAAY,GAAG,KAAK,IAAmB,EAAE;QAC7C,IAAI,KAAK;YAAE,OAAO;QAClB,KAAK,GAAG,IAAI,CAAC;QAEb,OAAO,CAAC,GAAG,CAAC,gCAAgC,CAAC,CAAC;QAE9C,IAAI,CAAC;YACH,MAAM,YAAY,GAAG,MAAM,OAAO,CAAC,YAAY,EAAE,CAAC;YAElD,aAAa,CACX,IAAI,CAAC,IAAI,CAAC,UAAU,EAAE,oBAAoB,CAAC,EAC3C,IAAI,CAAC,SAAS,CAAC,YAAY,EAAE,IAAI,EAAE,CAAC,CAAC,CACtC,CAAC;YAEF,0DAA0D;YAC1D,MAAM,OAAO,GAAa;gBACxB,GAAG,IAAI,GAAG,CACR,CAAC,YAAY,CAAC,OAAO,IAAI,EAAE,CAAC;qBACzB,GAAG,CAAC,CAAC,CAAM,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,MAAM,IAAI,EAAE,CAAC,CAAC,OAAO,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC;qBACpD,MAAM,CAAC,OAAO,CAAC,CACnB;aACF,CAAC;YAEF,MAAM,GAAG,GAAG,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;YACrC,MAAM,IAAI,GAAoB;gBAC5B,IAAI;gBACJ,OAAO,EAAE,GAAG;gBACZ,QAAQ,EAAE,GAAG;gBACb,OAAO;gBACP,GAAG,CAAC,WAAW,CAAC,CAAC,CAAC,EAAE,WAAW,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;aACxC,CAAC;YAEF,aAAa,CACX,IAAI,CAAC,IAAI,CAAC,UAAU,EAAE,eAAe,CAAC,EACtC,IAAI,CAAC,SAAS,CAAC,IAAI,EAAE,IAAI,EAAE,CAAC,CAAC,CAC9B,CAAC;YAEF,OAAO,CAAC,GAAG,CAAC,cAAc,IAAI,cAAc,UAAU,EAAE,CAAC,CAAC;YAC1D,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBACvB,OAAO,CAAC,GAAG,CAAC,cAAc,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;YAClD,CAAC;iBAAM,CAAC;gBACN,OAAO,CAAC,GAAG,CAAC,4CAA4C,CAAC,CAAC;YAC5D,CAAC;QACH,CAAC;QAAC,OAAO,CAAC,EAAE,CAAC;YACX,OAAO,CAAC,KAAK,CACX,wCAAwC,EACxC,CAAC,YAAY,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAC3C,CAAC;YACF,6BAA6B;YAC7B,IAAI,CAAC;gBACH,MAAM,CAAC,UAAU,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;YACvD,CAAC;YAAC,MAAM,CAAC;gBACP,YAAY;YACd,CAAC;QACH,CAAC;QAED,IAAI,CAAC;YACH,MAAM,OAAO,CAAC,KAAK,EAAE,CAAC;QACxB,CAAC;QAAC,MAAM,CAAC;YACP,4CAA4C;QAC9C,CAAC;IACH,CAAC,CAAC;IAEF,iFAAiF;IACjF,MAAM,IAAI,OAAO,CAAO,CAAC,OAAO,EAAE,EAAE;QAClC,OAAO,CAAC,EAAE,CAAC,cAAc,EAAE,KAAK,IAAI,EAAE;YACpC,MAAM,YAAY,EAAE,CAAC;YACrB,OAAO,EAAE,CAAC;QACZ,CAAC,CAAC,CAAC;QAEH,2BAA2B;QAC3B,MAAM,aAAa,GAAG,KAAK,IAAI,EAAE;YAC/B,MAAM,YAAY,EAAE,CAAC;YACrB,OAAO,EAAE,CAAC;QACZ,CAAC,CAAC;QACF,OAAO,CAAC,IAAI,CAAC,QAAQ,EAAE,aAAa,CAAC,CAAC;QAEtC,sDAAsD;QACtD,OAAO,CAAC,EAAE,CAAC,cAAc,EAAE,GAAG,EAAE;YAC9B,OAAO,CAAC,cAAc,CAAC,QAAQ,EAAE,aAAa,CAAC,CAAC;QAClD,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;AACL,CAAC"}

package/dist/core/schema-extraction.d.ts

@@ -0,0 +1,67 @@
+/**
+ * Schema-based extraction using CSS selectors.
+ *
+ * Each schema defines how to extract listings from a specific domain,
+ * inspired by Crawl4AI's JsonCssExtractionStrategy. Unlike generic
+ * auto-detection, schemas provide exact selectors for each site's DOM.
+ *
+ * @module schema-extraction
+ */
+export interface SchemaField {
+    /** Field name in output (e.g., "title", "price", "rating") */
+    name: string;
+    /** CSS selector relative to baseSelector. Empty string selects the base element itself. */
+    selector: string;
+    /** What to extract */
+    type: 'text' | 'attribute' | 'html' | 'exists';
+    /** For type='attribute', which attribute to read */
+    attribute?: string;
+    /** Extract all matches (returns array instead of first match) */
+    multiple?: boolean;
+    /** Optional transform to apply after extraction */
+    transform?: 'trim' | 'number' | 'stripCurrency';
+}
+export interface ExtractionSchema {
+    /** Human-readable schema name (e.g., "Booking.com Hotel Search") */
+    name: string;
+    /** Schema version string */
+    version: string;
+    /** Matching domains (e.g., ["booking.com", "www.booking.com"]) */
+    domains: string[];
+    /** Optional URL path patterns (regex strings) for more specific matching */
+    urlPatterns?: string[];
+    /** CSS selector for each listing item */
+    baseSelector: string;
+    /** Fields to extract from each item */
+    fields: SchemaField[];
+    /** Optional pagination config */
+    pagination?: {
+        nextSelector?: string;
+        pageParam?: string;
+    };
+}
+/** A single extracted item — field names map to extracted values */
+export interface ExtractedItem {
+    [key: string]: string | string[] | boolean | number | undefined;
+}
+/**
+ * Load all bundled schemas.
+ */
+export declare function loadBundledSchemas(): ExtractionSchema[];
+/**
+ * Find a matching schema for a given URL.
+ *
+ * Matches by domain first, then optionally by URL patterns (regex).
+ * Returns the first matching schema or null.
+ */
+export declare function findSchemaForUrl(url: string): ExtractionSchema | null;
+/**
+ * Extract listings from HTML using a schema's CSS selectors.
+ *
+ * @param html    - Raw HTML string to parse
+ * @param schema  - Extraction schema to use
+ * @param baseUrl - Optional base URL for resolving relative links
+ * @returns Array of extracted items (may be empty)
+ */
+export declare function extractWithSchema(html: string, schema: ExtractionSchema, baseUrl?: string): ExtractedItem[];
+//# sourceMappingURL=schema-extraction.d.ts.map

package/dist/core/schema-extraction.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"schema-extraction.d.ts","sourceRoot":"","sources":["../../src/core/schema-extraction.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AASH,MAAM,WAAW,WAAW;IAC1B,8DAA8D;IAC9D,IAAI,EAAE,MAAM,CAAC;IACb,2FAA2F;IAC3F,QAAQ,EAAE,MAAM,CAAC;IACjB,sBAAsB;IACtB,IAAI,EAAE,MAAM,GAAG,WAAW,GAAG,MAAM,GAAG,QAAQ,CAAC;IAC/C,oDAAoD;IACpD,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,iEAAiE;IACjE,QAAQ,CAAC,EAAE,OAAO,CAAC;IACnB,mDAAmD;IACnD,SAAS,CAAC,EAAE,MAAM,GAAG,QAAQ,GAAG,eAAe,CAAC;CACjD;AAED,MAAM,WAAW,gBAAgB;IAC/B,oEAAoE;IACpE,IAAI,EAAE,MAAM,CAAC;IACb,4BAA4B;IAC5B,OAAO,EAAE,MAAM,CAAC;IAChB,kEAAkE;IAClE,OAAO,EAAE,MAAM,EAAE,CAAC;IAClB,4EAA4E;IAC5E,WAAW,CAAC,EAAE,MAAM,EAAE,CAAC;IACvB,yCAAyC;IACzC,YAAY,EAAE,MAAM,CAAC;IACrB,uCAAuC;IACvC,MAAM,EAAE,WAAW,EAAE,CAAC;IACtB,iCAAiC;IACjC,UAAU,CAAC,EAAE;QACX,YAAY,CAAC,EAAE,MAAM,CAAC;QACtB,SAAS,CAAC,EAAE,MAAM,CAAC;KACpB,CAAC;CACH;AAED,oEAAoE;AACpE,MAAM,WAAW,aAAa;IAC5B,CAAC,GAAG,EAAE,MAAM,GAAG,MAAM,GAAG,MAAM,EAAE,GAAG,OAAO,GAAG,MAAM,GAAG,SAAS,CAAC;CACjE;AA4PD;;GAEG;AACH,wBAAgB,kBAAkB,IAAI,gBAAgB,EAAE,CAEvD;AAED;;;;;GAKG;AACH,wBAAgB,gBAAgB,CAAC,GAAG,EAAE,MAAM,GAAG,gBAAgB,GAAG,IAAI,CAsCrE;AAED;;;;;;;GAOG;AACH,wBAAgB,iBAAiB,CAC/B,IAAI,EAAE,MAAM,EACZ,MAAM,EAAE,gBAAgB,EACxB,OAAO,CAAC,EAAE,MAAM,GACf,aAAa,EAAE,CA0DjB"}