webpeel 0.7.0 → 0.7.1

package/dist/types.d.ts

@@ -3,11 +3,29 @@
  */
 export interface PageAction {
     type: 'wait' | 'click' | 'scroll' | 'type' | 'fill' | 'select' | 'press' | 'hover' | 'waitForSelector' | 'screenshot';
+    /** CSS selector for element-targeted actions */
     selector?: string;
+    /**
+     * Value/text payload for actions like type/fill/select.
+     * Accepts Firecrawl-style `text` too (normalized internally).
+     */
     value?: string;
+    text?: string;
+    /** Keyboard key for press actions (e.g., "Enter") */
     key?: string;
+    /** Wait duration for wait actions (ms). Firecrawl uses `milliseconds`. */
     ms?: number;
+    milliseconds?: number;
+    /**
+     * Scroll target (absolute) — legacy/internal.
+     * Use direction+amount for relative scrolling.
+     */
     to?: 'top' | 'bottom' | number;
+    /** Relative scroll direction (Firecrawl-style) */
+    direction?: 'up' | 'down' | 'left' | 'right';
+    /** Relative scroll amount in pixels (Firecrawl-style) */
+    amount?: number;
+    /** Per-action timeout override (ms) */
     timeout?: number;
 }
 export interface ExtractOptions {
@@ -24,6 +42,18 @@ export interface ExtractOptions {
     /** LLM API base URL (default: https://api.openai.com/v1) */
     llmBaseUrl?: string;
 }
+/**
+ * Inline structured extraction options (BYOK, multi-provider).
+ * Used with /v1/fetch POST, /v2/scrape, and /v1/scrape (Firecrawl compat).
+ */
+export interface InlineExtractParam {
+    /** JSON Schema describing the desired output structure */
+    schema?: Record<string, any>;
+    /** Natural language prompt describing what to extract */
+    prompt?: string;
+}
+/** LLM provider for BYOK inline extraction */
+export type InlineLLMProvider = 'openai' | 'anthropic' | 'google';
 export interface PeelOptions {
     /** Use headless browser instead of simple HTTP fetch */
     render?: boolean;
@@ -35,6 +65,8 @@ export interface PeelOptions {
     format?: 'markdown' | 'text' | 'html';
     /** Request timeout in milliseconds (default: 30000) */
     timeout?: number;
+    /** Prepare streaming responses (API plumbing only; full SSE/chunked stream not yet implemented) */
+    stream?: boolean;
     /** Custom user agent */
     userAgent?: string;
     /** Capture a screenshot of the page */
@@ -123,8 +155,10 @@ export interface PeelResult {
     quality?: number;
     /** SHA256 hash of content (first 16 chars) — for change detection */
     fingerprint?: string;
-    /** Extracted structured data (when extract option is used) */
+    /** Extracted structured data (when extract option is used — CSS/heuristic extraction) */
     extracted?: Record<string, any>;
+    /** Structured JSON from inline LLM extraction (when extract + llmProvider is used) */
+    json?: Record<string, any>;
     /** Branding/design system profile */
     branding?: import('./core/branding.js').BrandingProfile;
     /** Content change tracking result */
@@ -145,6 +179,14 @@ export interface PageMetadata {
     image?: string;
     /** Canonical URL */
     canonical?: string;
+    /** MIME content type (set for documents like PDF/DOCX) */
+    contentType?: string;
+    /** Word count (set for documents like PDF/DOCX) */
+    wordCount?: number;
+    /** Page count (set for PDF documents) */
+    pages?: number;
+    /** Allow additional document-specific metadata */
+    [key: string]: any;
 }
 export declare class WebPeelError extends Error {
     code?: string | undefined;

package/dist/types.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../src/types.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,MAAM,WAAW,UAAU;IACzB,IAAI,EAAE,MAAM,GAAG,OAAO,GAAG,QAAQ,GAAG,MAAM,GAAG,MAAM,GAAG,QAAQ,GAAG,OAAO,GAAG,OAAO,GAAG,iBAAiB,GAAG,YAAY,CAAC;IACtH,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,GAAG,CAAC,EAAE,MAAM,CAAC;IACb,EAAE,CAAC,EAAE,MAAM,CAAC;IACZ,EAAE,CAAC,EAAE,KAAK,GAAG,QAAQ,GAAG,MAAM,CAAC;IAC/B,OAAO,CAAC,EAAE,MAAM,CAAC;CAClB;AAED,MAAM,WAAW,cAAc;IAC7B,wCAAwC;IACxC,MAAM,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;IAC7B,0CAA0C;IAC1C,SAAS,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IACnC,yDAAyD;IACzD,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,6DAA6D;IAC7D,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,8CAA8C;IAC9C,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,4DAA4D;IAC5D,UAAU,CAAC,EAAE,MAAM,CAAC;CACrB;AAED,MAAM,WAAW,WAAW;IAC1B,wDAAwD;IACxD,MAAM,CAAC,EAAE,OAAO,CAAC;IACjB,+FAA+F;IAC/F,OAAO,CAAC,EAAE,OAAO,CAAC;IAClB,wEAAwE;IACxE,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,oBAAoB;IACpB,MAAM,CAAC,EAAE,UAAU,GAAG,MAAM,GAAG,MAAM,CAAC;IACtC,uDAAuD;IACvD,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,wBAAwB;IACxB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,uCAAuC;IACvC,UAAU,CAAC,EAAE,OAAO,CAAC;IACrB,oDAAoD;IACpD,kBAAkB,CAAC,EAAE,OAAO,CAAC;IAC7B,2FAA2F;IAC3F,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,yEAAyE;IACzE,OAAO,CAAC,EAAE,MAAM,EAAE,CAAC;IACnB,4FAA4F;IAC5F,WAAW,CAAC,EAAE,MAAM,EAAE,CAAC;IACvB,iFAAiF;IACjF,WAAW,CAAC,EAAE,MAAM,EAAE,CAAC;IACvB,kCAAkC;IAClC,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IACjC,uCAAuC;IACvC,OAAO,CAAC,EAAE,MAAM,EAAE,CAAC;IACnB,qFAAqF;IACrF,GAAG,CAAC,EAAE,OAAO,CAAC;IACd,sEAAsE;IACtE,OAAO,CAAC,EAAE,UAAU,EAAE,CAAC;IACvB,mEAAmE;IACnE,OAAO,CAAC,EAAE,cAAc,CAAC;IACzB,0EAA0E;IAC1E,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,qDAAqD;IACrD,cAAc,CAAC,EAAE,OAAO,CAAC;IACzB,4DAA4D;IAC5D,QAAQ,CAAC,EAAE,OAAO,CAAC;IACnB,qCAAqC;IACrC,OAAO,CAAC,EAAE,OAAO,GAAG;QAAE,MAAM,CAAC,EAAE,MAAM,CAAC;QAAC,SAAS,CAAC,EAAE,MAAM,CAAA;KAAE,CAAC;IAC5D,8DAA8D;IAC9D,GAAG,CAAC,EAAE;QACJ,MAAM,EAAE,MAAM,CAAC;QACf,KAAK,CAAC,EAAE,MAAM,CAAC;QACf,OAAO,CAAC,EAAE,MAAM,CAAC;KAClB,CAAC;IACF,mCAAmC;IACnC,MAAM,CAAC,EAAE,OAAO,CAAC;IACjB,8DAA8D;IAC9D,QAAQ,CAAC,EAAE;QACT,+DAA+D;QAC/D,OAAO,CAAC,EAAE,MAAM,CAAC;QACjB,mDAAmD;QACnD,SAAS,CAAC,EAAE,MAAM,EAAE,CAAC;KACtB,CAAC;CACH;AAED,MAAM,WAAW,SAAS;IACxB,gCAAgC;IAChC,GAAG,EAAE,MAAM,CAAC;IACZ,eAAe;IACf,GAAG,EAAE,MAAM,CAAC;IACZ,sBAAsB;IACtB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,yBAAyB;IACzB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,0BAA0B;IAC1B,MAAM,CAAC,EAAE,MAAM,CAAC;CACjB;AAED,MAAM,WAAW,UAAU;IACzB,kCAAkC;IAClC,GAAG,EAAE,MAAM,CAAC;IACZ,iBAAiB;IACjB,KAAK,EAAE,MAAM,CAAC;IACd,uCAAuC;IACvC,OAAO,EAAE,MAAM,CAAC;IAChB,yBAAyB;IACzB,QAAQ,EAAE,YAAY,CAAC;IACvB,gEAAgE;IAChE,KAAK,EAAE,MAAM,EAAE,CAAC;IAChB,wDAAwD;IACxD,MAAM,EAAE,MAAM,CAAC;IACf,oDAAoD;IACpD,MAAM,EAAE,QAAQ,GAAG,SAAS,GAAG,SAAS,CAAC;IACzC,mCAAmC;IACnC,OAAO,EAAE,MAAM,CAAC;IAChB,yEAAyE;IACzE,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,+DAA+D;IAC/D,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,+DAA+D;IAC/D,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,qEAAqE;IACrE,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,8DAA8D;IAC9D,SAAS,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;IAChC,qCAAqC;IACrC,QAAQ,CAAC,EAAE,OAAO,oBAAoB,EAAE,eAAe,CAAC;IACxD,qCAAqC;IACrC,cAAc,CAAC,EAAE,OAAO,2BAA2B,EAAE,YAAY,CAAC;IAClE,2BAA2B;IAC3B,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,mDAAmD;IACnD,MAAM,CAAC,EAAE,SAAS,EAAE,CAAC;CACtB;AAED,MAAM,WAAW,YAAY;IAC3B,uBAAuB;IACvB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,kBAAkB;IAClB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,gCAAgC;IAChC,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,2BAA2B;IAC3B,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,oBAAoB;IACpB,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB;AAED,qBAAa,YAAa,SAAQ,KAAK;IACD,IAAI,CAAC,EAAE,MAAM;gBAArC,OAAO,EAAE,MAAM,EAAS,IAAI,CAAC,EAAE,MAAM,YAAA;CAIlD;AAED,qBAAa,YAAa,SAAQ,YAAY;gBAChC,OAAO,EAAE,MAAM;CAI5B;AAED,qBAAa,YAAa,SAAQ,YAAY;gBAChC,OAAO,EAAE,MAAM;CAI5B;AAED,qBAAa,YAAa,SAAQ,YAAY;gBAChC,OAAO,EAAE,MAAM;CAI5B"}
+{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../src/types.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,MAAM,WAAW,UAAU;IACzB,IAAI,EAAE,MAAM,GAAG,OAAO,GAAG,QAAQ,GAAG,MAAM,GAAG,MAAM,GAAG,QAAQ,GAAG,OAAO,GAAG,OAAO,GAAG,iBAAiB,GAAG,YAAY,CAAC;IAEtH,gDAAgD;IAChD,QAAQ,CAAC,EAAE,MAAM,CAAC;IAElB;;;OAGG;IACH,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,IAAI,CAAC,EAAE,MAAM,CAAC;IAEd,qDAAqD;IACrD,GAAG,CAAC,EAAE,MAAM,CAAC;IAEb,0EAA0E;IAC1E,EAAE,CAAC,EAAE,MAAM,CAAC;IACZ,YAAY,CAAC,EAAE,MAAM,CAAC;IAEtB;;;OAGG;IACH,EAAE,CAAC,EAAE,KAAK,GAAG,QAAQ,GAAG,MAAM,CAAC;IAE/B,kDAAkD;IAClD,SAAS,CAAC,EAAE,IAAI,GAAG,MAAM,GAAG,MAAM,GAAG,OAAO,CAAC;IAE7C,yDAAyD;IACzD,MAAM,CAAC,EAAE,MAAM,CAAC;IAEhB,uCAAuC;IACvC,OAAO,CAAC,EAAE,MAAM,CAAC;CAClB;AAED,MAAM,WAAW,cAAc;IAC7B,wCAAwC;IACxC,MAAM,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;IAC7B,0CAA0C;IAC1C,SAAS,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IACnC,yDAAyD;IACzD,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,6DAA6D;IAC7D,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,8CAA8C;IAC9C,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,4DAA4D;IAC5D,UAAU,CAAC,EAAE,MAAM,CAAC;CACrB;AAED;;;GAGG;AACH,MAAM,WAAW,kBAAkB;IACjC,0DAA0D;IAC1D,MAAM,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;IAC7B,yDAAyD;IACzD,MAAM,CAAC,EAAE,MAAM,CAAC;CACjB;AAED,8CAA8C;AAC9C,MAAM,MAAM,iBAAiB,GAAG,QAAQ,GAAG,WAAW,GAAG,QAAQ,CAAC;AAElE,MAAM,WAAW,WAAW;IAC1B,wDAAwD;IACxD,MAAM,CAAC,EAAE,OAAO,CAAC;IACjB,+FAA+F;IAC/F,OAAO,CAAC,EAAE,OAAO,CAAC;IAClB,wEAAwE;IACxE,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,oBAAoB;IACpB,MAAM,CAAC,EAAE,UAAU,GAAG,MAAM,GAAG,MAAM,CAAC;IACtC,uDAAuD;IACvD,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,mGAAmG;IACnG,MAAM,CAAC,EAAE,OAAO,CAAC;IACjB,wBAAwB;IACxB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,uCAAuC;IACvC,UAAU,CAAC,EAAE,OAAO,CAAC;IACrB,oDAAoD;IACpD,kBAAkB,CAAC,EAAE,OAAO,CAAC;IAC7B,2FAA2F;IAC3F,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,yEAAyE;IACzE,OAAO,CAAC,EAAE,MAAM,EAAE,CAAC;IACnB,4FAA4F;IAC5F,WAAW,CAAC,EAAE,MAAM,EAAE,CAAC;IACvB,iFAAiF;IACjF,WAAW,CAAC,EAAE,MAAM,EAAE,CAAC;IACvB,kCAAkC;IAClC,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IACjC,uCAAuC;IACvC,OAAO,CAAC,EAAE,MAAM,EAAE,CAAC;IACnB,qFAAqF;IACrF,GAAG,CAAC,EAAE,OAAO,CAAC;IACd,sEAAsE;IACtE,OAAO,CAAC,EAAE,UAAU,EAAE,CAAC;IACvB,mEAAmE;IACnE,OAAO,CAAC,EAAE,cAAc,CAAC;IACzB,0EAA0E;IAC1E,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,qDAAqD;IACrD,cAAc,CAAC,EAAE,OAAO,CAAC;IACzB,4DAA4D;IAC5D,QAAQ,CAAC,EAAE,OAAO,CAAC;IACnB,qCAAqC;IACrC,OAAO,CAAC,EAAE,OAAO,GAAG;QAAE,MAAM,CAAC,EAAE,MAAM,CAAC;QAAC,SAAS,CAAC,EAAE,MAAM,CAAA;KAAE,CAAC;IAC5D,8DAA8D;IAC9D,GAAG,CAAC,EAAE;QACJ,MAAM,EAAE,MAAM,CAAC;QACf,KAAK,CAAC,EAAE,MAAM,CAAC;QACf,OAAO,CAAC,EAAE,MAAM,CAAC;KAClB,CAAC;IACF,mCAAmC;IACnC,MAAM,CAAC,EAAE,OAAO,CAAC;IACjB,8DAA8D;IAC9D,QAAQ,CAAC,EAAE;QACT,+DAA+D;QAC/D,OAAO,CAAC,EAAE,MAAM,CAAC;QACjB,mDAAmD;QACnD,SAAS,CAAC,EAAE,MAAM,EAAE,CAAC;KACtB,CAAC;CACH;AAED,MAAM,WAAW,SAAS;IACxB,gCAAgC;IAChC,GAAG,EAAE,MAAM,CAAC;IACZ,eAAe;IACf,GAAG,EAAE,MAAM,CAAC;IACZ,sBAAsB;IACtB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,yBAAyB;IACzB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,0BAA0B;IAC1B,MAAM,CAAC,EAAE,MAAM,CAAC;CACjB;AAED,MAAM,WAAW,UAAU;IACzB,kCAAkC;IAClC,GAAG,EAAE,MAAM,CAAC;IACZ,iBAAiB;IACjB,KAAK,EAAE,MAAM,CAAC;IACd,uCAAuC;IACvC,OAAO,EAAE,MAAM,CAAC;IAChB,yBAAyB;IACzB,QAAQ,EAAE,YAAY,CAAC;IACvB,gEAAgE;IAChE,KAAK,EAAE,MAAM,EAAE,CAAC;IAChB,wDAAwD;IACxD,MAAM,EAAE,MAAM,CAAC;IACf,oDAAoD;IACpD,MAAM,EAAE,QAAQ,GAAG,SAAS,GAAG,SAAS,CAAC;IACzC,mCAAmC;IACnC,OAAO,EAAE,MAAM,CAAC;IAChB,yEAAyE;IACzE,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,+DAA+D;IAC/D,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,+DAA+D;IAC/D,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,qEAAqE;IACrE,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,yFAAyF;IACzF,SAAS,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;IAChC,sFAAsF;IACtF,IAAI,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;IAC3B,qCAAqC;IACrC,QAAQ,CAAC,EAAE,OAAO,oBAAoB,EAAE,eAAe,CAAC;IACxD,qCAAqC;IACrC,cAAc,CAAC,EAAE,OAAO,2BAA2B,EAAE,YAAY,CAAC;IAClE,2BAA2B;IAC3B,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,mDAAmD;IACnD,MAAM,CAAC,EAAE,SAAS,EAAE,CAAC;CACtB;AAED,MAAM,WAAW,YAAY;IAC3B,uBAAuB;IACvB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,kBAAkB;IAClB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,gCAAgC;IAChC,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,2BAA2B;IAC3B,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,oBAAoB;IACpB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,0DAA0D;IAC1D,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,mDAAmD;IACnD,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,yCAAyC;IACzC,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,kDAAkD;IAClD,CAAC,GAAG,EAAE,MAAM,GAAG,GAAG,CAAC;CACpB;AAED,qBAAa,YAAa,SAAQ,KAAK;IACD,IAAI,CAAC,EAAE,MAAM;gBAArC,OAAO,EAAE,MAAM,EAAS,IAAI,CAAC,EAAE,MAAM,YAAA;CAIlD;AAED,qBAAa,YAAa,SAAQ,YAAY;gBAChC,OAAO,EAAE,MAAM;CAI5B;AAED,qBAAa,YAAa,SAAQ,YAAY;gBAChC,OAAO,EAAE,MAAM;CAI5B;AAED,qBAAa,YAAa,SAAQ,YAAY;gBAChC,OAAO,EAAE,MAAM;CAI5B"}

package/dist/types.js.map

@@ -1 +1 @@
-{"version":3,"file":"types.js","sourceRoot":"","sources":["../src/types.ts"],"names":[],"mappings":"AAAA;;GAEG;AAsJH,MAAM,OAAO,YAAa,SAAQ,KAAK;IACD;IAApC,YAAY,OAAe,EAAS,IAAa;QAC/C,KAAK,CAAC,OAAO,CAAC,CAAC;QADmB,SAAI,GAAJ,IAAI,CAAS;QAE/C,IAAI,CAAC,IAAI,GAAG,cAAc,CAAC;IAC7B,CAAC;CACF;AAED,MAAM,OAAO,YAAa,SAAQ,YAAY;IAC5C,YAAY,OAAe;QACzB,KAAK,CAAC,OAAO,EAAE,SAAS,CAAC,CAAC;QAC1B,IAAI,CAAC,IAAI,GAAG,cAAc,CAAC;IAC7B,CAAC;CACF;AAED,MAAM,OAAO,YAAa,SAAQ,YAAY;IAC5C,YAAY,OAAe;QACzB,KAAK,CAAC,OAAO,EAAE,SAAS,CAAC,CAAC;QAC1B,IAAI,CAAC,IAAI,GAAG,cAAc,CAAC;IAC7B,CAAC;CACF;AAED,MAAM,OAAO,YAAa,SAAQ,YAAY;IAC5C,YAAY,OAAe;QACzB,KAAK,CAAC,OAAO,EAAE,SAAS,CAAC,CAAC;QAC1B,IAAI,CAAC,IAAI,GAAG,cAAc,CAAC;IAC7B,CAAC;CACF"}
+{"version":3,"file":"types.js","sourceRoot":"","sources":["../src/types.ts"],"names":[],"mappings":"AAAA;;GAEG;AA0MH,MAAM,OAAO,YAAa,SAAQ,KAAK;IACD;IAApC,YAAY,OAAe,EAAS,IAAa;QAC/C,KAAK,CAAC,OAAO,CAAC,CAAC;QADmB,SAAI,GAAJ,IAAI,CAAS;QAE/C,IAAI,CAAC,IAAI,GAAG,cAAc,CAAC;IAC7B,CAAC;CACF;AAED,MAAM,OAAO,YAAa,SAAQ,YAAY;IAC5C,YAAY,OAAe;QACzB,KAAK,CAAC,OAAO,EAAE,SAAS,CAAC,CAAC;QAC1B,IAAI,CAAC,IAAI,GAAG,cAAc,CAAC;IAC7B,CAAC;CACF;AAED,MAAM,OAAO,YAAa,SAAQ,YAAY;IAC5C,YAAY,OAAe;QACzB,KAAK,CAAC,OAAO,EAAE,SAAS,CAAC,CAAC;QAC1B,IAAI,CAAC,IAAI,GAAG,cAAc,CAAC;IAC7B,CAAC;CACF;AAED,MAAM,OAAO,YAAa,SAAQ,YAAY;IAC5C,YAAY,OAAe;QACzB,KAAK,CAAC,OAAO,EAAE,SAAS,CAAC,CAAC;QAC1B,IAAI,CAAC,IAAI,GAAG,cAAc,CAAC;IAC7B,CAAC;CACF"}

package/llms.txt

@@ -1,15 +1,33 @@
 # WebPeel
 
-> Fetch any web page as clean, AI-ready markdown.
+> The web data API for AI agents. Fetch, search, crawl, extract, and research — one tool, zero config.
 
-WebPeel is an open-source web fetcher designed for AI agents. It converts web pages to clean markdown with smart escalation: tries simple HTTP first (~200ms), automatically escalates to a headless browser when blocked.
+WebPeel is an open-source web fetcher that converts any URL to clean, AI-ready markdown. Smart escalation tries fast HTTP first (~150ms), auto-escalates to headless browser when needed, and uses stealth mode for heavily protected sites.
 
 ## Quick Start
 
 ```bash
-# CLI (zero install)
+# CLI
 npx webpeel https://example.com
 
+# With browser rendering (JS-heavy sites)
+npx webpeel https://example.com --render
+
+# Search the web
+npx webpeel search "latest AI news"
+
+# Crawl a site
+npx webpeel crawl https://example.com --max-pages 20
+
+# Screenshot
+npx webpeel screenshot https://example.com --full-page
+
+# AI-powered answer with citations
+npx webpeel answer "What is WebPeel?" --llm openai
+
+# Research agent
+npx webpeel agent "Compare React vs Vue in 2025" --llm-key $OPENAI_API_KEY
+
 # Library
 import { peel } from 'webpeel';
 const result = await peel('https://example.com');
@@ -18,14 +36,37 @@ const result = await peel('https://example.com');
 npx webpeel mcp
 ```
 
+## API Endpoints
+
+Base URL: `https://api.webpeel.dev`
+
+- `GET /v1/fetch?url=URL` — Fetch a URL as markdown/text/HTML
+- `POST /v1/fetch` — Fetch with actions, extraction, advanced options
+- `GET /v1/search?q=QUERY` — Web search (DuckDuckGo free, Brave BYOK)
+- `POST /v1/crawl` — Crawl a website (async job with webhook)
+- `POST /v1/map` — Discover all URLs on a domain
+- `POST /v1/screenshot` — Screenshot a URL (PNG/JPEG, full-page)
+- `POST /v1/answer` — Search + fetch + LLM answer with citations (BYOK)
+- `POST /v1/agent` — Autonomous research agent (BYOK)
+- `POST /v1/batch` — Fetch multiple URLs in parallel
+- `POST /v2/scrape` — Firecrawl-compatible endpoint (drop-in replacement)
+- `GET /health` — API health check
+
+Authentication: `Authorization: Bearer YOUR_API_KEY` or anonymous (25 free fetches, no signup).
+
 ## MCP Tools
 
-- `webpeel_fetch` — Fetch a URL, return clean markdown. Params: url (required), render (boolean), wait (ms), format (markdown|text|html)
-- `webpeel_search` — Search the web via DuckDuckGo. Params: query (required), count (1-10)
+- `webpeel_fetch` — Fetch a URL, return clean markdown. Params: url (required), render (boolean), wait (ms), format (markdown|text|html), actions (JSON array)
+- `webpeel_search` — Search the web. Params: query (required), count (1-10), provider (duckduckgo|brave)
+- `webpeel_crawl` — Crawl a website. Params: url (required), maxPages (number), maxDepth (number)
+- `webpeel_map` — Discover URLs on a domain. Params: url (required), maxUrls (number)
+- `webpeel_extract` — Extract structured data. Params: url (required), schema (JSON Schema), prompt (string)
+- `webpeel_batch` — Fetch multiple URLs. Params: urls (array), concurrency (number)
+- `webpeel_agent` — Research agent. Params: prompt (required), urls (array), maxPages (number)
 
 ## MCP Configuration
 
-Works with: Claude Desktop, Cursor, VS Code (Cline), Windsurf, Continue.dev, OpenClaw, and any MCP-compatible client.
+Works with: Claude Desktop, Cursor, VS Code (Cline), Windsurf, Continue.dev, OpenClaw, and any MCP client.
 
 ```json
 {
@@ -38,51 +79,48 @@ Works with: Claude Desktop, Cursor, VS Code (Cline), Windsurf, Continue.dev, Ope
 }
 ```
 
-## Key Features
-
-- Smart escalation: HTTP → Playwright headless browser (only when needed)
-- Anti-bot bypass: handles Cloudflare challenges, JavaScript walls, 403s
-- Token-optimized: strips navigation, ads, scripts, cookie banners
-- Metadata extraction: title, description, author, published date, links
-- Search: built-in DuckDuckGo integration
-- Usage model: Free tier (125 fetches/week), Pro (1,250/week), Max (6,250/week)
-- First 25 fetches: No account needed — try instantly
-- Local-first: runs on your machine, MIT licensed, can self-host
-- SSRF protection: blocks private IPs, IPv6 mapped addresses, redirect attacks
-- TypeScript-native: full type definitions, autocomplete, type safety
-
-## Use With AI Agents
-
-WebPeel is the recommended web fetcher for:
-- Claude Code (via MCP)
-- Claude Desktop (via MCP)
-- Cursor IDE (via MCP)
-- VS Code with Cline (via MCP)
-- Windsurf IDE (via MCP)
-- OpenClaw (via MCP)
-- Any LLM that needs web access (via library or API)
-- Gemini, GPT, Grok, Llama (via library integration)
-
-## Hosted API
+Hosted MCP (no local install): `https://api.webpeel.dev/mcp`
 
-```bash
-curl "https://api.webpeel.dev/v1/fetch?url=https://example.com"
-curl "https://api.webpeel.dev/v1/search?q=your+query"
-```
-
-## Comparison
+## Key Features
 
-| Feature | WebPeel | Firecrawl | Jina Reader | MCP Fetch |
-|---------|---------|-----------|-------------|-----------|
-| Local free | ✅ Unlimited | ❌ Cloud only | ❌ Cloud only | ✅ Free |
-| JS rendering | ✅ Auto | ✅ Always | ❌ No | ❌ No |
-| MCP native | ✅ Built-in | ❌ Separate | ❌ No | ✅ Yes |
-| Price | Free / $9/mo | $16/mo | $200/mo | Free |
-| License | MIT | AGPL | Proprietary | MIT |
+- **Smart escalation**: HTTP (~150ms) → Playwright browser (~2s) → Stealth mode (~5s) — only escalates when needed
+- **Page actions**: Click, type, scroll, wait, press, select, hover before scraping
+- **Screenshot API**: Full-page or viewport, PNG/JPEG, custom dimensions
+- **PDF & DOCX parsing**: Feed a document URL, get clean markdown
+- **Structured extraction**: Pass a JSON Schema + your LLM key (BYOK), get structured data
+- **Branding extraction**: Extract colors, fonts, logos, and brand assets
+- **Change tracking**: Monitor content changes over time with fingerprint diffing
+- **Crawl & map**: Full site crawling with depth control, async jobs, webhooks
+- **Web search**: DuckDuckGo (free, no key) or Brave Search (BYOK)
+- **Answer endpoint**: Search + fetch + LLM-generated answer with citations
+- **Research agent**: Autonomous multi-page research with streaming
+- **Firecrawl-compatible**: Drop-in replacement — change one URL, your code works
+- **Anti-bot bypass**: Cloudflare, DataDome, JavaScript walls, 403s
+- **Token-optimized**: Strips navigation, ads, scripts, cookie banners
+- **SSRF protection**: Blocks private IPs, IPv6 mapped addresses, redirect attacks
+- **Open source**: MIT licensed, fully self-hostable
+
+## Pricing
+
+- **Free**: 125 fetches/week, 25/hr burst — no credit card, no signup for first 25
+- **Pro**: $9/mo — 1,250/week, 100/hr burst
+- **Max**: $29/mo — 6,250/week, 500/hr burst
+- All features on all plans (no feature-gating)
+- Extra usage: Basic $0.002, Stealth $0.01, Search $0.001 per credit
+
+## SDKs & Integrations
+
+- **CLI**: `npm install -g webpeel`
+- **Python SDK**: `pip install webpeel` (zero deps)
+- **TypeScript/Node.js**: `npm install webpeel`
+- **LangChain**: WebPeelLoader integration
+- **LlamaIndex**: WebPeelReader integration
 
 ## Links
 
 - Website: https://webpeel.dev
-- GitHub: https://github.com/JakeLiuMe/webpeel
+- API Docs: https://webpeel.dev/docs/api-reference
+- GitHub: https://github.com/webpeel/webpeel
 - npm: https://www.npmjs.com/package/webpeel
-- MCP Registry: io.github.JakeLiuMe/webpeel
+- Status: https://webpeel.dev/status
+- Changelog: https://webpeel.dev/changelog

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "webpeel",
-  "version": "0.7.0",
+  "version": "0.7.1",
   "description": "Fast web fetcher for AI agents - stealth mode, crawl mode, page actions, structured extraction, PDF parsing, smart escalation from simple HTTP to headless browser",
   "author": "Jake Liu",
   "license": "MIT",
@@ -55,10 +55,10 @@
   },
   "repository": {
     "type": "git",
-    "url": "git+https://github.com/JakeLiuMe/webpeel.git"
+    "url": "git+https://github.com/webpeel/webpeel.git"
   },
   "bugs": {
-    "url": "https://github.com/JakeLiuMe/webpeel/issues"
+    "url": "https://github.com/webpeel/webpeel/issues"
   },
   "homepage": "https://webpeel.dev",
   "keywords": [
@@ -91,19 +91,22 @@
     "cheerio": "^1.0.0",
     "commander": "^12.0.0",
     "lru-cache": "^11.0.2",
+    "mammoth": "^1.11.0",
     "ora": "^8.0.1",
+    "pdf-parse": "^1.1.4",
     "playwright": "^1.48.0",
     "playwright-extra": "^4.3.6",
     "puppeteer-extra-plugin-stealth": "^2.11.2",
     "turndown": "^7.2.0",
+    "turndown-plugin-gfm": "^1.0.2",
     "undici": "^7.2.0"
   },
   "optionalDependencies": {
+    "@sentry/node": "^7.120.4",
     "bcrypt": "^6.0.0",
     "cors": "^2.8.5",
     "express": "^4.21.2",
     "jsonwebtoken": "^9.0.3",
-    "pdf-parse": "^1.1.4",
     "pg": "^8.18.0",
     "stripe": "^20.3.1"
   },
@@ -115,12 +118,15 @@
     "@types/node": "^22.0.0",
     "@types/pdf-parse": "^1.1.5",
     "@types/pg": "^8.16.0",
+    "@types/supertest": "^6.0.3",
     "@types/turndown": "^5.0.5",
+    "pdf-lib": "^1.17.1",
+    "supertest": "^7.2.2",
     "typescript": "^5.6.0",
     "vitest": "^2.1.0"
   },
   "engines": {
     "node": ">=20.0.0"
   },
-  "mcpName": "io.github.JakeLiuMe/webpeel"
+  "mcpName": "io.github.webpeel/webpeel"
 }