webpeel 0.13.4 → 0.14.0

package/dist/server/routes/youtube.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"youtube.js","sourceRoot":"","sources":["../../../src/server/routes/youtube.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,EAAE,MAAM,EAAqB,MAAM,SAAS,CAAC;AACpD,OAAO,EAAE,oBAAoB,EAAE,eAAe,EAAE,MAAM,uBAAuB,CAAC;AAE9E,MAAM,UAAU,mBAAmB;IACjC,MAAM,MAAM,GAAG,MAAM,EAAE,CAAC;IAExB;;;;;;;;;;OAUG;IACH,MAAM,CAAC,GAAG,CAAC,aAAa,EAAE,KAAK,EAAE,GAAY,EAAE,GAAa,EAAE,EAAE;QAC9D,MAAM,EAAE,GAAG,EAAE,QAAQ,EAAE,GAAG,GAAG,CAAC,KAAK,CAAC;QAEpC,IAAI,CAAC,GAAG,IAAI,OAAO,GAAG,KAAK,QAAQ,EAAE,CAAC;YACpC,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC;gBACnB,KAAK,EAAE,iBAAiB;gBACxB,OAAO,EAAE,uGAAuG;gBAChH,OAAO,EAAE,4EAA4E;aACtF,CAAC,CAAC;YACH,OAAO;QACT,CAAC;QAED,MAAM,OAAO,GAAG,eAAe,CAAC,GAAG,CAAC,CAAC;QACrC,IAAI,CAAC,OAAO,EAAE,CAAC;YACb,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC;gBACnB,KAAK,EAAE,qBAAqB;gBAC5B,OAAO,EAAE,oDAAoD;gBAC7D,SAAS,EAAE;oBACT,0CAA0C;oBAC1C,2BAA2B;oBAC3B,wCAAwC;oBACxC,wCAAwC;iBACzC;aACF,CAAC,CAAC;YACH,OAAO;QACT,CAAC;QAED,IAAI,CAAC;YACH,MAAM,IAAI,GAAG,OAAO,QAAQ,KAAK,QAAQ,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC;YAE5D,MAAM,UAAU,GAAG,MAAM,oBAAoB,CAAC,GAAG,EAAE,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC,CAAC;YAEvE,GAAG,CAAC,IAAI,CAAC;gBACP,OAAO,EAAE,IAAI;gBACb,OAAO,EAAE,UAAU,CAAC,OAAO;gBAC3B,KAAK,EAAE,UAAU,CAAC,KAAK;gBACvB,OAAO,EAAE,UAAU,CAAC,OAAO;gBAC3B,QAAQ,EAAE,UAAU,CAAC,QAAQ;gBAC7B,QAAQ,EAAE,UAAU,CAAC,QAAQ;gBAC7B,kBAAkB,EAAE,UAAU,CAAC,kBAAkB;gBACjD,QAAQ,EAAE,UAAU,CAAC,QAAQ;gBAC7B,QAAQ,EAAE,UAAU,CAAC,QAAQ;gBAC7B,GAAG,EAAE,mCAAmC,OAAO,EAAE;aAClD,CAAC,CAAC;QACL,CAAC;QAAC,OAAO,KAAU,EAAE,CAAC;YACpB,MAAM,OAAO,GAAG,KAAK,EAAE,OAAO,IAAI,sCAAsC,CAAC;YAEzE,IAAI,OAAO,CAAC,QAAQ,CAAC,uBAAuB,CAAC,EAAE,CAAC;gBAC9C,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC;oBACnB,KAAK,EAAE,aAAa;oBACpB,OAAO,EAAE,6EAA6E;oBACtF,OAAO;iBACR,CAAC,CAAC;gBACH,OAAO;YACT,CAAC;YAED,IAAI,OAAO,CAAC,QAAQ,CAAC,yBAAyB,CAAC,EAAE,CAAC;gBAChD,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC;oBACnB,KAAK,EAAE,qBAAqB;oBAC5B,OAAO;iBACR,CAAC,CAAC;gBACH,OAAO;YACT,CAAC;YAED,OAAO,CAAC,KAAK,CAAC,wBAAwB,EAAE,KAAK,CAAC,CAAC;YAC/C,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC;gBACnB,KAAK,EAAE,mBAAmB;gBAC1B,OAAO,EAAE,oGAAoG;gBAC7G,MAAM,EAAE,OAAO,CAAC,GAAG,CAAC,QAAQ,KAAK,YAAY,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,SAAS;aACpE,CAAC,CAAC;QACL,CAAC;IACH,CAAC,CAAC,CAAC;IAEH,OAAO,MAAM,CAAC;AAChB,CAAC"}

package/dist/types.d.ts

@@ -169,6 +169,13 @@ export interface PeelOptions {
      * Set to true to opt out and receive the complete page.
      */
     fullPage?: boolean;
+    /**
+     * Reader mode — extract only the main article content, strip all noise.
+     * Like browser Reader Mode / Pocket / Instapaper but deterministic and fast.
+     * Returns clean markdown with metadata header (title, author, date, reading time).
+     * When enabled, readability metadata is included in result.readability.
+     */
+    readable?: boolean;
     /**
      * Intelligently scroll the page to load all lazy/infinite-scroll content
      * before extracting. Set to `true` for default settings or an object to
@@ -182,6 +189,8 @@ export interface PeelOptions {
      * { autoScroll: { maxScrolls: 10, scrollDelay: 2000, timeout: 60000 } }
      */
     autoScroll?: boolean | import('./core/actions.js').AutoScrollOptions;
+    /** Ask a question about the page content. Uses BM25 to find relevant passages — no LLM key needed. */
+    question?: string;
 }
 export interface ImageInfo {
     /** Absolute URL of the image */
@@ -234,6 +243,15 @@ export interface PeelResult {
     images?: ImageInfo[];
     /** Percentage of HTML pruned by content density scoring (0-100). Only present when pruning was applied. */
     prunedPercent?: number;
+    /**
+     * Readability extraction result (when readable option is true).
+     * Contains title, author, date, reading time, excerpt, and word count.
+     */
+    readability?: import('./core/readability.js').ReadabilityResult;
+    /** Domain-aware structured data (Twitter, Reddit, GitHub, HN). Present when URL matches a known domain. */
+    domainData?: import('./core/domain-extractors.js').DomainExtractResult;
+    /** Quick answer result (when question option is set). BM25-powered, no LLM needed. */
+    quickAnswer?: import('./core/quick-answer.js').QuickAnswerResult;
 }
 export interface PageMetadata {
     /** Meta description */

package/dist/types.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../src/types.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,MAAM,WAAW,UAAU;IACzB,IAAI,EAAE,MAAM,GAAG,OAAO,GAAG,QAAQ,GAAG,MAAM,GAAG,MAAM,GAAG,QAAQ,GAAG,OAAO,GAAG,OAAO,GAAG,iBAAiB,GAAG,YAAY,CAAC;IAEtH,gDAAgD;IAChD,QAAQ,CAAC,EAAE,MAAM,CAAC;IAElB;;;OAGG;IACH,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,IAAI,CAAC,EAAE,MAAM,CAAC;IAEd,qDAAqD;IACrD,GAAG,CAAC,EAAE,MAAM,CAAC;IAEb,0EAA0E;IAC1E,EAAE,CAAC,EAAE,MAAM,CAAC;IACZ,YAAY,CAAC,EAAE,MAAM,CAAC;IAEtB;;;OAGG;IACH,EAAE,CAAC,EAAE,KAAK,GAAG,QAAQ,GAAG,MAAM,CAAC;IAE/B,kDAAkD;IAClD,SAAS,CAAC,EAAE,IAAI,GAAG,MAAM,GAAG,MAAM,GAAG,OAAO,CAAC;IAE7C,yDAAyD;IACzD,MAAM,CAAC,EAAE,MAAM,CAAC;IAEhB,uCAAuC;IACvC,OAAO,CAAC,EAAE,MAAM,CAAC;CAClB;AAED,MAAM,WAAW,cAAc;IAC7B,wCAAwC;IACxC,MAAM,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;IAC7B,0CAA0C;IAC1C,SAAS,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IACnC,yDAAyD;IACzD,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,6DAA6D;IAC7D,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,8CAA8C;IAC9C,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,4DAA4D;IAC5D,UAAU,CAAC,EAAE,MAAM,CAAC;CACrB;AAED;;;GAGG;AACH,MAAM,WAAW,kBAAkB;IACjC,0DAA0D;IAC1D,MAAM,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;IAC7B,yDAAyD;IACzD,MAAM,CAAC,EAAE,MAAM,CAAC;CACjB;AAED,8CAA8C;AAC9C,MAAM,MAAM,iBAAiB,GAAG,QAAQ,GAAG,WAAW,GAAG,QAAQ,CAAC;AAElE,MAAM,WAAW,WAAW;IAC1B,wDAAwD;IACxD,MAAM,CAAC,EAAE,OAAO,CAAC;IACjB,+FAA+F;IAC/F,OAAO,CAAC,EAAE,OAAO,CAAC;IAClB,wEAAwE;IACxE,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,oBAAoB;IACpB,MAAM,CAAC,EAAE,UAAU,GAAG,MAAM,GAAG,MAAM,CAAC;IACtC,uDAAuD;IACvD,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,mGAAmG;IACnG,MAAM,CAAC,EAAE,OAAO,CAAC;IACjB,wBAAwB;IACxB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,uCAAuC;IACvC,UAAU,CAAC,EAAE,OAAO,CAAC;IACrB,oDAAoD;IACpD,kBAAkB,CAAC,EAAE,OAAO,CAAC;IAC7B,2FAA2F;IAC3F,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,yEAAyE;IACzE,OAAO,CAAC,EAAE,MAAM,EAAE,CAAC;IACnB,4FAA4F;IAC5F,WAAW,CAAC,EAAE,MAAM,EAAE,CAAC;IACvB,iFAAiF;IACjF,WAAW,CAAC,EAAE,MAAM,EAAE,CAAC;IACvB,kCAAkC;IAClC,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IACjC,uCAAuC;IACvC,OAAO,CAAC,EAAE,MAAM,EAAE,CAAC;IACnB,qFAAqF;IACrF,GAAG,CAAC,EAAE,OAAO,CAAC;IACd,sEAAsE;IACtE,OAAO,CAAC,EAAE,UAAU,EAAE,CAAC;IACvB,mEAAmE;IACnE,OAAO,CAAC,EAAE,cAAc,CAAC;IACzB,0EAA0E;IAC1E,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,qDAAqD;IACrD,cAAc,CAAC,EAAE,OAAO,CAAC;IACzB,4DAA4D;IAC5D,QAAQ,CAAC,EAAE,OAAO,CAAC;IACnB,qCAAqC;IACrC,OAAO,CAAC,EAAE,OAAO,GAAG;QAAE,MAAM,CAAC,EAAE,MAAM,CAAC;QAAC,SAAS,CAAC,EAAE,MAAM,CAAA;KAAE,CAAC;IAC5D,8DAA8D;IAC9D,GAAG,CAAC,EAAE;QACJ,MAAM,EAAE,MAAM,CAAC;QACf,KAAK,CAAC,EAAE,MAAM,CAAC;QACf,OAAO,CAAC,EAAE,MAAM,CAAC;KAClB,CAAC;IACF,mCAAmC;IACnC,MAAM,CAAC,EAAE,OAAO,CAAC;IACjB,8DAA8D;IAC9D,QAAQ,CAAC,EAAE;QACT,+DAA+D;QAC/D,OAAO,CAAC,EAAE,MAAM,CAAC;QACjB,mDAAmD;QACnD,SAAS,CAAC,EAAE,MAAM,EAAE,CAAC;KACtB,CAAC;IACF;;;;;OAKG;IACH,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB;;;;;;;;OAQG;IACH,KAAK,CAAC,EAAE,MAAM,CAAC;IACf;;;;;;OAMG;IACH,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB;;;;OAIG;IACH,MAAM,CAAC,EAAE,OAAO,CAAC;IACjB;;;;OAIG;IACH,YAAY,CAAC,EAAE,GAAG,CAAC;IACnB;;;;;;OAMG;IACH,SAAS,CAAC,EAAE,OAAO,CAAC;IACpB;;;;;OAKG;IACH,QAAQ,CAAC,EAAE,OAAO,CAAC;IACnB;;;;;;;;;;;OAWG;IACH,UAAU,CAAC,EAAE,OAAO,GAAG,OAAO,mBAAmB,EAAE,iBAAiB,CAAC;CACtE;AAED,MAAM,WAAW,SAAS;IACxB,gCAAgC;IAChC,GAAG,EAAE,MAAM,CAAC;IACZ,eAAe;IACf,GAAG,EAAE,MAAM,CAAC;IACZ,sBAAsB;IACtB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,yBAAyB;IACzB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,0BAA0B;IAC1B,MAAM,CAAC,EAAE,MAAM,CAAC;CACjB;AAED,MAAM,WAAW,UAAU;IACzB,kCAAkC;IAClC,GAAG,EAAE,MAAM,CAAC;IACZ,iBAAiB;IACjB,KAAK,EAAE,MAAM,CAAC;IACd,uCAAuC;IACvC,OAAO,EAAE,MAAM,CAAC;IAChB,yBAAyB;IACzB,QAAQ,EAAE,YAAY,CAAC;IACvB,gEAAgE;IAChE,KAAK,EAAE,MAAM,EAAE,CAAC;IAChB,wDAAwD;IACxD,MAAM,EAAE,MAAM,CAAC;IACf,oDAAoD;IACpD,MAAM,EAAE,QAAQ,GAAG,SAAS,GAAG,SAAS,CAAC;IACzC,mCAAmC;IACnC,OAAO,EAAE,MAAM,CAAC;IAChB,yEAAyE;IACzE,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,+DAA+D;IAC/D,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,+DAA+D;IAC/D,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,qEAAqE;IACrE,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,yFAAyF;IACzF,SAAS,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;IAChC,sFAAsF;IACtF,IAAI,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;IAC3B,qCAAqC;IACrC,QAAQ,CAAC,EAAE,OAAO,oBAAoB,EAAE,eAAe,CAAC;IACxD,qCAAqC;IACrC,cAAc,CAAC,EAAE,OAAO,2BAA2B,EAAE,YAAY,CAAC;IAClE,2BAA2B;IAC3B,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,mDAAmD;IACnD,MAAM,CAAC,EAAE,SAAS,EAAE,CAAC;IACrB,2GAA2G;IAC3G,aAAa,CAAC,EAAE,MAAM,CAAC;CACxB;AAED,MAAM,WAAW,YAAY;IAC3B,uBAAuB;IACvB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,kBAAkB;IAClB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,gCAAgC;IAChC,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,2BAA2B;IAC3B,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,oBAAoB;IACpB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,0DAA0D;IAC1D,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,mDAAmD;IACnD,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,yCAAyC;IACzC,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,kDAAkD;IAClD,CAAC,GAAG,EAAE,MAAM,GAAG,GAAG,CAAC;CACpB;AAED;;;;;;;GAOG;AACH,MAAM,WAAW,YAAY;IAC3B,kCAAkC;IAClC,GAAG,EAAE,MAAM,CAAC;IACZ,2DAA2D;IAC3D,MAAM,EAAE,MAAM,CAAC;IACf,2CAA2C;IAC3C,OAAO,EAAE,MAAM,CAAC;IAChB;;;OAGG;IACH,UAAU,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,EAAE,CAAC;IACvC,gEAAgE;IAChE,QAAQ,EAAE;QACR,KAAK,CAAC,EAAE,MAAM,CAAC;QACf,WAAW,CAAC,EAAE,MAAM,CAAC;QACrB,MAAM,CAAC,EAAE,MAAM,CAAC;QAChB,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC;KACxB,CAAC;IACF,mEAAmE;IACnE,MAAM,EAAE,MAAM,CAAC;IACf,0DAA0D;IAC1D,MAAM,EAAE,OAAO,CAAC;IAChB,yCAAyC;IACzC,OAAO,EAAE,MAAM,CAAC;IAChB;;;OAGG;IACH,SAAS,CAAC,EAAE,OAAO,CAAC;IACpB;;;OAGG;IACH,cAAc,CAAC,EAAE,MAAM,CAAC;CACzB;AAED,qBAAa,YAAa,SAAQ,KAAK;IACD,IAAI,CAAC,EAAE,MAAM;gBAArC,OAAO,EAAE,MAAM,EAAS,IAAI,CAAC,EAAE,MAAM,YAAA;CAIlD;AAED,qBAAa,YAAa,SAAQ,YAAY;gBAChC,OAAO,EAAE,MAAM;CAI5B;AAED,qBAAa,YAAa,SAAQ,YAAY;gBAChC,OAAO,EAAE,MAAM;CAI5B;AAED,qBAAa,YAAa,SAAQ,YAAY;gBAChC,OAAO,EAAE,MAAM;CAI5B"}
+{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../src/types.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,MAAM,WAAW,UAAU;IACzB,IAAI,EAAE,MAAM,GAAG,OAAO,GAAG,QAAQ,GAAG,MAAM,GAAG,MAAM,GAAG,QAAQ,GAAG,OAAO,GAAG,OAAO,GAAG,iBAAiB,GAAG,YAAY,CAAC;IAEtH,gDAAgD;IAChD,QAAQ,CAAC,EAAE,MAAM,CAAC;IAElB;;;OAGG;IACH,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,IAAI,CAAC,EAAE,MAAM,CAAC;IAEd,qDAAqD;IACrD,GAAG,CAAC,EAAE,MAAM,CAAC;IAEb,0EAA0E;IAC1E,EAAE,CAAC,EAAE,MAAM,CAAC;IACZ,YAAY,CAAC,EAAE,MAAM,CAAC;IAEtB;;;OAGG;IACH,EAAE,CAAC,EAAE,KAAK,GAAG,QAAQ,GAAG,MAAM,CAAC;IAE/B,kDAAkD;IAClD,SAAS,CAAC,EAAE,IAAI,GAAG,MAAM,GAAG,MAAM,GAAG,OAAO,CAAC;IAE7C,yDAAyD;IACzD,MAAM,CAAC,EAAE,MAAM,CAAC;IAEhB,uCAAuC;IACvC,OAAO,CAAC,EAAE,MAAM,CAAC;CAClB;AAED,MAAM,WAAW,cAAc;IAC7B,wCAAwC;IACxC,MAAM,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;IAC7B,0CAA0C;IAC1C,SAAS,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IACnC,yDAAyD;IACzD,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,6DAA6D;IAC7D,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,8CAA8C;IAC9C,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,4DAA4D;IAC5D,UAAU,CAAC,EAAE,MAAM,CAAC;CACrB;AAED;;;GAGG;AACH,MAAM,WAAW,kBAAkB;IACjC,0DAA0D;IAC1D,MAAM,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;IAC7B,yDAAyD;IACzD,MAAM,CAAC,EAAE,MAAM,CAAC;CACjB;AAED,8CAA8C;AAC9C,MAAM,MAAM,iBAAiB,GAAG,QAAQ,GAAG,WAAW,GAAG,QAAQ,CAAC;AAElE,MAAM,WAAW,WAAW;IAC1B,wDAAwD;IACxD,MAAM,CAAC,EAAE,OAAO,CAAC;IACjB,+FAA+F;IAC/F,OAAO,CAAC,EAAE,OAAO,CAAC;IAClB,wEAAwE;IACxE,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,oBAAoB;IACpB,MAAM,CAAC,EAAE,UAAU,GAAG,MAAM,GAAG,MAAM,CAAC;IACtC,uDAAuD;IACvD,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,mGAAmG;IACnG,MAAM,CAAC,EAAE,OAAO,CAAC;IACjB,wBAAwB;IACxB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,uCAAuC;IACvC,UAAU,CAAC,EAAE,OAAO,CAAC;IACrB,oDAAoD;IACpD,kBAAkB,CAAC,EAAE,OAAO,CAAC;IAC7B,2FAA2F;IAC3F,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,yEAAyE;IACzE,OAAO,CAAC,EAAE,MAAM,EAAE,CAAC;IACnB,4FAA4F;IAC5F,WAAW,CAAC,EAAE,MAAM,EAAE,CAAC;IACvB,iFAAiF;IACjF,WAAW,CAAC,EAAE,MAAM,EAAE,CAAC;IACvB,kCAAkC;IAClC,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IACjC,uCAAuC;IACvC,OAAO,CAAC,EAAE,MAAM,EAAE,CAAC;IACnB,qFAAqF;IACrF,GAAG,CAAC,EAAE,OAAO,CAAC;IACd,sEAAsE;IACtE,OAAO,CAAC,EAAE,UAAU,EAAE,CAAC;IACvB,mEAAmE;IACnE,OAAO,CAAC,EAAE,cAAc,CAAC;IACzB,0EAA0E;IAC1E,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,qDAAqD;IACrD,cAAc,CAAC,EAAE,OAAO,CAAC;IACzB,4DAA4D;IAC5D,QAAQ,CAAC,EAAE,OAAO,CAAC;IACnB,qCAAqC;IACrC,OAAO,CAAC,EAAE,OAAO,GAAG;QAAE,MAAM,CAAC,EAAE,MAAM,CAAC;QAAC,SAAS,CAAC,EAAE,MAAM,CAAA;KAAE,CAAC;IAC5D,8DAA8D;IAC9D,GAAG,CAAC,EAAE;QACJ,MAAM,EAAE,MAAM,CAAC;QACf,KAAK,CAAC,EAAE,MAAM,CAAC;QACf,OAAO,CAAC,EAAE,MAAM,CAAC;KAClB,CAAC;IACF,mCAAmC;IACnC,MAAM,CAAC,EAAE,OAAO,CAAC;IACjB,8DAA8D;IAC9D,QAAQ,CAAC,EAAE;QACT,+DAA+D;QAC/D,OAAO,CAAC,EAAE,MAAM,CAAC;QACjB,mDAAmD;QACnD,SAAS,CAAC,EAAE,MAAM,EAAE,CAAC;KACtB,CAAC;IACF;;;;;OAKG;IACH,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB;;;;;;;;OAQG;IACH,KAAK,CAAC,EAAE,MAAM,CAAC;IACf;;;;;;OAMG;IACH,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB;;;;OAIG;IACH,MAAM,CAAC,EAAE,OAAO,CAAC;IACjB;;;;OAIG;IACH,YAAY,CAAC,EAAE,GAAG,CAAC;IACnB;;;;;;OAMG;IACH,SAAS,CAAC,EAAE,OAAO,CAAC;IACpB;;;;;OAKG;IACH,QAAQ,CAAC,EAAE,OAAO,CAAC;IACnB;;;;;OAKG;IACH,QAAQ,CAAC,EAAE,OAAO,CAAC;IACnB;;;;;;;;;;;OAWG;IACH,UAAU,CAAC,EAAE,OAAO,GAAG,OAAO,mBAAmB,EAAE,iBAAiB,CAAC;IACrE,sGAAsG;IACtG,QAAQ,CAAC,EAAE,MAAM,CAAC;CACnB;AAED,MAAM,WAAW,SAAS;IACxB,gCAAgC;IAChC,GAAG,EAAE,MAAM,CAAC;IACZ,eAAe;IACf,GAAG,EAAE,MAAM,CAAC;IACZ,sBAAsB;IACtB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,yBAAyB;IACzB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,0BAA0B;IAC1B,MAAM,CAAC,EAAE,MAAM,CAAC;CACjB;AAED,MAAM,WAAW,UAAU;IACzB,kCAAkC;IAClC,GAAG,EAAE,MAAM,CAAC;IACZ,iBAAiB;IACjB,KAAK,EAAE,MAAM,CAAC;IACd,uCAAuC;IACvC,OAAO,EAAE,MAAM,CAAC;IAChB,yBAAyB;IACzB,QAAQ,EAAE,YAAY,CAAC;IACvB,gEAAgE;IAChE,KAAK,EAAE,MAAM,EAAE,CAAC;IAChB,wDAAwD;IACxD,MAAM,EAAE,MAAM,CAAC;IACf,oDAAoD;IACpD,MAAM,EAAE,QAAQ,GAAG,SAAS,GAAG,SAAS,CAAC;IACzC,mCAAmC;IACnC,OAAO,EAAE,MAAM,CAAC;IAChB,yEAAyE;IACzE,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,+DAA+D;IAC/D,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,+DAA+D;IAC/D,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,qEAAqE;IACrE,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,yFAAyF;IACzF,SAAS,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;IAChC,sFAAsF;IACtF,IAAI,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;IAC3B,qCAAqC;IACrC,QAAQ,CAAC,EAAE,OAAO,oBAAoB,EAAE,eAAe,CAAC;IACxD,qCAAqC;IACrC,cAAc,CAAC,EAAE,OAAO,2BAA2B,EAAE,YAAY,CAAC;IAClE,2BAA2B;IAC3B,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,mDAAmD;IACnD,MAAM,CAAC,EAAE,SAAS,EAAE,CAAC;IACrB,2GAA2G;IAC3G,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB;;;OAGG;IACH,WAAW,CAAC,EAAE,OAAO,uBAAuB,EAAE,iBAAiB,CAAC;IAChE,2GAA2G;IAC3G,UAAU,CAAC,EAAE,OAAO,6BAA6B,EAAE,mBAAmB,CAAC;IACvE,sFAAsF;IACtF,WAAW,CAAC,EAAE,OAAO,wBAAwB,EAAE,iBAAiB,CAAC;CAClE;AAED,MAAM,WAAW,YAAY;IAC3B,uBAAuB;IACvB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,kBAAkB;IAClB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,gCAAgC;IAChC,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,2BAA2B;IAC3B,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,oBAAoB;IACpB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,0DAA0D;IAC1D,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,mDAAmD;IACnD,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,yCAAyC;IACzC,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,kDAAkD;IAClD,CAAC,GAAG,EAAE,MAAM,GAAG,GAAG,CAAC;CACpB;AAED;;;;;;;GAOG;AACH,MAAM,WAAW,YAAY;IAC3B,kCAAkC;IAClC,GAAG,EAAE,MAAM,CAAC;IACZ,2DAA2D;IAC3D,MAAM,EAAE,MAAM,CAAC;IACf,2CAA2C;IAC3C,OAAO,EAAE,MAAM,CAAC;IAChB;;;OAGG;IACH,UAAU,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,EAAE,CAAC;IACvC,gEAAgE;IAChE,QAAQ,EAAE;QACR,KAAK,CAAC,EAAE,MAAM,CAAC;QACf,WAAW,CAAC,EAAE,MAAM,CAAC;QACrB,MAAM,CAAC,EAAE,MAAM,CAAC;QAChB,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC;KACxB,CAAC;IACF,mEAAmE;IACnE,MAAM,EAAE,MAAM,CAAC;IACf,0DAA0D;IAC1D,MAAM,EAAE,OAAO,CAAC;IAChB,yCAAyC;IACzC,OAAO,EAAE,MAAM,CAAC;IAChB;;;OAGG;IACH,SAAS,CAAC,EAAE,OAAO,CAAC;IACpB;;;OAGG;IACH,cAAc,CAAC,EAAE,MAAM,CAAC;CACzB;AAED,qBAAa,YAAa,SAAQ,KAAK;IACD,IAAI,CAAC,EAAE,MAAM;gBAArC,OAAO,EAAE,MAAM,EAAS,IAAI,CAAC,EAAE,MAAM,YAAA;CAIlD;AAED,qBAAa,YAAa,SAAQ,YAAY;gBAChC,OAAO,EAAE,MAAM;CAI5B;AAED,qBAAa,YAAa,SAAQ,YAAY;gBAChC,OAAO,EAAE,MAAM;CAI5B;AAED,qBAAa,YAAa,SAAQ,YAAY;gBAChC,OAAO,EAAE,MAAM;CAI5B"}

package/dist/types.js.map

@@ -1 +1 @@
-{"version":3,"file":"types.js","sourceRoot":"","sources":["../src/types.ts"],"names":[],"mappings":"AAAA;;GAEG;AA0TH,MAAM,OAAO,YAAa,SAAQ,KAAK;IACD;IAApC,YAAY,OAAe,EAAS,IAAa;QAC/C,KAAK,CAAC,OAAO,CAAC,CAAC;QADmB,SAAI,GAAJ,IAAI,CAAS;QAE/C,IAAI,CAAC,IAAI,GAAG,cAAc,CAAC;IAC7B,CAAC;CACF;AAED,MAAM,OAAO,YAAa,SAAQ,YAAY;IAC5C,YAAY,OAAe;QACzB,KAAK,CAAC,OAAO,EAAE,SAAS,CAAC,CAAC;QAC1B,IAAI,CAAC,IAAI,GAAG,cAAc,CAAC;IAC7B,CAAC;CACF;AAED,MAAM,OAAO,YAAa,SAAQ,YAAY;IAC5C,YAAY,OAAe;QACzB,KAAK,CAAC,OAAO,EAAE,SAAS,CAAC,CAAC;QAC1B,IAAI,CAAC,IAAI,GAAG,cAAc,CAAC;IAC7B,CAAC;CACF;AAED,MAAM,OAAO,YAAa,SAAQ,YAAY;IAC5C,YAAY,OAAe;QACzB,KAAK,CAAC,OAAO,EAAE,SAAS,CAAC,CAAC;QAC1B,IAAI,CAAC,IAAI,GAAG,cAAc,CAAC;IAC7B,CAAC;CACF"}
+{"version":3,"file":"types.js","sourceRoot":"","sources":["../src/types.ts"],"names":[],"mappings":"AAAA;;GAEG;AA4UH,MAAM,OAAO,YAAa,SAAQ,KAAK;IACD;IAApC,YAAY,OAAe,EAAS,IAAa;QAC/C,KAAK,CAAC,OAAO,CAAC,CAAC;QADmB,SAAI,GAAJ,IAAI,CAAS;QAE/C,IAAI,CAAC,IAAI,GAAG,cAAc,CAAC;IAC7B,CAAC;CACF;AAED,MAAM,OAAO,YAAa,SAAQ,YAAY;IAC5C,YAAY,OAAe;QACzB,KAAK,CAAC,OAAO,EAAE,SAAS,CAAC,CAAC;QAC1B,IAAI,CAAC,IAAI,GAAG,cAAc,CAAC;IAC7B,CAAC;CACF;AAED,MAAM,OAAO,YAAa,SAAQ,YAAY;IAC5C,YAAY,OAAe;QACzB,KAAK,CAAC,OAAO,EAAE,SAAS,CAAC,CAAC;QAC1B,IAAI,CAAC,IAAI,GAAG,cAAc,CAAC;IAC7B,CAAC;CACF;AAED,MAAM,OAAO,YAAa,SAAQ,YAAY;IAC5C,YAAY,OAAe;QACzB,KAAK,CAAC,OAAO,EAAE,SAAS,CAAC,CAAC;QAC1B,IAAI,CAAC,IAAI,GAAG,cAAc,CAAC;IAC7B,CAAC;CACF"}

package/llms.txt

@@ -1,8 +1,8 @@
 # WebPeel
 
-> The web data API for AI agents. Fetch, search, crawl, extract, and research — one tool, zero config.
+> Version: 0.14.0 | 18 MCP tools | 927 tests | The web data API for AI agents. Fetch, search, crawl, extract, and research — one tool, zero config.
 
-WebPeel is an open-source web fetcher that converts any URL to clean, AI-ready markdown. Smart escalation tries fast HTTP first (~150ms), auto-escalates to headless browser when needed, and uses stealth mode for heavily protected sites.
+WebPeel is an open-source web fetcher that converts any URL to clean, AI-ready markdown. Smart escalation tries fast HTTP first (~150ms), auto-escalates to headless browser when needed, and uses stealth mode for heavily protected sites. v0.14.0 adds YouTube transcript extraction, domain-aware extractors (Twitter/X, Reddit, GitHub, HN), LLM-free BM25 Q&A, reader mode, auto-extract, deep fetch intelligence, and URL monitoring.
 
 ## Quick Start
 
@@ -47,7 +47,12 @@ Base URL: `https://api.webpeel.dev`
 - `POST /v1/map` — Discover all URLs on a domain
 - `POST /v1/screenshot` — Screenshot a URL (PNG/JPEG, full-page)
 - `POST /v1/answer` — Search + fetch + LLM answer with citations (BYOK)
-- `POST /v1/deep_fetch` — Search + batch fetch + merge in one call. Params: query (string), count (number, default 5). Returns merged markdown with source attribution. No LLM key needed.
+- `GET /v1/answer/quick?url=URL&question=Q` — BM25 Q&A, no LLM key needed
+- `POST /v1/deep-fetch` — Search + batch fetch + BM25 merge + dedup. Params: query, count. No LLM key needed.
+- `GET /v1/youtube?url=YT_URL` — Extract YouTube video transcript. All URL formats. No API key.
+- `GET /v1/extract/auto?url=URL` — Auto-detect page type and extract structured JSON
+- `GET /v1/watch` — List URL watchers
+- `POST /v1/watch` — Create URL watcher with webhook notification
 - `POST /v1/agent` — Autonomous research agent (BYOK)
 - `POST /v1/batch` — Fetch multiple URLs in parallel
 - `POST /v2/scrape` — Firecrawl-compatible endpoint (drop-in replacement)
@@ -57,7 +62,7 @@ Authentication: `Authorization: Bearer YOUR_API_KEY` or anonymous (500 free fetc
 
 ## MCP Tools
 
-- `webpeel_fetch` — Fetch a URL, return clean markdown. Params: url (required), render (boolean), wait (ms), format (markdown|text|html), actions (JSON array)
+- `webpeel_fetch` — Fetch a URL, return clean markdown. Params: url (required), render (boolean), wait (ms), format (markdown|text|html), actions (JSON array), readable (boolean)
 - `webpeel_search` — Search the web. Params: query (required), count (1-10), provider (duckduckgo|brave)
 - `webpeel_crawl` — Crawl a website. Params: url (required), maxPages (number), maxDepth (number)
 - `webpeel_map` — Discover URLs on a domain. Params: url (required), maxUrls (number)
@@ -68,8 +73,12 @@ Authentication: `Authorization: Bearer YOUR_API_KEY` or anonymous (500 free fetc
 - `webpeel_answer` — Answer a question about any URL without BYOK. Params: url (required), question (required)
 - `webpeel_brand` — Extract brand info (name, logo, colors, social links). Params: url (required)
 - `webpeel_change_track` — Detect content changes (hash-based monitoring). Params: url (required), previousHash (string)
-- `webpeel_deep_fetch` — Search + batch fetch + merge in one call. Params: query (required), count (1-10)
+- `webpeel_deep_fetch` — Search + batch fetch + BM25 merge. Params: query (required), count (1-20). Comparison mode auto-detected.
 - `webpeel_research` — Research agent. Params: prompt (required), urls (array), maxPages (number)
+- `webpeel_youtube` — Extract YouTube video transcripts. Params: url (required). Supports all YouTube URL formats (watch, youtu.be, embed, shorts). No API key needed.
+- `webpeel_auto_extract` — Heuristic structured data extraction — auto-detects page type (pricing, product, contact, article, api-docs). Params: url (required)
+- `webpeel_quick_answer` — BM25-powered Q&A, no LLM key needed. Params: url (required), question (required)
+- `webpeel_watch` — Persistent URL change monitoring with webhook notifications. Params: url (required), webhookUrl (required), schedule (cron string)
 
 ## MCP Configuration
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "webpeel",
-  "version": "0.13.4",
+  "version": "0.14.0",
   "description": "Fast web fetcher for AI agents - stealth mode, crawl mode, page actions, structured extraction, PDF parsing, smart escalation from simple HTTP to headless browser",
   "author": "Jake Liu",
   "license": "AGPL-3.0-only",