webpeel 0.1.0

package/dist/core/markdown.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"markdown.js","sourceRoot":"","sources":["../../src/core/markdown.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,eAAe,MAAM,UAAU,CAAC;AACvC,OAAO,KAAK,OAAO,MAAM,SAAS,CAAC;AAEnC,MAAM,cAAc,GAAG;IACrB,QAAQ;IACR,OAAO;IACP,KAAK;IACL,QAAQ;IACR,oBAAoB;IACpB,OAAO;IACP,UAAU;IACV,gBAAgB;IAChB,KAAK;IACL,gBAAgB;IAChB,gBAAgB;IAChB,oBAAoB;IACpB,eAAe;IACf,gBAAgB;IAChB,WAAW;IACX,WAAW;IACX,iBAAiB;IACjB,mBAAmB;IACnB,gBAAgB;IAChB,mBAAmB;IACnB,kBAAkB;IAClB,kBAAkB;CACnB,CAAC;AAEF;;;GAGG;AACH,SAAS,SAAS,CAAC,IAAY;IAC7B,2CAA2C;IAC3C,IAAI,IAAI,CAAC,MAAM,GAAG,EAAE,GAAG,IAAI,GAAG,IAAI,EAAE,CAAC,CAAC,OAAO;QAC3C,MAAM,IAAI,KAAK,CAAC,sCAAsC,CAAC,CAAC;IAC1D,CAAC;IAED,MAAM,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAE7B,uBAAuB;IACvB,cAAc,CAAC,OAAO,CAAC,CAAC,QAAQ,EAAE,EAAE;QAClC,CAAC,CAAC,QAAQ,CAAC,CAAC,MAAM,EAAE,CAAC;IACvB,CAAC,CAAC,CAAC;IAEH,mCAAmC;IACnC,CAAC,CAAC,oBAAoB,CAAC,CAAC,MAAM,EAAE,CAAC;IAEjC,uCAAuC;IACvC,CAAC,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,EAAE;QACtB,MAAM,KAAK,GAAG,CAAC,CAAC,IAAI,CAAC,CAAC;QACtB,MAAM,IAAI,GAAG,KAAK,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE,CAAC;QACjC,IAAI,CAAC,IAAI,IAAI,KAAK,CAAC,QAAQ,EAAE,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YAC3C,KAAK,CAAC,MAAM,EAAE,CAAC;QACjB,CAAC;IACH,CAAC,CAAC,CAAC;IAEH,OAAO,CAAC,CAAC,IAAI,EAAE,CAAC;AAClB,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,cAAc,CAAC,IAAY;IACzC,MAAM,WAAW,GAAG,SAAS,CAAC,IAAI,CAAC,CAAC;IAEpC,MAAM,QAAQ,GAAG,IAAI,eAAe,CAAC;QACnC,YAAY,EAAE,KAAK;QACnB,cAAc,EAAE,QAAQ;QACxB,gBAAgB,EAAE,GAAG;QACrB,WAAW,EAAE,GAAG;QAChB,eAAe,EAAE,IAAI;KACtB,CAAC,CAAC;IAEH,kBAAkB;IAClB,QAAQ,CAAC,IAAI,CAAC,CAAC,OAAO,EAAE,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,CAAC,CAAC,CAAC;IAE7D,kDAAkD;IAClD,QAAQ,CAAC,OAAO,CAAC,QAAQ,EAAE;QACzB,MAAM,EAAE,KAAK;QACb,WAAW,EAAE,CAAC,QAAQ,EAAE,IAAI,EAAE,EAAE;YAC9B,MAAM,GAAG,GAAI,IAAY,CAAC,GAAG,CAAC;YAC9B,MAAM,GAAG,GAAI,IAAY,CAAC,GAAG,CAAC;YAC9B,IAAI,GAAG,EAAE,CAAC;gBACR,OAAO,KAAK,GAAG,KAAK,GAAG,GAAG,CAAC;YAC7B,CAAC;YACD,OAAO,EAAE,CAAC;QACZ,CAAC;KACF,CAAC,CAAC;IAEH,oCAAoC;IACpC,QAAQ,CAAC,OAAO,CAAC,YAAY,EAAE;QAC7B,MAAM,EAAE,CAAC,IAAI,EAAE,EAAE;YACf,OAAO,IAAI,CAAC,QAAQ,KAAK,KAAK,IAAI,IAAI,CAAC,UAAU,EAAE,QAAQ,KAAK,MAAM,CAAC;QACzE,CAAC;QACD,WAAW,EAAE,CAAC,QAAQ,EAAE,IAAI,EAAE,EAAE;YAC9B,MAAM,QAAQ,GAAG,IAAI,CAAC,UAAiB,CAAC;YACxC,MAAM,SAAS,GAAG,QAAQ,CAAC,YAAY,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC;YACvD,MAAM,QAAQ,GAAG,SAAS,CAAC,KAAK,CAAC,gBAAgB,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;YAC9D,OAAO,SAAS,GAAG,QAAQ,GAAG,IAAI,GAAG,QAAQ,CAAC,WAAW,GAAG,WAAW,CAAC;QAC1E,CAAC;KACF,CAAC,CAAC;IAEH,IAAI,QAAQ,GAAG,QAAQ,CAAC,QAAQ,CAAC,WAAW,CAAC,CAAC;IAE9C,kEAAkE;IAClE,IAAI,QAAQ,CAAC,MAAM,GAAG,IAAI,GAAG,IAAI,EAAE,CAAC,CAAC,yBAAyB;QAC5D,QAAQ,GAAG,QAAQ,CAAC,KAAK,CAAC,CAAC,EAAE,IAAI,GAAG,IAAI,CAAC,CAAC;IAC5C,CAAC;IAED,8DAA8D;IAC9D,QAAQ,GAAG,QAAQ,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,IAAI,EAAE,CAAC,EAAE,GAAG,EAAE,EAAE;QAC3D,IAAI,CAAC,KAAK,CAAC;YAAE,OAAO,IAAI,CAAC;QACzB,MAAM,SAAS,GAAG,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,EAAE,KAAK,EAAE,CAAC;QAC3C,MAAM,SAAS,GAAG,IAAI,CAAC,IAAI,EAAE,KAAK,EAAE,CAAC;QACrC,IAAI,SAAS,IAAI,SAAS;YAAE,OAAO,GAAG,CAAC;QACvC,OAAO,GAAG,GAAG,IAAI,GAAG,IAAI,CAAC;IAC3B,CAAC,EAAE,EAAE,CAAC,CAAC;IAEP,qCAAqC;IACrC,QAAQ,GAAG,QAAQ,CAAC,IAAI,EAAE,CAAC;IAE3B,OAAO,QAAQ,CAAC;AAClB,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,UAAU,CAAC,IAAY;IACrC,MAAM,WAAW,GAAG,SAAS,CAAC,IAAI,CAAC,CAAC;IACpC,MAAM,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;IAEpC,8CAA8C;IAC9C,IAAI,IAAI,GAAG,EAAE,CAAC;IACd,CAAC,CAAC,+BAA+B,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,EAAE;QAClD,MAAM,OAAO,GAAG,CAAC,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE,CAAC;QACtC,IAAI,OAAO,EAAE,CAAC;YACZ,IAAI,IAAI,OAAO,GAAG,MAAM,CAAC;QAC3B,CAAC;IACH,CAAC,CAAC,CAAC;IAEH,yDAAyD;IACzD,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,EAAE,CAAC;QACjB,IAAI,GAAG,CAAC,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,CAAC;IAC1B,CAAC;IAED,gCAAgC;IAChC,IAAI,GAAG,IAAI,CAAC,OAAO,CAAC,SAAS,EAAE,MAAM,CAAC,CAAC;IACvC,IAAI,GAAG,IAAI,CAAC,OAAO,CAAC,SAAS,EAAE,GAAG,CAAC,CAAC;IAEpC,OAAO,IAAI,CAAC,IAAI,EAAE,CAAC;AACrB,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,cAAc,CAAC,IAAY;IACzC,OAAO,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;AACpC,CAAC"}

package/dist/core/metadata.d.ts

@@ -0,0 +1,17 @@
+/**
+ * Extract structured metadata from HTML
+ */
+import type { PageMetadata } from '../types.js';
+/**
+ * Extract all links from page
+ * Returns absolute URLs, deduplicated
+ */
+export declare function extractLinks(html: string, baseUrl: string): string[];
+/**
+ * Extract all metadata from HTML
+ */
+export declare function extractMetadata(html: string, _url: string): {
+    title: string;
+    metadata: PageMetadata;
+};
+//# sourceMappingURL=metadata.d.ts.map

package/dist/core/metadata.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"metadata.d.ts","sourceRoot":"","sources":["../../src/core/metadata.ts"],"names":[],"mappings":"AAAA;;GAEG;AAGH,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AAsHhD;;;GAGG;AACH,wBAAgB,YAAY,CAAC,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,GAAG,MAAM,EAAE,CA4BpE;AAED;;GAEG;AACH,wBAAgB,eAAe,CAAC,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,GAAG;IAAE,KAAK,EAAE,MAAM,CAAC;IAAC,QAAQ,EAAE,YAAY,CAAA;CAAE,CAarG"}

package/dist/core/metadata.js

@@ -0,0 +1,159 @@
+/**
+ * Extract structured metadata from HTML
+ */
+import * as cheerio from 'cheerio';
+/**
+ * Extract page title using fallback chain:
+ * og:title → twitter:title → title tag → h1
+ */
+function extractTitle($) {
+    // Try Open Graph title
+    let title = $('meta[property="og:title"]').attr('content');
+    if (title)
+        return title.trim();
+    // Try Twitter title
+    title = $('meta[name="twitter:title"]').attr('content');
+    if (title)
+        return title.trim();
+    // Try title tag
+    title = $('title').text();
+    if (title)
+        return title.trim();
+    // Fallback to first h1
+    title = $('h1').first().text();
+    if (title)
+        return title.trim();
+    return '';
+}
+/**
+ * Extract page description using fallback chain:
+ * og:description → twitter:description → meta description
+ */
+function extractDescription($) {
+    // Try Open Graph description
+    let desc = $('meta[property="og:description"]').attr('content');
+    if (desc)
+        return desc.trim();
+    // Try Twitter description
+    desc = $('meta[name="twitter:description"]').attr('content');
+    if (desc)
+        return desc.trim();
+    // Try standard meta description
+    desc = $('meta[name="description"]').attr('content');
+    if (desc)
+        return desc.trim();
+    return undefined;
+}
+/**
+ * Extract author from meta tags
+ */
+function extractAuthor($) {
+    // Try article:author
+    let author = $('meta[property="article:author"]').attr('content');
+    if (author)
+        return author.trim();
+    // Try author meta tag
+    author = $('meta[name="author"]').attr('content');
+    if (author)
+        return author.trim();
+    return undefined;
+}
+/**
+ * Extract published date from meta tags
+ * Returns ISO 8601 date string if found
+ */
+function extractPublished($) {
+    // Try article:published_time
+    let published = $('meta[property="article:published_time"]').attr('content');
+    if (published) {
+        try {
+            return new Date(published).toISOString();
+        }
+        catch {
+            // Invalid date, continue
+        }
+    }
+    // Try datePublished schema.org
+    published = $('meta[itemprop="datePublished"]').attr('content');
+    if (published) {
+        try {
+            return new Date(published).toISOString();
+        }
+        catch {
+            // Invalid date, continue
+        }
+    }
+    return undefined;
+}
+/**
+ * Extract Open Graph image URL
+ */
+function extractImage($) {
+    // Try og:image
+    let image = $('meta[property="og:image"]').attr('content');
+    if (image)
+        return image.trim();
+    // Try twitter:image
+    image = $('meta[name="twitter:image"]').attr('content');
+    if (image)
+        return image.trim();
+    return undefined;
+}
+/**
+ * Extract canonical URL
+ */
+function extractCanonical($) {
+    const canonical = $('link[rel="canonical"]').attr('href');
+    if (canonical)
+        return canonical.trim();
+    // Fallback to og:url
+    const ogUrl = $('meta[property="og:url"]').attr('content');
+    if (ogUrl)
+        return ogUrl.trim();
+    return undefined;
+}
+/**
+ * Extract all links from page
+ * Returns absolute URLs, deduplicated
+ */
+export function extractLinks(html, baseUrl) {
+    const $ = cheerio.load(html);
+    const links = new Set();
+    $('a[href]').each((_, elem) => {
+        const href = $(elem).attr('href');
+        if (!href)
+            return;
+        try {
+            const absoluteUrl = new URL(href, baseUrl);
+            // SECURITY: Only allow HTTP and HTTPS protocols
+            if (!['http:', 'https:'].includes(absoluteUrl.protocol)) {
+                return;
+            }
+            // Skip anchor links
+            if (absoluteUrl.hash && absoluteUrl.href === baseUrl + absoluteUrl.hash) {
+                return;
+            }
+            links.add(absoluteUrl.href);
+        }
+        catch {
+            // Invalid URL, skip
+        }
+    });
+    return Array.from(links).sort();
+}
+/**
+ * Extract all metadata from HTML
+ */
+export function extractMetadata(html, _url) {
+    const $ = cheerio.load(html);
+    const title = extractTitle($);
+    const metadata = {
+        description: extractDescription($),
+        author: extractAuthor($),
+        published: extractPublished($),
+        image: extractImage($),
+        canonical: extractCanonical($),
+    };
+    return { title, metadata };
+}
+//# sourceMappingURL=metadata.js.map

package/dist/core/metadata.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"metadata.js","sourceRoot":"","sources":["../../src/core/metadata.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,KAAK,OAAO,MAAM,SAAS,CAAC;AAGnC;;;GAGG;AACH,SAAS,YAAY,CAAC,CAAqB;IACzC,uBAAuB;IACvB,IAAI,KAAK,GAAG,CAAC,CAAC,2BAA2B,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;IAC3D,IAAI,KAAK;QAAE,OAAO,KAAK,CAAC,IAAI,EAAE,CAAC;IAE/B,oBAAoB;IACpB,KAAK,GAAG,CAAC,CAAC,4BAA4B,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;IACxD,IAAI,KAAK;QAAE,OAAO,KAAK,CAAC,IAAI,EAAE,CAAC;IAE/B,gBAAgB;IAChB,KAAK,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,IAAI,EAAE,CAAC;IAC1B,IAAI,KAAK;QAAE,OAAO,KAAK,CAAC,IAAI,EAAE,CAAC;IAE/B,uBAAuB;IACvB,KAAK,GAAG,CAAC,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,CAAC,IAAI,EAAE,CAAC;IAC/B,IAAI,KAAK;QAAE,OAAO,KAAK,CAAC,IAAI,EAAE,CAAC;IAE/B,OAAO,EAAE,CAAC;AACZ,CAAC;AAED;;;GAGG;AACH,SAAS,kBAAkB,CAAC,CAAqB;IAC/C,6BAA6B;IAC7B,IAAI,IAAI,GAAG,CAAC,CAAC,iCAAiC,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;IAChE,IAAI,IAAI;QAAE,OAAO,IAAI,CAAC,IAAI,EAAE,CAAC;IAE7B,0BAA0B;IAC1B,IAAI,GAAG,CAAC,CAAC,kCAAkC,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;IAC7D,IAAI,IAAI;QAAE,OAAO,IAAI,CAAC,IAAI,EAAE,CAAC;IAE7B,gCAAgC;IAChC,IAAI,GAAG,CAAC,CAAC,0BAA0B,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;IACrD,IAAI,IAAI;QAAE,OAAO,IAAI,CAAC,IAAI,EAAE,CAAC;IAE7B,OAAO,SAAS,CAAC;AACnB,CAAC;AAED;;GAEG;AACH,SAAS,aAAa,CAAC,CAAqB;IAC1C,qBAAqB;IACrB,IAAI,MAAM,GAAG,CAAC,CAAC,iCAAiC,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;IAClE,IAAI,MAAM;QAAE,OAAO,MAAM,CAAC,IAAI,EAAE,CAAC;IAEjC,sBAAsB;IACtB,MAAM,GAAG,CAAC,CAAC,qBAAqB,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;IAClD,IAAI,MAAM;QAAE,OAAO,MAAM,CAAC,IAAI,EAAE,CAAC;IAEjC,OAAO,SAAS,CAAC;AACnB,CAAC;AAED;;;GAGG;AACH,SAAS,gBAAgB,CAAC,CAAqB;IAC7C,6BAA6B;IAC7B,IAAI,SAAS,GAAG,CAAC,CAAC,yCAAyC,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;IAC7E,IAAI,SAAS,EAAE,CAAC;QACd,IAAI,CAAC;YACH,OAAO,IAAI,IAAI,CAAC,SAAS,CAAC,CAAC,WAAW,EAAE,CAAC;QAC3C,CAAC;QAAC,MAAM,CAAC;YACP,yBAAyB;QAC3B,CAAC;IACH,CAAC;IAED,+BAA+B;IAC/B,SAAS,GAAG,CAAC,CAAC,gCAAgC,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;IAChE,IAAI,SAAS,EAAE,CAAC;QACd,IAAI,CAAC;YACH,OAAO,IAAI,IAAI,CAAC,SAAS,CAAC,CAAC,WAAW,EAAE,CAAC;QAC3C,CAAC;QAAC,MAAM,CAAC;YACP,yBAAyB;QAC3B,CAAC;IACH,CAAC;IAED,OAAO,SAAS,CAAC;AACnB,CAAC;AAED;;GAEG;AACH,SAAS,YAAY,CAAC,CAAqB;IACzC,eAAe;IACf,IAAI,KAAK,GAAG,CAAC,CAAC,2BAA2B,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;IAC3D,IAAI,KAAK;QAAE,OAAO,KAAK,CAAC,IAAI,EAAE,CAAC;IAE/B,oBAAoB;IACpB,KAAK,GAAG,CAAC,CAAC,4BAA4B,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;IACxD,IAAI,KAAK;QAAE,OAAO,KAAK,CAAC,IAAI,EAAE,CAAC;IAE/B,OAAO,SAAS,CAAC;AACnB,CAAC;AAED;;GAEG;AACH,SAAS,gBAAgB,CAAC,CAAqB;IAC7C,MAAM,SAAS,GAAG,CAAC,CAAC,uBAAuB,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;IAC1D,IAAI,SAAS;QAAE,OAAO,SAAS,CAAC,IAAI,EAAE,CAAC;IAEvC,qBAAqB;IACrB,MAAM,KAAK,GAAG,CAAC,CAAC,yBAAyB,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;IAC3D,IAAI,KAAK;QAAE,OAAO,KAAK,CAAC,IAAI,EAAE,CAAC;IAE/B,OAAO,SAAS,CAAC;AACnB,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,YAAY,CAAC,IAAY,EAAE,OAAe;IACxD,MAAM,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAC7B,MAAM,KAAK,GAAG,IAAI,GAAG,EAAU,CAAC;IAEhC,CAAC,CAAC,SAAS,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,EAAE;QAC5B,MAAM,IAAI,GAAG,CAAC,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QAClC,IAAI,CAAC,IAAI;YAAE,OAAO;QAElB,IAAI,CAAC;YACH,MAAM,WAAW,GAAG,IAAI,GAAG,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;YAE3C,gDAAgD;YAChD,IAAI,CAAC,CAAC,OAAO,EAAE,QAAQ,CAAC,CAAC,QAAQ,CAAC,WAAW,CAAC,QAAQ,CAAC,EAAE,CAAC;gBACxD,OAAO;YACT,CAAC;YAED,oBAAoB;YACpB,IAAI,WAAW,CAAC,IAAI,IAAI,WAAW,CAAC,IAAI,KAAK,OAAO,GAAG,WAAW,CAAC,IAAI,EAAE,CAAC;gBACxE,OAAO;YACT,CAAC;YAED,KAAK,CAAC,GAAG,CAAC,WAAW,CAAC,IAAI,CAAC,CAAC;QAC9B,CAAC;QAAC,MAAM,CAAC;YACP,oBAAoB;QACtB,CAAC;IACH,CAAC,CAAC,CAAC;IAEH,OAAO,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,IAAI,EAAE,CAAC;AAClC,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,eAAe,CAAC,IAAY,EAAE,IAAY;IACxD,MAAM,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAE7B,MAAM,KAAK,GAAG,YAAY,CAAC,CAAC,CAAC,CAAC;IAC9B,MAAM,QAAQ,GAAiB;QAC7B,WAAW,EAAE,kBAAkB,CAAC,CAAC,CAAC;QAClC,MAAM,EAAE,aAAa,CAAC,CAAC,CAAC;QACxB,SAAS,EAAE,gBAAgB,CAAC,CAAC,CAAC;QAC9B,KAAK,EAAE,YAAY,CAAC,CAAC,CAAC;QACtB,SAAS,EAAE,gBAAgB,CAAC,CAAC,CAAC;KAC/B,CAAC;IAEF,OAAO,EAAE,KAAK,EAAE,QAAQ,EAAE,CAAC;AAC7B,CAAC"}

package/dist/core/strategies.d.ts

@@ -0,0 +1,30 @@
+/**
+ * Smart escalation strategy: try simple fetch first, escalate to browser if needed
+ */
+import { type FetchResult } from './fetcher.js';
+export interface StrategyOptions {
+    /** Force browser mode (skip simple fetch) */
+    forceBrowser?: boolean;
+    /** Wait time after page load in browser mode (ms) */
+    waitMs?: number;
+    /** Custom user agent */
+    userAgent?: string;
+    /** Request timeout (ms) */
+    timeoutMs?: number;
+}
+export interface StrategyResult extends FetchResult {
+    /** Which strategy succeeded: 'simple' | 'browser' */
+    method: 'simple' | 'browser';
+}
+/**
+ * Smart fetch with automatic escalation
+ *
+ * Strategy:
+ * 1. Try simple HTTP fetch first (fast, ~200ms)
+ * 2. If blocked (403, 503, Cloudflare, empty body) → try browser
+ * 3. If browser encounters Cloudflare challenge → wait 5s and retry
+ *
+ * Returns the result along with which method worked
+ */
+export declare function smartFetch(url: string, options?: StrategyOptions): Promise<StrategyResult>;
+//# sourceMappingURL=strategies.d.ts.map

package/dist/core/strategies.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"strategies.d.ts","sourceRoot":"","sources":["../../src/core/strategies.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,EAAyC,KAAK,WAAW,EAAE,MAAM,cAAc,CAAC;AAGvF,MAAM,WAAW,eAAe;IAC9B,6CAA6C;IAC7C,YAAY,CAAC,EAAE,OAAO,CAAC;IACvB,qDAAqD;IACrD,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,wBAAwB;IACxB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,2BAA2B;IAC3B,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB;AAED,MAAM,WAAW,cAAe,SAAQ,WAAW;IACjD,qDAAqD;IACrD,MAAM,EAAE,QAAQ,GAAG,SAAS,CAAC;CAC9B;AAED;;;;;;;;;GASG;AACH,wBAAsB,UAAU,CAAC,GAAG,EAAE,MAAM,EAAE,OAAO,GAAE,eAAoB,GAAG,OAAO,CAAC,cAAc,CAAC,CAuDpG"}

package/dist/core/strategies.js

@@ -0,0 +1,67 @@
+/**
+ * Smart escalation strategy: try simple fetch first, escalate to browser if needed
+ */
+import { simpleFetch, browserFetch, retryFetch } from './fetcher.js';
+import { BlockedError, NetworkError } from '../types.js';
+/**
+ * Smart fetch with automatic escalation
+ *
+ * Strategy:
+ * 1. Try simple HTTP fetch first (fast, ~200ms)
+ * 2. If blocked (403, 503, Cloudflare, empty body) → try browser
+ * 3. If browser encounters Cloudflare challenge → wait 5s and retry
+ *
+ * Returns the result along with which method worked
+ */
+export async function smartFetch(url, options = {}) {
+    const { forceBrowser = false, waitMs = 0, userAgent, timeoutMs = 30000 } = options;
+    // Strategy 1: Simple fetch (unless browser is forced)
+    if (!forceBrowser) {
+        try {
+            const result = await retryFetch(() => simpleFetch(url, userAgent, timeoutMs), 3);
+            return {
+                ...result,
+                method: 'simple',
+            };
+        }
+        catch (error) {
+            // If blocked or needs JS, escalate to browser
+            if (error instanceof BlockedError) {
+                // Fall through to browser strategy
+            }
+            else {
+                // Re-throw other errors (timeout, network errors)
+                throw error;
+            }
+        }
+    }
+    // Strategy 2: Browser fetch
+    try {
+        const result = await browserFetch(url, {
+            userAgent,
+            waitMs,
+            timeoutMs,
+        });
+        return {
+            ...result,
+            method: 'browser',
+        };
+    }
+    catch (error) {
+        // If browser encounters Cloudflare, retry with extra wait time
+        if (error instanceof NetworkError &&
+            error.message.toLowerCase().includes('cloudflare')) {
+            const result = await browserFetch(url, {
+                userAgent,
+                waitMs: 5000, // Wait 5s for Cloudflare challenge
+                timeoutMs,
+            });
+            return {
+                ...result,
+                method: 'browser',
+            };
+        }
+        throw error;
+    }
+}
+//# sourceMappingURL=strategies.js.map

package/dist/core/strategies.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"strategies.js","sourceRoot":"","sources":["../../src/core/strategies.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,EAAE,WAAW,EAAE,YAAY,EAAE,UAAU,EAAoB,MAAM,cAAc,CAAC;AACvF,OAAO,EAAE,YAAY,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AAkBzD;;;;;;;;;GASG;AACH,MAAM,CAAC,KAAK,UAAU,UAAU,CAAC,GAAW,EAAE,UAA2B,EAAE;IACzE,MAAM,EAAE,YAAY,GAAG,KAAK,EAAE,MAAM,GAAG,CAAC,EAAE,SAAS,EAAE,SAAS,GAAG,KAAK,EAAE,GAAG,OAAO,CAAC;IAEnF,sDAAsD;IACtD,IAAI,CAAC,YAAY,EAAE,CAAC;QAClB,IAAI,CAAC;YACH,MAAM,MAAM,GAAG,MAAM,UAAU,CAC7B,GAAG,EAAE,CAAC,WAAW,CAAC,GAAG,EAAE,SAAS,EAAE,SAAS,CAAC,EAC5C,CAAC,CACF,CAAC;YACF,OAAO;gBACL,GAAG,MAAM;gBACT,MAAM,EAAE,QAAQ;aACjB,CAAC;QACJ,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,8CAA8C;YAC9C,IAAI,KAAK,YAAY,YAAY,EAAE,CAAC;gBAClC,mCAAmC;YACrC,CAAC;iBAAM,CAAC;gBACN,kDAAkD;gBAClD,MAAM,KAAK,CAAC;YACd,CAAC;QACH,CAAC;IACH,CAAC;IAED,4BAA4B;IAC5B,IAAI,CAAC;QACH,MAAM,MAAM,GAAG,MAAM,YAAY,CAAC,GAAG,EAAE;YACrC,SAAS;YACT,MAAM;YACN,SAAS;SACV,CAAC,CAAC;QACH,OAAO;YACL,GAAG,MAAM;YACT,MAAM,EAAE,SAAS;SAClB,CAAC;IACJ,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,+DAA+D;QAC/D,IACE,KAAK,YAAY,YAAY;YAC7B,KAAK,CAAC,OAAO,CAAC,WAAW,EAAE,CAAC,QAAQ,CAAC,YAAY,CAAC,EAClD,CAAC;YACD,MAAM,MAAM,GAAG,MAAM,YAAY,CAAC,GAAG,EAAE;gBACrC,SAAS;gBACT,MAAM,EAAE,IAAI,EAAE,mCAAmC;gBACjD,SAAS;aACV,CAAC,CAAC;YACH,OAAO;gBACL,GAAG,MAAM;gBACT,MAAM,EAAE,SAAS;aAClB,CAAC;QACJ,CAAC;QAED,MAAM,KAAK,CAAC;IACd,CAAC;AACH,CAAC"}

package/dist/index.d.ts

@@ -0,0 +1,31 @@
+/**
+ * WebPeel - Fast web fetcher for AI agents
+ *
+ * Main library export
+ */
+import { cleanup } from './core/fetcher.js';
+import type { PeelOptions, PeelResult } from './types.js';
+export * from './types.js';
+/**
+ * Fetch and extract content from a URL
+ *
+ * @param url - URL to fetch
+ * @param options - Fetch options
+ * @returns Extracted content and metadata
+ *
+ * @example
+ * ```typescript
+ * import { peel } from 'webpeel';
+ *
+ * const result = await peel('https://example.com');
+ * console.log(result.content); // Markdown content
+ * console.log(result.metadata); // Structured metadata
+ * ```
+ */
+export declare function peel(url: string, options?: PeelOptions): Promise<PeelResult>;
+/**
+ * Clean up any browser resources
+ * Call this when you're done using WebPeel
+ */
+export { cleanup };
+//# sourceMappingURL=index.d.ts.map

package/dist/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAKH,OAAO,EAAE,OAAO,EAAE,MAAM,mBAAmB,CAAC;AAC5C,OAAO,KAAK,EAAE,WAAW,EAAE,UAAU,EAAE,MAAM,YAAY,CAAC;AAE1D,cAAc,YAAY,CAAC;AAE3B;;;;;;;;;;;;;;;GAeG;AACH,wBAAsB,IAAI,CAAC,GAAG,EAAE,MAAM,EAAE,OAAO,GAAE,WAAgB,GAAG,OAAO,CAAC,UAAU,CAAC,CA4DtF;AAED;;;GAGG;AACH,OAAO,EAAE,OAAO,EAAE,CAAC"}

package/dist/index.js

@@ -0,0 +1,81 @@
+/**
+ * WebPeel - Fast web fetcher for AI agents
+ *
+ * Main library export
+ */
+import { smartFetch } from './core/strategies.js';
+import { htmlToMarkdown, htmlToText, estimateTokens } from './core/markdown.js';
+import { extractMetadata, extractLinks } from './core/metadata.js';
+import { cleanup } from './core/fetcher.js';
+export * from './types.js';
+/**
+ * Fetch and extract content from a URL
+ *
+ * @param url - URL to fetch
+ * @param options - Fetch options
+ * @returns Extracted content and metadata
+ *
+ * @example
+ * ```typescript
+ * import { peel } from 'webpeel';
+ *
+ * const result = await peel('https://example.com');
+ * console.log(result.content); // Markdown content
+ * console.log(result.metadata); // Structured metadata
+ * ```
+ */
+export async function peel(url, options = {}) {
+    const startTime = Date.now();
+    const { render = false, wait = 0, format = 'markdown', timeout = 30000, userAgent, } = options;
+    try {
+        // Fetch the page
+        const fetchResult = await smartFetch(url, {
+            forceBrowser: render,
+            waitMs: wait,
+            userAgent,
+            timeoutMs: timeout,
+        });
+        // Extract metadata and title
+        const { title, metadata } = extractMetadata(fetchResult.html, fetchResult.url);
+        // Extract links
+        const links = extractLinks(fetchResult.html, fetchResult.url);
+        // Convert content to requested format
+        let content;
+        switch (format) {
+            case 'html':
+                content = fetchResult.html;
+                break;
+            case 'text':
+                content = htmlToText(fetchResult.html);
+                break;
+            case 'markdown':
+            default:
+                content = htmlToMarkdown(fetchResult.html);
+                break;
+        }
+        // Calculate elapsed time and token estimate
+        const elapsed = Date.now() - startTime;
+        const tokens = estimateTokens(content);
+        return {
+            url: fetchResult.url,
+            title,
+            content,
+            metadata,
+            links,
+            tokens,
+            method: fetchResult.method,
+            elapsed,
+        };
+    }
+    catch (error) {
+        // Clean up browser resources on error
+        await cleanup();
+        throw error;
+    }
+}
+/**
+ * Clean up any browser resources
+ * Call this when you're done using WebPeel
+ */
+export { cleanup };
+//# sourceMappingURL=index.js.map

package/dist/index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,OAAO,EAAE,UAAU,EAAE,MAAM,sBAAsB,CAAC;AAClD,OAAO,EAAE,cAAc,EAAE,UAAU,EAAE,cAAc,EAAE,MAAM,oBAAoB,CAAC;AAChF,OAAO,EAAE,eAAe,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAC;AACnE,OAAO,EAAE,OAAO,EAAE,MAAM,mBAAmB,CAAC;AAG5C,cAAc,YAAY,CAAC;AAE3B;;;;;;;;;;;;;;;GAeG;AACH,MAAM,CAAC,KAAK,UAAU,IAAI,CAAC,GAAW,EAAE,UAAuB,EAAE;IAC/D,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IAE7B,MAAM,EACJ,MAAM,GAAG,KAAK,EACd,IAAI,GAAG,CAAC,EACR,MAAM,GAAG,UAAU,EACnB,OAAO,GAAG,KAAK,EACf,SAAS,GACV,GAAG,OAAO,CAAC;IAEZ,IAAI,CAAC;QACH,iBAAiB;QACjB,MAAM,WAAW,GAAG,MAAM,UAAU,CAAC,GAAG,EAAE;YACxC,YAAY,EAAE,MAAM;YACpB,MAAM,EAAE,IAAI;YACZ,SAAS;YACT,SAAS,EAAE,OAAO;SACnB,CAAC,CAAC;QAEH,6BAA6B;QAC7B,MAAM,EAAE,KAAK,EAAE,QAAQ,EAAE,GAAG,eAAe,CAAC,WAAW,CAAC,IAAI,EAAE,WAAW,CAAC,GAAG,CAAC,CAAC;QAE/E,gBAAgB;QAChB,MAAM,KAAK,GAAG,YAAY,CAAC,WAAW,CAAC,IAAI,EAAE,WAAW,CAAC,GAAG,CAAC,CAAC;QAE9D,sCAAsC;QACtC,IAAI,OAAe,CAAC;QACpB,QAAQ,MAAM,EAAE,CAAC;YACf,KAAK,MAAM;gBACT,OAAO,GAAG,WAAW,CAAC,IAAI,CAAC;gBAC3B,MAAM;YACR,KAAK,MAAM;gBACT,OAAO,GAAG,UAAU,CAAC,WAAW,CAAC,IAAI,CAAC,CAAC;gBACvC,MAAM;YACR,KAAK,UAAU,CAAC;YAChB;gBACE,OAAO,GAAG,cAAc,CAAC,WAAW,CAAC,IAAI,CAAC,CAAC;gBAC3C,MAAM;QACV,CAAC;QAED,4CAA4C;QAC5C,MAAM,OAAO,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS,CAAC;QACvC,MAAM,MAAM,GAAG,cAAc,CAAC,OAAO,CAAC,CAAC;QAEvC,OAAO;YACL,GAAG,EAAE,WAAW,CAAC,GAAG;YACpB,KAAK;YACL,OAAO;YACP,QAAQ;YACR,KAAK;YACL,MAAM;YACN,MAAM,EAAE,WAAW,CAAC,MAAM;YAC1B,OAAO;SACR,CAAC;IACJ,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,sCAAsC;QACtC,MAAM,OAAO,EAAE,CAAC;QAChB,MAAM,KAAK,CAAC;IACd,CAAC;AACH,CAAC;AAED;;;GAGG;AACH,OAAO,EAAE,OAAO,EAAE,CAAC"}

package/dist/mcp/server.d.ts

@@ -0,0 +1,7 @@
+#!/usr/bin/env node
+/**
+ * MCP Server for WebPeel
+ * Provides webpeel_fetch and webpeel_search tools for Claude Desktop / Cursor
+ */
+export {};
+//# sourceMappingURL=server.d.ts.map

package/dist/mcp/server.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"server.d.ts","sourceRoot":"","sources":["../../src/mcp/server.ts"],"names":[],"mappings":";AAEA;;;GAGG"}