webpeel 0.1.0

package/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 Jake Liu
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

package/README.md

@@ -0,0 +1,415 @@
+# WebPeel
+
+[![npm version](https://img.shields.io/npm/v/webpeel.svg)](https://www.npmjs.com/package/webpeel)
+[![npm downloads](https://img.shields.io/npm/dm/webpeel.svg)](https://www.npmjs.com/package/webpeel)
+[![CI](https://github.com/JakeLiuMe/webpeel/actions/workflows/ci.yml/badge.svg)](https://github.com/JakeLiuMe/webpeel/actions/workflows/ci.yml)
+[![TypeScript](https://img.shields.io/badge/TypeScript-5.6-blue.svg)](https://www.typescriptlang.org/)
+[![License: MIT](https://img.shields.io/badge/License-MIT-blue.svg)](https://opensource.org/licenses/MIT)
+
+Turn any web page into clean markdown. Zero config. Free forever.
+
+```bash
+npx webpeel https://news.ycombinator.com
+```
+
+**Output:**
+```markdown
+# Hacker News
+
+**New** | **Past** | **Comments** | **Ask** | **Show** | **Jobs** | **Submit**
+
+## Top Stories
+
+1. **Show HN: WebPeel – Turn any webpage into AI-ready markdown**
+   [https://github.com/JakeLiuMe/webpeel](https://github.com/JakeLiuMe/webpeel)
+   142 points by jakeliu 2 hours ago | 31 comments
+
+2. **The End of the API Era**
+   ...
+```
+
+---
+
+## Why WebPeel?
+
+|  | **WebPeel** | Firecrawl | Jina Reader | MCP Fetch |
+|---|:---:|:---:|:---:|:---:|
+| **Local execution** | ✅ Free forever | ❌ Cloud only | ❌ Cloud only | ✅ Free |
+| **JS rendering** | ✅ Auto-escalates | ✅ Always | ❌ No | ❌ No |
+| **Anti-bot handling** | ✅ Stealth mode | ✅ Yes | ⚠️ Limited | ❌ No |
+| **MCP Server** | ✅ Built-in | ✅ Separate repo | ❌ No | ✅ Yes |
+| **Zero config** | ✅ `npx webpeel` | ❌ API key required | ❌ API key required | ✅ Yes |
+| **Free tier** | ∞ Unlimited local | 500 pages/month | 1000 req/month | ∞ Local only |
+| **Hosted API** | Coming soon | $16/mo (Starter) | $200/mo (Starter) | N/A |
+| **Markdown output** | ✅ Optimized for AI | ✅ Yes | ✅ Yes | ⚠️ Basic |
+
+**WebPeel gives you Firecrawl's power without the price tag.** Run locally for free, or use our hosted API when you need scale.
+
+---
+
+## Quick Start
+
+### CLI (Zero Install)
+
+```bash
+# Basic usage
+npx webpeel https://example.com
+
+# JSON output with metadata
+npx webpeel https://example.com --json
+
+# Force browser rendering (for JS-heavy sites)
+npx webpeel https://x.com/elonmusk --render
+
+# Wait for dynamic content
+npx webpeel https://example.com --render --wait 3000
+```
+
+### Library (TypeScript)
+
+```bash
+npm install webpeel
+```
+
+```typescript
+import { peel } from 'webpeel';
+
+// Simple usage
+const result = await peel('https://example.com');
+console.log(result.content);    // Clean markdown
+console.log(result.metadata);   // { title, description, author, ... }
+console.log(result.tokens);     // Estimated token count
+
+// With options
+const result = await peel('https://example.com', {
+  format: 'markdown',  // 'markdown' | 'text' | 'html'
+  render: true,        // Force browser mode
+  wait: 3000,          // Wait 3s for dynamic content
+  timeout: 30000,      // Request timeout (ms)
+});
+```
+
+### MCP Server (Claude Desktop, Cursor, VS Code)
+
+WebPeel provides two MCP tools: `webpeel_fetch` (fetch a URL) and `webpeel_search` (DuckDuckGo search + fetch results).
+
+#### Claude Desktop
+
+Add to `~/Library/Application Support/Claude/claude_desktop_config.json`:
+
+```json
+{
+  "mcpServers": {
+    "webpeel": {
+      "command": "npx",
+      "args": ["-y", "webpeel", "mcp"]
+    }
+  }
+}
+```
+
+#### Cursor
+
+Add to Cursor Settings → MCP Servers:
+
+```json
+{
+  "mcpServers": {
+    "webpeel": {
+      "command": "npx",
+      "args": ["-y", "webpeel", "mcp"]
+    }
+  }
+}
+```
+
+#### VS Code (with Cline or other MCP clients)
+
+Create or edit `~/.vscode/mcp.json`:
+
+```json
+{
+  "mcpServers": {
+    "webpeel": {
+      "command": "npx",
+      "args": ["-y", "webpeel", "mcp"]
+    }
+  }
+}
+```
+
+Or install with one click:
+
+[![Install in Claude Desktop](https://img.shields.io/badge/Install-Claude%20Desktop-5B3FFF?style=for-the-badge&logo=anthropic)](https://mcp.so/install/webpeel?for=claude)
+[![Install in VS Code](https://img.shields.io/badge/Install-VS%20Code-007ACC?style=for-the-badge&logo=visualstudiocode)](https://mcp.so/install/webpeel?for=vscode)
+
+---
+
+## How It Works: Smart Escalation
+
+WebPeel tries the fastest method first, then escalates only when needed:
+
+```
+┌─────────────────────────────────────────────────────────────┐
+│                    Smart Escalation                          │
+└─────────────────────────────────────────────────────────────┘
+
+Simple HTTP Fetch          Browser Rendering         Stealth Mode
+    ~200ms                      ~2 seconds             ~5 seconds
+       │                            │                       │
+       ├─ User-Agent headers        ├─ Full JS execution   ├─ Anti-detect
+       ├─ Cheerio parsing           ├─ Wait for content    ├─ Proxy rotation
+       ├─ Fast & cheap              ├─ Screenshots         └─ Cloudflare bypass
+       │                            │
+       ▼                            ▼
+   Works for 80%              Works for 19%            Works for 1%
+   of websites                (JS-heavy sites)         (heavily protected)
+```
+
+**Why this matters:**
+- **Speed**: Don't waste 2 seconds rendering when 200ms will do
+- **Cost**: Headless browsers burn CPU and memory
+- **Reliability**: Auto-retry with browser if simple fetch fails
+
+WebPeel automatically detects blocked requests (403, 503, Cloudflare challenges) and retries with browser mode. You get the best of both worlds.
+
+---
+
+## API Reference
+
+### `peel(url, options?)`
+
+Fetch and extract content from a URL.
+
+```typescript
+interface PeelOptions {
+  render?: boolean;        // Force browser mode (default: false)
+  wait?: number;           // Wait time after page load in ms (default: 0)
+  format?: 'markdown' | 'text' | 'html';  // Output format (default: 'markdown')
+  timeout?: number;        // Request timeout in ms (default: 30000)
+  userAgent?: string;      // Custom user agent
+}
+
+interface PeelResult {
+  url: string;             // Final URL (after redirects)
+  title: string;           // Page title
+  content: string;         // Page content in requested format
+  metadata: {              // Extracted metadata
+    description?: string;
+    author?: string;
+    published?: string;    // ISO 8601 date
+    image?: string;        // Open Graph image
+    canonical?: string;
+  };
+  links: string[];         // All links on page (absolute URLs)
+  tokens: number;          // Estimated token count
+  method: 'simple' | 'browser';  // Method used
+  elapsed: number;         // Time taken (ms)
+}
+```
+
+### Error Types
+
+```typescript
+import { TimeoutError, BlockedError, NetworkError } from 'webpeel';
+
+try {
+  const result = await peel('https://example.com');
+} catch (error) {
+  if (error instanceof TimeoutError) {
+    // Request timed out
+  } else if (error instanceof BlockedError) {
+    // Site blocked the request (403, Cloudflare, etc.)
+  } else if (error instanceof NetworkError) {
+    // Network/DNS error
+  }
+}
+```
+
+### `cleanup()`
+
+Clean up browser resources. Call this when you're done using WebPeel in your application:
+
+```typescript
+import { peel, cleanup } from 'webpeel';
+
+// ... use peel() ...
+
+await cleanup();  // Close browser instances
+```
+
+---
+
+## Hosted API (Coming Soon)
+
+Run WebPeel locally for free, or use our hosted API for scale:
+
+| Plan | Price | Requests/Month | Features |
+|------|------:|---------------:|----------|
+| **Free** | $0 | Unlimited local | CLI, library, MCP server |
+| **Hosted Free** | $0 | 1,000 | API access, no credit card |
+| **Pro** | $9 | 50,000 | Priority queue, 99.9% SLA |
+| **Scale** | $29 | 250,000 | Dedicated instances, webhook support |
+
+**Compare:** Firecrawl Starter is $16/mo for 3,000 requests. Our Pro tier gives you 50,000 for $9/mo.
+
+Join the waitlist at [webpeel.dev](https://webpeel.dev)
+
+---
+
+## Examples
+
+### Extract blog post metadata
+
+```typescript
+const result = await peel('https://example.com/blog/post');
+
+console.log(result.metadata);
+// {
+//   title: "How We Built WebPeel",
+//   description: "A deep dive into smart escalation...",
+//   author: "Jake Liu",
+//   published: "2026-02-12T18:00:00Z",
+//   image: "https://example.com/og-image.png"
+// }
+```
+
+### Get all links from a page
+
+```typescript
+const result = await peel('https://news.ycombinator.com');
+
+console.log(result.links.slice(0, 5));
+// [
+//   "https://news.ycombinator.com/newest",
+//   "https://news.ycombinator.com/submit",
+//   "https://github.com/example/repo",
+//   ...
+// ]
+```
+
+### Force browser rendering for JavaScript-heavy sites
+
+```typescript
+// Twitter/X requires JavaScript
+const result = await peel('https://x.com/elonmusk', {
+  render: true,
+  wait: 2000,  // Wait for tweets to load
+});
+
+console.log(result.content);  // Rendered tweet content
+```
+
+### Token counting for LLM usage
+
+```typescript
+const result = await peel('https://example.com/long-article');
+
+console.log(`Content is ~${result.tokens} tokens`);
+// Content is ~3,247 tokens
+
+if (result.tokens > 4000) {
+  console.log('Too long for GPT-3.5 context window');
+}
+```
+
+---
+
+## Use Cases
+
+- **AI Agents**: Feed web content to Claude, GPT, or local LLMs
+- **Research**: Bulk extract articles, docs, or social media
+- **Monitoring**: Track content changes on websites
+- **Archiving**: Save web pages as clean markdown
+- **Data Pipelines**: Extract structured data from web sources
+
+---
+
+## Development
+
+```bash
+# Clone the repo
+git clone https://github.com/JakeLiuMe/webpeel.git
+cd webpeel
+
+# Install dependencies
+npm install
+
+# Build
+npm run build
+
+# Run tests
+npm test
+
+# Watch mode (auto-rebuild)
+npm run dev
+
+# Test the CLI locally
+node dist/cli.js https://example.com
+
+# Test the MCP server
+npm run mcp
+```
+
+See [CONTRIBUTING.md](CONTRIBUTING.md) for guidelines.
+
+---
+
+## Roadmap
+
+- [x] CLI with smart escalation
+- [x] TypeScript library
+- [x] MCP server for Claude/Cursor/VS Code
+- [ ] Hosted API with authentication
+- [ ] Rate limiting and caching
+- [ ] Batch processing API
+- [ ] Screenshot capture
+- [ ] PDF extraction
+- [ ] Webhook notifications for monitoring
+
+Vote on features and roadmap at [GitHub Discussions](https://github.com/JakeLiuMe/webpeel/discussions).
+
+---
+
+## FAQ
+
+**Q: How is this different from Firecrawl?**  
+A: WebPeel runs locally for free (Firecrawl is cloud-only). We also have smart escalation to avoid burning resources on simple pages.
+
+**Q: Can I self-host the API server?**  
+A: Yes! Run `npm run serve` to start the API server. See [docs/self-hosting.md](docs/self-hosting.md) (coming soon).
+
+**Q: Does this violate websites' Terms of Service?**  
+A: WebPeel respects `robots.txt` by default. Always check a site's ToS before scraping at scale.
+
+**Q: What about CAPTCHA and Cloudflare?**  
+A: WebPeel handles most Cloudflare challenges automatically. For CAPTCHAs, you'll need a solving service (not included).
+
+**Q: Can I use this in production?**  
+A: Yes, but be mindful of rate limits. The hosted API (coming soon) is better for high-volume production use.
+
+---
+
+## Credits
+
+Built with:
+- [Playwright](https://playwright.dev/) — Headless browser automation
+- [Cheerio](https://cheerio.js.org/) — Fast HTML parsing
+- [Turndown](https://github.com/mixmark-io/turndown) — HTML to Markdown conversion
+- [Commander](https://github.com/tj/commander.js) — CLI framework
+
+---
+
+## Contributing
+
+Contributions are welcome! See [CONTRIBUTING.md](CONTRIBUTING.md) for guidelines.
+
+---
+
+## License
+
+MIT © [Jake Liu](https://github.com/JakeLiuMe)
+
+---
+
+**Like WebPeel?** [⭐ Star us on GitHub](https://github.com/JakeLiuMe/webpeel) — it helps others discover the project!

package/dist/cli.d.ts

@@ -0,0 +1,16 @@
+#!/usr/bin/env node
+/**
+ * WebPeel CLI
+ *
+ * Usage:
+ *   npx webpeel <url>                  - Fetch and convert to markdown
+ *   npx webpeel <url> --json           - Output as JSON
+ *   npx webpeel <url> --html           - Output raw HTML
+ *   npx webpeel <url> --render         - Force browser mode
+ *   npx webpeel <url> --wait 5000      - Wait 5s for JS to load
+ *   npx webpeel search "query"         - DuckDuckGo search
+ *   npx webpeel serve                  - Start API server (future)
+ *   npx webpeel mcp                    - Start MCP server (future)
+ */
+export {};
+//# sourceMappingURL=cli.d.ts.map

package/dist/cli.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"cli.d.ts","sourceRoot":"","sources":["../src/cli.ts"],"names":[],"mappings":";AAEA;;;;;;;;;;;;GAYG"}

package/dist/cli.js

@@ -0,0 +1,140 @@
+#!/usr/bin/env node
+/**
+ * WebPeel CLI
+ *
+ * Usage:
+ *   npx webpeel <url>                  - Fetch and convert to markdown
+ *   npx webpeel <url> --json           - Output as JSON
+ *   npx webpeel <url> --html           - Output raw HTML
+ *   npx webpeel <url> --render         - Force browser mode
+ *   npx webpeel <url> --wait 5000      - Wait 5s for JS to load
+ *   npx webpeel search "query"         - DuckDuckGo search
+ *   npx webpeel serve                  - Start API server (future)
+ *   npx webpeel mcp                    - Start MCP server (future)
+ */
+import { Command } from 'commander';
+import ora from 'ora';
+import { peel, cleanup } from './index.js';
+const program = new Command();
+program
+    .name('webpeel')
+    .description('Fast web fetcher for AI agents')
+    .version('0.1.0');
+program
+    .argument('[url]', 'URL to fetch')
+    .option('-r, --render', 'Use headless browser (for JS-heavy sites)')
+    .option('-w, --wait <ms>', 'Wait time after page load (ms)', parseInt)
+    .option('--html', 'Output raw HTML instead of markdown')
+    .option('--text', 'Output plain text instead of markdown')
+    .option('--json', 'Output as JSON')
+    .option('-t, --timeout <ms>', 'Request timeout (ms)', parseInt, 30000)
+    .option('--ua <agent>', 'Custom user agent')
+    .option('-s, --silent', 'Silent mode (no spinner)')
+    .action(async (url, options) => {
+    if (!url) {
+        console.error('Error: URL is required\n');
+        program.help();
+        process.exit(1);
+    }
+    // SECURITY: Enhanced URL validation
+    if (url.length > 2048) {
+        console.error('Error: URL too long (max 2048 characters)');
+        process.exit(1);
+    }
+    // Check for control characters
+    if (/[\x00-\x1F\x7F]/.test(url)) {
+        console.error('Error: URL contains invalid control characters');
+        process.exit(1);
+    }
+    // Validate URL format
+    try {
+        const parsed = new URL(url);
+        if (!['http:', 'https:'].includes(parsed.protocol)) {
+            console.error('Error: Only HTTP and HTTPS protocols are allowed');
+            process.exit(1);
+        }
+    }
+    catch {
+        console.error(`Error: Invalid URL format: ${url}`);
+        process.exit(1);
+    }
+    const spinner = options.silent ? null : ora('Fetching...').start();
+    try {
+        // Validate options
+        if (options.wait && (options.wait < 0 || options.wait > 60000)) {
+            console.error('Error: Wait time must be between 0 and 60000ms');
+            process.exit(1);
+        }
+        // Build peel options
+        const peelOptions = {
+            render: options.render || false,
+            wait: options.wait || 0,
+            timeout: options.timeout,
+            userAgent: options.ua,
+        };
+        // Determine format
+        if (options.html) {
+            peelOptions.format = 'html';
+        }
+        else if (options.text) {
+            peelOptions.format = 'text';
+        }
+        else {
+            peelOptions.format = 'markdown';
+        }
+        // Fetch the page
+        const result = await peel(url, peelOptions);
+        if (spinner) {
+            spinner.succeed(`Fetched in ${result.elapsed}ms using ${result.method} method`);
+        }
+        // Output results
+        if (options.json) {
+            console.log(JSON.stringify(result, null, 2));
+        }
+        else {
+            console.log(result.content);
+        }
+        // Clean up and exit
+        await cleanup();
+        process.exit(0);
+    }
+    catch (error) {
+        if (spinner) {
+            spinner.fail('Failed to fetch');
+        }
+        if (error instanceof Error) {
+            console.error(`\nError: ${error.message}`);
+        }
+        else {
+            console.error('\nError: Unknown error occurred');
+        }
+        await cleanup();
+        process.exit(1);
+    }
+});
+// Future commands
+program
+    .command('search')
+    .argument('<query>', 'Search query')
+    .description('Search using DuckDuckGo (future)')
+    .action(() => {
+    console.log('Search command not yet implemented');
+    console.log('Coming soon: DuckDuckGo search integration');
+    process.exit(1);
+});
+program
+    .command('serve')
+    .description('Start API server')
+    .option('-p, --port <port>', 'Port number', '3000')
+    .action(async (options) => {
+    const { startServer } = await import('./server/app.js');
+    startServer({ port: parseInt(options.port, 10) });
+});
+program
+    .command('mcp')
+    .description('Start MCP server for Claude Desktop / Cursor')
+    .action(async () => {
+    await import('./mcp/server.js');
+});
+program.parse();
+//# sourceMappingURL=cli.js.map

package/dist/cli.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"cli.js","sourceRoot":"","sources":["../src/cli.ts"],"names":[],"mappings":";AAEA;;;;;;;;;;;;GAYG;AAEH,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AACpC,OAAO,GAAG,MAAM,KAAK,CAAC;AACtB,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,MAAM,YAAY,CAAC;AAG3C,MAAM,OAAO,GAAG,IAAI,OAAO,EAAE,CAAC;AAE9B,OAAO;KACJ,IAAI,CAAC,SAAS,CAAC;KACf,WAAW,CAAC,gCAAgC,CAAC;KAC7C,OAAO,CAAC,OAAO,CAAC,CAAC;AAEpB,OAAO;KACJ,QAAQ,CAAC,OAAO,EAAE,cAAc,CAAC;KACjC,MAAM,CAAC,cAAc,EAAE,2CAA2C,CAAC;KACnE,MAAM,CAAC,iBAAiB,EAAE,gCAAgC,EAAE,QAAQ,CAAC;KACrE,MAAM,CAAC,QAAQ,EAAE,qCAAqC,CAAC;KACvD,MAAM,CAAC,QAAQ,EAAE,uCAAuC,CAAC;KACzD,MAAM,CAAC,QAAQ,EAAE,gBAAgB,CAAC;KAClC,MAAM,CAAC,oBAAoB,EAAE,sBAAsB,EAAE,QAAQ,EAAE,KAAK,CAAC;KACrE,MAAM,CAAC,cAAc,EAAE,mBAAmB,CAAC;KAC3C,MAAM,CAAC,cAAc,EAAE,0BAA0B,CAAC;KAClD,MAAM,CAAC,KAAK,EAAE,GAAuB,EAAE,OAAO,EAAE,EAAE;IACjD,IAAI,CAAC,GAAG,EAAE,CAAC;QACT,OAAO,CAAC,KAAK,CAAC,0BAA0B,CAAC,CAAC;QAC1C,OAAO,CAAC,IAAI,EAAE,CAAC;QACf,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC;IAED,oCAAoC;IACpC,IAAI,GAAG,CAAC,MAAM,GAAG,IAAI,EAAE,CAAC;QACtB,OAAO,CAAC,KAAK,CAAC,2CAA2C,CAAC,CAAC;QAC3D,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC;IAED,+BAA+B;IAC/B,IAAI,iBAAiB,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC;QAChC,OAAO,CAAC,KAAK,CAAC,gDAAgD,CAAC,CAAC;QAChE,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC;IAED,sBAAsB;IACtB,IAAI,CAAC;QACH,MAAM,MAAM,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC;QAC5B,IAAI,CAAC,CAAC,OAAO,EAAE,QAAQ,CAAC,CAAC,QAAQ,CAAC,MAAM,CAAC,QAAQ,CAAC,EAAE,CAAC;YACnD,OAAO,CAAC,KAAK,CAAC,kDAAkD,CAAC,CAAC;YAClE,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QAClB,CAAC;IACH,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,CAAC,KAAK,CAAC,8BAA8B,GAAG,EAAE,CAAC,CAAC;QACnD,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC;IAED,MAAM,OAAO,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,GAAG,CAAC,aAAa,CAAC,CAAC,KAAK,EAAE,CAAC;IAEnE,IAAI,CAAC;QACH,mBAAmB;QACnB,IAAI,OAAO,CAAC,IAAI,IAAI,CAAC,OAAO,CAAC,IAAI,GAAG,CAAC,IAAI,OAAO,CAAC,IAAI,GAAG,KAAK,CAAC,EAAE,CAAC;YAC/D,OAAO,CAAC,KAAK,CAAC,gDAAgD,CAAC,CAAC;YAChE,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QAClB,CAAC;QAED,qBAAqB;QACrB,MAAM,WAAW,GAAgB;YAC/B,MAAM,EAAE,OAAO,CAAC,MAAM,IAAI,KAAK;YAC/B,IAAI,EAAE,OAAO,CAAC,IAAI,IAAI,CAAC;YACvB,OAAO,EAAE,OAAO,CAAC,OAAO;YACxB,SAAS,EAAE,OAAO,CAAC,EAAE;SACtB,CAAC;QAEF,mBAAmB;QACnB,IAAI,OAAO,CAAC,IAAI,EAAE,CAAC;YACjB,WAAW,CAAC,MAAM,GAAG,MAAM,CAAC;QAC9B,CAAC;aAAM,IAAI,OAAO,CAAC,IAAI,EAAE,CAAC;YACxB,WAAW,CAAC,MAAM,GAAG,MAAM,CAAC;QAC9B,CAAC;aAAM,CAAC;YACN,WAAW,CAAC,MAAM,GAAG,UAAU,CAAC;QAClC,CAAC;QAED,iBAAiB;QACjB,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,GAAG,EAAE,WAAW,CAAC,CAAC;QAE5C,IAAI,OAAO,EAAE,CAAC;YACZ,OAAO,CAAC,OAAO,CAAC,cAAc,MAAM,CAAC,OAAO,YAAY,MAAM,CAAC,MAAM,SAAS,CAAC,CAAC;QAClF,CAAC;QAED,iBAAiB;QACjB,IAAI,OAAO,CAAC,IAAI,EAAE,CAAC;YACjB,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,SAAS,CAAC,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC;QAC/C,CAAC;aAAM,CAAC;YACN,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;QAC9B,CAAC;QAED,oBAAoB;QACpB,MAAM,OAAO,EAAE,CAAC;QAChB,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,IAAI,OAAO,EAAE,CAAC;YACZ,OAAO,CAAC,IAAI,CAAC,iBAAiB,CAAC,CAAC;QAClC,CAAC;QAED,IAAI,KAAK,YAAY,KAAK,EAAE,CAAC;YAC3B,OAAO,CAAC,KAAK,CAAC,YAAY,KAAK,CAAC,OAAO,EAAE,CAAC,CAAC;QAC7C,CAAC;aAAM,CAAC;YACN,OAAO,CAAC,KAAK,CAAC,iCAAiC,CAAC,CAAC;QACnD,CAAC;QAED,MAAM,OAAO,EAAE,CAAC;QAChB,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC;AACH,CAAC,CAAC,CAAC;AAEL,kBAAkB;AAClB,OAAO;KACJ,OAAO,CAAC,QAAQ,CAAC;KACjB,QAAQ,CAAC,SAAS,EAAE,cAAc,CAAC;KACnC,WAAW,CAAC,kCAAkC,CAAC;KAC/C,MAAM,CAAC,GAAG,EAAE;IACX,OAAO,CAAC,GAAG,CAAC,oCAAoC,CAAC,CAAC;IAClD,OAAO,CAAC,GAAG,CAAC,4CAA4C,CAAC,CAAC;IAC1D,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;AAClB,CAAC,CAAC,CAAC;AAEL,OAAO;KACJ,OAAO,CAAC,OAAO,CAAC;KAChB,WAAW,CAAC,kBAAkB,CAAC;KAC/B,MAAM,CAAC,mBAAmB,EAAE,aAAa,EAAE,MAAM,CAAC;KAClD,MAAM,CAAC,KAAK,EAAE,OAAO,EAAE,EAAE;IACxB,MAAM,EAAE,WAAW,EAAE,GAAG,MAAM,MAAM,CAAC,iBAAiB,CAAC,CAAC;IACxD,WAAW,CAAC,EAAE,IAAI,EAAE,QAAQ,CAAC,OAAO,CAAC,IAAI,EAAE,EAAE,CAAC,EAAE,CAAC,CAAC;AACpD,CAAC,CAAC,CAAC;AAEL,OAAO;KACJ,OAAO,CAAC,KAAK,CAAC;KACd,WAAW,CAAC,8CAA8C,CAAC;KAC3D,MAAM,CAAC,KAAK,IAAI,EAAE;IACjB,MAAM,MAAM,CAAC,iBAAiB,CAAC,CAAC;AAClC,CAAC,CAAC,CAAC;AAEL,OAAO,CAAC,KAAK,EAAE,CAAC"}

package/dist/core/fetcher.d.ts

@@ -0,0 +1,32 @@
+/**
+ * Core fetching logic: simple HTTP and browser-based fetching
+ */
+export interface FetchResult {
+    html: string;
+    url: string;
+    statusCode?: number;
+}
+/**
+ * Simple HTTP fetch using native fetch + Cheerio
+ * Fast and lightweight, but can be blocked by Cloudflare/bot detection
+ * SECURITY: Manual redirect handling with SSRF re-validation
+ */
+export declare function simpleFetch(url: string, userAgent?: string, timeoutMs?: number): Promise<FetchResult>;
+/**
+ * Fetch using headless Chromium via Playwright
+ * Slower but can handle JavaScript-heavy sites and bypass some bot detection
+ */
+export declare function browserFetch(url: string, options?: {
+    userAgent?: string;
+    waitMs?: number;
+    timeoutMs?: number;
+}): Promise<FetchResult>;
+/**
+ * Retry a fetch operation with exponential backoff
+ */
+export declare function retryFetch<T>(fn: () => Promise<T>, maxAttempts?: number, baseDelayMs?: number): Promise<T>;
+/**
+ * Clean up browser resources
+ */
+export declare function cleanup(): Promise<void>;
+//# sourceMappingURL=fetcher.d.ts.map

package/dist/core/fetcher.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"fetcher.d.ts","sourceRoot":"","sources":["../../src/core/fetcher.ts"],"names":[],"mappings":"AAAA;;GAEG;AA2PH,MAAM,WAAW,WAAW;IAC1B,IAAI,EAAE,MAAM,CAAC;IACb,GAAG,EAAE,MAAM,CAAC;IACZ,UAAU,CAAC,EAAE,MAAM,CAAC;CACrB;AAED;;;;GAIG;AACH,wBAAsB,WAAW,CAC/B,GAAG,EAAE,MAAM,EACX,SAAS,CAAC,EAAE,MAAM,EAClB,SAAS,GAAE,MAAc,GACxB,OAAO,CAAC,WAAW,CAAC,CAyItB;AAuBD;;;GAGG;AACH,wBAAsB,YAAY,CAChC,GAAG,EAAE,MAAM,EACX,OAAO,GAAE;IACP,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,SAAS,CAAC,EAAE,MAAM,CAAC;CACf,GACL,OAAO,CAAC,WAAW,CAAC,CAoGtB;AAED;;GAEG;AACH,wBAAsB,UAAU,CAAC,CAAC,EAChC,EAAE,EAAE,MAAM,OAAO,CAAC,CAAC,CAAC,EACpB,WAAW,GAAE,MAAU,EACvB,WAAW,GAAE,MAAa,GACzB,OAAO,CAAC,CAAC,CAAC,CAsBZ;AAED;;GAEG;AACH,wBAAsB,OAAO,IAAI,OAAO,CAAC,IAAI,CAAC,CAK7C"}