webpeel 0.1.0__tar.gz

webpeel-0.1.0/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2025 Jake Liu
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

webpeel-0.1.0/PKG-INFO

@@ -0,0 +1,257 @@
+Metadata-Version: 2.4
+Name: webpeel
+Version: 0.1.0
+Summary: Fast web fetcher for AI agents — smart extraction, stealth mode, structured data
+Author-email: Jake Liu <jake@webpeel.dev>
+License: MIT
+Project-URL: Homepage, https://webpeel.dev
+Project-URL: Documentation, https://github.com/JakeLiuMe/webpeel
+Project-URL: Repository, https://github.com/JakeLiuMe/webpeel
+Project-URL: Issues, https://github.com/JakeLiuMe/webpeel/issues
+Keywords: web-scraping,ai,llm,mcp,web-fetcher,markdown,scraper,crawler,ai-agents
+Classifier: Development Status :: 4 - Beta
+Classifier: Intended Audience :: Developers
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.8
+Classifier: Programming Language :: Python :: 3.9
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Topic :: Internet :: WWW/HTTP
+Classifier: Topic :: Software Development :: Libraries
+Classifier: Topic :: Software Development :: Libraries :: Python Modules
+Requires-Python: >=3.8
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Dynamic: license-file
+
+# WebPeel Python SDK
+
+**Fast web fetcher for AI agents** — smart extraction, stealth mode, structured data.
+
+Zero dependencies. Pure Python 3.8+ stdlib.
+
+## Installation
+
+```bash
+pip install webpeel
+```
+
+## Quick Start
+
+### Basic Scraping
+
+```python
+from webpeel import WebPeel
+
+client = WebPeel()
+
+# Scrape a URL and get clean markdown
+result = client.scrape("https://example.com")
+print(result.title)
+print(result.content)  # Clean markdown content
+print(result.metadata)  # Structured metadata
+```
+
+### Search the Web
+
+```python
+# Search via DuckDuckGo
+results = client.search("python web scraping")
+
+for item in results.data.get("web", []):
+    print(f"{item['title']}: {item['url']}")
+```
+
+### JavaScript-Heavy Sites
+
+```python
+# Use browser rendering for SPAs and JS-heavy sites
+result = client.scrape(
+    "https://twitter.com/elonmusk",
+    render=True,  # Enable browser mode
+    wait=2000,    # Wait 2s for JS to load
+)
+```
+
+### Stealth Mode (Bypass Bot Detection)
+
+```python
+# Bypass Cloudflare, reCAPTCHA, and anti-bot systems
+result = client.scrape(
+    "https://protected-site.com",
+    stealth=True,  # Enable stealth mode
+)
+```
+
+### Structured Data Extraction
+
+```python
+# Extract specific data using CSS selectors
+result = client.scrape(
+    "https://amazon.com/product/...",
+    extract={
+        "selectors": {
+            "title": "h1#title",
+            "price": "span.price",
+            "rating": ".review-rating",
+        }
+    }
+)
+
+print(result.extracted)
+# {"title": "Product Name", "price": "$29.99", "rating": "4.5"}
+```
+
+### Crawl a Website
+
+```python
+# Start an async crawl job (requires API key)
+client = WebPeel(api_key="your-api-key")
+
+job = client.crawl(
+    "https://docs.example.com",
+    limit=100,
+    max_depth=3,
+)
+
+print(job.id)  # Job ID for tracking
+
+# Check status later
+status = client.get_job(job.id)
+print(status["status"])  # pending, running, completed, failed
+```
+
+### Map a Domain
+
+```python
+# Discover all URLs on a domain
+result = client.map("https://example.com")
+
+print(f"Found {result.total} URLs")
+for url in result.urls[:10]:
+    print(url)
+```
+
+### Batch Scraping
+
+```python
+# Scrape multiple URLs in batch (requires API key)
+client = WebPeel(api_key="your-api-key")
+
+urls = [
+    "https://example.com/page1",
+    "https://example.com/page2",
+    "https://example.com/page3",
+]
+
+job = client.batch_scrape(urls, max_tokens=5000)
+print(job.id)
+```
+
+## API Reference
+
+### WebPeel Class
+
+```python
+WebPeel(
+    api_key: Optional[str] = None,
+    base_url: str = "https://api.webpeel.dev",
+    timeout: int = 30,
+)
+```
+
+- **`api_key`**: API key for authentication (optional for free tier)
+- **`base_url`**: Base URL for the WebPeel API
+- **`timeout`**: Request timeout in seconds
+
+### Methods
+
+#### `scrape(url, **options) -> ScrapeResult`
+
+Scrape a URL and extract content.
+
+**Options:**
+- `formats`: Output formats (default: `["markdown"]`)
+- `max_tokens`: Maximum token count for output
+- `render`: Use headless browser (default: `False`)
+- `stealth`: Bypass bot detection (default: `False`)
+- `wait`: Wait time in ms after page load
+- `extract`: Structured data extraction config
+- `headers`: Custom HTTP headers
+
+#### `search(query, limit=5) -> SearchResult`
+
+Search the web via DuckDuckGo.
+
+#### `crawl(url, limit=50, max_depth=3) -> CrawlResult`
+
+Start an async crawl job (requires API key).
+
+#### `map(url) -> MapResult`
+
+Discover all URLs on a domain.
+
+#### `batch_scrape(urls, **options) -> BatchResult`
+
+Batch scrape multiple URLs (requires API key).
+
+#### `get_job(job_id) -> Dict`
+
+Check status of an async job.
+
+## WebPeel vs Firecrawl
+
+| Feature | WebPeel | Firecrawl |
+|---------|---------|-----------|
+| **Pricing** | $0 local / $9-$29 cloud | $16-$333/mo |
+| **Free Tier** | 125 fetches/week | 500 credits one-time |
+| **License** | MIT | AGPL-3.0 |
+| **Python SDK Deps** | Zero (pure stdlib) | httpx, pydantic |
+| **Smart Escalation** | ✅ Auto HTTP→Browser→Stealth | Manual mode selection |
+| **Token Budget** | ✅ `--max-tokens` | ❌ |
+| **Quality Scoring** | ✅ 0-1 per response | ❌ |
+| **Local CLI** | ✅ Free, unlimited | Requires API key |
+| **LangChain** | ✅ | ✅ |
+| **LlamaIndex** | ✅ | ✅ |
+
+**WebPeel is the free, fast, MIT-licensed alternative to Firecrawl.**
+
+## Authentication
+
+Free tier: No API key needed. Anonymous usage with rate limits.
+
+Paid tier: Get an API key at [webpeel.dev](https://webpeel.dev).
+
+```python
+client = WebPeel(api_key="wp_...")
+```
+
+## Error Handling
+
+```python
+from webpeel import WebPeel, WebPeelError, RateLimitError, TimeoutError
+
+client = WebPeel()
+
+try:
+    result = client.scrape("https://example.com")
+except RateLimitError:
+    print("Rate limit exceeded. Upgrade or wait.")
+except TimeoutError:
+    print("Request timeout. Try again.")
+except WebPeelError as e:
+    print(f"Error: {e}")
+```
+
+## License
+
+MIT © Jake Liu
+
+## Links
+
+- [Homepage](https://webpeel.dev)
+- [Documentation](https://github.com/JakeLiuMe/webpeel)
+- [GitHub](https://github.com/JakeLiuMe/webpeel)
+- [Issues](https://github.com/JakeLiuMe/webpeel/issues)

webpeel-0.1.0/README.md

@@ -0,0 +1,229 @@
+# WebPeel Python SDK
+
+**Fast web fetcher for AI agents** — smart extraction, stealth mode, structured data.
+
+Zero dependencies. Pure Python 3.8+ stdlib.
+
+## Installation
+
+```bash
+pip install webpeel
+```
+
+## Quick Start
+
+### Basic Scraping
+
+```python
+from webpeel import WebPeel
+
+client = WebPeel()
+
+# Scrape a URL and get clean markdown
+result = client.scrape("https://example.com")
+print(result.title)
+print(result.content)  # Clean markdown content
+print(result.metadata)  # Structured metadata
+```
+
+### Search the Web
+
+```python
+# Search via DuckDuckGo
+results = client.search("python web scraping")
+
+for item in results.data.get("web", []):
+    print(f"{item['title']}: {item['url']}")
+```
+
+### JavaScript-Heavy Sites
+
+```python
+# Use browser rendering for SPAs and JS-heavy sites
+result = client.scrape(
+    "https://twitter.com/elonmusk",
+    render=True,  # Enable browser mode
+    wait=2000,    # Wait 2s for JS to load
+)
+```
+
+### Stealth Mode (Bypass Bot Detection)
+
+```python
+# Bypass Cloudflare, reCAPTCHA, and anti-bot systems
+result = client.scrape(
+    "https://protected-site.com",
+    stealth=True,  # Enable stealth mode
+)
+```
+
+### Structured Data Extraction
+
+```python
+# Extract specific data using CSS selectors
+result = client.scrape(
+    "https://amazon.com/product/...",
+    extract={
+        "selectors": {
+            "title": "h1#title",
+            "price": "span.price",
+            "rating": ".review-rating",
+        }
+    }
+)
+
+print(result.extracted)
+# {"title": "Product Name", "price": "$29.99", "rating": "4.5"}
+```
+
+### Crawl a Website
+
+```python
+# Start an async crawl job (requires API key)
+client = WebPeel(api_key="your-api-key")
+
+job = client.crawl(
+    "https://docs.example.com",
+    limit=100,
+    max_depth=3,
+)
+
+print(job.id)  # Job ID for tracking
+
+# Check status later
+status = client.get_job(job.id)
+print(status["status"])  # pending, running, completed, failed
+```
+
+### Map a Domain
+
+```python
+# Discover all URLs on a domain
+result = client.map("https://example.com")
+
+print(f"Found {result.total} URLs")
+for url in result.urls[:10]:
+    print(url)
+```
+
+### Batch Scraping
+
+```python
+# Scrape multiple URLs in batch (requires API key)
+client = WebPeel(api_key="your-api-key")
+
+urls = [
+    "https://example.com/page1",
+    "https://example.com/page2",
+    "https://example.com/page3",
+]
+
+job = client.batch_scrape(urls, max_tokens=5000)
+print(job.id)
+```
+
+## API Reference
+
+### WebPeel Class
+
+```python
+WebPeel(
+    api_key: Optional[str] = None,
+    base_url: str = "https://api.webpeel.dev",
+    timeout: int = 30,
+)
+```
+
+- **`api_key`**: API key for authentication (optional for free tier)
+- **`base_url`**: Base URL for the WebPeel API
+- **`timeout`**: Request timeout in seconds
+
+### Methods
+
+#### `scrape(url, **options) -> ScrapeResult`
+
+Scrape a URL and extract content.
+
+**Options:**
+- `formats`: Output formats (default: `["markdown"]`)
+- `max_tokens`: Maximum token count for output
+- `render`: Use headless browser (default: `False`)
+- `stealth`: Bypass bot detection (default: `False`)
+- `wait`: Wait time in ms after page load
+- `extract`: Structured data extraction config
+- `headers`: Custom HTTP headers
+
+#### `search(query, limit=5) -> SearchResult`
+
+Search the web via DuckDuckGo.
+
+#### `crawl(url, limit=50, max_depth=3) -> CrawlResult`
+
+Start an async crawl job (requires API key).
+
+#### `map(url) -> MapResult`
+
+Discover all URLs on a domain.
+
+#### `batch_scrape(urls, **options) -> BatchResult`
+
+Batch scrape multiple URLs (requires API key).
+
+#### `get_job(job_id) -> Dict`
+
+Check status of an async job.
+
+## WebPeel vs Firecrawl
+
+| Feature | WebPeel | Firecrawl |
+|---------|---------|-----------|
+| **Pricing** | $0 local / $9-$29 cloud | $16-$333/mo |
+| **Free Tier** | 125 fetches/week | 500 credits one-time |
+| **License** | MIT | AGPL-3.0 |
+| **Python SDK Deps** | Zero (pure stdlib) | httpx, pydantic |
+| **Smart Escalation** | ✅ Auto HTTP→Browser→Stealth | Manual mode selection |
+| **Token Budget** | ✅ `--max-tokens` | ❌ |
+| **Quality Scoring** | ✅ 0-1 per response | ❌ |
+| **Local CLI** | ✅ Free, unlimited | Requires API key |
+| **LangChain** | ✅ | ✅ |
+| **LlamaIndex** | ✅ | ✅ |
+
+**WebPeel is the free, fast, MIT-licensed alternative to Firecrawl.**
+
+## Authentication
+
+Free tier: No API key needed. Anonymous usage with rate limits.
+
+Paid tier: Get an API key at [webpeel.dev](https://webpeel.dev).
+
+```python
+client = WebPeel(api_key="wp_...")
+```
+
+## Error Handling
+
+```python
+from webpeel import WebPeel, WebPeelError, RateLimitError, TimeoutError
+
+client = WebPeel()
+
+try:
+    result = client.scrape("https://example.com")
+except RateLimitError:
+    print("Rate limit exceeded. Upgrade or wait.")
+except TimeoutError:
+    print("Request timeout. Try again.")
+except WebPeelError as e:
+    print(f"Error: {e}")
+```
+
+## License
+
+MIT © Jake Liu
+
+## Links
+
+- [Homepage](https://webpeel.dev)
+- [Documentation](https://github.com/JakeLiuMe/webpeel)
+- [GitHub](https://github.com/JakeLiuMe/webpeel)
+- [Issues](https://github.com/JakeLiuMe/webpeel/issues)

webpeel-0.1.0/pyproject.toml

@@ -0,0 +1,51 @@
+[build-system]
+requires = ["setuptools>=64", "wheel"]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "webpeel"
+version = "0.1.0"
+description = "Fast web fetcher for AI agents — smart extraction, stealth mode, structured data"
+readme = "README.md"
+license = {text = "MIT"}
+requires-python = ">=3.8"
+authors = [
+    {name = "Jake Liu", email = "jake@webpeel.dev"}
+]
+keywords = [
+    "web-scraping",
+    "ai",
+    "llm",
+    "mcp",
+    "web-fetcher",
+    "markdown",
+    "scraper",
+    "crawler",
+    "ai-agents",
+]
+classifiers = [
+    "Development Status :: 4 - Beta",
+    "Intended Audience :: Developers",
+    "License :: OSI Approved :: MIT License",
+    "Programming Language :: Python :: 3",
+    "Programming Language :: Python :: 3.8",
+    "Programming Language :: Python :: 3.9",
+    "Programming Language :: Python :: 3.10",
+    "Programming Language :: Python :: 3.11",
+    "Programming Language :: Python :: 3.12",
+    "Topic :: Internet :: WWW/HTTP",
+    "Topic :: Software Development :: Libraries",
+    "Topic :: Software Development :: Libraries :: Python Modules",
+]
+dependencies = []
+
+[project.urls]
+Homepage = "https://webpeel.dev"
+Documentation = "https://github.com/JakeLiuMe/webpeel"
+Repository = "https://github.com/JakeLiuMe/webpeel"
+Issues = "https://github.com/JakeLiuMe/webpeel/issues"
+
+[tool.setuptools.packages.find]
+where = ["."]
+include = ["webpeel*"]
+exclude = ["tests*"]

webpeel-0.1.0/setup.cfg

@@ -0,0 +1,4 @@
+[egg_info]
+tag_build = 
+tag_date = 0
+

webpeel-0.1.0/tests/test_client.py

@@ -0,0 +1,96 @@
+"""Basic tests for WebPeel SDK."""
+
+import unittest
+from unittest.mock import patch, MagicMock
+import json
+
+from webpeel import WebPeel, ScrapeResult, WebPeelError
+
+
+class TestWebPeel(unittest.TestCase):
+    """Test WebPeel client."""
+    
+    def setUp(self):
+        """Set up test client."""
+        self.client = WebPeel(api_key="test-key")
+    
+    def test_init(self):
+        """Test client initialization."""
+        self.assertEqual(self.client.api_key, "test-key")
+        self.assertEqual(self.client.base_url, "https://api.webpeel.dev")
+        self.assertEqual(self.client.timeout, 30)
+    
+    def test_init_custom(self):
+        """Test client with custom settings."""
+        client = WebPeel(
+            api_key="custom-key",
+            base_url="https://custom.api",
+            timeout=60,
+        )
+        self.assertEqual(client.api_key, "custom-key")
+        self.assertEqual(client.base_url, "https://custom.api")
+        self.assertEqual(client.timeout, 60)
+    
+    @patch('urllib.request.urlopen')
+    def test_scrape_basic(self, mock_urlopen):
+        """Test basic scrape."""
+        # Mock response
+        mock_response = MagicMock()
+        mock_response.read.return_value = json.dumps({
+            "url": "https://example.com",
+            "title": "Example Domain",
+            "content": "# Example Domain\n\nThis is an example.",
+            "metadata": {"description": "Example site"},
+            "links": ["https://example.com/about"],
+            "tokens": 100,
+            "method": "simple",
+            "elapsed": 250,
+        }).encode('utf-8')
+        mock_response.__enter__ = MagicMock(return_value=mock_response)
+        mock_response.__exit__ = MagicMock(return_value=False)
+        
+        mock_urlopen.return_value = mock_response
+        
+        # Call scrape
+        result = self.client.scrape("https://example.com")
+        
+        # Verify result
+        self.assertIsInstance(result, ScrapeResult)
+        self.assertEqual(result.url, "https://example.com")
+        self.assertEqual(result.title, "Example Domain")
+        self.assertEqual(result.method, "simple")
+        self.assertEqual(result.tokens, 100)
+        self.assertIn("Example Domain", result.content)
+    
+    @patch('urllib.request.urlopen')
+    def test_scrape_with_options(self, mock_urlopen):
+        """Test scrape with options."""
+        mock_response = MagicMock()
+        mock_response.read.return_value = json.dumps({
+            "url": "https://example.com",
+            "title": "Example",
+            "content": "Content",
+            "metadata": {},
+            "links": [],
+            "tokens": 50,
+            "method": "browser",
+            "elapsed": 1500,
+        }).encode('utf-8')
+        mock_response.__enter__ = MagicMock(return_value=mock_response)
+        mock_response.__exit__ = MagicMock(return_value=False)
+        
+        mock_urlopen.return_value = mock_response
+        
+        result = self.client.scrape(
+            "https://example.com",
+            render=True,
+            wait=2000,
+            max_tokens=5000,
+        )
+        
+        self.assertEqual(result.method, "browser")
+        self.assertEqual(result.url, "https://example.com")
+
+
+if __name__ == '__main__':
+    unittest.main()

webpeel-0.1.0/webpeel/__init__.py

@@ -0,0 +1,35 @@
+"""
+WebPeel Python SDK
+
+Fast web fetcher for AI agents — smart extraction, stealth mode, structured data.
+"""
+
+from .client import WebPeel
+from .types import (
+    ScrapeResult,
+    SearchResult,
+    CrawlResult,
+    MapResult,
+    BatchResult,
+)
+from .exceptions import (
+    WebPeelError,
+    AuthError,
+    RateLimitError,
+    TimeoutError,
+)
+from ._version import __version__
+
+__all__ = [
+    "WebPeel",
+    "ScrapeResult",
+    "SearchResult",
+    "CrawlResult",
+    "MapResult",
+    "BatchResult",
+    "WebPeelError",
+    "AuthError",
+    "RateLimitError",
+    "TimeoutError",
+    "__version__",
+]

webpeel-0.1.0/webpeel/_version.py

@@ -0,0 +1,3 @@
+"""Version information for WebPeel SDK."""
+
+__version__ = "0.1.0"