datablue 1.0.0__tar.gz

datablue-1.0.0/.gitignore

@@ -0,0 +1,137 @@
+ADMIN_CLI.txt
+
+# Python
+__pycache__/
+*.py[cod]
+*$py.class
+*.so
+.Python
+env/
+venv/
+.venv/
+*.egg-info/
+dist/
+build/
+.eggs/
+
+# Node
+node_modules/
+.next/
+out/
+.turbo/
+
+# Environment
+.env
+.env.local
+.env.*.local
+
+# IDE
+.vscode/
+.idea/
+*.swp
+*.swo
+*~
+
+# OS
+.DS_Store
+Thumbs.db
+
+# Docker
+docker-compose.override.yml
+
+# Database
+*.sqlite3
+*.db
+
+# Logs
+*.log
+logs/
+
+# Coverage
+htmlcov/
+.coverage
+coverage/
+.nyc_output/
+
+# Celery
+celerybeat-schedule
+celerybeat.pid
+
+# AI tools
+.agent/
+.agents/
+.windsurf/
+
+# TypeScript build artifacts
+*.tsbuildinfo
+.claude/skills/
+.playwright-mcp/
+
+# Landing page prototypes
+datablue-landing-v2/
+.gitnexus
+
+# Deploy secrets (never commit env files with credentials)
+deploy/*.env
+deploy/generate-env.sh
+
+# Screenshots & design iterations (bloat)
+/*.png
+/logo/*.png
+
+# Embedded repos
+new-ui/
+
+# Research & benchmarking (ephemeral work)
+alt_engines_research/
+bing_research/
+brave_research/
+ddg_research/
+startpage_research/
+pow_capture/
+bench_results/
+
+# Temp scripts & result files at root
+/test_*.py
+/bench_*.py
+/analyze_*.py
+/capture_*.py
+/scrape_*.py
+/find_*.py
+/keyword_*.py
+/engine_*.py
+/cookie_replay*.py
+/pow_*.py
+/pow_*.js
+/extract_script*.py
+/*.csv
+/*.json
+/*.xlsx
+/cookie_test_*.html
+/google_js_shell_sample.html
+/datablue_*.xlsx
+/pow_shell_raw.html
+/client_scrape_export.json
+
+# Output & temp directories
+tmp/
+output/
+
+# MCP config (local)
+.mcp.json
+
+# Report/research docs (not product docs)
+BOT_DETECTION_RESEARCH.md
+COOKIE_REPLAY_REPORT.md
+MICROBROWSER_INTEGRATION.md
+MICROBROWSER_OSINT_REPORT.md
+MICROBROWSER_PLAN.md
+MULTI_ENGINE_DESIGN.md
+SEARCH_ENGINE_RESEARCH.md
+SERP_MISSION_REPORT.md
+V8_MINIRACER_REPORT.md
+MIGRATION_RUNBOOK.md
+datablue-architecture.html
+
+# Test results
+test-results/

datablue-1.0.0/PKG-INFO

@@ -0,0 +1,375 @@
+Metadata-Version: 2.4
+Name: datablue
+Version: 1.0.0
+Summary: Python SDK for the DataBlue web scraping platform — scrape, crawl, search, and map
+Project-URL: Homepage, https://datablue.dev
+Project-URL: Documentation, https://docs.datablue.dev
+Project-URL: Repository, https://github.com/datablue-dev/datablue-python
+License-Expression: MIT
+Keywords: crawling,datablue,firecrawl-alternative,scraping,web-data
+Classifier: Development Status :: 4 - Beta
+Classifier: Intended Audience :: Developers
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Topic :: Internet :: WWW/HTTP
+Classifier: Topic :: Software Development :: Libraries :: Python Modules
+Classifier: Typing :: Typed
+Requires-Python: >=3.10
+Requires-Dist: httpx>=0.27.0
+Requires-Dist: pydantic>=2.0.0
+Provides-Extra: dev
+Requires-Dist: pytest-asyncio>=0.23; extra == 'dev'
+Requires-Dist: pytest>=8.0; extra == 'dev'
+Requires-Dist: respx>=0.21; extra == 'dev'
+Description-Content-Type: text/markdown
+
+# DataBlue Python SDK
+
+The official Python SDK for [DataBlue](https://datablue.dev) — a self-hosted web scraping platform with anti-bot bypass, structured data extraction, and site crawling.
+
+## Installation
+
+```bash
+pip install datablue
+```
+
+## Quick Start
+
+```python
+from datablue import DataBlue
+
+client = DataBlue(
+    api_url="https://api.datablue.dev",
+    api_key="your_api_key",
+)
+
+# Scrape a page
+result = client.scrape("https://example.com")
+print(result.data.markdown)
+
+client.close()
+```
+
+### Context Manager
+
+```python
+with DataBlue(api_key="your_api_key") as client:
+    result = client.scrape("https://example.com")
+    print(result.data.markdown)
+```
+
+### Environment Variables
+
+```bash
+export DATABLUE_API_KEY=your_api_key
+export DATABLUE_API_URL=https://api.datablue.dev
+```
+
+```python
+client = DataBlue.from_env()
+```
+
+## Scrape
+
+Scrape a single URL and get structured content back.
+
+```python
+result = client.scrape("https://example.com")
+
+# Access content
+print(result.data.markdown)       # Markdown content
+print(result.data.html)           # HTML content
+print(result.data.links)          # Extracted links
+print(result.data.metadata.title) # Page title
+```
+
+### Advanced Scraping
+
+```python
+result = client.scrape(
+    "https://example.com",
+    formats=["markdown", "html", "links", "screenshot"],
+    only_main_content=True,
+    wait_for=2000,              # Wait 2s for JS to load
+    timeout=30000,              # 30s timeout
+    css_selector=".article",    # Target specific element
+    mobile=True,                # Mobile viewport
+    headers={"Accept-Language": "en-US"},
+    cookies={"session": "abc123"},
+)
+```
+
+### Browser Actions
+
+Execute browser actions before scraping:
+
+```python
+result = client.scrape(
+    "https://example.com",
+    actions=[
+        {"type": "click", "selector": "#load-more"},
+        {"type": "wait", "milliseconds": 2000},
+        {"type": "scroll", "direction": "down", "amount": 3},
+        {"type": "screenshot"},
+    ],
+)
+```
+
+### LLM Extraction
+
+Extract structured data using AI:
+
+```python
+result = client.scrape(
+    "https://example.com/product",
+    extract={
+        "prompt": "Extract the product name, price, and rating",
+        "schema": {
+            "type": "object",
+            "properties": {
+                "name": {"type": "string"},
+                "price": {"type": "number"},
+                "rating": {"type": "number"},
+            },
+        },
+    },
+)
+print(result.data.extract)
+```
+
+## Crawl
+
+Crawl an entire website. Returns results as pages are discovered.
+
+### Blocking (waits for completion)
+
+```python
+result = client.crawl(
+    "https://example.com",
+    max_pages=50,
+    max_depth=3,
+    timeout=300,
+)
+
+for page in result.data:
+    print(page.url, len(page.markdown or ""))
+```
+
+### Non-blocking (manual polling)
+
+```python
+job = client.start_crawl("https://example.com", max_pages=100)
+print(f"Job started: {job.job_id}")
+
+# Poll for status
+status = client.get_crawl_status(job.job_id)
+print(f"Progress: {status.completed_pages}/{status.total_pages}")
+
+# Cancel if needed
+client.cancel_crawl(job.job_id)
+```
+
+### Crawl Options
+
+```python
+result = client.crawl(
+    "https://example.com",
+    max_pages=200,
+    max_depth=5,
+    concurrency=5,
+    crawl_strategy="bfs",           # bfs, dfs, or bff (best-first)
+    include_paths=["/blog/*"],      # Only crawl matching paths
+    exclude_paths=["/admin/*"],     # Skip these paths
+    allow_external_links=False,
+    respect_robots_txt=True,
+    scrape_options={
+        "formats": ["markdown"],
+        "only_main_content": True,
+    },
+)
+```
+
+## Search
+
+Search the web and scrape each result page.
+
+```python
+result = client.search(
+    "best python web scraping libraries",
+    num_results=10,
+    formats=["markdown"],
+)
+
+for item in result.data:
+    print(f"{item.title}: {item.url}")
+    print(item.markdown[:200])
+```
+
+### Search Engines
+
+```python
+# Default: Google (via SearXNG)
+result = client.search("query", engine="google")
+
+# DuckDuckGo
+result = client.search("query", engine="duckduckgo")
+
+# Brave (requires API key)
+result = client.search("query", engine="brave", brave_api_key="...")
+```
+
+## Map
+
+Discover all URLs on a website using sitemaps and link crawling.
+
+```python
+result = client.map("https://example.com", limit=500)
+
+for link in result.links:
+    print(f"{link.url} - {link.title}")
+
+# Just the URLs
+print(result.urls)
+```
+
+### Filter URLs
+
+```python
+result = client.map(
+    "https://example.com",
+    search="blog",              # Filter by keyword
+    include_subdomains=True,
+    use_sitemap=True,
+    limit=1000,
+)
+```
+
+## Batch Scrape
+
+Scrape multiple URLs efficiently.
+
+```python
+results = client.batch_scrape(
+    [
+        "https://example.com/page1",
+        "https://example.com/page2",
+        "https://example.com/page3",
+    ],
+    scrape_options={"formats": ["markdown"], "only_main_content": True},
+)
+
+for r in results:
+    if r.success:
+        print(r.data.metadata.title)
+```
+
+## Async Client
+
+Full async support for high-performance applications.
+
+```python
+import asyncio
+from datablue import AsyncDataBlue
+
+async def main():
+    async with AsyncDataBlue(api_key="your_key") as client:
+        # Scrape
+        result = await client.scrape("https://example.com")
+
+        # Crawl
+        crawl = await client.crawl("https://example.com", max_pages=50)
+
+        # Search
+        search = await client.search("python scraping", num_results=5)
+
+        # Map
+        sitemap = await client.map("https://example.com")
+
+        # Batch scrape (concurrent)
+        results = await client.batch_scrape(urls, concurrency=10)
+
+        # Streaming batch (yields as completed)
+        async for result in client.batch_scrape_iter(urls, concurrency=10):
+            print(result.data.url)
+
+asyncio.run(main())
+```
+
+## Error Handling
+
+```python
+from datablue import (
+    DataBlueError,
+    AuthenticationError,
+    RateLimitError,
+    NotFoundError,
+    ServerError,
+    JobFailedError,
+    TimeoutError,
+)
+
+try:
+    result = client.scrape("https://example.com")
+except AuthenticationError:
+    print("Bad API key")
+except RateLimitError as e:
+    print(f"Rate limited. Retry after {e.retry_after}s")
+except TimeoutError as e:
+    print(f"Job timed out after {e.elapsed}s")
+except JobFailedError as e:
+    print(f"Job {e.job_id} failed")
+except ServerError:
+    print("Server error (auto-retried)")
+except DataBlueError as e:
+    print(f"Error {e.status_code}: {e.message}")
+```
+
+All errors include:
+- `e.message` — human-readable description
+- `e.status_code` — HTTP status code
+- `e.is_retryable` — whether the request can be retried
+- `e.retry_after` — seconds to wait (for 429s)
+- `e.docs_url` — link to error documentation
+
+## Configuration
+
+```python
+client = DataBlue(
+    api_url="https://api.datablue.dev",  # API base URL
+    api_key="your_key",                   # API key
+    timeout=60.0,                         # Request timeout (seconds)
+    max_retries=3,                        # Retry count for 5xx/429
+)
+```
+
+### Self-Hosted
+
+```python
+client = DataBlue(
+    api_url="http://localhost:8000",
+    api_key="your_key",
+)
+```
+
+### Login with Email/Password
+
+```python
+client = DataBlue(api_url="https://api.datablue.dev")
+client.login("you@email.com", "password")
+# JWT token is stored automatically
+result = client.scrape("https://example.com")
+```
+
+## Requirements
+
+- Python 3.10+
+- httpx
+- pydantic v2
+
+## License
+
+MIT