datablue 2.0.0__tar.gz → 2.1.0__tar.gz

{datablue-2.0.0 → datablue-2.1.0}/.gitignore

@@ -1,137 +1,172 @@
-ADMIN_CLI.txt
-
-# Python
-__pycache__/
-*.py[cod]
-*$py.class
-*.so
-.Python
-env/
-venv/
-.venv/
-*.egg-info/
-dist/
-build/
-.eggs/
-
-# Node
-node_modules/
-.next/
-out/
-.turbo/
-
-# Environment
-.env
-.env.local
-.env.*.local
-
-# IDE
-.vscode/
-.idea/
-*.swp
-*.swo
-*~
-
-# OS
-.DS_Store
-Thumbs.db
-
-# Docker
-docker-compose.override.yml
-
-# Database
-*.sqlite3
-*.db
-
-# Logs
-*.log
-logs/
-
-# Coverage
-htmlcov/
-.coverage
-coverage/
-.nyc_output/
-
-# Celery
-celerybeat-schedule
-celerybeat.pid
-
-# AI tools
-.agent/
-.agents/
-.windsurf/
-
-# TypeScript build artifacts
-*.tsbuildinfo
-.claude/skills/
-.playwright-mcp/
-
-# Landing page prototypes
-datablue-landing-v2/
-.gitnexus
-
-# Deploy secrets (never commit env files with credentials)
-deploy/*.env
-deploy/generate-env.sh
-
-# Screenshots & design iterations (bloat)
-/*.png
-/logo/*.png
-
-# Embedded repos
-new-ui/
-
-# Research & benchmarking (ephemeral work)
-alt_engines_research/
-bing_research/
-brave_research/
-ddg_research/
-startpage_research/
-pow_capture/
-bench_results/
-
-# Temp scripts & result files at root
-/test_*.py
-/bench_*.py
-/analyze_*.py
-/capture_*.py
-/scrape_*.py
-/find_*.py
-/keyword_*.py
-/engine_*.py
-/cookie_replay*.py
-/pow_*.py
-/pow_*.js
-/extract_script*.py
-/*.csv
-/*.json
-/*.xlsx
-/cookie_test_*.html
-/google_js_shell_sample.html
-/datablue_*.xlsx
-/pow_shell_raw.html
+ADMIN_CLI.txt
+
+# Python
+__pycache__/
+*.py[cod]
+*$py.class
+*.so
+.Python
+env/
+venv/
+.venv/
+*.egg-info/
+dist/
+build/
+.eggs/
+
+# Node
+node_modules/
+.next/
+out/
+.turbo/
+
+# Environment
+.env
+.env.local
+.env.*.local
+
+# IDE
+.vscode/
+.idea/
+*.swp
+*.swo
+*~
+
+# OS
+.DS_Store
+Thumbs.db
+
+# Docker
+docker-compose.override.yml
+
+# Database
+*.sqlite3
+*.db
+
+# Logs
+*.log
+logs/
+
+# Coverage
+htmlcov/
+.coverage
+coverage/
+.nyc_output/
+
+# Celery
+celerybeat-schedule
+celerybeat.pid
+
+# AI tools
+.agent/
+.agents/
+.codex/
+.windsurf/
+
+# TypeScript build artifacts
+*.tsbuildinfo
+.claude/skills/
+.playwright-mcp/
+
+# Landing page prototypes
+datablue-landing-v2/
+.gitnexus
+
+# Deploy secrets (never commit env files with credentials)
+deploy/*.env
+deploy/generate-env.sh
+docs/server_cred.md
+docs/server-envs/
+
+# Screenshots & design iterations (bloat)
+/*.png
+/logo/*.png
+
+# Embedded repos
+new-ui/
+
+# Research & benchmarking (ephemeral work)
+/experiments/
+alt_engines_research/
+bing_research/
+brave_research/
+ddg_research/
+startpage_research/
+pow_capture/
+bench_results/
+
+# Temp scripts & result files at root
+/test_*.py
+/bench_*.py
+/analyze_*.py
+/capture_*.py
+/scrape_*.py
+/find_*.py
+/keyword_*.py
+/engine_*.py
+/cookie_replay*.py
+/pow_*.py
+/pow_*.js
+/extract_script*.py
+/*.csv
+/*.json
+/*.xlsx
+/cookie_test_*.html
+/google_js_shell_sample.html
+/datablue_*.xlsx
+/pow_shell_raw.html
 /client_scrape_export.json
+/*.backup-*
+/instamart-*-network.txt
 
 # Output & temp directories
-tmp/
-output/
-
-# MCP config (local)
-.mcp.json
-
+tmp/
+.tmp/
+output/
+
+# Local POC and generated experiment output
+/poc/
+/poc_backups/
+/.local-legacy/
+/legacy/
+/artifacts/
+/reports/
+/docker-compose.rendered-google-poc*.yml
+/frontend-legacy/
+/serp-worker/
+/rendered-google-worker/
+/browser-execution-worker/app/None/
+/browser-execution-worker/profiles*/
+/browser-execution-worker/tools/multipage_tab_reuse_ab.py
+/browser-execution-worker/tools/us_asn_filter_canary.py
+/tools/*_latest.json
+/tools/native_data_api_*.json
+/tools/prod_serp_*.json
+/scrapingdog-*.txt
+/rendered-google-worker/tests/last-*.json
+/rendered-google-worker/tests/one-keyword-*.json
+/rendered-google-worker/tests/phase*-*.json
+/rendered-google-worker/tests/prod-*.json
+/rendered-google-worker/tests/prod-*.txt
+/rendered-google-worker/tests/prod-*.meta.txt
+/rendered-google-worker/tests/prod-*.raw.json
+/rendered-google-worker/tests/repair-*.json
+
+# MCP config (local)
+.mcp.json
+
 # Report/research docs (not product docs)
 BOT_DETECTION_RESEARCH.md
-COOKIE_REPLAY_REPORT.md
-MICROBROWSER_INTEGRATION.md
-MICROBROWSER_OSINT_REPORT.md
-MICROBROWSER_PLAN.md
-MULTI_ENGINE_DESIGN.md
-SEARCH_ENGINE_RESEARCH.md
-SERP_MISSION_REPORT.md
-V8_MINIRACER_REPORT.md
-MIGRATION_RUNBOOK.md
-datablue-architecture.html
-
-# Test results
-test-results/
+COOKIE_REPLAY_REPORT.md
+MICROBROWSER_INTEGRATION.md
+MICROBROWSER_OSINT_REPORT.md
+MICROBROWSER_PLAN.md
+MULTI_ENGINE_DESIGN.md
+SEARCH_ENGINE_RESEARCH.md
+SERP_MISSION_REPORT.md
+V8_MINIRACER_REPORT.md
+MIGRATION_RUNBOOK.md
+datablue-architecture.html
+
+# Test results
+test-results/

datablue-2.1.0/PKG-INFO

@@ -0,0 +1,12 @@
+Metadata-Version: 2.4
+Name: datablue
+Version: 2.1.0
+Summary: Python SDK for the DataBlue web scraping platform
+License-Expression: MIT
+Requires-Python: >=3.10
+Requires-Dist: httpx>=0.27.0
+Requires-Dist: pydantic>=2.0.0
+Provides-Extra: dev
+Requires-Dist: pytest-asyncio>=0.23; extra == 'dev'
+Requires-Dist: pytest>=8.0; extra == 'dev'
+Requires-Dist: respx>=0.21; extra == 'dev'

datablue-2.1.0/datablue/__init__.py

@@ -0,0 +1,7 @@
+"""DataBlue Python SDK."""
+
+from webharvest import *  # noqa: F403
+from webharvest import __all__ as __all__
+from webharvest import __version__ as _webharvest_version
+
+__version__ = _webharvest_version

datablue-2.1.0/datablue-mcp/.gitignore

@@ -0,0 +1,3 @@
+dist/
+node_modules/
+*.tgz

datablue-2.1.0/datablue-mcp/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 DataBlue
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

datablue-2.1.0/datablue-mcp/README.md

@@ -0,0 +1,135 @@
+# DataBlue MCP
+
+Official Model Context Protocol server for [DataBlue](https://www.datablue.dev). It gives MCP-compatible agents a scraper-first toolset for search, scrape, batch scrape, crawl, extraction, and DataBlue's stable public Data APIs.
+
+## Requirements
+
+- Node.js 18 or newer
+- A DataBlue API key from the DataBlue dashboard
+
+## Run
+
+```bash
+DATABLUE_API_KEY="<your-key>" npx --yes --prefer-online @datablue/mcp@latest
+```
+
+The server uses stdio. Do not print to or reuse its stdout stream for application logs.
+
+## Standard MCP JSON Configuration
+
+Claude Desktop, Cursor, Windsurf-style clients, and other clients that use an `mcpServers` JSON file can configure DataBlue like this:
+
+```json
+{
+  "mcpServers": {
+    "datablue": {
+      "command": "npx",
+      "args": ["--yes", "--prefer-online", "@datablue/mcp@latest"],
+      "env": {
+        "DATABLUE_API_KEY": "<your-key>"
+      }
+    }
+  }
+}
+```
+
+Restart the MCP client after changing its configuration. Keep the API key in the local client configuration or environment and never commit it to source control.
+
+## Codex
+
+Add the server with the Codex CLI:
+
+```bash
+codex mcp add datablue --env DATABLUE_API_KEY=<your-key> -- npx --yes --prefer-online @datablue/mcp@latest
+```
+
+Equivalent `~/.codex/config.toml` configuration:
+
+```toml
+[mcp_servers.datablue]
+command = "npx"
+args = ["--yes", "--prefer-online", "@datablue/mcp@latest"]
+
+[mcp_servers.datablue.env]
+DATABLUE_API_KEY = "<your-key>"
+```
+
+## Claude Code
+
+```bash
+claude mcp add --transport stdio datablue --env DATABLUE_API_KEY=<your-key> -- npx --yes --prefer-online @datablue/mcp@latest
+```
+
+## Tools
+
+Core tools:
+
+- `datablue_search_google`
+- `datablue_scrape_url`
+- `datablue_batch_scrape`
+- `datablue_extract`
+- `datablue_map_site`
+- `datablue_start_crawl`
+- `datablue_get_job_status`
+
+Catalog tools:
+
+- `datablue_list_data_apis`
+- `datablue_run_data_api`
+
+Call `datablue_list_data_apis` before using the generic runner when you need the current API ids, parameters, examples, limits, or live credit weights.
+
+`datablue_scrape_url` and `datablue_batch_scrape` return the full DataBlue scrape response. Screenshot capture is exposed through scrape formats, for example `formats: ["screenshot"]` or an object format such as `{ "type": "screenshot", "capture": "full_page", "full_page": true }`.
+
+## Resources
+
+- `datablue://apis`
+- `datablue://pricing`
+- `datablue://usage`
+- `datablue://docs/scrape`
+- `datablue://docs/crawl`
+- `datablue://docs/google-serp`
+- `datablue://examples/batch-scrape`
+- `datablue://examples/research-company`
+
+## Prompts
+
+- `research_company`
+- `competitor_serp_report`
+- `extract_products_from_url`
+- `find_leads_from_google`
+- `scrape_site_to_markdown`
+- `monitor_keyword`
+
+## Configuration
+
+| Variable | Default | Allowed |
+| --- | --- | --- |
+| `DATABLUE_API_KEY` | none | DataBlue API key; required for API actions |
+| `DATABLUE_BASE_URL` | `https://api.datablue.dev` | DataBlue API origin |
+| `DATABLUE_MCP_TIMEOUT_MS` | `60000` | `1000..300000` |
+| `DATABLUE_MCP_MAX_RESULTS` | `100` | `1..100` |
+| `DATABLUE_MCP_MAX_CRAWL_PAGES` | `100` | `1..1000` |
+
+The API key is read only from the environment. It is not accepted as a tool parameter and is redacted from tool errors.
+
+When a request exceeds a configured MCP guard such as `DATABLUE_MCP_MAX_RESULTS` or `DATABLUE_MCP_MAX_CRAWL_PAGES`, the server rejects the call before contacting DataBlue. It does not silently lower requested result counts.
+
+## Catalog Policy
+
+The MCP catalog contains only endpoints published in DataBlue's docs-ready endpoint list. SDK-only helpers and APIs still under rollout are intentionally unavailable. Credit weights are fetched from DataBlue's live public weight catalog and are omitted when that catalog cannot be reached; stale weights are never hardcoded.
+
+## Development
+
+```bash
+npm install
+npm run catalog:sync
+npm run verify
+npm pack --dry-run
+```
+
+`npm run catalog:sync` regenerates the package catalog from `web/src/app/docs/docs-data.ts`. The contract test fails if the generated catalog and docs-ready endpoint list diverge.
+
+## License
+
+MIT

datablue-2.1.0/datablue-mcp/eslint.config.js

@@ -0,0 +1,19 @@
+import eslint from "@eslint/js";
+import tseslint from "typescript-eslint";
+
+export default tseslint.config(
+  eslint.configs.recommended,
+  ...tseslint.configs.recommended,
+  {
+    files: ["src/**/*.ts", "test/**/*.ts"],
+    languageOptions: {
+      parserOptions: {
+        project: "./tsconfig.json",
+        tsconfigRootDir: import.meta.dirname,
+      },
+    },
+  },
+  {
+    ignores: ["dist/**", "node_modules/**"],
+  }
+);