crawlforge 0.1.0__tar.gz

crawlforge-0.1.0/.gitignore

@@ -0,0 +1,12 @@
+.venv*/
+dist/
+build/
+*.egg-info/
+__pycache__/
+*.pyc
+.env
+.env.*
+.pytest_cache/
+.mypy_cache/
+.ruff_cache/
+.DS_Store

crawlforge-0.1.0/CHANGELOG.md

@@ -0,0 +1,18 @@
+# Changelog
+
+## 0.1.0 (2026-09-07)
+
+Initial release.
+
+- `CrawlForge` (sync) and `AsyncCrawlForge` (async) clients on `httpx`.
+- One typed method per tool, 30 in all, generated from `openapi.json`; every
+  keyword argument is validated locally by a pydantic v2 request model before
+  a request is sent.
+- `call(tool, params)` for a raw body and `describe(tool)` for a tool's
+  self-description.
+- Error classes by status: `ValidationError`, `AuthenticationError`,
+  `InsufficientCreditsError`, `RateLimitError`, `ToolError`; `NETWORK_ERROR`
+  and `UNEXPECTED_RESPONSE` on the base `CrawlForgeError`.
+- Retries on 429 only, honouring `Retry-After` (seconds or HTTP date, capped
+  at 60 s) with jittered exponential backoff otherwise.
+- Python 3.9 to 3.13. Runtime dependencies: `httpx` and `pydantic` only.

crawlforge-0.1.0/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 CrawlForge
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

crawlforge-0.1.0/Makefile

@@ -0,0 +1,21 @@
+PYTHON ?= python
+
+.PHONY: generate lint typecheck test build clean
+
+generate:
+	$(PYTHON) scripts/generate.py
+
+lint:
+	ruff check .
+
+typecheck:
+	mypy crawlforge scripts
+
+test:
+	pytest
+
+build:
+	$(PYTHON) -m build
+
+clean:
+	rm -rf dist build *.egg-info .pytest_cache .mypy_cache .ruff_cache

crawlforge-0.1.0/PKG-INFO

@@ -0,0 +1,269 @@
+Metadata-Version: 2.5
+Name: crawlforge
+Version: 0.1.0
+Summary: Official Python SDK for the CrawlForge REST API: 30 metered web tools behind one API key.
+Project-URL: Homepage, https://www.crawlforge.dev
+Project-URL: Documentation, https://www.crawlforge.dev/docs/api-reference
+Project-URL: Source, https://github.com/mysleekdesigns/crawlforge-sdk-python
+Project-URL: Changelog, https://github.com/mysleekdesigns/crawlforge-sdk-python/blob/main/CHANGELOG.md
+Author: CrawlForge
+License-Expression: MIT
+License-File: LICENSE
+Keywords: crawler,crawlforge,extraction,mcp,sdk,search,web scraping
+Classifier: Development Status :: 4 - Beta
+Classifier: Intended Audience :: Developers
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Operating System :: OS Independent
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3 :: Only
+Classifier: Programming Language :: Python :: 3.9
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Topic :: Internet :: WWW/HTTP
+Classifier: Topic :: Software Development :: Libraries :: Python Modules
+Classifier: Typing :: Typed
+Requires-Python: >=3.9
+Requires-Dist: httpx<1,>=0.27
+Requires-Dist: pydantic<3,>=2
+Provides-Extra: dev
+Requires-Dist: build>=1.0; extra == 'dev'
+Requires-Dist: mypy>=1.8; extra == 'dev'
+Requires-Dist: pytest-asyncio>=0.23; extra == 'dev'
+Requires-Dist: pytest>=7.4; extra == 'dev'
+Requires-Dist: ruff>=0.5; extra == 'dev'
+Description-Content-Type: text/markdown
+
+# CrawlForge Python SDK
+
+The official Python client for the [CrawlForge REST API](https://www.crawlforge.dev/docs/api-reference):
+30 metered web tools (scrape, search, crawl, extract, research) behind one API key.
+
+- Sync `CrawlForge` and async `AsyncCrawlForge` clients on `httpx`
+- One typed method per tool, generated from the API's [OpenAPI spec](https://www.crawlforge.dev/openapi.json)
+- Arguments validated locally by pydantic v2 before any request is sent
+- Retries on rate limits only, honouring `Retry-After`
+- Python 3.9+, fully typed (`py.typed`), runtime dependencies `httpx` and `pydantic` only
+
+## Install
+
+```bash
+pip install crawlforge
+```
+
+Create an API key at https://www.crawlforge.dev/dashboard/keys and export it as
+`CRAWLFORGE_API_KEY`, or pass `api_key=` to the client.
+
+## Quick start
+
+```python
+from crawlforge import CrawlForge
+
+with CrawlForge() as client:  # reads CRAWLFORGE_API_KEY
+    result = client.scrape(url="https://example.com", formats=["markdown", "links"])
+
+print(result.data["formats"]["markdown"][:200])
+print(result.credits_used, result.credits_remaining)
+```
+
+Async:
+
+```python
+import asyncio
+from crawlforge import AsyncCrawlForge
+
+async def main() -> None:
+    async with AsyncCrawlForge() as client:
+        result = await client.search_web(query="web scraping", limit=5)
+        for hit in result.data["results"]:
+            print(hit["title"], hit["url"])
+
+asyncio.run(main())
+```
+
+Every tool method returns a `ToolResult`:
+
+| Field | Meaning |
+|---|---|
+| `data` | The tool's result. **Tool-specific and untyped (`dict`) in this release**; its shape is documented on the tool's docs page. |
+| `credits_used` | Credits charged for this call. |
+| `credits_remaining` | Your balance after the call. |
+| `processing_time` | Server-side processing time in milliseconds. |
+| `warnings` | Non-fatal notes, `[]` when there are none. |
+
+## Calling a tool
+
+Each tool is a method named exactly like the tool. Pass keyword arguments, a
+dict, or the tool's request model; all three send the same body, and only the
+fields you set are sent.
+
+```python
+from crawlforge.models import FetchUrlRequest
+
+client.fetch_url(url="https://example.com", timeout=15000)
+client.fetch_url({"url": "https://example.com", "timeout": 15000})
+client.fetch_url(FetchUrlRequest(url="https://example.com", timeout=15000))
+```
+
+Keyword names are the API's wire names, camelCase included (`onlyMainContent`,
+`maxUrls`). Nested objects can be passed as dicts. A misspelled keyword is a
+`TypeError` from the method signature; a wrong type, a value outside the spec's
+range, or an unknown key in a dict is a `pydantic.ValidationError`. Either way
+nothing is sent and nothing is charged.
+
+Two escape hatches:
+
+- `client.call(tool, params)` posts a raw body to any tool without local
+  validation, for a parameter the spec does not list or a tool newer than this
+  SDK.
+- `client.describe(tool)` fetches the tool's self-description (`ToolInfo`: price,
+  JSON Schema, example) with no API key.
+
+### Per-call timeout and retries
+
+`with_options()` returns a copy of the client sharing the same connection pool:
+
+```python
+client.with_options(timeout=10.0, max_retries=0).fetch_url(url="https://example.com")
+```
+
+The tool parameter `timeout` (milliseconds, on `fetch_url` and ten other tools)
+is unrelated to the HTTP timeout, which is why the HTTP one is not a keyword on
+the tool methods.
+
+## Tools and credits
+
+Prices are the `x-credits` of each operation in the spec. An error response,
+whatever its status, charges nothing: credits are deducted only after a tool
+succeeded.
+
+<!-- generated:tools:start -->
+| Method | Credits | Note | Docs |
+|---|---|---|---|
+| `agent` | 8 |  | [docs](https://www.crawlforge.dev/docs/api-reference/tools/agent) |
+| `analyze_content` | 3 |  | [docs](https://www.crawlforge.dev/docs/api-reference/tools/analyze-content) |
+| `batch_scrape` | 5 | 5 per URL attempted (skipped URLs are not charged) | [docs](https://www.crawlforge.dev/docs/api-reference/tools/batch-scrape) |
+| `crawl_deep` | 4 |  | [docs](https://www.crawlforge.dev/docs/api-reference/tools/crawl-deep) |
+| `deep_research` | 10 |  | [docs](https://www.crawlforge.dev/docs/api-reference/tools/deep-research) |
+| `extract_content` | 2 |  | [docs](https://www.crawlforge.dev/docs/api-reference/tools/extract-content) |
+| `extract_embedded_state` | 2 |  | [docs](https://www.crawlforge.dev/docs/api-reference/tools/extract-embedded-state) |
+| `extract_links` | 1 |  | [docs](https://www.crawlforge.dev/docs/api-reference/tools/extract-links) |
+| `extract_metadata` | 1 |  | [docs](https://www.crawlforge.dev/docs/api-reference/tools/extract-metadata) |
+| `extract_structured` | 3 |  | [docs](https://www.crawlforge.dev/docs/api-reference/tools/extract-structured) |
+| `extract_text` | 1 |  | [docs](https://www.crawlforge.dev/docs/api-reference/tools/extract-text) |
+| `extract_with_llm` | 3 |  | [docs](https://www.crawlforge.dev/docs/api-reference/tools/extract-with-llm) |
+| `fetch_url` | 1 |  | [docs](https://www.crawlforge.dev/docs/api-reference/tools/fetch-url) |
+| `generate_llms_txt` | 5 |  | [docs](https://www.crawlforge.dev/docs/api-reference/tools/generate-llms-txt) |
+| `get_batch_results` | 1 |  | [docs](https://www.crawlforge.dev/docs/api-reference/tools/get-batch-results) |
+| `list_ollama_models` | 1 |  | [docs](https://www.crawlforge.dev/docs/api-reference/tools/list-ollama-models) |
+| `localization` | 2 |  | [docs](https://www.crawlforge.dev/docs/api-reference/tools/localization) |
+| `map_site` | 2 |  | [docs](https://www.crawlforge.dev/docs/api-reference/tools/map-site) |
+| `process_document` | 2 |  | [docs](https://www.crawlforge.dev/docs/api-reference/tools/process-document) |
+| `read_result` | 1 |  | [docs](https://www.crawlforge.dev/docs/api-reference/tools/read-result) |
+| `reddit_search` | 5 |  | [docs](https://www.crawlforge.dev/docs/api-reference/tools/reddit-search) |
+| `scrape` | 2 |  | [docs](https://www.crawlforge.dev/docs/api-reference/tools/scrape) |
+| `scrape_structured` | 2 |  | [docs](https://www.crawlforge.dev/docs/api-reference/tools/scrape-structured) |
+| `scrape_template` | 1 |  | [docs](https://www.crawlforge.dev/docs/api-reference/tools/scrape-template) |
+| `scrape_with_actions` | 5 |  | [docs](https://www.crawlforge.dev/docs/api-reference/tools/scrape-with-actions) |
+| `search_web` | 5 |  | [docs](https://www.crawlforge.dev/docs/api-reference/tools/search-web) |
+| `serp_rank` | 5 |  | [docs](https://www.crawlforge.dev/docs/api-reference/tools/serp-rank) |
+| `stealth_mode` | 5 |  | [docs](https://www.crawlforge.dev/docs/api-reference/tools/stealth-mode) |
+| `summarize_content` | 4 |  | [docs](https://www.crawlforge.dev/docs/api-reference/tools/summarize-content) |
+| `track_changes` | 3 |  | [docs](https://www.crawlforge.dev/docs/api-reference/tools/track-changes) |
+<!-- generated:tools:end -->
+
+## Errors
+
+All errors derive from `crawlforge.CrawlForgeError`, with `status`, `code`,
+`message`, `details` and `request_id`. The class follows the HTTP status:
+
+| Status | Class | Extra attributes |
+|---|---|---|
+| 400 | `ValidationError` | `details` is the list of validation issues |
+| 401 | `AuthenticationError` | |
+| 402 | `InsufficientCreditsError` | `auto_recharge_triggered` (from the `X-Auto-Recharge-Triggered` header) |
+| 429 | `RateLimitError` | `retry_after` in seconds, `None` when absent; `code` is `RATE_LIMIT_EXCEEDED` or `HOST_BACKOFF` |
+| other | `ToolError` | `blocked` (vendor and evidence when a bot wall was met) and `escalated` |
+
+A response that is not JSON is a `CrawlForgeError` with code
+`UNEXPECTED_RESPONSE` and the raw text in `details`; a connection or timeout
+failure is code `NETWORK_ERROR` with `status` 0. No error message ever contains
+your API key.
+
+```python
+from crawlforge import CrawlForge, InsufficientCreditsError, RateLimitError, ToolError
+
+try:
+    result = client.scrape(url="https://example.com", escalate=True)
+except RateLimitError as e:
+    print("slow down", e.retry_after)
+except InsufficientCreditsError as e:
+    print("top up", e.auto_recharge_triggered)
+except ToolError as e:
+    print(e.code, e.blocked)
+```
+
+## Retries
+
+Only a 429 is retried, and only `max_retries` times (default 2, so up to three
+requests). The wait is the `Retry-After` header when present (seconds or an HTTP
+date, capped at 60 s), otherwise exponential backoff from 1 s with ±20 % jitter.
+A 5xx is never retried, because a failed call is not charged but a repeated
+one might be. After the last retry the `RateLimitError` is raised.
+
+## Configuration
+
+```python
+CrawlForge(
+    api_key=None,          # default: CRAWLFORGE_API_KEY
+    base_url=None,         # default: https://www.crawlforge.dev/api/v1
+    max_retries=2,
+    timeout=60.0,          # seconds, per request
+    http_client=None,      # your own httpx.Client (AsyncClient for AsyncCrawlForge)
+)
+```
+
+Passing `http_client` is also the test seam: an `httpx.Client` on an
+`httpx.MockTransport` gives a fixture server with no sockets. The SDK stores
+nothing: results live only in the `ToolResult` you hold.
+
+## Python 3.9+
+
+The generated models use `Optional[...]`, `List[...]` and `Dict[...]` rather
+than `X | Y`, which pydantic cannot evaluate on 3.9. Tested on 3.9 through 3.13.
+
+## Development
+
+```bash
+python -m venv .venv && . .venv/bin/activate
+pip install -e ".[dev]"
+python scripts/generate.py   # or: make generate
+ruff check . && mypy crawlforge scripts && pytest
+python -m build
+```
+
+`crawlforge/_generated/models.py`, `crawlforge/_generated/tools.py` and the
+table above are generated from `openapi.json` by `scripts/generate.py`
+(`scripts/gen_models.py` is the generator) and committed; CI regenerates and
+fails on any diff. `openapi.json` is copied verbatim from
+https://www.crawlforge.dev/openapi.json and never edited by hand.
+
+The generator is in-repo and standard-library only. It replaced
+`datamodel-code-generator` 0.76.2, which on this spec has no Python 3.9 target
+(it emits `X | None`), splits the nine request schemas with a top-level
+`anyOf`/`oneOf`/`allOf` cross-field rule into `RootModel` unions, maps
+`format: uri` to `AnyUrl` (which rewrites URLs before they are sent) and
+`format: email` to `EmailStr` (an extra dependency). The in-repo generator keeps
+wire names, ignores the cross-field rules locally (the server enforces them),
+skips `pattern` (the spec's lookaheads are not supported by pydantic's default
+regex engine) and forbids unknown keys on request models.
+
+## Links
+
+- API reference: https://www.crawlforge.dev/docs/api-reference
+- OpenAPI spec: https://www.crawlforge.dev/openapi.json
+- API keys: https://www.crawlforge.dev/dashboard/keys
+- Changelog: [CHANGELOG.md](CHANGELOG.md)
+
+MIT licensed.

crawlforge-0.1.0/README.md

@@ -0,0 +1,232 @@
+# CrawlForge Python SDK
+
+The official Python client for the [CrawlForge REST API](https://www.crawlforge.dev/docs/api-reference):
+30 metered web tools (scrape, search, crawl, extract, research) behind one API key.
+
+- Sync `CrawlForge` and async `AsyncCrawlForge` clients on `httpx`
+- One typed method per tool, generated from the API's [OpenAPI spec](https://www.crawlforge.dev/openapi.json)
+- Arguments validated locally by pydantic v2 before any request is sent
+- Retries on rate limits only, honouring `Retry-After`
+- Python 3.9+, fully typed (`py.typed`), runtime dependencies `httpx` and `pydantic` only
+
+## Install
+
+```bash
+pip install crawlforge
+```
+
+Create an API key at https://www.crawlforge.dev/dashboard/keys and export it as
+`CRAWLFORGE_API_KEY`, or pass `api_key=` to the client.
+
+## Quick start
+
+```python
+from crawlforge import CrawlForge
+
+with CrawlForge() as client:  # reads CRAWLFORGE_API_KEY
+    result = client.scrape(url="https://example.com", formats=["markdown", "links"])
+
+print(result.data["formats"]["markdown"][:200])
+print(result.credits_used, result.credits_remaining)
+```
+
+Async:
+
+```python
+import asyncio
+from crawlforge import AsyncCrawlForge
+
+async def main() -> None:
+    async with AsyncCrawlForge() as client:
+        result = await client.search_web(query="web scraping", limit=5)
+        for hit in result.data["results"]:
+            print(hit["title"], hit["url"])
+
+asyncio.run(main())
+```
+
+Every tool method returns a `ToolResult`:
+
+| Field | Meaning |
+|---|---|
+| `data` | The tool's result. **Tool-specific and untyped (`dict`) in this release**; its shape is documented on the tool's docs page. |
+| `credits_used` | Credits charged for this call. |
+| `credits_remaining` | Your balance after the call. |
+| `processing_time` | Server-side processing time in milliseconds. |
+| `warnings` | Non-fatal notes, `[]` when there are none. |
+
+## Calling a tool
+
+Each tool is a method named exactly like the tool. Pass keyword arguments, a
+dict, or the tool's request model; all three send the same body, and only the
+fields you set are sent.
+
+```python
+from crawlforge.models import FetchUrlRequest
+
+client.fetch_url(url="https://example.com", timeout=15000)
+client.fetch_url({"url": "https://example.com", "timeout": 15000})
+client.fetch_url(FetchUrlRequest(url="https://example.com", timeout=15000))
+```
+
+Keyword names are the API's wire names, camelCase included (`onlyMainContent`,
+`maxUrls`). Nested objects can be passed as dicts. A misspelled keyword is a
+`TypeError` from the method signature; a wrong type, a value outside the spec's
+range, or an unknown key in a dict is a `pydantic.ValidationError`. Either way
+nothing is sent and nothing is charged.
+
+Two escape hatches:
+
+- `client.call(tool, params)` posts a raw body to any tool without local
+  validation, for a parameter the spec does not list or a tool newer than this
+  SDK.
+- `client.describe(tool)` fetches the tool's self-description (`ToolInfo`: price,
+  JSON Schema, example) with no API key.
+
+### Per-call timeout and retries
+
+`with_options()` returns a copy of the client sharing the same connection pool:
+
+```python
+client.with_options(timeout=10.0, max_retries=0).fetch_url(url="https://example.com")
+```
+
+The tool parameter `timeout` (milliseconds, on `fetch_url` and ten other tools)
+is unrelated to the HTTP timeout, which is why the HTTP one is not a keyword on
+the tool methods.
+
+## Tools and credits
+
+Prices are the `x-credits` of each operation in the spec. An error response,
+whatever its status, charges nothing: credits are deducted only after a tool
+succeeded.
+
+<!-- generated:tools:start -->
+| Method | Credits | Note | Docs |
+|---|---|---|---|
+| `agent` | 8 |  | [docs](https://www.crawlforge.dev/docs/api-reference/tools/agent) |
+| `analyze_content` | 3 |  | [docs](https://www.crawlforge.dev/docs/api-reference/tools/analyze-content) |
+| `batch_scrape` | 5 | 5 per URL attempted (skipped URLs are not charged) | [docs](https://www.crawlforge.dev/docs/api-reference/tools/batch-scrape) |
+| `crawl_deep` | 4 |  | [docs](https://www.crawlforge.dev/docs/api-reference/tools/crawl-deep) |
+| `deep_research` | 10 |  | [docs](https://www.crawlforge.dev/docs/api-reference/tools/deep-research) |
+| `extract_content` | 2 |  | [docs](https://www.crawlforge.dev/docs/api-reference/tools/extract-content) |
+| `extract_embedded_state` | 2 |  | [docs](https://www.crawlforge.dev/docs/api-reference/tools/extract-embedded-state) |
+| `extract_links` | 1 |  | [docs](https://www.crawlforge.dev/docs/api-reference/tools/extract-links) |
+| `extract_metadata` | 1 |  | [docs](https://www.crawlforge.dev/docs/api-reference/tools/extract-metadata) |
+| `extract_structured` | 3 |  | [docs](https://www.crawlforge.dev/docs/api-reference/tools/extract-structured) |
+| `extract_text` | 1 |  | [docs](https://www.crawlforge.dev/docs/api-reference/tools/extract-text) |
+| `extract_with_llm` | 3 |  | [docs](https://www.crawlforge.dev/docs/api-reference/tools/extract-with-llm) |
+| `fetch_url` | 1 |  | [docs](https://www.crawlforge.dev/docs/api-reference/tools/fetch-url) |
+| `generate_llms_txt` | 5 |  | [docs](https://www.crawlforge.dev/docs/api-reference/tools/generate-llms-txt) |
+| `get_batch_results` | 1 |  | [docs](https://www.crawlforge.dev/docs/api-reference/tools/get-batch-results) |
+| `list_ollama_models` | 1 |  | [docs](https://www.crawlforge.dev/docs/api-reference/tools/list-ollama-models) |
+| `localization` | 2 |  | [docs](https://www.crawlforge.dev/docs/api-reference/tools/localization) |
+| `map_site` | 2 |  | [docs](https://www.crawlforge.dev/docs/api-reference/tools/map-site) |
+| `process_document` | 2 |  | [docs](https://www.crawlforge.dev/docs/api-reference/tools/process-document) |
+| `read_result` | 1 |  | [docs](https://www.crawlforge.dev/docs/api-reference/tools/read-result) |
+| `reddit_search` | 5 |  | [docs](https://www.crawlforge.dev/docs/api-reference/tools/reddit-search) |
+| `scrape` | 2 |  | [docs](https://www.crawlforge.dev/docs/api-reference/tools/scrape) |
+| `scrape_structured` | 2 |  | [docs](https://www.crawlforge.dev/docs/api-reference/tools/scrape-structured) |
+| `scrape_template` | 1 |  | [docs](https://www.crawlforge.dev/docs/api-reference/tools/scrape-template) |
+| `scrape_with_actions` | 5 |  | [docs](https://www.crawlforge.dev/docs/api-reference/tools/scrape-with-actions) |
+| `search_web` | 5 |  | [docs](https://www.crawlforge.dev/docs/api-reference/tools/search-web) |
+| `serp_rank` | 5 |  | [docs](https://www.crawlforge.dev/docs/api-reference/tools/serp-rank) |
+| `stealth_mode` | 5 |  | [docs](https://www.crawlforge.dev/docs/api-reference/tools/stealth-mode) |
+| `summarize_content` | 4 |  | [docs](https://www.crawlforge.dev/docs/api-reference/tools/summarize-content) |
+| `track_changes` | 3 |  | [docs](https://www.crawlforge.dev/docs/api-reference/tools/track-changes) |
+<!-- generated:tools:end -->
+
+## Errors
+
+All errors derive from `crawlforge.CrawlForgeError`, with `status`, `code`,
+`message`, `details` and `request_id`. The class follows the HTTP status:
+
+| Status | Class | Extra attributes |
+|---|---|---|
+| 400 | `ValidationError` | `details` is the list of validation issues |
+| 401 | `AuthenticationError` | |
+| 402 | `InsufficientCreditsError` | `auto_recharge_triggered` (from the `X-Auto-Recharge-Triggered` header) |
+| 429 | `RateLimitError` | `retry_after` in seconds, `None` when absent; `code` is `RATE_LIMIT_EXCEEDED` or `HOST_BACKOFF` |
+| other | `ToolError` | `blocked` (vendor and evidence when a bot wall was met) and `escalated` |
+
+A response that is not JSON is a `CrawlForgeError` with code
+`UNEXPECTED_RESPONSE` and the raw text in `details`; a connection or timeout
+failure is code `NETWORK_ERROR` with `status` 0. No error message ever contains
+your API key.
+
+```python
+from crawlforge import CrawlForge, InsufficientCreditsError, RateLimitError, ToolError
+
+try:
+    result = client.scrape(url="https://example.com", escalate=True)
+except RateLimitError as e:
+    print("slow down", e.retry_after)
+except InsufficientCreditsError as e:
+    print("top up", e.auto_recharge_triggered)
+except ToolError as e:
+    print(e.code, e.blocked)
+```
+
+## Retries
+
+Only a 429 is retried, and only `max_retries` times (default 2, so up to three
+requests). The wait is the `Retry-After` header when present (seconds or an HTTP
+date, capped at 60 s), otherwise exponential backoff from 1 s with ±20 % jitter.
+A 5xx is never retried, because a failed call is not charged but a repeated
+one might be. After the last retry the `RateLimitError` is raised.
+
+## Configuration
+
+```python
+CrawlForge(
+    api_key=None,          # default: CRAWLFORGE_API_KEY
+    base_url=None,         # default: https://www.crawlforge.dev/api/v1
+    max_retries=2,
+    timeout=60.0,          # seconds, per request
+    http_client=None,      # your own httpx.Client (AsyncClient for AsyncCrawlForge)
+)
+```
+
+Passing `http_client` is also the test seam: an `httpx.Client` on an
+`httpx.MockTransport` gives a fixture server with no sockets. The SDK stores
+nothing: results live only in the `ToolResult` you hold.
+
+## Python 3.9+
+
+The generated models use `Optional[...]`, `List[...]` and `Dict[...]` rather
+than `X | Y`, which pydantic cannot evaluate on 3.9. Tested on 3.9 through 3.13.
+
+## Development
+
+```bash
+python -m venv .venv && . .venv/bin/activate
+pip install -e ".[dev]"
+python scripts/generate.py   # or: make generate
+ruff check . && mypy crawlforge scripts && pytest
+python -m build
+```
+
+`crawlforge/_generated/models.py`, `crawlforge/_generated/tools.py` and the
+table above are generated from `openapi.json` by `scripts/generate.py`
+(`scripts/gen_models.py` is the generator) and committed; CI regenerates and
+fails on any diff. `openapi.json` is copied verbatim from
+https://www.crawlforge.dev/openapi.json and never edited by hand.
+
+The generator is in-repo and standard-library only. It replaced
+`datamodel-code-generator` 0.76.2, which on this spec has no Python 3.9 target
+(it emits `X | None`), splits the nine request schemas with a top-level
+`anyOf`/`oneOf`/`allOf` cross-field rule into `RootModel` unions, maps
+`format: uri` to `AnyUrl` (which rewrites URLs before they are sent) and
+`format: email` to `EmailStr` (an extra dependency). The in-repo generator keeps
+wire names, ignores the cross-field rules locally (the server enforces them),
+skips `pattern` (the spec's lookaheads are not supported by pydantic's default
+regex engine) and forbids unknown keys on request models.
+
+## Links
+
+- API reference: https://www.crawlforge.dev/docs/api-reference
+- OpenAPI spec: https://www.crawlforge.dev/openapi.json
+- API keys: https://www.crawlforge.dev/dashboard/keys
+- Changelog: [CHANGELOG.md](CHANGELOG.md)
+
+MIT licensed.

crawlforge-0.1.0/crawlforge/__init__.py

@@ -0,0 +1,38 @@
+"""CrawlForge Python SDK: 30 metered web tools behind one API key.
+
+from crawlforge import CrawlForge
+
+client = CrawlForge()  # reads CRAWLFORGE_API_KEY
+result = client.scrape(url="https://example.com", formats=["markdown"])
+print(result.data["formats"]["markdown"], result.credits_remaining)
+"""
+
+from crawlforge._client import AsyncCrawlForge, CrawlForge
+from crawlforge._generated.models import ToolInfo
+from crawlforge._generated.tools import TOOLS
+from crawlforge._version import __version__
+from crawlforge.errors import (
+    AuthenticationError,
+    CrawlForgeError,
+    InsufficientCreditsError,
+    RateLimitError,
+    ToolError,
+    ValidationError,
+)
+from crawlforge.types import ToolResult, ToolSpec
+
+__all__ = [
+    "AsyncCrawlForge",
+    "AuthenticationError",
+    "CrawlForge",
+    "CrawlForgeError",
+    "InsufficientCreditsError",
+    "RateLimitError",
+    "TOOLS",
+    "ToolError",
+    "ToolInfo",
+    "ToolResult",
+    "ToolSpec",
+    "ValidationError",
+    "__version__",
+]