datafuel 0.5.0__tar.gz

datafuel-0.5.0/.github/workflows/ci.yml

@@ -0,0 +1,21 @@
+name: CI
+on:
+  push:
+    branches: [main]
+  pull_request:
+jobs:
+  test:
+    runs-on: [self-hosted, datafuel]
+    strategy:
+      matrix:
+        python: ["3.10", "3.11", "3.12", "3.13"]
+    steps:
+      - uses: actions/checkout@v5
+      - uses: actions/setup-python@v5
+        with:
+          python-version: ${{ matrix.python }}
+      - run: pip install -e ".[dev]"
+      - run: ruff check .
+      - run: ruff format --check .
+      - run: mypy
+      - run: pytest -q

datafuel-0.5.0/.github/workflows/release.yml

@@ -0,0 +1,59 @@
+name: Release
+on:
+  push:
+    tags: ["v*"]
+permissions:
+  contents: read
+concurrency:
+  group: release
+  cancel-in-progress: false
+jobs:
+  build:
+    runs-on: [self-hosted, datafuel]
+    steps:
+      - uses: actions/checkout@v5
+        with:
+          fetch-depth: 0
+          persist-credentials: false
+      - env:
+          TAG: ${{ github.ref_name }}
+        run: |
+          git rev-parse --verify -q origin/main >/dev/null || { echo "::error::origin/main was not fetched"; exit 1; }
+          git merge-base --is-ancestor "$GITHUB_SHA" origin/main || { echo "::error::$TAG is not on main"; exit 1; }
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "3.12"
+      - env:
+          TAG: ${{ github.ref_name }}
+        run: |
+          version=$(python -c 'import tomllib; print(tomllib.load(open("pyproject.toml", "rb"))["project"]["version"])')
+          [ "${TAG#v}" = "$version" ] || { echo "::error::tag $TAG does not match version $version in pyproject.toml"; exit 1; }
+      - run: pip install -e ".[dev]" build twine
+      - run: ruff check .
+      - run: ruff format --check .
+      - run: mypy
+      - run: pytest -q
+      - run: python -m build
+      - run: twine check --strict dist/*
+      - uses: actions/upload-artifact@v7
+        with:
+          name: dist
+          path: dist/
+          if-no-files-found: error
+          retention-days: 7
+  publish:
+    needs: build
+    runs-on: [self-hosted, datafuel]
+    environment:
+      name: pypi
+      url: https://pypi.org/p/datafuel
+    permissions:
+      id-token: write
+    steps:
+      - uses: actions/download-artifact@v8
+        with:
+          name: dist
+          path: dist/
+      - uses: pypa/gh-action-pypi-publish@release/v1
+        with:
+          skip-existing: true

datafuel-0.5.0/.gitignore

@@ -0,0 +1,9 @@
+__pycache__/
+*.py[cod]
+.venv/
+dist/
+build/
+*.egg-info/
+.pytest_cache/
+.mypy_cache/
+.ruff_cache/

datafuel-0.5.0/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 Datafuel
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

datafuel-0.5.0/PKG-INFO

@@ -0,0 +1,329 @@
+Metadata-Version: 2.5
+Name: datafuel
+Version: 0.5.0
+Summary: Python client for the DataFuel scraping API
+Project-URL: Homepage, https://datafuel.ai
+Project-URL: Documentation, https://docs.datafuel.ai
+License: MIT License
+        
+        Copyright (c) 2026 Datafuel
+        
+        Permission is hereby granted, free of charge, to any person obtaining a copy
+        of this software and associated documentation files (the "Software"), to deal
+        in the Software without restriction, including without limitation the rights
+        to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+        copies of the Software, and to permit persons to whom the Software is
+        furnished to do so, subject to the following conditions:
+        
+        The above copyright notice and this permission notice shall be included in all
+        copies or substantial portions of the Software.
+        
+        THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+        IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+        FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+        AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+        LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+        OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+        SOFTWARE.
+License-File: LICENSE
+Classifier: Development Status :: 4 - Beta
+Classifier: Intended Audience :: Developers
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Topic :: Internet :: WWW/HTTP
+Classifier: Typing :: Typed
+Requires-Python: >=3.10
+Requires-Dist: httpx>=0.27
+Requires-Dist: pydantic>=2.7
+Provides-Extra: dev
+Requires-Dist: mypy>=1.11; extra == 'dev'
+Requires-Dist: pytest>=8; extra == 'dev'
+Requires-Dist: ruff>=0.6; extra == 'dev'
+Description-Content-Type: text/markdown
+
+# datafuel-py
+
+Python client for the [DataFuel](https://datafuel.ai) scraping API. Python 3.10+, sync and async.
+
+```bash
+pip install datafuel
+```
+
+```python
+from datafuel import DataFuel
+
+df = DataFuel()  # or DataFuel("df_key_..."); reads DATAFUEL_API_KEY
+
+markdown = df.markdown("https://example.com")
+```
+
+## Pick the call
+
+| You have | Call | Blocks? |
+|---|---|---|
+| One URL | `scrape` / `markdown` | yes |
+| A site, need its URL list | `map` | yes |
+| A start URL, need many pages | `crawl`, or `start_crawl` + `wait_crawl` + `crawl_pages` | `crawl` does |
+| A list of known URLs | `run_job`, or `create_job` + `wait_job` + `job_results` | `run_job` does |
+| A question for an AI engine | `ask` | yes |
+| A Google search | `search` | yes |
+| Many searches or prompts | `run_search_job` / `run_ask_job`, or `create_search_job` / `create_ask_job` + `wait_job` + `job_results` | `run_*` does |
+| Earlier jobs, tasks, usage | `list_jobs` / `list_tasks` / `analytics` / `transactions` | yes |
+| What a request may contain | `js_instructions` / `ai_providers` / `proxy_locations` / `proxy_asns` / `capabilities` | yes |
+
+Start with plain `scrape`. Turn on `js_rendering` only when the page comes back empty: it is slower and costs five times the credits on a Basic proxy. `map` a section before you `crawl` it, it costs one credit and tells you how big it is.
+
+Every method exists on both `DataFuel` and `AsyncDataFuel`, with the same arguments:
+
+```python
+from datafuel import AsyncDataFuel
+
+async with AsyncDataFuel() as df:
+    result = await df.scrape("https://example.com")
+```
+
+## Scrape
+
+```python
+from datafuel import DataFuel, Proxy, Blocked
+
+df = DataFuel()
+
+try:
+    res = df.scrape(
+        "https://shop.example.com/p/42",
+        proxy=Proxy(type="Premium", country="US"),
+        js_rendering=True,
+        wait_for="#price",
+        extract={"title": "h1", "price": "#price"},
+    )
+    fields = res.data  # {"title": [...], "price": [...]}
+except Blocked as e:
+    # 403/429/503 or an anti-bot wall. Refunded. e.protection names the vendor.
+    ...
+```
+
+`Result` carries the metadata next to the content. Read it first:
+
+| Field | Meaning |
+|---|---|
+| `status_code` | What the target answered. A 404 still completes and bills. |
+| `final_url`, `redirected` | Where the page really came from. |
+| `blocked`, `protection` | The target refused the request. The task failed and was refunded. |
+| `credits_used` | Charged for this task, 0 when it failed. |
+
+`res.text` returns html or markdown, `res.data` structured output, `res.image` screenshot bytes.
+
+Page options, shared by `scrape`, jobs and crawls: `format` (`html`, `markdown`, `json`, `png`, `jpeg`; `Format` lists the rest), `js_rendering`, `wait_for`, `wait_for_timeout_ms`, `js_instructions`, `block_resource`, `main_content_only`, `include_images`, `extract`, `extract_regex`, `template`, `method`, `body`, `content_type`, `headers`, `header_order`, `cookies`, `user_agent`, `user_agent_type`, `ai`.
+
+`js_instructions` is a list of single-action dicts. They run in the order you list them and an action can repeat:
+
+```python
+df.scrape(
+    url,
+    js_rendering=True,
+    js_instructions=[
+        {"fill": ["input[name=q]", "laptops"]},
+        {"click": "button[type=submit]"},
+        {"wait_ms": 1000},
+    ],
+    block_resource=["Image", "Font", "Media"],
+)
+```
+
+The older form, one dict keyed by action (`{"click": "#load-more"}`), still works, but its order is not guaranteed and an action cannot repeat. `df.js_instructions()` lists every action with an example. It needs `js_rendering=True`. `block_resource` takes one resource type or a list of them.
+
+`get_task(id)` reads any task; a task still running comes back with `pending` true. `wait_task(id)` polls until it is done.
+
+## AI after the scrape
+
+Point the page at an LLM with your own key and get structured data back:
+
+```python
+from datafuel import AI
+
+res = df.scrape(
+    url,
+    ai=AI(
+        prompt="extract the product name and its price",
+        format={"name": "string", "price": "number"},
+        provider="openai",  # required; df.ai_providers() lists the providers and their models
+        api_key=os.environ["OPENAI_API_KEY"],
+    ),
+)
+res.data  # {"name": "...", "price": ...}
+```
+
+Works on `scrape` and on URL jobs. Crawls reject it; the SDK says so before sending. `provider` is required. `model` is optional: leave it out for the provider's default, or pick one of the models `df.ai_providers()` lists; any other value is rejected with `InvalidAttributes`.
+
+## Crawl
+
+```python
+crawl_id = df.start_crawl(
+    "https://example.com/docs",
+    max_pages=200,
+    exclude_paths=[r"\.pdf$"],
+    format="markdown",
+)
+status = df.wait_crawl(crawl_id)  # status.stop_reason: max_pages, max_depth_exhausted, ...
+
+for page in df.crawl_pages(crawl_id):
+    if page.pending or not page.ok:
+        continue  # still running, or a failed page (refunded)
+    print(page.url, page.depth, len(page.text))
+```
+
+`df.crawl(url, ...)` does all three in one call:
+
+```python
+crawl = df.crawl("https://example.com/docs", max_pages=200)
+print(crawl.status.stop_reason, crawl.status.total_cost, len(crawl.pages))
+```
+
+Check `stop_reason`: `insufficient_credits` means the crawl ended early. If the wait times out, `WaitTimeout.id` still holds the crawl id — it keeps running and billing server side, so pick it up again with `wait_crawl` and `crawl_pages`.
+
+Unset limits use the API defaults: 100 pages, depth 3, 5 pages in flight.
+
+`df.cancel_crawl(crawl_id)` stops a crawl and returns what was refunded (`refunded_tasks`, `refunded_credits`). A crawl that already finished raises `JobNotCancellable`.
+
+## Jobs
+
+```python
+results = df.run_job(urls, format="markdown")
+for task in results.tasks:
+    if not task.ok:
+        continue  # this URL failed, the others did not
+    print(task.text)
+```
+
+`sequential=True` runs the URLs one after the other; the default runs them concurrently, bounded by your account's concurrency limit. `run_ask_job(prompts, engine=...)` does the same for a batch of prompts, `run_search_job(queries, ...)` for a batch of searches. A job needs at least two targets.
+
+`df.cancel_job(job_id)` stops a job; tasks not yet started are refunded.
+
+## Ask
+
+```python
+res = df.ask("best CRM for a 10-person team", engine="perplexity", websearch=True)
+print(res.data)  # the engine's answer and its sources
+```
+
+`engine` is `openai`, `gemini`, `google_ai_mode`, `perplexity`, `copilot` or `duck_ai` (`Engine.DUCK_AI`); `df.capabilities()` says which are on right now, and `Result.engine` is the engine the task asked for. Other options: `follow_up` (a second prompt in the same conversation, `openai` and `perplexity` only; other engines fail the task, refunded), `country` (exit country) and `location` (where the engine should answer for).
+
+`model` picks the duck.ai model and is valid only with `engine="duck_ai"`: `gpt-6-luna` (default), `gpt-5.4-mini`, `claude-haiku-4-5`, `mistral-small-2603`, `tinfoil/gpt-oss-120b` or `tinfoil/gemma4-31b`; anything else is a 400. A `duck_ai` answer is `{text, citations}` in `res.data`. On `duck_ai`, `websearch` only affects the `tinfoil/*` models; other `duck_ai` models ignore it.
+
+## Search
+
+```python
+res = df.search("best CRM for startups", country="us", language="en")
+results = res.data  # structured results with the default json format
+```
+
+10 credits per result page, whatever the proxy. `page=N` (1 to 10, more is capped at 10) returns N pages merged into one `organic` list with running positions, the other sections from page 1; pages Google did not serve are refunded and `res.data["page"]` is the number returned. `format` is `json` (default), `markdown` or `html`. Searches leave through DataFuel's own pool: pick the market with `country` and `language` (`proxy_country` is accepted but not used yet). Other options: `page`, `google_domain`, one of `location` / `uule` / `lat`+`lon` (+`radius`), and Google's own `cr`, `lr`, `tbs`, `safe`, `nfpr`, `filter`, `uds`, `kgmid`, `si`, `ludocid`, `lsig`, `ibp`. Set `device="mobile"` for mobile results (default `"desktop"`; mobile has no `total_results`) and `ai_overview=True` to also get Google's AI overview (default `False`) in `res.data["ai_overview"]`: `{text, products: [{title, price, source, rating, reviews, link}], references: [{title, source, snippet, url, redirect}]}`.
+
+## Map
+
+```python
+site = df.map("https://example.com", search="blog", limit=500)
+for link in site.links:
+    print(link.url)
+```
+
+An empty `site.links` comes with a `site.reason`. `no_links_on_page` usually means the navigation is rendered client-side.
+
+## Errors
+
+```python
+import datafuel
+
+try:
+    res = df.scrape(url)
+except datafuel.Blocked as e:  # subclass of TaskFailed
+    ...
+except datafuel.InsufficientCredits:
+    ...
+```
+
+- `NoAPIKey`: no key was passed and `DATAFUEL_API_KEY` is empty. Raised before any request.
+- `APIError`: the API refused the request. Subclasses: `Unauthorized`, `Forbidden`, `InsufficientCredits`, `RateLimited`, `NotFound`, `InvalidAttributes`, `IdempotencyKeyReused`, `JobNotCancellable`, `AlreadyExists` (a create collided with an existing task or job; nothing was charged, send it again).
+- `ModuleUnavailable`, `EngineUnavailable`: an operator switched a task type or LLM engine off, e.g. during a provider outage. The reason is in the message, nothing is charged, and the SDK does not retry. `df.capabilities()` lists what is on.
+- `Unavailable`: any other 503, e.g. `SERVICE_UNAVAILABLE` while the key store is down. Temporary, so the SDK retries it.
+- `TaskFailed`, and `Blocked` when the target refused: the API accepted the task but the page could not be scraped. The exception carries `.result`, so the envelope is still readable. Failed tasks are refunded.
+- `WaitTimeout`: a wait ran out of time. `.id` picks the work back up.
+
+Tasks inside a job or a crawl never raise on their own — check `task.ok`, or call `task.raise_for_status()`.
+
+## Usage and history
+
+```python
+usage = df.analytics(start_date="2026-09-01", end_date="2026-09-30")
+print(usage.summary.success_rate, usage.summary.avg_credits_per_request)
+
+page = df.list_jobs(type="crawl", limit=20)
+failed = df.list_tasks(job_id=page.jobs[0].id, status="failed")
+history = df.transactions(operation="refund", limit=50)
+```
+
+`list_jobs` and `list_tasks` run newest first; pass `next_cursor` back as `cursor` until it is `None`. Task items carry no result: call `get_task(id)` for it. Dates are `YYYY-MM-DD` strings or `date` objects in UTC, and `end_date` is inclusive. `transactions` pages with `page` and `limit`, and its `sums` total each operation over the whole range. In `analytics`, `status_code` 0 means the target never answered (timeout, DNS).
+
+## Account and config
+
+```python
+df.balance()  # remaining credits
+df.balance_split()  # the same, split into plan_balance and payg_balance
+df.me()  # profile, concurrency and credit limits
+df.capabilities()  # which task types and LLM engines are on
+df.analytics(start_date="2026-09-01", interval="weekly")
+df.check_protection(["https://shop.example.com/"])  # anti-bot vendor per URL, nothing scraped
+df.proxy_locations("Premium")  # countries, regions, cities
+df.proxy_asns("US", proxy_type="Premium")
+df.js_instructions()  # the browser actions js_instructions accepts
+df.ai_providers()  # the LLM providers and models AI accepts
+df.health(deep=True)  # .ok is False while a dependency is down
+df.reset_api_key()  # returns the new key, the only time it is shown
+```
+
+`balance` is what you can spend. It is made of plan credits and pay-as-you-go credits. Plan credits are spent first; unused ones roll over when the plan renews and expire if it is not renewed. Pay-as-you-go credits come from one-time credit packs (a `purchase` transaction), are spent after plan credits and never expire. Each transaction's `plan_amount` is the part of `amount` that moved plan credits.
+
+## Retries and idempotency
+
+Every write carries an `Idempotency-Key`, generated per request unless you pass `idempotency_key=`. That makes retries safe: network errors and 429/502/503/504 answers are retried with backoff, reusing the key, so a retry attaches to the task already running instead of charging twice. Other 4xx errors are never retried.
+
+The same key is how a synchronous task survives a slow page: when the API answers "still processing", the SDK re-sends the identical request under the identical key until the result is ready or the call's timeout is reached.
+
+A key replays its stored result, failures included. To run a failed scrape again, send a new request rather than the same key.
+
+## Options
+
+```python
+DataFuel(
+    api_key,
+    base_url="https://scraping-api.staging.datafuel.ai/api/v1",
+    timeout=300,  # seconds; scrape blocks until the page is ready
+    max_retries=4,
+    poll_interval=5,  # wait_job / wait_crawl
+    user_agent="my-app/1.2",
+    http_client=my_httpx_client,
+)
+```
+
+Keep `timeout` generous: `scrape` blocks until the page is ready, which can take minutes with `js_rendering`. Every call also takes its own `timeout=`.
+
+If you pass your own `http_client`, leave `follow_redirects` off. httpx keeps custom headers across a redirect, so a redirect to another host would carry your `X-API-Key` to it. The clients this SDK builds have redirects disabled.
+
+Both clients are safe to share across threads (or tasks): nothing mutates after construction.
+
+## Try it
+
+```bash
+DATAFUEL_API_KEY=df_key_... python examples/quickstart.py https://example.com
+```
+
+Guides and the full API reference: https://docs.datafuel.ai
+
+## License
+
+MIT, see the `LICENSE` file.