octocrawl-client 0.3.0__tar.gz

octocrawl_client-0.3.0/.gitignore

@@ -0,0 +1,41 @@
+# Node
+node_modules/
+dist/
+*.tsbuildinfo
+npm-debug.log*
+yarn-debug.log*
+yarn-error.log*
+pnpm-debug.log*
+
+# System
+.DS_Store
+Thumbs.db
+
+# IDE
+.vscode/
+.idea/
+*.swp
+*.swo
+*~
+
+# Claude Code
+.claude/worktrees/
+.claude/scheduled-tasks/
+
+# Testing
+coverage/
+.nyc_output/
+
+# Build artifacts
+*.log
+.cache/
+tmp/
+temp/
+output/
+.w2l/
+
+# Python client
+python/**/__pycache__/
+python/**/*.egg-info/
+python/.pytest_cache/
+python/dist/

octocrawl_client-0.3.0/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 W2L contributors
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

octocrawl_client-0.3.0/PKG-INFO

@@ -0,0 +1,47 @@
+Metadata-Version: 2.5
+Name: octocrawl-client
+Version: 0.3.0
+Summary: Python client for the Octocrawl API: web pages as Markdown, tables and JSON, each with its Evidence Record, into pandas.
+Project-URL: Homepage, https://github.com/77777R7/Octocrawl
+Project-URL: Source, https://github.com/77777R7/Octocrawl/tree/main/python
+License-Expression: MIT
+License-File: LICENSE
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Python :: 3
+Classifier: Topic :: Internet :: WWW/HTTP :: Indexing/Search
+Classifier: Topic :: Scientific/Engineering :: Information Analysis
+Requires-Python: >=3.9
+Requires-Dist: httpx>=0.27
+Provides-Extra: pandas
+Requires-Dist: pandas>=1.5; extra == 'pandas'
+Provides-Extra: test
+Requires-Dist: pandas>=1.5; extra == 'test'
+Requires-Dist: pytest>=7; extra == 'test'
+Description-Content-Type: text/markdown
+
+# octocrawl-client (Python)
+
+Python client for the Octocrawl API: web pages as Markdown, tables and JSON, each page with its **Evidence Record** (final URL, fetch time, HTTP status, robots.txt decision, hashes of what was read and delivered), into pandas.
+
+```bash
+pip install 'octocrawl-client[pandas]'
+npx octocrawl serve       # a local Octocrawl API on http://127.0.0.1:8787
+```
+
+```python
+import octocrawl_client
+
+result = octocrawl_client.batch(
+    ["https://books.toscrape.com/", "https://quotes.toscrape.com/"],
+    formats=["markdown", "tables"],
+)
+df = result.to_pandas()          # one row per page
+df[["url", "status", "fetched_at", "http_status", "raw_sha256"]]
+result.report["succeeded"], result.report["failed"]
+```
+
+`to_pandas()` has one row per page, failed pages included, with these columns: `url`, `status`, `reason`, `final_url`, `fetched_at`, `http_status`, `lane`, `robots_decision`, `raw_sha256`, `markdown_sha256`, `extractor`, `source_commit`, `cache_state`, `cached_at` and `markdown`. A value W2L did not observe is missing (`None`, or `<NA>` for `http_status`), never 0. `result.items` keeps each page as the API answered it, its tables (`formats=["tables"]`) and full `evidenceRecord` included.
+
+Options take the API's names in snake_case or camelCase: `max_age=3_600_000`, `only_main_content=False`, `include_tags=["main"]`, `parsers=[{"type": "pdf", "maxPages": 5}]`. `W2L(base_url=..., token=...)` names another API (default `W2L_API_URL`, else http://127.0.0.1:8787; `W2L_API_TOKEN`). A batch takes up to 1,000 URLs; `octocrawl_client.scrape`, `octocrawl_client.map` and `octocrawl_client.crawl` work the same way. (The name `octocrawl` on PyPI belongs to another project.)
+
+Licence: MIT. Source and the API reference: https://github.com/77777R7/Octocrawl

octocrawl_client-0.3.0/README.md

@@ -0,0 +1,26 @@
+# octocrawl-client (Python)
+
+Python client for the Octocrawl API: web pages as Markdown, tables and JSON, each page with its **Evidence Record** (final URL, fetch time, HTTP status, robots.txt decision, hashes of what was read and delivered), into pandas.
+
+```bash
+pip install 'octocrawl-client[pandas]'
+npx octocrawl serve       # a local Octocrawl API on http://127.0.0.1:8787
+```
+
+```python
+import octocrawl_client
+
+result = octocrawl_client.batch(
+    ["https://books.toscrape.com/", "https://quotes.toscrape.com/"],
+    formats=["markdown", "tables"],
+)
+df = result.to_pandas()          # one row per page
+df[["url", "status", "fetched_at", "http_status", "raw_sha256"]]
+result.report["succeeded"], result.report["failed"]
+```
+
+`to_pandas()` has one row per page, failed pages included, with these columns: `url`, `status`, `reason`, `final_url`, `fetched_at`, `http_status`, `lane`, `robots_decision`, `raw_sha256`, `markdown_sha256`, `extractor`, `source_commit`, `cache_state`, `cached_at` and `markdown`. A value W2L did not observe is missing (`None`, or `<NA>` for `http_status`), never 0. `result.items` keeps each page as the API answered it, its tables (`formats=["tables"]`) and full `evidenceRecord` included.
+
+Options take the API's names in snake_case or camelCase: `max_age=3_600_000`, `only_main_content=False`, `include_tags=["main"]`, `parsers=[{"type": "pdf", "maxPages": 5}]`. `W2L(base_url=..., token=...)` names another API (default `W2L_API_URL`, else http://127.0.0.1:8787; `W2L_API_TOKEN`). A batch takes up to 1,000 URLs; `octocrawl_client.scrape`, `octocrawl_client.map` and `octocrawl_client.crawl` work the same way. (The name `octocrawl` on PyPI belongs to another project.)
+
+Licence: MIT. Source and the API reference: https://github.com/77777R7/Octocrawl

octocrawl_client-0.3.0/pyproject.toml

@@ -0,0 +1,30 @@
+[build-system]
+requires = ["hatchling>=1.21"]
+build-backend = "hatchling.build"
+
+[project]
+name = "octocrawl-client"
+version = "0.3.0"
+description = "Python client for the Octocrawl API: web pages as Markdown, tables and JSON, each with its Evidence Record, into pandas."
+readme = "README.md"
+license = "MIT"
+license-files = ["LICENSE"]
+requires-python = ">=3.9"
+dependencies = ["httpx>=0.27"]
+classifiers = [
+  "Programming Language :: Python :: 3",
+  "License :: OSI Approved :: MIT License",
+  "Topic :: Internet :: WWW/HTTP :: Indexing/Search",
+  "Topic :: Scientific/Engineering :: Information Analysis",
+]
+
+[project.optional-dependencies]
+pandas = ["pandas>=1.5"]
+test = ["pytest>=7", "pandas>=1.5"]
+
+[project.urls]
+Homepage = "https://github.com/77777R7/Octocrawl"
+Source = "https://github.com/77777R7/Octocrawl/tree/main/python"
+
+[tool.hatch.build.targets.wheel]
+packages = ["src/octocrawl_client"]

octocrawl_client-0.3.0/src/octocrawl_client/__init__.py

@@ -0,0 +1,17 @@
+"""Octocrawl: web pages as Markdown, tables and JSON, each with its Evidence Record.
+
+    import octocrawl_client
+    df = octocrawl_client.batch(["https://example.com/a", "https://example.com/b"], formats=["markdown", "tables"]).to_pandas()
+
+The client talks to an Octocrawl API: `npx octocrawl serve` runs one locally (http://127.0.0.1:8787,
+or set W2L_API_URL; W2L_API_TOKEN for a hosted one).
+"""
+
+from ._version import __version__
+from .client import DEFAULT_BASE_URL, MAX_BATCH_URLS, ORIGIN, W2L, JobResult, W2LError, api_name, batch, crawl, map, scrape
+from .frame import EVIDENCE_COLUMNS, evidence_row, evidence_rows
+
+__all__ = [
+    "__version__", "W2L", "W2LError", "JobResult", "batch", "crawl", "map", "scrape", "api_name",
+    "EVIDENCE_COLUMNS", "evidence_row", "evidence_rows", "DEFAULT_BASE_URL", "MAX_BATCH_URLS", "ORIGIN",
+]

octocrawl_client-0.3.0/src/octocrawl_client/_version.py

@@ -0,0 +1 @@
+__version__ = "0.3.0"

octocrawl_client-0.3.0/src/octocrawl_client/client.py

@@ -0,0 +1,201 @@
+"""The W2L API over HTTP: scrape, map, batch and crawl, each page with its Evidence Record."""
+
+from __future__ import annotations
+
+import os
+import re
+import time
+from dataclasses import dataclass, field
+from typing import Any, Iterable, Mapping, Optional, Sequence
+
+import httpx
+
+from ._version import __version__
+from .frame import evidence_rows, to_pandas
+
+DEFAULT_BASE_URL = "http://127.0.0.1:8787"
+#: What the client records as the `origin` of every request, for W2L's own records; nothing goes to the target.
+ORIGIN = f"py-sdk@{__version__}"
+#: The most URLs one batch takes (the API's limit).
+MAX_BATCH_URLS = 1000
+TERMINAL = ("completed", "failed", "cancelled")
+
+# Option names whose acronym the plain snake_case → camelCase rule would spell wrong.
+_ACRONYMS = {
+    "regex_on_full_url": "regexOnFullURL",
+    "ignore_invalid_urls": "ignoreInvalidURLs",
+    "deduplicate_similar_urls": "deduplicateSimilarURLs",
+}
+
+
+def api_name(key: str) -> str:
+    """`max_age` → `maxAge`; a name already in camelCase is kept as it is."""
+    if key in _ACRONYMS:
+        return _ACRONYMS[key]
+    return re.sub(r"_([a-z0-9])", lambda m: m.group(1).upper(), key)
+
+
+def _options(options: Mapping[str, Any]) -> dict[str, Any]:
+    return {api_name(key): value for key, value in options.items() if value is not None}
+
+
+class W2LError(Exception):
+    """An answer of the API that is not a result: a refused request (400), an unknown id (404), a rate limit (429), ..."""
+
+    def __init__(self, status: int, message: str, code: Optional[str] = None, details: Any = None, agent_hints: Optional[list[str]] = None):
+        super().__init__(f"{status} {code or 'error'}: {message}")
+        self.status = status
+        self.code = code
+        self.message = message
+        self.details = details
+        self.agent_hints = agent_hints or []
+
+
+@dataclass
+class JobResult:
+    """A finished batch or crawl: its final report and every page it recorded, failed ones included."""
+
+    task_id: str
+    report: dict[str, Any]
+    items: list[dict[str, Any]] = field(default_factory=list)
+
+    def rows(self, include_markdown: bool = True) -> list[dict[str, Any]]:
+        """One row per page with the evidence columns (see `EVIDENCE_COLUMNS`)."""
+        return evidence_rows(self.items, include_markdown=include_markdown)
+
+    def to_pandas(self, include_markdown: bool = True):
+        """A pandas DataFrame with one row per page and the evidence columns; needs `pip install 'octocrawl-client[pandas]'`."""
+        return to_pandas(self.rows(include_markdown=include_markdown))
+
+
+class W2L:
+    """A client of one W2L API: a local `octocrawl serve` (default `W2L_API_URL`, else http://127.0.0.1:8787) or a hosted one with a token."""
+
+    def __init__(
+        self,
+        base_url: Optional[str] = None,
+        token: Optional[str] = None,
+        *,
+        timeout: float = 330.0,
+        poll_interval: float = 1.0,
+        transport: Optional[httpx.BaseTransport] = None,
+    ):
+        self.base_url = (base_url or os.environ.get("W2L_API_URL") or DEFAULT_BASE_URL).rstrip("/")
+        token = token if token is not None else os.environ.get("W2L_API_TOKEN") or None
+        headers = {"user-agent": ORIGIN}
+        if token:
+            headers["authorization"] = f"Bearer {token}"
+        # A scrape is answered at its deadline (up to 300 s), so reads wait a little longer than that.
+        self._http = httpx.Client(base_url=self.base_url, headers=headers, timeout=timeout, transport=transport)
+        self.poll_interval = poll_interval
+
+    def close(self) -> None:
+        self._http.close()
+
+    def __enter__(self) -> "W2L":
+        return self
+
+    def __exit__(self, *_exc: object) -> None:
+        self.close()
+
+    def _call(self, method: str, path: str, *, retries: int = 0, **kwargs: Any) -> Any:
+        """One API call; with `retries`, a network error, a 5xx or a 429 is tried again (a poll or a page read, never a start)."""
+        for attempt in range(retries + 1):
+            try:
+                response = self._http.request(method, path, **kwargs)
+            except httpx.TransportError:
+                if attempt == retries:
+                    raise
+                time.sleep(min(2 ** attempt, 10))
+                continue
+            if (response.status_code >= 500 or response.status_code == 429) and attempt < retries:
+                time.sleep(min(float(response.headers.get("retry-after", 2 ** attempt)), 30))
+                continue
+            break
+        if response.status_code >= 400:
+            try:
+                body = response.json()
+            except ValueError:
+                body = {"error": response.text}
+            raise W2LError(response.status_code, str(body.get("error", "")), body.get("code"), body.get("details"), body.get("agentHints"))
+        return response.json()
+
+    def scrape(self, url: str, **options: Any) -> dict[str, Any]:
+        """One page: the scrape response with its `evidenceRecord` (compact unless `debug=True`)."""
+        return self._call("POST", "/v1/scrape", json={"debug": False, **_options(options), "url": url, "origin": ORIGIN})
+
+    def map(self, url: str, **options: Any) -> dict[str, Any]:
+        """A site's URLs from its sitemaps and start page, without fetching each page."""
+        return self._call("POST", "/v1/map", json={**_options(options), "url": url, "origin": ORIGIN})
+
+    def batch(self, urls: Iterable[str], *, wait_timeout: Optional[float] = None, **options: Any) -> JobResult:
+        """Scrape up to 1,000 URLs as one batch, wait until it ends, and return its report and every item."""
+        urls = list(urls)
+        if not urls:
+            raise ValueError("batch takes at least one URL")
+        if len(urls) > MAX_BATCH_URLS:
+            raise ValueError(f"a batch takes at most {MAX_BATCH_URLS} URLs; split the list and call batch for each part")
+        accepted = self._call("POST", "/v1/batches", json={**_options(options), "urls": urls, "origin": ORIGIN})
+        task_id = accepted["taskId"]
+        report = self._wait(f"/v1/batches/{task_id}", wait_timeout)
+        return JobResult(task_id, report, self._items(f"/v1/batches/{task_id}/items"))
+
+    def crawl(self, url: str, *, wait_timeout: Optional[float] = None, **options: Any) -> JobResult:
+        """Crawl from `url`, wait until the crawl ends, and return its report and every page of its latest attempt."""
+        accepted = self._call("POST", "/v1/crawl", json={**_options(options), "url": url, "origin": ORIGIN})
+        task_id = accepted["taskId"]
+        report = self._wait(f"/v1/crawl/{task_id}", wait_timeout)
+        return JobResult(task_id, report, self._items(f"/v1/crawl/{task_id}/pages") + self._items(f"/v1/crawl/{task_id}/errors"))
+
+    def _wait(self, path: str, wait_timeout: Optional[float]) -> dict[str, Any]:
+        deadline = None if wait_timeout is None else time.monotonic() + wait_timeout
+        while True:
+            report = self._call("GET", path, retries=5)
+            if report.get("status") in TERMINAL:
+                return report
+            if deadline is not None and time.monotonic() >= deadline:
+                raise TimeoutError(f"{path} is still {report.get('status')} after {wait_timeout} s; it keeps running on the server")
+            time.sleep(self.poll_interval)
+
+    def _items(self, path: str) -> list[dict[str, Any]]:
+        items: list[dict[str, Any]] = []
+        cursor: Optional[str] = None
+        while True:
+            params: dict[str, Any] = {"limit": 50}
+            if cursor is not None:
+                params["cursor"] = cursor
+            page = self._call("GET", path, params=params, retries=5)
+            items.extend(page.get("items", []))
+            cursor = page.get("nextCursor")
+            if not page.get("hasMore"):
+                return items
+            if cursor is None:
+                raise W2LError(200, f"{path} says it has more items but gives no cursor; {len(items)} items read so far")
+
+
+def _default() -> W2L:
+    return W2L()
+
+
+def scrape(url: str, **options: Any) -> dict[str, Any]:
+    """`W2L().scrape(...)` on the API named by `W2L_API_URL` (default http://127.0.0.1:8787)."""
+    with _default() as client:
+        return client.scrape(url, **options)
+
+
+def map(url: str, **options: Any) -> dict[str, Any]:  # noqa: A001 - the API's own name
+    """`W2L().map(...)` on the API named by `W2L_API_URL`."""
+    with _default() as client:
+        return client.map(url, **options)
+
+
+def batch(urls: Sequence[str], **options: Any) -> JobResult:
+    """`W2L().batch(...)`: `octocrawl_client.batch(urls).to_pandas()` gives one row per page with its evidence."""
+    with _default() as client:
+        return client.batch(urls, **options)
+
+
+def crawl(url: str, **options: Any) -> JobResult:
+    """`W2L().crawl(...)` on the API named by `W2L_API_URL`."""
+    with _default() as client:
+        return client.crawl(url, **options)

octocrawl_client-0.3.0/src/octocrawl_client/frame.py

@@ -0,0 +1,73 @@
+"""Pages as rows with their evidence, for pandas or anything that takes a list of dicts."""
+
+from __future__ import annotations
+
+from typing import Any, Iterable, Optional
+
+#: The columns of `rows()` and `to_pandas()`, in order. A value W2L did not observe is None, never 0 or a guess.
+EVIDENCE_COLUMNS = (
+    "url",              # the URL the batch or crawl was given (or discovered)
+    "status",           # success, partial, empty_verified, blocked, failed, cancelled, budget_exceeded, duplicate
+    "reason",           # the failure, block or budget reason; None for other statuses
+    "final_url",        # the URL that answered, after redirects
+    "fetched_at",       # UTC ISO time the answer was received
+    "http_status",
+    "lane",             # http, browser_local, provider, ...
+    "robots_decision",  # allowed, disallowed, no_robots; None when none was made
+    "raw_sha256",       # SHA-256 of the page as read
+    "markdown_sha256",  # SHA-256 of the delivered Markdown
+    "extractor",        # the extractor version, e.g. extract-tf/5 or pdf-text/1
+    "source_commit",    # the W2L commit that ran, when the server declares it
+    "cache_state",      # hit or miss when the request asked the cache; None otherwise
+    "cached_at",        # on a hit, when the reused result was fetched
+    "markdown",
+)
+
+
+def _get(value: Any, *path: str) -> Any:
+    for key in path:
+        if not isinstance(value, dict):
+            return None
+        value = value.get(key)
+    return value
+
+
+def evidence_row(item: dict[str, Any], include_markdown: bool = True) -> dict[str, Any]:
+    """One batch item or crawl page as a row of EVIDENCE_COLUMNS, read from its `evidenceRecord`."""
+    record: Optional[dict[str, Any]] = item.get("evidenceRecord")
+    row = {
+        "url": item.get("url"),
+        "status": item.get("status"),
+        "reason": _get(record, "reason") if record is not None else (item.get("failureReason") or item.get("blockReason") or item.get("budgetExceeded")),
+        "final_url": _get(record, "finalUrl"),
+        "fetched_at": _get(record, "fetchedAt"),
+        "http_status": _get(record, "httpStatus"),
+        "lane": _get(record, "lane") if record is not None else item.get("lane"),
+        "robots_decision": _get(record, "robotsDecision", "decision"),
+        "raw_sha256": _get(record, "rawSha256"),
+        "markdown_sha256": _get(record, "outputSha256", "markdown"),
+        "extractor": _get(record, "extractor", "version"),
+        "source_commit": _get(record, "extractor", "commit"),
+        "cache_state": item.get("cacheState"),
+        "cached_at": item.get("cachedAt"),
+        "markdown": item.get("markdown"),
+    }
+    if not include_markdown:
+        row.pop("markdown")
+    return row
+
+
+def evidence_rows(items: Iterable[dict[str, Any]], include_markdown: bool = True) -> list[dict[str, Any]]:
+    return [evidence_row(item, include_markdown=include_markdown) for item in items]
+
+
+def to_pandas(rows: list[dict[str, Any]]):
+    try:
+        import pandas as pd
+    except ImportError as error:  # pragma: no cover - depends on the environment
+        raise ImportError("to_pandas needs pandas: pip install 'octocrawl-client[pandas]'") from error
+    columns = [column for column in EVIDENCE_COLUMNS if not rows or column in rows[0]]
+    frame = pd.DataFrame(rows, columns=columns)
+    # Unknown stays missing: an HTTP status W2L did not observe is <NA>, not 0.
+    frame["http_status"] = frame["http_status"].astype("Int64")
+    return frame

octocrawl_client-0.3.0/tests/test_client.py

@@ -0,0 +1,139 @@
+import json
+
+import httpx
+import pytest
+
+import octocrawl_client as w2l
+from octocrawl_client import EVIDENCE_COLUMNS, W2L, W2LError, api_name
+
+RECORD = {
+    "schemaVersion": "w2l.evidence/1", "requestedUrl": "https://a.example/", "finalUrl": "https://a.example/", "fetchedAt": "2026-10-03T08:00:00.000Z",
+    "httpStatus": 200, "status": "success", "reason": None, "lane": "http",
+    "robotsDecision": {"decision": "allowed"}, "rawSha256": "a" * 64, "outputSha256": {"markdown": "b" * 64, "json": None},
+    "extractor": {"name": "extract-tf", "version": "extract-tf/5", "commit": None},
+}
+ITEMS = [
+    {"id": "1", "url": "https://a.example/", "status": "success", "markdown": "# A", "evidenceRecord": RECORD, "cacheState": "hit", "cachedAt": "2026-10-03T08:00:00.000Z"},
+    {"id": "2", "url": "https://b.example/", "status": "failed", "failureReason": "dns_error", "markdown": None, "evidenceRecord": {**RECORD, "requestedUrl": "https://b.example/", "finalUrl": None, "fetchedAt": None, "httpStatus": None, "status": "failed", "reason": "dns_error", "robotsDecision": None, "rawSha256": None, "outputSha256": {"markdown": None, "json": None}}},
+    {"id": "3", "url": "https://c.example/", "status": "pending", "markdown": None, "evidenceRecord": None},
+]
+
+
+def api(calls):
+    """A fake W2L API: a batch that is running on the first poll and completed on the second, with its items on two pages."""
+    polls = {"n": 0}
+
+    def handle(request: httpx.Request) -> httpx.Response:
+        calls.append((request.method, request.url.path, dict(request.url.params), json.loads(request.content) if request.content else None, request.headers.get("authorization")))
+        if request.method == "POST" and request.url.path == "/v1/batches":
+            return httpx.Response(202, json={"taskId": "t1"})
+        if request.url.path == "/v1/batches/t1":
+            polls["n"] += 1
+            return httpx.Response(200, json={"taskId": "t1", "status": "running" if polls["n"] == 1 else "completed", "succeeded": 1, "failed": 1})
+        if request.url.path == "/v1/batches/t1/items":
+            first = request.url.params.get("cursor") is None
+            return httpx.Response(200, json={"items": ITEMS[:2] if first else ITEMS[2:], "hasMore": first, "nextCursor": "c2" if first else None})
+        if request.url.path == "/v1/scrape":
+            body = json.loads(request.content)
+            if body.get("maxAge") == -1:
+                return httpx.Response(400, json={"error": "maxAge must be an integer number of milliseconds from 0 to 315360000000", "code": "invalid_request"})
+            return httpx.Response(200, json={"status": "success", "markdown": "# A", "evidenceRecord": RECORD})
+        return httpx.Response(404, json={"error": "not found", "code": "not_found"})
+
+    return httpx.MockTransport(handle)
+
+
+def test_batch_waits_pages_and_gives_evidence_rows():
+    calls = []
+    client = W2L("http://w2l.test", token="t0k", poll_interval=0, transport=api(calls))
+    result = client.batch(["https://a.example/", "https://b.example/", "https://c.example/"], formats=["markdown", "tables"], max_age=3_600_000, ignore_invalid_urls=True)
+    method, path, _, body, auth = calls[0]
+    assert (method, path, auth) == ("POST", "/v1/batches", "Bearer t0k")
+    assert body == {"formats": ["markdown", "tables"], "maxAge": 3_600_000, "ignoreInvalidURLs": True, "urls": ["https://a.example/", "https://b.example/", "https://c.example/"], "origin": w2l.ORIGIN}
+    assert [c[1] for c in calls].count("/v1/batches/t1") == 2
+    assert [c[2] for c in calls if c[1] == "/v1/batches/t1/items"] == [{"limit": "50"}, {"limit": "50", "cursor": "c2"}]
+    assert result.task_id == "t1" and result.report["status"] == "completed" and len(result.items) == 3
+
+    rows = result.rows()
+    assert list(rows[0]) == list(EVIDENCE_COLUMNS)
+    assert rows[0] == {
+        "url": "https://a.example/", "status": "success", "reason": None, "final_url": "https://a.example/", "fetched_at": "2026-10-03T08:00:00.000Z",
+        "http_status": 200, "lane": "http", "robots_decision": "allowed", "raw_sha256": "a" * 64, "markdown_sha256": "b" * 64,
+        "extractor": "extract-tf/5", "source_commit": None, "cache_state": "hit", "cached_at": "2026-10-03T08:00:00.000Z", "markdown": "# A",
+    }
+    # Unknown stays None: no fetch time, no status, no hash for a page that was not read; no record at all for one without a result.
+    assert rows[1]["reason"] == "dns_error" and rows[1]["http_status"] is None and rows[1]["fetched_at"] is None
+    assert rows[2]["status"] == "pending" and rows[2]["final_url"] is None and rows[2]["lane"] is None
+
+
+def test_to_pandas_keeps_unknown_missing():
+    pd = pytest.importorskip("pandas")
+    client = W2L("http://w2l.test", poll_interval=0, transport=api([]))
+    frame = client.batch(["https://a.example/", "https://b.example/", "https://c.example/"]).to_pandas(include_markdown=False)
+    assert list(frame.columns) == [c for c in EVIDENCE_COLUMNS if c != "markdown"]
+    assert frame["http_status"].tolist()[0] == 200
+    assert pd.isna(frame["http_status"].tolist()[1]) and pd.isna(frame["http_status"].tolist()[2])
+    assert str(frame["http_status"].dtype) == "Int64"
+
+
+def test_errors_carry_the_api_code_and_options_are_named_as_the_api_names_them():
+    client = W2L("http://w2l.test", transport=api([]))
+    with pytest.raises(W2LError) as error:
+        client.scrape("https://a.example/", max_age=-1)
+    assert error.value.status == 400 and error.value.code == "invalid_request" and "maxAge must be" in error.value.message
+    assert client.scrape("https://a.example/")["evidenceRecord"]["status"] == "success"
+    assert api_name("only_main_content") == "onlyMainContent"
+    assert api_name("regex_on_full_url") == "regexOnFullURL"
+    assert api_name("deduplicate_similar_urls") == "deduplicateSimilarURLs"
+    assert api_name("maxAge") == "maxAge"
+    with pytest.raises(ValueError):
+        client.batch(["https://a.example/"] * 1001)
+    with pytest.raises(ValueError):
+        client.batch([])
+
+
+def test_crawl_reads_pages_and_errors_and_polls_through_a_transient_failure():
+    hits = {"status": 0}
+
+    def handle(request: httpx.Request) -> httpx.Response:
+        if request.method == "POST":
+            return httpx.Response(202, json={"taskId": "c1"})
+        if request.url.path == "/v1/crawl/c1":
+            hits["status"] += 1
+            if hits["status"] == 1:
+                raise httpx.ConnectError("blip")
+            if hits["status"] == 2:
+                return httpx.Response(503, json={"error": "busy"}, headers={"retry-after": "0"})
+            return httpx.Response(200, json={"taskId": "c1", "status": "completed"})
+        if request.url.path == "/v1/crawl/c1/pages":
+            return httpx.Response(200, json={"items": [ITEMS[0]], "hasMore": False, "nextCursor": None})
+        if request.url.path == "/v1/crawl/c1/errors":
+            return httpx.Response(200, json={"items": [ITEMS[1]], "hasMore": False, "nextCursor": None})
+        return httpx.Response(404, json={"error": "not found"})
+
+    client = W2L("http://w2l.test", poll_interval=0, transport=httpx.MockTransport(handle))
+    result = client.crawl("https://a.example/", max_pages=2, sitemap="skip")
+    assert result.report["status"] == "completed" and hits["status"] == 3
+    assert [row["status"] for row in result.rows()] == ["success", "failed"]
+
+
+def test_a_listing_that_has_more_but_no_cursor_is_an_error():
+    def handle(request: httpx.Request) -> httpx.Response:
+        if request.method == "POST":
+            return httpx.Response(202, json={"taskId": "t1"})
+        if request.url.path == "/v1/batches/t1":
+            return httpx.Response(200, json={"status": "completed"})
+        return httpx.Response(200, json={"items": [ITEMS[0]], "hasMore": True, "nextCursor": None})
+
+    with pytest.raises(W2LError, match="no cursor"):
+        W2L("http://w2l.test", poll_interval=0, transport=httpx.MockTransport(handle)).batch(["https://a.example/"])
+
+
+def test_the_base_url_and_token_come_from_the_environment(monkeypatch):
+    monkeypatch.setenv("W2L_API_URL", "http://env.test:9000/")
+    monkeypatch.setenv("W2L_API_TOKEN", "envtok")
+    calls = []
+    client = W2L(transport=api(calls))
+    assert client.base_url == "http://env.test:9000"
+    client.scrape("https://a.example/")
+    assert calls[0][4] == "Bearer envtok"