probnode-sdk 0.1.0__tar.gz

probnode_sdk-0.1.0/PKG-INFO

@@ -0,0 +1,50 @@
+Metadata-Version: 2.5
+Name: probnode-sdk
+Version: 0.1.0
+Summary: Python client and CLI for ProbNode: Polymarket order books, trades and gaps
+Project-URL: Homepage, https://probnode.com
+Project-URL: Documentation, https://probnode.com/docs
+License: MIT
+Requires-Python: >=3.10
+Requires-Dist: httpx>=0.27
+Requires-Dist: numpy>=1.24
+Requires-Dist: pyarrow>=14
+Provides-Extra: dev
+Requires-Dist: pytest>=8.3; extra == 'dev'
+Requires-Dist: ruff>=0.6; extra == 'dev'
+Provides-Extra: pandas
+Requires-Dist: pandas>=2.0; extra == 'pandas'
+Provides-Extra: polars
+Requires-Dist: polars>=0.20; extra == 'polars'
+Description-Content-Type: text/markdown
+
+# probnode-sdk
+
+Python client and command line for [ProbNode](https://probnode.com): Polymarket order books, trades
+and feed gaps as hourly and market-day files (lp2, parquet, csv.gz), with a sha256 for every file.
+
+```sh
+pip install ./packages/probnode        # from a checkout; on PyPI as probnode-sdk soon
+                                       # (not `pip install probnode`: that is someone else's package)
+probnode login
+probnode datasets
+probnode files books parquet 2026-11-08T12 2026-11-08T14
+probnode sync books parquet 2026-11-08T12 2026-11-08T14 -o data/   # again later: only changes
+probnode doctor data/                                              # re-check every sha256
+```
+
+```python
+import probnode
+
+pn = probnode.Client()  # PROBNODE_API_KEY, or the key `probnode login` saved
+files = pn.files("books", "parquet", "2026-11-08T12:00:00Z", "2026-11-08T13:00:00Z")
+paths = list(pn.download_all(files, "data/"))  # each checked against its sha256
+table = probnode.read(paths[0])  # pyarrow.Table -> .to_pandas() / polars.from_arrow()
+
+for batch in probnode.iter_lp2("hour.lp2"):  # native books, decoded in bounded memory
+    ...
+```
+
+Units: prices x10,000 (5400 = 0.54), sizes x1,000,000 shares, timestamps in ms UTC. Signed links
+last 15 minutes; the client signs again whenever it needs to. The import name is `probnode`; the
+PyPI name is `probnode-sdk` because `probnode` is taken.

probnode_sdk-0.1.0/README.md

@@ -0,0 +1,30 @@
+# probnode-sdk
+
+Python client and command line for [ProbNode](https://probnode.com): Polymarket order books, trades
+and feed gaps as hourly and market-day files (lp2, parquet, csv.gz), with a sha256 for every file.
+
+```sh
+pip install ./packages/probnode        # from a checkout; on PyPI as probnode-sdk soon
+                                       # (not `pip install probnode`: that is someone else's package)
+probnode login
+probnode datasets
+probnode files books parquet 2026-11-08T12 2026-11-08T14
+probnode sync books parquet 2026-11-08T12 2026-11-08T14 -o data/   # again later: only changes
+probnode doctor data/                                              # re-check every sha256
+```
+
+```python
+import probnode
+
+pn = probnode.Client()  # PROBNODE_API_KEY, or the key `probnode login` saved
+files = pn.files("books", "parquet", "2026-11-08T12:00:00Z", "2026-11-08T13:00:00Z")
+paths = list(pn.download_all(files, "data/"))  # each checked against its sha256
+table = probnode.read(paths[0])  # pyarrow.Table -> .to_pandas() / polars.from_arrow()
+
+for batch in probnode.iter_lp2("hour.lp2"):  # native books, decoded in bounded memory
+    ...
+```
+
+Units: prices x10,000 (5400 = 0.54), sizes x1,000,000 shares, timestamps in ms UTC. Signed links
+last 15 minutes; the client signs again whenever it needs to. The import name is `probnode`; the
+PyPI name is `probnode-sdk` because `probnode` is taken.

probnode_sdk-0.1.0/pyproject.toml

@@ -0,0 +1,43 @@
+[project]
+name = "probnode-sdk"
+version = "0.1.0"
+description = "Python client and CLI for ProbNode: Polymarket order books, trades and gaps"
+readme = "README.md"
+requires-python = ">=3.10"
+license = { text = "MIT" }
+dependencies = [
+    "httpx>=0.27",
+    # The lp2 reader decodes the native book format; parquet and csv.gz need nothing extra to
+    # download, but reading any of them into a table does.
+    "numpy>=1.24",
+    "pyarrow>=14",
+]
+
+[project.optional-dependencies]
+pandas = ["pandas>=2.0"]
+polars = ["polars>=0.20"]
+dev = ["pytest>=8.3", "ruff>=0.6"]
+
+[project.urls]
+Homepage = "https://probnode.com"
+Documentation = "https://probnode.com/docs"
+
+[project.scripts]
+probnode = "probnode.cli:main"
+
+[build-system]
+requires = ["hatchling"]
+build-backend = "hatchling.build"
+
+[tool.hatch.build.targets.wheel]
+packages = ["src/probnode"]
+
+[tool.pytest.ini_options]
+testpaths = ["tests"]
+
+[tool.ruff]
+line-length = 100
+target-version = "py310"
+
+[tool.ruff.lint]
+select = ["E", "F", "I", "UP", "B"]

probnode_sdk-0.1.0/src/probnode/__init__.py

@@ -0,0 +1,7 @@
+"""ProbNode: Polymarket order books, trades and feed gaps, as files and as a Python API."""
+
+from .client import Client, DataFile, ProbNodeError
+from .read import iter_lp2, read
+
+__all__ = ["Client", "DataFile", "ProbNodeError", "iter_lp2", "read"]
+__version__ = "0.1.0"

probnode_sdk-0.1.0/src/probnode/cli.py

@@ -0,0 +1,227 @@
+"""`probnode` on the command line.
+
+    probnode login                                   # email + password -> a key in ~/.config
+    probnode datasets
+    probnode files books parquet 2026-11-08T12 2026-11-08T14
+    probnode download books parquet 2026-11-08T12 2026-11-08T14 -o data/
+    probnode sync books parquet 2026-11-08T12 2026-11-08T14 -o data/   # only what changed
+    probnode doctor data/                            # every file against its sha256
+
+Times are UTC hours: `2026-11-08T12`, `2026-11-08T12:00`, or full ISO all work.
+Exit codes: 0 done, 1 a check failed (doctor), 2 the API refused or a request failed.
+"""
+
+from __future__ import annotations
+
+import argparse
+import getpass
+import json
+import sys
+from pathlib import Path
+
+from .client import Client, DataFile, ProbNodeError, load_config, save_config, sha256_of
+
+STATE = ".probnode-sync.json"
+
+
+def iso_hour(value: str) -> str:
+    """`2026-11-08T12` / `2026-11-08 12:00` / ISO -> `2026-11-08T12:00:00Z`."""
+    v = value.strip().replace(" ", "T").rstrip("Z")
+    date, _, time = v.partition("T")
+    hour = (time or "00")[:2]
+    if len(date) != 10 or not hour.isdigit():
+        raise argparse.ArgumentTypeError(f"not a UTC hour: {value!r} (use 2026-11-08T12)")
+    return f"{date}T{hour}:00:00Z"
+
+
+def human(n: int) -> str:
+    size = float(n)
+    for unit in ("B", "KB", "MB", "GB"):
+        if size < 1024 or unit == "GB":
+            return f"{size:.1f} {unit}" if unit != "B" else f"{int(size)} B"
+        size /= 1024
+    return f"{n} B"
+
+
+def _client(args: argparse.Namespace) -> Client:
+    return Client(api_key=args.key, base_url=args.base_url)
+
+
+def _listing(pn: Client, args: argparse.Namespace) -> list[DataFile]:
+    return pn.files(
+        args.dataset,
+        args.format,
+        args.start,
+        args.end,
+        granularity=args.granularity,
+        market=args.market or (),
+        series=args.series or (),
+    )
+
+
+def _read_state(dest: Path) -> dict[str, dict]:
+    try:
+        return json.loads((dest / STATE).read_text(encoding="utf-8"))
+    except FileNotFoundError:
+        return {}
+
+
+def _write_state(dest: Path, state: dict[str, dict]) -> None:
+    tmp = dest / (STATE + ".tmp")
+    tmp.write_text(json.dumps(state, indent=1, sort_keys=True), encoding="utf-8")
+    tmp.replace(dest / STATE)
+
+
+def _current(dest: Path, recorded: dict | None, f: DataFile) -> bool:
+    """A file is current when the listing still names the sha256 we fetched AND the file on disk
+    is the one we wrote: same size, same mtime. Anything touched since - truncated, overwritten,
+    restored from elsewhere - is fetched again, rather than trusted because the state says so.
+    (A full re-hash is `probnode doctor`.)"""
+    path = dest / f.object_path
+    if not recorded or recorded.get("sha256") != f.sha256 or not path.exists():
+        return False
+    st = path.stat()
+    return st.st_size == f.bytes and st.st_mtime_ns == recorded.get("mtime_ns")
+
+
+def cmd_login(args: argparse.Namespace) -> int:
+    email = args.email or input("Email: ")
+    password = getpass.getpass("Password: ")
+    pn = _client(args)
+    key = pn.login(email, password)
+    config = load_config()
+    config.update({"api_key": key, "base_url": pn.base_url})
+    save_config(config)
+    print(f"signed in as {email}; key saved")
+    return 0
+
+
+def cmd_datasets(args: argparse.Namespace) -> int:
+    for d in _client(args).datasets():
+        held = f"{d['first_hour']} .. {d['last_hour']}" if d.get("last_hour") else "nothing held"
+        print(f"{d['code']:<10} {d['tier']:<5} {','.join(d['formats']):<20} {held}")
+    return 0
+
+
+def cmd_files(args: argparse.Namespace) -> int:
+    files = _listing(_client(args), args)
+    for f in files:
+        print(f"{f.object_path}  {f.rows:>12,} rows  {human(f.bytes):>10}  {f.sha256[:12]}")
+    print(
+        f"{len(files)} files, {human(sum(f.bytes for f in files))}, "
+        f"{sum(f.rows for f in files):,} rows"
+    )
+    return 0
+
+
+def cmd_download(args: argparse.Namespace) -> int:
+    pn = _client(args)
+    files = _listing(pn, args)
+    dest = Path(args.out)
+    for path in pn.download_all(files, dest):
+        print(path)
+    return 0
+
+
+def cmd_sync(args: argparse.Namespace) -> int:
+    """Download what is new or changed since the last sync into the same directory; a file whose
+    sha256 already matches is left alone, so a second run moves nothing. Interrupted downloads
+    leave only a .part file, and the next run fetches that file again."""
+    pn = _client(args)
+    dest = Path(args.out)
+    dest.mkdir(parents=True, exist_ok=True)
+    state = _read_state(dest)
+    files = _listing(pn, args)
+    todo = [f for f in files if not _current(dest, state.get(f.object_path), f)]
+    moved = 0
+    for f, path in zip(todo, pn.download_all(todo, dest), strict=False):
+        st = path.stat()
+        state[f.object_path] = {
+            "sha256": f.sha256,
+            "bytes": f.bytes,
+            "rows": f.rows,
+            "mtime_ns": st.st_mtime_ns,
+        }
+        _write_state(dest, state)
+        moved += f.bytes
+        print(f"got {path}")
+    print(
+        f"{len(todo)} of {len(files)} files fetched, {human(moved)}; "
+        f"{len(files) - len(todo)} already current"
+    )
+    return 0
+
+
+def cmd_doctor(args: argparse.Namespace) -> int:
+    """Every file a sync recorded, re-hashed. Missing or changed files are named; exit 1 if any."""
+    dest = Path(args.dir)
+    state = _read_state(dest)
+    if not state:
+        print(f"no {STATE} in {dest}: run `probnode sync` into it first", file=sys.stderr)
+        return 1
+    bad = 0
+    for object_path, meta in sorted(state.items()):
+        path = dest / object_path
+        if not path.exists():
+            print(f"MISSING  {object_path}")
+            bad += 1
+        elif sha256_of(path) != meta["sha256"]:
+            print(f"CORRUPT  {object_path}")
+            bad += 1
+    print(f"{len(state) - bad} of {len(state)} files ok")
+    return 1 if bad else 0
+
+
+def parser() -> argparse.ArgumentParser:
+    p = argparse.ArgumentParser(prog="probnode", description="ProbNode data from the shell")
+    p.add_argument("--key", help="API key (default: PROBNODE_API_KEY or `probnode login`)")
+    p.add_argument("--base-url", help="API base (default: https://probnode.com)")
+    sub = p.add_subparsers(dest="command", required=True)
+
+    login = sub.add_parser("login", help="sign in with email and password, save the key")
+    login.add_argument("--email")
+    login.set_defaults(run=cmd_login)
+
+    sub.add_parser("datasets", help="datasets, formats and the hours held").set_defaults(
+        run=cmd_datasets
+    )
+
+    def selection(sp: argparse.ArgumentParser) -> None:
+        sp.add_argument("dataset", help="books, trades, gaps, markets, quotes, ohlcv_1m")
+        sp.add_argument("format", help="lp2, parquet or csv")
+        sp.add_argument("start", type=iso_hour, help="first UTC hour, inclusive")
+        sp.add_argument("end", type=iso_hour, help="last UTC hour, exclusive")
+        sp.add_argument("--granularity", default="hour", choices=["hour", "market_day"])
+        sp.add_argument("--market", action="append", help="narrow to a market slug (repeatable)")
+        sp.add_argument("--series", action="append", help="narrow to a series slug (repeatable)")
+
+    files = sub.add_parser("files", help="list the files in a selection")
+    selection(files)
+    files.set_defaults(run=cmd_files)
+
+    for name, run, text in (
+        ("download", cmd_download, "download a selection, sha256-checked"),
+        ("sync", cmd_sync, "download only what is new or changed"),
+    ):
+        sp = sub.add_parser(name, help=text)
+        selection(sp)
+        sp.add_argument("-o", "--out", default=".", help="directory (default: here)")
+        sp.set_defaults(run=run)
+
+    doctor = sub.add_parser("doctor", help="re-check every synced file against its sha256")
+    doctor.add_argument("dir", nargs="?", default=".")
+    doctor.set_defaults(run=cmd_doctor)
+    return p
+
+
+def main(argv: list[str] | None = None) -> int:
+    args = parser().parse_args(argv)
+    try:
+        return args.run(args)
+    except ProbNodeError as exc:
+        print(f"probnode: {exc}", file=sys.stderr)
+        return 2
+
+
+if __name__ == "__main__":
+    sys.exit(main())

probnode_sdk-0.1.0/src/probnode/client.py

@@ -0,0 +1,289 @@
+"""The ProbNode API, as a Python object.
+
+    import probnode
+    pn = probnode.Client()                      # key from PROBNODE_API_KEY or `probnode login`
+    files = pn.files("books", "parquet", "2026-11-08T12:00:00Z", "2026-11-08T14:00:00Z")
+    for f in files:
+        path = pn.download(f, "data/")          # sha256-checked against the listing
+    table = probnode.read(path)                 # pyarrow.Table, whatever the format
+
+Every method is one HTTP call (download is two: sign, then fetch). Errors are `ProbNodeError` with
+the API's own sentence in it, never a bare status code.
+"""
+
+from __future__ import annotations
+
+import hashlib
+import json
+import os
+from collections.abc import Iterable, Iterator
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import Any
+
+import httpx
+
+DEFAULT_BASE_URL = "https://probnode.com"
+CONFIG_PATH = Path(os.environ.get("PROBNODE_CONFIG", Path.home() / ".config" / "probnode.json"))
+SIGN_BATCH = 100  # POST /v1/data/urls takes at most 100 paths
+CHUNK = 1 << 20
+
+
+class ProbNodeError(Exception):
+    """A refusal or failure from the API, with its status and its sentence."""
+
+    def __init__(self, status: int, detail: str):
+        super().__init__(f"{status}: {detail}")
+        self.status = status
+        self.detail = detail
+
+
+@dataclass(frozen=True)
+class DataFile:
+    """One object in the lake, as GET /v1/data/files lists it."""
+
+    dataset: str
+    format: str
+    event_date: str
+    hour: int
+    src: str
+    object_path: str
+    rows: int
+    bytes: int
+    sha256: str
+    granularity: str = "hour"
+    market: str | None = None
+    extra: dict[str, Any] = field(default_factory=dict, compare=False, repr=False)
+
+    @classmethod
+    def from_api(cls, raw: dict[str, Any]) -> DataFile:
+        known = {k: raw[k] for k in cls.__dataclass_fields__ if k in raw and k != "extra"}
+        rest = {k: v for k, v in raw.items() if k not in cls.__dataclass_fields__}
+        return cls(**known, extra=rest)
+
+
+def load_config() -> dict[str, str]:
+    try:
+        return json.loads(CONFIG_PATH.read_text(encoding="utf-8"))
+    except FileNotFoundError:
+        return {}
+
+
+def save_config(values: dict[str, str]) -> None:
+    CONFIG_PATH.parent.mkdir(parents=True, exist_ok=True)
+    CONFIG_PATH.write_text(json.dumps(values, indent=2), encoding="utf-8")
+    try:
+        CONFIG_PATH.chmod(0o600)
+    except OSError:
+        pass  # Windows: the file lives in the user's profile, which is already private
+
+
+def sha256_of(path: Path) -> str:
+    h = hashlib.sha256()
+    with path.open("rb") as f:
+        for chunk in iter(lambda: f.read(CHUNK), b""):
+            h.update(chunk)
+    return h.hexdigest()
+
+
+class Client:
+    """A session against one ProbNode deployment.
+
+    `api_key` and `base_url` fall back to PROBNODE_API_KEY / PROBNODE_BASE_URL, then to what
+    `probnode login` saved, then (base URL only) to https://probnode.com. Public calls - datasets,
+    samples, the catalog - work without a key.
+    """
+
+    def __init__(
+        self,
+        api_key: str | None = None,
+        base_url: str | None = None,
+        *,
+        timeout: float = 60.0,
+        transport: httpx.BaseTransport | None = None,
+    ):
+        saved = load_config()
+        self.api_key = api_key or os.environ.get("PROBNODE_API_KEY") or saved.get("api_key")
+        self.base_url = (
+            base_url
+            or os.environ.get("PROBNODE_BASE_URL")
+            or saved.get("base_url")
+            or DEFAULT_BASE_URL
+        ).rstrip("/")
+        self._http = httpx.Client(
+            base_url=self.base_url, timeout=timeout, transport=transport, follow_redirects=False
+        )
+        # Signed URLs point at the lake, possibly another host; fetched without our key.
+        self._fetch = httpx.Client(timeout=timeout, transport=transport, follow_redirects=True)
+
+    def __enter__(self) -> Client:
+        return self
+
+    def __exit__(self, *exc: object) -> None:
+        self.close()
+
+    def close(self) -> None:
+        self._http.close()
+        self._fetch.close()
+
+    # -- plumbing ---------------------------------------------------------------------------
+
+    def _headers(self, need_key: bool) -> dict[str, str]:
+        if self.api_key:
+            return {"Authorization": f"Bearer {self.api_key}"}
+        if need_key:
+            raise ProbNodeError(
+                401, "no API key: pass api_key, set PROBNODE_API_KEY, or run `probnode login`"
+            )
+        return {}
+
+    def _request(self, method: str, path: str, *, need_key: bool = True, **kw: Any) -> Any:
+        response = self._http.request(method, path, headers=self._headers(need_key), **kw)
+        if response.status_code >= 400:
+            try:
+                detail = response.json().get("detail")
+            except ValueError:
+                detail = None
+            if not isinstance(detail, str):
+                detail = json.dumps(detail) if detail else response.reason_phrase
+            raise ProbNodeError(response.status_code, detail)
+        return response
+
+    def _json(self, method: str, path: str, **kw: Any) -> Any:
+        return self._request(method, path, **kw).json()
+
+    # -- account ----------------------------------------------------------------------------
+
+    def login(self, email: str, password: str) -> str:
+        """Mint a key with email and password; the client uses it from then on."""
+        out = self._json(
+            "POST", "/v1/auth/login", need_key=False, json={"email": email, "password": password}
+        )
+        self.api_key = out["key"]
+        return self.api_key
+
+    def me(self) -> dict[str, Any]:
+        return self._json("GET", "/v1/me")
+
+    def usage(self) -> dict[str, Any]:
+        return self._json("GET", "/v1/data/usage")
+
+    # -- catalog ----------------------------------------------------------------------------
+
+    def datasets(self) -> list[dict[str, Any]]:
+        """Every dataset, its formats, its tier and the hours held. Public."""
+        return self._json("GET", "/v1/data/datasets", need_key=False)
+
+    def series(self, category: str | None = None, q: str | None = None) -> list[dict[str, Any]]:
+        params = {k: v for k, v in {"category": category, "q": q}.items() if v}
+        return self._json("GET", "/v1/catalog/series", need_key=False, params=params)
+
+    def markets(self, series: str | None = None, **params: Any) -> Any:
+        if series:
+            params["series"] = series
+        return self._json("GET", "/v1/catalog/markets", need_key=False, params=params)
+
+    def coverage(self, token_id: str, start: str | None = None, end: str | None = None) -> Any:
+        params = {k: v for k, v in {"token_id": token_id, "start": start, "end": end}.items() if v}
+        return self._json("GET", "/v1/catalog/coverage", need_key=False, params=params)
+
+    def samples(self) -> Any:
+        """Free sample files, one market-day per dataset and format. Public."""
+        return self._json("GET", "/v1/samples", need_key=False)
+
+    # -- files ------------------------------------------------------------------------------
+
+    def files(
+        self,
+        dataset: str,
+        format: str,
+        start: str,
+        end: str,
+        *,
+        granularity: str = "hour",
+        market: Iterable[str] = (),
+        series: Iterable[str] = (),
+    ) -> list[DataFile]:
+        """The objects for one dataset and format over [start, end), from the manifests."""
+        params: list[tuple[str, str]] = [
+            ("dataset", dataset),
+            ("format", format),
+            ("start", start),
+            ("end", end),
+        ]
+        if granularity != "hour":
+            params.append(("granularity", granularity))
+        params += [("market", m) for m in market] + [("series", s) for s in series]
+        out = self._json("GET", "/v1/data/files", params=params)
+        return [DataFile.from_api(f) for f in out["files"]]
+
+    def sign(self, paths: Iterable[str]) -> dict[str, str]:
+        """object_path -> signed URL (valid for minutes; sign again any time)."""
+        paths = list(paths)
+        urls: dict[str, str] = {}
+        for i in range(0, len(paths), SIGN_BATCH):
+            out = self._json("POST", "/v1/data/urls", json={"paths": paths[i : i + SIGN_BATCH]})
+            urls.update({u["object_path"]: u["url"] for u in out["urls"]})
+        return urls
+
+    def download(
+        self,
+        file: DataFile,
+        dest: str | Path = ".",
+        *,
+        url: str | None = None,
+        verify: bool = True,
+    ) -> Path:
+        """Fetch one file to `dest/<object_path>`, written to a .part first and renamed only when
+        complete and (unless verify=False) its sha256 matches the listing."""
+        target = Path(dest) / file.object_path
+        target.parent.mkdir(parents=True, exist_ok=True)
+        url = url or self.sign([file.object_path])[file.object_path]
+        part = target.with_name(target.name + ".part")
+        h = hashlib.sha256()
+        with self._fetch.stream("GET", url) as response:
+            if response.status_code >= 400:
+                raise ProbNodeError(response.status_code, f"fetching {file.object_path} failed")
+            with part.open("wb") as out:
+                for chunk in response.iter_bytes(CHUNK):
+                    out.write(chunk)
+                    h.update(chunk)
+        if verify and h.hexdigest() != file.sha256:
+            part.unlink(missing_ok=True)
+            raise ProbNodeError(0, f"sha256 mismatch for {file.object_path}")
+        part.replace(target)
+        return target
+
+    def download_all(
+        self, files: Iterable[DataFile], dest: str | Path = ".", *, verify: bool = True
+    ) -> Iterator[Path]:
+        files = list(files)
+        urls = self.sign(f.object_path for f in files)
+        for f in files:
+            yield self.download(f, dest, url=urls[f.object_path], verify=verify)
+
+    # -- range query ------------------------------------------------------------------------
+
+    def timeseries(
+        self, dataset: str, market: str, start: str, end: str, *, format: str = "parquet"
+    ) -> Any:
+        """Rows for one market over a window (at most 7 days, 200,000 rows) as a pyarrow Table
+        (format="parquet") or a list of dicts (format="ndjson")."""
+        response = self._request(
+            "GET",
+            "/v1/timeseries",
+            params={
+                "dataset": dataset,
+                "market": market,
+                "start": start,
+                "end": end,
+                "format": format,
+            },
+        )
+        if format == "ndjson":
+            return [json.loads(line) for line in response.text.splitlines() if line]
+        import io
+
+        import pyarrow.parquet as pq
+
+        return pq.read_table(io.BytesIO(response.content))

probnode_sdk-0.1.0/src/probnode/lp2.py

@@ -0,0 +1,357 @@
+"""The recorder's LP2 book archive, decoded. Vendored, not imported.
+
+The reference reader for the LP2 format ProbNode's recorder writes, kept byte-for-byte equal to
+the copy the ProbNode ingest uses, so what a customer decodes is what the product exported.
+
+What changed against the original, and why:
+
+* `decode` returns the per-level columns (`bid_px0`, `bid_sz0`, ...) the decoder already holds,
+  instead of only folding them back into list columns. The exports want exactly those columns
+  (the CSV is flat), and rebuilding lists only to flatten them again costs a quarter of the time.
+* `to_table` is lobpack's `from_arrays`, for callers that want the books schema as the recorder
+  defines it - the round-trip tests compare against it.
+* `stream` is new: the same decoding a batch of whole markets at a time, in bounded memory, for
+  the ingest. `decode` stays as the reference the tests hold `stream` to.
+* LP1 (the xz prototype) is not here. The lake holds LP2 only.
+
+The format, in one paragraph (the Go file is the authority): `b"LOBP2\\n"`, a u32 LE header
+length, a JSON header, then one zstd frame per stream in header order. Rows are grouped per token
+and stored change-only - every field a zigzag delta against the previous row of the same token,
+written only when it changed, with a per-row bit mask saying which. The NO book of a market is the
+exact complement of the YES book, so a row of token 2k+1 that mirrors the row of token 2k at the
+same millisecond is not stored at all: its partner carries a `twin` bit. Prices are x10000, sizes
+x1e6, levels best first.
+"""
+
+import json
+import struct
+from collections.abc import Iterator
+from dataclasses import dataclass
+from typing import Any
+
+import numpy as np
+import pyarrow as pa
+
+MAGIC2 = b"LOBP2\n"
+VERSION = 2
+PX_ONE = 10000  # px ticks of 1e-4: price 1.0
+
+#: The books schema exactly as lobpack defines it, so `to_table` output compares equal to the
+#: reference reader's.
+SCHEMA = pa.schema(
+    [
+        ("ts_ms", pa.int64(), False),
+        ("ts_ex_ms", pa.int64(), False),
+        ("tok", pa.int32(), False),
+        ("bid_px", pa.list_(pa.field("element", pa.int32(), False)), False),
+        ("bid_sz", pa.list_(pa.field("element", pa.int64(), False)), False),
+        ("ask_px", pa.list_(pa.field("element", pa.int32(), False)), False),
+        ("ask_sz", pa.list_(pa.field("element", pa.int64(), False)), False),
+        ("snap", pa.bool_(), False),
+        ("shard", pa.int32(), False),
+        ("conn", pa.int32(), False),
+    ]
+)
+
+SIDES = ("bid_px", "bid_sz", "ask_px", "ask_sz")
+
+
+@dataclass(frozen=True)
+class Decoded:
+    """One LP2 object as columns, in (ts_ms, tok) order like the reference reader.
+
+    `columns` holds `ts_ms`, `ts_ex_ms`, `tok`, `snap`, `shard`, `conn`, the level counts `nb` and
+    `na`, and one array per side and level (`bid_px0` ... `ask_sz{depth-1}`). An absent level is 0
+    in its array and is outside `nb`/`na`; nothing else marks it.
+    """
+
+    header: dict[str, Any]
+    depth: int
+    columns: dict[str, np.ndarray]
+
+    @property
+    def rows(self) -> int:
+        return int(self.columns["ts_ms"].size)
+
+
+# ---------- varint / zigzag (vectorised), as in lobpack ----------
+
+
+def unzz(u: np.ndarray) -> np.ndarray:
+    u = u.astype(np.uint64)
+    return ((u >> np.uint64(1)).astype(np.int64)) ^ (-(u & np.uint64(1)).astype(np.int64))
+
+
+def vdec(buf: bytes, count: int) -> np.ndarray:
+    b = np.frombuffer(buf, np.uint8)
+    ends = np.flatnonzero(b < 0x80)
+    if ends.size != count:
+        raise ValueError(f"varint stream has {ends.size} values, header says {count}")
+    if count == 0:
+        return np.zeros(0, np.uint64)
+    starts = np.r_[0, ends[:-1] + 1]
+    lens = ends - starts + 1
+    out = np.zeros(count, np.uint64)
+    for k in range(int(lens.max())):
+        m = lens > k
+        out[m] |= (b[starts[m] + k] & np.uint64(0x7F)).astype(np.uint64) << np.uint64(7 * k)
+    return out
+
+
+def seg_cumsum(d: np.ndarray, run_start: np.ndarray, run_len: np.ndarray) -> np.ndarray:
+    c = np.cumsum(d)
+    before = np.r_[0, c[:-1]][run_start]
+    return c - np.repeat(before, run_len)
+
+
+def mirror_of(F: dict[str, np.ndarray], depth: int, i: np.ndarray) -> dict[str, np.ndarray]:
+    """Complement rows of rows i (token 2k -> 2k+1)."""
+    G = {
+        "ts_ms": F["ts_ms"][i],
+        "ts_ex_ms": F["ts_ex_ms"][i],
+        "tok": F["tok"][i] + 1,
+        "snap": F["snap"][i],
+        "shard": F["shard"][i],
+        "conn": F["conn"][i],
+        "nb": F["na"][i],
+        "na": F["nb"][i],
+    }
+    for k in range(depth):
+        G[f"bid_px{k}"] = np.where(F["na"][i] > k, PX_ONE - F[f"ask_px{k}"][i], 0)
+        G[f"bid_sz{k}"] = np.where(F["na"][i] > k, F[f"ask_sz{k}"][i], 0)
+        G[f"ask_px{k}"] = np.where(F["nb"][i] > k, PX_ONE - F[f"bid_px{k}"][i], 0)
+        G[f"ask_sz{k}"] = np.where(F["nb"][i] > k, F[f"bid_sz{k}"][i], 0)
+    return G
+
+
+def read_header(blob: bytes) -> tuple[dict[str, Any], int]:
+    """The JSON header and the offset of the first stream. Cheap: reads no stream."""
+    if blob[: len(MAGIC2)] != MAGIC2:
+        raise ValueError("not an LP2 object")
+    p = len(MAGIC2)
+    (hl,) = struct.unpack("<I", blob[p : p + 4])
+    hdr = json.loads(blob[p + 4 : p + 4 + hl])
+    if hdr["version"] != VERSION or hdr["codec"] != "zstd":
+        raise ValueError(f"LP2 version {hdr['version']} codec {hdr['codec']}")
+    return hdr, p + 4 + hl
+
+
+def decode(blob: bytes) -> Decoded:
+    """lobpack's `unpack2`, stopping before the list columns are rebuilt."""
+    hdr, p = read_header(blob)
+    codec = pa.Codec("zstd")
+    S = {}
+    for s in hdr["streams"]:
+        z = blob[p : p + s["z"]]
+        p += s["z"]
+        raw = codec.decompress(z, decompressed_size=s["raw"], asbytes=True) if s["raw"] else b""
+        S[s["name"]] = vdec(raw, s["count"])
+    if p != len(blob):
+        raise ValueError("trailing bytes after the last stream")
+    depth, names, m = hdr["depth"], hdr["fields"], hdr["rows_kept"]
+    run_tok = np.cumsum(S["run_tok"].astype(np.int64) + 1) - 1
+    run_len = S["run_len"].astype(np.int64) + 1
+    if int(run_len.sum()) != m:
+        raise ValueError("run lengths do not add up to rows_kept")
+    run_start = np.r_[0, np.cumsum(run_len)[:-1]].astype(np.int64)
+    new = np.zeros(m, bool)
+    new[run_start] = True
+    F = {"tok": np.repeat(run_tok, run_len)}
+    dts = np.zeros(m, np.int64)
+    dts[~new] = unzz(S["dts"])
+    dts[run_start] = unzz(S["run_ts0"]) + hdr["base_ts"]
+    F["ts_ms"] = seg_cumsum(dts, run_start, run_len)
+    F["ts_ex_ms"] = F["ts_ms"] - unzz(S["ex_lag"])
+    mask = S["mask"]
+    for bit, nm in enumerate(names):
+        ch = ((mask >> np.uint64(bit)) & np.uint64(1)).astype(bool)
+        d = np.zeros(m, np.int64)
+        d[ch] = unzz(S["f_" + nm])
+        F[nm] = seg_cumsum(d, run_start, run_len)
+    nf = len(names)
+    F["snap"] = ((mask >> np.uint64(nf)) & np.uint64(1)).astype(np.int64)
+    ti = np.flatnonzero(((mask >> np.uint64(nf + 1)) & np.uint64(1)).astype(bool))
+    if ti.size != hdr["rows_mirror"]:
+        raise ValueError(f"twin bits {ti.size} != rows_mirror {hdr['rows_mirror']}")
+    G = mirror_of(F, depth, ti)
+    for c in F:
+        F[c] = np.r_[F[c], G[c]]
+    o = np.lexsort((F["tok"], F["ts_ms"]))
+    for c in F:
+        F[c] = F[c][o]
+    if F["ts_ms"].size != hdr["rows"]:
+        raise ValueError("row count mismatch")
+    return Decoded(header=hdr, depth=depth, columns=F)
+
+
+def to_table(decoded: Decoded) -> pa.Table:
+    """lobpack's `from_arrays`: the books schema with list columns, as the recorder defines it."""
+    F, depth = decoded.columns, decoded.depth
+    cols = {}
+    for c, dt, cnt in (
+        ("bid_px", pa.int32(), "nb"),
+        ("bid_sz", pa.int64(), "nb"),
+        ("ask_px", pa.int32(), "na"),
+        ("ask_sz", pa.int64(), "na"),
+    ):
+        cols[c] = pa.ListArray.from_arrays(
+            list_offsets(F[cnt]), level_values(F, c, depth, F[cnt]).astype(dt.to_pandas_dtype())
+        ).cast(SCHEMA.field(c).type)
+    arrs = [
+        pa.array(F["ts_ms"], pa.int64()),
+        pa.array(F["ts_ex_ms"], pa.int64()),
+        pa.array(F["tok"].astype(np.int32), pa.int32()),
+        cols["bid_px"],
+        cols["bid_sz"],
+        cols["ask_px"],
+        cols["ask_sz"],
+        pa.array(F["snap"].astype(bool), pa.bool_()),
+        pa.array(F["shard"].astype(np.int32), pa.int32()),
+        pa.array(F["conn"].astype(np.int32), pa.int32()),
+    ]
+    return pa.Table.from_arrays(arrs, schema=SCHEMA)
+
+
+def list_offsets(lengths: np.ndarray) -> pa.Array:
+    """List offsets for per-row level counts."""
+    return pa.array(np.r_[0, np.cumsum(lengths)].astype(np.int32), pa.int32())
+
+
+def level_values(
+    F: dict[str, np.ndarray], side: str, depth: int, lengths: np.ndarray
+) -> np.ndarray:
+    """The present levels of one side, flattened row by row, best level first."""
+    if not depth:
+        return np.zeros(0, np.int64)
+    M = np.stack([F[f"{side}{k}"] for k in range(depth)], axis=1)
+    keep = np.arange(depth)[None, :] < lengths[:, None]
+    return M[keep]
+
+
+# ---------- streaming: one window in bounded memory ----------
+#
+# Not in lobpack. A real hour is one window of about 35 million rows, and `decode` holds all of
+# them at once (about 3 GB). The ingest runs on a box with far less, so it reads the window the
+# way it is laid out: token by token. Every stream is decompressed incrementally and its varints
+# are consumed through a cursor, a batch of whole token runs at a time. A batch never splits the
+# two tokens of one market, so a mirrored row always lands in the same batch as its partner and a
+# market's rows stay contiguous in the output.
+
+
+class _Varints:
+    """The varints of one zstd stream, read incrementally. `take(n)` returns the next n values."""
+
+    def __init__(self, name: str, frame: pa.Buffer, count: int, raw: int) -> None:
+        self._name = name
+        self._stream = pa.input_stream(frame, compression="zstd") if raw else None
+        self._buf = np.zeros(0, np.uint8)
+        self._left = count
+        self._raw_left = raw
+
+    def take(self, n: int) -> np.ndarray:
+        if n > self._left:
+            raise ValueError(f"stream {self._name}: {n} values asked, {self._left} left")
+        if n == 0:
+            return np.zeros(0, np.uint64)
+        while True:
+            ends = np.flatnonzero(self._buf < 0x80)
+            if ends.size >= n:
+                break
+            if self._stream is None:
+                raise ValueError(f"stream {self._name} ended early")
+            more = self._stream.read(max(1 << 20, 2 * (n - ends.size)))
+            if not more:
+                raise ValueError(f"stream {self._name} ended early")
+            self._raw_left -= len(more)
+            self._buf = np.concatenate([self._buf, np.frombuffer(more, np.uint8)])
+        cut = int(ends[n - 1]) + 1
+        values = vdec(self._buf[:cut].tobytes(), n)
+        self._buf = self._buf[cut:]
+        self._left -= n
+        return values
+
+    def finish(self) -> None:
+        """Every value consumed and not a byte more - the streaming form of lobpack's checks."""
+        trailing = self._stream.read() if self._stream is not None else b""
+        self._raw_left -= len(trailing)
+        if self._left or self._buf.size or trailing or self._raw_left:
+            raise ValueError(f"stream {self._name} does not match its header")
+
+
+def stream(buffer: pa.Buffer, max_rows: int) -> Iterator[Decoded]:
+    """The rows of one LP2 object in batches of whole markets, each in (tok, ts_ms) order.
+
+    `max_rows` bounds the stored rows of a batch; mirrored rows come on top, at most as many
+    again. A single market larger than that is still one batch, because splitting it would
+    separate a row from its mirror. Within a market the order is lobpack's tie order.
+    """
+    hdr, p = read_header(buffer.slice(0, min(buffer.size, 1 << 20)).to_pybytes())
+    cursors: dict[str, _Varints] = {}
+    for s in hdr["streams"]:
+        if p + s["z"] > buffer.size:
+            raise ValueError(f"stream {s['name']} runs past the end of the object")
+        cursors[s["name"]] = _Varints(s["name"], buffer.slice(p, s["z"]), s["count"], s["raw"])
+        p += s["z"]
+    if p != buffer.size:
+        raise ValueError("trailing bytes after the last stream")
+    depth, names = hdr["depth"], hdr["fields"]
+    runs = next(s["count"] for s in hdr["streams"] if s["name"] == "run_tok")
+    run_tok = np.cumsum(cursors["run_tok"].take(runs).astype(np.int64) + 1) - 1
+    run_len = cursors["run_len"].take(runs).astype(np.int64) + 1
+    run_ts0 = unzz(cursors["run_ts0"].take(runs)) + hdr["base_ts"]
+    if int(run_len.sum()) != hdr["rows_kept"]:
+        raise ValueError("run lengths do not add up to rows_kept")
+
+    # A batch may end only where the next run starts another market (tok // 2 changes).
+    market_start = np.r_[True, (run_tok[1:] // 2) != (run_tok[:-1] // 2)]
+    rows_seen = mirrors_seen = 0
+    a = 0
+    while a < runs:
+        b = a + 1
+        total = int(run_len[a])
+        while b < runs and (not market_start[b] or total + int(run_len[b]) <= max_rows):
+            total += int(run_len[b])
+            b += 1
+        lens = run_len[a:b]
+        m = int(lens.sum())
+        starts = np.r_[0, np.cumsum(lens)[:-1]].astype(np.int64)
+        new = np.zeros(m, bool)
+        new[starts] = True
+        F = {"tok": np.repeat(run_tok[a:b], lens)}
+        dts = np.zeros(m, np.int64)
+        dts[~new] = unzz(cursors["dts"].take(m - (b - a)))
+        dts[starts] = run_ts0[a:b]
+        F["ts_ms"] = seg_cumsum(dts, starts, lens)
+        F["ts_ex_ms"] = F["ts_ms"] - unzz(cursors["ex_lag"].take(m))
+        mask = cursors["mask"].take(m)
+        for bit, nm in enumerate(names):
+            ch = ((mask >> np.uint64(bit)) & np.uint64(1)).astype(bool)
+            d = np.zeros(m, np.int64)
+            d[ch] = unzz(cursors["f_" + nm].take(int(ch.sum())))
+            F[nm] = seg_cumsum(d, starts, lens)
+        nf = len(names)
+        F["snap"] = ((mask >> np.uint64(nf)) & np.uint64(1)).astype(np.int64)
+        ti = np.flatnonzero(((mask >> np.uint64(nf + 1)) & np.uint64(1)).astype(bool))
+        G = mirror_of(F, depth, ti)
+        for c in F:
+            F[c] = np.r_[F[c], G[c]]
+        o = np.lexsort((F["ts_ms"], F["tok"]))
+        for c in F:
+            F[c] = F[c][o]
+        rows_seen += F["ts_ms"].size
+        mirrors_seen += ti.size
+        yield Decoded(header=hdr, depth=depth, columns=F)
+        a = b
+
+    # Streams this reader decodes are held to their header exactly. A stream it does not know - an
+    # addition a newer writer makes, like the legacy archive's sub_ns - is skipped, the way
+    # lobpack's unpack2 skips it: an extension must not make an older reader refuse the file.
+    known = {"run_tok", "run_len", "run_ts0", "dts", "ex_lag", "mask"} | {f"f_{n}" for n in names}
+    for name, cursor in cursors.items():
+        if name in known:
+            cursor.finish()
+    if rows_seen != hdr["rows"] or mirrors_seen != hdr["rows_mirror"]:
+        raise ValueError(
+            f"rows {rows_seen} (mirror {mirrors_seen}) != header {hdr['rows']} "
+            f"({hdr['rows_mirror']})"
+        )

probnode_sdk-0.1.0/src/probnode/read.py

@@ -0,0 +1,47 @@
+"""Open any downloaded ProbNode file as a pyarrow Table: lp2, parquet or csv.gz."""
+
+from __future__ import annotations
+
+from collections.abc import Iterator
+from pathlib import Path
+
+import pyarrow as pa
+
+#: Stored rows per decode batch. About 0.5 GB of working memory on a real hour of books; a
+#: whole-object decode of the same hour needs ~8 GB, which is why lp2 is always read in batches.
+LP2_BATCH_ROWS = 200_000
+
+
+def iter_lp2(path: str | Path, batch_rows: int = LP2_BATCH_ROWS) -> Iterator[pa.Table]:
+    """The books in one lp2 file as tables of whole markets, in bounded memory.
+
+    Columns: ts_ms (our receive clock) and ts_ex_ms (the venue's), tok (index into the file's
+    token list), the bid/ask price and size lists - prices x10000, sizes x1e6, best level first -
+    snap, shard and conn.
+    """
+    from . import lp2
+
+    buffer = pa.memory_map(str(path)).read_buffer()
+    for decoded in lp2.stream(buffer, batch_rows):
+        yield lp2.to_table(decoded)
+
+
+def read(path: str | Path) -> pa.Table:
+    """A pyarrow Table for one file. `.to_pandas()` or `polars.from_arrow()` from there.
+
+    An hour of books is tens of millions of rows; for lp2 prefer `iter_lp2` unless the whole hour
+    fits in memory, and for parquet `pyarrow.parquet.ParquetFile(path).iter_batches()`.
+    """
+    path = Path(path)
+    name = path.name
+    if name.endswith(".lp2"):
+        return pa.concat_tables(iter_lp2(path))
+    if name.endswith(".parquet"):
+        import pyarrow.parquet as pq
+
+        return pq.read_table(path)
+    if name.endswith(".csv.gz") or name.endswith(".csv"):
+        import pyarrow.csv as pacsv
+
+        return pacsv.read_csv(path)
+    raise ValueError(f"not a ProbNode data file (lp2, parquet, csv.gz): {path}")

probnode_sdk-0.1.0/tests/test_client_cli.py

@@ -0,0 +1,204 @@
+"""The SDK and CLI against a fake ProbNode: an httpx MockTransport that serves a listing, signs
+paths and hands out the bytes behind each signed URL."""
+
+from __future__ import annotations
+
+import hashlib
+import json
+from pathlib import Path
+
+import httpx
+import pytest
+
+import probnode
+from probnode import cli
+from probnode.client import Client, ProbNodeError
+
+BLOBS = {
+    "x/parquet/books/event_date=2026-11-08/hour=12.parquet": b"twelve" * 1000,
+    "x/parquet/books/event_date=2026-11-08/hour=13.parquet": b"thirteen" * 1000,
+}
+
+
+class Fake:
+    def __init__(self) -> None:
+        self.blobs = dict(BLOBS)
+        self.fetches: list[str] = []
+        self.signed: list[str] = []
+
+    def listing(self) -> list[dict]:
+        return [
+            {
+                "dataset": "books",
+                "format": "parquet",
+                "event_date": "2026-11-08",
+                "hour": int(p.split("hour=")[1][:2]),
+                "src": "s6",
+                "object_path": p,
+                "rows": 10,
+                "bytes": len(b),
+                "sha256": hashlib.sha256(b).hexdigest(),
+                "granularity": "hour",
+                "market": None,
+                "future_field": 1,
+            }
+            for p, b in self.blobs.items()
+        ]
+
+    def __call__(self, request: httpx.Request) -> httpx.Response:
+        url = request.url
+        if url.host == "lake.test":
+            path = url.path.removeprefix("/")
+            self.fetches.append(path)
+            return httpx.Response(200, content=self.served(path))
+        if request.headers.get("Authorization") != "Bearer k":
+            return httpx.Response(401, json={"detail": "missing or invalid API key"})
+        if url.path == "/v1/data/files":
+            if url.params.get("dataset") == "nope":
+                return httpx.Response(400, json={"detail": "unknown dataset 'nope'"})
+            files = self.listing()
+            return httpx.Response(
+                200,
+                json={
+                    "files": files,
+                    "total_files": len(files),
+                    "total_bytes": sum(f["bytes"] for f in files),
+                },
+            )
+        if url.path == "/v1/data/urls":
+            paths = json.loads(request.content)["paths"]
+            self.signed += paths
+            return httpx.Response(
+                200,
+                json={
+                    "urls": [
+                        {"object_path": p, "url": f"https://lake.test/{p}", "expires_at": "x"}
+                        for p in paths
+                    ],
+                    "licence": "research",
+                    "licence_detail": "",
+                },
+            )
+        return httpx.Response(404, json={"detail": "not found"})
+
+    def served(self, path: str) -> bytes:
+        return self.blobs[path]
+
+
+@pytest.fixture
+def fake(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> Fake:
+    f = Fake()
+    monkeypatch.setenv("PROBNODE_CONFIG", str(tmp_path / "cfg.json"))
+    real_init = Client.__init__
+
+    def init(self, api_key=None, base_url=None, **kw):
+        real_init(
+            self,
+            api_key=api_key or "k",
+            base_url=base_url or "https://api.test",
+            transport=httpx.MockTransport(f),
+        )
+
+    monkeypatch.setattr(Client, "__init__", init)
+    return f
+
+
+ARGS = ["books", "parquet", "2026-11-08T12", "2026-11-08T14"]
+
+
+def test_files_keeps_unknown_fields_and_parses(fake: Fake) -> None:
+    files = Client().files("books", "parquet", "2026-11-08T12:00:00Z", "2026-11-08T14:00:00Z")
+    assert [f.hour for f in files] == [12, 13]
+    assert files[0].extra == {"future_field": 1}
+
+
+def test_api_refusal_is_the_api_sentence(fake: Fake) -> None:
+    with pytest.raises(ProbNodeError) as err:
+        Client().files("nope", "parquet", "a", "b")
+    assert err.value.status == 400 and "unknown dataset" in err.value.detail
+
+
+def test_download_verifies_sha256(fake: Fake, tmp_path: Path) -> None:
+    pn = Client()
+    f = pn.files(*ARGS[:2], "s", "e")[0]
+    fake.served = lambda path: b"tampered"  # type: ignore[method-assign]
+    with pytest.raises(ProbNodeError, match="sha256 mismatch"):
+        pn.download(f, tmp_path)
+    assert not (tmp_path / f.object_path).exists()
+    assert not list(tmp_path.rglob("*.part"))
+
+
+def test_sync_twice_moves_nothing_then_only_the_change(fake: Fake, tmp_path: Path) -> None:
+    out = tmp_path / "data"
+    assert cli.main(["sync", *ARGS, "-o", str(out)]) == 0
+    assert len(fake.fetches) == 2
+    assert cli.main(["sync", *ARGS, "-o", str(out)]) == 0
+    assert len(fake.fetches) == 2, "second sync fetched again"
+
+    changed = "x/parquet/books/event_date=2026-11-08/hour=13.parquet"
+    fake.blobs[changed] = b"corrected" * 1000
+    assert cli.main(["sync", *ARGS, "-o", str(out)]) == 0
+    assert fake.fetches[2:] == [changed]
+    assert (out / changed).read_bytes() == fake.blobs[changed]
+
+
+def test_doctor_names_a_corrupt_and_a_missing_file(
+    fake: Fake, tmp_path: Path, capsys: pytest.CaptureFixture[str]
+) -> None:
+    out = tmp_path / "data"
+    cli.main(["sync", *ARGS, "-o", str(out)])
+    assert cli.main(["doctor", str(out)]) == 0
+    paths = sorted(BLOBS)
+    (out / paths[0]).write_bytes(b"bitrot")
+    (out / paths[1]).unlink()
+    capsys.readouterr()
+    assert cli.main(["doctor", str(out)]) == 1
+    text = capsys.readouterr().out
+    assert f"CORRUPT  {paths[0]}" in text and f"MISSING  {paths[1]}" in text
+
+
+def test_cli_refusal_exits_2(fake: Fake, capsys: pytest.CaptureFixture[str]) -> None:
+    assert cli.main(["files", "nope", "parquet", "2026-11-08T12", "2026-11-08T13"]) == 2
+    assert "unknown dataset" in capsys.readouterr().err
+
+
+def test_iso_hour_accepts_the_short_forms() -> None:
+    assert cli.iso_hour("2026-11-08T12") == "2026-11-08T12:00:00Z"
+    assert cli.iso_hour("2026-11-08 07:00") == "2026-11-08T07:00:00Z"
+    assert cli.iso_hour("2026-11-08T12:00:00Z") == "2026-11-08T12:00:00Z"
+
+
+def test_read_parquet_and_csv(tmp_path: Path) -> None:
+    import pyarrow as pa
+    import pyarrow.csv as pacsv
+    import pyarrow.parquet as pq
+
+    t = pa.table({"a": [1, 2]})
+    pq.write_table(t, tmp_path / "x.parquet")
+    pacsv.write_csv(t, tmp_path / "x.csv")
+    assert probnode.read(tmp_path / "x.parquet").to_pydict() == {"a": [1, 2]}
+    assert probnode.read(tmp_path / "x.csv").to_pydict() == {"a": [1, 2]}
+    with pytest.raises(ValueError):
+        probnode.read(tmp_path / "x.txt")
+
+
+def test_lp2_reader_is_the_api_one() -> None:
+    """The SDK ships the same lp2 decoder the ingest uses; if they drift, a customer decodes
+    differently from the exports we made. Byte-identical, checked where both are present."""
+    api = Path(__file__).resolve().parents[3] / "apps/api/src/pmd_api/lp2.py"
+    if not api.exists():
+        pytest.skip("not in the monorepo")
+    ours = Path(probnode.__file__).with_name("lp2.py")
+    assert ours.read_bytes() == api.read_bytes()
+
+
+def test_sync_refetches_a_file_changed_on_disk(fake: Fake, tmp_path: Path) -> None:
+    """The state file says current, but the bytes on disk were changed: sync must notice and fetch
+    again, not trust its own bookkeeping."""
+    out = tmp_path / "data"
+    cli.main(["sync", *ARGS, "-o", str(out)])
+    victim = sorted(BLOBS)[0]
+    (out / victim).write_bytes(b"corrupt")
+    assert cli.main(["sync", *ARGS, "-o", str(out)]) == 0
+    assert fake.fetches[2:] == [victim]
+    assert cli.main(["doctor", str(out)]) == 0