quantpad-data 0.8.1__tar.gz → 0.8.4__tar.gz

{quantpad_data-0.8.1 → quantpad_data-0.8.4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: quantpad-data
-Version: 0.8.1
+Version: 0.8.4
 Summary: Official Python client for the QuantPad market-data API
 Project-URL: Documentation, https://api.quantpad.ai/external/docs
 Author: QuantPad

{quantpad_data-0.8.1 → quantpad_data-0.8.4}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
 
 [project]
 name = "quantpad-data"
-version = "0.8.1"
+version = "0.8.4"
 description = "Official Python client for the QuantPad market-data API"
 readme = "README.md"
 requires-python = ">=3.10"

{quantpad_data-0.8.1 → quantpad_data-0.8.4}/quantpad_data/__init__.py

@@ -11,6 +11,7 @@ import pyarrow as pa
 from .client import Client, ROLL_ADJUST_MODES, TICK_SCHEMAS, USER_AGENT, VERSION
 from .errors import (
     AuthenticationError,
+    IncompleteResponseError,
     NotFoundError,
     PermissionDeniedError,
     QuantPadDataError,
@@ -213,7 +214,7 @@ __all__ = [
     "Client", "QuantPadClient", "QuantPadDataError", "QuantPadDataNotReady", "AuthenticationError",
     "PermissionDeniedError", "RateLimitError", "QuotaExceededError",
     "NotFoundError", "V2NotAvailableError", "ValidationError", "UpstreamError",
-    "ResponseTooLargeError", "TruncatedChainWarning", "TICK_SCHEMAS",
+    "IncompleteResponseError", "ResponseTooLargeError", "TruncatedChainWarning", "TICK_SCHEMAS",
     "ROLL_ADJUST_MODES", "get_bars", "get_bars_with_retry", "get_ticks",
     "get_trades", "get_mbp1", "get_mbp10", "get_trades_rth", "get_mbp1_rth",
     "get_mbp10_rth", "get_spread_1m_rth", "symbology_resolve", "get_universe",

{quantpad_data-0.8.1 → quantpad_data-0.8.4}/quantpad_data/cache.py

@@ -349,6 +349,12 @@ def _columns_tag(columns: Sequence[str] | None) -> str:
 #: would serve the wrong rows.
 _DELIVERY_ONLY = frozenset({"compression", "max_decoded_bytes"})
 
+#: Changed whenever the service starts answering an unchanged body with
+#: different rows, so every entry cached under the old answer misses once.
+#: 2: until 2026-10, a window ending mid-day was answered without its last
+#: day, and that short answer was cached here as the window's answer.
+_RANGE_KEY_EPOCH = 2
+
 
 def _request_key(body: dict) -> str:
     """A stable key for the answer a request body asks for.
@@ -357,7 +363,9 @@ def _request_key(body: dict) -> str:
     which is what stops it drifting as ``get_range`` grows arguments.
     """
     material = {k: v for k, v in sorted(body.items()) if k not in _DELIVERY_ONLY}
-    return _hash_key([json.dumps(material, sort_keys=True, default=str)])
+    return _hash_key(
+        [str(_RANGE_KEY_EPOCH), json.dumps(material, sort_keys=True, default=str)]
+    )
 
 
 def range_path(body: dict) -> Path:
@@ -403,13 +411,23 @@ def read_meta(path: Path) -> dict | None:
     return _read_meta(path)
 
 
-def get_frame(path: Path) -> pd.DataFrame | None:
+def get_frame(path: Path, *, expect: dict | None = None) -> pd.DataFrame | None:
     """Read a cached frame, or None on miss.
 
     A corrupt or half-written entry is dropped and reported as a miss, so a
     damaged cache degrades to the network rather than failing the request.
+
+    ``expect`` names sidecar fields the entry must carry exactly, and a
+    mismatch is a miss.  This is how a partition the service has since
+    re-fetched stops being served from disk: the manifest's ``version``
+    moves when upstream corrects a day, and a settled day is otherwise
+    never asked about again.  An entry written before versions existed
+    carries none, so it misses once against a server that sends one.
     """
-    if not enabled() or _read_meta(path) is None:
+    meta = _read_meta(path) if enabled() else None
+    if meta is None:
+        return None
+    if any(meta.get(field) != value for field, value in (expect or {}).items()):
         return None
     try:
         frame = pd.read_parquet(path)

{quantpad_data-0.8.1 → quantpad_data-0.8.4}/quantpad_data/client.py

@@ -10,11 +10,14 @@ from datetime import date
 from typing import Iterator, Sequence
 
 import pandas as pd
+import pyarrow as pa
 import pyarrow.ipc as ipc
 import requests
+import urllib3
 
 from .errors import (
     AuthenticationError,
+    IncompleteResponseError,
     NotFoundError,
     PermissionDeniedError,
     QuantPadDataError,
@@ -84,7 +87,7 @@ V2_NOT_AVAILABLE = (
     "QuantPad support about /v2 access."
 )
 
-VERSION = "0.8.1"
+VERSION = "0.8.4"
 USER_AGENT = f"quantpad-data-python/{VERSION}"
 TICK_SCHEMAS = (
     "trades", "mbp-1", "mbp-10", "cmbp-1", "tcbbo", "cbbo-1s",
@@ -142,6 +145,50 @@ BARS_TIMEOUT = (30, 1800)
 #: stall detector and comes down to a couple of minutes.
 V2_DATA_TIMEOUT = (10, 1800)
 
+#: Backoff between retries: exponential, with jitter, so that the clients
+#: of one outage do not come back in lockstep and knock it over again.
+#:
+#: Was 0.5s, 1s, 2s plus at most 0.25s, with the server's ``Retry-After``
+#: ignored on every 5xx.  During the 2026-10-01/02 API-key outage the
+#: service answered every request "503, Retry-After: 5"; this client
+#: retried each call three more times inside four seconds, and one key
+#: kept that up for sixteen hours.
+RETRY_BACKOFF_BASE = 1.0
+RETRY_BACKOFF_MAX = 30.0
+
+#: The longest ``Retry-After`` honoured, from any status.  Past this the
+#: client would rather fail and let its caller decide than sleep silently:
+#: a daily-quota 429 can say "come back in twenty hours".
+RETRY_AFTER_MAX = 60.0
+
+
+#: What reading a stream raises when the connection ends early: urllib3's
+#: incomplete-chunk error from the socket, requests' wrapper of the same,
+#: and what pyarrow raises when a message is cut short (ArrowInvalid) or
+#: its read fails (OSError).
+_BROKEN_STREAM = (
+    urllib3.exceptions.HTTPError,
+    requests.exceptions.RequestException,
+    pa.ArrowInvalid,
+    OSError,
+)
+
+
+def _retry_delay(attempt: int, retry_after: float | None) -> float:
+    """Seconds to wait before retry number ``attempt + 1``.
+
+    Exponential with equal jitter -- half the step fixed, half random --
+    and never sooner than the server asked, though no later than
+    ``RETRY_AFTER_MAX``.  The server's figure is jittered too, because
+    every client of an outage receives the same one.
+    """
+    step = min(RETRY_BACKOFF_MAX, RETRY_BACKOFF_BASE * (2**attempt))
+    delay = step / 2 + random.uniform(0, step / 2)
+    if retry_after is not None and retry_after > 0:
+        asked = min(float(retry_after), RETRY_AFTER_MAX)
+        delay = max(delay, asked + random.uniform(0, min(1.0, asked / 4)))
+    return delay
+
 
 def _prose_with_hint(error: dict) -> str:
     """The sentence to raise from a structured refusal.
@@ -269,7 +316,7 @@ class Client:
             cls = QuotaExceededError if "quota" in message.lower() else RateLimitError
             raise cls(message, retry_after, status)
         if status >= 500:
-            raise UpstreamError(message, status)
+            raise UpstreamError(message, status, retry_after=retry_after)
         raise QuantPadDataError(message, status, detail)
 
     def _request(self, method: str, path: str, **kwargs) -> requests.Response:
@@ -322,9 +369,8 @@ class Client:
                 last_error = exc
                 if attempt >= self.max_retries:
                     raise
-                delay = None
-            base_delay = delay if delay is not None else min(8.0, 0.5 * (2**attempt))
-            time.sleep(max(0.0, base_delay) + random.uniform(0, 0.25))
+                delay = getattr(exc, "retry_after", None)
+            time.sleep(_retry_delay(attempt, delay))
         raise QuantPadDataError(str(last_error or "Request failed"))
 
     @staticmethod
@@ -417,8 +463,34 @@ class Client:
         response.raw.decode_content = True
         buffered: list[pd.DataFrame] = []
         buffered_rows = 0
+        received = 0
+
+        def broken(exc: BaseException) -> IncompleteResponseError:
+            return IncompleteResponseError(
+                f"The /v1/ticks stream for {params['symbol']} {schema} ended "
+                f"before the service finished sending it ({received:,} rows "
+                "had arrived). Nothing from this call is complete; retry the "
+                f"request. ({type(exc).__name__}: {exc})",
+                rows_received=received,
+            )
+
         try:
-            for batch in ipc.open_stream(response.raw):
+            # Read errors are told apart from a clean end because the
+            # service now ends a failed stream by closing the connection
+            # early -- its 200 has already gone out -- and the raw urllib3
+            # ProtocolError that surfaces says nothing about what it means.
+            try:
+                batches = iter(ipc.open_stream(response.raw))
+            except _BROKEN_STREAM as exc:
+                raise broken(exc) from exc
+            while True:
+                try:
+                    batch = next(batches)
+                except StopIteration:
+                    break
+                except _BROKEN_STREAM as exc:
+                    raise broken(exc) from exc
+                received += batch.num_rows
                 if not batch.num_rows:
                     continue
                 buffered.append(batch.to_pandas(split_blocks=True, self_destruct=True))

{quantpad_data-0.8.1 → quantpad_data-0.8.4}/quantpad_data/errors.py

@@ -134,7 +134,39 @@ class ValidationError(ValueError, QuantPadDataError):
 
 
 class UpstreamError(QuantPadDataError):
-    pass
+    """A 5xx from the service.
+
+    ``retry_after`` is the service's ``Retry-After`` in seconds when it sent
+    one: a 503 that says when to come back is a 503 worth believing, and the
+    client waits at least that long before retrying it.
+    """
+
+    def __init__(
+        self,
+        message: str,
+        status_code: int | None = None,
+        detail: dict | None = None,
+        *,
+        retry_after: float | None = None,
+    ):
+        super().__init__(message, status_code, detail)
+        self.retry_after = retry_after
+
+
+class IncompleteResponseError(UpstreamError):
+    """A streamed response ended before the service finished sending it.
+
+    The service now ends a tick stream this way when it fails partway --
+    the status was already 200, so the connection is the only signal
+    left.  Nothing this call returned can be taken as complete; retry the
+    request.  A subclass of :class:`UpstreamError`, so existing ``except
+    UpstreamError`` handlers keep catching it.  ``rows_received`` counts
+    the rows that had arrived when the stream broke.
+    """
+
+    def __init__(self, message: str, rows_received: int = 0):
+        super().__init__(message)
+        self.rows_received = rows_received
 
 
 class QuantPadDataNotReady(QuantPadDataError):

{quantpad_data-0.8.1 → quantpad_data-0.8.4}/quantpad_data/memory.py

@@ -42,6 +42,29 @@ import os
 #: returns to killing the process with no message.
 PEAK_PER_PRICED_BYTE = 6.0
 
+#: Measured factors for the schemas where the 6x worst case refused what
+#: fitted -- in 14 days of production sandboxes, 31 of 74 refusals were ~1 GB
+#: or less, mostly single days of 80-byte quote records.  Measured 2026-10-28
+#: (peak RSS over priced bytes): mbp-10 2.37x and 2.57x; mbp-1 3.17x-3.19x on
+#: full days near the budget and 4.32x on a 63 MB one; trades 4.73x, so trades
+#: keeps the default.  The other quote schemas share mbp-1's record shape.
+#: Kept identical to the sandbox client's ``_PEAK_BY_SCHEMA``.
+PEAK_BY_SCHEMA = {
+    "mbp-10": 3.0,
+    "mbp-1": 4.5,
+    "tbbo": 4.5,
+    "bbo-1s": 4.5,
+    "bbo-1m": 4.5,
+    "cmbp-1": 4.5,
+    "tcbbo": 4.5,
+    "cbbo-1s": 4.5,
+    "cbbo-1m": 4.5,
+}
+
+
+def peak_per_priced_byte(schema: str | None) -> float:
+    return PEAK_BY_SCHEMA.get(schema, PEAK_PER_PRICED_BYTE)
+
 #: cgroup v1 spells "no limit" as a huge number rather than a word, and the
 #: exact sentinel varies with page size, so anything absurd is read as absent.
 _NO_LIMIT = 1 << 53
@@ -172,7 +195,7 @@ def available_bytes() -> int | None:
     return min(candidates) if candidates else None
 
 
-def declared_budget(mode: str) -> int | None:
+def declared_budget(mode: str, schema: str | None = None) -> int | None:
     """The figure to send as ``max_decoded_bytes``, in the service's units.
 
     TWO NUMBERS, BECAUSE THE COST OF BEING WRONG DIFFERS BY MODE.
@@ -186,9 +209,12 @@ def declared_budget(mode: str) -> int | None:
     and there is nowhere to degrade to, so a refusal is a hard failure.  It
     therefore declares free memory UNDIVIDED, which refuses only what cannot
     possibly fit: peak is larger than the priced figure in every case ever
-    measured (2.49x at the lowest), so a request priced above all available
+    measured (2.37x at the lowest), so a request priced above all available
     memory is a certain MemoryError rather than a risk of one.  Nothing that
     works today starts failing.
+
+    Under ``"auto"`` the divisor is the schema's measured factor where one
+    exists (:data:`PEAK_BY_SCHEMA`), and the worst case otherwise.
     """
     override = os.getenv(BUDGET_ENV, "").strip()
     if override:
@@ -203,4 +229,4 @@ def declared_budget(mode: str) -> int | None:
         return None
     if mode == "interactive":
         return free
-    return int(free / PEAK_PER_PRICED_BYTE)
+    return int(free / peak_per_priced_byte(schema))

{quantpad_data-0.8.1 → quantpad_data-0.8.4}/quantpad_data/v2.py

@@ -23,10 +23,13 @@ request.
 
 from __future__ import annotations
 
+import collections
+import concurrent.futures
 import io
 import json
 import os
 import tempfile
+import threading
 import warnings
 from datetime import date, timedelta
 from typing import Any, Iterator, Sequence
@@ -83,6 +86,39 @@ STORAGE_SPOOL_BYTES = 32 * 1024 * 1024
 #: all -- it has no gaps, so it stays the single manifest it always was.
 HYDRATE_CHUNK_DAYS = 5
 
+#: How many of those runs hydrate at once.  One at a time, each waited for
+#: the one before it, so a cold window cost the SUM of its fetches: measured
+#: 2026-10-28 on production, a cold 12-day GF.v.0 window took 99.0s one at a
+#: time and 33.6s four at a time, with identical rows.  Hydration returns a
+#: manifest, not rows, so running ahead costs this process nothing; frames
+#: are still read and yielded one at a time, in order.  Kept identical to
+#: the sandbox client's ``V2_HYDRATE_PARALLEL``.
+HYDRATE_PARALLEL = 4
+
+
+def _in_background(fn, *args) -> "concurrent.futures.Future":
+    """Run ``fn(*args)`` on a DAEMON thread and return its future.
+
+    Not a ``ThreadPoolExecutor``: its workers are joined at interpreter
+    exit, and a hydration already in flight cannot be cancelled, so a
+    script that stopped iterating a cold window early would hang on exit
+    until those requests returned -- up to the data timeout.  A daemon
+    thread is abandoned at exit instead.  Concurrency stays bounded by the
+    caller, which keeps at most ``HYDRATE_PARALLEL`` unconsumed.
+    """
+    future: concurrent.futures.Future = concurrent.futures.Future()
+
+    def run() -> None:
+        if not future.set_running_or_notify_cancel():
+            return
+        try:
+            future.set_result(fn(*args))
+        except BaseException as exc:  # noqa: BLE001 - handed to the consumer
+            future.set_exception(exc)
+
+    threading.Thread(target=run, name="qpd-hydrate", daemon=True).start()
+    return future
+
 
 def _symbols(value: str | Sequence[str]) -> list[str]:
     return [value] if isinstance(value, str) else [str(v) for v in value]
@@ -341,7 +377,7 @@ class Timeseries(_Namespace):
         # service too old to know the field, which ignores unknown keys.
         budget = (
             max_decoded_bytes if max_decoded_bytes is not None
-            else memory.declared_budget(mode)
+            else memory.declared_budget(mode, schema)
         )
         if budget is not None:
             # A MEASURED ZERO is not the same as "could not measure", and
@@ -643,20 +679,43 @@ class Timeseries(_Namespace):
         # One cursor over the union of both sets, so a fetched day lands
         # between the cached days that surround it.
         fetched: dict[str, list[tuple[dict, dict]]] = {}
-        pending = list(_runs(sorted(missing), HYDRATE_CHUNK_DAYS))
-        for day in sorted(set(ready) | missing):
-            while pending and pending[0][0] <= day:
-                start, end = pending.pop(0)
-                for payload in self._manifests(
-                    {**manifest_body, "start": start, "end": end}
-                ):
-                    for item in payload["files"]:
-                        fetched.setdefault(item["day"], []).append((payload, item))
-            yield from fresh(ready.get(day, ()))
-            # A missing day that came back with nothing -- a holiday, or
-            # a genuinely quiet session -- simply has no entry here, and
-            # is skipped exactly as an ``empty`` day always was.
-            yield from fresh(fetched.pop(day, ()))
+        pending = collections.deque(_runs(sorted(missing), HYDRATE_CHUNK_DAYS))
+
+        def hydrate(start: str, end: str) -> list[dict]:
+            return list(self._manifests({**manifest_body, "start": start, "end": end}))
+
+        # Runs hydrate up to HYDRATE_PARALLEL at a time, ahead of the
+        # cursor, and are CONSUMED in order: a run's files join ``fetched``
+        # only when the cursor reaches its first day, exactly as when they
+        # were fetched there, so the order frames come out in is unchanged.
+        inflight: collections.deque = collections.deque()
+
+        def top_up() -> None:
+            while pending and len(inflight) < HYDRATE_PARALLEL:
+                start, end = pending.popleft()
+                inflight.append((start, _in_background(hydrate, start, end)))
+
+        try:
+            top_up()
+            for day in sorted(set(ready) | missing):
+                while inflight and inflight[0][0] <= day:
+                    _, future = inflight.popleft()
+                    # Re-raises the run's own error here, at the day it
+                    # covers, which is where the sequential loop raised it.
+                    for payload in future.result():
+                        for item in payload["files"]:
+                            fetched.setdefault(item["day"], []).append((payload, item))
+                    top_up()
+                yield from fresh(ready.get(day, ()))
+                # A missing day that came back with nothing -- a holiday, or
+                # a genuinely quiet session -- simply has no entry here, and
+                # is skipped exactly as an ``empty`` day always was.
+                yield from fresh(fetched.pop(day, ()))
+        finally:
+            # A caller that stops iterating early: runs already started finish
+            # on their daemon threads and are dropped; none is started after.
+            for _, future in inflight:
+                future.cancel()
 
         # Anything a fetch returned for a day the survey did not list.
         # It should be empty, and it is drained rather than asserted
@@ -979,6 +1038,14 @@ class Options(_Namespace):
         instrument_class: str | None = None,
         limit: int | None = None,
     ) -> pd.DataFrame:
+        """Outright calls and puts for ``underlier`` as of ``date``.
+
+        ``underlier`` is the parent (``SPY.OPT``, ``NQ.OPT``); a futures
+        option handle such as ``CL.FUT.OPT`` is accepted and sent as the
+        parent upstream knows (``LO.OPT``).  Spreads are left out unless
+        their class is asked for (``instrument_class="T"``), and
+        ``expiration`` is a ``YYYY-MM-DD`` UTC date.
+        """
         params = {
             "dataset": dataset,
             "underlier": underlier,
@@ -1330,7 +1397,13 @@ def _partition(body: dict, item: dict, columns) -> pd.DataFrame:
     hostile ``QPD_CACHE_DIR`` can contain, and the only other way it could
     fail is a manifest item missing ``day`` or ``instrument_id``, which the
     caller subscripts for its own labels two lines later regardless.
+
+    A settled day is not immutable: the service re-fetches a day its
+    vendor regenerates, and says so by changing the item's ``version``.  A
+    hit therefore has to be the version the manifest describes, or it is
+    the uncorrected copy served from disk for as long as the disk keeps it.
     """
+    expect = {"version": item.get("version")}
     path = None
     if cache.enabled() and cache.day_settled(item.get("day")):
         path = cache.day_path(
@@ -1343,14 +1416,15 @@ def _partition(body: dict, item: dict, columns) -> pd.DataFrame:
         )
 
     if path is not None:
-        hit = cache.get_frame(path)
+        hit = cache.get_frame(path, expect=expect)
         if hit is not None:
             return hit
 
     frame = _read_parquet_url(item["url"], columns)
     if path is not None:
         cache.put_frame(
-            path, frame, {"day": item["day"], "dataset": body["dataset"]}
+            path, frame,
+            {"day": item["day"], "dataset": body["dataset"], **expect},
         )
     return frame
 

{quantpad_data-0.8.1 → quantpad_data-0.8.4}/tests/conftest.py

@@ -24,3 +24,14 @@ def isolated_cache(tmp_path, monkeypatch):
     monkeypatch.delenv("QPD_CACHE", raising=False)
     monkeypatch.delenv("QPD_CACHE_MAX_BYTES", raising=False)
     return tmp_path / "qpd-cache"
+
+
+@pytest.fixture(autouse=True)
+def one_hydration_at_a_time(monkeypatch):
+    """The fake sessions replay responses in CALL order, and parallel
+    hydrations would race for them, so tests hydrate one run at a time.
+    The parallel path has its own tests, with a session keyed on the window
+    each request asks for (``test_v2.WindowSession``)."""
+    from quantpad_data import v2
+
+    monkeypatch.setattr(v2, "HYDRATE_PARALLEL", 1)

{quantpad_data-0.8.1 → quantpad_data-0.8.4}/tests/test_client.py

@@ -164,6 +164,128 @@ def test_rate_limit_exposes_retry_after():
     assert caught.value.retry_after == 7
 
 
+def _record_sleeps(monkeypatch) -> list[float]:
+    import quantpad_data.client as client_module
+
+    slept: list[float] = []
+    monkeypatch.setattr(client_module.time, "sleep", slept.append)
+    return slept
+
+
+def test_a_503_that_says_when_to_come_back_is_believed(monkeypatch):
+    """During the 2026-10-01/02 API-key outage every answer was "503,
+    Retry-After: 5", and this client retried each call three more times
+    inside four seconds -- one key kept that up for sixteen hours."""
+    from quantpad_data.errors import UpstreamError
+
+    slept = _record_sleeps(monkeypatch)
+    unavailable = b'{"error":"API-key validation is temporarily unavailable"}'
+    session = FakeSession(
+        [response(503, unavailable, {"Retry-After": "5"}) for _ in range(4)]
+    )
+    client = Client(api_key="secret", session=session, max_retries=3)
+
+    with pytest.raises(UpstreamError) as caught:
+        client.get_coverage("AAPL")
+
+    assert caught.value.retry_after == 5
+    assert len(session.calls) == 4, "retries are capped at max_retries"
+    assert len(slept) == 3
+    assert all(delay >= 5 for delay in slept), slept
+
+
+def test_backoff_doubles_with_jitter_when_the_server_names_no_time(monkeypatch):
+    from quantpad_data.client import RETRY_BACKOFF_BASE
+    from quantpad_data.errors import UpstreamError
+
+    slept = _record_sleeps(monkeypatch)
+    session = FakeSession([response(502, b"bad gateway") for _ in range(4)])
+    client = Client(api_key="secret", session=session, max_retries=3)
+
+    with pytest.raises(UpstreamError):
+        client.get_coverage("AAPL")
+
+    for attempt, delay in enumerate(slept):
+        step = RETRY_BACKOFF_BASE * 2**attempt
+        assert step / 2 <= delay <= step, (attempt, delay)
+
+
+def test_jitter_actually_spreads_clients_out():
+    from quantpad_data.client import _retry_delay
+
+    assert len({round(_retry_delay(2, 5), 6) for _ in range(50)}) > 1
+    assert len({round(_retry_delay(2, None), 6) for _ in range(50)}) > 1
+
+
+def test_an_absurd_retry_after_is_capped_rather_than_slept(monkeypatch):
+    """A daily-quota 429 can say "come back in twenty hours"; sleeping
+    that long without a word is worse than failing."""
+    from quantpad_data.client import RETRY_AFTER_MAX, _retry_delay
+
+    assert _retry_delay(0, 72_000) <= RETRY_AFTER_MAX + 1.0
+
+
+def _trades_stream(*batch_rows: int, complete: bool) -> bytes:
+    """An Arrow IPC tick stream; ``complete=False`` stops before the
+    end-of-stream marker, as a body cut off by the connection would."""
+    schema = pa.schema([("t", pa.int64()), ("price", pa.float64())])
+    sink = io.BytesIO()
+    writer = ipc.new_stream(sink, schema)
+    for rows in batch_rows:
+        writer.write_batch(pa.record_batch(
+            [pa.array(range(rows), pa.int64()), pa.array([1.0] * rows)],
+            schema=schema,
+        ))
+    if complete:
+        writer.close()
+    return sink.getvalue()
+
+
+class _CutConnection(io.BytesIO):
+    """A raw body that ends the way an aborted chunked response does:
+    after the bytes that arrived, the next read fails rather than
+    returning empty."""
+
+    def read(self, size=-1):
+        data = super().read(size)
+        if not data:
+            import urllib3
+
+            raise urllib3.exceptions.ProtocolError(
+                "Connection broken: IncompleteRead(0 bytes read)"
+            )
+        return data
+
+
+def test_a_tick_stream_cut_short_raises_an_error_that_says_so():
+    """The service ends a tick stream that fails partway by closing the
+    connection -- its 200 has already gone out.  What surfaced before was
+    urllib3's bare ProtocolError; a caller has to be told that what they
+    received is incomplete and that retrying is the answer."""
+    from quantpad_data.errors import IncompleteResponseError, UpstreamError
+
+    cut = response(200, b"")
+    cut.raw = _CutConnection(_trades_stream(4, 6, complete=False))
+    client = Client(api_key="secret", session=FakeSession([cut]), max_retries=0)
+
+    with pytest.raises(IncompleteResponseError) as caught:
+        list(client.get_ticks("NQ.FUT", "trades", 1, 2, chunksize=1000))
+
+    assert isinstance(caught.value, UpstreamError), "existing handlers must still catch it"
+    assert caught.value.rows_received == 10
+    assert "retry" in str(caught.value).lower()
+
+
+def test_a_complete_tick_stream_is_unaffected():
+    whole = response(200, b"")
+    whole.raw = io.BytesIO(_trades_stream(4, 6, complete=True))
+    client = Client(api_key="secret", session=FakeSession([whole]), max_retries=0)
+
+    frames = list(client.get_ticks("NQ.FUT", "trades", 1, 2, chunksize=1000))
+
+    assert sum(len(frame) for frame in frames) == 10
+
+
 def test_universe_and_coverage_paths():
     session = FakeSession(
         [response(body=b'{"results":[]}'), response(body=b'{"symbol":"AAPL"}')]

{quantpad_data-0.8.1 → quantpad_data-0.8.4}/tests/test_v2.py

@@ -10,6 +10,8 @@ month of depth data is a normal thing to do.
 
 from __future__ import annotations
 
+import threading
+
 import io
 import json
 from datetime import date, timedelta
@@ -473,11 +475,20 @@ class TestBulkDelivery:
             dataset="GLBX.MDP3", symbols="ESZ6", schema="mbp-10",
             start="2026-01-05", end="2026-01-07",
         )
+        # The fetch starts in the background straight away now, so it is
+        # held open: the cached frame must arrive without waiting for it.
+        gate = threading.Event()
+        replay = session.request
+
+        def held(method, url, **kwargs):
+            if session.calls:  # every call after the survey is the fetch
+                assert gate.wait(5), "the cached frame never arrived"
+            return replay(method, url, **kwargs)
+
+        session.request = held
         first = next(frames)
         assert first.attrs["quantpad"]["day"] == "2026-01-05"
-        # One call so far: the cached day was served without the fetch
-        # for the missing one having been made.
-        assert len(session.calls) == 1
+        gate.set()
         rest = list(frames)
         assert [f.attrs["quantpad"]["day"] for f in rest] == ["2026-01-06"]
         assert len(session.calls) == 2
@@ -1471,7 +1482,21 @@ class TestTheCallerDeclaresWhatItCanHold:
         )
 
         assert self._sent(session)["max_decoded_bytes"] == int(
-            self.FREE / memory.PEAK_PER_PRICED_BYTE
+            self.FREE / memory.peak_per_priced_byte("mbp-10")
+        )
+
+    def test_the_divisor_is_the_schemas_measured_factor(self):
+        """6x refused single quote-record days that peak at ~3.2x."""
+        for schema, factor in [("mbp-10", 3.0), ("mbp-1", 4.5), ("trades", 6.0),
+                               ("definition", 6.0), (None, 6.0)]:
+            assert memory.declared_budget("auto", schema) == int(self.FREE / factor)
+        assert memory.declared_budget("interactive", "mbp-1") == self.FREE
+
+    def test_every_measured_factor_stays_above_its_reading(self):
+        for schema, reading in {"mbp-10": 2.57, "mbp-1": 4.32}.items():
+            assert memory.peak_per_priced_byte(schema) > reading
+        assert all(
+            f <= memory.PEAK_PER_PRICED_BYTE for f in memory.PEAK_BY_SCHEMA.values()
         )
 
     def test_interactive_declares_the_undivided_figure(self):
@@ -2297,4 +2322,126 @@ class TestDerivations:
                 stype_in="continuous", start="2026-01-05", end="2026-02-05",
                 resample="4h",
             ))
-        assert session.calls == []
+        assert session.calls == []
+
+
+class WindowSession(FakeSession):
+    """Answers each manifest by the window it asks for, not by call order,
+    because parallel hydrations reach it in whatever order their threads
+    run.  The first ``parties`` fetches wait at a barrier until all of them
+    are in flight -- the proof that they overlap."""
+
+    def __init__(self, survey, parties=None):
+        super().__init__([])
+        self.survey = survey
+        self.lock = threading.Lock()
+        self.barrier = threading.Barrier(parties) if parties else None
+        self.fetches = []
+
+    def request(self, method, url, **kwargs):
+        body = kwargs.get("json") or {}
+        with self.lock:
+            self.calls.append((method, url, kwargs))
+            if body.get("hydrate") is False:
+                return self.survey
+            self.fetches.append((body["start"], body["end"]))
+            first_wave = len(self.fetches) <= (self.barrier.parties if self.barrier else 0)
+        if first_wave:
+            self.barrier.wait(5)
+        start, end = date.fromisoformat(body["start"]), date.fromisoformat(body["end"])
+        days = [(start + timedelta(days=i)).isoformat() for i in range((end - start).days)]
+        return json_response({
+            "format": "parquet",
+            "files": [{"day": d, "instrument_id": 1, "key": d, "size_bytes": 1,
+                       "url": f"https://r2.test/{d}"} for d in days],
+            "gaps": [],
+            "metadata": {},
+        })
+
+
+class TestParallelHydration:
+    """Sequential hydration made a cold window cost the SUM of its fetches
+    (99.0s against 33.6s four at a time, measured on production)."""
+
+    GAPS = [f"2026-01-{day:02d}" for day in range(5, 13)]  # runs of 1, 2, 4, 1
+
+    def _iter(self, monkeypatch, session, end="2026-01-13"):
+        from quantpad_data import v2
+
+        client = Client(api_key="k", base_url="https://api.test", session=session)
+        monkeypatch.setattr(
+            v2, "_read_parquet_url", lambda url, columns: pd.DataFrame({"url": [url]})
+        )
+        return Historical(client).timeseries.iter_range(
+            dataset="GLBX.MDP3", symbols="ESZ6", schema="mbp-10",
+            start="2026-01-05", end=end,
+        )
+
+    def _survey(self, gaps, cached=()):
+        return json_response({
+            "format": "parquet",
+            "files": [{"day": d, "instrument_id": 1, "key": d, "size_bytes": 1,
+                       "url": f"https://r2.test/{d}"} for d in cached],
+            "gaps": [{"day": d, "instrument_id": 1} for d in gaps],
+            "metadata": {},
+        })
+
+    def test_cold_runs_hydrate_at_the_same_time(self, monkeypatch):
+        from quantpad_data import v2
+
+        monkeypatch.setattr(v2, "HYDRATE_PARALLEL", 3)
+        session = WindowSession(self._survey(self.GAPS), parties=3)
+
+        days = [f.attrs["quantpad"]["day"] for f in self._iter(monkeypatch, session)]
+
+        assert days == self.GAPS
+        assert len(session.fetches) == 4
+
+    def test_the_window_stays_in_order_with_cached_days_in_place(self, monkeypatch):
+        from quantpad_data import v2
+
+        monkeypatch.setattr(v2, "HYDRATE_PARALLEL", 4)
+        gaps = ["2026-01-05", "2026-01-07", "2026-01-08", "2026-01-10"]
+        cached = ["2026-01-06", "2026-01-09"]
+        session = WindowSession(self._survey(gaps, cached))
+
+        days = [f.attrs["quantpad"]["day"]
+                for f in self._iter(monkeypatch, session, end="2026-01-11")]
+
+        assert days == sorted(gaps + cached)
+
+    def test_an_early_close_leaves_nothing_that_blocks_exit(self, monkeypatch):
+        """A hydration in flight cannot be cancelled, and pool workers are
+        joined at interpreter exit -- so hydrations run on daemon threads,
+        which a script that stops iterating early can exit past."""
+        from quantpad_data import v2
+
+        monkeypatch.setattr(v2, "HYDRATE_PARALLEL", 2)
+        gate, started = threading.Event(), threading.Event()
+        session = WindowSession(self._survey(self.GAPS))
+        replay = session.request
+
+        def hanging(method, url, **kwargs):
+            body = kwargs.get("json") or {}
+            if body.get("hydrate") is not False and body["start"] != "2026-01-05":
+                started.set()
+                gate.wait(10)
+            return replay(method, url, **kwargs)
+
+        session.request = hanging
+        frames = self._iter(monkeypatch, session)
+        assert next(frames).attrs["quantpad"]["day"] == "2026-01-05"
+        assert started.wait(5)
+        frames.close()
+        hydrating = [t for t in threading.enumerate() if t.name == "qpd-hydrate"]
+        assert hydrating and all(t.daemon for t in hydrating)
+        gate.set()
+
+    def test_stopping_early_does_not_wait_on_queued_hydrations(self, monkeypatch):
+        from quantpad_data import v2
+
+        monkeypatch.setattr(v2, "HYDRATE_PARALLEL", 2)
+        frames = self._iter(monkeypatch, WindowSession(self._survey(self.GAPS)))
+
+        assert next(frames).attrs["quantpad"]["day"] == "2026-01-05"
+        frames.close()

{quantpad_data-0.8.1 → quantpad_data-0.8.4}/tests/test_v2_cache.py

@@ -124,12 +124,13 @@ def api(*responses):
     return Historical(client).timeseries, session
 
 
-def manifest_for(days, instrument_id=1551):
+def manifest_for(days, instrument_id=1551, version=None):
     return response(200, json.dumps({
         "format": "parquet",
         "files": [
             {"day": day, "instrument_id": instrument_id, "symbol": "BA",
              "key": f"k/{day}", "size_bytes": 10,
+             **({"version": version} if version is not None else {}),
              "url": f"http://r2.test/{day}.parquet"}
             for day in days
         ],
@@ -276,6 +277,17 @@ class TestWhatTheKeyMustSeparate:
         base = _body(**ASK)
         assert cache.range_path(base) == cache.range_path({**base, field: value})
 
+    def test_an_answer_cached_before_the_service_changed_it_is_not_served(
+        self, monkeypatch
+    ):
+        # Until 2026-10 a window ending mid-day came back without its last
+        # day, and was cached that way. The body has not changed, so only the
+        # epoch can tell the old short answer from the new one.
+        body = _body(**{**ASK, "end": "2026-09-16T13:58:00Z"})
+        current = cache.range_path(body)
+        monkeypatch.setattr(cache, "_RANGE_KEY_EPOCH", cache._RANGE_KEY_EPOCH - 1)
+        assert cache.range_path(body) != current
+
 
 class TestWhatMustNotBeCached:
     def test_a_window_reaching_into_the_current_day_is_not_cached(self):
@@ -546,6 +558,70 @@ class TestPartitionsAreCachedIndividually:
         )
 
 
+class TestACorrectedDayIsNotServedFromDisk:
+    """A settled day is final for the calendar, not for the vendor.
+
+    The service re-fetches a day its vendor regenerates and changes the
+    manifest item's ``version`` when it does.  Before this, a day already on
+    disk was never asked about again, so the correction reached nobody who
+    had read the day once -- and nothing anywhere said the copy was old.
+    """
+
+    CORRECTED = ROWS.assign(price=[100.5, 101.5])
+
+    def test_the_same_version_is_still_a_hit(self, monkeypatch):
+        urls = UrlRecorder()
+        monkeypatch.setattr(requests, "get", urls)
+        ts, _ = api(
+            manifest_for(["2026-09-16"], version="v1"),
+            manifest_for(["2026-09-16"], version="v1"),
+        )
+        list(ts.iter_range(**_manifest_ask()))
+        list(ts.iter_range(**_manifest_ask()))
+        assert len(urls.urls) == 1
+
+    def test_a_new_version_is_fetched_and_replaces_the_copy(self, monkeypatch):
+        urls = UrlRecorder()
+        monkeypatch.setattr(requests, "get", urls)
+        ts, _ = api(
+            manifest_for(["2026-09-16"], version="v1"),
+            manifest_for(["2026-09-16"], version="v2"),
+            manifest_for(["2026-09-16"], version="v2"),
+        )
+        list(ts.iter_range(**_manifest_ask()))
+        urls.frame = self.CORRECTED
+        [corrected] = [f.copy() for f in ts.iter_range(**_manifest_ask())]
+        [again] = [f.copy() for f in ts.iter_range(**_manifest_ask())]
+
+        assert len(urls.urls) == 2, "fetched once per version, not per read"
+        assert list(corrected["price"]) == [100.5, 101.5]
+        pd.testing.assert_frame_equal(corrected, again)
+
+    def test_a_copy_written_before_versions_existed_is_refetched_once(
+        self, monkeypatch
+    ):
+        """Every entry on disk today carries no version, and some of them
+        are the uncorrected days this exists for."""
+        urls = UrlRecorder()
+        monkeypatch.setattr(requests, "get", urls)
+        ts, _ = api(
+            manifest_for(["2026-09-16"]),
+            manifest_for(["2026-09-16"], version="v1"),
+            manifest_for(["2026-09-16"], version="v1"),
+        )
+        for _ in range(3):
+            list(ts.iter_range(**_manifest_ask()))
+        assert len(urls.urls) == 2
+
+    def test_a_range_entry_is_not_asked_for_a_version_it_never_had(self):
+        """``expect`` is opt-in: the whole-response cache has no per-file
+        version to compare, and must keep its hits."""
+        path = cache.range_path(_body(**ASK))
+        assert cache.put_frame(path, ROWS, {"echo": {}})
+        assert cache.get_frame(path) is not None
+        assert cache.get_frame(path, expect={"version": "v1"}) is None
+
+
 class TestTheSettlementRuleItself:
     def test_a_past_day_is_settled(self):
         assert cache.day_settled("2026-09-16") is True