quantpad-data 0.8.3__tar.gz → 0.8.4__tar.gz

{quantpad_data-0.8.3 → quantpad_data-0.8.4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: quantpad-data
-Version: 0.8.3
+Version: 0.8.4
 Summary: Official Python client for the QuantPad market-data API
 Project-URL: Documentation, https://api.quantpad.ai/external/docs
 Author: QuantPad

{quantpad_data-0.8.3 → quantpad_data-0.8.4}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
 
 [project]
 name = "quantpad-data"
-version = "0.8.3"
+version = "0.8.4"
 description = "Official Python client for the QuantPad market-data API"
 readme = "README.md"
 requires-python = ">=3.10"

{quantpad_data-0.8.3 → quantpad_data-0.8.4}/quantpad_data/client.py

@@ -87,7 +87,7 @@ V2_NOT_AVAILABLE = (
     "QuantPad support about /v2 access."
 )
 
-VERSION = "0.8.3"
+VERSION = "0.8.4"
 USER_AGENT = f"quantpad-data-python/{VERSION}"
 TICK_SCHEMAS = (
     "trades", "mbp-1", "mbp-10", "cmbp-1", "tcbbo", "cbbo-1s",

{quantpad_data-0.8.3 → quantpad_data-0.8.4}/quantpad_data/memory.py

@@ -42,6 +42,29 @@ import os
 #: returns to killing the process with no message.
 PEAK_PER_PRICED_BYTE = 6.0
 
+#: Measured factors for the schemas where the 6x worst case refused what
+#: fitted -- in 14 days of production sandboxes, 31 of 74 refusals were ~1 GB
+#: or less, mostly single days of 80-byte quote records.  Measured 2026-10-28
+#: (peak RSS over priced bytes): mbp-10 2.37x and 2.57x; mbp-1 3.17x-3.19x on
+#: full days near the budget and 4.32x on a 63 MB one; trades 4.73x, so trades
+#: keeps the default.  The other quote schemas share mbp-1's record shape.
+#: Kept identical to the sandbox client's ``_PEAK_BY_SCHEMA``.
+PEAK_BY_SCHEMA = {
+    "mbp-10": 3.0,
+    "mbp-1": 4.5,
+    "tbbo": 4.5,
+    "bbo-1s": 4.5,
+    "bbo-1m": 4.5,
+    "cmbp-1": 4.5,
+    "tcbbo": 4.5,
+    "cbbo-1s": 4.5,
+    "cbbo-1m": 4.5,
+}
+
+
+def peak_per_priced_byte(schema: str | None) -> float:
+    return PEAK_BY_SCHEMA.get(schema, PEAK_PER_PRICED_BYTE)
+
 #: cgroup v1 spells "no limit" as a huge number rather than a word, and the
 #: exact sentinel varies with page size, so anything absurd is read as absent.
 _NO_LIMIT = 1 << 53
@@ -172,7 +195,7 @@ def available_bytes() -> int | None:
     return min(candidates) if candidates else None
 
 
-def declared_budget(mode: str) -> int | None:
+def declared_budget(mode: str, schema: str | None = None) -> int | None:
     """The figure to send as ``max_decoded_bytes``, in the service's units.
 
     TWO NUMBERS, BECAUSE THE COST OF BEING WRONG DIFFERS BY MODE.
@@ -186,9 +209,12 @@ def declared_budget(mode: str) -> int | None:
     and there is nowhere to degrade to, so a refusal is a hard failure.  It
     therefore declares free memory UNDIVIDED, which refuses only what cannot
     possibly fit: peak is larger than the priced figure in every case ever
-    measured (2.49x at the lowest), so a request priced above all available
+    measured (2.37x at the lowest), so a request priced above all available
     memory is a certain MemoryError rather than a risk of one.  Nothing that
     works today starts failing.
+
+    Under ``"auto"`` the divisor is the schema's measured factor where one
+    exists (:data:`PEAK_BY_SCHEMA`), and the worst case otherwise.
     """
     override = os.getenv(BUDGET_ENV, "").strip()
     if override:
@@ -203,4 +229,4 @@ def declared_budget(mode: str) -> int | None:
         return None
     if mode == "interactive":
         return free
-    return int(free / PEAK_PER_PRICED_BYTE)
+    return int(free / peak_per_priced_byte(schema))

{quantpad_data-0.8.3 → quantpad_data-0.8.4}/quantpad_data/v2.py

@@ -23,10 +23,13 @@ request.
 
 from __future__ import annotations
 
+import collections
+import concurrent.futures
 import io
 import json
 import os
 import tempfile
+import threading
 import warnings
 from datetime import date, timedelta
 from typing import Any, Iterator, Sequence
@@ -83,6 +86,39 @@ STORAGE_SPOOL_BYTES = 32 * 1024 * 1024
 #: all -- it has no gaps, so it stays the single manifest it always was.
 HYDRATE_CHUNK_DAYS = 5
 
+#: How many of those runs hydrate at once.  One at a time, each waited for
+#: the one before it, so a cold window cost the SUM of its fetches: measured
+#: 2026-10-28 on production, a cold 12-day GF.v.0 window took 99.0s one at a
+#: time and 33.6s four at a time, with identical rows.  Hydration returns a
+#: manifest, not rows, so running ahead costs this process nothing; frames
+#: are still read and yielded one at a time, in order.  Kept identical to
+#: the sandbox client's ``V2_HYDRATE_PARALLEL``.
+HYDRATE_PARALLEL = 4
+
+
+def _in_background(fn, *args) -> "concurrent.futures.Future":
+    """Run ``fn(*args)`` on a DAEMON thread and return its future.
+
+    Not a ``ThreadPoolExecutor``: its workers are joined at interpreter
+    exit, and a hydration already in flight cannot be cancelled, so a
+    script that stopped iterating a cold window early would hang on exit
+    until those requests returned -- up to the data timeout.  A daemon
+    thread is abandoned at exit instead.  Concurrency stays bounded by the
+    caller, which keeps at most ``HYDRATE_PARALLEL`` unconsumed.
+    """
+    future: concurrent.futures.Future = concurrent.futures.Future()
+
+    def run() -> None:
+        if not future.set_running_or_notify_cancel():
+            return
+        try:
+            future.set_result(fn(*args))
+        except BaseException as exc:  # noqa: BLE001 - handed to the consumer
+            future.set_exception(exc)
+
+    threading.Thread(target=run, name="qpd-hydrate", daemon=True).start()
+    return future
+
 
 def _symbols(value: str | Sequence[str]) -> list[str]:
     return [value] if isinstance(value, str) else [str(v) for v in value]
@@ -341,7 +377,7 @@ class Timeseries(_Namespace):
         # service too old to know the field, which ignores unknown keys.
         budget = (
             max_decoded_bytes if max_decoded_bytes is not None
-            else memory.declared_budget(mode)
+            else memory.declared_budget(mode, schema)
         )
         if budget is not None:
             # A MEASURED ZERO is not the same as "could not measure", and
@@ -643,20 +679,43 @@ class Timeseries(_Namespace):
         # One cursor over the union of both sets, so a fetched day lands
         # between the cached days that surround it.
         fetched: dict[str, list[tuple[dict, dict]]] = {}
-        pending = list(_runs(sorted(missing), HYDRATE_CHUNK_DAYS))
-        for day in sorted(set(ready) | missing):
-            while pending and pending[0][0] <= day:
-                start, end = pending.pop(0)
-                for payload in self._manifests(
-                    {**manifest_body, "start": start, "end": end}
-                ):
-                    for item in payload["files"]:
-                        fetched.setdefault(item["day"], []).append((payload, item))
-            yield from fresh(ready.get(day, ()))
-            # A missing day that came back with nothing -- a holiday, or
-            # a genuinely quiet session -- simply has no entry here, and
-            # is skipped exactly as an ``empty`` day always was.
-            yield from fresh(fetched.pop(day, ()))
+        pending = collections.deque(_runs(sorted(missing), HYDRATE_CHUNK_DAYS))
+
+        def hydrate(start: str, end: str) -> list[dict]:
+            return list(self._manifests({**manifest_body, "start": start, "end": end}))
+
+        # Runs hydrate up to HYDRATE_PARALLEL at a time, ahead of the
+        # cursor, and are CONSUMED in order: a run's files join ``fetched``
+        # only when the cursor reaches its first day, exactly as when they
+        # were fetched there, so the order frames come out in is unchanged.
+        inflight: collections.deque = collections.deque()
+
+        def top_up() -> None:
+            while pending and len(inflight) < HYDRATE_PARALLEL:
+                start, end = pending.popleft()
+                inflight.append((start, _in_background(hydrate, start, end)))
+
+        try:
+            top_up()
+            for day in sorted(set(ready) | missing):
+                while inflight and inflight[0][0] <= day:
+                    _, future = inflight.popleft()
+                    # Re-raises the run's own error here, at the day it
+                    # covers, which is where the sequential loop raised it.
+                    for payload in future.result():
+                        for item in payload["files"]:
+                            fetched.setdefault(item["day"], []).append((payload, item))
+                    top_up()
+                yield from fresh(ready.get(day, ()))
+                # A missing day that came back with nothing -- a holiday, or
+                # a genuinely quiet session -- simply has no entry here, and
+                # is skipped exactly as an ``empty`` day always was.
+                yield from fresh(fetched.pop(day, ()))
+        finally:
+            # A caller that stops iterating early: runs already started finish
+            # on their daemon threads and are dropped; none is started after.
+            for _, future in inflight:
+                future.cancel()
 
         # Anything a fetch returned for a day the survey did not list.
         # It should be empty, and it is drained rather than asserted
@@ -979,6 +1038,14 @@ class Options(_Namespace):
         instrument_class: str | None = None,
         limit: int | None = None,
     ) -> pd.DataFrame:
+        """Outright calls and puts for ``underlier`` as of ``date``.
+
+        ``underlier`` is the parent (``SPY.OPT``, ``NQ.OPT``); a futures
+        option handle such as ``CL.FUT.OPT`` is accepted and sent as the
+        parent upstream knows (``LO.OPT``).  Spreads are left out unless
+        their class is asked for (``instrument_class="T"``), and
+        ``expiration`` is a ``YYYY-MM-DD`` UTC date.
+        """
         params = {
             "dataset": dataset,
             "underlier": underlier,

{quantpad_data-0.8.3 → quantpad_data-0.8.4}/tests/conftest.py

@@ -24,3 +24,14 @@ def isolated_cache(tmp_path, monkeypatch):
     monkeypatch.delenv("QPD_CACHE", raising=False)
     monkeypatch.delenv("QPD_CACHE_MAX_BYTES", raising=False)
     return tmp_path / "qpd-cache"
+
+
+@pytest.fixture(autouse=True)
+def one_hydration_at_a_time(monkeypatch):
+    """The fake sessions replay responses in CALL order, and parallel
+    hydrations would race for them, so tests hydrate one run at a time.
+    The parallel path has its own tests, with a session keyed on the window
+    each request asks for (``test_v2.WindowSession``)."""
+    from quantpad_data import v2
+
+    monkeypatch.setattr(v2, "HYDRATE_PARALLEL", 1)

{quantpad_data-0.8.3 → quantpad_data-0.8.4}/tests/test_v2.py

@@ -10,6 +10,8 @@ month of depth data is a normal thing to do.
 
 from __future__ import annotations
 
+import threading
+
 import io
 import json
 from datetime import date, timedelta
@@ -473,11 +475,20 @@ class TestBulkDelivery:
             dataset="GLBX.MDP3", symbols="ESZ6", schema="mbp-10",
             start="2026-01-05", end="2026-01-07",
         )
+        # The fetch starts in the background straight away now, so it is
+        # held open: the cached frame must arrive without waiting for it.
+        gate = threading.Event()
+        replay = session.request
+
+        def held(method, url, **kwargs):
+            if session.calls:  # every call after the survey is the fetch
+                assert gate.wait(5), "the cached frame never arrived"
+            return replay(method, url, **kwargs)
+
+        session.request = held
         first = next(frames)
         assert first.attrs["quantpad"]["day"] == "2026-01-05"
-        # One call so far: the cached day was served without the fetch
-        # for the missing one having been made.
-        assert len(session.calls) == 1
+        gate.set()
         rest = list(frames)
         assert [f.attrs["quantpad"]["day"] for f in rest] == ["2026-01-06"]
         assert len(session.calls) == 2
@@ -1471,7 +1482,21 @@ class TestTheCallerDeclaresWhatItCanHold:
         )
 
         assert self._sent(session)["max_decoded_bytes"] == int(
-            self.FREE / memory.PEAK_PER_PRICED_BYTE
+            self.FREE / memory.peak_per_priced_byte("mbp-10")
+        )
+
+    def test_the_divisor_is_the_schemas_measured_factor(self):
+        """6x refused single quote-record days that peak at ~3.2x."""
+        for schema, factor in [("mbp-10", 3.0), ("mbp-1", 4.5), ("trades", 6.0),
+                               ("definition", 6.0), (None, 6.0)]:
+            assert memory.declared_budget("auto", schema) == int(self.FREE / factor)
+        assert memory.declared_budget("interactive", "mbp-1") == self.FREE
+
+    def test_every_measured_factor_stays_above_its_reading(self):
+        for schema, reading in {"mbp-10": 2.57, "mbp-1": 4.32}.items():
+            assert memory.peak_per_priced_byte(schema) > reading
+        assert all(
+            f <= memory.PEAK_PER_PRICED_BYTE for f in memory.PEAK_BY_SCHEMA.values()
         )
 
     def test_interactive_declares_the_undivided_figure(self):
@@ -2297,4 +2322,126 @@ class TestDerivations:
                 stype_in="continuous", start="2026-01-05", end="2026-02-05",
                 resample="4h",
             ))
-        assert session.calls == []
+        assert session.calls == []
+
+
+class WindowSession(FakeSession):
+    """Answers each manifest by the window it asks for, not by call order,
+    because parallel hydrations reach it in whatever order their threads
+    run.  The first ``parties`` fetches wait at a barrier until all of them
+    are in flight -- the proof that they overlap."""
+
+    def __init__(self, survey, parties=None):
+        super().__init__([])
+        self.survey = survey
+        self.lock = threading.Lock()
+        self.barrier = threading.Barrier(parties) if parties else None
+        self.fetches = []
+
+    def request(self, method, url, **kwargs):
+        body = kwargs.get("json") or {}
+        with self.lock:
+            self.calls.append((method, url, kwargs))
+            if body.get("hydrate") is False:
+                return self.survey
+            self.fetches.append((body["start"], body["end"]))
+            first_wave = len(self.fetches) <= (self.barrier.parties if self.barrier else 0)
+        if first_wave:
+            self.barrier.wait(5)
+        start, end = date.fromisoformat(body["start"]), date.fromisoformat(body["end"])
+        days = [(start + timedelta(days=i)).isoformat() for i in range((end - start).days)]
+        return json_response({
+            "format": "parquet",
+            "files": [{"day": d, "instrument_id": 1, "key": d, "size_bytes": 1,
+                       "url": f"https://r2.test/{d}"} for d in days],
+            "gaps": [],
+            "metadata": {},
+        })
+
+
+class TestParallelHydration:
+    """Sequential hydration made a cold window cost the SUM of its fetches
+    (99.0s against 33.6s four at a time, measured on production)."""
+
+    GAPS = [f"2026-01-{day:02d}" for day in range(5, 13)]  # runs of 1, 2, 4, 1
+
+    def _iter(self, monkeypatch, session, end="2026-01-13"):
+        from quantpad_data import v2
+
+        client = Client(api_key="k", base_url="https://api.test", session=session)
+        monkeypatch.setattr(
+            v2, "_read_parquet_url", lambda url, columns: pd.DataFrame({"url": [url]})
+        )
+        return Historical(client).timeseries.iter_range(
+            dataset="GLBX.MDP3", symbols="ESZ6", schema="mbp-10",
+            start="2026-01-05", end=end,
+        )
+
+    def _survey(self, gaps, cached=()):
+        return json_response({
+            "format": "parquet",
+            "files": [{"day": d, "instrument_id": 1, "key": d, "size_bytes": 1,
+                       "url": f"https://r2.test/{d}"} for d in cached],
+            "gaps": [{"day": d, "instrument_id": 1} for d in gaps],
+            "metadata": {},
+        })
+
+    def test_cold_runs_hydrate_at_the_same_time(self, monkeypatch):
+        from quantpad_data import v2
+
+        monkeypatch.setattr(v2, "HYDRATE_PARALLEL", 3)
+        session = WindowSession(self._survey(self.GAPS), parties=3)
+
+        days = [f.attrs["quantpad"]["day"] for f in self._iter(monkeypatch, session)]
+
+        assert days == self.GAPS
+        assert len(session.fetches) == 4
+
+    def test_the_window_stays_in_order_with_cached_days_in_place(self, monkeypatch):
+        from quantpad_data import v2
+
+        monkeypatch.setattr(v2, "HYDRATE_PARALLEL", 4)
+        gaps = ["2026-01-05", "2026-01-07", "2026-01-08", "2026-01-10"]
+        cached = ["2026-01-06", "2026-01-09"]
+        session = WindowSession(self._survey(gaps, cached))
+
+        days = [f.attrs["quantpad"]["day"]
+                for f in self._iter(monkeypatch, session, end="2026-01-11")]
+
+        assert days == sorted(gaps + cached)
+
+    def test_an_early_close_leaves_nothing_that_blocks_exit(self, monkeypatch):
+        """A hydration in flight cannot be cancelled, and pool workers are
+        joined at interpreter exit -- so hydrations run on daemon threads,
+        which a script that stops iterating early can exit past."""
+        from quantpad_data import v2
+
+        monkeypatch.setattr(v2, "HYDRATE_PARALLEL", 2)
+        gate, started = threading.Event(), threading.Event()
+        session = WindowSession(self._survey(self.GAPS))
+        replay = session.request
+
+        def hanging(method, url, **kwargs):
+            body = kwargs.get("json") or {}
+            if body.get("hydrate") is not False and body["start"] != "2026-01-05":
+                started.set()
+                gate.wait(10)
+            return replay(method, url, **kwargs)
+
+        session.request = hanging
+        frames = self._iter(monkeypatch, session)
+        assert next(frames).attrs["quantpad"]["day"] == "2026-01-05"
+        assert started.wait(5)
+        frames.close()
+        hydrating = [t for t in threading.enumerate() if t.name == "qpd-hydrate"]
+        assert hydrating and all(t.daemon for t in hydrating)
+        gate.set()
+
+    def test_stopping_early_does_not_wait_on_queued_hydrations(self, monkeypatch):
+        from quantpad_data import v2
+
+        monkeypatch.setattr(v2, "HYDRATE_PARALLEL", 2)
+        frames = self._iter(monkeypatch, WindowSession(self._survey(self.GAPS)))
+
+        assert next(frames).attrs["quantpad"]["day"] == "2026-01-05"
+        frames.close()