quantpad-data 0.8.3__tar.gz → 0.8.5__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (24) hide show
  1. {quantpad_data-0.8.3 → quantpad_data-0.8.5}/PKG-INFO +2 -2
  2. {quantpad_data-0.8.3 → quantpad_data-0.8.5}/README.md +1 -1
  3. {quantpad_data-0.8.3 → quantpad_data-0.8.5}/pyproject.toml +1 -1
  4. {quantpad_data-0.8.3 → quantpad_data-0.8.5}/quantpad_data/client.py +1 -1
  5. {quantpad_data-0.8.3 → quantpad_data-0.8.5}/quantpad_data/memory.py +29 -3
  6. {quantpad_data-0.8.3 → quantpad_data-0.8.5}/quantpad_data/v2.py +195 -19
  7. {quantpad_data-0.8.3 → quantpad_data-0.8.5}/tests/conftest.py +11 -0
  8. {quantpad_data-0.8.3 → quantpad_data-0.8.5}/tests/test_v2.py +322 -5
  9. {quantpad_data-0.8.3 → quantpad_data-0.8.5}/.gitignore +0 -0
  10. {quantpad_data-0.8.3 → quantpad_data-0.8.5}/quantpad_data/__init__.py +0 -0
  11. {quantpad_data-0.8.3 → quantpad_data-0.8.5}/quantpad_data/cache.py +0 -0
  12. {quantpad_data-0.8.3 → quantpad_data-0.8.5}/quantpad_data/errors.py +0 -0
  13. {quantpad_data-0.8.3 → quantpad_data-0.8.5}/quantpad_data/lean.py +0 -0
  14. {quantpad_data-0.8.3 → quantpad_data-0.8.5}/quantpad_data/py.typed +0 -0
  15. {quantpad_data-0.8.3 → quantpad_data-0.8.5}/quantpad_data/rth.py +0 -0
  16. {quantpad_data-0.8.3 → quantpad_data-0.8.5}/tests/test_client.py +0 -0
  17. {quantpad_data-0.8.3 → quantpad_data-0.8.5}/tests/test_lean.py +0 -0
  18. {quantpad_data-0.8.3 → quantpad_data-0.8.5}/tests/test_message_encoding.py +0 -0
  19. {quantpad_data-0.8.3 → quantpad_data-0.8.5}/tests/test_packaging.py +0 -0
  20. {quantpad_data-0.8.3 → quantpad_data-0.8.5}/tests/test_refusal_hint.py +0 -0
  21. {quantpad_data-0.8.3 → quantpad_data-0.8.5}/tests/test_rth.py +0 -0
  22. {quantpad_data-0.8.3 → quantpad_data-0.8.5}/tests/test_v2_cache.py +0 -0
  23. {quantpad_data-0.8.3 → quantpad_data-0.8.5}/tests/test_v2_gate.py +0 -0
  24. {quantpad_data-0.8.3 → quantpad_data-0.8.5}/tests/test_version.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.5
2
2
  Name: quantpad-data
3
- Version: 0.8.3
3
+ Version: 0.8.5
4
4
  Summary: Official Python client for the QuantPad market-data API
5
5
  Project-URL: Documentation, https://api.quantpad.ai/external/docs
6
6
  Author: QuantPad
@@ -170,7 +170,7 @@ the cache so ordinary `get_range` calls serve it from storage:
170
170
 
171
171
  ```python
172
172
  job = client.batch.submit(
173
- dataset="GLBX.MDP3", symbols="ESZ6", schema="mbo",
173
+ dataset="GLBX.MDP3", symbols="ESZ6", schema="mbp-10",
174
174
  start="2026-01-05", end="2026-02-05", hydrate=True,
175
175
  )
176
176
  client.batch.hydrate(job["job_id"])
@@ -148,7 +148,7 @@ the cache so ordinary `get_range` calls serve it from storage:
148
148
 
149
149
  ```python
150
150
  job = client.batch.submit(
151
- dataset="GLBX.MDP3", symbols="ESZ6", schema="mbo",
151
+ dataset="GLBX.MDP3", symbols="ESZ6", schema="mbp-10",
152
152
  start="2026-01-05", end="2026-02-05", hydrate=True,
153
153
  )
154
154
  client.batch.hydrate(job["job_id"])
@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
4
4
 
5
5
  [project]
6
6
  name = "quantpad-data"
7
- version = "0.8.3"
7
+ version = "0.8.5"
8
8
  description = "Official Python client for the QuantPad market-data API"
9
9
  readme = "README.md"
10
10
  requires-python = ">=3.10"
@@ -87,7 +87,7 @@ V2_NOT_AVAILABLE = (
87
87
  "QuantPad support about /v2 access."
88
88
  )
89
89
 
90
- VERSION = "0.8.3"
90
+ VERSION = "0.8.5"
91
91
  USER_AGENT = f"quantpad-data-python/{VERSION}"
92
92
  TICK_SCHEMAS = (
93
93
  "trades", "mbp-1", "mbp-10", "cmbp-1", "tcbbo", "cbbo-1s",
@@ -42,6 +42,29 @@ import os
42
42
  #: returns to killing the process with no message.
43
43
  PEAK_PER_PRICED_BYTE = 6.0
44
44
 
45
+ #: Measured factors for the schemas where the 6x worst case refused what
46
+ #: fitted -- in 14 days of production sandboxes, 31 of 74 refusals were ~1 GB
47
+ #: or less, mostly single days of 80-byte quote records. Measured 2026-10-28
48
+ #: (peak RSS over priced bytes): mbp-10 2.37x and 2.57x; mbp-1 3.17x-3.19x on
49
+ #: full days near the budget and 4.32x on a 63 MB one; trades 4.73x, so trades
50
+ #: keeps the default. The other quote schemas share mbp-1's record shape.
51
+ #: Kept identical to the sandbox client's ``_PEAK_BY_SCHEMA``.
52
+ PEAK_BY_SCHEMA = {
53
+ "mbp-10": 3.0,
54
+ "mbp-1": 4.5,
55
+ "tbbo": 4.5,
56
+ "bbo-1s": 4.5,
57
+ "bbo-1m": 4.5,
58
+ "cmbp-1": 4.5,
59
+ "tcbbo": 4.5,
60
+ "cbbo-1s": 4.5,
61
+ "cbbo-1m": 4.5,
62
+ }
63
+
64
+
65
+ def peak_per_priced_byte(schema: str | None) -> float:
66
+ return PEAK_BY_SCHEMA.get(schema, PEAK_PER_PRICED_BYTE)
67
+
45
68
  #: cgroup v1 spells "no limit" as a huge number rather than a word, and the
46
69
  #: exact sentinel varies with page size, so anything absurd is read as absent.
47
70
  _NO_LIMIT = 1 << 53
@@ -172,7 +195,7 @@ def available_bytes() -> int | None:
172
195
  return min(candidates) if candidates else None
173
196
 
174
197
 
175
- def declared_budget(mode: str) -> int | None:
198
+ def declared_budget(mode: str, schema: str | None = None) -> int | None:
176
199
  """The figure to send as ``max_decoded_bytes``, in the service's units.
177
200
 
178
201
  TWO NUMBERS, BECAUSE THE COST OF BEING WRONG DIFFERS BY MODE.
@@ -186,9 +209,12 @@ def declared_budget(mode: str) -> int | None:
186
209
  and there is nowhere to degrade to, so a refusal is a hard failure. It
187
210
  therefore declares free memory UNDIVIDED, which refuses only what cannot
188
211
  possibly fit: peak is larger than the priced figure in every case ever
189
- measured (2.49x at the lowest), so a request priced above all available
212
+ measured (2.37x at the lowest), so a request priced above all available
190
213
  memory is a certain MemoryError rather than a risk of one. Nothing that
191
214
  works today starts failing.
215
+
216
+ Under ``"auto"`` the divisor is the schema's measured factor where one
217
+ exists (:data:`PEAK_BY_SCHEMA`), and the worst case otherwise.
192
218
  """
193
219
  override = os.getenv(BUDGET_ENV, "").strip()
194
220
  if override:
@@ -203,4 +229,4 @@ def declared_budget(mode: str) -> int | None:
203
229
  return None
204
230
  if mode == "interactive":
205
231
  return free
206
- return int(free / PEAK_PER_PRICED_BYTE)
232
+ return int(free / peak_per_priced_byte(schema))
@@ -23,10 +23,13 @@ request.
23
23
 
24
24
  from __future__ import annotations
25
25
 
26
+ import collections
27
+ import concurrent.futures
26
28
  import io
27
29
  import json
28
30
  import os
29
31
  import tempfile
32
+ import threading
30
33
  import warnings
31
34
  from datetime import date, timedelta
32
35
  from typing import Any, Iterator, Sequence
@@ -63,6 +66,19 @@ STORAGE_TIMEOUT = (10, 120)
63
66
  #: decoded table at full size at once is worth avoiding.
64
67
  STORAGE_SPOOL_BYTES = 32 * 1024 * 1024
65
68
 
69
+
70
+ class _StorageUrlRejected(requests.HTTPError):
71
+ """Object storage refused a presigned partition URL (401/403).
72
+
73
+ Almost always expiry: a manifest's URLs live 15 minutes and
74
+ ``iter_range`` surveys its whole window up front, so any walk that
75
+ takes longer than that reaches days whose URLs died while earlier
76
+ days were being processed (measured 2026-11-11: ES trades 2010-2026
77
+ died with a 403 at day 600, 18 minutes in). An ``HTTPError``
78
+ subclass so callers that caught the old ``raise_for_status``
79
+ failure still do.
80
+ """
81
+
66
82
  #: Widest window of missing days :meth:`Timeseries.iter_range` fetches
67
83
  #: in one request, once it has ramped up to it.
68
84
  #:
@@ -83,6 +99,39 @@ STORAGE_SPOOL_BYTES = 32 * 1024 * 1024
83
99
  #: all -- it has no gaps, so it stays the single manifest it always was.
84
100
  HYDRATE_CHUNK_DAYS = 5
85
101
 
102
+ #: How many of those runs hydrate at once. One at a time, each waited for
103
+ #: the one before it, so a cold window cost the SUM of its fetches: measured
104
+ #: 2026-10-28 on production, a cold 12-day GF.v.0 window took 99.0s one at a
105
+ #: time and 33.6s four at a time, with identical rows. Hydration returns a
106
+ #: manifest, not rows, so running ahead costs this process nothing; frames
107
+ #: are still read and yielded one at a time, in order. Kept identical to
108
+ #: the sandbox client's ``V2_HYDRATE_PARALLEL``.
109
+ HYDRATE_PARALLEL = 4
110
+
111
+
112
+ def _in_background(fn, *args) -> "concurrent.futures.Future":
113
+ """Run ``fn(*args)`` on a DAEMON thread and return its future.
114
+
115
+ Not a ``ThreadPoolExecutor``: its workers are joined at interpreter
116
+ exit, and a hydration already in flight cannot be cancelled, so a
117
+ script that stopped iterating a cold window early would hang on exit
118
+ until those requests returned -- up to the data timeout. A daemon
119
+ thread is abandoned at exit instead. Concurrency stays bounded by the
120
+ caller, which keeps at most ``HYDRATE_PARALLEL`` unconsumed.
121
+ """
122
+ future: concurrent.futures.Future = concurrent.futures.Future()
123
+
124
+ def run() -> None:
125
+ if not future.set_running_or_notify_cancel():
126
+ return
127
+ try:
128
+ future.set_result(fn(*args))
129
+ except BaseException as exc: # noqa: BLE001 - handed to the consumer
130
+ future.set_exception(exc)
131
+
132
+ threading.Thread(target=run, name="qpd-hydrate", daemon=True).start()
133
+ return future
134
+
86
135
 
87
136
  def _symbols(value: str | Sequence[str]) -> list[str]:
88
137
  return [value] if isinstance(value, str) else [str(v) for v in value]
@@ -341,7 +390,7 @@ class Timeseries(_Namespace):
341
390
  # service too old to know the field, which ignores unknown keys.
342
391
  budget = (
343
392
  max_decoded_bytes if max_decoded_bytes is not None
344
- else memory.declared_budget(mode)
393
+ else memory.declared_budget(mode, schema)
345
394
  )
346
395
  if budget is not None:
347
396
  # A MEASURED ZERO is not the same as "could not measure", and
@@ -449,9 +498,11 @@ class Timeseries(_Namespace):
449
498
  # day, instrument, projection), so the whole-window body is the right
450
499
  # thing to hand every piece.
451
500
  manifest_body = _manifest_body(columns=columns, **kwargs)
501
+ resigner = _Resigner(self, manifest_body)
452
502
  for payload, item in self._partitions(manifest_body):
453
503
  frame = _labelled(
454
- _partition(manifest_body, item, columns), item, columns
504
+ _partition(manifest_body, item, columns, resigner),
505
+ item, columns,
455
506
  )
456
507
  frame.attrs["quantpad"] = {
457
508
  **_manifest_attrs(payload),
@@ -643,20 +694,43 @@ class Timeseries(_Namespace):
643
694
  # One cursor over the union of both sets, so a fetched day lands
644
695
  # between the cached days that surround it.
645
696
  fetched: dict[str, list[tuple[dict, dict]]] = {}
646
- pending = list(_runs(sorted(missing), HYDRATE_CHUNK_DAYS))
647
- for day in sorted(set(ready) | missing):
648
- while pending and pending[0][0] <= day:
649
- start, end = pending.pop(0)
650
- for payload in self._manifests(
651
- {**manifest_body, "start": start, "end": end}
652
- ):
653
- for item in payload["files"]:
654
- fetched.setdefault(item["day"], []).append((payload, item))
655
- yield from fresh(ready.get(day, ()))
656
- # A missing day that came back with nothing -- a holiday, or
657
- # a genuinely quiet session -- simply has no entry here, and
658
- # is skipped exactly as an ``empty`` day always was.
659
- yield from fresh(fetched.pop(day, ()))
697
+ pending = collections.deque(_runs(sorted(missing), HYDRATE_CHUNK_DAYS))
698
+
699
+ def hydrate(start: str, end: str) -> list[dict]:
700
+ return list(self._manifests({**manifest_body, "start": start, "end": end}))
701
+
702
+ # Runs hydrate up to HYDRATE_PARALLEL at a time, ahead of the
703
+ # cursor, and are CONSUMED in order: a run's files join ``fetched``
704
+ # only when the cursor reaches its first day, exactly as when they
705
+ # were fetched there, so the order frames come out in is unchanged.
706
+ inflight: collections.deque = collections.deque()
707
+
708
+ def top_up() -> None:
709
+ while pending and len(inflight) < HYDRATE_PARALLEL:
710
+ start, end = pending.popleft()
711
+ inflight.append((start, _in_background(hydrate, start, end)))
712
+
713
+ try:
714
+ top_up()
715
+ for day in sorted(set(ready) | missing):
716
+ while inflight and inflight[0][0] <= day:
717
+ _, future = inflight.popleft()
718
+ # Re-raises the run's own error here, at the day it
719
+ # covers, which is where the sequential loop raised it.
720
+ for payload in future.result():
721
+ for item in payload["files"]:
722
+ fetched.setdefault(item["day"], []).append((payload, item))
723
+ top_up()
724
+ yield from fresh(ready.get(day, ()))
725
+ # A missing day that came back with nothing -- a holiday, or
726
+ # a genuinely quiet session -- simply has no entry here, and
727
+ # is skipped exactly as an ``empty`` day always was.
728
+ yield from fresh(fetched.pop(day, ()))
729
+ finally:
730
+ # A caller that stops iterating early: runs already started finish
731
+ # on their daemon threads and are dropped; none is started after.
732
+ for _, future in inflight:
733
+ future.cancel()
660
734
 
661
735
  # Anything a fetch returned for a day the survey did not list.
662
736
  # It should be empty, and it is drained rather than asserted
@@ -730,6 +804,9 @@ class Timeseries(_Namespace):
730
804
  columns = body.get("columns")
731
805
  frames = []
732
806
  served: list[dict] = []
807
+ # Bulk downloads its partitions one after another too, so a large
808
+ # enough window outlives its URLs exactly as iter_range does.
809
+ resigner = _Resigner(self, manifest_body)
733
810
  for payload in self._manifests(manifest_body):
734
811
  served.append(payload)
735
812
  frames.extend(
@@ -739,7 +816,10 @@ class Timeseries(_Namespace):
739
816
  # for a later bulk read of the same days and the other way
740
817
  # round. Reading the urls directly here would make the cache
741
818
  # depend on which delivery mode happened to fetch first.
742
- _labelled(_partition(manifest_body, item, columns), item, columns)
819
+ _labelled(
820
+ _partition(manifest_body, item, columns, resigner),
821
+ item, columns,
822
+ )
743
823
  for item in payload["files"]
744
824
  )
745
825
  if not frames:
@@ -979,6 +1059,14 @@ class Options(_Namespace):
979
1059
  instrument_class: str | None = None,
980
1060
  limit: int | None = None,
981
1061
  ) -> pd.DataFrame:
1062
+ """Outright calls and puts for ``underlier`` as of ``date``.
1063
+
1064
+ ``underlier`` is the parent (``SPY.OPT``, ``NQ.OPT``); a futures
1065
+ option handle such as ``CL.FUT.OPT`` is accepted and sent as the
1066
+ parent upstream knows (``LO.OPT``). Spreads are left out unless
1067
+ their class is asked for (``instrument_class="T"``), and
1068
+ ``expiration`` is a ``YYYY-MM-DD`` UTC date.
1069
+ """
982
1070
  params = {
983
1071
  "dataset": dataset,
984
1072
  "underlier": underlier,
@@ -1296,6 +1384,12 @@ def _read_parquet_url(url: str, columns: Sequence[str] | None) -> pd.DataFrame:
1296
1384
  with requests.get(
1297
1385
  url, timeout=STORAGE_TIMEOUT, stream=True
1298
1386
  ) as response:
1387
+ if response.status_code in (401, 403):
1388
+ raise _StorageUrlRejected(
1389
+ f"object storage refused a partition URL "
1390
+ f"(HTTP {response.status_code})",
1391
+ response=response,
1392
+ )
1299
1393
  response.raise_for_status()
1300
1394
  # Spooled rather than ``io.BytesIO(response.content)`` so the
1301
1395
  # compressed bytes and the decoded table are not both resident
@@ -1315,7 +1409,86 @@ def _read_parquet_url(url: str, columns: Sequence[str] | None) -> pd.DataFrame:
1315
1409
  return table.to_pandas()
1316
1410
 
1317
1411
 
1318
- def _partition(body: dict, item: dict, columns) -> pd.DataFrame:
1412
+ class _Resigner:
1413
+ """Fresh URLs for a manifest walk that outlives its signatures.
1414
+
1415
+ A manifest's URLs expire 15 minutes after it is issued and a walk
1416
+ over years of ticks takes hours, so the URLs handed out at the start
1417
+ are dead long before the cursor reaches them. Re-signing is lazy,
1418
+ on the first refusal: one survey (``hydrate=False``, so nothing is
1419
+ fetched upstream) of the window from the refused day to the end
1420
+ re-signs every partition still ahead, and the walk carries on -- one
1421
+ refusal and one survey per 15 minutes of reading, not a manifest per
1422
+ day or a timer guessing at the expiry.
1423
+
1424
+ Looked up by :func:`_identity` (the object key), which is what stays
1425
+ the same between two manifests; the URL never does. A day the
1426
+ survey no longer lists (evicted between the two manifests) is asked
1427
+ for on its own WITH hydration, as it would have been had it been a
1428
+ gap from the start. Bounded per partition: a URL signed a moment
1429
+ ago that is refused too is not expiry, so it raises rather than
1430
+ looping over a real auth or storage failure.
1431
+ """
1432
+
1433
+ def __init__(self, timeseries: "Timeseries", body: dict):
1434
+ self._timeseries = timeseries
1435
+ self._body = body
1436
+ self._fresh: dict[tuple, str] = {}
1437
+
1438
+ def read(self, item: dict, columns) -> pd.DataFrame:
1439
+ try:
1440
+ return _read_parquet_url(
1441
+ self._fresh.get(_identity(item), item["url"]), columns
1442
+ )
1443
+ except _StorageUrlRejected:
1444
+ pass
1445
+ url = self._resign(item)
1446
+ try:
1447
+ return _read_parquet_url(url, columns)
1448
+ except _StorageUrlRejected as exc:
1449
+ raise _StorageUrlRejected(
1450
+ f"object storage refused a freshly signed URL for "
1451
+ f"{item['day']} (instrument {item['instrument_id']}), so "
1452
+ "this is not expiry. Retry shortly; if it persists, "
1453
+ "contact QuantPad support.",
1454
+ response=exc.response,
1455
+ ) from exc
1456
+
1457
+ @staticmethod
1458
+ def _urls(payloads, into: dict) -> dict:
1459
+ for payload in payloads:
1460
+ for entry in payload.get("files") or ():
1461
+ into[_identity(entry)] = entry["url"]
1462
+ return into
1463
+
1464
+ def _resign(self, item: dict) -> str:
1465
+ identity = _identity(item)
1466
+ day = str(item["day"])[:10]
1467
+ # Judged against THIS survey only, then swapped in whole: the survey
1468
+ # covers every day still ahead, and merging into an earlier wave's
1469
+ # URLs would make a day this survey no longer lists look present,
1470
+ # skip the single-day hydrate, and retry a dead URL.
1471
+ latest = self._urls(self._timeseries._manifests(
1472
+ {**self._body, "start": day}, hydrate=False
1473
+ ), {})
1474
+ if identity not in latest:
1475
+ next_day = (date.fromisoformat(day) + timedelta(days=1)).isoformat()
1476
+ self._urls(self._timeseries._manifests(
1477
+ {**self._body, "start": day, "end": next_day}
1478
+ ), latest)
1479
+ self._fresh = latest
1480
+ if identity not in self._fresh:
1481
+ raise _StorageUrlRejected(
1482
+ f"partition URL for {item['day']} expired and the service "
1483
+ "no longer lists that partition, so it could not be "
1484
+ "re-signed. Re-run from this day."
1485
+ )
1486
+ return self._fresh[identity]
1487
+
1488
+
1489
+ def _partition(
1490
+ body: dict, item: dict, columns, resigner: "_Resigner | None" = None
1491
+ ) -> pd.DataFrame:
1319
1492
  """One manifest partition, from local disk if it is already there.
1320
1493
 
1321
1494
  The unit ``iter_range`` already deals in -- one file per instrument-day --
@@ -1353,7 +1526,10 @@ def _partition(body: dict, item: dict, columns) -> pd.DataFrame:
1353
1526
  if hit is not None:
1354
1527
  return hit
1355
1528
 
1356
- frame = _read_parquet_url(item["url"], columns)
1529
+ if resigner is None:
1530
+ frame = _read_parquet_url(item["url"], columns)
1531
+ else:
1532
+ frame = resigner.read(item, columns)
1357
1533
  if path is not None:
1358
1534
  cache.put_frame(
1359
1535
  path, frame,
@@ -24,3 +24,14 @@ def isolated_cache(tmp_path, monkeypatch):
24
24
  monkeypatch.delenv("QPD_CACHE", raising=False)
25
25
  monkeypatch.delenv("QPD_CACHE_MAX_BYTES", raising=False)
26
26
  return tmp_path / "qpd-cache"
27
+
28
+
29
+ @pytest.fixture(autouse=True)
30
+ def one_hydration_at_a_time(monkeypatch):
31
+ """The fake sessions replay responses in CALL order, and parallel
32
+ hydrations would race for them, so tests hydrate one run at a time.
33
+ The parallel path has its own tests, with a session keyed on the window
34
+ each request asks for (``test_v2.WindowSession``)."""
35
+ from quantpad_data import v2
36
+
37
+ monkeypatch.setattr(v2, "HYDRATE_PARALLEL", 1)
@@ -10,6 +10,8 @@ month of depth data is a normal thing to do.
10
10
 
11
11
  from __future__ import annotations
12
12
 
13
+ import threading
14
+
13
15
  import io
14
16
  import json
15
17
  from datetime import date, timedelta
@@ -473,11 +475,20 @@ class TestBulkDelivery:
473
475
  dataset="GLBX.MDP3", symbols="ESZ6", schema="mbp-10",
474
476
  start="2026-01-05", end="2026-01-07",
475
477
  )
478
+ # The fetch starts in the background straight away now, so it is
479
+ # held open: the cached frame must arrive without waiting for it.
480
+ gate = threading.Event()
481
+ replay = session.request
482
+
483
+ def held(method, url, **kwargs):
484
+ if session.calls: # every call after the survey is the fetch
485
+ assert gate.wait(5), "the cached frame never arrived"
486
+ return replay(method, url, **kwargs)
487
+
488
+ session.request = held
476
489
  first = next(frames)
477
490
  assert first.attrs["quantpad"]["day"] == "2026-01-05"
478
- # One call so far: the cached day was served without the fetch
479
- # for the missing one having been made.
480
- assert len(session.calls) == 1
491
+ gate.set()
481
492
  rest = list(frames)
482
493
  assert [f.attrs["quantpad"]["day"] for f in rest] == ["2026-01-06"]
483
494
  assert len(session.calls) == 2
@@ -1471,7 +1482,21 @@ class TestTheCallerDeclaresWhatItCanHold:
1471
1482
  )
1472
1483
 
1473
1484
  assert self._sent(session)["max_decoded_bytes"] == int(
1474
- self.FREE / memory.PEAK_PER_PRICED_BYTE
1485
+ self.FREE / memory.peak_per_priced_byte("mbp-10")
1486
+ )
1487
+
1488
+ def test_the_divisor_is_the_schemas_measured_factor(self):
1489
+ """6x refused single quote-record days that peak at ~3.2x."""
1490
+ for schema, factor in [("mbp-10", 3.0), ("mbp-1", 4.5), ("trades", 6.0),
1491
+ ("definition", 6.0), (None, 6.0)]:
1492
+ assert memory.declared_budget("auto", schema) == int(self.FREE / factor)
1493
+ assert memory.declared_budget("interactive", "mbp-1") == self.FREE
1494
+
1495
+ def test_every_measured_factor_stays_above_its_reading(self):
1496
+ for schema, reading in {"mbp-10": 2.57, "mbp-1": 4.32}.items():
1497
+ assert memory.peak_per_priced_byte(schema) > reading
1498
+ assert all(
1499
+ f <= memory.PEAK_PER_PRICED_BYTE for f in memory.PEAK_BY_SCHEMA.values()
1475
1500
  )
1476
1501
 
1477
1502
  def test_interactive_declares_the_undivided_figure(self):
@@ -2297,4 +2322,296 @@ class TestDerivations:
2297
2322
  stype_in="continuous", start="2026-01-05", end="2026-02-05",
2298
2323
  resample="4h",
2299
2324
  ))
2300
- assert session.calls == []
2325
+ assert session.calls == []
2326
+
2327
+
2328
+ class WindowSession(FakeSession):
2329
+ """Answers each manifest by the window it asks for, not by call order,
2330
+ because parallel hydrations reach it in whatever order their threads
2331
+ run. The first ``parties`` fetches wait at a barrier until all of them
2332
+ are in flight -- the proof that they overlap."""
2333
+
2334
+ def __init__(self, survey, parties=None):
2335
+ super().__init__([])
2336
+ self.survey = survey
2337
+ self.lock = threading.Lock()
2338
+ self.barrier = threading.Barrier(parties) if parties else None
2339
+ self.fetches = []
2340
+
2341
+ def request(self, method, url, **kwargs):
2342
+ body = kwargs.get("json") or {}
2343
+ with self.lock:
2344
+ self.calls.append((method, url, kwargs))
2345
+ if body.get("hydrate") is False:
2346
+ return self.survey
2347
+ self.fetches.append((body["start"], body["end"]))
2348
+ first_wave = len(self.fetches) <= (self.barrier.parties if self.barrier else 0)
2349
+ if first_wave:
2350
+ self.barrier.wait(5)
2351
+ start, end = date.fromisoformat(body["start"]), date.fromisoformat(body["end"])
2352
+ days = [(start + timedelta(days=i)).isoformat() for i in range((end - start).days)]
2353
+ return json_response({
2354
+ "format": "parquet",
2355
+ "files": [{"day": d, "instrument_id": 1, "key": d, "size_bytes": 1,
2356
+ "url": f"https://r2.test/{d}"} for d in days],
2357
+ "gaps": [],
2358
+ "metadata": {},
2359
+ })
2360
+
2361
+
2362
+ class TestParallelHydration:
2363
+ """Sequential hydration made a cold window cost the SUM of its fetches
2364
+ (99.0s against 33.6s four at a time, measured on production)."""
2365
+
2366
+ GAPS = [f"2026-01-{day:02d}" for day in range(5, 13)] # runs of 1, 2, 4, 1
2367
+
2368
+ def _iter(self, monkeypatch, session, end="2026-01-13"):
2369
+ from quantpad_data import v2
2370
+
2371
+ client = Client(api_key="k", base_url="https://api.test", session=session)
2372
+ monkeypatch.setattr(
2373
+ v2, "_read_parquet_url", lambda url, columns: pd.DataFrame({"url": [url]})
2374
+ )
2375
+ return Historical(client).timeseries.iter_range(
2376
+ dataset="GLBX.MDP3", symbols="ESZ6", schema="mbp-10",
2377
+ start="2026-01-05", end=end,
2378
+ )
2379
+
2380
+ def _survey(self, gaps, cached=()):
2381
+ return json_response({
2382
+ "format": "parquet",
2383
+ "files": [{"day": d, "instrument_id": 1, "key": d, "size_bytes": 1,
2384
+ "url": f"https://r2.test/{d}"} for d in cached],
2385
+ "gaps": [{"day": d, "instrument_id": 1} for d in gaps],
2386
+ "metadata": {},
2387
+ })
2388
+
2389
+ def test_cold_runs_hydrate_at_the_same_time(self, monkeypatch):
2390
+ from quantpad_data import v2
2391
+
2392
+ monkeypatch.setattr(v2, "HYDRATE_PARALLEL", 3)
2393
+ session = WindowSession(self._survey(self.GAPS), parties=3)
2394
+
2395
+ days = [f.attrs["quantpad"]["day"] for f in self._iter(monkeypatch, session)]
2396
+
2397
+ assert days == self.GAPS
2398
+ assert len(session.fetches) == 4
2399
+
2400
+ def test_the_window_stays_in_order_with_cached_days_in_place(self, monkeypatch):
2401
+ from quantpad_data import v2
2402
+
2403
+ monkeypatch.setattr(v2, "HYDRATE_PARALLEL", 4)
2404
+ gaps = ["2026-01-05", "2026-01-07", "2026-01-08", "2026-01-10"]
2405
+ cached = ["2026-01-06", "2026-01-09"]
2406
+ session = WindowSession(self._survey(gaps, cached))
2407
+
2408
+ days = [f.attrs["quantpad"]["day"]
2409
+ for f in self._iter(monkeypatch, session, end="2026-01-11")]
2410
+
2411
+ assert days == sorted(gaps + cached)
2412
+
2413
+ def test_an_early_close_leaves_nothing_that_blocks_exit(self, monkeypatch):
2414
+ """A hydration in flight cannot be cancelled, and pool workers are
2415
+ joined at interpreter exit -- so hydrations run on daemon threads,
2416
+ which a script that stops iterating early can exit past."""
2417
+ from quantpad_data import v2
2418
+
2419
+ monkeypatch.setattr(v2, "HYDRATE_PARALLEL", 2)
2420
+ gate, started = threading.Event(), threading.Event()
2421
+ session = WindowSession(self._survey(self.GAPS))
2422
+ replay = session.request
2423
+
2424
+ def hanging(method, url, **kwargs):
2425
+ body = kwargs.get("json") or {}
2426
+ if body.get("hydrate") is not False and body["start"] != "2026-01-05":
2427
+ started.set()
2428
+ gate.wait(10)
2429
+ return replay(method, url, **kwargs)
2430
+
2431
+ session.request = hanging
2432
+ frames = self._iter(monkeypatch, session)
2433
+ assert next(frames).attrs["quantpad"]["day"] == "2026-01-05"
2434
+ assert started.wait(5)
2435
+ frames.close()
2436
+ hydrating = [t for t in threading.enumerate() if t.name == "qpd-hydrate"]
2437
+ assert hydrating and all(t.daemon for t in hydrating)
2438
+ gate.set()
2439
+
2440
+ def test_stopping_early_does_not_wait_on_queued_hydrations(self, monkeypatch):
2441
+ from quantpad_data import v2
2442
+
2443
+ monkeypatch.setattr(v2, "HYDRATE_PARALLEL", 2)
2444
+ frames = self._iter(monkeypatch, WindowSession(self._survey(self.GAPS)))
2445
+
2446
+ assert next(frames).attrs["quantpad"]["day"] == "2026-01-05"
2447
+ frames.close()
2448
+
2449
+
2450
+ class TestExpiredUrlsAreResigned:
2451
+ """A manifest's presigned URLs expire 15 minutes after it is issued, and
2452
+ iter_range surveys its whole window up front, so on a long walk the
2453
+ URLs for later days are dead by the time the cursor reaches them
2454
+ (measured 2026-11-11: ES trades 2010-2026 died with a 403 at day 600,
2455
+ 18 minutes in). A refusal re-signs the rest of the window."""
2456
+
2457
+ @staticmethod
2458
+ def _signed(days, sig, gaps=()):
2459
+ return {
2460
+ "format": "parquet",
2461
+ "files": [
2462
+ {"day": f"2026-01-{d:02d}", "instrument_id": 42,
2463
+ "key": f"GLBX/trades/42/2026-01-{d:02d}.parquet",
2464
+ "size_bytes": 10,
2465
+ "url": f"https://r2.test/{d:02d}.parquet?sig={sig}"}
2466
+ for d in days
2467
+ ],
2468
+ "gaps": [{"day": f"2026-01-{d:02d}"} for d in gaps],
2469
+ "metadata": {"dataset": "GLBX.MDP3"},
2470
+ }
2471
+
2472
+ @staticmethod
2473
+ def _storage(monkeypatch, dead, alive_days=()):
2474
+ from quantpad_data import v2 as v2_module
2475
+
2476
+ reads = []
2477
+
2478
+ def reader(url, columns):
2479
+ reads.append(url)
2480
+ if not any(f"/{d:02d}.parquet" in url for d in alive_days) and any(
2481
+ f"sig={sig}" in url for sig in dead
2482
+ ):
2483
+ raise v2_module._StorageUrlRejected("refused (HTTP 403)")
2484
+ return pd.DataFrame({"url": [url]})
2485
+
2486
+ monkeypatch.setattr("quantpad_data.v2._read_parquet_url", reader)
2487
+ return reads
2488
+
2489
+ @staticmethod
2490
+ def _walk(api):
2491
+ return list(api.timeseries.iter_range(
2492
+ dataset="GLBX.MDP3", symbols="ES.v.0", schema="trades",
2493
+ stype_in="continuous", start="2026-01-05", end="2026-01-09",
2494
+ ))
2495
+
2496
+ def test_an_expired_url_is_resigned_and_the_walk_carries_on(
2497
+ self, monkeypatch
2498
+ ):
2499
+ api, session = client_with([
2500
+ json_response(self._signed([5, 6, 7, 8], "old")),
2501
+ json_response(self._signed([6, 7, 8], "new")),
2502
+ ])
2503
+ reads = self._storage(monkeypatch, dead=["old"], alive_days=[5])
2504
+
2505
+ frames = self._walk(api)
2506
+
2507
+ assert [f.attrs["quantpad"]["day"] for f in frames] == [
2508
+ "2026-01-05", "2026-01-06", "2026-01-07", "2026-01-08",
2509
+ ]
2510
+ assert all("sig=new" in f["url"][0] for f in frames[1:])
2511
+ # One survey, ONE re-sign for everything ahead -- not one per day.
2512
+ assert len(session.calls) == 2
2513
+ resign = session.calls[1][2]["json"]
2514
+ assert (resign["start"], resign["end"]) == ("2026-01-06", "2026-01-09")
2515
+ assert resign["hydrate"] is False
2516
+ assert [u for u in reads if "sig=old" in u and "05" not in u] == [
2517
+ "https://r2.test/06.parquet?sig=old",
2518
+ ]
2519
+
2520
+ def test_a_freshly_signed_url_refused_too_is_not_mistaken_for_expiry(
2521
+ self, monkeypatch
2522
+ ):
2523
+ api, session = client_with([
2524
+ json_response(self._signed([5, 6], "old")),
2525
+ json_response(self._signed([5, 6], "new")),
2526
+ ])
2527
+ self._storage(monkeypatch, dead=["old", "new"])
2528
+
2529
+ with pytest.raises(requests.HTTPError, match="not expiry"):
2530
+ self._walk(api)
2531
+ assert len(session.calls) == 2
2532
+
2533
+ def test_a_day_the_resurvey_no_longer_lists_is_fetched_on_its_own(
2534
+ self, monkeypatch
2535
+ ):
2536
+ api, session = client_with([
2537
+ json_response(self._signed([5, 6], "old")),
2538
+ json_response(self._signed([6], "new", gaps=[5])),
2539
+ json_response(self._signed([5], "hydrated")),
2540
+ ])
2541
+ self._storage(monkeypatch, dead=["old"])
2542
+
2543
+ frames = self._walk(api)
2544
+
2545
+ assert "sig=hydrated" in frames[0]["url"][0]
2546
+ assert "sig=new" in frames[1]["url"][0]
2547
+ single = session.calls[2][2]["json"]
2548
+ assert (single["start"], single["end"]) == ("2026-01-05", "2026-01-06")
2549
+ assert single["hydrate"] is True
2550
+
2551
+ def test_a_bulk_read_resigns_too(self, monkeypatch):
2552
+ api, _ = client_with([
2553
+ json_response(self._signed([5, 6], "old")),
2554
+ json_response(self._signed([5, 6], "new")),
2555
+ ])
2556
+ self._storage(monkeypatch, dead=["old"])
2557
+
2558
+ frame = api.timeseries.get_range(
2559
+ dataset="GLBX.MDP3", symbols="ES.v.0", schema="trades",
2560
+ stype_in="continuous", start="2026-01-05", end="2026-01-07",
2561
+ mode="bulk",
2562
+ )
2563
+
2564
+ assert list(frame["url"]) == [
2565
+ "https://r2.test/05.parquet?sig=new",
2566
+ "https://r2.test/06.parquet?sig=new",
2567
+ ]
2568
+
2569
+ def test_a_refused_storage_read_is_classified_as_rejected(
2570
+ self, monkeypatch
2571
+ ):
2572
+ from quantpad_data import v2 as v2_module
2573
+
2574
+ for status in (401, 403):
2575
+ spy = _StreamSpy(status, b"expired")
2576
+ monkeypatch.setattr(requests, "get", lambda url, **kw: spy)
2577
+ with pytest.raises(v2_module._StorageUrlRejected):
2578
+ v2_module._read_parquet_url("https://r2.test/d.parquet", None)
2579
+ assert spy.closed
2580
+
2581
+ def test_any_other_storage_failure_is_not_resigned(self, monkeypatch):
2582
+ api, session = client_with([json_response(self._signed([5], "old"))])
2583
+ spy = _StreamSpy(500, b"boom")
2584
+ monkeypatch.setattr(requests, "get", lambda url, **kw: spy)
2585
+
2586
+ with pytest.raises(requests.HTTPError, match="500"):
2587
+ self._walk(api)
2588
+ assert len(session.calls) == 1
2589
+
2590
+ def test_a_day_evicted_before_a_later_resign_is_still_hydrated(
2591
+ self, monkeypatch
2592
+ ):
2593
+ """Two re-sign waves: the first lists day 7, the second no longer
2594
+ does. Day 7 is hydrated alone, not retried on the first wave's URL."""
2595
+ from quantpad_data import v2 as v2_module
2596
+
2597
+ api, session = client_with([
2598
+ json_response(self._signed([5, 6, 7], "old")),
2599
+ json_response(self._signed([6, 7], "mid")),
2600
+ json_response(self._signed([], "late", gaps=[7])),
2601
+ json_response(self._signed([7], "hydrated")),
2602
+ ])
2603
+
2604
+ def reader(url, columns):
2605
+ if "sig=old" in url and "/05." not in url:
2606
+ raise v2_module._StorageUrlRejected("refused (HTTP 403)")
2607
+ if "/07." in url and "sig=mid" in url:
2608
+ raise v2_module._StorageUrlRejected("refused (HTTP 403)")
2609
+ return pd.DataFrame({"url": [url]})
2610
+
2611
+ monkeypatch.setattr("quantpad_data.v2._read_parquet_url", reader)
2612
+
2613
+ frames = self._walk(api)
2614
+
2615
+ assert "sig=hydrated" in frames[2]["url"][0]
2616
+ single = session.calls[3][2]["json"]
2617
+ assert (single["start"], single["hydrate"]) == ("2026-01-07", True)
File without changes