quantpad-data 0.8.4__tar.gz → 0.8.5__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (24) hide show
  1. {quantpad_data-0.8.4 → quantpad_data-0.8.5}/PKG-INFO +2 -2
  2. {quantpad_data-0.8.4 → quantpad_data-0.8.5}/README.md +1 -1
  3. {quantpad_data-0.8.4 → quantpad_data-0.8.5}/pyproject.toml +1 -1
  4. {quantpad_data-0.8.4 → quantpad_data-0.8.5}/quantpad_data/client.py +1 -1
  5. {quantpad_data-0.8.4 → quantpad_data-0.8.5}/quantpad_data/v2.py +113 -4
  6. {quantpad_data-0.8.4 → quantpad_data-0.8.5}/tests/test_v2.py +170 -0
  7. {quantpad_data-0.8.4 → quantpad_data-0.8.5}/.gitignore +0 -0
  8. {quantpad_data-0.8.4 → quantpad_data-0.8.5}/quantpad_data/__init__.py +0 -0
  9. {quantpad_data-0.8.4 → quantpad_data-0.8.5}/quantpad_data/cache.py +0 -0
  10. {quantpad_data-0.8.4 → quantpad_data-0.8.5}/quantpad_data/errors.py +0 -0
  11. {quantpad_data-0.8.4 → quantpad_data-0.8.5}/quantpad_data/lean.py +0 -0
  12. {quantpad_data-0.8.4 → quantpad_data-0.8.5}/quantpad_data/memory.py +0 -0
  13. {quantpad_data-0.8.4 → quantpad_data-0.8.5}/quantpad_data/py.typed +0 -0
  14. {quantpad_data-0.8.4 → quantpad_data-0.8.5}/quantpad_data/rth.py +0 -0
  15. {quantpad_data-0.8.4 → quantpad_data-0.8.5}/tests/conftest.py +0 -0
  16. {quantpad_data-0.8.4 → quantpad_data-0.8.5}/tests/test_client.py +0 -0
  17. {quantpad_data-0.8.4 → quantpad_data-0.8.5}/tests/test_lean.py +0 -0
  18. {quantpad_data-0.8.4 → quantpad_data-0.8.5}/tests/test_message_encoding.py +0 -0
  19. {quantpad_data-0.8.4 → quantpad_data-0.8.5}/tests/test_packaging.py +0 -0
  20. {quantpad_data-0.8.4 → quantpad_data-0.8.5}/tests/test_refusal_hint.py +0 -0
  21. {quantpad_data-0.8.4 → quantpad_data-0.8.5}/tests/test_rth.py +0 -0
  22. {quantpad_data-0.8.4 → quantpad_data-0.8.5}/tests/test_v2_cache.py +0 -0
  23. {quantpad_data-0.8.4 → quantpad_data-0.8.5}/tests/test_v2_gate.py +0 -0
  24. {quantpad_data-0.8.4 → quantpad_data-0.8.5}/tests/test_version.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.5
2
2
  Name: quantpad-data
3
- Version: 0.8.4
3
+ Version: 0.8.5
4
4
  Summary: Official Python client for the QuantPad market-data API
5
5
  Project-URL: Documentation, https://api.quantpad.ai/external/docs
6
6
  Author: QuantPad
@@ -170,7 +170,7 @@ the cache so ordinary `get_range` calls serve it from storage:
170
170
 
171
171
  ```python
172
172
  job = client.batch.submit(
173
- dataset="GLBX.MDP3", symbols="ESZ6", schema="mbo",
173
+ dataset="GLBX.MDP3", symbols="ESZ6", schema="mbp-10",
174
174
  start="2026-01-05", end="2026-02-05", hydrate=True,
175
175
  )
176
176
  client.batch.hydrate(job["job_id"])
@@ -148,7 +148,7 @@ the cache so ordinary `get_range` calls serve it from storage:
148
148
 
149
149
  ```python
150
150
  job = client.batch.submit(
151
- dataset="GLBX.MDP3", symbols="ESZ6", schema="mbo",
151
+ dataset="GLBX.MDP3", symbols="ESZ6", schema="mbp-10",
152
152
  start="2026-01-05", end="2026-02-05", hydrate=True,
153
153
  )
154
154
  client.batch.hydrate(job["job_id"])
@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
4
4
 
5
5
  [project]
6
6
  name = "quantpad-data"
7
- version = "0.8.4"
7
+ version = "0.8.5"
8
8
  description = "Official Python client for the QuantPad market-data API"
9
9
  readme = "README.md"
10
10
  requires-python = ">=3.10"
@@ -87,7 +87,7 @@ V2_NOT_AVAILABLE = (
87
87
  "QuantPad support about /v2 access."
88
88
  )
89
89
 
90
- VERSION = "0.8.4"
90
+ VERSION = "0.8.5"
91
91
  USER_AGENT = f"quantpad-data-python/{VERSION}"
92
92
  TICK_SCHEMAS = (
93
93
  "trades", "mbp-1", "mbp-10", "cmbp-1", "tcbbo", "cbbo-1s",
@@ -66,6 +66,19 @@ STORAGE_TIMEOUT = (10, 120)
66
66
  #: decoded table at full size at once is worth avoiding.
67
67
  STORAGE_SPOOL_BYTES = 32 * 1024 * 1024
68
68
 
69
+
70
+ class _StorageUrlRejected(requests.HTTPError):
71
+ """Object storage refused a presigned partition URL (401/403).
72
+
73
+ Almost always expiry: a manifest's URLs live 15 minutes and
74
+ ``iter_range`` surveys its whole window up front, so any walk that
75
+ takes longer than that reaches days whose URLs died while earlier
76
+ days were being processed (measured 2026-11-11: ES trades 2010-2026
77
+ died with a 403 at day 600, 18 minutes in). An ``HTTPError``
78
+ subclass so callers that caught the old ``raise_for_status``
79
+ failure still do.
80
+ """
81
+
69
82
  #: Widest window of missing days :meth:`Timeseries.iter_range` fetches
70
83
  #: in one request, once it has ramped up to it.
71
84
  #:
@@ -485,9 +498,11 @@ class Timeseries(_Namespace):
485
498
  # day, instrument, projection), so the whole-window body is the right
486
499
  # thing to hand every piece.
487
500
  manifest_body = _manifest_body(columns=columns, **kwargs)
501
+ resigner = _Resigner(self, manifest_body)
488
502
  for payload, item in self._partitions(manifest_body):
489
503
  frame = _labelled(
490
- _partition(manifest_body, item, columns), item, columns
504
+ _partition(manifest_body, item, columns, resigner),
505
+ item, columns,
491
506
  )
492
507
  frame.attrs["quantpad"] = {
493
508
  **_manifest_attrs(payload),
@@ -789,6 +804,9 @@ class Timeseries(_Namespace):
789
804
  columns = body.get("columns")
790
805
  frames = []
791
806
  served: list[dict] = []
807
+ # Bulk downloads its partitions one after another too, so a large
808
+ # enough window outlives its URLs exactly as iter_range does.
809
+ resigner = _Resigner(self, manifest_body)
792
810
  for payload in self._manifests(manifest_body):
793
811
  served.append(payload)
794
812
  frames.extend(
@@ -798,7 +816,10 @@ class Timeseries(_Namespace):
798
816
  # for a later bulk read of the same days and the other way
799
817
  # round. Reading the urls directly here would make the cache
800
818
  # depend on which delivery mode happened to fetch first.
801
- _labelled(_partition(manifest_body, item, columns), item, columns)
819
+ _labelled(
820
+ _partition(manifest_body, item, columns, resigner),
821
+ item, columns,
822
+ )
802
823
  for item in payload["files"]
803
824
  )
804
825
  if not frames:
@@ -1363,6 +1384,12 @@ def _read_parquet_url(url: str, columns: Sequence[str] | None) -> pd.DataFrame:
1363
1384
  with requests.get(
1364
1385
  url, timeout=STORAGE_TIMEOUT, stream=True
1365
1386
  ) as response:
1387
+ if response.status_code in (401, 403):
1388
+ raise _StorageUrlRejected(
1389
+ f"object storage refused a partition URL "
1390
+ f"(HTTP {response.status_code})",
1391
+ response=response,
1392
+ )
1366
1393
  response.raise_for_status()
1367
1394
  # Spooled rather than ``io.BytesIO(response.content)`` so the
1368
1395
  # compressed bytes and the decoded table are not both resident
@@ -1382,7 +1409,86 @@ def _read_parquet_url(url: str, columns: Sequence[str] | None) -> pd.DataFrame:
1382
1409
  return table.to_pandas()
1383
1410
 
1384
1411
 
1385
- def _partition(body: dict, item: dict, columns) -> pd.DataFrame:
1412
+ class _Resigner:
1413
+ """Fresh URLs for a manifest walk that outlives its signatures.
1414
+
1415
+ A manifest's URLs expire 15 minutes after it is issued and a walk
1416
+ over years of ticks takes hours, so the URLs handed out at the start
1417
+ are dead long before the cursor reaches them. Re-signing is lazy,
1418
+ on the first refusal: one survey (``hydrate=False``, so nothing is
1419
+ fetched upstream) of the window from the refused day to the end
1420
+ re-signs every partition still ahead, and the walk carries on -- one
1421
+ refusal and one survey per 15 minutes of reading, not a manifest per
1422
+ day or a timer guessing at the expiry.
1423
+
1424
+ Looked up by :func:`_identity` (the object key), which is what stays
1425
+ the same between two manifests; the URL never does. A day the
1426
+ survey no longer lists (evicted between the two manifests) is asked
1427
+ for on its own WITH hydration, as it would have been had it been a
1428
+ gap from the start. Bounded per partition: a URL signed a moment
1429
+ ago that is refused too is not expiry, so it raises rather than
1430
+ looping over a real auth or storage failure.
1431
+ """
1432
+
1433
+ def __init__(self, timeseries: "Timeseries", body: dict):
1434
+ self._timeseries = timeseries
1435
+ self._body = body
1436
+ self._fresh: dict[tuple, str] = {}
1437
+
1438
+ def read(self, item: dict, columns) -> pd.DataFrame:
1439
+ try:
1440
+ return _read_parquet_url(
1441
+ self._fresh.get(_identity(item), item["url"]), columns
1442
+ )
1443
+ except _StorageUrlRejected:
1444
+ pass
1445
+ url = self._resign(item)
1446
+ try:
1447
+ return _read_parquet_url(url, columns)
1448
+ except _StorageUrlRejected as exc:
1449
+ raise _StorageUrlRejected(
1450
+ f"object storage refused a freshly signed URL for "
1451
+ f"{item['day']} (instrument {item['instrument_id']}), so "
1452
+ "this is not expiry. Retry shortly; if it persists, "
1453
+ "contact QuantPad support.",
1454
+ response=exc.response,
1455
+ ) from exc
1456
+
1457
+ @staticmethod
1458
+ def _urls(payloads, into: dict) -> dict:
1459
+ for payload in payloads:
1460
+ for entry in payload.get("files") or ():
1461
+ into[_identity(entry)] = entry["url"]
1462
+ return into
1463
+
1464
+ def _resign(self, item: dict) -> str:
1465
+ identity = _identity(item)
1466
+ day = str(item["day"])[:10]
1467
+ # Judged against THIS survey only, then swapped in whole: the survey
1468
+ # covers every day still ahead, and merging into an earlier wave's
1469
+ # URLs would make a day this survey no longer lists look present,
1470
+ # skip the single-day hydrate, and retry a dead URL.
1471
+ latest = self._urls(self._timeseries._manifests(
1472
+ {**self._body, "start": day}, hydrate=False
1473
+ ), {})
1474
+ if identity not in latest:
1475
+ next_day = (date.fromisoformat(day) + timedelta(days=1)).isoformat()
1476
+ self._urls(self._timeseries._manifests(
1477
+ {**self._body, "start": day, "end": next_day}
1478
+ ), latest)
1479
+ self._fresh = latest
1480
+ if identity not in self._fresh:
1481
+ raise _StorageUrlRejected(
1482
+ f"partition URL for {item['day']} expired and the service "
1483
+ "no longer lists that partition, so it could not be "
1484
+ "re-signed. Re-run from this day."
1485
+ )
1486
+ return self._fresh[identity]
1487
+
1488
+
1489
+ def _partition(
1490
+ body: dict, item: dict, columns, resigner: "_Resigner | None" = None
1491
+ ) -> pd.DataFrame:
1386
1492
  """One manifest partition, from local disk if it is already there.
1387
1493
 
1388
1494
  The unit ``iter_range`` already deals in -- one file per instrument-day --
@@ -1420,7 +1526,10 @@ def _partition(body: dict, item: dict, columns) -> pd.DataFrame:
1420
1526
  if hit is not None:
1421
1527
  return hit
1422
1528
 
1423
- frame = _read_parquet_url(item["url"], columns)
1529
+ if resigner is None:
1530
+ frame = _read_parquet_url(item["url"], columns)
1531
+ else:
1532
+ frame = resigner.read(item, columns)
1424
1533
  if path is not None:
1425
1534
  cache.put_frame(
1426
1535
  path, frame,
@@ -2445,3 +2445,173 @@ class TestParallelHydration:
2445
2445
 
2446
2446
  assert next(frames).attrs["quantpad"]["day"] == "2026-01-05"
2447
2447
  frames.close()
2448
+
2449
+
2450
+ class TestExpiredUrlsAreResigned:
2451
+ """A manifest's presigned URLs expire 15 minutes after it is issued, and
2452
+ iter_range surveys its whole window up front, so on a long walk the
2453
+ URLs for later days are dead by the time the cursor reaches them
2454
+ (measured 2026-11-11: ES trades 2010-2026 died with a 403 at day 600,
2455
+ 18 minutes in). A refusal re-signs the rest of the window."""
2456
+
2457
+ @staticmethod
2458
+ def _signed(days, sig, gaps=()):
2459
+ return {
2460
+ "format": "parquet",
2461
+ "files": [
2462
+ {"day": f"2026-01-{d:02d}", "instrument_id": 42,
2463
+ "key": f"GLBX/trades/42/2026-01-{d:02d}.parquet",
2464
+ "size_bytes": 10,
2465
+ "url": f"https://r2.test/{d:02d}.parquet?sig={sig}"}
2466
+ for d in days
2467
+ ],
2468
+ "gaps": [{"day": f"2026-01-{d:02d}"} for d in gaps],
2469
+ "metadata": {"dataset": "GLBX.MDP3"},
2470
+ }
2471
+
2472
+ @staticmethod
2473
+ def _storage(monkeypatch, dead, alive_days=()):
2474
+ from quantpad_data import v2 as v2_module
2475
+
2476
+ reads = []
2477
+
2478
+ def reader(url, columns):
2479
+ reads.append(url)
2480
+ if not any(f"/{d:02d}.parquet" in url for d in alive_days) and any(
2481
+ f"sig={sig}" in url for sig in dead
2482
+ ):
2483
+ raise v2_module._StorageUrlRejected("refused (HTTP 403)")
2484
+ return pd.DataFrame({"url": [url]})
2485
+
2486
+ monkeypatch.setattr("quantpad_data.v2._read_parquet_url", reader)
2487
+ return reads
2488
+
2489
+ @staticmethod
2490
+ def _walk(api):
2491
+ return list(api.timeseries.iter_range(
2492
+ dataset="GLBX.MDP3", symbols="ES.v.0", schema="trades",
2493
+ stype_in="continuous", start="2026-01-05", end="2026-01-09",
2494
+ ))
2495
+
2496
+ def test_an_expired_url_is_resigned_and_the_walk_carries_on(
2497
+ self, monkeypatch
2498
+ ):
2499
+ api, session = client_with([
2500
+ json_response(self._signed([5, 6, 7, 8], "old")),
2501
+ json_response(self._signed([6, 7, 8], "new")),
2502
+ ])
2503
+ reads = self._storage(monkeypatch, dead=["old"], alive_days=[5])
2504
+
2505
+ frames = self._walk(api)
2506
+
2507
+ assert [f.attrs["quantpad"]["day"] for f in frames] == [
2508
+ "2026-01-05", "2026-01-06", "2026-01-07", "2026-01-08",
2509
+ ]
2510
+ assert all("sig=new" in f["url"][0] for f in frames[1:])
2511
+ # One survey, ONE re-sign for everything ahead -- not one per day.
2512
+ assert len(session.calls) == 2
2513
+ resign = session.calls[1][2]["json"]
2514
+ assert (resign["start"], resign["end"]) == ("2026-01-06", "2026-01-09")
2515
+ assert resign["hydrate"] is False
2516
+ assert [u for u in reads if "sig=old" in u and "05" not in u] == [
2517
+ "https://r2.test/06.parquet?sig=old",
2518
+ ]
2519
+
2520
+ def test_a_freshly_signed_url_refused_too_is_not_mistaken_for_expiry(
2521
+ self, monkeypatch
2522
+ ):
2523
+ api, session = client_with([
2524
+ json_response(self._signed([5, 6], "old")),
2525
+ json_response(self._signed([5, 6], "new")),
2526
+ ])
2527
+ self._storage(monkeypatch, dead=["old", "new"])
2528
+
2529
+ with pytest.raises(requests.HTTPError, match="not expiry"):
2530
+ self._walk(api)
2531
+ assert len(session.calls) == 2
2532
+
2533
+ def test_a_day_the_resurvey_no_longer_lists_is_fetched_on_its_own(
2534
+ self, monkeypatch
2535
+ ):
2536
+ api, session = client_with([
2537
+ json_response(self._signed([5, 6], "old")),
2538
+ json_response(self._signed([6], "new", gaps=[5])),
2539
+ json_response(self._signed([5], "hydrated")),
2540
+ ])
2541
+ self._storage(monkeypatch, dead=["old"])
2542
+
2543
+ frames = self._walk(api)
2544
+
2545
+ assert "sig=hydrated" in frames[0]["url"][0]
2546
+ assert "sig=new" in frames[1]["url"][0]
2547
+ single = session.calls[2][2]["json"]
2548
+ assert (single["start"], single["end"]) == ("2026-01-05", "2026-01-06")
2549
+ assert single["hydrate"] is True
2550
+
2551
+ def test_a_bulk_read_resigns_too(self, monkeypatch):
2552
+ api, _ = client_with([
2553
+ json_response(self._signed([5, 6], "old")),
2554
+ json_response(self._signed([5, 6], "new")),
2555
+ ])
2556
+ self._storage(monkeypatch, dead=["old"])
2557
+
2558
+ frame = api.timeseries.get_range(
2559
+ dataset="GLBX.MDP3", symbols="ES.v.0", schema="trades",
2560
+ stype_in="continuous", start="2026-01-05", end="2026-01-07",
2561
+ mode="bulk",
2562
+ )
2563
+
2564
+ assert list(frame["url"]) == [
2565
+ "https://r2.test/05.parquet?sig=new",
2566
+ "https://r2.test/06.parquet?sig=new",
2567
+ ]
2568
+
2569
+ def test_a_refused_storage_read_is_classified_as_rejected(
2570
+ self, monkeypatch
2571
+ ):
2572
+ from quantpad_data import v2 as v2_module
2573
+
2574
+ for status in (401, 403):
2575
+ spy = _StreamSpy(status, b"expired")
2576
+ monkeypatch.setattr(requests, "get", lambda url, **kw: spy)
2577
+ with pytest.raises(v2_module._StorageUrlRejected):
2578
+ v2_module._read_parquet_url("https://r2.test/d.parquet", None)
2579
+ assert spy.closed
2580
+
2581
+ def test_any_other_storage_failure_is_not_resigned(self, monkeypatch):
2582
+ api, session = client_with([json_response(self._signed([5], "old"))])
2583
+ spy = _StreamSpy(500, b"boom")
2584
+ monkeypatch.setattr(requests, "get", lambda url, **kw: spy)
2585
+
2586
+ with pytest.raises(requests.HTTPError, match="500"):
2587
+ self._walk(api)
2588
+ assert len(session.calls) == 1
2589
+
2590
+ def test_a_day_evicted_before_a_later_resign_is_still_hydrated(
2591
+ self, monkeypatch
2592
+ ):
2593
+ """Two re-sign waves: the first lists day 7, the second no longer
2594
+ does. Day 7 is hydrated alone, not retried on the first wave's URL."""
2595
+ from quantpad_data import v2 as v2_module
2596
+
2597
+ api, session = client_with([
2598
+ json_response(self._signed([5, 6, 7], "old")),
2599
+ json_response(self._signed([6, 7], "mid")),
2600
+ json_response(self._signed([], "late", gaps=[7])),
2601
+ json_response(self._signed([7], "hydrated")),
2602
+ ])
2603
+
2604
+ def reader(url, columns):
2605
+ if "sig=old" in url and "/05." not in url:
2606
+ raise v2_module._StorageUrlRejected("refused (HTTP 403)")
2607
+ if "/07." in url and "sig=mid" in url:
2608
+ raise v2_module._StorageUrlRejected("refused (HTTP 403)")
2609
+ return pd.DataFrame({"url": [url]})
2610
+
2611
+ monkeypatch.setattr("quantpad_data.v2._read_parquet_url", reader)
2612
+
2613
+ frames = self._walk(api)
2614
+
2615
+ assert "sig=hydrated" in frames[2]["url"][0]
2616
+ single = session.calls[3][2]["json"]
2617
+ assert (single["start"], single["hydrate"]) == ("2026-01-07", True)
File without changes