quantpad-data 0.8.4__tar.gz → 0.9.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {quantpad_data-0.8.4 → quantpad_data-0.9.0}/PKG-INFO +35 -4
- {quantpad_data-0.8.4 → quantpad_data-0.9.0}/README.md +34 -3
- {quantpad_data-0.8.4 → quantpad_data-0.9.0}/pyproject.toml +1 -1
- {quantpad_data-0.8.4 → quantpad_data-0.9.0}/quantpad_data/__init__.py +7 -1
- {quantpad_data-0.8.4 → quantpad_data-0.9.0}/quantpad_data/client.py +111 -1
- {quantpad_data-0.8.4 → quantpad_data-0.9.0}/quantpad_data/v2.py +113 -4
- {quantpad_data-0.8.4 → quantpad_data-0.9.0}/tests/test_client.py +60 -0
- {quantpad_data-0.8.4 → quantpad_data-0.9.0}/tests/test_v2.py +170 -0
- {quantpad_data-0.8.4 → quantpad_data-0.9.0}/.gitignore +0 -0
- {quantpad_data-0.8.4 → quantpad_data-0.9.0}/quantpad_data/cache.py +0 -0
- {quantpad_data-0.8.4 → quantpad_data-0.9.0}/quantpad_data/errors.py +0 -0
- {quantpad_data-0.8.4 → quantpad_data-0.9.0}/quantpad_data/lean.py +0 -0
- {quantpad_data-0.8.4 → quantpad_data-0.9.0}/quantpad_data/memory.py +0 -0
- {quantpad_data-0.8.4 → quantpad_data-0.9.0}/quantpad_data/py.typed +0 -0
- {quantpad_data-0.8.4 → quantpad_data-0.9.0}/quantpad_data/rth.py +0 -0
- {quantpad_data-0.8.4 → quantpad_data-0.9.0}/tests/conftest.py +0 -0
- {quantpad_data-0.8.4 → quantpad_data-0.9.0}/tests/test_lean.py +0 -0
- {quantpad_data-0.8.4 → quantpad_data-0.9.0}/tests/test_message_encoding.py +0 -0
- {quantpad_data-0.8.4 → quantpad_data-0.9.0}/tests/test_packaging.py +0 -0
- {quantpad_data-0.8.4 → quantpad_data-0.9.0}/tests/test_refusal_hint.py +0 -0
- {quantpad_data-0.8.4 → quantpad_data-0.9.0}/tests/test_rth.py +0 -0
- {quantpad_data-0.8.4 → quantpad_data-0.9.0}/tests/test_v2_cache.py +0 -0
- {quantpad_data-0.8.4 → quantpad_data-0.9.0}/tests/test_v2_gate.py +0 -0
- {quantpad_data-0.8.4 → quantpad_data-0.9.0}/tests/test_version.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.5
|
|
2
2
|
Name: quantpad-data
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.9.0
|
|
4
4
|
Summary: Official Python client for the QuantPad market-data API
|
|
5
5
|
Project-URL: Documentation, https://api.quantpad.ai/external/docs
|
|
6
6
|
Author: QuantPad
|
|
@@ -88,8 +88,39 @@ levels 0-9. It is available for supported CME futures, individual CME
|
|
|
88
88
|
options-on-futures contracts, and US equities, subject to `get_coverage()` and
|
|
89
89
|
the plan lookback window (currently 30 days).
|
|
90
90
|
|
|
91
|
-
This SDK intentionally exposes only bars, ticks, universe, coverage,
|
|
92
|
-
symbology. It does not proxy FRED or SEC EDGAR.
|
|
91
|
+
This SDK intentionally exposes only bars, ticks, universe, coverage,
|
|
92
|
+
symbology, and SEC insider transactions. It does not proxy FRED or SEC EDGAR.
|
|
93
|
+
|
|
94
|
+
## SEC insider transactions
|
|
95
|
+
|
|
96
|
+
`get_insider_transactions` returns QuantPad's cleaned copy of the SEC's public
|
|
97
|
+
Form 3/4/5 filings: every transaction line since 2006, refreshed nightly. The
|
|
98
|
+
filings themselves are public domain; what the service adds is the cleaning:
|
|
99
|
+
|
|
100
|
+
- amendments (4/A) applied, so a corrected trade replaces the original;
|
|
101
|
+
- a trade filed separately by several members of a group (a fund, its general
|
|
102
|
+
partner, its principals) counted once;
|
|
103
|
+
- weighted-average price ranges and Rule 10b5-1 plans read from the footnotes;
|
|
104
|
+
- price typos flagged or corrected (a total value or a share count typed as
|
|
105
|
+
the price), checked within each security class (ADS vs ordinary shares);
|
|
106
|
+
- tickers that follow a company through renames.
|
|
107
|
+
|
|
108
|
+
```python
|
|
109
|
+
# open-market buys and sells at one company
|
|
110
|
+
df = qpd.get_insider_transactions("NVDA", start="2020-01-01", signal=True)
|
|
111
|
+
|
|
112
|
+
# everything one insider filed, at every company
|
|
113
|
+
df = qpd.get_insider_transactions(owner_cik=1214156)
|
|
114
|
+
|
|
115
|
+
# market-wide open-market purchases (fetched a year at a time)
|
|
116
|
+
buys = qpd.get_insider_transactions(start="2025-01-01", end="2025-04-01",
|
|
117
|
+
codes="P", signal=True)
|
|
118
|
+
```
|
|
119
|
+
|
|
120
|
+
`signal=True` keeps open-market purchases (`P`) and sales (`S`) that are
|
|
121
|
+
current, counted once, and sanely priced. For a backtest, use `accepted_at`
|
|
122
|
+
(UTC), or the end of `filed_date` where it is null, as the event time; never
|
|
123
|
+
`trans_date`, because insiders have two business days to file.
|
|
93
124
|
|
|
94
125
|
## Explicit addressing: `qpd.v2`
|
|
95
126
|
|
|
@@ -170,7 +201,7 @@ the cache so ordinary `get_range` calls serve it from storage:
|
|
|
170
201
|
|
|
171
202
|
```python
|
|
172
203
|
job = client.batch.submit(
|
|
173
|
-
dataset="GLBX.MDP3", symbols="ESZ6", schema="
|
|
204
|
+
dataset="GLBX.MDP3", symbols="ESZ6", schema="mbp-10",
|
|
174
205
|
start="2026-01-05", end="2026-02-05", hydrate=True,
|
|
175
206
|
)
|
|
176
207
|
client.batch.hydrate(job["job_id"])
|
|
@@ -66,8 +66,39 @@ levels 0-9. It is available for supported CME futures, individual CME
|
|
|
66
66
|
options-on-futures contracts, and US equities, subject to `get_coverage()` and
|
|
67
67
|
the plan lookback window (currently 30 days).
|
|
68
68
|
|
|
69
|
-
This SDK intentionally exposes only bars, ticks, universe, coverage,
|
|
70
|
-
symbology. It does not proxy FRED or SEC EDGAR.
|
|
69
|
+
This SDK intentionally exposes only bars, ticks, universe, coverage,
|
|
70
|
+
symbology, and SEC insider transactions. It does not proxy FRED or SEC EDGAR.
|
|
71
|
+
|
|
72
|
+
## SEC insider transactions
|
|
73
|
+
|
|
74
|
+
`get_insider_transactions` returns QuantPad's cleaned copy of the SEC's public
|
|
75
|
+
Form 3/4/5 filings: every transaction line since 2006, refreshed nightly. The
|
|
76
|
+
filings themselves are public domain; what the service adds is the cleaning:
|
|
77
|
+
|
|
78
|
+
- amendments (4/A) applied, so a corrected trade replaces the original;
|
|
79
|
+
- a trade filed separately by several members of a group (a fund, its general
|
|
80
|
+
partner, its principals) counted once;
|
|
81
|
+
- weighted-average price ranges and Rule 10b5-1 plans read from the footnotes;
|
|
82
|
+
- price typos flagged or corrected (a total value or a share count typed as
|
|
83
|
+
the price), checked within each security class (ADS vs ordinary shares);
|
|
84
|
+
- tickers that follow a company through renames.
|
|
85
|
+
|
|
86
|
+
```python
|
|
87
|
+
# open-market buys and sells at one company
|
|
88
|
+
df = qpd.get_insider_transactions("NVDA", start="2020-01-01", signal=True)
|
|
89
|
+
|
|
90
|
+
# everything one insider filed, at every company
|
|
91
|
+
df = qpd.get_insider_transactions(owner_cik=1214156)
|
|
92
|
+
|
|
93
|
+
# market-wide open-market purchases (fetched a year at a time)
|
|
94
|
+
buys = qpd.get_insider_transactions(start="2025-01-01", end="2025-04-01",
|
|
95
|
+
codes="P", signal=True)
|
|
96
|
+
```
|
|
97
|
+
|
|
98
|
+
`signal=True` keeps open-market purchases (`P`) and sales (`S`) that are
|
|
99
|
+
current, counted once, and sanely priced. For a backtest, use `accepted_at`
|
|
100
|
+
(UTC), or the end of `filed_date` where it is null, as the event time; never
|
|
101
|
+
`trans_date`, because insiders have two business days to file.
|
|
71
102
|
|
|
72
103
|
## Explicit addressing: `qpd.v2`
|
|
73
104
|
|
|
@@ -148,7 +179,7 @@ the cache so ordinary `get_range` calls serve it from storage:
|
|
|
148
179
|
|
|
149
180
|
```python
|
|
150
181
|
job = client.batch.submit(
|
|
151
|
-
dataset="GLBX.MDP3", symbols="ESZ6", schema="
|
|
182
|
+
dataset="GLBX.MDP3", symbols="ESZ6", schema="mbp-10",
|
|
152
183
|
start="2026-01-05", end="2026-02-05", hydrate=True,
|
|
153
184
|
)
|
|
154
185
|
client.batch.hydrate(job["job_id"])
|
|
@@ -89,6 +89,12 @@ def get_coverage(symbol: str):
|
|
|
89
89
|
return _client().get_coverage(symbol)
|
|
90
90
|
|
|
91
91
|
|
|
92
|
+
def get_insider_transactions(ticker=None, **kwargs):
|
|
93
|
+
"""SEC insider transactions (Forms 3/4/5); see
|
|
94
|
+
:meth:`Client.get_insider_transactions`."""
|
|
95
|
+
return _client().get_insider_transactions(ticker, **kwargs)
|
|
96
|
+
|
|
97
|
+
|
|
92
98
|
def _iter_rth_windows_ms(start_ms: int, end_ms: int):
|
|
93
99
|
try:
|
|
94
100
|
import exchange_calendars as ec
|
|
@@ -218,7 +224,7 @@ __all__ = [
|
|
|
218
224
|
"ROLL_ADJUST_MODES", "get_bars", "get_bars_with_retry", "get_ticks",
|
|
219
225
|
"get_trades", "get_mbp1", "get_mbp10", "get_trades_rth", "get_mbp1_rth",
|
|
220
226
|
"get_mbp10_rth", "get_spread_1m_rth", "symbology_resolve", "get_universe",
|
|
221
|
-
"get_coverage", "iter_trades", "iter_mbp1", "iter_l1_df_chunks",
|
|
227
|
+
"get_coverage", "get_insider_transactions", "iter_trades", "iter_mbp1", "iter_l1_df_chunks",
|
|
222
228
|
"iter_l1_trades_df_chunks", "iter_l1_mbp1_df_chunks", "cache", "lean",
|
|
223
229
|
"v2", "Historical", "np", "pd", "pa",
|
|
224
230
|
]
|
|
@@ -87,7 +87,7 @@ V2_NOT_AVAILABLE = (
|
|
|
87
87
|
"QuantPad support about /v2 access."
|
|
88
88
|
)
|
|
89
89
|
|
|
90
|
-
VERSION = "0.
|
|
90
|
+
VERSION = "0.9.0"
|
|
91
91
|
USER_AGENT = f"quantpad-data-python/{VERSION}"
|
|
92
92
|
TICK_SCHEMAS = (
|
|
93
93
|
"trades", "mbp-1", "mbp-10", "cmbp-1", "tcbbo", "cbbo-1s",
|
|
@@ -209,6 +209,30 @@ def _prose_with_hint(error: dict) -> str:
|
|
|
209
209
|
return str(message)
|
|
210
210
|
|
|
211
211
|
|
|
212
|
+
_INSIDER_DATE_COLS = (
|
|
213
|
+
"filed_date", "trans_date", "deemed_execution_date", "exercise_date", "expiration_date",
|
|
214
|
+
)
|
|
215
|
+
_INSIDER_NUMERIC_COLS = (
|
|
216
|
+
"shares", "price", "price_clean", "price_low", "price_high", "value",
|
|
217
|
+
"shares_owned_after", "conv_exercise_price", "underlying_shares",
|
|
218
|
+
)
|
|
219
|
+
|
|
220
|
+
|
|
221
|
+
def _insider_frame(rows: list[dict]) -> pd.DataFrame:
|
|
222
|
+
df = pd.DataFrame(rows)
|
|
223
|
+
if df.empty:
|
|
224
|
+
return df
|
|
225
|
+
for c in _INSIDER_DATE_COLS:
|
|
226
|
+
if c in df:
|
|
227
|
+
df[c] = pd.to_datetime(df[c], errors="coerce")
|
|
228
|
+
if "accepted_at" in df:
|
|
229
|
+
df["accepted_at"] = pd.to_datetime(df["accepted_at"], utc=True, errors="coerce")
|
|
230
|
+
for c in _INSIDER_NUMERIC_COLS:
|
|
231
|
+
if c in df:
|
|
232
|
+
df[c] = pd.to_numeric(df[c], errors="coerce")
|
|
233
|
+
return df
|
|
234
|
+
|
|
235
|
+
|
|
212
236
|
class Client:
|
|
213
237
|
def __init__(
|
|
214
238
|
self,
|
|
@@ -541,3 +565,89 @@ class Client:
|
|
|
541
565
|
|
|
542
566
|
def get_coverage(self, symbol: str) -> dict:
|
|
543
567
|
return self._request("GET", "/v1/coverage", params={"symbol": symbol}).json()
|
|
568
|
+
|
|
569
|
+
def get_insider_transactions(
|
|
570
|
+
self,
|
|
571
|
+
ticker: str | None = None,
|
|
572
|
+
*,
|
|
573
|
+
issuer_cik: int | None = None,
|
|
574
|
+
owner_cik: int | None = None,
|
|
575
|
+
start: str | None = None,
|
|
576
|
+
end: str | None = None,
|
|
577
|
+
date_field: str = "filed",
|
|
578
|
+
codes: str | None = None,
|
|
579
|
+
kinds: str | None = None,
|
|
580
|
+
signal: bool = False,
|
|
581
|
+
include_superseded: bool = False,
|
|
582
|
+
include_duplicates: bool = False,
|
|
583
|
+
max_rows: int = 1_000_000,
|
|
584
|
+
) -> pd.DataFrame:
|
|
585
|
+
"""SEC insider transactions (Forms 3/4/5), one row per transaction line.
|
|
586
|
+
|
|
587
|
+
QuantPad's cleaned copy of the SEC's public filings, every line since
|
|
588
|
+
2006, refreshed nightly: amendments applied, a trade filed by several
|
|
589
|
+
members of a group counted once, weighted-average price ranges and
|
|
590
|
+
10b5-1 plans read from the footnotes, and price typos flagged.
|
|
591
|
+
|
|
592
|
+
Examples::
|
|
593
|
+
|
|
594
|
+
client.get_insider_transactions("NVDA", start="2020-01-01", signal=True)
|
|
595
|
+
client.get_insider_transactions(owner_cik=1214156)
|
|
596
|
+
client.get_insider_transactions(start="2025-01-01", end="2025-04-01",
|
|
597
|
+
codes="P", signal=True)
|
|
598
|
+
|
|
599
|
+
``ticker`` follows the company through renames (by CIK). ``start``
|
|
600
|
+
is inclusive and ``end`` exclusive, on ``date_field`` ("filed", the
|
|
601
|
+
default, or "trans"). ``signal=True`` keeps open-market purchases
|
|
602
|
+
(P) and sales (S) that are current, counted once and sanely priced.
|
|
603
|
+
A market-wide pull (no ticker or CIK) needs ``start`` and ``end``
|
|
604
|
+
and is fetched a year at a time. Pages of up to 20,000 rows are
|
|
605
|
+
followed for you.
|
|
606
|
+
|
|
607
|
+
For a backtest, the event time is ``accepted_at`` (UTC), or the end
|
|
608
|
+
of ``filed_date`` where it is null. Never ``trans_date``: insiders
|
|
609
|
+
have two business days to file.
|
|
610
|
+
"""
|
|
611
|
+
windows: list[tuple[str | None, str | None]] = [(start, end)]
|
|
612
|
+
if ticker is None and issuer_cik is None and owner_cik is None:
|
|
613
|
+
if not (start and end):
|
|
614
|
+
raise ValueError("a market-wide pull needs start and end")
|
|
615
|
+
s, e = pd.Timestamp(start), pd.Timestamp(end)
|
|
616
|
+
windows = []
|
|
617
|
+
while s < e:
|
|
618
|
+
n = min(s + pd.DateOffset(years=1), e)
|
|
619
|
+
windows.append((s.strftime("%Y-%m-%d"), n.strftime("%Y-%m-%d")))
|
|
620
|
+
s = n
|
|
621
|
+
rows: list[dict] = []
|
|
622
|
+
for ws, we in windows:
|
|
623
|
+
after = None
|
|
624
|
+
while len(rows) < max_rows:
|
|
625
|
+
params = {
|
|
626
|
+
"ticker": ticker, "issuer_cik": issuer_cik, "owner_cik": owner_cik,
|
|
627
|
+
"start": ws, "end": we, "date_field": date_field,
|
|
628
|
+
"codes": codes, "kinds": kinds,
|
|
629
|
+
"signal": str(bool(signal)).lower(),
|
|
630
|
+
"include_superseded": str(bool(include_superseded)).lower(),
|
|
631
|
+
"include_duplicates": str(bool(include_duplicates)).lower(),
|
|
632
|
+
"limit": min(20_000, max_rows - len(rows)),
|
|
633
|
+
"after": after,
|
|
634
|
+
}
|
|
635
|
+
page = self._request(
|
|
636
|
+
"GET", "/v1/insiders/transactions",
|
|
637
|
+
params={k: v for k, v in params.items() if v is not None},
|
|
638
|
+
).json()
|
|
639
|
+
notes = page.get("footnotes") or {}
|
|
640
|
+
for r in page.get("rows") or []:
|
|
641
|
+
texts = notes.get(r.get("accession")) or {}
|
|
642
|
+
ids = r.pop("footnote_ids", None) or {}
|
|
643
|
+
r["footnotes"] = {
|
|
644
|
+
field: [texts[i] for i in fids if i in texts]
|
|
645
|
+
for field, fids in ids.items()
|
|
646
|
+
} or None
|
|
647
|
+
rows.append(r)
|
|
648
|
+
after = page.get("next_cursor")
|
|
649
|
+
if not after:
|
|
650
|
+
break
|
|
651
|
+
if len(rows) >= max_rows:
|
|
652
|
+
break
|
|
653
|
+
return _insider_frame(rows)
|
|
@@ -66,6 +66,19 @@ STORAGE_TIMEOUT = (10, 120)
|
|
|
66
66
|
#: decoded table at full size at once is worth avoiding.
|
|
67
67
|
STORAGE_SPOOL_BYTES = 32 * 1024 * 1024
|
|
68
68
|
|
|
69
|
+
|
|
70
|
+
class _StorageUrlRejected(requests.HTTPError):
|
|
71
|
+
"""Object storage refused a presigned partition URL (401/403).
|
|
72
|
+
|
|
73
|
+
Almost always expiry: a manifest's URLs live 15 minutes and
|
|
74
|
+
``iter_range`` surveys its whole window up front, so any walk that
|
|
75
|
+
takes longer than that reaches days whose URLs died while earlier
|
|
76
|
+
days were being processed (measured 2026-11-11: ES trades 2010-2026
|
|
77
|
+
died with a 403 at day 600, 18 minutes in). An ``HTTPError``
|
|
78
|
+
subclass so callers that caught the old ``raise_for_status``
|
|
79
|
+
failure still do.
|
|
80
|
+
"""
|
|
81
|
+
|
|
69
82
|
#: Widest window of missing days :meth:`Timeseries.iter_range` fetches
|
|
70
83
|
#: in one request, once it has ramped up to it.
|
|
71
84
|
#:
|
|
@@ -485,9 +498,11 @@ class Timeseries(_Namespace):
|
|
|
485
498
|
# day, instrument, projection), so the whole-window body is the right
|
|
486
499
|
# thing to hand every piece.
|
|
487
500
|
manifest_body = _manifest_body(columns=columns, **kwargs)
|
|
501
|
+
resigner = _Resigner(self, manifest_body)
|
|
488
502
|
for payload, item in self._partitions(manifest_body):
|
|
489
503
|
frame = _labelled(
|
|
490
|
-
_partition(manifest_body, item, columns
|
|
504
|
+
_partition(manifest_body, item, columns, resigner),
|
|
505
|
+
item, columns,
|
|
491
506
|
)
|
|
492
507
|
frame.attrs["quantpad"] = {
|
|
493
508
|
**_manifest_attrs(payload),
|
|
@@ -789,6 +804,9 @@ class Timeseries(_Namespace):
|
|
|
789
804
|
columns = body.get("columns")
|
|
790
805
|
frames = []
|
|
791
806
|
served: list[dict] = []
|
|
807
|
+
# Bulk downloads its partitions one after another too, so a large
|
|
808
|
+
# enough window outlives its URLs exactly as iter_range does.
|
|
809
|
+
resigner = _Resigner(self, manifest_body)
|
|
792
810
|
for payload in self._manifests(manifest_body):
|
|
793
811
|
served.append(payload)
|
|
794
812
|
frames.extend(
|
|
@@ -798,7 +816,10 @@ class Timeseries(_Namespace):
|
|
|
798
816
|
# for a later bulk read of the same days and the other way
|
|
799
817
|
# round. Reading the urls directly here would make the cache
|
|
800
818
|
# depend on which delivery mode happened to fetch first.
|
|
801
|
-
_labelled(
|
|
819
|
+
_labelled(
|
|
820
|
+
_partition(manifest_body, item, columns, resigner),
|
|
821
|
+
item, columns,
|
|
822
|
+
)
|
|
802
823
|
for item in payload["files"]
|
|
803
824
|
)
|
|
804
825
|
if not frames:
|
|
@@ -1363,6 +1384,12 @@ def _read_parquet_url(url: str, columns: Sequence[str] | None) -> pd.DataFrame:
|
|
|
1363
1384
|
with requests.get(
|
|
1364
1385
|
url, timeout=STORAGE_TIMEOUT, stream=True
|
|
1365
1386
|
) as response:
|
|
1387
|
+
if response.status_code in (401, 403):
|
|
1388
|
+
raise _StorageUrlRejected(
|
|
1389
|
+
f"object storage refused a partition URL "
|
|
1390
|
+
f"(HTTP {response.status_code})",
|
|
1391
|
+
response=response,
|
|
1392
|
+
)
|
|
1366
1393
|
response.raise_for_status()
|
|
1367
1394
|
# Spooled rather than ``io.BytesIO(response.content)`` so the
|
|
1368
1395
|
# compressed bytes and the decoded table are not both resident
|
|
@@ -1382,7 +1409,86 @@ def _read_parquet_url(url: str, columns: Sequence[str] | None) -> pd.DataFrame:
|
|
|
1382
1409
|
return table.to_pandas()
|
|
1383
1410
|
|
|
1384
1411
|
|
|
1385
|
-
|
|
1412
|
+
class _Resigner:
|
|
1413
|
+
"""Fresh URLs for a manifest walk that outlives its signatures.
|
|
1414
|
+
|
|
1415
|
+
A manifest's URLs expire 15 minutes after it is issued and a walk
|
|
1416
|
+
over years of ticks takes hours, so the URLs handed out at the start
|
|
1417
|
+
are dead long before the cursor reaches them. Re-signing is lazy,
|
|
1418
|
+
on the first refusal: one survey (``hydrate=False``, so nothing is
|
|
1419
|
+
fetched upstream) of the window from the refused day to the end
|
|
1420
|
+
re-signs every partition still ahead, and the walk carries on -- one
|
|
1421
|
+
refusal and one survey per 15 minutes of reading, not a manifest per
|
|
1422
|
+
day or a timer guessing at the expiry.
|
|
1423
|
+
|
|
1424
|
+
Looked up by :func:`_identity` (the object key), which is what stays
|
|
1425
|
+
the same between two manifests; the URL never does. A day the
|
|
1426
|
+
survey no longer lists (evicted between the two manifests) is asked
|
|
1427
|
+
for on its own WITH hydration, as it would have been had it been a
|
|
1428
|
+
gap from the start. Bounded per partition: a URL signed a moment
|
|
1429
|
+
ago that is refused too is not expiry, so it raises rather than
|
|
1430
|
+
looping over a real auth or storage failure.
|
|
1431
|
+
"""
|
|
1432
|
+
|
|
1433
|
+
def __init__(self, timeseries: "Timeseries", body: dict):
|
|
1434
|
+
self._timeseries = timeseries
|
|
1435
|
+
self._body = body
|
|
1436
|
+
self._fresh: dict[tuple, str] = {}
|
|
1437
|
+
|
|
1438
|
+
def read(self, item: dict, columns) -> pd.DataFrame:
|
|
1439
|
+
try:
|
|
1440
|
+
return _read_parquet_url(
|
|
1441
|
+
self._fresh.get(_identity(item), item["url"]), columns
|
|
1442
|
+
)
|
|
1443
|
+
except _StorageUrlRejected:
|
|
1444
|
+
pass
|
|
1445
|
+
url = self._resign(item)
|
|
1446
|
+
try:
|
|
1447
|
+
return _read_parquet_url(url, columns)
|
|
1448
|
+
except _StorageUrlRejected as exc:
|
|
1449
|
+
raise _StorageUrlRejected(
|
|
1450
|
+
f"object storage refused a freshly signed URL for "
|
|
1451
|
+
f"{item['day']} (instrument {item['instrument_id']}), so "
|
|
1452
|
+
"this is not expiry. Retry shortly; if it persists, "
|
|
1453
|
+
"contact QuantPad support.",
|
|
1454
|
+
response=exc.response,
|
|
1455
|
+
) from exc
|
|
1456
|
+
|
|
1457
|
+
@staticmethod
|
|
1458
|
+
def _urls(payloads, into: dict) -> dict:
|
|
1459
|
+
for payload in payloads:
|
|
1460
|
+
for entry in payload.get("files") or ():
|
|
1461
|
+
into[_identity(entry)] = entry["url"]
|
|
1462
|
+
return into
|
|
1463
|
+
|
|
1464
|
+
def _resign(self, item: dict) -> str:
|
|
1465
|
+
identity = _identity(item)
|
|
1466
|
+
day = str(item["day"])[:10]
|
|
1467
|
+
# Judged against THIS survey only, then swapped in whole: the survey
|
|
1468
|
+
# covers every day still ahead, and merging into an earlier wave's
|
|
1469
|
+
# URLs would make a day this survey no longer lists look present,
|
|
1470
|
+
# skip the single-day hydrate, and retry a dead URL.
|
|
1471
|
+
latest = self._urls(self._timeseries._manifests(
|
|
1472
|
+
{**self._body, "start": day}, hydrate=False
|
|
1473
|
+
), {})
|
|
1474
|
+
if identity not in latest:
|
|
1475
|
+
next_day = (date.fromisoformat(day) + timedelta(days=1)).isoformat()
|
|
1476
|
+
self._urls(self._timeseries._manifests(
|
|
1477
|
+
{**self._body, "start": day, "end": next_day}
|
|
1478
|
+
), latest)
|
|
1479
|
+
self._fresh = latest
|
|
1480
|
+
if identity not in self._fresh:
|
|
1481
|
+
raise _StorageUrlRejected(
|
|
1482
|
+
f"partition URL for {item['day']} expired and the service "
|
|
1483
|
+
"no longer lists that partition, so it could not be "
|
|
1484
|
+
"re-signed. Re-run from this day."
|
|
1485
|
+
)
|
|
1486
|
+
return self._fresh[identity]
|
|
1487
|
+
|
|
1488
|
+
|
|
1489
|
+
def _partition(
|
|
1490
|
+
body: dict, item: dict, columns, resigner: "_Resigner | None" = None
|
|
1491
|
+
) -> pd.DataFrame:
|
|
1386
1492
|
"""One manifest partition, from local disk if it is already there.
|
|
1387
1493
|
|
|
1388
1494
|
The unit ``iter_range`` already deals in -- one file per instrument-day --
|
|
@@ -1420,7 +1526,10 @@ def _partition(body: dict, item: dict, columns) -> pd.DataFrame:
|
|
|
1420
1526
|
if hit is not None:
|
|
1421
1527
|
return hit
|
|
1422
1528
|
|
|
1423
|
-
|
|
1529
|
+
if resigner is None:
|
|
1530
|
+
frame = _read_parquet_url(item["url"], columns)
|
|
1531
|
+
else:
|
|
1532
|
+
frame = resigner.read(item, columns)
|
|
1424
1533
|
if path is not None:
|
|
1425
1534
|
cache.put_frame(
|
|
1426
1535
|
path, frame,
|
|
@@ -332,3 +332,63 @@ def test_get_mbp10_exposes_l2_schema_and_column_projection(monkeypatch):
|
|
|
332
332
|
)
|
|
333
333
|
assert captured["args"][:2] == ("ES.FUT", "mbp-10")
|
|
334
334
|
assert captured["kwargs"] == {"columns": columns, "chunksize": 50_000}
|
|
335
|
+
|
|
336
|
+
|
|
337
|
+
def test_get_insider_transactions_pages_and_attaches_footnotes():
|
|
338
|
+
import json
|
|
339
|
+
|
|
340
|
+
page1 = {
|
|
341
|
+
"rows": [{"txn_key": "a:N:1", "accession": "a", "filed_date": "2026-11-16",
|
|
342
|
+
"accepted_at": "2026-11-16T21:05:00+00:00", "shares": "2700.0",
|
|
343
|
+
"price_clean": "63.917", "footnote_ids": {"price": ["F1"]}}],
|
|
344
|
+
"next_cursor": "2026-11-16|a:N:1",
|
|
345
|
+
"footnotes": {"a": {"F1": "weighted average, $63.50 to $64.20"}},
|
|
346
|
+
}
|
|
347
|
+
page2 = {"rows": [{"txn_key": "b:N:1", "accession": "b", "filed_date": "2026-11-17"}],
|
|
348
|
+
"next_cursor": None, "footnotes": {}}
|
|
349
|
+
session = FakeSession([response(body=json.dumps(page1).encode()),
|
|
350
|
+
response(body=json.dumps(page2).encode())])
|
|
351
|
+
client = Client(api_key="secret", session=session)
|
|
352
|
+
df = client.get_insider_transactions("pam", start="2026-11-01", signal=True)
|
|
353
|
+
|
|
354
|
+
assert list(df["txn_key"]) == ["a:N:1", "b:N:1"]
|
|
355
|
+
assert df.loc[0, "footnotes"] == {"price": ["weighted average, $63.50 to $64.20"]}
|
|
356
|
+
assert df.loc[0, "shares"] == 2700.0 and str(df["accepted_at"].dt.tz) == "UTC"
|
|
357
|
+
method, url, kwargs = session.calls[0]
|
|
358
|
+
assert method == "GET" and url.endswith("/v1/insiders/transactions")
|
|
359
|
+
assert kwargs["params"]["ticker"] == "pam" and kwargs["params"]["signal"] == "true"
|
|
360
|
+
assert "end" not in kwargs["params"] and "after" not in kwargs["params"]
|
|
361
|
+
assert session.calls[1][2]["params"]["after"] == "2026-11-16|a:N:1"
|
|
362
|
+
assert kwargs["headers"]["X-API-Key"] == "secret"
|
|
363
|
+
|
|
364
|
+
|
|
365
|
+
def test_get_insider_transactions_market_wide_needs_a_window_and_splits_by_year():
|
|
366
|
+
import json
|
|
367
|
+
|
|
368
|
+
client = Client(api_key="secret", session=FakeSession([]))
|
|
369
|
+
with pytest.raises(ValueError):
|
|
370
|
+
client.get_insider_transactions(codes="P")
|
|
371
|
+
empty = json.dumps({"rows": [], "next_cursor": None}).encode()
|
|
372
|
+
session = FakeSession([response(body=empty), response(body=empty)])
|
|
373
|
+
client = Client(api_key="secret", session=session)
|
|
374
|
+
df = client.get_insider_transactions(start="2024-01-01", end="2025-06-01", codes="P")
|
|
375
|
+
assert df.empty
|
|
376
|
+
windows = [(c[2]["params"]["start"], c[2]["params"]["end"]) for c in session.calls]
|
|
377
|
+
assert windows == [("2024-01-01", "2025-01-01"), ("2025-01-01", "2025-06-01")]
|
|
378
|
+
|
|
379
|
+
|
|
380
|
+
def test_module_level_insider_alias(monkeypatch):
|
|
381
|
+
captured = {}
|
|
382
|
+
|
|
383
|
+
class Stub:
|
|
384
|
+
def get_insider_transactions(self, ticker, **kwargs):
|
|
385
|
+
captured.update(ticker=ticker, kwargs=kwargs)
|
|
386
|
+
return "df"
|
|
387
|
+
|
|
388
|
+
monkeypatch.setattr(quantpad_data, "_client", lambda **kwargs: Stub())
|
|
389
|
+
assert quantpad_data.get_insider_transactions("NVDA", signal=True) == "df"
|
|
390
|
+
assert captured == {"ticker": "NVDA", "kwargs": {"signal": True}}
|
|
391
|
+
|
|
392
|
+
|
|
393
|
+
def test_insider_helper_is_public():
|
|
394
|
+
assert "get_insider_transactions" in quantpad_data.__all__
|
|
@@ -2445,3 +2445,173 @@ class TestParallelHydration:
|
|
|
2445
2445
|
|
|
2446
2446
|
assert next(frames).attrs["quantpad"]["day"] == "2026-01-05"
|
|
2447
2447
|
frames.close()
|
|
2448
|
+
|
|
2449
|
+
|
|
2450
|
+
class TestExpiredUrlsAreResigned:
|
|
2451
|
+
"""A manifest's presigned URLs expire 15 minutes after it is issued, and
|
|
2452
|
+
iter_range surveys its whole window up front, so on a long walk the
|
|
2453
|
+
URLs for later days are dead by the time the cursor reaches them
|
|
2454
|
+
(measured 2026-11-11: ES trades 2010-2026 died with a 403 at day 600,
|
|
2455
|
+
18 minutes in). A refusal re-signs the rest of the window."""
|
|
2456
|
+
|
|
2457
|
+
@staticmethod
|
|
2458
|
+
def _signed(days, sig, gaps=()):
|
|
2459
|
+
return {
|
|
2460
|
+
"format": "parquet",
|
|
2461
|
+
"files": [
|
|
2462
|
+
{"day": f"2026-01-{d:02d}", "instrument_id": 42,
|
|
2463
|
+
"key": f"GLBX/trades/42/2026-01-{d:02d}.parquet",
|
|
2464
|
+
"size_bytes": 10,
|
|
2465
|
+
"url": f"https://r2.test/{d:02d}.parquet?sig={sig}"}
|
|
2466
|
+
for d in days
|
|
2467
|
+
],
|
|
2468
|
+
"gaps": [{"day": f"2026-01-{d:02d}"} for d in gaps],
|
|
2469
|
+
"metadata": {"dataset": "GLBX.MDP3"},
|
|
2470
|
+
}
|
|
2471
|
+
|
|
2472
|
+
@staticmethod
|
|
2473
|
+
def _storage(monkeypatch, dead, alive_days=()):
|
|
2474
|
+
from quantpad_data import v2 as v2_module
|
|
2475
|
+
|
|
2476
|
+
reads = []
|
|
2477
|
+
|
|
2478
|
+
def reader(url, columns):
|
|
2479
|
+
reads.append(url)
|
|
2480
|
+
if not any(f"/{d:02d}.parquet" in url for d in alive_days) and any(
|
|
2481
|
+
f"sig={sig}" in url for sig in dead
|
|
2482
|
+
):
|
|
2483
|
+
raise v2_module._StorageUrlRejected("refused (HTTP 403)")
|
|
2484
|
+
return pd.DataFrame({"url": [url]})
|
|
2485
|
+
|
|
2486
|
+
monkeypatch.setattr("quantpad_data.v2._read_parquet_url", reader)
|
|
2487
|
+
return reads
|
|
2488
|
+
|
|
2489
|
+
@staticmethod
|
|
2490
|
+
def _walk(api):
|
|
2491
|
+
return list(api.timeseries.iter_range(
|
|
2492
|
+
dataset="GLBX.MDP3", symbols="ES.v.0", schema="trades",
|
|
2493
|
+
stype_in="continuous", start="2026-01-05", end="2026-01-09",
|
|
2494
|
+
))
|
|
2495
|
+
|
|
2496
|
+
def test_an_expired_url_is_resigned_and_the_walk_carries_on(
|
|
2497
|
+
self, monkeypatch
|
|
2498
|
+
):
|
|
2499
|
+
api, session = client_with([
|
|
2500
|
+
json_response(self._signed([5, 6, 7, 8], "old")),
|
|
2501
|
+
json_response(self._signed([6, 7, 8], "new")),
|
|
2502
|
+
])
|
|
2503
|
+
reads = self._storage(monkeypatch, dead=["old"], alive_days=[5])
|
|
2504
|
+
|
|
2505
|
+
frames = self._walk(api)
|
|
2506
|
+
|
|
2507
|
+
assert [f.attrs["quantpad"]["day"] for f in frames] == [
|
|
2508
|
+
"2026-01-05", "2026-01-06", "2026-01-07", "2026-01-08",
|
|
2509
|
+
]
|
|
2510
|
+
assert all("sig=new" in f["url"][0] for f in frames[1:])
|
|
2511
|
+
# One survey, ONE re-sign for everything ahead -- not one per day.
|
|
2512
|
+
assert len(session.calls) == 2
|
|
2513
|
+
resign = session.calls[1][2]["json"]
|
|
2514
|
+
assert (resign["start"], resign["end"]) == ("2026-01-06", "2026-01-09")
|
|
2515
|
+
assert resign["hydrate"] is False
|
|
2516
|
+
assert [u for u in reads if "sig=old" in u and "05" not in u] == [
|
|
2517
|
+
"https://r2.test/06.parquet?sig=old",
|
|
2518
|
+
]
|
|
2519
|
+
|
|
2520
|
+
def test_a_freshly_signed_url_refused_too_is_not_mistaken_for_expiry(
|
|
2521
|
+
self, monkeypatch
|
|
2522
|
+
):
|
|
2523
|
+
api, session = client_with([
|
|
2524
|
+
json_response(self._signed([5, 6], "old")),
|
|
2525
|
+
json_response(self._signed([5, 6], "new")),
|
|
2526
|
+
])
|
|
2527
|
+
self._storage(monkeypatch, dead=["old", "new"])
|
|
2528
|
+
|
|
2529
|
+
with pytest.raises(requests.HTTPError, match="not expiry"):
|
|
2530
|
+
self._walk(api)
|
|
2531
|
+
assert len(session.calls) == 2
|
|
2532
|
+
|
|
2533
|
+
def test_a_day_the_resurvey_no_longer_lists_is_fetched_on_its_own(
|
|
2534
|
+
self, monkeypatch
|
|
2535
|
+
):
|
|
2536
|
+
api, session = client_with([
|
|
2537
|
+
json_response(self._signed([5, 6], "old")),
|
|
2538
|
+
json_response(self._signed([6], "new", gaps=[5])),
|
|
2539
|
+
json_response(self._signed([5], "hydrated")),
|
|
2540
|
+
])
|
|
2541
|
+
self._storage(monkeypatch, dead=["old"])
|
|
2542
|
+
|
|
2543
|
+
frames = self._walk(api)
|
|
2544
|
+
|
|
2545
|
+
assert "sig=hydrated" in frames[0]["url"][0]
|
|
2546
|
+
assert "sig=new" in frames[1]["url"][0]
|
|
2547
|
+
single = session.calls[2][2]["json"]
|
|
2548
|
+
assert (single["start"], single["end"]) == ("2026-01-05", "2026-01-06")
|
|
2549
|
+
assert single["hydrate"] is True
|
|
2550
|
+
|
|
2551
|
+
def test_a_bulk_read_resigns_too(self, monkeypatch):
|
|
2552
|
+
api, _ = client_with([
|
|
2553
|
+
json_response(self._signed([5, 6], "old")),
|
|
2554
|
+
json_response(self._signed([5, 6], "new")),
|
|
2555
|
+
])
|
|
2556
|
+
self._storage(monkeypatch, dead=["old"])
|
|
2557
|
+
|
|
2558
|
+
frame = api.timeseries.get_range(
|
|
2559
|
+
dataset="GLBX.MDP3", symbols="ES.v.0", schema="trades",
|
|
2560
|
+
stype_in="continuous", start="2026-01-05", end="2026-01-07",
|
|
2561
|
+
mode="bulk",
|
|
2562
|
+
)
|
|
2563
|
+
|
|
2564
|
+
assert list(frame["url"]) == [
|
|
2565
|
+
"https://r2.test/05.parquet?sig=new",
|
|
2566
|
+
"https://r2.test/06.parquet?sig=new",
|
|
2567
|
+
]
|
|
2568
|
+
|
|
2569
|
+
def test_a_refused_storage_read_is_classified_as_rejected(
|
|
2570
|
+
self, monkeypatch
|
|
2571
|
+
):
|
|
2572
|
+
from quantpad_data import v2 as v2_module
|
|
2573
|
+
|
|
2574
|
+
for status in (401, 403):
|
|
2575
|
+
spy = _StreamSpy(status, b"expired")
|
|
2576
|
+
monkeypatch.setattr(requests, "get", lambda url, **kw: spy)
|
|
2577
|
+
with pytest.raises(v2_module._StorageUrlRejected):
|
|
2578
|
+
v2_module._read_parquet_url("https://r2.test/d.parquet", None)
|
|
2579
|
+
assert spy.closed
|
|
2580
|
+
|
|
2581
|
+
def test_any_other_storage_failure_is_not_resigned(self, monkeypatch):
|
|
2582
|
+
api, session = client_with([json_response(self._signed([5], "old"))])
|
|
2583
|
+
spy = _StreamSpy(500, b"boom")
|
|
2584
|
+
monkeypatch.setattr(requests, "get", lambda url, **kw: spy)
|
|
2585
|
+
|
|
2586
|
+
with pytest.raises(requests.HTTPError, match="500"):
|
|
2587
|
+
self._walk(api)
|
|
2588
|
+
assert len(session.calls) == 1
|
|
2589
|
+
|
|
2590
|
+
def test_a_day_evicted_before_a_later_resign_is_still_hydrated(
|
|
2591
|
+
self, monkeypatch
|
|
2592
|
+
):
|
|
2593
|
+
"""Two re-sign waves: the first lists day 7, the second no longer
|
|
2594
|
+
does. Day 7 is hydrated alone, not retried on the first wave's URL."""
|
|
2595
|
+
from quantpad_data import v2 as v2_module
|
|
2596
|
+
|
|
2597
|
+
api, session = client_with([
|
|
2598
|
+
json_response(self._signed([5, 6, 7], "old")),
|
|
2599
|
+
json_response(self._signed([6, 7], "mid")),
|
|
2600
|
+
json_response(self._signed([], "late", gaps=[7])),
|
|
2601
|
+
json_response(self._signed([7], "hydrated")),
|
|
2602
|
+
])
|
|
2603
|
+
|
|
2604
|
+
def reader(url, columns):
|
|
2605
|
+
if "sig=old" in url and "/05." not in url:
|
|
2606
|
+
raise v2_module._StorageUrlRejected("refused (HTTP 403)")
|
|
2607
|
+
if "/07." in url and "sig=mid" in url:
|
|
2608
|
+
raise v2_module._StorageUrlRejected("refused (HTTP 403)")
|
|
2609
|
+
return pd.DataFrame({"url": [url]})
|
|
2610
|
+
|
|
2611
|
+
monkeypatch.setattr("quantpad_data.v2._read_parquet_url", reader)
|
|
2612
|
+
|
|
2613
|
+
frames = self._walk(api)
|
|
2614
|
+
|
|
2615
|
+
assert "sig=hydrated" in frames[2]["url"][0]
|
|
2616
|
+
single = session.calls[3][2]["json"]
|
|
2617
|
+
assert (single["start"], single["hydrate"]) == ("2026-01-07", True)
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|