quantpad-data 0.10.0__tar.gz → 0.10.1__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {quantpad_data-0.10.0 → quantpad_data-0.10.1}/PKG-INFO +53 -5
- {quantpad_data-0.10.0 → quantpad_data-0.10.1}/README.md +52 -4
- {quantpad_data-0.10.0 → quantpad_data-0.10.1}/pyproject.toml +1 -1
- {quantpad_data-0.10.0 → quantpad_data-0.10.1}/quantpad_data/__init__.py +12 -1
- {quantpad_data-0.10.0 → quantpad_data-0.10.1}/quantpad_data/client.py +226 -1
- {quantpad_data-0.10.0 → quantpad_data-0.10.1}/quantpad_data/v2.py +62 -2
- {quantpad_data-0.10.0 → quantpad_data-0.10.1}/tests/test_client.py +155 -0
- {quantpad_data-0.10.0 → quantpad_data-0.10.1}/tests/test_v2.py +51 -0
- {quantpad_data-0.10.0 → quantpad_data-0.10.1}/.gitignore +0 -0
- {quantpad_data-0.10.0 → quantpad_data-0.10.1}/quantpad_data/cache.py +0 -0
- {quantpad_data-0.10.0 → quantpad_data-0.10.1}/quantpad_data/errors.py +0 -0
- {quantpad_data-0.10.0 → quantpad_data-0.10.1}/quantpad_data/lean.py +0 -0
- {quantpad_data-0.10.0 → quantpad_data-0.10.1}/quantpad_data/memory.py +0 -0
- {quantpad_data-0.10.0 → quantpad_data-0.10.1}/quantpad_data/py.typed +0 -0
- {quantpad_data-0.10.0 → quantpad_data-0.10.1}/quantpad_data/rth.py +0 -0
- {quantpad_data-0.10.0 → quantpad_data-0.10.1}/tests/conftest.py +0 -0
- {quantpad_data-0.10.0 → quantpad_data-0.10.1}/tests/test_in_app_only.py +0 -0
- {quantpad_data-0.10.0 → quantpad_data-0.10.1}/tests/test_lean.py +0 -0
- {quantpad_data-0.10.0 → quantpad_data-0.10.1}/tests/test_message_encoding.py +0 -0
- {quantpad_data-0.10.0 → quantpad_data-0.10.1}/tests/test_packaging.py +0 -0
- {quantpad_data-0.10.0 → quantpad_data-0.10.1}/tests/test_refusal_hint.py +0 -0
- {quantpad_data-0.10.0 → quantpad_data-0.10.1}/tests/test_rth.py +0 -0
- {quantpad_data-0.10.0 → quantpad_data-0.10.1}/tests/test_v2_cache.py +0 -0
- {quantpad_data-0.10.0 → quantpad_data-0.10.1}/tests/test_v2_gate.py +0 -0
- {quantpad_data-0.10.0 → quantpad_data-0.10.1}/tests/test_version.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.5
|
|
2
2
|
Name: quantpad-data
|
|
3
|
-
Version: 0.10.
|
|
3
|
+
Version: 0.10.1
|
|
4
4
|
Summary: Official Python client for the QuantPad market-data API
|
|
5
5
|
Project-URL: Documentation, https://api.quantpad.ai/external/docs
|
|
6
6
|
Author: QuantPad
|
|
@@ -105,7 +105,8 @@ levels 0-9. With an API key it is available for US equities, subject to
|
|
|
105
105
|
`get_coverage()` and the plan lookback window (currently 30 days).
|
|
106
106
|
|
|
107
107
|
This SDK intentionally exposes only bars, ticks, universe, coverage,
|
|
108
|
-
symbology,
|
|
108
|
+
symbology, SEC insider transactions, and company fundamentals and earnings
|
|
109
|
+
events from SEC filings. It does not proxy FRED or SEC EDGAR.
|
|
109
110
|
|
|
110
111
|
## SEC insider transactions
|
|
111
112
|
|
|
@@ -138,6 +139,44 @@ current, counted once, and sanely priced. For a backtest, use `accepted_at`
|
|
|
138
139
|
(UTC), or the end of `filed_date` where it is null, as the event time; never
|
|
139
140
|
`trans_date`, because insiders have two business days to file.
|
|
140
141
|
|
|
142
|
+
## Company fundamentals and earnings
|
|
143
|
+
|
|
144
|
+
`get_fundamentals` returns 32 standard fields (revenue, net income, EPS,
|
|
145
|
+
total assets, operating cash flow, ...) for US companies that file 10-Ks and
|
|
146
|
+
10-Qs, from SEC filings since 2009, refreshed nightly. Every value keeps the
|
|
147
|
+
time it became public, and restatements are kept as later versions, so a
|
|
148
|
+
backtest can see exactly what was known on a given day.
|
|
149
|
+
|
|
150
|
+
```python
|
|
151
|
+
# quarters, one column per field
|
|
152
|
+
df = qpd.get_fundamentals("AAPL", fields=["revenue", "eps_diluted", "total_assets"],
|
|
153
|
+
start="2020-01-01")
|
|
154
|
+
|
|
155
|
+
# fiscal years
|
|
156
|
+
df = qpd.get_fundamentals("AAPL", period="annual")
|
|
157
|
+
|
|
158
|
+
# what a reader had on a day: the newest restatement known by then
|
|
159
|
+
df = qpd.get_fundamentals("AAPL", version="latest", as_of="2024-06-01")
|
|
160
|
+
|
|
161
|
+
# one row per value, with when it became public and which filing carried it
|
|
162
|
+
df = qpd.get_fundamentals("AAPL", wide=False)
|
|
163
|
+
```
|
|
164
|
+
|
|
165
|
+
A fourth quarter the company did not file on its own is the year less nine
|
|
166
|
+
months, marked `derived` in the long form. Per-share values are never derived,
|
|
167
|
+
so such a quarter has no EPS.
|
|
168
|
+
|
|
169
|
+
`get_earnings` returns one row per period a company reported: when the results
|
|
170
|
+
went public (`announced_at`, UTC to the second, from the 8-K that released them
|
|
171
|
+
or else the 10-Q/10-K), before, during or after the session, and the filed
|
|
172
|
+
figures with the same quarter a year earlier. Upcoming dates and analyst
|
|
173
|
+
estimates are not included.
|
|
174
|
+
|
|
175
|
+
```python
|
|
176
|
+
df = qpd.get_earnings("AAPL")
|
|
177
|
+
week = qpd.get_earnings(start="2026-10-26", end="2026-11-07") # every company
|
|
178
|
+
```
|
|
179
|
+
|
|
141
180
|
## Explicit addressing: `qpd.v2`
|
|
142
181
|
|
|
143
182
|
> **Not open yet.** `/v2` is limited to a small allowlist while it is
|
|
@@ -216,13 +255,22 @@ For ranges too large even for that, submit a batch job, then hydrate it into
|
|
|
216
255
|
the cache so ordinary `get_range` calls serve it from storage:
|
|
217
256
|
|
|
218
257
|
```python
|
|
219
|
-
|
|
258
|
+
submitted = client.batch.submit(
|
|
220
259
|
dataset="XNAS.ITCH", symbols="AAPL", schema="trades",
|
|
221
|
-
start="2026-01-05", end="2026-02-05",
|
|
260
|
+
start="2026-01-05", end="2026-02-05",
|
|
222
261
|
)
|
|
223
|
-
|
|
262
|
+
job_id = submitted["job"]["job_id"]
|
|
263
|
+
# ...once client.batch.list_jobs() shows the job as "done":
|
|
264
|
+
result = client.batch.hydrate(job_id) # waits; returns written/empty/skipped
|
|
224
265
|
```
|
|
225
266
|
|
|
267
|
+
The hydrate runs on the service, not in your process. `hydrate()` polls it
|
|
268
|
+
until it finishes (pass `timeout=` to bound the wait, or `wait=False` to return
|
|
269
|
+
at once), and `client.batch.hydrate_status(job_id)` reports `state`,
|
|
270
|
+
`files_done` and `files_total` at any time. If it is `interrupted`, call
|
|
271
|
+
`hydrate()` again: it resumes after the days already written. Submitting with
|
|
272
|
+
`hydrate=True` instead starts the hydrate as soon as the job has rendered.
|
|
273
|
+
|
|
226
274
|
### Regular trading hours
|
|
227
275
|
|
|
228
276
|
`get_range` returns the whole published session, which for US equities includes
|
|
@@ -83,7 +83,8 @@ levels 0-9. With an API key it is available for US equities, subject to
|
|
|
83
83
|
`get_coverage()` and the plan lookback window (currently 30 days).
|
|
84
84
|
|
|
85
85
|
This SDK intentionally exposes only bars, ticks, universe, coverage,
|
|
86
|
-
symbology,
|
|
86
|
+
symbology, SEC insider transactions, and company fundamentals and earnings
|
|
87
|
+
events from SEC filings. It does not proxy FRED or SEC EDGAR.
|
|
87
88
|
|
|
88
89
|
## SEC insider transactions
|
|
89
90
|
|
|
@@ -116,6 +117,44 @@ current, counted once, and sanely priced. For a backtest, use `accepted_at`
|
|
|
116
117
|
(UTC), or the end of `filed_date` where it is null, as the event time; never
|
|
117
118
|
`trans_date`, because insiders have two business days to file.
|
|
118
119
|
|
|
120
|
+
## Company fundamentals and earnings
|
|
121
|
+
|
|
122
|
+
`get_fundamentals` returns 32 standard fields (revenue, net income, EPS,
|
|
123
|
+
total assets, operating cash flow, ...) for US companies that file 10-Ks and
|
|
124
|
+
10-Qs, from SEC filings since 2009, refreshed nightly. Every value keeps the
|
|
125
|
+
time it became public, and restatements are kept as later versions, so a
|
|
126
|
+
backtest can see exactly what was known on a given day.
|
|
127
|
+
|
|
128
|
+
```python
|
|
129
|
+
# quarters, one column per field
|
|
130
|
+
df = qpd.get_fundamentals("AAPL", fields=["revenue", "eps_diluted", "total_assets"],
|
|
131
|
+
start="2020-01-01")
|
|
132
|
+
|
|
133
|
+
# fiscal years
|
|
134
|
+
df = qpd.get_fundamentals("AAPL", period="annual")
|
|
135
|
+
|
|
136
|
+
# what a reader had on a day: the newest restatement known by then
|
|
137
|
+
df = qpd.get_fundamentals("AAPL", version="latest", as_of="2024-06-01")
|
|
138
|
+
|
|
139
|
+
# one row per value, with when it became public and which filing carried it
|
|
140
|
+
df = qpd.get_fundamentals("AAPL", wide=False)
|
|
141
|
+
```
|
|
142
|
+
|
|
143
|
+
A fourth quarter the company did not file on its own is the year less nine
|
|
144
|
+
months, marked `derived` in the long form. Per-share values are never derived,
|
|
145
|
+
so such a quarter has no EPS.
|
|
146
|
+
|
|
147
|
+
`get_earnings` returns one row per period a company reported: when the results
|
|
148
|
+
went public (`announced_at`, UTC to the second, from the 8-K that released them
|
|
149
|
+
or else the 10-Q/10-K), before, during or after the session, and the filed
|
|
150
|
+
figures with the same quarter a year earlier. Upcoming dates and analyst
|
|
151
|
+
estimates are not included.
|
|
152
|
+
|
|
153
|
+
```python
|
|
154
|
+
df = qpd.get_earnings("AAPL")
|
|
155
|
+
week = qpd.get_earnings(start="2026-10-26", end="2026-11-07") # every company
|
|
156
|
+
```
|
|
157
|
+
|
|
119
158
|
## Explicit addressing: `qpd.v2`
|
|
120
159
|
|
|
121
160
|
> **Not open yet.** `/v2` is limited to a small allowlist while it is
|
|
@@ -194,13 +233,22 @@ For ranges too large even for that, submit a batch job, then hydrate it into
|
|
|
194
233
|
the cache so ordinary `get_range` calls serve it from storage:
|
|
195
234
|
|
|
196
235
|
```python
|
|
197
|
-
|
|
236
|
+
submitted = client.batch.submit(
|
|
198
237
|
dataset="XNAS.ITCH", symbols="AAPL", schema="trades",
|
|
199
|
-
start="2026-01-05", end="2026-02-05",
|
|
238
|
+
start="2026-01-05", end="2026-02-05",
|
|
200
239
|
)
|
|
201
|
-
|
|
240
|
+
job_id = submitted["job"]["job_id"]
|
|
241
|
+
# ...once client.batch.list_jobs() shows the job as "done":
|
|
242
|
+
result = client.batch.hydrate(job_id) # waits; returns written/empty/skipped
|
|
202
243
|
```
|
|
203
244
|
|
|
245
|
+
The hydrate runs on the service, not in your process. `hydrate()` polls it
|
|
246
|
+
until it finishes (pass `timeout=` to bound the wait, or `wait=False` to return
|
|
247
|
+
at once), and `client.batch.hydrate_status(job_id)` reports `state`,
|
|
248
|
+
`files_done` and `files_total` at any time. If it is `interrupted`, call
|
|
249
|
+
`hydrate()` again: it resumes after the days already written. Submitting with
|
|
250
|
+
`hydrate=True` instead starts the hydrate as soon as the job has rendered.
|
|
251
|
+
|
|
204
252
|
### Regular trading hours
|
|
205
253
|
|
|
206
254
|
`get_range` returns the whole published session, which for US equities includes
|
|
@@ -90,6 +90,17 @@ def get_coverage(symbol: str):
|
|
|
90
90
|
return _client().get_coverage(symbol)
|
|
91
91
|
|
|
92
92
|
|
|
93
|
+
def get_fundamentals(ticker=None, **kwargs):
|
|
94
|
+
"""Company fundamentals from SEC filings, point in time; see
|
|
95
|
+
:meth:`Client.get_fundamentals`."""
|
|
96
|
+
return _client().get_fundamentals(ticker, **kwargs)
|
|
97
|
+
|
|
98
|
+
|
|
99
|
+
def get_earnings(ticker=None, **kwargs):
|
|
100
|
+
"""Earnings events from SEC filings; see :meth:`Client.get_earnings`."""
|
|
101
|
+
return _client().get_earnings(ticker, **kwargs)
|
|
102
|
+
|
|
103
|
+
|
|
93
104
|
def get_insider_transactions(ticker=None, **kwargs):
|
|
94
105
|
"""SEC insider transactions (Forms 3/4/5); see
|
|
95
106
|
:meth:`Client.get_insider_transactions`."""
|
|
@@ -225,7 +236,7 @@ __all__ = [
|
|
|
225
236
|
"ROLL_ADJUST_MODES", "get_bars", "get_bars_with_retry", "get_ticks",
|
|
226
237
|
"get_trades", "get_mbp1", "get_mbp10", "get_trades_rth", "get_mbp1_rth",
|
|
227
238
|
"get_mbp10_rth", "get_spread_1m_rth", "symbology_resolve", "get_universe",
|
|
228
|
-
"get_coverage", "get_insider_transactions", "iter_trades", "iter_mbp1", "iter_l1_df_chunks",
|
|
239
|
+
"get_coverage", "get_earnings", "get_fundamentals", "get_insider_transactions", "iter_trades", "iter_mbp1", "iter_l1_df_chunks",
|
|
229
240
|
"iter_l1_trades_df_chunks", "iter_l1_mbp1_df_chunks", "cache", "lean",
|
|
230
241
|
"v2", "Historical", "np", "pd", "pa",
|
|
231
242
|
]
|
|
@@ -6,6 +6,7 @@ import io
|
|
|
6
6
|
import os
|
|
7
7
|
import random
|
|
8
8
|
import time
|
|
9
|
+
import warnings
|
|
9
10
|
from datetime import date
|
|
10
11
|
from typing import Iterator, Sequence
|
|
11
12
|
|
|
@@ -88,7 +89,7 @@ V2_NOT_AVAILABLE = (
|
|
|
88
89
|
"QuantPad support about /v2 access."
|
|
89
90
|
)
|
|
90
91
|
|
|
91
|
-
VERSION = "0.10.
|
|
92
|
+
VERSION = "0.10.1"
|
|
92
93
|
USER_AGENT = f"quantpad-data-python/{VERSION}"
|
|
93
94
|
TICK_SCHEMAS = (
|
|
94
95
|
"trades", "mbp-1", "mbp-10", "cmbp-1", "tcbbo", "cbbo-1s",
|
|
@@ -234,6 +235,94 @@ def _insider_frame(rows: list[dict]) -> pd.DataFrame:
|
|
|
234
235
|
return df
|
|
235
236
|
|
|
236
237
|
|
|
238
|
+
_EARNINGS_TIME_COLS = ("announced_at", "report_accepted_at", "candidate_accepted_at")
|
|
239
|
+
_EARNINGS_NUMERIC_COLS = (
|
|
240
|
+
"revenue", "net_income", "eps_basic", "eps_diluted",
|
|
241
|
+
"fy_revenue", "fy_net_income", "fy_eps_basic", "fy_eps_diluted",
|
|
242
|
+
"revenue_year_ago", "net_income_year_ago", "eps_basic_year_ago", "eps_diluted_year_ago",
|
|
243
|
+
)
|
|
244
|
+
|
|
245
|
+
|
|
246
|
+
def _year_windows(start: str | None, end: str | None) -> list[tuple[str | None, str | None]]:
|
|
247
|
+
"""``[start, end)`` cut into windows of at most a year, for a market-wide
|
|
248
|
+
pull; a single open window when there is nothing to cut."""
|
|
249
|
+
if not (start and end):
|
|
250
|
+
return [(start, end)]
|
|
251
|
+
s, e = pd.Timestamp(start), pd.Timestamp(end)
|
|
252
|
+
out = []
|
|
253
|
+
while s < e:
|
|
254
|
+
n = min(s + pd.DateOffset(years=1), e)
|
|
255
|
+
out.append((s.strftime("%Y-%m-%d"), n.strftime("%Y-%m-%d")))
|
|
256
|
+
s = n
|
|
257
|
+
return out
|
|
258
|
+
|
|
259
|
+
|
|
260
|
+
def _fundamentals_long(rows: list[dict]) -> pd.DataFrame:
|
|
261
|
+
df = pd.DataFrame(rows)
|
|
262
|
+
if df.empty:
|
|
263
|
+
return df
|
|
264
|
+
for c in ("period_start", "period_end", "filed_date"):
|
|
265
|
+
if c in df:
|
|
266
|
+
df[c] = pd.to_datetime(df[c], errors="coerce")
|
|
267
|
+
for c in ("accepted_at", "known_at"):
|
|
268
|
+
if c in df:
|
|
269
|
+
df[c] = pd.to_datetime(df[c], utc=True, errors="coerce")
|
|
270
|
+
df["value"] = pd.to_numeric(df["value"], errors="coerce")
|
|
271
|
+
return df
|
|
272
|
+
|
|
273
|
+
|
|
274
|
+
def _fundamentals_wide(long: pd.DataFrame, period: str = "quarter") -> pd.DataFrame:
|
|
275
|
+
"""One row per company and period end, one column per field, plus
|
|
276
|
+
``known_at``: when the last of the row's values became public.
|
|
277
|
+
|
|
278
|
+
A balance-sheet value and a flow for the same period end share a row.
|
|
279
|
+
``fiscal_quarter`` is left out of an annual frame: there the year's rows
|
|
280
|
+
carry none and the year-end balance sheet carries 4.
|
|
281
|
+
"""
|
|
282
|
+
if long.empty:
|
|
283
|
+
return long
|
|
284
|
+
index = [c for c in ("cik", "ticker", "period_end") if c in long]
|
|
285
|
+
values = long.groupby(index + ["field"], dropna=False)["value"].first().unstack("field")
|
|
286
|
+
values.columns.name = None
|
|
287
|
+
grouped = long.groupby(index, dropna=False)
|
|
288
|
+
extra = {"known_at": grouped["known_at"].max(), "fiscal_year": grouped["fiscal_year"].max()}
|
|
289
|
+
if period != "annual":
|
|
290
|
+
extra["fiscal_quarter"] = grouped["fiscal_quarter"].max()
|
|
291
|
+
for name, series in extra.items():
|
|
292
|
+
values[name] = series.reindex(values.index)
|
|
293
|
+
wide = values.reset_index()
|
|
294
|
+
head = index + [c for c in ("fiscal_year", "fiscal_quarter") if c in wide]
|
|
295
|
+
fields = sorted(c for c in wide.columns if c not in head and c != "known_at")
|
|
296
|
+
return wide[head + fields + ["known_at"]].sort_values(["period_end", "cik"]).reset_index(drop=True)
|
|
297
|
+
|
|
298
|
+
|
|
299
|
+
#: Shown when events come back with announcement times but no figures.
|
|
300
|
+
EARNINGS_NO_FIGURES_NOTE = (
|
|
301
|
+
"{n} of {total} earnings events have no figures: their reports were not "
|
|
302
|
+
"machine-readable (SEC filings were tagged from mid-2009 for large companies "
|
|
303
|
+
"and mid-2011 for smaller ones). Announcement times are exact."
|
|
304
|
+
)
|
|
305
|
+
|
|
306
|
+
|
|
307
|
+
def _earnings_frame(rows: list[dict]) -> pd.DataFrame:
|
|
308
|
+
df = pd.DataFrame(rows)
|
|
309
|
+
if df.empty:
|
|
310
|
+
return df
|
|
311
|
+
df["period_end"] = pd.to_datetime(df["period_end"], errors="coerce")
|
|
312
|
+
for c in _EARNINGS_TIME_COLS:
|
|
313
|
+
if c in df:
|
|
314
|
+
df[c] = pd.to_datetime(df[c], utc=True, errors="coerce")
|
|
315
|
+
for c in _EARNINGS_NUMERIC_COLS:
|
|
316
|
+
if c in df:
|
|
317
|
+
df[c] = pd.to_numeric(df[c], errors="coerce")
|
|
318
|
+
figures = [c for c in _EARNINGS_NUMERIC_COLS if c in df and not c.endswith("_year_ago")]
|
|
319
|
+
if figures:
|
|
320
|
+
empty = int(df[figures].isna().all(axis=1).sum())
|
|
321
|
+
if empty:
|
|
322
|
+
warnings.warn(EARNINGS_NO_FIGURES_NOTE.format(n=empty, total=len(df)), stacklevel=3)
|
|
323
|
+
return df
|
|
324
|
+
|
|
325
|
+
|
|
237
326
|
class Client:
|
|
238
327
|
def __init__(
|
|
239
328
|
self,
|
|
@@ -654,3 +743,139 @@ class Client:
|
|
|
654
743
|
if len(rows) >= max_rows:
|
|
655
744
|
break
|
|
656
745
|
return _insider_frame(rows)
|
|
746
|
+
|
|
747
|
+
def _pages(self, path: str, base: dict, windows, max_rows: int) -> tuple[list[dict], bool]:
|
|
748
|
+
"""Every row of every window, up to ``max_rows``. Returns the rows
|
|
749
|
+
and whether the cap stopped the read inside a window (the last rows
|
|
750
|
+
may then be part of a company's period). A capped read warns."""
|
|
751
|
+
rows: list[dict] = []
|
|
752
|
+
for i, (ws, we) in enumerate(windows):
|
|
753
|
+
after = None
|
|
754
|
+
while True:
|
|
755
|
+
params = {**base, "start": ws, "end": we, "after": after,
|
|
756
|
+
"limit": min(20_000, max_rows - len(rows))}
|
|
757
|
+
page = self._request(
|
|
758
|
+
"GET", path, params={k: v for k, v in params.items() if v is not None},
|
|
759
|
+
).json()
|
|
760
|
+
rows.extend(page.get("rows") or [])
|
|
761
|
+
after = page.get("next_cursor")
|
|
762
|
+
if not after:
|
|
763
|
+
break
|
|
764
|
+
if len(rows) >= max_rows:
|
|
765
|
+
warnings.warn(f"{path}: stopped at max_rows={max_rows}; raise it for the rest",
|
|
766
|
+
stacklevel=3)
|
|
767
|
+
return rows, True
|
|
768
|
+
if len(rows) >= max_rows and i + 1 < len(windows):
|
|
769
|
+
warnings.warn(f"{path}: stopped at max_rows={max_rows}; raise it for the rest",
|
|
770
|
+
stacklevel=3)
|
|
771
|
+
return rows, False
|
|
772
|
+
return rows, False
|
|
773
|
+
|
|
774
|
+
def get_fundamentals(
|
|
775
|
+
self,
|
|
776
|
+
ticker: str | None = None,
|
|
777
|
+
*,
|
|
778
|
+
cik: int | None = None,
|
|
779
|
+
fields: list[str] | str | None = None,
|
|
780
|
+
period: str = "quarter",
|
|
781
|
+
start: str | None = None,
|
|
782
|
+
end: str | None = None,
|
|
783
|
+
as_of: str | None = None,
|
|
784
|
+
version: str = "first",
|
|
785
|
+
wide: bool = True,
|
|
786
|
+
max_rows: int = 2_000_000,
|
|
787
|
+
) -> pd.DataFrame:
|
|
788
|
+
"""Company fundamentals from SEC filings, point in time.
|
|
789
|
+
|
|
790
|
+
32 standard fields (revenue, net_income, eps_diluted, total_assets,
|
|
791
|
+
operating_cash_flow, ...) for US companies that file 10-Ks and 10-Qs,
|
|
792
|
+
from 2009 on, refreshed nightly.
|
|
793
|
+
|
|
794
|
+
Examples::
|
|
795
|
+
|
|
796
|
+
client.get_fundamentals("AAPL", fields=["revenue", "eps_diluted"], start="2020-01-01")
|
|
797
|
+
client.get_fundamentals("AAPL", period="annual")
|
|
798
|
+
client.get_fundamentals(start="2025-01-01", end="2025-04-01", fields="revenue,net_income")
|
|
799
|
+
|
|
800
|
+
``period`` is "quarter" (fiscal quarters, with the balance sheet at
|
|
801
|
+
each quarter end) or "annual". ``start`` is inclusive and ``end``
|
|
802
|
+
exclusive, on ``period_end``. A market-wide pull (no ticker or
|
|
803
|
+
cik) needs both and is fetched a year at a time.
|
|
804
|
+
|
|
805
|
+
``version="first"`` (default) gives each value as first reported;
|
|
806
|
+
"latest" gives the newest restatement. ``as_of`` keeps only values
|
|
807
|
+
public by that moment: ``version="latest", as_of=...`` is what a
|
|
808
|
+
reader had then, which is what a backtest should use.
|
|
809
|
+
|
|
810
|
+
``wide=True`` (default) returns one row per company and period end
|
|
811
|
+
with a column per field, and ``known_at``: when the last of the
|
|
812
|
+
row's values became public (UTC). ``wide=False`` returns one row
|
|
813
|
+
per value, with its own ``known_at``, filing, SEC tag, and whether
|
|
814
|
+
it was ``derived`` (a fourth quarter as the year less nine months,
|
|
815
|
+
a cash-flow quarter from year-to-date figures). Per-share values
|
|
816
|
+
are never derived: a fourth quarter the company did not file on its
|
|
817
|
+
own has none.
|
|
818
|
+
|
|
819
|
+
``max_rows`` caps the values read, not wide rows. A capped read
|
|
820
|
+
warns, and the wide frame then leaves out the company-period the cap
|
|
821
|
+
cut short.
|
|
822
|
+
"""
|
|
823
|
+
if isinstance(fields, (list, tuple)):
|
|
824
|
+
fields = ",".join(fields)
|
|
825
|
+
windows = [(start, end)]
|
|
826
|
+
if ticker is None and cik is None:
|
|
827
|
+
if not (start and end):
|
|
828
|
+
raise ValueError("a market-wide pull needs start and end")
|
|
829
|
+
windows = _year_windows(start, end)
|
|
830
|
+
base = {"ticker": ticker, "cik": cik, "fields": fields, "period": period,
|
|
831
|
+
"as_of": as_of, "version": version}
|
|
832
|
+
rows, split = self._pages("/v1/fundamentals", base, windows, max_rows)
|
|
833
|
+
if split and wide:
|
|
834
|
+
# rows come in (period_end, cik, field) order: the cap may have cut
|
|
835
|
+
# the last company-period short, and a wide row needs all its fields
|
|
836
|
+
last = (rows[-1].get("period_end"), rows[-1].get("cik"))
|
|
837
|
+
rows = [r for r in rows if (r.get("period_end"), r.get("cik")) != last]
|
|
838
|
+
long = _fundamentals_long(rows)
|
|
839
|
+
return _fundamentals_wide(long, period) if wide else long
|
|
840
|
+
|
|
841
|
+
def get_earnings(
|
|
842
|
+
self,
|
|
843
|
+
ticker: str | None = None,
|
|
844
|
+
*,
|
|
845
|
+
cik: int | None = None,
|
|
846
|
+
start: str | None = None,
|
|
847
|
+
end: str | None = None,
|
|
848
|
+
date_field: str = "announced",
|
|
849
|
+
include_candidates: bool = False,
|
|
850
|
+
max_rows: int = 1_000_000,
|
|
851
|
+
) -> pd.DataFrame:
|
|
852
|
+
"""Earnings events from SEC filings: one row per period a company
|
|
853
|
+
reported, oldest first.
|
|
854
|
+
|
|
855
|
+
``announced_at`` is when the results went public (UTC, to the
|
|
856
|
+
second): the acceptance time of the 8-K that released them, or of the
|
|
857
|
+
10-Q/10-K itself when there was none (``announcement_source``).
|
|
858
|
+
``session`` is "pre", "regular" or "post" by New York clock time.
|
|
859
|
+
The figures (``revenue``, ``eps_diluted``, ...) are the report's own,
|
|
860
|
+
as public when it was accepted, with the same quarter a year earlier;
|
|
861
|
+
most reports from before 2011 were not machine-readable and have
|
|
862
|
+
times but no figures (the call warns). Upcoming dates and analyst
|
|
863
|
+
estimates are not included.
|
|
864
|
+
|
|
865
|
+
Examples::
|
|
866
|
+
|
|
867
|
+
client.get_earnings("AAPL")
|
|
868
|
+
client.get_earnings(start="2026-10-26", end="2026-11-07") # a market-wide window
|
|
869
|
+
|
|
870
|
+
``start``/``end`` bound ``date_field``: "announced" (New York dates)
|
|
871
|
+
or "period" (the period end). A market-wide pull needs both and is
|
|
872
|
+
fetched a year at a time.
|
|
873
|
+
"""
|
|
874
|
+
windows = [(start, end)]
|
|
875
|
+
if ticker is None and cik is None:
|
|
876
|
+
if not (start and end):
|
|
877
|
+
raise ValueError("a market-wide pull needs start and end")
|
|
878
|
+
windows = _year_windows(start, end)
|
|
879
|
+
base = {"ticker": ticker, "cik": cik, "date_field": date_field,
|
|
880
|
+
"include_candidates": str(bool(include_candidates)).lower()}
|
|
881
|
+
return _earnings_frame(self._pages("/v1/earnings", base, windows, max_rows)[0])
|
|
@@ -33,6 +33,7 @@ import json
|
|
|
33
33
|
import os
|
|
34
34
|
import tempfile
|
|
35
35
|
import threading
|
|
36
|
+
import time
|
|
36
37
|
import warnings
|
|
37
38
|
from datetime import date, timedelta
|
|
38
39
|
from typing import Any, Iterator, Sequence
|
|
@@ -1127,15 +1128,74 @@ class Batch(_Namespace):
|
|
|
1127
1128
|
"GET", f"/v2/batch/{job_id}/files"
|
|
1128
1129
|
).json()["files"]
|
|
1129
1130
|
|
|
1130
|
-
def hydrate(
|
|
1131
|
+
def hydrate(
|
|
1132
|
+
self,
|
|
1133
|
+
job_id: str,
|
|
1134
|
+
*,
|
|
1135
|
+
wait: bool = True,
|
|
1136
|
+
poll_seconds: float = 15.0,
|
|
1137
|
+
timeout: float | None = None,
|
|
1138
|
+
) -> dict:
|
|
1131
1139
|
"""Pull a finished extract into the cache.
|
|
1132
1140
|
|
|
1133
1141
|
After this, the same days are served by ``get_range`` from
|
|
1134
1142
|
storage — the job is a one-time cost, not a per-read one.
|
|
1143
|
+
|
|
1144
|
+
The service hydrates in the background and this returns its
|
|
1145
|
+
progress record. With ``wait`` (the default) it polls every
|
|
1146
|
+
``poll_seconds`` until the hydrate is ``done`` and returns the
|
|
1147
|
+
final record (``written``, ``empty``, ``skipped``); a hydrate
|
|
1148
|
+
that ``failed`` or was ``interrupted`` raises. ``timeout``
|
|
1149
|
+
bounds the wait in seconds (default: none). With ``wait=False``
|
|
1150
|
+
it returns at once; poll :meth:`hydrate_status` yourself.
|
|
1151
|
+
Calling it again while one is running is harmless and reports
|
|
1152
|
+
that one.
|
|
1135
1153
|
"""
|
|
1136
|
-
|
|
1154
|
+
# The full data timeout, as before 0.10.1, not the short one the
|
|
1155
|
+
# polls use: a service from before background hydrate still runs
|
|
1156
|
+
# the hydrate inside this request. A current one answers at once.
|
|
1157
|
+
status = self._client._request(
|
|
1137
1158
|
"POST", f"/v2/batch/{job_id}/hydrate", timeout=V2_DATA_TIMEOUT,
|
|
1138
1159
|
).json()
|
|
1160
|
+
if not wait or "state" not in status:
|
|
1161
|
+
# A service from before background hydrate answers with the
|
|
1162
|
+
# finished report and no ``state``.
|
|
1163
|
+
return status
|
|
1164
|
+
deadline = None if timeout is None else time.monotonic() + float(timeout)
|
|
1165
|
+
while True:
|
|
1166
|
+
state = status.get("state")
|
|
1167
|
+
if state == "done":
|
|
1168
|
+
return status
|
|
1169
|
+
if state in ("failed", "interrupted"):
|
|
1170
|
+
raise QuantPadDataError(
|
|
1171
|
+
f"hydrate of {job_id} {state}: "
|
|
1172
|
+
f"{status.get('error') or 'call hydrate() again to resume'}",
|
|
1173
|
+
None,
|
|
1174
|
+
status,
|
|
1175
|
+
)
|
|
1176
|
+
if deadline is not None and time.monotonic() >= deadline:
|
|
1177
|
+
raise TimeoutError(
|
|
1178
|
+
f"hydrate of {job_id} still {state} after {timeout}s "
|
|
1179
|
+
f"({status.get('files_done')}/{status.get('files_total')} "
|
|
1180
|
+
"files); it keeps running on the service -- poll "
|
|
1181
|
+
"hydrate_status()"
|
|
1182
|
+
)
|
|
1183
|
+
time.sleep(max(1.0, float(poll_seconds)))
|
|
1184
|
+
status = self.hydrate_status(job_id)
|
|
1185
|
+
|
|
1186
|
+
def hydrate_status(self, job_id: str) -> dict:
|
|
1187
|
+
"""Where this job's hydrate stands: ``state`` is
|
|
1188
|
+
``waiting_for_job``, ``running``, ``done``, ``failed`` or
|
|
1189
|
+
``interrupted``, with ``files_done`` / ``files_total`` and the
|
|
1190
|
+
counts so far."""
|
|
1191
|
+
return self._client._request(
|
|
1192
|
+
"GET", f"/v2/batch/{job_id}/hydrate", timeout=_HYDRATE_STATUS_TIMEOUT,
|
|
1193
|
+
).json()
|
|
1194
|
+
|
|
1195
|
+
|
|
1196
|
+
#: (connect, read) for a hydrate status poll, which the service answers at
|
|
1197
|
+
#: once. Short so one stalled poll cannot outlast the caller's ``timeout``.
|
|
1198
|
+
_HYDRATE_STATUS_TIMEOUT = (10, 120)
|
|
1139
1199
|
|
|
1140
1200
|
|
|
1141
1201
|
class Historical:
|
|
@@ -392,3 +392,158 @@ def test_module_level_insider_alias(monkeypatch):
|
|
|
392
392
|
|
|
393
393
|
def test_insider_helper_is_public():
|
|
394
394
|
assert "get_insider_transactions" in quantpad_data.__all__
|
|
395
|
+
|
|
396
|
+
|
|
397
|
+
def _fact(cik, ticker, field, end, value, known, kind="Q", fq=1, fy=2026):
|
|
398
|
+
return {"cik": cik, "ticker": ticker, "field": field, "period_kind": kind, "period_start": None,
|
|
399
|
+
"period_end": end, "fiscal_year": fy, "fiscal_quarter": fq, "value": value, "unit": "USD",
|
|
400
|
+
"version": 1, "derived": False, "source_tag": "T", "accession": "a", "form": "10-Q",
|
|
401
|
+
"filed_date": known[:10], "accepted_at": known, "known_at": known}
|
|
402
|
+
|
|
403
|
+
|
|
404
|
+
def test_get_fundamentals_pages_and_pivots_to_a_column_per_field():
|
|
405
|
+
import json
|
|
406
|
+
|
|
407
|
+
page1 = {"rows": [_fact(320193, "AAPL", "revenue", "2025-12-27", 143756000000, "2026-01-30T11:01:32+00:00"),
|
|
408
|
+
_fact(320193, "AAPL", "total_assets", "2025-12-27", "379297000000", "2026-01-30T11:01:32+00:00",
|
|
409
|
+
kind="I")],
|
|
410
|
+
"next_cursor": "2025-12-27|320193|total_assets"}
|
|
411
|
+
page2 = {"rows": [_fact(320193, "AAPL", "eps_diluted", "2026-03-28", 2.01, "2026-05-01T10:01:00+00:00", fq=2),
|
|
412
|
+
_fact(320193, "AAPL", "revenue", "2026-03-28", 111184000000, "2026-05-01T10:01:00+00:00", fq=2)],
|
|
413
|
+
"next_cursor": None}
|
|
414
|
+
session = FakeSession([response(body=json.dumps(page1).encode()), response(body=json.dumps(page2).encode())])
|
|
415
|
+
client = Client(api_key="secret", session=session)
|
|
416
|
+
|
|
417
|
+
df = client.get_fundamentals("aapl", fields=["revenue", "eps_diluted", "total_assets"], start="2025-10-01")
|
|
418
|
+
|
|
419
|
+
assert list(df["period_end"].dt.strftime("%Y-%m-%d")) == ["2025-12-27", "2026-03-28"]
|
|
420
|
+
assert df.loc[0, "revenue"] == 143756000000 and df.loc[0, "total_assets"] == 379297000000
|
|
421
|
+
assert pd.isna(df.loc[0, "eps_diluted"]) and df.loc[1, "eps_diluted"] == 2.01
|
|
422
|
+
assert str(df["known_at"].dt.tz) == "UTC" and df.loc[1, "known_at"] == pd.Timestamp("2026-05-01T10:01:00Z")
|
|
423
|
+
assert list(df.columns) == ["cik", "ticker", "period_end", "fiscal_year", "fiscal_quarter",
|
|
424
|
+
"eps_diluted", "revenue", "total_assets", "known_at"]
|
|
425
|
+
method, url, kwargs = session.calls[0]
|
|
426
|
+
assert method == "GET" and url.endswith("/v1/fundamentals")
|
|
427
|
+
params = kwargs["params"]
|
|
428
|
+
assert params["ticker"] == "aapl" and params["fields"] == "revenue,eps_diluted,total_assets"
|
|
429
|
+
assert params["period"] == "quarter" and params["version"] == "first" and params["start"] == "2025-10-01"
|
|
430
|
+
assert "end" not in params and "after" not in params and "as_of" not in params
|
|
431
|
+
assert session.calls[1][2]["params"]["after"] == "2025-12-27|320193|total_assets"
|
|
432
|
+
|
|
433
|
+
|
|
434
|
+
def test_an_annual_frame_keeps_the_year_and_its_balance_sheet_on_one_row():
|
|
435
|
+
import json
|
|
436
|
+
|
|
437
|
+
rows = [_fact(320193, "AAPL", "revenue", "2025-09-27", 416161000000, "2025-10-31T10:01:26+00:00", kind="FY", fq=None, fy=2025),
|
|
438
|
+
_fact(320193, "AAPL", "total_assets", "2025-09-27", 359241000000, "2025-10-31T10:01:26+00:00", kind="I", fq=4, fy=2025),
|
|
439
|
+
_fact(None, None, "revenue", "2025-09-27", 5, "2025-12-01T00:00:00+00:00", kind="FY", fq=None, fy=2025)]
|
|
440
|
+
rows[2]["cik"] = 999
|
|
441
|
+
session = FakeSession([response(body=json.dumps({"rows": rows, "next_cursor": None}).encode())])
|
|
442
|
+
df = Client(api_key="secret", session=session).get_fundamentals(start="2025-01-01", end="2026-01-01", period="annual")
|
|
443
|
+
assert len(df) == 2 and "fiscal_quarter" not in df
|
|
444
|
+
apple = df[df["cik"] == 320193].iloc[0]
|
|
445
|
+
assert apple["revenue"] == 416161000000 and apple["total_assets"] == 359241000000 and apple["fiscal_year"] == 2025
|
|
446
|
+
other = df[df["cik"] == 999].iloc[0]
|
|
447
|
+
assert pd.isna(other["ticker"]) and other["revenue"] == 5
|
|
448
|
+
|
|
449
|
+
|
|
450
|
+
def test_get_fundamentals_long_keeps_one_row_per_value():
|
|
451
|
+
import json
|
|
452
|
+
|
|
453
|
+
rows = [_fact(320193, "AAPL", "revenue", "2025-12-27", 1, "2026-01-30T11:01:32+00:00")]
|
|
454
|
+
session = FakeSession([response(body=json.dumps({"rows": rows, "next_cursor": None}).encode())])
|
|
455
|
+
df = Client(api_key="secret", session=session).get_fundamentals(
|
|
456
|
+
cik=320193, version="latest", as_of="2026-02-01T00:00:00Z", wide=False)
|
|
457
|
+
assert list(df.columns)[:3] == ["cik", "ticker", "field"] and len(df) == 1
|
|
458
|
+
params = session.calls[0][2]["params"]
|
|
459
|
+
assert params["cik"] == 320193 and params["version"] == "latest" and params["as_of"] == "2026-02-01T00:00:00Z"
|
|
460
|
+
|
|
461
|
+
|
|
462
|
+
def test_get_fundamentals_market_wide_needs_a_window_and_splits_by_year():
|
|
463
|
+
import json
|
|
464
|
+
|
|
465
|
+
with pytest.raises(ValueError):
|
|
466
|
+
Client(api_key="secret", session=FakeSession([])).get_fundamentals(fields="revenue")
|
|
467
|
+
empty = json.dumps({"rows": [], "next_cursor": None}).encode()
|
|
468
|
+
session = FakeSession([response(body=empty), response(body=empty)])
|
|
469
|
+
df = Client(api_key="secret", session=session).get_fundamentals(start="2024-01-01", end="2025-06-01")
|
|
470
|
+
assert df.empty
|
|
471
|
+
assert [(c[2]["params"]["start"], c[2]["params"]["end"]) for c in session.calls] == [
|
|
472
|
+
("2024-01-01", "2025-01-01"), ("2025-01-01", "2025-06-01")]
|
|
473
|
+
|
|
474
|
+
|
|
475
|
+
def test_a_capped_wide_read_leaves_out_the_period_it_cut_short():
|
|
476
|
+
import json
|
|
477
|
+
|
|
478
|
+
rows = [_fact(1, "A", "revenue", "2025-06-28", 1, "2025-08-01T10:00:00+00:00"),
|
|
479
|
+
_fact(1, "A", "total_assets", "2025-06-28", 2, "2025-08-01T10:00:00+00:00", kind="I"),
|
|
480
|
+
_fact(1, "A", "revenue", "2025-09-27", 3, "2025-10-31T10:00:00+00:00")]
|
|
481
|
+
page = json.dumps({"rows": rows, "next_cursor": "2025-09-27|1|revenue"}).encode()
|
|
482
|
+
session = FakeSession([response(body=page)])
|
|
483
|
+
with pytest.warns(UserWarning, match="max_rows=3"):
|
|
484
|
+
df = Client(api_key="secret", session=session).get_fundamentals(cik=1, max_rows=3)
|
|
485
|
+
assert len(session.calls) == 1 and session.calls[0][2]["params"]["limit"] == 3
|
|
486
|
+
assert list(df["period_end"].dt.strftime("%Y-%m-%d")) == ["2025-06-28"] and df.loc[0, "total_assets"] == 2
|
|
487
|
+
|
|
488
|
+
session = FakeSession([response(body=page)])
|
|
489
|
+
with pytest.warns(UserWarning):
|
|
490
|
+
long = Client(api_key="secret", session=session).get_fundamentals(cik=1, max_rows=3, wide=False)
|
|
491
|
+
assert len(long) == 3
|
|
492
|
+
|
|
493
|
+
|
|
494
|
+
def test_an_exact_fit_does_not_warn(recwarn):
|
|
495
|
+
import json
|
|
496
|
+
|
|
497
|
+
rows = [_fact(1, "A", "revenue", "2025-06-28", 1, "2025-08-01T10:00:00+00:00")]
|
|
498
|
+
session = FakeSession([response(body=json.dumps({"rows": rows, "next_cursor": None}).encode())])
|
|
499
|
+
assert len(Client(api_key="secret", session=session).get_fundamentals(cik=1, max_rows=1)) == 1
|
|
500
|
+
assert not [w for w in recwarn if "max_rows" in str(w.message)]
|
|
501
|
+
|
|
502
|
+
|
|
503
|
+
def test_get_earnings_parses_times_and_figures():
|
|
504
|
+
import json
|
|
505
|
+
|
|
506
|
+
row = {"cik": 320193, "ticker": "AAPL", "period_end": "2026-09-26", "fiscal_year": 2026, "fiscal_quarter": 4,
|
|
507
|
+
"announced_at": "2026-11-02T21:30:23+00:00", "report_accepted_at": "2026-11-03T13:05:15+00:00",
|
|
508
|
+
"session": "post", "announcement_source": "8-K 2.02", "revenue": 114477000000, "eps_diluted": None,
|
|
509
|
+
"fy_eps_diluted": "8.98", "eps_diluted_year_ago": None}
|
|
510
|
+
session = FakeSession([response(body=json.dumps({"rows": [row], "next_cursor": None}).encode())])
|
|
511
|
+
df = Client(api_key="secret", session=session).get_earnings("AAPL", include_candidates=True)
|
|
512
|
+
assert df.loc[0, "announced_at"] == pd.Timestamp("2026-11-02T21:30:23Z")
|
|
513
|
+
assert df.loc[0, "fy_eps_diluted"] == 8.98 and pd.isna(df.loc[0, "eps_diluted"])
|
|
514
|
+
method, url, kwargs = session.calls[0]
|
|
515
|
+
assert url.endswith("/v1/earnings")
|
|
516
|
+
assert kwargs["params"]["include_candidates"] == "true" and kwargs["params"]["date_field"] == "announced"
|
|
517
|
+
with pytest.raises(ValueError):
|
|
518
|
+
Client(api_key="secret", session=FakeSession([])).get_earnings()
|
|
519
|
+
|
|
520
|
+
|
|
521
|
+
def test_events_without_figures_say_why():
|
|
522
|
+
import json
|
|
523
|
+
|
|
524
|
+
rows = [{"cik": 1, "period_end": "2006-03-31", "announced_at": "2006-04-20T20:05:00+00:00",
|
|
525
|
+
"revenue": None, "eps_diluted": None, "revenue_year_ago": None},
|
|
526
|
+
{"cik": 1, "period_end": "2012-03-31", "announced_at": "2012-04-20T20:05:00+00:00",
|
|
527
|
+
"revenue": "100", "eps_diluted": "0.5", "revenue_year_ago": None}]
|
|
528
|
+
session = FakeSession([response(body=json.dumps({"rows": rows, "next_cursor": None}).encode())])
|
|
529
|
+
with pytest.warns(UserWarning, match="1 of 2 earnings events have no figures"):
|
|
530
|
+
Client(api_key="secret", session=session).get_earnings(cik=1)
|
|
531
|
+
|
|
532
|
+
|
|
533
|
+
def test_fundamentals_helpers_are_public(monkeypatch):
|
|
534
|
+
captured = {}
|
|
535
|
+
|
|
536
|
+
class Stub:
|
|
537
|
+
def get_fundamentals(self, ticker, **kwargs):
|
|
538
|
+
captured["f"] = (ticker, kwargs)
|
|
539
|
+
return "f"
|
|
540
|
+
|
|
541
|
+
def get_earnings(self, ticker, **kwargs):
|
|
542
|
+
captured["e"] = (ticker, kwargs)
|
|
543
|
+
return "e"
|
|
544
|
+
|
|
545
|
+
monkeypatch.setattr(quantpad_data, "_client", lambda **kwargs: Stub())
|
|
546
|
+
assert quantpad_data.get_fundamentals("AAPL", period="annual") == "f"
|
|
547
|
+
assert quantpad_data.get_earnings("AAPL") == "e"
|
|
548
|
+
assert captured == {"f": ("AAPL", {"period": "annual"}), "e": ("AAPL", {})}
|
|
549
|
+
assert {"get_fundamentals", "get_earnings"} <= set(quantpad_data.__all__)
|
|
@@ -2615,3 +2615,54 @@ class TestExpiredUrlsAreResigned:
|
|
|
2615
2615
|
assert "sig=hydrated" in frames[2]["url"][0]
|
|
2616
2616
|
single = session.calls[3][2]["json"]
|
|
2617
2617
|
assert (single["start"], single["hydrate"]) == ("2026-01-07", True)
|
|
2618
|
+
|
|
2619
|
+
|
|
2620
|
+
class TestBatchHydrate:
|
|
2621
|
+
"""2026-11-20: hydrate ran inside one request, and a two-hour options
|
|
2622
|
+
hydrate timed out at thirty minutes while the service carried on."""
|
|
2623
|
+
|
|
2624
|
+
def _no_sleep(self, monkeypatch):
|
|
2625
|
+
import quantpad_data.v2 as v2
|
|
2626
|
+
|
|
2627
|
+
monkeypatch.setattr(v2.time, "sleep", lambda _s: None)
|
|
2628
|
+
|
|
2629
|
+
def test_it_polls_until_the_hydrate_is_done(self, monkeypatch):
|
|
2630
|
+
self._no_sleep(monkeypatch)
|
|
2631
|
+
api, session = client_with([
|
|
2632
|
+
json_response({"job_id": "j", "state": "running"}, status=202),
|
|
2633
|
+
json_response({"job_id": "j", "state": "running", "files_done": 1}),
|
|
2634
|
+
json_response({"job_id": "j", "state": "done", "written": 3}),
|
|
2635
|
+
])
|
|
2636
|
+
result = api.batch.hydrate("j")
|
|
2637
|
+
assert result["written"] == 3
|
|
2638
|
+
assert [(m, u.rsplit("/v2", 1)[1]) for m, u, _ in session.calls] == [
|
|
2639
|
+
("POST", "/batch/j/hydrate"),
|
|
2640
|
+
("GET", "/batch/j/hydrate"),
|
|
2641
|
+
("GET", "/batch/j/hydrate"),
|
|
2642
|
+
]
|
|
2643
|
+
# An older service hydrates inside the POST, so it keeps the full
|
|
2644
|
+
# budget; the polls are short.
|
|
2645
|
+
from quantpad_data.client import V2_DATA_TIMEOUT
|
|
2646
|
+
|
|
2647
|
+
assert session.calls[0][2]["timeout"] == V2_DATA_TIMEOUT
|
|
2648
|
+
assert [c[2]["timeout"] for c in session.calls[1:]] == [(10, 120)] * 2
|
|
2649
|
+
|
|
2650
|
+
def test_without_wait_it_returns_the_first_record(self):
|
|
2651
|
+
api, session = client_with([
|
|
2652
|
+
json_response({"job_id": "j", "state": "running"}, status=202),
|
|
2653
|
+
])
|
|
2654
|
+
assert api.batch.hydrate("j", wait=False)["state"] == "running"
|
|
2655
|
+
assert len(session.calls) == 1
|
|
2656
|
+
|
|
2657
|
+
def test_a_failed_hydrate_raises_with_its_reason(self, monkeypatch):
|
|
2658
|
+
self._no_sleep(monkeypatch)
|
|
2659
|
+
api, _ = client_with([
|
|
2660
|
+
json_response({"job_id": "j", "state": "running"}, status=202),
|
|
2661
|
+
json_response({"job_id": "j", "state": "failed", "error": "boom"}),
|
|
2662
|
+
])
|
|
2663
|
+
with pytest.raises(QuantPadDataError, match="boom"):
|
|
2664
|
+
api.batch.hydrate("j")
|
|
2665
|
+
|
|
2666
|
+
def test_an_older_service_s_finished_report_is_returned_as_it_is(self):
|
|
2667
|
+
api, _ = client_with([json_response({"job_id": "j", "written": 2})])
|
|
2668
|
+
assert api.batch.hydrate("j") == {"job_id": "j", "written": 2}
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|