quantpad-data 0.5.1__tar.gz → 0.7.0__tar.gz

{quantpad_data-0.5.1 → quantpad_data-0.7.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: quantpad-data
-Version: 0.5.1
+Version: 0.7.0
 Summary: Official Python client for the QuantPad market-data API
 Project-URL: Documentation, https://api.quantpad.ai/external/docs
 Author: QuantPad
@@ -73,6 +73,11 @@ coverage = qpd.get_coverage("AAPL")
 `QuantPadClient` (also available as `Client`) accepts `api_key=`, `base_url=`,
 and `max_retries=`. The default
 client honors `Retry-After` and uses exponential backoff for transient failures.
+A read timeout is the exception and is never retried: the budget was already
+spent in full, so the request is probably still running on the server, and
+repeating it adds load rather than replacing it. Connection failures and
+connect timeouts are still retried, because failing to reach the service
+costs the service nothing.
 QuantPad notebooks remain compatible through `QUANTPAD_NOTEBOOK_DATA_TOKEN`.
 
 Bars preserve the notebook helper's OHLCV aliases and smart futures

{quantpad_data-0.5.1 → quantpad_data-0.7.0}/README.md

@@ -51,6 +51,11 @@ coverage = qpd.get_coverage("AAPL")
 `QuantPadClient` (also available as `Client`) accepts `api_key=`, `base_url=`,
 and `max_retries=`. The default
 client honors `Retry-After` and uses exponential backoff for transient failures.
+A read timeout is the exception and is never retried: the budget was already
+spent in full, so the request is probably still running on the server, and
+repeating it adds load rather than replacing it. Connection failures and
+connect timeouts are still retried, because failing to reach the service
+costs the service nothing.
 QuantPad notebooks remain compatible through `QUANTPAD_NOTEBOOK_DATA_TOKEN`.
 
 Bars preserve the notebook helper's OHLCV aliases and smart futures

{quantpad_data-0.5.1 → quantpad_data-0.7.0}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
 
 [project]
 name = "quantpad-data"
-version = "0.5.1"
+version = "0.7.0"
 description = "Official Python client for the QuantPad market-data API"
 readme = "README.md"
 requires-python = ">=3.10"

{quantpad_data-0.5.1 → quantpad_data-0.7.0}/quantpad_data/client.py

@@ -84,7 +84,7 @@ V2_NOT_AVAILABLE = (
     "QuantPad support about /v2 access."
 )
 
-VERSION = "0.5.1"
+VERSION = "0.7.0"
 USER_AGENT = f"quantpad-data-python/{VERSION}"
 TICK_SCHEMAS = (
     "trades", "mbp-1", "mbp-10", "cmbp-1", "tcbbo", "cbbo-1s",
@@ -114,6 +114,34 @@ ROLL_ADJUST_MODES = ("none", "back", "ratio")
 #: server latency and belongs in the engine, not in a client's patience.
 BARS_TIMEOUT = (30, 1800)
 
+#: (connect, read) budget for a ``/v2`` route that has to fetch.
+#:
+#: ``/v2`` inherited the bare ``self.timeout`` -- 120 seconds, a SCALAR --
+#: which is the mistake ``BARS_TIMEOUT`` above exists to correct, made
+#: again on the surface meant to replace bars.
+#:
+#: It lands harder here.  A scalar timeout is a cap on TIME TO FIRST
+#: BYTE, and ``/v2`` sends no first byte until its fetch is finished:
+#: ``get_range`` plans, prices, fetches every missing day from upstream
+#: and only then primes the first Arrow batch.  So the number is not a
+#: patience setting, it is a guess about how long upstream takes --
+#: made once, for every schema and every width.
+#:
+#: Measured against that guess: a cold fourteen-day crude-options
+#: ``mbp-1`` window spent its entire budget inside the fetch and failed
+#: work that was going fine.  And since the service learned to stop
+#: serving a caller who has left, the guess is now what *ends* the work
+#: rather than merely what stops waiting for it.
+#:
+#: 1800 is what ``/v1/ticks`` and ``/v1/bars`` already allow for the same
+#: kind of work, so this is ``/v2`` catching up with ``/v1`` rather than
+#: a new indulgence.  It has to be this large only because nothing on
+#: this socket yet says "still working": a read budget can be short when
+#: silence means death and must be long when silence is normal.  Once
+#: the service reports progress while it fetches, this becomes a real
+#: stall detector and comes down to a couple of minutes.
+V2_DATA_TIMEOUT = (10, 1800)
+
 
 class Client:
     def __init__(
@@ -236,6 +264,31 @@ class Client:
                 if attempt >= self.max_retries:
                     raise
                 delay = exc.retry_after
+            except requests.ReadTimeout:
+                # The one thing never retried, and the exception proves the
+                # rule rather than breaking it: everything else in these
+                # clauses is cheap to repeat, and this is the opposite.
+                #
+                # A read timeout means the whole budget above was spent
+                # without the server going quiet by choice, so the work is
+                # very likely still running.  Repeating the request does
+                # not replace that work, it adds to it -- the comment on
+                # BARS_TIMEOUT measured where that leads: three retries
+                # turning one slow call into four concurrent copies, each
+                # holding one of the eight stream slots a key is allowed,
+                # which is a self-inflicted 429.  Worse now than when that
+                # was written, because the service cancels for a caller
+                # who has left, so each retry also destroys the progress
+                # the one before it had made.
+                #
+                # A caller is better served by the truth after one budget
+                # than by the same answer after four.  ConnectTimeout is
+                # subclassed from ConnectionError as well and so is still
+                # caught below, which is right: failing to reach the
+                # service at all costs the service nothing.
+                if response is not None:
+                    response.close()
+                raise
             except (requests.ConnectionError, requests.Timeout, UpstreamError) as exc:
                 if response is not None:
                     response.close()

{quantpad_data-0.5.1 → quantpad_data-0.7.0}/quantpad_data/v2.py

@@ -37,6 +37,7 @@ from __future__ import annotations
 import io
 import json
 import os
+import tempfile
 from datetime import date, timedelta
 from typing import Any, Iterator, Sequence
 
@@ -47,7 +48,7 @@ import pyarrow.parquet as pq
 import requests
 
 from . import cache, memory, rth
-from .client import Client
+from .client import V2_DATA_TIMEOUT, Client
 from .errors import QuantPadDataError, ResponseTooLargeError, ValidationError
 
 #: Above this, ``get_range`` stops buffering a single response and reads
@@ -56,6 +57,42 @@ from .errors import QuantPadDataError, ResponseTooLargeError, ValidationError
 #: in a notebook degrades into a slower read rather than a failure.
 _BULK_FALLBACK_STATUS = 413
 
+#: (connect, read) budget for a presigned read from object storage.
+#:
+#: Deliberately far shorter than :data:`V2_DATA_TIMEOUT`.  There is no
+#: work behind an object download -- no planning, no upstream fetch, no
+#: transcode -- so nothing here can legitimately take minutes to begin,
+#: and silence really is a stall.  This is the one place a read budget
+#: does the job it is named for; handing it the service's budget would
+#: let a dead transfer hang a notebook for half an hour.
+STORAGE_TIMEOUT = (10, 120)
+
+#: Bytes of a partition held in memory before the download spills to
+#: disk.  Most partitions are far under this; a depth day is not, and
+#: that is exactly the case where holding the compressed body and the
+#: decoded table at full size at once is worth avoiding.
+STORAGE_SPOOL_BYTES = 32 * 1024 * 1024
+
+#: Widest window of missing days :meth:`Timeseries.iter_range` fetches
+#: in one request, once it has ramped up to it.
+#:
+#: A cadence, not a capacity.  The service's own ceilings decide what
+#: fits and refuse what does not, and :func:`_split_window` already
+#: halves a window that gets refused, so nothing here is protecting
+#: memory.  What it buys is the caller seeing frames while the fetch is
+#: still going, which one request for the whole window cannot give
+#: however fast it is: measured on a cold month of niche-equity
+#: ``mbp-10``, 129.5 of the 132.4 seconds elapsed before the first
+#: frame, because one manifest was hydrating nineteen days inside the
+#: request.  That is indistinguishable from a hang, and it is the shape
+#: of the "all projects hang, no error" report this exists to answer.
+#:
+#: Five caps a ramp that starts at one day (see :func:`_runs`), so a
+#: cold month is a handful of requests rather than one or nineteen, and
+#: the first of them is small.  A warm window never reaches this at
+#: all -- it has no gaps, so it stays the single manifest it always was.
+HYDRATE_CHUNK_DAYS = 5
+
 
 def _symbols(value: str | Sequence[str]) -> list[str]:
     return [value] if isinstance(value, str) else [str(v) for v in value]
@@ -130,6 +167,66 @@ def _split_window(start: str, end: str) -> tuple[tuple[str, str], ...] | None:
     return ((start, middle), (middle, end))
 
 
+def _identity(item: dict) -> tuple:
+    """What makes two manifest entries the same stored partition.
+
+    The object key, because it is what the service stores the partition
+    under and is therefore stable from one manifest to the next.  The
+    obvious alternative, ``url``, is a freshly signed URL on every call
+    and would make every entry unique -- which is to say it would look
+    like it worked and deduplicate nothing.
+
+    Day and instrument ride along so that a service which omits ``key``
+    still distinguishes one instrument-day from another, rather than
+    collapsing a whole window onto a single ``None``.
+    """
+    return (item.get("key"), item["day"], item["instrument_id"])
+
+
+def _runs(days: Sequence[str], size: int) -> Iterator[tuple[str, str]]:
+    """Windows covering ``days``, small at first and then up to ``size``.
+
+    Consecutive days are grouped, because a run of missing days is one
+    fetch rather than several: the days between two gaps are already
+    cached and asking for them again would re-fetch what the caller
+    has.  Groups are then cut into windows, because a run can be a
+    whole month and that is the silence being removed.
+
+    **The first window is one day, and each is twice the last up to**
+    ``size``.  A flat cut makes the first frame wait for a full chunk,
+    which measured 29.0s on a cold 8-day window of equity depth -- an
+    improvement on the 58.0s it replaced and still long enough to read
+    as nothing happening.  Ramping puts the first frame after a single
+    day, and by the time the caller is watching output arrive the
+    windows are full width, so the round trips are spent where they buy
+    reassurance rather than spread over the whole fetch.
+
+    Yields half-open ``(start, end)`` pairs, ``end`` being the midnight
+    after the window's last day.  Day-aligned for :func:`_split_window`'s
+    reason -- a partition is a whole day, so a boundary inside one would
+    put that day in both windows and return its rows twice.
+    """
+    ordered = sorted({date.fromisoformat(day) for day in days})
+    groups: list[list[date]] = []
+    for day in ordered:
+        if groups and day == groups[-1][-1] + timedelta(days=1):
+            groups[-1].append(day)
+        else:
+            groups.append([day])
+
+    # Carried ACROSS groups, not reset per group: the ramp is about how
+    # long the caller has been waiting, and a second group is reached
+    # only after the first has already produced frames.
+    width = 1
+    for group in groups:
+        index = 0
+        while index < len(group):
+            piece = group[index : index + width]
+            index += width
+            width = min(width * 2, size)
+            yield piece[0].isoformat(), (piece[-1] + timedelta(days=1)).isoformat()
+
+
 def _table_from_arrow(payload: bytes) -> pa.Table:
     with ipc.open_stream(io.BytesIO(payload)) as reader:
         return reader.read_all()
@@ -271,7 +368,8 @@ class Timeseries(_Namespace):
 
         try:
             response = self._client._request(
-                "POST", "/v2/timeseries/get_range", json=body
+                "POST", "/v2/timeseries/get_range", json=body,
+                timeout=V2_DATA_TIMEOUT,
             )
         except ResponseTooLargeError as exc:
             # Bulk answers OUR ceiling and does not answer the caller's own
@@ -331,6 +429,7 @@ class Timeseries(_Namespace):
                 start=start, end=end, stype_in=stype_in,
                 stype_out=stype_out, columns=columns,
             ),
+            timeout=V2_DATA_TIMEOUT,
         ).json()
 
     def iter_range(self, **kwargs) -> Iterator[pd.DataFrame]:
@@ -339,8 +438,13 @@ class Timeseries(_Namespace):
         For ranges that do not fit in memory.  Each yielded frame is one
         (day, instrument) partition, in chronological order.
 
-        Asks for the window in as few pieces as the service will allow,
-        which is one piece whenever the days are already cached.
+        **Streams the fetch, not only the answer.**  The first call asks
+        the manifest not to hydrate, so it returns in about a second
+        however cold the window is and says which days it does not have.
+        Cached days are then yielded straight away and the missing ones
+        fetched in runs of :data:`HYDRATE_CHUNK_DAYS` as the iteration
+        reaches them.  A warm window is still one round trip; a cold one
+        produces frames throughout instead of after a single silence.
         """
         if kwargs.get("roll_adjust", "none") != "none" or kwargs.get("resample"):
             raise ValidationError(
@@ -355,18 +459,17 @@ class Timeseries(_Namespace):
         # day, instrument, projection), so the whole-window body is the right
         # thing to hand every piece.
         manifest_body = _manifest_body(columns=columns, **kwargs)
-        for payload in self._manifests(manifest_body):
-            for item in payload["files"]:
-                frame = _labelled(
-                    _partition(manifest_body, item, columns), item, columns
-                )
-                frame.attrs["quantpad"] = {
-                    **_manifest_attrs(payload),
-                    "day": item["day"],
-                    "instrument_id": item["instrument_id"],
-                    "symbol": item.get("symbol"),
-                }
-                yield frame
+        for payload, item in self._partitions(manifest_body):
+            frame = _labelled(
+                _partition(manifest_body, item, columns), item, columns
+            )
+            frame.attrs["quantpad"] = {
+                **_manifest_attrs(payload),
+                "day": item["day"],
+                "instrument_id": item["instrument_id"],
+                "symbol": item.get("symbol"),
+            }
+            yield frame
 
     def iter_range_rth(self, **kwargs) -> Iterator[pd.DataFrame]:
         """:meth:`iter_range`, restricted to XNYS regular trading hours.
@@ -480,7 +583,103 @@ class Timeseries(_Namespace):
             exc.detail,
         )
 
-    def _manifests(self, manifest_body: dict) -> Iterator[dict]:
+    def _partitions(self, manifest_body: dict) -> Iterator[tuple[dict, dict]]:
+        """``(manifest, file)`` pairs in day order, fetching as it goes.
+
+        The window is surveyed once without hydrating -- cheap whatever
+        the cache holds, measured at 0.9s against 29.6s for the same
+        cold window fetched -- and the survey says which days are
+        missing.  Days already present are handed over immediately;
+        missing ones are fetched only when the walk reaches them, so a
+        caller whose window is half warm sees frames before any fetch
+        starts at all.
+
+        **Chronological across the join**, which is the constraint that
+        shapes this: yielding every cached day and then every fetched
+        one would be simpler and would reorder the window, and
+        ``iter_range`` promises day order.  So the two are interleaved
+        by day rather than concatenated.
+
+        **A calendar day is not the unit of cache state**, which is the
+        other constraint and much the less obvious one.  A manifest
+        speaks in instrument-days, so asking for two symbols whose
+        histories were fetched at different times puts one date in
+        ``files`` for one symbol and in ``gaps`` for the other.  The
+        fetch for that date then answers with BOTH instruments -- it is
+        given a window, not a list of holes -- so the already-cached one
+        arrives a second time, and a caller concatenating the frames
+        would silently double those rows.  Hence :func:`_identity`: the
+        day decides WHEN a partition is handed over, the partition
+        itself decides whether it already has been.
+
+        Degrades rather than breaks against a service that predates the
+        flag: an older manifest ignores ``hydrate`` (the request models
+        do not forbid unknown fields, so it is dropped rather than
+        rejected), fetches anyway and answers with no gaps, which this
+        reads as a fully cached window and streams directly.  Slower,
+        never wrong -- and it keeps the splitting it would otherwise
+        have lost, since the survey goes through :meth:`_manifests`
+        rather than asking once.  Against a service that honours the
+        flag that costs nothing: nothing is fetched, so there is no
+        hydration ceiling to breach and the splitter loops once.
+        """
+        surveyed = list(self._manifests(manifest_body, hydrate=False))
+
+        # No early return for the all-cached case, though one reads
+        # naturally here and was written first.  The walk below already
+        # handles it in one round trip -- there is nothing to fetch, so
+        # nothing is fetched -- and it additionally puts the days in
+        # order, where handing back ``survey["files"]`` would inherit
+        # whatever order the service happened to list them in.  A
+        # mutation battery is what established the two were equivalent:
+        # the shortcut turned out to be the weaker of the two.
+        ready: dict[str, list[tuple[dict, dict]]] = {}
+        missing: set[str] = set()
+        for payload in surveyed:
+            for item in payload["files"]:
+                ready.setdefault(item["day"], []).append((payload, item))
+            for gap in payload.get("gaps") or ():
+                missing.add(gap["day"])
+
+        seen: set[tuple] = set()
+
+        def fresh(pairs) -> Iterator[tuple[dict, dict]]:
+            for payload, item in pairs:
+                identity = _identity(item)
+                if identity not in seen:
+                    seen.add(identity)
+                    yield payload, item
+
+        # One cursor over the union of both sets, so a fetched day lands
+        # between the cached days that surround it.
+        fetched: dict[str, list[tuple[dict, dict]]] = {}
+        pending = list(_runs(sorted(missing), HYDRATE_CHUNK_DAYS))
+        for day in sorted(set(ready) | missing):
+            while pending and pending[0][0] <= day:
+                start, end = pending.pop(0)
+                for payload in self._manifests(
+                    {**manifest_body, "start": start, "end": end}
+                ):
+                    for item in payload["files"]:
+                        fetched.setdefault(item["day"], []).append((payload, item))
+            yield from fresh(ready.get(day, ()))
+            # A missing day that came back with nothing -- a holiday, or
+            # a genuinely quiet session -- simply has no entry here, and
+            # is skipped exactly as an ``empty`` day always was.
+            yield from fresh(fetched.pop(day, ()))
+
+        # Anything a fetch returned for a day the survey did not list.
+        # It should be empty, and it is drained rather than asserted
+        # about, because the two calls resolve symbols independently and
+        # the failure of the cursor to reach a day would otherwise drop
+        # its rows in silence -- the one outcome worth ruling out by
+        # construction.
+        for day in sorted(fetched):
+            yield from fresh(fetched[day])
+
+    def _manifests(
+        self, manifest_body: dict, *, hydrate: bool = True
+    ) -> Iterator[dict]:
         """Manifests covering the window, splitting it when the service says to.
 
         The manifest hydrates its missing days synchronously, under its own
@@ -497,6 +696,13 @@ class Timeseries(_Namespace):
         a property of the data rather than of the calendar -- a month of
         options trades and a month of equity depth differ by orders of
         magnitude -- and because the warm case must stay one round trip.
+
+        ``hydrate`` rides through the split rather than being applied by
+        the caller afterwards, so a survey that has to be halved stays a
+        survey.  Only a service predating the flag can put it in that
+        position -- one that honours it fetches nothing and so cannot
+        refuse on a hydration ceiling -- but that is exactly the service
+        whose refusal this exists to absorb.
         """
         pending = [(manifest_body["start"], manifest_body["end"])]
         while pending:
@@ -504,7 +710,13 @@ class Timeseries(_Namespace):
             try:
                 payload = self._client._request(
                     "POST", "/v2/timeseries/manifest",
-                    json={**manifest_body, "start": start, "end": end},
+                    json={
+                        **manifest_body,
+                        "start": start,
+                        "end": end,
+                        "hydrate": hydrate,
+                    },
+                    timeout=V2_DATA_TIMEOUT,
                 ).json()
             except ResponseTooLargeError as exc:
                 if exc.reason != "hydration_limit":
@@ -793,7 +1005,7 @@ class Batch(_Namespace):
         storage — the job is a one-time cost, not a per-read one.
         """
         return self._client._request(
-            "POST", f"/v2/batch/{job_id}/hydrate"
+            "POST", f"/v2/batch/{job_id}/hydrate", timeout=V2_DATA_TIMEOUT,
         ).json()
 
 
@@ -1040,12 +1252,28 @@ def _read_parquet_url(url: str, columns: Sequence[str] | None) -> pd.DataFrame:
     point of the manifest path is that these bytes never traverse the
     data service.
     """
-    response = requests.get(url, timeout=300)
-    response.raise_for_status()
-    table = pq.read_table(
-        io.BytesIO(response.content),
-        columns=list(columns) if columns else None,
-    )
+    # ``with``, because ``stream=True`` leaves the body unread until it
+    # is: a ``raise_for_status`` on the next line would otherwise abandon
+    # the connection holding one, and a manifest walks hundreds of these.
+    with requests.get(
+        url, timeout=STORAGE_TIMEOUT, stream=True
+    ) as response:
+        response.raise_for_status()
+        # Spooled rather than ``io.BytesIO(response.content)`` so the
+        # compressed bytes and the decoded table are not both resident
+        # at their full size: past the threshold the download goes to
+        # disk and only the table is in memory.  Streaming without this
+        # buys nothing -- ``.content`` materialises the whole body
+        # anyway, just later.
+        with tempfile.SpooledTemporaryFile(
+            max_size=STORAGE_SPOOL_BYTES
+        ) as spool:
+            for chunk in response.iter_content(chunk_size=1024 * 1024):
+                spool.write(chunk)
+            spool.seek(0)
+            table = pq.read_table(
+                spool, columns=list(columns) if columns else None
+            )
     return table.to_pandas()
 
 

{quantpad_data-0.5.1 → quantpad_data-0.7.0}/tests/test_client.py

@@ -104,6 +104,56 @@ def test_the_csv_fallback_keeps_the_long_read_budget():
     assert session.calls[1][2]["params"]["format"] == "csv"
 
 
+class _RaisingSession:
+    """A session that raises the same exception on every attempt."""
+
+    def __init__(self, exc):
+        self.headers = {}
+        self.exc = exc
+        self.calls = 0
+
+    def request(self, method, url, **kwargs):
+        self.calls += 1
+        raise self.exc
+
+
+def test_a_read_timeout_is_not_retried():
+    """Retrying a read timeout adds work rather than replacing it.
+
+    The budget was spent in full without the server choosing to go quiet,
+    so the request is very likely still running. ``BARS_TIMEOUT``'s own
+    comment measured where repeating it leads: three retries turn one
+    slow call into four concurrent copies, each holding one of the eight
+    stream slots a key is allowed, which is a self-inflicted 429. Worse
+    since the service cancels for a caller who left, because each retry
+    now also destroys the progress the one before it made.
+    """
+    session = _RaisingSession(requests.ReadTimeout("read timed out"))
+    client = Client(api_key="secret", session=session, max_retries=3)
+
+    with pytest.raises(requests.ReadTimeout):
+        client.get_coverage("AAPL")
+
+    assert session.calls == 1, "a read timeout must cost exactly one attempt"
+
+
+def test_a_connect_timeout_is_still_retried():
+    """The other half, and the reason this is a rule rather than a mood.
+
+    Failing to reach the service at all costs the service nothing, so it
+    is cheap to repeat -- which is exactly what a read timeout is not.
+    ConnectTimeout subclasses Timeout, so a rule written on Timeout alone
+    would have stopped retrying the one case that should be.
+    """
+    session = _RaisingSession(requests.ConnectTimeout("connect timed out"))
+    client = Client(api_key="secret", session=session, max_retries=2)
+
+    with pytest.raises(requests.ConnectTimeout):
+        client.get_coverage("AAPL")
+
+    assert session.calls == 3, "connect failures are cheap and still retried"
+
+
 def test_rate_limit_exposes_retry_after():
     session = FakeSession(
         [response(429, b'{"detail":"rate limit"}', {"Retry-After": "7"})]