quantpad-data 0.5.1__tar.gz → 0.6.0__tar.gz

{quantpad_data-0.5.1 → quantpad_data-0.6.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: quantpad-data
-Version: 0.5.1
+Version: 0.6.0
 Summary: Official Python client for the QuantPad market-data API
 Project-URL: Documentation, https://api.quantpad.ai/external/docs
 Author: QuantPad
@@ -73,6 +73,11 @@ coverage = qpd.get_coverage("AAPL")
 `QuantPadClient` (also available as `Client`) accepts `api_key=`, `base_url=`,
 and `max_retries=`. The default
 client honors `Retry-After` and uses exponential backoff for transient failures.
+A read timeout is the exception and is never retried: the budget was already
+spent in full, so the request is probably still running on the server, and
+repeating it adds load rather than replacing it. Connection failures and
+connect timeouts are still retried, because failing to reach the service
+costs the service nothing.
 QuantPad notebooks remain compatible through `QUANTPAD_NOTEBOOK_DATA_TOKEN`.
 
 Bars preserve the notebook helper's OHLCV aliases and smart futures

{quantpad_data-0.5.1 → quantpad_data-0.6.0}/README.md

@@ -51,6 +51,11 @@ coverage = qpd.get_coverage("AAPL")
 `QuantPadClient` (also available as `Client`) accepts `api_key=`, `base_url=`,
 and `max_retries=`. The default
 client honors `Retry-After` and uses exponential backoff for transient failures.
+A read timeout is the exception and is never retried: the budget was already
+spent in full, so the request is probably still running on the server, and
+repeating it adds load rather than replacing it. Connection failures and
+connect timeouts are still retried, because failing to reach the service
+costs the service nothing.
 QuantPad notebooks remain compatible through `QUANTPAD_NOTEBOOK_DATA_TOKEN`.
 
 Bars preserve the notebook helper's OHLCV aliases and smart futures

{quantpad_data-0.5.1 → quantpad_data-0.6.0}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
 
 [project]
 name = "quantpad-data"
-version = "0.5.1"
+version = "0.6.0"
 description = "Official Python client for the QuantPad market-data API"
 readme = "README.md"
 requires-python = ">=3.10"

{quantpad_data-0.5.1 → quantpad_data-0.6.0}/quantpad_data/client.py

@@ -84,7 +84,7 @@ V2_NOT_AVAILABLE = (
     "QuantPad support about /v2 access."
 )
 
-VERSION = "0.5.1"
+VERSION = "0.6.0"
 USER_AGENT = f"quantpad-data-python/{VERSION}"
 TICK_SCHEMAS = (
     "trades", "mbp-1", "mbp-10", "cmbp-1", "tcbbo", "cbbo-1s",
@@ -114,6 +114,34 @@ ROLL_ADJUST_MODES = ("none", "back", "ratio")
 #: server latency and belongs in the engine, not in a client's patience.
 BARS_TIMEOUT = (30, 1800)
 
+#: (connect, read) budget for a ``/v2`` route that has to fetch.
+#:
+#: ``/v2`` inherited the bare ``self.timeout`` -- 120 seconds, a SCALAR --
+#: which is the mistake ``BARS_TIMEOUT`` above exists to correct, made
+#: again on the surface meant to replace bars.
+#:
+#: It lands harder here.  A scalar timeout is a cap on TIME TO FIRST
+#: BYTE, and ``/v2`` sends no first byte until its fetch is finished:
+#: ``get_range`` plans, prices, fetches every missing day from upstream
+#: and only then primes the first Arrow batch.  So the number is not a
+#: patience setting, it is a guess about how long upstream takes --
+#: made once, for every schema and every width.
+#:
+#: Measured against that guess: a cold fourteen-day crude-options
+#: ``mbp-1`` window spent its entire budget inside the fetch and failed
+#: work that was going fine.  And since the service learned to stop
+#: serving a caller who has left, the guess is now what *ends* the work
+#: rather than merely what stops waiting for it.
+#:
+#: 1800 is what ``/v1/ticks`` and ``/v1/bars`` already allow for the same
+#: kind of work, so this is ``/v2`` catching up with ``/v1`` rather than
+#: a new indulgence.  It has to be this large only because nothing on
+#: this socket yet says "still working": a read budget can be short when
+#: silence means death and must be long when silence is normal.  Once
+#: the service reports progress while it fetches, this becomes a real
+#: stall detector and comes down to a couple of minutes.
+V2_DATA_TIMEOUT = (10, 1800)
+
 
 class Client:
     def __init__(
@@ -236,6 +264,31 @@ class Client:
                 if attempt >= self.max_retries:
                     raise
                 delay = exc.retry_after
+            except requests.ReadTimeout:
+                # The one thing never retried, and the exception proves the
+                # rule rather than breaking it: everything else in these
+                # clauses is cheap to repeat, and this is the opposite.
+                #
+                # A read timeout means the whole budget above was spent
+                # without the server going quiet by choice, so the work is
+                # very likely still running.  Repeating the request does
+                # not replace that work, it adds to it -- the comment on
+                # BARS_TIMEOUT measured where that leads: three retries
+                # turning one slow call into four concurrent copies, each
+                # holding one of the eight stream slots a key is allowed,
+                # which is a self-inflicted 429.  Worse now than when that
+                # was written, because the service cancels for a caller
+                # who has left, so each retry also destroys the progress
+                # the one before it had made.
+                #
+                # A caller is better served by the truth after one budget
+                # than by the same answer after four.  ConnectTimeout is
+                # subclassed from ConnectionError as well and so is still
+                # caught below, which is right: failing to reach the
+                # service at all costs the service nothing.
+                if response is not None:
+                    response.close()
+                raise
             except (requests.ConnectionError, requests.Timeout, UpstreamError) as exc:
                 if response is not None:
                     response.close()

{quantpad_data-0.5.1 → quantpad_data-0.6.0}/quantpad_data/v2.py

@@ -37,6 +37,7 @@ from __future__ import annotations
 import io
 import json
 import os
+import tempfile
 from datetime import date, timedelta
 from typing import Any, Iterator, Sequence
 
@@ -47,7 +48,7 @@ import pyarrow.parquet as pq
 import requests
 
 from . import cache, memory, rth
-from .client import Client
+from .client import V2_DATA_TIMEOUT, Client
 from .errors import QuantPadDataError, ResponseTooLargeError, ValidationError
 
 #: Above this, ``get_range`` stops buffering a single response and reads
@@ -56,6 +57,22 @@ from .errors import QuantPadDataError, ResponseTooLargeError, ValidationError
 #: in a notebook degrades into a slower read rather than a failure.
 _BULK_FALLBACK_STATUS = 413
 
+#: (connect, read) budget for a presigned read from object storage.
+#:
+#: Deliberately far shorter than :data:`V2_DATA_TIMEOUT`.  There is no
+#: work behind an object download -- no planning, no upstream fetch, no
+#: transcode -- so nothing here can legitimately take minutes to begin,
+#: and silence really is a stall.  This is the one place a read budget
+#: does the job it is named for; handing it the service's budget would
+#: let a dead transfer hang a notebook for half an hour.
+STORAGE_TIMEOUT = (10, 120)
+
+#: Bytes of a partition held in memory before the download spills to
+#: disk.  Most partitions are far under this; a depth day is not, and
+#: that is exactly the case where holding the compressed body and the
+#: decoded table at full size at once is worth avoiding.
+STORAGE_SPOOL_BYTES = 32 * 1024 * 1024
+
 
 def _symbols(value: str | Sequence[str]) -> list[str]:
     return [value] if isinstance(value, str) else [str(v) for v in value]
@@ -271,7 +288,8 @@ class Timeseries(_Namespace):
 
         try:
             response = self._client._request(
-                "POST", "/v2/timeseries/get_range", json=body
+                "POST", "/v2/timeseries/get_range", json=body,
+                timeout=V2_DATA_TIMEOUT,
             )
         except ResponseTooLargeError as exc:
             # Bulk answers OUR ceiling and does not answer the caller's own
@@ -331,6 +349,7 @@ class Timeseries(_Namespace):
                 start=start, end=end, stype_in=stype_in,
                 stype_out=stype_out, columns=columns,
             ),
+            timeout=V2_DATA_TIMEOUT,
         ).json()
 
     def iter_range(self, **kwargs) -> Iterator[pd.DataFrame]:
@@ -505,6 +524,7 @@ class Timeseries(_Namespace):
                 payload = self._client._request(
                     "POST", "/v2/timeseries/manifest",
                     json={**manifest_body, "start": start, "end": end},
+                    timeout=V2_DATA_TIMEOUT,
                 ).json()
             except ResponseTooLargeError as exc:
                 if exc.reason != "hydration_limit":
@@ -793,7 +813,7 @@ class Batch(_Namespace):
         storage — the job is a one-time cost, not a per-read one.
         """
         return self._client._request(
-            "POST", f"/v2/batch/{job_id}/hydrate"
+            "POST", f"/v2/batch/{job_id}/hydrate", timeout=V2_DATA_TIMEOUT,
         ).json()
 
 
@@ -1040,12 +1060,28 @@ def _read_parquet_url(url: str, columns: Sequence[str] | None) -> pd.DataFrame:
     point of the manifest path is that these bytes never traverse the
     data service.
     """
-    response = requests.get(url, timeout=300)
-    response.raise_for_status()
-    table = pq.read_table(
-        io.BytesIO(response.content),
-        columns=list(columns) if columns else None,
-    )
+    # ``with``, because ``stream=True`` leaves the body unread until it
+    # is: a ``raise_for_status`` on the next line would otherwise abandon
+    # the connection holding one, and a manifest walks hundreds of these.
+    with requests.get(
+        url, timeout=STORAGE_TIMEOUT, stream=True
+    ) as response:
+        response.raise_for_status()
+        # Spooled rather than ``io.BytesIO(response.content)`` so the
+        # compressed bytes and the decoded table are not both resident
+        # at their full size: past the threshold the download goes to
+        # disk and only the table is in memory.  Streaming without this
+        # buys nothing -- ``.content`` materialises the whole body
+        # anyway, just later.
+        with tempfile.SpooledTemporaryFile(
+            max_size=STORAGE_SPOOL_BYTES
+        ) as spool:
+            for chunk in response.iter_content(chunk_size=1024 * 1024):
+                spool.write(chunk)
+            spool.seek(0)
+            table = pq.read_table(
+                spool, columns=list(columns) if columns else None
+            )
     return table.to_pandas()
 
 

{quantpad_data-0.5.1 → quantpad_data-0.6.0}/tests/test_client.py

@@ -104,6 +104,56 @@ def test_the_csv_fallback_keeps_the_long_read_budget():
     assert session.calls[1][2]["params"]["format"] == "csv"
 
 
+class _RaisingSession:
+    """A session that raises the same exception on every attempt."""
+
+    def __init__(self, exc):
+        self.headers = {}
+        self.exc = exc
+        self.calls = 0
+
+    def request(self, method, url, **kwargs):
+        self.calls += 1
+        raise self.exc
+
+
+def test_a_read_timeout_is_not_retried():
+    """Retrying a read timeout adds work rather than replacing it.
+
+    The budget was spent in full without the server choosing to go quiet,
+    so the request is very likely still running. ``BARS_TIMEOUT``'s own
+    comment measured where repeating it leads: three retries turn one
+    slow call into four concurrent copies, each holding one of the eight
+    stream slots a key is allowed, which is a self-inflicted 429. Worse
+    since the service cancels for a caller who left, because each retry
+    now also destroys the progress the one before it made.
+    """
+    session = _RaisingSession(requests.ReadTimeout("read timed out"))
+    client = Client(api_key="secret", session=session, max_retries=3)
+
+    with pytest.raises(requests.ReadTimeout):
+        client.get_coverage("AAPL")
+
+    assert session.calls == 1, "a read timeout must cost exactly one attempt"
+
+
+def test_a_connect_timeout_is_still_retried():
+    """The other half, and the reason this is a rule rather than a mood.
+
+    Failing to reach the service at all costs the service nothing, so it
+    is cheap to repeat -- which is exactly what a read timeout is not.
+    ConnectTimeout subclasses Timeout, so a rule written on Timeout alone
+    would have stopped retrying the one case that should be.
+    """
+    session = _RaisingSession(requests.ConnectTimeout("connect timed out"))
+    client = Client(api_key="secret", session=session, max_retries=2)
+
+    with pytest.raises(requests.ConnectTimeout):
+        client.get_coverage("AAPL")
+
+    assert session.calls == 3, "connect failures are cheap and still retried"
+
+
 def test_rate_limit_exposes_retry_after():
     session = FakeSession(
         [response(429, b'{"detail":"rate limit"}', {"Retry-After": "7"})]

{quantpad_data-0.5.1 → quantpad_data-0.6.0}/tests/test_v2.py

@@ -121,6 +121,171 @@ class TestNothingIsInferred:
         assert session.calls == []
 
 
+class TestWhatAV2CallIsAllowedToWaitFor:
+    """A read budget on /v2 is a guess about upstream, not about patience.
+
+    /v2 sends no first byte until its fetch is finished -- get_range
+    plans, prices, fetches every missing day and only then primes the
+    first Arrow batch -- so a scalar timeout is a cap on time to FIRST
+    BYTE. Inheriting the client's 120s default put that cap below the
+    normal cost of a cold window, and a cold fourteen-day crude-options
+    mbp-1 request spent its whole budget inside the fetch.
+    """
+
+    def _budget(self, kwargs):
+        return kwargs["timeout"]
+
+    def test_get_range_is_given_the_long_read_budget(self):
+        from quantpad_data.client import V2_DATA_TIMEOUT
+
+        api, session = client_with([response(200, arrow_bytes(bars_table()))])
+        api.timeseries.get_range(
+            dataset="GLBX.MDP3", symbols="ESZ6", schema="ohlcv-1m",
+            start="2026-01-05", end="2026-01-06",
+        )
+
+        connect, read = self._budget(session.calls[0][2])
+        assert (connect, read) == V2_DATA_TIMEOUT
+        assert read >= 1800, "a cold window does not reliably fetch in 120s"
+
+    def test_the_manifest_is_given_it_too(self):
+        """The manifest hydrates inside the request, so it is the *more*
+        exposed of the two rather than the less: the whole of a cold
+        month's fetch happens before it answers."""
+        from quantpad_data.client import V2_DATA_TIMEOUT
+
+        api, session = client_with([json_response({"files": [], "gaps": []})])
+        api.timeseries.manifest(
+            dataset="GLBX.MDP3", symbols="ESZ6", schema="mbp-10",
+            start="2026-01-05", end="2026-02-05",
+        )
+
+        assert self._budget(session.calls[0][2]) == V2_DATA_TIMEOUT
+
+    def test_the_budget_is_a_pair_not_a_number(self):
+        """Connect and read are different questions and sharing one
+        answer is what made this wrong in both directions at once: 120s
+        is far too long to spend on a TCP handshake and far too short for
+        an upstream fetch."""
+        from quantpad_data.client import V2_DATA_TIMEOUT
+
+        connect, read = V2_DATA_TIMEOUT
+        assert connect <= 30, "a handshake that slow is a dead host"
+        assert read > connect * 10
+
+    def test_a_storage_read_keeps_a_short_budget(self, monkeypatch):
+        """Presigned partitions are a plain object download with no work
+        behind them, so silence there really is a stall -- the one case a
+        read budget is supposed to detect. Giving it get_range's budget
+        would make a dead transfer hang a notebook for half an hour."""
+        seen = {}
+
+        def fake_get(url, **kwargs):
+            seen.update(kwargs)
+            return response(200, _parquet_bytes(bars_table()))
+
+        monkeypatch.setattr(requests, "get", fake_get)
+        from quantpad_data import v2 as v2_module
+
+        v2_module._read_parquet_url("https://storage.test/day.parquet", None)
+
+        connect, read = seen["timeout"]
+        assert connect <= 30 and read <= 300
+
+    def test_a_storage_read_consumes_the_body_a_chunk_at_a_time(
+        self, monkeypatch
+    ):
+        """``stream=True`` alone buys nothing.
+
+        Reading ``.content`` behind it still materialises the whole
+        object before the parquet is decoded, so peak memory stays the
+        HTTP buffer plus the decoded frame -- the pair this was meant to
+        break up. The body has to actually be consumed in pieces.
+        """
+        spy = _StreamSpy(200, _parquet_bytes(bars_table()))
+        monkeypatch.setattr(requests, "get", lambda url, **kw: spy)
+        from quantpad_data import v2 as v2_module
+
+        v2_module._read_parquet_url("https://storage.test/day.parquet", None)
+
+        assert spy.streamed
+
+    def test_a_refused_storage_read_still_closes_the_connection(
+        self, monkeypatch
+    ):
+        """With ``stream=True`` the body is unread until someone reads
+        it, so a ``raise_for_status`` that escapes leaves a connection
+        holding one. A manifest walk makes hundreds of these calls, so a
+        leak per failure exhausts the pool rather than being untidy.
+        """
+        spy = _StreamSpy(403, b"expired")
+        monkeypatch.setattr(requests, "get", lambda url, **kw: spy)
+        from quantpad_data import v2 as v2_module
+
+        with pytest.raises(requests.HTTPError):
+            v2_module._read_parquet_url(
+                "https://storage.test/day.parquet", None
+            )
+
+        assert spy.closed
+
+    def test_a_storage_read_returns_the_rows_it_downloaded(self, monkeypatch):
+        """Asserted on the data rather than the mechanism: a chunked
+        download reassembled wrongly would still stream and still
+        close."""
+        spy = _StreamSpy(200, _parquet_bytes(bars_table()))
+        monkeypatch.setattr(requests, "get", lambda url, **kw: spy)
+        from quantpad_data import v2 as v2_module
+
+        frame = v2_module._read_parquet_url(
+            "https://storage.test/day.parquet", None
+        )
+
+        assert list(frame["close"]) == [100.0, 101.0]
+
+
+class _StreamSpy:
+    """A response that records how its body was taken and when it closed.
+
+    Both are invisible from outside a real ``requests.Response``, and
+    both are the behaviour under test: ``.content`` and a chunked read
+    return the same bytes, and a leaked connection returns the right
+    answer right up until the pool runs out.
+    """
+
+    def __init__(self, status, body):
+        self.status_code = status
+        self.content = body
+        self.headers = {}
+        self.streamed = False
+        self.closed = False
+
+    def raise_for_status(self):
+        if self.status_code >= 400:
+            raise requests.HTTPError(f"HTTP {self.status_code}")
+
+    def iter_content(self, chunk_size=1):
+        self.streamed = True
+        for start in range(0, len(self.content), chunk_size):
+            yield self.content[start:start + chunk_size]
+
+    def close(self):
+        self.closed = True
+
+    def __enter__(self):
+        return self
+
+    def __exit__(self, *exc):
+        self.close()
+        return False
+
+
+def _parquet_bytes(table):
+    sink = io.BytesIO()
+    pq.write_table(table, sink)
+    return sink.getvalue()
+
+
 class TestTheResolvedAddressIsReported:
     def test_echo_headers_land_on_the_frame(self):
         """A caller has to be able to confirm which contract answered,