quantpad-data 0.8.1__tar.gz → 0.8.3__tar.gz

{quantpad_data-0.8.1 → quantpad_data-0.8.3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: quantpad-data
-Version: 0.8.1
+Version: 0.8.3
 Summary: Official Python client for the QuantPad market-data API
 Project-URL: Documentation, https://api.quantpad.ai/external/docs
 Author: QuantPad

{quantpad_data-0.8.1 → quantpad_data-0.8.3}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
 
 [project]
 name = "quantpad-data"
-version = "0.8.1"
+version = "0.8.3"
 description = "Official Python client for the QuantPad market-data API"
 readme = "README.md"
 requires-python = ">=3.10"

{quantpad_data-0.8.1 → quantpad_data-0.8.3}/quantpad_data/__init__.py

@@ -11,6 +11,7 @@ import pyarrow as pa
 from .client import Client, ROLL_ADJUST_MODES, TICK_SCHEMAS, USER_AGENT, VERSION
 from .errors import (
     AuthenticationError,
+    IncompleteResponseError,
     NotFoundError,
     PermissionDeniedError,
     QuantPadDataError,
@@ -213,7 +214,7 @@ __all__ = [
     "Client", "QuantPadClient", "QuantPadDataError", "QuantPadDataNotReady", "AuthenticationError",
     "PermissionDeniedError", "RateLimitError", "QuotaExceededError",
     "NotFoundError", "V2NotAvailableError", "ValidationError", "UpstreamError",
-    "ResponseTooLargeError", "TruncatedChainWarning", "TICK_SCHEMAS",
+    "IncompleteResponseError", "ResponseTooLargeError", "TruncatedChainWarning", "TICK_SCHEMAS",
     "ROLL_ADJUST_MODES", "get_bars", "get_bars_with_retry", "get_ticks",
     "get_trades", "get_mbp1", "get_mbp10", "get_trades_rth", "get_mbp1_rth",
     "get_mbp10_rth", "get_spread_1m_rth", "symbology_resolve", "get_universe",

{quantpad_data-0.8.1 → quantpad_data-0.8.3}/quantpad_data/cache.py

@@ -349,6 +349,12 @@ def _columns_tag(columns: Sequence[str] | None) -> str:
 #: would serve the wrong rows.
 _DELIVERY_ONLY = frozenset({"compression", "max_decoded_bytes"})
 
+#: Changed whenever the service starts answering an unchanged body with
+#: different rows, so every entry cached under the old answer misses once.
+#: 2: until 2026-10, a window ending mid-day was answered without its last
+#: day, and that short answer was cached here as the window's answer.
+_RANGE_KEY_EPOCH = 2
+
 
 def _request_key(body: dict) -> str:
     """A stable key for the answer a request body asks for.
@@ -357,7 +363,9 @@ def _request_key(body: dict) -> str:
     which is what stops it drifting as ``get_range`` grows arguments.
     """
     material = {k: v for k, v in sorted(body.items()) if k not in _DELIVERY_ONLY}
-    return _hash_key([json.dumps(material, sort_keys=True, default=str)])
+    return _hash_key(
+        [str(_RANGE_KEY_EPOCH), json.dumps(material, sort_keys=True, default=str)]
+    )
 
 
 def range_path(body: dict) -> Path:
@@ -403,13 +411,23 @@ def read_meta(path: Path) -> dict | None:
     return _read_meta(path)
 
 
-def get_frame(path: Path) -> pd.DataFrame | None:
+def get_frame(path: Path, *, expect: dict | None = None) -> pd.DataFrame | None:
     """Read a cached frame, or None on miss.
 
     A corrupt or half-written entry is dropped and reported as a miss, so a
     damaged cache degrades to the network rather than failing the request.
+
+    ``expect`` names sidecar fields the entry must carry exactly, and a
+    mismatch is a miss.  This is how a partition the service has since
+    re-fetched stops being served from disk: the manifest's ``version``
+    moves when upstream corrects a day, and a settled day is otherwise
+    never asked about again.  An entry written before versions existed
+    carries none, so it misses once against a server that sends one.
     """
-    if not enabled() or _read_meta(path) is None:
+    meta = _read_meta(path) if enabled() else None
+    if meta is None:
+        return None
+    if any(meta.get(field) != value for field, value in (expect or {}).items()):
         return None
     try:
         frame = pd.read_parquet(path)

{quantpad_data-0.8.1 → quantpad_data-0.8.3}/quantpad_data/client.py

@@ -10,11 +10,14 @@ from datetime import date
 from typing import Iterator, Sequence
 
 import pandas as pd
+import pyarrow as pa
 import pyarrow.ipc as ipc
 import requests
+import urllib3
 
 from .errors import (
     AuthenticationError,
+    IncompleteResponseError,
     NotFoundError,
     PermissionDeniedError,
     QuantPadDataError,
@@ -84,7 +87,7 @@ V2_NOT_AVAILABLE = (
     "QuantPad support about /v2 access."
 )
 
-VERSION = "0.8.1"
+VERSION = "0.8.3"
 USER_AGENT = f"quantpad-data-python/{VERSION}"
 TICK_SCHEMAS = (
     "trades", "mbp-1", "mbp-10", "cmbp-1", "tcbbo", "cbbo-1s",
@@ -142,6 +145,50 @@ BARS_TIMEOUT = (30, 1800)
 #: stall detector and comes down to a couple of minutes.
 V2_DATA_TIMEOUT = (10, 1800)
 
+#: Backoff between retries: exponential, with jitter, so that the clients
+#: of one outage do not come back in lockstep and knock it over again.
+#:
+#: Was 0.5s, 1s, 2s plus at most 0.25s, with the server's ``Retry-After``
+#: ignored on every 5xx.  During the 2026-10-01/02 API-key outage the
+#: service answered every request "503, Retry-After: 5"; this client
+#: retried each call three more times inside four seconds, and one key
+#: kept that up for sixteen hours.
+RETRY_BACKOFF_BASE = 1.0
+RETRY_BACKOFF_MAX = 30.0
+
+#: The longest ``Retry-After`` honoured, from any status.  Past this the
+#: client would rather fail and let its caller decide than sleep silently:
+#: a daily-quota 429 can say "come back in twenty hours".
+RETRY_AFTER_MAX = 60.0
+
+
+#: What reading a stream raises when the connection ends early: urllib3's
+#: incomplete-chunk error from the socket, requests' wrapper of the same,
+#: and what pyarrow raises when a message is cut short (ArrowInvalid) or
+#: its read fails (OSError).
+_BROKEN_STREAM = (
+    urllib3.exceptions.HTTPError,
+    requests.exceptions.RequestException,
+    pa.ArrowInvalid,
+    OSError,
+)
+
+
+def _retry_delay(attempt: int, retry_after: float | None) -> float:
+    """Seconds to wait before retry number ``attempt + 1``.
+
+    Exponential with equal jitter -- half the step fixed, half random --
+    and never sooner than the server asked, though no later than
+    ``RETRY_AFTER_MAX``.  The server's figure is jittered too, because
+    every client of an outage receives the same one.
+    """
+    step = min(RETRY_BACKOFF_MAX, RETRY_BACKOFF_BASE * (2**attempt))
+    delay = step / 2 + random.uniform(0, step / 2)
+    if retry_after is not None and retry_after > 0:
+        asked = min(float(retry_after), RETRY_AFTER_MAX)
+        delay = max(delay, asked + random.uniform(0, min(1.0, asked / 4)))
+    return delay
+
 
 def _prose_with_hint(error: dict) -> str:
     """The sentence to raise from a structured refusal.
@@ -269,7 +316,7 @@ class Client:
             cls = QuotaExceededError if "quota" in message.lower() else RateLimitError
             raise cls(message, retry_after, status)
         if status >= 500:
-            raise UpstreamError(message, status)
+            raise UpstreamError(message, status, retry_after=retry_after)
         raise QuantPadDataError(message, status, detail)
 
     def _request(self, method: str, path: str, **kwargs) -> requests.Response:
@@ -322,9 +369,8 @@ class Client:
                 last_error = exc
                 if attempt >= self.max_retries:
                     raise
-                delay = None
-            base_delay = delay if delay is not None else min(8.0, 0.5 * (2**attempt))
-            time.sleep(max(0.0, base_delay) + random.uniform(0, 0.25))
+                delay = getattr(exc, "retry_after", None)
+            time.sleep(_retry_delay(attempt, delay))
         raise QuantPadDataError(str(last_error or "Request failed"))
 
     @staticmethod
@@ -417,8 +463,34 @@ class Client:
         response.raw.decode_content = True
         buffered: list[pd.DataFrame] = []
         buffered_rows = 0
+        received = 0
+
+        def broken(exc: BaseException) -> IncompleteResponseError:
+            return IncompleteResponseError(
+                f"The /v1/ticks stream for {params['symbol']} {schema} ended "
+                f"before the service finished sending it ({received:,} rows "
+                "had arrived). Nothing from this call is complete; retry the "
+                f"request. ({type(exc).__name__}: {exc})",
+                rows_received=received,
+            )
+
         try:
-            for batch in ipc.open_stream(response.raw):
+            # Read errors are told apart from a clean end because the
+            # service now ends a failed stream by closing the connection
+            # early -- its 200 has already gone out -- and the raw urllib3
+            # ProtocolError that surfaces says nothing about what it means.
+            try:
+                batches = iter(ipc.open_stream(response.raw))
+            except _BROKEN_STREAM as exc:
+                raise broken(exc) from exc
+            while True:
+                try:
+                    batch = next(batches)
+                except StopIteration:
+                    break
+                except _BROKEN_STREAM as exc:
+                    raise broken(exc) from exc
+                received += batch.num_rows
                 if not batch.num_rows:
                     continue
                 buffered.append(batch.to_pandas(split_blocks=True, self_destruct=True))

{quantpad_data-0.8.1 → quantpad_data-0.8.3}/quantpad_data/errors.py

@@ -134,7 +134,39 @@ class ValidationError(ValueError, QuantPadDataError):
 
 
 class UpstreamError(QuantPadDataError):
-    pass
+    """A 5xx from the service.
+
+    ``retry_after`` is the service's ``Retry-After`` in seconds when it sent
+    one: a 503 that says when to come back is a 503 worth believing, and the
+    client waits at least that long before retrying it.
+    """
+
+    def __init__(
+        self,
+        message: str,
+        status_code: int | None = None,
+        detail: dict | None = None,
+        *,
+        retry_after: float | None = None,
+    ):
+        super().__init__(message, status_code, detail)
+        self.retry_after = retry_after
+
+
+class IncompleteResponseError(UpstreamError):
+    """A streamed response ended before the service finished sending it.
+
+    The service now ends a tick stream this way when it fails partway --
+    the status was already 200, so the connection is the only signal
+    left.  Nothing this call returned can be taken as complete; retry the
+    request.  A subclass of :class:`UpstreamError`, so existing ``except
+    UpstreamError`` handlers keep catching it.  ``rows_received`` counts
+    the rows that had arrived when the stream broke.
+    """
+
+    def __init__(self, message: str, rows_received: int = 0):
+        super().__init__(message)
+        self.rows_received = rows_received
 
 
 class QuantPadDataNotReady(QuantPadDataError):

{quantpad_data-0.8.1 → quantpad_data-0.8.3}/quantpad_data/v2.py

@@ -1330,7 +1330,13 @@ def _partition(body: dict, item: dict, columns) -> pd.DataFrame:
     hostile ``QPD_CACHE_DIR`` can contain, and the only other way it could
     fail is a manifest item missing ``day`` or ``instrument_id``, which the
     caller subscripts for its own labels two lines later regardless.
+
+    A settled day is not immutable: the service re-fetches a day its
+    vendor regenerates, and says so by changing the item's ``version``.  A
+    hit therefore has to be the version the manifest describes, or it is
+    the uncorrected copy served from disk for as long as the disk keeps it.
     """
+    expect = {"version": item.get("version")}
     path = None
     if cache.enabled() and cache.day_settled(item.get("day")):
         path = cache.day_path(
@@ -1343,14 +1349,15 @@ def _partition(body: dict, item: dict, columns) -> pd.DataFrame:
         )
 
     if path is not None:
-        hit = cache.get_frame(path)
+        hit = cache.get_frame(path, expect=expect)
         if hit is not None:
             return hit
 
     frame = _read_parquet_url(item["url"], columns)
     if path is not None:
         cache.put_frame(
-            path, frame, {"day": item["day"], "dataset": body["dataset"]}
+            path, frame,
+            {"day": item["day"], "dataset": body["dataset"], **expect},
         )
     return frame
 

{quantpad_data-0.8.1 → quantpad_data-0.8.3}/tests/test_client.py

@@ -164,6 +164,128 @@ def test_rate_limit_exposes_retry_after():
     assert caught.value.retry_after == 7
 
 
+def _record_sleeps(monkeypatch) -> list[float]:
+    import quantpad_data.client as client_module
+
+    slept: list[float] = []
+    monkeypatch.setattr(client_module.time, "sleep", slept.append)
+    return slept
+
+
+def test_a_503_that_says_when_to_come_back_is_believed(monkeypatch):
+    """During the 2026-10-01/02 API-key outage every answer was "503,
+    Retry-After: 5", and this client retried each call three more times
+    inside four seconds -- one key kept that up for sixteen hours."""
+    from quantpad_data.errors import UpstreamError
+
+    slept = _record_sleeps(monkeypatch)
+    unavailable = b'{"error":"API-key validation is temporarily unavailable"}'
+    session = FakeSession(
+        [response(503, unavailable, {"Retry-After": "5"}) for _ in range(4)]
+    )
+    client = Client(api_key="secret", session=session, max_retries=3)
+
+    with pytest.raises(UpstreamError) as caught:
+        client.get_coverage("AAPL")
+
+    assert caught.value.retry_after == 5
+    assert len(session.calls) == 4, "retries are capped at max_retries"
+    assert len(slept) == 3
+    assert all(delay >= 5 for delay in slept), slept
+
+
+def test_backoff_doubles_with_jitter_when_the_server_names_no_time(monkeypatch):
+    from quantpad_data.client import RETRY_BACKOFF_BASE
+    from quantpad_data.errors import UpstreamError
+
+    slept = _record_sleeps(monkeypatch)
+    session = FakeSession([response(502, b"bad gateway") for _ in range(4)])
+    client = Client(api_key="secret", session=session, max_retries=3)
+
+    with pytest.raises(UpstreamError):
+        client.get_coverage("AAPL")
+
+    for attempt, delay in enumerate(slept):
+        step = RETRY_BACKOFF_BASE * 2**attempt
+        assert step / 2 <= delay <= step, (attempt, delay)
+
+
+def test_jitter_actually_spreads_clients_out():
+    from quantpad_data.client import _retry_delay
+
+    assert len({round(_retry_delay(2, 5), 6) for _ in range(50)}) > 1
+    assert len({round(_retry_delay(2, None), 6) for _ in range(50)}) > 1
+
+
+def test_an_absurd_retry_after_is_capped_rather_than_slept(monkeypatch):
+    """A daily-quota 429 can say "come back in twenty hours"; sleeping
+    that long without a word is worse than failing."""
+    from quantpad_data.client import RETRY_AFTER_MAX, _retry_delay
+
+    assert _retry_delay(0, 72_000) <= RETRY_AFTER_MAX + 1.0
+
+
+def _trades_stream(*batch_rows: int, complete: bool) -> bytes:
+    """An Arrow IPC tick stream; ``complete=False`` stops before the
+    end-of-stream marker, as a body cut off by the connection would."""
+    schema = pa.schema([("t", pa.int64()), ("price", pa.float64())])
+    sink = io.BytesIO()
+    writer = ipc.new_stream(sink, schema)
+    for rows in batch_rows:
+        writer.write_batch(pa.record_batch(
+            [pa.array(range(rows), pa.int64()), pa.array([1.0] * rows)],
+            schema=schema,
+        ))
+    if complete:
+        writer.close()
+    return sink.getvalue()
+
+
+class _CutConnection(io.BytesIO):
+    """A raw body that ends the way an aborted chunked response does:
+    after the bytes that arrived, the next read fails rather than
+    returning empty."""
+
+    def read(self, size=-1):
+        data = super().read(size)
+        if not data:
+            import urllib3
+
+            raise urllib3.exceptions.ProtocolError(
+                "Connection broken: IncompleteRead(0 bytes read)"
+            )
+        return data
+
+
+def test_a_tick_stream_cut_short_raises_an_error_that_says_so():
+    """The service ends a tick stream that fails partway by closing the
+    connection -- its 200 has already gone out.  What surfaced before was
+    urllib3's bare ProtocolError; a caller has to be told that what they
+    received is incomplete and that retrying is the answer."""
+    from quantpad_data.errors import IncompleteResponseError, UpstreamError
+
+    cut = response(200, b"")
+    cut.raw = _CutConnection(_trades_stream(4, 6, complete=False))
+    client = Client(api_key="secret", session=FakeSession([cut]), max_retries=0)
+
+    with pytest.raises(IncompleteResponseError) as caught:
+        list(client.get_ticks("NQ.FUT", "trades", 1, 2, chunksize=1000))
+
+    assert isinstance(caught.value, UpstreamError), "existing handlers must still catch it"
+    assert caught.value.rows_received == 10
+    assert "retry" in str(caught.value).lower()
+
+
+def test_a_complete_tick_stream_is_unaffected():
+    whole = response(200, b"")
+    whole.raw = io.BytesIO(_trades_stream(4, 6, complete=True))
+    client = Client(api_key="secret", session=FakeSession([whole]), max_retries=0)
+
+    frames = list(client.get_ticks("NQ.FUT", "trades", 1, 2, chunksize=1000))
+
+    assert sum(len(frame) for frame in frames) == 10
+
+
 def test_universe_and_coverage_paths():
     session = FakeSession(
         [response(body=b'{"results":[]}'), response(body=b'{"symbol":"AAPL"}')]

{quantpad_data-0.8.1 → quantpad_data-0.8.3}/tests/test_v2_cache.py

@@ -124,12 +124,13 @@ def api(*responses):
     return Historical(client).timeseries, session
 
 
-def manifest_for(days, instrument_id=1551):
+def manifest_for(days, instrument_id=1551, version=None):
     return response(200, json.dumps({
         "format": "parquet",
         "files": [
             {"day": day, "instrument_id": instrument_id, "symbol": "BA",
              "key": f"k/{day}", "size_bytes": 10,
+             **({"version": version} if version is not None else {}),
              "url": f"http://r2.test/{day}.parquet"}
             for day in days
         ],
@@ -276,6 +277,17 @@ class TestWhatTheKeyMustSeparate:
         base = _body(**ASK)
         assert cache.range_path(base) == cache.range_path({**base, field: value})
 
+    def test_an_answer_cached_before_the_service_changed_it_is_not_served(
+        self, monkeypatch
+    ):
+        # Until 2026-10 a window ending mid-day came back without its last
+        # day, and was cached that way. The body has not changed, so only the
+        # epoch can tell the old short answer from the new one.
+        body = _body(**{**ASK, "end": "2026-09-16T13:58:00Z"})
+        current = cache.range_path(body)
+        monkeypatch.setattr(cache, "_RANGE_KEY_EPOCH", cache._RANGE_KEY_EPOCH - 1)
+        assert cache.range_path(body) != current
+
 
 class TestWhatMustNotBeCached:
     def test_a_window_reaching_into_the_current_day_is_not_cached(self):
@@ -546,6 +558,70 @@ class TestPartitionsAreCachedIndividually:
         )
 
 
+class TestACorrectedDayIsNotServedFromDisk:
+    """A settled day is final for the calendar, not for the vendor.
+
+    The service re-fetches a day its vendor regenerates and changes the
+    manifest item's ``version`` when it does.  Before this, a day already on
+    disk was never asked about again, so the correction reached nobody who
+    had read the day once -- and nothing anywhere said the copy was old.
+    """
+
+    CORRECTED = ROWS.assign(price=[100.5, 101.5])
+
+    def test_the_same_version_is_still_a_hit(self, monkeypatch):
+        urls = UrlRecorder()
+        monkeypatch.setattr(requests, "get", urls)
+        ts, _ = api(
+            manifest_for(["2026-09-16"], version="v1"),
+            manifest_for(["2026-09-16"], version="v1"),
+        )
+        list(ts.iter_range(**_manifest_ask()))
+        list(ts.iter_range(**_manifest_ask()))
+        assert len(urls.urls) == 1
+
+    def test_a_new_version_is_fetched_and_replaces_the_copy(self, monkeypatch):
+        urls = UrlRecorder()
+        monkeypatch.setattr(requests, "get", urls)
+        ts, _ = api(
+            manifest_for(["2026-09-16"], version="v1"),
+            manifest_for(["2026-09-16"], version="v2"),
+            manifest_for(["2026-09-16"], version="v2"),
+        )
+        list(ts.iter_range(**_manifest_ask()))
+        urls.frame = self.CORRECTED
+        [corrected] = [f.copy() for f in ts.iter_range(**_manifest_ask())]
+        [again] = [f.copy() for f in ts.iter_range(**_manifest_ask())]
+
+        assert len(urls.urls) == 2, "fetched once per version, not per read"
+        assert list(corrected["price"]) == [100.5, 101.5]
+        pd.testing.assert_frame_equal(corrected, again)
+
+    def test_a_copy_written_before_versions_existed_is_refetched_once(
+        self, monkeypatch
+    ):
+        """Every entry on disk today carries no version, and some of them
+        are the uncorrected days this exists for."""
+        urls = UrlRecorder()
+        monkeypatch.setattr(requests, "get", urls)
+        ts, _ = api(
+            manifest_for(["2026-09-16"]),
+            manifest_for(["2026-09-16"], version="v1"),
+            manifest_for(["2026-09-16"], version="v1"),
+        )
+        for _ in range(3):
+            list(ts.iter_range(**_manifest_ask()))
+        assert len(urls.urls) == 2
+
+    def test_a_range_entry_is_not_asked_for_a_version_it_never_had(self):
+        """``expect`` is opt-in: the whole-response cache has no per-file
+        version to compare, and must keep its hits."""
+        path = cache.range_path(_body(**ASK))
+        assert cache.put_frame(path, ROWS, {"echo": {}})
+        assert cache.get_frame(path) is not None
+        assert cache.get_frame(path, expect={"version": "v1"}) is None
+
+
 class TestTheSettlementRuleItself:
     def test_a_past_day_is_settled(self):
         assert cache.day_settled("2026-09-16") is True