quantpad-data 0.6.0__tar.gz → 0.7.1__tar.gz

{quantpad_data-0.6.0 → quantpad_data-0.7.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: quantpad-data
-Version: 0.6.0
+Version: 0.7.1
 Summary: Official Python client for the QuantPad market-data API
 Project-URL: Documentation, https://api.quantpad.ai/external/docs
 Author: QuantPad

{quantpad_data-0.6.0 → quantpad_data-0.7.1}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
 
 [project]
 name = "quantpad-data"
-version = "0.6.0"
+version = "0.7.1"
 description = "Official Python client for the QuantPad market-data API"
 readme = "README.md"
 requires-python = ">=3.10"

{quantpad_data-0.6.0 → quantpad_data-0.7.1}/quantpad_data/client.py

@@ -84,7 +84,7 @@ V2_NOT_AVAILABLE = (
     "QuantPad support about /v2 access."
 )
 
-VERSION = "0.6.0"
+VERSION = "0.7.1"
 USER_AGENT = f"quantpad-data-python/{VERSION}"
 TICK_SCHEMAS = (
     "trades", "mbp-1", "mbp-10", "cmbp-1", "tcbbo", "cbbo-1s",
@@ -143,6 +143,25 @@ BARS_TIMEOUT = (30, 1800)
 V2_DATA_TIMEOUT = (10, 1800)
 
 
+def _prose_with_hint(error: dict) -> str:
+    """The sentence to raise from a structured refusal.
+
+    ``message`` first; a ``hint`` follows it as a second sentence when the
+    service sent one.  Kept out of the message on the wire so a caller
+    reading the body can still tell the refusal from the advice, and put
+    back here because the exception text is the only thing most callers
+    ever see.  A body with a hint and no message is stringified whole, as
+    it always was.
+    """
+    message = error.get("message")
+    if not message:
+        return str(error)
+    hint = error.get("hint")
+    if isinstance(hint, str) and hint.strip():
+        return f"{message} {hint.strip()}"
+    return str(message)
+
+
 class Client:
     def __init__(
         self,
@@ -189,6 +208,14 @@ class Client:
         user where the service had written a sentence.  So the prose is
         unwrapped from ``message`` when there is one, and the object is
         kept beside it for the callers that need to branch on it.
+
+        A ``hint`` rides with the prose.  The /v2 404 for a symbol nothing
+        resolves carries what to send instead -- ``NQ.FUT`` with
+        ``stype_in='parent'`` for a bare ``NQ`` -- as its own field so a
+        caller branching on the body can tell it from the message; but the
+        exception a notebook prints is the message, and advice a client
+        drops on the floor is advice nobody gets (the shape
+        ``options.chain`` had with ``truncated``).
         """
         try:
             body = response.json()
@@ -198,7 +225,7 @@ class Client:
             return str(body), None
         error = body.get("error") or body.get("detail") or body
         if isinstance(error, dict):
-            return str(error.get("message") or error), error
+            return _prose_with_hint(error), error
         return str(error), None
 
     @classmethod

{quantpad_data-0.6.0 → quantpad_data-0.7.1}/quantpad_data/v2.py

@@ -73,6 +73,26 @@ STORAGE_TIMEOUT = (10, 120)
 #: decoded table at full size at once is worth avoiding.
 STORAGE_SPOOL_BYTES = 32 * 1024 * 1024
 
+#: Widest window of missing days :meth:`Timeseries.iter_range` fetches
+#: in one request, once it has ramped up to it.
+#:
+#: A cadence, not a capacity.  The service's own ceilings decide what
+#: fits and refuse what does not, and :func:`_split_window` already
+#: halves a window that gets refused, so nothing here is protecting
+#: memory.  What it buys is the caller seeing frames while the fetch is
+#: still going, which one request for the whole window cannot give
+#: however fast it is: measured on a cold month of niche-equity
+#: ``mbp-10``, 129.5 of the 132.4 seconds elapsed before the first
+#: frame, because one manifest was hydrating nineteen days inside the
+#: request.  That is indistinguishable from a hang, and it is the shape
+#: of the "all projects hang, no error" report this exists to answer.
+#:
+#: Five caps a ramp that starts at one day (see :func:`_runs`), so a
+#: cold month is a handful of requests rather than one or nineteen, and
+#: the first of them is small.  A warm window never reaches this at
+#: all -- it has no gaps, so it stays the single manifest it always was.
+HYDRATE_CHUNK_DAYS = 5
+
 
 def _symbols(value: str | Sequence[str]) -> list[str]:
     return [value] if isinstance(value, str) else [str(v) for v in value]
@@ -147,6 +167,66 @@ def _split_window(start: str, end: str) -> tuple[tuple[str, str], ...] | None:
     return ((start, middle), (middle, end))
 
 
+def _identity(item: dict) -> tuple:
+    """What makes two manifest entries the same stored partition.
+
+    The object key, because it is what the service stores the partition
+    under and is therefore stable from one manifest to the next.  The
+    obvious alternative, ``url``, is a freshly signed URL on every call
+    and would make every entry unique -- which is to say it would look
+    like it worked and deduplicate nothing.
+
+    Day and instrument ride along so that a service which omits ``key``
+    still distinguishes one instrument-day from another, rather than
+    collapsing a whole window onto a single ``None``.
+    """
+    return (item.get("key"), item["day"], item["instrument_id"])
+
+
+def _runs(days: Sequence[str], size: int) -> Iterator[tuple[str, str]]:
+    """Windows covering ``days``, small at first and then up to ``size``.
+
+    Consecutive days are grouped, because a run of missing days is one
+    fetch rather than several: the days between two gaps are already
+    cached and asking for them again would re-fetch what the caller
+    has.  Groups are then cut into windows, because a run can be a
+    whole month and that is the silence being removed.
+
+    **The first window is one day, and each is twice the last up to**
+    ``size``.  A flat cut makes the first frame wait for a full chunk,
+    which measured 29.0s on a cold 8-day window of equity depth -- an
+    improvement on the 58.0s it replaced and still long enough to read
+    as nothing happening.  Ramping puts the first frame after a single
+    day, and by the time the caller is watching output arrive the
+    windows are full width, so the round trips are spent where they buy
+    reassurance rather than spread over the whole fetch.
+
+    Yields half-open ``(start, end)`` pairs, ``end`` being the midnight
+    after the window's last day.  Day-aligned for :func:`_split_window`'s
+    reason -- a partition is a whole day, so a boundary inside one would
+    put that day in both windows and return its rows twice.
+    """
+    ordered = sorted({date.fromisoformat(day) for day in days})
+    groups: list[list[date]] = []
+    for day in ordered:
+        if groups and day == groups[-1][-1] + timedelta(days=1):
+            groups[-1].append(day)
+        else:
+            groups.append([day])
+
+    # Carried ACROSS groups, not reset per group: the ramp is about how
+    # long the caller has been waiting, and a second group is reached
+    # only after the first has already produced frames.
+    width = 1
+    for group in groups:
+        index = 0
+        while index < len(group):
+            piece = group[index : index + width]
+            index += width
+            width = min(width * 2, size)
+            yield piece[0].isoformat(), (piece[-1] + timedelta(days=1)).isoformat()
+
+
 def _table_from_arrow(payload: bytes) -> pa.Table:
     with ipc.open_stream(io.BytesIO(payload)) as reader:
         return reader.read_all()
@@ -358,8 +438,13 @@ class Timeseries(_Namespace):
         For ranges that do not fit in memory.  Each yielded frame is one
         (day, instrument) partition, in chronological order.
 
-        Asks for the window in as few pieces as the service will allow,
-        which is one piece whenever the days are already cached.
+        **Streams the fetch, not only the answer.**  The first call asks
+        the manifest not to hydrate, so it returns in about a second
+        however cold the window is and says which days it does not have.
+        Cached days are then yielded straight away and the missing ones
+        fetched in runs of :data:`HYDRATE_CHUNK_DAYS` as the iteration
+        reaches them.  A warm window is still one round trip; a cold one
+        produces frames throughout instead of after a single silence.
         """
         if kwargs.get("roll_adjust", "none") != "none" or kwargs.get("resample"):
             raise ValidationError(
@@ -374,18 +459,17 @@ class Timeseries(_Namespace):
         # day, instrument, projection), so the whole-window body is the right
         # thing to hand every piece.
         manifest_body = _manifest_body(columns=columns, **kwargs)
-        for payload in self._manifests(manifest_body):
-            for item in payload["files"]:
-                frame = _labelled(
-                    _partition(manifest_body, item, columns), item, columns
-                )
-                frame.attrs["quantpad"] = {
-                    **_manifest_attrs(payload),
-                    "day": item["day"],
-                    "instrument_id": item["instrument_id"],
-                    "symbol": item.get("symbol"),
-                }
-                yield frame
+        for payload, item in self._partitions(manifest_body):
+            frame = _labelled(
+                _partition(manifest_body, item, columns), item, columns
+            )
+            frame.attrs["quantpad"] = {
+                **_manifest_attrs(payload),
+                "day": item["day"],
+                "instrument_id": item["instrument_id"],
+                "symbol": item.get("symbol"),
+            }
+            yield frame
 
     def iter_range_rth(self, **kwargs) -> Iterator[pd.DataFrame]:
         """:meth:`iter_range`, restricted to XNYS regular trading hours.
@@ -499,7 +583,103 @@ class Timeseries(_Namespace):
             exc.detail,
         )
 
-    def _manifests(self, manifest_body: dict) -> Iterator[dict]:
+    def _partitions(self, manifest_body: dict) -> Iterator[tuple[dict, dict]]:
+        """``(manifest, file)`` pairs in day order, fetching as it goes.
+
+        The window is surveyed once without hydrating -- cheap whatever
+        the cache holds, measured at 0.9s against 29.6s for the same
+        cold window fetched -- and the survey says which days are
+        missing.  Days already present are handed over immediately;
+        missing ones are fetched only when the walk reaches them, so a
+        caller whose window is half warm sees frames before any fetch
+        starts at all.
+
+        **Chronological across the join**, which is the constraint that
+        shapes this: yielding every cached day and then every fetched
+        one would be simpler and would reorder the window, and
+        ``iter_range`` promises day order.  So the two are interleaved
+        by day rather than concatenated.
+
+        **A calendar day is not the unit of cache state**, which is the
+        other constraint and much the less obvious one.  A manifest
+        speaks in instrument-days, so asking for two symbols whose
+        histories were fetched at different times puts one date in
+        ``files`` for one symbol and in ``gaps`` for the other.  The
+        fetch for that date then answers with BOTH instruments -- it is
+        given a window, not a list of holes -- so the already-cached one
+        arrives a second time, and a caller concatenating the frames
+        would silently double those rows.  Hence :func:`_identity`: the
+        day decides WHEN a partition is handed over, the partition
+        itself decides whether it already has been.
+
+        Degrades rather than breaks against a service that predates the
+        flag: an older manifest ignores ``hydrate`` (the request models
+        do not forbid unknown fields, so it is dropped rather than
+        rejected), fetches anyway and answers with no gaps, which this
+        reads as a fully cached window and streams directly.  Slower,
+        never wrong -- and it keeps the splitting it would otherwise
+        have lost, since the survey goes through :meth:`_manifests`
+        rather than asking once.  Against a service that honours the
+        flag that costs nothing: nothing is fetched, so there is no
+        hydration ceiling to breach and the splitter loops once.
+        """
+        surveyed = list(self._manifests(manifest_body, hydrate=False))
+
+        # No early return for the all-cached case, though one reads
+        # naturally here and was written first.  The walk below already
+        # handles it in one round trip -- there is nothing to fetch, so
+        # nothing is fetched -- and it additionally puts the days in
+        # order, where handing back ``survey["files"]`` would inherit
+        # whatever order the service happened to list them in.  A
+        # mutation battery is what established the two were equivalent:
+        # the shortcut turned out to be the weaker of the two.
+        ready: dict[str, list[tuple[dict, dict]]] = {}
+        missing: set[str] = set()
+        for payload in surveyed:
+            for item in payload["files"]:
+                ready.setdefault(item["day"], []).append((payload, item))
+            for gap in payload.get("gaps") or ():
+                missing.add(gap["day"])
+
+        seen: set[tuple] = set()
+
+        def fresh(pairs) -> Iterator[tuple[dict, dict]]:
+            for payload, item in pairs:
+                identity = _identity(item)
+                if identity not in seen:
+                    seen.add(identity)
+                    yield payload, item
+
+        # One cursor over the union of both sets, so a fetched day lands
+        # between the cached days that surround it.
+        fetched: dict[str, list[tuple[dict, dict]]] = {}
+        pending = list(_runs(sorted(missing), HYDRATE_CHUNK_DAYS))
+        for day in sorted(set(ready) | missing):
+            while pending and pending[0][0] <= day:
+                start, end = pending.pop(0)
+                for payload in self._manifests(
+                    {**manifest_body, "start": start, "end": end}
+                ):
+                    for item in payload["files"]:
+                        fetched.setdefault(item["day"], []).append((payload, item))
+            yield from fresh(ready.get(day, ()))
+            # A missing day that came back with nothing -- a holiday, or
+            # a genuinely quiet session -- simply has no entry here, and
+            # is skipped exactly as an ``empty`` day always was.
+            yield from fresh(fetched.pop(day, ()))
+
+        # Anything a fetch returned for a day the survey did not list.
+        # It should be empty, and it is drained rather than asserted
+        # about, because the two calls resolve symbols independently and
+        # the failure of the cursor to reach a day would otherwise drop
+        # its rows in silence -- the one outcome worth ruling out by
+        # construction.
+        for day in sorted(fetched):
+            yield from fresh(fetched[day])
+
+    def _manifests(
+        self, manifest_body: dict, *, hydrate: bool = True
+    ) -> Iterator[dict]:
         """Manifests covering the window, splitting it when the service says to.
 
         The manifest hydrates its missing days synchronously, under its own
@@ -516,6 +696,13 @@ class Timeseries(_Namespace):
         a property of the data rather than of the calendar -- a month of
         options trades and a month of equity depth differ by orders of
         magnitude -- and because the warm case must stay one round trip.
+
+        ``hydrate`` rides through the split rather than being applied by
+        the caller afterwards, so a survey that has to be halved stays a
+        survey.  Only a service predating the flag can put it in that
+        position -- one that honours it fetches nothing and so cannot
+        refuse on a hydration ceiling -- but that is exactly the service
+        whose refusal this exists to absorb.
         """
         pending = [(manifest_body["start"], manifest_body["end"])]
         while pending:
@@ -523,7 +710,12 @@ class Timeseries(_Namespace):
             try:
                 payload = self._client._request(
                     "POST", "/v2/timeseries/manifest",
-                    json={**manifest_body, "start": start, "end": end},
+                    json={
+                        **manifest_body,
+                        "start": start,
+                        "end": end,
+                        "hydrate": hydrate,
+                    },
                     timeout=V2_DATA_TIMEOUT,
                 ).json()
             except ResponseTooLargeError as exc:

quantpad_data-0.7.1/tests/test_refusal_hint.py

@@ -0,0 +1,71 @@
+"""A structured refusal's ``hint`` reaches the exception text.
+
+The /v2 404 for a symbol nothing resolves carries what to send instead as
+its own ``hint`` field -- ``NQ.FUT with stype_in='parent'`` for a bare
+``NQ`` -- so a caller reading the body can tell the refusal from the
+advice.  The client used to raise ``message`` alone, which is the shape
+``options.chain`` had with ``truncated``: the server said the useful thing
+and the SDK dropped it before anyone could read it.
+"""
+
+from __future__ import annotations
+
+import json
+
+import pytest
+import requests
+
+from quantpad_data.client import Client
+from quantpad_data.errors import NotFoundError
+
+MESSAGE = "None of the requested symbols resolve to an instrument on GLBX.MDP3 over this window."
+HINT = "For NQ try NQ.FUT with stype_in='parent' (every contract in the chain)."
+
+
+def _response(body: dict, status: int = 404) -> requests.Response:
+    response = requests.Response()
+    response.status_code = status
+    response.url = "https://example.test/v2/timeseries/get_range"
+    response._content = json.dumps({"error": body}).encode()
+    response.headers["Content-Type"] = "application/json"
+    return response
+
+
+def _client() -> Client:
+    return Client(api_key="k", base_url="https://example.test")
+
+
+class TestTheHintIsRaisedWithTheMessage:
+    def test_the_exception_text_carries_both_sentences_in_order(self):
+        with pytest.raises(NotFoundError) as caught:
+            _client()._raise_for_status(
+                _response({"message": MESSAGE, "hint": HINT, "not_found": ["NQ"]}),
+                "/v2/timeseries/get_range",
+            )
+        text = str(caught.value)
+        assert text == f"{MESSAGE} {HINT}"
+
+    def test_the_structured_body_is_still_available_beside_it(self):
+        """Branching callers read the fields; the hint joining the prose
+        must not take the fields away."""
+        message, detail = Client._envelope(
+            _response({"message": MESSAGE, "hint": HINT, "not_found": ["NQ"]})
+        )
+        assert detail["hint"] == HINT
+        assert detail["not_found"] == ["NQ"]
+        assert message.startswith(MESSAGE)
+
+    def test_a_refusal_without_a_hint_reads_exactly_as_before(self):
+        message, _ = Client._envelope(_response({"message": MESSAGE}))
+        assert message == MESSAGE
+
+    @pytest.mark.parametrize("hint", ["", "   ", None, 7, ["not", "prose"]])
+    def test_a_blank_or_non_prose_hint_adds_nothing(self, hint):
+        message, _ = Client._envelope(_response({"message": MESSAGE, "hint": hint}))
+        assert message == MESSAGE
+
+    def test_a_hint_with_no_message_does_not_become_the_message(self):
+        """A body with advice but no refusal is malformed; stringify it
+        whole as the client always has rather than promote the hint."""
+        message, _ = Client._envelope(_response({"hint": HINT}))
+        assert message == str({"hint": HINT})

{quantpad_data-0.6.0 → quantpad_data-0.7.1}/tests/test_v2.py

@@ -12,6 +12,7 @@ from __future__ import annotations
 
 import io
 import json
+from datetime import date, timedelta
 
 import pandas as pd
 import pyarrow as pa
@@ -405,6 +406,392 @@ class TestBulkDelivery:
         assert len(frames) == 2
         assert frames[0].attrs["quantpad"]["day"] == "2026-01-05"
 
+    def test_a_warm_window_is_still_one_round_trip(self, monkeypatch):
+        """The survey is free when there is nothing to survey.
+
+        Asking not to hydrate costs an extra call only if it then has to
+        fetch.  A window whose days are all present must not pay one.
+        """
+        api, session = client_with([json_response(self._manifest("https://r2.test/a.parquet"))])
+        monkeypatch.setattr(
+            "quantpad_data.v2._read_parquet_url",
+            lambda url, columns: pd.DataFrame({"url": [url]}),
+        )
+        list(api.timeseries.iter_range(
+            dataset="GLBX.MDP3", symbols="ESZ6", schema="mbp-10",
+            start="2026-01-05", end="2026-01-06",
+        ))
+        assert len(session.calls) == 1
+        assert session.calls[0][2]["json"]["hydrate"] is False
+
+    def test_the_first_manifest_asks_not_to_fetch(self, monkeypatch):
+        """The whole point: the opening call must not hydrate.
+
+        A cold month spent 129.5 of its 132.4 seconds inside this one
+        call, which is what made it read as a hang.
+        """
+        api, session = client_with([json_response(self._manifest("https://r2.test/a.parquet"))])
+        monkeypatch.setattr(
+            "quantpad_data.v2._read_parquet_url",
+            lambda url, columns: pd.DataFrame({"url": [url]}),
+        )
+        list(api.timeseries.iter_range(
+            dataset="GLBX.MDP3", symbols="ESZ6", schema="mbp-10",
+            start="2026-01-05", end="2026-01-06",
+        ))
+        assert session.calls[0][2]["json"]["hydrate"] is False
+
+    def _survey(self, cached_days, gap_days):
+        return {
+            "format": "parquet",
+            "files": [
+                {"day": day, "instrument_id": 42, "symbol": "ESZ6",
+                 "key": day, "size_bytes": 10, "url": f"https://r2.test/{day}"}
+                for day in cached_days
+            ],
+            "gaps": [{"day": day, "instrument_id": 42, "symbol": "ESZ6"}
+                     for day in gap_days],
+            "metadata": {"dataset": "GLBX.MDP3"},
+        }
+
+    def test_a_cached_day_arrives_before_any_fetch_happens(self, monkeypatch):
+        """The latency claim, expressed as ordering.
+
+        A half-warm window must hand over what it has before it goes and
+        gets what it does not -- that is the difference between a first
+        frame in a second and a first frame in two minutes.
+        """
+        api, session = client_with([
+            json_response(self._survey(["2026-01-05"], ["2026-01-06"])),
+            json_response(self._survey(["2026-01-06"], [])),
+        ])
+        monkeypatch.setattr(
+            "quantpad_data.v2._read_parquet_url",
+            lambda url, columns: pd.DataFrame({"url": [url]}),
+        )
+        frames = api.timeseries.iter_range(
+            dataset="GLBX.MDP3", symbols="ESZ6", schema="mbp-10",
+            start="2026-01-05", end="2026-01-07",
+        )
+        first = next(frames)
+        assert first.attrs["quantpad"]["day"] == "2026-01-05"
+        # One call so far: the cached day was served without the fetch
+        # for the missing one having been made.
+        assert len(session.calls) == 1
+        rest = list(frames)
+        assert [f.attrs["quantpad"]["day"] for f in rest] == ["2026-01-06"]
+        assert len(session.calls) == 2
+
+    def test_fetched_days_keep_their_place_in_the_window(self, monkeypatch):
+        """Chronological ACROSS the join, not cached-then-fetched.
+
+        Yielding everything cached and then everything fetched would be
+        the simpler loop and would silently reorder the window, which
+        iter_range's contract does not allow.
+        """
+        api, _ = client_with([
+            json_response(self._survey(["2026-01-05", "2026-01-07"], ["2026-01-06"])),
+            json_response(self._survey(["2026-01-06"], [])),
+        ])
+        monkeypatch.setattr(
+            "quantpad_data.v2._read_parquet_url",
+            lambda url, columns: pd.DataFrame({"url": [url]}),
+        )
+        days = [f.attrs["quantpad"]["day"] for f in api.timeseries.iter_range(
+            dataset="GLBX.MDP3", symbols="ESZ6", schema="mbp-10",
+            start="2026-01-05", end="2026-01-08",
+        )]
+        assert days == ["2026-01-05", "2026-01-06", "2026-01-07"]
+
+    def test_missing_days_are_fetched_in_a_widening_ramp(self, monkeypatch):
+        """A cold run starts with one day and widens to the cap.
+
+        A flat chunk makes the first frame wait for a whole chunk, which
+        measured 29.0s on a cold eight-day window of equity depth.  The
+        ramp puts the first frame after a single day and is at full
+        width by the time anyone is watching frames arrive.
+        """
+        gaps = [f"2026-01-{day:02d}" for day in range(5, 17)]  # 12 days
+        widths = [1, 2, 4, 5]
+        pieces, at = [], 0
+        for width in widths:
+            pieces.append(gaps[at:at + width])
+            at += width
+        api, session = client_with(
+            [json_response(self._survey([], gaps))]
+            + [json_response(self._survey(piece, [])) for piece in pieces]
+        )
+        monkeypatch.setattr(
+            "quantpad_data.v2._read_parquet_url",
+            lambda url, columns: pd.DataFrame({"url": [url]}),
+        )
+        days = [f.attrs["quantpad"]["day"] for f in api.timeseries.iter_range(
+            dataset="GLBX.MDP3", symbols="ESZ6", schema="mbp-10",
+            start="2026-01-05", end="2026-01-17",
+        )]
+        assert days == gaps
+        asked = [(c[2]["json"]["start"], c[2]["json"]["end"])
+                 for c in session.calls[1:]]
+        assert asked == [
+            ("2026-01-05", "2026-01-06"),   # one day, so a frame arrives
+            ("2026-01-06", "2026-01-08"),   # two
+            ("2026-01-08", "2026-01-12"),   # four
+            ("2026-01-12", "2026-01-17"),   # capped at HYDRATE_CHUNK_DAYS
+        ]
+
+    def test_the_ramp_stops_widening_at_the_cap(self, monkeypatch):
+        """Doubling without a ceiling ends where it started: one request.
+
+        Needs a long window to show at all.  Over twelve days the capped
+        and uncapped ramps happen to agree, because the remainder after
+        1+2+4 is exactly the cap -- so the first version of this test
+        passed against an unbounded ramp and proved nothing.
+        """
+        gaps = [f"2026-01-{day:02d}" for day in range(5, 25)]  # 20 days
+        api, session = client_with(
+            [json_response(self._survey([], gaps))]
+            + [json_response(self._survey([], [])) for _ in range(12)]
+        )
+        monkeypatch.setattr(
+            "quantpad_data.v2._read_parquet_url",
+            lambda url, columns: pd.DataFrame({"url": [url]}),
+        )
+        list(api.timeseries.iter_range(
+            dataset="GLBX.MDP3", symbols="ESZ6", schema="mbp-10",
+            start="2026-01-05", end="2026-01-25",
+        ))
+        widths = [
+            (date.fromisoformat(c[2]["json"]["end"])
+             - date.fromisoformat(c[2]["json"]["start"])).days
+            for c in session.calls[1:]
+        ]
+        from quantpad_data import v2 as v2_module
+
+        assert widths == [1, 2, 4, 5, 5, 3]
+        assert max(widths) == v2_module.HYDRATE_CHUNK_DAYS
+
+    def test_the_ramp_does_not_restart_for_a_later_group_of_gaps(
+        self, monkeypatch
+    ):
+        """The ramp tracks how long the caller has waited, not the group.
+
+        Restarting it per run of consecutive days would drop back to
+        one-day requests deep into a window whose frames are already
+        flowing, spending round trips where they buy nothing.
+        """
+        gaps = ["2026-01-05", "2026-01-06", "2026-01-09", "2026-01-10",
+                "2026-01-11"]
+        api, session = client_with([
+            json_response(self._survey(["2026-01-07"], gaps)),
+            json_response(self._survey(["2026-01-05"], [])),
+            json_response(self._survey(["2026-01-06"], [])),
+            json_response(self._survey(["2026-01-09", "2026-01-10",
+                                        "2026-01-11"], [])),
+        ])
+        monkeypatch.setattr(
+            "quantpad_data.v2._read_parquet_url",
+            lambda url, columns: pd.DataFrame({"url": [url]}),
+        )
+        list(api.timeseries.iter_range(
+            dataset="GLBX.MDP3", symbols="ESZ6", schema="mbp-10",
+            start="2026-01-05", end="2026-01-12",
+        ))
+        asked = [(c[2]["json"]["start"], c[2]["json"]["end"])
+                 for c in session.calls[1:]]
+        # 1 then 2 within the first group; the second group continues at
+        # 4 rather than dropping back to 1.
+        assert asked == [
+            ("2026-01-05", "2026-01-06"),
+            ("2026-01-06", "2026-01-07"),
+            ("2026-01-09", "2026-01-12"),
+        ]
+
+    def test_cached_days_between_two_gaps_are_not_asked_for_again(
+        self, monkeypatch
+    ):
+        """A gap run stops at the first day already held.
+
+        Asking for one window spanning every missing day is the obvious
+        simplification and it drags the cached days in the middle back
+        through the service, which is the cost the survey just avoided.
+        """
+        api, session = client_with([
+            json_response(self._survey(
+                cached_days=["2026-01-06", "2026-01-07"],
+                gap_days=["2026-01-05", "2026-01-08"],
+            )),
+            json_response(self._survey(["2026-01-05"], [])),
+            json_response(self._survey(["2026-01-08"], [])),
+        ])
+        monkeypatch.setattr(
+            "quantpad_data.v2._read_parquet_url",
+            lambda url, columns: pd.DataFrame({"url": [url]}),
+        )
+        days = [f.attrs["quantpad"]["day"] for f in api.timeseries.iter_range(
+            dataset="GLBX.MDP3", symbols="ESZ6", schema="mbp-10",
+            start="2026-01-05", end="2026-01-09",
+        )]
+        assert days == ["2026-01-05", "2026-01-06", "2026-01-07", "2026-01-08"]
+        asked = [(c[2]["json"]["start"], c[2]["json"]["end"])
+                 for c in session.calls[1:]]
+        assert asked == [("2026-01-05", "2026-01-06"),
+                         ("2026-01-08", "2026-01-09")]
+
+    def test_days_are_yielded_in_order_whatever_order_they_are_listed_in(
+        self, monkeypatch
+    ):
+        """Day order is the iterator's promise, not the manifest's.
+
+        Handing back the manifest's own file list would inherit whatever
+        order the service used.  Nothing says that is chronological, and
+        a caller accumulating a rolling statistic would get it wrong
+        without anything failing.
+        """
+        survey = self._survey(["2026-01-07", "2026-01-05", "2026-01-06"], [])
+        api, session = client_with([json_response(survey)])
+        monkeypatch.setattr(
+            "quantpad_data.v2._read_parquet_url",
+            lambda url, columns: pd.DataFrame({"url": [url]}),
+        )
+        days = [f.attrs["quantpad"]["day"] for f in api.timeseries.iter_range(
+            dataset="GLBX.MDP3", symbols="ESZ6", schema="mbp-10",
+            start="2026-01-05", end="2026-01-08",
+        )]
+        assert days == ["2026-01-05", "2026-01-06", "2026-01-07"]
+        assert len(session.calls) == 1
+
+    def test_a_missing_day_that_has_no_data_is_simply_absent(self, monkeypatch):
+        """A holiday comes back empty and must not stall the walk."""
+        api, _ = client_with([
+            json_response(self._survey(["2026-01-05"], ["2026-01-06"])),
+            json_response(self._survey([], [])),
+        ])
+        monkeypatch.setattr(
+            "quantpad_data.v2._read_parquet_url",
+            lambda url, columns: pd.DataFrame({"url": [url]}),
+        )
+        days = [f.attrs["quantpad"]["day"] for f in api.timeseries.iter_range(
+            dataset="GLBX.MDP3", symbols="ESZ6", schema="mbp-10",
+            start="2026-01-05", end="2026-01-07",
+        )]
+        assert days == ["2026-01-05"]
+
+    def test_an_older_service_that_ignores_the_flag_still_works(self, monkeypatch):
+        """Compatibility, and it is the likely case during a rollout.
+
+        A service predating the flag hydrates regardless and answers
+        with no gaps.  That is indistinguishable from a warm window, so
+        the walk streams it directly: slower than it could be, never
+        wrong.
+        """
+        api, session = client_with([
+            json_response(self._survey(["2026-01-05", "2026-01-06"], []))
+        ])
+        monkeypatch.setattr(
+            "quantpad_data.v2._read_parquet_url",
+            lambda url, columns: pd.DataFrame({"url": [url]}),
+        )
+        days = [f.attrs["quantpad"]["day"] for f in api.timeseries.iter_range(
+            dataset="GLBX.MDP3", symbols="ESZ6", schema="mbp-10",
+            start="2026-01-05", end="2026-01-07",
+        )]
+        assert days == ["2026-01-05", "2026-01-06"]
+        assert len(session.calls) == 1
+
+    def test_a_day_no_survey_mentioned_is_still_delivered(self, monkeypatch):
+        """Rows must not vanish because two calls disagreed.
+
+        The cursor walks days the survey named.  If a fetch answers for
+        a day it did not, the obvious loop drops those rows in silence,
+        which is the one outcome worth ruling out by construction.
+        """
+        api, _ = client_with([
+            json_response(self._survey([], ["2026-01-05"])),
+            json_response(self._survey(["2026-01-05", "2026-01-09"], [])),
+        ])
+        monkeypatch.setattr(
+            "quantpad_data.v2._read_parquet_url",
+            lambda url, columns: pd.DataFrame({"url": [url]}),
+        )
+        days = [f.attrs["quantpad"]["day"] for f in api.timeseries.iter_range(
+            dataset="GLBX.MDP3", symbols="ESZ6", schema="mbp-10",
+            start="2026-01-05", end="2026-01-10",
+        )]
+        assert days == ["2026-01-05", "2026-01-09"]
+
+    def _mixed(self, day, cached_ids, gap_ids):
+        """One date, cached for some instruments and missing for others.
+
+        The state the walk used to get wrong, and it needs no contrivance
+        to arrive: two symbols fetched at different times, or a chain
+        whose members were not all asked for at once.
+        """
+        return {
+            "format": "parquet",
+            "files": [
+                {"day": day, "instrument_id": ref, "symbol": f"S{ref}",
+                 "key": f"{day}/{ref}", "size_bytes": 10,
+                 "url": f"https://r2.test/{day}/{ref}"}
+                for ref in cached_ids
+            ],
+            "gaps": [{"day": day, "instrument_id": ref, "symbol": f"S{ref}"}
+                     for ref in gap_ids],
+            "metadata": {"dataset": "GLBX.MDP3"},
+        }
+
+    def test_a_day_cached_for_one_symbol_and_missing_for_another(
+        self, monkeypatch
+    ):
+        """Cache state is per instrument-day, not per day.
+
+        So one date can be in ``files`` for the symbol already fetched
+        and in ``gaps`` for the one that was not.  The fetch is given a
+        WINDOW rather than a list of holes, so it answers with both
+        instruments -- and the cached one would be handed over twice,
+        which a caller concatenating the frames reads as real rows.
+        """
+        api, _ = client_with([
+            json_response(self._mixed("2026-01-05", [42], [43])),
+            json_response(self._mixed("2026-01-05", [42, 43], [])),
+        ])
+        monkeypatch.setattr(
+            "quantpad_data.v2._read_parquet_url",
+            lambda url, columns: pd.DataFrame({"url": [url]}),
+        )
+        refs = [f.attrs["quantpad"]["instrument_id"]
+                for f in api.timeseries.iter_range(
+                    dataset="GLBX.MDP3", symbols=["ESZ6", "ESH7"],
+                    schema="mbp-10", start="2026-01-05", end="2026-01-06",
+                )]
+        assert refs == [42, 43]
+
+    def test_a_partition_is_identified_by_its_key_not_its_url(
+        self, monkeypatch
+    ):
+        """A presigned URL is signed afresh on every call.
+
+        Deduplicating on it would compile, run, and remove nothing --
+        the failure mode worth a test of its own, because the two
+        manifests below describe the same stored object and agree on
+        everything except the signature.
+        """
+        first = self._mixed("2026-01-05", [42], [43])
+        second = self._mixed("2026-01-05", [42, 43], [])
+        second["files"][0]["url"] += "?sig=second"
+        api, _ = client_with([json_response(first), json_response(second)])
+        monkeypatch.setattr(
+            "quantpad_data.v2._read_parquet_url",
+            lambda url, columns: pd.DataFrame({"url": [url]}),
+        )
+        frames = list(api.timeseries.iter_range(
+            dataset="GLBX.MDP3", symbols=["ESZ6", "ESH7"], schema="mbp-10",
+            start="2026-01-05", end="2026-01-06",
+        ))
+        assert [f.attrs["quantpad"]["instrument_id"] for f in frames] == [42, 43]
+        # The first signature won, i.e. the survey's entry was the one
+        # served -- not the refetched duplicate wearing a new URL.
+        assert frames[0]["url"][0].endswith("/2026-01-05/42")
+
     def test_a_bulk_frame_is_labelled_like_an_interactive_one(self, monkeypatch):
         """The bug this pins: storage keys partitions by instrument id and
         stores no name, so bulk came back a column short.  Since
@@ -533,24 +920,100 @@ class TestAWindowTooColdToFetchInOneGo:
             lambda url, columns: pd.DataFrame({"url": [url]}),
         )
 
-    def test_a_window_refused_for_hydration_is_asked_for_in_halves(
-        self, monkeypatch
-    ):
+    def _survey(self, gap_days):
+        """What the opening, non-hydrating call answers.
+
+        Since the survey does not fetch, it is not the call that can be
+        refused for hydration any more -- the fetch of a run of missing
+        days is.  So every case below reaches the splitter through one
+        of these rather than on the first request, which is where the
+        refusal now actually arrives.
+        """
+        return json_response({
+            "format": "parquet",
+            "files": [],
+            "gaps": [{"day": day, "instrument_id": 42, "symbol": "NVDA"}
+                     for day in gap_days],
+            "metadata": {"dataset": "XNAS.ITCH"},
+        })
+
+    def _days(self, first, count):
+        base = date.fromisoformat(first)
+        return [(base + timedelta(days=i)).isoformat() for i in range(count)]
+
+    def _ramped(self, *after):
+        """The survey plus the two narrow windows the ramp asks for first.
+
+        ``iter_range`` opens at one day and widens, and a one-day window
+        cannot be halved, so the splitter is not reachable on the first
+        fetch any more.  These cases therefore let the ramp run up to
+        its four-day window and refuse THAT, which is where a real
+        hydration refusal would now land.
+        """
+        return [
+            self._survey(self._days("2026-08-20", 8)),
+            json_response(self._manifest("d20", day="2026-08-20")),
+            json_response(self._manifest("d21", day="2026-08-21")),
+            *after,
+        ]
+
+    def _run(self, api):
+        return list(api.timeseries.iter_range(
+            dataset="XNAS.ITCH", symbols="NVDA", schema="mbp-10",
+            start="2026-08-20", end="2026-08-28",
+        ))
+
+    def test_an_older_service_still_gets_its_survey_split(self, monkeypatch):
+        """The survey can be refused, but only by a service that ignores it.
+
+        One that honours ``hydrate=false`` fetches nothing and so has no
+        hydration ceiling to breach.  One that predates the flag drops
+        it and fetches the whole window -- and that is exactly the
+        service the splitting was written for, so the survey has to go
+        through the splitter rather than ask once.  Asking once reads as
+        a pure win against today's deployment and quietly withdraws the
+        earlier fix everywhere else.
+        """
         api, session = client_with([
             self._refusal(),
-            json_response(self._manifest("https://r2.test/a.parquet", day="2026-08-21")),
-            json_response(self._manifest("https://r2.test/b.parquet", day="2026-08-22")),
+            json_response(self._manifest("first", day="2026-08-20")),
+            json_response(self._manifest("second", day="2026-08-25")),
         ])
         self._reading(monkeypatch)
-        frames = list(api.timeseries.iter_range(
-            dataset="XNAS.ITCH", symbols="NVDA", schema="mbp-10",
-            start="2026-08-20", end="2026-09-17",
-        ))
-        assert len(frames) == 2
+        frames = self._run(api)
+
         assert self._windows(session) == [
-            ("2026-08-20", "2026-09-17"),
-            ("2026-08-20", "2026-09-03"),
-            ("2026-09-03", "2026-09-17"),
+            ("2026-08-20", "2026-08-28"),   # refused
+            ("2026-08-20", "2026-08-24"),   # so halved
+            ("2026-08-24", "2026-08-28"),
+        ]
+        # Both halves are surveys still, not fetches: splitting a survey
+        # must not turn it into the hydrating call it was avoiding.
+        assert [call[2]["json"]["hydrate"] for call in session.calls] == [
+            False, False, False,
+        ]
+        # An older service reports no gaps, so the whole window reads as
+        # cached and both halves' files are streamed in day order.
+        assert [f["url"][0] for f in frames] == ["first", "second"]
+
+    def test_a_window_refused_for_hydration_is_asked_for_in_halves(
+        self, monkeypatch
+    ):
+        api, session = client_with(self._ramped(
+            self._refusal(),
+            json_response(self._manifest("d23", day="2026-08-23")),
+            json_response(self._manifest("d25", day="2026-08-25")),
+            json_response(self._manifest("d27", day="2026-08-27")),
+        ))
+        self._reading(monkeypatch)
+        self._run(api)
+        assert self._windows(session)[1:] == [
+            ("2026-08-20", "2026-08-21"),   # ramp: one day
+            ("2026-08-21", "2026-08-23"),   # two
+            ("2026-08-23", "2026-08-27"),   # four -- refused
+            ("2026-08-23", "2026-08-25"),   # so halved
+            ("2026-08-25", "2026-08-27"),
+            ("2026-08-27", "2026-08-28"),   # and the tail of the run
         ]
 
     def test_the_halves_meet_exactly_once_so_no_day_is_read_twice(
@@ -562,41 +1025,35 @@ class TestAWindowTooColdToFetchInOneGo:
         yielded twice — a silently doubled series, worse than the refusal
         the splitting exists to avoid.
         """
-        api, session = client_with([
+        api, session = client_with(self._ramped(
             self._refusal(),
-            json_response(self._manifest("https://r2.test/a.parquet", day="2026-08-21")),
-            json_response(self._manifest("https://r2.test/b.parquet", day="2026-08-22")),
-        ])
-        self._reading(monkeypatch)
-        list(api.timeseries.iter_range(
-            dataset="XNAS.ITCH", symbols="NVDA", schema="mbp-10",
-            start="2026-08-20T09:30", end="2026-09-17T16:00",
+            json_response(self._manifest("d23", day="2026-08-23")),
+            json_response(self._manifest("d25", day="2026-08-25")),
+            json_response(self._manifest("d27", day="2026-08-27")),
         ))
-        served = self._windows(session)[1:]
-        assert served[0][1] == served[1][0]
-        # The caller's own edges survive, times and all; only the internal
-        # boundary is invented.
-        assert served[0][0] == "2026-08-20T09:30"
-        assert served[-1][1] == "2026-09-17T16:00"
+        self._reading(monkeypatch)
+        self._run(api)
+        halves = self._windows(session)[4:6]
+        assert halves[0][1] == halves[1][0]
+        # And together they still span exactly the window that was refused.
+        assert (halves[0][0], halves[1][1]) == ("2026-08-23", "2026-08-27")
 
     def test_a_half_still_too_large_is_split_again(self, monkeypatch):
-        api, session = client_with([
+        api, session = client_with(self._ramped(
             self._refusal(),
             self._refusal(),
-            json_response(self._manifest("https://r2.test/a.parquet", day="2026-08-21")),
-            json_response(self._manifest("https://r2.test/b.parquet", day="2026-08-22")),
-            json_response(self._manifest("https://r2.test/c.parquet", day="2026-08-23")),
-        ])
-        self._reading(monkeypatch)
-        frames = list(api.timeseries.iter_range(
-            dataset="XNAS.ITCH", symbols="NVDA", schema="mbp-10",
-            start="2026-08-20", end="2026-09-17",
+            json_response(self._manifest("d23", day="2026-08-23")),
+            json_response(self._manifest("d24", day="2026-08-24")),
+            json_response(self._manifest("d25", day="2026-08-25")),
+            json_response(self._manifest("d27", day="2026-08-27")),
         ))
-        assert len(frames) == 3
-        assert self._windows(session)[-3:] == [
-            ("2026-08-20", "2026-08-27"),
-            ("2026-08-27", "2026-09-03"),
-            ("2026-09-03", "2026-09-17"),
+        self._reading(monkeypatch)
+        self._run(api)
+        assert self._windows(session)[3:7] == [
+            ("2026-08-23", "2026-08-27"),   # refused
+            ("2026-08-23", "2026-08-25"),   # refused again
+            ("2026-08-23", "2026-08-24"),   # so halved once more
+            ("2026-08-24", "2026-08-25"),
         ]
 
     def test_the_pieces_arrive_in_order(self, monkeypatch):
@@ -606,18 +1063,17 @@ class TestAWindowTooColdToFetchInOneGo:
         before the remainder of the first, which a caller accumulating a
         series or a rolling statistic would silently get wrong.
         """
-        api, _ = client_with([
-            self._refusal(), self._refusal(),
-            json_response(self._manifest("first", day="2026-08-21")),
-            json_response(self._manifest("second", day="2026-08-22")),
-            json_response(self._manifest("third", day="2026-08-23")),
-        ])
-        self._reading(monkeypatch)
-        frames = list(api.timeseries.iter_range(
-            dataset="XNAS.ITCH", symbols="NVDA", schema="mbp-10",
-            start="2026-08-20", end="2026-09-17",
+        api, _ = client_with(self._ramped(
+            self._refusal(),
+            json_response(self._manifest("d23", day="2026-08-23")),
+            json_response(self._manifest("d25", day="2026-08-25")),
+            json_response(self._manifest("d27", day="2026-08-27")),
         ))
-        assert [f["url"][0] for f in frames] == ["first", "second", "third"]
+        self._reading(monkeypatch)
+        frames = self._run(api)
+        assert [f["url"][0] for f in frames] == [
+            "d20", "d21", "d23", "d25", "d27",
+        ]
 
     def test_a_single_day_that_cannot_be_fetched_is_reported_not_looped(self):
         """Nothing smaller can be asked for, so the refusal has to surface.
@@ -625,14 +1081,17 @@ class TestAWindowTooColdToFetchInOneGo:
         Halving a one-day window returns the same window, so a splitter
         that did not stop here would ask for it for ever.
         """
-        api, session = client_with([self._refusal()])
+        api, session = client_with([
+            self._survey(["2026-08-20"]),
+            self._refusal(),
+        ])
         with pytest.raises(ResponseTooLargeError) as excinfo:
             list(api.timeseries.iter_range(
                 dataset="XNAS.ITCH", symbols="NVDA", schema="mbp-10",
                 start="2026-08-20", end="2026-08-21",
             ))
         assert excinfo.value.reason == "hydration_limit"
-        assert len(session.calls) == 1
+        assert len(session.calls) == 2
 
     def test_a_window_carrying_times_across_two_days_is_still_split(
         self, monkeypatch
@@ -643,18 +1102,25 @@ class TestAWindowTooColdToFetchInOneGo:
         spans two daily partitions, so there is a midnight between them to
         cut at.  Reading it as "already one day" refused a window that could
         have been served, on the retry path that exists to serve it.
+
+        Asked through ``mode='bulk'`` rather than ``iter_range``: the
+        iterator now fetches missing days in day-aligned runs, so a
+        caller's times never reach the splitter from there.  Bulk still
+        hands it the window exactly as written, so this is the path that
+        still has the question — and the reason the time handling in
+        :func:`_split_window` is load-bearing rather than vestigial.
         """
         api, session = client_with([
             self._refusal(),
-            json_response(self._manifest("https://r2.test/a.parquet", day="2026-08-21")),
-            json_response(self._manifest("https://r2.test/b.parquet", day="2026-08-22")),
+            json_response(self._manifest("https://r2.test/a.parquet", day="2026-08-20")),
+            json_response(self._manifest("https://r2.test/b.parquet", day="2026-08-21")),
         ])
         self._reading(monkeypatch)
-        frames = list(api.timeseries.iter_range(
+        frame = api.timeseries.get_range(
             dataset="XNAS.ITCH", symbols="NVDA", schema="mbp-10",
-            start="2026-08-20T13:00", end="2026-08-21T13:00",
-        ))
-        assert len(frames) == 2
+            start="2026-08-20T13:00", end="2026-08-21T13:00", mode="bulk",
+        )
+        assert len(frame) == 2
         assert self._windows(session) == [
             ("2026-08-20T13:00", "2026-08-21T13:00"),   # refused
             ("2026-08-20T13:00", "2026-08-21"),         # up to the midnight