quantpad-data 0.3.2__tar.gz → 0.4.1__tar.gz

{quantpad_data-0.3.2 → quantpad_data-0.4.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: quantpad-data
-Version: 0.3.2
+Version: 0.4.1
 Summary: Official Python client for the QuantPad market-data API
 Project-URL: Documentation, https://api.quantpad.ai/external/docs
 Author: QuantPad
@@ -131,10 +131,22 @@ covering a different period than you asked for.
 
 ### Large ranges
 
-`get_range` buffers a single response. When that would be too large, it falls
-back automatically to reading parquet straight from object storage — the bytes
-never traverse the data service. Force either behaviour with
-`mode="interactive"` or `mode="bulk"`, or stream day by day:
+`get_range` buffers a single response, so it tells the service up front how much
+this process can hold. Two different things can then be too large, and they are
+answered differently:
+
+- **Too large for the service to assemble in one response.** `get_range` falls
+  back automatically to reading parquet straight from object storage — the bytes
+  never traverse the data service.
+- **Too large for this process to hold.** You get a `ResponseTooLargeError`
+  naming the estimated size, because no delivery mode makes a frame fit in
+  memory it does not fit in. Stream it with `iter_range` instead, or pass
+  `max_decoded_bytes=` if the memory really is available.
+
+The budget is derived from the memory visible to the process (a container's
+limit, if there is one). Override it per call with `max_decoded_bytes=`, or
+globally with the `QUANTPAD_MAX_DECODED_BYTES` environment variable. Force a
+delivery path with `mode="interactive"` or `mode="bulk"`, or stream day by day:
 
 ```python
 for day in client.timeseries.iter_range(
@@ -144,6 +156,11 @@ for day in client.timeseries.iter_range(
     process(day)
 ```
 
+`iter_range` holds one day at a time rather than the whole window, and splits
+the window itself when a single request would have to fetch too much uncached
+data at once. On a cold window the first frame can take a while to arrive, since
+the days behind it are fetched before it is handed over.
+
 For ranges too large even for that, submit a batch job, then hydrate it into
 the cache so ordinary `get_range` calls serve it from storage:
 

{quantpad_data-0.3.2 → quantpad_data-0.4.1}/README.md

@@ -110,10 +110,22 @@ covering a different period than you asked for.
 
 ### Large ranges
 
-`get_range` buffers a single response. When that would be too large, it falls
-back automatically to reading parquet straight from object storage — the bytes
-never traverse the data service. Force either behaviour with
-`mode="interactive"` or `mode="bulk"`, or stream day by day:
+`get_range` buffers a single response, so it tells the service up front how much
+this process can hold. Two different things can then be too large, and they are
+answered differently:
+
+- **Too large for the service to assemble in one response.** `get_range` falls
+  back automatically to reading parquet straight from object storage — the bytes
+  never traverse the data service.
+- **Too large for this process to hold.** You get a `ResponseTooLargeError`
+  naming the estimated size, because no delivery mode makes a frame fit in
+  memory it does not fit in. Stream it with `iter_range` instead, or pass
+  `max_decoded_bytes=` if the memory really is available.
+
+The budget is derived from the memory visible to the process (a container's
+limit, if there is one). Override it per call with `max_decoded_bytes=`, or
+globally with the `QUANTPAD_MAX_DECODED_BYTES` environment variable. Force a
+delivery path with `mode="interactive"` or `mode="bulk"`, or stream day by day:
 
 ```python
 for day in client.timeseries.iter_range(
@@ -123,6 +135,11 @@ for day in client.timeseries.iter_range(
     process(day)
 ```
 
+`iter_range` holds one day at a time rather than the whole window, and splits
+the window itself when a single request would have to fetch too much uncached
+data at once. On a cold window the first frame can take a while to arrive, since
+the days behind it are fetched before it is handed over.
+
 For ranges too large even for that, submit a batch job, then hydrate it into
 the cache so ordinary `get_range` calls serve it from storage:
 

{quantpad_data-0.3.2 → quantpad_data-0.4.1}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
 
 [project]
 name = "quantpad-data"
-version = "0.3.2"
+version = "0.4.1"
 description = "Official Python client for the QuantPad market-data API"
 readme = "README.md"
 requires-python = ">=3.10"

{quantpad_data-0.3.2 → quantpad_data-0.4.1}/quantpad_data/client.py

@@ -20,6 +20,7 @@ from .errors import (
     QuantPadDataError,
     QuotaExceededError,
     RateLimitError,
+    ResponseTooLargeError,
     UpstreamError,
     V2NotAvailableError,
     ValidationError,
@@ -89,7 +90,7 @@ V2_NOT_AVAILABLE = (
     "QuantPad support about /v2 access."
 )
 
-VERSION = "0.3.2"
+VERSION = "0.4.1"
 USER_AGENT = f"quantpad-data-python/{VERSION}"
 TICK_SCHEMAS = (
     "trades", "mbp-1", "mbp-10", "cmbp-1", "tcbbo", "cbbo-1s",
@@ -157,19 +158,35 @@ class Client:
         )
 
     @staticmethod
-    def _message(response: requests.Response) -> str:
+    def _envelope(response: requests.Response) -> tuple[str, dict | None]:
+        """The prose and the structured body of a refusal, separately.
+
+        Some refusals answer with an object rather than a string -- a size
+        estimate, a breadth count, which of two limits was breached -- and
+        stringifying the whole thing put a Python dict repr in front of the
+        user where the service had written a sentence.  So the prose is
+        unwrapped from ``message`` when there is one, and the object is
+        kept beside it for the callers that need to branch on it.
+        """
         try:
             body = response.json()
-            if isinstance(body, dict):
-                return str(body.get("error") or body.get("detail") or body)
         except ValueError:
-            pass
-        return (response.text or f"HTTP {response.status_code}").strip()
+            return (response.text or f"HTTP {response.status_code}").strip(), None
+        if not isinstance(body, dict):
+            return str(body), None
+        error = body.get("error") or body.get("detail") or body
+        if isinstance(error, dict):
+            return str(error.get("message") or error), error
+        return str(error), None
+
+    @classmethod
+    def _message(cls, response: requests.Response) -> str:
+        return cls._envelope(response)[0]
 
     def _raise_for_status(self, response: requests.Response, path: str = "") -> None:
         if response.ok:
             return
-        message = self._message(response)
+        message, detail = self._envelope(response)
         retry_after: float | None = None
         try:
             retry_after = float(response.headers["Retry-After"])
@@ -194,12 +211,17 @@ class Client:
             raise PermissionDeniedError(message, status)
         if status == 404:
             raise NotFoundError(message)
+        if status == 413:
+            # Typed, because this is the one refusal a caller has to act on
+            # differently depending on WHY, and the reason is in the body
+            # rather than the status.  See ResponseTooLargeError.
+            raise ResponseTooLargeError(message, status, detail)
         if status == 429:
             cls = QuotaExceededError if "quota" in message.lower() else RateLimitError
             raise cls(message, retry_after, status)
         if status >= 500:
             raise UpstreamError(message, status)
-        raise QuantPadDataError(message, status)
+        raise QuantPadDataError(message, status, detail)
 
     def _request(self, method: str, path: str, **kwargs) -> requests.Response:
         kwargs.setdefault("timeout", self.timeout)

{quantpad_data-0.3.2 → quantpad_data-0.4.1}/quantpad_data/errors.py

@@ -16,9 +16,22 @@ class QuantPadDataError(RuntimeError):
     #: run this __init__.
     status_code: int | None = None
 
-    def __init__(self, message: str, status_code: int | None = None):
+    #: The structured body, when the service sent one.  Some refusals carry
+    #: machine-readable fields beside their prose -- a size estimate, or
+    #: which of two limits was breached -- and a client that has to match
+    #: on message text to find them is a client that breaks when the text
+    #: is improved.
+    detail: dict | None = None
+
+    def __init__(
+        self,
+        message: str,
+        status_code: int | None = None,
+        detail: dict | None = None,
+    ):
         super().__init__(message)
         self.status_code = status_code
+        self.detail = detail
 
 
 class AuthenticationError(QuantPadDataError):
@@ -72,6 +85,50 @@ class V2NotAvailableError(NotFoundError, PermissionDeniedError):
         self.status_code = status_code
 
 
+class ResponseTooLargeError(QuantPadDataError):
+    """The answer will not fit somewhere, and the two somewheres differ.
+
+    ``reason`` says which:
+
+    ``"service_limit"``
+        Above a ceiling this service enforces to protect itself.  Bulk
+        delivery genuinely answers this, which is why ``mode="auto"``
+        switches to it rather than failing.
+
+    ``"client_budget"``
+        Above what this process said it could hold.  Bulk delivery does
+        NOT answer this -- it ends in one ``concat`` and returns a frame
+        just as large, holding every day's frame at once to build it -- so
+        falling back would be a second route to the same MemoryError.
+        ``iter_range`` is the answer, because it never holds more than one
+        partition.
+
+    A subclass of :class:`QuantPadDataError` and raised for the same 413 as
+    before, so ``except QuantPadDataError`` and any code branching on
+    ``status_code == 413`` keeps working.
+    """
+
+    def __init__(
+        self,
+        message: str,
+        status_code: int | None = None,
+        detail: dict | None = None,
+    ):
+        super().__init__(message, status_code, detail)
+
+    @property
+    def reason(self) -> str | None:
+        return (self.detail or {}).get("reason")
+
+    @property
+    def estimated_bytes(self) -> int | None:
+        return ((self.detail or {}).get("estimate") or {}).get("bytes")
+
+    @property
+    def estimated_rows(self) -> int | None:
+        return ((self.detail or {}).get("estimate") or {}).get("rows")
+
+
 class ValidationError(ValueError, QuantPadDataError):
     pass
 

quantpad_data-0.4.1/quantpad_data/memory.py

@@ -0,0 +1,150 @@
+"""How much of this answer this process can afford to hold.
+
+WHY THIS EXISTS
+``get_range`` returns one DataFrame, so it holds the whole answer.  The
+service's own ceilings do not protect that: the materializing path caps at
+2 GiB and the streaming path at 64 GiB, and both numbers exist to keep the
+SERVICE alive.  A notebook with 4 GB is not protected by either, and a month of
+liquid-equity depth prices at about 17 GB -- so it was served, buffered, and
+killed the kernel with no error to read.  That is the shape of the "all
+projects hang, no error" report.
+
+So the caller declares what it can hold and the service refuses above it.  Only
+the caller can know the number, and only the service can price the request
+before fetching it, which is why it takes both.
+
+THE UNITS ARE NOT THE SAME, AND THAT IS THE WHOLE DIFFICULTY
+The service prices in DBN bytes: record count times record width, which is
+Databento's own billing arithmetic.  This process pays for the compressed wire
+bytes, an Arrow table decoded from them, and a pandas frame copied out of that
+table -- partly all three at once.  Measured with
+scripts/measure_client_memory.py against production, peak RSS per priced byte
+was:
+
+    equity mbp-10, 1 day    693 MB priced    1.7 GB peak    2.49x
+    equity mbp-1,  1 day     69 MB priced    276 MB peak    4.02x
+    OPRA trades,   1 day     73 MB priced    370 MB peak    5.05x
+
+The ratio is LARGER for narrow records, which is the opposite of the intuition:
+DBN packs a trade into 48 bytes while pandas gives every column at least 8, so
+a narrow record inflates more than a wide one.  Hence a single factor taken
+from the worst case rather than a per-schema table.
+"""
+
+from __future__ import annotations
+
+import os
+
+#: Peak process bytes per byte the service prices.  Worst measured was 5.05x;
+#: rounded up because three schemas are not the whole catalogue and the two
+#: errors are not symmetric -- too high refuses a request that would have
+#: fitted, which under ``mode="auto"`` costs a slower path, while too low
+#: returns to killing the process with no message.
+PEAK_PER_PRICED_BYTE = 6.0
+
+#: cgroup v1 spells "no limit" as a huge number rather than a word, and the
+#: exact sentinel varies with page size, so anything absurd is read as absent.
+_NO_LIMIT = 1 << 53
+
+#: Overrides the detected figure.  For an operator who has to correct this
+#: from outside the notebook, which is the only lever available in a sandbox.
+BUDGET_ENV = "QUANTPAD_MAX_DECODED_BYTES"
+
+
+def _read_int(path: str) -> int | None:
+    try:
+        with open(path) as handle:
+            return int(handle.read().strip())
+    except (OSError, ValueError):
+        return None
+
+
+def _cgroup_free() -> int | None:
+    """Headroom under this container's memory limit, if it has one.
+
+    Consulted BEFORE /proc/meminfo and allowed to win, because meminfo
+    reports the HOST's memory: in a sandbox on a large machine that is wrong
+    by an order of magnitude and wrong in the generous direction, which is
+    the direction that kills the process.
+    """
+    for limit_path, usage_path in (
+        ("/sys/fs/cgroup/memory.max", "/sys/fs/cgroup/memory.current"),
+        (
+            "/sys/fs/cgroup/memory/memory.limit_in_bytes",
+            "/sys/fs/cgroup/memory/memory.usage_in_bytes",
+        ),
+    ):
+        limit = _read_int(limit_path)
+        # v2 writes the literal string "max", which _read_int reports as
+        # None; v1 writes a sentinel.  Both mean unlimited.
+        if limit is None or limit >= _NO_LIMIT:
+            continue
+        return max(limit - (_read_int(usage_path) or 0), 0)
+    return None
+
+
+def _meminfo_available() -> int | None:
+    """MemAvailable, the kernel's own estimate of what can be allocated.
+
+    Preferred over MemFree, which excludes reclaimable cache and so
+    understates by gigabytes on any machine that has been up a while.
+    """
+    try:
+        with open("/proc/meminfo") as handle:
+            for line in handle:
+                if line.startswith("MemAvailable:"):
+                    return int(line.split()[1]) * 1024
+    except (OSError, ValueError, IndexError):
+        pass
+    return None
+
+
+def available_bytes() -> int | None:
+    """What this process can still allocate, or None if it cannot be told.
+
+    None matters as much as a number: it means declare nothing and leave the
+    service's own ceilings in charge, which is the behaviour before this
+    existed.  Guessing instead would refuse real requests on platforms where
+    the guess is wrong, and the platform this cannot read is somebody's
+    laptop rather than the sandbox this exists for.
+    """
+    candidates = [
+        value
+        for value in (_cgroup_free(), _meminfo_available())
+        if value is not None
+    ]
+    return min(candidates) if candidates else None
+
+
+def declared_budget(mode: str) -> int | None:
+    """The figure to send as ``max_decoded_bytes``, in the service's units.
+
+    TWO NUMBERS, BECAUSE THE COST OF BEING WRONG DIFFERS BY MODE.
+
+    Under ``"auto"`` a refusal is cheap: the caller asked for a frame and can
+    be told to iterate instead, before anything is fetched.  So it declares
+    the conservative figure -- free memory divided by the measured expansion
+    -- and errs towards refusing something that would have fitted.
+
+    Under ``"interactive"`` the caller has insisted on one buffered response
+    and there is nowhere to degrade to, so a refusal is a hard failure.  It
+    therefore declares free memory UNDIVIDED, which refuses only what cannot
+    possibly fit: peak is larger than the priced figure in every case ever
+    measured (2.49x at the lowest), so a request priced above all available
+    memory is a certain MemoryError rather than a risk of one.  Nothing that
+    works today starts failing.
+    """
+    override = os.getenv(BUDGET_ENV, "").strip()
+    if override:
+        try:
+            parsed = int(override)
+        except ValueError:
+            parsed = 0
+        return parsed if parsed > 0 else None
+
+    free = available_bytes()
+    if free is None:
+        return None
+    if mode == "interactive":
+        return free
+    return int(free / PEAK_PER_PRICED_BYTE)