quantpad-data 0.5.0__tar.gz → 0.5.1__tar.gz

{quantpad_data-0.5.0 → quantpad_data-0.5.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: quantpad-data
-Version: 0.5.0
+Version: 0.5.1
 Summary: Official Python client for the QuantPad market-data API
 Project-URL: Documentation, https://api.quantpad.ai/external/docs
 Author: QuantPad

{quantpad_data-0.5.0 → quantpad_data-0.5.1}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
 
 [project]
 name = "quantpad-data"
-version = "0.5.0"
+version = "0.5.1"
 description = "Official Python client for the QuantPad market-data API"
 readme = "README.md"
 requires-python = ">=3.10"

{quantpad_data-0.5.0 → quantpad_data-0.5.1}/quantpad_data/client.py

@@ -84,7 +84,7 @@ V2_NOT_AVAILABLE = (
     "QuantPad support about /v2 access."
 )
 
-VERSION = "0.5.0"
+VERSION = "0.5.1"
 USER_AGENT = f"quantpad-data-python/{VERSION}"
 TICK_SCHEMAS = (
     "trades", "mbp-1", "mbp-10", "cmbp-1", "tcbbo", "cbbo-1s",

{quantpad_data-0.5.0 → quantpad_data-0.5.1}/quantpad_data/memory.py

@@ -59,6 +59,34 @@ def _read_int(path: str) -> int | None:
         return None
 
 
+def _stat_map(path: str) -> dict[str, int]:
+    """A cgroup ``memory.stat`` file as a mapping, empty if unreadable."""
+    out: dict[str, int] = {}
+    try:
+        with open(path) as handle:
+            for line in handle:
+                key, _, value = line.partition(" ")
+                try:
+                    out[key] = int(value)
+                except ValueError:
+                    continue
+    except OSError:
+        pass
+    return out
+
+
+def _unreclaimable(stat: dict[str, int], keys: tuple[str, ...]) -> int | None:
+    """What this cgroup holds that the kernel cannot take back.
+
+    Everything charged to a cgroup is *not* equally spent.  Page cache is
+    charged, and is handed back the moment anything needs the memory, so
+    subtracting it from the limit describes a process that is nowhere near
+    its ceiling as though it were at it.
+    """
+    present = [stat[key] for key in keys if key in stat]
+    return sum(present) if present else None
+
+
 def _cgroup_free() -> int | None:
     """Headroom under this container's memory limit, if it has one.
 
@@ -66,12 +94,34 @@ def _cgroup_free() -> int | None:
     reports the HOST's memory: in a sandbox on a large machine that is wrong
     by an order of magnitude and wrong in the generous direction, which is
     the direction that kills the process.
+
+    HEADROOM IS THE LIMIT LESS WHAT CANNOT BE RECLAIMED, NOT LESS WHAT IS
+    CHARGED, and the difference is the whole reason this function is not two
+    lines.  ``memory.current`` includes page cache, so *writing a file* looks
+    identical to *allocating memory*: measured on a 4 GiB Daytona sandbox,
+    2 GiB of files on disk took this from 3.91 GiB to 1.93 GiB while the
+    process's own ``anon`` never left 0.09 GiB, and deleting the files put it
+    straight back.  Under ``mode="auto"`` that halved the declared budget and
+    refused windows that fitted twice over, with a message naming memory the
+    caller had.  The disk cache makes it worse by design: every partition it
+    writes, and every one it reads back, is more page cache.
+
+    Note this is also what makes the two sources answer the SAME question --
+    ``MemAvailable`` already counts reclaimable cache as available, so the old
+    subtraction had them disagreeing and ``min()`` let the wrong one win.
     """
-    for limit_path, usage_path in (
-        ("/sys/fs/cgroup/memory.max", "/sys/fs/cgroup/memory.current"),
+    for limit_path, usage_path, stat_path, keys in (
+        (
+            "/sys/fs/cgroup/memory.max",
+            "/sys/fs/cgroup/memory.current",
+            "/sys/fs/cgroup/memory.stat",
+            ("anon", "unevictable", "slab_unreclaimable"),
+        ),
         (
             "/sys/fs/cgroup/memory/memory.limit_in_bytes",
             "/sys/fs/cgroup/memory/memory.usage_in_bytes",
+            "/sys/fs/cgroup/memory/memory.stat",
+            ("total_rss", "total_unevictable"),
         ),
     ):
         limit = _read_int(limit_path)
@@ -79,7 +129,13 @@ def _cgroup_free() -> int | None:
         # None; v1 writes a sentinel.  Both mean unlimited.
         if limit is None or limit >= _NO_LIMIT:
             continue
-        return max(limit - (_read_int(usage_path) or 0), 0)
+        spent = _unreclaimable(_stat_map(stat_path), keys)
+        if spent is None:
+            # No usable stat file.  Fall back to the charged figure, which is
+            # pessimistic rather than dangerous: it can refuse a request that
+            # would have fitted, and never admits one that will not.
+            spent = _read_int(usage_path) or 0
+        return max(limit - spent, 0)
     return None
 
 

{quantpad_data-0.5.0 → quantpad_data-0.5.1}/tests/test_v2.py

@@ -1059,6 +1059,107 @@ class TestWhatThisProcessCanHold:
         assert memory.available_bytes() is None
         assert memory.declared_budget("auto") is None
 
+    @staticmethod
+    def _fake_cgroup(monkeypatch, *, v2=True, limit=4 * 2**30, stat=None,
+                     charged=None):
+        """Stand in for the cgroup files, since their paths are absolute."""
+        limit_name = "memory.max" if v2 else "memory.limit_in_bytes"
+        usage_name = "memory.current" if v2 else "memory.usage_in_bytes"
+
+        def read_int(path):
+            if limit_name in path:
+                return limit
+            if usage_name in path:
+                return charged
+            return None
+
+        monkeypatch.setattr(memory, "_read_int", read_int)
+        monkeypatch.setattr(memory, "_stat_map", lambda path: dict(stat or {}))
+
+    def test_page_cache_is_not_charged_to_the_caller(self, monkeypatch):
+        """The defect this arithmetic exists to stop, measured not imagined.
+
+        On a 4 GiB Daytona sandbox, writing 2 GiB of files took the detected
+        figure from 3.91 GiB to 1.93 GiB while the process's own ``anon``
+        stayed at 0.09 GiB, and deleting the files restored it exactly.  Under
+        ``mode="auto"`` that halved the declared budget and refused windows
+        that fitted twice over, blaming memory the caller had.  The disk cache
+        feeds it: every partition written and every one read back is page
+        cache.
+        """
+        limit = 4 * 2**30
+        self._fake_cgroup(
+            monkeypatch,
+            limit=limit,
+            # 2 GiB of page cache on top of a 100 MiB process.
+            charged=100 * 2**20 + 2 * 2**30,
+            stat={
+                "anon": 100 * 2**20,
+                "file": 2 * 2**30,
+                "unevictable": 0,
+                "slab_unreclaimable": 8 * 2**20,
+            },
+        )
+
+        free = memory._cgroup_free()
+
+        assert free == limit - (100 * 2**20 + 8 * 2**20)
+        # Emphatically not the charged figure, which is what it used to be.
+        assert free > 3 * 2**30
+
+    def test_a_cgroup_v1_container_reads_its_resident_set(self, monkeypatch):
+        """v1 spells the same quantity ``total_rss`` rather than ``anon``."""
+        limit = 2 * 2**30
+        self._fake_cgroup(
+            monkeypatch,
+            v2=False,
+            limit=limit,
+            charged=limit,  # wholly charged, almost all of it cache
+            stat={
+                "total_rss": 300 * 2**20,
+                "total_cache": limit - 300 * 2**20,
+                "total_unevictable": 0,
+            },
+        )
+
+        assert memory._cgroup_free() == limit - 300 * 2**20
+
+    def test_a_process_that_really_is_near_its_limit_still_reads_as_full(
+        self, monkeypatch
+    ):
+        """The guard must still guard.
+
+        Discounting page cache is only correct because it is reclaimable;
+        anonymous memory is not, so a process that has genuinely filled its
+        cgroup must still report almost nothing free.
+        """
+        limit = 4 * 2**30
+        self._fake_cgroup(
+            monkeypatch,
+            limit=limit,
+            charged=limit,
+            stat={"anon": limit - 64 * 2**20, "file": 32 * 2**20},
+        )
+
+        assert memory._cgroup_free() == 64 * 2**20
+        assert memory.declared_budget("auto") < 16 * 2**20
+
+    def test_an_unreadable_stat_file_falls_back_to_the_charged_figure(
+        self, monkeypatch
+    ):
+        """Pessimistic rather than dangerous.
+
+        Without ``memory.stat`` there is no way to separate cache from
+        anonymous memory, and the old subtraction is the safe direction: it
+        can refuse something that would have fitted, but never admits
+        something that will not.
+        """
+        limit = 4 * 2**30
+        self._fake_cgroup(monkeypatch, limit=limit, stat={},
+                          charged=3 * 2**30)
+
+        assert memory._cgroup_free() == limit - 3 * 2**30
+
     def test_an_unlimited_cgroup_is_not_read_as_a_limit(self, monkeypatch):
         """v2 writes the literal "max" and v1 writes a huge sentinel.