matrx-orm 3.1.232__tar.gz → 3.1.233__tar.gz

{matrx_orm-3.1.232 → matrx_orm-3.1.233}/.gitignore

@@ -97,7 +97,7 @@ db.sqlite3
 db.sqlite3-journal
 
 # Flask stuff:
-instance/
+/instance/
 .webassets-cache
 
 # Scrapy stuff:

{matrx_orm-3.1.232 → matrx_orm-3.1.233}/CLAUDE.md

@@ -164,7 +164,11 @@ enforced in code:
    `bulk_create` (or uses `insert_returning`) so its rows are neither read back
    nor cached. Before this an unread entry lived for the whole process, and a
    workflow worker's checkpoints/outcomes/events grew it until ECS killed the
-   task for memory (MKT-111, 2026-10-15). Guard: `tests/test_state_cache_sweeps_stale.py`.
+   task for memory (MKT-111, 2026-10-15). The sweep pops oldest-first and stops at
+   the first fresh entry, and `cache_bulk` yields the loop once per sweep window, so
+   caching a big read never stalls the server (a full-scan sweep froze it 2-6 s on a
+   21k-row read, 2026-10-17). Guards: `tests/test_state_cache_sweeps_stale.py`,
+   `tests/test_a_bulk_cache_never_stalls_the_loop.py`.
 
 This is not theoretical hygiene. On 2026-07-25 rule 1 was missing: `session/flush.py`
 builds a `{pk + payload}` stub per queued UPDATE and `bulk_update` cached it, so a

{matrx_orm-3.1.232 → matrx_orm-3.1.233}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: matrx-orm
-Version: 3.1.232
+Version: 3.1.233
 Summary: Async-first PostgreSQL ORM with bidirectional migrations, schema introspection, many-to-many relationships, and built-in state caching
 Project-URL: Homepage, https://github.com/AI-Matrix-Engine/aidream-current
 Project-URL: Repository, https://github.com/AI-Matrix-Engine/aidream-current
@@ -26,7 +26,7 @@ Requires-Dist: asyncpg>=0.31.0
 Requires-Dist: cbor2<7,>=6.1.3
 Requires-Dist: cryptography>=43.0
 Requires-Dist: gitpython>=3.1.40
-Requires-Dist: matrx-utils>=2.0.74
+Requires-Dist: matrx-utils>=2.0.75
 Requires-Dist: psycopg-pool>=3.2.5
 Requires-Dist: psycopg[binary]>=3.2.5
 Requires-Dist: python-dotenv>=1.0

{matrx_orm-3.1.232 → matrx_orm-3.1.233}/matrx_orm/operations/catalog_select.py

@@ -58,6 +58,65 @@ class UnregisteredRelationError(ValueError):
     """
 
 
+# JSON path selections on a json/jsonb column, e.g. ``provider_entry->>version``.
+#
+# Accepted forms (the ONLY ones; anything else is refused naming this list):
+#   col->key           jsonb value        col->>key            text value
+#   col->k1->k2->>k3   chained steps      col#>{k1,k2}         jsonb at path
+#   col#>>{k1,k2}      text at path       (keys may be quoted: col->>'a-b')
+# A key is letters, digits, ``_``, ``-`` or ``.``; an all-digit key is an array
+# index. Keys are NEVER interpolated: they travel as one bound ``text[]`` parameter
+# to ``#>`` / ``#>>``, and the column itself must be a live json/jsonb column.
+JSON_PATH_FORMS = (
+    "col->key, col->>key, col->k1->k2->>k3, col#>{k1,k2}, col#>>{k1,k2} "
+    "(col must be a json/jsonb column; keys are letters, digits, _ - .)"
+)
+_BARE_KEY = r"[A-Za-z0-9_.]+(?:-(?!>)[A-Za-z0-9_.]+)*"  # a "-" inside a key, never the "->" arrow
+_JSON_KEY = rf"(?:'{_BARE_KEY}'|{_BARE_KEY})"
+_JSON_ARROWS = re.compile(
+    rf"^(?P<col>[A-Za-z_][A-Za-z0-9_$]*)(?P<steps>(?:->>?{_JSON_KEY})+)$"
+)
+_JSON_HASH = re.compile(
+    rf"^(?P<col>[A-Za-z_][A-Za-z0-9_$]*)(?P<op>#>>?)\{{(?P<keys>{_JSON_KEY}(?:,{_JSON_KEY})*)\}}$"
+)
+_JSON_STEP = re.compile(rf"(->>?)({_JSON_KEY})")
+_JSON_TYPES = {"json", "jsonb"}
+
+
+def looks_like_json_path(spec: str) -> bool:
+    return "->" in spec or "#>" in spec
+
+
+def parse_json_path(spec: str) -> tuple[str, bool, list[str]]:
+    """``spec`` -> ``(column, as_text, keys)`` or raise naming the accepted forms."""
+    text = spec.strip()
+    m = _JSON_HASH.match(text)
+    if m:
+        keys = [k.strip("'") for k in m.group("keys").split(",")]
+        return m.group("col"), m.group("op") == "#>>", keys
+    m = _JSON_ARROWS.match(text)
+    if m:
+        steps = _JSON_STEP.findall(m.group("steps"))
+        # ``->>`` returns text, so it can only be the LAST step.
+        if any(op == "->>" for op, _ in steps[:-1]):
+            raise UnregisteredRelationError(
+                f"Field {spec!r}: ->> returns text, so it can only be the last step. "
+                f"Accepted JSON path forms: {JSON_PATH_FORMS}."
+            )
+        return m.group("col"), steps[-1][0] == "->>", [k.strip("'") for _, k in steps]
+    raise UnregisteredRelationError(
+        f"Field {spec!r} is not a column or a supported JSON path. "
+        f"Accepted JSON path forms: {JSON_PATH_FORMS}."
+    )
+
+
+def _quote_alias(spec: str) -> str:
+    """Quote a validated JSON-path spec as the result column's name."""
+    if '"' in spec or "\x00" in spec:
+        raise UnregisteredRelationError(f"Unsafe field name: {spec!r}")
+    return f'"{spec}"'
+
+
 def _quote(ident: str) -> str:
     if not _SAFE_IDENT.match(ident):
         raise UnregisteredRelationError(f"Unsafe SQL identifier: {ident!r}")
@@ -98,6 +157,13 @@ async def resolve_relation(table: str, *, database: str | None = None) -> tuple[
     )
 
 
+async def _describe(schema: str, name: str, database: str) -> list[dict[str, Any]]:
+    from matrx_orm.catalog import describe_relation_columns
+
+    rows = await describe_relation_columns(database, schema=schema, table=name)
+    return [r for r in rows if isinstance(r, dict) and r.get("column_name")]
+
+
 async def live_columns(schema: str, name: str, *, database: str | None = None) -> list[str]:
     """Ordered live column names for a relation, straight from pg_catalog."""
     from matrx_orm.catalog import describe_relation_columns
@@ -202,7 +268,9 @@ async def catalog_select(
         raise UnregisteredRelationError(f'relation "{schema}.{name}" does not exist')
     live_set = set(live)
 
-    requested = [c for c in columns if c != "*"]
+    requested_all = [c for c in columns if c != "*"]
+    json_specs = {c: parse_json_path(c) for c in requested_all if looks_like_json_path(c)}
+    requested = [c for c in requested_all if c not in json_specs]
     order_fields = [f.removeprefix("-") for f in order_by]
     filter_fields = [getattr(f, "field", "") for f in filters]
     unknown = [
@@ -211,7 +279,27 @@ async def catalog_select(
         if c and c not in live_set
     ]
     if unknown:
-        raise _unknown_columns_error(schema, name, unknown, live, registered_fields)
+        err = _unknown_columns_error(schema, name, unknown, live, registered_fields)
+        if any(looks_like_json_path(u) for u in unknown):
+            raise UnregisteredRelationError(
+                f"{err}\nJSON paths are accepted in `fields` only (not match/order_by). "
+                f"Accepted forms: {JSON_PATH_FORMS}."
+            )
+        raise err
+    if json_specs:
+        type_by_col = {
+            str(r["column_name"]): str(r.get("data_type") or "").lower()
+            for r in await _describe(schema, name, db)
+        }
+        for spec, (col, _as_text, _keys) in json_specs.items():
+            if col not in live_set:
+                raise _unknown_columns_error(schema, name, [col], live, registered_fields)
+            dtype = type_by_col.get(col)
+            if dtype and dtype not in _JSON_TYPES:
+                raise UnregisteredRelationError(
+                    f"Field {spec!r}: column {col!r} is {dtype}, not json/jsonb, so a JSON "
+                    f"path cannot be taken from it. Accepted JSON path forms: {JSON_PATH_FORMS}."
+                )
 
     params: list[Any] = []
 
@@ -219,7 +307,15 @@ async def catalog_select(
         params.append(value)
         return f"${len(params)}"
 
-    select_list = ", ".join(_quote(c) for c in requested) if requested else "*"
+    parts_select: list[str] = []
+    for c in requested_all:
+        if c in json_specs:
+            col, as_text, keys = json_specs[c]
+            op = "#>>" if as_text else "#>"
+            parts_select.append(f"{_quote(col)} {op} {next_param(keys)}::text[] AS {_quote_alias(c)}")
+        else:
+            parts_select.append(_quote(c))
+    select_list = ", ".join(parts_select) if parts_select else "*"
     sql = f"SELECT {select_list} FROM {_quote(schema)}.{_quote(name)}"
 
     predicates = [

{matrx_orm-3.1.232 → matrx_orm-3.1.233}/matrx_orm/state.py

@@ -269,25 +269,33 @@ class ModelState:
             )
         return matches[0] if matches else None
 
+    def _stale_before(self, now: datetime | None = None) -> datetime | None:
+        """The cache time an entry must be newer than to be fresh; None = never stale.
+
+        One model has ONE age rule, so staleness is a pure function of the
+        entry's cache time — which is what lets :meth:`sweep_stale` stop at the
+        first fresh entry instead of re-checking every row.
+        """
+        now = now or datetime.now()
+        if self.timeout is not None:
+            return now - timedelta(seconds=self.timeout)
+        if self.policy == CachePolicy.PERMANENT:
+            return None
+        if self.policy == CachePolicy.LONG_TERM:
+            return now - timedelta(hours=4)
+        if self.policy == CachePolicy.SHORT_TERM:
+            return now - timedelta(minutes=10)
+        if self.policy == CachePolicy.INSTANT:
+            return now - timedelta(minutes=1)
+        return datetime.max
+
     def _is_stale(self, record):
         cache_key = self._get_record_cache_key(record)
         cache_time = self._cache_times.get(cache_key)
         if not cache_time:
             return True
-
-        if self.timeout is not None:
-            return (datetime.now() - cache_time).total_seconds() > self.timeout
-
-        now = datetime.now()
-        if self.policy == CachePolicy.PERMANENT:
-            return False
-        elif self.policy == CachePolicy.LONG_TERM:
-            return now - cache_time > timedelta(hours=4)
-        elif self.policy == CachePolicy.SHORT_TERM:
-            return now - cache_time > timedelta(minutes=10)
-        elif self.policy == CachePolicy.INSTANT:
-            return now - cache_time > timedelta(minutes=1)
-        return True
+        cutoff = self._stale_before()
+        return cutoff is not None and cache_time < cutoff
 
     async def cache(self, record):
         """Cache a record with proper error handling.
@@ -336,7 +344,7 @@ class ModelState:
                     return
 
             self._cache[cache_key] = record
-            self._cache_times[cache_key] = datetime.now()
+            self._stamp(cache_key)
             self._writes_since_sweep = getattr(self, "_writes_since_sweep", 0) + 1
             if self._writes_since_sweep >= STALE_SWEEP_EVERY_WRITES:
                 self.sweep_stale()
@@ -356,6 +364,15 @@ class ModelState:
                 original_error=e,
             ) from e
 
+    def _stamp(self, cache_key) -> None:
+        """Record ``cache_key`` as cached NOW, at the END of ``_cache_times``.
+
+        ``_cache_times`` is kept in cache-time order (a re-cache moves the key to
+        the end), so the oldest entries are always at the front.
+        """
+        self._cache_times.pop(cache_key, None)
+        self._cache_times[cache_key] = datetime.now()
+
     def sweep_stale(self) -> int:
         """Drop every entry its policy already treats as stale. Returns how many.
 
@@ -366,18 +383,30 @@ class ModelState:
         cache) accumulated for the life of the process, multi-megabyte JSON
         included, until ECS killed the task for memory. A stale entry is
         already a miss, so dropping it changes no read.
+
+        THE STALL THIS CLOSES (2026-10-17): the first version re-checked EVERY
+        entry on every sweep. A sweep runs every ``STALE_SWEEP_EVERY_WRITES``
+        writes, so one bulk read of N fresh rows did N/200 full scans — the AI
+        catalog's 21,677-row rule-cell reload held the event loop 2-6 s and
+        froze every request on the server. ``_cache_times`` is in cache-time
+        order and one model has one age rule, so the sweep pops from the front
+        and stops at the first fresh entry: its cost is what it drops, never
+        what is cached.
         """
         self._writes_since_sweep = 0
+        cutoff = self._stale_before()
+        times = self._cache_times
         dropped = 0
-        for key in list(self._cache_times):
-            record = self._cache.get(key)
-            if record is None or self._is_stale(record):
-                self._cache.pop(key, None)
-                self._cache_times.pop(key, None)
-                lock = self._locks.get(key)
-                if lock is not None and not lock.locked():
-                    self._locks.pop(key, None)
-                dropped += 1
+        while times:
+            key = next(iter(times))
+            if key in self._cache and (cutoff is None or times[key] >= cutoff):
+                break
+            times.pop(key, None)
+            self._cache.pop(key, None)
+            lock = self._locks.get(key)
+            if lock is not None and not lock.locked():
+                self._locks.pop(key, None)
+            dropped += 1
         return dropped
 
     def _downgraded_fields(self, existing, incoming) -> list[str]:
@@ -419,7 +448,7 @@ class ModelState:
         record = self.model_class.from_db_result(data)
         cache_key = self._get_record_cache_key(record)
         self._cache[cache_key] = record
-        self._cache_times[cache_key] = datetime.now()
+        self._stamp(cache_key)
 
     def get_all_cached(self):
         """Returns all cached records as a list."""
@@ -620,7 +649,7 @@ class StateManager:
                     reason="Model not registered with StateManager",
                 )
 
-            for record in records:
+            for index, record in enumerate(records, start=1):
                 try:
                     pk_kwargs = {pk: getattr(record, pk) for pk in record._meta.primary_keys}
                     if not await state.get(**pk_kwargs):
@@ -628,6 +657,11 @@ class StateManager:
                 except Exception as e:
                     vcprint(f"Failed to cache record: {str(e)}", color="yellow")
                     continue
+                # An uncontended lock never yields, so a 20k-row read cached in
+                # one go held the loop for its whole length. Hand the loop back
+                # once per sweep window: other requests run between chunks.
+                if index % STALE_SWEEP_EVERY_WRITES == 0:
+                    await asyncio.sleep(0)
         except StateError:
             raise
         except Exception as e:

{matrx_orm-3.1.232 → matrx_orm-3.1.233}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "matrx-orm"
-version = "3.1.232"
+version = "3.1.233"
 description = "Async-first PostgreSQL ORM with bidirectional migrations, schema introspection, many-to-many relationships, and built-in state caching"
 readme = "README.md"
 license = { text = "MIT" }
@@ -41,7 +41,7 @@ dependencies = [
     "gitpython>=3.1.40",
     "psycopg[binary]>=3.2.5",
     "psycopg-pool>=3.2.5",
-    "matrx-utils>=2.0.74",
+    "matrx-utils>=2.0.75",
     "pyyaml>=6.0.3",
     "python-dotenv>=1.0",
     "cryptography>=43.0",

matrx_orm-3.1.233/tests/test_a_bulk_cache_never_stalls_the_loop.py

@@ -0,0 +1,113 @@
+"""A bulk read never freezes the server while its rows are cached (2026-10-17).
+
+The stale sweep (MKT-111) re-checked EVERY cached entry each time it ran, and it ran
+every ``STALE_SWEEP_EVERY_WRITES`` writes — so caching N fresh rows did N/200 full
+scans. The AI catalog's 21,677-row rule-cell reload held the event loop 2-6 s in
+production (ops.system_error EventLoopStall, frame ``state.py:342 sweep_stale``) and
+froze every request on the server. ``cache_bulk`` also never yielded, so even a
+linear pass over 20k rows held the loop for its whole length.
+"""
+
+from __future__ import annotations
+
+import asyncio
+import types
+from datetime import datetime, timedelta
+
+import pytest
+
+from matrx_orm import state as state_module
+from matrx_orm.state import CachePolicy, ModelState, StateManager
+
+
+def _model(policy: CachePolicy, name: str):
+    meta = types.SimpleNamespace(primary_keys=["id"])
+    return type(
+        name,
+        (),
+        {
+            "_cache_policy": policy,
+            "_fields": {},
+            "_meta": meta,
+            "get_database_name": staticmethod(lambda: "bulk-cache-test"),
+        },
+    )
+
+
+def _record(model, i: int) -> object:
+    rec = types.SimpleNamespace(id=f"row-{i}")
+    rec._meta = model._meta
+    return rec
+
+
+@pytest.mark.asyncio
+async def test_caching_fresh_rows_costs_one_key_per_row(monkeypatch: pytest.MonkeyPatch) -> None:
+    model = _model(CachePolicy.SHORT_TERM, "_LinearModel")
+    state = ModelState(model)
+    calls = 0
+    real = ModelState._get_record_cache_key
+
+    def counting(self, record):
+        nonlocal calls
+        calls += 1
+        return real(self, record)
+
+    monkeypatch.setattr(ModelState, "_get_record_cache_key", counting)
+    n = 4000
+    for i in range(n):
+        await state.cache(_record(model, i))
+    assert state.count() == n
+    # The quadratic sweep made this ~n + n*n/(2*200) = 44,000; the ordered sweep is n.
+    assert calls <= 2 * n, f"{calls} per-row checks to cache {n} fresh rows"
+
+
+@pytest.mark.asyncio
+async def test_bulk_cache_hands_the_loop_back() -> None:
+    model = _model(CachePolicy.SHORT_TERM, "_YieldModel")
+    StateManager.register_model(model)
+    turns = 0
+    done = False
+
+    async def other_request() -> None:
+        nonlocal turns
+        while not done:
+            turns += 1
+            await asyncio.sleep(0)
+
+    other = asyncio.create_task(other_request())
+    await asyncio.sleep(0)
+    start = turns
+    await StateManager.cache_bulk(model, [_record(model, i) for i in range(2000)])
+    ran_during = turns - start
+    done = True
+    await other
+    assert await StateManager.count(model) == 2000
+    assert ran_during >= 5, f"another request ran {ran_during} times during a 2000-row cache"
+
+
+@pytest.mark.asyncio
+async def test_the_sweep_drops_only_what_aged_out(monkeypatch: pytest.MonkeyPatch) -> None:
+    monkeypatch.setattr(state_module, "STALE_SWEEP_EVERY_WRITES", 10_000)
+    model = _model(CachePolicy.SHORT_TERM, "_AgeModel")
+    state = ModelState(model)
+    for i in range(6):
+        await state.cache(_record(model, i))
+    old = datetime.now() - timedelta(minutes=11)  # SHORT_TERM is 10 minutes
+    for key in list(state._cache_times):
+        state._cache_times[key] = old
+    # row-2 is cached again now: it moves behind the old ones and must survive.
+    await state.cache(_record(model, 2))
+    await state.cache(_record(model, 9))
+    assert state.sweep_stale() == 5
+    assert sorted(state._cache) == ["row-2", "row-9"]
+    assert state.sweep_stale() == 0
+
+
+@pytest.mark.asyncio
+async def test_a_permanent_model_is_never_swept() -> None:
+    model = _model(CachePolicy.PERMANENT, "_PermanentModel")
+    state = ModelState(model)
+    await state.cache(_record(model, 1))
+    state._cache_times["row-1"] = datetime.now() - timedelta(days=30)
+    assert state.sweep_stale() == 0
+    assert "row-1" in state._cache

{matrx_orm-3.1.232 → matrx_orm-3.1.233}/tests/test_dynamic_select_unregistered.py

@@ -166,3 +166,70 @@ async def test_unsafe_identifier_never_reaches_postgres(monkeypatch) -> None:
 
     with pytest.raises(UnregisteredRelationError):
         await dynamic_select("ai.provider", columns=('evil"; drop table x; --',), database=DB)
+
+
+def _wire_typed(monkeypatch, columns: dict[str, str], rows: list[dict[str, Any]]) -> dict[str, Any]:
+    seen = _wire_catalog(monkeypatch, columns=list(columns), rows=rows)
+
+    async def describe_relation_columns(database: str, *, schema: str, table: str):
+        return [{"column_name": c, "data_type": t} for c, t in columns.items()]
+
+    monkeypatch.setattr("matrx_orm.catalog.describe_relation_columns", describe_relation_columns)
+    return seen
+
+
+@pytest.mark.asyncio
+async def test_json_path_selection_is_built_with_bound_keys(monkeypatch) -> None:
+    """2026-10-17: an agent asked for ``provider_entry->>version`` and was told
+    the column did not exist. Paths are validated against the live json column and
+    the keys travel as a bound text[] — never interpolated."""
+    monkeypatch.setattr(
+        dynamic_crud,
+        "resolve_dynamic_model",
+        lambda table, *, database=None: (_ for _ in ()).throw(ValueError("none")),
+    )
+    seen = _wire_typed(
+        monkeypatch,
+        {"model_id": "text", "provider_entry": "jsonb"},
+        rows=[{"model_id": "m", "provider_entry->>version": "001"}],
+    )
+
+    rows = await dynamic_select(
+        "ai.provider_sync_candidates",
+        columns=("model_id", "provider_entry->>version", "provider_entry->a->>'b-c'", "provider_entry#>{x,0}"),
+        database=DB,
+    )
+
+    assert rows == [{"model_id": "m", "provider_entry->>version": "001"}]
+    assert seen["sql"] == (
+        'SELECT "model_id", "provider_entry" #>> $1::text[] AS "provider_entry->>version", '
+        '"provider_entry" #>> $2::text[] AS "provider_entry->a->>\'b-c\'", '
+        '"provider_entry" #> $3::text[] AS "provider_entry#>{x,0}" '
+        'FROM "ai"."provider_sync_candidates"'
+    )
+    assert seen["params"] == [["version"], ["a", "b-c"], ["x", "0"]]
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize(
+    "spec, needle",
+    [
+        ("provider_entry->>ver sion", "Accepted JSON path forms"),
+        ("provider_entry->>a->b", "only be the last step"),
+        ("provider_entry->>x; drop table t", "Accepted JSON path forms"),
+        ("model_id->>version", "not json/jsonb"),
+        ("nope->>version", "does not exist"),
+    ],
+)
+async def test_bad_json_path_names_the_accepted_forms(monkeypatch, spec, needle) -> None:
+    monkeypatch.setattr(
+        dynamic_crud,
+        "resolve_dynamic_model",
+        lambda table, *, database=None: (_ for _ in ()).throw(ValueError("none")),
+    )
+    _wire_typed(monkeypatch, {"model_id": "text", "provider_entry": "jsonb"}, rows=[])
+
+    with pytest.raises(UnregisteredRelationError) as excinfo:
+        await dynamic_select("ai.provider_sync_candidates", columns=(spec,), database=DB)
+
+    assert needle in str(excinfo.value)