matrx-orm 3.1.237__tar.gz → 3.1.239__tar.gz

{matrx_orm-3.1.237 → matrx_orm-3.1.239}/.gitignore

@@ -355,3 +355,8 @@ apps/shared/zz-proof-*/
 
 # tsup writes this temp file while it builds a config; never commit it
 tsup.config.bundled_*.mjs
+
+# db/tests/test_provision_pull.py writes a projection for a spec row it ALWAYS rolls back;
+# if the test is interrupted before its scrub, a sweep commit must never pick it up
+# (2026-10-19: one was swept in and failed 5 aidream releases with identity.token.taken).
+db/migrations/*_provision_provision_pull_selftest.sql

{matrx_orm-3.1.237 → matrx_orm-3.1.239}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: matrx-orm
-Version: 3.1.237
+Version: 3.1.239
 Summary: Async-first PostgreSQL ORM with bidirectional migrations, schema introspection, many-to-many relationships, and built-in state caching
 Project-URL: Homepage, https://github.com/AI-Matrix-Engine/aidream-current
 Project-URL: Repository, https://github.com/AI-Matrix-Engine/aidream-current
@@ -26,7 +26,7 @@ Requires-Dist: asyncpg>=0.31.0
 Requires-Dist: cbor2<7,>=6.1.3
 Requires-Dist: cryptography>=43.0
 Requires-Dist: gitpython>=3.1.40
-Requires-Dist: matrx-utils>=2.0.78
+Requires-Dist: matrx-utils>=2.0.79
 Requires-Dist: psycopg-pool>=3.2.5
 Requires-Dist: psycopg[binary]>=3.2.5
 Requires-Dist: python-dotenv>=1.0

{matrx_orm-3.1.237 → matrx_orm-3.1.239}/matrx_orm/core/extended.py

@@ -36,6 +36,9 @@ class BaseManager(Generic[ModelT]):
     # for that manager.  Has no effect when Pydantic is not installed.
     validate_input: bool = True
 
+    # Knob: hard ceiling for fetch-on-init (rows). Reaching it raises; never truncates.
+    fetch_on_init_max_rows: int = 50_000
+
     def __init__(
         self,
         model: type[ModelT],
@@ -1744,6 +1747,43 @@ class BaseManager(Generic[ModelT]):
 
         run_sync(self._auto_fetch_on_init_async())
 
+    async def _fetch_whole_table(self) -> list[ModelT]:
+        """Return EVERY row of the table, never a truncated prefix.
+
+        ``fetch_on_init_limit`` is the PAGE SIZE (a knob), not a cap: a
+        fetch-on-init cache that claims to hold the table must hold all of it
+        (2026-10 defect: a 300 cap silently dropped 84 of 384 ai.model_definition
+        rows). Single-PK tables page by keyset (``pk > last``); composite-PK
+        tables page by offset over a full-PK order. Past
+        ``fetch_on_init_max_rows`` the fetch RAISES (loud at boot) instead of
+        continuing with partial data — switch such a manager to load-on-miss.
+        """
+        page = int(self.fetch_on_init_limit)
+        pks = list(self.model._meta.primary_keys or ["id"])
+        ceiling = int(self.fetch_on_init_max_rows)
+        rows: list[ModelT] = []
+        last: Any = None
+        while True:
+            qb = self.model.filter()
+            if len(pks) == 1:
+                if last is not None:
+                    qb = qb.filter(**{f"{pks[0]}__gt": last})
+                qb = qb.order_by(pks[0]).limit(page)
+            else:
+                qb = qb.order_by(*pks).limit(page).offset(len(rows))
+            batch = await qb.all()
+            rows.extend(batch)
+            if len(rows) > ceiling:
+                raise RuntimeError(
+                    f"[{self.model.__name__}] fetch-on-init exceeded fetch_on_init_max_rows="
+                    f"{ceiling}; this table is too large to cache whole — make the manager "
+                    "load-on-miss (fetch_on_init_limit=0) or raise the knob deliberately."
+                )
+            if len(batch) < page:
+                return rows
+            if len(pks) == 1:
+                last = getattr(batch[-1], pks[0])
+
     async def _auto_fetch_on_init_async(self) -> None:
         """Fetch items on initialization (async version with full DTO support)."""
         if int(self.fetch_on_init_limit) <= 0:
@@ -1751,7 +1791,7 @@ class BaseManager(Generic[ModelT]):
 
         start_time = time.perf_counter()
 
-        items = await self.model.filter().limit(self.fetch_on_init_limit).all()
+        items = await self._fetch_whole_table()
         initialized_items: list[ModelT] = []
         for item in items:
             result = await self._initialize_item_runtime(item)
@@ -1797,9 +1837,6 @@ class BaseManager(Generic[ModelT]):
                         color="red",
                     )
 
-        if count >= (self.fetch_on_init_limit - 5):
-            self._trigger_limit_reached_warning(count, initialized_items)
-
         if not warnings_suppressed and count > warning_limit_threshold:
             self._trigger_fetch_warnings(count, initialized_items, warning_limit_threshold)
 

{matrx_orm-3.1.237 → matrx_orm-3.1.239}/matrx_orm/session/flush.py

@@ -168,13 +168,17 @@ async def _execute_group(
         pks = getattr(meta, "primary_keys", None) if meta else None
         pk_field = pks[0] if pks else ops[0].pk_field
 
-        # Union of all payload keys across this group — bulk_update wants
-        # ONE fields list applied to all instances. Per-row, matrx-orm's
-        # bulk_update skips None values, so clearing a field via this path
-        # is unsupported (acceptable for our use cases).
-        all_fields: list[str] = sorted({key for op in ops for key in op.payload.keys()})
-
-        instances = []
+        # bulk_update applies ONE fields list to every instance it is given,
+        # reading each field off the instance. A stub carries the Model's
+        # Python DEFAULT in every column its op never set, so a single call
+        # with the UNION of the group's keys wrote those defaults over real
+        # values (PB-08: a completed chat.tool_call row reset to
+        # status='pending', output_type='text', output_chars=0 because the
+        # parallel call's completion flushed in the same group). One call per
+        # distinct field set: every row writes exactly the fields its op set.
+        # Per-row, bulk_update skips None values, so clearing a field via this
+        # path is unsupported (acceptable for our use cases).
+        by_fields: dict[tuple[str, ...], list[object]] = {}
         for op in ops:
             kwargs = {pk_field: op.pk_value}
             kwargs.update(op.payload)
@@ -184,10 +188,15 @@ async def _execute_group(
             # entry and blank every other column for the TTL (this is the bug
             # that made a saved conversation read back with no organization_id
             # on its next turn). See matrx_orm.state.mark_partial.
-            instances.append(mark_partial(model_cls(**kwargs), kwargs))
-
-        await model_cls.bulk_update(instances, fields=all_fields)
-        return len(instances)
+            fields = tuple(sorted(op.payload.keys()))
+            by_fields.setdefault(fields, []).append(mark_partial(model_cls(**kwargs), kwargs))
+
+        written = 0
+        for fields, instances in by_fields.items():
+            if fields:
+                await model_cls.bulk_update(instances, fields=list(fields))
+            written += len(instances)
+        return written
 
     if op_type == "delete":
         meta = getattr(model_cls, "_meta", None)

{matrx_orm-3.1.237 → matrx_orm-3.1.239}/matrx_orm/session/reads.py

@@ -179,19 +179,63 @@ def merge_pending_with_db_one(
 def _matches_simple_filter(payload: dict[str, Any], filters: dict[str, Any]) -> bool:
     """Return True if `payload` matches every (field, value) in `filters`.
 
-    Only literal-equality filters supported. Anything with operator
-    suffixes (``foo__lt``, ``foo__in``, etc.) is treated as "unknown" and
-    we say "the payload matches" — letting the DB result be authoritative
-    rather than dropping a potentially-valid pending row. This is the
-    safe direction (false positive is correctable on next flush; false
-    negative is a missed read).
+    Literal equality, plus the operators that can be judged exactly on an
+    in-memory payload: ``in`` / ``nin`` / ``not_in`` / ``ne`` / ``isnull`` and
+    ordered comparisons on comparable values. Only an operator that cannot be
+    judged here (JSON paths, text search, a field the payload does not carry)
+    answers "might match".
+
+    "Might match" for EVERY operator was a lie with a victim: an ``exists()``
+    asking "is a run live on conversation X?" (``id__in=[…],
+    status__in=['pending','processing']``) answered yes for ANY conversation
+    while the caller's own request INSERT was pending in its Session, so a
+    finished helper was steered instead of continued and never answered
+    (PB-08, conversation 6e3f85a3…, 2026-10-19).
     """
     for key, expected in filters.items():
         if "__" in key:
-            # Operator-suffixed filter — skip the check; treat as "might match."
+            field, _, op = key.rpartition("__")
+            verdict = _judge_operator(payload, field, op, expected)
+            if verdict is False:
+                return False
             continue
         if key not in payload:
             return False
         if payload[key] != expected:
             return False
     return True
+
+
+def _same(a: Any, b: Any) -> bool:
+    """Equality that treats a uuid and its string as the same value."""
+    return a == b or (a is not None and b is not None and str(a) == str(b))
+
+
+def _judge_operator(payload: dict[str, Any], field: str, op: str, expected: Any) -> bool | None:
+    """True / False when the operator can be judged on ``payload``; None = cannot."""
+    if not field or "__" in field or field not in payload:
+        return None
+    value = payload[field]
+    if op == "isnull":
+        return (value is None) == bool(expected)
+    if op in ("in", "nin", "not_in"):
+        if isinstance(expected, (str, bytes)) or not hasattr(expected, "__iter__"):
+            return None
+        member = any(_same(value, e) for e in expected)
+        return member if op == "in" else not member
+    if op == "ne":
+        return not _same(value, expected)
+    if op in ("gt", "gte", "lt", "lte"):
+        if value is None or expected is None:
+            return None
+        try:
+            if op == "gt":
+                return value > expected
+            if op == "gte":
+                return value >= expected
+            if op == "lt":
+                return value < expected
+            return value <= expected
+        except TypeError:
+            return None
+    return None

{matrx_orm-3.1.237 → matrx_orm-3.1.239}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "matrx-orm"
-version = "3.1.237"
+version = "3.1.239"
 description = "Async-first PostgreSQL ORM with bidirectional migrations, schema introspection, many-to-many relationships, and built-in state caching"
 readme = "README.md"
 license = { text = "MIT" }
@@ -41,7 +41,7 @@ dependencies = [
     "gitpython>=3.1.40",
     "psycopg[binary]>=3.2.5",
     "psycopg-pool>=3.2.5",
-    "matrx-utils>=2.0.78",
+    "matrx-utils>=2.0.79",
     "pyyaml>=6.0.3",
     "python-dotenv>=1.0",
     "cryptography>=43.0",

matrx_orm-3.1.239/tests/level1/test_fetch_on_init_holds_whole_table.py

@@ -0,0 +1,116 @@
+"""A fetch-on-init cache holds the WHOLE table or fails loudly — never a silent prefix.
+
+Pins the 2026-10 defect: fetch_on_init_limit=300 on a 384-row table loaded 300
+rows and only printed a banner. Plus a census of every fetch-on-init manager.
+"""
+
+import re
+from pathlib import Path
+
+import pytest
+from matrx_orm.core.base import Model, ModelMeta
+from matrx_orm.core.extended import BaseManager
+from matrx_orm.core.fields import CharField, IntegerField
+from matrx_orm.core.registry import ModelRegistry
+
+
+@pytest.fixture(autouse=True)
+def _clean_registry():
+    ModelRegistry.clear()
+    yield
+    ModelRegistry.clear()
+
+
+class _FakeQB:
+    def __init__(self, rows, pk):
+        self.rows, self.pk = rows, pk
+        self._gt = None
+        self._limit = None
+        self._offset = 0
+
+    def filter(self, **kw):
+        for k, v in kw.items():
+            assert k == f"{self.pk}__gt"
+            self._gt = v
+        return self
+
+    def order_by(self, *a):
+        return self
+
+    def limit(self, n):
+        self._limit = n
+        return self
+
+    def offset(self, n):
+        self._offset = n
+        return self
+
+    async def all(self):
+        rows = sorted(self.rows, key=lambda r: getattr(r, self.pk))
+        if self._gt is not None:
+            rows = [r for r in rows if getattr(r, self.pk) > self._gt]
+        rows = rows[self._offset :]
+        return rows[: self._limit] if self._limit is not None else rows
+
+
+def _manager(n, limit, max_rows=None):
+    M = ModelMeta(
+        "BigTable",
+        (Model,),
+        {"id": IntegerField(primary_key=True), "name": CharField(), "_table_name": "big", "_database": "t"},
+    )
+    rows = [M(id=i, name=str(i)) for i in range(n)]
+    M.filter = classmethod(lambda cls, **kw: _FakeQB(rows, "id"))
+
+    class Mgr(BaseManager):
+        async def _initialize_runtime_data(self, item):
+            pass
+
+    if max_rows is not None:
+        Mgr.fetch_on_init_max_rows = max_rows
+    return Mgr(M, fetch_on_init_limit=limit)
+
+
+@pytest.mark.asyncio
+async def test_more_rows_than_limit_are_all_loaded():
+    mgr = _manager(n=387, limit=300)
+    await mgr._auto_fetch_on_init_async()
+    assert len(mgr._active_items) == 387
+
+
+@pytest.mark.asyncio
+async def test_exact_multiple_of_page_size_loads_all():
+    mgr = _manager(n=600, limit=300)
+    await mgr._auto_fetch_on_init_async()
+    assert len(mgr._active_items) == 600
+
+
+@pytest.mark.asyncio
+async def test_ceiling_raises_instead_of_partial():
+    mgr = _manager(n=50, limit=10, max_rows=30)
+    with pytest.raises(RuntimeError, match="fetch_on_init_max_rows"):
+        await mgr._auto_fetch_on_init_async()
+
+
+# ---- census: every fetch-on-init manager, its limit, and its recorded size class ----
+# size class = the largest table the limit is expected to cover per page; the page size
+# is a knob only, because the loader pages through the whole table.
+CENSUS = {
+    # file (relative to aidream/packages): (page size, table, size class)
+    "matrx-ai/matrx_ai/db/ai_models/_ai_model_manager_orm.py": (300, "ai.model_definition", "hundreds"),
+}
+
+
+def test_census_lists_every_fetch_on_init_manager():
+    root = Path(__file__).resolve().parents[3]  # aidream/packages
+    found = {}
+    for pkg in root.iterdir():
+        for f in pkg.rglob("*.py"):
+            if "/tests/" in str(f) or f.name.startswith("test_") or "/node_modules/" in str(f):
+                continue
+            for m in re.finditer(r"fetch_on_init_limit\s*=\s*([1-9]\d*)", f.read_text(errors="ignore")):
+                found[str(f.relative_to(root))] = int(m.group(1))
+    assert found == {k: v[0] for k, v in CENSUS.items()}, (
+        "A new fetch-on-init manager appeared (or a limit changed): add it to CENSUS with its "
+        "table and size class."
+    )

matrx_orm-3.1.239/tests/session/test_a_grouped_update_writes_only_each_rows_own_fields.py

@@ -0,0 +1,36 @@
+"""A flushed UPDATE group writes, for each row, only the fields that row's op set.
+
+PB-08 defect 58 (conversation 6e3f85a3…, 2026-10-19): two parallel ``agent``
+calls flushed their ``chat.tool_call`` UPDATEs in one group. The flush handed
+``bulk_update`` ONE field list — the union of every op's keys — and a stub
+instance per row whose untouched columns held the Model's Python defaults. Tidewell's
+row, whose op never set ``status``, got ``status='pending'``,
+``output_type='text'``, ``output_chars=0``, ``execution_events=[]`` written over
+its completed values (version 4, same ``updated_at`` as Ares's row), and no
+sweep ever finalized a ``pending`` row.
+"""
+
+from __future__ import annotations
+
+import pytest
+from matrx_orm.session import Session
+
+
+@pytest.mark.asyncio
+async def test_each_row_is_updated_with_only_its_own_fields(FakeMessage) -> None:
+    async with Session() as s:
+        s.defer_update(
+            FakeMessage,
+            "ares",
+            {"status": "completed", "output": "{}", "output_type": "json", "output_chars": 2},
+        )
+        s.defer_update(FakeMessage, "tidewell", {"message_id": "msg-1"})
+
+    written: dict[str, set[str]] = {}
+    for instances, fields in FakeMessage._bulk_update_calls:
+        for inst in instances:
+            written.setdefault(inst.id, set()).update(fields)
+    assert written.get("tidewell") == {"message_id"}, (
+        f"row 'tidewell' was written fields it never set: {sorted(written.get('tidewell', set()))}"
+    )
+    assert written.get("ares") == {"status", "output", "output_type", "output_chars"}

matrx_orm-3.1.239/tests/session/test_pending_insert_honors_operator_filters.py

@@ -0,0 +1,65 @@
+"""A pending INSERT joins a read only when it matches the read's operator filters.
+
+PB-08 defects 56/57 (conversation 6e3f85a3…, 2026-10-19): ``send_message`` to a
+helper that had FINISHED answered ``delivered_to_running_agent``, and the steer
+row stayed ``pending`` forever. The "is it running?" read is
+``user_request.exists(id__in=[<the helper's request>], status__in=['pending',
+'processing'])``. It ran inside the lead's own turn, whose own ``user_request``
+INSERT was still pending in the Session, and the pending-aware merger treated
+every operator-suffixed filter as "might match" — so the lead's own
+unflushed request answered "yes, a run is live" for any conversation at all.
+
+Operators the merger can evaluate exactly (in / nin / not_in / ne / isnull, and
+ordered comparisons on comparable values) are evaluated; only a truly unknown
+operator keeps the "might match" answer.
+"""
+
+from __future__ import annotations
+
+import uuid
+
+from matrx_orm.session.op import make_insert
+from matrx_orm.session.reads import merge_pending_with_db
+
+OWN = str(uuid.uuid4())
+HELPERS = str(uuid.uuid4())
+
+
+def _pending_own_request(model):
+    return [make_insert(model, {"id": OWN, "status": "processing"})]
+
+
+def test_an_in_filter_on_another_id_does_not_match_a_pending_insert(FakeMessage) -> None:
+    result = merge_pending_with_db(
+        [],
+        _pending_own_request(FakeMessage),
+        {"id__in": [HELPERS], "status__in": ["pending", "processing"]},
+    )
+    assert result == [], "a pending row matched an id__in filter that does not name it"
+
+
+def test_an_in_filter_naming_the_pending_row_matches(FakeMessage) -> None:
+    result = merge_pending_with_db(
+        [],
+        _pending_own_request(FakeMessage),
+        {"id__in": [uuid.UUID(OWN)], "status__in": ["pending", "processing"]},
+    )
+    assert [r["id"] for r in result] == [OWN]
+
+
+def test_status_in_excludes_a_pending_row_in_another_status(FakeMessage) -> None:
+    pending = [make_insert(FakeMessage, {"id": OWN, "status": "completed"})]
+    assert merge_pending_with_db([], pending, {"status__in": ["pending", "processing"]}) == []
+
+
+def test_ne_nin_and_isnull_are_evaluated(FakeMessage) -> None:
+    pending = [make_insert(FakeMessage, {"id": OWN, "status": "running", "deleted_at": None})]
+    assert merge_pending_with_db([], pending, {"status__ne": "running"}) == []
+    assert merge_pending_with_db([], pending, {"status__nin": ["running"]}) == []
+    assert merge_pending_with_db([], pending, {"deleted_at__isnull": False}) == []
+    assert len(merge_pending_with_db([], pending, {"deleted_at__isnull": True})) == 1
+
+
+def test_an_unknown_operator_still_might_match(FakeMessage) -> None:
+    pending = [make_insert(FakeMessage, {"id": OWN, "title": "x"})]
+    assert len(merge_pending_with_db([], pending, {"title__json_path": "$.a"})) == 1