matrx-orm 3.1.185__tar.gz → 3.1.187__tar.gz

{matrx_orm-3.1.185 → matrx_orm-3.1.187}/BASE_CLASS_METHODS.md

@@ -60,6 +60,7 @@ The following methods are available directly from all manager classes:
 - `upsert_items(items_data, conflict_fields, update_fields=None)`
 - `count(**filters)`
 - `update_where(filters, *, expected_version=None, lock_rows_in_pk_order=False, **updates)`
+- `insert_returning(data, *, returning)` — INSERT only the named columns (the rest take DB defaults), RETURNING only `returning`; the acting-as-user create on a table whose system-only columns the person is not granted. Update twin: `Model.filter(...).returning(*cols).update(...)`.
 - `delete_where(*, expected_version=None, **filters)`
 
 ### A multi-row UPDATE that racing callers share — `lock_rows_in_pk_order=True`

{matrx_orm-3.1.185 → matrx_orm-3.1.187}/CHANGELOG.md

@@ -2,6 +2,11 @@
 
 ## Unreleased
 
+- `Model.insert_returning` and `QueryBuilder.returning(*fields)` — a create/update
+  write can now ask Postgres to hand back only the columns the caller is
+  actually granted, instead of the row the ORM already had in hand (which may
+  carry columns the writer's RLS/column-scope never authorised it to read
+  back). Guard: `packages/matrx-orm/tests/level1/test_column_scoped_writes.py`.
 - Pool creation always ends. A pool `init` failure now aborts the connection
   (asyncpg's own failure path awaited an untimed graceful close), and
   `asyncpg.create_pool` runs under one deadline (`pool_create_timeout`, default

{matrx_orm-3.1.185 → matrx_orm-3.1.187}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: matrx-orm
-Version: 3.1.185
+Version: 3.1.187
 Summary: Async-first PostgreSQL ORM with bidirectional migrations, schema introspection, many-to-many relationships, and built-in state caching
 Project-URL: Homepage, https://github.com/AI-Matrix-Engine/aidream-current
 Project-URL: Repository, https://github.com/AI-Matrix-Engine/aidream-current

{matrx_orm-3.1.185 → matrx_orm-3.1.187}/matrx_orm/core/base.py

@@ -673,6 +673,24 @@ class Model(RuntimeMixin, metaclass=ModelMeta):
         await StateManager.cache(cls, instance)
         return cast(Self, instance)
 
+    @classmethod
+    async def insert_returning(
+        cls,
+        data: dict[str, Any],
+        *,
+        returning: Sequence[str],
+        statement_timeout: float | None = None,
+    ) -> dict[str, Any]:
+        """INSERT only the columns in ``data`` (the rest take their DB defaults)
+        and return only ``returning`` — the write door for an acting-as-user
+        insert into a table whose system-only columns the person is not granted.
+        See ``matrx_orm.operations.create.insert_returning``."""
+        from matrx_orm.operations.create import insert_returning as _insert_returning
+
+        return await _insert_returning(
+            cls, data, returning=returning, statement_timeout=statement_timeout
+        )
+
     @classmethod
     async def bulk_create(cls, objects_data: list[dict[str, Any]]) -> list[Self]:
         """

{matrx_orm-3.1.185 → matrx_orm-3.1.187}/matrx_orm/core/types.py

@@ -228,6 +228,15 @@ class UpdateResult:
     rows_affected: int
     updated_rows: list[dict[str, Any]]
 
+    def __bool__(self) -> bool:
+        """True only when the UPDATE changed a row.
+
+        A dataclass without this is ALWAYS truthy, so ``if await q.update(...):``
+        read as "it changed something" even at zero rows — the runtime reaper
+        soft-deleted a stream's frames on that false yes (found 2026-09-26).
+        """
+        return self.rows_affected > 0
+
     def to_dict(self) -> dict[str, Any]:
         """Serialise to the legacy dict shape for JSON / API boundaries."""
         return {

{matrx_orm-3.1.185 → matrx_orm-3.1.187}/matrx_orm/operations/create.py

@@ -1,6 +1,7 @@
 from __future__ import annotations
 
 import asyncio
+from collections.abc import Sequence
 from typing import Any, TYPE_CHECKING, Union
 
 from matrx_orm.state import StateManager
@@ -122,6 +123,51 @@ async def save(instance: Model, *, statement_timeout: float | None = None) -> Mo
     return instance
 
 
+async def insert_returning(
+    model_cls: type[Model],
+    data: dict[str, Any],
+    *,
+    returning: Sequence[str],
+    statement_timeout: float | None = None,
+) -> dict[str, Any]:
+    """INSERT exactly the columns in ``data`` and hand back ``returning`` only.
+
+    ``create()`` materializes every model default and asks for ``RETURNING *``,
+    so it needs INSERT and SELECT on every column. A governed table may withhold
+    a system-only column from the acting role (``agent.term_list.metadata`` is
+    neither insertable nor readable by ``authenticated``), and then an
+    acting-as-user create fails with 42501 though the caller never set that
+    column. Here the columns the caller did not name take their DATABASE
+    defaults, and only the named, granted columns come back. The result is a
+    plain dict (python-coerced); a partial row is never cached as a model.
+    """
+    if not returning:
+        raise ValueError("insert_returning needs at least one returning column")
+    fields = model_cls._fields
+    unknown = [
+        k
+        for k in [*data, *returning]
+        if not isinstance(fields.get(k), Field) or not getattr(fields[k], "is_native", True)
+    ]
+    if unknown:
+        raise ValueError(f"insert_returning: {unknown} are not columns of {model_cls.__name__}")
+    prepared = {
+        getattr(fields[k], "field_name", k): fields[k].get_db_prep_value(v)
+        for k, v in data.items()
+        if v is not None
+    }
+    query = QueryBuilder(model_cls)._build_query()
+    query["data"] = prepared
+    query["returning"] = [getattr(fields[k], "field_name", k) for k in returning]
+    if statement_timeout is not None:
+        query["statement_timeout"] = statement_timeout
+    row = await QueryExecutor(query).insert(query)
+    return {
+        key: (fields[key].to_python(value) if isinstance(fields.get(key), Field) else value)
+        for key, value in dict(row).items()
+    }
+
+
 async def bulk_create(model_cls: type[Model], objects_data: list[dict[str, Any]]) -> list[Model]:
     """Bulk insert with field coercion. Returns hydrated model instances."""
     if not objects_data:

{matrx_orm-3.1.185 → matrx_orm-3.1.187}/matrx_orm/query/builder.py

@@ -35,6 +35,19 @@ _SQL_IDENTIFIER_RE = re.compile(r"^[A-Za-z_][A-Za-z0-9_]*$")
 _JOIN_TYPES = {"INNER", "LEFT", "RIGHT", "FULL"}
 
 
+async def _evict_updated_rows(model_cls: type, rows: list[Any]) -> None:
+    """Evict the cache entries for rows an UPDATE ... RETURNING <named columns>
+    touched. Keyed by pk when the caller returned it; otherwise the model's
+    cache is flushed (losing warmth is free, serving a stale row is not)."""
+    if not rows or not StateManager.is_registered(model_cls):
+        return
+    pks = model_cls._meta.primary_keys or ["id"]
+    if all(isinstance(r, dict) and all(pk in r for pk in pks) for r in rows):
+        await StateManager.refresh_from_rows(model_cls, rows, max_rows=-1)  # evict-only
+    else:
+        await StateManager.clear_cache(model_cls)
+
+
 def _merge_pending_into_models(model_cls: type, db_models: list, filters: dict[str, Any]) -> list:
     """Apply pending Session state to a list of Model instances from DB.
 
@@ -216,6 +229,8 @@ class QueryBuilder(Generic[ModelT]):
     verify_empty_val: bool
     #: lock_rows_in_pk_order() — take an UPDATE's row locks ascending by pk.
     lock_rows_in_pk_order_val: bool
+    #: returning() — the columns an UPDATE hands back (None = every column).
+    returning_fields: tuple[str, ...] | None
 
     def __init__(self, model_cls: type[ModelT], database: str | None = None) -> None:
         self.model = model_cls
@@ -244,6 +259,7 @@ class QueryBuilder(Generic[ModelT]):
         self.timeout_val: float | None = None
         self.verify_empty_val = True
         self.lock_rows_in_pk_order_val = False
+        self.returning_fields = None
 
     def _set_database(self, model_cls: type[ModelT]) -> str:
         if hasattr(model_cls, "_database") and model_cls._database:
@@ -566,6 +582,37 @@ class QueryBuilder(Generic[ModelT]):
         new.lock_rows_in_pk_order_val = True
         return new
 
+    def returning(self, *fields: str) -> QueryBuilder[ModelT]:
+        """Name the columns an ``update()`` hands back (``RETURNING a, b``).
+
+        The default is ``RETURNING *``, which needs SELECT on EVERY column. A
+        governed table can withhold a column from the acting role (a system-only
+        ``metadata`` the person may neither read nor write), and then every
+        acting-as-user update fails with 42501 even though it touched only
+        granted columns. Name the granted columns instead::
+
+            await TermList.filter(id=x).returning("id", "name", "version").update(name=n)
+
+        Rows come back as dicts carrying just these columns; the identity cache
+        entries for them are EVICTED (a partial row is never cached as a model).
+        """
+        if not fields:
+            raise ValueError("returning() needs at least one column")
+        unknown = [
+            f
+            for f in fields
+            if f not in self.model._fields or not getattr(self.model._fields[f], "is_native", True)
+        ]
+        if unknown:
+            raise ValueError(
+                f"returning(): {unknown} are not columns of {self.model.__name__}"
+            )
+        new = self._clone()
+        new.returning_fields = tuple(
+            getattr(self.model._fields[f], "field_name", f) for f in fields
+        )
+        return new
+
     def using(self, database: str) -> QueryBuilder[ModelT]:
         """Override the database for this query.
 
@@ -782,6 +829,7 @@ class QueryBuilder(Generic[ModelT]):
             "timeout": self.timeout_val,
             "verify_empty": self.verify_empty_val,
             "lock_rows_in_pk_order": self.lock_rows_in_pk_order_val,
+            "returning": self.returning_fields,
         }
         return query
 
@@ -980,7 +1028,12 @@ class QueryBuilder(Generic[ModelT]):
                 raise ValidationError(model=self.model, reason="No update data provided")
             executor = await self._get_executor_async()
             result = await executor.update(**kwargs)
-            await StateManager.refresh_from_rows(self.model, result.updated_rows)
+            if self.returning_fields is None:
+                await StateManager.refresh_from_rows(self.model, result.updated_rows)
+            else:
+                # Partial rows (named columns only) must never be cached as a
+                # model — drop whatever copy the cache held instead.
+                await _evict_updated_rows(self.model, result.updated_rows)
             return result
         except ValidationError:
             raise

{matrx_orm-3.1.185 → matrx_orm-3.1.187}/matrx_orm/query/executor.py

@@ -1216,10 +1216,14 @@ class QueryExecutor:
         columns = list(data.keys())
         values = list(data.values())
         placeholders = [f"${i + 1}" for i in range(len(values))]
+        # ``returning`` (Model.insert_returning) names the columns handed back;
+        # the default ``*`` needs SELECT on every column (see update()).
+        named = query.get("returning")
+        returning_clause = ", ".join(named) if named else "*"
         sql = (
             f"INSERT INTO {table} ({', '.join(columns)}) "
             f"VALUES ({', '.join(placeholders)}) "
-            f"RETURNING *"
+            f"RETURNING {returning_clause}"
         )
 
         try:
@@ -1650,7 +1654,12 @@ class QueryExecutor:
                 rows = dict_result  # type: ignore[assignment]
                 return UpdateResult(rows_affected=len(rows or []), updated_rows=rows or [])
 
-            result = await self.db.execute_write(sql + " RETURNING *", *params)
+            # returning() on the builder names the columns handed back; the
+            # default ``*`` needs SELECT on every column, which an acting-as-user
+            # write on a column-governed table does not hold (42501).
+            named = self._full_query_dict.get("returning")
+            returning_clause = ", ".join(named) if named else "*"
+            result = await self.db.execute_write(sql + f" RETURNING {returning_clause}", *params)
             return UpdateResult(rows_affected=len(result), updated_rows=result)
 
         except (ValidationError, IntegrityError, DatabaseError, ParameterError):

{matrx_orm-3.1.185 → matrx_orm-3.1.187}/matrx_orm/secrets_battery/FEATURE.md

@@ -33,22 +33,20 @@ charset-validated here — `provider_key`, `display_name`, `tags`, `status`,
 alias (nullable). A DB trigger (`trg_user_secrets_owner_match`) rejects any field
 row whose owner columns don't match its item's owner.
 
-**History foundation (capture disabled).** Migration 1034 adds a nonnegative,
-server-managed `history_revision` item counter, but the ORM does not map it until
-the history migrations are installed on every target: mapping it early makes
-ordinary item creation fail with a missing-column error. No battery mutation
-currently writes history. Migrations 1035 and 1036 keep the private
-`vault_history_writer` personal-only and enforce the closed v1 envelope: exact
-keys, typed identifiers and timestamp, locked original ownership, and empty
-component arrays. Organization-owned history refuses until the host supplies the
-grant/offboarding parent-lock contract. Future aggregate capture may use the
-revision-1/revision-2 bootstrap only after its producer, authority, retention,
-rotation, and reader gates are independently accepted. Gate B's source-only
-1047 validator freezes one personal, active ordinary password field shape: its
-first recorded update is an A=A bootstrap followed by A→B, and later records
-are one prior-after-matched delta each. Application capture is explicitly off
-until every producer and reader gate is accepted; 1047 is never applied outside
-the named runner's reviewed rehearsal.
+**Personal-password history.** The installed history counter is mapped and the
+ordinary personal-password capture gate is on. Battery mutations record the
+closed v1 chain under the private writer: revision 1 is an A=A bootstrap and
+later revisions are prior-after-matched deltas. Migrations 1035 and 1036 keep the private
+`vault_history_writer` personal-only and establish the closed v1 envelope.
+Installed migration 1047 requires exact keys, typed identifiers and timestamp,
+locked original ownership, and exactly one manifest and one changed component
+bound to the locked active personal password field. Organization-owned history refuses until the host supplies the
+grant/offboarding parent-lock contract. Capture, list/reveal, and restore are
+backend capabilities for this bounded personal-password chain. Migration 1166
+is reviewed and enabled in source; installed behavior requires the migration
+ledger and exact live validator bodies. The physical history timestamp remains
+the destination audit time, while authenticated backup source dates are retained
+separately when the installed contract supports them.
 
 Items may also own N encrypted **file attachments**
 (`users.credential_attachments`). An attachment has its own stable id,
@@ -391,9 +389,11 @@ delete).
   replacement, matching the host bulk reader's revocation order. A focused
   ordering test fails if deletion moves ahead of the parent lock.
 
-- **2026-09-24** — Migration 1036 closes the personal-only Vault revision envelope and refuses organization history pending the canonical grant/offboarding parent lock; capture remains disabled.
-
-- **2026-09-24** — Kept the history counter out of the ORM until migrations are installed; the branch database lacks the column and a premature model mapping broke ordinary Vault creation. Capture remains disabled.
+- **2026-09-26** — The personal-password capture gate is on. The battery maps
+  the installed history counter and maintains the bounded A=A then A→B chain;
+  organization history remains refused pending the canonical grant/offboarding
+  parent lock. Guarded migration 1166 does not yet provide backup source-date
+  fidelity.
 
 - **2026-09-19** — Restored native lifecycle metadata with the parsed, timezone-aware manifest instant at the typed mutation boundary, preserving the manifest string for aggregate integrity checks.
 

{matrx_orm-3.1.185 → matrx_orm-3.1.187}/matrx_orm/secrets_battery/items.py

@@ -90,9 +90,9 @@ ITEM_SOURCES: tuple[str, ...] = (
     "system_import",
 )
 
-# Gate B is source-complete only.  Turning this on would make history appear
-# complete while sibling producers and readers still omit revisions.
-_PASSWORD_HISTORY_CAPTURE_ENABLED = False
+# Eligible personal password writes capture their prior value and the new
+# value atomically. Organization history remains outside this boundary.
+_PASSWORD_HISTORY_CAPTURE_ENABLED = True
 
 
 class CredentialItemNotFoundError(LookupError):
@@ -1296,9 +1296,8 @@ def _captures_personal_password_history(
 ) -> bool:
     """Return whether this locked row belongs in the private password history.
 
-    The module-local gate remains deliberately false in production. Keeping
-    eligibility here gives every existing-field ciphertext writer the same
-    future activation boundary without adding a caller-controlled switch.
+    The module-local gate keeps every existing-field ciphertext writer behind
+    the same eligibility boundary without adding a caller-controlled switch.
     """
     return (
         _PASSWORD_HISTORY_CAPTURE_ENABLED

{matrx_orm-3.1.185 → matrx_orm-3.1.187}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "matrx-orm"
-version = "3.1.185"
+version = "3.1.187"
 description = "Async-first PostgreSQL ORM with bidirectional migrations, schema introspection, many-to-many relationships, and built-in state caching"
 readme = "README.md"
 license = { text = "MIT" }

matrx_orm-3.1.187/tests/level1/test_column_scoped_writes.py

@@ -0,0 +1,126 @@
+"""Level 1: a write on a column-governed table names only the columns it may touch.
+
+REGRESSION — 2026-09-26, aidream term lists. ``agent.term_list.metadata`` (vendor
+caches) is withheld from ``authenticated``: no INSERT, no SELECT. ``create()``
+materializes every model default (so it INSERTs ``metadata``) and asks for
+``RETURNING *`` (so it SELECTs ``metadata``); ``update()`` also asked for
+``RETURNING *``. Every acting-as-user create and edit failed with 42501 though the
+person never set that column. ``Model.insert_returning`` and
+``QueryBuilder.returning()`` are the doors that name the columns instead.
+"""
+
+from __future__ import annotations
+
+from typing import Any
+
+import pytest
+from matrx_orm.core.fields import JSONBField, TextField
+
+
+
+class _Meta:
+    def __init__(self, qualified: str) -> None:
+        self.qualified_table_name = qualified
+        self.primary_keys = ["id"]
+
+
+@pytest.fixture
+def captured_sql(monkeypatch: pytest.MonkeyPatch) -> list[tuple[str, tuple[Any, ...]]]:
+    """Intercept at the executor's DB boundary so the real insert path runs."""
+    calls: list[tuple[str, tuple[Any, ...]]] = []
+
+    from matrx_orm.query import builder as builder_mod
+    from matrx_orm.query import executor as executor_mod
+
+    class _DBStub:
+        async def execute_write(self, sql: str, *params: Any, **_kw: Any) -> list[dict[str, Any]]:
+            calls.append((sql, params))
+            return [{"id": "row-1"}]
+
+    def _fake_init(self: Any, query: dict[str, Any]) -> None:
+        self.query = query
+        self.model = query.get("model")
+        self.db = _DBStub()
+
+    async def _no_dict(self: Any, _query: dict[str, Any]) -> Any:
+        return NotImplemented
+
+    monkeypatch.setattr(executor_mod.QueryExecutor, "__init__", _fake_init)
+    monkeypatch.setattr(executor_mod.QueryExecutor, "_try_write_dict", _no_dict)
+    monkeypatch.setattr(executor_mod.QueryExecutor, "_guard_writable", lambda self, _op: None)
+    monkeypatch.setattr(
+        builder_mod.QueryBuilder,
+        "_build_query",
+        lambda self: {"table": self.model._meta.qualified_table_name, "model": self.model},
+    )
+    return calls
+
+
+def _term_list_model() -> type:
+    fields: dict[str, Any] = {
+        "id": TextField(),
+        "organization_id": TextField(),
+        "name": TextField(),
+        "entries": JSONBField(default=[]),
+        # The system-only column: a Python default the person cannot write.
+        "metadata": JSONBField(default={}),
+    }
+
+    class _TermList:
+        _meta = _Meta("agent.term_list")
+        _fields = fields
+        _read_only = False
+        _database = "test_db"
+
+        def __init__(self, **kw: Any) -> None:
+            for name in fields:
+                setattr(self, name, None)
+            for k, v in kw.items():
+                setattr(self, k, v)
+
+        @classmethod
+        def get_database_name(cls) -> str:
+            return "test_db"
+
+    return _TermList
+
+
+@pytest.mark.asyncio
+async def test_insert_returning_writes_only_named_columns_and_returns_only_granted_ones(
+    captured_sql,
+) -> None:
+    from matrx_orm.operations.create import insert_returning
+
+    model = _term_list_model()
+    row = await insert_returning(
+        model,
+        {"organization_id": "org-1", "name": "All Green brand terms", "entries": []},
+        returning=["id", "name"],
+    )
+    sql, params = captured_sql[0]
+    assert "metadata" not in sql, sql  # the DB default applies; the person never names it
+    assert sql.endswith("RETURNING id, name"), sql
+    assert "RETURNING *" not in sql
+    assert params[1] == "All Green brand terms"
+    assert row == {"id": "row-1"}
+
+
+@pytest.mark.asyncio
+async def test_insert_returning_refuses_unknown_columns(captured_sql) -> None:
+    from matrx_orm.operations.create import insert_returning
+
+    with pytest.raises(ValueError, match="not columns"):
+        await insert_returning(_term_list_model(), {"name": "x"}, returning=["id", "nope"])
+    assert captured_sql == []
+
+
+def test_builder_returning_names_columns_for_update() -> None:
+    from matrx_orm.query.builder import QueryBuilder
+
+    model = _term_list_model()
+    qb = QueryBuilder(model)
+    scoped = qb.returning("id", "name")
+    assert scoped.returning_fields == ("id", "name")
+    assert qb.returning_fields is None  # chainable clone, never mutates the source
+    with pytest.raises(ValueError, match="not columns"):
+        qb.returning("metadatax")

{matrx_orm-3.1.185 → matrx_orm-3.1.187}/tests/test_secrets_battery.py

@@ -1324,17 +1324,17 @@ async def test_second_history_write_failure_propagates_from_atomic_transaction(m
     assert updates  # The real transaction owns this statement and rolls it back on the propagated failure.
 
 
-def test_password_history_capture_is_off_and_not_a_public_writer_option():
+def test_password_history_capture_is_on_and_not_a_public_writer_option():
     from matrx_orm.secrets_battery import items
 
-    assert items._PASSWORD_HISTORY_CAPTURE_ENABLED is False
+    assert items._PASSWORD_HISTORY_CAPTURE_ENABLED is True
     for writer in (items.update_credential_field_value, items.rotate_credential_fields):
         assert "capture" not in inspect.signature(writer).parameters
 
 
 @pytest.mark.asyncio
 async def test_enabled_history_capture_chains_two_writers_and_rolls_back(monkeypatch):
-    """The private switch is test-only: a failed append restores field and chain."""
+    """An active capture writer propagates append failure so the transaction rolls back."""
     from matrx_orm.secrets_battery import items
 
     monkeypatch.setenv("CREDENTIALS_ENCRYPTION_KEY", _fernet_key())

matrx_orm-3.1.187/tests/test_update_result_truthiness.py

@@ -0,0 +1,6 @@
+from matrx_orm.core.types import UpdateResult
+
+
+def test_an_update_that_changed_nothing_is_falsy() -> None:
+    assert not UpdateResult(rows_affected=0, updated_rows=[])
+    assert UpdateResult(rows_affected=2, updated_rows=[{}, {}])