matrx-orm 3.1.52__tar.gz → 3.1.53__tar.gz

{matrx_orm-3.1.52 → matrx_orm-3.1.53}/CLAUDE.md

@@ -113,6 +113,16 @@ Tests must run against a Postgres instance the developer provides (or a testcont
 Host generators may expose a focused schema pass by forwarding
 `only_schemas={...}` to `run_schema_generation`; aidream's focused package-model
 command is `uv run python db/generate.py --schema <schema> --models-only`.
+🚨 **A scoped run must NEVER shrink the host models aggregator.** `db/models/__init__.py`
+is the entire `from db.models import X` surface; it is always written from the FULL,
+unfiltered `generate[]` list, never from the `only_schemas` subset — and model names for
+schemas the run did not regenerate are read back from their modules on disk (AST, no
+import) so cross-schema collision de-flattening stays complete. Writing it from a subset
+once truncated it to one schema and broke imports repo-wide, silently at generation time
+(hand-repaired in 79505fee8). `_write_host_models_aggregator` now RAISES
+`AggregatorShrinkError` naming every schema that would be lost, and writes nothing.
+Guard: `tests/schema_builder/test_host_models_aggregator.py`.
+
 `models_only=True` skips unrelated schema/type/helper outputs but still performs
 live introspection and writes the canonical generated Model file. The unfiltered
 command remains the release-wide regeneration path. A package target may declare

{matrx_orm-3.1.52 → matrx_orm-3.1.53}/PKG-INFO

@@ -1,6 +1,6 @@
-Metadata-Version: 2.4
+Metadata-Version: 2.5
 Name: matrx-orm
-Version: 3.1.52
+Version: 3.1.53
 Summary: Async-first PostgreSQL ORM with bidirectional migrations, schema introspection, many-to-many relationships, and built-in state caching
 Project-URL: Homepage, https://github.com/AI-Matrix-Engine/aidream-current
 Project-URL: Repository, https://github.com/AI-Matrix-Engine/aidream-current

{matrx_orm-3.1.52 → matrx_orm-3.1.53}/matrx_orm/schema_builder/runner.py

@@ -8,7 +8,9 @@ Usage:
 
 from __future__ import annotations
 
+import ast
 import os
+import re
 import sys
 from pathlib import Path
 from typing import Any
@@ -408,8 +410,17 @@ def run_schema_generation(
             raise
 
     if not models_only:
+        # ALWAYS pass the FULL, unfiltered generate[] list. A scoped run
+        # (only_schemas={"hindsight"}) regenerates one schema module, but the
+        # aggregator must keep re-exporting EVERY host schema — it is the whole
+        # `from db.models import X` surface. Handing it the filtered list once
+        # truncated db/models/__init__.py to a single schema and broke imports
+        # repo-wide, silently at generation time (repaired by hand in 79505fee8).
         _write_host_models_aggregator(
-            generate_list, python_root, output_config, host_schema_model_names
+            cfg.get("generate", []) or [],
+            python_root,
+            output_config,
+            host_schema_model_names,
         )
 
     # Declarative host→package DB wiring: resolve each incorporated package's
@@ -438,8 +449,73 @@ def run_schema_generation(
     _close_pools()
 
 
+_AGGREGATOR_IMPORT_RE = re.compile(r"^from \.(\w+) import \*", re.MULTILINE)
+
+
+class AggregatorShrinkError(RuntimeError):
+    """The host models aggregator was about to lose schemas or exported names."""
+
+
+def _existing_aggregator_schemas(init_path: str) -> list[str]:
+    """Schemas currently re-exported by db/models/__init__.py (empty if absent)."""
+    try:
+        with open(init_path) as f:
+            return _AGGREGATOR_IMPORT_RE.findall(f.read())
+    except FileNotFoundError:
+        return []
+
+
+_MODEL_BASES = {"Model", "MatrxEntity"}
+
+
+def _module_model_names(models_dir: str, schema: str) -> list[str]:
+    """Read a generated schema module's MODEL class names WITHOUT importing it.
+
+    A scoped generation run only holds fresh names for the schema it regenerated;
+    every other schema's names come from its module on disk, so cross-schema
+    collision detection stays complete instead of scoped-down.
+
+    Mirrors ``Schema.model_export_names`` exactly: real Model/MatrxEntity classes
+    (tables, read-only views, the public ``Users`` stub) — NOT enums, NOT DTOs,
+    NOT managers, and NOT ``_unfetchable`` cross-schema FK stubs (a stub of a
+    sibling schema's table would otherwise fake a collision).
+    """
+    path = os.path.join(models_dir, f"{schema}.py")
+    try:
+        with open(path) as f:
+            tree = ast.parse(f.read(), filename=path)
+    except FileNotFoundError:
+        return []
+    except SyntaxError as e:
+        # Unreadable module ⇒ its names go undetected ⇒ a colliding name would
+        # silently stay in the flat namespace and bind whichever schema imported
+        # last. Never let that pass quietly.
+        vcprint(
+            f"[MATRX ORM] Could NOT parse {path} ({e}) — its model names are invisible "
+            "to cross-schema collision detection. Fix or regenerate that module.",
+            color="red",
+        )
+        return []
+    names: list[str] = []
+    for node in tree.body:
+        if not isinstance(node, ast.ClassDef):
+            continue
+        if not any(isinstance(b, ast.Name) and b.id in _MODEL_BASES for b in node.bases):
+            continue
+        is_stub = any(
+            isinstance(stmt, ast.Assign)
+            and any(isinstance(t, ast.Name) and t.id == "_unfetchable" for t in stmt.targets)
+            and isinstance(stmt.value, ast.Constant)
+            and stmt.value.value is True
+            for stmt in node.body
+        )
+        if not is_stub:
+            names.append(node.name)
+    return names
+
+
 def _write_host_models_aggregator(
-    generate_list: list[dict[str, Any]],
+    all_generate_entries: list[dict[str, Any]],
     python_root: str,
     output_config: OutputConfig,
     host_schema_model_names: dict[str, list[str]] | None = None,
@@ -464,6 +540,13 @@ def _write_host_models_aggregator(
     single-file layout, so it is skipped here. ``public`` is imported first so
     its canonical models win over any cross-schema FK stub registered with
     skip_existing in a sibling schema.
+
+    ``all_generate_entries`` MUST be the full, unfiltered ``generate[]`` list —
+    NEVER the subset selected by ``only_schemas``. This file is the entire
+    ``from db.models import X`` surface; writing it from a scoped run's subset
+    truncates it to that one schema and breaks imports across the whole host
+    repo, with no error at generation time. The shrink guard below makes that
+    outcome impossible rather than merely discouraged.
     """
     if not (output_config.python and output_config.save_direct):
         return
@@ -471,7 +554,7 @@ def _write_host_models_aggregator(
     host_schema_model_names = host_schema_model_names or {}
 
     schemas: list[str] = []
-    for entry in generate_list:
+    for entry in all_generate_entries:
         output_block = entry.get("output") or {}
         if output_block.get("python_root"):
             continue  # package tree — not part of the host db/models package
@@ -485,13 +568,52 @@ def _write_host_models_aggregator(
     # public first (canonical models register before sibling stubs)
     schemas.sort(key=lambda s: (s != "public", s))
 
-    # Detect model class names that exist in >1 host schema.
+    models_dir = os.path.join(python_root, "db", "models")
+    os.makedirs(models_dir, exist_ok=True)
+    init_path = os.path.join(models_dir, "__init__.py")
+
+    # Detect model class names that exist in >1 host schema. Names for schemas
+    # this run did NOT regenerate (a scoped `--schema X` pass) are read from
+    # their modules on disk, so collision detection is never scoped-down either.
     name_to_schemas: dict[str, set[str]] = {}
     for schema_name in schemas:
-        for model_name in host_schema_model_names.get(schema_name, []):
+        model_names = host_schema_model_names.get(schema_name) or _module_model_names(
+            models_dir, schema_name
+        )
+        for model_name in model_names:
             name_to_schemas.setdefault(model_name, set()).add(schema_name)
     colliding = sorted(n for n, owners in name_to_schemas.items() if len(owners) > 1)
 
+    # -------------------------------------------------------------------------
+    # SHRINK GUARD — silent shrinkage of this file is repo-wide breakage.
+    # `from db.models import X` resolves through here; a lost schema import
+    # explodes far from generation time, at some unrelated module's import.
+    # -------------------------------------------------------------------------
+    previous_schemas = _existing_aggregator_schemas(init_path)
+    lost_schemas = [s for s in previous_schemas if s not in set(schemas)]
+    if lost_schemas:
+        vcprint(
+            "\n" + "=" * 80 + "\n"
+            "[MATRX ORM] REFUSING to shrink the host models aggregator.\n"
+            f"  File: {init_path}\n"
+            f"  Currently re-exports {len(previous_schemas)} schema(s); this run would "
+            f"write {len(schemas)}.\n"
+            f"  Would LOSE: {', '.join(sorted(lost_schemas))}\n"
+            "  Every `from db.models import X` for those schemas would break at import\n"
+            "  time, far from here. The aggregator must always be written from the FULL\n"
+            "  generate[] list in db/matrx_orm.yaml — never a scoped subset.\n"
+            "  If a schema was intentionally removed from matrx_orm.yaml, delete its\n"
+            "  db/models/<schema>.py and the stale import line by hand, then rerun.\n"
+            + "=" * 80
+            + "\n",
+            color="red",
+        )
+        raise AggregatorShrinkError(
+            f"host models aggregator would lose {len(lost_schemas)} schema(s): "
+            f"{', '.join(sorted(lost_schemas))} (had {len(previous_schemas)}, "
+            f"writing {len(schemas)})"
+        )
+
     lines = [
         "# File: db/models/__init__.py",
         "# AUTO-GENERATED by matrx-orm schema generation — do not edit by hand.",
@@ -522,9 +644,6 @@ def _write_host_models_aggregator(
             ]
         )
 
-    models_dir = os.path.join(python_root, "db", "models")
-    os.makedirs(models_dir, exist_ok=True)
-    init_path = os.path.join(models_dir, "__init__.py")
     with open(init_path, "w") as f:
         f.write("\n".join(lines))
 

{matrx_orm-3.1.52 → matrx_orm-3.1.53}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "matrx-orm"
-version = "3.1.52"
+version = "3.1.53"
 description = "Async-first PostgreSQL ORM with bidirectional migrations, schema introspection, many-to-many relationships, and built-in state caching"
 readme = "README.md"
 license = { text = "MIT" }

matrx_orm-3.1.53/tests/schema_builder/test_host_models_aggregator.py

@@ -0,0 +1,171 @@
+"""The host db/models/__init__.py aggregator must never shrink.
+
+A scoped generation run (`--schema hindsight`) once rewrote the aggregator down
+to that one schema, breaking `from db.models import X` across the entire host
+repo — silently, with the explosion arriving later at some unrelated import.
+These tests exercise the real writer, not a stand-in.
+"""
+
+from __future__ import annotations
+
+import pytest
+from matrx_orm.schema_builder.common import OutputConfig
+from matrx_orm.schema_builder.runner import (
+    AggregatorShrinkError,
+    _select_generation_targets,
+    _write_host_models_aggregator,
+)
+
+FULL_GENERATE = [
+    {"database": "main", "schema": "public"},
+    {"database": "main", "schema": "agent"},
+    {"database": "main", "schema": "chat"},
+    {"database": "main", "schema": "hindsight"},
+    {"database": "main", "schema": "workflow"},
+    # A package-tree target: keeps its own layout, never joins the aggregator.
+    {"database": "main", "schema": "seo", "output": {"python_root": "packages/matrx-seo"}},
+]
+
+HOST_SCHEMAS = {"public", "agent", "chat", "hindsight", "workflow"}
+
+
+def _output_config() -> OutputConfig:
+    return OutputConfig(python=True, save_direct=True, typescript=False)
+
+
+def _seed_models_tree(python_root, schema_models: dict[str, list[str]]) -> None:
+    """Write generated-shaped schema modules: real model classes, an enum, a DTO,
+    a manager, and an _unfetchable cross-schema FK stub — only the model classes
+    may ever count toward collision detection."""
+    models_dir = python_root / "db" / "models"
+    models_dir.mkdir(parents=True)
+    for schema, names in schema_models.items():
+        parts = [f'class {n}(Model):\n    _table_name = "{n.lower()}"\n' for n in names]
+        parts.append('class Visibility(str, Enum):\n    PUBLIC = "public"\n')
+        parts.append("class SomethingDTO(BaseDTO):\n    id: str\n")
+        parts.append("class SomethingManager(BaseManager):\n    pass\n")
+        parts.append(
+            'class ForeignStub(Model):\n    _table_name = "foreign"\n    _unfetchable = True\n'
+        )
+        exports = "\n".join(f'    "{n}",' for n in [*names, "Visibility"])
+        (models_dir / f"{schema}.py").write_text(
+            "\n".join(parts) + f"\n__all__ = [\n{exports}\n]\n"
+        )
+
+
+def _write_aggregator(python_root, host_schema_model_names) -> str:
+    _write_host_models_aggregator(
+        FULL_GENERATE,
+        str(python_root),
+        _output_config(),
+        host_schema_model_names,
+    )
+    return (python_root / "db" / "models" / "__init__.py").read_text()
+
+
+def test_scoped_generation_keeps_every_schema(tmp_path) -> None:
+    """The failure that actually happened: `--schema hindsight` must not truncate."""
+    _seed_models_tree(
+        tmp_path,
+        {
+            "public": ["Conversation", "AgentRun"],
+            "agent": ["Definition"],
+            "chat": ["AgentRun"],
+            "hindsight": ["Enrollment", "Review"],
+            "workflow": ["Definition"],
+        },
+    )
+
+    # Full run first — establishes the complete aggregator.
+    full_text = _write_aggregator(
+        tmp_path,
+        {
+            "public": ["Conversation", "AgentRun"],
+            "agent": ["Definition"],
+            "chat": ["AgentRun"],
+            "hindsight": ["Enrollment", "Review"],
+            "workflow": ["Definition"],
+        },
+    )
+    for schema in HOST_SCHEMAS:
+        assert f"from .{schema} import *" in full_text
+
+    # Now the scoped pass, exactly as db/generate.py --schema hindsight runs it:
+    # only the hindsight entry is selected, and only its fresh model names exist.
+    selected, _ = _select_generation_targets(FULL_GENERATE, {"hindsight"})
+    assert [e["schema"] for e in selected] == ["hindsight"]
+
+    scoped_text = _write_aggregator(tmp_path, {"hindsight": ["Enrollment", "Review", "Replay"]})
+
+    for schema in HOST_SCHEMAS:
+        assert f"from .{schema} import *" in scoped_text, (
+            f"scoped run dropped schema {schema!r} from the aggregator"
+        )
+    assert scoped_text.count("import *") == len(HOST_SCHEMAS)
+    # A package-tree entry never joins the host aggregator.
+    assert "from .seo import *" not in scoped_text
+
+
+def test_scoped_generation_preserves_cross_schema_collisions(tmp_path) -> None:
+    """Names for un-regenerated schemas come from disk, so de-flattening survives."""
+    _seed_models_tree(
+        tmp_path,
+        {
+            "public": ["Conversation", "AgentRun"],
+            "agent": ["Definition"],
+            "chat": ["AgentRun"],
+            "hindsight": ["Enrollment"],
+            "workflow": ["Definition"],
+        },
+    )
+    text = _write_aggregator(tmp_path, {"hindsight": ["Enrollment"]})
+
+    assert '"AgentRun",' in text  # public/chat
+    assert '"Definition",' in text  # agent/workflow
+    assert '"Enrollment",' not in text  # unique — stays in the flat namespace
+    # Enums, DTOs, managers and _unfetchable FK stubs exist in EVERY seeded
+    # module; counting them would fabricate collisions across all 5 schemas.
+    for non_model in ('"Visibility",', '"SomethingDTO",', '"SomethingManager",', '"ForeignStub",'):
+        assert non_model not in text
+
+
+def test_shrinking_the_aggregator_raises_and_names_the_losses(tmp_path) -> None:
+    """A subset generate[] list is refused outright, not written."""
+    _seed_models_tree(tmp_path, {s: [f"Model{s.title()}"] for s in HOST_SCHEMAS})
+    _write_aggregator(tmp_path, {s: [f"Model{s.title()}"] for s in HOST_SCHEMAS})
+
+    init_path = tmp_path / "db" / "models" / "__init__.py"
+    before = init_path.read_text()
+
+    subset = [{"database": "main", "schema": "hindsight"}]
+    with pytest.raises(AggregatorShrinkError) as excinfo:
+        _write_host_models_aggregator(
+            subset,
+            str(tmp_path),
+            _output_config(),
+            {"hindsight": ["Enrollment"]},
+        )
+
+    message = str(excinfo.value)
+    for lost in HOST_SCHEMAS - {"hindsight"}:
+        assert lost in message
+    # Refused means REFUSED — the file on disk is untouched.
+    assert init_path.read_text() == before
+
+
+def test_adding_a_new_schema_still_registers(tmp_path) -> None:
+    """The guard blocks loss, never growth."""
+    _seed_models_tree(tmp_path, {s: [f"Model{s.title()}"] for s in HOST_SCHEMAS})
+    _write_aggregator(tmp_path, {s: [f"Model{s.title()}"] for s in HOST_SCHEMAS})
+
+    grown = FULL_GENERATE + [{"database": "main", "schema": "brandnew"}]
+    _write_host_models_aggregator(
+        grown,
+        str(tmp_path),
+        _output_config(),
+        {"brandnew": ["Shiny"]},
+    )
+    text = (tmp_path / "db" / "models" / "__init__.py").read_text()
+    assert "from .brandnew import *" in text
+    for schema in HOST_SCHEMAS:
+        assert f"from .{schema} import *" in text