etlantic-sqlmodel 0.40.0__tar.gz → 0.41.0__tar.gz

{etlantic_sqlmodel-0.40.0 → etlantic_sqlmodel-0.41.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: etlantic-sqlmodel
-Version: 0.40.0
+Version: 0.41.0
 Summary: ContractModel ↔ SQLModel bridge for ETLantic.
 Project-URL: Homepage, https://github.com/eddiethedean/etlantic
 Project-URL: Documentation, https://github.com/eddiethedean/etlantic/tree/main/docs
@@ -18,7 +18,7 @@ Classifier: Programming Language :: Python :: 3.12
 Classifier: Programming Language :: Python :: 3.13
 Classifier: Typing :: Typed
 Requires-Python: >=3.11
-Requires-Dist: etlantic<0.41,>=0.40.0
+Requires-Dist: etlantic<0.42,>=0.41.0
 Requires-Dist: sqlmodel<1,>=0.0.22
 Description-Content-Type: text/markdown
 
@@ -28,13 +28,13 @@ Optional bridge between ETLantic `Data` contracts and
 [SQLModel](https://sqlmodel.tiangolo.com/) table models, plus optional CP1
 control-plane reference stores. Install when you need `contract_to_sqlmodel`
 helpers or SQLModel-backed definition/submission stores for local CP1 demos.
-Package version is **0.39.0** — pin with core.
+Package version is **0.41.0** — pin with core.
 
 ## Install
 
 ```bash
-pip install 'etlantic-sqlmodel==0.40.0'
-# pip install 'etlantic==0.40.0'
+pip install 'etlantic-sqlmodel==0.41.0'
+# pip install 'etlantic==0.41.0'
 ```
 
 ## Schema bridge
@@ -93,6 +93,6 @@ by a durable database URL. They are not a production multi-tenant claim.
 
 ## Links
 
-[Optional packages](https://etlantic.readthedocs.io/en/v0.40.0/10_REFERENCE/OPTIONAL_PACKAGES/) ·
+[Optional packages](https://etlantic.readthedocs.io/en/v0.41.0/10_REFERENCE/OPTIONAL_PACKAGES/) ·
 [Source](https://github.com/eddiethedean/etlantic/tree/main/packages/etlantic-sqlmodel) ·
 [Issues](https://github.com/eddiethedean/etlantic/issues)

{etlantic_sqlmodel-0.40.0 → etlantic_sqlmodel-0.41.0}/README.md

@@ -4,13 +4,13 @@ Optional bridge between ETLantic `Data` contracts and
 [SQLModel](https://sqlmodel.tiangolo.com/) table models, plus optional CP1
 control-plane reference stores. Install when you need `contract_to_sqlmodel`
 helpers or SQLModel-backed definition/submission stores for local CP1 demos.
-Package version is **0.39.0** — pin with core.
+Package version is **0.41.0** — pin with core.
 
 ## Install
 
 ```bash
-pip install 'etlantic-sqlmodel==0.40.0'
-# pip install 'etlantic==0.40.0'
+pip install 'etlantic-sqlmodel==0.41.0'
+# pip install 'etlantic==0.41.0'
 ```
 
 ## Schema bridge
@@ -69,6 +69,6 @@ by a durable database URL. They are not a production multi-tenant claim.
 
 ## Links
 
-[Optional packages](https://etlantic.readthedocs.io/en/v0.40.0/10_REFERENCE/OPTIONAL_PACKAGES/) ·
+[Optional packages](https://etlantic.readthedocs.io/en/v0.41.0/10_REFERENCE/OPTIONAL_PACKAGES/) ·
 [Source](https://github.com/eddiethedean/etlantic/tree/main/packages/etlantic-sqlmodel) ·
 [Issues](https://github.com/eddiethedean/etlantic/issues)

{etlantic_sqlmodel-0.40.0 → etlantic_sqlmodel-0.41.0}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "etlantic-sqlmodel"
-version = "0.40.0"
+version = "0.41.0"
 description = "ContractModel ↔ SQLModel bridge for ETLantic."
 readme = "README.md"
 license = "MIT"
@@ -17,7 +17,7 @@ classifiers = [
     "Typing :: Typed",
 ]
 dependencies = [
-    "etlantic>=0.40.0,<0.41",
+    "etlantic>=0.41.0,<0.42",
     "sqlmodel>=0.0.22,<1",
 ]
 

{etlantic_sqlmodel-0.40.0 → etlantic_sqlmodel-0.41.0}/src/etlantic_sqlmodel/__init__.py

@@ -52,7 +52,7 @@ from etlantic_sqlmodel.migrations import (
 )
 from sqlmodel import Field, SQLModel
 
-__version__ = "0.40.0"
+__version__ = "0.41.0"
 
 __all__ = [
     "AliasRow",

{etlantic_sqlmodel-0.40.0 → etlantic_sqlmodel-0.41.0}/src/etlantic_sqlmodel/control_plane/__init__.py

@@ -9,9 +9,15 @@ tests/demos — production must apply versioned migrations under
 
 from __future__ import annotations
 
+from etlantic_sqlmodel.control_plane.durable_stores import (
+    DURABLE_TABLES,
+    SQLModelDurableWorkStore,
+    create_durable_tables,
+)
 from etlantic_sqlmodel.control_plane.models import (
     AliasRow,
     DefinitionRow,
+    DurableSnapshotRow,
     EnvironmentRow,
     EventRow,
     LogicalIdentityRow,
@@ -55,16 +61,19 @@ from etlantic_sqlmodel.control_plane.stores import (
 
 __all__ = [
     "BACKUP_SCHEMA",
+    "DURABLE_TABLES",
     "REGISTRY_TABLES",
     "AliasRow",
     "BackupTranscript",
     "DefinitionRow",
+    "DurableSnapshotRow",
     "EnvironmentRow",
     "EventRow",
     "LogicalIdentityRow",
     "PromotionRow",
     "RevisionRow",
     "SQLModelDefinitionRepository",
+    "SQLModelDurableWorkStore",
     "SQLModelSubmissionStore",
     "SecurityDomainRow",
     "SqlModelEventStore",
@@ -78,6 +87,7 @@ __all__ = [
     "backup_round_trip",
     "collect_revision_hits",
     "create_control_plane_tables",
+    "create_durable_tables",
     "create_registry_tables",
     "create_sqlite_engine",
     "dump_registry_sqlite",

etlantic_sqlmodel-0.41.0/src/etlantic_sqlmodel/control_plane/durable_stores.py

@@ -0,0 +1,319 @@
+"""SQLModel-backed DurableWorkStore (CP3 / 041-P).
+
+Reference provider: each mutating call loads the durable snapshot inside a
+database transaction, applies MemoryDurableWorkStore semantics, and writes the
+snapshot back with an optimistic ``payload_version`` check before commit.
+
+Production note: apply versioned migrations — do not rely on
+``create_durable_tables`` as the sole schema path.
+"""
+
+from __future__ import annotations
+
+import json
+from collections.abc import Callable, Mapping
+from dataclasses import asdict
+from datetime import UTC, datetime
+from typing import Any, TypeVar
+
+from sqlalchemy.engine import Engine
+
+from etlantic.control_plane.durable_memory import MemoryDurableWorkStore
+from etlantic.control_plane.durable_models import (
+    AttemptRecord,
+    BaselineAcknowledgement,
+    CheckpointRecord,
+    DiffRecord,
+    EffectRecord,
+    LeaseRecord,
+    OutboxRecord,
+    PreviewWorkspace,
+    ShadowRunRecord,
+    StateDiagnostic,
+    SubmissionRecord,
+)
+from etlantic.control_plane.errors import ControlPlaneError
+from etlantic.control_plane.models import ControlPlaneContext
+from etlantic_sqlmodel.control_plane.models import DurableSnapshotRow
+from etlantic_sqlmodel.control_plane.session import session_scope
+from sqlmodel import Session, SQLModel, select
+
+T = TypeVar("T")
+
+DURABLE_TABLES = (DurableSnapshotRow,)
+
+
+def create_durable_tables(engine: Engine) -> None:
+    """Create CP3 durable tables (tests/demos only)."""
+    SQLModel.metadata.create_all(
+        engine,
+        tables=[cls.__table__ for cls in DURABLE_TABLES],  # type: ignore[list-item]
+    )
+
+
+def _utcnow_iso() -> str:
+    return datetime.now(UTC).isoformat().replace("+00:00", "Z")
+
+
+def _encode_key(parts: tuple[str, ...]) -> str:
+    return json.dumps(list(parts), separators=(",", ":"))
+
+
+def _decode_key(text: str) -> tuple[str, ...]:
+    raw = json.loads(text)
+    if not isinstance(raw, list) or not all(isinstance(p, str) for p in raw):
+        raise ValueError("durable snapshot key must be a JSON string array")
+    return tuple(raw)
+
+
+def _dump_store(store: MemoryDurableWorkStore) -> dict[str, Any]:
+    return {
+        "admission_limit": store.admission_limit,
+        "submissions": {
+            _encode_key(k): asdict(v) for k, v in store._submissions.items()
+        },
+        "idempotency": {_encode_key(k): v for k, v in store._idempotency.items()},
+        "outbox": {_encode_key(k): asdict(v) for k, v in store._outbox.items()},
+        "leases": {_encode_key(k): asdict(v) for k, v in store._leases.items()},
+        "attempts": {_encode_key(k): asdict(v) for k, v in store._attempts.items()},
+        "checkpoints": {
+            _encode_key(k): asdict(v) for k, v in store._checkpoints.items()
+        },
+        "effects": {_encode_key(k): asdict(v) for k, v in store._effects.items()},
+        "previews": {_encode_key(k): asdict(v) for k, v in store._previews.items()},
+        "diffs": {_encode_key(k): asdict(v) for k, v in store._diffs.items()},
+        "shadows": {_encode_key(k): asdict(v) for k, v in store._shadows.items()},
+        "baselines": {_encode_key(k): asdict(v) for k, v in store._baselines.items()},
+        "diagnostics": [asdict(d) for d in store._diagnostics],
+    }
+
+
+def _load_store(payload: Mapping[str, Any]) -> MemoryDurableWorkStore:
+    store = MemoryDurableWorkStore(admission_limit=payload.get("admission_limit"))
+    store._submissions = {
+        _decode_key(k): SubmissionRecord(**v)  # type: ignore[arg-type]
+        for k, v in dict(payload.get("submissions") or {}).items()
+    }
+    store._idempotency = {
+        _decode_key(k): v for k, v in dict(payload.get("idempotency") or {}).items()
+    }
+    store._outbox = {
+        _decode_key(k): OutboxRecord(**v)  # type: ignore[arg-type]
+        for k, v in dict(payload.get("outbox") or {}).items()
+    }
+    store._leases = {
+        _decode_key(k): LeaseRecord(**v)  # type: ignore[arg-type]
+        for k, v in dict(payload.get("leases") or {}).items()
+    }
+    store._attempts = {
+        _decode_key(k): AttemptRecord(**v)  # type: ignore[arg-type]
+        for k, v in dict(payload.get("attempts") or {}).items()
+    }
+    store._checkpoints = {
+        _decode_key(k): CheckpointRecord(**v)  # type: ignore[arg-type]
+        for k, v in dict(payload.get("checkpoints") or {}).items()
+    }
+    store._effects = {
+        _decode_key(k): EffectRecord(**v)  # type: ignore[arg-type]
+        for k, v in dict(payload.get("effects") or {}).items()
+    }
+    store._previews = {
+        _decode_key(k): PreviewWorkspace(**v)  # type: ignore[arg-type]
+        for k, v in dict(payload.get("previews") or {}).items()
+    }
+    store._diffs = {
+        _decode_key(k): DiffRecord(**v)  # type: ignore[arg-type]
+        for k, v in dict(payload.get("diffs") or {}).items()
+    }
+    store._shadows = {
+        _decode_key(k): ShadowRunRecord(**v)  # type: ignore[arg-type]
+        for k, v in dict(payload.get("shadows") or {}).items()
+    }
+    store._baselines = {
+        _decode_key(k): BaselineAcknowledgement(**v)  # type: ignore[arg-type]
+        for k, v in dict(payload.get("baselines") or {}).items()
+    }
+    store._diagnostics = [
+        StateDiagnostic(**d)  # type: ignore[arg-type]
+        for d in list(payload.get("diagnostics") or [])
+    ]
+    return store
+
+
+class SQLModelDurableWorkStore:
+    """Transactional DurableWorkStore backed by a SQL snapshot row."""
+
+    def __init__(
+        self,
+        engine: Engine,
+        *,
+        admission_limit: int | None = None,
+        store_id: str = "default",
+    ) -> None:
+        self.engine = engine
+        self.admission_limit = admission_limit
+        self.store_id = store_id
+
+    def _txn(self, fn: Callable[[MemoryDurableWorkStore], T]) -> T:
+        with session_scope(self.engine) as session:
+            mem, version = self._read(session, for_update=True)
+            if self.admission_limit is not None:
+                mem.admission_limit = self.admission_limit
+            result = fn(mem)
+            self._write(session, mem, expected_version=version)
+            return result
+
+    def _read_only(self, fn: Callable[[MemoryDurableWorkStore], T]) -> T:
+        with session_scope(self.engine) as session:
+            mem, _version = self._read(session, for_update=False)
+            if self.admission_limit is not None:
+                mem.admission_limit = self.admission_limit
+            return fn(mem)
+
+    def _read(
+        self, session: Session, *, for_update: bool
+    ) -> tuple[MemoryDurableWorkStore, int]:
+        stmt = select(DurableSnapshotRow).where(
+            DurableSnapshotRow.store_id == self.store_id
+        )
+        if for_update:
+            stmt = stmt.with_for_update()
+        row = session.exec(stmt).first()
+        if row is None:
+            return MemoryDurableWorkStore(admission_limit=self.admission_limit), 0
+        return _load_store(json.loads(row.payload_json or "{}")), int(
+            row.payload_version or 0
+        )
+
+    def _write(
+        self,
+        session: Session,
+        store: MemoryDurableWorkStore,
+        *,
+        expected_version: int,
+    ) -> None:
+        payload = json.dumps(_dump_store(store), sort_keys=True)
+        row = session.exec(
+            select(DurableSnapshotRow)
+            .where(DurableSnapshotRow.store_id == self.store_id)
+            .with_for_update()
+        ).first()
+        if row is None:
+            if expected_version != 0:
+                raise ControlPlaneError.conflict(
+                    "Durable snapshot version conflict (missing row)"
+                )
+            session.add(
+                DurableSnapshotRow(
+                    store_id=self.store_id,
+                    payload_json=payload,
+                    payload_version=1,
+                    updated_at=_utcnow_iso(),
+                )
+            )
+            return
+        current = int(row.payload_version or 0)
+        if current != expected_version:
+            raise ControlPlaneError.conflict(
+                "Durable snapshot version conflict (stale write)"
+            )
+        row.payload_json = payload
+        row.payload_version = current + 1
+        row.updated_at = _utcnow_iso()
+        session.add(row)
+
+    def accept(self, ctx: ControlPlaneContext, **kwargs: Any):
+        return self._txn(lambda m: m.accept(ctx, **kwargs))
+
+    def pending_outbox(self, ctx: ControlPlaneContext, *, limit: int = 100):
+        return self._read_only(lambda m: m.pending_outbox(ctx, limit=limit))
+
+    def mark_published(self, ctx: ControlPlaneContext, outbox_id: str):
+        return self._txn(lambda m: m.mark_published(ctx, outbox_id))
+
+    def cancel_submission(self, ctx: ControlPlaneContext, submission_id: str):
+        return self._txn(lambda m: m.cancel_submission(ctx, submission_id))
+
+    def acquire_lease(
+        self, ctx: ControlPlaneContext, submission_id: str, **kwargs: Any
+    ):
+        return self._txn(lambda m: m.acquire_lease(ctx, submission_id, **kwargs))
+
+    def heartbeat(self, ctx: ControlPlaneContext, submission_id: str, **kwargs: Any):
+        return self._txn(lambda m: m.heartbeat(ctx, submission_id, **kwargs))
+
+    def release_lease(
+        self, ctx: ControlPlaneContext, submission_id: str, **kwargs: Any
+    ):
+        return self._txn(lambda m: m.release_lease(ctx, submission_id, **kwargs))
+
+    def start_attempt(
+        self, ctx: ControlPlaneContext, submission_id: str, **kwargs: Any
+    ):
+        return self._txn(lambda m: m.start_attempt(ctx, submission_id, **kwargs))
+
+    def finish_attempt(self, ctx: ControlPlaneContext, attempt_id: str, **kwargs: Any):
+        return self._txn(lambda m: m.finish_attempt(ctx, attempt_id, **kwargs))
+
+    def compare_and_swap_checkpoint(
+        self, ctx: ControlPlaneContext, checkpoint_id: str, **kwargs: Any
+    ):
+        return self._txn(
+            lambda m: m.compare_and_swap_checkpoint(ctx, checkpoint_id, **kwargs)
+        )
+
+    def explain_transition(
+        self, ctx: ControlPlaneContext, checkpoint_id: str, **kwargs: Any
+    ):
+        return self._read_only(
+            lambda m: m.explain_transition(ctx, checkpoint_id, **kwargs)
+        )
+
+    def diagnose_checkpoint(
+        self, ctx: ControlPlaneContext, checkpoint_id: str, **kwargs: Any
+    ):
+        return self._txn(lambda m: m.diagnose_checkpoint(ctx, checkpoint_id, **kwargs))
+
+    def acknowledge_baseline(self, ctx: ControlPlaneContext, **kwargs: Any):
+        return self._txn(lambda m: m.acknowledge_baseline(ctx, **kwargs))
+
+    def record_effect(self, ctx: ControlPlaneContext, effect: EffectRecord):
+        return self._txn(lambda m: m.record_effect(ctx, effect))
+
+    def replay(self, ctx: ControlPlaneContext, submission_id: str, **kwargs: Any):
+        return self._read_only(lambda m: m.replay(ctx, submission_id, **kwargs))
+
+    def plan_resume(self, ctx: ControlPlaneContext, submission_id: str, **kwargs: Any):
+        return self._read_only(lambda m: m.plan_resume(ctx, submission_id, **kwargs))
+
+    def plan_repair(self, ctx: ControlPlaneContext, submission_id: str, **kwargs: Any):
+        return self._read_only(lambda m: m.plan_repair(ctx, submission_id, **kwargs))
+
+    def plan_backfill(
+        self, ctx: ControlPlaneContext, submission_id: str, **kwargs: Any
+    ):
+        return self._read_only(lambda m: m.plan_backfill(ctx, submission_id, **kwargs))
+
+    def create_preview(self, ctx: ControlPlaneContext, preview: PreviewWorkspace):
+        return self._txn(lambda m: m.create_preview(ctx, preview))
+
+    def mark_preview_stale(
+        self, ctx: ControlPlaneContext, preview_id: str, **kwargs: Any
+    ):
+        return self._txn(lambda m: m.mark_preview_stale(ctx, preview_id, **kwargs))
+
+    def record_preview_diff(self, ctx: ControlPlaneContext, diff: DiffRecord):
+        return self._txn(lambda m: m.record_preview_diff(ctx, diff))
+
+    def authorize_shadow_run(self, ctx: ControlPlaneContext, shadow: ShadowRunRecord):
+        return self._txn(lambda m: m.authorize_shadow_run(ctx, shadow))
+
+    def cleanup_expired_previews(self, ctx: ControlPlaneContext):
+        return self._txn(lambda m: m.cleanup_expired_previews(ctx))
+
+
+__all__ = [
+    "DURABLE_TABLES",
+    "SQLModelDurableWorkStore",
+    "create_durable_tables",
+]

{etlantic_sqlmodel-0.40.0 → etlantic_sqlmodel-0.41.0}/src/etlantic_sqlmodel/control_plane/models.py

@@ -6,7 +6,7 @@ pipeline contracts.
 
 from __future__ import annotations
 
-from sqlalchemy import Column, String, UniqueConstraint
+from sqlalchemy import Column, Integer, String, UniqueConstraint
 
 from sqlmodel import Field, SQLModel
 
@@ -266,9 +266,29 @@ class SecurityDomainRow(SQLModel, table=True):
     metadata_json: str = "{}"
 
 
+class DurableSnapshotRow(SQLModel, table=True):
+    """Transactional CP3 durable-work snapshot (reference provider).
+
+    One logical store per engine; mutations load/save under a DB transaction so
+    accept+outbox and fencing CAS share commit boundaries.
+    """
+
+    __tablename__ = "cp_durable_snapshot"
+    __table_args__ = (UniqueConstraint("store_id", name="uq_cp_durable_snapshot"),)
+
+    id: int | None = Field(default=None, primary_key=True)
+    store_id: str = Field(
+        sa_column=Column(String(), index=True, nullable=False), default="default"
+    )
+    payload_json: str = "{}"
+    payload_version: int = Field(default=0, sa_column=Column(Integer(), nullable=False))
+    updated_at: str | None = None
+
+
 __all__ = [
     "AliasRow",
     "DefinitionRow",
+    "DurableSnapshotRow",
     "EnvironmentRow",
     "EventRow",
     "LogicalIdentityRow",

{etlantic_sqlmodel-0.40.0 → etlantic_sqlmodel-0.41.0}/src/etlantic_sqlmodel/control_plane/registry_backup.py

@@ -9,7 +9,7 @@ from __future__ import annotations
 
 import json
 from collections.abc import Mapping
-from dataclasses import dataclass
+from dataclasses import dataclass, replace
 from datetime import UTC, datetime
 from pathlib import Path
 from typing import Any
@@ -21,6 +21,7 @@ from etlantic.control_plane import (
     ControlPlaneContext,
     EnvironmentRecord,
     EnvironmentRef,
+    LifecycleState,
     LogicalIdentity,
     Principal,
     PromotionRecord,
@@ -32,7 +33,9 @@ from etlantic.control_plane import (
     WorkspaceRecord,
     WorkspaceRef,
     content_fingerprint,
+    redact_control_plane_payload,
 )
+from etlantic.control_plane.registry_memory import safe_registry_content
 from etlantic_sqlmodel.control_plane.models import (
     AliasRow,
     EnvironmentRow,
@@ -147,23 +150,26 @@ def dump_registry_sqlite(engine: Engine) -> BackupTranscript:
             }
             for row in session.exec(select(WorkspaceRow)).all()
         ]
-        revisions = [
-            {
-                "tenant_id": row.tenant_id,
-                "workspace_id": row.workspace_id,
-                "logical_id": row.logical_id,
-                "revision_id": row.revision_id,
-                "content_fingerprint": row.content_fingerprint,
-                "content": json.loads(row.content_json or "{}"),
-                "created_at": row.created_at,
-                "kind": row.kind,
-                "signature_placeholder": row.signature_placeholder,
-                "provenance": (
-                    json.loads(row.provenance_json) if row.provenance_json else {}
-                ),
-            }
-            for row in session.exec(select(RevisionRow)).all()
-        ]
+        revisions = []
+        for row in session.exec(select(RevisionRow)).all():
+            content = safe_registry_content(json.loads(row.content_json or "{}"))
+            provenance = redact_control_plane_payload(
+                json.loads(row.provenance_json) if row.provenance_json else {}
+            )
+            revisions.append(
+                {
+                    "tenant_id": row.tenant_id,
+                    "workspace_id": row.workspace_id,
+                    "logical_id": row.logical_id,
+                    "revision_id": row.revision_id,
+                    "content_fingerprint": content_fingerprint(content),
+                    "content": content,
+                    "created_at": row.created_at,
+                    "kind": row.kind,
+                    "signature_placeholder": row.signature_placeholder,
+                    "provenance": provenance if isinstance(provenance, dict) else {},
+                }
+            )
         logicals = [
             {
                 "tenant_id": row.tenant_id,
@@ -286,7 +292,13 @@ def load_registry_sqlite(
             workspace_id="__backup__",
             domain_id=domain_id,
         )
-        provider.tenants.put(ctx, TenantRecord.from_dict(tenant))
+        provider.tenants.put(
+            ctx,
+            replace(
+                TenantRecord.from_dict(tenant),
+                lifecycle=LifecycleState.ACTIVE,
+            ),
+        )
 
     for workspace in transcript.workspaces:
         tenant_id = str(workspace["tenant_id"])
@@ -304,7 +316,13 @@ def load_registry_sqlite(
             workspace_id=workspace_id,
             domain_id=domain_id,
         )
-        provider.workspaces.put(ctx, WorkspaceRecord.from_dict(workspace))
+        provider.workspaces.put(
+            ctx,
+            replace(
+                WorkspaceRecord.from_dict(workspace),
+                lifecycle=LifecycleState.ACTIVE,
+            ),
+        )
 
     for environment in transcript.environments:
         tenant_id = str(environment["tenant_id"])
@@ -318,13 +336,14 @@ def load_registry_sqlite(
             ),
             "backup-domain",
         )
+        environment_record = EnvironmentRecord.from_dict(environment)
         provider.put_environment(
             _admin_ctx(
                 tenant_id=tenant_id,
                 workspace_id=workspace_id,
                 domain_id=domain_id,
             ),
-            EnvironmentRecord.from_dict(environment),
+            replace(environment_record, lifecycle=LifecycleState.ACTIVE),
         )
 
     for logical in transcript.logicals:
@@ -416,10 +435,72 @@ def load_registry_sqlite(
                     from_environment=record.from_environment,
                     to_environment=record.to_environment,
                     created_at=record.created_at,
-                    metadata_json=json.dumps(dict(record.metadata), sort_keys=True),
+                    metadata_json=json.dumps(
+                        redact_control_plane_payload(dict(record.metadata)),
+                        sort_keys=True,
+                    ),
                 )
             )
 
+    # Apply archived/suspended lifecycle states only after dependent rows exist.
+    for environment in transcript.environments:
+        record = EnvironmentRecord.from_dict(environment)
+        if record.lifecycle == LifecycleState.ACTIVE:
+            continue
+        workspace_id = record.workspace_id or "__backup__"
+        domain_id = next(
+            (
+                str(tenant["security_domain_id"])
+                for tenant in transcript.tenants
+                if tenant.get("tenant_id") == record.tenant_id
+                and tenant.get("security_domain_id")
+            ),
+            "backup-domain",
+        )
+        provider.put_environment(
+            _admin_ctx(
+                tenant_id=record.tenant_id,
+                workspace_id=workspace_id,
+                domain_id=domain_id,
+            ),
+            record,
+        )
+
+    for workspace in transcript.workspaces:
+        record = WorkspaceRecord.from_dict(workspace)
+        if record.lifecycle != LifecycleState.ACTIVE:
+            domain_id = next(
+                (
+                    str(tenant["security_domain_id"])
+                    for tenant in transcript.tenants
+                    if tenant.get("tenant_id") == record.tenant_id
+                    and tenant.get("security_domain_id")
+                ),
+                "backup-domain",
+            )
+            provider.workspaces.set_lifecycle(
+                _admin_ctx(
+                    tenant_id=record.tenant_id,
+                    workspace_id=record.workspace_id,
+                    domain_id=domain_id,
+                ),
+                record.workspace_id,
+                record.lifecycle,
+            )
+
+    for tenant in transcript.tenants:
+        record = TenantRecord.from_dict(tenant)
+        if record.lifecycle != LifecycleState.ACTIVE:
+            provider.tenants.set_lifecycle(
+                _admin_ctx(
+                    tenant_id=record.tenant_id,
+                    workspace_id="__backup__",
+                    domain_id=record.security_domain_id or "backup-domain",
+                ),
+                record.tenant_id,
+                record.lifecycle,
+            )
+
     return provider
 
 

{etlantic_sqlmodel-0.40.0 → etlantic_sqlmodel-0.41.0}/src/etlantic_sqlmodel/control_plane/registry_stores.py

@@ -32,6 +32,8 @@ from etlantic.control_plane import (
     content_fingerprint,
     redact_control_plane_payload,
 )
+from etlantic.control_plane.redaction import redact_control_plane_text
+from etlantic.control_plane.registry_memory import safe_registry_content
 from etlantic_sqlmodel.control_plane.models import (
     AliasRow,
     EnvironmentRow,
@@ -192,58 +194,33 @@ class SqlModelTenantDirectory:
     def get(self, ctx: ControlPlaneContext, tenant_id: str) -> TenantRecord:
         with session_scope(self.engine) as session:
             row = self._by_id(session, tenant_id)
-            if row is None:
+            if row is None or ctx.tenant.tenant_id != tenant_id:
                 raise ControlPlaneError.not_found(
                     "Tenant not found",
                     extensions={"tenant_id": tenant_id},
                 )
             record = _tenant_from_row(row)
-            in_scope = ctx.tenant.tenant_id == tenant_id
-            admin_scope = (
-                record.security_domain_id is not None
-                and record.security_domain_id == ctx.security_domain.domain_id
-            )
-            if not in_scope and not admin_scope:
-                raise ControlPlaneError.not_found(
-                    "Tenant not found",
-                    extensions={"tenant_id": tenant_id},
-                )
             _require_active(record.lifecycle, resource=f"tenant:{tenant_id}")
             return deepcopy(record)
 
     def put(self, ctx: ControlPlaneContext, record: TenantRecord) -> None:
         with session_scope(self.engine) as session:
+            if ctx.tenant.tenant_id != record.tenant_id:
+                raise ControlPlaneError.not_found(
+                    "Tenant not found",
+                    extensions={"tenant_id": record.tenant_id},
+                )
             existing = self._by_id(session, record.tenant_id)
             if existing is not None:
                 _require_active(
                     LifecycleState(existing.lifecycle),
                     resource=f"tenant:{record.tenant_id}",
                 )
-                existing_in_scope = ctx.tenant.tenant_id == record.tenant_id
-                existing_admin_scope = (
-                    existing.security_domain_id is not None
-                    and existing.security_domain_id == ctx.security_domain.domain_id
-                )
-                if not existing_in_scope and not existing_admin_scope:
-                    raise ControlPlaneError.not_found(
-                        "Tenant not found",
-                        extensions={"tenant_id": record.tenant_id},
-                    )
                 if existing.security_domain_id != record.security_domain_id:
                     raise ControlPlaneError.conflict(
                         "Tenant security domain is immutable through directory put",
                         extensions={"tenant_id": record.tenant_id},
                     )
-            in_scope = ctx.tenant.tenant_id == record.tenant_id
-            admin_scope = (
-                record.security_domain_id is not None
-                and record.security_domain_id == ctx.security_domain.domain_id
-            )
-            if not in_scope and not admin_scope:
-                raise ControlPlaneError.not_found(
-                    "Tenant not found",
-                    extensions={"tenant_id": record.tenant_id},
-                )
             now = _utcnow_iso()
             if existing is None:
                 session.add(
@@ -267,16 +244,10 @@ class SqlModelTenantDirectory:
 
     def list(self, ctx: ControlPlaneContext) -> Sequence[TenantRecord]:
         with session_scope(self.engine) as session:
-            rows = session.exec(select(TenantRow)).all()
-            out: list[TenantRecord] = []
-            for row in rows:
-                record = _tenant_from_row(row)
-                if record.tenant_id == ctx.tenant.tenant_id or (
-                    record.security_domain_id is not None
-                    and record.security_domain_id == ctx.security_domain.domain_id
-                ):
-                    out.append(deepcopy(record))
-            return sorted(out, key=lambda r: r.tenant_id)
+            row = self._by_id(session, ctx.tenant.tenant_id)
+            if row is None:
+                return ()
+            return (deepcopy(_tenant_from_row(row)),)
 
     def set_lifecycle(
         self,
@@ -286,10 +257,7 @@ class SqlModelTenantDirectory:
     ) -> TenantRecord:
         with session_scope(self.engine) as session:
             row = self._by_id(session, tenant_id)
-            if row is None or (
-                ctx.tenant.tenant_id != tenant_id
-                and row.security_domain_id != ctx.security_domain.domain_id
-            ):
+            if row is None or ctx.tenant.tenant_id != tenant_id:
                 raise ControlPlaneError.not_found(
                     "Tenant not found",
                     extensions={"tenant_id": tenant_id},
@@ -327,6 +295,11 @@ class SqlModelWorkspaceDirectory:
             _require_active(tenant.lifecycle, resource=f"tenant:{tenant_id}")
 
     def get(self, ctx: ControlPlaneContext, workspace_id: str) -> WorkspaceRecord:
+        if workspace_id != ctx.workspace.workspace_id:
+            raise ControlPlaneError.not_found(
+                "Workspace not found",
+                extensions={"workspace_id": workspace_id},
+            )
         with session_scope(self.engine) as session:
             self._assert_tenant_active(ctx.tenant.tenant_id)
             row = self._by_key(session, ctx.tenant.tenant_id, workspace_id)
@@ -343,7 +316,10 @@ class SqlModelWorkspaceDirectory:
             return deepcopy(record)
 
     def put(self, ctx: ControlPlaneContext, record: WorkspaceRecord) -> None:
-        if record.tenant_id != ctx.tenant.tenant_id:
+        if (
+            record.tenant_id != ctx.tenant.tenant_id
+            or record.workspace_id != ctx.workspace.workspace_id
+        ):
             raise ControlPlaneError.not_found(
                 "Workspace not found",
                 extensions={"workspace_id": record.workspace_id},
@@ -394,6 +370,11 @@ class SqlModelWorkspaceDirectory:
         workspace_id: str,
         state: LifecycleState,
     ) -> WorkspaceRecord:
+        if workspace_id != ctx.workspace.workspace_id:
+            raise ControlPlaneError.not_found(
+                "Workspace not found",
+                extensions={"workspace_id": workspace_id},
+            )
         with session_scope(self.engine) as session:
             self._assert_tenant_active(ctx.tenant.tenant_id)
             row = self._by_key(session, ctx.tenant.tenant_id, workspace_id)
@@ -549,6 +530,7 @@ class SqlModelRevisionRegistry:
                     "recorded": revision.content_fingerprint,
                 },
             )
+        safe_content = safe_registry_content(revision.content)
         try:
             with session_scope(self.engine) as session:
                 self._assert_scope_active(ctx)
@@ -597,15 +579,17 @@ class SqlModelRevisionRegistry:
                         workspace_id=revision.workspace_id,
                         logical_id=revision.logical_id,
                         revision_id=revision.revision_id,
-                        content_fingerprint=revision.content_fingerprint,
-                        content_json=json.dumps(dict(revision.content), sort_keys=True),
+                        content_fingerprint=content_fingerprint(safe_content),
+                        content_json=json.dumps(safe_content, sort_keys=True),
                         created_at=revision.created_at or _utcnow_iso(),
                         kind=revision.kind,
-                        signature_placeholder=revision.signature_placeholder,
+                        signature_placeholder=(
+                            redact_control_plane_text(revision.signature_placeholder)
+                            if revision.signature_placeholder is not None
+                            else None
+                        ),
                         provenance_json=(
-                            json.dumps(
-                                dict(revision.provenance_placeholder), sort_keys=True
-                            )
+                            _meta(revision.provenance_placeholder)
                             if revision.provenance_placeholder is not None
                             else None
                         ),
@@ -653,7 +637,7 @@ class SqlModelRevisionRegistry:
                 key=lambda r: r.revision_id,
             )
 
-    def put_alias(self, ctx: ControlPlaneContext, alias: AliasRecord) -> None:
+    def put_alias(self, ctx: ControlPlaneContext, alias: AliasRecord) -> AliasRecord:
         if (
             alias.tenant_id != ctx.tenant.tenant_id
             or alias.workspace_id != ctx.workspace.workspace_id
@@ -706,6 +690,12 @@ class SqlModelRevisionRegistry:
                 existing.created_at = created
                 existing.metadata_json = _meta(alias.metadata)
                 session.add(existing)
+            session.flush()
+            row = self._alias_row(
+                session, ctx.tenant.tenant_id, ctx.workspace.workspace_id, alias.alias
+            )
+            assert row is not None
+            return deepcopy(_alias_from_row(row))
 
     def resolve_alias(self, ctx: ControlPlaneContext, alias: str) -> RegistryRevision:
         with session_scope(self.engine) as session:
@@ -757,11 +747,12 @@ class SqlModelRevisionRegistry:
                 )
             source = _revision_from_row(source_row)
             source_snapshot = deepcopy(source)
-            body = (
+            requested_body = (
                 deepcopy(dict(content))
                 if content is not None
                 else deepcopy(dict(source.content))
             )
+            body = safe_registry_content(requested_body)
             new_revision_id = f"rev-{uuid.uuid4().hex[:16]}"
             created = _utcnow_iso()
             new_rev = RegistryRevision(
@@ -819,7 +810,7 @@ class SqlModelRevisionRegistry:
                 from_environment=from_environment,
                 to_environment=to_environment,
                 created_at=created,
-                metadata=dict(metadata or {}),
+                metadata=json.loads(_meta(metadata)),
             )
             session.add(
                 PromotionRow(
@@ -948,6 +939,15 @@ class SqlModelRegistryProvider:
                 _require_active(
                     tenant.lifecycle, resource=f"tenant:{ctx.tenant.tenant_id}"
                 )
+            assert self.workspaces is not None
+            workspace = self.workspaces.peek(
+                ctx.tenant.tenant_id, ctx.workspace.workspace_id
+            )
+            if workspace is not None:
+                _require_active(
+                    workspace.lifecycle,
+                    resource=f"workspace:{ctx.workspace.workspace_id}",
+                )
             now = _utcnow_iso()
             existing = session.exec(
                 select(EnvironmentRow).where(
@@ -955,6 +955,20 @@ class SqlModelRegistryProvider:
                     EnvironmentRow.environment_id == record.environment_id,
                 )
             ).first()
+            if (
+                existing is not None
+                and existing.workspace_id is not None
+                and existing.workspace_id != ctx.workspace.workspace_id
+            ):
+                raise ControlPlaneError.not_found(
+                    "Environment not found",
+                    extensions={"environment_id": record.environment_id},
+                )
+            if existing is not None:
+                _require_active(
+                    LifecycleState(existing.lifecycle),
+                    resource=f"environment:{record.environment_id}",
+                )
             if existing is None:
                 session.add(
                     EnvironmentRow(
@@ -988,29 +1002,44 @@ class SqlModelRegistryProvider:
                 _require_active(
                     tenant.lifecycle, resource=f"tenant:{ctx.tenant.tenant_id}"
                 )
+            assert self.workspaces is not None
+            workspace = self.workspaces.peek(
+                ctx.tenant.tenant_id, ctx.workspace.workspace_id
+            )
+            if workspace is not None:
+                _require_active(
+                    workspace.lifecycle,
+                    resource=f"workspace:{ctx.workspace.workspace_id}",
+                )
             row = session.exec(
                 select(EnvironmentRow).where(
                     EnvironmentRow.tenant_id == ctx.tenant.tenant_id,
                     EnvironmentRow.environment_id == environment_id,
                 )
             ).first()
-            if row is None:
+            if row is None or (
+                row.workspace_id is not None
+                and row.workspace_id != ctx.workspace.workspace_id
+            ):
                 raise ControlPlaneError.not_found(
                     "Environment not found",
                     extensions={"environment_id": environment_id},
                 )
-            return deepcopy(
-                EnvironmentRecord(
-                    tenant_id=row.tenant_id,
-                    environment_id=row.environment_id,
-                    name=row.name,
-                    workspace_id=row.workspace_id,
-                    lifecycle=LifecycleState(row.lifecycle),
-                    created_at=row.created_at,
-                    updated_at=row.updated_at,
-                    metadata=_load_meta(row.metadata_json),
-                )
+            record = EnvironmentRecord(
+                tenant_id=row.tenant_id,
+                environment_id=row.environment_id,
+                name=row.name,
+                workspace_id=row.workspace_id,
+                lifecycle=LifecycleState(row.lifecycle),
+                created_at=row.created_at,
+                updated_at=row.updated_at,
+                metadata=_load_meta(row.metadata_json),
             )
+            _require_active(
+                record.lifecycle,
+                resource=f"environment:{environment_id}",
+            )
+            return deepcopy(record)
 
     def put_security_domain(
         self,

{etlantic_sqlmodel-0.40.0 → etlantic_sqlmodel-0.41.0}/src/etlantic_sqlmodel/control_plane/stores.py

@@ -5,6 +5,7 @@ from __future__ import annotations
 import json
 import uuid
 from collections.abc import Callable, Mapping, Sequence
+from copy import deepcopy
 from datetime import UTC, datetime
 from typing import Any
 
@@ -16,6 +17,7 @@ from etlantic.control_plane import (
     ControlPlaneContext,
     ControlPlaneError,
     ControlPlaneEvent,
+    redact_control_plane_payload,
 )
 from etlantic_sqlmodel.control_plane.models import (
     DefinitionRow,
@@ -130,6 +132,9 @@ class SQLModelSubmissionStore:
     ) -> AcceptResult:
         from sqlalchemy.exc import IntegrityError
 
+        safe_payload = redact_control_plane_payload(deepcopy(dict(payload)))
+        if not isinstance(safe_payload, dict):
+            safe_payload = {}
         try:
             with session_scope(self._engine) as session:
                 existing = self._by_idem(
@@ -137,7 +142,7 @@ class SQLModelSubmissionStore:
                 )
                 if existing is not None:
                     prior = json.loads(existing.payload_json)
-                    if prior != dict(payload):
+                    if prior != safe_payload:
                         raise ControlPlaneError.conflict(
                             "Idempotency key reuse with a different payload",
                             extensions={"idempotency_key": idempotency_key},
@@ -162,12 +167,12 @@ class SQLModelSubmissionStore:
                     status="accepted",
                     resource_type=resource_type,
                     resource_id=run_id,
-                    payload_json=json.dumps(dict(payload), sort_keys=True),
+                    payload_json=json.dumps(safe_payload, sort_keys=True),
                     run_status="accepted",
                     updated_at=created,
                     definition_id=(
-                        str(payload["definition_id"])
-                        if payload.get("definition_id") is not None
+                        str(safe_payload["definition_id"])
+                        if safe_payload.get("definition_id") is not None
                         else None
                     ),
                 )
@@ -185,7 +190,7 @@ class SQLModelSubmissionStore:
                         extensions={"idempotency_key": idempotency_key},
                     ) from exc
                 prior = json.loads(winner.payload_json)
-                if prior != dict(payload):
+                if prior != safe_payload:
                     raise ControlPlaneError.conflict(
                         "Idempotency key reuse with a different payload",
                         extensions={"idempotency_key": idempotency_key},
@@ -311,6 +316,9 @@ class SqlModelEventStore:
     ) -> ControlPlaneEvent:
         import hashlib
 
+        safe_payload = redact_control_plane_payload(deepcopy(dict(payload or {})))
+        if not isinstance(safe_payload, dict):
+            safe_payload = {}
         with session_scope(self._engine) as session:
             statement = (
                 select(EventRow)
@@ -339,7 +347,7 @@ class SqlModelEventStore:
                 cursor=cursor,
                 kind=kind,
                 created_at=created,
-                payload_json=json.dumps(dict(payload or {}), sort_keys=True),
+                payload_json=json.dumps(safe_payload, sort_keys=True),
                 correlation_id=correlation_id,
             )
             session.add(row)
@@ -350,7 +358,7 @@ class SqlModelEventStore:
                 cursor=cursor,
                 kind=kind,
                 created_at=created,
-                payload=dict(payload or {}),
+                payload=safe_payload,
                 correlation_id=correlation_id,
                 scope={
                     "tenant_id": ctx.tenant.tenant_id,

{etlantic_sqlmodel-0.40.0 → etlantic_sqlmodel-0.41.0}/src/etlantic_sqlmodel/migrations/__init__.py

@@ -14,7 +14,7 @@ from typing import Any
 from sqlalchemy import text
 from sqlalchemy.engine import Engine
 
-VERSIONS: Sequence[str] = ("001_registry_cp2",)
+VERSIONS: Sequence[str] = ("001_registry_cp2", "002_durable_cp3")
 
 
 def _load(version: str) -> Any:

etlantic_sqlmodel-0.41.0/src/etlantic_sqlmodel/migrations/versions/002_durable_cp3.py

@@ -0,0 +1,33 @@
+"""002 — CP3 durable work snapshot tables.
+
+Production note: apply via ``etlantic_sqlmodel.migrations.upgrade``.
+``create_all`` is for tests/demos only.
+"""
+
+from __future__ import annotations
+
+from sqlalchemy.engine import Engine
+
+from etlantic_sqlmodel.control_plane.models import DurableSnapshotRow
+from sqlmodel import SQLModel
+
+TABLES = (DurableSnapshotRow,)
+
+
+def upgrade(engine: Engine) -> None:
+    """Create CP3 durable tables."""
+    SQLModel.metadata.create_all(
+        engine,
+        tables=[cls.__table__ for cls in TABLES],  # type: ignore[list-item]
+    )
+
+
+def downgrade(engine: Engine) -> None:
+    """Drop CP3 durable tables."""
+    SQLModel.metadata.drop_all(
+        engine,
+        tables=[cls.__table__ for cls in reversed(TABLES)],  # type: ignore[list-item]
+    )
+
+
+__all__ = ["downgrade", "upgrade"]