nl2data-semantic-catalog-postgres 0.1.0__py3-none-any.whl

nl2data_semantic_catalog_postgres/fake_postgres/pool.py

@@ -0,0 +1,152 @@
+"""Thread-safe in-memory pool executing the catalog's SQL statement set.
+
+Expose ``snapshots``/``snapshot_pointers``/``proposal_sets``/``publications``/
+``bundle_pointers``/``bundle_history``/``events``/``clock`` for direct
+assertions in tests.  Statement dispatch goes through the domain handler
+registry in :mod:`.registry`; unrecognized statements fail loudly with
+``AssertionError``.
+"""
+
+from __future__ import annotations
+
+import contextlib
+import threading
+from collections.abc import Iterator
+from datetime import datetime
+from typing import Any
+
+from ..sql import SQL_TEMPLATES
+from .driver import (
+    FakeClock,
+    OperationalError,
+    PoolClosed,
+    _FakeConnection,
+    _FakeCursor,
+    _monotonic,
+    _normalize,
+    _PassFailure,
+)
+from .registry import HANDLERS
+
+
+class FakePostgresPool:
+    """Thread-safe in-memory pool executing the catalog's SQL statement set."""
+
+    def __init__(self, *, start: datetime | None = None) -> None:
+        self.clock = FakeClock(start)
+        self.snapshots: dict[tuple[str, str], dict[str, Any]] = {}
+        self.snapshot_pointers: dict[tuple[str, str], dict[str, Any]] = {}
+        self.proposal_sets: dict[tuple[str, str], dict[str, Any]] = {}
+        self.assembly_drafts: dict[tuple[str, str], dict[str, Any]] = {}
+        self.publications: dict[tuple[str, str, str], dict[str, Any]] = {}
+        self.accepted_manifests: dict[tuple[str, str, str], dict[str, Any]] = {}
+        self.verification_evidence: dict[tuple[str, str, str], dict[str, Any]] = {}
+        self.publish_audits: dict[tuple[str, str, str], dict[str, Any]] = {}
+        self.published_versions: dict[tuple[str, str, str], dict[str, Any]] = {}
+        self.supersession_edges: dict[tuple[str, str, str], dict[str, Any]] = {}
+        self.bundle_pointers: dict[tuple[str, str], dict[str, Any]] = {}
+        self.bundle_history: dict[tuple[str, str], dict[int, dict[str, Any]]] = {}
+        self.events: dict[tuple[str, str], dict[str, Any]] = {}
+        self.publication_audit_evidence: dict[
+            tuple[str, str, str], dict[str, Any]
+        ] = {}
+        self.audit_entries: dict[tuple[str, str], dict[str, Any]] = {}
+        self.schema_metadata: dict[str, str] = {}
+        self._closed = False
+        self._struct_lock = threading.RLock()
+        self._lock_owner: dict[tuple[Any, ...], str] = {}
+        self._lock_cond = threading.Condition(self._struct_lock)
+        self._pending_failures: list[BaseException | _PassFailure] = []
+        self.statement_journal: list[str] = []
+        self._statement_names = {
+            _normalize(template.format(schema='"catalog"')): name
+            for name, template in SQL_TEMPLATES.items()
+        }
+
+    # -- lifecycle --------------------------------------------------------
+
+    @contextlib.contextmanager
+    def connection(self) -> Iterator[_FakeConnection]:
+        """One transaction-backed connection; ``commit``/``rollback`` explicit."""
+        with self._struct_lock:
+            if self._closed:
+                raise PoolClosed("fake pool is closed")
+            connection = _FakeConnection(self)
+        try:
+            yield connection
+        finally:
+            connection._finish()
+
+    def close(self) -> None:
+        """Close the pool; later checkouts fail with ``PoolClosed``."""
+        with self._struct_lock:
+            self._closed = True
+
+    def set_schema_version(self, version: int) -> None:
+        """Seed ``schema_metadata`` so catalog initialization sees a version."""
+        with self._struct_lock:
+            self.schema_metadata["schema_version"] = str(version)
+
+    def fail_next(
+        self, error: BaseException, *, count: int = 1, after: int = 0
+    ) -> None:
+        """Fail the next ``count`` statements, after ``after`` pass-throughs.
+
+        ``after`` lets a multi-statement operation commit earlier writes and
+        then fail mid-transaction, so store rollback of partial work can be
+        asserted deterministically.
+        """
+        with self._struct_lock:
+            self._pending_failures.extend([_PassFailure()] * after)
+            self._pending_failures.extend([error] * count)
+
+    # -- row locking ------------------------------------------------------
+
+    def _lock(
+        self, connection: _FakeConnection, key: tuple[Any, ...], timeout: float
+    ) -> None:
+        """Acquire a row lock, blocking while another connection holds it."""
+        deadline = _monotonic() + timeout
+        with self._lock_cond:
+            while self._lock_owner.get(key) not in (None, connection.id):
+                remaining = deadline - _monotonic()
+                if remaining <= 0:
+                    raise OperationalError("lock wait timeout exceeded")
+                self._lock_cond.wait(remaining)
+            self._lock_owner[key] = connection.id
+            connection._owned_keys.add(key)
+
+    def _release(self, connection: _FakeConnection) -> None:
+        """Release every row lock owned by the connection."""
+        with self._lock_cond:
+            for key in connection._owned_keys:
+                if self._lock_owner.get(key) == connection.id:
+                    del self._lock_owner[key]
+            connection._owned_keys.clear()
+            self._lock_cond.notify_all()
+
+    # -- statement dispatch -----------------------------------------------
+
+    def _execute(
+        self,
+        connection: _FakeConnection,
+        sql: str,
+        params: tuple[Any, ...],
+        timeout: float,
+    ) -> _FakeCursor:
+        with self._struct_lock:
+            while self._pending_failures:
+                failure = self._pending_failures.pop(0)
+                if not isinstance(failure, _PassFailure):
+                    raise failure
+        normalized = _normalize(sql)
+        if normalized.startswith("CREATE "):
+            # DDL is emulated: tables exist by construction.
+            return _FakeCursor(self, connection, [], 0)
+        name = self._statement_names.get(normalized)
+        if name is None:
+            raise AssertionError(f"fake pool does not recognize statement: {sql[:240]}")
+        self.statement_journal.append(name)
+        handler = HANDLERS[name]
+        rows, rowcount = handler(self, connection, params or (), timeout)
+        return _FakeCursor(self, connection, rows, rowcount)

nl2data_semantic_catalog_postgres/fake_postgres/registry.py

@@ -0,0 +1,144 @@
+"""Statement-name to handler registry for the fake pool.
+
+Each entry maps one production SQL template name (the exact key set of
+:data:`SQL_TEMPLATES`) to its domain handler, so template drift between
+the store and the fake is caught at the first unrecognized statement.
+"""
+
+from __future__ import annotations
+
+from collections.abc import Callable
+from typing import Any
+
+from .handlers_audit import (
+    _h_count_audit_evidence,
+    _h_insert_audit_entry,
+    _h_insert_publication_audit_evidence,
+    _h_list_audit_evidence,
+    _h_read_audit_entry,
+    _h_read_latest_publication_entry,
+    _h_read_publication_audit_entry,
+    _h_read_publication_audit_evidence,
+)
+from .handlers_drafts import (
+    _h_insert_assembly_draft,
+    _h_lock_assembly_draft,
+    _h_read_assembly_draft,
+    _h_replace_assembly_draft,
+)
+from .handlers_maintenance import (
+    _h_delete_expired_audit_entries,
+    _h_delete_expired_events,
+    _h_delete_expired_publications,
+    _h_delete_expired_snapshots,
+    _h_insert_event,
+)
+from .handlers_publications import (
+    _h_insert_accepted_manifest,
+    _h_insert_publication,
+    _h_insert_publish_audit,
+    _h_insert_verification_evidence,
+    _h_list_publications,
+    _h_lock_publication_series,
+    _h_read_accepted_manifest,
+    _h_read_publication,
+    _h_read_publication_by_fingerprint,
+    _h_read_publication_fingerprint,
+    _h_read_publish_audit,
+    _h_read_publish_by_idempotency_key,
+    _h_read_verification_evidence,
+)
+from .handlers_schema import _h_read_schema_version, _h_write_schema_version
+from .handlers_snapshots import (
+    _h_list_snapshot_pointers,
+    _h_lock_snapshot_row,
+    _h_read_proposal_set,
+    _h_read_snapshot_envelope,
+    _h_read_snapshot_pointer,
+    _h_set_snapshot_state,
+    _h_snapshot_exists,
+    _h_upsert_proposal_set,
+    _h_upsert_snapshot,
+    _h_upsert_snapshot_pointer,
+)
+from .handlers_versions import (
+    _h_delete_history_top,
+    _h_insert_history,
+    _h_insert_published_version,
+    _h_insert_supersession_edge,
+    _h_list_bundle_pointers,
+    _h_list_orphan_active_versions,
+    _h_list_published_versions,
+    _h_lock_bundle_pointer,
+    _h_next_history_position,
+    _h_read_bundle_pointer,
+    _h_read_history_top,
+    _h_read_latest_version,
+    _h_read_published_version,
+    _h_set_published_version_state,
+    _h_trim_history,
+    _h_update_version_successor,
+    _h_upsert_bundle_pointer,
+)
+
+HANDLERS: dict[str, Callable[..., tuple[list[dict[str, Any]], int]]] = {
+    "read_schema_version": _h_read_schema_version,
+    "write_schema_version": _h_write_schema_version,
+    "upsert_snapshot": _h_upsert_snapshot,
+    "read_snapshot_envelope": _h_read_snapshot_envelope,
+    "lock_snapshot_row": _h_lock_snapshot_row,
+    "set_snapshot_state": _h_set_snapshot_state,
+    "snapshot_exists": _h_snapshot_exists,
+    "upsert_snapshot_pointer": _h_upsert_snapshot_pointer,
+    "read_snapshot_pointer": _h_read_snapshot_pointer,
+    "list_snapshot_pointers": _h_list_snapshot_pointers,
+    "upsert_proposal_set": _h_upsert_proposal_set,
+    "read_proposal_set": _h_read_proposal_set,
+    "insert_assembly_draft": _h_insert_assembly_draft,
+    "read_assembly_draft": _h_read_assembly_draft,
+    "lock_assembly_draft": _h_lock_assembly_draft,
+    "replace_assembly_draft": _h_replace_assembly_draft,
+    "insert_publication": _h_insert_publication,
+    "read_publication": _h_read_publication,
+    "read_publication_fingerprint": _h_read_publication_fingerprint,
+    "read_publication_by_fingerprint": _h_read_publication_by_fingerprint,
+    "lock_publication_series": _h_lock_publication_series,
+    "list_publications": _h_list_publications,
+    "insert_accepted_manifest": _h_insert_accepted_manifest,
+    "read_accepted_manifest": _h_read_accepted_manifest,
+    "insert_verification_evidence": _h_insert_verification_evidence,
+    "read_verification_evidence": _h_read_verification_evidence,
+    "insert_publish_audit": _h_insert_publish_audit,
+    "read_publish_audit": _h_read_publish_audit,
+    "read_publish_by_idempotency_key": _h_read_publish_by_idempotency_key,
+    "read_latest_version": _h_read_latest_version,
+    "insert_published_version": _h_insert_published_version,
+    "update_version_successor": _h_update_version_successor,
+    "insert_supersession_edge": _h_insert_supersession_edge,
+    "read_published_version": _h_read_published_version,
+    "list_published_versions": _h_list_published_versions,
+    "set_published_version_state": _h_set_published_version_state,
+    "upsert_bundle_pointer": _h_upsert_bundle_pointer,
+    "read_bundle_pointer": _h_read_bundle_pointer,
+    "lock_bundle_pointer": _h_lock_bundle_pointer,
+    "next_history_position": _h_next_history_position,
+    "insert_history": _h_insert_history,
+    "read_history_top": _h_read_history_top,
+    "delete_history_top": _h_delete_history_top,
+    "trim_history": _h_trim_history,
+    "list_bundle_pointers": _h_list_bundle_pointers,
+    "list_orphan_active_versions": _h_list_orphan_active_versions,
+    "insert_event": _h_insert_event,
+    "insert_publication_audit_evidence": _h_insert_publication_audit_evidence,
+    "read_publication_audit_evidence": _h_read_publication_audit_evidence,
+    "insert_audit_entry": _h_insert_audit_entry,
+    "read_audit_entry": _h_read_audit_entry,
+    "read_publication_audit_entry": _h_read_publication_audit_entry,
+    "read_latest_publication_entry": _h_read_latest_publication_entry,
+    "count_audit_evidence": _h_count_audit_evidence,
+    "list_audit_evidence": _h_list_audit_evidence,
+    "delete_expired_snapshots": _h_delete_expired_snapshots,
+    "delete_expired_publications": _h_delete_expired_publications,
+    "delete_expired_events": _h_delete_expired_events,
+    "delete_expired_audit_entries": _h_delete_expired_audit_entries,
+}

nl2data_semantic_catalog_postgres/maintenance.py

@@ -0,0 +1,235 @@
+"""Bounded maintenance and startup revalidation for the durable catalog.
+
+The store facade delegates these cross-domain sweeps here: ``cleanup``
+removes expired inactive records in bounded batches while preserving
+active content and required dependencies, and ``reload_active``
+revalidates every active snapshot/Bundle pointer after startup so a
+newer persisted schema or envelope version fails closed.
+"""
+
+from __future__ import annotations
+
+from datetime import datetime, timedelta
+
+from nl2data_core.bundles.publication import (
+    LifecycleWitnessError,
+    PublishedVersionState,
+    validate_lifecycle_witness,
+    witness_cause_type,
+)
+from nl2data_core.metadata.catalog import CatalogReloadIssue, CatalogReloadReport
+
+from .envelope import ArtifactKind
+from .errors import SemanticCatalogError, SemanticCatalogErrorCode
+from .repositories.evidence import EvidenceRepository
+from .unit_of_work import CatalogUnitOfWork
+
+__all__ = ["cleanup", "reload_active"]
+
+
+def cleanup(uow: CatalogUnitOfWork, *, now: datetime | None = None) -> int:
+    """Remove expired inactive records; preserves active content.
+
+    Expired snapshots that no pointer references, expired publications
+    that neither pointer nor history references (and that no publication
+    depends on), expired lifecycle events, and expired audit-evidence
+    entries (unless their bundle fingerprint belongs to a non-retired
+    published version, or a protected entry references them as a
+    predecessor) are removed in bounded batches.  Active snapshots,
+    active Bundles, and required dependencies are never removed.
+    """
+    config = uow.config
+    current = uow.now() if now is None else now
+    event_cutoff = current - timedelta(seconds=config.event_retention_seconds)
+    audit_cutoff = current - timedelta(seconds=config.audit_retention_seconds)
+    total = 0
+    with uow.transaction() as conn:
+        cursor = uow.execute(
+            conn,
+            "delete_expired_snapshots",
+            (current, config.cleanup_batch_size),
+        )
+        total += int(cursor.rowcount or 0)
+        cursor = uow.execute(
+            conn,
+            "delete_expired_publications",
+            (current, config.cleanup_batch_size),
+        )
+        total += int(cursor.rowcount or 0)
+        cursor = uow.execute(
+            conn,
+            "delete_expired_events",
+            (event_cutoff, config.cleanup_batch_size),
+        )
+        total += int(cursor.rowcount or 0)
+        cursor = uow.execute(
+            conn,
+            "delete_expired_audit_entries",
+            (audit_cutoff, config.cleanup_batch_size),
+        )
+        total += int(cursor.rowcount or 0)
+        uow.insert_event(
+            conn,
+            "cleanup",
+            None,
+            namespace="",
+            occurred_at=current,
+        )
+    return total
+
+
+def reload_active(uow: CatalogUnitOfWork, *, now: datetime | None = None) -> CatalogReloadReport:
+    """Revalidate every active snapshot/Bundle pointer after startup.
+
+    A newer persisted schema or envelope version fails closed; active
+    pointers whose artifact no longer revalidates are reported as
+    rejected and are never exposed for query-time resolution (reads
+    revalidate independently and fail closed).  When an active Bundle
+    publication carries verification evidence, its immutable frozen
+    binding, audit, and manifest cross-links are revalidated too, so a
+    corrupted or legacy evidence record is reported as rejected instead
+    of silently passing startup revalidation.
+    """
+    checked_at = uow.now() if now is None else now
+    issues: list[CatalogReloadIssue] = []
+    snapshots_revalidated = 0
+    bundles_revalidated = 0
+    evidence = EvidenceRepository(uow)
+    with uow.transaction() as conn:
+        for pointer in uow.execute(conn, "list_snapshot_pointers").fetchall():
+            if len(issues) >= 16:
+                break
+            row = uow.execute(
+                conn,
+                "read_snapshot_envelope",
+                (
+                    pointer["scope_namespace"],
+                    pointer["snapshot_fingerprint"],
+                ),
+            ).fetchone()
+            try:
+                if row is None:
+                    raise SemanticCatalogError(
+                        SemanticCatalogErrorCode.ENVELOPE_REJECTED,
+                        "active snapshot artifact is missing",
+                        details={"cause_type": "MissingArtifact"},
+                    )
+                envelope = uow.decode(
+                    row["envelope"],
+                    ArtifactKind.SNAPSHOT,
+                    row_schema_version=row["schema_version"],
+                )
+                uow.snapshot_from_envelope(
+                    envelope, discovered_at=row["discovered_at"]
+                )
+                snapshots_revalidated += 1
+            except SemanticCatalogError as error:
+                issues.append(
+                    CatalogReloadIssue(
+                        code=error.code.value,
+                        message=error.message,
+                        member_id=pointer["source_id"],
+                    )
+                )
+        for pointer in uow.execute(conn, "list_bundle_pointers").fetchall():
+            if len(issues) >= 16:
+                break
+            row = uow.execute(
+                conn,
+                "read_publication",
+                (
+                    pointer["scope_namespace"],
+                    pointer["bundle_id"],
+                    pointer["model_version"],
+                ),
+            ).fetchone()
+            try:
+                if row is None:
+                    raise SemanticCatalogError(
+                        SemanticCatalogErrorCode.ENVELOPE_REJECTED,
+                        "active bundle artifact is missing",
+                        details={"cause_type": "MissingArtifact"},
+                    )
+                envelope = uow.decode(
+                    row["envelope"],
+                    ArtifactKind.BUNDLE,
+                    row_schema_version=row["schema_version"],
+                )
+                bundle = uow.bundle_from_envelope(envelope)
+                version_row = uow.execute(
+                    conn,
+                    "read_published_version",
+                    (
+                        pointer["scope_namespace"],
+                        pointer["bundle_id"],
+                        bundle.fingerprint,
+                    ),
+                ).fetchone()
+                if version_row is None:
+                    # The pointer is a witness that this version was
+                    # activated; a publication without its lifecycle row
+                    # is corruption.
+                    raise SemanticCatalogError(
+                        SemanticCatalogErrorCode.ENVELOPE_REJECTED,
+                        "active publication has no published version record",
+                        details={"cause_type": "PublicationVersionMissing"},
+                    )
+                try:
+                    # The pointer carries redundant fingerprint and version
+                    # witnesses and must agree with an ACTIVE version row.
+                    validate_lifecycle_witness(
+                        bundle,
+                        witness="pointer",
+                        witness_fingerprint=pointer["bundle_fingerprint"],
+                        witness_model_version=pointer["model_version"],
+                        lifecycle_state=PublishedVersionState(
+                            version_row["lifecycle_state"]
+                        ),
+                        require_state=PublishedVersionState.ACTIVE,
+                    )
+                except LifecycleWitnessError as error:
+                    raise SemanticCatalogError(
+                        SemanticCatalogErrorCode.ENVELOPE_REJECTED,
+                        error.message,
+                        details={"cause_type": witness_cause_type(error.code)},
+                    ) from error
+                # Active publications must still satisfy their immutable
+                # frozen binding, audit, and manifest cross-links through
+                # the centralized integrity rule set; legacy compatibility
+                # records (plain Bundle, manifest-only) revalidate through
+                # the Bundle envelope and manifest bundle-match alone.
+                evidence.validated_publication_records(
+                    conn,
+                    pointer["scope_namespace"],
+                    pointer["bundle_id"],
+                    bundle.fingerprint,
+                    audit_id=version_row["audit_id"],
+                )
+                bundles_revalidated += 1
+            except SemanticCatalogError as error:
+                issues.append(
+                    CatalogReloadIssue(
+                        code=error.code.value,
+                        message=error.message,
+                        member_id=pointer["bundle_id"],
+                    )
+                )
+        # An ACTIVE version row without a pointer is invisible to the
+        # pointer sweep above; report it so orphaned lifecycle rows are
+        # never silently carried across a restart.
+        for row in uow.execute(conn, "list_orphan_active_versions", ()).fetchall():
+            if len(issues) >= 16:
+                break
+            issues.append(
+                CatalogReloadIssue(
+                    code="orphan_active_version",
+                    message="an active version exists without an active pointer",
+                    member_id=row["bundle_id"],
+                )
+            )
+    return CatalogReloadReport(
+        checked_at=checked_at,
+        active_snapshots_revalidated=snapshots_revalidated,
+        active_bundles_revalidated=bundles_revalidated,
+        rejected=tuple(issues[:16]),
+    )

nl2data_semantic_catalog_postgres/repositories/__init__.py

@@ -0,0 +1,26 @@
+"""Focused repositories over the shared catalog unit of work.
+
+Repositories persist one domain each (metadata snapshots/proposal sets,
+assembly drafts, Bundle publications, verification/audit evidence, and
+activation/history).  They operate on connections handed to them by a
+transaction owner and never commit independently during atomic
+cross-domain operations such as publication, activation, or rollback.
+"""
+
+from __future__ import annotations
+
+from .activation import ActivationRepository
+from .audit_evidence import AuditEvidenceRepository
+from .drafts import DraftRepository
+from .evidence import EvidenceRepository
+from .publications import PublicationRepository
+from .snapshots import SnapshotRepository
+
+__all__ = [
+    "ActivationRepository",
+    "AuditEvidenceRepository",
+    "DraftRepository",
+    "EvidenceRepository",
+    "PublicationRepository",
+    "SnapshotRepository",
+]