serverless-data-mesh 0.2.0__py3-none-any.whl

serverless_data_mesh/local/runtime.py

@@ -0,0 +1,380 @@
+"""Run the Vaquar Pattern (PVDM) lifecycle on local disk without AWS."""
+
+from __future__ import annotations
+
+import json
+import tempfile
+import time
+from dataclasses import asdict, dataclass
+from datetime import datetime, timezone
+from pathlib import Path
+from typing import Any
+
+from serverless_data_mesh.types.workload import (
+    DataWriteWorkload,
+    DomainTransactionBoundary,
+    WriteOutcome,
+)
+from serverless_data_mesh.metrics.mesh_trust import publish_vrp_metric
+from serverless_data_mesh.orchestration.reprocess import attempt_vrp_repair
+from serverless_data_mesh.verification.backend import create_proof_generator
+from serverless_data_mesh.verification.vrp import validate_then_commit
+
+
+@dataclass(frozen=True, slots=True)
+class LocalWriteResult:
+    """Outcome of a local PVDM chunk write."""
+
+    outcome: str
+    workload_id: str
+    records_written: int
+    proof_verdict: str
+    snapshot_id: str | None
+    proof_path: str | None
+    consumer_row_count: int
+    message: str | None = None
+
+    def to_dict(self) -> dict[str, Any]:
+        return asdict(self)
+
+
+def _default_boundary() -> DomainTransactionBoundary:
+    return DomainTransactionBoundary(
+        domain_id="orders-domain",
+        source_namespace="raw_orders",
+        target_table="orders_curated",
+        partition_spec={"dt": "2026-06-14"},
+    )
+
+
+def _default_workload(root: Path, *, workload_id: str, total_records: int) -> DataWriteWorkload:
+    boundary = _default_boundary()
+    return DataWriteWorkload(
+        workload_id=workload_id,
+        boundary=boundary,
+        source_uri=f"file://{root}/source/",
+        target_uri=f"file://{root}/lakehouse/orders_curated/",
+        total_records=total_records,
+        checkpoint_bucket=str(root / "checkpoints"),
+        proof_bucket=str(root / "proofs"),
+    )
+
+
+def _records(n: int, *, corrupt_last: bool = False) -> list[dict[str, str]]:
+    rows = [{"id": str(i), "payload_hash": f"h{i}"} for i in range(n)]
+    if corrupt_last and rows:
+        rows[-1] = {"id": rows[-1]["id"], "payload_hash": "CORRUPT"}
+    return rows
+
+
+class LocalPVDMRuntime:
+    """Simulate Physical → Verify → Durable → Metadata on a laptop.
+
+    Uses veridata-recon when available; pure-Python fallback on Windows/Mac without wheels.
+    Checkpoints, proofs, and catalog snapshots are stored on local disk.
+    """
+
+    def __init__(self, root: Path | None = None) -> None:
+        self.root = root or Path(tempfile.mkdtemp(prefix="sdm-demo-"))
+        self.checkpoints = self.root / "checkpoints"
+        self.proofs = self.root / "proofs"
+        self.lakehouse = self.root / "lakehouse" / "orders_curated" / "dt=2026-06-14"
+        self.catalog = self.root / "catalog"
+        for path in (self.checkpoints, self.proofs, self.lakehouse, self.catalog):
+            path.mkdir(parents=True, exist_ok=True)
+        self._snapshot_file = self.catalog / "snapshots.json"
+        if not self._snapshot_file.exists():
+            self._snapshot_file.write_text("[]", encoding="utf-8")
+
+    @property
+    def consumer_row_count(self) -> int:
+        """Rows visible to consumers from the latest committed snapshot."""
+        snapshots = json.loads(self._snapshot_file.read_text(encoding="utf-8"))
+        if not snapshots:
+            return 0
+        return int(snapshots[-1]["row_count"])
+
+    def _persist_proof(
+        self,
+        proof: dict[str, Any],
+        *,
+        workload: DataWriteWorkload,
+        chunk_index: int,
+    ) -> Path:
+        rel = f"{workload.boundary.domain_id}/{workload.workload_id}"
+        dest_dir = self.proofs / rel
+        dest_dir.mkdir(parents=True, exist_ok=True)
+        path = dest_dir / f"chunk-{chunk_index:06d}.vrp.json"
+        path.write_text(json.dumps(proof, indent=2, sort_keys=True), encoding="utf-8")
+        return path
+
+    def _write_physical(self, records: list[dict[str, str]], *, part_name: str) -> Path:
+        part = self.lakehouse / f"{part_name}.jsonl"
+        with part.open("w", encoding="utf-8") as handle:
+            for row in records:
+                handle.write(json.dumps(row) + "\n")
+        return part
+
+    def _commit_metadata(self, *, workload: DataWriteWorkload, row_count: int, proof_id: str) -> str:
+        snapshots = json.loads(self._snapshot_file.read_text(encoding="utf-8"))
+        snapshot_id = f"snap-{len(snapshots) + 1:06d}"
+        snapshots.append(
+            {
+                "snapshot_id": snapshot_id,
+                "table": workload.boundary.target_table,
+                "partition": workload.boundary.partition_spec,
+                "row_count": row_count,
+                "proof_id": proof_id,
+                "committed_at": datetime.now(timezone.utc).replace(microsecond=0).isoformat(),
+            }
+        )
+        self._snapshot_file.write_text(json.dumps(snapshots, indent=2), encoding="utf-8")
+        checkpoint = self.checkpoints / f"{workload.workload_id}.json"
+        checkpoint.write_text(
+            json.dumps({"workload_id": workload.workload_id, "snapshot_id": snapshot_id}),
+            encoding="utf-8",
+        )
+        return snapshot_id
+
+    def run_write(
+        self,
+        *,
+        workload_id: str = "local-demo-001",
+        record_count: int = 1000,
+        corrupt_sink: bool = False,
+        proof_generator: Any | None = None,
+        defer_snapshot: bool = False,
+    ) -> LocalWriteResult:
+        """Execute one PVDM write cycle on local disk."""
+        workload = _default_workload(self.root, workload_id=workload_id, total_records=record_count)
+        if proof_generator is None:
+            gen, self._last_backend = create_proof_generator()
+        else:
+            gen = proof_generator
+            self._last_backend = getattr(gen, "producer", "custom")
+
+        source = _records(record_count)
+        sink = _records(record_count, corrupt_last=corrupt_sink)
+
+        self._write_physical(sink, part_name=f"{workload_id}-part-00000")
+
+        proof = gen.build_proof(
+            source_records=source,
+            sink_records=sink,
+            workload=workload,
+            chunk_start=0,
+            chunk_end=record_count,
+        )
+        verification = validate_then_commit(proof)
+        proof_path = self._persist_proof(proof, workload=workload, chunk_index=0)
+        verdict = proof["reconciliation"]["verdict"]
+        publish_vrp_metric(
+            domain_id=workload.boundary.domain_id,
+            verdict=verdict,
+            row_count=record_count,
+            workload_id=workload_id,
+        )
+
+        if verification.outcome != "PASS":
+            return LocalWriteResult(
+                outcome=WriteOutcome.VERIFICATION_FAILED.value,
+                workload_id=workload_id,
+                records_written=0,
+                proof_verdict=verdict,
+                snapshot_id=None,
+                proof_path=str(proof_path),
+                consumer_row_count=self.consumer_row_count,
+                message=verification.reason,
+            )
+
+        if defer_snapshot:
+            pending = self.catalog / "pending.json"
+            pending_rows = []
+            if pending.exists():
+                pending_rows = json.loads(pending.read_text(encoding="utf-8"))
+            pending_rows.append(
+                {
+                    "workload_id": workload_id,
+                    "row_count": record_count,
+                    "proof_id": proof["proof_id"],
+                }
+            )
+            pending.write_text(json.dumps(pending_rows, indent=2), encoding="utf-8")
+            return LocalWriteResult(
+                outcome=WriteOutcome.COMMITTED.value,
+                workload_id=workload_id,
+                records_written=record_count,
+                proof_verdict=verdict,
+                snapshot_id=None,
+                proof_path=str(proof_path),
+                consumer_row_count=self.consumer_row_count,
+                message="VRP PASS; snapshot deferred for mesh leader commit",
+            )
+
+        snapshot_id = self._commit_metadata(
+            workload=workload,
+            row_count=record_count,
+            proof_id=proof["proof_id"],
+        )
+        return LocalWriteResult(
+            outcome=WriteOutcome.COMMITTED.value,
+            workload_id=workload_id,
+            records_written=record_count,
+            proof_verdict=verdict,
+            snapshot_id=snapshot_id,
+            proof_path=str(proof_path),
+            consumer_row_count=self.consumer_row_count,
+        )
+
+    def finalize_mesh_transaction(self, domain_results: list[LocalWriteResult]) -> dict[str, Any]:
+        """Leader commit: all domains must VRP PASS or no consumer snapshot."""
+        pending_file = self.catalog / "pending.json"
+        if any(r.outcome != WriteOutcome.COMMITTED.value for r in domain_results):
+            if pending_file.exists():
+                pending_file.unlink()
+            return {
+                "mesh_outcome": WriteOutcome.VERIFICATION_FAILED.value,
+                "consumer_row_count": self.consumer_row_count,
+                "message": "At least one domain failed VRP; pending snapshots discarded",
+            }
+
+        if not pending_file.exists():
+            return {
+                "mesh_outcome": WriteOutcome.VERIFICATION_FAILED.value,
+                "consumer_row_count": self.consumer_row_count,
+                "message": "No pending domain writes",
+            }
+
+        pending_rows = json.loads(pending_file.read_text(encoding="utf-8"))
+        total_rows = sum(int(row["row_count"]) for row in pending_rows)
+        proof_id = pending_rows[-1]["proof_id"]
+        workload = _default_workload(self.root, workload_id="mesh-txn", total_records=total_rows)
+        snapshot_id = self._commit_metadata(
+            workload=workload,
+            row_count=total_rows,
+            proof_id=proof_id,
+        )
+        pending_file.unlink()
+        return {
+            "mesh_outcome": WriteOutcome.COMMITTED.value,
+            "snapshot_id": snapshot_id,
+            "consumer_row_count": self.consumer_row_count,
+            "domains_committed": len(domain_results),
+        }
+
+    def run_write_with_auto_repair(
+        self,
+        *,
+        workload_id: str = "auto-repair-demo",
+        record_count: int = 100,
+        drop_count: int = 5,
+        proof_generator: Any | None = None,
+    ) -> dict[str, Any]:
+        """Simulate dropped records, auto-repair via VRP reprocessing, then commit."""
+        workload = _default_workload(self.root, workload_id=workload_id, total_records=record_count)
+        if proof_generator is None:
+            gen, backend = create_proof_generator()
+        else:
+            gen = proof_generator
+            backend = getattr(gen, "producer", "custom")
+
+        source = _records(record_count)
+        sink = _records(record_count - drop_count)
+        repaired_sink = list(sink)
+
+        def _merge_missing(missing: list[dict[str, str]]) -> list[dict[str, str]]:
+            nonlocal repaired_sink
+            repaired_sink = repaired_sink + missing
+            return repaired_sink
+
+        repair = attempt_vrp_repair(
+            source_records=source,
+            sink_records=sink,
+            workload=workload,
+            chunk_start=0,
+            chunk_end=record_count,
+            proof_generator=gen,
+            write_repair_fn=_merge_missing,
+        )
+
+        if repair.outcome != "repaired_pass" or repair.proof is None:
+            return {
+                "outcome": repair.outcome,
+                "backend": backend,
+                "repair": {
+                    "outcome": repair.outcome,
+                    "attempts": repair.attempts,
+                    "missing_before": repair.missing_before,
+                    "missing_after": repair.missing_after,
+                    "message": repair.message,
+                },
+                "consumer_row_count": self.consumer_row_count,
+            }
+
+        self._write_physical(repaired_sink, part_name=f"{workload_id}-repaired")
+        proof_path = self._persist_proof(repair.proof, workload=workload, chunk_index=0)
+        publish_vrp_metric(
+            domain_id=workload.boundary.domain_id,
+            verdict="PASS",
+            row_count=record_count,
+            workload_id=workload_id,
+        )
+        snapshot_id = self._commit_metadata(
+            workload=workload,
+            row_count=record_count,
+            proof_id=repair.proof["proof_id"],
+        )
+        return {
+            "outcome": "repaired_and_committed",
+            "backend": backend,
+            "repair": {
+                "outcome": repair.outcome,
+                "attempts": repair.attempts,
+                "missing_before": repair.missing_before,
+                "missing_after": repair.missing_after,
+                "message": repair.message,
+            },
+            "snapshot_id": snapshot_id,
+            "proof_path": str(proof_path),
+            "consumer_row_count": self.consumer_row_count,
+        }
+
+    def run_demo_sequence(self) -> dict[str, Any]:
+        """Run clean write, corrupt write, and consumer visibility check."""
+        gen, backend = create_proof_generator()
+
+        started = time.perf_counter()
+        clean = self.run_write(
+            workload_id="demo-clean",
+            record_count=1000,
+            corrupt_sink=False,
+            proof_generator=gen,
+        )
+        corrupt = self.run_write(
+            workload_id="demo-corrupt",
+            record_count=1000,
+            corrupt_sink=True,
+            proof_generator=gen,
+        )
+        elapsed_ms = round((time.perf_counter() - started) * 1000, 1)
+
+        return {
+            "mode": "local-pvdm",
+            "verifier_backend": backend,
+            "root": str(self.root),
+            "elapsed_ms": elapsed_ms,
+            "phases": {
+                "clean_write": clean.to_dict(),
+                "corrupt_write": corrupt.to_dict(),
+            },
+            "consumer": {
+                "visible_row_count": self.consumer_row_count,
+                "corrupt_data_visible": corrupt.outcome == WriteOutcome.COMMITTED.value,
+                "gate_blocked_bad_data": corrupt.outcome == WriteOutcome.VERIFICATION_FAILED.value,
+            },
+            "summary": (
+                "VRP gate blocked corrupt write; consumers see only clean snapshot."
+                if corrupt.outcome == WriteOutcome.VERIFICATION_FAILED.value
+                else "Unexpected: corrupt write committed."
+            ),
+        }

serverless_data_mesh/metrics/__init__.py

@@ -0,0 +1,5 @@
+"""Mesh trust metrics for CloudWatch and Grafana."""
+
+from serverless_data_mesh.metrics.mesh_trust import publish_vrp_metric
+
+__all__ = ["publish_vrp_metric"]

serverless_data_mesh/metrics/mesh_trust.py

@@ -0,0 +1,56 @@
+"""Publish VRP trust metrics to CloudWatch for live dashboards."""
+
+from __future__ import annotations
+
+import logging
+import os
+from typing import Any
+
+logger = logging.getLogger(__name__)
+
+NAMESPACE = "ServerlessDataMesh/Trust"
+
+
+def publish_vrp_metric(
+    *,
+    domain_id: str,
+    verdict: str,
+    row_count: int = 0,
+    workload_id: str | None = None,
+    cloudwatch_client: Any | None = None,
+) -> None:
+    """Emit VRP PASS/FAIL metric for CloudWatch / Grafana dashboards."""
+    if os.environ.get("SDM_DISABLE_METRICS", "").lower() in ("1", "true", "yes"):
+        return
+
+    try:
+        import boto3
+    except ImportError:
+        logger.debug("boto3 unavailable; skip CloudWatch metric")
+        return
+
+    client = cloudwatch_client or boto3.client("cloudwatch")
+    value = 1.0 if verdict == "PASS" else 0.0
+
+    dimensions = [{"Name": "Domain", "Value": domain_id}]
+    if workload_id:
+        dimensions.append({"Name": "WorkloadId", "Value": workload_id})
+
+    client.put_metric_data(
+        Namespace=NAMESPACE,
+        MetricData=[
+            {
+                "MetricName": "VRPTrustScore",
+                "Dimensions": dimensions,
+                "Value": value,
+                "Unit": "None",
+            },
+            {
+                "MetricName": "VRPRowCount",
+                "Dimensions": dimensions,
+                "Value": float(row_count),
+                "Unit": "Count",
+            },
+        ],
+    )
+    logger.info("Published VRP metric domain=%s verdict=%s", domain_id, verdict)

serverless_data_mesh/orchestration/__init__.py

@@ -0,0 +1,28 @@
+"""Durable orchestration bridging IceGuard and AWS Lambda Durable Execution."""
+
+from __future__ import annotations
+
+from typing import Any
+
+__all__ = [
+    "IceGuardDurableCoordinator",
+    "OrchestrationState",
+    "durable_commit_metadata",
+    "durable_write_chunk",
+]
+
+
+def __getattr__(name: str) -> Any:
+    if name == "IceGuardDurableCoordinator":
+        from serverless_data_mesh.orchestration.coordinator import IceGuardDurableCoordinator
+
+        return IceGuardDurableCoordinator
+    if name == "OrchestrationState":
+        from serverless_data_mesh.orchestration.state import OrchestrationState
+
+        return OrchestrationState
+    if name in ("durable_commit_metadata", "durable_write_chunk"):
+        from serverless_data_mesh.orchestration import durable_steps
+
+        return getattr(durable_steps, name)
+    raise AttributeError(f"module {__name__!r} has no attribute {name!r}")

serverless_data_mesh/orchestration/canary.py

@@ -0,0 +1,127 @@
+"""Canary write comparison using VRP proof divergence (roadmap 11/10)."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from typing import Any, Callable
+
+from serverless_data_mesh.verification.backend import create_proof_generator
+from serverless_data_mesh.verification.vrp import validate_then_commit
+
+
+@dataclass(frozen=True, slots=True)
+class CanaryResult:
+    outcome: str  # canary_approved | canary_diverged
+    production_verdict: str
+    canary_verdict: str
+    divergence_pct: float
+    message: str
+
+
+def run_canary_comparison(
+    *,
+    production_source: list[dict[str, Any]],
+    production_sink: list[dict[str, Any]],
+    canary_sink: list[dict[str, Any]],
+    workload_factory: Callable[[], Any],
+    max_divergence_pct: float = 1.0,
+) -> CanaryResult:
+    """Compare production vs canary VRP proofs before promoting a schema/logic change."""
+    gen, _ = create_proof_generator()
+    workload = workload_factory()
+    n = len(production_source)
+
+    prod_proof = gen.build_proof(
+        source_records=production_source,
+        sink_records=production_sink,
+        workload=workload,
+        chunk_start=0,
+        chunk_end=n,
+    )
+    canary_proof = gen.build_proof(
+        source_records=production_source,
+        sink_records=canary_sink,
+        workload=workload,
+        chunk_start=0,
+        chunk_end=n,
+    )
+    prod_v = validate_then_commit(prod_proof).outcome
+    canary_v = validate_then_commit(canary_proof).outcome
+
+    if prod_v != "PASS" or canary_v != "PASS":
+        return CanaryResult(
+            outcome="canary_diverged",
+            production_verdict=prod_v,
+            canary_verdict=canary_v,
+            divergence_pct=100.0,
+            message="One or both proofs failed VRP",
+        )
+
+    prod_count = prod_proof["reconciliation"].get("sink_count", n)
+    canary_count = canary_proof["reconciliation"].get("sink_count", n)
+    divergence = abs(canary_count - prod_count) / max(prod_count, 1) * 100
+
+    if divergence > max_divergence_pct:
+        return CanaryResult(
+            outcome="canary_diverged",
+            production_verdict=prod_v,
+            canary_verdict=canary_v,
+            divergence_pct=round(divergence, 2),
+            message=f"Row count divergence {divergence:.2f}% exceeds {max_divergence_pct}%",
+        )
+
+    return CanaryResult(
+        outcome="canary_approved",
+        production_verdict=prod_v,
+        canary_verdict=canary_v,
+        divergence_pct=round(divergence, 2),
+        message="Canary within tolerance",
+    )
+
+
+def run_canary(
+    *,
+    record_count: int = 1000,
+    inject_canary_drift: bool = False,
+    max_divergence_pct: float = 1.0,
+) -> dict[str, object]:
+    """End-to-end canary promotion check with sample production vs canary sinks."""
+    from serverless_data_mesh.types.workload import DataWriteWorkload, DomainTransactionBoundary
+
+    source = [{"id": str(i), "payload_hash": f"h{i}"} for i in range(record_count)]
+    production_sink = list(source)
+    canary_sink = list(source)
+    if inject_canary_drift:
+        canary_sink = canary_sink[: int(record_count * 0.98)]
+
+    boundary = DomainTransactionBoundary(
+        domain_id="canary-demo",
+        source_namespace="raw_canary",
+        target_table="orders_curated",
+        partition_spec={"dt": "2026-06-14"},
+    )
+    workload = DataWriteWorkload(
+        workload_id="canary-001",
+        boundary=boundary,
+        source_uri="s3://publisher/source/",
+        target_uri="s3://publisher/lakehouse/orders_curated/",
+        total_records=record_count,
+        checkpoint_bucket="s3://steward/checkpoints",
+        proof_bucket="s3://steward/proofs",
+    )
+
+    result = run_canary_comparison(
+        production_source=source,
+        production_sink=production_sink,
+        canary_sink=canary_sink,
+        workload_factory=lambda: workload,
+        max_divergence_pct=max_divergence_pct,
+    )
+    return {
+        "outcome": result.outcome,
+        "production_verdict": result.production_verdict,
+        "canary_verdict": result.canary_verdict,
+        "divergence_pct": result.divergence_pct,
+        "message": result.message,
+        "promote": result.outcome == "canary_approved",
+    }