serverless-data-mesh 0.2.0__py3-none-any.whl

serverless_data_mesh/__init__.py

@@ -0,0 +1,93 @@
+"""Serverless Data Mesh: cross-domain lakehouse write coordination on AWS Lambda."""
+
+from __future__ import annotations
+
+from pathlib import Path
+from typing import Any
+
+from serverless_data_mesh.config import MeshSettings
+from serverless_data_mesh.exceptions import (
+    CatalogCommitError,
+    RuleEvaluationError,
+    ServerlessDataMeshError,
+    VerificationRejectedError,
+    WorkloadConfigurationError,
+)
+from serverless_data_mesh.types import (
+    ChunkWriteResult,
+    DataProductContract,
+    DataWriteWorkload,
+    DomainTransactionBoundary,
+    WriteOutcome,
+)
+
+def _read_version() -> str:
+    try:
+        from importlib.metadata import version
+
+        return version("serverless-data-mesh")
+    except Exception:
+        root = Path(__file__).resolve().parents[2]
+        return (root / "VERSION").read_text(encoding="utf-8").strip()
+
+
+__version__ = _read_version()
+
+__all__ = [
+    "CatalogCommitError",
+    "ChunkWriteResult",
+    "DataProductContract",
+    "DataWriteWorkload",
+    "DomainTransactionBoundary",
+    "GlueCatalogConnector",
+    "GlueRestCatalogAdapter",
+    "IceGuardDurableCoordinator",
+    "MeshSettings",
+    "OrchestrationState",
+    "RuleEvaluationError",
+    "RuleFireSummary",
+    "SparkRulesConnector",
+    "ServerlessDataMeshError",
+    "VRPProofGenerator",
+    "VerificationRejectedError",
+    "WorkloadConfigurationError",
+    "WriteOutcome",
+    "validate_then_commit",
+    "__version__",
+]
+
+
+def __getattr__(name: str) -> Any:
+    """Lazy-load heavy integrations (IceGuard, veridata-recon, PyIceberg)."""
+    if name in ("GlueCatalogConnector", "GlueRestCatalogAdapter"):
+        from serverless_data_mesh.catalog import GlueCatalogConnector, GlueRestCatalogAdapter
+
+        return {
+            "GlueCatalogConnector": GlueCatalogConnector,
+            "GlueRestCatalogAdapter": GlueRestCatalogAdapter,
+        }[name]
+    if name in ("SparkRulesConnector", "RuleFireSummary"):
+        from serverless_data_mesh.rules import RuleFireSummary, SparkRulesConnector
+
+        return {
+            "SparkRulesConnector": SparkRulesConnector,
+            "RuleFireSummary": RuleFireSummary,
+        }[name]
+    if name in ("IceGuardDurableCoordinator", "OrchestrationState"):
+        from serverless_data_mesh.orchestration import (
+            IceGuardDurableCoordinator,
+            OrchestrationState,
+        )
+
+        return {
+            "IceGuardDurableCoordinator": IceGuardDurableCoordinator,
+            "OrchestrationState": OrchestrationState,
+        }[name]
+    if name in ("VRPProofGenerator", "validate_then_commit"):
+        from serverless_data_mesh.verification import VRPProofGenerator, validate_then_commit
+
+        return {
+            "VRPProofGenerator": VRPProofGenerator,
+            "validate_then_commit": validate_then_commit,
+        }[name]
+    raise AttributeError(f"module {__name__!r} has no attribute {name!r}")

serverless_data_mesh/catalog/__init__.py

@@ -0,0 +1,6 @@
+"""Iceberg catalog adapters for serverless metadata commits."""
+
+from serverless_data_mesh.catalog.glue_connector import GlueCatalogConnector
+from serverless_data_mesh.catalog.glue_rest import GlueRestCatalogAdapter
+
+__all__ = ["GlueCatalogConnector", "GlueRestCatalogAdapter"]

serverless_data_mesh/catalog/glue_connector.py

@@ -0,0 +1,17 @@
+"""Glue Catalog Connector: metadata-only integration for Lambda domain writers.
+
+AWS Glue **ETL jobs** (managed Spark runners) do not run inside Lambda containers.
+Domain writers execute **physical** transforms on Lambda (PySpark-on-Lambda, Polars,
+PyArrow, DuckDB, etc.) and use this connector for **metadata** commits against the
+Glue Data Catalog Iceberg REST endpoint.
+
+The connector is a thin SigV4 HTTPS client via PyIceberg: no Glue job runtime, no JVM
+Spark catalog session, no Glue Studio dependency.
+"""
+
+from serverless_data_mesh.catalog.glue_rest import GlueRestCatalogAdapter
+
+# Public alias: "Glue connector" in docs maps to this class.
+GlueCatalogConnector = GlueRestCatalogAdapter
+
+__all__ = ["GlueCatalogConnector", "GlueRestCatalogAdapter"]

serverless_data_mesh/catalog/glue_rest.py

@@ -0,0 +1,134 @@
+"""Zero-config AWS Glue Iceberg REST catalog adapter for serverless 2PC commits."""
+
+from __future__ import annotations
+
+import logging
+import os
+from dataclasses import dataclass, field
+from typing import Any
+
+import boto3
+from pyiceberg.catalog import load_catalog
+from pyiceberg.table import Table
+
+from serverless_data_mesh.exceptions import CatalogCommitError
+
+logger = logging.getLogger(__name__)
+
+
+@dataclass(slots=True)
+class GlueRestCatalogAdapter:
+    """Native Python catalog commit adapter using Glue's Iceberg REST endpoint.
+
+    Replaces heavy PySpark JVM catalog operations with lightweight HTTPS REST
+    calls authenticated via AWS SigV4. IceGuard performs physical file writes;
+    this adapter executes the final metadata commit in the two-phase commit (2PC)
+    sequence purely over REST.
+    """
+
+    catalog_name: str
+    namespace: str
+    table_name: str
+    region: str
+    warehouse: str | None = None
+    catalog: Any = field(default=None, repr=False)
+    _prepared_files: list[str] = field(default_factory=list, init=False, repr=False)
+
+    @classmethod
+    def from_environment(
+        cls,
+        *,
+        namespace: str,
+        table_name: str,
+        catalog_name: str = "glue_rest",
+        aws_account_id: str | None = None,
+        warehouse: str | None = None,
+    ) -> GlueRestCatalogAdapter:
+        """Construct adapter from standard Lambda environment variables."""
+        region = os.environ.get("AWS_REGION", os.environ.get("AWS_DEFAULT_REGION", "us-east-1"))
+        account_id = aws_account_id or os.environ.get("AWS_ACCOUNT_ID")
+        if not account_id:
+            account_id = boto3.client("sts").get_caller_identity()["Account"]
+        resolved_warehouse = warehouse or os.environ.get(
+            "ICEBERG_WAREHOUSE",
+            f"{account_id}:s3tablescatalog/{os.environ.get('ICEBERG_TABLE_BUCKET', 'default')}",
+        )
+        return cls(
+            catalog_name=catalog_name,
+            namespace=namespace,
+            table_name=table_name,
+            region=region,
+            warehouse=resolved_warehouse,
+        )
+
+    def _rest_properties(self) -> dict[str, str]:
+        """Build pyiceberg REST catalog properties with SigV4 signing."""
+        if not self.warehouse:
+            raise ValueError("warehouse is required for Glue REST catalog access")
+        return {
+            "type": "rest",
+            "uri": f"https://glue.{self.region}.amazonaws.com/iceberg",
+            "warehouse": self.warehouse,
+            "rest.sigv4-enabled": "true",
+            "rest.signing-name": "glue",
+            "rest.signing-region": self.region,
+        }
+
+    def connect(self) -> Table:
+        """Authenticate via SigV4 and load the target Iceberg table handle."""
+        if self.catalog is None:
+            self.catalog = load_catalog(self.catalog_name, **self._rest_properties())
+        identifier = f"{self.namespace}.{self.table_name}"
+        table = self.catalog.load_table(identifier)
+        logger.info("Connected to Iceberg table %s via Glue REST", identifier)
+        return table
+
+    def prepare_commit(self, parquet_paths: list[str]) -> None:
+        """Phase-1 prepare: stage file paths for the pending metadata transaction."""
+        if not parquet_paths:
+            raise ValueError("prepare_commit requires at least one parquet path")
+        self._prepared_files = list(dict.fromkeys(parquet_paths))
+        logger.info("Prepared %d data files for REST metadata commit", len(self._prepared_files))
+
+    def commit(self, *, snapshot_properties: dict[str, str] | None = None) -> int:
+        """Phase-2 commit: publish a new Iceberg snapshot via HTTPS REST."""
+        if not self._prepared_files:
+            raise CatalogCommitError("commit called before prepare_commit")
+
+        table = self.connect()
+        props = snapshot_properties or {
+            "write.format.default": "parquet",
+            "app-id": "serverless-data-mesh",
+        }
+        try:
+            with table.transaction() as tx:
+                tx.add_files(self._prepared_files, snapshot_properties=props)
+        except Exception as exc:
+            raise CatalogCommitError(f"Glue REST commit failed: {exc}") from exc
+
+        snapshot_id = table.metadata.current_snapshot_id()
+        logger.info(
+            "Committed snapshot %s with %d files to %s.%s",
+            snapshot_id,
+            len(self._prepared_files),
+            self.namespace,
+            self.table_name,
+        )
+        self._prepared_files.clear()
+        return int(snapshot_id or 0)
+
+    def abort(self) -> None:
+        """Abort the in-flight metadata transaction without catalog side effects."""
+        self._prepared_files.clear()
+        logger.info("Aborted pending REST catalog commit for %s.%s", self.namespace, self.table_name)
+
+    def rollback_to_snapshot(self, snapshot_id: int) -> None:
+        """Rollback table metadata to a prior snapshot (IceGuard timeout recovery)."""
+        table = self.connect()
+        table.manage_snapshots().rollback_to_snapshot(snapshot_id).commit()
+        logger.warning(
+            "Rolled back %s.%s to snapshot %s",
+            self.namespace,
+            self.table_name,
+            snapshot_id,
+        )

serverless_data_mesh/cli.py

@@ -0,0 +1,165 @@
+#!/usr/bin/env python3
+"""CLI entry points for Serverless Data Mesh."""
+
+from __future__ import annotations
+
+import argparse
+import json
+import sys
+
+
+def _cmd_demo(args: argparse.Namespace) -> int:
+    from serverless_data_mesh.local.runtime import LocalPVDMRuntime
+    from serverless_data_mesh.verification.backend import veridata_available
+
+    runtime = LocalPVDMRuntime()
+    result = runtime.run_demo_sequence()
+
+    if args.json:
+        print(json.dumps(result, indent=2))
+    else:
+        backend = result.get("verifier_backend", "unknown")
+        fallback_note = ""
+        if backend == "pure-python-fallback":
+            fallback_note = "  (pure-Python verifier - no Rust wheel needed)\n"
+        print("\n  Serverless Data Mesh - local PVDM demo (no AWS)\n")
+        print(f"  Verifier:  {backend}{fallback_note}")
+        print(f"  Workspace: {result['root']}")
+        print(f"  Elapsed:   {result['elapsed_ms']} ms\n")
+        clean = result["phases"]["clean_write"]
+        corrupt = result["phases"]["corrupt_write"]
+        consumer = result["consumer"]
+        print(f"  Phase 1 - clean write:    {clean['outcome']} ({clean['records_written']} rows)")
+        print(f"  Phase 2 - corrupt write:  {corrupt['outcome']} (VRP {corrupt['proof_verdict']})")
+        print(f"  Consumer visible rows:      {consumer['visible_row_count']}")
+        print(f"  Gate blocked bad data:      {consumer['gate_blocked_bad_data']}\n")
+        print(f"  {result['summary']}\n")
+        print("  Vaquar Pattern (PVDM): Physical -> Verify -> Durable -> Metadata")
+        if not veridata_available():
+            print("  Tip: pip install veridata-recon on Linux for cryptographic VRP proofs.")
+        print("  Docs: docs/vaquar-pattern.md\n")
+
+    return 0 if result["consumer"]["gate_blocked_bad_data"] else 1
+
+
+def _cmd_init(args: argparse.Namespace) -> int:
+    from serverless_data_mesh.scaffold.init_domain import scaffold_domain
+
+    root = scaffold_domain(
+        domain=args.domain,
+        table=args.table,
+        account_id=args.account,
+        output_dir=args.output,
+    )
+    print(f"\n  Domain scaffold created: {root}\n")
+    print("  Next steps:")
+    print(f"    1. Edit {root}/handler.py")
+    print(f"    2. Review {root}/contract.yaml")
+    print(f"    3. Deploy {root}/terraform/ (copy tfvars.example -> tfvars)")
+    print(f"    4. Run tests: pytest {root}/tests/\n")
+    return 0
+
+
+def _cmd_dashboard(args: argparse.Namespace) -> int:
+    from serverless_data_mesh.dashboard.trust import render_trust_dashboard
+
+    path = render_trust_dashboard(
+        proofs_dir=args.proofs_dir,
+        output=args.output,
+        demo=not args.proofs_dir and not args.cloudwatch,
+        cloudwatch=args.cloudwatch,
+        cloudwatch_region=args.region,
+    )
+    print(f"Trust dashboard written: {path}")
+    if args.open_browser:
+        import webbrowser
+
+        webbrowser.open(f"file://{path}")
+    return 0
+
+
+def _cmd_canary(args: argparse.Namespace) -> int:
+    from serverless_data_mesh.orchestration.canary import run_canary
+
+    result = run_canary(
+        record_count=args.records,
+        inject_canary_drift=args.drift,
+        max_divergence_pct=args.max_divergence,
+    )
+    if args.json:
+        print(json.dumps(result, indent=2))
+    else:
+        print(f"\n  Canary outcome: {result['outcome']}")
+        print(f"  Production VRP: {result['production_verdict']}")
+        print(f"  Canary VRP:     {result['canary_verdict']}")
+        print(f"  Divergence:     {result['divergence_pct']}%")
+        print(f"  Promote:        {result['promote']}\n")
+        print(f"  {result['message']}\n")
+    return 0 if result["promote"] else 1
+
+
+def _cmd_reprocess_demo(args: argparse.Namespace) -> int:
+    from serverless_data_mesh.local.runtime import LocalPVDMRuntime
+
+    runtime = LocalPVDMRuntime()
+    result = runtime.run_write_with_auto_repair(
+        record_count=args.records,
+        drop_count=args.drop,
+    )
+    if args.json:
+        print(json.dumps(result, indent=2, default=str))
+    else:
+        print("\n  Auto VRP reprocessing demo\n")
+        print(f"  Outcome:          {result['outcome']}")
+        repair = result.get("repair", {})
+        print(f"  Missing before:   {repair.get('missing_before', '?')}")
+        print(f"  Repair attempts:  {repair.get('attempts', '?')}")
+        print(f"  Consumer rows:    {result.get('consumer_row_count', 0)}\n")
+    return 0 if result.get("outcome") == "repaired_and_committed" else 1
+
+
+def main(argv: list[str] | None = None) -> int:
+    parser = argparse.ArgumentParser(
+        prog="serverless-data-mesh",
+        description="Serverless Data Mesh - federated lakehouse writes on AWS Lambda",
+    )
+    sub = parser.add_subparsers(dest="command", required=True)
+
+    demo = sub.add_parser("demo", help="Run local PVDM demo in <60s without AWS")
+    demo.add_argument("--json", action="store_true", help="Emit machine-readable JSON")
+    demo.set_defaults(func=_cmd_demo)
+
+    init_p = sub.add_parser("init", help="Scaffold a new proof-gated domain writer")
+    init_p.add_argument("--domain", required=True, help="Domain id (e.g. payments)")
+    init_p.add_argument("--table", required=True, help="Target Iceberg table")
+    init_p.add_argument("--account", required=True, help="Producer AWS account ID")
+    init_p.add_argument("--output", default="domains", help="Output parent directory")
+    init_p.set_defaults(func=_cmd_init)
+
+    dash = sub.add_parser("dashboard", help="Generate mesh trust dashboard HTML")
+    dash.add_argument("--proofs-dir", help="Steward proofs directory (local or mounted S3)")
+    dash.add_argument("--cloudwatch", action="store_true", help="Pull live metrics from CloudWatch")
+    dash.add_argument("--region", help="AWS region for CloudWatch")
+    dash.add_argument("--output", default="mesh-trust-dashboard.html")
+    dash.add_argument("--open", dest="open_browser", action="store_true")
+    dash.set_defaults(func=_cmd_dashboard)
+
+    canary = sub.add_parser("canary", help="Run VRP canary comparison before promotion")
+    canary.add_argument("--records", type=int, default=1000)
+    canary.add_argument("--drift", action="store_true", help="Inject canary row-count drift")
+    canary.add_argument("--max-divergence", type=float, default=1.0)
+    canary.add_argument("--json", action="store_true")
+    canary.set_defaults(func=_cmd_canary)
+
+    repro = sub.add_parser("reprocess-demo", help="Demo auto VRP repair after dropped records")
+    repro.add_argument("--records", type=int, default=100)
+    repro.add_argument("--drop", type=int, default=5)
+    repro.add_argument("--json", action="store_true")
+    repro.set_defaults(func=_cmd_reprocess_demo)
+
+    args = parser.parse_args(argv)
+    return int(args.func(args))
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

serverless_data_mesh/config.py

@@ -0,0 +1,42 @@
+"""Runtime configuration loaded from environment variables."""
+
+from __future__ import annotations
+
+import os
+from dataclasses import dataclass
+
+
+@dataclass(frozen=True, slots=True)
+class MeshSettings:
+    """Operational settings for Lambda domain writers."""
+
+    checkpoint_bucket: str
+    proof_bucket: str
+    iceberg_table_bucket: str
+    aws_region: str
+    checkpoint_interval: int = 5000
+    rollback_threshold_ms: int = 30_000
+    lambda_timeout_seconds: int = 900
+    iceberg_warehouse: str | None = None
+    aws_account_id: str | None = None
+
+    @classmethod
+    def from_environment(cls) -> MeshSettings:
+        checkpoint = os.environ.get("ICEGUARD_CHECKPOINT_BUCKET")
+        if not checkpoint:
+            raise ValueError("ICEGUARD_CHECKPOINT_BUCKET is required")
+
+        region = os.environ.get("AWS_REGION", os.environ.get("AWS_DEFAULT_REGION", "us-east-1"))
+        table_bucket = os.environ.get("ICEBERG_TABLE_BUCKET", "default")
+
+        return cls(
+            checkpoint_bucket=checkpoint,
+            proof_bucket=os.environ.get("VRP_PROOF_BUCKET", checkpoint),
+            iceberg_table_bucket=table_bucket,
+            aws_region=region,
+    checkpoint_interval=int(os.environ.get("ICEGUARD_CHECKPOINT_INTERVAL", "5000")),
+    rollback_threshold_ms=int(os.environ.get("ICEGUARD_ROLLBACK_THRESHOLD_MS", "30000")),
+    lambda_timeout_seconds=int(os.environ.get("LAMBDA_TIMEOUT_SECONDS", "900")),
+            iceberg_warehouse=os.environ.get("ICEBERG_WAREHOUSE"),
+            aws_account_id=os.environ.get("AWS_ACCOUNT_ID"),
+        )

serverless_data_mesh/dashboard/__init__.py

@@ -0,0 +1,5 @@
+"""Mesh trust dashboard."""
+
+from serverless_data_mesh.dashboard.trust import render_trust_dashboard
+
+__all__ = ["render_trust_dashboard"]

serverless_data_mesh/dashboard/cloudwatch.py

@@ -0,0 +1,80 @@
+"""Fetch live VRP trust rows from CloudWatch metrics."""
+
+from __future__ import annotations
+
+import logging
+from datetime import datetime, timedelta, timezone
+from typing import Any
+
+logger = logging.getLogger(__name__)
+
+NAMESPACE = "ServerlessDataMesh/Trust"
+
+
+def fetch_cloudwatch_trust_rows(
+    *,
+    hours: int = 24,
+    region: str | None = None,
+    cloudwatch_client: Any | None = None,
+) -> list[dict[str, Any]]:
+    """Pull latest VRPTrustScore per domain from CloudWatch for dashboard rendering."""
+    try:
+        import boto3
+    except ImportError:
+        logger.debug("boto3 unavailable for CloudWatch dashboard")
+        return []
+
+    client = cloudwatch_client or boto3.client("cloudwatch", region_name=region)
+    end = datetime.now(timezone.utc)
+    start = end - timedelta(hours=hours)
+
+    response = client.list_metrics(Namespace=NAMESPACE, MetricName="VRPTrustScore")
+    domains = sorted(
+        {
+            dim["Value"]
+            for metric in response.get("Metrics", [])
+            for dim in metric.get("Dimensions", [])
+            if dim.get("Name") == "Domain"
+        }
+    )
+
+    rows: list[dict[str, Any]] = []
+    for domain in domains:
+        trust = client.get_metric_statistics(
+            Namespace=NAMESPACE,
+            MetricName="VRPTrustScore",
+            Dimensions=[{"Name": "Domain", "Value": domain}],
+            StartTime=start,
+            EndTime=end,
+            Period=3600,
+            Statistics=["Maximum"],
+        )
+        count = client.get_metric_statistics(
+            Namespace=NAMESPACE,
+            MetricName="VRPRowCount",
+            Dimensions=[{"Name": "Domain", "Value": domain}],
+            StartTime=start,
+            EndTime=end,
+            Period=3600,
+            Statistics=["Maximum"],
+        )
+        trust_points = trust.get("Datapoints", [])
+        count_points = count.get("Datapoints", [])
+        latest_trust = max(trust_points, key=lambda p: p["Timestamp"]) if trust_points else None
+        latest_count = max(count_points, key=lambda p: p["Timestamp"]) if count_points else None
+
+        score = latest_trust["Maximum"] if latest_trust else 0.0
+        rows.append(
+            {
+                "domain": domain,
+                "last_vrp": (
+                    latest_trust["Timestamp"].strftime("%I:%M %p")
+                    if latest_trust
+                    else "no data"
+                ),
+                "status": "PASS" if score >= 1.0 else "FAIL",
+                "rows": str(int(latest_count["Maximum"])) if latest_count else "?",
+                "detail": "" if score >= 1.0 else "VRP trust score below 1.0",
+            }
+        )
+    return rows