serverless-data-mesh 0.2.0__py3-none-any.whl

serverless_data_mesh/rules/sparkrules_connector.py

@@ -0,0 +1,193 @@
+"""SparkRules engine connector for Lambda domain writers.
+
+SparkRules runs on Lambda in two modes:
+
+1. **Pure Python (default)**: ``LocalRuleExecutor`` evaluates DRL per chunk without
+   a Spark cluster or Glue ETL job. Ideal for enrichment and quality gates before VRP.
+
+2. **Spark-on-Lambda (optional)**: ``apply_drl`` when PySpark is bundled in the
+   Lambda layer/container.
+
+Install: ``pip install serverless-data-mesh[rules]`` or ``[spark]`` for PySpark path.
+"""
+
+from __future__ import annotations
+
+import json
+import logging
+import os
+from dataclasses import dataclass, field
+from typing import Any
+
+from serverless_data_mesh.exceptions import RuleEvaluationError
+
+logger = logging.getLogger(__name__)
+
+
+def _require_sparkrules() -> Any:
+    try:
+        import sparkrules  # noqa: F401
+    except ImportError as exc:
+        raise ImportError(
+            "sparkrules is required for SparkRulesConnector. "
+            "Install with: pip install serverless-data-mesh[rules]"
+        ) from exc
+    return sparkrules
+
+
+@dataclass(frozen=True, slots=True)
+class RuleFireSummary:
+    """Audit-friendly summary of one rule evaluation on a fact."""
+
+    fact_index: int
+    rule_name: str
+    fired: bool
+    reason_codes: tuple[str, ...]
+    action_output: dict[str, Any]
+
+
+@dataclass(slots=True)
+class SparkRulesConnector:
+    """Lambda-native business rules connector using SparkRules ``LocalRuleExecutor``.
+
+    Use between ``source_reader`` and ``batch_writer`` to enrich or filter records
+    with Drools-style DRL before VRP verification and Iceberg commit.
+    """
+
+    drl: str
+    policy_id: str = "mesh-default"
+    _executor: Any = field(default=None, repr=False)
+
+    @classmethod
+    def from_drl(cls, drl: str, *, policy_id: str = "mesh-default") -> SparkRulesConnector:
+        """Compile DRL and build a pure-Python executor (no JVM, no Glue ETL)."""
+        _require_sparkrules()
+        from sparkrules.executor.local_executor import LocalRuleExecutor
+
+        connector = cls(drl=drl, policy_id=policy_id)
+        connector._executor = LocalRuleExecutor.from_drl(drl)
+        return connector
+
+    @classmethod
+    def from_environment(cls) -> SparkRulesConnector:
+        """Load DRL from ``SPARKRULES_DRL`` (inline) or ``SPARKRULES_DRL_S3_URI``."""
+        inline = os.environ.get("SPARKRULES_DRL")
+        if inline:
+            return cls.from_drl(inline, policy_id=os.environ.get("SPARKRULES_POLICY_ID", "mesh-default"))
+
+        s3_uri = os.environ.get("SPARKRULES_DRL_S3_URI")
+        if s3_uri:
+            return cls.from_s3(s3_uri, policy_id=os.environ.get("SPARKRULES_POLICY_ID", "mesh-default"))
+
+        raise ValueError("Set SPARKRULES_DRL or SPARKRULES_DRL_S3_URI for SparkRulesConnector")
+
+    @classmethod
+    def from_s3(cls, s3_uri: str, *, policy_id: str = "mesh-default") -> SparkRulesConnector:
+        """Load rule pack DRL from S3 (Steward governance bucket pattern)."""
+        import boto3
+
+        if not s3_uri.startswith("s3://"):
+            raise ValueError(f"Expected s3:// URI, got {s3_uri!r}")
+        path = s3_uri[5:]
+        bucket, _, key = path.partition("/")
+        body = boto3.client("s3").get_object(Bucket=bucket, Key=key)["Body"].read()
+        return cls.from_drl(body.decode("utf-8"), policy_id=policy_id)
+
+    def _ensure_executor(self) -> Any:
+        if self._executor is None:
+            _require_sparkrules()
+            from sparkrules.executor.local_executor import LocalRuleExecutor
+
+            self._executor = LocalRuleExecutor.from_drl(self.drl)
+        return self._executor
+
+    def apply_chunk(
+        self,
+        records: list[dict[str, Any]],
+        *,
+        fact_binding: str = "row",
+    ) -> tuple[list[dict[str, Any]], list[RuleFireSummary]]:
+        """Evaluate DRL against each record; merge rule actions into enriched rows.
+
+        Each record is wrapped as ``{fact_binding: record}`` for DRL binding, e.g.
+        ``$row : Row ( $row.amount > 0 )``.
+        """
+        executor = self._ensure_executor()
+        enriched: list[dict[str, Any]] = []
+        audit: list[RuleFireSummary] = []
+
+        for idx, record in enumerate(records):
+            fact: dict[str, Any] = {fact_binding: dict(record)}
+            result = executor.score(fact)
+            merged = dict(record)
+            merged.update(result.merged_actions)
+            merged["_sparkrules_fired"] = result.fired_any
+            enriched.append(merged)
+
+            for fire in result.fires:
+                if fire.fired:
+                    audit.append(
+                        RuleFireSummary(
+                            fact_index=idx,
+                            rule_name=fire.rule_name,
+                            fired=True,
+                            reason_codes=fire.reason_codes,
+                            action_output=dict(fire.action_output),
+                        )
+                    )
+        return enriched, audit
+
+    def quality_gate(
+        self,
+        records: list[dict[str, Any]],
+        *,
+        require_any_rule_fired: bool = False,
+        reject_field: str = "_mesh_reject",
+    ) -> list[dict[str, Any]]:
+        """Filter or mark records that fail the rules policy before physical write."""
+        enriched, audit = self.apply_chunk(records)
+        if not require_any_rule_fired:
+            return enriched
+
+        fired_indices = {a.fact_index for a in audit if a.fired}
+        passed: list[dict[str, Any]] = []
+        for idx, row in enumerate(enriched):
+            if idx in fired_indices:
+                passed.append(row)
+            else:
+                logger.warning("SparkRules quality gate rejected record index=%d", idx)
+        if not passed:
+            raise RuleEvaluationError(
+                f"No records passed SparkRules policy {self.policy_id!r} "
+                f"(require_any_rule_fired=True, chunk_size={len(records)})"
+            )
+        return passed
+
+    def audit_json(self, audit: list[RuleFireSummary]) -> str:
+        """Serialize rule fires for Steward proof / lineage buckets."""
+        payload = [
+            {
+                "fact_index": a.fact_index,
+                "rule_name": a.rule_name,
+                "fired": a.fired,
+                "reason_codes": list(a.reason_codes),
+                "action_output": a.action_output,
+                "policy_id": self.policy_id,
+            }
+            for a in audit
+        ]
+        return json.dumps(payload, default=str)
+
+    @staticmethod
+    def apply_drl_spark(
+        spark: Any,
+        dataframe: Any,
+        drl: str,
+        *,
+        fact_id_field: str = "id",
+    ) -> Any:
+        """Spark-on-Lambda path: distributed ``apply_drl`` over a DataFrame."""
+        _require_sparkrules()
+        from sparkrules.spark.dataframe import apply_drl
+
+        return apply_drl(dataframe, drl, fact_id_field=fact_id_field)

serverless_data_mesh/scaffold/__init__.py

@@ -0,0 +1,5 @@
+"""Scaffold new Vaquar Pattern domain writers."""
+
+from serverless_data_mesh.scaffold.init_domain import scaffold_domain
+
+__all__ = ["scaffold_domain"]

serverless_data_mesh/scaffold/init_domain.py

@@ -0,0 +1,210 @@
+"""Scaffold new domain writers for the Vaquar Pattern."""
+
+from __future__ import annotations
+
+from pathlib import Path
+
+
+def scaffold_domain(
+    *,
+    domain: str,
+    table: str,
+    account_id: str,
+    output_dir: str = "domains",
+) -> Path:
+    """Create handler, contract, terraform stub, and tests for a new domain."""
+    root = Path(output_dir) / domain
+    root.mkdir(parents=True, exist_ok=True)
+
+    (root / "tests").mkdir(exist_ok=True)
+    (root / "terraform").mkdir(exist_ok=True)
+
+    handler = root / "handler.py"
+    handler.write_text(
+        HANDLER_TEMPLATE.format(domain=domain, table=table),
+        encoding="utf-8",
+    )
+
+    contract = root / "contract.yaml"
+    contract.write_text(
+        CONTRACT_TEMPLATE.format(domain=domain, table=table, account_id=account_id),
+        encoding="utf-8",
+    )
+
+    (root / "terraform" / "main.tf").write_text(
+        TERRAFORM_TEMPLATE.format(domain=domain, account_id=account_id),
+        encoding="utf-8",
+    )
+    (root / "terraform" / "terraform.tfvars.example").write_text(
+        TFVARS_TEMPLATE.format(domain=domain, account_id=account_id),
+        encoding="utf-8",
+    )
+
+    (root / "tests" / f"test_{domain}.py").write_text(
+        TEST_TEMPLATE.format(domain=domain, table=table),
+        encoding="utf-8",
+    )
+
+    (root / "consumer_sla.yaml").write_text(
+        CONSUMER_SLA_TEMPLATE.format(domain=domain, table=table),
+        encoding="utf-8",
+    )
+
+    (root / "step_function.asl.json").write_text(
+        STEP_FUNCTION_TEMPLATE.format(domain=domain, table=table),
+        encoding="utf-8",
+    )
+
+    (root / "README.md").write_text(
+        README_TEMPLATE.format(domain=domain, table=table),
+        encoding="utf-8",
+    )
+
+    return root
+
+
+HANDLER_TEMPLATE = '''"""Domain writer: {domain} -> {table} (Vaquar Pattern PVDM)."""
+
+from __future__ import annotations
+
+from typing import Any
+
+from serverless_data_mesh.governance.consumer_sla import enforce_consumer_sla
+from serverless_data_mesh.metrics.mesh_trust import publish_vrp_metric
+from serverless_data_mesh.orchestration.reprocess import attempt_vrp_repair
+from serverless_data_mesh.types.workload import ConsumerSLAContract, DataWriteWorkload
+from serverless_data_mesh.verification.backend import create_proof_generator
+from serverless_data_mesh.verification.vrp import validate_then_commit
+
+
+def source_reader(start: int, end: int) -> list[dict[str, Any]]:
+    return [{{"id": str(i), "payload_hash": f"h{{i}}"}} for i in range(start, end)]
+
+
+def sink_reader(start: int, end: int) -> list[dict[str, Any]]:
+    """Read physical sink for VRP; replace with Parquet reader in production."""
+    return source_reader(start, end)
+
+
+def batch_writer(start: int, end: int) -> list[str]:
+    base = "s3://publisher-lakehouse/{table}/dt=PARTITION"
+    return [f"{{base}}/part-{{i:08d}}.parquet" for i in range(start, end)]
+
+
+def lambda_handler(event: dict[str, Any], context: Any) -> dict[str, Any]:
+    """Wire IceGuardDurableCoordinator with auto-repair and consumer SLA gate."""
+    raise NotImplementedError("Copy wiring from examples/domain_writer/handler.py")
+'''
+
+CONTRACT_TEMPLATE = """# Data product contract: {domain}
+product_id: {domain}-{table}
+owner_team: {domain}-platform
+domain_id: {domain}
+target_table: {table}
+source_namespace: raw_{domain}
+producer_account_id: \"{account_id}\"
+sla_freshness_hours: 2
+schema_version: \"1.0.0\"
+quality_policy_id: strict-zero-drop
+vaquar_pattern: PVDM
+"""
+
+TERRAFORM_TEMPLATE = """# Producer Terraform stub for domain: {domain}
+variable "producer_account_id" {{
+  default = "{account_id}"
+}}
+
+variable "domain_id" {{
+  default = "{domain}"
+}}
+
+# Copy modules from infrastructure/terraform/environments/multi-account/producer/
+"""
+
+TFVARS_TEMPLATE = """aws_region = \"us-east-2\"
+name_prefix = \"sdm-{domain}\"
+producer_account_id = \"{account_id}\"
+# steward_account_id = \"STEWARD_ACCOUNT\"
+# publisher_account_id = \"PUBLISHER_ACCOUNT\"
+"""
+
+TEST_TEMPLATE = '''"""Tests for {domain} domain writer."""
+
+from __future__ import annotations
+
+
+def test_boundary_declared() -> None:
+    from serverless_data_mesh import DomainTransactionBoundary
+
+    boundary = DomainTransactionBoundary(
+        domain_id="{domain}",
+        source_namespace="raw_{domain}",
+        target_table="{table}",
+        partition_spec={{"dt": "2026-06-14"}},
+    )
+    assert boundary.domain_id == "{domain}"
+'''
+
+README_TEMPLATE = """# {domain} domain writer
+
+Target table: `{table}`
+
+## Scaffolded by
+
+```bash
+serverless-data-mesh init --domain {domain} --table {table} --account YOUR_ACCOUNT_ID
+```
+
+## Next steps
+
+1. Implement `handler.py` (copy from `examples/domain_writer/handler.py`)
+2. Review `consumer_sla.yaml` for Lake Formation read gates
+3. Deploy `step_function.asl.json` durable workflow
+4. Deploy Terraform in `terraform/`
+5. Run `make demo` locally to verify PVDM gate
+"""
+
+CONSUMER_SLA_TEMPLATE = """# Consumer SLA for {table} (VRP-backed Lake Formation gate)
+consumer_id: analytics-team
+target_table: {table}
+max_freshness_minutes: 60
+min_completeness_pct: 99.9
+required_columns:
+  - id
+  - payload_hash
+enforcement: vrp_backed
+"""
+
+STEP_FUNCTION_TEMPLATE = """{{
+  "Comment": "PVDM durable write for {domain} -> {table}",
+  "StartAt": "WriteChunk",
+  "States": {{
+    "WriteChunk": {{
+      "Type": "Task",
+      "Resource": "arn:aws:states:::lambda:invoke",
+      "Parameters": {{
+        "FunctionName": "${{DomainWriterArn}}",
+        "Payload.$": "$"
+      }},
+      "Retry": [
+        {{
+          "ErrorEquals": ["VerificationRejectedError"],
+          "IntervalSeconds": 30,
+          "MaxAttempts": 2,
+          "BackoffRate": 2.0
+        }}
+      ],
+      "Next": "CommitMetadata"
+    }},
+    "CommitMetadata": {{
+      "Type": "Task",
+      "Resource": "arn:aws:states:::lambda:invoke",
+      "Parameters": {{
+        "FunctionName": "${{CatalogCommitArn}}",
+        "Payload.$": "$"
+      }},
+      "End": true
+    }}
+  }}
+}}
+"""

serverless_data_mesh/types/__init__.py

@@ -0,0 +1,21 @@
+"""Domain contracts and workload types."""
+
+from serverless_data_mesh.types.workload import (
+    BatchWriterFn,
+    ChunkWriteResult,
+    DataProductContract,
+    DataWriteWorkload,
+    DomainTransactionBoundary,
+    SourceReaderFn,
+    WriteOutcome,
+)
+
+__all__ = [
+    "BatchWriterFn",
+    "ChunkWriteResult",
+    "DataProductContract",
+    "DataWriteWorkload",
+    "DomainTransactionBoundary",
+    "SourceReaderFn",
+    "WriteOutcome",
+]

serverless_data_mesh/types/workload.py

@@ -0,0 +1,123 @@
+"""Shared types for multi-domain write coordination and contract enforcement."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from enum import Enum
+from typing import Any, Callable
+
+
+class WriteOutcome(str, Enum):
+    """Terminal state of a domain write transaction."""
+
+    COMMITTED = "committed"
+    ROLLED_BACK = "rolled_back"
+    RESUMED = "resumed"
+    VERIFICATION_FAILED = "verification_failed"
+
+
+@dataclass(frozen=True, slots=True)
+class DomainTransactionBoundary:
+    """Declarative contract for a single domain team's write scope.
+
+    In a federated data mesh, each domain owns its tables and publishes
+    consumption contracts. This boundary encodes the partition scope and
+    quality gates that the central coordinator enforces before metadata commit.
+    """
+
+    domain_id: str
+    source_namespace: str
+    target_table: str
+    partition_spec: dict[str, str]
+    quality_policy_id: str = "default:strict"
+    max_chunk_records: int = 5000
+
+
+@dataclass(frozen=True, slots=True)
+class DataWriteWorkload:
+    """Description of a large backfill or cross-account copy job.
+
+    The durable orchestrator shards ``total_records`` into checkpoint-sized
+    chunks. Each chunk is an independently resumable unit guarded by IceGuard.
+    """
+
+    workload_id: str
+    boundary: DomainTransactionBoundary
+    source_uri: str
+    target_uri: str
+    total_records: int
+    checkpoint_bucket: str
+    proof_bucket: str
+    content_fields: tuple[str, ...] = ("id", "payload_hash")
+    identity_fields: tuple[str, ...] = ("id",)
+
+
+@dataclass(frozen=True, slots=True)
+class DataProductContract:
+    """Published data product contract for the federated mesh registry.
+
+    Extends ``DomainTransactionBoundary`` with ownership and SLA metadata that
+    Steward governance and Publisher consumers rely on for discovery and audit.
+    """
+
+    product_id: str
+    owner_team: str
+    boundary: DomainTransactionBoundary
+    sla_freshness_hours: int = 24
+    schema_version: str = "1.0"
+    description: str = ""
+
+    def to_registry_entry(self) -> dict[str, str | int | dict[str, str]]:
+        """Serialize for mesh catalog / Backstage / internal registry APIs."""
+        return {
+            "product_id": self.product_id,
+            "owner_team": self.owner_team,
+            "domain_id": self.boundary.domain_id,
+            "target_table": self.boundary.target_table,
+            "quality_policy_id": self.boundary.quality_policy_id,
+            "sla_freshness_hours": self.sla_freshness_hours,
+            "schema_version": self.schema_version,
+            "description": self.description,
+            "partition_spec": dict(self.boundary.partition_spec),
+        }
+
+
+@dataclass(frozen=True, slots=True)
+class ConsumerSLAContract:
+    """Consumer-declared SLA enforced via VRP-backed proofs before read access."""
+
+    consumer_id: str
+    target_table: str
+    max_freshness_minutes: int = 60
+    min_completeness_pct: float = 99.9
+    required_columns: tuple[str, ...] = ()
+    enforcement: str = "vrp_backed"
+
+    def to_dict(self) -> dict[str, str | int | float | tuple[str, ...]]:
+        return {
+            "consumer_id": self.consumer_id,
+            "target_table": self.target_table,
+            "max_freshness_minutes": self.max_freshness_minutes,
+            "min_completeness_pct": self.min_completeness_pct,
+            "required_columns": self.required_columns,
+            "enforcement": self.enforcement,
+        }
+
+
+@dataclass(slots=True)
+class ChunkWriteResult:
+    """Outcome of one durable chunk step."""
+
+    chunk_index: int
+    record_offset: int
+    record_end: int
+    parquet_paths: list[str]
+    proof_s3_uri: str | None = None
+    verification_passed: bool = False
+
+
+BatchWriterFn = Callable[[int, int], list[str]]
+"""Write records ``[start, end)`` and return newly created Parquet S3 URIs."""
+
+SourceReaderFn = Callable[[int, int], list[dict[str, Any]]]
+"""Read source records ``[start, end)`` for VRP fingerprinting."""

serverless_data_mesh/verification/__init__.py

@@ -0,0 +1,21 @@
+"""Cryptographic verification and reconciliation proofs."""
+
+from serverless_data_mesh.verification.vrp import (
+    ValidateResult,
+    VRPProofGenerator,
+    VerifyResult,
+    validate_then_commit,
+)
+from serverless_data_mesh.verification.backend import create_proof_generator, veridata_available
+from serverless_data_mesh.verification.fallback import FallbackProofGenerator, reconcile_multiset
+
+__all__ = [
+    "VRPProofGenerator",
+    "FallbackProofGenerator",
+    "ValidateResult",
+    "VerifyResult",
+    "validate_then_commit",
+    "create_proof_generator",
+    "veridata_available",
+    "reconcile_multiset",
+]

serverless_data_mesh/verification/backend.py

@@ -0,0 +1,41 @@
+"""Select veridata-recon or pure-Python fallback proof backend."""
+
+from __future__ import annotations
+
+from typing import Any, Protocol
+
+
+class ProofGeneratorProtocol(Protocol):
+    def build_proof(self, **kwargs: Any) -> dict[str, Any]: ...
+    def persist_proof(self, **kwargs: Any) -> str: ...
+
+
+def veridata_available() -> bool:
+    try:
+        import veridata_recon  # noqa: F401
+
+        return True
+    except ImportError:
+        return False
+
+
+def create_proof_generator() -> tuple[Any, str]:
+    """Return (generator, backend_name). Prefers veridata-recon; falls back to pure Python."""
+    if veridata_available():
+        import veridata_recon as vr
+
+        from serverless_data_mesh.verification.vrp import VRPProofGenerator
+
+        keys = vr.generate_keypair()
+        return (
+            VRPProofGenerator(
+                private_key_b64=keys["private_key"],
+                public_key_b64=keys["public_key"],
+                salt_hex=vr.generate_salt(),
+            ),
+            "veridata-recon",
+        )
+
+    from serverless_data_mesh.verification.fallback import FallbackProofGenerator
+
+    return FallbackProofGenerator(), "pure-python-fallback"