serverless-data-mesh 0.2.0__py3-none-any.whl

serverless_data_mesh/dashboard/trust.py

@@ -0,0 +1,162 @@
+"""Mesh trust dashboard: real-time VRP status per domain."""
+
+from __future__ import annotations
+
+import json
+from datetime import datetime, timezone
+from pathlib import Path
+from typing import Any
+
+from serverless_data_mesh.dashboard.cloudwatch import fetch_cloudwatch_trust_rows
+
+
+def _demo_rows() -> list[dict[str, Any]]:
+    now = datetime.now(timezone.utc).strftime("%I:%M %p")
+    return [
+        {"domain": "orders", "last_vrp": now, "status": "PASS", "rows": "5.2M", "detail": ""},
+        {"domain": "payments", "last_vrp": now, "status": "PASS", "rows": "1.1M", "detail": ""},
+        {
+            "domain": "inventory",
+            "last_vrp": "09:45 AM",
+            "status": "FAIL",
+            "rows": "0",
+            "detail": "3 drops detected",
+        },
+        {"domain": "shipping", "last_vrp": now, "status": "PASS", "rows": "800K", "detail": ""},
+    ]
+
+
+def _scan_proofs(proofs_dir: Path) -> list[dict[str, Any]]:
+    rows: list[dict[str, Any]] = []
+    for proof_file in sorted(proofs_dir.rglob("*.vrp.json")):
+        try:
+            data = json.loads(proof_file.read_text(encoding="utf-8"))
+        except (json.JSONDecodeError, OSError):
+            continue
+        recon = data.get("reconciliation", {})
+        verdict = recon.get("verdict", "UNKNOWN")
+        domain = proof_file.parts[-3] if len(proof_file.parts) >= 3 else "unknown"
+        created = data.get("created_at", "")[:16].replace("T", " ")
+        rows.append(
+            {
+                "domain": domain,
+                "last_vrp": created or "unknown",
+                "status": verdict,
+                "rows": str(recon.get("sink_count", "?")),
+                "detail": _fail_detail(recon) if verdict == "FAIL" else "",
+            }
+        )
+    return rows or _demo_rows()
+
+
+def _fail_detail(recon: dict[str, Any]) -> str:
+    missing = len(recon.get("missing", []))
+    mutated = len(recon.get("mutated", []))
+    dup = len(recon.get("duplicated", []))
+    parts = []
+    if missing:
+        parts.append(f"{missing} drops")
+    if mutated:
+        parts.append(f"{mutated} mutations")
+    if dup:
+        parts.append(f"{dup} duplicates")
+    return ", ".join(parts) or "reconciliation failed"
+
+
+def render_trust_dashboard(
+    *,
+    proofs_dir: str | None = None,
+    output: str = "mesh-trust-dashboard.html",
+    demo: bool = False,
+    cloudwatch: bool = False,
+    cloudwatch_region: str | None = None,
+) -> Path:
+    """Render HTML trust dashboard from proofs, CloudWatch, or demo data."""
+    if cloudwatch:
+        rows = fetch_cloudwatch_trust_rows(region=cloudwatch_region)
+        mode = "cloudwatch"
+        if not rows:
+            rows = _demo_rows()
+            mode = "cloudwatch-fallback-demo"
+    elif demo or not proofs_dir:
+        rows = _demo_rows()
+        mode = "demo"
+    else:
+        rows = _scan_proofs(Path(proofs_dir))
+        mode = "live-proofs"
+
+    html = HTML_TEMPLATE.format(
+        generated_at=datetime.now(timezone.utc).isoformat(),
+        mode=mode,
+        rows=_render_rows(rows),
+        pass_count=sum(1 for r in rows if r["status"] == "PASS"),
+        fail_count=sum(1 for r in rows if r["status"] == "FAIL"),
+        total=len(rows),
+    )
+    out = Path(output)
+    out.write_text(html, encoding="utf-8")
+    return out.resolve()
+
+
+def _render_rows(rows: list[dict[str, Any]]) -> str:
+    lines = []
+    for row in rows:
+        if row["status"] == "PASS":
+            icon = "PASS"
+            cls = "pass"
+        elif row["status"] == "FAIL":
+            icon = "FAIL"
+            cls = "fail"
+        else:
+            icon = "PENDING"
+            cls = "pending"
+        detail = f" ({row['detail']})" if row.get("detail") else ""
+        lines.append(
+            f"<tr class='{cls}'><td>{row['domain']}</td>"
+            f"<td>{row['last_vrp']}</td>"
+            f"<td><span class='badge {cls}'>{icon}</span></td>"
+            f"<td>{row['rows']}{detail}</td></tr>"
+        )
+    return "\n".join(lines)
+
+
+HTML_TEMPLATE = """<!DOCTYPE html>
+<html lang="en">
+<head>
+  <meta charset="utf-8"/>
+  <title>Mesh Trust Dashboard</title>
+  <style>
+    body {{ font-family: system-ui, sans-serif; background: #0f172a; color: #e2e8f0; margin: 2rem; }}
+    h1 {{ color: #38bdf8; }}
+    .summary {{ display: flex; gap: 1.5rem; margin: 1.5rem 0; }}
+    .card {{ background: #1e293b; padding: 1rem 1.5rem; border-radius: 8px; }}
+    table {{ width: 100%; border-collapse: collapse; margin-top: 1rem; }}
+    th, td {{ padding: 0.75rem 1rem; text-align: left; border-bottom: 1px solid #334155; }}
+    th {{ color: #94a3b8; }}
+    tr.pass td {{ }}
+    tr.fail {{ background: #450a0a33; }}
+    .badge {{ padding: 0.2rem 0.6rem; border-radius: 4px; font-weight: 600; font-size: 0.85rem; }}
+    .badge.pass {{ background: #14532d; color: #86efac; }}
+    .badge.fail {{ background: #7f1d1d; color: #fca5a5; }}
+    .badge.pending {{ background: #713f12; color: #fde68a; }}
+    .meta {{ color: #64748b; font-size: 0.9rem; }}
+  </style>
+</head>
+<body>
+  <h1>Mesh Trust Dashboard</h1>
+  <p class="meta">Vaquar Pattern (PVDM) · mode: {mode} · generated {generated_at}</p>
+  <div class="summary">
+    <div class="card"><strong>{total}</strong> domains</div>
+    <div class="card"><strong>{pass_count}</strong> PASS</div>
+    <div class="card"><strong>{fail_count}</strong> FAIL</div>
+  </div>
+  <table>
+    <thead><tr><th>Domain</th><th>Last VRP</th><th>Status</th><th>Rows / Detail</th></tr></thead>
+    <tbody>
+    {rows}
+    </tbody>
+  </table>
+  <p class="meta">Invariant: commit_metadata implies VRP = PASS</p>
+</body>
+</html>
+"""

serverless_data_mesh/exceptions.py

@@ -0,0 +1,23 @@
+"""Framework-specific exceptions for transaction boundary enforcement."""
+
+from __future__ import annotations
+
+
+class ServerlessDataMeshError(Exception):
+    """Base error for all serverless-data-mesh failures."""
+
+
+class VerificationRejectedError(ServerlessDataMeshError):
+    """Raised when VRP validation blocks a chunk from committing."""
+
+
+class CatalogCommitError(ServerlessDataMeshError):
+    """Raised when the Glue REST metadata commit fails."""
+
+
+class WorkloadConfigurationError(ServerlessDataMeshError):
+    """Raised when a domain workload or boundary contract is invalid."""
+
+
+class RuleEvaluationError(ServerlessDataMeshError):
+    """Raised when SparkRules quality gate or policy evaluation fails."""

serverless_data_mesh/governance/__init__.py

@@ -0,0 +1,9 @@
+"""Federated governance: consumer SLA and Lake Formation enforcement."""
+
+from serverless_data_mesh.governance.consumer_sla import (
+    ConsumerAccessDecision,
+    enforce_consumer_sla,
+    grant_read_if_sla_met,
+)
+
+__all__ = ["ConsumerAccessDecision", "enforce_consumer_sla", "grant_read_if_sla_met"]

serverless_data_mesh/governance/consumer_sla.py

@@ -0,0 +1,109 @@
+"""Consumer SLA contracts backed by VRP proofs before read access."""
+
+from __future__ import annotations
+
+import logging
+from dataclasses import dataclass
+from datetime import datetime, timezone
+from typing import Any
+
+from serverless_data_mesh.types.workload import ConsumerSLAContract
+
+logger = logging.getLogger(__name__)
+
+
+@dataclass(frozen=True, slots=True)
+class ConsumerAccessDecision:
+    """Whether a consumer may read a table given proof + SLA."""
+
+    granted: bool
+    consumer_id: str
+    target_table: str
+    reason: str
+    checks: dict[str, bool]
+
+
+def enforce_consumer_sla(
+    contract: ConsumerSLAContract,
+    *,
+    proof: dict[str, Any],
+    snapshot_committed_at: datetime | None = None,
+) -> ConsumerAccessDecision:
+    """Verify producer VRP proof meets consumer SLA before granting read access."""
+    checks: dict[str, bool] = {}
+    recon = proof.get("reconciliation", {})
+    verdict = recon.get("verdict", "FAIL")
+
+    checks["vrp_pass"] = verdict == "PASS"
+    if not checks["vrp_pass"]:
+        return ConsumerAccessDecision(
+            granted=False,
+            consumer_id=contract.consumer_id,
+            target_table=contract.target_table,
+            reason="VRP verdict is not PASS",
+            checks=checks,
+        )
+
+    source_count = int(recon.get("source_count", 0))
+    sink_count = int(recon.get("sink_count", 0))
+    if source_count > 0:
+        completeness = (sink_count / source_count) * 100.0
+    else:
+        completeness = 0.0
+    checks["completeness"] = completeness >= contract.min_completeness_pct
+
+    content_fields = set(proof.get("content_fields", []))
+    checks["required_columns"] = all(col in content_fields for col in contract.required_columns)
+
+    committed_at = snapshot_committed_at
+    if committed_at is None and proof.get("created_at"):
+        committed_at = datetime.fromisoformat(proof["created_at"].replace("Z", "+00:00"))
+    freshness_ok = True
+    if committed_at is not None:
+        age_min = (datetime.now(timezone.utc) - committed_at).total_seconds() / 60.0
+        freshness_ok = age_min <= contract.max_freshness_minutes
+    checks["freshness"] = freshness_ok
+
+    granted = all(checks.values())
+    failed = [k for k, v in checks.items() if not v]
+    reason = "All SLA checks passed" if granted else f"SLA failed: {', '.join(failed)}"
+
+    logger.info(
+        "Consumer SLA %s for %s: granted=%s checks=%s",
+        contract.consumer_id,
+        contract.target_table,
+        granted,
+        checks,
+    )
+
+    return ConsumerAccessDecision(
+        granted=granted,
+        consumer_id=contract.consumer_id,
+        target_table=contract.target_table,
+        reason=reason,
+        checks=checks,
+    )
+
+
+def grant_read_if_sla_met(
+    contract: ConsumerSLAContract,
+    *,
+    proof: dict[str, Any],
+    snapshot_committed_at: datetime | None = None,
+) -> dict[str, Any]:
+    """Lake Formation / steward automation hook: return grant payload or denial."""
+    decision = enforce_consumer_sla(
+        contract,
+        proof=proof,
+        snapshot_committed_at=snapshot_committed_at,
+    )
+    return {
+        "consumer_id": decision.consumer_id,
+        "target_table": decision.target_table,
+        "grant_read": decision.granted,
+        "enforcement": contract.enforcement,
+        "reason": decision.reason,
+        "checks": decision.checks,
+        "lf_action": "GrantPermissions" if decision.granted else "Deny",
+        "proof_id": proof.get("proof_id"),
+    }

serverless_data_mesh/lineage/__init__.py

@@ -0,0 +1,5 @@
+"""OpenLineage integration for mesh data product discovery."""
+
+from serverless_data_mesh.lineage.openlineage import emit_openlineage_event
+
+__all__ = ["emit_openlineage_event"]

serverless_data_mesh/lineage/openlineage.py

@@ -0,0 +1,96 @@
+"""Emit OpenLineage RunEvent after successful Vaquar Pattern commits."""
+
+from __future__ import annotations
+
+import json
+import logging
+import os
+import urllib.error
+import urllib.request
+import uuid
+from datetime import datetime, timezone
+from typing import Any
+
+logger = logging.getLogger(__name__)
+
+OPENLINEAGE_PRODUCER = "https://github.com/vaquarkhan/aws-serverless-datamesh-framework"
+
+
+def _dataset(name: str, namespace: str = "s3") -> dict[str, str]:
+    return {"namespace": namespace, "name": name}
+
+
+def emit_openlineage_event(
+    *,
+    job_name: str,
+    run_id: str | None = None,
+    inputs: list[str],
+    outputs: list[str],
+    facets: dict[str, Any] | None = None,
+    event_type: str = "COMPLETE",
+    endpoint: str | None = None,
+) -> dict[str, Any]:
+    """Build and optionally POST an OpenLineage RunEvent.
+
+    Set ``OPENLINEAGE_URL`` (or pass ``endpoint``) to POST JSON to Marquez/DataHub.
+    Without an endpoint, returns the event dict for logging or local persistence.
+    """
+    run_id = run_id or str(uuid.uuid4())
+    now = datetime.now(timezone.utc).replace(microsecond=0).isoformat()
+
+    event: dict[str, Any] = {
+        "eventType": event_type,
+        "eventTime": now,
+        "producer": OPENLINEAGE_PRODUCER,
+        "schemaURL": "https://openlineage.io/spec/1-0-5/OpenLineage.json",
+        "run": {"runId": run_id},
+        "job": {"namespace": "serverless-data-mesh", "name": job_name},
+        "inputs": [_dataset(name) for name in inputs],
+        "outputs": [_dataset(name) for name in outputs],
+        "facets": facets or {},
+    }
+
+    url = endpoint or os.environ.get("OPENLINEAGE_URL")
+    if url:
+        body = json.dumps(event).encode("utf-8")
+        request = urllib.request.Request(
+            url.rstrip("/") + "/api/v1/lineage",
+            data=body,
+            headers={"Content-Type": "application/json"},
+            method="POST",
+        )
+        try:
+            with urllib.request.urlopen(request, timeout=10) as response:
+                logger.info("OpenLineage event posted (%s)", response.status)
+        except urllib.error.URLError as exc:
+            logger.warning("OpenLineage POST failed: %s", exc)
+
+    return event
+
+
+def emit_from_commit_result(
+    *,
+    domain_id: str,
+    target_table: str,
+    source_namespace: str,
+    commit_result: dict[str, Any],
+    proof_id: str | None = None,
+    checkpoint_path: str | None = None,
+) -> dict[str, Any]:
+    """Convenience wrapper after ``coordinator.execute_workload()`` returns committed."""
+    if commit_result.get("outcome") != "committed":
+        return {}
+
+    return emit_openlineage_event(
+        job_name=f"{domain_id}.{target_table}",
+        run_id=commit_result.get("workload_id"),
+        inputs=[source_namespace],
+        outputs=[target_table],
+        facets={
+            "vrp_proof_id": proof_id or commit_result.get("proof_chain_tail"),
+            "iceguard_checkpoint": checkpoint_path,
+            "row_count": commit_result.get("records_written"),
+            "snapshot_id": commit_result.get("snapshot_id"),
+            "vaquar_pattern": "PVDM",
+        },
+    )

serverless_data_mesh/local/__init__.py

@@ -0,0 +1,5 @@
+"""Local-first PVDM runtime (no AWS credentials required)."""
+
+from serverless_data_mesh.local.runtime import LocalPVDMRuntime, LocalWriteResult
+
+__all__ = ["LocalPVDMRuntime", "LocalWriteResult"]