agentflow-runtime 1.1.0__py3-none-any.whl

src/quality/monitors/metrics_collector.py

@@ -0,0 +1,367 @@
+"""Collects and exposes pipeline health metrics.
+
+Aggregates metrics from Kafka consumer groups, Flink jobs,
+and quality checks into a unified health status.
+"""
+
+import os
+from dataclasses import dataclass
+from datetime import UTC, datetime
+from enum import StrEnum
+from pathlib import Path
+
+import duckdb
+import httpx
+import structlog
+import yaml  # type: ignore[import-untyped]
+from confluent_kafka import KafkaException
+from prometheus_client import Gauge
+from pyiceberg.exceptions import NoSuchPropertyException, RESTError, ValidationError
+
+logger = structlog.get_logger()
+
+
+class HealthStatus(StrEnum):
+    HEALTHY = "healthy"
+    DEGRADED = "degraded"
+    UNHEALTHY = "unhealthy"
+
+
+PIPELINE_HEALTH = Gauge(
+    "agentflow_pipeline_health",
+    "Pipeline health status (1=healthy, 0.5=degraded, 0=unhealthy)",
+    ["component"],
+)
+
+CONSUMER_LAG = Gauge(
+    "agentflow_consumer_lag",
+    "Kafka consumer group lag",
+    ["group_id", "topic", "partition"],
+)
+
+
+class CheckSource(StrEnum):
+    LIVE = "live"
+    PLACEHOLDER = "placeholder"
+
+
+@dataclass
+class ComponentHealth:
+    name: str
+    status: HealthStatus
+    message: str
+    last_check: datetime
+    metrics: dict
+    source: CheckSource = CheckSource.LIVE
+
+
+@dataclass
+class PipelineHealth:
+    overall: HealthStatus
+    components: list[ComponentHealth]
+    checked_at: datetime
+
+    def to_dict(self) -> dict:
+        return {
+            "status": self.overall,
+            "checked_at": self.checked_at.isoformat(),
+            "components": [
+                {
+                    "name": c.name,
+                    "status": c.status,
+                    "message": c.message,
+                    "metrics": c.metrics,
+                    "source": c.source.value,
+                }
+                for c in self.components
+            ],
+        }
+
+
+class HealthCollector:
+    """Aggregates health from all pipeline components."""
+
+    def __init__(self):
+        self._checks: list = [
+            self._check_kafka,
+            self._check_flink,
+            self._check_freshness,
+            self._check_quality_score,
+            self._check_iceberg,
+        ]
+
+    def collect(self) -> PipelineHealth:
+        components = []
+        for check in self._checks:
+            components.append(check())
+
+        # Overall status: worst component determines it
+        statuses = [c.status for c in components]
+        if HealthStatus.UNHEALTHY in statuses:
+            overall = HealthStatus.UNHEALTHY
+        elif HealthStatus.DEGRADED in statuses:
+            overall = HealthStatus.DEGRADED
+        else:
+            overall = HealthStatus.HEALTHY
+
+        for c in components:
+            val = {"healthy": 1.0, "degraded": 0.5, "unhealthy": 0.0}[c.status]
+            PIPELINE_HEALTH.labels(component=c.name).set(val)
+
+        return PipelineHealth(
+            overall=overall,
+            components=components,
+            checked_at=datetime.now(UTC),
+        )
+
+    def _check_kafka(self) -> ComponentHealth:
+        """Check Kafka broker connectivity and consumer lag."""
+        from confluent_kafka.admin import AdminClient
+
+        bootstrap = os.getenv("KAFKA_BOOTSTRAP_SERVERS", "localhost:9092")
+        try:
+            admin = AdminClient({"bootstrap.servers": bootstrap})
+            cluster_meta = admin.list_topics(timeout=5)
+        except (KafkaException, OSError) as exc:
+            logger.warning(
+                "kafka_check_unavailable",
+                bootstrap_servers=bootstrap,
+                error=str(exc),
+                exc_info=True,
+            )
+            return ComponentHealth(
+                name="kafka",
+                status=HealthStatus.UNHEALTHY,
+                message=f"Kafka unavailable: {exc}",
+                last_check=datetime.now(UTC),
+                metrics={"brokers": 0, "topics": 0},
+                source=CheckSource.PLACEHOLDER,
+            )
+        topic_count = len(cluster_meta.topics)
+        broker_count = len(cluster_meta.brokers)
+
+        if broker_count == 0:
+            return ComponentHealth(
+                name="kafka",
+                status=HealthStatus.UNHEALTHY,
+                message="No brokers available",
+                last_check=datetime.now(UTC),
+                metrics={"brokers": 0},
+            )
+
+        return ComponentHealth(
+            name="kafka",
+            status=HealthStatus.HEALTHY,
+            message=f"{broker_count} brokers, {topic_count} topics",
+            last_check=datetime.now(UTC),
+            metrics={"brokers": broker_count, "topics": topic_count},
+        )
+
+    def _check_flink(self) -> ComponentHealth:
+        """Check Flink JobManager and running jobs."""
+        flink_url = os.getenv("FLINK_JOBMANAGER_URL", "http://localhost:8081")
+        try:
+            resp = httpx.get(f"{flink_url}/overview", timeout=5)
+            resp.raise_for_status()
+            data = resp.json()
+        except (httpx.HTTPError, ValueError) as exc:
+            logger.warning(
+                "flink_check_unavailable",
+                flink_url=flink_url,
+                error=str(exc),
+                exc_info=True,
+            )
+            return ComponentHealth(
+                name="flink",
+                status=HealthStatus.UNHEALTHY,
+                message=f"Flink unavailable: {exc}",
+                last_check=datetime.now(UTC),
+                metrics={"running_jobs": None, "failed_jobs": None},
+                source=CheckSource.PLACEHOLDER,
+            )
+
+        running = data.get("jobs-running", 0)
+        failed = data.get("jobs-failed", 0)
+
+        if failed > 0:
+            status = HealthStatus.DEGRADED
+            msg = f"{running} running, {failed} failed"
+        elif running == 0:
+            status = HealthStatus.DEGRADED
+            msg = "No running jobs"
+        else:
+            status = HealthStatus.HEALTHY
+            msg = f"{running} jobs running"
+
+        return ComponentHealth(
+            name="flink",
+            status=status,
+            message=msg,
+            last_check=datetime.now(UTC),
+            metrics={"running_jobs": running, "failed_jobs": failed},
+        )
+
+    def _check_freshness(self) -> ComponentHealth:
+        """Check data freshness from the most recent pipeline event."""
+        try:
+            db_path = os.getenv("DUCKDB_PATH", "agentflow_demo.duckdb")
+            conn = duckdb.connect(db_path, read_only=True)
+            row = conn.execute("SELECT MAX(processed_at) FROM pipeline_events").fetchone()
+            conn.close()
+
+            if row and row[0]:
+                last_event = row[0]
+                if hasattr(last_event, "timestamp"):
+                    age_s = (datetime.now(UTC) - last_event.replace(tzinfo=UTC)).total_seconds()
+                else:
+                    age_s = -1.0
+
+                sla = int(os.getenv("FRESHNESS_SLA_SECONDS", "30"))
+                if age_s <= sla:
+                    status = HealthStatus.HEALTHY
+                    msg = f"Last event {age_s:.0f}s ago (SLA: {sla}s)"
+                elif age_s <= sla * 3:
+                    status = HealthStatus.DEGRADED
+                    msg = f"Last event {age_s:.0f}s ago (SLA: {sla}s)"
+                else:
+                    status = HealthStatus.UNHEALTHY
+                    msg = f"Last event {age_s:.0f}s ago (SLA: {sla}s)"
+
+                return ComponentHealth(
+                    name="freshness",
+                    status=status,
+                    message=msg,
+                    last_check=datetime.now(UTC),
+                    metrics={
+                        "last_event_age_seconds": round(age_s, 1),
+                        "sla_seconds": sla,
+                    },
+                    source=CheckSource.LIVE,
+                )
+        except duckdb.Error as exc:
+            logger.warning(
+                "freshness_check_unavailable",
+                db_path=db_path,
+                error=str(exc),
+                exc_info=True,
+            )
+
+        return ComponentHealth(
+            name="freshness",
+            status=HealthStatus.DEGRADED,
+            message="No pipeline events found (run local pipeline first)",
+            last_check=datetime.now(UTC),
+            metrics={"last_event_age_seconds": None},
+            source=CheckSource.PLACEHOLDER,
+        )
+
+    def _check_quality_score(self) -> ComponentHealth:
+        """Check data quality from dead letter ratio in pipeline events."""
+        try:
+            db_path = os.getenv("DUCKDB_PATH", "agentflow_demo.duckdb")
+            conn = duckdb.connect(db_path, read_only=True)
+            row = conn.execute("""
+                SELECT
+                    COUNT(*) as total,
+                    COUNT(*) FILTER (
+                        WHERE topic = 'events.deadletter'
+                    ) as dead
+                FROM pipeline_events
+                WHERE processed_at >= NOW() - INTERVAL '1 hour'
+            """).fetchone()
+            conn.close()
+
+            if row and row[0] and row[0] > 0:
+                total, dead = row[0], row[1]
+                pass_rate = (total - dead) / total
+                if pass_rate >= 0.99:
+                    status = HealthStatus.HEALTHY
+                elif pass_rate >= 0.95:
+                    status = HealthStatus.DEGRADED
+                else:
+                    status = HealthStatus.UNHEALTHY
+
+                return ComponentHealth(
+                    name="quality",
+                    status=status,
+                    message=f"Pass rate: {pass_rate:.1%} ({dead}/{total} rejected)",
+                    last_check=datetime.now(UTC),
+                    metrics={
+                        "pass_rate": round(pass_rate, 4),
+                        "total_events": total,
+                        "rejected_events": dead,
+                    },
+                    source=CheckSource.LIVE,
+                )
+        except duckdb.Error as exc:
+            logger.warning(
+                "quality_check_unavailable",
+                db_path=db_path,
+                error=str(exc),
+                exc_info=True,
+            )
+
+        return ComponentHealth(
+            name="quality",
+            status=HealthStatus.DEGRADED,
+            message="No pipeline events found (run local pipeline first)",
+            last_check=datetime.now(UTC),
+            metrics={"pass_rate": None},
+            source=CheckSource.PLACEHOLDER,
+        )
+
+    def _check_iceberg(self) -> ComponentHealth:
+        """Check Iceberg catalog accessibility and row counts."""
+        config_path = Path(os.getenv("AGENTFLOW_ICEBERG_CONFIG", "config/iceberg.yaml"))
+        if not config_path.exists():
+            return ComponentHealth(
+                name="iceberg",
+                status=HealthStatus.DEGRADED,
+                message="Iceberg config not found",
+                last_check=datetime.now(UTC),
+                metrics={"row_counts": {}},
+                source=CheckSource.PLACEHOLDER,
+            )
+
+        try:
+            from src.processing.iceberg_sink import IcebergSink
+
+            sink = IcebergSink(config_path=config_path)
+            row_counts = sink.row_counts()
+        except (
+            ImportError,
+            OSError,
+            KeyError,
+            ValueError,
+            yaml.YAMLError,
+            NoSuchPropertyException,
+            RESTError,
+            ValidationError,
+        ) as exc:
+            logger.warning(
+                "iceberg_check_unavailable",
+                config_path=str(config_path),
+                error=str(exc),
+                exc_info=True,
+            )
+            return ComponentHealth(
+                name="iceberg",
+                status=HealthStatus.DEGRADED,
+                message=f"Iceberg unavailable: {exc}",
+                last_check=datetime.now(UTC),
+                metrics={"row_counts": {}},
+                source=CheckSource.PLACEHOLDER,
+            )
+
+        total_rows = sum(row_counts.values())
+        return ComponentHealth(
+            name="iceberg",
+            status=HealthStatus.HEALTHY,
+            message=f"{len(row_counts)} tables, {total_rows} rows",
+            last_check=datetime.now(UTC),
+            metrics={
+                "row_counts": row_counts,
+                "total_rows": total_rows,
+            },
+            source=CheckSource.LIVE,
+        )

src/quality/validators/schema_validator.py

@@ -0,0 +1,119 @@
+"""Schema validation for incoming events.
+
+Validates events against their Pydantic schemas before they enter the storage layer.
+Returns structured validation results with error details for observability.
+"""
+
+from dataclasses import dataclass, field
+from datetime import UTC, datetime
+
+from pydantic import ValidationError
+
+from src.ingestion.schemas.events import (
+    CdcEvent,
+    ClickstreamEvent,
+    OrderEvent,
+    PaymentEvent,
+    ProductEvent,
+)
+
+
+@dataclass
+class ValidationResult:
+    is_valid: bool
+    event_id: str
+    event_type: str
+    errors: list[dict] = field(default_factory=list)
+    validated_at: datetime = field(default_factory=lambda: datetime.now(UTC))
+
+    def to_dict(self) -> dict:
+        return {
+            "is_valid": self.is_valid,
+            "event_id": self.event_id,
+            "event_type": self.event_type,
+            "errors": self.errors,
+            "validated_at": self.validated_at.isoformat(),
+        }
+
+
+# Map event type prefixes to their Pydantic models
+_SCHEMA_MAP: dict = {
+    "order.": OrderEvent,
+    "payment.": PaymentEvent,
+    "click": ClickstreamEvent,
+    "page_view": ClickstreamEvent,
+    "add_to_cart": ClickstreamEvent,
+    "product.": ProductEvent,
+}
+
+_CDC_SOURCES = {"postgres_cdc", "mysql_cdc"}
+
+
+def _get_model_for_event(event_type: str):  # -> BaseModel subclass | None
+    for prefix, model in _SCHEMA_MAP.items():
+        if event_type.startswith(prefix) or event_type == prefix:
+            return model
+    return None
+
+
+def validate_event(raw_event: dict) -> ValidationResult:
+    """Validate a single event against its schema.
+
+    Args:
+        raw_event: Raw event dict (already parsed from JSON).
+
+    Returns:
+        ValidationResult with is_valid=True if the event passes,
+        or is_valid=False with structured error details.
+    """
+    event_id = raw_event.get("event_id", "unknown")
+    event_type = raw_event.get("event_type", "unknown")
+
+    model = CdcEvent if _is_cdc_event(raw_event) else _get_model_for_event(event_type)
+    if model is None:
+        return ValidationResult(
+            is_valid=False,
+            event_id=event_id,
+            event_type=event_type,
+            errors=[{"type": "unknown_event_type", "msg": f"No schema for: {event_type}"}],
+        )
+
+    try:
+        model.model_validate(raw_event)
+        return ValidationResult(is_valid=True, event_id=event_id, event_type=event_type)
+    except ValidationError as e:
+        errors = [
+            {
+                "type": err["type"],
+                "loc": list(err["loc"]),
+                "msg": err["msg"],
+            }
+            for err in e.errors()
+        ]
+        return ValidationResult(
+            is_valid=False,
+            event_id=event_id,
+            event_type=event_type,
+            errors=errors,
+        )
+
+
+def _is_cdc_event(raw_event: dict) -> bool:
+    return (
+        raw_event.get("source") in _CDC_SOURCES
+        and "operation" in raw_event
+        and "source_metadata" in raw_event
+    )
+
+
+def validate_batch(events: list[dict]) -> tuple[list[dict], list[ValidationResult]]:
+    """Validate a batch of events. Returns (valid_events, failed_results)."""
+    valid = []
+    failed = []
+    for event in events:
+        result = validate_event(event)
+        if result.is_valid:
+            valid.append(event)
+        else:
+            failed.append(result)
+    return valid, failed

src/quality/validators/semantic_validator.py

@@ -0,0 +1,202 @@
+"""Semantic validation: business rules that go beyond schema correctness.
+
+Schema validation checks structure. Semantic validation checks meaning:
+- Does the order total actually match line items?
+- Is the payment amount within reasonable bounds?
+- Does the user_id reference a plausible user?
+
+These rules catch data quality issues that pass schema validation
+but would cause AI agents to give wrong answers.
+"""
+
+import argparse
+from dataclasses import dataclass, field
+from datetime import UTC, datetime
+from decimal import Decimal
+
+
+@dataclass
+class SemanticIssue:
+    rule: str
+    severity: str  # "error" | "warning"
+    field: str
+    message: str
+    actual_value: str | None = None
+    expected: str | None = None
+
+
+@dataclass
+class SemanticResult:
+    event_id: str
+    event_type: str
+    is_clean: bool
+    issues: list[SemanticIssue] = field(default_factory=list)
+    checked_at: datetime = field(default_factory=lambda: datetime.now(UTC))
+
+    def to_dict(self) -> dict:
+        return {
+            "event_id": self.event_id,
+            "event_type": self.event_type,
+            "is_clean": self.is_clean,
+            "issues": [
+                {
+                    "rule": i.rule,
+                    "severity": i.severity,
+                    "field": i.field,
+                    "message": i.message,
+                }
+                for i in self.issues
+            ],
+            "checked_at": self.checked_at.isoformat(),
+        }
+
+
+# ── Rule definitions ────────────────────────────────────────────
+
+
+def _check_order_total_consistency(event: dict) -> list[SemanticIssue]:
+    """Order total must match sum of (quantity * unit_price) for all items."""
+    issues = []
+    items = event.get("items", [])
+    stated_total = Decimal(str(event.get("total_amount", 0)))
+
+    computed_total = sum(
+        Decimal(str(i.get("quantity", 0))) * Decimal(str(i.get("unit_price", 0))) for i in items
+    )
+
+    if abs(stated_total - computed_total) > Decimal("0.01"):
+        issues.append(
+            SemanticIssue(
+                rule="order_total_consistency",
+                severity="error",
+                field="total_amount",
+                message=f"Stated total {stated_total} != computed {computed_total}",
+                actual_value=str(stated_total),
+                expected=str(computed_total),
+            )
+        )
+    return issues
+
+
+def _check_payment_amount_bounds(event: dict) -> list[SemanticIssue]:
+    """Payment amount should be between $0.50 and $50,000."""
+    issues = []
+    amount = Decimal(str(event.get("amount", 0)))
+
+    if amount < Decimal("0.50"):
+        issues.append(
+            SemanticIssue(
+                rule="payment_min_amount",
+                severity="error",
+                field="amount",
+                message=f"Payment amount {amount} below minimum $0.50",
+                actual_value=str(amount),
+            )
+        )
+    elif amount > Decimal("50000"):
+        issues.append(
+            SemanticIssue(
+                rule="payment_max_amount",
+                severity="warning",
+                field="amount",
+                message=f"Payment amount {amount} exceeds $50,000 — needs manual review",
+                actual_value=str(amount),
+            )
+        )
+    return issues
+
+
+def _check_payment_failure_reason(event: dict) -> list[SemanticIssue]:
+    """Failed payments must have a failure_reason."""
+    issues = []
+    if event.get("status") == "failed" and not event.get("failure_reason"):
+        issues.append(
+            SemanticIssue(
+                rule="payment_failure_reason_required",
+                severity="warning",
+                field="failure_reason",
+                message="Failed payment missing failure_reason",
+            )
+        )
+    return issues
+
+
+def _check_clickstream_session_id(event: dict) -> list[SemanticIssue]:
+    """Clickstream events must have a session_id."""
+    issues = []
+    if not event.get("session_id"):
+        issues.append(
+            SemanticIssue(
+                rule="clickstream_session_required",
+                severity="error",
+                field="session_id",
+                message="Clickstream event missing session_id",
+            )
+        )
+    return issues
+
+
+def _check_product_price_sanity(event: dict) -> list[SemanticIssue]:
+    """Product price should be between $0 and $100,000."""
+    issues = []
+    price = Decimal(str(event.get("price", 0)))
+    if price > Decimal("100000"):
+        issues.append(
+            SemanticIssue(
+                rule="product_price_sanity",
+                severity="warning",
+                field="price",
+                message=f"Product price {price} seems unreasonably high",
+                actual_value=str(price),
+            )
+        )
+    return issues
+
+
+# ── Rule registry ───────────────────────────────────────────────
+
+_RULES: dict[str, list] = {
+    "order.": [_check_order_total_consistency],
+    "payment.": [_check_payment_amount_bounds, _check_payment_failure_reason],
+    "click": [_check_clickstream_session_id],
+    "page_view": [_check_clickstream_session_id],
+    "add_to_cart": [_check_clickstream_session_id],
+    "product.": [_check_product_price_sanity],
+}
+
+
+def validate_semantics(event: dict) -> SemanticResult:
+    """Run all applicable semantic rules on an event."""
+    event_id = event.get("event_id", "unknown")
+    event_type = event.get("event_type", "unknown")
+    all_issues: list[SemanticIssue] = []
+
+    for prefix, rules in _RULES.items():
+        if event_type.startswith(prefix) or event_type == prefix:
+            for rule_fn in rules:
+                all_issues.extend(rule_fn(event))
+
+    has_errors = any(i.severity == "error" for i in all_issues)
+
+    return SemanticResult(
+        event_id=event_id,
+        event_type=event_type,
+        is_clean=not has_errors,
+        issues=all_issues,
+    )
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser(description="Run semantic validation checks")
+    parser.add_argument("--check-all", action="store_true", help="Run all checks on sample data")
+    args = parser.parse_args()
+
+    if args.check_all:
+        sample_order = {
+            "event_id": "test-001",
+            "event_type": "order.created",
+            "total_amount": "100.00",
+            "items": [{"quantity": 2, "unit_price": "50.00", "product_id": "P1"}],
+        }
+        result = validate_semantics(sample_order)
+        print(f"Order check: is_clean={result.is_clean}, issues={len(result.issues)}")