agentflow-runtime 1.1.0__py3-none-any.whl

src/processing/outbox.py

@@ -0,0 +1,273 @@
+from __future__ import annotations
+
+import asyncio
+import json
+import os
+from collections.abc import Callable
+from contextlib import nullcontext
+from datetime import UTC, datetime, timedelta
+
+import duckdb
+import structlog
+from confluent_kafka import KafkaException
+from opentelemetry import trace
+
+from src.processing.tracing import inject_trace_to_kafka_headers, telemetry_disabled
+
+logger = structlog.get_logger()
+tracer = trace.get_tracer("agentflow.outbox")
+
+DEFAULT_KAFKA_BOOTSTRAP = os.getenv("KAFKA_BOOTSTRAP_SERVERS", "localhost:9092")
+
+
+def ensure_outbox_table(conn) -> None:
+    conn.execute(
+        """
+        CREATE TABLE IF NOT EXISTS outbox (
+            id TEXT PRIMARY KEY,
+            event_id TEXT NOT NULL,
+            payload JSON NOT NULL,
+            topic TEXT NOT NULL,
+            created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
+            sent_at TIMESTAMP,
+            status TEXT DEFAULT 'pending',
+            retry_count INTEGER DEFAULT 0,
+            next_attempt_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
+            last_error TEXT
+        )
+        """
+    )
+
+
+class OutboxProcessor:
+    def __init__(
+        self,
+        duckdb_path: str | None = None,
+        conn=None,
+        producer: Callable[[str, dict], None] | None = None,
+        bootstrap_servers: str | None = None,
+        max_retries: int = 5,
+    ) -> None:
+        if conn is None and duckdb_path is None:
+            raise ValueError("duckdb_path or conn is required")
+        self._owns_conn = conn is None
+        self._conn = conn if conn is not None else duckdb.connect(str(duckdb_path))
+        self._producer = producer or self._produce_to_kafka
+        self._bootstrap_servers = bootstrap_servers or DEFAULT_KAFKA_BOOTSTRAP
+        self._max_retries = max_retries
+        ensure_outbox_table(self._conn)
+
+    def close(self) -> None:
+        if self._owns_conn and self._conn is not None:
+            self._conn.close()
+            self._conn = None
+
+    async def run_forever(self) -> None:
+        try:
+            while True:
+                await asyncio.sleep(2)
+                try:
+                    self.process_pending()
+                except duckdb.Error as exc:
+                    logger.warning(
+                        "outbox_processing_failed",
+                        error=str(exc),
+                        bootstrap_servers=self._bootstrap_servers,
+                        owns_connection=self._owns_conn,
+                        exc_info=True,
+                    )
+        finally:
+            self.close()
+
+    def process_pending(self, limit: int = 100) -> int:
+        rows = self._conn.execute(
+            """
+            SELECT id, event_id, payload, topic, retry_count
+            FROM outbox
+            WHERE status = 'pending'
+              AND (next_attempt_at IS NULL OR next_attempt_at <= ?)
+            ORDER BY created_at
+            LIMIT ?
+            """,
+            [datetime.now(UTC), limit],
+        ).fetchall()
+        processed = 0
+        for row in rows:
+            if self._process_row(row):
+                processed += 1
+        return processed
+
+    def process_entry(self, outbox_id: str) -> bool:
+        row = self._conn.execute(
+            """
+            SELECT id, event_id, payload, topic, retry_count
+            FROM outbox
+            WHERE id = ?
+              AND status = 'pending'
+            """,
+            [outbox_id],
+        ).fetchone()
+        if row is None:
+            return False
+        return self._process_row(row)
+
+    def _process_row(self, row) -> bool:
+        outbox_id, event_id, payload, topic, retry_count = row
+        decoded_payload = self._decode_payload(payload)
+        try:
+            self._producer(topic, decoded_payload)
+        except (BufferError, ConnectionError, TimeoutError, KafkaException, RuntimeError) as exc:
+            error_message = str(exc)
+            if isinstance(exc, RuntimeError) and not (
+                error_message.startswith("KafkaError{")
+                or "Kafka message(s) were not delivered" in error_message
+            ):
+                raise
+            next_retry_count = int(retry_count or 0) + 1
+            logger.warning(
+                "outbox_delivery_retry_scheduled",
+                outbox_id=outbox_id,
+                event_id=event_id,
+                topic=topic,
+                retry_count=next_retry_count,
+                error=error_message,
+                exc_info=True,
+            )
+            self._schedule_retry(
+                outbox_id=outbox_id,
+                event_id=event_id,
+                retry_count=next_retry_count,
+                error_message=error_message,
+            )
+            return False
+        self._mark_sent(outbox_id=outbox_id, event_id=event_id)
+        return True
+
+    def _mark_sent(self, outbox_id: str, event_id: str) -> None:
+        sent_at = datetime.now(UTC)
+        self._conn.execute("BEGIN TRANSACTION")
+        try:
+            self._conn.execute(
+                """
+                UPDATE outbox
+                SET status = 'sent',
+                    sent_at = ?,
+                    last_error = NULL
+                WHERE id = ?
+                """,
+                [sent_at, outbox_id],
+            )
+            self._conn.execute(
+                "UPDATE dead_letter_events SET status = 'replayed' WHERE event_id = ?",
+                [event_id],
+            )
+            self._conn.execute("COMMIT")
+        except Exception:  # nosec B110 - rollback must preserve the original replay failure
+            # Transaction rollback must happen before unexpected errors propagate.
+            self._conn.execute("ROLLBACK")
+            raise
+
+    def _schedule_retry(
+        self,
+        outbox_id: str,
+        event_id: str,
+        retry_count: int,
+        error_message: str,
+    ) -> None:
+        status = "pending"
+        retry_delay_seconds = 2**retry_count
+        is_kafka_error = (
+            error_message.startswith("KafkaError{")
+            or "Kafka message(s) were not delivered" in error_message
+        )
+        if is_kafka_error:
+            retry_delay_seconds = max(retry_delay_seconds, 30)
+        next_attempt_at: datetime | None = datetime.now(UTC) + timedelta(
+            seconds=retry_delay_seconds
+        )
+        self._conn.execute("BEGIN TRANSACTION")
+        try:
+            if retry_count >= self._max_retries:
+                status = "failed"
+                next_attempt_at = None
+            self._conn.execute(
+                """
+                UPDATE outbox
+                SET status = ?,
+                    retry_count = ?,
+                    next_attempt_at = ?,
+                    last_error = ?
+                WHERE id = ?
+                """,
+                [status, retry_count, next_attempt_at, error_message, outbox_id],
+            )
+            if status == "failed":
+                self._conn.execute(
+                    "UPDATE dead_letter_events SET status = 'failed' WHERE event_id = ?",
+                    [event_id],
+                )
+            self._conn.execute("COMMIT")
+        except Exception:  # nosec B110 - rollback must preserve the original retry scheduling failure
+            # Transaction rollback must happen before unexpected errors propagate.
+            self._conn.execute("ROLLBACK")
+            raise
+
+    def _decode_payload(self, payload) -> dict:
+        if isinstance(payload, dict):
+            return payload
+        if isinstance(payload, str):
+            decoded = json.loads(payload)
+            if isinstance(decoded, dict):
+                return decoded
+        raise ValueError("Outbox payload must be a JSON object.")
+
+    def _produce_to_kafka(self, topic: str, payload: dict) -> None:
+        from confluent_kafka import Producer
+
+        delivery_errors: list[str] = []
+
+        def on_delivery(err, msg) -> None:
+            del msg
+            if err is not None:
+                delivery_errors.append(str(err))
+
+        producer = Producer({"bootstrap.servers": self._bootstrap_servers})
+        produce_span = (
+            tracer.start_as_current_span("kafka.produce")
+            if not telemetry_disabled()
+            else nullcontext()
+        )
+        with produce_span as span:
+            if span is not None and span.is_recording():
+                span.set_attribute("topic", topic)
+                event_type = payload.get("event_type")
+                if event_type is not None:
+                    span.set_attribute("event_type", str(event_type))
+                tenant_id = payload.get("tenant_id") or structlog.contextvars.get_contextvars().get(
+                    "tenant_id"
+                )
+                if tenant_id is not None:
+                    span.set_attribute("tenant_id", str(tenant_id))
+            headers = inject_trace_to_kafka_headers({})
+            try:
+                producer.produce(
+                    topic,
+                    key=str(payload.get("event_id", "")),
+                    value=json.dumps(payload).encode("utf-8"),
+                    headers=list(headers.items()) or None,
+                    on_delivery=on_delivery,
+                )
+            except TypeError as exc:
+                if "on_delivery" not in str(exc):
+                    raise
+                producer.produce(
+                    topic,
+                    key=str(payload.get("event_id", "")),
+                    value=json.dumps(payload).encode("utf-8"),
+                    headers=list(headers.items()) or None,
+                )
+            remaining = producer.flush(10)
+            if delivery_errors:
+                raise RuntimeError(delivery_errors[0])
+            if remaining != 0:
+                raise RuntimeError(f"{remaining} Kafka message(s) were not delivered")

src/processing/tracing.py

@@ -0,0 +1,36 @@
+from __future__ import annotations
+
+import os
+from collections.abc import Mapping, Sequence
+
+from opentelemetry.context import Context
+from opentelemetry.propagate import extract, inject
+
+
+def telemetry_disabled() -> bool:
+    return os.getenv("OTEL_SDK_DISABLED", "").lower() == "true"
+
+
+def inject_trace_to_kafka_headers(
+    headers: Mapping[str, bytes] | None = None,
+) -> dict[str, bytes]:
+    injected_headers = dict(headers or {})
+    if telemetry_disabled():
+        return injected_headers
+
+    carrier: dict[str, str] = {}
+    inject(carrier)
+    for key, value in carrier.items():
+        injected_headers[key] = value.encode("utf-8")
+    return injected_headers
+
+
+def extract_trace_from_kafka_headers(
+    headers: Mapping[str, bytes] | Sequence[tuple[str, bytes]] | None,
+) -> Context:
+    if telemetry_disabled() or headers is None:
+        return extract({})
+
+    items = headers.items() if isinstance(headers, Mapping) else headers
+    carrier = {str(key): value.decode("utf-8") for key, value in items if isinstance(value, bytes)}
+    return extract(carrier)

src/processing/transformations/enrichment.py

@@ -0,0 +1,125 @@
+"""Event enrichment functions for the processing layer.
+
+Pure functions that add derived fields to events. Used by Flink jobs
+and batch transformations alike — keeping logic DRY across streaming and batch.
+"""
+
+from decimal import Decimal
+
+ORDER_SIZE_SMALL_MAX_TOTAL = Decimal("50")
+ORDER_SIZE_MEDIUM_MAX_TOTAL = Decimal("200")
+ORDER_SIZE_LARGE_MAX_TOTAL = Decimal("1000")
+MOBILE_VIEWPORT_MAX_WIDTH = 768
+PAYMENT_RISK_HIGH_AMOUNT_THRESHOLD = 500
+PAYMENT_RISK_MEDIUM_AMOUNT_THRESHOLD = 200
+PAYMENT_RISK_HIGH_SCORE_THRESHOLD = 0.5
+PAYMENT_RISK_MEDIUM_SCORE_THRESHOLD = 0.2
+
+
+def enrich_order(event: dict) -> dict:
+    """Add derived fields to an order event.
+
+    Adds:
+    - item_count: total number of items
+    - unique_products: number of distinct products
+    - avg_item_price: average price per item
+    - order_size_bucket: small/medium/large/whale
+    """
+    items = event.get("items", [])
+    total = Decimal(str(event.get("total_amount", 0)))
+
+    item_count = sum(i.get("quantity", 0) for i in items)
+    unique_products = len({i["product_id"] for i in items if "product_id" in i})
+    avg_price = total / item_count if item_count > 0 else Decimal("0")
+
+    if total < ORDER_SIZE_SMALL_MAX_TOTAL:
+        bucket = "small"
+    elif total < ORDER_SIZE_MEDIUM_MAX_TOTAL:
+        bucket = "medium"
+    elif total < ORDER_SIZE_LARGE_MAX_TOTAL:
+        bucket = "large"
+    else:
+        bucket = "whale"
+
+    event["_derived"] = {
+        "item_count": item_count,
+        "unique_products": unique_products,
+        "avg_item_price": float(avg_price.quantize(Decimal("0.01"))),
+        "order_size_bucket": bucket,
+    }
+    return event
+
+
+def enrich_clickstream(event: dict) -> dict:
+    """Add derived fields to a clickstream event.
+
+    Adds:
+    - is_mobile: viewport < 768px
+    - page_category: derived from URL path
+    - is_product_page: bool
+    """
+    viewport = event.get("viewport_width")
+    page_url = event.get("page_url", "")
+
+    if "/products/" in page_url:
+        page_category = "product_detail"
+        is_product_page = True
+    elif "/cart" in page_url:
+        page_category = "cart"
+        is_product_page = False
+    elif "/checkout" in page_url:
+        page_category = "checkout"
+        is_product_page = False
+    elif "/search" in page_url:
+        page_category = "search"
+        is_product_page = False
+    elif page_url == "/":
+        page_category = "home"
+        is_product_page = False
+    else:
+        page_category = "other"
+        is_product_page = False
+
+    event["_derived"] = {
+        "is_mobile": viewport is not None and viewport < MOBILE_VIEWPORT_MAX_WIDTH,
+        "page_category": page_category,
+        "is_product_page": is_product_page,
+    }
+    return event
+
+
+def compute_payment_risk_score(event: dict) -> dict:
+    """Add a simple fraud risk score to payment events.
+
+    Heuristic scoring (0.0 - 1.0):
+    - High amount → higher risk
+    - Bank transfer → lower risk than card
+    - Missing user_id → higher risk
+    """
+    score = 0.0
+    amount = float(event.get("amount", 0))
+
+    if amount > PAYMENT_RISK_HIGH_AMOUNT_THRESHOLD:
+        score += 0.3
+    elif amount > PAYMENT_RISK_MEDIUM_AMOUNT_THRESHOLD:
+        score += 0.1
+
+    if event.get("method") == "card":
+        score += 0.1
+    elif event.get("method") == "wallet":
+        score += 0.15
+
+    if not event.get("user_id"):
+        score += 0.3
+
+    event["_derived"] = {
+        "risk_score": min(score, 1.0),
+        "risk_level": (
+            "high"
+            if score > PAYMENT_RISK_HIGH_SCORE_THRESHOLD
+            else "medium"
+            if score >= PAYMENT_RISK_MEDIUM_SCORE_THRESHOLD
+            else "low"
+        ),
+    }
+    return event

src/quality/monitors/freshness_monitor.py

@@ -0,0 +1,166 @@
+"""Monitors data freshness across all pipeline stages.
+
+Checks that events flow through the pipeline within SLA bounds.
+Exposes metrics to Prometheus and triggers alerts on SLA breaches.
+
+SLA: end-to-end latency (ingestion → serving) < 30 seconds for p99.
+"""
+
+import json
+import os
+from collections import defaultdict
+from datetime import UTC, datetime
+
+import structlog
+from confluent_kafka import Consumer, KafkaError
+from prometheus_client import Gauge, Histogram, start_http_server
+
+logger = structlog.get_logger()
+
+FRESHNESS_SLA_SECONDS = int(os.getenv("FRESHNESS_SLA_SECONDS", "30"))
+
+# Prometheus metrics
+PIPELINE_LATENCY = Histogram(
+    "agentflow_pipeline_latency_seconds",
+    "End-to-end pipeline latency in seconds",
+    ["topic", "event_type"],
+    buckets=[0.1, 0.5, 1.0, 2.0, 5.0, 10.0, 30.0, 60.0],
+)
+
+SLA_COMPLIANCE = Gauge(
+    "agentflow_sla_compliance_ratio",
+    "Ratio of events within SLA (rolling 5-min window)",
+    ["topic"],
+)
+
+EVENTS_PROCESSED = Gauge(
+    "agentflow_freshness_events_total",
+    "Total events checked by freshness monitor",
+    ["topic"],
+)
+
+
+class FreshnessMonitor:
+    """Consumes from validated topics and measures pipeline latency."""
+
+    def __init__(self, bootstrap_servers: str, topics: list[str]):
+        self.consumer = Consumer(
+            {
+                "bootstrap.servers": bootstrap_servers,
+                "group.id": "agentflow-freshness-monitor",
+                "auto.offset.reset": "latest",
+                "enable.auto.commit": True,
+            }
+        )
+        self.topics = topics
+        self._sla_window: dict[str, list[bool]] = defaultdict(list)
+        self._window_size = 1000  # last N events per topic
+
+    def start(self, metrics_port: int = 8001):
+        """Start monitoring loop with Prometheus metrics endpoint."""
+        start_http_server(metrics_port)
+        logger.info("freshness_monitor_started", topics=self.topics, port=metrics_port)
+
+        self.consumer.subscribe(self.topics)
+
+        try:
+            while True:
+                msg = self.consumer.poll(timeout=1.0)
+                if msg is None:
+                    continue
+                err = msg.error()
+                if err:
+                    if err.code() != KafkaError._PARTITION_EOF:
+                        logger.error("kafka_error", error=str(err))
+                    continue
+
+                self._process_message(msg)
+        except KeyboardInterrupt:
+            logger.info("freshness_monitor_stopping")
+        finally:
+            self.consumer.close()
+
+    def _process_message(self, msg):
+        topic = msg.topic()
+        try:
+            event = json.loads(msg.value().decode())
+        except (json.JSONDecodeError, UnicodeDecodeError) as exc:
+            logger.warning(
+                "freshness_message_skipped",
+                topic=topic,
+                partition=msg.partition(),
+                offset=msg.offset(),
+                reason="invalid_payload",
+                error=str(exc),
+                exc_info=True,
+            )
+            return
+
+        # Calculate latency from event timestamp to now
+        event_ts_str = event.get("timestamp")
+        if not event_ts_str:
+            logger.warning(
+                "freshness_message_skipped",
+                topic=topic,
+                partition=msg.partition(),
+                offset=msg.offset(),
+                reason="missing_timestamp",
+                event_id=event.get("event_id"),
+                event_type=event.get("event_type", "unknown"),
+            )
+            return
+
+        try:
+            event_ts = datetime.fromisoformat(event_ts_str)
+            if event_ts.tzinfo is None:
+                event_ts = event_ts.replace(tzinfo=UTC)
+            now = datetime.now(UTC)
+            latency = (now - event_ts).total_seconds()
+        except (ValueError, TypeError) as exc:
+            logger.warning(
+                "freshness_message_skipped",
+                topic=topic,
+                partition=msg.partition(),
+                offset=msg.offset(),
+                reason="invalid_timestamp",
+                event_id=event.get("event_id"),
+                event_type=event.get("event_type", "unknown"),
+                timestamp=event_ts_str,
+                error=str(exc),
+                exc_info=True,
+            )
+            return
+
+        event_type = event.get("event_type", "unknown")
+
+        # Record metrics
+        PIPELINE_LATENCY.labels(topic=topic, event_type=event_type).observe(latency)
+        EVENTS_PROCESSED.labels(topic=topic).inc()
+
+        # Track SLA compliance
+        within_sla = latency <= FRESHNESS_SLA_SECONDS
+        window = self._sla_window[topic]
+        window.append(within_sla)
+        if len(window) > self._window_size:
+            window.pop(0)
+
+        compliance = sum(window) / len(window)
+        SLA_COMPLIANCE.labels(topic=topic).set(compliance)
+
+        if not within_sla:
+            logger.warning(
+                "sla_breach",
+                topic=topic,
+                event_type=event_type,
+                latency_seconds=round(latency, 2),
+                sla_seconds=FRESHNESS_SLA_SECONDS,
+                event_id=event.get("event_id"),
+            )
+
+
+if __name__ == "__main__":
+    monitor = FreshnessMonitor(
+        bootstrap_servers=os.getenv("KAFKA_BOOTSTRAP_SERVERS", "localhost:9092"),
+        topics=["events.validated", "sessions.aggregated"],
+    )
+    monitor.start()