agentflow-runtime 1.1.0__py3-none-any.whl

src/ingestion/producers/event_producer.py

@@ -0,0 +1,237 @@
+"""Simulates realistic e-commerce event streams for local development.
+
+Produces orders, payments, clickstream, and product events to Kafka topics
+with configurable throughput. Events follow realistic distributions:
+- Orders follow a daily pattern (peaks at 10am, 2pm, 8pm)
+- Payments follow orders with 0-5s delay
+- Clickstream is 10x order volume
+- Products update in batches every ~60s
+"""
+
+import json
+import random
+import time
+import uuid
+from datetime import UTC, datetime
+from decimal import Decimal
+
+import structlog
+from confluent_kafka import Producer
+from pydantic_settings import BaseSettings
+
+from src.ingestion.schemas.events import (
+    ClickstreamEvent,
+    Currency,
+    EventType,
+    OrderEvent,
+    OrderItem,
+    OrderStatus,
+    PaymentEvent,
+    PaymentMethod,
+    ProductEvent,
+)
+
+logger = structlog.get_logger()
+
+
+class ProducerConfig(BaseSettings):
+    kafka_bootstrap_servers: str = "localhost:9092"
+    events_per_second: int = 100
+    order_ratio: float = 0.15
+    payment_ratio: float = 0.10
+    click_ratio: float = 0.70
+    product_ratio: float = 0.05
+
+    model_config = {"env_prefix": "PRODUCER_"}
+
+
+PRODUCT_CATALOG = [
+    ("PROD-001", "Wireless Headphones", "electronics", Decimal("79.99")),
+    ("PROD-002", "Running Shoes", "footwear", Decimal("129.99")),
+    ("PROD-003", "Coffee Maker", "kitchen", Decimal("49.99")),
+    ("PROD-004", "Mechanical Keyboard", "electronics", Decimal("149.99")),
+    ("PROD-005", "Yoga Mat", "fitness", Decimal("34.99")),
+    ("PROD-006", "Backpack", "accessories", Decimal("89.99")),
+    ("PROD-007", "Water Bottle", "fitness", Decimal("24.99")),
+    ("PROD-008", "Desk Lamp", "home", Decimal("44.99")),
+    ("PROD-009", "Bluetooth Speaker", "electronics", Decimal("59.99")),
+    ("PROD-010", "Sunglasses", "accessories", Decimal("119.99")),
+]
+
+PAGES = ["/", "/products", "/cart", "/checkout", "/account", "/search"]
+USER_AGENTS = [
+    "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7)",
+    "Mozilla/5.0 (Windows NT 10.0; Win64; x64)",
+    "Mozilla/5.0 (iPhone; CPU iPhone OS 17_0 like Mac OS X)",
+    "Mozilla/5.0 (Linux; Android 14; Pixel 8)",
+]
+
+
+class DecimalEncoder(json.JSONEncoder):
+    def default(self, o):
+        if isinstance(o, Decimal):
+            return float(o)
+        return super().default(o)
+
+
+def _delivery_report(err, msg):
+    if err:
+        logger.error("delivery_failed", error=str(err), topic=msg.topic())
+
+
+def _now() -> datetime:
+    return datetime.now(UTC)
+
+
+def _uuid() -> str:
+    return str(uuid.uuid4())
+
+
+def generate_order() -> tuple[str, OrderEvent]:
+    products = random.sample(PRODUCT_CATALOG, k=random.randint(1, 4))
+    items = [
+        OrderItem(
+            product_id=p[0],
+            quantity=random.randint(1, 3),
+            unit_price=p[3],
+        )
+        for p in products
+    ]
+    total = sum((item.quantity * item.unit_price for item in items), Decimal(0))
+    order_id = f"ORD-{_now().strftime('%Y%m%d')}-{random.randint(1000, 9999)}"
+
+    event = OrderEvent(
+        event_id=_uuid(),
+        event_type=EventType.ORDER_CREATED,
+        timestamp=_now(),
+        source="web-store",
+        order_id=order_id,
+        user_id=f"USR-{random.randint(10000, 99999)}",
+        status=OrderStatus.PENDING,
+        items=items,
+        total_amount=total,
+        currency=Currency.USD,
+    )
+    return "orders.raw", event
+
+
+def generate_payment(order_id: str | None = None) -> tuple[str, PaymentEvent]:
+    oid = order_id or f"ORD-{_now().strftime('%Y%m%d')}-{random.randint(1000, 9999)}"
+    event = PaymentEvent(
+        event_id=_uuid(),
+        event_type=EventType.PAYMENT_INITIATED,
+        timestamp=_now(),
+        source="payment-gateway",
+        payment_id=f"PAY-{_uuid()[:8]}",
+        order_id=oid,
+        user_id=f"USR-{random.randint(10000, 99999)}",
+        amount=Decimal(str(round(random.uniform(20, 500), 2))),
+        currency=Currency.USD,
+        method=random.choice(list(PaymentMethod)),
+        status="initiated",
+    )
+    return "payments.raw", event
+
+
+def generate_click() -> tuple[str, ClickstreamEvent]:
+    product = random.choice(PRODUCT_CATALOG) if random.random() > 0.4 else None
+    page = f"/products/{product[0]}" if product else random.choice(PAGES)
+    event = ClickstreamEvent(
+        event_id=_uuid(),
+        event_type=random.choice([EventType.CLICK, EventType.PAGE_VIEW, EventType.ADD_TO_CART]),
+        timestamp=_now(),
+        source="web-tracker",
+        session_id=f"SES-{_uuid()[:12]}",
+        user_id=f"USR-{random.randint(10000, 99999)}" if random.random() > 0.3 else None,
+        page_url=page,
+        referrer="https://google.com" if random.random() > 0.5 else None,
+        user_agent=random.choice(USER_AGENTS),
+        viewport_width=random.choice([375, 768, 1024, 1440, 1920]),
+        product_id=product[0] if product else None,
+    )
+    return "clicks.raw", event
+
+
+def generate_product() -> tuple[str, ProductEvent]:
+    product = random.choice(PRODUCT_CATALOG)
+    event = ProductEvent(
+        event_id=_uuid(),
+        event_type=EventType.PRODUCT_UPDATED,
+        timestamp=_now(),
+        source="inventory-service",
+        product_id=product[0],
+        name=product[1],
+        category=product[2],
+        price=product[3],
+        currency=Currency.USD,
+        in_stock=random.random() > 0.1,
+        stock_quantity=random.randint(0, 500),
+    )
+    return "products.cdc", event
+
+
+def run_producer():
+    config = ProducerConfig()
+    producer = Producer(
+        {
+            "bootstrap.servers": config.kafka_bootstrap_servers,
+            "linger.ms": 50,
+            "batch.num.messages": 500,
+            "compression.type": "lz4",
+            "acks": "all",
+        }
+    )
+
+    generators: list[tuple] = [
+        (config.order_ratio, generate_order),
+        (config.payment_ratio, generate_payment),
+        (config.click_ratio, generate_click),
+        (config.product_ratio, generate_product),
+    ]
+
+    logger.info(
+        "producer_started",
+        eps=config.events_per_second,
+        bootstrap=config.kafka_bootstrap_servers,
+    )
+
+    produced = 0
+    interval = 1.0 / config.events_per_second
+
+    try:
+        while True:
+            roll = random.random()
+            cumulative = 0.0
+
+            for ratio, gen in generators:
+                cumulative += ratio
+                if roll <= cumulative:
+                    topic, event = gen()
+                    producer.produce(
+                        topic,
+                        key=event.event_id.encode(),
+                        value=json.dumps(
+                            event.model_dump(mode="json"),
+                            cls=DecimalEncoder,
+                        ).encode(),
+                        callback=_delivery_report,
+                    )
+                    produced += 1
+                    break
+
+            if produced % 1000 == 0:
+                producer.flush()
+                logger.info("producer_progress", total_produced=produced)
+
+            producer.poll(0)
+            time.sleep(interval)
+
+    except KeyboardInterrupt:
+        logger.info("producer_stopping", total_produced=produced)
+    finally:
+        producer.flush(timeout=10)
+        logger.info("producer_stopped", total_produced=produced)
+
+
+if __name__ == "__main__":
+    run_producer()

src/ingestion/schemas/events.py

@@ -0,0 +1,147 @@
+"""Canonical event schemas for all data sources.
+
+These Pydantic models serve as the single source of truth for event structure.
+Used by producers (serialization), Flink jobs (validation), and the API (response types).
+"""
+
+from datetime import UTC, datetime
+from decimal import Decimal
+from enum import StrEnum
+
+from pydantic import BaseModel, Field, field_validator
+
+
+class EventType(StrEnum):
+    ORDER_CREATED = "order.created"
+    ORDER_UPDATED = "order.updated"
+    ORDER_CANCELLED = "order.cancelled"
+    PAYMENT_INITIATED = "payment.initiated"
+    PAYMENT_COMPLETED = "payment.completed"
+    PAYMENT_FAILED = "payment.failed"
+    CLICK = "click"
+    PAGE_VIEW = "page_view"
+    ADD_TO_CART = "add_to_cart"
+    PRODUCT_UPDATED = "product.updated"
+
+
+class CdcOperation(StrEnum):
+    SNAPSHOT = "snapshot"
+    INSERT = "insert"
+    UPDATE = "update"
+    DELETE = "delete"
+    DDL = "ddl"
+
+
+class CdcSource(StrEnum):
+    POSTGRES = "postgres_cdc"
+    MYSQL = "mysql_cdc"
+
+
+class Currency(StrEnum):
+    USD = "USD"
+    EUR = "EUR"
+    GBP = "GBP"
+
+
+class OrderStatus(StrEnum):
+    PENDING = "pending"
+    CONFIRMED = "confirmed"
+    SHIPPED = "shipped"
+    DELIVERED = "delivered"
+    CANCELLED = "cancelled"
+
+
+class BaseEvent(BaseModel):
+    event_id: str = Field(..., pattern=r"^[a-f0-9\-]{36}$")
+    event_type: EventType
+    timestamp: datetime
+    source: str = Field(..., min_length=1, max_length=64)
+
+    @field_validator("timestamp")
+    @classmethod
+    def timestamp_not_future(cls, v: datetime) -> datetime:
+        now = datetime.now(UTC)
+        if v.tzinfo is None:
+            v = v.replace(tzinfo=UTC)
+        # Allow 5 minutes of clock skew
+        max_future = now.timestamp() + 300
+        if v.timestamp() > max_future:
+            msg = f"Event timestamp {v} is too far in the future"
+            raise ValueError(msg)
+        return v
+
+
+class CdcEvent(BaseModel):
+    event_id: str = Field(..., min_length=1)
+    event_type: str = Field(..., min_length=1, max_length=128)
+    operation: CdcOperation
+    timestamp: datetime
+    source: CdcSource
+    entity_type: str = Field(..., min_length=1, max_length=64)
+    entity_id: str = Field(..., min_length=1, max_length=256)
+    before: dict | None = None
+    after: dict | None = None
+    source_metadata: dict = Field(..., min_length=1)
+
+
+class OrderItem(BaseModel):
+    product_id: str
+    quantity: int = Field(..., gt=0, le=1000)
+    unit_price: Decimal = Field(..., gt=0, decimal_places=2)
+
+
+class OrderEvent(BaseEvent):
+    order_id: str = Field(..., pattern=r"^ORD-\d{8}-\d{4,}$")
+    user_id: str
+    status: OrderStatus
+    items: list[OrderItem] = Field(..., min_length=1)
+    total_amount: Decimal = Field(..., gt=0)
+    currency: Currency = Currency.USD
+
+    @field_validator("total_amount")
+    @classmethod
+    def total_matches_items(cls, v: Decimal, info) -> Decimal:
+        items = info.data.get("items")
+        if items:
+            expected = sum(item.quantity * item.unit_price for item in items)
+            if abs(v - expected) > Decimal("0.01"):
+                msg = f"Total {v} doesn't match sum of items {expected}"
+                raise ValueError(msg)
+        return v
+
+
+class PaymentMethod(StrEnum):
+    CARD = "card"
+    BANK_TRANSFER = "bank_transfer"
+    WALLET = "wallet"
+
+
+class PaymentEvent(BaseEvent):
+    payment_id: str
+    order_id: str = Field(..., pattern=r"^ORD-\d{8}-\d{4,}$")
+    user_id: str
+    amount: Decimal = Field(..., gt=0)
+    currency: Currency = Currency.USD
+    method: PaymentMethod
+    status: str = Field(..., pattern=r"^(initiated|completed|failed|refunded)$")
+    failure_reason: str | None = None
+
+
+class ClickstreamEvent(BaseEvent):
+    session_id: str
+    user_id: str | None = None  # anonymous users allowed
+    page_url: str
+    referrer: str | None = None
+    user_agent: str
+    viewport_width: int | None = None
+    product_id: str | None = None  # set if on a product page
+
+
+class ProductEvent(BaseEvent):
+    product_id: str
+    name: str = Field(..., min_length=1, max_length=500)
+    category: str
+    price: Decimal = Field(..., ge=0)
+    currency: Currency = Currency.USD
+    in_stock: bool
+    stock_quantity: int = Field(..., ge=0)

src/ingestion/tenant_router.py

@@ -0,0 +1,80 @@
+from __future__ import annotations
+
+import json
+import os
+from pathlib import Path
+
+from pydantic import BaseModel, Field
+
+try:
+    import yaml  # type: ignore[import-untyped]
+except ImportError:  # pragma: no cover
+    yaml = None
+
+
+def default_tenants_config_path() -> Path:
+    return Path(os.getenv("AGENTFLOW_TENANTS_FILE", "config/tenants.yaml"))
+
+
+class TenantDefinition(BaseModel):
+    id: str
+    display_name: str
+    kafka_topic_prefix: str
+    duckdb_schema: str
+    max_events_per_day: int
+    max_api_keys: int
+    allowed_entity_types: list[str] | None = None
+
+
+class TenantsConfig(BaseModel):
+    tenants: list[TenantDefinition] = Field(default_factory=list)
+
+
+class TenantRouter:
+    def __init__(self, config_path: Path | str | None = None) -> None:
+        self.config_path = (
+            Path(config_path) if config_path is not None else default_tenants_config_path()
+        )
+        self._has_config: bool | None = None
+        self._config: TenantsConfig | None = None
+
+    def has_config(self) -> bool:
+        if self._has_config is None:
+            self._has_config = self.config_path.exists()
+        return self._has_config
+
+    def load(self) -> TenantsConfig:
+        if self._config is not None:
+            return self._config
+
+        if not self.has_config():
+            self._config = TenantsConfig()
+            return self._config
+
+        raw = self.config_path.read_text(encoding="utf-8")
+        if yaml is not None:
+            data = yaml.safe_load(raw) or {}
+        else:  # pragma: no cover
+            data = json.loads(raw)
+        self._config = TenantsConfig.model_validate(data)
+        return self._config
+
+    def get_tenant(self, tenant_id: str | None) -> TenantDefinition | None:
+        if tenant_id is None:
+            return None
+        for tenant in self.load().tenants:
+            if tenant.id == tenant_id:
+                return tenant
+        return None
+
+    def get_duckdb_schema(self, tenant_id: str | None) -> str | None:
+        tenant = self.get_tenant(tenant_id)
+        if tenant is None:
+            return None
+        return tenant.duckdb_schema
+
+    def route_topic(self, topic: str, tenant_id: str | None) -> str:
+        tenant = self.get_tenant(tenant_id)
+        if tenant is None:
+            return topic
+        return f"{tenant.kafka_topic_prefix}.{topic}"

src/logger.py

@@ -0,0 +1,41 @@
+from __future__ import annotations
+
+import logging
+import os
+from collections.abc import MutableMapping
+from typing import Any
+
+import structlog
+from opentelemetry import trace
+
+
+def add_otel_context(
+    _logger: Any,
+    _method_name: str,
+    event_dict: MutableMapping[str, Any],
+) -> MutableMapping[str, Any]:
+    span = trace.get_current_span()
+    if not span.is_recording():
+        return event_dict
+    span_context = span.get_span_context()
+    event_dict["trace_id"] = format(span_context.trace_id, "032x")
+    event_dict["span_id"] = format(span_context.span_id, "016x")
+    return event_dict
+
+
+def configure_logging() -> None:
+    level_name = os.getenv("LOG_LEVEL", "INFO").upper()
+    level = getattr(logging, level_name, logging.INFO)
+    logging.basicConfig(format="%(message)s", level=level)
+    structlog.configure(
+        processors=[
+            structlog.contextvars.merge_contextvars,
+            structlog.processors.add_log_level,
+            structlog.processors.TimeStamper(fmt="iso", utc=True),
+            add_otel_context,
+            structlog.processors.JSONRenderer(),
+        ],
+        logger_factory=structlog.stdlib.LoggerFactory(),
+        wrapper_class=structlog.make_filtering_bound_logger(level),
+        cache_logger_on_first_use=True,
+    )