agentomatic 0.1.0__py3-none-any.whl

agentomatic/middleware/feedback.py

@@ -0,0 +1,207 @@
+"""Feedback collection middleware and decorator.
+
+Auto-adds feedback endpoints to every agent and provides
+a decorator for automatic input/output recording.
+
+Usage::
+
+    # In agent code:
+    from agentomatic.middleware.feedback import collect_feedback
+
+    @collect_feedback(store=True)
+    async def invoke(state):
+        ...
+
+    # Via platform:
+    platform = AgentPlatform.from_folder(
+        "agents/",
+        enable_feedback=True,
+        store=MemoryStore(),
+    )
+"""
+
+from __future__ import annotations
+
+import asyncio
+import functools
+import uuid
+from collections.abc import Callable
+from dataclasses import dataclass, field
+from datetime import UTC
+from typing import Any, TypeVar, cast
+
+from loguru import logger
+
+F = TypeVar("F", bound=Callable[..., Any])
+
+
+@dataclass
+class FeedbackRecord:
+    """A feedback entry."""
+
+    feedback_id: str = field(default_factory=lambda: uuid.uuid4().hex[:12])
+    agent_name: str = ""
+    user_id: str = ""
+    thread_id: str | None = None
+    message_id: int | None = None
+    query: str = ""
+    response: str = ""
+    rating: int | None = None  # 1 (thumbs down) or 5 (thumbs up)
+    comment: str | None = None  # Free-text comment
+    correction: str | None = None  # User-provided correct answer
+    feedback_type: str = "thumbs"  # thumbs, rating, correction, comment
+    timestamp: str = ""
+    metadata: dict[str, Any] = field(default_factory=dict)
+
+    def to_dict(self) -> dict[str, Any]:
+        return {k: v for k, v in self.__dict__.items() if v is not None and v != "" and v != {}}
+
+
+class FeedbackCollector:
+    """Async feedback collector with background storage.
+
+    Can be used standalone or auto-attached to agents.
+    """
+
+    def __init__(self, store: Any = None, buffer_size: int = 100):
+        self._store = store
+        self._buffer: list[FeedbackRecord] = []
+        self._buffer_size = buffer_size
+        self._lock = asyncio.Lock()
+
+    async def record(
+        self,
+        agent_name: str,
+        user_id: str = "",
+        *,
+        query: str = "",
+        response: str = "",
+        rating: int | None = None,
+        comment: str | None = None,
+        correction: str | None = None,
+        feedback_type: str = "thumbs",
+        thread_id: str | None = None,
+        metadata: dict[str, Any] | None = None,
+    ) -> FeedbackRecord:
+        """Record a feedback entry."""
+        from datetime import datetime
+
+        record = FeedbackRecord(
+            agent_name=agent_name,
+            user_id=user_id,
+            query=query,
+            response=response,
+            rating=rating,
+            comment=comment,
+            correction=correction,
+            feedback_type=feedback_type,
+            thread_id=thread_id,
+            timestamp=datetime.now(UTC).isoformat(),
+            metadata=metadata or {},
+        )
+
+        # Store via backend
+        if self._store and hasattr(self._store, "add_feedback"):
+            try:
+                await self._store.add_feedback(
+                    thread_id=thread_id or "",
+                    user_id=user_id,
+                    agent_name=agent_name,
+                    rating=rating,
+                    comment=comment,
+                    feedback_type=feedback_type,
+                )
+            except Exception as exc:
+                logger.warning(f"Failed to store feedback: {exc}")
+
+        # Buffer for batch export
+        async with self._lock:
+            self._buffer.append(record)
+            if len(self._buffer) > self._buffer_size:
+                self._buffer = self._buffer[-self._buffer_size :]
+
+        logger.debug(f"📝 Feedback recorded for {agent_name} (rating={rating})")
+        return record
+
+    async def get_feedback(
+        self,
+        agent_name: str | None = None,
+        limit: int = 50,
+    ) -> list[dict[str, Any]]:
+        """Get stored feedback."""
+        if self._store and hasattr(self._store, "get_feedback"):
+            return cast(
+                list[dict[str, Any]],
+                await self._store.get_feedback(
+                    agent_name=agent_name,
+                    limit=limit,
+                ),
+            )
+        # Fall back to buffer
+        async with self._lock:
+            items = self._buffer
+            if agent_name:
+                items = [f for f in items if f.agent_name == agent_name]
+            return [f.to_dict() for f in items[-limit:]]
+
+    async def export_jsonl(self, agent_name: str | None = None) -> str:
+        """Export feedback as JSONL string (for optimization datasets)."""
+        import json
+
+        records = await self.get_feedback(agent_name=agent_name, limit=10000)
+        lines = []
+        for r in records:
+            # Convert to optimization-friendly format
+            entry = {
+                "query": r.get("query", ""),
+                "expected_answer": r.get("correction") or r.get("response", ""),
+                "metadata": {
+                    "rating": r.get("rating"),
+                    "comment": r.get("comment"),
+                    "feedback_type": r.get("feedback_type"),
+                },
+            }
+            if entry["query"]:  # Only include entries with queries
+                lines.append(json.dumps(entry))
+        return "\n".join(lines)
+
+
+def collect_feedback(
+    store: bool = True,
+    log: bool = True,
+) -> Callable[[F], F]:
+    """Decorator to auto-record agent inputs/outputs for feedback.
+
+    Records every invocation's query and response.
+    Useful for building optimization datasets from production traffic.
+    """
+
+    def decorator(fn: F) -> F:
+        @functools.wraps(fn)
+        async def wrapper(state: dict, *args, **kwargs):
+            result = await fn(state, *args, **kwargs)
+            if log:
+                query = state.get("current_query", "")
+                response = result.get("response", "") if isinstance(result, dict) else str(result)
+                logger.info(f"📊 [feedback] Q={query[:50]}... A={response[:50]}...")
+            return result
+
+        return wrapper  # type: ignore
+
+    return decorator
+
+
+# Module-level singleton
+_collector: FeedbackCollector | None = None
+
+
+def get_collector() -> FeedbackCollector:
+    global _collector
+    if _collector is None:
+        _collector = FeedbackCollector()
+    return _collector
+
+
+def set_collector(collector: FeedbackCollector) -> None:
+    global _collector
+    _collector = collector

agentomatic/middleware/logging.py

@@ -0,0 +1,40 @@
+"""Request logging middleware with X-Request-ID."""
+
+from __future__ import annotations
+
+import time
+import uuid
+
+from loguru import logger
+from starlette.middleware.base import BaseHTTPMiddleware
+from starlette.requests import Request
+from starlette.responses import Response
+
+_SKIP_PATHS = {"/health", "/healthz", "/readiness", "/metrics", "/favicon.ico"}
+
+
+class LoggingMiddleware(BaseHTTPMiddleware):
+    """Logs every request with timing and request ID."""
+
+    async def dispatch(self, request: Request, call_next) -> Response:
+        """Process request with logging and timing."""
+        if request.url.path in _SKIP_PATHS:
+            response: Response = await call_next(request)
+            return response
+
+        request_id = request.headers.get("X-Request-ID", uuid.uuid4().hex[:12])
+        t0 = time.perf_counter()
+
+        logger.info(f"→ {request.method} {request.url.path} [{request_id}]")
+
+        response = await call_next(request)
+
+        elapsed_ms = (time.perf_counter() - t0) * 1000
+        logger.info(
+            f"← {request.method} {request.url.path} → {response.status_code} "
+            f"({elapsed_ms:.1f}ms) [{request_id}]"
+        )
+
+        response.headers["X-Request-ID"] = request_id
+        response.headers["X-Process-Time-Ms"] = f"{elapsed_ms:.1f}"
+        return response

agentomatic/middleware/metrics.py

@@ -0,0 +1,93 @@
+"""Prometheus metrics middleware.
+
+Enabled via ``FEATURES__ENABLE_METRICS=true``.
+Automatically tracks request count, latency histogram, and active requests.
+"""
+
+from __future__ import annotations
+
+import time
+
+from starlette.middleware.base import BaseHTTPMiddleware
+from starlette.requests import Request
+from starlette.responses import Response
+
+try:
+    from prometheus_client import CONTENT_TYPE_LATEST, Counter, Gauge, Histogram, generate_latest
+
+    HAS_PROMETHEUS = True
+except ImportError:
+    HAS_PROMETHEUS = False
+
+_SKIP_PATHS = {"/health", "/healthz", "/readiness", "/metrics"}
+
+
+class MetricsMiddleware(BaseHTTPMiddleware):
+    """Prometheus metrics collection per request."""
+
+    def __init__(self, app, *, prefix: str = "agentomatic") -> None:
+        super().__init__(app)
+        self._requests: Counter | None = None
+        self._duration: Histogram | None = None
+        self._active: Gauge | None = None
+        if HAS_PROMETHEUS:
+            self._requests = Counter(
+                f"{prefix}_http_requests_total",
+                "Total HTTP requests",
+                ["method", "path", "status"],
+            )
+            self._duration = Histogram(
+                f"{prefix}_http_request_duration_seconds",
+                "HTTP request duration",
+                ["method", "path"],
+                buckets=(0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1.0, 2.5, 5.0, 10.0),
+            )
+            self._active = Gauge(
+                f"{prefix}_http_requests_active",
+                "Active HTTP requests",
+            )
+
+    async def dispatch(self, request: Request, call_next) -> Response:
+        if request.url.path in _SKIP_PATHS:
+            # Serve /metrics endpoint
+            if request.url.path == "/metrics" and HAS_PROMETHEUS:
+                from starlette.responses import Response as StarletteResponse
+
+                body = generate_latest()
+                return StarletteResponse(
+                    content=body,
+                    media_type=CONTENT_TYPE_LATEST,
+                )
+            response: Response = await call_next(request)
+            return response
+
+        if self._active:
+            self._active.inc()
+
+        t0 = time.perf_counter()
+        response = await call_next(request)
+        duration = time.perf_counter() - t0
+
+        # Normalize path for cardinality control
+        path = request.url.path
+        # Collapse UUIDs and hex IDs to reduce cardinality
+        parts = path.split("/")
+        normalized = "/".join(
+            "{id}" if (len(p) > 8 and any(c.isdigit() for c in p)) else p for p in parts
+        )
+
+        if self._requests:
+            self._requests.labels(
+                method=request.method,
+                path=normalized,
+                status=str(response.status_code),
+            ).inc()
+        if self._duration:
+            self._duration.labels(
+                method=request.method,
+                path=normalized,
+            ).observe(duration)
+        if self._active:
+            self._active.dec()
+
+        return response

agentomatic/middleware/rate_limit.py

@@ -0,0 +1,70 @@
+"""In-memory sliding-window rate limiter.
+
+Enabled via ``FEATURES__ENABLE_RATE_LIMIT=true``.
+Configured via ``RATE_LIMIT__REQUESTS`` and ``RATE_LIMIT__WINDOW_SECONDS``.
+"""
+
+from __future__ import annotations
+
+import time
+from collections import defaultdict
+
+from starlette.middleware.base import BaseHTTPMiddleware
+from starlette.requests import Request
+from starlette.responses import JSONResponse, Response
+
+_SKIP_PATHS = {"/health", "/healthz", "/readiness"}
+
+
+class RateLimitMiddleware(BaseHTTPMiddleware):
+    """Token-bucket style rate limiter per client IP.
+
+    Args:
+        app: ASGI application.
+        max_requests: Maximum requests per window.
+        window_seconds: Sliding window duration.
+    """
+
+    def __init__(
+        self,
+        app,
+        *,
+        max_requests: int = 100,
+        window_seconds: int = 60,
+    ) -> None:
+        super().__init__(app)
+        self._max = max_requests
+        self._window = window_seconds
+        self._hits: dict[str, list[float]] = defaultdict(list)
+
+    def _client_key(self, request: Request) -> str:
+        forwarded = request.headers.get("X-Forwarded-For")
+        if forwarded:
+            return forwarded.split(",")[0].strip()
+        return request.client.host if request.client else "unknown"
+
+    async def dispatch(self, request: Request, call_next) -> Response:
+        if request.url.path in _SKIP_PATHS:
+            response: Response = await call_next(request)
+            return response
+
+        key = self._client_key(request)
+        now = time.monotonic()
+
+        # Purge expired entries
+        self._hits[key] = [t for t in self._hits[key] if now - t < self._window]
+
+        if len(self._hits[key]) >= self._max:
+            retry_after = int(self._window - (now - self._hits[key][0]))
+            return JSONResponse(
+                {"detail": "Rate limit exceeded", "retry_after": max(retry_after, 1)},
+                status_code=429,
+                headers={"Retry-After": str(max(retry_after, 1))},
+            )
+
+        self._hits[key].append(now)
+        response = await call_next(request)
+        remaining = self._max - len(self._hits[key])
+        response.headers["X-RateLimit-Limit"] = str(self._max)
+        response.headers["X-RateLimit-Remaining"] = str(max(remaining, 0))
+        return response

agentomatic/observability/__init__.py

@@ -0,0 +1,11 @@
+"""Observability (metrics, health, concurrency, telemetry)."""
+
+from __future__ import annotations
+
+from .telemetry import get_tracer, setup_telemetry, traced
+
+__all__ = [
+    "get_tracer",
+    "setup_telemetry",
+    "traced",
+]

agentomatic/observability/concurrency.py

@@ -0,0 +1,109 @@
+"""Circuit breaker and concurrency control."""
+
+from __future__ import annotations
+
+import asyncio
+import time
+from contextlib import asynccontextmanager
+from enum import Enum
+
+from loguru import logger
+
+
+class CircuitState(Enum):
+    """States for the circuit breaker."""
+
+    CLOSED = "closed"
+    OPEN = "open"
+    HALF_OPEN = "half_open"
+
+
+class CircuitBreakerOpen(Exception):
+    """Raised when circuit breaker is open."""
+
+
+class CircuitBreaker:
+    """Async circuit breaker for protecting external services."""
+
+    def __init__(
+        self,
+        name: str = "default",
+        failure_threshold: int = 5,
+        reset_timeout: float = 60.0,
+    ) -> None:
+        self.name = name
+        self.failure_threshold = failure_threshold
+        self.reset_timeout = reset_timeout
+        self._state = CircuitState.CLOSED
+        self._failure_count = 0
+        self._last_failure_time: float = 0
+
+    @property
+    def state(self) -> CircuitState:
+        """Get current state, auto-transitioning from OPEN to HALF_OPEN after timeout."""
+        if self._state == CircuitState.OPEN:
+            if time.monotonic() - self._last_failure_time >= self.reset_timeout:
+                self._state = CircuitState.HALF_OPEN
+        return self._state
+
+    @asynccontextmanager
+    async def __call__(self):
+        """Use as an async context manager around protected calls."""
+        if self.state == CircuitState.OPEN:
+            raise CircuitBreakerOpen(f"Circuit breaker '{self.name}' is open")
+
+        try:
+            yield
+            self._on_success()
+        except Exception:
+            self._on_failure()
+            raise
+
+    def _on_success(self) -> None:
+        """Record a successful call."""
+        self._failure_count = 0
+        self._state = CircuitState.CLOSED
+
+    def _on_failure(self) -> None:
+        """Record a failed call and potentially open the circuit."""
+        self._failure_count += 1
+        self._last_failure_time = time.monotonic()
+        if self._failure_count >= self.failure_threshold:
+            self._state = CircuitState.OPEN
+            logger.warning(
+                f"Circuit breaker '{self.name}' opened after {self._failure_count} failures"
+            )
+
+
+class AgentSemaphore:
+    """Global semaphore to limit concurrent agent invocations."""
+
+    _instance: AgentSemaphore | None = None
+
+    def __init__(self, max_concurrent: int = 10) -> None:
+        self._semaphore = asyncio.Semaphore(max_concurrent)
+        self._max = max_concurrent
+        self._active = 0
+
+    @classmethod
+    def get(cls, max_concurrent: int = 10) -> AgentSemaphore:
+        """Get or create the singleton semaphore."""
+        if cls._instance is None:
+            cls._instance = cls(max_concurrent)
+        return cls._instance
+
+    @asynccontextmanager
+    async def acquire(self):
+        """Acquire the semaphore, limiting concurrency."""
+        await self._semaphore.acquire()
+        self._active += 1
+        try:
+            yield
+        finally:
+            self._active -= 1
+            self._semaphore.release()
+
+    @property
+    def active(self) -> int:
+        """Number of currently active acquisitions."""
+        return self._active

agentomatic/observability/metrics.py

@@ -0,0 +1,101 @@
+"""Prometheus metrics with graceful fallback."""
+
+from __future__ import annotations
+
+from contextlib import asynccontextmanager
+from typing import Any
+
+try:
+    from prometheus_client import Counter, Gauge, Histogram
+
+    HAS_PROMETHEUS = True
+except ImportError:
+    HAS_PROMETHEUS = False
+
+
+class _DummyMetric:
+    """No-op metric when prometheus_client is not installed."""
+
+    def labels(self, **kw: Any) -> _DummyMetric:
+        return self
+
+    def inc(self, amount: float = 1) -> None:
+        pass
+
+    def dec(self, amount: float = 1) -> None:
+        pass
+
+    def set(self, value: float) -> None:
+        pass
+
+    def observe(self, value: float) -> None:
+        pass
+
+    def info(self, val: dict) -> None:
+        pass
+
+
+def _counter(name: str, doc: str, labels: list[str] | None = None) -> Any:
+    """Create a Prometheus Counter or dummy fallback."""
+    if HAS_PROMETHEUS:
+        return Counter(name, doc, labels or [])
+    return _DummyMetric()
+
+
+def _histogram(name: str, doc: str, labels: list[str] | None = None, buckets: Any = None) -> Any:
+    """Create a Prometheus Histogram or dummy fallback."""
+    if HAS_PROMETHEUS:
+        kw = {"buckets": buckets} if buckets else {}
+        return Histogram(name, doc, labels or [], **kw)
+    return _DummyMetric()
+
+
+def _gauge(name: str, doc: str, labels: list[str] | None = None) -> Any:
+    """Create a Prometheus Gauge or dummy fallback."""
+    if HAS_PROMETHEUS:
+        return Gauge(name, doc, labels or [])
+    return _DummyMetric()
+
+
+# Counters
+REQUEST_COUNT = _counter(
+    "agentomatic_requests_total", "Total requests", ["method", "endpoint", "status_code"]
+)
+AGENT_INVOCATION_COUNT = _counter(
+    "agentomatic_agent_invocations_total", "Agent invocations", ["agent_name", "status"]
+)
+ERROR_COUNT = _counter("agentomatic_errors_total", "Errors", ["error_type", "agent_name"])
+
+# Histograms
+REQUEST_DURATION = _histogram(
+    "agentomatic_request_duration_seconds", "Request duration", ["method", "endpoint"]
+)
+AGENT_DURATION = _histogram("agentomatic_agent_duration_seconds", "Agent duration", ["agent_name"])
+LLM_DURATION = _histogram(
+    "agentomatic_llm_duration_seconds", "LLM call duration", ["provider", "model"]
+)
+
+# Gauges
+ACTIVE_REQUESTS = _gauge("agentomatic_active_requests", "Active requests")
+ACTIVE_AGENTS = _gauge("agentomatic_active_agents", "Active agent invocations")
+REGISTERED_AGENTS = _gauge("agentomatic_registered_agents", "Registered agents")
+
+
+@asynccontextmanager
+async def track_agent_invocation(agent_name: str):
+    """Context manager to track agent invocation metrics."""
+    import time
+
+    ACTIVE_AGENTS.inc()
+    AGENT_INVOCATION_COUNT.labels(agent_name=agent_name, status="started").inc()
+    t0 = time.perf_counter()
+    try:
+        yield
+        AGENT_INVOCATION_COUNT.labels(agent_name=agent_name, status="success").inc()
+    except Exception:
+        AGENT_INVOCATION_COUNT.labels(agent_name=agent_name, status="error").inc()
+        raise
+    finally:
+        duration = time.perf_counter() - t0
+        AGENT_DURATION.labels(agent_name=agent_name).observe(duration)
+        ACTIVE_AGENTS.dec()