frugal-sdk-python 0.1.0__py3-none-any.whl

frugal_metrics/prompt_analyzer.py

@@ -0,0 +1,273 @@
+"""Lightweight per-request analyzer.
+
+Inspects messages, tools, and response_format kwargs to derive low-cardinality
+signals for the AI cost-trap detection catalogue. Never emits raw request
+content; emits stats and (in-process-only) hashes that the instrument layer
+turns into metrics.
+"""
+
+from __future__ import annotations
+
+import hashlib
+import json
+import re
+from dataclasses import dataclass
+from typing import Any, Callable, TypeVar
+
+from frugal_metrics import semconv
+
+T = TypeVar("T")
+
+
+@dataclass
+class RequestPayload:
+    """Captured kwargs from the wrapped SDK call."""
+
+    messages: Any = None
+    tools: Any = None
+    response_format: Any = None
+    max_tokens: int | None = None
+    system: Any = None
+
+
+@dataclass
+class AnalysisResult:
+    system_hash: str | None = None
+    user_hash: str | None = None
+    tools_hash: str | None = None
+    schema_hash: str | None = None
+    history_depth: int | None = None
+    few_shot_count: int | None = None
+    noise_ratio: float | None = None
+    structured_output_hint: bool = False
+
+
+def analyze_request(payload: RequestPayload) -> AnalysisResult:
+    return AnalysisResult(
+        system_hash=_safe(lambda: _hash_system_block(payload), None),
+        user_hash=_safe(lambda: _hash_user_block(payload), None),
+        tools_hash=_safe(lambda: _hash_json(payload.tools), None),
+        schema_hash=_safe(lambda: _hash_json(payload.response_format), None),
+        history_depth=_safe(lambda: _history_depth(payload), None),
+        few_shot_count=_safe(lambda: _few_shot_count(payload), None),
+        noise_ratio=_safe(lambda: _noise_ratio(payload), None),
+        structured_output_hint=_safe(
+            lambda: _structured_output_hint(payload), False
+        ),
+    )
+
+
+def structured_output_mode(payload: RequestPayload) -> str:
+    if isinstance(payload.tools, list) and len(payload.tools) > 0:
+        return semconv.STRUCTURED_OUTPUT_TOOLS
+    rf = payload.response_format
+    if isinstance(rf, dict):
+        rf_type = rf.get("type")
+        if rf_type == "json_object":
+            return semconv.STRUCTURED_OUTPUT_JSON_OBJECT
+        if rf_type == "json_schema":
+            return semconv.STRUCTURED_OUTPUT_JSON_SCHEMA
+    elif rf is not None:
+        rf_type = getattr(rf, "type", None)
+        if rf_type == "json_object":
+            return semconv.STRUCTURED_OUTPUT_JSON_OBJECT
+        if rf_type == "json_schema":
+            return semconv.STRUCTURED_OUTPUT_JSON_SCHEMA
+    return semconv.STRUCTURED_OUTPUT_NONE
+
+
+def _safe(fn: Callable[[], T], fallback: T) -> T:
+    try:
+        return fn()
+    except Exception:  # noqa: BLE001
+        return fallback
+
+
+# --- hashing ---------------------------------------------------------------
+
+
+def _sha256(data: str) -> str:
+    return hashlib.sha256(data.encode("utf-8")).hexdigest()
+
+
+def _canonicalize(value: Any) -> str:
+    """Deterministic JSON serialization with sorted keys.
+
+    Pydantic-style objects are converted to dicts via __dict__ (best-effort);
+    anything else falls back to json.dumps(default=str).
+    """
+    if hasattr(value, "model_dump") and callable(value.model_dump):
+        try:
+            value = value.model_dump()
+        except Exception:  # noqa: BLE001
+            pass
+    return json.dumps(value, sort_keys=True, default=str, separators=(",", ":"))
+
+
+def _hash_json(value: Any) -> str | None:
+    if value is None:
+        return None
+    return _sha256(_canonicalize(value))
+
+
+def _hash_system_block(payload: RequestPayload) -> str | None:
+    blocks: list[Any] = []
+    if payload.system is not None:
+        blocks.append(payload.system)
+    if isinstance(payload.messages, list):
+        for m in payload.messages:
+            if _attr(m, "role") == "system":
+                blocks.append(_attr(m, "content"))
+    if not blocks:
+        return None
+    return _hash_json(blocks)
+
+
+def _hash_user_block(payload: RequestPayload) -> str | None:
+    if not isinstance(payload.messages, list):
+        return None
+    blocks = [
+        _attr(m, "content")
+        for m in payload.messages
+        if _attr(m, "role") == "user"
+    ]
+    if not blocks:
+        return None
+    return _hash_json(blocks)
+
+
+# --- structural counts -----------------------------------------------------
+
+
+def _history_depth(payload: RequestPayload) -> int | None:
+    if not isinstance(payload.messages, list):
+        return None
+    return len(payload.messages)
+
+
+def _few_shot_count(payload: RequestPayload) -> int | None:
+    if not isinstance(payload.messages, list):
+        return None
+    msgs = [m for m in payload.messages if m is not None]
+    if not msgs or _attr(msgs[-1], "role") != "user":
+        return None
+    # If any tool-call turns exist, treat this as a real conversation.
+    for m in msgs:
+        if _attr(m, "role") == "tool":
+            return None
+        tool_calls = _attr(m, "tool_calls")
+        if isinstance(tool_calls, list) and len(tool_calls) > 0:
+            return None
+
+    # Skip leading system messages.
+    i = 0
+    while i < len(msgs) and _attr(msgs[i], "role") == "system":
+        i += 1
+
+    pairs = 0
+    last = len(msgs) - 1
+    while i + 1 < last:
+        if _attr(msgs[i], "role") == "user" and _attr(msgs[i + 1], "role") == "assistant":
+            pairs += 1
+            i += 2
+        else:
+            break
+    return pairs
+
+
+# --- noise ratio -----------------------------------------------------------
+
+_HTML_TAG = re.compile(r"<[^>]+>")
+_BASE64_RUN = re.compile(r"[A-Za-z0-9+/]{40,}={0,2}")
+_WHITESPACE_RUN = re.compile(r"\s{4,}")
+
+
+def _noise_ratio(payload: RequestPayload) -> float | None:
+    text = _collect_text(payload)
+    if not text:
+        return None
+    noise = 0
+    for match in _HTML_TAG.findall(text):
+        noise += len(match)
+    for match in _BASE64_RUN.findall(text):
+        noise += len(match)
+    for match in _WHITESPACE_RUN.findall(text):
+        noise += len(match)
+    return min(1.0, noise / len(text))
+
+
+def _collect_text(payload: RequestPayload) -> str:
+    parts: list[str] = []
+    if isinstance(payload.system, str):
+        parts.append(payload.system)
+    if isinstance(payload.messages, list):
+        for m in payload.messages:
+            if m is None:
+                continue
+            content = _attr(m, "content")
+            if isinstance(content, str):
+                parts.append(content)
+            elif isinstance(content, list):
+                for c in content:
+                    text = _attr(c, "text")
+                    if isinstance(text, str):
+                        parts.append(text)
+    return "\n".join(parts)
+
+
+# --- structured output hint ------------------------------------------------
+
+_STRUCTURED_HINTS = [
+    re.compile(r"format\s+as\s+JSON", re.IGNORECASE),
+    re.compile(r"return\s+a\s+(JSON\s+)?(list|array|object)", re.IGNORECASE),
+    re.compile(r"respond\s+with\s+a\s+JSON", re.IGNORECASE),
+    re.compile(r"output\s+(must\s+be|should\s+be|in)\s+JSON", re.IGNORECASE),
+]
+
+
+def _structured_output_hint(payload: RequestPayload) -> bool:
+    if isinstance(payload.tools, list) and len(payload.tools) > 0:
+        return False
+    rf = payload.response_format
+    if rf is not None:
+        rf_type = rf.get("type") if isinstance(rf, dict) else getattr(rf, "type", None)
+        if rf_type is not None:
+            return False
+    text = _collect_text(payload)
+    if not text:
+        return False
+    return any(p.search(text) for p in _STRUCTURED_HINTS)
+
+
+# --- attribute helper ------------------------------------------------------
+
+
+def _attr(obj: Any, name: str, default: Any = None) -> Any:
+    if obj is None:
+        return default
+    if isinstance(obj, dict):
+        return obj.get(name, default)
+    return getattr(obj, name, default)
+
+
+# --- sampling --------------------------------------------------------------
+
+_BUCKET_MS = 60_000
+
+
+def should_sample(call_site_id: str, sample_rate: float, *, now_ms: int | None = None) -> bool:
+    if sample_rate >= 1:
+        return True
+    if sample_rate <= 0:
+        return False
+    if now_ms is None:
+        import time
+
+        now_ms = int(time.time() * 1000)
+    bucket = now_ms // _BUCKET_MS
+    key = f"{call_site_id}:{bucket}"
+    h = 0x811C9DC5
+    for ch in key.encode("utf-8"):
+        h ^= ch
+        h = (h + ((h << 1) + (h << 4) + (h << 7) + (h << 8) + (h << 24))) & 0xFFFFFFFF
+    return h / 0xFFFFFFFF < sample_rate

frugal_metrics/safe.py

@@ -0,0 +1,34 @@
+"""Safe-call helper — wraps any function so it never throws.
+
+Tries to emit an internal_errors metric on failure; if even that fails,
+swallows silently. The customer's code path must never be interrupted by our
+instrumentation.
+"""
+
+from __future__ import annotations
+
+import logging
+from typing import Callable
+
+logger = logging.getLogger("frugal_metrics")
+
+
+def safe_call(fn: Callable[[], None]) -> None:
+    """Call fn, recording an internal error metric on failure."""
+    try:
+        fn()
+    except Exception as exc:  # noqa: BLE001
+        logger.debug("frugal-metrics: metric emission failed", exc_info=True)
+        try:
+            from frugal_metrics.otel import _instruments
+
+            if _instruments is not None:
+                _instruments.internal_errors.add(
+                    1,
+                    {
+                        "frugal.internal_error.stage": "emission",
+                        "frugal.internal_error.type": type(exc).__name__,
+                    },
+                )
+        except Exception:  # noqa: BLE001
+            pass

frugal_metrics/semconv.py

@@ -0,0 +1,70 @@
+"""OpenTelemetry GenAI semantic convention constants + Frugal extensions."""
+
+# GenAI semconv attribute names (stable as of semconv v1.28+).
+GEN_AI_SYSTEM = "gen_ai.system"
+GEN_AI_REQUEST_MODEL = "gen_ai.request.model"
+GEN_AI_RESPONSE_MODEL = "gen_ai.response.model"
+GEN_AI_OPERATION_NAME = "gen_ai.operation.name"
+GEN_AI_TOKEN_TYPE = "gen_ai.token.type"
+
+# GenAI semconv system identifiers.
+SYSTEM_OPENAI = "openai"
+SYSTEM_ANTHROPIC = "anthropic"
+SYSTEM_AZURE_OPENAI = "az.ai.openai"
+SYSTEM_AWS_BEDROCK = "aws.bedrock"
+SYSTEM_VERTEX_AI = "vertex_ai"
+
+# GenAI semconv operation names.
+OP_CHAT = "chat"
+OP_TEXT_COMPLETION = "text_completion"
+OP_EMBEDDINGS = "embeddings"
+
+# GenAI semconv token types.
+TOKEN_INPUT = "input"
+TOKEN_OUTPUT = "output"
+
+# GenAI semconv metric names.
+METRIC_OPERATION_DURATION = "gen_ai.client.operation.duration"
+METRIC_TOKEN_USAGE = "gen_ai.client.token.usage"
+
+# Frugal-namespaced metric extensions.
+METRIC_CALLS = "frugal.gen_ai.calls"
+METRIC_CACHE_READ_TOKENS = "frugal.gen_ai.cache_read_tokens"
+METRIC_CACHE_WRITE_TOKENS = "frugal.gen_ai.cache_write_tokens"
+METRIC_REASONING_TOKENS = "frugal.gen_ai.reasoning_tokens"
+METRIC_ERRORS = "frugal.gen_ai.errors"
+METRIC_INTERNAL_ERRORS = "frugal.gen_ai.internal_errors"
+METRIC_OUTPUT_CAP_UTILIZATION = "frugal.gen_ai.output_cap_utilization"
+METRIC_CACHE_PREFIX_STABLE = "frugal.gen_ai.cache_prefix_stable"
+METRIC_HISTORY_DEPTH = "frugal.gen_ai.history_depth"
+METRIC_FEW_SHOT_COUNT = "frugal.gen_ai.few_shot_count"
+METRIC_NOISE_RATIO = "frugal.gen_ai.noise_ratio"
+METRIC_STRUCTURED_OUTPUT_HINT = "frugal.gen_ai.structured_output_hint"
+
+# Frugal-namespaced attributes.
+FRUGAL_CUSTOMER_ID = "frugal.customer_id"
+FRUGAL_PROJECT_ID = "frugal.project_id"
+FRUGAL_COMPONENT = "frugal.component"
+FRUGAL_CALLER_FILE = "frugal.caller_file"
+FRUGAL_CALLER_FUNCTION = "frugal.caller_function"
+FRUGAL_ERROR_TYPE = "frugal.error.type"
+FRUGAL_BATCHED = "frugal.batched"
+FRUGAL_STRUCTURED_OUTPUT_MODE = "frugal.structured_output_mode"
+FRUGAL_BLOCK = "frugal.block"
+FRUGAL_STABLE = "frugal.stable"
+
+STRUCTURED_OUTPUT_NONE = "none"
+STRUCTURED_OUTPUT_JSON_OBJECT = "json_object"
+STRUCTURED_OUTPUT_JSON_SCHEMA = "json_schema"
+STRUCTURED_OUTPUT_TOOLS = "tools"
+
+BLOCK_SYSTEM = "system"
+BLOCK_USER = "user"
+BLOCK_TOOLS = "tools"
+BLOCK_SCHEMA = "schema"
+
+# Sentinel for calls whose caller could not be resolved to a file inside the repo.
+CALLER_FILE_EXTERNAL = "<external>"
+CALLER_FILE_UNKNOWN = "<unknown>"
+# Placeholder for cardinality overflow (both file and function use the same value).
+CALLER_OVERFLOW = "<overflow>"

frugal_metrics/streams.py

@@ -0,0 +1,172 @@
+"""Stream proxies that pass events through while accumulating usage metrics.
+
+Used by both OpenAI (stream=True returns an iterator of chunks) and Anthropic
+(stream=True returns an iterator of MessageStreamEvent). The provider-specific
+extractor inspects each event and mutates the accumulating UsageReport in
+place. Metrics are emitted once the underlying iterator terminates, the proxy
+is explicitly closed, or the customer exits a context manager we wrap.
+"""
+
+from __future__ import annotations
+
+import logging
+from typing import Any, Callable, Iterator
+
+from frugal_metrics.instrument import CallEmitter, UsageReport
+
+logger = logging.getLogger("frugal_metrics")
+
+
+StreamEventHandler = Callable[[Any, UsageReport], None]
+"""Invoked for each event/chunk — mutates the passed UsageReport in place."""
+
+
+class SyncStreamProxy:
+    def __init__(
+        self,
+        wrapped: Any,
+        emit: CallEmitter,
+        handler: StreamEventHandler,
+    ) -> None:
+        self._wrapped = wrapped
+        self._emit = emit
+        self._handler = handler
+        self._usage = UsageReport()
+        self._finalized = False
+
+    # --- iteration ---
+
+    def __iter__(self) -> Iterator[Any]:
+        return self
+
+    def __next__(self) -> Any:
+        try:
+            chunk = next(self._wrapped)
+        except StopIteration:
+            self._finalize_success()
+            raise
+        except BaseException as exc:
+            self._finalize_error(exc)
+            raise
+        self._safe_handle(chunk)
+        return chunk
+
+    # --- context manager ---
+
+    def __enter__(self) -> "SyncStreamProxy":
+        enter = getattr(self._wrapped, "__enter__", None)
+        if enter is not None:
+            enter()
+        return self
+
+    def __exit__(self, exc_type, exc, tb) -> Any:
+        exit_ = getattr(self._wrapped, "__exit__", None)
+        suppress = exit_(exc_type, exc, tb) if exit_ is not None else None
+        if exc is not None:
+            self._finalize_error(exc)
+        else:
+            self._finalize_success()
+        return suppress
+
+    # --- plumbing ---
+
+    def close(self) -> None:
+        close = getattr(self._wrapped, "close", None)
+        if callable(close):
+            close()
+        self._finalize_success()
+
+    def __getattr__(self, name: str) -> Any:
+        # Delegate everything else (e.g., .response, .text_stream helpers).
+        return getattr(self._wrapped, name)
+
+    def _safe_handle(self, chunk: Any) -> None:
+        try:
+            self._handler(chunk, self._usage)
+        except Exception:  # noqa: BLE001
+            logger.debug("frugal-metrics: stream handler error", exc_info=True)
+
+    def _finalize_success(self) -> None:
+        if self._finalized:
+            return
+        self._finalized = True
+        self._emit.success(self._usage)
+
+    def _finalize_error(self, exc: BaseException) -> None:
+        if self._finalized:
+            return
+        self._finalized = True
+        self._emit.error(exc)
+
+
+class AsyncStreamProxy:
+    def __init__(
+        self,
+        wrapped: Any,
+        emit: CallEmitter,
+        handler: StreamEventHandler,
+    ) -> None:
+        self._wrapped = wrapped
+        self._emit = emit
+        self._handler = handler
+        self._usage = UsageReport()
+        self._finalized = False
+
+    def __aiter__(self) -> "AsyncStreamProxy":
+        return self
+
+    async def __anext__(self) -> Any:
+        try:
+            chunk = await self._wrapped.__anext__()
+        except StopAsyncIteration:
+            self._finalize_success()
+            raise
+        except BaseException as exc:
+            self._finalize_error(exc)
+            raise
+        self._safe_handle(chunk)
+        return chunk
+
+    async def __aenter__(self) -> "AsyncStreamProxy":
+        aenter = getattr(self._wrapped, "__aenter__", None)
+        if aenter is not None:
+            await aenter()
+        return self
+
+    async def __aexit__(self, exc_type, exc, tb) -> Any:
+        aexit = getattr(self._wrapped, "__aexit__", None)
+        suppress = (
+            await aexit(exc_type, exc, tb) if aexit is not None else None
+        )
+        if exc is not None:
+            self._finalize_error(exc)
+        else:
+            self._finalize_success()
+        return suppress
+
+    async def aclose(self) -> None:
+        aclose = getattr(self._wrapped, "aclose", None)
+        if callable(aclose):
+            await aclose()
+        self._finalize_success()
+
+    def __getattr__(self, name: str) -> Any:
+        return getattr(self._wrapped, name)
+
+    def _safe_handle(self, chunk: Any) -> None:
+        try:
+            self._handler(chunk, self._usage)
+        except Exception:  # noqa: BLE001
+            logger.debug("frugal-metrics: async stream handler error", exc_info=True)
+
+    def _finalize_success(self) -> None:
+        if self._finalized:
+            return
+        self._finalized = True
+        self._emit.success(self._usage)
+
+    def _finalize_error(self, exc: BaseException) -> None:
+        if self._finalized:
+            return
+        self._finalized = True
+        self._emit.error(exc)

frugal_metrics/wrappers/__init__.py

@@ -0,0 +1 @@
+"""Client wrappers for supported AI SDKs."""