frugal-sdk-python 0.1.0__py3-none-any.whl

frugal_metrics/instrument.py

@@ -0,0 +1,268 @@
+"""Shared instrumentation helpers used by per-provider wrappers.
+
+A wrapper calls `start_call()` at the top of its closure to resolve the caller
+and start a timer, then calls `emit.success(...)` on return or `emit.error(...)`
+on exception. We deliberately avoid `@contextmanager` — a contextlib frame on
+the stack between the wrapper and the caller would be picked as the call site.
+"""
+
+from __future__ import annotations
+
+import logging
+import time
+from dataclasses import dataclass
+from typing import Any, Callable
+
+from frugal_metrics import semconv
+from frugal_metrics.block_hash_lru import BlockHashes, get_shared_block_hash_lru
+from frugal_metrics.caller import (
+    CallerInfo,
+    consume_cap_hit_event,
+    resolve_caller,
+)
+from frugal_metrics.config import Config
+from frugal_metrics.otel import Instruments, get_instruments
+from frugal_metrics.prompt_analyzer import (
+    RequestPayload,
+    analyze_request,
+    should_sample,
+    structured_output_mode,
+)
+
+logger = logging.getLogger("frugal_metrics")
+
+
+@dataclass(frozen=True)
+class CallContext:
+    system: str
+    operation: str
+    request_model: str
+    skip_package_dirs: tuple[str, ...]
+    # True when this call is a batch-API submission, False for normal sync/streaming.
+    batched: bool = False
+    # Captured request kwargs for prompt analysis + cap utilization.
+    request_payload: RequestPayload | None = None
+
+
+@dataclass
+class UsageReport:
+    response_model: str | None = None
+    input_tokens: int | None = None
+    output_tokens: int | None = None
+    cache_read_tokens: int | None = None
+    cache_write_tokens: int | None = None
+    reasoning_tokens: int | None = None
+
+
+def _base_attrs(config: Config, ctx: CallContext, caller: CallerInfo) -> dict[str, Any]:
+    attrs: dict[str, Any] = {
+        semconv.GEN_AI_SYSTEM: ctx.system,
+        semconv.GEN_AI_OPERATION_NAME: ctx.operation,
+        semconv.GEN_AI_REQUEST_MODEL: ctx.request_model,
+        semconv.FRUGAL_PROJECT_ID: config.project_id,
+        semconv.FRUGAL_CALLER_FILE: caller.file,
+        semconv.FRUGAL_CALLER_FUNCTION: caller.function,
+        semconv.FRUGAL_BATCHED: ctx.batched,
+    }
+    if config.customer_id:
+        attrs[semconv.FRUGAL_CUSTOMER_ID] = config.customer_id
+    return attrs
+
+
+def start_call(
+    config: Config,
+    ctx: CallContext,
+    *,
+    caller_resolver: Callable[[], CallerInfo] | None = None,
+) -> "CallEmitter":
+    """Resolve the caller, start the timer, return an emitter.
+
+    Must be called directly from the wrapper closure so the stack walk sees the
+    user's frame immediately after our wrapper frame (no intermediate helpers).
+    """
+    instruments = get_instruments(config)
+    resolver = caller_resolver or default_caller_resolver(config, ctx)
+    caller = resolver()
+    if consume_cap_hit_event():
+        try:
+            cap_attrs: dict[str, Any] = {
+                semconv.FRUGAL_PROJECT_ID: config.project_id,
+                "frugal.internal_error.stage": "cardinality_cap_hit",
+            }
+            if config.customer_id:
+                cap_attrs[semconv.FRUGAL_CUSTOMER_ID] = config.customer_id
+            instruments.internal_errors.add(1, cap_attrs)
+        except Exception:  # noqa: BLE001
+            # Never let our own metric emission break a customer call.
+            pass
+    return CallEmitter(
+        config=config,
+        ctx=ctx,
+        caller=caller,
+        instruments=instruments,
+        start=time.perf_counter(),
+    )
+
+
+def default_caller_resolver(
+    config: Config, ctx: CallContext
+) -> Callable[[], CallerInfo]:
+    def resolve() -> CallerInfo:
+        return resolve_caller(
+            skip_package_dirs=ctx.skip_package_dirs,
+            block_list=config.path_block_list,
+            repo_root=config.repo_root,
+            max_call_sites=config.max_call_sites,
+        )
+
+    return resolve
+
+
+@dataclass
+class CallEmitter:
+    config: Config
+    ctx: CallContext
+    caller: CallerInfo
+    instruments: Instruments
+    start: float
+    finalized: bool = False
+
+    def success(self, usage: UsageReport) -> None:
+        if self.finalized:
+            return
+        self.finalized = True
+        attrs = _base_attrs(self.config, self.ctx, self.caller)
+        if usage.response_model:
+            attrs[semconv.GEN_AI_RESPONSE_MODEL] = usage.response_model
+
+        duration = time.perf_counter() - self.start
+        self.instruments.operation_duration.record(duration, attributes=attrs)
+        self.instruments.calls.add(1, attributes=attrs)
+
+        if usage.input_tokens is not None:
+            self.instruments.token_usage.record(
+                usage.input_tokens,
+                attributes={**attrs, semconv.GEN_AI_TOKEN_TYPE: semconv.TOKEN_INPUT},
+            )
+        if usage.output_tokens is not None:
+            self.instruments.token_usage.record(
+                usage.output_tokens,
+                attributes={**attrs, semconv.GEN_AI_TOKEN_TYPE: semconv.TOKEN_OUTPUT},
+            )
+        if usage.cache_read_tokens is not None:
+            self.instruments.cache_read_tokens.record(
+                usage.cache_read_tokens, attributes=attrs
+            )
+        if usage.cache_write_tokens is not None:
+            self.instruments.cache_write_tokens.record(
+                usage.cache_write_tokens, attributes=attrs
+            )
+        if usage.reasoning_tokens is not None:
+            self.instruments.reasoning_tokens.record(
+                usage.reasoning_tokens, attributes=attrs
+            )
+
+        self._record_cap_utilization(usage, attrs)
+        self._maybe_analyze(attrs)
+
+    def _record_cap_utilization(
+        self, usage: UsageReport, attrs: dict[str, Any]
+    ) -> None:
+        payload = self.ctx.request_payload
+        if (
+            payload is None
+            or payload.max_tokens is None
+            or payload.max_tokens <= 0
+            or usage.output_tokens is None
+        ):
+            return
+        ratio = min(1.0, usage.output_tokens / payload.max_tokens)
+        cap_attrs = {
+            **attrs,
+            semconv.FRUGAL_STRUCTURED_OUTPUT_MODE: structured_output_mode(payload),
+        }
+        self.instruments.output_cap_utilization.record(ratio, attributes=cap_attrs)
+
+    def _maybe_analyze(self, attrs: dict[str, Any]) -> None:
+        payload = self.ctx.request_payload
+        if payload is None:
+            return
+        call_site_id = f"{self.caller.file}:{self.caller.function}"
+        if not should_sample(
+            call_site_id, self.config.prompt_analysis_sample_rate
+        ):
+            return
+        try:
+            result = analyze_request(payload)
+        except Exception:  # noqa: BLE001
+            return
+
+        if result.history_depth is not None:
+            self.instruments.history_depth.record(
+                result.history_depth, attributes=attrs
+            )
+        if result.few_shot_count is not None:
+            self.instruments.few_shot_count.record(
+                result.few_shot_count, attributes=attrs
+            )
+        if result.noise_ratio is not None:
+            self.instruments.noise_ratio.record(
+                result.noise_ratio, attributes=attrs
+            )
+        if result.structured_output_hint:
+            self.instruments.structured_output_hint.add(1, attributes=attrs)
+
+        lru = get_shared_block_hash_lru(self.config.max_call_sites)
+        previous = lru.get(call_site_id)
+        current = BlockHashes(
+            system_hash=result.system_hash,
+            user_hash=result.user_hash,
+            tools_hash=result.tools_hash,
+            schema_hash=result.schema_hash,
+        )
+        if previous is not None:
+            self._record_stable(
+                attrs, semconv.BLOCK_SYSTEM, previous.system_hash, current.system_hash
+            )
+            self._record_stable(
+                attrs, semconv.BLOCK_USER, previous.user_hash, current.user_hash
+            )
+            self._record_stable(
+                attrs, semconv.BLOCK_TOOLS, previous.tools_hash, current.tools_hash
+            )
+            self._record_stable(
+                attrs, semconv.BLOCK_SCHEMA, previous.schema_hash, current.schema_hash
+            )
+        lru.set(call_site_id, current)
+
+    def _record_stable(
+        self,
+        attrs: dict[str, Any],
+        block: str,
+        prev: str | None,
+        curr: str | None,
+    ) -> None:
+        # Only emit when both calls had this block — comparing absent-vs-present
+        # would conflate "user didn't send a system prompt this time" with churn.
+        if prev is None or curr is None:
+            return
+        self.instruments.cache_prefix_stable.add(
+            1,
+            attributes={
+                **attrs,
+                semconv.FRUGAL_BLOCK: block,
+                semconv.FRUGAL_STABLE: prev == curr,
+            },
+        )
+
+    def error(self, exc: BaseException) -> None:
+        if self.finalized:
+            return
+        self.finalized = True
+        attrs = _base_attrs(self.config, self.ctx, self.caller)
+        attrs[semconv.FRUGAL_ERROR_TYPE] = type(exc).__name__
+
+        duration = time.perf_counter() - self.start
+        self.instruments.operation_duration.record(duration, attributes=attrs)
+        self.instruments.calls.add(1, attributes=attrs)
+        self.instruments.errors.add(1, attributes=attrs)

frugal_metrics/otel.py

@@ -0,0 +1,321 @@
+"""OpenTelemetry meter provider + OTLP exporter setup.
+
+Lazy init on first wrap call. We own our own MeterProvider (not the global one)
+so we don't conflict with whatever the customer may have already configured.
+Metrics go to a Frugal-namespaced provider with its own OTLP exporter.
+"""
+
+from __future__ import annotations
+
+import logging
+import threading
+from dataclasses import dataclass, replace
+from typing import TYPE_CHECKING, Any
+
+from opentelemetry.exporter.otlp.proto.http.metric_exporter import (
+    OTLPMetricExporter,
+)
+from opentelemetry.sdk.metrics import MeterProvider
+from opentelemetry.sdk.metrics.export import (
+    ConsoleMetricExporter,
+    MetricExporter,
+    PeriodicExportingMetricReader,
+)
+from opentelemetry.sdk.metrics.view import (
+    ExplicitBucketHistogramAggregation,
+    View,
+)
+from opentelemetry.sdk.resources import Resource
+
+from frugal_metrics import semconv
+
+if TYPE_CHECKING:
+    from opentelemetry.metrics import Counter, Histogram, Meter
+
+    from frugal_metrics.config import Config
+
+logger = logging.getLogger("frugal_metrics")
+
+_INSTRUMENTATION_NAME = "frugal-metrics"
+_INSTRUMENTATION_VERSION = "0.1.0"
+
+_lock = threading.Lock()
+_instruments: "Instruments | None" = None
+
+
+@dataclass(frozen=True)
+class Instruments:
+    """The metric instruments we emit to."""
+
+    meter_provider: MeterProvider
+    operation_duration: "Histogram"
+    token_usage: "Histogram"
+    calls: "Counter"
+    cache_read_tokens: "Histogram"
+    cache_write_tokens: "Histogram"
+    reasoning_tokens: "Histogram"
+    errors: "Counter"
+    internal_errors: "Counter"
+    output_cap_utilization: "Histogram"
+    cache_prefix_stable: "Counter"
+    history_depth: "Histogram"
+    few_shot_count: "Histogram"
+    noise_ratio: "Histogram"
+    structured_output_hint: "Counter"
+
+
+class _FanoutCounter:
+    """Wraps a Counter so add() emits the base series plus one per component."""
+
+    def __init__(self, inner: "Counter", components: tuple[str, ...]) -> None:
+        self._inner = inner
+        self._components = components
+
+    def add(
+        self,
+        amount: int | float,
+        attributes: dict[str, Any] | None = None,
+        context: Any = None,
+    ) -> None:
+        self._inner.add(amount, attributes, context)
+        for component in self._components:
+            self._inner.add(
+                amount,
+                {**(attributes or {}), semconv.FRUGAL_COMPONENT: component},
+                context,
+            )
+
+
+class _FanoutHistogram:
+    """Wraps a Histogram so record() emits the base series plus one per component."""
+
+    def __init__(self, inner: "Histogram", components: tuple[str, ...]) -> None:
+        self._inner = inner
+        self._components = components
+
+    def record(
+        self,
+        amount: int | float,
+        attributes: dict[str, Any] | None = None,
+        context: Any = None,
+    ) -> None:
+        self._inner.record(amount, attributes, context)
+        for component in self._components:
+            self._inner.record(
+                amount,
+                {**(attributes or {}), semconv.FRUGAL_COMPONENT: component},
+                context,
+            )
+
+
+def _apply_component_fanout(
+    raw: Instruments, components: tuple[str, ...]
+) -> Instruments:
+    """When FRUGAL_COMPONENTS is set, every per-call gen-AI metric is emitted once
+    without a component label (the canonical project-level series) AND once per
+    component (tagged frugal.component). So usage can be queried for the whole
+    project (component absent) or split by component, with no double-count of the
+    project total. internal_errors is excluded — it's a process-level diagnostic.
+    """
+    if not components:
+        return raw
+    return replace(
+        raw,
+        operation_duration=_FanoutHistogram(raw.operation_duration, components),
+        token_usage=_FanoutHistogram(raw.token_usage, components),
+        calls=_FanoutCounter(raw.calls, components),
+        cache_read_tokens=_FanoutHistogram(raw.cache_read_tokens, components),
+        cache_write_tokens=_FanoutHistogram(raw.cache_write_tokens, components),
+        reasoning_tokens=_FanoutHistogram(raw.reasoning_tokens, components),
+        errors=_FanoutCounter(raw.errors, components),
+        output_cap_utilization=_FanoutHistogram(raw.output_cap_utilization, components),
+        cache_prefix_stable=_FanoutCounter(raw.cache_prefix_stable, components),
+        history_depth=_FanoutHistogram(raw.history_depth, components),
+        few_shot_count=_FanoutHistogram(raw.few_shot_count, components),
+        noise_ratio=_FanoutHistogram(raw.noise_ratio, components),
+        structured_output_hint=_FanoutCounter(raw.structured_output_hint, components),
+    )
+
+
+def get_instruments(config: "Config") -> Instruments:
+    """Return cached instruments, initializing the MeterProvider on first call."""
+    global _instruments
+    if _instruments is not None:
+        return _instruments
+    with _lock:
+        if _instruments is not None:
+            return _instruments
+        _instruments = _build_instruments(config)
+        return _instruments
+
+
+def _build_instruments(config: "Config") -> Instruments:
+    exporter = _build_exporter(config)
+    reader = PeriodicExportingMetricReader(
+        exporter=exporter,
+        export_interval_millis=config.export_interval_ms,
+    )
+    # service.name takes the customer suffix when configured, otherwise stays
+    # flat — the tenant label (auto-injected by VMAuth from the bearer token)
+    # is the canonical customer identifier on the server side, so a missing
+    # customer_id is not a problem.
+    service_name = (
+        f"frugal-metrics-{config.customer_id}"
+        if config.customer_id
+        else "frugal-metrics"
+    )
+    resource_attrs: dict[str, str] = {
+        "service.name": service_name,
+        semconv.FRUGAL_PROJECT_ID: config.project_id,
+    }
+    if config.customer_id:
+        resource_attrs[semconv.FRUGAL_CUSTOMER_ID] = config.customer_id
+    resource = Resource.create(resource_attrs)
+    # Override OTel's default histogram bucket boundaries for operation
+    # duration. The default `(0, 5, 10, 25, 50, 75, 100, 250, 500, 750, 1000,
+    # 2500, 5000, 7500, 10000)` is tuned for long-running operations; LLM
+    # calls almost always finish in 0.05s-30s, so the default sticks every
+    # sample in the (0, 5] bucket and histogram_quantile() returns the same
+    # P50/P95 for every call site. These boundaries give visible P50/P95/P99
+    # spread for realistic LLM workloads.
+    provider = MeterProvider(
+        metric_readers=[reader],
+        resource=resource,
+        views=[
+            View(
+                instrument_name=semconv.METRIC_OPERATION_DURATION,
+                aggregation=ExplicitBucketHistogramAggregation(
+                    (0.05, 0.1, 0.25, 0.5, 0.75, 1, 1.5, 2, 3, 5, 7.5, 10, 15, 20, 30, 60),
+                ),
+            ),
+        ],
+    )
+    meter = provider.get_meter(_INSTRUMENTATION_NAME, _INSTRUMENTATION_VERSION)
+
+    raw = Instruments(
+        meter_provider=provider,
+        operation_duration=meter.create_histogram(
+            name=semconv.METRIC_OPERATION_DURATION,
+            unit="s",
+            description="Duration of AI client operations.",
+        ),
+        token_usage=meter.create_histogram(
+            name=semconv.METRIC_TOKEN_USAGE,
+            unit="{token}",
+            description="Token usage reported by the provider.",
+        ),
+        calls=meter.create_counter(
+            name=semconv.METRIC_CALLS,
+            unit="{call}",
+            description="Count of AI client operation calls.",
+        ),
+        cache_read_tokens=meter.create_histogram(
+            name=semconv.METRIC_CACHE_READ_TOKENS,
+            unit="{token}",
+            description="Tokens served from provider prompt cache.",
+        ),
+        cache_write_tokens=meter.create_histogram(
+            name=semconv.METRIC_CACHE_WRITE_TOKENS,
+            unit="{token}",
+            description="Tokens written to provider prompt cache.",
+        ),
+        reasoning_tokens=meter.create_histogram(
+            name=semconv.METRIC_REASONING_TOKENS,
+            unit="{token}",
+            description="Reasoning / extended thinking tokens billed at output rate.",
+        ),
+        errors=meter.create_counter(
+            name=semconv.METRIC_ERRORS,
+            unit="{error}",
+            description="Count of errors raised by AI client operations.",
+        ),
+        internal_errors=meter.create_counter(
+            name=semconv.METRIC_INTERNAL_ERRORS,
+            unit="{error}",
+            description="Count of frugal-metrics internal failures (instrumentation bugs).",
+        ),
+        output_cap_utilization=meter.create_histogram(
+            name=semconv.METRIC_OUTPUT_CAP_UTILIZATION,
+            unit="1",
+            description="Ratio of output_tokens to the request's max_tokens cap (0 to 1).",
+        ),
+        cache_prefix_stable=meter.create_counter(
+            name=semconv.METRIC_CACHE_PREFIX_STABLE,
+            unit="{call}",
+            description=(
+                "Per-call-site comparison of a request block's hash to the previous call. "
+                "Stable=true when the hash matches."
+            ),
+        ),
+        history_depth=meter.create_histogram(
+            name=semconv.METRIC_HISTORY_DEPTH,
+            unit="{message}",
+            description="Number of messages in the request payload.",
+        ),
+        few_shot_count=meter.create_histogram(
+            name=semconv.METRIC_FEW_SHOT_COUNT,
+            unit="{example}",
+            description=(
+                "Count of consecutive user/assistant pairs preceding the final user message in "
+                "single-turn calls."
+            ),
+        ),
+        noise_ratio=meter.create_histogram(
+            name=semconv.METRIC_NOISE_RATIO,
+            unit="1",
+            description=(
+                "Ratio of noise characters (HTML tags, base64 runs, large whitespace) to total "
+                "input characters."
+            ),
+        ),
+        structured_output_hint=meter.create_counter(
+            name=semconv.METRIC_STRUCTURED_OUTPUT_HINT,
+            unit="{call}",
+            description=(
+                "Count of calls whose prompt asks for structured output in prose without using "
+                "tools or response_format."
+            ),
+        ),
+    )
+    return _apply_component_fanout(raw, config.components)
+
+
+def _build_exporter(config: "Config") -> MetricExporter:
+    if not config.otlp_endpoint:
+        logger.info(
+            "frugal-metrics: no OTLP endpoint configured; falling back to console exporter"
+        )
+        return ConsoleMetricExporter()
+
+    headers = _parse_headers(config.otlp_headers)
+    return OTLPMetricExporter(
+        endpoint=config.otlp_endpoint,
+        headers=headers or None,
+    )
+
+
+def _parse_headers(raw: str | None) -> dict[str, str]:
+    """Parse OTEL-style header strings: key1=value1,key2=value2."""
+    if not raw:
+        return {}
+    headers: dict[str, str] = {}
+    for part in raw.split(","):
+        if not part.strip():
+            continue
+        key, sep, value = part.partition("=")
+        if not sep:
+            continue
+        headers[key.strip()] = value.strip()
+    return headers
+
+
+def reset_instruments_for_tests() -> None:
+    """Test-only hook — wipe the cached instruments between tests."""
+    global _instruments
+    with _lock:
+        if _instruments is not None:
+            try:
+                _instruments.meter_provider.shutdown()
+            except Exception:  # noqa: BLE001
+                pass
+        _instruments = None