tracemeter 0.1.0__py3-none-any.whl

tracemeter/__init__.py

@@ -0,0 +1,41 @@
+"""TraceMeter: local-first, zero-infra cost & latency tracing for LLM pipelines.
+
+Emits OpenTelemetry GenAI-compliant spans (`gen_ai.*` attributes) to a
+local SQLite store -- no collector, no exporter config, no account
+signup required to see a cost-annotated trace.
+
+Quickstart:
+
+    import tracemeter
+
+    with tracemeter.span("my_pipeline"):
+        ...
+
+    @tracemeter.trace()
+    def my_step():
+        ...
+"""
+
+from tracemeter.tracer import Tracer, get_default_tracer, trace
+from tracemeter.pricing.engine import compute_cost
+from tracemeter.integrations.openai_wrap import instrument_openai
+from tracemeter.integrations.anthropic_wrap import instrument_anthropic
+from tracemeter.integrations.litellm_wrap import instrument_litellm
+
+__version__ = "0.1.0"
+
+__all__ = [
+    "Tracer",
+    "get_default_tracer",
+    "trace",
+    "span",
+    "compute_cost",
+    "instrument_openai",
+    "instrument_anthropic",
+    "instrument_litellm",
+]
+
+
+def span(name: str, **attributes):
+    """Shorthand for tracemeter.get_default_tracer().span(name, **attrs)."""
+    return get_default_tracer().span(name, **attributes)

tracemeter/cli.py

@@ -0,0 +1,68 @@
+"""tracemeter CLI: `tracemeter serve` launches the local dashboard.
+
+Zero required config: run it, get a URL, see your traces. No collector,
+no exporter, no account.
+"""
+
+from __future__ import annotations
+
+import argparse
+import sys
+
+from tracemeter.storage.sqlite_store import default_db_path
+
+
+def _cmd_serve(args: argparse.Namespace) -> int:
+    try:
+        import uvicorn
+    except ImportError:
+        print(
+            "The dashboard server requires extra dependencies.\n"
+            "Install them with: pip install 'tracemeter[server]'",
+            file=sys.stderr,
+        )
+        return 1
+
+    from tracemeter.server.app import create_app
+    from tracemeter.storage.sqlite_store import SqliteStore
+
+    store = SqliteStore(args.db) if args.db else SqliteStore.default()
+    app = create_app(store=store)
+
+    print(f"TraceMeter dashboard: reading {store.db_path}")
+    print(f"Serving at http://{args.host}:{args.port}")
+    uvicorn.run(app, host=args.host, port=args.port, log_level="warning")
+    return 0
+
+
+def _cmd_where(args: argparse.Namespace) -> int:
+    print(default_db_path())
+    return 0
+
+
+def build_parser() -> argparse.ArgumentParser:
+    parser = argparse.ArgumentParser(prog="tracemeter")
+    subparsers = parser.add_subparsers(dest="command", required=True)
+
+    serve = subparsers.add_parser("serve", help="Launch the local dashboard")
+    serve.add_argument("--host", default="127.0.0.1")
+    serve.add_argument("--port", type=int, default=8765)
+    serve.add_argument(
+        "--db", default=None, help="Path to the SQLite trace DB (default: ~/.tracemeter/traces.db)"
+    )
+    serve.set_defaults(func=_cmd_serve)
+
+    where = subparsers.add_parser("where", help="Print the default trace DB path")
+    where.set_defaults(func=_cmd_where)
+
+    return parser
+
+
+def main() -> None:
+    parser = build_parser()
+    args = parser.parse_args()
+    sys.exit(args.func(args))
+
+
+if __name__ == "__main__":
+    main()

tracemeter/compare.py

@@ -0,0 +1,72 @@
+"""Run comparison: cost/latency diff between two traces (e.g. prompt v2
+vs v1), matching steps by span name within each trace."""
+
+from __future__ import annotations
+
+from typing import Any
+
+from tracemeter.storage.sqlite_store import SqliteStore
+
+
+def _trace_totals(spans: list[dict[str, Any]]) -> dict[str, Any]:
+    cost = sum(s["attributes"].get("tracemeter.cost.usd") or 0.0 for s in spans)
+    if spans:
+        start = min(s["start_time"] for s in spans)
+        end = max(s["end_time"] or s["start_time"] for s in spans)
+        latency_ms = (end - start) * 1000.0
+    else:
+        latency_ms = 0.0
+    return {"cost_usd": cost, "latency_ms": latency_ms, "span_count": len(spans)}
+
+
+def _steps_by_name(spans: list[dict[str, Any]]) -> dict[str, dict[str, Any]]:
+    """Groups non-root spans by name. The root span represents the whole
+    run and is already reflected in the totals, so including it here
+    would just add a noisy "only in A/B" row for every comparison."""
+    steps: dict[str, dict[str, Any]] = {}
+    for s in spans:
+        if s["parent_span_id"] is None:
+            continue
+        step = steps.setdefault(s["name"], {"cost_usd": 0.0, "latency_ms": 0.0, "count": 0})
+        step["cost_usd"] += s["attributes"].get("tracemeter.cost.usd") or 0.0
+        step["latency_ms"] += s["attributes"].get("tracemeter.latency_ms") or 0.0
+        step["count"] += 1
+    return steps
+
+
+def compare_traces(store: SqliteStore, trace_id_a: str, trace_id_b: str) -> dict[str, Any]:
+    spans_a = store.get_trace_spans(trace_id_a)
+    spans_b = store.get_trace_spans(trace_id_b)
+
+    totals_a = _trace_totals(spans_a)
+    totals_b = _trace_totals(spans_b)
+
+    steps_a = _steps_by_name(spans_a)
+    steps_b = _steps_by_name(spans_b)
+
+    step_rows = []
+    for name in sorted(set(steps_a) | set(steps_b)):
+        a = steps_a.get(name, {"cost_usd": 0.0, "latency_ms": 0.0, "count": 0})
+        b = steps_b.get(name, {"cost_usd": 0.0, "latency_ms": 0.0, "count": 0})
+        step_rows.append(
+            {
+                "name": name,
+                "a_cost_usd": a["cost_usd"],
+                "b_cost_usd": b["cost_usd"],
+                "cost_delta_usd": b["cost_usd"] - a["cost_usd"],
+                "a_latency_ms": a["latency_ms"],
+                "b_latency_ms": b["latency_ms"],
+                "latency_delta_ms": b["latency_ms"] - a["latency_ms"],
+                "only_in": "b" if a["count"] == 0 else ("a" if b["count"] == 0 else None),
+            }
+        )
+
+    return {
+        "a": {"trace_id": trace_id_a, **totals_a},
+        "b": {"trace_id": trace_id_b, **totals_b},
+        "delta": {
+            "cost_usd": totals_b["cost_usd"] - totals_a["cost_usd"],
+            "latency_ms": totals_b["latency_ms"] - totals_a["latency_ms"],
+        },
+        "steps": step_rows,
+    }

tracemeter/integrations/__init__.py

@@ -0,0 +1 @@
+"""Auto-instrumentation for common LLM clients: openai, anthropic, litellm."""

tracemeter/integrations/_common.py

@@ -0,0 +1,186 @@
+"""Shared plumbing for provider auto-instrumentation.
+
+Wrapping strategy: patch the bound `create` method on a *client instance*
+(not the class globally) so instrumenting one client never affects
+another, and tests can instrument a fake client with no monkeypatching
+of real SDK internals.
+"""
+
+from __future__ import annotations
+
+import inspect
+import time
+from typing import Any, Callable, Optional
+
+from tracemeter import semconv
+from tracemeter.pricing.engine import compute_cost
+from tracemeter.tracer import Tracer, get_default_tracer
+
+
+class _StreamSpanWrapper:
+    """Wraps a streaming response iterator so the span stays open across
+    iteration and closes (with usage/cost attrs, if available) once the
+    stream is exhausted or errors."""
+
+    def __init__(self, iterator: Any, span_ctx: Any, span: Any, on_chunk: Callable, on_done: Callable):
+        self._iterator = iterator
+        self._span_ctx = span_ctx
+        self._span = span
+        self._on_chunk = on_chunk
+        self._on_done = on_done
+        self._first_chunk_seen = False
+        self._closed = False
+
+    def __iter__(self):
+        return self
+
+    def __next__(self):
+        try:
+            chunk = next(self._iterator)
+        except StopIteration:
+            self._close()
+            raise
+        except BaseException as exc:
+            self._close(exc)
+            raise
+        if not self._first_chunk_seen:
+            self._first_chunk_seen = True
+            self._span.set_attribute(
+                semconv.TRACEMETER_TTFT_MS, (time.time() - self._span.start_time) * 1000.0
+            )
+        self._on_chunk(chunk)
+        return chunk
+
+    def _close(self, exc: Optional[BaseException] = None):
+        if self._closed:
+            return
+        self._closed = True
+        self._on_done()
+        self._span_ctx.__exit__(type(exc) if exc else None, exc, None)
+
+
+def instrument_create_method(
+    client: Any,
+    attr_path: str,
+    system: str,
+    operation: str,
+    extract_model: Callable[[dict], Optional[str]],
+    extract_usage: Callable[[Any], tuple[int, int, int]],
+    is_streaming: Callable[[dict], bool],
+    on_stream_chunk_usage: Optional[Callable[[Any], Optional[tuple[int, int, int]]]] = None,
+    tracer: Optional[Tracer] = None,
+) -> None:
+    """Monkeypatch `client.<attr_path>` (e.g. "chat.completions.create") to
+    emit an OTel-GenAI-compliant span around every call.
+
+    extract_usage(response) -> (input_tokens, output_tokens, reasoning_tokens)
+    on_stream_chunk_usage(chunk) -> usage tuple or None; called per chunk to
+        find the final chunk's usage in streaming responses.
+    """
+    tracer = tracer or get_default_tracer()
+    parts = attr_path.split(".")
+    parent = client
+    for p in parts[:-1]:
+        parent = getattr(parent, p)
+    method_name = parts[-1]
+    original = getattr(parent, method_name)
+
+    if getattr(original, "_tracemeter_wrapped", False):
+        return  # already instrumented
+
+    def _start_span(kwargs: dict):
+        model = extract_model(kwargs)
+        span_ctx = tracer.span(
+            f"{system}.{operation}",
+            **{
+                semconv.GEN_AI_SYSTEM: system,
+                semconv.GEN_AI_OPERATION_NAME: operation,
+                semconv.GEN_AI_REQUEST_MODEL: model,
+            },
+        )
+        span = span_ctx.__enter__()
+        if "max_tokens" in kwargs:
+            span.set_attribute(semconv.GEN_AI_REQUEST_MAX_TOKENS, kwargs["max_tokens"])
+        if "temperature" in kwargs:
+            span.set_attribute(semconv.GEN_AI_REQUEST_TEMPERATURE, kwargs["temperature"])
+        return span_ctx, span, model
+
+    def _finish_non_streaming(span_ctx, span, model, response):
+        in_tok, out_tok, reason_tok = extract_usage(response)
+        response_model = getattr(response, "model", None) or model
+        span.set_attributes(
+            {
+                semconv.GEN_AI_RESPONSE_MODEL: response_model,
+                semconv.GEN_AI_USAGE_INPUT_TOKENS: in_tok,
+                semconv.GEN_AI_USAGE_OUTPUT_TOKENS: out_tok,
+            }
+        )
+        if reason_tok:
+            span.set_attribute(semconv.GEN_AI_USAGE_REASONING_TOKENS, reason_tok)
+        _set_cost(span, system, response_model or model, in_tok, out_tok, reason_tok)
+        span_ctx.__exit__(None, None, None)
+
+    def _make_stream_wrapper(span_ctx, span, model, iterator):
+        usage_holder: dict[str, Any] = {}
+
+        def on_chunk(chunk):
+            if on_stream_chunk_usage:
+                usage = on_stream_chunk_usage(chunk)
+                if usage:
+                    usage_holder["usage"] = usage
+
+        def on_done():
+            in_tok, out_tok, reason_tok = usage_holder.get("usage", (0, 0, 0))
+            span.set_attributes(
+                {
+                    semconv.GEN_AI_RESPONSE_MODEL: model,
+                    semconv.GEN_AI_USAGE_INPUT_TOKENS: in_tok,
+                    semconv.GEN_AI_USAGE_OUTPUT_TOKENS: out_tok,
+                }
+            )
+            if reason_tok:
+                span.set_attribute(semconv.GEN_AI_USAGE_REASONING_TOKENS, reason_tok)
+            _set_cost(span, system, model, in_tok, out_tok, reason_tok)
+
+        return _StreamSpanWrapper(iterator, span_ctx, span, on_chunk, on_done)
+
+    if inspect.iscoroutinefunction(original):
+
+        async def wrapper(*args: Any, **kwargs: Any) -> Any:
+            span_ctx, span, model = _start_span(kwargs)
+            try:
+                response = await original(*args, **kwargs)
+            except BaseException as exc:
+                span_ctx.__exit__(type(exc), exc, None)
+                raise
+            if is_streaming(kwargs):
+                return _make_stream_wrapper(span_ctx, span, model, response)
+            _finish_non_streaming(span_ctx, span, model, response)
+            return response
+
+    else:
+
+        def wrapper(*args: Any, **kwargs: Any) -> Any:
+            span_ctx, span, model = _start_span(kwargs)
+            try:
+                response = original(*args, **kwargs)
+            except BaseException as exc:
+                span_ctx.__exit__(type(exc), exc, None)
+                raise
+            if is_streaming(kwargs):
+                return _make_stream_wrapper(span_ctx, span, model, response)
+            _finish_non_streaming(span_ctx, span, model, response)
+            return response
+
+    wrapper._tracemeter_wrapped = True  # type: ignore[attr-defined]
+    setattr(parent, method_name, wrapper)
+
+
+def _set_cost(span, system: str, model: Optional[str], in_tok: int, out_tok: int, reason_tok: int) -> None:
+    cost = compute_cost(
+        model, input_tokens=in_tok, output_tokens=out_tok, reasoning_tokens=reason_tok, system=system
+    )
+    if cost is None:
+        span.set_attribute(semconv.TRACEMETER_COST_UNKNOWN, True)
+    else:
+        span.set_attribute(semconv.TRACEMETER_COST_USD, cost)

tracemeter/integrations/anthropic_wrap.py

@@ -0,0 +1,69 @@
+"""Auto-instrumentation for the `anthropic` client.
+
+    from anthropic import Anthropic
+    from tracemeter.integrations.anthropic_wrap import instrument_anthropic
+
+    client = Anthropic()
+    instrument_anthropic(client)
+
+    client.messages.create(...)  # now traced automatically
+"""
+
+from __future__ import annotations
+
+from typing import Any, Optional
+
+from tracemeter.integrations._common import instrument_create_method
+from tracemeter.tracer import Tracer
+
+
+def _extract_model(kwargs: dict) -> Optional[str]:
+    return kwargs.get("model")
+
+
+def _extract_usage(response: Any) -> tuple[int, int, int]:
+    usage = getattr(response, "usage", None)
+    if usage is None:
+        return (0, 0, 0)
+    input_tokens = getattr(usage, "input_tokens", 0) or 0
+    output_tokens = getattr(usage, "output_tokens", 0) or 0
+    return (input_tokens, output_tokens, 0)
+
+
+def _is_streaming(kwargs: dict) -> bool:
+    return bool(kwargs.get("stream"))
+
+
+def _on_stream_chunk_usage(chunk: Any) -> Optional[tuple[int, int, int]]:
+    # Anthropic streams a message_delta event carrying cumulative output
+    # usage near the end of the stream; message_start carries input usage.
+    usage = getattr(chunk, "usage", None)
+    if usage is None:
+        message = getattr(chunk, "message", None)
+        usage = getattr(message, "usage", None) if message else None
+    if usage is None:
+        return None
+    input_tokens = getattr(usage, "input_tokens", 0) or 0
+    output_tokens = getattr(usage, "output_tokens", 0) or 0
+    if not input_tokens and not output_tokens:
+        return None
+    return (input_tokens, output_tokens, 0)
+
+
+def instrument_anthropic(client: Any, tracer: Optional[Tracer] = None) -> Any:
+    """Instrument an Anthropic (or AsyncAnthropic) client instance in place.
+
+    Wraps `client.messages.create`. Returns the same client for convenience.
+    """
+    instrument_create_method(
+        client,
+        "messages.create",
+        system="anthropic",
+        operation="chat",
+        extract_model=_extract_model,
+        extract_usage=_extract_usage,
+        is_streaming=_is_streaming,
+        on_stream_chunk_usage=_on_stream_chunk_usage,
+        tracer=tracer,
+    )
+    return client

tracemeter/integrations/litellm_wrap.py

@@ -0,0 +1,68 @@
+"""Auto-instrumentation for `litellm`.
+
+Unlike the openai/anthropic SDKs, litellm's primary interface is a
+module-level function (`litellm.completion`, `litellm.acompletion`)
+rather than a client instance, so this patches the module directly
+rather than a per-instance bound method.
+
+    import litellm
+    from tracemeter.integrations.litellm_wrap import instrument_litellm
+
+    instrument_litellm(litellm)
+
+    litellm.completion(model="gpt-4o-mini", messages=[...])  # traced
+"""
+
+from __future__ import annotations
+
+from typing import Any, Optional
+
+from tracemeter.integrations._common import instrument_create_method
+from tracemeter.tracer import Tracer
+
+
+def _extract_model(kwargs: dict) -> Optional[str]:
+    return kwargs.get("model")
+
+
+def _extract_usage(response: Any) -> tuple[int, int, int]:
+    usage = getattr(response, "usage", None)
+    if usage is None and isinstance(response, dict):
+        usage = response.get("usage")
+    if usage is None:
+        return (0, 0, 0)
+    get = (lambda k: usage.get(k)) if isinstance(usage, dict) else (lambda k: getattr(usage, k, None))
+    return (get("prompt_tokens") or 0, get("completion_tokens") or 0, 0)
+
+
+def _is_streaming(kwargs: dict) -> bool:
+    return bool(kwargs.get("stream"))
+
+
+def instrument_litellm(litellm_module: Any, tracer: Optional[Tracer] = None) -> Any:
+    """Instrument the `litellm` module's `completion` (and `acompletion`,
+    if present) in place. litellm normalizes providers internally, so
+    `gen_ai.system` is reported as "litellm" -- the underlying provider is
+    still visible via the model string."""
+    instrument_create_method(
+        litellm_module,
+        "completion",
+        system="litellm",
+        operation="chat",
+        extract_model=_extract_model,
+        extract_usage=_extract_usage,
+        is_streaming=_is_streaming,
+        tracer=tracer,
+    )
+    if hasattr(litellm_module, "acompletion"):
+        instrument_create_method(
+            litellm_module,
+            "acompletion",
+            system="litellm",
+            operation="chat",
+            extract_model=_extract_model,
+            extract_usage=_extract_usage,
+            is_streaming=_is_streaming,
+            tracer=tracer,
+        )
+    return litellm_module

tracemeter/integrations/openai_wrap.py

@@ -0,0 +1,79 @@
+"""Auto-instrumentation for the `openai` client (>=1.0 SDK).
+
+    from openai import OpenAI
+    from tracemeter.integrations.openai_wrap import instrument_openai
+
+    client = OpenAI()
+    instrument_openai(client)
+
+    client.chat.completions.create(...)  # now traced automatically
+"""
+
+from __future__ import annotations
+
+from typing import Any, Optional
+
+from tracemeter.integrations._common import instrument_create_method
+from tracemeter.tracer import Tracer
+
+
+def _extract_model(kwargs: dict) -> Optional[str]:
+    return kwargs.get("model")
+
+
+def _extract_usage(response: Any) -> tuple[int, int, int]:
+    usage = getattr(response, "usage", None)
+    if usage is None:
+        return (0, 0, 0)
+    input_tokens = getattr(usage, "prompt_tokens", 0) or 0
+    output_tokens = getattr(usage, "completion_tokens", 0) or 0
+    reasoning_tokens = 0
+    details = getattr(usage, "completion_tokens_details", None)
+    if details is not None:
+        reasoning_tokens = getattr(details, "reasoning_tokens", 0) or 0
+    return (input_tokens, output_tokens, reasoning_tokens)
+
+
+def _is_streaming(kwargs: dict) -> bool:
+    return bool(kwargs.get("stream"))
+
+
+def _on_stream_chunk_usage(chunk: Any) -> Optional[tuple[int, int, int]]:
+    # Only populated on the final chunk when `stream_options={"include_usage": True}`.
+    usage = getattr(chunk, "usage", None)
+    if usage is None:
+        return None
+    input_tokens = getattr(usage, "prompt_tokens", 0) or 0
+    output_tokens = getattr(usage, "completion_tokens", 0) or 0
+    return (input_tokens, output_tokens, 0)
+
+
+def instrument_openai(client: Any, tracer: Optional[Tracer] = None) -> Any:
+    """Instrument an OpenAI (or AsyncOpenAI) client instance in place.
+
+    Wraps `client.chat.completions.create` and `client.embeddings.create`.
+    Returns the same client for convenience.
+    """
+    instrument_create_method(
+        client,
+        "chat.completions.create",
+        system="openai",
+        operation="chat",
+        extract_model=_extract_model,
+        extract_usage=_extract_usage,
+        is_streaming=_is_streaming,
+        on_stream_chunk_usage=_on_stream_chunk_usage,
+        tracer=tracer,
+    )
+    if hasattr(client, "embeddings"):
+        instrument_create_method(
+            client,
+            "embeddings.create",
+            system="openai",
+            operation="embeddings",
+            extract_model=_extract_model,
+            extract_usage=_extract_usage,
+            is_streaming=lambda kwargs: False,
+            tracer=tracer,
+        )
+    return client

tracemeter/pricing/__init__.py

@@ -0,0 +1,3 @@
+from tracemeter.pricing.engine import PricingEngine, compute_cost
+
+__all__ = ["PricingEngine", "compute_cost"]