toad-eye 0.2.1 → 1.1.0

package/README.md

@@ -18,13 +18,19 @@ One-line instrumentation for any LLM service — get traces, metrics, and Grafan
 
 LLM APIs are black boxes — you don't know what they cost, how slow they are, or why they fail. toad-eye gives you full visibility with one line of code.
 
+## Install
+
+```bash
+npm install toad-eye
+```
+
 ## Architecture
 
 ```
 Your LLM service
        │
        ▼
-@toad-eye/instrumentation
+toad-eye
        │
        ▼ OTLP/HTTP
 OTel Collector
@@ -51,7 +57,7 @@ npm run load --workspace=demo       # send test traffic
 ## Usage
 
 ```typescript
-import { initObservability, traceLLMCall } from "@toad-eye/instrumentation";
+import { initObservability, traceLLMCall } from "toad-eye";
 
 // one-line init
 initObservability({
@@ -108,7 +114,7 @@ All metrics are labeled with `provider` and `model` for filtering and grouping.
 ## Project structure
 
 ```
-packages/instrumentation/   — NPM package (@toad-eye/instrumentation)
+packages/instrumentation/   — NPM package (toad-eye)
 demo/                       — mock LLM service for testing
 infra/                      — docker-compose stack (OTel + Prometheus + Jaeger + Grafana)
 ```

package/dist/cli.d.ts

@@ -0,0 +1,2 @@
+#!/usr/bin/env node
+export {};

package/dist/cli.js

@@ -0,0 +1,123 @@
+#!/usr/bin/env node
+import { execSync } from "node:child_process";
+import { cpSync, existsSync, mkdirSync } from "node:fs";
+import { join, resolve } from "node:path";
+import { fileURLToPath } from "node:url";
+const INFRA_DIR = "infra/toad-eye";
+const SERVICES = [
+    { name: "Grafana", url: "http://localhost:3100", login: "admin / admin" },
+    { name: "Jaeger UI", url: "http://localhost:16686" },
+    { name: "Prometheus", url: "http://localhost:9090" },
+    { name: "OTel Collector", url: "http://localhost:4318" },
+];
+function getTemplatesDir() {
+    const thisFile = fileURLToPath(import.meta.url);
+    // dist/cli.js -> package root -> templates/
+    return join(thisFile, "..", "..", "templates");
+}
+function getComposeFile() {
+    return join(process.cwd(), INFRA_DIR, "docker-compose.yml");
+}
+function init() {
+    const dest = join(process.cwd(), INFRA_DIR);
+    if (existsSync(dest)) {
+        console.log(`\u2705 ${INFRA_DIR}/ already exists, skipping.`);
+        return;
+    }
+    const templatesDir = getTemplatesDir();
+    if (!existsSync(templatesDir)) {
+        console.error(`\u274c Templates not found at ${templatesDir}. Reinstall toad-eye.`);
+        process.exit(1);
+    }
+    mkdirSync(dest, { recursive: true });
+    cpSync(templatesDir, dest, { recursive: true });
+    console.log(`\u2705 Created ${INFRA_DIR}/ with observability stack config.`);
+    console.log();
+    console.log("Next steps:");
+    console.log("  npx toad-eye up      Start the stack");
+    console.log("  npx toad-eye status  Check running services");
+}
+function up() {
+    const composeFile = getComposeFile();
+    if (!existsSync(composeFile)) {
+        console.error(`\u274c ${INFRA_DIR}/ not found. Run \`npx toad-eye init\` first.`);
+        process.exit(1);
+    }
+    console.log("\u{1f438} Starting observability stack...");
+    execSync(`docker compose -f ${composeFile} up -d`, { stdio: "inherit" });
+    console.log();
+    status();
+}
+function down() {
+    const composeFile = getComposeFile();
+    if (!existsSync(composeFile)) {
+        console.error(`\u274c ${INFRA_DIR}/ not found. Nothing to stop.`);
+        process.exit(1);
+    }
+    console.log("\u{1f44b} Stopping observability stack...");
+    execSync(`docker compose -f ${composeFile} down`, { stdio: "inherit" });
+    console.log("\u2705 Stack stopped.");
+}
+function status() {
+    const composeFile = getComposeFile();
+    if (!existsSync(composeFile)) {
+        console.error(`\u274c ${INFRA_DIR}/ not found. Run \`npx toad-eye init\` first.`);
+        process.exit(1);
+    }
+    try {
+        const output = execSync(`docker compose -f ${composeFile} ps --format json`, {
+            encoding: "utf-8",
+        });
+        const lines = output.trim().split("\n").filter(Boolean);
+        const containers = lines.map((line) => JSON.parse(line));
+        console.log("\u{1f438} toad-eye stack:");
+        console.log();
+        for (const svc of SERVICES) {
+            const container = containers.find((c) => svc.name.toLowerCase().includes(c.Service.toLowerCase()));
+            const state = container?.State === "running" ? "\u{1f7e2}" : "\u{1f534}";
+            const loginInfo = "login" in svc ? ` (${svc.login})` : "";
+            console.log(`  ${state} ${svc.name.padEnd(16)} ${svc.url}${loginInfo}`);
+        }
+        console.log();
+    }
+    catch {
+        console.log("Could not check container status. Is Docker running?");
+    }
+}
+function help() {
+    console.log(`
+\u{1f438} toad-eye CLI — observability stack for LLM services
+
+Commands:
+  init     Copy observability configs into your project
+  up       Start the stack (OTel Collector + Prometheus + Jaeger + Grafana)
+  down     Stop the stack
+  status   Show running services and URLs
+  help     Show this message
+`);
+}
+const command = process.argv[2];
+switch (command) {
+    case "init":
+        init();
+        break;
+    case "up":
+        up();
+        break;
+    case "down":
+        down();
+        break;
+    case "status":
+        status();
+        break;
+    case "help":
+    case "--help":
+    case "-h":
+    case undefined:
+        help();
+        break;
+    default:
+        console.error(`Unknown command: ${command}`);
+        help();
+        process.exit(1);
+}

package/{src/index.ts → dist/index.d.ts}

@@ -1,11 +1,5 @@
 export { initObservability, shutdown, getConfig } from "./tracer.js";
 export { traceLLMCall } from "./spans.js";
 export type { LLMCallInput, LLMCallOutput } from "./spans.js";
-export type {
-  ToadEyeConfig,
-  LLMSpanAttributes,
-  LLMProvider,
-  SpanStatus,
-  MetricName,
-} from "./types.js";
+export type { ToadEyeConfig, LLMSpanAttributes, LLMProvider, SpanStatus, MetricName, } from "./types.js";
 export { LLM_METRICS, LLM_ATTRS } from "./types.js";

package/dist/index.js

@@ -0,0 +1,3 @@
+export { initObservability, shutdown, getConfig } from "./tracer.js";
+export { traceLLMCall } from "./spans.js";
+export { LLM_METRICS, LLM_ATTRS } from "./types.js";

package/dist/metrics.d.ts

@@ -0,0 +1,6 @@
+export declare function initMetrics(): void;
+export declare function recordRequestDuration(ms: number, provider: string, model: string): void;
+export declare function recordRequestCost(usd: number, provider: string, model: string): void;
+export declare function recordTokens(count: number, provider: string, model: string): void;
+export declare function recordRequest(provider: string, model: string): void;
+export declare function recordError(provider: string, model: string): void;

package/dist/metrics.js

@@ -0,0 +1,46 @@
+import { metrics } from "@opentelemetry/api";
+import { LLM_METRICS, INSTRUMENTATION_NAME } from "./types.js";
+let requestDuration;
+let requestCost;
+let tokensTotal;
+let requestsTotal;
+let errorsTotal;
+let initialized = false;
+export function initMetrics() {
+    if (initialized)
+        return;
+    const meter = metrics.getMeter(INSTRUMENTATION_NAME);
+    requestDuration = meter.createHistogram(LLM_METRICS.REQUEST_DURATION, {
+        description: "LLM request duration in milliseconds",
+        unit: "ms",
+    });
+    requestCost = meter.createHistogram(LLM_METRICS.REQUEST_COST, {
+        description: "LLM request cost in USD",
+        unit: "USD",
+    });
+    tokensTotal = meter.createCounter(LLM_METRICS.TOKENS, {
+        description: "Total tokens consumed",
+    });
+    requestsTotal = meter.createCounter(LLM_METRICS.REQUESTS, {
+        description: "Total LLM requests",
+    });
+    errorsTotal = meter.createCounter(LLM_METRICS.ERRORS, {
+        description: "Total failed LLM requests",
+    });
+    initialized = true;
+}
+export function recordRequestDuration(ms, provider, model) {
+    requestDuration.record(ms, { provider, model });
+}
+export function recordRequestCost(usd, provider, model) {
+    requestCost.record(usd, { provider, model });
+}
+export function recordTokens(count, provider, model) {
+    tokensTotal.add(count, { provider, model });
+}
+export function recordRequest(provider, model) {
+    requestsTotal.add(1, { provider, model });
+}
+export function recordError(provider, model) {
+    errorsTotal.add(1, { provider, model });
+}

package/dist/spans.d.ts

@@ -0,0 +1,16 @@
+import type { LLMSpanAttributes } from "./types.js";
+/** Input for traceLLMCall — what the user knows before calling the LLM */
+export interface LLMCallInput {
+    readonly provider: LLMSpanAttributes["provider"];
+    readonly model: string;
+    readonly prompt: string;
+    readonly temperature?: number | undefined;
+}
+/** Output from the LLM call — what comes back */
+export interface LLMCallOutput {
+    readonly completion: string;
+    readonly inputTokens: number;
+    readonly outputTokens: number;
+    readonly cost: number;
+}
+export declare function traceLLMCall(input: LLMCallInput, fn: () => Promise<LLMCallOutput>): Promise<LLMCallOutput>;

package/dist/spans.js

@@ -0,0 +1,61 @@
+import { trace, SpanStatusCode } from "@opentelemetry/api";
+import { LLM_ATTRS, INSTRUMENTATION_NAME } from "./types.js";
+import { recordRequestDuration, recordRequestCost, recordTokens, recordRequest, recordError, } from "./metrics.js";
+import { getConfig } from "./tracer.js";
+const tracer = trace.getTracer(INSTRUMENTATION_NAME);
+function shouldRecordContent() {
+    return getConfig()?.recordContent !== false;
+}
+function setBaseAttributes(span, input) {
+    span.setAttributes({
+        [LLM_ATTRS.PROVIDER]: input.provider,
+        [LLM_ATTRS.MODEL]: input.model,
+        [LLM_ATTRS.TEMPERATURE]: input.temperature ?? 1.0,
+        ...(shouldRecordContent() && { [LLM_ATTRS.PROMPT]: input.prompt }),
+    });
+}
+export async function traceLLMCall(input, fn) {
+    return tracer.startActiveSpan(`llm.${input.provider}.${input.model}`, async (span) => {
+        const start = performance.now();
+        setBaseAttributes(span, input);
+        try {
+            const output = await fn();
+            const duration = performance.now() - start;
+            span.setAttributes({
+                ...(shouldRecordContent() && {
+                    [LLM_ATTRS.COMPLETION]: output.completion,
+                }),
+                [LLM_ATTRS.INPUT_TOKENS]: output.inputTokens,
+                [LLM_ATTRS.OUTPUT_TOKENS]: output.outputTokens,
+                [LLM_ATTRS.COST]: output.cost,
+                [LLM_ATTRS.STATUS]: "success",
+            });
+            span.setStatus({ code: SpanStatusCode.OK });
+            recordRequest(input.provider, input.model);
+            recordRequestDuration(duration, input.provider, input.model);
+            recordRequestCost(output.cost, input.provider, input.model);
+            recordTokens(output.inputTokens + output.outputTokens, input.provider, input.model);
+            return output;
+        }
+        catch (error) {
+            const duration = performance.now() - start;
+            const message = error instanceof Error ? error.message : String(error);
+            span.setAttributes({
+                [LLM_ATTRS.COMPLETION]: "",
+                [LLM_ATTRS.INPUT_TOKENS]: 0,
+                [LLM_ATTRS.OUTPUT_TOKENS]: 0,
+                [LLM_ATTRS.COST]: 0,
+                [LLM_ATTRS.STATUS]: "error",
+                [LLM_ATTRS.ERROR]: message,
+            });
+            span.setStatus({ code: SpanStatusCode.ERROR, message });
+            recordRequest(input.provider, input.model);
+            recordRequestDuration(duration, input.provider, input.model);
+            recordError(input.provider, input.model);
+            throw error;
+        }
+        finally {
+            span.end();
+        }
+    });
+}

package/dist/tracer.d.ts

@@ -0,0 +1,4 @@
+import type { ToadEyeConfig } from "./types.js";
+export declare function getConfig(): ToadEyeConfig | null;
+export declare function initObservability(config: ToadEyeConfig): void;
+export declare function shutdown(): Promise<void>;

package/dist/tracer.js

@@ -0,0 +1,46 @@
+import { NodeSDK } from "@opentelemetry/sdk-node";
+import { resourceFromAttributes } from "@opentelemetry/resources";
+import { OTLPTraceExporter } from "@opentelemetry/exporter-trace-otlp-http";
+import { OTLPMetricExporter } from "@opentelemetry/exporter-metrics-otlp-http";
+import { PeriodicExportingMetricReader } from "@opentelemetry/sdk-metrics";
+import { ATTR_SERVICE_NAME } from "@opentelemetry/semantic-conventions";
+import { initMetrics } from "./metrics.js";
+const DEFAULT_ENDPOINT = "http://localhost:4318";
+let sdk = null;
+let currentConfig = null;
+export function getConfig() {
+    return currentConfig;
+}
+export function initObservability(config) {
+    if (sdk)
+        return;
+    const endpoint = config.endpoint ?? DEFAULT_ENDPOINT;
+    const resource = resourceFromAttributes({
+        [ATTR_SERVICE_NAME]: config.serviceName,
+    });
+    const traceExporter = new OTLPTraceExporter({
+        url: `${endpoint}/v1/traces`,
+    });
+    const metricExporter = new OTLPMetricExporter({
+        url: `${endpoint}/v1/metrics`,
+    });
+    const metricReader = new PeriodicExportingMetricReader({
+        exporter: metricExporter,
+        exportIntervalMillis: 5000,
+    });
+    sdk = new NodeSDK({
+        resource,
+        traceExporter,
+        metricReader,
+    });
+    sdk.start();
+    currentConfig = config;
+    initMetrics();
+}
+export async function shutdown() {
+    if (!sdk)
+        return;
+    await sdk.shutdown();
+    sdk = null;
+    currentConfig = null;
+}

package/dist/types.d.ts

@@ -0,0 +1,69 @@
+export declare const INSTRUMENTATION_NAME = "toad-eye";
+/**
+ * LLM providers supported by toad-eye
+ */
+export type LLMProvider = "anthropic" | "gemini" | "openai";
+/**
+ * Span status
+ */
+export type SpanStatus = "success" | "error";
+/**
+ * Attributes attached to every LLM span (trace).
+ * These become searchable fields in Jaeger and filterable dimensions in Grafana.
+ */
+export interface LLMSpanAttributes {
+    readonly provider: LLMProvider;
+    readonly model: string;
+    readonly prompt: string;
+    readonly completion: string;
+    readonly inputTokens: number;
+    readonly outputTokens: number;
+    readonly cost: number;
+    readonly temperature: number;
+    readonly status: SpanStatus;
+    readonly error?: string | undefined;
+}
+/**
+ * Metric names exported to Prometheus.
+ *
+ * Naming convention follows OpenTelemetry semantic conventions:
+ * - dots as separators: "llm.request.duration"
+ * - Prometheus auto-converts dots to underscores: llm_request_duration
+ * - Prometheus adds _total suffix to counters automatically — do NOT include it in the name
+ */
+export declare const LLM_METRICS: {
+    readonly REQUEST_DURATION: "llm.request.duration";
+    readonly REQUEST_COST: "llm.request.cost";
+    readonly TOKENS: "llm.tokens";
+    readonly REQUESTS: "llm.requests";
+    readonly ERRORS: "llm.errors";
+};
+/**
+ * Span attribute keys attached to every LLM span.
+ * Using constants prevents typos and enables rename-refactoring.
+ */
+export declare const LLM_ATTRS: {
+    readonly PROVIDER: "llm.provider";
+    readonly MODEL: "llm.model";
+    readonly PROMPT: "llm.prompt";
+    readonly COMPLETION: "llm.completion";
+    readonly INPUT_TOKENS: "llm.input_tokens";
+    readonly OUTPUT_TOKENS: "llm.output_tokens";
+    readonly COST: "llm.cost";
+    readonly TEMPERATURE: "llm.temperature";
+    readonly STATUS: "llm.status";
+    readonly ERROR: "llm.error";
+};
+/**
+ * Type-safe metric name — only values from LLM_METRICS are allowed
+ */
+export type MetricName = (typeof LLM_METRICS)[keyof typeof LLM_METRICS];
+/**
+ * Configuration for initObservability().
+ * This is what the user passes when connecting toad-eye to their service.
+ */
+export interface ToadEyeConfig {
+    readonly serviceName: string;
+    readonly endpoint?: string | undefined;
+    readonly recordContent?: boolean | undefined;
+}

package/dist/types.js

@@ -0,0 +1,32 @@
+export const INSTRUMENTATION_NAME = "toad-eye";
+/**
+ * Metric names exported to Prometheus.
+ *
+ * Naming convention follows OpenTelemetry semantic conventions:
+ * - dots as separators: "llm.request.duration"
+ * - Prometheus auto-converts dots to underscores: llm_request_duration
+ * - Prometheus adds _total suffix to counters automatically — do NOT include it in the name
+ */
+export const LLM_METRICS = {
+    REQUEST_DURATION: "llm.request.duration",
+    REQUEST_COST: "llm.request.cost",
+    TOKENS: "llm.tokens",
+    REQUESTS: "llm.requests",
+    ERRORS: "llm.errors",
+};
+/**
+ * Span attribute keys attached to every LLM span.
+ * Using constants prevents typos and enables rename-refactoring.
+ */
+export const LLM_ATTRS = {
+    PROVIDER: "llm.provider",
+    MODEL: "llm.model",
+    PROMPT: "llm.prompt",
+    COMPLETION: "llm.completion",
+    INPUT_TOKENS: "llm.input_tokens",
+    OUTPUT_TOKENS: "llm.output_tokens",
+    COST: "llm.cost",
+    TEMPERATURE: "llm.temperature",
+    STATUS: "llm.status",
+    ERROR: "llm.error",
+};

package/package.json

@@ -1,15 +1,27 @@
 {
   "name": "toad-eye",
-  "version": "0.2.1",
+  "version": "1.1.0",
   "description": "One-line OpenTelemetry instrumentation for LLM services",
   "type": "module",
-  "main": "src/index.ts",
-  "types": "src/index.ts",
+  "main": "dist/index.js",
+  "types": "dist/index.d.ts",
+  "bin": {
+    "toad-eye": "dist/cli.js"
+  },
+  "files": [
+    "dist",
+    "templates",
+    "README.md"
+  ],
+  "scripts": {
+    "build": "tsc -p tsconfig.build.json",
+    "prepublishOnly": "npm run build"
+  },
   "repository": {
     "type": "git",
-    "url": "https://github.com/albertalov/toad-eye"
+    "url": "https://github.com/vola-trebla/toad-eye"
   },
-  "homepage": "https://github.com/albertalov/toad-eye#readme",
+  "homepage": "https://github.com/vola-trebla/toad-eye#readme",
   "author": "albertalov",
   "keywords": [
     "opentelemetry",

package/templates/docker-compose.yml

@@ -0,0 +1,40 @@
+services:
+  collector:
+    image: otel/opentelemetry-collector-contrib:latest
+    volumes:
+      - ./otel-collector.yml:/etc/otelcol-contrib/config.yaml
+    ports:
+      - "4318:4318"
+    depends_on:
+      - jaeger
+      - prometheus
+
+  prometheus:
+    image: prom/prometheus:latest
+    volumes:
+      - ./prometheus.yml:/etc/prometheus/prometheus.yml
+    ports:
+      - "9090:9090"
+
+  jaeger:
+    image: jaegertracing/all-in-one:latest
+    environment:
+      - COLLECTOR_OTLP_ENABLED=true
+    ports:
+      - "16686:16686"
+      - "4317:4317"
+
+  grafana:
+    image: grafana/grafana:latest
+    volumes:
+      - ./grafana/datasources.yml:/etc/grafana/provisioning/datasources/datasources.yml
+      - ./grafana/dashboards/dashboards.yml:/etc/grafana/provisioning/dashboards/dashboards.yml
+      - ./grafana/dashboards/llm-overview.json:/etc/grafana/provisioning/dashboards/llm-overview.json
+    environment:
+      - GF_SECURITY_ADMIN_PASSWORD=admin
+      - GF_DASHBOARDS_DEFAULT_HOME_DASHBOARD_PATH=/etc/grafana/provisioning/dashboards/llm-overview.json
+    ports:
+      - "3100:3000"
+    depends_on:
+      - prometheus
+      - jaeger

package/templates/grafana/dashboards/dashboards.yml

@@ -0,0 +1,8 @@
+apiVersion: 1
+
+providers:
+  - name: "toad-eye"
+    type: file
+    options:
+      path: /etc/grafana/provisioning/dashboards
+      foldersFromFilesStructure: false

package/templates/grafana/dashboards/llm-overview.json

@@ -0,0 +1,173 @@
+{
+  "annotations": { "list": [] },
+  "editable": true,
+  "fiscalYearStartMonth": 0,
+  "graphTooltip": 1,
+  "id": null,
+  "links": [],
+  "panels": [
+    {
+      "title": "Request rate (per minute)",
+      "type": "timeseries",
+      "gridPos": { "h": 8, "w": 12, "x": 0, "y": 0 },
+      "datasource": { "type": "prometheus", "uid": "prometheus" },
+      "targets": [
+        {
+          "expr": "sum(rate(llm_requests_total[1m])) by (provider) * 60",
+          "legendFormat": "{{ provider }}"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "unit": "reqpm",
+          "custom": { "drawStyle": "line", "fillOpacity": 15, "lineWidth": 2 }
+        },
+        "overrides": []
+      }
+    },
+    {
+      "title": "Error rate (per minute)",
+      "type": "timeseries",
+      "gridPos": { "h": 8, "w": 12, "x": 12, "y": 0 },
+      "datasource": { "type": "prometheus", "uid": "prometheus" },
+      "targets": [
+        {
+          "expr": "sum(rate(llm_errors_total[1m])) by (provider) * 60",
+          "legendFormat": "{{ provider }}"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "unit": "reqpm",
+          "custom": { "drawStyle": "line", "fillOpacity": 15, "lineWidth": 2 },
+          "color": { "mode": "palette-classic" }
+        },
+        "overrides": []
+      }
+    },
+    {
+      "title": "Latency p50 / p95 (ms)",
+      "type": "timeseries",
+      "gridPos": { "h": 8, "w": 12, "x": 0, "y": 8 },
+      "datasource": { "type": "prometheus", "uid": "prometheus" },
+      "targets": [
+        {
+          "expr": "histogram_quantile(0.50, sum(rate(llm_request_duration_milliseconds_bucket[1m])) by (le, provider))",
+          "legendFormat": "p50 {{ provider }}"
+        },
+        {
+          "expr": "histogram_quantile(0.95, sum(rate(llm_request_duration_milliseconds_bucket[1m])) by (le, provider))",
+          "legendFormat": "p95 {{ provider }}"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "unit": "ms",
+          "custom": { "drawStyle": "line", "fillOpacity": 10, "lineWidth": 2 }
+        },
+        "overrides": []
+      }
+    },
+    {
+      "title": "Cost per minute (USD)",
+      "type": "timeseries",
+      "gridPos": { "h": 8, "w": 12, "x": 12, "y": 8 },
+      "datasource": { "type": "prometheus", "uid": "prometheus" },
+      "targets": [
+        {
+          "expr": "sum(rate(llm_request_cost_USD_sum[1m])) by (provider) * 60",
+          "legendFormat": "{{ provider }}"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "unit": "currencyUSD",
+          "custom": { "drawStyle": "bars", "fillOpacity": 50, "lineWidth": 1 }
+        },
+        "overrides": []
+      }
+    },
+    {
+      "title": "Total tokens consumed",
+      "type": "stat",
+      "gridPos": { "h": 4, "w": 8, "x": 0, "y": 16 },
+      "datasource": { "type": "prometheus", "uid": "prometheus" },
+      "targets": [
+        {
+          "expr": "sum(llm_tokens_total)"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "unit": "short",
+          "thresholds": {
+            "steps": [
+              { "color": "green", "value": null },
+              { "color": "yellow", "value": 100000 },
+              { "color": "red", "value": 1000000 }
+            ]
+          }
+        },
+        "overrides": []
+      }
+    },
+    {
+      "title": "Total requests",
+      "type": "stat",
+      "gridPos": { "h": 4, "w": 8, "x": 8, "y": 16 },
+      "datasource": { "type": "prometheus", "uid": "prometheus" },
+      "targets": [
+        {
+          "expr": "sum(llm_requests_total)"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "unit": "short",
+          "thresholds": {
+            "steps": [
+              { "color": "green", "value": null },
+              { "color": "yellow", "value": 1000 },
+              { "color": "red", "value": 10000 }
+            ]
+          }
+        },
+        "overrides": []
+      }
+    },
+    {
+      "title": "Total errors",
+      "type": "stat",
+      "gridPos": { "h": 4, "w": 8, "x": 16, "y": 16 },
+      "datasource": { "type": "prometheus", "uid": "prometheus" },
+      "targets": [
+        {
+          "expr": "sum(llm_errors_total)"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "unit": "short",
+          "thresholds": {
+            "steps": [
+              { "color": "green", "value": null },
+              { "color": "orange", "value": 10 },
+              { "color": "red", "value": 100 }
+            ]
+          }
+        },
+        "overrides": []
+      }
+    }
+  ],
+  "refresh": "5s",
+  "schemaVersion": 39,
+  "tags": ["llm", "observability", "toad-eye"],
+  "templating": { "list": [] },
+  "time": { "from": "now-15m", "to": "now" },
+  "timepicker": {},
+  "timezone": "browser",
+  "title": "toad-eye: LLM observability",
+  "uid": "toad-eye-llm-overview",
+  "version": 2
+}

package/templates/grafana/datasources.yml

@@ -0,0 +1,15 @@
+apiVersion: 1
+
+datasources:
+  - name: Prometheus
+    type: prometheus
+    uid: prometheus
+    access: proxy
+    url: http://prometheus:9090
+    isDefault: true
+
+  - name: Jaeger
+    type: jaeger
+    uid: jaeger
+    access: proxy
+    url: http://jaeger:16686

package/templates/otel-collector.yml

@@ -0,0 +1,29 @@
+receivers:
+  otlp:
+    protocols:
+      http:
+        endpoint: 0.0.0.0:4318
+
+processors:
+  batch:
+    timeout: 1s
+
+exporters:
+  otlphttp/jaeger:
+    endpoint: http://jaeger:4318
+    tls:
+      insecure: true
+
+  prometheus:
+    endpoint: 0.0.0.0:8889
+
+service:
+  pipelines:
+    traces:
+      receivers: [otlp]
+      processors: [batch]
+      exporters: [otlphttp/jaeger]
+    metrics:
+      receivers: [otlp]
+      processors: [batch]
+      exporters: [prometheus]

package/templates/prometheus.yml

@@ -0,0 +1,7 @@
+global:
+  scrape_interval: 5s
+
+scrape_configs:
+  - job_name: "otel-collector"
+    static_configs:
+      - targets: ["collector:8889"]

package/src/metrics.ts

@@ -1,69 +0,0 @@
-import { metrics } from "@opentelemetry/api";
-import type { Counter, Histogram } from "@opentelemetry/api";
-import { LLM_METRICS, INSTRUMENTATION_NAME } from "./types.js";
-
-let requestDuration: Histogram;
-let requestCost: Histogram;
-let tokensTotal: Counter;
-let requestsTotal: Counter;
-let errorsTotal: Counter;
-
-let initialized = false;
-
-export function initMetrics() {
-  if (initialized) return;
-
-  const meter = metrics.getMeter(INSTRUMENTATION_NAME);
-
-  requestDuration = meter.createHistogram(LLM_METRICS.REQUEST_DURATION, {
-    description: "LLM request duration in milliseconds",
-    unit: "ms",
-  });
-
-  requestCost = meter.createHistogram(LLM_METRICS.REQUEST_COST, {
-    description: "LLM request cost in USD",
-    unit: "USD",
-  });
-
-  tokensTotal = meter.createCounter(LLM_METRICS.TOKENS, {
-    description: "Total tokens consumed",
-  });
-
-  requestsTotal = meter.createCounter(LLM_METRICS.REQUESTS, {
-    description: "Total LLM requests",
-  });
-
-  errorsTotal = meter.createCounter(LLM_METRICS.ERRORS, {
-    description: "Total failed LLM requests",
-  });
-
-  initialized = true;
-}
-
-export function recordRequestDuration(
-  ms: number,
-  provider: string,
-  model: string,
-) {
-  requestDuration.record(ms, { provider, model });
-}
-
-export function recordRequestCost(
-  usd: number,
-  provider: string,
-  model: string,
-) {
-  requestCost.record(usd, { provider, model });
-}
-
-export function recordTokens(count: number, provider: string, model: string) {
-  tokensTotal.add(count, { provider, model });
-}
-
-export function recordRequest(provider: string, model: string) {
-  requestsTotal.add(1, { provider, model });
-}
-
-export function recordError(provider: string, model: string) {
-  errorsTotal.add(1, { provider, model });
-}

package/src/spans.ts

@@ -1,103 +0,0 @@
-import { trace, type Span, SpanStatusCode } from "@opentelemetry/api";
-import type { LLMSpanAttributes } from "./types.js";
-import { LLM_ATTRS, INSTRUMENTATION_NAME } from "./types.js";
-import {
-  recordRequestDuration,
-  recordRequestCost,
-  recordTokens,
-  recordRequest,
-  recordError,
-} from "./metrics.js";
-import { getConfig } from "./tracer.js";
-
-/** Input for traceLLMCall — what the user knows before calling the LLM */
-export interface LLMCallInput {
-  readonly provider: LLMSpanAttributes["provider"];
-  readonly model: string;
-  readonly prompt: string;
-  readonly temperature?: number | undefined;
-}
-
-/** Output from the LLM call — what comes back */
-export interface LLMCallOutput {
-  readonly completion: string;
-  readonly inputTokens: number;
-  readonly outputTokens: number;
-  readonly cost: number;
-}
-
-const tracer = trace.getTracer(INSTRUMENTATION_NAME);
-
-function shouldRecordContent() {
-  return getConfig()?.recordContent !== false;
-}
-
-function setBaseAttributes(span: Span, input: LLMCallInput) {
-  span.setAttributes({
-    [LLM_ATTRS.PROVIDER]: input.provider,
-    [LLM_ATTRS.MODEL]: input.model,
-    [LLM_ATTRS.TEMPERATURE]: input.temperature ?? 1.0,
-    ...(shouldRecordContent() && { [LLM_ATTRS.PROMPT]: input.prompt }),
-  });
-}
-
-export async function traceLLMCall(
-  input: LLMCallInput,
-  fn: () => Promise<LLMCallOutput>,
-): Promise<LLMCallOutput> {
-  return tracer.startActiveSpan(
-    `llm.${input.provider}.${input.model}`,
-    async (span) => {
-      const start = performance.now();
-      setBaseAttributes(span, input);
-
-      try {
-        const output = await fn();
-        const duration = performance.now() - start;
-
-        span.setAttributes({
-          ...(shouldRecordContent() && {
-            [LLM_ATTRS.COMPLETION]: output.completion,
-          }),
-          [LLM_ATTRS.INPUT_TOKENS]: output.inputTokens,
-          [LLM_ATTRS.OUTPUT_TOKENS]: output.outputTokens,
-          [LLM_ATTRS.COST]: output.cost,
-          [LLM_ATTRS.STATUS]: "success",
-        });
-        span.setStatus({ code: SpanStatusCode.OK });
-
-        recordRequest(input.provider, input.model);
-        recordRequestDuration(duration, input.provider, input.model);
-        recordRequestCost(output.cost, input.provider, input.model);
-        recordTokens(
-          output.inputTokens + output.outputTokens,
-          input.provider,
-          input.model,
-        );
-
-        return output;
-      } catch (error) {
-        const duration = performance.now() - start;
-        const message = error instanceof Error ? error.message : String(error);
-
-        span.setAttributes({
-          [LLM_ATTRS.COMPLETION]: "",
-          [LLM_ATTRS.INPUT_TOKENS]: 0,
-          [LLM_ATTRS.OUTPUT_TOKENS]: 0,
-          [LLM_ATTRS.COST]: 0,
-          [LLM_ATTRS.STATUS]: "error",
-          [LLM_ATTRS.ERROR]: message,
-        });
-        span.setStatus({ code: SpanStatusCode.ERROR, message });
-
-        recordRequest(input.provider, input.model);
-        recordRequestDuration(duration, input.provider, input.model);
-        recordError(input.provider, input.model);
-
-        throw error;
-      } finally {
-        span.end();
-      }
-    },
-  );
-}

package/src/tracer.ts

@@ -1,57 +0,0 @@
-import { NodeSDK } from "@opentelemetry/sdk-node";
-import { resourceFromAttributes } from "@opentelemetry/resources";
-import { OTLPTraceExporter } from "@opentelemetry/exporter-trace-otlp-http";
-import { OTLPMetricExporter } from "@opentelemetry/exporter-metrics-otlp-http";
-import { PeriodicExportingMetricReader } from "@opentelemetry/sdk-metrics";
-import { ATTR_SERVICE_NAME } from "@opentelemetry/semantic-conventions";
-import type { ToadEyeConfig } from "./types.js";
-import { initMetrics } from "./metrics.js";
-
-const DEFAULT_ENDPOINT = "http://localhost:4318";
-
-let sdk: NodeSDK | null = null;
-let currentConfig: ToadEyeConfig | null = null;
-
-export function getConfig() {
-  return currentConfig;
-}
-
-export function initObservability(config: ToadEyeConfig) {
-  if (sdk) return;
-
-  const endpoint = config.endpoint ?? DEFAULT_ENDPOINT;
-
-  const resource = resourceFromAttributes({
-    [ATTR_SERVICE_NAME]: config.serviceName,
-  });
-
-  const traceExporter = new OTLPTraceExporter({
-    url: `${endpoint}/v1/traces`,
-  });
-
-  const metricExporter = new OTLPMetricExporter({
-    url: `${endpoint}/v1/metrics`,
-  });
-
-  const metricReader = new PeriodicExportingMetricReader({
-    exporter: metricExporter,
-    exportIntervalMillis: 5000,
-  });
-
-  sdk = new NodeSDK({
-    resource,
-    traceExporter,
-    metricReader,
-  });
-
-  sdk.start();
-  currentConfig = config;
-  initMetrics();
-}
-
-export async function shutdown() {
-  if (!sdk) return;
-  await sdk.shutdown();
-  sdk = null;
-  currentConfig = null;
-}

package/src/types.ts

@@ -1,76 +0,0 @@
-export const INSTRUMENTATION_NAME = "toad-eye";
-
-/**
- * LLM providers supported by toad-eye
- */
-export type LLMProvider = "anthropic" | "gemini" | "openai";
-
-/**
- * Span status
- */
-export type SpanStatus = "success" | "error";
-
-/**
- * Attributes attached to every LLM span (trace).
- * These become searchable fields in Jaeger and filterable dimensions in Grafana.
- */
-export interface LLMSpanAttributes {
-  readonly provider: LLMProvider;
-  readonly model: string;
-  readonly prompt: string;
-  readonly completion: string;
-  readonly inputTokens: number;
-  readonly outputTokens: number;
-  readonly cost: number;
-  readonly temperature: number;
-  readonly status: SpanStatus;
-  readonly error?: string | undefined;
-}
-
-/**
- * Metric names exported to Prometheus.
- *
- * Naming convention follows OpenTelemetry semantic conventions:
- * - dots as separators: "llm.request.duration"
- * - Prometheus auto-converts dots to underscores: llm_request_duration
- * - Prometheus adds _total suffix to counters automatically — do NOT include it in the name
- */
-export const LLM_METRICS = {
-  REQUEST_DURATION: "llm.request.duration",
-  REQUEST_COST: "llm.request.cost",
-  TOKENS: "llm.tokens",
-  REQUESTS: "llm.requests",
-  ERRORS: "llm.errors",
-} as const;
-
-/**
- * Span attribute keys attached to every LLM span.
- * Using constants prevents typos and enables rename-refactoring.
- */
-export const LLM_ATTRS = {
-  PROVIDER: "llm.provider",
-  MODEL: "llm.model",
-  PROMPT: "llm.prompt",
-  COMPLETION: "llm.completion",
-  INPUT_TOKENS: "llm.input_tokens",
-  OUTPUT_TOKENS: "llm.output_tokens",
-  COST: "llm.cost",
-  TEMPERATURE: "llm.temperature",
-  STATUS: "llm.status",
-  ERROR: "llm.error",
-} as const;
-
-/**
- * Type-safe metric name — only values from LLM_METRICS are allowed
- */
-export type MetricName = (typeof LLM_METRICS)[keyof typeof LLM_METRICS];
-
-/**
- * Configuration for initObservability().
- * This is what the user passes when connecting toad-eye to their service.
- */
-export interface ToadEyeConfig {
-  readonly serviceName: string;
-  readonly endpoint?: string | undefined;
-  readonly recordContent?: boolean | undefined;
-}

package/tsconfig.json

@@ -1,5 +0,0 @@
-{
-  "extends": "../../tsconfig.json",
-  "include": ["src"],
-  "exclude": ["node_modules", "dist"]
-}