toad-eye 0.1.0

package/package.json

@@ -0,0 +1,24 @@
+{
+  "name": "toad-eye",
+  "version": "0.1.0",
+  "description": "One-line OpenTelemetry instrumentation for LLM services",
+  "type": "module",
+  "main": "src/index.ts",
+  "types": "src/index.ts",
+  "scripts": {},
+  "keywords": [
+    "opentelemetry",
+    "llm",
+    "instrumentation"
+  ],
+  "license": "ISC",
+  "dependencies": {
+    "@opentelemetry/api": "^1.9.0",
+    "@opentelemetry/exporter-metrics-otlp-http": "^0.213.0",
+    "@opentelemetry/exporter-trace-otlp-http": "^0.213.0",
+    "@opentelemetry/resources": "^2.6.0",
+    "@opentelemetry/sdk-metrics": "^2.6.0",
+    "@opentelemetry/sdk-node": "^0.213.0",
+    "@opentelemetry/semantic-conventions": "^1.40.0"
+  }
+}

package/src/index.ts

@@ -0,0 +1,11 @@
+export { initObservability, shutdown, getConfig } from "./tracer.js";
+export { traceLLMCall } from "./spans.js";
+export type { LLMCallInput, LLMCallOutput } from "./spans.js";
+export type {
+  ToadEyeConfig,
+  LLMSpanAttributes,
+  LLMProvider,
+  SpanStatus,
+  MetricName,
+} from "./types.js";
+export { LLM_METRICS, LLM_ATTRS } from "./types.js";

package/src/metrics.ts

@@ -0,0 +1,69 @@
+import { metrics } from "@opentelemetry/api";
+import type { Counter, Histogram } from "@opentelemetry/api";
+import { LLM_METRICS, INSTRUMENTATION_NAME } from "./types.js";
+
+let requestDuration: Histogram;
+let requestCost: Histogram;
+let tokensTotal: Counter;
+let requestsTotal: Counter;
+let errorsTotal: Counter;
+
+let initialized = false;
+
+export function initMetrics() {
+  if (initialized) return;
+
+  const meter = metrics.getMeter(INSTRUMENTATION_NAME);
+
+  requestDuration = meter.createHistogram(LLM_METRICS.REQUEST_DURATION, {
+    description: "LLM request duration in milliseconds",
+    unit: "ms",
+  });
+
+  requestCost = meter.createHistogram(LLM_METRICS.REQUEST_COST, {
+    description: "LLM request cost in USD",
+    unit: "USD",
+  });
+
+  tokensTotal = meter.createCounter(LLM_METRICS.TOKENS, {
+    description: "Total tokens consumed",
+  });
+
+  requestsTotal = meter.createCounter(LLM_METRICS.REQUESTS, {
+    description: "Total LLM requests",
+  });
+
+  errorsTotal = meter.createCounter(LLM_METRICS.ERRORS, {
+    description: "Total failed LLM requests",
+  });
+
+  initialized = true;
+}
+
+export function recordRequestDuration(
+  ms: number,
+  provider: string,
+  model: string,
+) {
+  requestDuration.record(ms, { provider, model });
+}
+
+export function recordRequestCost(
+  usd: number,
+  provider: string,
+  model: string,
+) {
+  requestCost.record(usd, { provider, model });
+}
+
+export function recordTokens(count: number, provider: string, model: string) {
+  tokensTotal.add(count, { provider, model });
+}
+
+export function recordRequest(provider: string, model: string) {
+  requestsTotal.add(1, { provider, model });
+}
+
+export function recordError(provider: string, model: string) {
+  errorsTotal.add(1, { provider, model });
+}

package/src/spans.ts

@@ -0,0 +1,103 @@
+import { trace, type Span, SpanStatusCode } from "@opentelemetry/api";
+import type { LLMSpanAttributes } from "./types.js";
+import { LLM_ATTRS, INSTRUMENTATION_NAME } from "./types.js";
+import {
+  recordRequestDuration,
+  recordRequestCost,
+  recordTokens,
+  recordRequest,
+  recordError,
+} from "./metrics.js";
+import { getConfig } from "./tracer.js";
+
+/** Input for traceLLMCall — what the user knows before calling the LLM */
+export interface LLMCallInput {
+  readonly provider: LLMSpanAttributes["provider"];
+  readonly model: string;
+  readonly prompt: string;
+  readonly temperature?: number | undefined;
+}
+
+/** Output from the LLM call — what comes back */
+export interface LLMCallOutput {
+  readonly completion: string;
+  readonly inputTokens: number;
+  readonly outputTokens: number;
+  readonly cost: number;
+}
+
+const tracer = trace.getTracer(INSTRUMENTATION_NAME);
+
+function shouldRecordContent() {
+  return getConfig()?.recordContent !== false;
+}
+
+function setBaseAttributes(span: Span, input: LLMCallInput) {
+  span.setAttributes({
+    [LLM_ATTRS.PROVIDER]: input.provider,
+    [LLM_ATTRS.MODEL]: input.model,
+    [LLM_ATTRS.TEMPERATURE]: input.temperature ?? 1.0,
+    ...(shouldRecordContent() && { [LLM_ATTRS.PROMPT]: input.prompt }),
+  });
+}
+
+export async function traceLLMCall(
+  input: LLMCallInput,
+  fn: () => Promise<LLMCallOutput>,
+): Promise<LLMCallOutput> {
+  return tracer.startActiveSpan(
+    `llm.${input.provider}.${input.model}`,
+    async (span) => {
+      const start = performance.now();
+      setBaseAttributes(span, input);
+
+      try {
+        const output = await fn();
+        const duration = performance.now() - start;
+
+        span.setAttributes({
+          ...(shouldRecordContent() && {
+            [LLM_ATTRS.COMPLETION]: output.completion,
+          }),
+          [LLM_ATTRS.INPUT_TOKENS]: output.inputTokens,
+          [LLM_ATTRS.OUTPUT_TOKENS]: output.outputTokens,
+          [LLM_ATTRS.COST]: output.cost,
+          [LLM_ATTRS.STATUS]: "success",
+        });
+        span.setStatus({ code: SpanStatusCode.OK });
+
+        recordRequest(input.provider, input.model);
+        recordRequestDuration(duration, input.provider, input.model);
+        recordRequestCost(output.cost, input.provider, input.model);
+        recordTokens(
+          output.inputTokens + output.outputTokens,
+          input.provider,
+          input.model,
+        );
+
+        return output;
+      } catch (error) {
+        const duration = performance.now() - start;
+        const message = error instanceof Error ? error.message : String(error);
+
+        span.setAttributes({
+          [LLM_ATTRS.COMPLETION]: "",
+          [LLM_ATTRS.INPUT_TOKENS]: 0,
+          [LLM_ATTRS.OUTPUT_TOKENS]: 0,
+          [LLM_ATTRS.COST]: 0,
+          [LLM_ATTRS.STATUS]: "error",
+          [LLM_ATTRS.ERROR]: message,
+        });
+        span.setStatus({ code: SpanStatusCode.ERROR, message });
+
+        recordRequest(input.provider, input.model);
+        recordRequestDuration(duration, input.provider, input.model);
+        recordError(input.provider, input.model);
+
+        throw error;
+      } finally {
+        span.end();
+      }
+    },
+  );
+}

package/src/tracer.ts

@@ -0,0 +1,57 @@
+import { NodeSDK } from "@opentelemetry/sdk-node";
+import { resourceFromAttributes } from "@opentelemetry/resources";
+import { OTLPTraceExporter } from "@opentelemetry/exporter-trace-otlp-http";
+import { OTLPMetricExporter } from "@opentelemetry/exporter-metrics-otlp-http";
+import { PeriodicExportingMetricReader } from "@opentelemetry/sdk-metrics";
+import { ATTR_SERVICE_NAME } from "@opentelemetry/semantic-conventions";
+import type { ToadEyeConfig } from "./types.js";
+import { initMetrics } from "./metrics.js";
+
+const DEFAULT_ENDPOINT = "http://localhost:4318";
+
+let sdk: NodeSDK | null = null;
+let currentConfig: ToadEyeConfig | null = null;
+
+export function getConfig() {
+  return currentConfig;
+}
+
+export function initObservability(config: ToadEyeConfig) {
+  if (sdk) return;
+
+  const endpoint = config.endpoint ?? DEFAULT_ENDPOINT;
+
+  const resource = resourceFromAttributes({
+    [ATTR_SERVICE_NAME]: config.serviceName,
+  });
+
+  const traceExporter = new OTLPTraceExporter({
+    url: `${endpoint}/v1/traces`,
+  });
+
+  const metricExporter = new OTLPMetricExporter({
+    url: `${endpoint}/v1/metrics`,
+  });
+
+  const metricReader = new PeriodicExportingMetricReader({
+    exporter: metricExporter,
+    exportIntervalMillis: 5000,
+  });
+
+  sdk = new NodeSDK({
+    resource,
+    traceExporter,
+    metricReader,
+  });
+
+  sdk.start();
+  currentConfig = config;
+  initMetrics();
+}
+
+export async function shutdown() {
+  if (!sdk) return;
+  await sdk.shutdown();
+  sdk = null;
+  currentConfig = null;
+}

package/src/types.ts

@@ -0,0 +1,76 @@
+export const INSTRUMENTATION_NAME = "toad-eye";
+
+/**
+ * LLM providers supported by toad-eye
+ */
+export type LLMProvider = "anthropic" | "gemini" | "openai";
+
+/**
+ * Span status
+ */
+export type SpanStatus = "success" | "error";
+
+/**
+ * Attributes attached to every LLM span (trace).
+ * These become searchable fields in Jaeger and filterable dimensions in Grafana.
+ */
+export interface LLMSpanAttributes {
+  readonly provider: LLMProvider;
+  readonly model: string;
+  readonly prompt: string;
+  readonly completion: string;
+  readonly inputTokens: number;
+  readonly outputTokens: number;
+  readonly cost: number;
+  readonly temperature: number;
+  readonly status: SpanStatus;
+  readonly error?: string | undefined;
+}
+
+/**
+ * Metric names exported to Prometheus.
+ *
+ * Naming convention follows OpenTelemetry semantic conventions:
+ * - dots as separators: "llm.request.duration"
+ * - Prometheus auto-converts dots to underscores: llm_request_duration
+ * - Prometheus adds _total suffix to counters automatically — do NOT include it in the name
+ */
+export const LLM_METRICS = {
+  REQUEST_DURATION: "llm.request.duration",
+  REQUEST_COST: "llm.request.cost",
+  TOKENS: "llm.tokens",
+  REQUESTS: "llm.requests",
+  ERRORS: "llm.errors",
+} as const;
+
+/**
+ * Span attribute keys attached to every LLM span.
+ * Using constants prevents typos and enables rename-refactoring.
+ */
+export const LLM_ATTRS = {
+  PROVIDER: "llm.provider",
+  MODEL: "llm.model",
+  PROMPT: "llm.prompt",
+  COMPLETION: "llm.completion",
+  INPUT_TOKENS: "llm.input_tokens",
+  OUTPUT_TOKENS: "llm.output_tokens",
+  COST: "llm.cost",
+  TEMPERATURE: "llm.temperature",
+  STATUS: "llm.status",
+  ERROR: "llm.error",
+} as const;
+
+/**
+ * Type-safe metric name — only values from LLM_METRICS are allowed
+ */
+export type MetricName = (typeof LLM_METRICS)[keyof typeof LLM_METRICS];
+
+/**
+ * Configuration for initObservability().
+ * This is what the user passes when connecting toad-eye to their service.
+ */
+export interface ToadEyeConfig {
+  readonly serviceName: string;
+  readonly endpoint?: string | undefined;
+  readonly recordContent?: boolean | undefined;
+}

package/tsconfig.json

@@ -0,0 +1,5 @@
+{
+  "extends": "../../tsconfig.json",
+  "include": ["src"],
+  "exclude": ["node_modules", "dist"]
+}