toad-eye 1.1.0 → 2.1.0

package/dist/instrumentations/openai.js

@@ -0,0 +1,94 @@
+import { register } from "./registry.js";
+import { createInstrumentation } from "./create.js";
+function extractMessages(messages) {
+    if (!Array.isArray(messages))
+        return "";
+    return messages
+        .map((m) => {
+        if (typeof m.content === "string")
+            return m.content;
+        if (Array.isArray(m.content)) {
+            return m.content
+                .map((part) => {
+                if (part.type === "text")
+                    return part.text ?? "";
+                if (part.type === "image_url")
+                    return "[image]";
+                if (part.type === "input_audio")
+                    return "[audio]";
+                return "";
+            })
+                .filter(Boolean)
+                .join("");
+        }
+        return "";
+    })
+        .filter(Boolean)
+        .join("\n");
+}
+const chatCompletions = {
+    getPrototype: (sdk) => (sdk?.Chat?.Completions ?? sdk?.Completions)?.prototype,
+    method: "create",
+    extractRequest: (body) => {
+        const b = body;
+        return {
+            prompt: extractMessages(b?.messages),
+            model: b?.model ?? "unknown",
+            temperature: b?.temperature ?? 1.0,
+        };
+    },
+    extractResponse: (response) => {
+        const r = response;
+        return {
+            completion: r?.choices?.[0]?.message?.content ?? "",
+            inputTokens: r?.usage?.prompt_tokens ?? 0,
+            outputTokens: r?.usage?.completion_tokens ?? 0,
+        };
+    },
+    isStreaming: (body) => !!body?.stream,
+    extractStreamResponse: (chunks) => {
+        // OpenAI stream chunks: { choices: [{ delta: { content } }], usage?: { ... } }
+        let completion = "";
+        let inputTokens = 0;
+        let outputTokens = 0;
+        for (const chunk of chunks) {
+            const c = chunk;
+            const delta = c?.choices?.[0]?.delta?.content;
+            if (delta)
+                completion += delta;
+            // Usage is typically in the final chunk (when stream_options.include_usage is set)
+            if (c?.usage) {
+                inputTokens = c.usage.prompt_tokens ?? inputTokens;
+                outputTokens = c.usage.completion_tokens ?? outputTokens;
+            }
+        }
+        return { completion, inputTokens, outputTokens };
+    },
+};
+const embeddings = {
+    getPrototype: (sdk) => sdk?.Embeddings?.prototype,
+    method: "create",
+    extractRequest: (body) => {
+        const b = body;
+        const input = b?.input;
+        const prompt = typeof input === "string"
+            ? input
+            : Array.isArray(input)
+                ? input.join("\n")
+                : "";
+        return { prompt, model: b?.model ?? "unknown" };
+    },
+    extractResponse: (response) => {
+        const r = response;
+        return {
+            completion: "[embedding]",
+            inputTokens: r?.usage?.prompt_tokens ?? 0,
+            outputTokens: 0,
+        };
+    },
+};
+register(createInstrumentation({
+    name: "openai",
+    moduleName: "openai",
+    patches: [chatCompletions, embeddings],
+}));

package/dist/instrumentations/registry.d.ts

@@ -0,0 +1,5 @@
+import type { LLMProvider } from "../types/index.js";
+import type { Instrumentation } from "./types.js";
+export declare function register(inst: Instrumentation): void;
+export declare function enableAll(providers: readonly LLMProvider[]): void;
+export declare function disableAll(): void;

package/dist/instrumentations/registry.js

@@ -0,0 +1,31 @@
+import { diag } from "@opentelemetry/api";
+const instrumentations = new Map();
+const active = new Set();
+export function register(inst) {
+    instrumentations.set(inst.name, inst);
+}
+export function enableAll(providers) {
+    for (const name of providers) {
+        if (active.has(name))
+            continue;
+        const inst = instrumentations.get(name);
+        if (!inst) {
+            diag.warn(`toad-eye: unknown provider "${name}", skipping`);
+            continue;
+        }
+        const patched = inst.enable();
+        if (patched) {
+            active.add(name);
+            diag.debug(`toad-eye: auto-instrumented ${name}`);
+        }
+        else {
+            diag.debug(`toad-eye: ${name} SDK not found, skipping`);
+        }
+    }
+}
+export function disableAll() {
+    for (const name of active) {
+        instrumentations.get(name)?.disable();
+    }
+    active.clear();
+}

package/dist/instrumentations/types.d.ts

@@ -0,0 +1,37 @@
+import type { LLMProvider } from "../types/index.js";
+export interface Instrumentation {
+    readonly name: LLMProvider;
+    /** Try to find and patch the SDK. Returns true if patched, false if SDK not found. */
+    enable(): boolean;
+    /** Restore original methods. */
+    disable(): void;
+}
+/** Describes a single method to monkey-patch on an SDK prototype. */
+export interface PatchTarget {
+    /** How to find the prototype from the loaded SDK module */
+    getPrototype: (sdk: any) => any | undefined;
+    /** Method name on the prototype to patch */
+    method: string;
+    /** Extract LLMCallInput fields from the request arguments */
+    extractRequest: (body: unknown) => {
+        prompt: string;
+        model: string;
+        temperature?: number;
+    };
+    /** Extract LLMCallOutput fields from the SDK response */
+    extractResponse: (response: unknown, model: string) => {
+        completion: string;
+        inputTokens: number;
+        outputTokens: number;
+    };
+    /** Return true if this call should skip patching entirely */
+    shouldSkip?: (body: unknown) => boolean;
+    /** Return true if this call is a streaming request */
+    isStreaming?: (body: unknown) => boolean;
+    /** Extract final stats from accumulated stream chunks. Required when isStreaming is set. */
+    extractStreamResponse?: (chunks: unknown[]) => {
+        completion: string;
+        inputTokens: number;
+        outputTokens: number;
+    };
+}

package/dist/instrumentations/types.js

@@ -0,0 +1 @@
+export {};

package/dist/types/attributes.d.ts

@@ -0,0 +1,48 @@
+/**
+ * Span attribute keys following OTel GenAI semantic conventions.
+ * See: https://opentelemetry.io/docs/specs/semconv/gen-ai/
+ *
+ * Standard attributes use `gen_ai.*` prefix.
+ * toad-eye extensions use `gen_ai.toad_eye.*` prefix.
+ */
+export declare const GEN_AI_ATTRS: {
+    readonly PROVIDER: "gen_ai.provider.name";
+    readonly REQUEST_MODEL: "gen_ai.request.model";
+    readonly RESPONSE_MODEL: "gen_ai.response.model";
+    readonly INPUT_TOKENS: "gen_ai.usage.input_tokens";
+    readonly OUTPUT_TOKENS: "gen_ai.usage.output_tokens";
+    readonly TEMPERATURE: "gen_ai.request.temperature";
+    readonly OPERATION: "gen_ai.operation.name";
+    readonly FINISH_REASONS: "gen_ai.response.finish_reasons";
+    readonly ERROR: "error.type";
+    readonly AGENT_STEP_TYPE: "gen_ai.agent.step.type";
+    readonly AGENT_STEP_NUMBER: "gen_ai.agent.step.number";
+    readonly AGENT_TOOL_NAME: "gen_ai.agent.tool.name";
+    readonly AGENT_STEP_CONTENT: "gen_ai.agent.step.content";
+    readonly GUARD_MODE: "gen_ai.toad_eye.guard.mode";
+    readonly GUARD_PASSED: "gen_ai.toad_eye.guard.passed";
+    readonly GUARD_FAILURE_REASON: "gen_ai.toad_eye.guard.failure_reason";
+    readonly GUARD_RULE_NAME: "gen_ai.toad_eye.guard.rule_name";
+    readonly PROMPT: "gen_ai.toad_eye.prompt";
+    readonly COMPLETION: "gen_ai.toad_eye.completion";
+    readonly COST: "gen_ai.toad_eye.cost";
+    readonly STATUS: "gen_ai.toad_eye.status";
+    readonly SESSION_ID: "session.id";
+    readonly TEAM: "toad_eye.team";
+    readonly USER_ID: "toad_eye.user_id";
+    readonly FEATURE: "toad_eye.feature";
+    readonly ENVIRONMENT: "toad_eye.environment";
+};
+/** @deprecated Use GEN_AI_ATTRS instead. Kept for backward compatibility. */
+export declare const LLM_ATTRS: {
+    readonly PROVIDER: "gen_ai.provider.name";
+    readonly MODEL: "gen_ai.request.model";
+    readonly PROMPT: "gen_ai.toad_eye.prompt";
+    readonly COMPLETION: "gen_ai.toad_eye.completion";
+    readonly INPUT_TOKENS: "gen_ai.usage.input_tokens";
+    readonly OUTPUT_TOKENS: "gen_ai.usage.output_tokens";
+    readonly COST: "gen_ai.toad_eye.cost";
+    readonly TEMPERATURE: "gen_ai.request.temperature";
+    readonly STATUS: "gen_ai.toad_eye.status";
+    readonly ERROR: "error.type";
+};

package/dist/types/attributes.js

@@ -0,0 +1,53 @@
+/**
+ * Span attribute keys following OTel GenAI semantic conventions.
+ * See: https://opentelemetry.io/docs/specs/semconv/gen-ai/
+ *
+ * Standard attributes use `gen_ai.*` prefix.
+ * toad-eye extensions use `gen_ai.toad_eye.*` prefix.
+ */
+export const GEN_AI_ATTRS = {
+    // OTel GenAI standard
+    PROVIDER: "gen_ai.provider.name",
+    REQUEST_MODEL: "gen_ai.request.model",
+    RESPONSE_MODEL: "gen_ai.response.model",
+    INPUT_TOKENS: "gen_ai.usage.input_tokens",
+    OUTPUT_TOKENS: "gen_ai.usage.output_tokens",
+    TEMPERATURE: "gen_ai.request.temperature",
+    OPERATION: "gen_ai.operation.name",
+    FINISH_REASONS: "gen_ai.response.finish_reasons",
+    ERROR: "error.type",
+    // Agent step attributes
+    AGENT_STEP_TYPE: "gen_ai.agent.step.type",
+    AGENT_STEP_NUMBER: "gen_ai.agent.step.number",
+    AGENT_TOOL_NAME: "gen_ai.agent.tool.name",
+    AGENT_STEP_CONTENT: "gen_ai.agent.step.content",
+    // Guard (shadow mode) attributes
+    GUARD_MODE: "gen_ai.toad_eye.guard.mode",
+    GUARD_PASSED: "gen_ai.toad_eye.guard.passed",
+    GUARD_FAILURE_REASON: "gen_ai.toad_eye.guard.failure_reason",
+    GUARD_RULE_NAME: "gen_ai.toad_eye.guard.rule_name",
+    // toad-eye extensions
+    PROMPT: "gen_ai.toad_eye.prompt",
+    COMPLETION: "gen_ai.toad_eye.completion",
+    COST: "gen_ai.toad_eye.cost",
+    STATUS: "gen_ai.toad_eye.status",
+    SESSION_ID: "session.id",
+    // FinOps attribution
+    TEAM: "toad_eye.team",
+    USER_ID: "toad_eye.user_id",
+    FEATURE: "toad_eye.feature",
+    ENVIRONMENT: "toad_eye.environment",
+};
+/** @deprecated Use GEN_AI_ATTRS instead. Kept for backward compatibility. */
+export const LLM_ATTRS = {
+    PROVIDER: GEN_AI_ATTRS.PROVIDER,
+    MODEL: GEN_AI_ATTRS.REQUEST_MODEL,
+    PROMPT: GEN_AI_ATTRS.PROMPT,
+    COMPLETION: GEN_AI_ATTRS.COMPLETION,
+    INPUT_TOKENS: GEN_AI_ATTRS.INPUT_TOKENS,
+    OUTPUT_TOKENS: GEN_AI_ATTRS.OUTPUT_TOKENS,
+    COST: GEN_AI_ATTRS.COST,
+    TEMPERATURE: GEN_AI_ATTRS.TEMPERATURE,
+    STATUS: GEN_AI_ATTRS.STATUS,
+    ERROR: GEN_AI_ATTRS.ERROR,
+};

package/dist/types/config.d.ts

@@ -0,0 +1,34 @@
+import type { LLMProvider } from "./providers.js";
+import type { BudgetConfig, BudgetExceededMode, DowngradeCallback } from "../budget/types.js";
+/**
+ * Configuration for initObservability().
+ * This is what the user passes when connecting toad-eye to their service.
+ */
+export interface ToadEyeConfig {
+    readonly serviceName: string;
+    readonly endpoint?: string | undefined;
+    /** API key for toad-eye cloud. When set, transport switches to HTTPS cloud endpoint automatically. */
+    readonly apiKey?: string | undefined;
+    /** Custom cloud endpoint override. Defaults to https://cloud.toad-eye.dev. */
+    readonly cloudEndpoint?: string | undefined;
+    readonly attributes?: Readonly<Record<string, string>> | undefined;
+    /** Set to false to disable recording prompt/completion text in spans. */
+    readonly recordContent?: boolean | undefined;
+    /** Record SHA-256 hash of content instead of plain text. Allows prompt comparison without reading. */
+    readonly hashContent?: boolean | undefined;
+    /** Salt for hashContent — prepended before hashing to prevent rainbow table attacks on short strings. */
+    readonly salt?: string | undefined;
+    /** Regex patterns to redact from prompt/completion text before recording. */
+    readonly redactPatterns?: readonly RegExp[] | undefined;
+    readonly instrument?: readonly LLMProvider[] | undefined;
+    /** Budget limits — daily, per-user, per-model spend caps in USD. */
+    readonly budgets?: BudgetConfig | undefined;
+    /** What to do when budget is exceeded: 'warn' (log), 'block' (throw), 'downgrade' (callback). */
+    readonly onBudgetExceeded?: BudgetExceededMode | undefined;
+    /** Callback for 'downgrade' mode — receives original provider/model, returns replacement. */
+    readonly downgradeCallback?: DowngradeCallback | undefined;
+    /** Static session ID — all spans will carry this value as `session.id`. */
+    readonly sessionId?: string | undefined;
+    /** Dynamic session ID extractor — called per traceLLMCall to resolve session ID. */
+    readonly sessionExtractor?: (() => string | undefined) | undefined;
+}

package/dist/types/config.js

@@ -0,0 +1 @@
+export {};

package/dist/types/index.d.ts

@@ -0,0 +1,17 @@
+/**
+ * Central type definitions for toad-eye.
+ *
+ * Organized by concern:
+ * - providers.ts  — supported LLM provider names
+ * - config.ts     — ToadEyeConfig (initObservability options)
+ * - attributes.ts — OTel span attribute keys (GEN_AI_ATTRS)
+ * - metrics.ts    — OTel metric names (GEN_AI_METRICS)
+ * - spans.ts      — span-related types (LLMSpanAttributes, SpanStatus)
+ */
+export declare const INSTRUMENTATION_NAME = "toad-eye";
+export type { LLMProvider } from "./providers.js";
+export type { ToadEyeConfig } from "./config.js";
+export { GEN_AI_ATTRS, LLM_ATTRS } from "./attributes.js";
+export { GEN_AI_METRICS, LLM_METRICS } from "./metrics.js";
+export type { MetricName } from "./metrics.js";
+export type { SpanStatus, LLMSpanAttributes, AgentStepType, AgentStepInput, GuardMode, GuardResult, } from "./spans.js";

package/dist/types/index.js

@@ -0,0 +1,13 @@
+/**
+ * Central type definitions for toad-eye.
+ *
+ * Organized by concern:
+ * - providers.ts  — supported LLM provider names
+ * - config.ts     — ToadEyeConfig (initObservability options)
+ * - attributes.ts — OTel span attribute keys (GEN_AI_ATTRS)
+ * - metrics.ts    — OTel metric names (GEN_AI_METRICS)
+ * - spans.ts      — span-related types (LLMSpanAttributes, SpanStatus)
+ */
+export const INSTRUMENTATION_NAME = "toad-eye";
+export { GEN_AI_ATTRS, LLM_ATTRS } from "./attributes.js";
+export { GEN_AI_METRICS, LLM_METRICS } from "./metrics.js";

package/dist/types/metrics.d.ts

@@ -0,0 +1,33 @@
+/**
+ * Metric names exported to Prometheus via OTel.
+ *
+ * Naming follows OTel GenAI semantic conventions:
+ * - `gen_ai.client.*` prefix for client-side metrics
+ * - Prometheus auto-converts dots to underscores
+ * - Prometheus adds _total suffix to counters automatically
+ */
+export declare const GEN_AI_METRICS: {
+    readonly REQUEST_DURATION: "gen_ai.client.operation.duration";
+    readonly TOKEN_USAGE: "gen_ai.client.token.usage";
+    readonly REQUEST_COST: "gen_ai.client.request.cost";
+    readonly REQUESTS: "gen_ai.client.requests";
+    readonly ERRORS: "gen_ai.client.errors";
+    readonly AGENT_STEPS_PER_QUERY: "gen_ai.agent.steps_per_query";
+    readonly AGENT_TOOL_USAGE: "gen_ai.agent.tool_usage";
+    readonly GUARD_EVALUATIONS: "gen_ai.toad_eye.guard.evaluations";
+    readonly GUARD_WOULD_BLOCK: "gen_ai.toad_eye.guard.would_block";
+    readonly SEMANTIC_DRIFT: "gen_ai.toad_eye.semantic_drift";
+    readonly BUDGET_EXCEEDED: "gen_ai.toad_eye.budget.exceeded";
+    readonly BUDGET_BLOCKED: "gen_ai.toad_eye.budget.blocked";
+    readonly BUDGET_DOWNGRADED: "gen_ai.toad_eye.budget.downgraded";
+};
+/** @deprecated Use GEN_AI_METRICS instead. Kept for backward compatibility. */
+export declare const LLM_METRICS: {
+    readonly REQUEST_DURATION: "gen_ai.client.operation.duration";
+    readonly REQUEST_COST: "gen_ai.client.request.cost";
+    readonly TOKENS: "gen_ai.client.token.usage";
+    readonly REQUESTS: "gen_ai.client.requests";
+    readonly ERRORS: "gen_ai.client.errors";
+};
+/** Type-safe metric name — only values from GEN_AI_METRICS are allowed. */
+export type MetricName = (typeof GEN_AI_METRICS)[keyof typeof GEN_AI_METRICS];

package/dist/types/metrics.js

@@ -0,0 +1,35 @@
+/**
+ * Metric names exported to Prometheus via OTel.
+ *
+ * Naming follows OTel GenAI semantic conventions:
+ * - `gen_ai.client.*` prefix for client-side metrics
+ * - Prometheus auto-converts dots to underscores
+ * - Prometheus adds _total suffix to counters automatically
+ */
+export const GEN_AI_METRICS = {
+    REQUEST_DURATION: "gen_ai.client.operation.duration",
+    TOKEN_USAGE: "gen_ai.client.token.usage",
+    REQUEST_COST: "gen_ai.client.request.cost",
+    REQUESTS: "gen_ai.client.requests",
+    ERRORS: "gen_ai.client.errors",
+    // Agent metrics
+    AGENT_STEPS_PER_QUERY: "gen_ai.agent.steps_per_query",
+    AGENT_TOOL_USAGE: "gen_ai.agent.tool_usage",
+    // Guard metrics
+    GUARD_EVALUATIONS: "gen_ai.toad_eye.guard.evaluations",
+    GUARD_WOULD_BLOCK: "gen_ai.toad_eye.guard.would_block",
+    // Drift metrics
+    SEMANTIC_DRIFT: "gen_ai.toad_eye.semantic_drift",
+    // Budget metrics
+    BUDGET_EXCEEDED: "gen_ai.toad_eye.budget.exceeded",
+    BUDGET_BLOCKED: "gen_ai.toad_eye.budget.blocked",
+    BUDGET_DOWNGRADED: "gen_ai.toad_eye.budget.downgraded",
+};
+/** @deprecated Use GEN_AI_METRICS instead. Kept for backward compatibility. */
+export const LLM_METRICS = {
+    REQUEST_DURATION: GEN_AI_METRICS.REQUEST_DURATION,
+    REQUEST_COST: GEN_AI_METRICS.REQUEST_COST,
+    TOKENS: GEN_AI_METRICS.TOKEN_USAGE,
+    REQUESTS: GEN_AI_METRICS.REQUESTS,
+    ERRORS: GEN_AI_METRICS.ERRORS,
+};

package/dist/types/providers.d.ts

@@ -0,0 +1,5 @@
+/**
+ * LLM providers supported by toad-eye.
+ * Values follow OTel GenAI semconv `gen_ai.provider.name`.
+ */
+export type LLMProvider = "anthropic" | "gemini" | "openai";

package/dist/types/providers.js

@@ -0,0 +1 @@
+export {};

package/dist/types/spans.d.ts

@@ -0,0 +1,40 @@
+import type { LLMProvider } from "./providers.js";
+/** Span status values used in trace attributes. */
+export type SpanStatus = "success" | "error";
+/** ReAct agent step types: think → act → observe → answer */
+export type AgentStepType = "think" | "act" | "observe" | "answer";
+/** Input for traceAgentStep — describes a single agent step */
+export interface AgentStepInput {
+    readonly type: AgentStepType;
+    readonly stepNumber: number;
+    readonly content?: string | undefined;
+    readonly toolName?: string | undefined;
+}
+/** Guard execution mode */
+export type GuardMode = "shadow" | "enforce";
+/**
+ * Result of a toad-guard validation — the contract between toad-guard and toad-eye.
+ * toad-guard produces this, toad-eye consumes it via recordGuardResult().
+ */
+export interface GuardResult {
+    readonly mode: GuardMode;
+    readonly passed: boolean;
+    readonly ruleName: string;
+    readonly failureReason?: string | undefined;
+}
+/**
+ * Attributes attached to every LLM span (trace).
+ * These become searchable fields in Jaeger and filterable dimensions in Grafana.
+ */
+export interface LLMSpanAttributes {
+    readonly provider: LLMProvider;
+    readonly model: string;
+    readonly prompt: string;
+    readonly completion: string;
+    readonly inputTokens: number;
+    readonly outputTokens: number;
+    readonly cost: number;
+    readonly temperature: number;
+    readonly status: SpanStatus;
+    readonly error?: string | undefined;
+}

package/dist/types/spans.js

@@ -0,0 +1 @@
+export {};

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "toad-eye",
-  "version": "1.1.0",
-  "description": "One-line OpenTelemetry instrumentation for LLM services",
+  "version": "2.1.0",
+  "description": "Auto-instrumentation, cost tracking, budget guards, and observability for LLM services",
   "type": "module",
   "main": "dist/index.js",
   "types": "dist/index.d.ts",
@@ -15,6 +15,8 @@
   ],
   "scripts": {
     "build": "tsc -p tsconfig.build.json",
+    "test": "vitest run",
+    "test:watch": "vitest",
     "prepublishOnly": "npm run build"
   },
   "repository": {
@@ -30,7 +32,16 @@
     "observability",
     "tracing",
     "metrics",
-    "grafana"
+    "grafana",
+    "openai",
+    "anthropic",
+    "gemini",
+    "cost-tracking",
+    "auto-instrumentation",
+    "gen-ai",
+    "budget-guards",
+    "finops",
+    "streaming"
   ],
   "license": "ISC",
   "dependencies": {
@@ -40,6 +51,29 @@
     "@opentelemetry/resources": "^2.6.0",
     "@opentelemetry/sdk-metrics": "^2.6.0",
     "@opentelemetry/sdk-node": "^0.213.0",
-    "@opentelemetry/semantic-conventions": "^1.40.0"
+    "@opentelemetry/semantic-conventions": "^1.40.0",
+    "nodemailer": "^8.0.3",
+    "yaml": "^2.7.0"
+  },
+  "peerDependencies": {
+    "@anthropic-ai/sdk": ">=0.30.0",
+    "@google/generative-ai": ">=0.20.0",
+    "openai": ">=4.0.0"
+  },
+  "peerDependenciesMeta": {
+    "openai": {
+      "optional": true
+    },
+    "@anthropic-ai/sdk": {
+      "optional": true
+    },
+    "@google/generative-ai": {
+      "optional": true
+    }
+  },
+  "devDependencies": {
+    "@types/nodemailer": "^7.0.11",
+    "playwright": "^1.58.2",
+    "vitest": "^4.1.0"
   }
 }

package/templates/alerts.example.yml

@@ -0,0 +1,54 @@
+# toad-eye alerts configuration
+# Copy to alerts.yml and customize
+
+# prometheusUrl: http://localhost:9090
+# grafanaUrl: http://localhost:3100
+# evalIntervalSeconds: 30
+# cooldownMinutes: 30
+
+# channels:
+#   telegram:
+#     type: telegram
+#     token: "BOT_TOKEN"
+#     chatId: "CHAT_ID"
+#   slack:
+#     type: slack_webhook
+#     url: "https://hooks.slack.com/services/..."
+
+alerts:
+  # Cost spike: $10 in 1 hour
+  - name: cost_spike
+    metric: gen_ai.client.request.cost
+    condition: sum_1h > 10
+    channels: [slack]
+
+  # Daily budget: $50 in 24 hours
+  - name: budget_daily
+    metric: gen_ai.client.request.cost
+    condition: sum_24h > 50
+    channels: [slack]
+
+  # Latency anomaly: p95 increased >50% vs 7-day baseline
+  - name: latency_anomaly
+    metric: gen_ai.client.operation.duration
+    condition: p95_pct_5m_7d > 50
+    channels: [slack]
+
+  # Latency critical: p95 increased >100% vs 7-day baseline
+  - name: latency_critical
+    metric: gen_ai.client.operation.duration
+    condition: p95_pct_5m_7d > 100
+    channels: [slack]
+
+  # Error rate warning: >5% over 15 minutes
+  - name: error_rate_warning
+    metric: gen_ai.client.errors
+    condition: ratio_15m > 0.05
+    channels: [slack]
+
+  # Error rate critical: >20% over 5 minutes (provider likely down)
+  - name: error_rate_critical
+    metric: gen_ai.client.errors
+    condition: ratio_5m > 0.2
+    channels: [slack]
+    cooldown: 15

package/templates/docker-compose.yml

@@ -28,8 +28,7 @@ services:
     image: grafana/grafana:latest
     volumes:
       - ./grafana/datasources.yml:/etc/grafana/provisioning/datasources/datasources.yml
-      - ./grafana/dashboards/dashboards.yml:/etc/grafana/provisioning/dashboards/dashboards.yml
-      - ./grafana/dashboards/llm-overview.json:/etc/grafana/provisioning/dashboards/llm-overview.json
+      - ./grafana/dashboards:/etc/grafana/provisioning/dashboards
     environment:
       - GF_SECURITY_ADMIN_PASSWORD=admin
       - GF_DASHBOARDS_DEFAULT_HOME_DASHBOARD_PATH=/etc/grafana/provisioning/dashboards/llm-overview.json