toad-eye 1.1.0 → 2.1.0

package/README.md

@@ -1,126 +1,171 @@
 # toad-eye 🐸👁️
 
-![TypeScript](https://img.shields.io/badge/TypeScript-3178C6?logo=typescript&logoColor=white)
-![OpenTelemetry](https://img.shields.io/badge/OpenTelemetry-000000?logo=opentelemetry&logoColor=white)
-![Hono](https://img.shields.io/badge/Hono-E36002?logo=hono&logoColor=white)
-![Prometheus](https://img.shields.io/badge/Prometheus-E6522C?logo=prometheus&logoColor=white)
-![Grafana](https://img.shields.io/badge/Grafana-F46800?logo=grafana&logoColor=white)
-![Docker](https://img.shields.io/badge/Docker-2496ED?logo=docker&logoColor=white)
-![npm](https://img.shields.io/npm/v/toad-eye?color=CB3837&logo=npm&logoColor=white)
-![CI](https://github.com/vola-trebla/toad-eye/actions/workflows/ci.yml/badge.svg)
-![License](https://img.shields.io/badge/License-ISC-blue)
+[![npm version](https://img.shields.io/npm/v/toad-eye?color=CB3837&logo=npm&logoColor=white)](https://www.npmjs.com/package/toad-eye)
+[![npm downloads](https://img.shields.io/npm/dm/toad-eye?color=CB3837&logo=npm&logoColor=white)](https://www.npmjs.com/package/toad-eye)
+[![CI](https://github.com/vola-trebla/toad-eye/actions/workflows/ci.yml/badge.svg)](https://github.com/vola-trebla/toad-eye/actions)
+[![License: ISC](https://img.shields.io/badge/License-ISC-blue.svg)](https://opensource.org/licenses/ISC)
 
-OpenTelemetry-based observability toolkit for LLM systems.
+**Full observability for LLM services — traces, metrics, and Grafana dashboards in one line of code.**
 
-One-line instrumentation for any LLM service — get traces, metrics, and Grafana dashboards out of the box.
+> LLM APIs are black boxes. You don't know what they cost, how slow they are, or why they fail.
+> toad-eye gives you complete visibility with zero manual instrumentation.
 
-## Why toad-eye?
+<!-- TODO: record GIF — terminal: npx toad-eye init → up → demo → Grafana with live data -->
+<!-- ![toad-eye demo](https://raw.githubusercontent.com/vola-trebla/toad-eye/main/demo/toad-eye-demo.gif) -->
 
-LLM APIs are black boxes — you don't know what they cost, how slow they are, or why they fail. toad-eye gives you full visibility with one line of code.
+---
 
-## Install
+## Quick Start
 
 ```bash
 npm install toad-eye
+npx toad-eye init   # scaffold observability stack configs
+npx toad-eye up     # start OTel Collector + Prometheus + Jaeger + Grafana
 ```
 
-## Architecture
+```typescript
+import { initObservability } from "toad-eye";
 
-```
-Your LLM service
-       │
-       ▼
-toad-eye
-       │
-       ▼ OTLP/HTTP
-OTel Collector
-       ├──▶ Prometheus ──▶ Grafana
-       └──▶ Jaeger
+initObservability({
+  serviceName: "my-app",
+  instrument: ["anthropic", "openai"],
+});
+// Every LLM call is now automatically traced. Open http://localhost:3100
 ```
 
-## Quick start
+That's it. **No wrappers. No manual spans.** All your Anthropic and OpenAI calls are traced automatically.
 
-```bash
-npm install
-cd infra && docker compose up -d   # start observability stack
-npm run demo                        # start mock LLM service
-npm run load --workspace=demo       # send test traffic
+---
+
+## What you get
+
+![toad-eye Grafana dashboard](https://raw.githubusercontent.com/vola-trebla/toad-eye/main/demo/grafana-dashboard.png)
+
+| Feature                  | Details                                                                           |
+| ------------------------ | --------------------------------------------------------------------------------- |
+| **Auto-instrumentation** | Patches Anthropic, OpenAI, Gemini SDKs — no code changes needed                   |
+| **5 Grafana dashboards** | Overview · Cost Breakdown · Latency Analysis · Model Comparison · Error Drilldown |
+| **Cost tracking**        | Per-request USD cost, hourly/daily totals, top models by spend                    |
+| **Cost alerts**          | Threshold alerts via Telegram, Slack, email, or webhook                           |
+| **Distributed traces**   | Full spans in Jaeger with prompts, completions, tokens                            |
+| **One-command stack**    | `npx toad-eye up` starts the entire observability stack                           |
+
+---
+
+## Observability stack
+
+```mermaid
+flowchart LR
+    app["Your LLM App\n(toad-eye SDK)"]
+    col["OTel Collector\n:4318"]
+    prom["Prometheus\n:9090"]
+    jaeger["Jaeger\n:16686"]
+    grafana["Grafana\n:3100"]
+
+    app -->|OTLP/HTTP| col
+    col -->|metrics| prom
+    col -->|traces| jaeger
+    prom --> grafana
+    jaeger --> grafana
 ```
 
-| Service        | URL                                       |
-| -------------- | ----------------------------------------- |
-| Grafana        | [localhost:3000](http://localhost:3000)   |
-| Jaeger UI      | [localhost:16686](http://localhost:16686) |
-| Prometheus     | [localhost:9090](http://localhost:9090)   |
-| OTel Collector | [localhost:4318](http://localhost:4318)   |
+| Service    | URL                                   |
+| ---------- | ------------------------------------- |
+| Grafana    | http://localhost:3100 (admin / admin) |
+| Jaeger     | http://localhost:16686                |
+| Prometheus | http://localhost:9090                 |
 
-## Usage
+---
 
-```typescript
-import { initObservability, traceLLMCall } from "toad-eye";
+## Metrics
 
-// one-line init
-initObservability({
-  serviceName: "my-llm-service",
-  endpoint: "http://localhost:4318",
-});
+| Metric                 | Type      | Description                                 |
+| ---------------------- | --------- | ------------------------------------------- |
+| `llm.request.duration` | Histogram | Request latency (ms), by provider + model   |
+| `llm.request.cost`     | Histogram | Cost per request (USD), by provider + model |
+| `llm.tokens`           | Counter   | Total tokens consumed (input + output)      |
+| `llm.requests`         | Counter   | Total requests made                         |
+| `llm.errors`           | Counter   | Total failed requests                       |
+
+---
+
+## Advanced
+
+### Manual tracing
+
+If you need to trace a provider not yet auto-instrumented:
+
+```typescript
+import { traceLLMCall } from "toad-eye";
 
-// wrap any LLM call
 const result = await traceLLMCall(
-  { provider: "anthropic", model: "claude-sonnet-4-20250514", prompt: "hello" },
-  () => callYourLLM(),
+  { provider: "anthropic", model: "claude-sonnet-4-20250514", prompt: input, temperature: 0.7 },
+  () => client.messages.create({ ... }),
+  (res) => ({
+    completion: res.content[0].text,
+    inputTokens: res.usage.input_tokens,
+    outputTokens: res.usage.output_tokens,
+    cost: calcCost(res.usage),
+  }),
 );
 ```
 
-> **Privacy mode:** pass `recordContent: false` to `initObservability()` to stop recording prompt/completion text in spans. Useful in production when prompts contain sensitive data.
+### Privacy mode
 
-## What it tracks
+Stop recording prompt/completion content in spans — useful in production:
 
-### Metrics
+```typescript
+initObservability({ serviceName: "my-app", recordContent: false });
+```
 
-| Metric                 | Type      | Description                            |
-| ---------------------- | --------- | -------------------------------------- |
-| `llm.request.duration` | Histogram | Request latency in milliseconds        |
-| `llm.request.cost`     | Histogram | Cost per request in USD                |
-| `llm.tokens`           | Counter   | Total tokens consumed (input + output) |
-| `llm.requests`         | Counter   | Total requests made                    |
-| `llm.errors`           | Counter   | Total failed requests                  |
+### Cost alerts
 
-All metrics are labeled with `provider` and `model` for filtering and grouping.
+Fire alerts when spend crosses a threshold:
 
-### Span attributes
+```typescript
+import { AlertManager } from "toad-eye";
+
+const alerts = new AlertManager({
+  prometheusUrl: "http://localhost:9090",
+  channels: {
+    telegram: {
+      type: "telegram",
+      token: process.env.TG_TOKEN!,
+      chatId: process.env.TG_CHAT!,
+    },
+  },
+  alerts: [
+    {
+      name: "cost_spike",
+      metric: "llm.request.cost",
+      condition: "sum_1h > 10",
+      channels: ["telegram"],
+    },
+  ],
+});
 
-| Attribute           | Type   | Description                     |
-| ------------------- | ------ | ------------------------------- |
-| `llm.provider`      | string | `anthropic`, `gemini`, `openai` |
-| `llm.model`         | string | Model identifier                |
-| `llm.prompt`        | string | Prompt sent to the LLM          |
-| `llm.completion`    | string | Response from the LLM           |
-| `llm.input_tokens`  | number | Tokens in the prompt            |
-| `llm.output_tokens` | number | Tokens in the completion        |
-| `llm.cost`          | number | Cost in USD                     |
-| `llm.temperature`   | number | Temperature parameter           |
-| `llm.status`        | string | `success` or `error`            |
-| `llm.error`         | string | Error message (if failed)       |
+alerts.start();
+```
 
-## Grafana dashboard
+Or configure via YAML — see [`infra/toad-eye/alerts.yml`](https://github.com/vola-trebla/toad-eye/blob/main/infra/toad-eye/alerts.yml).
 
-![toad-eye Grafana dashboard](https://raw.githubusercontent.com/vola-trebla/toad-eye/main/demo/grafana-dashboard.png)
+### Custom OTLP endpoint
 
-## Jaeger traces
+```typescript
+initObservability({
+  serviceName: "my-app",
+  endpoint: "http://your-collector:4318",
+});
+```
 
-![toad-eye Jaeger traces](https://raw.githubusercontent.com/vola-trebla/toad-eye/main/demo/jaeger-traces.png)
+---
 
-## Project structure
+## CLI reference
 
+```bash
+npx toad-eye init     # scaffold infra/toad-eye/ configs into your project
+npx toad-eye up       # docker compose up -d (OTel + Prometheus + Jaeger + Grafana)
+npx toad-eye down     # docker compose down
+npx toad-eye status   # show running services
+npx toad-eye demo     # start built-in demo with mock LLM traffic
 ```
-packages/instrumentation/   — NPM package (toad-eye)
-demo/                       — mock LLM service for testing
-infra/                      — docker-compose stack (OTel + Prometheus + Jaeger + Grafana)
-```
-
-## Tech stack
-
-- TypeScript, OpenTelemetry SDK 2.x, OTLP exporters
-- Hono (demo server)
-- Docker Compose (Prometheus, Jaeger, Grafana, OTel Collector)

package/dist/agent.d.ts

@@ -0,0 +1,41 @@
+import type { AgentStepInput } from "./types/index.js";
+/**
+ * Record a single agent step as a child span (ReAct pattern: think → act → observe → answer).
+ *
+ * Creates a short-lived span under the current active context. Call this inside
+ * {@link traceAgentQuery} (preferred) or any other active span (e.g. traceLLMCall).
+ *
+ * - Respects `recordContent` config — content is omitted when recording is disabled.
+ * - For `act` steps with a `toolName`, automatically increments the `agent.tool_usage` metric.
+ */
+declare function traceAgentStep(input: AgentStepInput): void;
+/**
+ * Wrap an entire agent query in a parent span, producing structured ReAct traces.
+ *
+ * The callback receives a `step` function to record individual steps.
+ * When the callback completes, the wrapper automatically records the
+ * `agent.steps_per_query` histogram metric with the total step count.
+ *
+ * Span hierarchy in Jaeger:
+ * ```
+ * [parent] agent.query "Is anything dangerous near Earth?"
+ *   ├── agent.step.think   "I need to check asteroids"
+ *   ├── agent.step.act     tool=near-earth-asteroids
+ *   ├── agent.step.observe "7 asteroids found"
+ *   └── agent.step.answer  "7 asteroids passing safely..."
+ * ```
+ *
+ * @example
+ * ```ts
+ * const result = await traceAgentQuery("Is anything dangerous near Earth?", async (step) => {
+ *   step({ type: "think", stepNumber: 1, content: "I need to check asteroids" });
+ *   const data = await callTool("near-earth-asteroids");
+ *   step({ type: "act", stepNumber: 2, toolName: "near-earth-asteroids" });
+ *   step({ type: "observe", stepNumber: 3, content: `${data.length} asteroids found` });
+ *   step({ type: "answer", stepNumber: 4, content: "7 asteroids passing safely..." });
+ *   return { answer: "7 asteroids passing safely..." };
+ * });
+ * ```
+ */
+export declare function traceAgentQuery<T>(query: string, fn: (step: (input: AgentStepInput) => void) => Promise<T>): Promise<T>;
+export { traceAgentStep };

package/dist/agent.js

@@ -0,0 +1,91 @@
+import { trace, SpanStatusCode } from "@opentelemetry/api";
+import { GEN_AI_ATTRS, INSTRUMENTATION_NAME } from "./types/index.js";
+import { recordAgentSteps, recordAgentToolUsage } from "./core/metrics.js";
+import { getConfig } from "./core/tracer.js";
+const tracer = trace.getTracer(INSTRUMENTATION_NAME);
+/**
+ * Record a single agent step as a child span (ReAct pattern: think → act → observe → answer).
+ *
+ * Creates a short-lived span under the current active context. Call this inside
+ * {@link traceAgentQuery} (preferred) or any other active span (e.g. traceLLMCall).
+ *
+ * - Respects `recordContent` config — content is omitted when recording is disabled.
+ * - For `act` steps with a `toolName`, automatically increments the `agent.tool_usage` metric.
+ */
+function traceAgentStep(input) {
+    const span = tracer.startSpan(`agent.step.${input.type}`);
+    const config = getConfig();
+    const recordContent = config?.recordContent !== false;
+    span.setAttributes({
+        [GEN_AI_ATTRS.AGENT_STEP_TYPE]: input.type,
+        [GEN_AI_ATTRS.AGENT_STEP_NUMBER]: input.stepNumber,
+        ...(input.toolName !== undefined && {
+            [GEN_AI_ATTRS.AGENT_TOOL_NAME]: input.toolName,
+        }),
+        ...(recordContent &&
+            input.content !== undefined && {
+            [GEN_AI_ATTRS.AGENT_STEP_CONTENT]: input.content,
+        }),
+    });
+    if (input.type === "act" && input.toolName !== undefined) {
+        recordAgentToolUsage(input.toolName);
+    }
+    span.setStatus({ code: SpanStatusCode.OK });
+    span.end();
+}
+/**
+ * Wrap an entire agent query in a parent span, producing structured ReAct traces.
+ *
+ * The callback receives a `step` function to record individual steps.
+ * When the callback completes, the wrapper automatically records the
+ * `agent.steps_per_query` histogram metric with the total step count.
+ *
+ * Span hierarchy in Jaeger:
+ * ```
+ * [parent] agent.query "Is anything dangerous near Earth?"
+ *   ├── agent.step.think   "I need to check asteroids"
+ *   ├── agent.step.act     tool=near-earth-asteroids
+ *   ├── agent.step.observe "7 asteroids found"
+ *   └── agent.step.answer  "7 asteroids passing safely..."
+ * ```
+ *
+ * @example
+ * ```ts
+ * const result = await traceAgentQuery("Is anything dangerous near Earth?", async (step) => {
+ *   step({ type: "think", stepNumber: 1, content: "I need to check asteroids" });
+ *   const data = await callTool("near-earth-asteroids");
+ *   step({ type: "act", stepNumber: 2, toolName: "near-earth-asteroids" });
+ *   step({ type: "observe", stepNumber: 3, content: `${data.length} asteroids found` });
+ *   step({ type: "answer", stepNumber: 4, content: "7 asteroids passing safely..." });
+ *   return { answer: "7 asteroids passing safely..." };
+ * });
+ * ```
+ */
+export async function traceAgentQuery(query, fn) {
+    return tracer.startActiveSpan(`agent.query`, async (span) => {
+        const config = getConfig();
+        const recordContent = config?.recordContent !== false;
+        if (recordContent) {
+            span.setAttribute(GEN_AI_ATTRS.AGENT_STEP_CONTENT, query);
+        }
+        let stepCount = 0;
+        try {
+            const result = await fn((input) => {
+                stepCount++;
+                traceAgentStep(input);
+            });
+            span.setStatus({ code: SpanStatusCode.OK });
+            return result;
+        }
+        catch (error) {
+            const message = error instanceof Error ? error.message : String(error);
+            span.setStatus({ code: SpanStatusCode.ERROR, message });
+            throw error;
+        }
+        finally {
+            recordAgentSteps(stepCount);
+            span.end();
+        }
+    });
+}
+export { traceAgentStep };

package/dist/alerts/channels.d.ts

@@ -0,0 +1,2 @@
+import type { AlertChannelConfig, FiredAlert } from "./types.js";
+export declare function sendToChannel(channel: AlertChannelConfig, alert: FiredAlert): Promise<void>;

package/dist/alerts/channels.js

@@ -0,0 +1,81 @@
+import nodemailer from "nodemailer";
+function formatMessage(alert) {
+    const topModelsText = alert.topModels.length > 0
+        ? `\nTop models by spend:\n${alert.topModels.map((m) => `  • ${m.model}: $${m.value.toFixed(4)}`).join("\n")}`
+        : "";
+    return [
+        `🚨 Alert: ${alert.rule.name}`,
+        `Metric: ${alert.rule.metric}`,
+        `Value: ${alert.value.toFixed(4)} (threshold: ${alert.threshold})`,
+        `Condition: ${alert.rule.condition}`,
+        topModelsText,
+    ]
+        .filter(Boolean)
+        .join("\n");
+}
+async function sendTelegram(config, alert) {
+    const res = await fetch(`https://api.telegram.org/bot${config.token}/sendMessage`, {
+        method: "POST",
+        headers: { "Content-Type": "application/json" },
+        body: JSON.stringify({
+            chat_id: config.chatId,
+            text: formatMessage(alert),
+        }),
+    });
+    if (!res.ok)
+        throw new Error(`Telegram send failed: ${res.status}`);
+}
+async function sendSlackWebhook(config, alert) {
+    const res = await fetch(config.url, {
+        method: "POST",
+        headers: { "Content-Type": "application/json" },
+        body: JSON.stringify({ text: formatMessage(alert) }),
+    });
+    if (!res.ok)
+        throw new Error(`Slack webhook failed: ${res.status}`);
+}
+async function sendWebhook(config, alert) {
+    const res = await fetch(config.url, {
+        method: "POST",
+        headers: { "Content-Type": "application/json", ...config.headers },
+        body: JSON.stringify({
+            alertName: alert.rule.name,
+            metric: alert.rule.metric,
+            value: alert.value,
+            threshold: alert.threshold,
+            condition: alert.rule.condition,
+            topModels: alert.topModels,
+            firedAt: alert.firedAt.toISOString(),
+        }),
+    });
+    if (!res.ok)
+        throw new Error(`Webhook delivery failed: ${res.status}`);
+}
+async function sendEmail(config, alert) {
+    const transporter = nodemailer.createTransport({
+        host: config.host,
+        port: config.port,
+        auth: { user: config.user, pass: config.password },
+    });
+    const to = typeof config.to === "string"
+        ? config.to
+        : Array.from(config.to).join(", ");
+    await transporter.sendMail({
+        from: config.from,
+        to,
+        subject: `🚨 toad-eye alert: ${alert.rule.name}`,
+        text: formatMessage(alert),
+    });
+}
+export async function sendToChannel(channel, alert) {
+    switch (channel.type) {
+        case "telegram":
+            return sendTelegram(channel, alert);
+        case "slack_webhook":
+            return sendSlackWebhook(channel, alert);
+        case "webhook":
+            return sendWebhook(channel, alert);
+        case "email":
+            return sendEmail(channel, alert);
+    }
+}

package/dist/alerts/conditions.d.ts

@@ -0,0 +1,29 @@
+import type { AlertRule } from "./types.js";
+type ConditionOperator = "sum" | "avg" | "rate" | "max" | "p95_pct" | "ratio";
+type Comparator = ">" | ">=" | "<" | "<=";
+export interface ParsedCondition {
+    readonly operator: ConditionOperator;
+    readonly window: string;
+    readonly baselineWindow?: string | undefined;
+    readonly comparator: Comparator;
+    readonly threshold: number;
+}
+/**
+ * Parse condition string into structured form.
+ *
+ * Standard: "sum_1h > 10", "rate_5m > 0.05"
+ * Baseline: "p95_pct_5m_7d > 50" — p95 over 5m increased >50% vs 7d baseline
+ * Ratio:    "ratio_15m > 0.05"   — errors/requests ratio over 15m
+ */
+export declare function parseCondition(condition: string): ParsedCondition;
+export interface EvalResult {
+    readonly triggered: boolean;
+    readonly value: number;
+    readonly threshold: number;
+    readonly topModels: Array<{
+        model: string;
+        value: number;
+    }>;
+}
+export declare function evaluateCondition(prometheusUrl: string, rule: AlertRule): Promise<EvalResult | null>;
+export {};

package/dist/alerts/conditions.js

@@ -0,0 +1,147 @@
+// Maps OTel metric names to Prometheus metric names.
+const METRIC_MAP = {
+    "gen_ai.client.request.cost": "gen_ai_client_request_cost_USD_sum",
+    "gen_ai.client.operation.duration": "gen_ai_client_operation_duration_milliseconds",
+    "gen_ai.client.requests": "gen_ai_client_requests_total",
+    "gen_ai.client.errors": "gen_ai_client_errors_total",
+    "gen_ai.client.token.usage": "gen_ai_client_token_usage_total",
+    // Legacy names (backward compat)
+    "llm.request.cost": "gen_ai_client_request_cost_USD_sum",
+    "llm.request.duration": "gen_ai_client_operation_duration_milliseconds",
+    "llm.requests": "gen_ai_client_requests_total",
+    "llm.errors": "gen_ai_client_errors_total",
+    "llm.tokens": "gen_ai_client_token_usage_total",
+};
+function toPromMetric(metric) {
+    return METRIC_MAP[metric] ?? metric.replace(/\./g, "_");
+}
+/**
+ * Parse condition string into structured form.
+ *
+ * Standard: "sum_1h > 10", "rate_5m > 0.05"
+ * Baseline: "p95_pct_5m_7d > 50" — p95 over 5m increased >50% vs 7d baseline
+ * Ratio:    "ratio_15m > 0.05"   — errors/requests ratio over 15m
+ */
+export function parseCondition(condition) {
+    // Baseline percentage: p95_pct_<window>_<baseline> <cmp> <threshold>
+    const baselineMatch = /^p95_pct_(\d+[smhd])_(\d+[smhd])\s*(>=|<=|>|<)\s*([\d.]+)$/.exec(condition.trim());
+    if (baselineMatch) {
+        return {
+            operator: "p95_pct",
+            window: baselineMatch[1],
+            baselineWindow: baselineMatch[2],
+            comparator: baselineMatch[3],
+            threshold: parseFloat(baselineMatch[4]),
+        };
+    }
+    // Standard: sum_1h > 10, ratio_15m > 0.05
+    const match = /^(sum|avg|rate|max|ratio)_(\d+[smhd])\s*(>=|<=|>|<)\s*([\d.]+)$/.exec(condition.trim());
+    if (!match) {
+        throw new Error(`Invalid alert condition: "${condition}". Expected format: "sum_1h > 10" or "p95_pct_5m_7d > 50"`);
+    }
+    return {
+        operator: match[1],
+        window: match[2],
+        comparator: match[3],
+        threshold: parseFloat(match[4]),
+    };
+}
+function buildPromQL(metric, operator, window) {
+    const m = toPromMetric(metric);
+    switch (operator) {
+        case "sum":
+            return `sum(increase(${m}[${window}]))`;
+        case "avg":
+            return `avg(rate(${m}[${window}])) * 60`;
+        case "rate":
+            return `sum(rate(${m}[${window}])) * 60`;
+        case "max":
+            return `max(increase(${m}[${window}]))`;
+        case "p95_pct":
+        case "ratio":
+            // Handled separately in evaluateCondition
+            return "";
+    }
+}
+function buildTopModelsQuery(metric, window) {
+    const m = toPromMetric(metric);
+    return `topk(5, sum by (gen_ai_request_model) (increase(${m}[${window}])))`;
+}
+function matches(value, comparator, threshold) {
+    switch (comparator) {
+        case ">":
+            return value > threshold;
+        case ">=":
+            return value >= threshold;
+        case "<":
+            return value < threshold;
+        case "<=":
+            return value <= threshold;
+    }
+}
+async function queryScalar(prometheusUrl, promql) {
+    const url = `${prometheusUrl}/api/v1/query?query=${encodeURIComponent(promql)}`;
+    const res = await fetch(url);
+    if (!res.ok)
+        throw new Error(`Prometheus query failed: ${res.status} ${res.statusText}`);
+    const data = (await res.json());
+    const first = data.data.result[0];
+    return first ? parseFloat(first.value[1]) : 0;
+}
+async function queryTopModels(prometheusUrl, metric, window) {
+    const promql = buildTopModelsQuery(metric, window);
+    const url = `${prometheusUrl}/api/v1/query?query=${encodeURIComponent(promql)}`;
+    const res = await fetch(url);
+    if (!res.ok)
+        return [];
+    const data = (await res.json());
+    return data.data.result
+        .map((r) => ({
+        model: r.metric.gen_ai_request_model ?? "unknown",
+        value: parseFloat(r.value[1]),
+    }))
+        .sort((a, b) => b.value - a.value);
+}
+/** Evaluate p95 current vs baseline, return % increase */
+async function evalP95Baseline(prometheusUrl, metric, window, baselineWindow) {
+    const m = toPromMetric(metric);
+    const bucketMetric = `${m}_bucket`;
+    const currentP95 = await queryScalar(prometheusUrl, `histogram_quantile(0.95, sum(rate(${bucketMetric}[${window}])) by (le))`);
+    const baselineP95 = await queryScalar(prometheusUrl, `histogram_quantile(0.95, sum(rate(${bucketMetric}[${baselineWindow}])) by (le))`);
+    if (baselineP95 === 0)
+        return 0;
+    return ((currentP95 - baselineP95) / baselineP95) * 100;
+}
+/** Evaluate error ratio: errors / requests over window */
+async function evalErrorRatio(prometheusUrl, window) {
+    const errors = await queryScalar(prometheusUrl, `sum(increase(gen_ai_client_errors_total[${window}]))`);
+    const requests = await queryScalar(prometheusUrl, `sum(increase(gen_ai_client_requests_total[${window}]))`);
+    if (requests === 0)
+        return 0;
+    return errors / requests;
+}
+export async function evaluateCondition(prometheusUrl, rule) {
+    try {
+        const parsed = parseCondition(rule.condition);
+        let value;
+        if (parsed.operator === "p95_pct") {
+            value = await evalP95Baseline(prometheusUrl, rule.metric, parsed.window, parsed.baselineWindow);
+        }
+        else if (parsed.operator === "ratio") {
+            value = await evalErrorRatio(prometheusUrl, parsed.window);
+        }
+        else {
+            const promql = buildPromQL(rule.metric, parsed.operator, parsed.window);
+            value = await queryScalar(prometheusUrl, promql);
+        }
+        const triggered = matches(value, parsed.comparator, parsed.threshold);
+        const topModels = triggered
+            ? await queryTopModels(prometheusUrl, rule.metric, parsed.window)
+            : [];
+        return { triggered, value, threshold: parsed.threshold, topModels };
+    }
+    catch (err) {
+        console.error(`[toad-eye alerts] Failed to evaluate "${rule.name}":`, err);
+        return null;
+    }
+}

package/dist/alerts/grafana.d.ts

@@ -0,0 +1,5 @@
+export declare function postGrafanaAnnotation(grafanaUrl: string, alert: {
+    name: string;
+    value: number;
+    threshold: number;
+}, apiKey: string | undefined): Promise<void>;