toad-eye 1.1.0 → 2.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (90) hide show
  1. package/README.md +132 -87
  2. package/dist/agent.d.ts +41 -0
  3. package/dist/agent.js +91 -0
  4. package/dist/alerts/channels.d.ts +2 -0
  5. package/dist/alerts/channels.js +81 -0
  6. package/dist/alerts/conditions.d.ts +29 -0
  7. package/dist/alerts/conditions.js +147 -0
  8. package/dist/alerts/grafana.d.ts +5 -0
  9. package/dist/alerts/grafana.js +19 -0
  10. package/dist/alerts/index.d.ts +5 -0
  11. package/dist/alerts/index.js +12 -0
  12. package/dist/alerts/manager.d.ts +13 -0
  13. package/dist/alerts/manager.js +87 -0
  14. package/dist/alerts/types.d.ts +46 -0
  15. package/dist/alerts/types.js +1 -0
  16. package/dist/budget/error.d.ts +10 -0
  17. package/dist/budget/error.js +22 -0
  18. package/dist/budget/index.d.ts +3 -0
  19. package/dist/budget/index.js +2 -0
  20. package/dist/budget/tracker.d.ts +35 -0
  21. package/dist/budget/tracker.js +121 -0
  22. package/dist/budget/types.d.ts +34 -0
  23. package/dist/budget/types.js +1 -0
  24. package/dist/cli.js +127 -18
  25. package/dist/core/metrics.d.ts +14 -0
  26. package/dist/core/metrics.js +119 -0
  27. package/dist/core/pricing.d.ts +24 -0
  28. package/dist/core/pricing.js +52 -0
  29. package/dist/{spans.d.ts → core/spans.d.ts} +3 -1
  30. package/dist/core/spans.js +143 -0
  31. package/dist/core/tracer.d.ts +9 -0
  32. package/dist/core/tracer.js +93 -0
  33. package/dist/drift/baseline.d.ts +16 -0
  34. package/dist/drift/baseline.js +16 -0
  35. package/dist/drift/cosine.d.ts +13 -0
  36. package/dist/drift/cosine.js +39 -0
  37. package/dist/drift/embedding-provider.d.ts +21 -0
  38. package/dist/drift/embedding-provider.js +8 -0
  39. package/dist/drift/index.d.ts +9 -0
  40. package/dist/drift/index.js +8 -0
  41. package/dist/drift/monitor.d.ts +29 -0
  42. package/dist/drift/monitor.js +47 -0
  43. package/dist/drift/openai-embeddings.d.ts +6 -0
  44. package/dist/drift/openai-embeddings.js +26 -0
  45. package/dist/export.d.ts +46 -0
  46. package/dist/export.js +108 -0
  47. package/dist/guard.d.ts +26 -0
  48. package/dist/guard.js +45 -0
  49. package/dist/index.d.ts +17 -5
  50. package/dist/index.js +10 -3
  51. package/dist/instrumentations/anthropic.d.ts +1 -0
  52. package/dist/instrumentations/anthropic.js +73 -0
  53. package/dist/instrumentations/create.d.ts +11 -0
  54. package/dist/instrumentations/create.js +165 -0
  55. package/dist/instrumentations/gemini.d.ts +1 -0
  56. package/dist/instrumentations/gemini.js +48 -0
  57. package/dist/instrumentations/openai.d.ts +1 -0
  58. package/dist/instrumentations/openai.js +94 -0
  59. package/dist/instrumentations/registry.d.ts +5 -0
  60. package/dist/instrumentations/registry.js +31 -0
  61. package/dist/instrumentations/types.d.ts +37 -0
  62. package/dist/instrumentations/types.js +1 -0
  63. package/dist/types/attributes.d.ts +48 -0
  64. package/dist/types/attributes.js +53 -0
  65. package/dist/types/config.d.ts +34 -0
  66. package/dist/types/config.js +1 -0
  67. package/dist/types/index.d.ts +17 -0
  68. package/dist/types/index.js +13 -0
  69. package/dist/types/metrics.d.ts +33 -0
  70. package/dist/types/metrics.js +35 -0
  71. package/dist/types/providers.d.ts +5 -0
  72. package/dist/types/providers.js +1 -0
  73. package/dist/types/spans.d.ts +40 -0
  74. package/dist/types/spans.js +1 -0
  75. package/package.json +38 -4
  76. package/templates/alerts.example.yml +54 -0
  77. package/templates/docker-compose.yml +1 -2
  78. package/templates/grafana/dashboards/cost-breakdown.json +208 -0
  79. package/templates/grafana/dashboards/error-drilldown.json +188 -0
  80. package/templates/grafana/dashboards/finops-attribution.json +278 -0
  81. package/templates/grafana/dashboards/latency-analysis.json +134 -0
  82. package/templates/grafana/dashboards/llm-overview.json +161 -30
  83. package/templates/grafana/dashboards/model-comparison.json +143 -0
  84. package/dist/metrics.d.ts +0 -6
  85. package/dist/metrics.js +0 -46
  86. package/dist/spans.js +0 -61
  87. package/dist/tracer.d.ts +0 -4
  88. package/dist/tracer.js +0 -46
  89. package/dist/types.d.ts +0 -69
  90. package/dist/types.js +0 -32
@@ -0,0 +1,19 @@
1
+ export async function postGrafanaAnnotation(grafanaUrl, alert, apiKey) {
2
+ const headers = {
3
+ "Content-Type": "application/json",
4
+ };
5
+ if (apiKey)
6
+ headers["Authorization"] = `Bearer ${apiKey}`;
7
+ const res = await fetch(`${grafanaUrl}/api/annotations`, {
8
+ method: "POST",
9
+ headers,
10
+ body: JSON.stringify({
11
+ time: Date.now(),
12
+ tags: ["toad-eye", "alert", alert.name],
13
+ text: `🚨 ${alert.name}: ${alert.value.toFixed(4)} > ${alert.threshold}`,
14
+ }),
15
+ });
16
+ if (!res.ok) {
17
+ throw new Error(`Grafana annotation failed: ${res.status} ${res.statusText}`);
18
+ }
19
+ }
@@ -0,0 +1,5 @@
1
+ export { AlertManager } from "./manager.js";
2
+ export { parseCondition } from "./conditions.js";
3
+ export type { AlertsConfig, AlertRule, AlertChannelConfig, FiredAlert, } from "./types.js";
4
+ import { AlertManager } from "./manager.js";
5
+ export declare function startAlertsFromFile(configPath: string): AlertManager;
@@ -0,0 +1,12 @@
1
+ export { AlertManager } from "./manager.js";
2
+ export { parseCondition } from "./conditions.js";
3
+ import { readFileSync } from "node:fs";
4
+ import yaml from "yaml";
5
+ import { AlertManager } from "./manager.js";
6
+ export function startAlertsFromFile(configPath) {
7
+ const raw = readFileSync(configPath, "utf-8");
8
+ const config = yaml.parse(raw);
9
+ const manager = new AlertManager(config);
10
+ manager.start();
11
+ return manager;
12
+ }
@@ -0,0 +1,13 @@
1
+ import type { AlertsConfig } from "./types.js";
2
+ export declare class AlertManager {
3
+ private readonly config;
4
+ private readonly cooldowns;
5
+ private intervalId;
6
+ constructor(config: AlertsConfig);
7
+ start(): void;
8
+ stop(): void;
9
+ private isInCooldown;
10
+ private evaluate;
11
+ private fireChannels;
12
+ private postAnnotation;
13
+ }
@@ -0,0 +1,87 @@
1
+ import { evaluateCondition } from "./conditions.js";
2
+ import { sendToChannel } from "./channels.js";
3
+ import { postGrafanaAnnotation } from "./grafana.js";
4
+ const DEFAULT_PROMETHEUS_URL = "http://localhost:9090";
5
+ const DEFAULT_GRAFANA_URL = "http://localhost:3100";
6
+ const DEFAULT_EVAL_INTERVAL_SECONDS = 60;
7
+ const DEFAULT_COOLDOWN_MINUTES = 30;
8
+ export class AlertManager {
9
+ config;
10
+ cooldowns = new Map();
11
+ intervalId;
12
+ constructor(config) {
13
+ this.config = config;
14
+ }
15
+ start() {
16
+ const intervalMs = (this.config.evalIntervalSeconds ?? DEFAULT_EVAL_INTERVAL_SECONDS) * 1000;
17
+ this.intervalId = setInterval(() => void this.evaluate(), intervalMs);
18
+ void this.evaluate();
19
+ console.log(`[toad-eye alerts] Started — ${this.config.alerts.length} rule(s), eval every ${intervalMs / 1000}s`);
20
+ }
21
+ stop() {
22
+ if (this.intervalId !== undefined) {
23
+ clearInterval(this.intervalId);
24
+ this.intervalId = undefined;
25
+ }
26
+ }
27
+ isInCooldown(rule) {
28
+ const lastFired = this.cooldowns.get(rule.name);
29
+ if (lastFired === undefined)
30
+ return false;
31
+ const cooldownMs = (rule.cooldown ??
32
+ this.config.cooldownMinutes ??
33
+ DEFAULT_COOLDOWN_MINUTES) *
34
+ 60 *
35
+ 1000;
36
+ return Date.now() - lastFired < cooldownMs;
37
+ }
38
+ async evaluate() {
39
+ const prometheusUrl = this.config.prometheusUrl ?? DEFAULT_PROMETHEUS_URL;
40
+ const grafanaUrl = this.config.grafanaUrl ?? DEFAULT_GRAFANA_URL;
41
+ for (const rule of this.config.alerts) {
42
+ if (this.isInCooldown(rule))
43
+ continue;
44
+ const result = await evaluateCondition(prometheusUrl, rule);
45
+ if (!result?.triggered)
46
+ continue;
47
+ this.cooldowns.set(rule.name, Date.now());
48
+ const firedAlert = {
49
+ rule,
50
+ value: result.value,
51
+ threshold: result.threshold,
52
+ topModels: result.topModels,
53
+ firedAt: new Date(),
54
+ };
55
+ console.log(`[toad-eye alerts] 🚨 "${rule.name}" fired — value=${result.value.toFixed(4)}, threshold=${result.threshold}`);
56
+ await Promise.allSettled([
57
+ this.fireChannels(firedAlert),
58
+ this.postAnnotation(grafanaUrl, firedAlert.rule.name, firedAlert.value, firedAlert.threshold),
59
+ ]);
60
+ }
61
+ }
62
+ async fireChannels(alert) {
63
+ const channels = this.config.channels ?? {};
64
+ for (const channelName of alert.rule.channels) {
65
+ const channelConfig = channels[channelName];
66
+ if (!channelConfig) {
67
+ console.warn(`[toad-eye alerts] Channel "${channelName}" not configured, skipping`);
68
+ continue;
69
+ }
70
+ try {
71
+ await sendToChannel(channelConfig, alert);
72
+ console.log(`[toad-eye alerts] ✓ Sent to channel "${channelName}"`);
73
+ }
74
+ catch (err) {
75
+ console.error(`[toad-eye alerts] ✗ Failed to send to channel "${channelName}":`, err);
76
+ }
77
+ }
78
+ }
79
+ async postAnnotation(grafanaUrl, name, value, threshold) {
80
+ try {
81
+ await postGrafanaAnnotation(grafanaUrl, { name, value, threshold }, this.config.grafanaApiKey);
82
+ }
83
+ catch (err) {
84
+ console.error(`[toad-eye alerts] Failed to post Grafana annotation:`, err);
85
+ }
86
+ }
87
+ }
@@ -0,0 +1,46 @@
1
+ export type AlertChannelConfig = {
2
+ readonly type: "telegram";
3
+ readonly token: string;
4
+ readonly chatId: string;
5
+ } | {
6
+ readonly type: "slack_webhook";
7
+ readonly url: string;
8
+ } | {
9
+ readonly type: "webhook";
10
+ readonly url: string;
11
+ readonly headers?: Record<string, string> | undefined;
12
+ } | {
13
+ readonly type: "email";
14
+ readonly host: string;
15
+ readonly port: number;
16
+ readonly user: string;
17
+ readonly password: string;
18
+ readonly from: string;
19
+ readonly to: string | readonly string[];
20
+ };
21
+ export interface AlertRule {
22
+ readonly name: string;
23
+ readonly metric: string;
24
+ readonly condition: string;
25
+ readonly channels: readonly string[];
26
+ readonly cooldown?: number | undefined;
27
+ }
28
+ export interface AlertsConfig {
29
+ readonly prometheusUrl?: string | undefined;
30
+ readonly grafanaUrl?: string | undefined;
31
+ readonly grafanaApiKey?: string | undefined;
32
+ readonly evalIntervalSeconds?: number | undefined;
33
+ readonly cooldownMinutes?: number | undefined;
34
+ readonly channels?: Record<string, AlertChannelConfig> | undefined;
35
+ readonly alerts: readonly AlertRule[];
36
+ }
37
+ export interface FiredAlert {
38
+ readonly rule: AlertRule;
39
+ readonly value: number;
40
+ readonly threshold: number;
41
+ readonly topModels: ReadonlyArray<{
42
+ model: string;
43
+ value: number;
44
+ }>;
45
+ readonly firedAt: Date;
46
+ }
@@ -0,0 +1 @@
1
+ export {};
@@ -0,0 +1,10 @@
1
+ import type { BudgetExceededInfo } from "./types.js";
2
+ /** Thrown when onBudgetExceeded is 'block' and a budget limit is hit. */
3
+ export declare class ToadBudgetExceededError extends Error {
4
+ readonly budget: BudgetExceededInfo["budget"];
5
+ readonly limit: number;
6
+ readonly current: number;
7
+ readonly model?: string | undefined;
8
+ readonly userId?: string | undefined;
9
+ constructor(info: BudgetExceededInfo);
10
+ }
@@ -0,0 +1,22 @@
1
+ /** Thrown when onBudgetExceeded is 'block' and a budget limit is hit. */
2
+ export class ToadBudgetExceededError extends Error {
3
+ budget;
4
+ limit;
5
+ current;
6
+ model;
7
+ userId;
8
+ constructor(info) {
9
+ const target = info.budget === "perModel"
10
+ ? ` for model ${info.model}`
11
+ : info.budget === "perUser"
12
+ ? ` for user ${info.userId}`
13
+ : "";
14
+ super(`toad-eye: ${info.budget} budget exceeded${target} — limit $${info.limit}, current $${info.current.toFixed(2)}`);
15
+ this.name = "ToadBudgetExceededError";
16
+ this.budget = info.budget;
17
+ this.limit = info.limit;
18
+ this.current = info.current;
19
+ this.model = info.model;
20
+ this.userId = info.userId;
21
+ }
22
+ }
@@ -0,0 +1,3 @@
1
+ export { BudgetTracker } from "./tracker.js";
2
+ export { ToadBudgetExceededError } from "./error.js";
3
+ export type { BudgetConfig, BudgetExceededMode, BudgetExceededInfo, BudgetState, DowngradeCallback, } from "./types.js";
@@ -0,0 +1,2 @@
1
+ export { BudgetTracker } from "./tracker.js";
2
+ export { ToadBudgetExceededError } from "./error.js";
@@ -0,0 +1,35 @@
1
+ import type { BudgetConfig, BudgetExceededMode, BudgetState, BudgetExceededInfo, DowngradeCallback } from "./types.js";
2
+ export declare class BudgetTracker {
3
+ private readonly config;
4
+ private readonly mode;
5
+ private readonly downgrade?;
6
+ private state;
7
+ constructor(config: BudgetConfig, mode: BudgetExceededMode, downgrade?: DowngradeCallback);
8
+ /** Reset counters if the day has changed. */
9
+ private resetIfNewDay;
10
+ /**
11
+ * Check budget BEFORE making an LLM call.
12
+ * Returns modified provider/model if downgrade mode triggers,
13
+ * throws ToadBudgetExceededError if block mode triggers,
14
+ * returns null if no action needed.
15
+ */
16
+ checkBefore(provider: string, model: string, userId?: string): {
17
+ provider: string;
18
+ model: string;
19
+ } | null;
20
+ /** Record cost AFTER a successful LLM call. Returns exceeded info if budget was just crossed. */
21
+ recordCost(cost: number, model: string, userId?: string): BudgetExceededInfo | null;
22
+ /** Find which budget (if any) is exceeded. */
23
+ private findExceeded;
24
+ /** Get current budget usage as percentage (0-100+). */
25
+ getUsagePercent(): number;
26
+ /** Get current state snapshot (for persistence or debugging). */
27
+ getState(): Readonly<BudgetState>;
28
+ /** Restore state from persistence (e.g., on startup). */
29
+ restoreState(saved: {
30
+ date: string;
31
+ totalCost: number;
32
+ perUser: Record<string, number>;
33
+ perModel: Record<string, number>;
34
+ }): void;
35
+ }
@@ -0,0 +1,121 @@
1
+ import { ToadBudgetExceededError } from "./error.js";
2
+ function todayUTC() {
3
+ return new Date().toISOString().slice(0, 10);
4
+ }
5
+ export class BudgetTracker {
6
+ config;
7
+ mode;
8
+ downgrade;
9
+ state;
10
+ constructor(config, mode, downgrade) {
11
+ this.config = config;
12
+ this.mode = mode;
13
+ this.downgrade = downgrade;
14
+ this.state = {
15
+ date: todayUTC(),
16
+ totalCost: 0,
17
+ perUser: new Map(),
18
+ perModel: new Map(),
19
+ };
20
+ }
21
+ /** Reset counters if the day has changed. */
22
+ resetIfNewDay() {
23
+ const today = todayUTC();
24
+ if (this.state.date !== today) {
25
+ this.state = {
26
+ date: today,
27
+ totalCost: 0,
28
+ perUser: new Map(),
29
+ perModel: new Map(),
30
+ };
31
+ }
32
+ }
33
+ /**
34
+ * Check budget BEFORE making an LLM call.
35
+ * Returns modified provider/model if downgrade mode triggers,
36
+ * throws ToadBudgetExceededError if block mode triggers,
37
+ * returns null if no action needed.
38
+ */
39
+ checkBefore(provider, model, userId) {
40
+ this.resetIfNewDay();
41
+ const exceeded = this.findExceeded(model, userId);
42
+ if (!exceeded)
43
+ return null;
44
+ if (this.mode === "block") {
45
+ throw new ToadBudgetExceededError(exceeded);
46
+ }
47
+ if (this.mode === "downgrade" && this.downgrade) {
48
+ return this.downgrade({ provider, model });
49
+ }
50
+ // 'warn' mode — continue, warning emitted after call
51
+ return null;
52
+ }
53
+ /** Record cost AFTER a successful LLM call. Returns exceeded info if budget was just crossed. */
54
+ recordCost(cost, model, userId) {
55
+ this.resetIfNewDay();
56
+ this.state.totalCost += cost;
57
+ this.state.perModel.set(model, (this.state.perModel.get(model) ?? 0) + cost);
58
+ if (userId) {
59
+ this.state.perUser.set(userId, (this.state.perUser.get(userId) ?? 0) + cost);
60
+ }
61
+ return this.findExceeded(model, userId);
62
+ }
63
+ /** Find which budget (if any) is exceeded. */
64
+ findExceeded(model, userId) {
65
+ if (this.config.daily !== undefined &&
66
+ this.state.totalCost >= this.config.daily) {
67
+ return {
68
+ budget: "daily",
69
+ limit: this.config.daily,
70
+ current: this.state.totalCost,
71
+ };
72
+ }
73
+ const modelLimit = this.config.perModel?.[model];
74
+ if (modelLimit !== undefined) {
75
+ const modelCost = this.state.perModel.get(model) ?? 0;
76
+ if (modelCost >= modelLimit) {
77
+ return {
78
+ budget: "perModel",
79
+ limit: modelLimit,
80
+ current: modelCost,
81
+ model,
82
+ };
83
+ }
84
+ }
85
+ if (this.config.perUser !== undefined && userId) {
86
+ const userCost = this.state.perUser.get(userId) ?? 0;
87
+ if (userCost >= this.config.perUser) {
88
+ return {
89
+ budget: "perUser",
90
+ limit: this.config.perUser,
91
+ current: userCost,
92
+ userId,
93
+ };
94
+ }
95
+ }
96
+ return null;
97
+ }
98
+ /** Get current budget usage as percentage (0-100+). */
99
+ getUsagePercent() {
100
+ if (this.config.daily === undefined || this.config.daily === 0)
101
+ return 0;
102
+ this.resetIfNewDay();
103
+ return (this.state.totalCost / this.config.daily) * 100;
104
+ }
105
+ /** Get current state snapshot (for persistence or debugging). */
106
+ getState() {
107
+ this.resetIfNewDay();
108
+ return this.state;
109
+ }
110
+ /** Restore state from persistence (e.g., on startup). */
111
+ restoreState(saved) {
112
+ if (saved.date !== todayUTC())
113
+ return; // stale state, ignore
114
+ this.state = {
115
+ date: saved.date,
116
+ totalCost: saved.totalCost,
117
+ perUser: new Map(Object.entries(saved.perUser)),
118
+ perModel: new Map(Object.entries(saved.perModel)),
119
+ };
120
+ }
121
+ }
@@ -0,0 +1,34 @@
1
+ /** Budget configuration for initObservability(). */
2
+ export interface BudgetConfig {
3
+ /** Max daily spend in USD for the entire application. */
4
+ readonly daily?: number | undefined;
5
+ /** Max daily spend in USD per user. Requires userId in traceLLMCall attributes. */
6
+ readonly perUser?: number | undefined;
7
+ /** Max daily spend in USD per model. */
8
+ readonly perModel?: Readonly<Record<string, number>> | undefined;
9
+ }
10
+ /** What happens when a budget is exceeded. */
11
+ export type BudgetExceededMode = "warn" | "block" | "downgrade";
12
+ /** Callback for 'downgrade' mode — receives original input, returns modified input. */
13
+ export type DowngradeCallback = (original: {
14
+ readonly provider: string;
15
+ readonly model: string;
16
+ }) => {
17
+ readonly provider: string;
18
+ readonly model: string;
19
+ };
20
+ /** Internal state tracking daily spend. */
21
+ export interface BudgetState {
22
+ date: string;
23
+ totalCost: number;
24
+ perUser: Map<string, number>;
25
+ perModel: Map<string, number>;
26
+ }
27
+ /** Info about which budget was exceeded. */
28
+ export interface BudgetExceededInfo {
29
+ readonly budget: "daily" | "perUser" | "perModel";
30
+ readonly limit: number;
31
+ readonly current: number;
32
+ readonly model?: string | undefined;
33
+ readonly userId?: string | undefined;
34
+ }
@@ -0,0 +1 @@
1
+ export {};
package/dist/cli.js CHANGED
@@ -1,9 +1,16 @@
1
1
  #!/usr/bin/env node
2
- import { execSync } from "node:child_process";
3
- import { cpSync, existsSync, mkdirSync } from "node:fs";
2
+ import { execSync, spawn } from "node:child_process";
3
+ import { cpSync, existsSync, mkdirSync, writeFileSync } from "node:fs";
4
4
  import { join, resolve } from "node:path";
5
5
  import { fileURLToPath } from "node:url";
6
+ import { initObservability, traceLLMCall, shutdown, calculateCost, } from "./index.js";
6
7
  const INFRA_DIR = "infra/toad-eye";
8
+ const DEMO_DELAY_MIN_MS = 200;
9
+ const DEMO_DELAY_MAX_MS = 2000;
10
+ const DEMO_ERROR_RATE = 0.1;
11
+ const DEMO_TICK_INTERVAL_MS = 2000;
12
+ const DEMO_TOKEN_INPUT_RANGE = [50, 500];
13
+ const DEMO_TOKEN_OUTPUT_RANGE = [20, 300];
7
14
  const SERVICES = [
8
15
  { name: "Grafana", url: "http://localhost:3100", login: "admin / admin" },
9
16
  { name: "Jaeger UI", url: "http://localhost:16686" },
@@ -18,6 +25,14 @@ function getTemplatesDir() {
18
25
  function getComposeFile() {
19
26
  return join(process.cwd(), INFRA_DIR, "docker-compose.yml");
20
27
  }
28
+ function requireInfra(message = `Run \`npx toad-eye init\` first.`) {
29
+ const composeFile = getComposeFile();
30
+ if (!existsSync(composeFile)) {
31
+ console.error(`\u274c ${INFRA_DIR}/ not found. ${message}`);
32
+ process.exit(1);
33
+ }
34
+ return composeFile;
35
+ }
21
36
  function init() {
22
37
  const dest = join(process.cwd(), INFRA_DIR);
23
38
  if (existsSync(dest)) {
@@ -38,32 +53,20 @@ function init() {
38
53
  console.log(" npx toad-eye status Check running services");
39
54
  }
40
55
  function up() {
41
- const composeFile = getComposeFile();
42
- if (!existsSync(composeFile)) {
43
- console.error(`\u274c ${INFRA_DIR}/ not found. Run \`npx toad-eye init\` first.`);
44
- process.exit(1);
45
- }
56
+ const composeFile = requireInfra();
46
57
  console.log("\u{1f438} Starting observability stack...");
47
58
  execSync(`docker compose -f ${composeFile} up -d`, { stdio: "inherit" });
48
59
  console.log();
49
60
  status();
50
61
  }
51
62
  function down() {
52
- const composeFile = getComposeFile();
53
- if (!existsSync(composeFile)) {
54
- console.error(`\u274c ${INFRA_DIR}/ not found. Nothing to stop.`);
55
- process.exit(1);
56
- }
63
+ const composeFile = requireInfra("Nothing to stop.");
57
64
  console.log("\u{1f44b} Stopping observability stack...");
58
65
  execSync(`docker compose -f ${composeFile} down`, { stdio: "inherit" });
59
66
  console.log("\u2705 Stack stopped.");
60
67
  }
61
68
  function status() {
62
- const composeFile = getComposeFile();
63
- if (!existsSync(composeFile)) {
64
- console.error(`\u274c ${INFRA_DIR}/ not found. Run \`npx toad-eye init\` first.`);
65
- process.exit(1);
66
- }
69
+ const composeFile = requireInfra();
67
70
  try {
68
71
  const output = execSync(`docker compose -f ${composeFile} ps --format json`, {
69
72
  encoding: "utf-8",
@@ -84,6 +87,104 @@ function status() {
84
87
  console.log("Could not check container status. Is Docker running?");
85
88
  }
86
89
  }
90
+ const DEMO_MODELS = [
91
+ { provider: "anthropic", model: "claude-sonnet-4-20250514" },
92
+ { provider: "gemini", model: "gemini-2.5-flash" },
93
+ { provider: "openai", model: "gpt-4o" },
94
+ ];
95
+ function randomBetween(min, max) {
96
+ return Math.floor(Math.random() * (max - min + 1)) + min;
97
+ }
98
+ const DEMO_PROMPTS = [
99
+ "Explain quantum computing",
100
+ "Write a haiku about frogs",
101
+ "What is the capital of Uruguay?",
102
+ "How does TCP work?",
103
+ "Translate hello to Japanese",
104
+ ];
105
+ function pickRandom(arr) {
106
+ return arr[randomBetween(0, arr.length - 1)];
107
+ }
108
+ async function simulateLLMCall(provider, model, prompt) {
109
+ await new Promise((r) => setTimeout(r, randomBetween(DEMO_DELAY_MIN_MS, DEMO_DELAY_MAX_MS)));
110
+ if (Math.random() < DEMO_ERROR_RATE) {
111
+ throw new Error(`${provider} API error: rate limit exceeded`);
112
+ }
113
+ const inputTokens = randomBetween(...DEMO_TOKEN_INPUT_RANGE);
114
+ const outputTokens = randomBetween(...DEMO_TOKEN_OUTPUT_RANGE);
115
+ return {
116
+ completion: `Mock response for: "${prompt}"`,
117
+ inputTokens,
118
+ outputTokens,
119
+ cost: calculateCost(model, inputTokens, outputTokens),
120
+ };
121
+ }
122
+ async function demo() {
123
+ requireInfra();
124
+ initObservability({
125
+ serviceName: "toad-eye-demo",
126
+ endpoint: "http://localhost:4318",
127
+ });
128
+ console.log("\u{1f438}\u{1f441}\u{fe0f} toad-eye demo — sending mock LLM traffic");
129
+ console.log(" Press Ctrl+C to stop\n");
130
+ process.on("SIGINT", async () => {
131
+ console.log("\n\u{1f44b} Shutting down...");
132
+ await shutdown();
133
+ process.exit(0);
134
+ });
135
+ const tick = async () => {
136
+ const { provider, model } = pickRandom(DEMO_MODELS);
137
+ const prompt = pickRandom(DEMO_PROMPTS);
138
+ try {
139
+ await traceLLMCall({ provider, model, prompt, temperature: 0.7 }, () => simulateLLMCall(provider, model, prompt));
140
+ console.log(` \u2705 [${provider}/${model}] ${prompt}`);
141
+ }
142
+ catch (err) {
143
+ const msg = err instanceof Error ? err.message : String(err);
144
+ console.log(` \u274c [${provider}/${model}] ${msg}`);
145
+ }
146
+ };
147
+ const run = () => {
148
+ tick().then(() => setTimeout(run, DEMO_TICK_INTERVAL_MS));
149
+ };
150
+ run();
151
+ }
152
+ async function exportTraceCommand() {
153
+ const traceId = process.argv[3];
154
+ if (!traceId) {
155
+ console.error("✗ Usage: npx toad-eye export-trace <trace_id> [--output dir] [--jaeger-url url]");
156
+ process.exit(1);
157
+ }
158
+ const args = process.argv.slice(4);
159
+ let outputDir = process.cwd();
160
+ let jaegerUrl;
161
+ for (let i = 0; i < args.length; i++) {
162
+ if (args[i] === "--output" && args[i + 1]) {
163
+ outputDir = resolve(args[i + 1]);
164
+ i++;
165
+ }
166
+ else if (args[i] === "--jaeger-url" && args[i + 1]) {
167
+ jaegerUrl = args[i + 1];
168
+ i++;
169
+ }
170
+ }
171
+ const { exportTrace } = await import("./export.js");
172
+ try {
173
+ const yaml = await exportTrace(traceId, { jaegerUrl });
174
+ const filename = `trace-${traceId.slice(0, 8)}.eval.yaml`;
175
+ const filepath = join(outputDir, filename);
176
+ if (!existsSync(outputDir)) {
177
+ mkdirSync(outputDir, { recursive: true });
178
+ }
179
+ writeFileSync(filepath, yaml, "utf-8");
180
+ console.log(`✅ Exported trace ${traceId} → ${filepath}`);
181
+ }
182
+ catch (error) {
183
+ const msg = error instanceof Error ? error.message : String(error);
184
+ console.error(`✗ ${msg}`);
185
+ process.exit(1);
186
+ }
187
+ }
87
188
  function help() {
88
189
  console.log(`
89
190
  \u{1f438} toad-eye CLI — observability stack for LLM services
@@ -93,7 +194,9 @@ Commands:
93
194
  up Start the stack (OTel Collector + Prometheus + Jaeger + Grafana)
94
195
  down Stop the stack
95
196
  status Show running services and URLs
96
- help Show this message
197
+ demo Send mock LLM traffic to see data in Grafana
198
+ export-trace Export a Jaeger trace to toad-eval YAML
199
+ help Show this message
97
200
  `);
98
201
  }
99
202
  const command = process.argv[2];
@@ -110,6 +213,12 @@ switch (command) {
110
213
  case "status":
111
214
  status();
112
215
  break;
216
+ case "demo":
217
+ demo();
218
+ break;
219
+ case "export-trace":
220
+ exportTraceCommand();
221
+ break;
113
222
  case "help":
114
223
  case "--help":
115
224
  case "-h":
@@ -0,0 +1,14 @@
1
+ export declare function initMetrics(): void;
2
+ export declare function recordRequestDuration(ms: number, provider: string, model: string, attrs?: Record<string, string>): void;
3
+ export declare function recordRequestCost(usd: number, provider: string, model: string, attrs?: Record<string, string>): void;
4
+ export declare function recordTokens(count: number, provider: string, model: string, attrs?: Record<string, string>): void;
5
+ export declare function recordRequest(provider: string, model: string, attrs?: Record<string, string>): void;
6
+ export declare function recordError(provider: string, model: string, attrs?: Record<string, string>): void;
7
+ export declare function recordAgentSteps(stepCount: number): void;
8
+ export declare function recordAgentToolUsage(toolName: string): void;
9
+ export declare function recordGuardEvaluation(ruleName: string): void;
10
+ export declare function recordGuardWouldBlock(ruleName: string): void;
11
+ export declare function recordSemanticDrift(drift: number, provider: string, model: string): void;
12
+ export declare function recordBudgetExceeded(budgetType: string): void;
13
+ export declare function recordBudgetBlocked(budgetType: string): void;
14
+ export declare function recordBudgetDowngraded(budgetType: string): void;