toad-eye 2.2.0 → 2.4.0

package/dist/agent.js

@@ -1,30 +1,60 @@
 import { trace, SpanStatusCode } from "@opentelemetry/api";
 import { GEN_AI_ATTRS, INSTRUMENTATION_NAME } from "./types/index.js";
 import { recordAgentSteps, recordAgentToolUsage } from "./core/metrics.js";
-import { getConfig } from "./core/tracer.js";
+import { getConfig, shouldEmitDeprecatedAttrs } from "./core/tracer.js";
 const tracer = trace.getTracer(INSTRUMENTATION_NAME);
+const DEFAULT_MAX_STEPS = 25;
 /**
- * Record a single agent step as a child span (ReAct pattern: think → act → observe → answer).
- *
- * Creates a short-lived span under the current active context. Call this inside
- * {@link traceAgentQuery} (preferred) or any other active span (e.g. traceLLMCall).
- *
- * - Respects `recordContent` config — content is omitted when recording is disabled.
- * - For `act` steps with a `toolName`, automatically increments the `agent.tool_usage` metric.
+ * Record a single agent step as a child span (ReAct pattern).
+ * Supports: think, act, observe, answer, handoff.
  */
 function traceAgentStep(input) {
-    const span = tracer.startSpan(`agent.step.${input.type}`);
+    const spanName = input.type === "act" && input.toolName !== undefined
+        ? `execute_tool ${input.toolName}`
+        : `gen_ai.agent.step.${input.type}`;
+    const span = tracer.startSpan(spanName);
     const config = getConfig();
     const recordContent = config?.recordContent !== false;
+    const emitDeprecated = shouldEmitDeprecatedAttrs();
     span.setAttributes({
-        [GEN_AI_ATTRS.AGENT_STEP_TYPE]: input.type,
-        [GEN_AI_ATTRS.AGENT_STEP_NUMBER]: input.stepNumber,
+        // Canonical toad_eye namespace
+        [GEN_AI_ATTRS.TOAD_AGENT_STEP_TYPE]: input.type,
+        [GEN_AI_ATTRS.TOAD_AGENT_STEP_NUMBER]: input.stepNumber,
+        // Deprecated aliases (skipped when OTEL_SEMCONV_STABILITY_OPT_IN=gen_ai_latest_experimental)
+        ...(emitDeprecated && {
+            [GEN_AI_ATTRS.AGENT_STEP_TYPE]: input.type,
+            [GEN_AI_ATTRS.AGENT_STEP_NUMBER]: input.stepNumber,
+        }),
         ...(input.toolName !== undefined && {
-            [GEN_AI_ATTRS.AGENT_TOOL_NAME]: input.toolName,
+            // OTel spec attribute
+            [GEN_AI_ATTRS.TOOL_NAME]: input.toolName,
+            // Deprecated alias
+            ...(emitDeprecated && {
+                [GEN_AI_ATTRS.AGENT_TOOL_NAME]: input.toolName,
+            }),
+        }),
+        ...(input.toolType !== undefined && {
+            [GEN_AI_ATTRS.TOOL_TYPE]: input.toolType,
         }),
         ...(recordContent &&
             input.content !== undefined && {
-            [GEN_AI_ATTRS.AGENT_STEP_CONTENT]: input.content,
+            [GEN_AI_ATTRS.TOAD_AGENT_STEP_CONTENT]: input.content,
+            ...(emitDeprecated && {
+                [GEN_AI_ATTRS.AGENT_STEP_CONTENT]: input.content,
+            }),
+        }),
+        // Handoff attributes
+        ...(input.toAgent !== undefined && {
+            [GEN_AI_ATTRS.TOAD_AGENT_HANDOFF_TO]: input.toAgent,
+            ...(emitDeprecated && {
+                [GEN_AI_ATTRS.AGENT_HANDOFF_TO]: input.toAgent,
+            }),
+        }),
+        ...(input.handoffReason !== undefined && {
+            [GEN_AI_ATTRS.TOAD_AGENT_HANDOFF_REASON]: input.handoffReason,
+            ...(emitDeprecated && {
+                [GEN_AI_ATTRS.AGENT_HANDOFF_REASON]: input.handoffReason,
+            }),
         }),
     });
     if (input.type === "act" && input.toolName !== undefined) {
@@ -34,51 +64,102 @@ function traceAgentStep(input) {
     span.end();
 }
 /**
- * Wrap an entire agent query in a parent span, producing structured ReAct traces.
+ * Wrap an entire agent query in a parent span with ReAct tracing.
  *
- * The callback receives a `step` function to record individual steps.
- * When the callback completes, the wrapper automatically records the
- * `agent.steps_per_query` histogram metric with the total step count.
+ * Accepts either a string query (backward-compatible) or an object with
+ * agent metadata per OTel GenAI spec (agentName, agentId).
  *
- * Span hierarchy in Jaeger:
- * ```
- * [parent] agent.query "Is anything dangerous near Earth?"
- *   ├── agent.step.think   "I need to check asteroids"
- *   ├── agent.step.act     tool=near-earth-asteroids
- *   ├── agent.step.observe "7 asteroids found"
- *   └── agent.step.answer  "7 asteroids passing safely..."
- * ```
+ * Supports multi-agent via nesting — child traceAgentQuery calls
+ * automatically become child spans in Jaeger.
+ *
+ * Loop detection: counts observe→think transitions. Records loop count
+ * as span attribute. Emits warning when maxSteps exceeded.
  *
  * @example
  * ```ts
- * const result = await traceAgentQuery("Is anything dangerous near Earth?", async (step) => {
- *   step({ type: "think", stepNumber: 1, content: "I need to check asteroids" });
- *   const data = await callTool("near-earth-asteroids");
- *   step({ type: "act", stepNumber: 2, toolName: "near-earth-asteroids" });
- *   step({ type: "observe", stepNumber: 3, content: `${data.length} asteroids found` });
- *   step({ type: "answer", stepNumber: 4, content: "7 asteroids passing safely..." });
- *   return { answer: "7 asteroids passing safely..." };
+ * // Simple form (backward-compatible)
+ * await traceAgentQuery('Is anything dangerous?', async (step) => { ... });
+ *
+ * // Object form with agent metadata (OTel GenAI spec)
+ * await traceAgentQuery(
+ *   { query: 'Is anything dangerous?', agentName: 'space-monitor', agentId: 'agent-001' },
+ *   async (step) => { ... }
+ * );
+ *
+ * // Multi-agent: orchestrator delegates to specialist
+ * await traceAgentQuery({ query: 'Analyze data', agentName: 'orchestrator' }, async (step) => {
+ *   step({ type: 'think', stepNumber: 1, content: 'Need domain expert' });
+ *   step({ type: 'handoff', stepNumber: 2, toAgent: 'specialist', handoffReason: 'domain expertise' });
+ *
+ *   const result = await traceAgentQuery({ query: 'Analyze', agentName: 'specialist' }, async (step) => {
+ *     step({ type: 'act', stepNumber: 1, toolName: 'analyze', toolType: 'function' });
+ *     step({ type: 'answer', stepNumber: 2, content: 'analysis complete' });
+ *     return { answer: 'done' };
+ *   });
  * });
  * ```
  */
-export async function traceAgentQuery(query, fn) {
-    return tracer.startActiveSpan(`agent.query`, async (span) => {
+export async function traceAgentQuery(queryOrInput, fn, options) {
+    const resolved = typeof queryOrInput === "string"
+        ? { query: queryOrInput, agentName: undefined, agentId: undefined }
+        : queryOrInput;
+    const spanName = resolved.agentName
+        ? `invoke_agent ${resolved.agentName}`
+        : `invoke_agent`;
+    return tracer.startActiveSpan(spanName, async (span) => {
         const config = getConfig();
         const recordContent = config?.recordContent !== false;
+        const emitDeprecated = shouldEmitDeprecatedAttrs();
+        const maxSteps = options?.maxSteps ?? DEFAULT_MAX_STEPS;
+        // OTel GenAI agent attributes
+        if (resolved.agentName !== undefined) {
+            span.setAttribute(GEN_AI_ATTRS.AGENT_NAME, resolved.agentName);
+        }
+        if (resolved.agentId !== undefined) {
+            span.setAttribute(GEN_AI_ATTRS.AGENT_ID, resolved.agentId);
+        }
         if (recordContent) {
-            span.setAttribute(GEN_AI_ATTRS.AGENT_STEP_CONTENT, query);
+            span.setAttribute(GEN_AI_ATTRS.TOAD_AGENT_STEP_CONTENT, resolved.query);
+            if (emitDeprecated) {
+                span.setAttribute(GEN_AI_ATTRS.AGENT_STEP_CONTENT, resolved.query);
+            }
         }
         let stepCount = 0;
+        let loopCount = 0;
+        let lastStepType;
+        let maxStepsWarned = false;
         try {
             const result = await fn((input) => {
                 stepCount++;
+                // Loop detection: observe → think = one loop iteration
+                if (input.type === "think" && lastStepType === "observe") {
+                    loopCount++;
+                }
+                lastStepType = input.type;
+                // Max steps guard
+                if (stepCount > maxSteps && !maxStepsWarned) {
+                    maxStepsWarned = true;
+                    console.warn(`toad-eye: agent exceeded maxSteps (${maxSteps}). Current: ${stepCount}`);
+                    span.addEvent("agent.max_steps_exceeded", {
+                        "agent.max_steps": maxSteps,
+                        "agent.current_steps": stepCount,
+                    });
+                }
                 traceAgentStep(input);
             });
+            span.setAttribute(GEN_AI_ATTRS.TOAD_AGENT_LOOP_COUNT, loopCount);
+            if (emitDeprecated) {
+                span.setAttribute(GEN_AI_ATTRS.AGENT_LOOP_COUNT, loopCount);
+            }
             span.setStatus({ code: SpanStatusCode.OK });
             return result;
         }
         catch (error) {
             const message = error instanceof Error ? error.message : String(error);
+            span.setAttribute(GEN_AI_ATTRS.TOAD_AGENT_LOOP_COUNT, loopCount);
+            if (emitDeprecated) {
+                span.setAttribute(GEN_AI_ATTRS.AGENT_LOOP_COUNT, loopCount);
+            }
             span.setStatus({ code: SpanStatusCode.ERROR, message });
             throw error;
         }

package/dist/alerts/channels.js

@@ -1,4 +1,10 @@
-import nodemailer from "nodemailer";
+// Cache nodemailer transporters by connection identity to avoid creating a new
+// SMTP connection for every alert (frequent alerts would exhaust connection pools).
+// eslint-disable-next-line @typescript-eslint/no-explicit-any
+const smtpTransporterCache = new Map();
+function smtpCacheKey(config) {
+    return `${config.host}:${config.port}:${config.user}`;
+}
 function formatMessage(alert) {
     const topModelsText = alert.topModels.length > 0
         ? `\nTop models by spend:\n${alert.topModels.map((m) => `  • ${m.model}: $${m.value.toFixed(4)}`).join("\n")}`
@@ -52,11 +58,23 @@ async function sendWebhook(config, alert) {
         throw new Error(`Webhook delivery failed: ${res.status}`);
 }
 async function sendEmail(config, alert) {
-    const transporter = nodemailer.createTransport({
-        host: config.host,
-        port: config.port,
-        auth: { user: config.user, pass: config.password },
-    });
+    const key = smtpCacheKey(config);
+    let transporter = smtpTransporterCache.get(key);
+    if (!transporter) {
+        let createTransport;
+        try {
+            ({ createTransport } = await import("nodemailer"));
+        }
+        catch {
+            throw new Error("toad-eye: email alerts require nodemailer — install it: npm install nodemailer");
+        }
+        transporter = createTransport({
+            host: config.host,
+            port: config.port,
+            auth: { user: config.user, pass: config.password },
+        });
+        smtpTransporterCache.set(key, transporter);
+    }
     const to = typeof config.to === "string"
         ? config.to
         : Array.from(config.to).join(", ");

package/dist/alerts/index.js

@@ -3,10 +3,54 @@ export { parseCondition } from "./conditions.js";
 import { readFileSync } from "node:fs";
 import yaml from "yaml";
 import { AlertManager } from "./manager.js";
+/** Resolve ${ENV_VAR} references in string values throughout the config. */
+function resolveEnvVars(value) {
+    return value.replace(/\$\{([^}]+)\}/g, (_, name) => {
+        return process.env[name] ?? `\${${name}}`;
+    });
+}
+function interpolateConfig(obj) {
+    if (typeof obj === "string")
+        return resolveEnvVars(obj);
+    if (Array.isArray(obj))
+        return obj.map(interpolateConfig);
+    if (obj !== null && typeof obj === "object") {
+        return Object.fromEntries(Object.entries(obj).map(([k, v]) => [
+            k,
+            interpolateConfig(v),
+        ]));
+    }
+    return obj;
+}
+function validateConfig(config) {
+    if (typeof config !== "object" || config === null) {
+        throw new Error("toad-eye: alerts config must be an object");
+    }
+    const c = config;
+    if (!Array.isArray(c["alerts"])) {
+        throw new Error('toad-eye: alerts config missing required "alerts" array');
+    }
+    for (const rule of c["alerts"]) {
+        if (typeof rule !== "object" || rule === null) {
+            throw new Error("toad-eye: each alert rule must be an object");
+        }
+        const r = rule;
+        for (const field of ["name", "metric", "condition"]) {
+            if (typeof r[field] !== "string" || !r[field]) {
+                throw new Error(`toad-eye: alert rule missing required string field "${field}"`);
+            }
+        }
+        if (!Array.isArray(r["channels"]) || r["channels"].length === 0) {
+            throw new Error(`toad-eye: alert rule "${String(r["name"])}" must have at least one channel`);
+        }
+    }
+}
 export function startAlertsFromFile(configPath) {
     const raw = readFileSync(configPath, "utf-8");
-    const config = yaml.parse(raw);
-    const manager = new AlertManager(config);
+    const parsed = yaml.parse(raw);
+    const interpolated = interpolateConfig(parsed);
+    validateConfig(interpolated);
+    const manager = new AlertManager(interpolated);
     manager.start();
     return manager;
 }

package/dist/alerts/manager.d.ts

@@ -8,6 +8,7 @@ export declare class AlertManager {
     stop(): void;
     private isInCooldown;
     private evaluate;
+    private evaluateRule;
     private fireChannels;
     private postAnnotation;
 }

package/dist/alerts/manager.js

@@ -5,6 +5,7 @@ const DEFAULT_PROMETHEUS_URL = "http://localhost:9090";
 const DEFAULT_GRAFANA_URL = "http://localhost:3100";
 const DEFAULT_EVAL_INTERVAL_SECONDS = 60;
 const DEFAULT_COOLDOWN_MINUTES = 30;
+const MIN_EVAL_INTERVAL_SECONDS = 10;
 export class AlertManager {
     config;
     cooldowns = new Map();
@@ -13,10 +14,15 @@ export class AlertManager {
         this.config = config;
     }
     start() {
-        const intervalMs = (this.config.evalIntervalSeconds ?? DEFAULT_EVAL_INTERVAL_SECONDS) * 1000;
+        const configured = this.config.evalIntervalSeconds ?? DEFAULT_EVAL_INTERVAL_SECONDS;
+        const clampedSeconds = Math.max(configured, MIN_EVAL_INTERVAL_SECONDS);
+        if (configured < MIN_EVAL_INTERVAL_SECONDS) {
+            console.warn(`[toad-eye alerts] evalIntervalSeconds ${configured} is below minimum ${MIN_EVAL_INTERVAL_SECONDS}s — using ${clampedSeconds}s`);
+        }
+        const intervalMs = clampedSeconds * 1000;
         this.intervalId = setInterval(() => void this.evaluate(), intervalMs);
         void this.evaluate();
-        console.log(`[toad-eye alerts] Started — ${this.config.alerts.length} rule(s), eval every ${intervalMs / 1000}s`);
+        console.log(`[toad-eye alerts] Started — ${this.config.alerts.length} rule(s), eval every ${clampedSeconds}s`);
     }
     stop() {
         if (this.intervalId !== undefined) {
@@ -38,26 +44,35 @@ export class AlertManager {
     async evaluate() {
         const prometheusUrl = this.config.prometheusUrl ?? DEFAULT_PROMETHEUS_URL;
         const grafanaUrl = this.config.grafanaUrl ?? DEFAULT_GRAFANA_URL;
-        for (const rule of this.config.alerts) {
-            if (this.isInCooldown(rule))
-                continue;
-            const result = await evaluateCondition(prometheusUrl, rule);
-            if (!result?.triggered)
-                continue;
-            this.cooldowns.set(rule.name, Date.now());
-            const firedAlert = {
-                rule,
-                value: result.value,
-                threshold: result.threshold,
-                topModels: result.topModels,
-                firedAt: new Date(),
-            };
-            console.log(`[toad-eye alerts] 🚨 "${rule.name}" fired — value=${result.value.toFixed(4)}, threshold=${result.threshold}`);
-            await Promise.allSettled([
-                this.fireChannels(firedAlert),
-                this.postAnnotation(grafanaUrl, firedAlert.rule.name, firedAlert.value, firedAlert.threshold),
-            ]);
+        // Evaluate all rules in parallel — each rule is independent
+        await Promise.allSettled(this.config.alerts.map((rule) => this.evaluateRule(prometheusUrl, grafanaUrl, rule)));
+    }
+    async evaluateRule(prometheusUrl, grafanaUrl, rule) {
+        if (this.isInCooldown(rule))
+            return;
+        let result;
+        try {
+            result = await evaluateCondition(prometheusUrl, rule);
+        }
+        catch (err) {
+            console.error(`[toad-eye alerts] Failed to evaluate rule "${rule.name}":`, err);
+            return;
         }
+        if (!result?.triggered)
+            return;
+        this.cooldowns.set(rule.name, Date.now());
+        const firedAlert = {
+            rule,
+            value: result.value,
+            threshold: result.threshold,
+            topModels: result.topModels,
+            firedAt: new Date(),
+        };
+        console.log(`[toad-eye alerts] 🚨 "${rule.name}" fired — value=${result.value.toFixed(4)}, threshold=${result.threshold}`);
+        await Promise.allSettled([
+            this.fireChannels(firedAlert),
+            this.postAnnotation(grafanaUrl, firedAlert.rule.name, firedAlert.value, firedAlert.threshold),
+        ]);
     }
     async fireChannels(alert) {
         const channels = this.config.channels ?? {};

package/dist/budget/tracker.d.ts

@@ -9,18 +9,28 @@ export declare class BudgetTracker {
     private resetIfNewDay;
     /**
      * Check budget BEFORE making an LLM call.
-     * Returns modified provider/model if downgrade mode triggers,
-     * throws ToadBudgetExceededError if block mode triggers,
-     * returns null if no action needed.
+     * - block mode: throws ToadBudgetExceededError if budget is exceeded.
+     * - downgrade mode: returns modified provider/model + budget type that triggered it.
+     * - warn mode: returns null (warning emitted after call via recordCost).
+     *
+     * When estimatedCost is provided, it is added to reservedCost so concurrent
+     * in-flight requests are counted against the budget before their actual cost is known.
+     * Call recordCost with the same estimatedCost to release the reservation.
      */
-    checkBefore(provider: string, model: string, userId?: string): {
+    checkBefore(provider: string, model: string, userId?: string, estimatedCost?: number): {
         provider: string;
         model: string;
+        budget: BudgetExceededInfo["budget"];
     } | null;
-    /** Record cost AFTER a successful LLM call. Returns exceeded info if budget was just crossed. */
-    recordCost(cost: number, model: string, userId?: string): BudgetExceededInfo | null;
+    /**
+     * Record cost AFTER a successful LLM call. Returns exceeded info if budget was just crossed.
+     * Pass the same estimatedCost used in checkBefore() to release the reservation atomically.
+     */
+    recordCost(cost: number, model: string, userId?: string, reservedAmount?: number): BudgetExceededInfo | null;
     /** Find which budget (if any) is exceeded. */
     private findExceeded;
+    /** Release a cost reservation when an LLM call fails (no cost was incurred). */
+    releaseReservation(reservedAmount: number): void;
     /** Get current budget usage as percentage (0-100+). */
     getUsagePercent(): number;
     /** Get current state snapshot (for persistence or debugging). */

package/dist/budget/tracker.js

@@ -14,6 +14,7 @@ export class BudgetTracker {
         this.state = {
             date: todayUTC(),
             totalCost: 0,
+            reservedCost: 0,
             perUser: new Map(),
             perModel: new Map(),
         };
@@ -25,6 +26,7 @@ export class BudgetTracker {
             this.state = {
                 date: today,
                 totalCost: 0,
+                reservedCost: 0,
                 perUser: new Map(),
                 perModel: new Map(),
             };
@@ -32,27 +34,40 @@ export class BudgetTracker {
     }
     /**
      * Check budget BEFORE making an LLM call.
-     * Returns modified provider/model if downgrade mode triggers,
-     * throws ToadBudgetExceededError if block mode triggers,
-     * returns null if no action needed.
+     * - block mode: throws ToadBudgetExceededError if budget is exceeded.
+     * - downgrade mode: returns modified provider/model + budget type that triggered it.
+     * - warn mode: returns null (warning emitted after call via recordCost).
+     *
+     * When estimatedCost is provided, it is added to reservedCost so concurrent
+     * in-flight requests are counted against the budget before their actual cost is known.
+     * Call recordCost with the same estimatedCost to release the reservation.
      */
-    checkBefore(provider, model, userId) {
+    checkBefore(provider, model, userId, estimatedCost = 0) {
         this.resetIfNewDay();
-        const exceeded = this.findExceeded(model, userId);
-        if (!exceeded)
-            return null;
-        if (this.mode === "block") {
-            throw new ToadBudgetExceededError(exceeded);
-        }
-        if (this.mode === "downgrade" && this.downgrade) {
-            return this.downgrade({ provider, model });
+        const exceeded = this.findExceeded(model, userId, estimatedCost);
+        if (exceeded) {
+            if (this.mode === "block") {
+                throw new ToadBudgetExceededError(exceeded);
+            }
+            if (this.mode === "downgrade" && this.downgrade) {
+                this.state.reservedCost += estimatedCost;
+                return {
+                    ...this.downgrade({ provider, model }),
+                    budget: exceeded.budget,
+                };
+            }
         }
-        // 'warn' mode — continue, warning emitted after call
+        // warn mode or no budget exceeded: reserve estimated cost and proceed
+        this.state.reservedCost += estimatedCost;
         return null;
     }
-    /** Record cost AFTER a successful LLM call. Returns exceeded info if budget was just crossed. */
-    recordCost(cost, model, userId) {
+    /**
+     * Record cost AFTER a successful LLM call. Returns exceeded info if budget was just crossed.
+     * Pass the same estimatedCost used in checkBefore() to release the reservation atomically.
+     */
+    recordCost(cost, model, userId, reservedAmount = 0) {
         this.resetIfNewDay();
+        this.state.reservedCost = Math.max(0, this.state.reservedCost - reservedAmount);
         this.state.totalCost += cost;
         this.state.perModel.set(model, (this.state.perModel.get(model) ?? 0) + cost);
         if (userId) {
@@ -61,9 +76,10 @@ export class BudgetTracker {
         return this.findExceeded(model, userId);
     }
     /** Find which budget (if any) is exceeded. */
-    findExceeded(model, userId) {
+    findExceeded(model, userId, additionalCost = 0) {
         if (this.config.daily !== undefined &&
-            this.state.totalCost >= this.config.daily) {
+            this.state.totalCost + this.state.reservedCost + additionalCost >=
+                this.config.daily) {
             return {
                 budget: "daily",
                 limit: this.config.daily,
@@ -95,6 +111,10 @@ export class BudgetTracker {
         }
         return null;
     }
+    /** Release a cost reservation when an LLM call fails (no cost was incurred). */
+    releaseReservation(reservedAmount) {
+        this.state.reservedCost = Math.max(0, this.state.reservedCost - reservedAmount);
+    }
     /** Get current budget usage as percentage (0-100+). */
     getUsagePercent() {
         if (this.config.daily === undefined || this.config.daily === 0)
@@ -114,6 +134,7 @@ export class BudgetTracker {
         this.state = {
             date: saved.date,
             totalCost: saved.totalCost,
+            reservedCost: 0,
             perUser: new Map(Object.entries(saved.perUser)),
             perModel: new Map(Object.entries(saved.perModel)),
         };

package/dist/budget/types.d.ts

@@ -21,6 +21,8 @@ export type DowngradeCallback = (original: {
 export interface BudgetState {
     date: string;
     totalCost: number;
+    /** Sum of estimated costs for in-flight requests (released when recordCost is called). */
+    reservedCost: number;
     perUser: Map<string, number>;
     perModel: Map<string, number>;
 }

package/dist/cli.d.ts

@@ -1,2 +1,9 @@
 #!/usr/bin/env node
-export {};
+/** Find the docker container matching a service by exact service name. Exported for testing. */
+export declare function findContainerByService(containers: ReadonlyArray<{
+    Service: string;
+    State: string;
+}>, serviceKey: string): {
+    Service: string;
+    State: string;
+} | undefined;