pi-pignon 0.1.1

package/src/deciders/laya-serve.ts

@@ -0,0 +1,53 @@
+/**
+ * `laya-serve` decider: a Laya model served by the official `laya-serve`
+ * (`pip install "laya[serve]"`), which speaks TypeSafe's Jev API on
+ * `POST /v1/systemone`. It reuses the Jev client, pointed at the server.
+ *
+ * The user runs the server; pignon only connects. While it is down or still
+ * loading, connections are refused within milliseconds, so a prompt is never
+ * held: the decision fails and the prompt keeps the current model.
+ */
+
+import type { LayaServeDeciderSpec } from "../types.js";
+import type { Fetch } from "@typesafe-ai/sdk";
+
+import { JevDecider } from "./jev.js";
+
+/** laya-serve's default port, on the loopback interface. */
+export const LAYA_SERVE_DEFAULT_URL = "http://127.0.0.1:8000";
+
+export function createLayaServeDecider(
+  spec: Omit<LayaServeDeciderSpec, "type">,
+  env: NodeJS.ProcessEnv = process.env,
+  fetch?: Fetch,
+): JevDecider {
+  const url = spec.url ?? LAYA_SERVE_DEFAULT_URL;
+  return new JevDecider({
+    id: "laya-serve",
+    baseURL: url,
+    requireApiKey: false,
+    unreachableHint: `is laya-serve running at ${url}?`,
+    // Only the server's own key: never TYPESAFE_API_KEY or other TYPESAFE_* settings.
+    env: spec.apiKeyEnv ? { [spec.apiKeyEnv]: env[spec.apiKeyEnv] } : {},
+    ...(spec.apiKeyEnv ? { apiKeyEnv: spec.apiKeyEnv } : {}),
+    ...(spec.model !== undefined ? { model: spec.model } : {}),
+    ...(spec.timeoutMs !== undefined ? { timeoutMs: spec.timeoutMs } : {}),
+    ...(fetch ? { fetch } : {}),
+  });
+}
+
+/** Whether a laya-serve answers at `url` (GET /health), within `timeoutMs`. */
+export async function probeLayaServe(
+  url: string = LAYA_SERVE_DEFAULT_URL,
+  timeoutMs = 500,
+  fetchFn: typeof fetch = fetch,
+): Promise<boolean> {
+  try {
+    const response = await fetchFn(`${url.replace(/\/+$/, "")}/health`, { signal: AbortSignal.timeout(timeoutMs) });
+    if (!response.ok) return false;
+    const body = (await response.json()) as { status?: unknown };
+    return body.status === "ok";
+  } catch {
+    return false;
+  }
+}

package/src/deciders/parse.ts

@@ -0,0 +1,48 @@
+/**
+ * Turn raw decider answers into the routing-relevant `RoutingDecision`.
+ *
+ * Shared by every decider, so the policy never sees decider-specific shapes.
+ * Tolerant of malformed input: a missing or invalid answer reads as "no
+ * signal", which the policy treats as fail-open.
+ */
+
+import { DEFAULT_CONFIG } from "../config/defaults.js";
+import type { ConfidenceSource, RouterConfig, RoutingDecision } from "../types.js";
+import { EXPLORATION_QUESTION, TIER_QUESTION } from "./questions.js";
+import type { RawAnswers } from "./types.js";
+
+export function parseDecision(
+  answers: RawAnswers,
+  latencyMs: number,
+  config: Pick<RouterConfig, "table" | "confidenceSource"> = DEFAULT_CONFIG,
+): RoutingDecision {
+  const tierAnswer = choiceOf(answers[TIER_QUESTION], config.confidenceSource);
+  const tier = tierAnswer && config.table.some((t) => t.id === tierAnswer.choice) ? tierAnswer.choice : null;
+
+  const explorationAnswer = choiceOf(answers[EXPLORATION_QUESTION], config.confidenceSource);
+
+  return {
+    tier,
+    tierConfidence: tier ? tierAnswer!.confidence : 0,
+    needsExploration: explorationAnswer?.choice === "yes",
+    explorationConfidence: explorationAnswer?.confidence ?? 0,
+    latencyMs,
+  };
+}
+
+/**
+ * Extract a choice answer. The Laya worker tags answers with `type`, the
+ * TypeSafe SDK does not, so the tag is only checked when present.
+ */
+function choiceOf(value: unknown, source: ConfidenceSource): { choice: string; confidence: number } | null {
+  if (!isRecord(value) || typeof value.choice !== "string") return null;
+  if (value.type !== undefined && value.type !== "choice") return null;
+  const choice = value.choice;
+  const confidence =
+    source === "top-probability" && isRecord(value.probabilities) ? value.probabilities[choice] : value.confidence;
+  return { choice, confidence: typeof confidence === "number" && Number.isFinite(confidence) ? confidence : 0 };
+}
+
+function isRecord(value: unknown): value is Record<string, unknown> {
+  return typeof value === "object" && value !== null && !Array.isArray(value);
+}

package/src/deciders/questions.ts

@@ -0,0 +1,34 @@
+/**
+ * The two questions every decider is asked about a prompt, built from the
+ * config: the tier criteria come from the routing table, the rest from
+ * `config.questions`.
+ *
+ * Changing the wording changes what the confidences mean, so thresholds
+ * calibrated on one wording do not carry over to another; see
+ * `QuestionWording.version`.
+ */
+
+import type { RouterConfig } from "../types.js";
+import type { DecisionRequest } from "./types.js";
+
+/** Question whose choice is the tier id. */
+export const TIER_QUESTION = "reasoning_demand";
+
+/** Question whose `yes` choice means the task needs exploration. */
+export const EXPLORATION_QUESTION = "needs_exploration";
+
+export function buildQuestions(config: Pick<RouterConfig, "table" | "questions">): DecisionRequest["questions"] {
+  const { table, questions } = config;
+  return {
+    [TIER_QUESTION]: {
+      type: "choice",
+      instructions: questions.tierInstructions,
+      criteria: Object.fromEntries(table.map((tier) => [tier.id, tier.criterion])),
+    },
+    [EXPLORATION_QUESTION]: {
+      type: "choice",
+      instructions: questions.explorationInstructions,
+      criteria: { yes: questions.explorationCriteria.yes, no: questions.explorationCriteria.no },
+    },
+  };
+}

package/src/deciders/strategy.ts

@@ -0,0 +1,225 @@
+/**
+ * Several deciders behind one: the `strategy` setting.
+ *
+ * - sequential: ask them in order, moving on when one is not ready, fails,
+ *   or answers with tier confidence below `escalateBelow`. Remote deciders
+ *   are only called when needed.
+ * - parallel: ask every ready decider at once, then route on one answer
+ *   (`pick`). Every answer is recorded, which is how deciders are compared.
+ *
+ * Every attempt is returned in `DeciderResult.attempts`, so both modes log
+ * the same data. A shared `budgetMs` bounds the wall time of one decision.
+ */
+
+import type { DeciderAttempt, RoutingDecision, StrategyConfig } from "../types.js";
+import { type Decider, type DeciderResult, type DecisionRequest, type RawAnswers, DeciderError } from "./types.js";
+
+/** Reads answers the way the router will (tier ids, confidence source). */
+export type ParseAnswers = (answers: RawAnswers, latencyMs: number) => RoutingDecision;
+
+interface Answered {
+  index: number;
+  result: DeciderResult;
+  decision: RoutingDecision;
+}
+
+export class StrategyDecider implements Decider {
+  readonly id: string;
+  readonly remote: boolean;
+
+  /** Warmups in flight, so a decider that is not ready is warmed once, not once per prompt. */
+  private readonly warming = new Map<Decider, Promise<void>>();
+
+  constructor(
+    private readonly deciders: readonly Decider[],
+    private readonly strategy: StrategyConfig,
+    private readonly parse: ParseAnswers,
+  ) {
+    if (deciders.length < 2) throw new Error("StrategyDecider needs at least two deciders");
+    this.id = `${strategy.mode}(${deciders.map((d) => d.id).join(",")})`;
+    this.remote = deciders.some((d) => d.remote);
+  }
+
+  /** The combined deciders, in configured order. */
+  get members(): readonly Decider[] {
+    return this.deciders;
+  }
+
+  /** Ready when any decider can answer: the others are skipped. */
+  get isReady(): boolean {
+    return this.deciders.some((d) => d.isReady);
+  }
+
+  get model(): string | undefined {
+    const models = this.deciders.map((d) => d.model ?? d.id);
+    return models.join(this.strategy.mode === "sequential" ? " → " : " + ");
+  }
+
+  get recentLogs(): readonly string[] {
+    return this.deciders.flatMap((d) => d.recentLogs.map((line) => `[${d.id}] ${line}`));
+  }
+
+  /**
+   * Warm every decider up; resolves as soon as one is ready, so a fast
+   * decider can route while a slow one (the local model) keeps loading.
+   */
+  async warmup(signal?: AbortSignal): Promise<void> {
+    const pending = this.deciders.filter((d) => !d.isReady).map((d) => this.warm(d, signal));
+    if (pending.length === this.deciders.length) {
+      try {
+        await Promise.any(pending);
+      } catch (err) {
+        const reasons = err instanceof AggregateError ? err.errors : [err];
+        throw new DeciderError(reasons.map((e) => (e instanceof Error ? e.message : String(e))).join("; "), err);
+      }
+    } else {
+      // Something is ready already; keep loading the rest in the background.
+      for (const p of pending) p.catch(() => {});
+    }
+  }
+
+  async decide(request: DecisionRequest, signal?: AbortSignal): Promise<DeciderResult> {
+    // The router only warms us up while nothing is ready; bring back a
+    // decider that is still loading or crashed without holding this prompt.
+    for (const d of this.deciders) if (!d.isReady) this.warm(d).catch(() => {});
+
+    const started = Date.now();
+    const budget = AbortSignal.timeout(this.strategy.budgetMs);
+    const bounded = signal ? AbortSignal.any([signal, budget]) : budget;
+    const attempts: DeciderAttempt[] = this.deciders.map((d) => ({
+      decider: d.id,
+      remote: d.remote,
+      outcome: d.isReady ? "not-asked" : "not-ready",
+      used: false,
+    }));
+
+    const answered =
+      this.strategy.mode === "sequential"
+        ? await this.sequential(request, bounded, budget, attempts)
+        : await this.parallel(request, bounded, budget, attempts);
+
+    const chosen = this.choose(answered);
+    if (!chosen) {
+      const reasons = attempts.map((a) => `${a.decider}: ${a.error ?? a.outcome}`).join("; ");
+      throw new DeciderError(`no decider answered (${reasons})`);
+    }
+    attempts[chosen.index]!.used = true;
+    return {
+      ...chosen.result,
+      // What the user waited for, not only the chosen decider's share.
+      latencyMs: Date.now() - started,
+      remote: attempts.some((a) => a.remote && (a.outcome === "answered" || a.outcome === "failed")),
+      attempts,
+      ...(sumCost(attempts) !== undefined ? { costUsd: sumCost(attempts) } : {}),
+    };
+  }
+
+  stop(): void {
+    for (const d of this.deciders) d.stop();
+  }
+
+  // -------------------------------------------------------------------------
+
+  private warm(decider: Decider, signal?: AbortSignal): Promise<void> {
+    let pending = this.warming.get(decider);
+    if (!pending) {
+      pending = decider.warmup(signal).finally(() => this.warming.delete(decider));
+      this.warming.set(decider, pending);
+    }
+    return pending;
+  }
+
+  private async sequential(
+    request: DecisionRequest,
+    signal: AbortSignal,
+    budget: AbortSignal,
+    attempts: DeciderAttempt[],
+  ): Promise<Answered[]> {
+    const answered: Answered[] = [];
+    for (const [index, decider] of this.deciders.entries()) {
+      if (signal.aborted) break;
+      if (!decider.isReady) continue;
+      const answer = await this.attempt(index, decider, request, signal, budget, attempts);
+      if (!answer) continue;
+      answered.push(answer);
+      if (answer.decision.tier !== null && answer.decision.tierConfidence >= this.strategy.escalateBelow) break;
+    }
+    return answered;
+  }
+
+  private async parallel(
+    request: DecisionRequest,
+    signal: AbortSignal,
+    budget: AbortSignal,
+    attempts: DeciderAttempt[],
+  ): Promise<Answered[]> {
+    const results = await Promise.all(
+      this.deciders.map((decider, index) =>
+        decider.isReady ? this.attempt(index, decider, request, signal, budget, attempts) : Promise.resolve(null),
+      ),
+    );
+    return results.filter((r): r is Answered => r !== null);
+  }
+
+  /** Call one decider and record the outcome. Never throws. */
+  private async attempt(
+    index: number,
+    decider: Decider,
+    request: DecisionRequest,
+    signal: AbortSignal,
+    budget: AbortSignal,
+    attempts: DeciderAttempt[],
+  ): Promise<Answered | null> {
+    const started = Date.now();
+    try {
+      const result = await decider.decide(request, signal);
+      const decision = this.parse(result.answers, result.latencyMs);
+      attempts[index] = {
+        decider: decider.id,
+        remote: decider.remote,
+        outcome: "answered",
+        used: false,
+        model: result.model,
+        tier: decision.tier,
+        tierConfidence: decision.tierConfidence,
+        needsExploration: decision.needsExploration,
+        explorationConfidence: decision.explorationConfidence,
+        latencyMs: result.latencyMs,
+        ...(result.costUsd !== undefined ? { costUsd: result.costUsd } : {}),
+      };
+      return { index, result, decision };
+    } catch (err) {
+      const error = budget.aborted
+        ? `over the ${this.strategy.budgetMs} ms budget`
+        : err instanceof Error
+          ? err.message
+          : String(err);
+      attempts[index] = {
+        decider: decider.id,
+        remote: decider.remote,
+        outcome: "failed",
+        used: false,
+        latencyMs: Date.now() - started,
+        error,
+      };
+      return null;
+    }
+  }
+
+  /** The answer to route on. Answers without a usable tier only win when nothing else answered. */
+  private choose(answered: Answered[]): Answered | undefined {
+    const usable = answered.filter((a) => a.decision.tier !== null);
+    const pool = usable.length > 0 ? usable : answered;
+    if (pool.length === 0) return undefined;
+    // "first" (parallel): list order decides. Sequential keeps the most
+    // confident answer, since a later decider may be less sure than an
+    // earlier one it was asked to double-check.
+    if (this.strategy.mode === "parallel" && this.strategy.pick === "first") return pool[0];
+    return pool.reduce((best, a) => (a.decision.tierConfidence > best.decision.tierConfidence ? a : best));
+  }
+}
+
+function sumCost(attempts: DeciderAttempt[]): number | undefined {
+  const costs = attempts.map((a) => a.costUsd).filter((c): c is number => c !== undefined);
+  return costs.length > 0 ? costs.reduce((a, b) => a + b, 0) : undefined;
+}

package/src/deciders/types.ts

@@ -0,0 +1,70 @@
+/**
+ * The seam between the router and whatever classifies a prompt.
+ *
+ * A decider answers typed questions (Laya/Jev format) about a prompt. It may
+ * run locally (the Laya stdio worker) or remotely (TypeSafe's Jev); the router
+ * only sees this interface, and `parse.ts` turns the raw answers into a
+ * `RoutingDecision`.
+ */
+
+import type { DeciderAttempt, LayaQuestion } from "../types.js";
+
+/** What a decider is asked about one prompt. */
+export interface DecisionRequest {
+  /** The prompt, already truncated by the router. */
+  text: string;
+  /** Typed questions keyed by name; see `questions.ts`. */
+  questions: Readonly<Record<string, LayaQuestion>>;
+}
+
+/**
+ * Answers keyed by question name, as the decider returned them.
+ *
+ * Deliberately untyped: answers come from another process or the network and
+ * are validated by `parseDecision`.
+ */
+export type RawAnswers = Readonly<Record<string, unknown>>;
+
+/** One decider's reply to a `DecisionRequest`. */
+export interface DeciderResult {
+  /** `Decider.id` of the decider that answered. */
+  deciderId: string;
+  /** Checkpoint or remote model version that produced the answers. */
+  model: string;
+  answers: RawAnswers;
+  /** Wall time of the call as seen by the caller. */
+  latencyMs: number;
+  /** Price of the call in USD, for remote deciders that report it. */
+  costUsd?: number;
+  /** Whether the prompt left the machine for this call. Defaults to `Decider.remote`. */
+  remote?: boolean;
+  /** Each decider tried, when several were (see `strategy.ts`). */
+  attempts?: DeciderAttempt[];
+}
+
+export interface Decider {
+  /** Stable identifier, e.g. `laya-local`. */
+  readonly id: string;
+  /** Whether prompts leave the machine. */
+  readonly remote: boolean;
+  /** Whether a `decide` call can be answered now without waiting to load. */
+  readonly isReady: boolean;
+  /** Model in use once known, for status lines and log entries. */
+  readonly model: string | undefined;
+  /** Recent diagnostics, oldest first, for `/laya log`. */
+  readonly recentLogs: readonly string[];
+
+  /** Prepare the decider (load a model, check credentials). */
+  warmup(signal?: AbortSignal): Promise<void>;
+  decide(request: DecisionRequest, signal?: AbortSignal): Promise<DeciderResult>;
+  /** Release resources. The decider cannot be used afterwards. */
+  stop(): void;
+}
+
+/** Error raised by a decider that could not produce answers. */
+export class DeciderError extends Error {
+  constructor(message: string, cause?: unknown) {
+    super(message, cause === undefined ? undefined : { cause });
+    this.name = "DeciderError";
+  }
+}