waypoi 0.0.0

package/src/benchmark/config.ts

@@ -0,0 +1,337 @@
+import { promises as fs } from "fs";
+import path from "path";
+import YAML from "yaml";
+import { StoragePaths } from "../storage/files";
+import {
+  BenchmarkCliOptions,
+  BenchmarkConfigFile,
+  BenchmarkDefaults,
+  BenchmarkExecutionMode,
+  BenchmarkGateConfig,
+  BenchmarkProfileSettings,
+  EffectiveBenchmarkConfig,
+} from "./types";
+
+const DEFAULT_VERSION = 1;
+const DEFAULT_CAP_TTL_DAYS = 7;
+
+const DEFAULTS: BenchmarkDefaults = {
+  requestTimeoutMs: 120000,
+  toolTimeoutMs: 15000,
+  maxIterations: 6,
+  temperature: 0,
+  top_p: 1,
+  max_tokens: 512,
+  presence_penalty: 0,
+  frequency_penalty: 0,
+};
+
+const DEFAULT_PROFILES: Record<string, BenchmarkProfileSettings> = {
+  local: {
+    warmupRuns: 1,
+    measuredRuns: 3,
+    minScenarioPassRate: 1.0,
+  },
+  ci: {
+    warmupRuns: 2,
+    measuredRuns: 5,
+    minScenarioPassRate: 1.0,
+  },
+};
+
+const DEFAULT_GATES: BenchmarkGateConfig = {
+  hard: {
+    smokeMinSuccessRate: 1.0,
+  },
+  soft: {
+    maxP95RegressionPct: 20,
+    maxThroughputDropPct: 20,
+  },
+};
+
+export async function resolveBenchmarkConfig(
+  paths: StoragePaths,
+  cli: BenchmarkCliOptions
+): Promise<EffectiveBenchmarkConfig> {
+  const { fileConfig, configSource } = await loadConfigFile(paths, cli.configPath);
+
+  const mergedDefaults: BenchmarkDefaults = {
+    ...DEFAULTS,
+    ...(fileConfig?.defaults ?? {}),
+  };
+
+  const mergedProfiles: Record<string, BenchmarkProfileSettings> = {
+    ...DEFAULT_PROFILES,
+  };
+  for (const [profileName, profilePatch] of Object.entries(fileConfig?.profiles ?? {})) {
+    mergedProfiles[profileName] = {
+      ...(mergedProfiles[profileName] ?? DEFAULT_PROFILES.local),
+      ...profilePatch,
+    };
+  }
+
+  const selectedProfile =
+    cli.profile ?? fileConfig?.run?.profile ?? "local";
+  const profileSettings = mergedProfiles[selectedProfile];
+  if (!profileSettings) {
+    const names = Object.keys(mergedProfiles).sort().join(", ");
+    throw new Error(
+      `Unknown benchmark profile '${selectedProfile}'. Available profiles: ${names}`
+    );
+  }
+
+  const mergedGates: BenchmarkGateConfig = {
+    hard: {
+      ...DEFAULT_GATES.hard,
+      ...(fileConfig?.gates?.hard ?? {}),
+    },
+    soft: {
+      ...DEFAULT_GATES.soft,
+      ...(fileConfig?.gates?.soft ?? {}),
+    },
+  };
+
+  const resolved: EffectiveBenchmarkConfig = {
+    version: fileConfig?.version ?? DEFAULT_VERSION,
+    profile: selectedProfile,
+    defaults: validateDefaults(mergedDefaults),
+    profileSettings: validateProfileSettings(profileSettings, selectedProfile),
+    gates: validateGates(mergedGates),
+    run: {
+      suite: cli.suite ?? fileConfig?.run?.suite ?? "showcase",
+      exampleId: cli.exampleId ?? fileConfig?.run?.exampleId,
+      scenarioPath: cli.scenarioPath ?? fileConfig?.run?.scenarioPath,
+      modelOverride: cli.modelOverride ?? fileConfig?.run?.model,
+      outPath: cli.outPath ?? fileConfig?.run?.outPath,
+      baselinePath: cli.baselinePath ?? fileConfig?.run?.baselinePath,
+      executionMode: resolveExecutionMode(cli, fileConfig),
+      listExamples: cli.listExamples ?? fileConfig?.run?.listExamples ?? false,
+      updateCapCache: cli.updateCapCache ?? fileConfig?.run?.updateCapCache ?? false,
+      capTtlDays: intField(
+        cli.capTtlDays ?? fileConfig?.run?.capTtlDays ?? DEFAULT_CAP_TTL_DAYS,
+        "run.capTtlDays",
+        1
+      ),
+      temperature: optionalNumberField(
+        cli.temperature ?? fileConfig?.run?.temperature,
+        "run.temperature"
+      ),
+      top_p: optionalBoundedField(
+        cli.top_p ?? fileConfig?.run?.top_p,
+        "run.top_p",
+        0,
+        1
+      ),
+      max_tokens: optionalIntField(
+        cli.max_tokens ?? fileConfig?.run?.max_tokens,
+        "run.max_tokens",
+        1
+      ),
+      presence_penalty: optionalBoundedField(
+        cli.presence_penalty ?? fileConfig?.run?.presence_penalty,
+        "run.presence_penalty",
+        -2,
+        2
+      ),
+      frequency_penalty: optionalBoundedField(
+        cli.frequency_penalty ?? fileConfig?.run?.frequency_penalty,
+        "run.frequency_penalty",
+        -2,
+        2
+      ),
+      seed: optionalIntField(cli.seed ?? fileConfig?.run?.seed, "run.seed", 0),
+      stop: optionalStopField(cli.stop ?? fileConfig?.run?.stop, "run.stop"),
+    },
+    configSource,
+  };
+
+  return resolved;
+}
+
+async function loadConfigFile(
+  paths: StoragePaths,
+  explicitPath?: string
+): Promise<{ fileConfig?: BenchmarkConfigFile; configSource?: string }> {
+  const candidatePath = explicitPath
+    ? path.resolve(explicitPath)
+    : path.join(paths.baseDir, "benchmark.config.yaml");
+
+  try {
+    const raw = await fs.readFile(candidatePath, "utf8");
+    const parsed = parseConfigDocument(candidatePath, raw);
+    return { fileConfig: parsed, configSource: candidatePath };
+  } catch (error) {
+    const code = (error as NodeJS.ErrnoException).code;
+    if (code === "ENOENT") {
+      if (explicitPath) {
+        throw new Error(`Benchmark config not found: ${candidatePath}`);
+      }
+      return {};
+    }
+    throw error;
+  }
+}
+
+function parseConfigDocument(filePath: string, raw: string): BenchmarkConfigFile {
+  const ext = path.extname(filePath).toLowerCase();
+  try {
+    if (ext === ".json") {
+      return JSON.parse(raw) as BenchmarkConfigFile;
+    }
+    return YAML.parse(raw) as BenchmarkConfigFile;
+  } catch (error) {
+    throw new Error(
+      `Failed to parse benchmark config ${filePath}: ${(error as Error).message}`
+    );
+  }
+}
+
+function validateDefaults(defaults: BenchmarkDefaults): BenchmarkDefaults {
+  return {
+    requestTimeoutMs: intField(defaults.requestTimeoutMs, "defaults.requestTimeoutMs", 1),
+    toolTimeoutMs: intField(defaults.toolTimeoutMs, "defaults.toolTimeoutMs", 1),
+    maxIterations: intField(defaults.maxIterations, "defaults.maxIterations", 1),
+    temperature: numberField(defaults.temperature, "defaults.temperature"),
+    top_p: boundedField(defaults.top_p, "defaults.top_p", 0, 1),
+    max_tokens: intField(defaults.max_tokens, "defaults.max_tokens", 1),
+    presence_penalty: boundedField(
+      defaults.presence_penalty,
+      "defaults.presence_penalty",
+      -2,
+      2
+    ),
+    frequency_penalty: boundedField(
+      defaults.frequency_penalty,
+      "defaults.frequency_penalty",
+      -2,
+      2
+    ),
+    seed: optionalIntField(defaults.seed, "defaults.seed", 0),
+    stop: optionalStopField(defaults.stop, "defaults.stop"),
+  };
+}
+
+function resolveExecutionMode(
+  cli: BenchmarkCliOptions,
+  fileConfig?: BenchmarkConfigFile
+): BenchmarkExecutionMode {
+  const explicit = cli.executionMode ?? fileConfig?.run?.executionMode;
+  if (explicit === "showcase" || explicit === "diagnostic") {
+    return explicit;
+  }
+  const suite = cli.suite ?? fileConfig?.run?.suite ?? "showcase";
+  return suite === "showcase" ? "showcase" : "diagnostic";
+}
+
+function validateProfileSettings(
+  profile: BenchmarkProfileSettings,
+  profileName: string
+): BenchmarkProfileSettings {
+  return {
+    warmupRuns: intField(profile.warmupRuns, `profiles.${profileName}.warmupRuns`, 0),
+    measuredRuns: intField(profile.measuredRuns, `profiles.${profileName}.measuredRuns`, 1),
+    minScenarioPassRate: boundedField(
+      profile.minScenarioPassRate,
+      `profiles.${profileName}.minScenarioPassRate`,
+      0,
+      1
+    ),
+  };
+}
+
+function validateGates(gates: BenchmarkGateConfig): BenchmarkGateConfig {
+  return {
+    hard: {
+      smokeMinSuccessRate: boundedField(
+        gates.hard.smokeMinSuccessRate,
+        "gates.hard.smokeMinSuccessRate",
+        0,
+        1
+      ),
+    },
+    soft: {
+      maxP95RegressionPct: numberField(
+        gates.soft.maxP95RegressionPct,
+        "gates.soft.maxP95RegressionPct",
+        0
+      ),
+      maxThroughputDropPct: numberField(
+        gates.soft.maxThroughputDropPct,
+        "gates.soft.maxThroughputDropPct",
+        0
+      ),
+    },
+  };
+}
+
+function intField(value: number, field: string, min: number): number {
+  if (!Number.isInteger(value) || value < min) {
+    throw new Error(`${field} must be an integer >= ${min}`);
+  }
+  return value;
+}
+
+function numberField(value: number, field: string, min?: number): number {
+  if (!Number.isFinite(value)) {
+    throw new Error(`${field} must be a finite number`);
+  }
+  if (typeof min === "number" && value < min) {
+    throw new Error(`${field} must be >= ${min}`);
+  }
+  return value;
+}
+
+function boundedField(value: number, field: string, min: number, max: number): number {
+  if (!Number.isFinite(value) || value < min || value > max) {
+    throw new Error(`${field} must be between ${min} and ${max}`);
+  }
+  return value;
+}
+
+function optionalIntField(value: number | undefined, field: string, min: number): number | undefined {
+  if (value === undefined) return undefined;
+  return intField(value, field, min);
+}
+
+function optionalNumberField(value: number | undefined, field: string, min?: number): number | undefined {
+  if (value === undefined) return undefined;
+  return numberField(value, field, min);
+}
+
+function optionalBoundedField(
+  value: number | undefined,
+  field: string,
+  min: number,
+  max: number
+): number | undefined {
+  if (value === undefined) return undefined;
+  return boundedField(value, field, min, max);
+}
+
+function optionalStopField(value: string | string[] | undefined, field: string): string | string[] | undefined {
+  if (value === undefined) return undefined;
+  if (typeof value === "string") {
+    const trimmed = value.trim();
+    if (trimmed.length === 0) {
+      throw new Error(`${field} must not be empty`);
+    }
+    return trimmed;
+  }
+  if (Array.isArray(value)) {
+    if (value.length === 0) {
+      throw new Error(`${field} must include at least one stop sequence`);
+    }
+    const normalized = value.map((item, index) => {
+      if (typeof item !== "string") {
+        throw new Error(`${field}[${index}] must be a string`);
+      }
+      const trimmed = item.trim();
+      if (trimmed.length === 0) {
+        throw new Error(`${field}[${index}] must not be empty`);
+      }
+      return trimmed;
+    });
+    return normalized;
+  }
+  throw new Error(`${field} must be a string or string[]`);
+}

package/src/benchmark/gates.ts

@@ -0,0 +1,164 @@
+import { promises as fs } from "fs";
+import {
+  BenchmarkGateResults,
+  BenchmarkReport,
+  EffectiveBenchmarkConfig,
+  ScenarioResult,
+} from "./types";
+
+export async function evaluateGates(
+  report: Omit<BenchmarkReport, "gateResults">,
+  effective: EffectiveBenchmarkConfig
+): Promise<BenchmarkGateResults> {
+  const hardMessages: string[] = [];
+  const softMessages: string[] = [];
+
+  if (effective.run.suite === "smoke") {
+    const min = effective.gates.hard.smokeMinSuccessRate;
+    if (report.executed > 0 && report.successRate < min) {
+      hardMessages.push(
+        `Smoke suite success rate ${toPct(report.successRate)} is below required ${toPct(min)}.`
+      );
+    }
+  }
+
+  const minScenarioPassRate = effective.profileSettings.minScenarioPassRate;
+  const failingScenarios = report.results.filter(
+    (scenario) => scenario.status !== "skipped" && scenario.passRate < minScenarioPassRate
+  );
+  for (const scenario of failingScenarios) {
+    hardMessages.push(
+      `Scenario '${scenario.id}' pass rate ${toPct(scenario.passRate)} is below required ${toPct(minScenarioPassRate)}.`
+    );
+  }
+
+  if (effective.run.baselinePath) {
+    const baseline = await loadBaseline(effective.run.baselinePath);
+    const baselineById = new Map(
+      baseline.results
+        .map((scenario) => normalizeBaselineScenario(scenario))
+        .filter((scenario): scenario is BaselineScenario => scenario !== null)
+        .map((scenario) => [scenario.id, scenario])
+    );
+
+    for (const current of report.results) {
+      if (current.status === "skipped") {
+        continue;
+      }
+      const ref = baselineById.get(current.id);
+      if (!ref) {
+        continue;
+      }
+      const maxP95 = effective.gates.soft.maxP95RegressionPct;
+      if (ref.p95LatencyMs > 0) {
+        const p95Threshold = ref.p95LatencyMs * (1 + maxP95 / 100);
+        if (current.p95LatencyMs > p95Threshold) {
+          softMessages.push(
+            `Scenario '${current.id}' p95 latency regressed ${pctDelta(current.p95LatencyMs, ref.p95LatencyMs)} (${current.p95LatencyMs}ms vs baseline ${ref.p95LatencyMs}ms).`
+          );
+        }
+      }
+
+      const maxThroughputDrop = effective.gates.soft.maxThroughputDropPct;
+      if (ref.avgThroughputTokensPerSec > 0) {
+        const throughputThreshold = ref.avgThroughputTokensPerSec * (1 - maxThroughputDrop / 100);
+        if (current.avgThroughputTokensPerSec < throughputThreshold) {
+          softMessages.push(
+            `Scenario '${current.id}' throughput dropped ${throughputDropPct(current.avgThroughputTokensPerSec, ref.avgThroughputTokensPerSec)} (${current.avgThroughputTokensPerSec.toFixed(2)} t/s vs baseline ${ref.avgThroughputTokensPerSec.toFixed(2)} t/s).`
+          );
+        }
+      }
+    }
+  }
+
+  return {
+    hard: {
+      passed: hardMessages.length === 0,
+      messages: hardMessages,
+    },
+    soft: {
+      passed: softMessages.length === 0,
+      messages: softMessages,
+    },
+  };
+}
+
+interface BaselineScenario {
+  id: string;
+  p95LatencyMs: number;
+  avgThroughputTokensPerSec: number;
+}
+
+async function loadBaseline(pathLike: string): Promise<{ results: unknown[] }> {
+  let raw: string;
+  try {
+    raw = await fs.readFile(pathLike, "utf8");
+  } catch (error) {
+    throw new Error(
+      `Failed to read baseline file '${pathLike}': ${(error as Error).message}`
+    );
+  }
+
+  let parsed: unknown;
+  try {
+    parsed = JSON.parse(raw);
+  } catch (error) {
+    throw new Error(
+      `Failed to parse baseline file '${pathLike}' as JSON: ${(error as Error).message}`
+    );
+  }
+
+  if (!parsed || typeof parsed !== "object" || !Array.isArray((parsed as { results?: unknown[] }).results)) {
+    throw new Error(
+      `Baseline file '${pathLike}' must contain a top-level 'results' array.`
+    );
+  }
+
+  return parsed as { results: unknown[] };
+}
+
+function normalizeBaselineScenario(raw: unknown): BaselineScenario | null {
+  if (!raw || typeof raw !== "object") {
+    return null;
+  }
+  const scenario = raw as Partial<ScenarioResult>;
+  if (typeof scenario.id !== "string") {
+    return null;
+  }
+  const p95LatencyMs =
+    typeof scenario.p95LatencyMs === "number" && Number.isFinite(scenario.p95LatencyMs)
+      ? scenario.p95LatencyMs
+      : 0;
+  const avgThroughputTokensPerSec =
+    typeof scenario.avgThroughputTokensPerSec === "number" &&
+    Number.isFinite(scenario.avgThroughputTokensPerSec)
+      ? scenario.avgThroughputTokensPerSec
+      : 0;
+
+  return {
+    id: scenario.id,
+    p95LatencyMs,
+    avgThroughputTokensPerSec,
+  };
+}
+
+function toPct(value: number): string {
+  return `${(value * 100).toFixed(1)}%`;
+}
+
+function pctDelta(current: number, baseline: number): string {
+  if (baseline === 0) {
+    return "n/a";
+  }
+  const delta = ((current - baseline) / baseline) * 100;
+  const sign = delta >= 0 ? "+" : "";
+  return `${sign}${delta.toFixed(1)}%`;
+}
+
+function throughputDropPct(current: number, baseline: number): string {
+  if (baseline === 0) {
+    return "n/a";
+  }
+  const delta = ((baseline - current) / baseline) * 100;
+  return `${delta.toFixed(1)}%`;
+}