waypoi 0.0.0

package/scripts/manual-test.sh

@@ -0,0 +1,11 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+BASE_URL=${BASE_URL:-http://localhost:9469}
+MODEL=${MODEL:-local-default}
+
+curl -sS "$BASE_URL/v1/models" | jq .
+
+curl -sS "$BASE_URL/v1/chat/completions" \
+  -H "Content-Type: application/json" \
+  -d '{"model":"'"$MODEL"'","messages":[{"role":"user","content":"Hello"}]}' | jq .

package/scripts/version-from-git.js

@@ -0,0 +1,23 @@
+#!/usr/bin/env node
+const { execSync } = require("child_process");
+const { writeFileSync } = require("fs");
+const { join } = require("path");
+
+const root = join(__dirname, "..");
+
+let version;
+try {
+  version = execSync("git describe --tags --abbrev=0", {
+    cwd: root,
+    encoding: "utf8",
+  }).trim();
+  if (version.startsWith("v")) version = version.slice(1);
+} catch {
+  version = "0.0.0";
+}
+
+const content = `// Auto-generated from git tags. Do not edit manually.
+export const VERSION = ${JSON.stringify(version)};
+`;
+
+writeFileSync(join(root, "src", "version.ts"), content, "utf8");

package/src/benchmark/artifacts.ts

@@ -0,0 +1,149 @@
+import { promises as fs } from "fs";
+import path from "path";
+import { StoragePaths } from "../storage/files";
+import { BenchmarkReport } from "./types";
+
+export async function writeBenchmarkArtifacts(
+  paths: StoragePaths,
+  report: BenchmarkReport,
+  outPath?: string
+): Promise<{ jsonPath: string; textPath: string }> {
+  const timestamp = timestampForFileName();
+  const defaultDir = path.join(paths.baseDir, "benchmarks");
+  const destination = outPath && outPath.trim().length > 0 ? path.resolve(outPath) : defaultDir;
+
+  let jsonPath: string;
+  let textPath: string;
+
+  if (destination.endsWith(".json")) {
+    jsonPath = destination;
+    textPath = destination.replace(/\.json$/i, ".txt");
+  } else if (destination.endsWith(".txt")) {
+    textPath = destination;
+    jsonPath = destination.replace(/\.txt$/i, ".json");
+  } else {
+    jsonPath = path.join(destination, `bench-${timestamp}.json`);
+    textPath = path.join(destination, `bench-${timestamp}.txt`);
+  }
+
+  await fs.mkdir(path.dirname(jsonPath), { recursive: true });
+  await fs.mkdir(path.dirname(textPath), { recursive: true });
+
+  await fs.writeFile(jsonPath, JSON.stringify(report, null, 2), "utf8");
+  await fs.writeFile(textPath, renderTextSummary(report), "utf8");
+
+  return { jsonPath, textPath };
+}
+
+function renderTextSummary(report: BenchmarkReport): string {
+  const lines: string[] = [];
+
+  lines.push("Waypoi Benchmark Report");
+  lines.push(`Run ID: ${report.id}`);
+  lines.push(`Created: ${report.createdAt}`);
+  lines.push(`Profile: ${report.profile}`);
+  lines.push(`Mode: ${report.executionMode}`);
+  if (report.suite) lines.push(`Suite: ${report.suite}`);
+  if (report.exampleId) lines.push(`Example: ${report.exampleId}`);
+  if (report.scenarioPath) lines.push(`Scenario File: ${report.scenarioPath}`);
+  if (report.modelOverride) lines.push(`Model Override: ${report.modelOverride}`);
+  lines.push("");
+
+  lines.push("Overall");
+  lines.push(`- Scenarios: ${report.total}`);
+  lines.push(`- Executed: ${report.executed}`);
+  lines.push(`- Skipped: ${report.skipped}`);
+  lines.push(`- Success: ${report.succeeded}`);
+  lines.push(`- Failed: ${report.failed}`);
+  lines.push(`- Success Rate: ${(report.successRate * 100).toFixed(1)}%`);
+  lines.push(`- Avg Latency: ${report.avgLatencyMs}ms`);
+  lines.push(`- P95 Latency: ${report.p95LatencyMs}ms`);
+  lines.push(`- Tokens: ${report.totalTokens}`);
+  lines.push(`- Tool Calls: ${report.totalToolCalls}`);
+  lines.push(`- Throughput: ${report.avgThroughputTokensPerSec.toFixed(2)} tokens/s`);
+  lines.push("");
+
+  lines.push("Gates");
+  lines.push(`- Hard: ${report.gateResults.hard.passed ? "PASS" : "FAIL"}`);
+  for (const message of report.gateResults.hard.messages) {
+    lines.push(`  - ${message}`);
+  }
+  lines.push(`- Soft: ${report.gateResults.soft.passed ? "PASS" : "WARN"}`);
+  for (const message of report.gateResults.soft.messages) {
+    lines.push(`  - ${message}`);
+  }
+  lines.push("");
+
+  lines.push("By Mode");
+  for (const [mode, summary] of Object.entries(report.modeSummary)) {
+    lines.push(
+      `- ${mode}: total=${summary.total} executed=${summary.executed} skipped=${summary.skipped} passed=${summary.passed} failed=${summary.failed}`
+    );
+  }
+  lines.push("");
+
+  lines.push("Scenario Results");
+  for (const scenario of report.results) {
+    if (scenario.status === "skipped") {
+      lines.push(`- ${scenario.id} [${scenario.mode}] SKIPPED: ${scenario.skippedReason ?? "not applicable"}`);
+      continue;
+    }
+    lines.push(
+      `- ${scenario.id} [${scenario.mode}] ${scenario.status.toUpperCase()} passRate=${(scenario.passRate * 100).toFixed(1)}% p95=${scenario.p95LatencyMs}ms tokens=${scenario.totalTokens} tools=${scenario.totalToolCalls} attempts=${scenario.candidateAttempts} failovers=${scenario.failovers} rateLimitSwitches=${scenario.rateLimitSwitches}`
+    );
+    for (const reason of scenario.errorReasons) {
+      lines.push(`  - ${reason}`);
+    }
+  }
+
+  if (report.scenarioDetails.length > 0) {
+    lines.push("");
+    lines.push("Showcase Details");
+    for (const detail of report.scenarioDetails) {
+      lines.push(`- ${detail.example?.title ?? detail.id} [${detail.status}]`);
+      lines.push(`  model=${detail.model}`);
+      lines.push(`  verdict=${detail.verdict}`);
+      if (detail.usedToolNames.length > 0) {
+        lines.push(`  tools=${detail.usedToolNames.join(", ")}`);
+      }
+      if (detail.finalResponsePreview) {
+        lines.push(`  final=${detail.finalResponsePreview}`);
+      }
+    }
+  }
+
+  if (report.topFailureReasons.length > 0) {
+    lines.push("");
+    lines.push("Top Failure Reasons");
+    for (const item of report.topFailureReasons) {
+      lines.push(`- ${item.reason} (${item.count})`);
+    }
+  }
+
+  if (report.warnings.length > 0) {
+    lines.push("");
+    lines.push("Warnings");
+    for (const warning of report.warnings) {
+      lines.push(`- ${warning}`);
+    }
+  }
+
+  if (report.capabilityMatrix && report.capabilityMatrix.models.length > 0) {
+    lines.push("");
+    lines.push("Capability Matrix");
+    lines.push(`- TTL Days: ${report.capabilityMatrix.ttlDays}`);
+    for (const model of report.capabilityMatrix.models) {
+      lines.push(`- ${model.model} (${model.freshness}) verified=${model.lastVerifiedAt}`);
+      lines.push(
+        `  chat=${model.findings.chat_basic.status} tools=${model.findings.chat_tool_calls.status} embed=${model.findings.embeddings.status} image=${model.findings.images_generation.status} audio_in=${model.findings.audio_transcription.status} audio_out=${model.findings.audio_speech.status}`
+      );
+    }
+  }
+
+  lines.push("");
+  return `${lines.join("\n")}\n`;
+}
+
+function timestampForFileName(): string {
+  return new Date().toISOString().replace(/[:.]/g, "-");
+}

package/src/benchmark/capabilityClassifier.ts

@@ -0,0 +1,99 @@
+import { BenchmarkCapabilityStatus } from "./types";
+
+const UNSUPPORTED_SIGNAL_PATTERNS = [
+  /unsupported/i,
+  /not\s+supported/i,
+  /not\s+implemented/i,
+  /unknown\s+parameter/i,
+  /invalid\s+parameter/i,
+  /unrecognized\s+parameter/i,
+  /does\s+not\s+support/i,
+  /modality\s+not\s+supported/i,
+  /endpoint\s+not\s+found/i,
+  /route\s+not\s+found/i,
+];
+
+const MISCONFIGURED_SIGNAL_PATTERNS = [
+  /api\s*key/i,
+  /missing\s+api\s*key/i,
+  /authorization/i,
+  /unauthorized/i,
+  /forbidden/i,
+  /invalid\s+auth/i,
+  /invalid\s+api\s*key/i,
+  /model\s+not\s+found/i,
+  /unknown\s+model/i,
+  /no\s+such\s+model/i,
+  /credential/i,
+  /token/i,
+];
+
+const TRANSIENT_SIGNAL_PATTERNS = [
+  /timeout/i,
+  /timed\s*out/i,
+  /econnreset/i,
+  /econnrefused/i,
+  /enotfound/i,
+  /network/i,
+  /temporar/i,
+  /rate\s*limit/i,
+  /too\s+many\s+requests/i,
+];
+
+export interface CapabilityClassificationInput {
+  success: boolean;
+  statusCode?: number;
+  error?: string;
+}
+
+export function classifyCapabilityStatus(input: CapabilityClassificationInput): BenchmarkCapabilityStatus {
+  if (input.success) {
+    return "supported";
+  }
+
+  const statusCode = Number.isFinite(input.statusCode) ? input.statusCode : undefined;
+  const error = (input.error ?? "").toLowerCase();
+
+  if (statusCode === 401 || statusCode === 403) {
+    return "misconfigured";
+  }
+
+  if (statusCode === 404) {
+    if (MISCONFIGURED_SIGNAL_PATTERNS.some((pattern) => pattern.test(error))) {
+      return "misconfigured";
+    }
+    return "unsupported";
+  }
+
+  if (statusCode === 429) {
+    return "unknown";
+  }
+
+  if (statusCode !== undefined && statusCode >= 500) {
+    return "unknown";
+  }
+
+  if (statusCode === 400) {
+    if (MISCONFIGURED_SIGNAL_PATTERNS.some((pattern) => pattern.test(error))) {
+      return "misconfigured";
+    }
+    if (UNSUPPORTED_SIGNAL_PATTERNS.some((pattern) => pattern.test(error))) {
+      return "unsupported";
+    }
+    return "unknown";
+  }
+
+  if (MISCONFIGURED_SIGNAL_PATTERNS.some((pattern) => pattern.test(error))) {
+    return "misconfigured";
+  }
+
+  if (UNSUPPORTED_SIGNAL_PATTERNS.some((pattern) => pattern.test(error))) {
+    return "unsupported";
+  }
+
+  if (TRANSIENT_SIGNAL_PATTERNS.some((pattern) => pattern.test(error))) {
+    return "unknown";
+  }
+
+  return "unknown";
+}

package/src/benchmark/capabilityStore.ts

@@ -0,0 +1,174 @@
+import { createHash } from "crypto";
+import { promises as fs } from "fs";
+import path from "path";
+import { StoragePaths } from "../storage/files";
+import {
+  BenchmarkCapabilityFreshness,
+  BenchmarkCapabilityMatrix,
+  BenchmarkCapabilityStatus,
+  BenchmarkModelCapabilitySnapshot,
+} from "./types";
+
+const DEFAULT_TTL_DAYS = 7;
+
+export interface CapabilityStoreListResult {
+  generatedAt: string;
+  ttlDays: number;
+  models: BenchmarkModelCapabilitySnapshot[];
+}
+
+export function capabilityCacheDir(paths: StoragePaths): string {
+  return path.join(paths.baseDir, "capabilities");
+}
+
+export function computeConfigFingerprint(input: unknown): string {
+  const raw = JSON.stringify(input ?? {});
+  return createHash("sha256").update(raw).digest("hex").slice(0, 16);
+}
+
+export async function writeCapabilitySnapshots(
+  paths: StoragePaths,
+  snapshots: BenchmarkModelCapabilitySnapshot[]
+): Promise<void> {
+  const dir = capabilityCacheDir(paths);
+  await fs.mkdir(dir, { recursive: true });
+
+  for (const snapshot of snapshots) {
+    const filePath = path.join(dir, capabilityFileName(snapshot.providerId, snapshot.modelId));
+    await fs.writeFile(filePath, JSON.stringify(snapshot, null, 2), "utf8");
+  }
+}
+
+export async function listCapabilitySnapshots(
+  paths: StoragePaths,
+  ttlDays = DEFAULT_TTL_DAYS
+): Promise<CapabilityStoreListResult> {
+  const dir = capabilityCacheDir(paths);
+  let files: string[] = [];
+  try {
+    files = await fs.readdir(dir);
+  } catch (error) {
+    if ((error as NodeJS.ErrnoException).code === "ENOENT") {
+      return {
+        generatedAt: new Date().toISOString(),
+        ttlDays,
+        models: [],
+      };
+    }
+    throw error;
+  }
+
+  const snapshots: BenchmarkModelCapabilitySnapshot[] = [];
+  for (const file of files) {
+    if (!file.endsWith(".json")) {
+      continue;
+    }
+
+    try {
+      const raw = await fs.readFile(path.join(dir, file), "utf8");
+      const parsed = JSON.parse(raw) as Partial<BenchmarkModelCapabilitySnapshot>;
+      const normalized = normalizeSnapshot(parsed, ttlDays);
+      if (normalized) {
+        snapshots.push(normalized);
+      }
+    } catch {
+      // Ignore malformed cache entries.
+    }
+  }
+
+  snapshots.sort((a, b) => `${a.providerId}/${a.modelId}`.localeCompare(`${b.providerId}/${b.modelId}`));
+
+  return {
+    generatedAt: new Date().toISOString(),
+    ttlDays,
+    models: snapshots,
+  };
+}
+
+export async function getCapabilitySnapshotByModel(
+  paths: StoragePaths,
+  model: string,
+  ttlDays = DEFAULT_TTL_DAYS
+): Promise<BenchmarkModelCapabilitySnapshot | null> {
+  const all = await listCapabilitySnapshots(paths, ttlDays);
+  return all.models.find((item) => item.model === model || `${item.providerId}/${item.modelId}` === model) ?? null;
+}
+
+function capabilityFileName(providerId: string, modelId: string): string {
+  return `${sanitizeSegment(providerId)}__${sanitizeSegment(modelId)}.json`;
+}
+
+function sanitizeSegment(value: string): string {
+  return value.replace(/[^a-zA-Z0-9._-]/g, "_");
+}
+
+function normalizeSnapshot(
+  input: Partial<BenchmarkModelCapabilitySnapshot>,
+  ttlDays: number
+): BenchmarkModelCapabilitySnapshot | null {
+  if (!input || typeof input !== "object") {
+    return null;
+  }
+  if (!input.providerId || !input.modelId || !input.model || !input.lastVerifiedAt) {
+    return null;
+  }
+
+  const verifiedAtMs = Date.parse(input.lastVerifiedAt);
+  if (!Number.isFinite(verifiedAtMs)) {
+    return null;
+  }
+
+  const ttlMs = ttlDays * 24 * 60 * 60 * 1000;
+  const expiresAt = input.expiresAt && Number.isFinite(Date.parse(input.expiresAt))
+    ? input.expiresAt
+    : new Date(verifiedAtMs + ttlMs).toISOString();
+  const freshness = computeFreshness(expiresAt);
+
+  return {
+    model: input.model,
+    providerId: input.providerId,
+    modelId: input.modelId,
+    configFingerprint: input.configFingerprint ?? "unknown",
+    confidence: clamp01(input.confidence ?? 0),
+    lastVerifiedAt: input.lastVerifiedAt,
+    expiresAt,
+    freshness,
+    findings: (input.findings ?? {}) as BenchmarkModelCapabilitySnapshot["findings"],
+  };
+}
+
+function computeFreshness(expiresAt: string): BenchmarkCapabilityFreshness {
+  const expiryMs = Date.parse(expiresAt);
+  if (!Number.isFinite(expiryMs)) {
+    return "stale";
+  }
+  return Date.now() <= expiryMs ? "fresh" : "stale";
+}
+
+function clamp01(value: number): number {
+  if (!Number.isFinite(value)) return 0;
+  if (value < 0) return 0;
+  if (value > 1) return 1;
+  return Number(value.toFixed(3));
+}
+
+export function statusRank(status: BenchmarkCapabilityStatus): number {
+  switch (status) {
+    case "supported":
+      return 4;
+    case "unsupported":
+      return 3;
+    case "misconfigured":
+      return 2;
+    case "unknown":
+      return 1;
+  }
+}
+
+export function toCapabilityMatrix(result: CapabilityStoreListResult): BenchmarkCapabilityMatrix {
+  return {
+    generatedAt: result.generatedAt,
+    ttlDays: result.ttlDays,
+    models: result.models,
+  };
+}