waypoi 0.0.0

package/src/benchmark/types.ts

@@ -0,0 +1,442 @@
+import { ModelModality } from "../types";
+
+export type BenchmarkMode =
+  | "chat"
+  | "agent"
+  | "responses"
+  | "embeddings"
+  | "image_generation"
+  | "audio_transcription"
+  | "audio_speech"
+  | "omni_call";
+
+export type BenchmarkExecutionMode = "showcase" | "diagnostic";
+
+export type BenchmarkExampleSource = "opencode" | "builtin" | "file" | "huggingface";
+
+export type BenchmarkCapabilityKey =
+  | "chat_basic"
+  | "chat_streaming"
+  | "chat_tool_calls"
+  | "chat_vision_input"
+  | "images_generation"
+  | "images_edit"
+  | "embeddings"
+  | "audio_transcription"
+  | "audio_speech"
+  | "responses_compat";
+
+export type BenchmarkCapabilityStatus =
+  | "supported"
+  | "unsupported"
+  | "unknown"
+  | "misconfigured";
+
+export type BenchmarkCapabilityFreshness = "fresh" | "stale";
+
+export interface BenchmarkAssertions {
+  contains?: string[];
+  notContains?: string[];
+  requiredToolNames?: string[];
+  minToolCalls?: number;
+  maxToolCalls?: number;
+  maxLatencyMs?: number;
+  statusCode: number;
+  minItems?: number;
+  minVectorLength?: number;
+  minImages?: number;
+  containsText?: string[];
+  notContainsText?: string[];
+  minBytes?: number;
+  contentType?: string;
+}
+
+export interface BenchmarkScenario {
+  id: string;
+  mode: BenchmarkMode;
+  title?: string;
+  summary?: string;
+  userVisibleGoal?: string;
+  exampleSource?: BenchmarkExampleSource;
+  inputPreview?: string;
+  successCriteria?: string;
+  expectedHighlights?: string[];
+  capability?: BenchmarkCapabilityKey;
+  model?: string;
+  timeoutMs?: number;
+  requiresAvailableTools?: boolean;
+  assertions: BenchmarkAssertions;
+
+  // chat / agent
+  prompt?: string;
+  tools?: string[];
+  maxIterations?: number;
+  temperature?: number;
+  top_p?: number;
+  max_tokens?: number;
+  presence_penalty?: number;
+  frequency_penalty?: number;
+  seed?: number;
+  stop?: string | string[];
+
+  // embeddings
+  input?: string | string[];
+
+  // image generation
+  n?: number;
+  size?: string;
+
+  // audio transcription
+  audioFile?: string;
+
+  // audio speech
+  inputText?: string;
+  voice?: string;
+  response_format?: string;
+}
+
+export interface BenchmarkCliOptions {
+  suite?: string;
+  exampleId?: string;
+  scenarioPath?: string;
+  modelOverride?: string;
+  outPath?: string;
+  configPath?: string;
+  profile?: string;
+  baselinePath?: string;
+  executionMode?: BenchmarkExecutionMode;
+  listExamples?: boolean;
+  updateCapCache?: boolean;
+  capTtlDays?: number;
+  temperature?: number;
+  top_p?: number;
+  max_tokens?: number;
+  presence_penalty?: number;
+  frequency_penalty?: number;
+  seed?: number;
+  stop?: string | string[];
+}
+
+export interface BenchmarkDefaults {
+  requestTimeoutMs: number;
+  toolTimeoutMs: number;
+  maxIterations: number;
+  temperature: number;
+  top_p: number;
+  max_tokens: number;
+  presence_penalty: number;
+  frequency_penalty: number;
+  seed?: number;
+  stop?: string | string[];
+}
+
+export interface BenchmarkProfileSettings {
+  warmupRuns: number;
+  measuredRuns: number;
+  minScenarioPassRate: number;
+}
+
+export interface BenchmarkGateHardConfig {
+  smokeMinSuccessRate: number;
+}
+
+export interface BenchmarkGateSoftConfig {
+  maxP95RegressionPct: number;
+  maxThroughputDropPct: number;
+}
+
+export interface BenchmarkGateConfig {
+  hard: BenchmarkGateHardConfig;
+  soft: BenchmarkGateSoftConfig;
+}
+
+export interface BenchmarkConfigFile {
+  version?: number;
+  defaults?: Partial<BenchmarkDefaults>;
+  profiles?: Record<string, Partial<BenchmarkProfileSettings>>;
+  gates?: {
+    hard?: Partial<BenchmarkGateHardConfig>;
+    soft?: Partial<BenchmarkGateSoftConfig>;
+  };
+  run?: {
+    suite?: string;
+    exampleId?: string;
+    scenarioPath?: string;
+    model?: string;
+    outPath?: string;
+    profile?: string;
+    baselinePath?: string;
+    executionMode?: BenchmarkExecutionMode;
+    listExamples?: boolean;
+    updateCapCache?: boolean;
+    capTtlDays?: number;
+    temperature?: number;
+    top_p?: number;
+    max_tokens?: number;
+    presence_penalty?: number;
+    frequency_penalty?: number;
+    seed?: number;
+    stop?: string | string[];
+  };
+}
+
+export interface BenchmarkRunPlan {
+  suite?: string;
+  exampleId?: string;
+  scenarioPath?: string;
+  modelOverride?: string;
+  outPath?: string;
+  baselinePath?: string;
+  executionMode?: BenchmarkExecutionMode;
+  listExamples?: boolean;
+  updateCapCache?: boolean;
+  capTtlDays?: number;
+  temperature?: number;
+  top_p?: number;
+  max_tokens?: number;
+  presence_penalty?: number;
+  frequency_penalty?: number;
+  seed?: number;
+  stop?: string | string[];
+}
+
+export interface EffectiveBenchmarkConfig {
+  version: number;
+  profile: string;
+  defaults: BenchmarkDefaults;
+  profileSettings: BenchmarkProfileSettings;
+  gates: BenchmarkGateConfig;
+  run: BenchmarkRunPlan;
+  configSource?: string;
+}
+
+export interface ValidationOutcome {
+  scenarios: BenchmarkScenario[];
+  warnings: string[];
+}
+
+export interface ScenarioRunSample {
+  runIndex: number;
+  success: boolean;
+  latencyMs: number;
+  statusCode: number;
+  tokens: number;
+  toolCalls: number;
+  throughputTokensPerSec: number;
+  finalOutput: string;
+  outputPreview: string;
+  verdict: string;
+  usedToolNames: string[];
+  error?: string;
+  candidateAttempts?: number;
+  failovers?: number;
+  rateLimitSwitches?: number;
+  distinctProviders?: number;
+  distinctModels?: number;
+  audioOutputPresent?: boolean;
+}
+
+export interface ScenarioResult {
+  id: string;
+  mode: BenchmarkMode;
+  title?: string;
+  summary?: string;
+  userVisibleGoal?: string;
+  exampleSource?: BenchmarkExampleSource;
+  inputPreview?: string;
+  successCriteria?: string;
+  expectedHighlights?: string[];
+  model: string;
+  status: "passed" | "failed" | "skipped";
+  success: boolean;
+  skippedReason?: string;
+  passRate: number;
+  passedRuns: number;
+  failedRuns: number;
+  avgLatencyMs: number;
+  p50LatencyMs: number;
+  p95LatencyMs: number;
+  p99LatencyMs: number;
+  totalTokens: number;
+  totalToolCalls: number;
+  avgThroughputTokensPerSec: number;
+  candidateAttempts: number;
+  failovers: number;
+  rateLimitSwitches: number;
+  distinctProviders: number;
+  distinctModels: number;
+  audioOutputRuns: number;
+  usedToolNames: string[];
+  verdict: string;
+  errorReasons: string[];
+  outputPreview: string;
+}
+
+export interface BenchmarkScenarioSummary {
+  id: string;
+  suite: string;
+  mode: BenchmarkMode;
+  title: string;
+  summary: string;
+  userVisibleGoal: string;
+  exampleSource: BenchmarkExampleSource;
+  inputPreview: string;
+  successCriteria: string;
+  expectedHighlights: string[];
+  requiresAvailableTools: boolean;
+  model?: string;
+}
+
+export interface BenchmarkToolTraceStep {
+  kind: "tool_call" | "tool_result";
+  toolName: string;
+  toolCallId?: string;
+  argumentsText?: string;
+  contentText?: string;
+}
+
+export interface BenchmarkExchangeSummary {
+  timestamp?: string;
+  mode: BenchmarkMode;
+  model: string;
+  requestPath: string;
+  statusCode: number;
+  contentType: string;
+  requestSanitized: unknown;
+  responseSanitized: unknown;
+  requestRaw?: unknown;
+  responseRaw?: unknown;
+  requestPreview: string;
+  responsePreview: string;
+  endpointId?: string;
+  endpointName?: string;
+  upstreamModel?: string;
+  toolTrace: BenchmarkToolTraceStep[];
+}
+
+export interface BenchmarkScenarioDetail {
+  id: string;
+  suite?: string;
+  example?: BenchmarkScenarioSummary;
+  model: string;
+  status: "passed" | "failed" | "skipped";
+  verdict: string;
+  exchanges: BenchmarkExchangeSummary[];
+  finalResponsePreview: string;
+  usedToolNames: string[];
+}
+
+export interface BenchmarkGateResult {
+  passed: boolean;
+  messages: string[];
+}
+
+export interface BenchmarkGateResults {
+  hard: BenchmarkGateResult;
+  soft: BenchmarkGateResult;
+}
+
+export interface BenchmarkReport {
+  id: string;
+  createdAt: string;
+  profile: string;
+  executionMode: BenchmarkExecutionMode;
+  suite?: string;
+  exampleId?: string;
+  scenarioPath?: string;
+  modelOverride?: string;
+  configSource?: string;
+  total: number;
+  executed: number;
+  skipped: number;
+  succeeded: number;
+  failed: number;
+  successRate: number;
+  totalTokens: number;
+  totalToolCalls: number;
+  avgLatencyMs: number;
+  p50LatencyMs: number;
+  p95LatencyMs: number;
+  p99LatencyMs: number;
+  avgThroughputTokensPerSec: number;
+  modeSummary: Record<
+    BenchmarkMode,
+    { total: number; executed: number; skipped: number; passed: number; failed: number }
+  >;
+  effectiveConfig: {
+    defaults: BenchmarkDefaults;
+    profileSettings: BenchmarkProfileSettings;
+    gates: BenchmarkGateConfig;
+  };
+  results: ScenarioResult[];
+  scenarioDetails: BenchmarkScenarioDetail[];
+  scenarioRuns: Array<{ id: string; samples: ScenarioRunSample[] }>;
+  gateResults: BenchmarkGateResults;
+  warnings: string[];
+  topFailureReasons: Array<{ reason: string; count: number }>;
+  capabilityMatrix?: BenchmarkCapabilityMatrix;
+}
+
+export interface BenchmarkRunOutput {
+  report: BenchmarkReport;
+  artifactPath: string;
+  textArtifactPath: string;
+}
+
+export const BENCHMARK_MODES: BenchmarkMode[] = [
+  "chat",
+  "agent",
+  "responses",
+  "embeddings",
+  "image_generation",
+  "audio_transcription",
+  "audio_speech",
+  "omni_call",
+];
+
+export const BENCHMARK_CAPABILITY_KEYS: BenchmarkCapabilityKey[] = [
+  "chat_basic",
+  "chat_streaming",
+  "chat_tool_calls",
+  "chat_vision_input",
+  "images_generation",
+  "images_edit",
+  "embeddings",
+  "audio_transcription",
+  "audio_speech",
+  "responses_compat",
+];
+
+export interface BenchmarkCapabilityFinding {
+  capability: BenchmarkCapabilityKey;
+  status: BenchmarkCapabilityStatus;
+  confidence: number;
+  evidence: string;
+  scenarioId?: string;
+  statusCode?: number;
+  observedAt: string;
+}
+
+export interface BenchmarkModelCapabilitySnapshot {
+  model: string;
+  providerId: string;
+  modelId: string;
+  configFingerprint: string;
+  confidence: number;
+  lastVerifiedAt: string;
+  expiresAt: string;
+  freshness: BenchmarkCapabilityFreshness;
+  findings: Record<BenchmarkCapabilityKey, BenchmarkCapabilityFinding>;
+}
+
+export interface BenchmarkCapabilityMatrix {
+  generatedAt: string;
+  ttlDays: number;
+  models: BenchmarkModelCapabilitySnapshot[];
+}
+
+export interface BenchmarkModeRequirements {
+  requiredInput: ModelModality[];
+  requiredOutput: ModelModality[];
+  preferredEndpointType?: "llm" | "diffusion" | "audio" | "embedding";
+}

package/src/config.ts

@@ -0,0 +1,44 @@
+import path from "path";
+
+function resolveBinaryName(): string {
+  if (process.argv.length >= 2) {
+    const bin = path.basename(process.argv[1]);
+    if (bin === "waypoi" || bin === "waypoi-dev") return bin;
+  }
+  if (process.env.WAYPOI_DEV === "1") return "waypoi-dev";
+  return "waypoi";
+}
+
+function isDev(): boolean {
+  return resolveBinaryName() === "waypoi-dev";
+}
+
+function resolvePort(): number {
+  if (process.env.PORT) return Number(process.env.PORT);
+  return isDev() ? 9470 : 9469;
+}
+
+function resolveStorageDir(): string {
+  if (process.env.WAYPOI_DIR) return process.env.WAYPOI_DIR;
+  if (isDev()) {
+    const home = process.env.HOME ?? process.env.USERPROFILE ?? ".";
+    return path.join(home, ".config", "waypoi-dev");
+  }
+  return "";
+}
+
+function resolvePidFileName(): string {
+  return isDev() ? "waypoi-dev.pid" : "waypoi.pid";
+}
+
+function resolveAppName(): string {
+  return isDev() ? "waypoi-dev" : "waypoi";
+}
+
+export const appConfig = {
+  isDev: isDev(),
+  port: resolvePort(),
+  appName: resolveAppName(),
+  storageDirOverride: resolveStorageDir(),
+  pidFileName: resolvePidFileName(),
+};

package/src/index.ts

@@ -0,0 +1,195 @@
+import Fastify from "fastify";
+import { registerChatRoutes } from "./routes/chat";
+import { registerEmbeddingsRoutes } from "./routes/embeddings";
+import { registerModelsRoutes } from "./routes/models";
+import { registerAdminRoutes } from "./routes/admin";
+import { registerImageRoutes } from "./routes/images";
+import { registerVideoRoutes } from "./routes/videos";
+import { registerAudioRoutes } from "./routes/audio";
+import { registerResponsesRoutes } from "./routes/responses";
+import { registerStatsRoutes } from "./routes/stats";
+import { registerSessionRoutes } from "./routes/sessions";
+import { registerMcpRoutes } from "./routes/mcp";
+import { closeMcpServiceRoutes, registerMcpServiceRoutes } from "./routes/mcpService";
+import { registerUiRoutes } from "./routes/ui";
+import { registerRequestStatsMiddleware } from "./middleware/requestStats";
+import { registerRequestCaptureMiddleware } from "./middleware/requestCapture";
+import { registerAuthHooks, loadAuthConfig, updateAuthConfig } from "./middleware/auth";
+import { startHealthChecker, stopHealthChecker } from "./workers/healthChecker";
+import { startStatsRotation, stopStatsRotation } from "./workers/statsRotation";
+import { startConfigWatcher, stopConfigWatcher } from "./workers/configWatcher";
+import { startCaptureRetentionWorker, stopCaptureRetentionWorker } from "./workers/captureRetention";
+import { ensureStorageDir, resolveStoragePaths } from "./storage/files";
+import { invalidateConfigCache } from "./storage/repositories";
+import { discoverAllTools, disconnectAllServers, summarizeMcpError, discoverBuiltinTools } from "./mcp/discovery";
+import { listProviders } from "./providers/repository";
+import { listModelsForApi } from "./providers/modelRegistry";
+import { listPools } from "./pools/repository";
+import { rebuildDefaultPools } from "./pools/builder";
+import { appConfig } from "./config";
+import { VERSION } from "./version.js";
+
+if (!process.env.WAYPOI_DIR && appConfig.storageDirOverride) {
+  process.env.WAYPOI_DIR = appConfig.storageDirOverride;
+}
+
+const PORT = appConfig.port;
+const ADMIN_TOKEN = process.env.ADMIN_TOKEN;
+const APP_NAME = appConfig.appName;
+
+async function start(): Promise<void> {
+  const app = Fastify({ logger: true });
+  const paths = resolveStoragePaths();
+  await ensureStorageDir(paths);
+  try {
+    await rebuildDefaultPools(paths);
+  } catch (error) {
+    console.error(`[${APP_NAME}] Failed to rebuild smart pool: ${(error as Error).message}`);
+    if (process.env.WAYPOI_DEBUG_ERRORS === "1") {
+      console.error(error);
+    }
+  }
+
+  // Register middleware
+  await registerRequestStatsMiddleware(app, paths);
+  await registerRequestCaptureMiddleware(app, paths);
+  
+  // Register auth hooks (no-op by default, enable via config.authEnabled)
+  await registerAuthHooks(app, paths, ["/admin", "/ui"]);
+
+  // OpenAI-compatible routes
+  await registerChatRoutes(app, paths);
+  await registerEmbeddingsRoutes(app, paths);
+  await registerModelsRoutes(app, paths);
+  await registerImageRoutes(app, paths);
+  await registerVideoRoutes(app, paths);
+  await registerAudioRoutes(app, paths);
+  await registerResponsesRoutes(app, paths);
+  
+  // Admin routes
+  await registerAdminRoutes(app, paths, { adminToken: ADMIN_TOKEN, version: VERSION });
+  await registerStatsRoutes(app, paths);
+  await registerSessionRoutes(app);
+  await registerMcpRoutes(app, paths);
+  await registerMcpServiceRoutes(app, paths);
+  
+  // UI routes (serve React frontend)
+  await registerUiRoutes(app);
+
+  // Start background workers
+  startHealthChecker(paths);
+  startStatsRotation(paths);
+  startCaptureRetentionWorker(paths);
+  
+  // Auto-connect to enabled MCP servers and discover tools
+  discoverAllTools(paths).then((tools) => {
+    if (tools.length > 0) {
+      console.log(`[${APP_NAME}] Connected to MCP servers, discovered ${tools.length} tools`);
+    }
+  }).catch((error) => {
+    console.error(`[${APP_NAME}] Failed to auto-connect to MCP servers: ${summarizeMcpError(error)}`);
+    if (process.env.WAYPOI_DEBUG_ERRORS === "1") {
+      console.error(error);
+    }
+  });
+
+  // Print model capability source summary for operators.
+  try {
+    const models = await listModelsForApi(paths);
+    const configured = models.filter((model) => model.capabilities.source === "configured").length;
+    const inferred = models.filter((model) => model.capabilities.source === "inferred").length;
+    const enabledModels = models.filter((model) => model.enabled).length;
+    console.log(
+      `[${APP_NAME}] Models: total=${models.length}, enabled=${enabledModels}, configured=${configured}, inferred=${inferred}`
+    );
+  } catch (error) {
+    console.error(`[${APP_NAME}] Failed to summarize model capabilities: ${(error as Error).message}`);
+    if (process.env.WAYPOI_DEBUG_ERRORS === "1") {
+      console.error(error);
+    }
+  }
+
+  try {
+    const providers = await listProviders(paths);
+    const pools = await listPools(paths);
+    const enabledProviders = providers.filter((provider) => provider.enabled).length;
+    const byProtocol = providers.reduce<Record<string, number>>((acc, provider) => {
+      const protocol = provider.protocolRaw ?? provider.protocol;
+      acc[protocol] = (acc[protocol] ?? 0) + 1;
+      return acc;
+    }, {});
+    console.log(
+      `[${APP_NAME}] Providers: total=${providers.length}, enabled=${enabledProviders}, pools=${pools.length}`
+    );
+    console.log(`[${APP_NAME}] Provider protocols: ${JSON.stringify(byProtocol)}`);
+  } catch (error) {
+    console.error(`[${APP_NAME}] Failed to summarize providers: ${(error as Error).message}`);
+    if (process.env.WAYPOI_DEBUG_ERRORS === "1") {
+      console.error(error);
+    }
+  }
+  
+  // Start config watcher for hot-reload
+  const configWatcher = startConfigWatcher(paths);
+  configWatcher.on("config:updated", async () => {
+    invalidateConfigCache();
+    // Reload auth config on config change
+    const authConfig = await loadAuthConfig(paths);
+    updateAuthConfig(authConfig);
+    console.log("[${APP_NAME}] Config reloaded - no restart needed");
+  });
+
+  // Graceful shutdown
+  let shuttingDown = false;
+  const shutdown = async (signal: string) => {
+    if (shuttingDown) {
+      console.log(`\n[${signal}] Forced exit.`);
+      process.exit(1);
+    }
+    shuttingDown = true;
+    console.log(`\n[${signal}] Shutting down gracefully...`);
+
+    // Force exit after 5 seconds if graceful shutdown stalls
+    const forceTimer = setTimeout(() => {
+      console.error("[shutdown] Timed out, forcing exit.");
+      process.exit(1);
+    }, 5_000);
+    forceTimer.unref();
+
+    stopConfigWatcher();
+    stopHealthChecker();
+    stopStatsRotation();
+    stopCaptureRetentionWorker();
+    await closeMcpServiceRoutes();
+    await disconnectAllServers();
+    await app.close();
+    clearTimeout(forceTimer);
+    process.exit(0);
+  };
+
+  process.on("SIGTERM", () => shutdown("SIGTERM"));
+  process.on("SIGINT", () => shutdown("SIGINT"));
+
+  await app.listen({ port: PORT, host: "0.0.0.0" });
+  
+  // Auto-connect to the built-in /mcp endpoint so playground tools are available by default
+  discoverBuiltinTools(paths, `http://localhost:${PORT}/mcp`).then((tools) => {
+    console.log(`[${APP_NAME}] Built-in MCP connected: ${tools.map((t) => t.name).join(", ") || "no tools"}`);
+  }).catch((error) => {
+    console.error(`[${APP_NAME}] Failed to connect to built-in MCP: ${summarizeMcpError(error)}`);
+    if (process.env.WAYPOI_DEBUG_ERRORS === "1") {
+      console.error(error);
+    }
+  });
+
+  console.log(`\n🚀 ${APP_NAME} running on http://localhost:${PORT}`);
+  console.log(`   Endpoints: /v1/chat/completions, /v1/embeddings, /v1/images/*, /v1/videos/*, /v1/audio/*`);
+  console.log(`   MCP Service: /mcp`);
+  console.log(`   Admin: /admin/*, /admin/stats, /admin/sessions, /admin/mcp, /admin/benchmarks/runs`);
+  console.log(`   UI: http://localhost:${PORT}/ui\n`);
+}
+
+start().catch((error) => {
+  console.error(error);
+  process.exit(1);
+});