waypoi 0.0.0

package/src/workers/configWatcher.ts

@@ -0,0 +1,115 @@
+import { watch, FSWatcher } from "fs";
+import { EventEmitter } from "events";
+import { StoragePaths } from "../storage/files";
+
+/**
+ * Config Watcher
+ * 
+ * Watches config.yaml for changes and emits events when the file is modified.
+ * Uses debouncing to handle rapid successive changes (common with text editors).
+ */
+
+export interface ConfigWatcher extends EventEmitter {
+  on(event: "config:updated", listener: () => void): this;
+  on(event: "error", listener: (error: Error) => void): this;
+  emit(event: "config:updated"): boolean;
+  emit(event: "error", error: Error): boolean;
+  stop(): void;
+}
+
+const DEBOUNCE_MS = 500;
+
+class ConfigWatcherImpl extends EventEmitter implements ConfigWatcher {
+  private watcher: FSWatcher | null = null;
+  private debounceTimer: NodeJS.Timeout | null = null;
+  private isWatching = false;
+  private configPath: string;
+
+  constructor(configPath: string) {
+    super();
+    this.configPath = configPath;
+  }
+
+  start(): void {
+    if (this.isWatching) return;
+
+    try {
+      this.watcher = watch(this.configPath, (eventType) => {
+        // Handle both 'change' and 'rename' events
+        // Some editors do atomic saves (write temp → rename)
+        if (eventType === "change" || eventType === "rename") {
+          this.debouncedEmit();
+        }
+      });
+
+      this.watcher.on("error", (error) => {
+        this.emit("error", error);
+      });
+
+      this.isWatching = true;
+      console.log("[config-watcher] Watching for config changes");
+    } catch (error) {
+      // File might not exist yet, that's okay
+      console.log("[config-watcher] Config file not found, will retry on next access");
+    }
+  }
+
+  private debouncedEmit(): void {
+    if (this.debounceTimer) {
+      clearTimeout(this.debounceTimer);
+    }
+
+    this.debounceTimer = setTimeout(() => {
+      console.log("[config-watcher] Config file changed, triggering reload");
+      this.emit("config:updated");
+      this.debounceTimer = null;
+    }, DEBOUNCE_MS);
+  }
+
+  stop(): void {
+    if (this.debounceTimer) {
+      clearTimeout(this.debounceTimer);
+      this.debounceTimer = null;
+    }
+    if (this.watcher) {
+      this.watcher.close();
+      this.watcher = null;
+    }
+    this.isWatching = false;
+    console.log("[config-watcher] Stopped watching");
+  }
+}
+
+let globalWatcher: ConfigWatcher | null = null;
+
+/**
+ * Start watching the config file for changes.
+ * Returns an EventEmitter that emits "config:updated" when the file changes.
+ */
+export function startConfigWatcher(paths: StoragePaths): ConfigWatcher {
+  if (globalWatcher) {
+    return globalWatcher;
+  }
+
+  const watcher = new ConfigWatcherImpl(paths.configPath);
+  watcher.start();
+  globalWatcher = watcher;
+  return watcher;
+}
+
+/**
+ * Stop the config watcher.
+ */
+export function stopConfigWatcher(): void {
+  if (globalWatcher) {
+    globalWatcher.stop();
+    globalWatcher = null;
+  }
+}
+
+/**
+ * Get the current config watcher instance (if running).
+ */
+export function getConfigWatcher(): ConfigWatcher | null {
+  return globalWatcher;
+}

package/src/workers/healthChecker.ts

@@ -0,0 +1,22 @@
+import { StoragePaths } from "../storage/files";
+import { probeProviderModels } from "../providers/health";
+
+let healthTimer: NodeJS.Timeout | null = null;
+
+export function startHealthChecker(paths: StoragePaths): void {
+  const intervalMs = 30_000;
+  const run = async () => {
+    await probeProviderModels(paths);
+  };
+
+  healthTimer = setInterval(run, intervalMs);
+  healthTimer.unref();
+  void run();
+}
+
+export function stopHealthChecker(): void {
+  if (healthTimer) {
+    clearInterval(healthTimer);
+    healthTimer = null;
+  }
+}

package/src/workers/statsRotation.ts

@@ -0,0 +1,49 @@
+import { rotateStats } from "../storage/statsRepository";
+import { StoragePaths } from "../storage/files";
+
+/**
+ * Stats Rotation Worker
+ * 
+ * Periodically cleans up stats files older than retention period.
+ * Runs daily by default.
+ */
+
+const ROTATION_INTERVAL_MS = 24 * 60 * 60 * 1000; // 24 hours
+const DEFAULT_RETENTION_DAYS = 30;
+
+let rotationTimer: NodeJS.Timeout | null = null;
+
+export function startStatsRotation(
+  paths: StoragePaths,
+  retentionDays: number = DEFAULT_RETENTION_DAYS
+): void {
+  if (rotationTimer) {
+    return; // Already running
+  }
+
+  async function runRotation(): Promise<void> {
+    try {
+      const deleted = await rotateStats(paths, retentionDays);
+      if (deleted > 0) {
+        console.log(`[stats-rotation] Deleted ${deleted} stats file(s) older than ${retentionDays} days`);
+      }
+    } catch (error) {
+      console.error("[stats-rotation] Error rotating stats:", error);
+    }
+  }
+
+  // Run immediately on startup, then periodically
+  runRotation();
+  
+  rotationTimer = setInterval(runRotation, ROTATION_INTERVAL_MS);
+  
+  // Prevent timer from keeping process alive
+  rotationTimer.unref();
+}
+
+export function stopStatsRotation(): void {
+  if (rotationTimer) {
+    clearInterval(rotationTimer);
+    rotationTimer = null;
+  }
+}

package/tests/benchmarkAdminRoutes.test.ts

@@ -0,0 +1,82 @@
+import test from "node:test";
+import assert from "node:assert/strict";
+import path from "path";
+import { promises as fs } from "fs";
+import Fastify from "fastify";
+import { registerAdminRoutes } from "../src/routes/admin";
+import type { StoragePaths } from "../src/storage/files";
+
+function makePaths(baseDir: string): StoragePaths {
+  return {
+    baseDir,
+    configPath: path.join(baseDir, "config.yaml"),
+    healthPath: path.join(baseDir, "health.json"),
+    providerHealthPath: path.join(baseDir, "providers_health.json"),
+    requestLogPath: path.join(baseDir, "request_logs.jsonl"),
+    providersPath: path.join(baseDir, "providers.json"),
+    poolsPath: path.join(baseDir, "pools.json"),
+    poolStatePath: path.join(baseDir, "pool_state.json"),
+  };
+}
+
+async function makeWorkspaceTempDir(prefix: string): Promise<string> {
+  const base = path.join(process.cwd(), "tmp");
+  await fs.mkdir(base, { recursive: true });
+  return fs.mkdtemp(path.join(base, prefix));
+}
+
+test("/admin/benchmarks/runs forwards execution + tuning fields", async () => {
+  const baseDir = await makeWorkspaceTempDir("waypoi-benchmark-admin-test-");
+  const paths = makePaths(baseDir);
+  const app = Fastify();
+  await registerAdminRoutes(app, paths, { adminToken: "test-token", version: "0.0.0" });
+
+  const res = await app.inject({
+    method: "POST",
+    url: "/admin/benchmarks/runs",
+    headers: {
+      authorization: "Bearer test-token",
+      "content-type": "application/json",
+    },
+    payload: {
+      suite: "showcase",
+      exampleId: "showcase-tinyqa-001",
+      executionMode: "showcase",
+      profile: "local",
+      temperature: 0.25,
+      top_p: 0.8,
+      max_tokens: 128,
+      presence_penalty: 0.3,
+      frequency_penalty: -0.2,
+      seed: 7,
+      stop: ["END", "STOP"],
+    },
+  });
+
+  assert.equal(res.statusCode, 202);
+  const body = res.json() as {
+    request: {
+      exampleId?: string;
+      executionMode?: string;
+      temperature?: number;
+      top_p?: number;
+      max_tokens?: number;
+      presence_penalty?: number;
+      frequency_penalty?: number;
+      seed?: number;
+      stop?: string[];
+    };
+  };
+
+  assert.equal(body.request.exampleId, "showcase-tinyqa-001");
+  assert.equal(body.request.executionMode, "showcase");
+  assert.equal(body.request.temperature, 0.25);
+  assert.equal(body.request.top_p, 0.8);
+  assert.equal(body.request.max_tokens, 128);
+  assert.equal(body.request.presence_penalty, 0.3);
+  assert.equal(body.request.frequency_penalty, -0.2);
+  assert.equal(body.request.seed, 7);
+  assert.deepEqual(body.request.stop, ["END", "STOP"]);
+
+  await app.close();
+});

package/tests/benchmarkBasics.test.ts

@@ -0,0 +1,116 @@
+import test from 'node:test'
+import assert from 'node:assert/strict'
+import os from 'os'
+import path from 'path'
+import { promises as fs } from 'fs'
+import { resolveBenchmarkConfig } from '../src/benchmark/config'
+import { validateScenarioCollection } from '../src/benchmark/schema'
+import { listBuiltInSuites, listSuiteExamples } from '../src/benchmark/suites'
+import type { StoragePaths } from '../src/storage/files'
+
+function makePaths(baseDir: string): StoragePaths {
+  return {
+    baseDir,
+    configPath: path.join(baseDir, 'config.yaml'),
+    healthPath: path.join(baseDir, 'health.json'),
+    providerHealthPath: path.join(baseDir, 'providers_health.json'),
+    requestLogPath: path.join(baseDir, 'request_logs.jsonl'),
+    providersPath: path.join(baseDir, 'providers.json'),
+    poolsPath: path.join(baseDir, 'pools.json'),
+    poolStatePath: path.join(baseDir, 'pool_state.json'),
+  }
+}
+
+test('benchmark config defaults to showcase execution mode', async () => {
+  const baseDir = await fs.mkdtemp(path.join(os.tmpdir(), 'waypoi-bench-config-'))
+  const resolved = await resolveBenchmarkConfig(makePaths(baseDir), {})
+  assert.equal(resolved.run.suite, 'showcase')
+  assert.equal(resolved.run.executionMode, 'showcase')
+  assert.equal(resolved.defaults.maxIterations, 6)
+  assert.equal(resolved.defaults.top_p, 1)
+  assert.equal(resolved.defaults.presence_penalty, 0)
+  assert.equal(resolved.defaults.frequency_penalty, 0)
+})
+
+test('showcase example catalog is tiny-qa-backed and capability suite has no concurrent probes', () => {
+  const suites = listBuiltInSuites()
+  assert.ok(suites.includes('showcase'))
+
+  const showcase = listSuiteExamples('showcase')
+  assert.ok(showcase.length >= 50)
+  assert.ok(showcase.every((example) => example.exampleSource === 'huggingface'))
+  assert.ok(showcase.some((example) => example.id === 'showcase-tinyqa-001'))
+  assert.ok(showcase.every((example) => example.mode === 'chat'))
+
+  const capabilityIds = listSuiteExamples('capabilities').map((example) => example.id)
+  assert.ok(!capabilityIds.some((id) => id.includes('concurrent') || id.includes('under_load')))
+})
+
+test('benchmark config accepts run-level generation overrides', async () => {
+  const baseDir = await fs.mkdtemp(path.join(os.tmpdir(), 'waypoi-bench-config-'))
+  const resolved = await resolveBenchmarkConfig(makePaths(baseDir), {
+    temperature: 0.4,
+    top_p: 0.8,
+    max_tokens: 256,
+    presence_penalty: 0.3,
+    frequency_penalty: -0.4,
+    seed: 9,
+    stop: ['END', 'STOP'],
+  })
+
+  assert.equal(resolved.run.temperature, 0.4)
+  assert.equal(resolved.run.top_p, 0.8)
+  assert.equal(resolved.run.max_tokens, 256)
+  assert.equal(resolved.run.presence_penalty, 0.3)
+  assert.equal(resolved.run.frequency_penalty, -0.4)
+  assert.equal(resolved.run.seed, 9)
+  assert.deepEqual(resolved.run.stop, ['END', 'STOP'])
+})
+
+test('scenario validation accepts showcase metadata, responses mode, and tool-name assertions', () => {
+  const outcome = validateScenarioCollection(
+    [
+      {
+        id: 'responses-demo',
+        mode: 'responses',
+        title: 'Responses Demo',
+        summary: 'Compatibility example',
+        userVisibleGoal: 'Show Responses API behavior',
+        exampleSource: 'huggingface',
+        inputPreview: 'Say hello from responses.',
+        successCriteria: 'Returns HTTP 200',
+        expectedHighlights: ['wire request', 'output_text'],
+        prompt: 'Say hello from responses.',
+        temperature: 0.3,
+        top_p: 0.9,
+        max_tokens: 128,
+        presence_penalty: 0.2,
+        frequency_penalty: -0.1,
+        seed: 42,
+        stop: ['END'],
+        assertions: {
+          statusCode: 200,
+        },
+      },
+      {
+        id: 'agent-demo',
+        mode: 'agent',
+        prompt: 'Use the weather tool.',
+        requiresAvailableTools: true,
+        assertions: {
+          statusCode: 200,
+          minToolCalls: 1,
+          requiredToolNames: ['weather'],
+        },
+      },
+    ],
+    'inline'
+  )
+
+  assert.equal(outcome.scenarios.length, 2)
+  assert.equal(outcome.scenarios[0].mode, 'responses')
+  assert.equal(outcome.scenarios[0].top_p, 0.9)
+  assert.deepEqual(outcome.scenarios[0].stop, ['END'])
+  assert.deepEqual(outcome.scenarios[1].assertions.requiredToolNames, ['weather'])
+  assert.equal(outcome.scenarios[1].requiresAvailableTools, true)
+})