waypoi 0.0.0

package/dist/src/benchmark/config.js

@@ -0,0 +1,238 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.resolveBenchmarkConfig = resolveBenchmarkConfig;
+const fs_1 = require("fs");
+const path_1 = __importDefault(require("path"));
+const yaml_1 = __importDefault(require("yaml"));
+const DEFAULT_VERSION = 1;
+const DEFAULT_CAP_TTL_DAYS = 7;
+const DEFAULTS = {
+    requestTimeoutMs: 120000,
+    toolTimeoutMs: 15000,
+    maxIterations: 6,
+    temperature: 0,
+    top_p: 1,
+    max_tokens: 512,
+    presence_penalty: 0,
+    frequency_penalty: 0,
+};
+const DEFAULT_PROFILES = {
+    local: {
+        warmupRuns: 1,
+        measuredRuns: 3,
+        minScenarioPassRate: 1.0,
+    },
+    ci: {
+        warmupRuns: 2,
+        measuredRuns: 5,
+        minScenarioPassRate: 1.0,
+    },
+};
+const DEFAULT_GATES = {
+    hard: {
+        smokeMinSuccessRate: 1.0,
+    },
+    soft: {
+        maxP95RegressionPct: 20,
+        maxThroughputDropPct: 20,
+    },
+};
+async function resolveBenchmarkConfig(paths, cli) {
+    const { fileConfig, configSource } = await loadConfigFile(paths, cli.configPath);
+    const mergedDefaults = {
+        ...DEFAULTS,
+        ...(fileConfig?.defaults ?? {}),
+    };
+    const mergedProfiles = {
+        ...DEFAULT_PROFILES,
+    };
+    for (const [profileName, profilePatch] of Object.entries(fileConfig?.profiles ?? {})) {
+        mergedProfiles[profileName] = {
+            ...(mergedProfiles[profileName] ?? DEFAULT_PROFILES.local),
+            ...profilePatch,
+        };
+    }
+    const selectedProfile = cli.profile ?? fileConfig?.run?.profile ?? "local";
+    const profileSettings = mergedProfiles[selectedProfile];
+    if (!profileSettings) {
+        const names = Object.keys(mergedProfiles).sort().join(", ");
+        throw new Error(`Unknown benchmark profile '${selectedProfile}'. Available profiles: ${names}`);
+    }
+    const mergedGates = {
+        hard: {
+            ...DEFAULT_GATES.hard,
+            ...(fileConfig?.gates?.hard ?? {}),
+        },
+        soft: {
+            ...DEFAULT_GATES.soft,
+            ...(fileConfig?.gates?.soft ?? {}),
+        },
+    };
+    const resolved = {
+        version: fileConfig?.version ?? DEFAULT_VERSION,
+        profile: selectedProfile,
+        defaults: validateDefaults(mergedDefaults),
+        profileSettings: validateProfileSettings(profileSettings, selectedProfile),
+        gates: validateGates(mergedGates),
+        run: {
+            suite: cli.suite ?? fileConfig?.run?.suite ?? "showcase",
+            exampleId: cli.exampleId ?? fileConfig?.run?.exampleId,
+            scenarioPath: cli.scenarioPath ?? fileConfig?.run?.scenarioPath,
+            modelOverride: cli.modelOverride ?? fileConfig?.run?.model,
+            outPath: cli.outPath ?? fileConfig?.run?.outPath,
+            baselinePath: cli.baselinePath ?? fileConfig?.run?.baselinePath,
+            executionMode: resolveExecutionMode(cli, fileConfig),
+            listExamples: cli.listExamples ?? fileConfig?.run?.listExamples ?? false,
+            updateCapCache: cli.updateCapCache ?? fileConfig?.run?.updateCapCache ?? false,
+            capTtlDays: intField(cli.capTtlDays ?? fileConfig?.run?.capTtlDays ?? DEFAULT_CAP_TTL_DAYS, "run.capTtlDays", 1),
+            temperature: optionalNumberField(cli.temperature ?? fileConfig?.run?.temperature, "run.temperature"),
+            top_p: optionalBoundedField(cli.top_p ?? fileConfig?.run?.top_p, "run.top_p", 0, 1),
+            max_tokens: optionalIntField(cli.max_tokens ?? fileConfig?.run?.max_tokens, "run.max_tokens", 1),
+            presence_penalty: optionalBoundedField(cli.presence_penalty ?? fileConfig?.run?.presence_penalty, "run.presence_penalty", -2, 2),
+            frequency_penalty: optionalBoundedField(cli.frequency_penalty ?? fileConfig?.run?.frequency_penalty, "run.frequency_penalty", -2, 2),
+            seed: optionalIntField(cli.seed ?? fileConfig?.run?.seed, "run.seed", 0),
+            stop: optionalStopField(cli.stop ?? fileConfig?.run?.stop, "run.stop"),
+        },
+        configSource,
+    };
+    return resolved;
+}
+async function loadConfigFile(paths, explicitPath) {
+    const candidatePath = explicitPath
+        ? path_1.default.resolve(explicitPath)
+        : path_1.default.join(paths.baseDir, "benchmark.config.yaml");
+    try {
+        const raw = await fs_1.promises.readFile(candidatePath, "utf8");
+        const parsed = parseConfigDocument(candidatePath, raw);
+        return { fileConfig: parsed, configSource: candidatePath };
+    }
+    catch (error) {
+        const code = error.code;
+        if (code === "ENOENT") {
+            if (explicitPath) {
+                throw new Error(`Benchmark config not found: ${candidatePath}`);
+            }
+            return {};
+        }
+        throw error;
+    }
+}
+function parseConfigDocument(filePath, raw) {
+    const ext = path_1.default.extname(filePath).toLowerCase();
+    try {
+        if (ext === ".json") {
+            return JSON.parse(raw);
+        }
+        return yaml_1.default.parse(raw);
+    }
+    catch (error) {
+        throw new Error(`Failed to parse benchmark config ${filePath}: ${error.message}`);
+    }
+}
+function validateDefaults(defaults) {
+    return {
+        requestTimeoutMs: intField(defaults.requestTimeoutMs, "defaults.requestTimeoutMs", 1),
+        toolTimeoutMs: intField(defaults.toolTimeoutMs, "defaults.toolTimeoutMs", 1),
+        maxIterations: intField(defaults.maxIterations, "defaults.maxIterations", 1),
+        temperature: numberField(defaults.temperature, "defaults.temperature"),
+        top_p: boundedField(defaults.top_p, "defaults.top_p", 0, 1),
+        max_tokens: intField(defaults.max_tokens, "defaults.max_tokens", 1),
+        presence_penalty: boundedField(defaults.presence_penalty, "defaults.presence_penalty", -2, 2),
+        frequency_penalty: boundedField(defaults.frequency_penalty, "defaults.frequency_penalty", -2, 2),
+        seed: optionalIntField(defaults.seed, "defaults.seed", 0),
+        stop: optionalStopField(defaults.stop, "defaults.stop"),
+    };
+}
+function resolveExecutionMode(cli, fileConfig) {
+    const explicit = cli.executionMode ?? fileConfig?.run?.executionMode;
+    if (explicit === "showcase" || explicit === "diagnostic") {
+        return explicit;
+    }
+    const suite = cli.suite ?? fileConfig?.run?.suite ?? "showcase";
+    return suite === "showcase" ? "showcase" : "diagnostic";
+}
+function validateProfileSettings(profile, profileName) {
+    return {
+        warmupRuns: intField(profile.warmupRuns, `profiles.${profileName}.warmupRuns`, 0),
+        measuredRuns: intField(profile.measuredRuns, `profiles.${profileName}.measuredRuns`, 1),
+        minScenarioPassRate: boundedField(profile.minScenarioPassRate, `profiles.${profileName}.minScenarioPassRate`, 0, 1),
+    };
+}
+function validateGates(gates) {
+    return {
+        hard: {
+            smokeMinSuccessRate: boundedField(gates.hard.smokeMinSuccessRate, "gates.hard.smokeMinSuccessRate", 0, 1),
+        },
+        soft: {
+            maxP95RegressionPct: numberField(gates.soft.maxP95RegressionPct, "gates.soft.maxP95RegressionPct", 0),
+            maxThroughputDropPct: numberField(gates.soft.maxThroughputDropPct, "gates.soft.maxThroughputDropPct", 0),
+        },
+    };
+}
+function intField(value, field, min) {
+    if (!Number.isInteger(value) || value < min) {
+        throw new Error(`${field} must be an integer >= ${min}`);
+    }
+    return value;
+}
+function numberField(value, field, min) {
+    if (!Number.isFinite(value)) {
+        throw new Error(`${field} must be a finite number`);
+    }
+    if (typeof min === "number" && value < min) {
+        throw new Error(`${field} must be >= ${min}`);
+    }
+    return value;
+}
+function boundedField(value, field, min, max) {
+    if (!Number.isFinite(value) || value < min || value > max) {
+        throw new Error(`${field} must be between ${min} and ${max}`);
+    }
+    return value;
+}
+function optionalIntField(value, field, min) {
+    if (value === undefined)
+        return undefined;
+    return intField(value, field, min);
+}
+function optionalNumberField(value, field, min) {
+    if (value === undefined)
+        return undefined;
+    return numberField(value, field, min);
+}
+function optionalBoundedField(value, field, min, max) {
+    if (value === undefined)
+        return undefined;
+    return boundedField(value, field, min, max);
+}
+function optionalStopField(value, field) {
+    if (value === undefined)
+        return undefined;
+    if (typeof value === "string") {
+        const trimmed = value.trim();
+        if (trimmed.length === 0) {
+            throw new Error(`${field} must not be empty`);
+        }
+        return trimmed;
+    }
+    if (Array.isArray(value)) {
+        if (value.length === 0) {
+            throw new Error(`${field} must include at least one stop sequence`);
+        }
+        const normalized = value.map((item, index) => {
+            if (typeof item !== "string") {
+                throw new Error(`${field}[${index}] must be a string`);
+            }
+            const trimmed = item.trim();
+            if (trimmed.length === 0) {
+                throw new Error(`${field}[${index}] must not be empty`);
+            }
+            return trimmed;
+        });
+        return normalized;
+    }
+    throw new Error(`${field} must be a string or string[]`);
+}

package/dist/src/benchmark/gates.js

@@ -0,0 +1,118 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.evaluateGates = evaluateGates;
+const fs_1 = require("fs");
+async function evaluateGates(report, effective) {
+    const hardMessages = [];
+    const softMessages = [];
+    if (effective.run.suite === "smoke") {
+        const min = effective.gates.hard.smokeMinSuccessRate;
+        if (report.executed > 0 && report.successRate < min) {
+            hardMessages.push(`Smoke suite success rate ${toPct(report.successRate)} is below required ${toPct(min)}.`);
+        }
+    }
+    const minScenarioPassRate = effective.profileSettings.minScenarioPassRate;
+    const failingScenarios = report.results.filter((scenario) => scenario.status !== "skipped" && scenario.passRate < minScenarioPassRate);
+    for (const scenario of failingScenarios) {
+        hardMessages.push(`Scenario '${scenario.id}' pass rate ${toPct(scenario.passRate)} is below required ${toPct(minScenarioPassRate)}.`);
+    }
+    if (effective.run.baselinePath) {
+        const baseline = await loadBaseline(effective.run.baselinePath);
+        const baselineById = new Map(baseline.results
+            .map((scenario) => normalizeBaselineScenario(scenario))
+            .filter((scenario) => scenario !== null)
+            .map((scenario) => [scenario.id, scenario]));
+        for (const current of report.results) {
+            if (current.status === "skipped") {
+                continue;
+            }
+            const ref = baselineById.get(current.id);
+            if (!ref) {
+                continue;
+            }
+            const maxP95 = effective.gates.soft.maxP95RegressionPct;
+            if (ref.p95LatencyMs > 0) {
+                const p95Threshold = ref.p95LatencyMs * (1 + maxP95 / 100);
+                if (current.p95LatencyMs > p95Threshold) {
+                    softMessages.push(`Scenario '${current.id}' p95 latency regressed ${pctDelta(current.p95LatencyMs, ref.p95LatencyMs)} (${current.p95LatencyMs}ms vs baseline ${ref.p95LatencyMs}ms).`);
+                }
+            }
+            const maxThroughputDrop = effective.gates.soft.maxThroughputDropPct;
+            if (ref.avgThroughputTokensPerSec > 0) {
+                const throughputThreshold = ref.avgThroughputTokensPerSec * (1 - maxThroughputDrop / 100);
+                if (current.avgThroughputTokensPerSec < throughputThreshold) {
+                    softMessages.push(`Scenario '${current.id}' throughput dropped ${throughputDropPct(current.avgThroughputTokensPerSec, ref.avgThroughputTokensPerSec)} (${current.avgThroughputTokensPerSec.toFixed(2)} t/s vs baseline ${ref.avgThroughputTokensPerSec.toFixed(2)} t/s).`);
+                }
+            }
+        }
+    }
+    return {
+        hard: {
+            passed: hardMessages.length === 0,
+            messages: hardMessages,
+        },
+        soft: {
+            passed: softMessages.length === 0,
+            messages: softMessages,
+        },
+    };
+}
+async function loadBaseline(pathLike) {
+    let raw;
+    try {
+        raw = await fs_1.promises.readFile(pathLike, "utf8");
+    }
+    catch (error) {
+        throw new Error(`Failed to read baseline file '${pathLike}': ${error.message}`);
+    }
+    let parsed;
+    try {
+        parsed = JSON.parse(raw);
+    }
+    catch (error) {
+        throw new Error(`Failed to parse baseline file '${pathLike}' as JSON: ${error.message}`);
+    }
+    if (!parsed || typeof parsed !== "object" || !Array.isArray(parsed.results)) {
+        throw new Error(`Baseline file '${pathLike}' must contain a top-level 'results' array.`);
+    }
+    return parsed;
+}
+function normalizeBaselineScenario(raw) {
+    if (!raw || typeof raw !== "object") {
+        return null;
+    }
+    const scenario = raw;
+    if (typeof scenario.id !== "string") {
+        return null;
+    }
+    const p95LatencyMs = typeof scenario.p95LatencyMs === "number" && Number.isFinite(scenario.p95LatencyMs)
+        ? scenario.p95LatencyMs
+        : 0;
+    const avgThroughputTokensPerSec = typeof scenario.avgThroughputTokensPerSec === "number" &&
+        Number.isFinite(scenario.avgThroughputTokensPerSec)
+        ? scenario.avgThroughputTokensPerSec
+        : 0;
+    return {
+        id: scenario.id,
+        p95LatencyMs,
+        avgThroughputTokensPerSec,
+    };
+}
+function toPct(value) {
+    return `${(value * 100).toFixed(1)}%`;
+}
+function pctDelta(current, baseline) {
+    if (baseline === 0) {
+        return "n/a";
+    }
+    const delta = ((current - baseline) / baseline) * 100;
+    const sign = delta >= 0 ? "+" : "";
+    return `${sign}${delta.toFixed(1)}%`;
+}
+function throughputDropPct(current, baseline) {
+    if (baseline === 0) {
+        return "n/a";
+    }
+    const delta = ((baseline - current) / baseline) * 100;
+    return `${delta.toFixed(1)}%`;
+}

package/dist/src/benchmark/jobs.js

@@ -0,0 +1,252 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.hasRunningBenchmarkRun = hasRunningBenchmarkRun;
+exports.getBenchmarkRun = getBenchmarkRun;
+exports.listBenchmarkRunEvents = listBenchmarkRunEvents;
+exports.subscribeBenchmarkRunEvents = subscribeBenchmarkRunEvents;
+exports.startBenchmarkRun = startBenchmarkRun;
+exports.listBenchmarkRuns = listBenchmarkRuns;
+exports.getArtifactBenchmarkRun = getArtifactBenchmarkRun;
+const fs_1 = require("fs");
+const path_1 = __importDefault(require("path"));
+const events_1 = require("events");
+const crypto_1 = require("crypto");
+const runner_1 = require("./runner");
+const runs = new Map();
+const runEmitter = new events_1.EventEmitter();
+function hasRunningBenchmarkRun() {
+    return Array.from(runs.values()).some((run) => run.status === "running");
+}
+function getBenchmarkRun(runId) {
+    return runs.get(runId);
+}
+function listBenchmarkRunEvents(runId) {
+    return [...(runs.get(runId)?.events ?? [])];
+}
+function subscribeBenchmarkRunEvents(runId, handler) {
+    const channel = eventChannel(runId);
+    runEmitter.on(channel, handler);
+    return () => {
+        runEmitter.off(channel, handler);
+    };
+}
+async function startBenchmarkRun(paths, request) {
+    const runId = (0, crypto_1.randomUUID)();
+    const now = new Date().toISOString();
+    const run = {
+        id: runId,
+        status: "running",
+        createdAt: now,
+        startedAt: now,
+        request,
+        progress: {
+            totalScenarios: 0,
+            completedScenarios: 0,
+        },
+        events: [],
+    };
+    runs.set(runId, run);
+    void executeBenchmarkRun(paths, runId);
+    return run;
+}
+async function listBenchmarkRuns(paths) {
+    const inMemory = Array.from(runs.values()).map(toSummary);
+    const artifactBacked = await listArtifactBackedRuns(paths);
+    const merged = new Map();
+    for (const item of artifactBacked) {
+        merged.set(item.id, item);
+    }
+    for (const item of inMemory) {
+        merged.set(item.id, item);
+    }
+    return Array.from(merged.values()).sort((a, b) => {
+        const aTs = Date.parse(a.createdAt || a.startedAt || a.finishedAt || "");
+        const bTs = Date.parse(b.createdAt || b.startedAt || b.finishedAt || "");
+        return bTs - aTs;
+    });
+}
+async function getArtifactBenchmarkRun(paths, runId) {
+    const benchmarksDir = path_1.default.join(paths.baseDir, "benchmarks");
+    let files = [];
+    try {
+        files = await fs_1.promises.readdir(benchmarksDir);
+    }
+    catch (error) {
+        if (error.code === "ENOENT") {
+            return null;
+        }
+        throw error;
+    }
+    const jsonFiles = files
+        .filter((file) => file.startsWith("bench-") && file.endsWith(".json"))
+        .sort((a, b) => b.localeCompare(a));
+    for (const file of jsonFiles) {
+        const filePath = path_1.default.join(benchmarksDir, file);
+        try {
+            const raw = await fs_1.promises.readFile(filePath, "utf8");
+            const report = JSON.parse(raw);
+            if (report.id !== runId) {
+                continue;
+            }
+            return {
+                id: report.id,
+                status: "completed",
+                createdAt: report.createdAt,
+                startedAt: report.createdAt,
+                finishedAt: report.createdAt,
+                request: {
+                    suite: report.suite,
+                    exampleId: report.exampleId,
+                    scenarioPath: report.scenarioPath,
+                    modelOverride: report.modelOverride,
+                    profile: report.profile,
+                    executionMode: report.executionMode,
+                },
+                progress: {
+                    totalScenarios: report.total,
+                    completedScenarios: report.total,
+                },
+                report,
+                artifactPath: filePath,
+                textArtifactPath: filePath.replace(/\.json$/i, ".txt"),
+                events: [],
+            };
+        }
+        catch {
+            // Ignore malformed artifact.
+        }
+    }
+    return null;
+}
+async function executeBenchmarkRun(paths, runId) {
+    const run = runs.get(runId);
+    if (!run) {
+        return;
+    }
+    try {
+        const output = await (0, runner_1.runBenchmark)(paths, run.request, {
+            runId,
+            onEvent: (event) => {
+                updateProgress(run, event);
+                pushEvent(run, event);
+            },
+        });
+        run.status = "completed";
+        run.finishedAt = new Date().toISOString();
+        run.report = output.report;
+        run.artifactPath = output.artifactPath;
+        run.textArtifactPath = output.textArtifactPath;
+    }
+    catch (error) {
+        run.status = "failed";
+        run.finishedAt = new Date().toISOString();
+        run.error = error.message;
+        const failureEvent = {
+            type: "warning",
+            timestamp: new Date().toISOString(),
+            runId,
+            warning: run.error,
+        };
+        pushEvent(run, failureEvent);
+    }
+}
+function updateProgress(run, event) {
+    if (event.type === "run_started") {
+        run.progress.totalScenarios = event.totalScenarios ?? run.progress.totalScenarios;
+        return;
+    }
+    if (event.type === "scenario_started") {
+        run.progress.currentScenarioId = event.scenarioId;
+        run.progress.currentScenarioIndex = event.scenarioIndex;
+        run.progress.totalScenarios = event.totalScenarios ?? run.progress.totalScenarios;
+        return;
+    }
+    if (event.type === "sample_completed") {
+        run.progress.currentRunIndex = event.runIndex;
+        run.progress.totalRuns = event.totalRuns;
+        run.progress.phase = event.phase;
+        return;
+    }
+    if (event.type === "scenario_completed") {
+        run.progress.completedScenarios += 1;
+        run.progress.currentRunIndex = undefined;
+        run.progress.totalRuns = undefined;
+        run.progress.phase = undefined;
+        return;
+    }
+    if (event.type === "run_completed") {
+        run.progress.completedScenarios = run.progress.totalScenarios;
+    }
+}
+function pushEvent(run, event) {
+    run.events.push(event);
+    if (run.events.length > 1000) {
+        run.events.splice(0, run.events.length - 1000);
+    }
+    runEmitter.emit(eventChannel(run.id), event);
+}
+function eventChannel(runId) {
+    return `benchmark:${runId}`;
+}
+function toSummary(run) {
+    return {
+        id: run.id,
+        status: run.status,
+        createdAt: run.createdAt,
+        startedAt: run.startedAt,
+        finishedAt: run.finishedAt,
+        suite: run.request.suite,
+        exampleId: run.request.exampleId,
+        profile: run.request.profile,
+        scenarioPath: run.request.scenarioPath,
+        succeeded: run.report?.succeeded,
+        failed: run.report?.failed,
+        successRate: run.report?.successRate,
+        artifactPath: run.artifactPath,
+    };
+}
+async function listArtifactBackedRuns(paths) {
+    const benchmarksDir = path_1.default.join(paths.baseDir, "benchmarks");
+    let files = [];
+    try {
+        files = await fs_1.promises.readdir(benchmarksDir);
+    }
+    catch (error) {
+        if (error.code === "ENOENT") {
+            return [];
+        }
+        throw error;
+    }
+    const jsonFiles = files
+        .filter((file) => file.startsWith("bench-") && file.endsWith(".json"))
+        .sort((a, b) => b.localeCompare(a))
+        .slice(0, 100);
+    const summaries = [];
+    for (const file of jsonFiles) {
+        const filePath = path_1.default.join(benchmarksDir, file);
+        try {
+            const raw = await fs_1.promises.readFile(filePath, "utf8");
+            const report = JSON.parse(raw);
+            summaries.push({
+                id: report.id,
+                status: "completed",
+                createdAt: report.createdAt,
+                finishedAt: report.createdAt,
+                suite: report.suite,
+                profile: report.profile,
+                scenarioPath: report.scenarioPath,
+                succeeded: report.succeeded,
+                failed: report.failed,
+                successRate: report.successRate,
+                artifactPath: filePath,
+            });
+        }
+        catch {
+            // Skip malformed artifacts.
+        }
+    }
+    return summaries;
+}