waypoi 0.0.0

package/dist/src/benchmark/schema.js

@@ -0,0 +1,353 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.validateScenarioCollection = validateScenarioCollection;
+const types_1 = require("./types");
+const SCENARIO_KEYS = new Set([
+    "id",
+    "mode",
+    "title",
+    "summary",
+    "userVisibleGoal",
+    "exampleSource",
+    "inputPreview",
+    "successCriteria",
+    "expectedHighlights",
+    "capability",
+    "model",
+    "timeoutMs",
+    "requiresAvailableTools",
+    "assertions",
+    "prompt",
+    "tools",
+    "maxIterations",
+    "temperature",
+    "top_p",
+    "max_tokens",
+    "presence_penalty",
+    "frequency_penalty",
+    "seed",
+    "stop",
+    "input",
+    "n",
+    "size",
+    "audioFile",
+    "inputText",
+    "voice",
+    "response_format",
+]);
+const ASSERTION_KEYS = new Set([
+    "contains",
+    "notContains",
+    "requiredToolNames",
+    "minToolCalls",
+    "maxToolCalls",
+    "maxLatencyMs",
+    "statusCode",
+    "minItems",
+    "minVectorLength",
+    "minImages",
+    "containsText",
+    "notContainsText",
+    "minBytes",
+    "contentType",
+]);
+function validateScenarioCollection(rawScenarios, sourceLabel) {
+    const warnings = [];
+    const scenarios = rawScenarios.map((raw, index) => validateScenario(raw, sourceLabel, index, warnings));
+    const ids = new Set();
+    for (const scenario of scenarios) {
+        if (ids.has(scenario.id)) {
+            throw new Error(`${sourceLabel} scenario '${scenario.id}' is duplicated.`);
+        }
+        ids.add(scenario.id);
+    }
+    return { scenarios, warnings };
+}
+function validateScenario(raw, sourceLabel, index, warnings) {
+    const ctx = `${sourceLabel} scenario[${index}]`;
+    if (!raw || typeof raw !== "object" || Array.isArray(raw)) {
+        throw new Error(`${ctx}: expected object.`);
+    }
+    const input = raw;
+    for (const key of Object.keys(input)) {
+        if (!SCENARIO_KEYS.has(key)) {
+            warnings.push(`${ctx}: unknown field '${key}' is ignored.`);
+        }
+    }
+    const id = requiredString(input.id, `${ctx}.id`);
+    const mode = validateMode(input.mode, `${ctx}.mode`);
+    const capability = optionalCapabilityKey(input.capability, `${ctx}.capability`);
+    const model = optionalString(input.model, `${ctx}.model`);
+    const timeoutMs = optionalInteger(input.timeoutMs, `${ctx}.timeoutMs`, 1);
+    const assertions = validateAssertions(input.assertions, `${ctx}.assertions`, warnings);
+    const scenario = {
+        id,
+        mode,
+        capability,
+        model,
+        timeoutMs,
+        assertions,
+    };
+    scenario.title = optionalString(input.title, `${ctx}.title`);
+    scenario.summary = optionalString(input.summary, `${ctx}.summary`);
+    scenario.userVisibleGoal = optionalString(input.userVisibleGoal, `${ctx}.userVisibleGoal`);
+    scenario.exampleSource = optionalExampleSource(input.exampleSource, `${ctx}.exampleSource`);
+    scenario.inputPreview = optionalString(input.inputPreview, `${ctx}.inputPreview`);
+    scenario.successCriteria = optionalString(input.successCriteria, `${ctx}.successCriteria`);
+    scenario.expectedHighlights = optionalStringArray(input.expectedHighlights, `${ctx}.expectedHighlights`);
+    scenario.prompt = optionalString(input.prompt, `${ctx}.prompt`);
+    scenario.tools = optionalStringArray(input.tools, `${ctx}.tools`);
+    scenario.maxIterations = optionalInteger(input.maxIterations, `${ctx}.maxIterations`, 1, 20);
+    scenario.temperature = optionalFiniteNumber(input.temperature, `${ctx}.temperature`);
+    scenario.top_p = optionalFiniteNumber(input.top_p, `${ctx}.top_p`, 0, 1);
+    scenario.max_tokens = optionalInteger(input.max_tokens, `${ctx}.max_tokens`, 1);
+    scenario.presence_penalty = optionalFiniteNumber(input.presence_penalty, `${ctx}.presence_penalty`, -2, 2);
+    scenario.frequency_penalty = optionalFiniteNumber(input.frequency_penalty, `${ctx}.frequency_penalty`, -2, 2);
+    scenario.seed = optionalInteger(input.seed, `${ctx}.seed`, 0);
+    scenario.stop = optionalStopField(input.stop, `${ctx}.stop`);
+    scenario.requiresAvailableTools = optionalBoolean(input.requiresAvailableTools, `${ctx}.requiresAvailableTools`);
+    scenario.input = optionalInputValue(input.input, `${ctx}.input`);
+    scenario.n = optionalInteger(input.n, `${ctx}.n`, 1);
+    scenario.size = optionalString(input.size, `${ctx}.size`);
+    scenario.audioFile = optionalString(input.audioFile, `${ctx}.audioFile`);
+    scenario.inputText = optionalString(input.inputText, `${ctx}.inputText`);
+    scenario.voice = optionalString(input.voice, `${ctx}.voice`);
+    scenario.response_format = optionalString(input.response_format, `${ctx}.response_format`);
+    validateScenarioByMode(scenario, ctx);
+    return scenario;
+}
+function validateScenarioByMode(scenario, ctx) {
+    if (scenario.mode === "chat" ||
+        scenario.mode === "agent" ||
+        scenario.mode === "responses" ||
+        scenario.mode === "image_generation") {
+        if (!scenario.prompt) {
+            throw new Error(`${ctx}.prompt: required for mode '${scenario.mode}'.`);
+        }
+    }
+    if (scenario.mode === "embeddings") {
+        if (scenario.input === undefined) {
+            throw new Error(`${ctx}.input: required for mode 'embeddings'.`);
+        }
+    }
+    if (scenario.mode === "audio_transcription") {
+        if (!scenario.audioFile) {
+            throw new Error(`${ctx}.audioFile: required for mode 'audio_transcription'.`);
+        }
+    }
+    if (scenario.mode === "omni_call") {
+        if (!scenario.audioFile) {
+            throw new Error(`${ctx}.audioFile: required for mode 'omni_call'.`);
+        }
+    }
+    if (scenario.mode === "audio_speech") {
+        if (!scenario.inputText) {
+            throw new Error(`${ctx}.inputText: required for mode 'audio_speech'.`);
+        }
+        if (!scenario.voice) {
+            throw new Error(`${ctx}.voice: required for mode 'audio_speech'.`);
+        }
+    }
+}
+function validateAssertions(raw, field, warnings) {
+    if (raw === undefined || raw === null) {
+        return { statusCode: 200 };
+    }
+    if (typeof raw !== "object" || Array.isArray(raw)) {
+        throw new Error(`${field}: expected object.`);
+    }
+    const input = raw;
+    for (const key of Object.keys(input)) {
+        if (!ASSERTION_KEYS.has(key)) {
+            warnings.push(`${field}: unknown field '${key}' is ignored.`);
+        }
+    }
+    const contains = optionalStringArray(input.contains, `${field}.contains`);
+    const notContains = optionalStringArray(input.notContains, `${field}.notContains`);
+    const requiredToolNames = optionalStringArray(input.requiredToolNames, `${field}.requiredToolNames`);
+    const minToolCalls = optionalInteger(input.minToolCalls, `${field}.minToolCalls`, 0);
+    const maxToolCalls = optionalInteger(input.maxToolCalls, `${field}.maxToolCalls`, 0);
+    const maxLatencyMs = optionalInteger(input.maxLatencyMs, `${field}.maxLatencyMs`, 1);
+    const statusCode = optionalInteger(input.statusCode, `${field}.statusCode`, 100, 599) ?? 200;
+    const minItems = optionalInteger(input.minItems, `${field}.minItems`, 1);
+    const minVectorLength = optionalInteger(input.minVectorLength, `${field}.minVectorLength`, 1);
+    const minImages = optionalInteger(input.minImages, `${field}.minImages`, 1);
+    const containsText = optionalStringArray(input.containsText, `${field}.containsText`);
+    const notContainsText = optionalStringArray(input.notContainsText, `${field}.notContainsText`);
+    const minBytes = optionalInteger(input.minBytes, `${field}.minBytes`, 1);
+    const contentType = optionalString(input.contentType, `${field}.contentType`);
+    if (typeof minToolCalls === "number" &&
+        typeof maxToolCalls === "number" &&
+        maxToolCalls < minToolCalls) {
+        throw new Error(`${field}: maxToolCalls must be >= minToolCalls.`);
+    }
+    return {
+        contains,
+        notContains,
+        requiredToolNames,
+        minToolCalls,
+        maxToolCalls,
+        maxLatencyMs,
+        statusCode,
+        minItems,
+        minVectorLength,
+        minImages,
+        containsText,
+        notContainsText,
+        minBytes,
+        contentType,
+    };
+}
+function optionalExampleSource(value, field) {
+    const parsed = optionalString(value, field);
+    if (!parsed) {
+        return undefined;
+    }
+    if (parsed !== "opencode" && parsed !== "builtin" && parsed !== "file" && parsed !== "huggingface") {
+        throw new Error(`${field}: expected 'opencode', 'builtin', 'file', or 'huggingface'.`);
+    }
+    return parsed;
+}
+function optionalBoolean(value, field) {
+    if (value === undefined || value === null) {
+        return undefined;
+    }
+    if (typeof value !== "boolean") {
+        throw new Error(`${field}: expected boolean.`);
+    }
+    return value;
+}
+function validateMode(value, field) {
+    if (typeof value !== "string") {
+        throw new Error(`${field}: expected mode string.`);
+    }
+    if (!types_1.BENCHMARK_MODES.includes(value)) {
+        throw new Error(`${field}: unsupported mode '${value}'.`);
+    }
+    return value;
+}
+function requiredString(value, field) {
+    const parsed = optionalString(value, field);
+    if (!parsed) {
+        throw new Error(`${field}: required non-empty string.`);
+    }
+    return parsed;
+}
+function optionalString(value, field) {
+    if (value === undefined || value === null) {
+        return undefined;
+    }
+    if (typeof value !== "string" || value.trim().length === 0) {
+        throw new Error(`${field}: expected non-empty string.`);
+    }
+    return value.trim();
+}
+function optionalStringArray(value, field) {
+    if (value === undefined || value === null) {
+        return undefined;
+    }
+    if (typeof value === "string") {
+        const single = value.trim();
+        if (!single) {
+            throw new Error(`${field}: string must be non-empty.`);
+        }
+        return [single];
+    }
+    if (!Array.isArray(value)) {
+        throw new Error(`${field}: expected string or string array.`);
+    }
+    return value.map((entry, index) => {
+        if (typeof entry !== "string" || entry.trim().length === 0) {
+            throw new Error(`${field}[${index}]: expected non-empty string.`);
+        }
+        return entry.trim();
+    });
+}
+function optionalInputValue(value, field) {
+    if (value === undefined || value === null) {
+        return undefined;
+    }
+    if (typeof value === "string") {
+        if (!value.trim()) {
+            throw new Error(`${field}: expected non-empty string.`);
+        }
+        return value;
+    }
+    if (!Array.isArray(value)) {
+        throw new Error(`${field}: expected string or string array.`);
+    }
+    const normalized = value.map((entry, index) => {
+        if (typeof entry !== "string" || !entry.trim()) {
+            throw new Error(`${field}[${index}]: expected non-empty string.`);
+        }
+        return entry;
+    });
+    return normalized;
+}
+function optionalCapabilityKey(value, field) {
+    if (value === undefined || value === null) {
+        return undefined;
+    }
+    if (typeof value !== "string") {
+        throw new Error(`${field}: expected capability key string.`);
+    }
+    if (!types_1.BENCHMARK_CAPABILITY_KEYS.includes(value)) {
+        throw new Error(`${field}: unsupported capability '${value}'.`);
+    }
+    return value;
+}
+function optionalInteger(value, field, min, max) {
+    if (value === undefined || value === null) {
+        return undefined;
+    }
+    if (!Number.isInteger(value)) {
+        throw new Error(`${field}: expected integer.`);
+    }
+    const num = value;
+    if (num < min) {
+        throw new Error(`${field}: must be >= ${min}.`);
+    }
+    if (typeof max === "number" && num > max) {
+        throw new Error(`${field}: must be <= ${max}.`);
+    }
+    return num;
+}
+function optionalFiniteNumber(value, field, min, max) {
+    if (value === undefined || value === null) {
+        return undefined;
+    }
+    if (typeof value !== "number" || !Number.isFinite(value)) {
+        throw new Error(`${field}: expected finite number.`);
+    }
+    if (typeof min === "number" && value < min) {
+        throw new Error(`${field}: must be >= ${min}.`);
+    }
+    if (typeof max === "number" && value > max) {
+        throw new Error(`${field}: must be <= ${max}.`);
+    }
+    return value;
+}
+function optionalStopField(value, field) {
+    if (value === undefined || value === null) {
+        return undefined;
+    }
+    if (typeof value === "string") {
+        const trimmed = value.trim();
+        if (!trimmed) {
+            throw new Error(`${field}: expected non-empty string.`);
+        }
+        return trimmed;
+    }
+    if (!Array.isArray(value)) {
+        throw new Error(`${field}: expected string or string array.`);
+    }
+    if (value.length === 0) {
+        throw new Error(`${field}: expected at least one stop sequence.`);
+    }
+    return value.map((entry, index) => {
+        if (typeof entry !== "string" || entry.trim().length === 0) {
+            throw new Error(`${field}[${index}]: expected non-empty string.`);
+        }
+        return entry.trim();
+    });
+}

package/dist/src/benchmark/suites.js

@@ -0,0 +1,314 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.builtInSuite = builtInSuite;
+exports.listBuiltInSuites = listBuiltInSuites;
+exports.listSuiteExamples = listSuiteExamples;
+const tinyQaDataset_1 = require("./tinyQaDataset");
+const SHOWCASE_SUITE = tinyQaDataset_1.TINY_QA_BENCHMARK.map((row) => toTinyQaShowcaseScenario(row));
+const SUITES = {
+    showcase: SHOWCASE_SUITE,
+    smoke: [
+        {
+            id: "smoke-chat-exact",
+            mode: "chat",
+            prompt: "Reply exactly with: WAYPOI_SMOKE_OK",
+            assertions: {
+                contains: ["WAYPOI_SMOKE_OK"],
+                statusCode: 200,
+            },
+        },
+        {
+            id: "smoke-agent-loop",
+            mode: "agent",
+            prompt: "If tools are available, call exactly one and summarize. If no tools are available, output NO_TOOLS_AVAILABLE. Prefix final answer with WAYPOI_AGENT_DONE:",
+            maxIterations: 4,
+            assertions: {
+                contains: ["WAYPOI_AGENT_DONE"],
+                statusCode: 200,
+            },
+        },
+        {
+            id: "smoke-embeddings-basic",
+            mode: "embeddings",
+            input: "waypoi benchmark smoke",
+            assertions: {
+                minItems: 1,
+                minVectorLength: 1,
+                statusCode: 200,
+            },
+        },
+        {
+            id: "smoke-image-generation",
+            mode: "image_generation",
+            prompt: "A tiny blue square on white background",
+            assertions: {
+                minImages: 1,
+                statusCode: 200,
+            },
+        },
+        {
+            id: "smoke-audio-speech",
+            mode: "audio_speech",
+            inputText: "Waypoi benchmark smoke",
+            voice: "alloy",
+            assertions: {
+                minBytes: 1,
+                statusCode: 200,
+            },
+        },
+    ],
+    proxy: [
+        {
+            id: "proxy-chat-short",
+            mode: "chat",
+            prompt: "Answer with one word: waypoi",
+            assertions: {
+                contains: ["waypoi"],
+                statusCode: 200,
+            },
+        },
+        {
+            id: "proxy-embeddings",
+            mode: "embeddings",
+            input: ["waypoi", "proxy", "benchmark"],
+            assertions: {
+                minItems: 3,
+                minVectorLength: 1,
+                statusCode: 200,
+            },
+        },
+        {
+            id: "proxy-image",
+            mode: "image_generation",
+            prompt: "A minimal icon of a gateway",
+            assertions: {
+                minImages: 1,
+                statusCode: 200,
+            },
+        },
+    ],
+    agent: [
+        {
+            id: "agent-tool-loop-basic",
+            mode: "agent",
+            prompt: "Use available tools if useful, then provide a concise final answer prefixed with WAYPOI_AGENT_DONE:",
+            maxIterations: 6,
+            assertions: {
+                contains: ["WAYPOI_AGENT_DONE"],
+                statusCode: 200,
+            },
+        },
+        {
+            id: "agent-tool-required",
+            mode: "agent",
+            prompt: "Use at least one tool before answering.",
+            maxIterations: 6,
+            requiresAvailableTools: true,
+            assertions: {
+                minToolCalls: 1,
+                statusCode: 200,
+            },
+        },
+    ],
+    pool_smoke: [
+        {
+            id: "pool-smart-chat",
+            mode: "chat",
+            model: "smart",
+            prompt: "Reply exactly with: WAYPOI_POOL_SMOKE_OK",
+            assertions: {
+                contains: ["WAYPOI_POOL_SMOKE_OK"],
+                statusCode: 200,
+            },
+        },
+        {
+            id: "pool-smart-agent",
+            mode: "agent",
+            model: "smart",
+            prompt: "Answer with prefix WAYPOI_POOL_AGENT_DONE:",
+            assertions: {
+                contains: ["WAYPOI_POOL_AGENT_DONE:"],
+                statusCode: 200,
+            },
+        },
+    ],
+    omni_call_smoke: [
+        {
+            id: "omni-call-basic",
+            mode: "omni_call",
+            prompt: "Please transcribe this audio and summarize it in one sentence.",
+            audioFile: "examples/scenarios/assets/omni-call-sample.wav",
+            assertions: {
+                statusCode: 200,
+            },
+        },
+    ],
+    capabilities: [
+        {
+            id: "cap.chat_basic",
+            mode: "chat",
+            capability: "chat_basic",
+            prompt: "Reply exactly with: WAYPOI_CAP_CHAT_BASIC_OK",
+            assertions: {
+                contains: ["WAYPOI_CAP_CHAT_BASIC_OK"],
+                statusCode: 200,
+            },
+        },
+        {
+            id: "cap.chat_streaming",
+            mode: "chat",
+            capability: "chat_streaming",
+            prompt: "Reply exactly with: WAYPOI_CAP_STREAMING_OK",
+            assertions: {
+                contains: ["WAYPOI_CAP_STREAMING_OK"],
+                statusCode: 200,
+            },
+        },
+        {
+            id: "cap.chat_tool_calls",
+            mode: "agent",
+            capability: "chat_tool_calls",
+            prompt: "Use at least one tool if available, then output WAYPOI_CAP_TOOL_CALLS_OK.",
+            maxIterations: 4,
+            requiresAvailableTools: true,
+            assertions: {
+                contains: ["WAYPOI_CAP_TOOL_CALLS_OK"],
+                minToolCalls: 1,
+                statusCode: 200,
+            },
+        },
+        {
+            id: "cap.chat_vision_input",
+            mode: "chat",
+            capability: "chat_vision_input",
+            prompt: "Vision probe placeholder: reply with WAYPOI_CAP_VISION_UNKNOWN when image input is unavailable.",
+            assertions: {
+                statusCode: 200,
+            },
+        },
+        {
+            id: "cap.images_generation",
+            mode: "image_generation",
+            capability: "images_generation",
+            prompt: "A monochrome square icon.",
+            assertions: {
+                minImages: 1,
+                statusCode: 200,
+            },
+        },
+        {
+            id: "cap.images_edit",
+            mode: "image_generation",
+            capability: "images_edit",
+            prompt: "Image edit probe placeholder",
+            assertions: {
+                statusCode: 200,
+            },
+        },
+        {
+            id: "cap.embeddings",
+            mode: "embeddings",
+            capability: "embeddings",
+            input: "waypoi capability embeddings probe",
+            assertions: {
+                minItems: 1,
+                minVectorLength: 1,
+                statusCode: 200,
+            },
+        },
+        {
+            id: "cap.audio_transcription",
+            mode: "audio_transcription",
+            capability: "audio_transcription",
+            audioFile: "examples/scenarios/assets/omni-call-sample.wav",
+            assertions: {
+                statusCode: 200,
+            },
+        },
+        {
+            id: "cap.audio_speech",
+            mode: "audio_speech",
+            capability: "audio_speech",
+            inputText: "Waypoi capability speech probe",
+            voice: "alloy",
+            assertions: {
+                minBytes: 1,
+                statusCode: 200,
+            },
+        },
+        {
+            id: "cap.responses_compat",
+            mode: "responses",
+            capability: "responses_compat",
+            prompt: "Summarize why the Responses API compatibility route matters in one sentence.",
+            assertions: {
+                statusCode: 200,
+            },
+        },
+    ],
+};
+function toTinyQaShowcaseScenario(row) {
+    const padded = String(row.id).padStart(3, "0");
+    return {
+        id: `showcase-tinyqa-${padded}`,
+        mode: "chat",
+        title: `Tiny QA #${padded}`,
+        summary: "Single-question QA probe from vincentkoc/tiny_qa_benchmark.",
+        userVisibleGoal: "Answer a tiny QA question with a concise factual response.",
+        exampleSource: "huggingface",
+        inputPreview: row.question,
+        successCriteria: `HTTP 200 and answer includes: ${row.answer}`,
+        expectedHighlights: [`category:${row.category}`, `difficulty:${row.difficulty}`, "gold-answer check"],
+        prompt: [
+            "Answer with only the final short answer.",
+            `Question: ${row.question}`,
+            `Reference: ${row.context}`,
+        ].join("\n"),
+        assertions: {
+            statusCode: 200,
+            contains: [row.answer],
+        },
+    };
+}
+function builtInSuite(name) {
+    const suite = SUITES[name];
+    if (!suite) {
+        const available = Object.keys(SUITES).sort().join(", ");
+        throw new Error(`Unknown benchmark suite '${name}'. Available: ${available}`);
+    }
+    return suite.map(cloneScenario);
+}
+function listBuiltInSuites() {
+    return Object.keys(SUITES).sort();
+}
+function listSuiteExamples(name) {
+    return builtInSuite(name).map((scenario) => toScenarioSummary(name, scenario));
+}
+function cloneScenario(scenario) {
+    return {
+        ...scenario,
+        assertions: { ...scenario.assertions },
+        expectedHighlights: scenario.expectedHighlights ? [...scenario.expectedHighlights] : undefined,
+        tools: scenario.tools ? [...scenario.tools] : undefined,
+        input: Array.isArray(scenario.input) ? [...scenario.input] : scenario.input,
+    };
+}
+function toScenarioSummary(suite, scenario) {
+    return {
+        id: scenario.id,
+        suite,
+        mode: scenario.mode,
+        title: scenario.title ?? scenario.id,
+        summary: scenario.summary ?? "Built-in benchmark scenario.",
+        userVisibleGoal: scenario.userVisibleGoal ?? "Exercise the configured model path and inspect the result.",
+        exampleSource: scenario.exampleSource ?? "builtin",
+        inputPreview: scenario.inputPreview ??
+            scenario.prompt ??
+            scenario.inputText ??
+            (typeof scenario.input === "string" ? scenario.input : Array.isArray(scenario.input) ? scenario.input.join(" | ") : ""),
+        successCriteria: scenario.successCriteria ?? "All configured assertions pass.",
+        expectedHighlights: scenario.expectedHighlights ?? [],
+        requiresAvailableTools: scenario.requiresAvailableTools === true,
+        model: scenario.model,
+    };
+}