waypoi 0.0.0

package/dist/cli/legacyRewrite.js

@@ -0,0 +1,92 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.rewriteLegacyArgv = rewriteLegacyArgv;
+function hasPositional(value) {
+    return typeof value === "string" && value.length > 0 && !value.startsWith("-");
+}
+function joinModelRef(providerId, modelRef) {
+    if (modelRef.includes("/")) {
+        return modelRef;
+    }
+    return `${providerId}/${modelRef}`;
+}
+function rewriteLegacyArgv(argv) {
+    const args = [...argv];
+    const [a0, a1, a2, a3, a4] = args;
+    if (a0 === "provider" && a1 === "model" && (a2 === "ls" || a2 === "list") && hasPositional(a3)) {
+        return {
+            argv: ["models", a3, ...args.slice(4)],
+            legacyUsed: true,
+            ruleId: "provider-model-list",
+            oldCmd: args.join(" "),
+            newCmd: `models ${a3}`,
+        };
+    }
+    if (a0 === "provider" && a1 === "models" && hasPositional(a2)) {
+        return {
+            argv: ["models", a2, ...args.slice(3)],
+            legacyUsed: true,
+            ruleId: "provider-models-list",
+            oldCmd: args.join(" "),
+            newCmd: `models ${a2}`,
+        };
+    }
+    if (a0 === "provider" && (a1 === "ls" || a1 === "list")) {
+        return {
+            argv: ["providers", "list", ...args.slice(2)],
+            legacyUsed: true,
+            ruleId: "provider-list",
+            oldCmd: args.join(" "),
+            newCmd: "providers list",
+        };
+    }
+    if (a0 === "provider" && a1 === "show" && hasPositional(a2)) {
+        return {
+            argv: ["providers", "show", a2, ...args.slice(3)],
+            legacyUsed: true,
+            ruleId: "provider-show",
+            oldCmd: args.join(" "),
+            newCmd: `providers show ${a2}`,
+        };
+    }
+    if (a0 === "provider" && (a1 === "enable" || a1 === "disable") && hasPositional(a2)) {
+        return {
+            argv: ["providers", a1, a2, ...args.slice(3)],
+            legacyUsed: true,
+            ruleId: `provider-${a1}`,
+            oldCmd: args.join(" "),
+            newCmd: `providers ${a1} ${a2}`,
+        };
+    }
+    if (a0 === "provider" && a1 === "model" && a2 === "show" && hasPositional(a3) && hasPositional(a4)) {
+        const modelRef = joinModelRef(a3, a4);
+        return {
+            argv: ["models", "show", modelRef, ...args.slice(5)],
+            legacyUsed: true,
+            ruleId: "provider-model-show",
+            oldCmd: args.join(" "),
+            newCmd: `models show ${modelRef}`,
+        };
+    }
+    if (a0 === "provider" && a1 === "model" && (a2 === "enable" || a2 === "disable") && hasPositional(a3) && hasPositional(a4)) {
+        const modelRef = joinModelRef(a3, a4);
+        return {
+            argv: ["models", a2, modelRef, ...args.slice(5)],
+            legacyUsed: true,
+            ruleId: `provider-model-${a2}`,
+            oldCmd: args.join(" "),
+            newCmd: `models ${a2} ${modelRef}`,
+        };
+    }
+    if (a0 === "provider" && a1 === "model" && a2 === "set-key" && hasPositional(a3) && hasPositional(a4)) {
+        const modelRef = joinModelRef(a3, a4);
+        return {
+            argv: ["models", "set-key", modelRef, ...args.slice(5)],
+            legacyUsed: true,
+            ruleId: "provider-model-set-key",
+            oldCmd: args.join(" "),
+            newCmd: `models set-key ${modelRef}`,
+        };
+    }
+    return { argv, legacyUsed: false };
+}

package/dist/cli/modelRef.js

@@ -0,0 +1,20 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.parseModelRef = parseModelRef;
+function parseModelRef(input) {
+    const trimmed = input.trim();
+    const slashIndex = trimmed.indexOf("/");
+    if (slashIndex <= 0 || slashIndex === trimmed.length - 1) {
+        return { modelId: trimmed };
+    }
+    const providerId = trimmed.slice(0, slashIndex).trim();
+    const modelId = trimmed.slice(slashIndex + 1).trim();
+    if (!providerId || !modelId) {
+        return { modelId: trimmed };
+    }
+    return {
+        providerId,
+        modelId,
+        canonical: `${providerId}/${modelId}`,
+    };
+}

package/dist/src/benchmark/artifacts.js

@@ -0,0 +1,131 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.writeBenchmarkArtifacts = writeBenchmarkArtifacts;
+const fs_1 = require("fs");
+const path_1 = __importDefault(require("path"));
+async function writeBenchmarkArtifacts(paths, report, outPath) {
+    const timestamp = timestampForFileName();
+    const defaultDir = path_1.default.join(paths.baseDir, "benchmarks");
+    const destination = outPath && outPath.trim().length > 0 ? path_1.default.resolve(outPath) : defaultDir;
+    let jsonPath;
+    let textPath;
+    if (destination.endsWith(".json")) {
+        jsonPath = destination;
+        textPath = destination.replace(/\.json$/i, ".txt");
+    }
+    else if (destination.endsWith(".txt")) {
+        textPath = destination;
+        jsonPath = destination.replace(/\.txt$/i, ".json");
+    }
+    else {
+        jsonPath = path_1.default.join(destination, `bench-${timestamp}.json`);
+        textPath = path_1.default.join(destination, `bench-${timestamp}.txt`);
+    }
+    await fs_1.promises.mkdir(path_1.default.dirname(jsonPath), { recursive: true });
+    await fs_1.promises.mkdir(path_1.default.dirname(textPath), { recursive: true });
+    await fs_1.promises.writeFile(jsonPath, JSON.stringify(report, null, 2), "utf8");
+    await fs_1.promises.writeFile(textPath, renderTextSummary(report), "utf8");
+    return { jsonPath, textPath };
+}
+function renderTextSummary(report) {
+    const lines = [];
+    lines.push("Waypoi Benchmark Report");
+    lines.push(`Run ID: ${report.id}`);
+    lines.push(`Created: ${report.createdAt}`);
+    lines.push(`Profile: ${report.profile}`);
+    lines.push(`Mode: ${report.executionMode}`);
+    if (report.suite)
+        lines.push(`Suite: ${report.suite}`);
+    if (report.exampleId)
+        lines.push(`Example: ${report.exampleId}`);
+    if (report.scenarioPath)
+        lines.push(`Scenario File: ${report.scenarioPath}`);
+    if (report.modelOverride)
+        lines.push(`Model Override: ${report.modelOverride}`);
+    lines.push("");
+    lines.push("Overall");
+    lines.push(`- Scenarios: ${report.total}`);
+    lines.push(`- Executed: ${report.executed}`);
+    lines.push(`- Skipped: ${report.skipped}`);
+    lines.push(`- Success: ${report.succeeded}`);
+    lines.push(`- Failed: ${report.failed}`);
+    lines.push(`- Success Rate: ${(report.successRate * 100).toFixed(1)}%`);
+    lines.push(`- Avg Latency: ${report.avgLatencyMs}ms`);
+    lines.push(`- P95 Latency: ${report.p95LatencyMs}ms`);
+    lines.push(`- Tokens: ${report.totalTokens}`);
+    lines.push(`- Tool Calls: ${report.totalToolCalls}`);
+    lines.push(`- Throughput: ${report.avgThroughputTokensPerSec.toFixed(2)} tokens/s`);
+    lines.push("");
+    lines.push("Gates");
+    lines.push(`- Hard: ${report.gateResults.hard.passed ? "PASS" : "FAIL"}`);
+    for (const message of report.gateResults.hard.messages) {
+        lines.push(`  - ${message}`);
+    }
+    lines.push(`- Soft: ${report.gateResults.soft.passed ? "PASS" : "WARN"}`);
+    for (const message of report.gateResults.soft.messages) {
+        lines.push(`  - ${message}`);
+    }
+    lines.push("");
+    lines.push("By Mode");
+    for (const [mode, summary] of Object.entries(report.modeSummary)) {
+        lines.push(`- ${mode}: total=${summary.total} executed=${summary.executed} skipped=${summary.skipped} passed=${summary.passed} failed=${summary.failed}`);
+    }
+    lines.push("");
+    lines.push("Scenario Results");
+    for (const scenario of report.results) {
+        if (scenario.status === "skipped") {
+            lines.push(`- ${scenario.id} [${scenario.mode}] SKIPPED: ${scenario.skippedReason ?? "not applicable"}`);
+            continue;
+        }
+        lines.push(`- ${scenario.id} [${scenario.mode}] ${scenario.status.toUpperCase()} passRate=${(scenario.passRate * 100).toFixed(1)}% p95=${scenario.p95LatencyMs}ms tokens=${scenario.totalTokens} tools=${scenario.totalToolCalls} attempts=${scenario.candidateAttempts} failovers=${scenario.failovers} rateLimitSwitches=${scenario.rateLimitSwitches}`);
+        for (const reason of scenario.errorReasons) {
+            lines.push(`  - ${reason}`);
+        }
+    }
+    if (report.scenarioDetails.length > 0) {
+        lines.push("");
+        lines.push("Showcase Details");
+        for (const detail of report.scenarioDetails) {
+            lines.push(`- ${detail.example?.title ?? detail.id} [${detail.status}]`);
+            lines.push(`  model=${detail.model}`);
+            lines.push(`  verdict=${detail.verdict}`);
+            if (detail.usedToolNames.length > 0) {
+                lines.push(`  tools=${detail.usedToolNames.join(", ")}`);
+            }
+            if (detail.finalResponsePreview) {
+                lines.push(`  final=${detail.finalResponsePreview}`);
+            }
+        }
+    }
+    if (report.topFailureReasons.length > 0) {
+        lines.push("");
+        lines.push("Top Failure Reasons");
+        for (const item of report.topFailureReasons) {
+            lines.push(`- ${item.reason} (${item.count})`);
+        }
+    }
+    if (report.warnings.length > 0) {
+        lines.push("");
+        lines.push("Warnings");
+        for (const warning of report.warnings) {
+            lines.push(`- ${warning}`);
+        }
+    }
+    if (report.capabilityMatrix && report.capabilityMatrix.models.length > 0) {
+        lines.push("");
+        lines.push("Capability Matrix");
+        lines.push(`- TTL Days: ${report.capabilityMatrix.ttlDays}`);
+        for (const model of report.capabilityMatrix.models) {
+            lines.push(`- ${model.model} (${model.freshness}) verified=${model.lastVerifiedAt}`);
+            lines.push(`  chat=${model.findings.chat_basic.status} tools=${model.findings.chat_tool_calls.status} embed=${model.findings.embeddings.status} image=${model.findings.images_generation.status} audio_in=${model.findings.audio_transcription.status} audio_out=${model.findings.audio_speech.status}`);
+        }
+    }
+    lines.push("");
+    return `${lines.join("\n")}\n`;
+}
+function timestampForFileName() {
+    return new Date().toISOString().replace(/[:.]/g, "-");
+}

package/dist/src/benchmark/capabilityClassifier.js

@@ -0,0 +1,81 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.classifyCapabilityStatus = classifyCapabilityStatus;
+const UNSUPPORTED_SIGNAL_PATTERNS = [
+    /unsupported/i,
+    /not\s+supported/i,
+    /not\s+implemented/i,
+    /unknown\s+parameter/i,
+    /invalid\s+parameter/i,
+    /unrecognized\s+parameter/i,
+    /does\s+not\s+support/i,
+    /modality\s+not\s+supported/i,
+    /endpoint\s+not\s+found/i,
+    /route\s+not\s+found/i,
+];
+const MISCONFIGURED_SIGNAL_PATTERNS = [
+    /api\s*key/i,
+    /missing\s+api\s*key/i,
+    /authorization/i,
+    /unauthorized/i,
+    /forbidden/i,
+    /invalid\s+auth/i,
+    /invalid\s+api\s*key/i,
+    /model\s+not\s+found/i,
+    /unknown\s+model/i,
+    /no\s+such\s+model/i,
+    /credential/i,
+    /token/i,
+];
+const TRANSIENT_SIGNAL_PATTERNS = [
+    /timeout/i,
+    /timed\s*out/i,
+    /econnreset/i,
+    /econnrefused/i,
+    /enotfound/i,
+    /network/i,
+    /temporar/i,
+    /rate\s*limit/i,
+    /too\s+many\s+requests/i,
+];
+function classifyCapabilityStatus(input) {
+    if (input.success) {
+        return "supported";
+    }
+    const statusCode = Number.isFinite(input.statusCode) ? input.statusCode : undefined;
+    const error = (input.error ?? "").toLowerCase();
+    if (statusCode === 401 || statusCode === 403) {
+        return "misconfigured";
+    }
+    if (statusCode === 404) {
+        if (MISCONFIGURED_SIGNAL_PATTERNS.some((pattern) => pattern.test(error))) {
+            return "misconfigured";
+        }
+        return "unsupported";
+    }
+    if (statusCode === 429) {
+        return "unknown";
+    }
+    if (statusCode !== undefined && statusCode >= 500) {
+        return "unknown";
+    }
+    if (statusCode === 400) {
+        if (MISCONFIGURED_SIGNAL_PATTERNS.some((pattern) => pattern.test(error))) {
+            return "misconfigured";
+        }
+        if (UNSUPPORTED_SIGNAL_PATTERNS.some((pattern) => pattern.test(error))) {
+            return "unsupported";
+        }
+        return "unknown";
+    }
+    if (MISCONFIGURED_SIGNAL_PATTERNS.some((pattern) => pattern.test(error))) {
+        return "misconfigured";
+    }
+    if (UNSUPPORTED_SIGNAL_PATTERNS.some((pattern) => pattern.test(error))) {
+        return "unsupported";
+    }
+    if (TRANSIENT_SIGNAL_PATTERNS.some((pattern) => pattern.test(error))) {
+        return "unknown";
+    }
+    return "unknown";
+}

package/dist/src/benchmark/capabilityStore.js

@@ -0,0 +1,144 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.capabilityCacheDir = capabilityCacheDir;
+exports.computeConfigFingerprint = computeConfigFingerprint;
+exports.writeCapabilitySnapshots = writeCapabilitySnapshots;
+exports.listCapabilitySnapshots = listCapabilitySnapshots;
+exports.getCapabilitySnapshotByModel = getCapabilitySnapshotByModel;
+exports.statusRank = statusRank;
+exports.toCapabilityMatrix = toCapabilityMatrix;
+const crypto_1 = require("crypto");
+const fs_1 = require("fs");
+const path_1 = __importDefault(require("path"));
+const DEFAULT_TTL_DAYS = 7;
+function capabilityCacheDir(paths) {
+    return path_1.default.join(paths.baseDir, "capabilities");
+}
+function computeConfigFingerprint(input) {
+    const raw = JSON.stringify(input ?? {});
+    return (0, crypto_1.createHash)("sha256").update(raw).digest("hex").slice(0, 16);
+}
+async function writeCapabilitySnapshots(paths, snapshots) {
+    const dir = capabilityCacheDir(paths);
+    await fs_1.promises.mkdir(dir, { recursive: true });
+    for (const snapshot of snapshots) {
+        const filePath = path_1.default.join(dir, capabilityFileName(snapshot.providerId, snapshot.modelId));
+        await fs_1.promises.writeFile(filePath, JSON.stringify(snapshot, null, 2), "utf8");
+    }
+}
+async function listCapabilitySnapshots(paths, ttlDays = DEFAULT_TTL_DAYS) {
+    const dir = capabilityCacheDir(paths);
+    let files = [];
+    try {
+        files = await fs_1.promises.readdir(dir);
+    }
+    catch (error) {
+        if (error.code === "ENOENT") {
+            return {
+                generatedAt: new Date().toISOString(),
+                ttlDays,
+                models: [],
+            };
+        }
+        throw error;
+    }
+    const snapshots = [];
+    for (const file of files) {
+        if (!file.endsWith(".json")) {
+            continue;
+        }
+        try {
+            const raw = await fs_1.promises.readFile(path_1.default.join(dir, file), "utf8");
+            const parsed = JSON.parse(raw);
+            const normalized = normalizeSnapshot(parsed, ttlDays);
+            if (normalized) {
+                snapshots.push(normalized);
+            }
+        }
+        catch {
+            // Ignore malformed cache entries.
+        }
+    }
+    snapshots.sort((a, b) => `${a.providerId}/${a.modelId}`.localeCompare(`${b.providerId}/${b.modelId}`));
+    return {
+        generatedAt: new Date().toISOString(),
+        ttlDays,
+        models: snapshots,
+    };
+}
+async function getCapabilitySnapshotByModel(paths, model, ttlDays = DEFAULT_TTL_DAYS) {
+    const all = await listCapabilitySnapshots(paths, ttlDays);
+    return all.models.find((item) => item.model === model || `${item.providerId}/${item.modelId}` === model) ?? null;
+}
+function capabilityFileName(providerId, modelId) {
+    return `${sanitizeSegment(providerId)}__${sanitizeSegment(modelId)}.json`;
+}
+function sanitizeSegment(value) {
+    return value.replace(/[^a-zA-Z0-9._-]/g, "_");
+}
+function normalizeSnapshot(input, ttlDays) {
+    if (!input || typeof input !== "object") {
+        return null;
+    }
+    if (!input.providerId || !input.modelId || !input.model || !input.lastVerifiedAt) {
+        return null;
+    }
+    const verifiedAtMs = Date.parse(input.lastVerifiedAt);
+    if (!Number.isFinite(verifiedAtMs)) {
+        return null;
+    }
+    const ttlMs = ttlDays * 24 * 60 * 60 * 1000;
+    const expiresAt = input.expiresAt && Number.isFinite(Date.parse(input.expiresAt))
+        ? input.expiresAt
+        : new Date(verifiedAtMs + ttlMs).toISOString();
+    const freshness = computeFreshness(expiresAt);
+    return {
+        model: input.model,
+        providerId: input.providerId,
+        modelId: input.modelId,
+        configFingerprint: input.configFingerprint ?? "unknown",
+        confidence: clamp01(input.confidence ?? 0),
+        lastVerifiedAt: input.lastVerifiedAt,
+        expiresAt,
+        freshness,
+        findings: (input.findings ?? {}),
+    };
+}
+function computeFreshness(expiresAt) {
+    const expiryMs = Date.parse(expiresAt);
+    if (!Number.isFinite(expiryMs)) {
+        return "stale";
+    }
+    return Date.now() <= expiryMs ? "fresh" : "stale";
+}
+function clamp01(value) {
+    if (!Number.isFinite(value))
+        return 0;
+    if (value < 0)
+        return 0;
+    if (value > 1)
+        return 1;
+    return Number(value.toFixed(3));
+}
+function statusRank(status) {
+    switch (status) {
+        case "supported":
+            return 4;
+        case "unsupported":
+            return 3;
+        case "misconfigured":
+            return 2;
+        case "unknown":
+            return 1;
+    }
+}
+function toCapabilityMatrix(result) {
+    return {
+        generatedAt: result.generatedAt,
+        ttlDays: result.ttlDays,
+        models: result.models,
+    };
+}