tewip-ai 0.4.0

package/dist/engine/llm/client.js

@@ -0,0 +1,139 @@
+// LLM provider access. The engine only sees LlmClient, so the provider or model can change
+// (or be compared in the eval) without touching diagnosis code.
+import { createHash } from 'node:crypto';
+import { existsSync } from 'node:fs';
+import { join } from 'node:path';
+import { appendJsonl, DATA_DIR, PROJECT_ROOT, readJsonl } from '../../reporters/store.js';
+import { truncate } from '../signature.js';
+import { clientFor, modelChoiceFromEnv } from './providers.js';
+import { recordCacheHit, recordCall, recordRefusal } from './usage.js';
+export const DEFAULT_MODEL = 'gpt-5.4-mini';
+export const DEFAULT_REASONING = 'low';
+const REQUEST_TIMEOUT_MS = 90_000;
+const MAX_ATTEMPTS = 3;
+/** TEWIP_LLM_CACHE moves the cache, e.g. into the project the command line runs in. */
+const CACHE_FILE = process.env.TEWIP_LLM_CACHE || join(DATA_DIR, 'llm-cache.jsonl');
+let envLoaded = false;
+/** Loads <project>/.env once. Variables already set in the environment win. */
+export function loadDotEnv() {
+    if (envLoaded)
+        return;
+    envLoaded = true;
+    const file = join(PROJECT_ROOT, '.env');
+    if (existsSync(file))
+        process.loadEnvFile(file);
+}
+export function openAiClient({ apiKey, model, reasoningEffort, baseUrl = 'https://api.openai.com/v1' }) {
+    return {
+        model: reasoningEffort ? `${model} (reasoning ${reasoningEffort})` : model,
+        async complete({ system, user, schemaName, schema }) {
+            const body = JSON.stringify({
+                model,
+                messages: [
+                    { role: 'system', content: system },
+                    { role: 'user', content: user },
+                ],
+                response_format: { type: 'json_schema', json_schema: { name: schemaName, strict: true, schema } },
+                ...(reasoningEffort ? { reasoning_effort: reasoningEffort } : {}),
+            });
+            const startedAt = Date.now();
+            for (let attempt = 1;; attempt++) {
+                let res;
+                try {
+                    res = await fetch(`${baseUrl}/chat/completions`, {
+                        method: 'POST',
+                        headers: { Authorization: `Bearer ${apiKey}`, 'Content-Type': 'application/json' },
+                        body,
+                        signal: AbortSignal.timeout(REQUEST_TIMEOUT_MS),
+                    });
+                }
+                catch (error) {
+                    if (attempt >= MAX_ATTEMPTS)
+                        throw new Error(`OpenAI request failed: ${String(error)}`);
+                    await sleep(2000 * attempt);
+                    continue;
+                }
+                if (res.ok) {
+                    const json = (await res.json());
+                    const u = json.usage;
+                    recordCall(Date.now() - startedAt, { prompt: u?.prompt_tokens, completion: u?.completion_tokens, cached: u?.prompt_tokens_details?.cached_tokens }, model);
+                    const message = json.choices?.[0]?.message;
+                    if (message?.refusal)
+                        throw new Error(`model refused: ${truncate(message.refusal, 200)}`);
+                    if (typeof message?.content !== 'string')
+                        throw new Error('model returned no content');
+                    return message.content;
+                }
+                const detail = truncate(await res.text(), 300);
+                if ((res.status !== 429 && res.status < 500) || attempt >= MAX_ATTEMPTS)
+                    throw new Error(`OpenAI API ${res.status}: ${detail}`);
+                await sleep(2000 * attempt);
+            }
+        },
+    };
+}
+const sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
+/**
+ * Caches responses in data/llm-cache.jsonl (gitignored), keyed by model + full request, so
+ * re-running the eval costs nothing and any prompt or context change misses the cache.
+ */
+export function cachedClient(inner, file = CACHE_FILE) {
+    let cache;
+    const entries = () => (cache ??= new Map(readJsonl(file).map((e) => [e.key, e.response])));
+    return {
+        model: inner.model,
+        async complete(request) {
+            const key = createHash('sha256').update(JSON.stringify([inner.model, request])).digest('hex');
+            const hit = entries().get(key);
+            if (hit !== undefined) {
+                recordCacheHit();
+                return hit;
+            }
+            const response = await inner.complete(request);
+            entries().set(key, response);
+            appendJsonl(file, [{ key, model: inner.model, response, at: new Date().toISOString() }]);
+            return response;
+        },
+    };
+}
+/**
+ * The configured client, or null when the LLM layer is off (no OPENAI_API_KEY, or
+ * TRIAGE_LLM=off). Settings: TRIAGE_LLM_MODEL, TRIAGE_LLM_REASONING, TRIAGE_LLM_CACHE=off.
+ */
+export function llmFromEnv() {
+    loadDotEnv();
+    if (process.env.TRIAGE_LLM === 'off')
+        return null;
+    const choice = modelChoiceFromEnv('triage');
+    if (!choice)
+        return null;
+    const client = clientFor(choice);
+    // The budget sits inside the cache: a cached answer never reaches it, so it is free, and only
+    // real calls to the model count against TRIAGE_LLM_MAX_CALLS.
+    const budget = Number(process.env.TRIAGE_LLM_MAX_CALLS);
+    const capped = Number.isFinite(budget) && budget > 0 ? budgetedClient(client, budget) : client;
+    return process.env.TRIAGE_LLM_CACHE === 'off' ? capped : cachedClient(capped);
+}
+/** Raised when the per-process call budget is used up; the pipeline keeps the rules' answer. */
+export class LlmBudgetError extends Error {
+}
+/**
+ * A cap on how many model calls one process may make (TRIAGE_LLM_MAX_CALLS). Cache hits are
+ * free and do not count. A guardrail for CI: a huge failed run cannot run up an unbounded bill.
+ */
+export function budgetedClient(inner, maxCalls) {
+    // Counted when the call starts, not when it returns: several requests run in parallel, and a
+    // budget that only counts finished calls would let a burst overshoot it.
+    let started = 0;
+    return {
+        model: inner.model,
+        async complete(request) {
+            if (started >= maxCalls) {
+                recordRefusal();
+                throw new LlmBudgetError(`AI call budget reached (TRIAGE_LLM_MAX_CALLS=${maxCalls}); keeping the rules' answer`);
+            }
+            started++;
+            return inner.complete(request);
+        },
+    };
+}

package/dist/engine/llm/context.js

@@ -0,0 +1,141 @@
+// Builds the structured summary sent to the model for one failure. Everything is clipped
+// and summarised; raw logs, traces and full DOM never leave this function.
+import { existsSync, readFileSync } from 'node:fs';
+import { join } from 'node:path';
+import { PROJECT_ROOT } from '../../reporters/store.js';
+import { errorShape, truncate } from '../signature.js';
+import { errorContextPath, pageSnapshotYaml } from '../snapshot.js';
+import { summarizeTrace } from '../trace/trace-summary.js';
+const CALL_LOG = /\n\s*Call log:/;
+const MAX_ARIA_LINES = 60;
+const ERROR_SCENARIO = /\b(rejects?|rejected|errors?|fail(s|ed|ures?)?|retr(y|ies|ying)|offline|timeouts?|invalid|denied|unauthori[sz]ed|forbidden|not allowed|unavailable|exhausting|transient|broken)\b/gi;
+/**
+ * The words in a test's name that show it deliberately provokes a failure (e.g. "reports a
+ * server failure", "recovers from a transient upload failure"), or null. In such tests a
+ * failed request is the scenario under test, not evidence of a broken environment.
+ */
+export function errorScenarioOf(testId) {
+    const title = testId.split(' > ').slice(-2).join(' > ');
+    const words = [...new Set((title.match(ERROR_SCENARIO) ?? []).map((w) => w.toLowerCase()))];
+    return words.length > 0 ? `test name mentions: ${words.join(', ')}` : null;
+}
+function errorPart(message) {
+    const [head, log = ''] = message.split(CALL_LOG);
+    const callLog = [...new Set(log.split('\n').map((l) => l.trim().replace(/^-\s*/, '')).filter(Boolean))];
+    return { message: truncate(head.trim(), 800), callLog: callLog.slice(0, 10).map((l) => truncate(l, 150)) };
+}
+function markdownBlock(markdown, heading) {
+    const block = markdown.match(new RegExp(`# ${heading}\\s*\`\`\`\\w*\\r?\\n([\\s\\S]*?)\`\`\``))?.[1];
+    return block === undefined ? null : block.split(/\r?\n/).filter((l) => l.trim() !== '');
+}
+/**
+ * Comments are removed from test code: in the synthetic suite they state the intended
+ * failure outright, which would turn the eval into reading the answer key.
+ */
+function stripComment(line) {
+    return line.replace(/(^|\s)\/\/.*$/, '').replace(/\/\*.*?\*\//g, '').trimEnd();
+}
+function failingCode(markdown) {
+    const lines = markdownBlock(markdown, 'Test source');
+    if (!lines)
+        return null;
+    const at = lines.findIndex((l) => /^>\s*\d+\s*\|/.test(l));
+    const window = at < 0 ? lines : lines.slice(Math.max(0, at - 5), at + 4);
+    return window
+        .filter((l) => !/^\s*\|\s*\^/.test(l)) // caret line repeats the error message
+        .map((l) => {
+        const [gutter, ...code] = l.split('|');
+        return `${gutter}|${stripComment(code.join('|'))}`;
+    })
+        .filter((l) => l.split('|')[1]?.trim());
+}
+/** Lines around the failing line of the record's test (comments removed), or null. */
+export function failingCodeOf(record) {
+    const file = errorContextPath(record);
+    return file ? failingCode(readFileSync(file, 'utf8')) : null;
+}
+function traceSummary(record) {
+    const rel = record.attachments.find((a) => a.name === 'trace')?.path;
+    if (!rel)
+        return { unavailable: 'no trace attached to this failure' };
+    const file = join(PROJECT_ROOT, rel);
+    if (!existsSync(file))
+        return { unavailable: 'trace file no longer on disk' };
+    try {
+        return summarizeTrace(file);
+    }
+    catch (error) {
+        return { unavailable: `trace could not be read: ${truncate(String(error), 120)}` };
+    }
+}
+/**
+ * Separates "flaky on the same code" from "broken by this commit": an overall failure rate
+ * that mixes commits reads as flakiness even when every attempt on this commit failed.
+ */
+export function historyPattern(h) {
+    if (h.currentRunFailures < h.currentRunAttempts)
+        return 'another attempt of the same code passed in this run: the failure is not deterministic';
+    if (h.sameCommitAttempts === 0)
+        return 'commit unknown: cannot compare with other commits';
+    if (h.sameCommitFailures < h.sameCommitAttempts) {
+        return `this commit both failed and passed (${h.sameCommitFailures}/${h.sameCommitAttempts} attempts failed): the failure is not deterministic`;
+    }
+    if (h.passesOnOtherCommits > 0 && h.failingRunsOnOtherCommits >= 2) {
+        // Saying only "it passed on other commits" invited REAL_BUG for tests that fail on and off
+        // all over the project, which was the AI layer's commonest mistake on real CI data (llm-6).
+        return `every attempt on this commit failed (${h.sameCommitFailures}/${h.sameCommitAttempts}); the test also failed in ${h.failingRunsOnOtherCommits} run(s) on other commits and passed ${h.passesOnOtherCommits} time(s) there`;
+    }
+    if (h.passesOnOtherCommits > 0) {
+        const alsoFailed = h.failingRunsOnOtherCommits > 0 ? `, though it also failed in ${h.failingRunsOnOtherCommits} run on another commit` : ' and never failed there';
+        return `every attempt on this commit failed (${h.sameCommitFailures}/${h.sameCommitAttempts}) while the test passed ${h.passesOnOtherCommits} time(s) on other commits${alsoFailed}: something in this commit likely changed behaviour`;
+    }
+    return `every recorded attempt failed (${h.sameCommitFailures}/${h.sameCommitAttempts}) and there is no passing run to compare with`;
+}
+export function buildFailureContext(record, history, run, rules) {
+    const contextFile = errorContextPath(record);
+    const markdown = contextFile ? readFileSync(contextFile, 'utf8') : '';
+    const aria = pageSnapshotYaml(markdown)
+        ?.split(/\r?\n/)
+        .filter((l) => l.trim() !== '')
+        .map((l) => l.replace(/\s*\[ref=[^\]]+\]/g, '')) ?? null;
+    const others = run.failures.filter((f) => f.testId !== record.testId);
+    return {
+        test: {
+            id: record.testId,
+            file: record.file,
+            status: record.status,
+            durationMs: record.duration,
+            retry: record.retry,
+            exercisesErrorHandling: errorScenarioOf(record.testId),
+        },
+        error: errorPart(record.errorMessage),
+        failingCode: markdown ? failingCode(markdown) : null,
+        history: {
+            pattern: historyPattern(history),
+            sameCommitAttempts: history.sameCommitAttempts,
+            sameCommitFailures: history.sameCommitFailures,
+            passesOnOtherCommits: history.passesOnOtherCommits,
+            failingRunsOnOtherCommits: history.failingRunsOnOtherCommits,
+            runs: history.runs,
+            failures: history.failures,
+            failureRatePct: Math.round(history.failureRate * 100),
+            distinctErrors: history.distinctErrors,
+            failingStreak: history.failingStreak,
+            passesBeforeStreak: history.passesBeforeStreak,
+            failuresBeforeStreak: history.failuresBeforeStreak,
+        },
+        thisRun: {
+            attempts: history.currentRunAttempts,
+            failedAttempts: history.currentRunFailures,
+            anotherAttemptPassed: history.currentRunFailures < history.currentRunAttempts,
+        },
+        run: {
+            testsAttempted: run.testsAttempted,
+            testsFailed: new Set(run.failures.map((f) => f.testId)).size,
+            otherFailures: others.slice(0, 8).map((f) => `${f.testId}: ${truncate(errorShape(f.errorMessage), 100)}`),
+        },
+        rules: rules.evidence.map((e) => `[${e.source}] ${e.message}`),
+        ariaSnapshot: aria ? aria.slice(0, MAX_ARIA_LINES) : null,
+        trace: traceSummary(record),
+    };
+}

package/dist/engine/llm/llm-diagnose.js

@@ -0,0 +1,48 @@
+// Phase-3 step for one failure the rules left UNKNOWN: ask the model, validate its answer
+// against the fixed taxonomy, and return a diagnosis. Any failure keeps the rules' UNKNOWN.
+import { recordFailure } from './usage.js';
+import { ROOT_CAUSES } from '../../reporters/types.js';
+import { truncate } from '../signature.js';
+import { OUTPUT_SCHEMA, OUTPUT_SCHEMA_NAME, PROMPT_VERSION, SYSTEM_PROMPT } from './prompt.js';
+/** Validates the model's JSON. Throws on anything outside the contract. */
+export function parseLlmOutput(raw) {
+    const value = JSON.parse(raw);
+    if (!ROOT_CAUSES.includes(value.category))
+        throw new Error(`category outside the taxonomy: ${String(value.category)}`);
+    if (typeof value.confidence !== 'number' || !Number.isFinite(value.confidence))
+        throw new Error('confidence is not a number');
+    if (typeof value.reasoning !== 'string')
+        throw new Error('reasoning is not a string');
+    if (!Array.isArray(value.evidence) || !value.evidence.every((e) => typeof e === 'string'))
+        throw new Error('evidence is not a string list');
+    return {
+        category: value.category,
+        confidence: Math.round(Math.min(1, Math.max(0, value.confidence)) * 100) / 100,
+        reasoning: truncate(value.reasoning, 600),
+        evidence: value.evidence.slice(0, 4).map((e) => truncate(e, 300)),
+    };
+}
+export async function llmDiagnose(context, rules, client, now = new Date()) {
+    try {
+        const raw = await client.complete({ system: SYSTEM_PROMPT, user: JSON.stringify(context), schemaName: OUTPUT_SCHEMA_NAME, schema: OUTPUT_SCHEMA });
+        const out = parseLlmOutput(raw);
+        return {
+            category: out.category,
+            confidence: out.confidence,
+            source: 'llm',
+            model: client.model,
+            engineVersion: `${rules.engineVersion}+${PROMPT_VERSION}`,
+            diagnosedAt: now.toISOString(),
+            evidence: [
+                { source: 'llm', message: out.reasoning },
+                ...out.evidence.map((message) => ({ source: 'llm', message })),
+                // Keep what the rules ruled out: the LLM decision sits on top of it.
+                ...rules.evidence,
+            ],
+        };
+    }
+    catch (error) {
+        recordFailure();
+        return { ...rules, evidence: [...rules.evidence, { source: 'llm', message: `LLM layer failed, left UNKNOWN: ${truncate(String(error), 200)}` }] };
+    }
+}

package/dist/engine/llm/prompt.js

@@ -0,0 +1,40 @@
+// Prompt and output contract for the phase-3 LLM layer. The schema is enforced by the
+// provider (strict JSON schema) and validated again locally before anything is stored.
+import { ROOT_CAUSES } from '../../reporters/types.js';
+/** Bump whenever the prompt, schema or context format changes; it is part of the cache key. */
+export const PROMPT_VERSION = 'llm-6';
+export const SYSTEM_PROMPT = `You are the root-cause classifier of a Playwright test-failure triage agent.
+You receive a structured summary of ONE failed test: the error, the test's run history, what the deterministic rules already checked, the test steps, the page at failure time (accessibility snapshot + notable DOM elements), failed network requests and console errors.
+
+Classify the root cause into exactly one of these categories:
+- REAL_BUG: the application under test behaves incorrectly. Examples: a wrong value is displayed; a feature or element the test needs is absent from the page and nothing equivalent exists; the app never reaches the expected state even though nothing external failed; uncaught page errors from application code.
+- LOCATOR_DRIFT: the element the test wants is on the page, but the selector no longer matches it (renamed id, class, test id or text). The page summary must show an element with the same purpose.
+- TIMING: the expected element or state does arrive, but not in time, or a race makes the result vary between attempts or runs. Signs: another attempt of the same code passed, intermittent history with varying errors, hard-coded waits, the element resolved but was not yet visible, stable or enabled.
+- ENV: something outside the app code and the test broke: failed or 5xx/502/503 responses from a backend or third-party service, connection or DNS errors, expired auth or credentials, or many tests in the same run failing the same way.
+- TEST_DATA: the test's preconditions or fixture data are wrong: missing or stale seeded data, a user or account in the wrong state, an expected value that depends on data the test did not set up, a 404 for a record the test assumed exists.
+
+How to decide:
+- Use only facts present in the summary. Do not invent page content, requests or history.
+- Prefer the category the evidence directly shows over one that is merely possible.
+- An element that is missing with no similar element on the page is not LOCATOR_DRIFT.
+- Playwright matches accessible names by substring, so a locator can match several elements at once ("strict mode violation"). That is LOCATOR_DRIFT only when one of the matched elements is the one the test means, which usually carries the locator's name as a whole word ("Edit" -> "Edit link"). When every match only contains the text inside a longer word or phrase ("Edit" -> "Editor canvas", "Gift" -> "Gifting options"), the element the test means is absent: judge it as REAL_BUG, TIMING or TEST_DATA on the other evidence, not LOCATOR_DRIFT.
+- A failed request or server error is only ENV evidence if the test did not provoke it. When "test.exercisesErrorHandling" is set, the test deliberately triggers failures (rejections, server errors, retries), so a failed request is the scenario under test: judge whether the app handled it as the test expects. If it did not, that is REAL_BUG, not ENV. A real environment outage also tends to hit unrelated tests and to pass on retry, rather than failing every attempt on one commit while other commits pass.
+- Check "thisRun" first. If anotherAttemptPassed is true, the same code both failed and passed, so the cause is not deterministic: a real bug or a drifted locator would fail every attempt. Choose among TIMING, ENV and TEST_DATA.
+- Then read "history.pattern", which compares this commit's attempts with other commits. If every attempt on this commit failed while other commits passed, something in this commit changed behaviour: choose among REAL_BUG, LOCATOR_DRIFT and TEST_DATA using the page summary, not TIMING. An overall failure rate that mixes different commits is not evidence of flakiness; only failing and passing on the same code is.
+- The "rules" section lists what the deterministic rules already examined and why they declined; treat those as checked facts.
+
+Confidence (0 to 1): 0.9 or higher only when the summary directly shows the cause; 0.6-0.8 when it is the most likely reading but another category remains plausible; below 0.5 when you are mostly guessing. Low confidence routes the failure to a human, so do not inflate it.
+
+Return JSON only: the category, the confidence, a short reasoning (at most 3 sentences), and 1-4 evidence items quoting or pointing at specific facts from the summary.`;
+export const OUTPUT_SCHEMA_NAME = 'triage_diagnosis';
+export const OUTPUT_SCHEMA = {
+    type: 'object',
+    additionalProperties: false,
+    required: ['category', 'confidence', 'reasoning', 'evidence'],
+    properties: {
+        category: { type: 'string', enum: [...ROOT_CAUSES] },
+        confidence: { type: 'number' },
+        reasoning: { type: 'string' },
+        evidence: { type: 'array', items: { type: 'string' } },
+    },
+};

package/dist/engine/llm/providers.js

@@ -0,0 +1,54 @@
+// Which model Tewip talks to, and how. The engine only ever sees LlmClient, so a team can run
+// Tewip on whatever they already pay for - or on nothing at all:
+//
+//   TEWIP_PROVIDER=openai|anthropic|google|azure|ollama|custom  (default: openai)
+//   TEWIP_API_KEY=...        the key for that provider (not needed for a local Ollama)
+//   TEWIP_MODEL=...          the model id, when the provider's default is not wanted
+//   TEWIP_BASE_URL=...       any OpenAI-compatible endpoint (OpenRouter, Groq, vLLM, LM Studio)
+//
+// The older OPENAI_API_KEY / OPENAI_BASE_URL / TRIAGE_LLM_MODEL names still work, so nothing that
+// already runs needs changing. With no key and no local server, the deterministic rules still
+// answer most failures on their own.
+import { anthropicClient } from './anthropic.js';
+import { openAiClient } from './client.js';
+export const PROVIDERS = {
+    openai: { defaultModel: 'gpt-5.4-mini', hint: 'create a key at platform.openai.com and set TEWIP_API_KEY' },
+    anthropic: { defaultModel: 'claude-haiku-4-5-20251001', hint: 'create a key at console.anthropic.com and set TEWIP_API_KEY' },
+    // Google publishes an OpenAI-compatible endpoint for Gemini.
+    google: { baseUrl: 'https://generativelanguage.googleapis.com/v1beta/openai', defaultModel: 'gemini-2.5-flash', hint: 'create a key at aistudio.google.com and set TEWIP_API_KEY' },
+    // Azure needs the full deployment URL, which only the team knows.
+    azure: { defaultModel: 'gpt-4o-mini', hint: 'set TEWIP_BASE_URL to your Azure endpoint (https://<resource>.openai.azure.com/openai/deployments/<deployment>) and TEWIP_API_KEY to its key' },
+    // Ollama runs models on your own machine: no account, no key, nothing leaves the computer.
+    ollama: { baseUrl: 'http://127.0.0.1:11434/v1', defaultModel: 'qwen3:8b', keyless: true, hint: 'install Ollama from ollama.com, then run: ollama pull qwen3:8b' },
+    custom: { defaultModel: 'gpt-4o-mini', hint: 'set TEWIP_BASE_URL to any OpenAI-compatible endpoint (OpenRouter, Groq, Together, vLLM, LM Studio) and TEWIP_API_KEY if it needs one' },
+};
+const providerOf = (name) => (name && name.toLowerCase() in PROVIDERS ? name.toLowerCase() : 'openai');
+/**
+ * What the environment asks for, or null when no model is configured at all. `purpose` only picks
+ * which model setting to read: triage uses a small fast model, writing a stronger one.
+ */
+export function modelChoiceFromEnv(purpose) {
+    const provider = providerOf(process.env.TEWIP_PROVIDER);
+    const spec = PROVIDERS[provider];
+    const apiKey = (process.env.TEWIP_API_KEY || process.env.OPENAI_API_KEY || '').trim();
+    const baseUrl = (process.env.TEWIP_BASE_URL || process.env.OPENAI_BASE_URL || spec.baseUrl || '').trim() || undefined;
+    if (!apiKey && !spec.keyless)
+        return null;
+    const model = purpose === 'write'
+        ? process.env.TEWIP_WRITE_MODEL || process.env.TEWIP_MODEL || (provider === 'openai' ? 'gpt-5.5' : spec.defaultModel)
+        : process.env.TRIAGE_LLM_MODEL || process.env.TEWIP_MODEL || spec.defaultModel;
+    const reasoningEffort = purpose === 'write' ? process.env.TEWIP_WRITE_REASONING || 'medium' : process.env.TRIAGE_LLM_REASONING || 'low';
+    // Reasoning effort is an OpenAI setting; other providers reject it.
+    return { provider, apiKey, model, baseUrl, reasoningEffort: provider === 'openai' || provider === 'azure' ? reasoningEffort : undefined };
+}
+/** A client for that choice: Anthropic has its own API shape, everything else speaks OpenAI's. */
+export function clientFor(choice) {
+    if (choice.provider === 'anthropic')
+        return anthropicClient({ apiKey: choice.apiKey, model: choice.model, baseUrl: choice.baseUrl });
+    return openAiClient({ apiKey: choice.apiKey || 'local', model: choice.model, reasoningEffort: choice.reasoningEffort, baseUrl: choice.baseUrl });
+}
+/** What to tell someone who has no model configured, naming every way to get one. */
+export function noModelHelp() {
+    const lines = Object.entries(PROVIDERS).map(([id, p]) => `  TEWIP_PROVIDER=${id.padEnd(9)} ${p.hint}`);
+    return ['No model is configured. Set TEWIP_API_KEY (and TEWIP_PROVIDER if not OpenAI):', ...lines, 'Free and private: TEWIP_PROVIDER=ollama runs a model on this machine, with no account and no data leaving it.'].join('\n');
+}

package/dist/engine/llm/usage.js

@@ -0,0 +1,69 @@
+const usage = { calls: 0, cacheHits: 0, failures: 0, promptTokens: 0, completionTokens: 0, totalMs: 0, refused: 0, costUsd: 0, unpriced: 0 };
+/**
+ * USD per million tokens (input, cached input, output), standard tier, from
+ * https://developers.openai.com/api/docs/pricing as read on 2026-09-19. The first
+ * matching prefix wins, so more specific names come first.
+ */
+const PRICES = [
+    ['gpt-5.6-astra', 10, 1, 50],
+    ['gpt-5.6-sol', 4, 0.4, 20],
+    ['gpt-5.6-terra', 2, 0.2, 12],
+    ['gpt-5.6-luna', 0.2, 0.02, 1.2],
+    ['gpt-5.5', 5, 0.5, 30],
+    ['gpt-5.4-mini', 0.75, 0.075, 4.5],
+    ['gpt-5.4-nano', 0.2, 0.02, 1.25],
+    ['gpt-5.4', 2.5, 0.25, 15],
+];
+export const llmUsage = () => ({ ...usage });
+export const resetLlmUsage = () => void Object.keys(usage).forEach((k) => (usage[k] = 0));
+export function recordCall(ms, tokens, model) {
+    usage.calls++;
+    usage.totalMs += ms;
+    const prompt = tokens?.prompt ?? 0;
+    const completion = tokens?.completion ?? 0;
+    usage.promptTokens += prompt;
+    usage.completionTokens += completion;
+    const price = priceOf(model);
+    if (!price) {
+        usage.unpriced++;
+        return;
+    }
+    const cached = Math.min(tokens?.cached ?? 0, prompt);
+    usage.costUsd += ((prompt - cached) * price.in + cached * price.cached + completion * price.out) / 1_000_000;
+}
+export const recordCacheHit = () => void usage.cacheHits++;
+export const recordFailure = () => void usage.failures++;
+export const recordRefusal = () => void usage.refused++;
+/** A team's own price (TRIAGE_LLM_PRICE_IN/OUT) wins; otherwise the published price of the model. */
+export function priceOf(model) {
+    const input = Number(process.env.TRIAGE_LLM_PRICE_IN);
+    const output = Number(process.env.TRIAGE_LLM_PRICE_OUT);
+    if (Number.isFinite(input) && Number.isFinite(output) && (input > 0 || output > 0))
+        return { in: input, cached: input, out: output };
+    const row = model ? PRICES.find(([prefix]) => model.startsWith(prefix)) : undefined;
+    return row ? { in: row[1], cached: row[2], out: row[3] } : null;
+}
+/** Dollars spent, or null when no call had a known price. */
+export function estimatedCostUsd(u = usage) {
+    return u.calls > u.unpriced ? u.costUsd : null;
+}
+/** One line for a report or job summary. Says "no AI calls" rather than printing zeros. */
+export function usageLine(u = llmUsage()) {
+    if (u.calls === 0 && u.cacheHits === 0)
+        return 'AI layer: no calls (rules only)';
+    const seconds = (u.totalMs / 1000).toFixed(1);
+    const perCall = u.calls > 0 ? ` (${Math.round(u.totalMs / u.calls)} ms each)` : '';
+    const tokens = `${u.promptTokens.toLocaleString('en-US')} in / ${u.completionTokens.toLocaleString('en-US')} out tokens`;
+    const cost = estimatedCostUsd(u);
+    const parts = [
+        `AI layer: ${u.calls} call(s) in ${seconds}s${perCall}`,
+        `${u.cacheHits} cache hit(s)`,
+        tokens,
+        cost === null ? (u.calls ? 'cost: unknown model price, set TRIAGE_LLM_PRICE_IN/OUT' : 'cost: $0 (all from cache)') : `cost: ~$${cost.toFixed(4)}${u.unpriced ? ` plus ${u.unpriced} call(s) at an unknown price` : ''}`,
+    ];
+    if (u.failures)
+        parts.push(`${u.failures} call(s) failed, kept the rules' answer`);
+    if (u.refused)
+        parts.push(`${u.refused} call(s) skipped: budget TRIAGE_LLM_MAX_CALLS reached`);
+    return parts.join(' · ');
+}

package/dist/engine/pipeline.js

@@ -0,0 +1,33 @@
+import { applyCorrection, correctionFor } from './corrections.js';
+import { diagnoseRecords } from './engine.js';
+import { buildFailureContext } from './llm/context.js';
+import { llmDiagnose } from './llm/llm-diagnose.js';
+async function mapLimit(items, limit, fn) {
+    const results = new Array(items.length);
+    let next = 0;
+    const worker = async () => {
+        while (next < items.length) {
+            const i = next++;
+            results[i] = await fn(items[i]);
+        }
+    };
+    await Promise.all(Array.from({ length: Math.min(limit, items.length) }, worker));
+    return results;
+}
+export async function diagnosePipeline(targets, index, llm, { concurrency = 4, now = new Date(), corrections = [] } = {}) {
+    const rules = diagnoseRecords(targets, index, now);
+    const corrected = targets.map((record) => (corrections.length > 0 ? correctionFor(record, corrections) : null));
+    const settled = rules.map((d, i) => (corrected[i] ? applyCorrection(d, corrected[i]) : d));
+    if (!llm)
+        return settled;
+    const unknown = targets.map((record, i) => ({ record, i })).filter(({ i }) => settled[i].category === 'UNKNOWN' && !corrected[i]);
+    const final = [...settled];
+    await mapLimit(unknown, concurrency, async ({ record, i }) => {
+        const context = buildFailureContext(record, index.historyAsOf(record.testId, record.runId), index.runContext(record.runId), rules[i]);
+        // A call that fails or is refused (budget, outage, bad response) keeps the rules' answer
+        // and records why: one bad call must never lose the whole run's triage.
+        const answer = await llmDiagnose(context, rules[i], llm, now);
+        final[i] = corrected[i] ? applyCorrection(answer, corrected[i]) : answer;
+    });
+    return final;
+}