tewip-ai 0.20.0 → 0.22.0

package/dist/cli/help.js

@@ -100,6 +100,17 @@ Usage:
                                     same way more than once, which rule decided it, and the failures
                                     it came from. Says "not yet" rather than inventing a pattern
                                     from two cases. No model, no cost.
+  tewip trace [--only "<ticket or title>"] [--out trace.md]
+                                    Is this requirement covered, and is it passing right now? Joins
+                                    the plans Tewip wrote to the tests that cover them and the runs
+                                    that attempted them. A scenario counts as covered only when a
+                                    real test can be named, and passing only when a real run
+                                    attempted it. No model, no cost.
+  tewip recall "<an error, or a question>" [--test "<id>"] [--limit 5]
+                                    What this project already concluded: earlier failures that look
+                                    like this one, what was decided, by which rule or which person,
+                                    and the run each came from. Searches the history Tewip keeps -
+                                    no model, no network, nothing to set up.
   tewip incident <sentry.json | otel.json | stack.txt> [--framework <id>] [--out incident.md]
                                     Turn a production incident into a test that proves it: reads
                                     Sentry, OpenTelemetry or a pasted stack trace, writes a test

package/dist/cli/tewip.js

@@ -177,6 +177,19 @@ async function main() {
             runLearn({ root: process.cwd(), name: flag(args, 'name'), labels: flag(args, 'labels'), out: flag(args, 'out') });
             return;
         }
+        case 'trace': {
+            const { runTrace } = await import('../engine/trace/trace-command.js');
+            runTrace({ root: process.cwd(), only: flag(args, 'only'), name: flag(args, 'name'), out: flag(args, 'out') });
+            return;
+        }
+        case 'recall': {
+            const question = args.filter((a, i) => !a.startsWith('--') && !['--test', '--limit', '--name', '--out'].includes(args[i - 1] ?? '')).join(' ').trim();
+            if (!question)
+                throw new Error('usage: tewip recall "<an error, or a question>" [--test "<test id>"] [--limit 5]');
+            const { runRecall } = await import('../engine/recall/recall-command.js');
+            runRecall({ root: process.cwd(), question, testId: flag(args, 'test'), limit: Number(flag(args, 'limit')) || undefined, name: flag(args, 'name'), out: flag(args, 'out') });
+            return;
+        }
         case 'incident': {
             const file = args.find((a, i) => !a.startsWith('--') && !['--framework', '--out'].includes(args[i - 1] ?? ''));
             if (!file)

package/dist/engine/recall/recall-command.js

@@ -0,0 +1,39 @@
+// `tewip recall "<an error, or a question>"`: what this project already concluded.
+import { existsSync, mkdirSync, readdirSync, writeFileSync } from 'node:fs';
+import { dirname, join, resolve } from 'node:path';
+import { datasetFiles, readTriageRecords } from '../../reporters/store.js';
+import { loadCorrections } from '../corrections.js';
+import { memories, recall, renderRecall } from './recall.js';
+/** Everything kept for this project: its failures, and the times a person overruled Tewip. */
+function kept(root, only) {
+    const dir = join(root, '.tewip', 'history');
+    if (!existsSync(dir))
+        return { records: [], corrections: [] };
+    const records = [];
+    const corrections = [];
+    for (const project of only ? [only] : readdirSync(dir)) {
+        const where = join(dir, project);
+        if (!existsSync(where))
+            continue;
+        records.push(...readTriageRecords(datasetFiles(where).triage));
+        corrections.push(...loadCorrections(where));
+    }
+    return { records, corrections };
+}
+export function runRecall(cmd) {
+    const { records, corrections } = kept(cmd.root, cmd.name);
+    if (records.length === 0) {
+        const empty = 'Nothing has been kept here yet, so there is no history to search. Run `tewip explain <report>` first.';
+        console.log(empty);
+        return empty;
+    }
+    const found = recall(cmd.question, memories(records, corrections), { testId: cmd.testId, limit: cmd.limit ?? 5 });
+    const report = renderRecall(cmd.question, found);
+    console.log(report);
+    console.error(`\nSearched ${records.length} kept failure(s) and ${corrections.length} correction(s). No model was called.`);
+    if (cmd.out) {
+        mkdirSync(dirname(resolve(cmd.out)), { recursive: true });
+        writeFileSync(resolve(cmd.out), `${report}\n`, 'utf8');
+    }
+    return report;
+}

package/dist/engine/recall/recall.js

@@ -0,0 +1,109 @@
+import { errorShape } from '../signature.js';
+/** Words worth matching on: rare, specific, not the furniture of every stack trace. */
+const COMMON = new Set([
+    'error', 'failed', 'failure', 'test', 'expected', 'received', 'actual', 'value', 'object', 'null', 'undefined', 'true', 'false', 'none', 'with', 'from', 'this', 'that', 'have', 'been', 'were', 'when', 'then', 'there', 'which', 'while', 'after', 'before', 'timeout', 'waiting', 'call', 'log', 'line', 'file', 'at', 'in', 'on', 'of', 'to', 'for', 'and', 'the', 'a', 'an', 'is', 'it', 'was', 'not', 'no', 'yes',
+]);
+const words = (text) => (errorShape(text).toLowerCase().match(/[a-z][a-z0-9_-]{2,}/g) ?? []).filter((w) => !COMMON.has(w));
+/** How rare each word is across the history: a word in every failure tells you nothing. */
+function rarity(all) {
+    const seen = new Map();
+    for (const doc of all)
+        for (const w of new Set(doc))
+            seen.set(w, (seen.get(w) ?? 0) + 1);
+    const total = Math.max(all.length, 1);
+    const out = new Map();
+    for (const [w, n] of seen)
+        out.set(w, Math.log(total / n) + 1);
+    return out;
+}
+/** Everything Tewip has kept about this project, as memories that can be cited. */
+export function memories(records, corrections = []) {
+    const byShape = new Map();
+    for (const c of corrections)
+        byShape.set(`${c.testId}::${c.errorShape}`, c);
+    const out = [];
+    for (const record of records) {
+        const diagnosis = record.diagnosis;
+        if (!diagnosis?.category)
+            continue;
+        const correction = byShape.get(`${record.testId}::${errorShape(record.errorMessage)}`);
+        out.push({
+            testId: record.testId,
+            category: correction?.category ?? diagnosis.category,
+            decidedBy: correction ? `${correction.by} (a person)` : String(diagnosis.source ?? 'unknown'),
+            when: record.timestamp,
+            runId: record.runId,
+            error: (record.errorMessage ?? '').replace(/\s+/g, ' ').slice(0, 220),
+            full: (record.errorMessage ?? '').replace(/\s+/g, ' ').slice(0, 4000),
+            ...(correction ? { correctedBy: correction.by, correctionReason: correction.reason } : {}),
+        });
+    }
+    return out;
+}
+/**
+ * The memories that answer a question, most useful first.
+ *
+ * Three things decide the order, and each is stated in `why` so a person can disagree with it: the
+ * same error shape is the strongest signal there is, the same test is next, and shared rare words
+ * carry the rest. A memory a person corrected is lifted above one only Tewip decided, because what
+ * the team concluded outranks what the tool concluded.
+ */
+export function recall(question, all, { testId, limit = 5 } = {}) {
+    if (all.length === 0)
+        return [];
+    const docs = all.map((m) => words(`${m.testId} ${m.full ?? m.error}`));
+    const weight = rarity(docs);
+    const asked = new Set(words(question));
+    const askedShape = errorShape(question);
+    const scored = all.map((memory, i) => {
+        const shared = docs[i].filter((w) => asked.has(w));
+        const mass = [...new Set(shared)].reduce((sum, w) => sum + (weight.get(w) ?? 1), 0);
+        const possible = [...asked].reduce((sum, w) => sum + (weight.get(w) ?? 1), 0) || 1;
+        let score = Math.min(mass / possible, 1);
+        const reasons = [];
+        if (askedShape && errorShape(memory.full ?? memory.error) === askedShape) {
+            score = Math.max(score, 0.95);
+            reasons.push('the same error, word for word once the changing parts are removed');
+        }
+        else if (shared.length > 0) {
+            reasons.push(`shares ${[...new Set(shared)].slice(0, 4).join(', ')}`);
+        }
+        if (testId && memory.testId === testId) {
+            score = Math.min(score + 0.25, 1);
+            reasons.push('the same test');
+        }
+        if (memory.correctedBy) {
+            score = Math.min(score + 0.1, 1);
+            reasons.push(`${memory.correctedBy} corrected this one`);
+        }
+        return { ...memory, score, sameTest: Boolean(testId && memory.testId === testId), why: reasons.join('; ') || 'nothing in common but the project' };
+    });
+    // Two memories can both reach the ceiling - an identical error recorded twice does - so the
+    // ordering needs a tie-break of its own. When the question is about one test, that test's own
+    // history comes first; otherwise the most recent, because the project changes underneath you.
+    return scored
+        .filter((m) => m.score > 0.15)
+        .sort((a, b) => b.score - a.score || Number(b.sameTest) - Number(a.sameTest) || (b.when > a.when ? 1 : -1))
+        .map(({ sameTest, ...rest }) => rest)
+        .slice(0, limit);
+}
+/** What a person reads. Every line names the run it came from, so nothing has to be believed. */
+export function renderRecall(question, found) {
+    const lines = [`# What this project already knows`, '', `Asked: ${question.replace(/\s+/g, ' ').slice(0, 160)}`, ''];
+    if (found.length === 0) {
+        lines.push('Nothing in this project\'s history resembles it. That is an answer, not a failure: it means', 'this is new here, and there is no earlier decision to follow.');
+        return lines.join('\n');
+    }
+    const corrected = found.filter((f) => f.correctedBy);
+    lines.push(`${found.length} earlier failure(s) look like this one.${corrected.length > 0 ? ` ${corrected.length} of them a person judged themselves.` : ''}`, '');
+    for (const memory of found) {
+        lines.push(`- **${memory.category}** — ${memory.testId}`);
+        lines.push(`  - Decided by ${memory.decidedBy}${memory.when ? ` on ${memory.when.slice(0, 10)}` : ''}, run \`${memory.runId}\``);
+        if (memory.correctionReason)
+            lines.push(`  - They said: ${memory.correctionReason}`);
+        lines.push(`  - Why it matched: ${memory.why}`);
+        lines.push(`  - The error then: ${memory.error.slice(0, 140)}`);
+    }
+    lines.push('', 'No model was called to answer this, and nothing above is a guess: each line is a record kept from a real run of this project.');
+    return lines.join('\n');
+}

package/dist/engine/trace/trace-command.js

@@ -0,0 +1,63 @@
+// `tewip trace`: is this requirement covered, and is it passing right now?
+import { existsSync, mkdirSync, readFileSync, readdirSync, writeFileSync } from 'node:fs';
+import { dirname, join, resolve } from 'node:path';
+import { datasetFiles, readJsonl, readTriageRecords } from '../../reporters/store.js';
+import { renderTrace, tracePlan } from './trace.js';
+/** The plans this project has kept, as `tewip plan` saved them. */
+function plans(root, only) {
+    const dir = join(root, '.tewip', 'plans');
+    if (!existsSync(dir))
+        return [];
+    const out = [];
+    for (const file of readdirSync(dir).filter((f) => f.endsWith('.json'))) {
+        try {
+            // `tewip plan` saves { plan, feature, urls, model }: the scenarios live under `plan`, and the
+            // feature - the words someone actually asked for - sits beside it. Reading the top level
+            // alone found no plans at all in a project that had one.
+            const saved = JSON.parse(readFileSync(join(dir, file), 'utf8'));
+            const inner = saved.plan ?? saved;
+            if (!Array.isArray(inner?.scenarios))
+                continue;
+            const parsed = { ...inner, feature: saved.feature ?? inner.feature, ticket: saved.ticket ?? inner.ticket };
+            const haystack = `${parsed.title ?? ''} ${parsed.feature ?? ''} ${parsed.ticket ?? ''} ${file}`.toLowerCase();
+            if (only && !haystack.includes(only.toLowerCase()))
+                continue;
+            out.push(parsed);
+        }
+        catch {
+            // A plan that is not readable JSON is not a plan; the Markdown beside it is still there.
+        }
+    }
+    return out;
+}
+function history(root, only) {
+    const dir = join(root, '.tewip', 'history');
+    if (!existsSync(dir))
+        return { runs: [], records: [] };
+    const runs = [];
+    const records = [];
+    for (const project of only ? [only] : readdirSync(dir)) {
+        const where = join(dir, project);
+        if (!existsSync(where))
+            continue;
+        runs.push(...readJsonl(datasetFiles(where).runs));
+        records.push(...readTriageRecords(datasetFiles(where).triage));
+    }
+    runs.sort((a, b) => (a.finishedAt > b.finishedAt ? 1 : -1));
+    return { runs, records };
+}
+export function runTrace(cmd) {
+    const found = plans(cmd.root, cmd.only);
+    const { runs, records } = history(cmd.root, cmd.name);
+    const traced = found.map((plan) => tracePlan(plan, runs, records));
+    const report = renderTrace(traced);
+    console.log(report);
+    if (found.length > 0) {
+        console.error(`\nTraced ${found.length} plan(s) against ${runs.length} kept run(s). No model was called.`);
+    }
+    if (cmd.out) {
+        mkdirSync(dirname(resolve(cmd.out)), { recursive: true });
+        writeFileSync(resolve(cmd.out), `${report}\n`, 'utf8');
+    }
+    return traced;
+}

package/dist/engine/trace/trace.js

@@ -0,0 +1,93 @@
+const clean = (s) => s.toLowerCase().replace(/[^a-z0-9]+/g, '');
+/**
+ * The test that covers a scenario, named from evidence rather than hope.
+ *
+ * Two ways a link is real: the plan says which existing test covers it - and that claim was already
+ * checked against the project's real test names when the plan was written - or a test file carries
+ * the scenario's own id, which is how `tewip write` names what it creates.
+ */
+export function testFor(scenario, attempted) {
+    const claim = (scenario.covered_by ?? '').trim();
+    if (claim) {
+        const exact = attempted.find((t) => t === claim);
+        if (exact)
+            return exact;
+        // A plan names a test as a person writes it; a report names it as the runner prints it.
+        const loose = attempted.find((t) => clean(t).includes(clean(claim.split(':')[0])) && clean(claim.split(':')[0]).length > 6);
+        if (loose)
+            return loose;
+        return claim; // named, but this project has not attempted it in any kept run
+    }
+    const id = clean(scenario.id);
+    return id.length >= 4 ? (attempted.find((t) => clean(t).includes(id)) ?? '') : '';
+}
+/** Joins one plan to what the runs actually say. Nothing here is inferred from anything else. */
+export function tracePlan(plan, runs, records) {
+    const newest = runs.at(-1);
+    const attempted = newest ? Object.keys(newest.attempts ?? {}) : [];
+    const everAttempted = new Set(runs.flatMap((r) => Object.keys(r.attempts ?? {})));
+    const failures = new Map();
+    for (const record of records)
+        if (record.runId === newest?.runId)
+            failures.set(record.testId, record);
+    const scenarios = plan.scenarios.map((scenario) => {
+        const test = testFor(scenario, attempted);
+        const base = { id: scenario.id, title: scenario.title, priority: scenario.priority ?? '', test, runId: newest?.runId ?? '', when: newest?.finishedAt ?? '' };
+        if (!test) {
+            return { ...base, standing: 'not-covered', diagnosis: '', why: 'no test in this project can be named for it, so nothing is checking it' };
+        }
+        if (!everAttempted.has(test)) {
+            return { ...base, standing: 'never-run', runId: '', when: '', diagnosis: '', why: `${test} is named as the cover, but no run kept here has attempted it` };
+        }
+        const failed = failures.get(test);
+        if (failed) {
+            const d = failed.diagnosis;
+            const diagnosis = d?.category ? `${d.category} at ${Number(d.confidence ?? 0).toFixed(2)} (${d.source ?? 'unknown'})` : 'failed, undiagnosed';
+            return { ...base, standing: 'failing', diagnosis, why: `${test} failed in the last run, and Tewip calls it ${d?.category ?? 'undiagnosed'}` };
+        }
+        if (!attempted.includes(test)) {
+            return { ...base, standing: 'never-run', diagnosis: '', why: `${test} ran before but not in the last run, so its standing now is unknown` };
+        }
+        return { ...base, standing: 'passing', diagnosis: '', why: `${test} was attempted in the last run and recorded no failure` };
+    });
+    return { requirement: plan.title || plan.feature || 'an unnamed requirement', ticket: plan.ticket ?? '', scenarios };
+}
+const MARK = { passing: 'passing', failing: 'FAILING', 'never-run': 'never run', 'not-covered': 'NOT COVERED' };
+/** The answer to "is this covered, and is it passing", written so it can be disagreed with. */
+export function renderTrace(traced) {
+    const all = traced.flatMap((t) => t.scenarios);
+    const count = (s) => all.filter((x) => x.standing === s).length;
+    const lines = ['# From what was asked for, to what is passing', ''];
+    if (all.length === 0) {
+        return [
+            '# From what was asked for, to what is passing',
+            '',
+            'No plan has been kept here, so there is nothing to trace. `tewip plan "<what it should do>"`',
+            'writes one, and `tewip write` turns its scenarios into tests.',
+        ].join('\n');
+    }
+    lines.push(`${all.length} scenario(s) across ${traced.length} requirement(s): ${count('passing')} passing, ${count('failing')} failing, ${count('never-run')} never run, ${count('not-covered')} not covered.`, '', 'A scenario counts as covered only when a real test can be named, and as passing only when a real', 'run attempted it. Everything else says what is missing instead.', '');
+    for (const requirement of traced) {
+        lines.push(`## ${requirement.requirement}${requirement.ticket ? ` (${requirement.ticket})` : ''}`, '');
+        for (const s of requirement.scenarios) {
+            lines.push(`- **${MARK[s.standing]}** — ${s.id}${s.priority ? ` (${s.priority})` : ''}: ${s.title}`);
+            lines.push(`  - ${s.why}`);
+            if (s.diagnosis)
+                lines.push(`  - Diagnosis: ${s.diagnosis}`);
+            // A run is only worth quoting when it says something about this scenario. Printing the last
+            // run beside "no test can be named for it" invited the reader to think one had checked it.
+            if (s.runId && s.test)
+                lines.push(`  - From run \`${s.runId}\`${s.when ? ` on ${s.when.slice(0, 10)}` : ''}`);
+        }
+        lines.push('');
+    }
+    const worst = all.filter((s) => s.standing === 'failing' || s.standing === 'not-covered');
+    if (worst.length > 0) {
+        lines.push('## What to do first', '');
+        for (const s of worst.slice(0, 5))
+            lines.push(`- ${s.standing === 'failing' ? `fix or triage ${s.test}` : `write a test for ${s.id}: ${s.title}`}`);
+        lines.push('');
+    }
+    lines.push('Every line above comes from a plan Tewip wrote and a run it recorded. Nothing is inferred.');
+    return lines.join('\n');
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "tewip-ai",
-  "version": "0.20.0",
+  "version": "0.22.0",
   "description": "Tewip finds why each failing test failed - real bug, flaky test, environment, test data or changed selector - for any stack, from the command line, CI, or your editor.",
   "keywords": [
     "test-failures",