tewip-ai 0.21.0 → 0.22.0

package/dist/cli/help.js

@@ -100,6 +100,12 @@ Usage:
                                     same way more than once, which rule decided it, and the failures
                                     it came from. Says "not yet" rather than inventing a pattern
                                     from two cases. No model, no cost.
+  tewip trace [--only "<ticket or title>"] [--out trace.md]
+                                    Is this requirement covered, and is it passing right now? Joins
+                                    the plans Tewip wrote to the tests that cover them and the runs
+                                    that attempted them. A scenario counts as covered only when a
+                                    real test can be named, and passing only when a real run
+                                    attempted it. No model, no cost.
   tewip recall "<an error, or a question>" [--test "<id>"] [--limit 5]
                                     What this project already concluded: earlier failures that look
                                     like this one, what was decided, by which rule or which person,

package/dist/cli/tewip.js

@@ -177,6 +177,11 @@ async function main() {
             runLearn({ root: process.cwd(), name: flag(args, 'name'), labels: flag(args, 'labels'), out: flag(args, 'out') });
             return;
         }
+        case 'trace': {
+            const { runTrace } = await import('../engine/trace/trace-command.js');
+            runTrace({ root: process.cwd(), only: flag(args, 'only'), name: flag(args, 'name'), out: flag(args, 'out') });
+            return;
+        }
         case 'recall': {
             const question = args.filter((a, i) => !a.startsWith('--') && !['--test', '--limit', '--name', '--out'].includes(args[i - 1] ?? '')).join(' ').trim();
             if (!question)

package/dist/engine/trace/trace-command.js

@@ -0,0 +1,63 @@
+// `tewip trace`: is this requirement covered, and is it passing right now?
+import { existsSync, mkdirSync, readFileSync, readdirSync, writeFileSync } from 'node:fs';
+import { dirname, join, resolve } from 'node:path';
+import { datasetFiles, readJsonl, readTriageRecords } from '../../reporters/store.js';
+import { renderTrace, tracePlan } from './trace.js';
+/** The plans this project has kept, as `tewip plan` saved them. */
+function plans(root, only) {
+    const dir = join(root, '.tewip', 'plans');
+    if (!existsSync(dir))
+        return [];
+    const out = [];
+    for (const file of readdirSync(dir).filter((f) => f.endsWith('.json'))) {
+        try {
+            // `tewip plan` saves { plan, feature, urls, model }: the scenarios live under `plan`, and the
+            // feature - the words someone actually asked for - sits beside it. Reading the top level
+            // alone found no plans at all in a project that had one.
+            const saved = JSON.parse(readFileSync(join(dir, file), 'utf8'));
+            const inner = saved.plan ?? saved;
+            if (!Array.isArray(inner?.scenarios))
+                continue;
+            const parsed = { ...inner, feature: saved.feature ?? inner.feature, ticket: saved.ticket ?? inner.ticket };
+            const haystack = `${parsed.title ?? ''} ${parsed.feature ?? ''} ${parsed.ticket ?? ''} ${file}`.toLowerCase();
+            if (only && !haystack.includes(only.toLowerCase()))
+                continue;
+            out.push(parsed);
+        }
+        catch {
+            // A plan that is not readable JSON is not a plan; the Markdown beside it is still there.
+        }
+    }
+    return out;
+}
+function history(root, only) {
+    const dir = join(root, '.tewip', 'history');
+    if (!existsSync(dir))
+        return { runs: [], records: [] };
+    const runs = [];
+    const records = [];
+    for (const project of only ? [only] : readdirSync(dir)) {
+        const where = join(dir, project);
+        if (!existsSync(where))
+            continue;
+        runs.push(...readJsonl(datasetFiles(where).runs));
+        records.push(...readTriageRecords(datasetFiles(where).triage));
+    }
+    runs.sort((a, b) => (a.finishedAt > b.finishedAt ? 1 : -1));
+    return { runs, records };
+}
+export function runTrace(cmd) {
+    const found = plans(cmd.root, cmd.only);
+    const { runs, records } = history(cmd.root, cmd.name);
+    const traced = found.map((plan) => tracePlan(plan, runs, records));
+    const report = renderTrace(traced);
+    console.log(report);
+    if (found.length > 0) {
+        console.error(`\nTraced ${found.length} plan(s) against ${runs.length} kept run(s). No model was called.`);
+    }
+    if (cmd.out) {
+        mkdirSync(dirname(resolve(cmd.out)), { recursive: true });
+        writeFileSync(resolve(cmd.out), `${report}\n`, 'utf8');
+    }
+    return traced;
+}

package/dist/engine/trace/trace.js

@@ -0,0 +1,93 @@
+const clean = (s) => s.toLowerCase().replace(/[^a-z0-9]+/g, '');
+/**
+ * The test that covers a scenario, named from evidence rather than hope.
+ *
+ * Two ways a link is real: the plan says which existing test covers it - and that claim was already
+ * checked against the project's real test names when the plan was written - or a test file carries
+ * the scenario's own id, which is how `tewip write` names what it creates.
+ */
+export function testFor(scenario, attempted) {
+    const claim = (scenario.covered_by ?? '').trim();
+    if (claim) {
+        const exact = attempted.find((t) => t === claim);
+        if (exact)
+            return exact;
+        // A plan names a test as a person writes it; a report names it as the runner prints it.
+        const loose = attempted.find((t) => clean(t).includes(clean(claim.split(':')[0])) && clean(claim.split(':')[0]).length > 6);
+        if (loose)
+            return loose;
+        return claim; // named, but this project has not attempted it in any kept run
+    }
+    const id = clean(scenario.id);
+    return id.length >= 4 ? (attempted.find((t) => clean(t).includes(id)) ?? '') : '';
+}
+/** Joins one plan to what the runs actually say. Nothing here is inferred from anything else. */
+export function tracePlan(plan, runs, records) {
+    const newest = runs.at(-1);
+    const attempted = newest ? Object.keys(newest.attempts ?? {}) : [];
+    const everAttempted = new Set(runs.flatMap((r) => Object.keys(r.attempts ?? {})));
+    const failures = new Map();
+    for (const record of records)
+        if (record.runId === newest?.runId)
+            failures.set(record.testId, record);
+    const scenarios = plan.scenarios.map((scenario) => {
+        const test = testFor(scenario, attempted);
+        const base = { id: scenario.id, title: scenario.title, priority: scenario.priority ?? '', test, runId: newest?.runId ?? '', when: newest?.finishedAt ?? '' };
+        if (!test) {
+            return { ...base, standing: 'not-covered', diagnosis: '', why: 'no test in this project can be named for it, so nothing is checking it' };
+        }
+        if (!everAttempted.has(test)) {
+            return { ...base, standing: 'never-run', runId: '', when: '', diagnosis: '', why: `${test} is named as the cover, but no run kept here has attempted it` };
+        }
+        const failed = failures.get(test);
+        if (failed) {
+            const d = failed.diagnosis;
+            const diagnosis = d?.category ? `${d.category} at ${Number(d.confidence ?? 0).toFixed(2)} (${d.source ?? 'unknown'})` : 'failed, undiagnosed';
+            return { ...base, standing: 'failing', diagnosis, why: `${test} failed in the last run, and Tewip calls it ${d?.category ?? 'undiagnosed'}` };
+        }
+        if (!attempted.includes(test)) {
+            return { ...base, standing: 'never-run', diagnosis: '', why: `${test} ran before but not in the last run, so its standing now is unknown` };
+        }
+        return { ...base, standing: 'passing', diagnosis: '', why: `${test} was attempted in the last run and recorded no failure` };
+    });
+    return { requirement: plan.title || plan.feature || 'an unnamed requirement', ticket: plan.ticket ?? '', scenarios };
+}
+const MARK = { passing: 'passing', failing: 'FAILING', 'never-run': 'never run', 'not-covered': 'NOT COVERED' };
+/** The answer to "is this covered, and is it passing", written so it can be disagreed with. */
+export function renderTrace(traced) {
+    const all = traced.flatMap((t) => t.scenarios);
+    const count = (s) => all.filter((x) => x.standing === s).length;
+    const lines = ['# From what was asked for, to what is passing', ''];
+    if (all.length === 0) {
+        return [
+            '# From what was asked for, to what is passing',
+            '',
+            'No plan has been kept here, so there is nothing to trace. `tewip plan "<what it should do>"`',
+            'writes one, and `tewip write` turns its scenarios into tests.',
+        ].join('\n');
+    }
+    lines.push(`${all.length} scenario(s) across ${traced.length} requirement(s): ${count('passing')} passing, ${count('failing')} failing, ${count('never-run')} never run, ${count('not-covered')} not covered.`, '', 'A scenario counts as covered only when a real test can be named, and as passing only when a real', 'run attempted it. Everything else says what is missing instead.', '');
+    for (const requirement of traced) {
+        lines.push(`## ${requirement.requirement}${requirement.ticket ? ` (${requirement.ticket})` : ''}`, '');
+        for (const s of requirement.scenarios) {
+            lines.push(`- **${MARK[s.standing]}** — ${s.id}${s.priority ? ` (${s.priority})` : ''}: ${s.title}`);
+            lines.push(`  - ${s.why}`);
+            if (s.diagnosis)
+                lines.push(`  - Diagnosis: ${s.diagnosis}`);
+            // A run is only worth quoting when it says something about this scenario. Printing the last
+            // run beside "no test can be named for it" invited the reader to think one had checked it.
+            if (s.runId && s.test)
+                lines.push(`  - From run \`${s.runId}\`${s.when ? ` on ${s.when.slice(0, 10)}` : ''}`);
+        }
+        lines.push('');
+    }
+    const worst = all.filter((s) => s.standing === 'failing' || s.standing === 'not-covered');
+    if (worst.length > 0) {
+        lines.push('## What to do first', '');
+        for (const s of worst.slice(0, 5))
+            lines.push(`- ${s.standing === 'failing' ? `fix or triage ${s.test}` : `write a test for ${s.id}: ${s.title}`}`);
+        lines.push('');
+    }
+    lines.push('Every line above comes from a plan Tewip wrote and a run it recorded. Nothing is inferred.');
+    return lines.join('\n');
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "tewip-ai",
-  "version": "0.21.0",
+  "version": "0.22.0",
   "description": "Tewip finds why each failing test failed - real bug, flaky test, environment, test data or changed selector - for any stack, from the command line, CI, or your editor.",
   "keywords": [
     "test-failures",