tewip-ai 0.13.0 → 0.14.0

package/README.md

@@ -181,6 +181,22 @@ Each reader was checked against real reports published in public repositories, w
 defects were found before release - a Robot timestamp that is not a date, k6's numbers living under
 `values`, and a JMeter line that read "mostly 200" when an assertion had failed.
 
+## Tell it when it is wrong (new)
+
+```bash
+tewip correct diagnosis "checkout charges twice" TEST_DATA "the fixture has stale prices"
+tewip correct flaky "adds an item" fix-now "it is a real race, not flakiness"
+tewip correct security "src/config.ts:12" noise "that is a sample key in a fixture"
+tewip correct --list
+```
+
+Tewip answers with **your** decision from then on, and says it was yours: *"TEST_DATA, confidence
+1.00, source: corrected by a person"*. Corrections live in your project, so they belong to the team;
+the newest one wins, so changing your mind is just saying so again.
+
+**A correction changes what Tewip says, never what it does on its own** — it will not change code,
+quarantine a test or file a ticket because you corrected it.
+
 ## Build a feature, test first (new)
 
 ```bash

package/TEAM-GUIDE.md

@@ -127,6 +127,7 @@ collects environment failures in one issue, and (optionally) opens a fix pull re
 | `tewip spec "<feature>"` | An epic, its user stories and acceptance criteria, grounded in your code; files them in Jira with `--project PROJ --create` | Before the work is broken down |
 | `tewip report report.xml --gate` | The CI command for GitLab, Jenkins, CircleCI, Azure, Bitbucket and the rest: diagnoses the run, comments on the GitLab merge request, and fails the build only on failures that need a person | Your CI is not GitHub Actions |
 | `tewip watch --cmd "npm test"` | Runs your tests when you save a file and explains each failure straight away | While you are writing code |
+| `tewip correct <kind> "<what>" <verdict> "why"` | Tell Tewip it was wrong about anything it decides; it answers with your decision from then on | Whenever you disagree with it |
 | `tewip build "<what it should do>"` | Writes a feature test first, then the code, and keeps it only if the test failed without it and passes with it | You want a small feature built and proved |
 | `tewip review --diff main...HEAD` | Which tests cover your change, which have flaked before, what nothing tests, and what to run first | Before you open a pull request |
 | `tewip flaky --days 30` | Which tests to quarantine, which to fix, what they cost in CI minutes - and when a quarantined one can come back | Monday morning |

package/dist/cli/tewip.js

@@ -102,6 +102,11 @@ Usage:
                                     Compare a Figma frame against the running page: the words the
                                     design specifies and the controls its layer names imply. Says
                                     "the design says X, the page says Y". No model, no cost.
+  tewip correct <kind> "<what>" <verdict> ["why"] | --list
+                                    Tell Tewip it was wrong, about anything it decides: a diagnosis,
+                                    a flaky verdict, a security finding, a design difference, a
+                                    refused build. It answers with your decision from then on, and
+                                    says it was yours. It never acts on a correction by itself.
   tewip mcp                         Serve Tewip over MCP (stdio) for an editor or AI assistant.
   tewip --version                   Print the version.
 
@@ -397,6 +402,20 @@ ${proposed.status === 'opened' ? 'Opened a draft pull request.' : proposed.statu
             // it in CI without the build going red over a design decision nobody has made yet.
             return;
         }
+        case 'correct': {
+            const { runCorrect } = await import('../engine/learn/correct-command.js');
+            const words = args.filter((a) => !a.startsWith('--'));
+            console.log(runCorrect({
+                root: process.cwd(),
+                kind: words[0] ?? '',
+                key: words[1],
+                verdict: words[2],
+                reason: words.slice(3).join(' ') || undefined,
+                list: args.includes('--list'),
+                by: flag(args, 'by'),
+            }));
+            return;
+        }
         case 'mcp':
             // An editor starts this from any folder, and an installed tool's own folder may be read-only:
             // the AI answer cache goes to the user's home folder, which is always writable.

package/dist/engine/design/design-command.js

@@ -8,6 +8,7 @@ import { readPages } from '../quality/grounding.js';
 import { Figma, figmaHelp, figmaTokenFromEnv, parseFigmaUrl, topFrames } from './figma.js';
 import { frameChecks } from './frame.js';
 import { compareDesign, renderDrift } from './drift.js';
+import { loadVerdicts, saidByAPerson, verdictFor } from '../learn/verdicts.js';
 /** A saved Figma response, in any of the shapes Figma's API returns. */
 function fromFile(path) {
     const raw = JSON.parse(readFileSync(path, 'utf8'));
@@ -70,8 +71,22 @@ export async function runDesign(cmd) {
         console.log(`\nThe page was not read, so nothing was compared: ${pageNote ?? 'unknown reason'}`);
         return [];
     }
-    const drifts = compareDesign(checks, pages[0].aria);
-    const report = renderDrift(drifts, design, cmd.url);
+    const found = compareDesign(checks, pages[0].aria);
+    // A difference a person has already looked at and accepted is not reported again as a surprise.
+    const corrections = loadVerdicts(cmd.root);
+    const settled = [];
+    const drifts = found.map((d) => {
+        const key = `${d.check.role ?? d.check.kind} ${d.check.name}`.trim();
+        const said = verdictFor('design', key, corrections) ?? verdictFor('design', d.check.name, corrections);
+        if (!said || said.verdict !== 'not-a-difference')
+            return d;
+        settled.push(`${key}: ${saidByAPerson(said)}`);
+        return { ...d, verdict: 'met' };
+    });
+    const alreadySettled = settled.length
+        ? ['', `${settled.length} difference(s) your team has already settled, so they are not reported:`, ...settled.map((s) => `  - ${s}`)].join('\n')
+        : '';
+    const report = [renderDrift(drifts, design, cmd.url), alreadySettled].filter(Boolean).join('\n');
     console.log(report);
     if (note)
         console.log(`\n${note}`);

package/dist/engine/flaky/flaky-command.js

@@ -6,6 +6,7 @@ import { join, resolve } from 'node:path';
 import { datasetFiles, readRunSummaries, readTriageRecords } from '../../reporters/store.js';
 import { flakyTests, readyToRelease, weeklyCost } from './flaky.js';
 import { applyQuarantine, frameworkOf, quarantineFor } from './quarantine.js';
+import { loadVerdicts, saidByAPerson, verdictFor } from '../learn/verdicts.js';
 const DAY_MS = 24 * 60 * 60 * 1000;
 const minutes = (ms) => `${(ms / 60_000).toFixed(1)} CI min`;
 function projects(root, only) {
@@ -31,7 +32,14 @@ export async function runFlaky(cmd) {
         const history = datasetFiles(join(cmd.root, '.tewip', 'history', name));
         const records = readTriageRecords(history.triage);
         const runs = readRunSummaries(history.runs);
-        const tests = flakyTests(records, runs, { since });
+        const computed = flakyTests(records, runs, { since });
+        // What a person already decided about a test wins over what Tewip works out, and is shown as
+        // theirs: the argument is had once, not every Monday.
+        const corrections = loadVerdicts(cmd.root);
+        const tests = computed.map((test) => {
+            const said = verdictFor('flaky', test.title, corrections) ?? verdictFor('flaky', test.testId, corrections);
+            return said ? { ...test, verdict: said.verdict, why: `${saidByAPerson(said)} (Tewip had said ${test.verdict})` } : test;
+        });
         if (tests.length === 0)
             continue;
         all.push(...tests);

package/dist/engine/learn/correct-command.js

@@ -0,0 +1,88 @@
+// `tewip correct`: the one place a person tells Tewip it was wrong, about anything.
+//
+// A diagnosis can already be corrected where Tewip posts it in CI. This is the same thing for
+// everything else it decides, and on a person's own machine, where most of the disagreeing happens.
+import { existsSync, readdirSync } from 'node:fs';
+import { join } from 'node:path';
+import { ROOT_CAUSES } from '../../reporters/types.js';
+import { datasetFiles, readTriageRecords } from '../../reporters/store.js';
+import { loadCorrections, saveCorrections } from '../corrections.js';
+import { errorShape } from '../signature.js';
+import { isVerdictWord, loadVerdicts, saidByAPerson, saveVerdict, verdictHelp, VERDICT_WORDS } from './verdicts.js';
+const KINDS = Object.keys(VERDICT_WORDS);
+/**
+ * The failure a title names, and the history folder it lives in. The folder matters: corrections are
+ * kept beside the runs of that project, which is where the engine looks for them - an earlier
+ * version wrote them one directory up, where nothing ever read them.
+ */
+function findRecord(root, key) {
+    const dir = join(root, '.tewip', 'history');
+    if (!existsSync(dir))
+        return null;
+    for (const project of readdirSync(dir)) {
+        const historyDir = join(dir, project);
+        const records = readTriageRecords(datasetFiles(historyDir).triage);
+        const found = [...records].reverse().find((r) => r.testId.toLowerCase().includes(key.trim().toLowerCase()));
+        if (found)
+            return { record: found, historyDir };
+    }
+    return null;
+}
+export function runCorrect(cmd) {
+    const who = cmd.by ?? process.env.USER ?? process.env.USERNAME ?? 'someone on the team';
+    if (cmd.list) {
+        const verdicts = loadVerdicts(cmd.root);
+        const historyRoot = join(cmd.root, '.tewip', 'history');
+        const diagnoses = (existsSync(historyRoot) ? readdirSync(historyRoot) : []).flatMap((project) => loadCorrections(join(historyRoot, project)));
+        if (verdicts.length === 0 && diagnoses.length === 0)
+            return 'Nothing has been corrected yet.\n\n' + verdictHelp();
+        const lines = [`${verdicts.length + diagnoses.length} correction(s), newest last:`, ''];
+        for (const v of verdicts)
+            lines.push(`  ${v.kind.padEnd(9)} ${v.key} -> ${v.verdict}  (${saidByAPerson(v)})`);
+        for (const d of diagnoses)
+            lines.push(`  diagnosis ${d.testId} -> ${d.category}  (${d.by}${d.reason ? `: ${d.reason}` : ''})`);
+        return lines.join('\n');
+    }
+    const kind = cmd.kind?.trim().toLowerCase() ?? '';
+    if (!cmd.key || !cmd.verdict)
+        throw new Error(`usage: tewip correct <${[...KINDS, 'diagnosis'].join('|')}> "<what it is about>" <verdict> ["why"]\n\n${verdictHelp()}`);
+    // A diagnosis is corrected into the same store the CI path writes, so both ends agree.
+    if (kind === 'diagnosis') {
+        const category = cmd.verdict.trim().toUpperCase();
+        if (!ROOT_CAUSES.includes(category))
+            throw new Error(`"${cmd.verdict}" is not a category. They are: ${ROOT_CAUSES.join(', ')}`);
+        const found = findRecord(cmd.root, cmd.key);
+        if (!found)
+            throw new Error(`no failure of a test called "${cmd.key}" is in the history Tewip has kept here. Run \`tewip explain\` on the report first, so there is something to correct.`);
+        const { record, historyDir } = found;
+        const correction = {
+            schemaVersion: 1,
+            testId: record.testId,
+            // Stored exactly as the matcher compares it: a truncated shape can never equal a real one.
+            errorShape: errorShape(record.errorMessage),
+            category,
+            by: who,
+            at: new Date().toISOString(),
+            reason: cmd.reason,
+            source: 'tewip correct',
+            wasCategory: record.diagnosis?.category,
+        };
+        saveCorrections(historyDir, [correction]);
+        return [
+            `Noted: "${record.testId}" is ${category}${record.diagnosis?.category ? `, not ${record.diagnosis.category}` : ''}.`,
+            'Tewip will answer with your decision the next time that test fails the same way, and say it was yours.',
+        ].join('\n');
+    }
+    if (!KINDS.includes(kind))
+        throw new Error(`"${cmd.kind}" is not something Tewip decides. It decides: ${[...KINDS, 'diagnosis'].join(', ')}.\n\n${verdictHelp()}`);
+    const verdictKind = kind;
+    if (!isVerdictWord(verdictKind, cmd.verdict)) {
+        throw new Error(`"${cmd.verdict}" is not one of the answers a ${verdictKind} verdict takes. They are: ${VERDICT_WORDS[verdictKind].join(', ')}.`);
+    }
+    const saved = saveVerdict(cmd.root, { kind: verdictKind, key: cmd.key, verdict: cmd.verdict.trim().toLowerCase(), reason: cmd.reason ?? '', by: who });
+    return [
+        `Noted: ${verdictKind} "${cmd.key}" is ${saved.verdict}.`,
+        'Tewip will say that from now on, and say it was your decision rather than its own.',
+        'It will not act on it by itself: a correction changes what it says, never what it does.',
+    ].join('\n');
+}

package/dist/engine/learn/verdicts.js

@@ -0,0 +1,71 @@
+// The LEARN step, everywhere.
+//
+// A person could already correct a diagnosis by replying under Tewip's own comment in CI, and Tewip
+// answers with their decision from then on. Every other verdict it gives - this test should be
+// quarantined, this secret is a real risk, the page has drifted from the design, this feature
+// cannot be built here - had no such path, so the same argument had to be had again every week.
+//
+// This is that path, for all of them, and it keeps the properties that made the first one worth
+// having: a correction is written by a person, it is stored with the project rather than inside
+// Tewip, it says who and why, and **it never authorises an action on its own**. Deciding that a
+// failure is a real bug does not make Tewip change code; deciding a flaky test is really broken
+// does not make it un-quarantine anything. It changes what Tewip *says*, which is what a person
+// was disagreeing with.
+import { existsSync } from 'node:fs';
+import { join } from 'node:path';
+import { appendJsonl, readJsonl } from '../../reporters/store.js';
+/** The words each kind accepts, so a typo is caught rather than stored and silently ignored. */
+export const VERDICT_WORDS = {
+    flaky: ['quarantine', 'fix-now', 'watch', 'release'],
+    security: ['real-risk', 'not-reachable', 'noise', 'accepted'],
+    design: ['real', 'not-a-difference'],
+    build: ['belongs-here', 'not-this-project'],
+    review: ['covers', 'does-not-cover'],
+};
+export const VERDICTS_FILE = 'verdicts.jsonl';
+const fileIn = (root) => join(root, '.tewip', VERDICTS_FILE);
+export function loadVerdicts(root) {
+    const path = fileIn(root);
+    return existsSync(path) ? readJsonl(path) : [];
+}
+/** Adds one correction, replacing an earlier one about the same thing. */
+export function saveVerdict(root, correction) {
+    const entry = { schemaVersion: 1, at: correction.at ?? new Date().toISOString(), ...correction };
+    appendJsonl(fileIn(root), [entry]);
+    return entry;
+}
+const same = (a, b) => a.trim().toLowerCase() === b.trim().toLowerCase();
+/**
+ * The person's answer about this thing, or null. The newest wins, so a team that changes its mind
+ * simply says so again rather than editing a file.
+ */
+export function verdictFor(kind, key, corrections) {
+    const matching = corrections.filter((c) => c.kind === kind && (same(c.key, key) || key.toLowerCase().endsWith(c.key.trim().toLowerCase())));
+    return matching.length > 0 ? matching[matching.length - 1] : null;
+}
+/** What a corrected verdict is shown as, so nobody mistakes a person's decision for Tewip's. */
+export function saidByAPerson(correction) {
+    const when = correction.at.slice(0, 10);
+    return `${correction.by} decided this on ${when}${correction.reason ? `: ${correction.reason}` : ''}`;
+}
+/** Whether a word is one this kind of verdict uses. */
+export function isVerdictWord(kind, word) {
+    return VERDICT_WORDS[kind].includes(word.trim().toLowerCase());
+}
+export function verdictHelp() {
+    return [
+        'Correct something Tewip said, and it will answer with your decision from now on:',
+        '',
+        '  tewip correct flaky "adds an item to the cart" fix-now "it is a real race, not flakiness"',
+        '  tewip correct security "src/config.ts:12" noise "that is a sample key in a fixture"',
+        '  tewip correct design "button Sign in" not-a-difference "we renamed it on purpose"',
+        '  tewip correct build "an HTTP sink" belongs-here "this really is the right repository"',
+        '',
+        'The words each kind takes:',
+        ...Object.entries(VERDICT_WORDS).map(([kind, words]) => `  ${kind.padEnd(9)} ${words.join(', ')}`),
+        '',
+        'A correction changes what Tewip says, never what it does on its own: it will not change code,',
+        'quarantine a test or file a ticket because you corrected it.',
+        'They are kept in .tewip/verdicts.jsonl, with the project, so the whole team shares them.',
+    ].join('\n');
+}

package/dist/engine/secure/secure-command.js

@@ -9,6 +9,7 @@ import { auditDependencies } from './deps.js';
 import { scanSecrets, sourceFiles } from './secrets.js';
 import { readAccepted, triageDependency, triageSecret, triageWeb, worstFirst } from './triage.js';
 import { isLocal, scanWeb } from './web.js';
+import { loadVerdicts, saidByAPerson, verdictFor } from '../learn/verdicts.js';
 import { readZapReport, zapAvailable, zapInstructions } from './zap.js';
 const CODE = /\.(?:[cm]?[jt]sx?|py|rb|go|java|kt|cs|php|rs|vue|svelte)$/;
 export function renderSecurity(items, checked, notChecked, root) {
@@ -97,6 +98,14 @@ export async function runSecure(cmd) {
     else if (cmd.url) {
         notChecked.push(`a full web scan. Tewip's own check is passive; for the deep one: ${zapInstructions(cmd.url, zapAvailable(root))}`);
     }
+    // A person's decision about a finding wins over Tewip's, and says whose it was.
+    const corrections = loadVerdicts(root);
+    const decided = items.map((item) => {
+        const said = verdictFor('security', item.where, corrections) ?? verdictFor('security', item.title, corrections);
+        return said ? { ...item, verdict: said.verdict, why: `${saidByAPerson(said)} (Tewip had said ${item.verdict})` } : item;
+    });
+    items.length = 0;
+    items.push(...decided);
     items.sort(worstFirst);
     const report = renderSecurity(items, checked, notChecked, root);
     console.log(report);

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "tewip-ai",
-  "version": "0.13.0",
+  "version": "0.14.0",
   "description": "Tewip finds why each failing test failed - real bug, flaky test, environment, test data or changed selector - for any stack, from the command line, CI, or your editor.",
   "keywords": [
     "test-failures",