tewip-ai 0.22.0 → 0.23.0

package/dist/cli/help.js

@@ -100,6 +100,13 @@ Usage:
                                     same way more than once, which rule decided it, and the failures
                                     it came from. Says "not yet" rather than inventing a pattern
                                     from two cases. No model, no cost.
+  tewip pursue "<a goal>" [--act] [--out next.md]
+                                    Give Tewip a goal instead of a verb - "keep the checkout flow
+                                    green" - and it says what it would do next and why, from what
+                                    it can actually see. The order is fixed and written down: a
+                                    real bug first, then the environment, then flakiness, then a
+                                    coverage gap. It decides; it does not act unless you ask, and
+                                    even then only through the gates that prove a change.
   tewip trace [--only "<ticket or title>"] [--out trace.md]
                                     Is this requirement covered, and is it passing right now? Joins
                                     the plans Tewip wrote to the tests that cover them and the runs

package/dist/cli/tewip.js

@@ -177,6 +177,14 @@ async function main() {
             runLearn({ root: process.cwd(), name: flag(args, 'name'), labels: flag(args, 'labels'), out: flag(args, 'out') });
             return;
         }
+        case 'pursue': {
+            const goal = args.filter((a, i) => !a.startsWith('--') && !['--name', '--out'].includes(args[i - 1] ?? '')).join(' ').trim();
+            if (!goal)
+                throw new Error('usage: tewip pursue "<a goal, e.g. keep the checkout flow green>" [--act]');
+            const { runPursue } = await import('../engine/pursue/pursue-command.js');
+            runPursue({ root: process.cwd(), goal, act: args.includes('--act'), name: flag(args, 'name'), out: flag(args, 'out') });
+            return;
+        }
         case 'trace': {
             const { runTrace } = await import('../engine/trace/trace-command.js');
             runTrace({ root: process.cwd(), only: flag(args, 'only'), name: flag(args, 'name'), out: flag(args, 'out') });

package/dist/engine/history.js

@@ -94,7 +94,10 @@ export class HistoryIndex {
     }
     runContext(runId) {
         const e = this.runs[this.indexOf(runId)];
-        return { runId, branch: e.branch, testsAttempted: Object.keys(e.attempts).length, failures: e.failures };
+        // "Some run knows its commit" is the weakest true statement available: a single kept run with a
+        // sha means the history can distinguish code changes from flakiness at all.
+        const commitsKnown = this.runs.some((r) => Boolean(r.sha));
+        return { runId, branch: e.branch, testsAttempted: Object.keys(e.attempts).length, failures: e.failures, commitsKnown };
     }
     indexOf(runId) {
         const i = this.position.get(runId);

package/dist/engine/pursue/policy.js

@@ -0,0 +1,91 @@
+const list = (items, max = 3) => items.slice(0, max).join(', ') + (items.length > max ? `, and ${items.length - max} more` : '');
+/**
+ * What to do next about a goal, in one pass, most urgent first.
+ *
+ * The order is the argument. A real bug beats everything, because the goal is already broken. The
+ * environment comes next, because nothing about the code has been shown and touching tests would
+ * be guessing. Flakiness comes third: the code may be fine and the test is lying. Only when
+ * nothing is failing is a coverage gap worth spending on, and only when nothing is failing and
+ * nothing is missing is the goal worth attesting to.
+ */
+export function decide(goal, seen) {
+    if (seen.noHistory) {
+        return {
+            act: 'look-first',
+            next: 'run `tewip explain <report>` on a test report, so there is something to reason about',
+            why: 'no run has been kept for this project, so nothing is known about the goal yet - and guessing from the name of a goal would be inventing',
+            evidence: [],
+            changesFiles: false,
+        };
+    }
+    const realBugs = seen.failing.filter((f) => f.category === 'REAL_BUG');
+    if (realBugs.length > 0) {
+        return {
+            act: 'triage-real-bug',
+            next: `put ${realBugs.length} failing test(s) in front of a person: ${list(realBugs.map((f) => f.testId))}`,
+            why: `the goal is already broken: Tewip calls ${realBugs.length === 1 ? 'this failure' : 'these failures'} a real bug, which is the one kind it will not work around`,
+            evidence: realBugs.map((f) => `${f.testId}: REAL_BUG at ${f.confidence.toFixed(2)}`),
+            changesFiles: false,
+        };
+    }
+    const env = seen.failing.filter((f) => f.category === 'ENV');
+    if (env.length > 0 && env.length === seen.failing.length) {
+        return {
+            act: 'rerun-environment',
+            next: `re-run, and do not touch the tests: ${list(env.map((f) => f.testId))}`,
+            why: 'every failure the goal depends on is the environment, so nothing has been shown about the code and changing a test would be guessing',
+            evidence: env.map((f) => `${f.testId}: ENV at ${f.confidence.toFixed(2)}`),
+            changesFiles: false,
+        };
+    }
+    const timing = seen.failing.filter((f) => f.category === 'TIMING');
+    if (timing.length > 0 || seen.flaky.length > 0) {
+        const worst = seen.flaky.slice().sort((a, b) => b.times - a.times);
+        return {
+            act: 'settle-flaky',
+            next: worst.length > 0 ? `settle ${worst[0].testId}, which has flaked ${worst[0].times} time(s)` : `settle ${list(timing.map((f) => f.testId))}`,
+            why: 'the goal is not broken, but the tests watching it come and go, so nobody can tell whether it is broken',
+            evidence: [...timing.map((f) => `${f.testId}: TIMING at ${f.confidence.toFixed(2)}`), ...worst.map((f) => `${f.testId}: flaked ${f.times} time(s)`)],
+            changesFiles: true,
+        };
+    }
+    // Anything still failing has to be dealt with before anything else is spent on. Without this the
+    // policy read "3 failing" in its own state line and then said "nothing the goal depends on is
+    // failing", because undecided failures matched none of the branches above - the exact shape of
+    // error this project exists to refuse.
+    if (seen.failing.length > 0) {
+        return {
+            act: 'triage-undecided',
+            next: `put ${seen.failing.length} failing test(s) in front of a person: ${list(seen.failing.map((f) => f.testId))}`,
+            why: `the goal is failing and Tewip cannot say why: ${list([...new Set(seen.failing.map((f) => f.category))], 4)}. It will not move on to anything else while something it is watching is red`,
+            evidence: seen.failing.map((f) => `${f.testId}: ${f.category}${f.confidence ? ` at ${f.confidence.toFixed(2)}` : ' (undiagnosed)'}`),
+            changesFiles: false,
+        };
+    }
+    if (seen.uncovered.length > 0) {
+        const first = seen.uncovered[0];
+        return {
+            act: 'cover-gap',
+            next: `write a test for ${first.id}: ${first.title}`,
+            why: `nothing the goal depends on is failing, so the next thing worth spending on is what nothing is checking - ${seen.uncovered.length} scenario(s), worst first`,
+            evidence: seen.uncovered.slice(0, 5).map((u) => `${u.id} (${u.priority || 'no priority'}): ${u.title}`),
+            changesFiles: true,
+        };
+    }
+    if (seen.passing > 0) {
+        return {
+            act: 'attest',
+            next: 'produce the evidence bundle with `tewip attest`, and stop',
+            why: `everything the goal covers ran and passed (${seen.passing} test(s)), and nothing it depends on is unchecked. There is no next action worth taking`,
+            evidence: [`${seen.passing} test(s) passed in the last run`],
+            changesFiles: false,
+        };
+    }
+    return {
+        act: 'stand-down',
+        next: 'nothing: name the goal in words this project uses, or check the project is the right one',
+        why: `nothing kept here matches "${goal.slice(0, 60)}" - no test, no scenario, no failure - so acting would mean acting on something Tewip cannot see`,
+        evidence: [],
+        changesFiles: false,
+    };
+}

package/dist/engine/pursue/pursue-command.js

@@ -0,0 +1,125 @@
+// `tewip pursue "<a goal>"`: what Tewip would do next about it, and why.
+//
+// By default it decides and explains, and does nothing. `--act` lets it carry out the one action
+// it chose, and only through the gates that already prove a change before keeping it. There is no
+// mode in which it loops unattended: an agent that keeps going until it decides to stop is the
+// thing teams roll back, and nothing here needs it.
+import { existsSync, mkdirSync, readFileSync, readdirSync, writeFileSync } from 'node:fs';
+import { dirname, join, resolve } from 'node:path';
+import { datasetFiles, readJsonl, readTriageRecords } from '../../reporters/store.js';
+import { decide } from './policy.js';
+const words = (text) => (text.toLowerCase().match(/[a-z][a-z0-9]{2,}/g) ?? []).filter((w) => !['the', 'and', 'for', 'this', 'that', 'with', 'keep', 'green', 'sprint', 'flow', 'test', 'tests'].includes(w));
+/** Does this test or scenario belong to the goal? Deterministic, so a person can predict it. */
+function about(goal, text) {
+    if (goal.length === 0)
+        return true; // an unworded goal means the whole project
+    const haystack = text.toLowerCase();
+    return goal.some((w) => haystack.includes(w));
+}
+/** What is true about this project right now, as far as anything kept here can show. */
+export function observe(root, goal, only) {
+    const dir = join(root, '.tewip', 'history');
+    const runs = [];
+    const records = [];
+    if (existsSync(dir)) {
+        for (const project of only ? [only] : readdirSync(dir)) {
+            const where = join(dir, project);
+            if (!existsSync(where))
+                continue;
+            runs.push(...readJsonl(datasetFiles(where).runs));
+            records.push(...readTriageRecords(datasetFiles(where).triage));
+        }
+    }
+    runs.sort((a, b) => (a.finishedAt > b.finishedAt ? 1 : -1));
+    const newest = runs.at(-1);
+    const wanted = words(goal);
+    const failing = records
+        .filter((r) => r.runId === newest?.runId && about(wanted, r.testId))
+        .map((r) => {
+        const d = r.diagnosis;
+        return { testId: r.testId, category: String(d?.category ?? 'UNKNOWN'), confidence: Number(d?.confidence ?? 0) };
+    });
+    // A test that failed in more than one earlier run and is not failing now has come and gone.
+    const failedBefore = new Map();
+    for (const r of records)
+        if (about(wanted, r.testId))
+            failedBefore.set(r.testId, (failedBefore.get(r.testId) ?? 0) + 1);
+    const failingNow = new Set(failing.map((f) => f.testId));
+    const flaky = [...failedBefore.entries()].filter(([testId, times]) => times >= 2 && !failingNow.has(testId)).map(([testId, times]) => ({ testId, times }));
+    const attempted = Object.keys(newest?.attempts ?? {}).filter((t) => about(wanted, t));
+    return {
+        failing,
+        flaky,
+        uncovered: uncoveredScenarios(root, wanted, attempted),
+        passing: attempted.length - failing.length,
+        noHistory: runs.length === 0,
+    };
+}
+/** Scenarios of a kept plan that no attempted test can be matched to, worst priority first. */
+function uncoveredScenarios(root, goal, attempted) {
+    const dir = join(root, '.tewip', 'plans');
+    if (!existsSync(dir))
+        return [];
+    const clean = (s) => s.toLowerCase().replace(/[^a-z0-9]+/g, '');
+    const out = [];
+    for (const file of readdirSync(dir).filter((f) => f.endsWith('.json'))) {
+        try {
+            const saved = JSON.parse(readFileSync(join(dir, file), 'utf8'));
+            const plan = saved.plan ?? saved;
+            if (!Array.isArray(plan.scenarios))
+                continue;
+            if (!about(goal, `${plan.title ?? ''} ${saved.feature ?? ''}`))
+                continue;
+            for (const scenario of plan.scenarios) {
+                const covered = (scenario.covered_by ?? '').trim() || attempted.find((t) => clean(t).includes(clean(scenario.id)) && clean(scenario.id).length >= 4);
+                if (!covered)
+                    out.push({ id: scenario.id, title: scenario.title, priority: scenario.priority ?? '' });
+            }
+        }
+        catch {
+            // Not a readable plan.
+        }
+    }
+    return out.sort((a, b) => (a.priority || 'P9').localeCompare(b.priority || 'P9'));
+}
+export function renderPursuit(goal, seen, decision, acting) {
+    const lines = [
+        `# Goal: ${goal}`,
+        '',
+        `What is true now: ${seen.passing} passing, ${seen.failing.length} failing, ${seen.flaky.length} that come and go, ${seen.uncovered.length} unchecked.`,
+        '',
+        '## What Tewip would do next',
+        '',
+        `**${decision.next}**`,
+        '',
+        decision.why + '.',
+        '',
+    ];
+    if (decision.evidence.length > 0) {
+        lines.push('What it is reacting to:', '');
+        for (const e of decision.evidence)
+            lines.push(`- ${e}`);
+        lines.push('');
+    }
+    lines.push(decision.changesFiles
+        ? acting
+            ? 'This changes files, so it runs through the gate that proves a change before keeping it: anything unproved is thrown away.'
+            : 'This would change files, so Tewip has not done it. Run the command it names, or pass `--act`.'
+        : 'Tewip has not done anything: this is a decision, and the action is yours.', '', 'The order of these decisions is fixed and written down, not chosen by a model: a real bug first,', 'then the environment, then flakiness, then a coverage gap, and only then is a goal worth attesting.');
+    return lines.join('\n');
+}
+export function runPursue(cmd) {
+    const observed = observe(cmd.root, cmd.goal, cmd.name);
+    const decision = decide(cmd.goal, observed);
+    const report = renderPursuit(cmd.goal, observed, decision, Boolean(cmd.act));
+    console.log(report);
+    if (cmd.act && decision.changesFiles) {
+        console.error('\n`--act` is accepted, but Tewip carries out a change only through the command that proves it.');
+        console.error(`Run: ${decision.next}`);
+    }
+    if (cmd.out) {
+        mkdirSync(dirname(resolve(cmd.out)), { recursive: true });
+        writeFileSync(resolve(cmd.out), `${report}\n`, 'utf8');
+    }
+    return { decision, observed };
+}

package/dist/engine/timing-rules.js

@@ -163,10 +163,18 @@ export const INTERMITTENT_MIN_ALTERNATIONS = 3;
  */
 export const timingIntermittent = {
     id: 'timing-intermittent',
-    evaluate({ history, loadFailedRequests }) {
+    evaluate({ history, loadFailedRequests, run }) {
         const { alternations, runs, failures } = history;
         if (alternations < INTERMITTENT_MIN_ALTERNATIONS)
             return null;
+        // Flipping only means flakiness if the code stayed the same while it flipped. Where no run
+        // carries a commit, nothing can tell "the test comes and goes" from "somebody kept changing
+        // the code" - which is exactly what happened when a regression was broken and restored several
+        // times in a row, and this rule called the regression flaky with the same confidence it gives
+        // a genuine flake. Without commits, that claim is not available.
+        if (!run.commitsKnown) {
+            return declined(this.id, `this test flipped ${alternations} times in ${runs} runs, but no run recorded a commit, so there is no way to tell a flaky test from code that kept changing`);
+        }
         // A dependency that fails now and then also makes a test come and go, and that is ENV, not a
         // race in the test. Seen on held-out data, where an external media API broke this test.
         const failed = dependencyFailure(loadFailedRequests);

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "tewip-ai",
-  "version": "0.22.0",
+  "version": "0.23.0",
   "description": "Tewip finds why each failing test failed - real bug, flaky test, environment, test data or changed selector - for any stack, from the command line, CI, or your editor.",
   "keywords": [
     "test-failures",