tewip-ai 0.20.0 → 0.21.0

package/dist/cli/help.js

@@ -100,6 +100,11 @@ Usage:
                                     same way more than once, which rule decided it, and the failures
                                     it came from. Says "not yet" rather than inventing a pattern
                                     from two cases. No model, no cost.
+  tewip recall "<an error, or a question>" [--test "<id>"] [--limit 5]
+                                    What this project already concluded: earlier failures that look
+                                    like this one, what was decided, by which rule or which person,
+                                    and the run each came from. Searches the history Tewip keeps -
+                                    no model, no network, nothing to set up.
   tewip incident <sentry.json | otel.json | stack.txt> [--framework <id>] [--out incident.md]
                                     Turn a production incident into a test that proves it: reads
                                     Sentry, OpenTelemetry or a pasted stack trace, writes a test

package/dist/cli/tewip.js

@@ -177,6 +177,14 @@ async function main() {
             runLearn({ root: process.cwd(), name: flag(args, 'name'), labels: flag(args, 'labels'), out: flag(args, 'out') });
             return;
         }
+        case 'recall': {
+            const question = args.filter((a, i) => !a.startsWith('--') && !['--test', '--limit', '--name', '--out'].includes(args[i - 1] ?? '')).join(' ').trim();
+            if (!question)
+                throw new Error('usage: tewip recall "<an error, or a question>" [--test "<test id>"] [--limit 5]');
+            const { runRecall } = await import('../engine/recall/recall-command.js');
+            runRecall({ root: process.cwd(), question, testId: flag(args, 'test'), limit: Number(flag(args, 'limit')) || undefined, name: flag(args, 'name'), out: flag(args, 'out') });
+            return;
+        }
         case 'incident': {
             const file = args.find((a, i) => !a.startsWith('--') && !['--framework', '--out'].includes(args[i - 1] ?? ''));
             if (!file)

package/dist/engine/recall/recall-command.js

@@ -0,0 +1,39 @@
+// `tewip recall "<an error, or a question>"`: what this project already concluded.
+import { existsSync, mkdirSync, readdirSync, writeFileSync } from 'node:fs';
+import { dirname, join, resolve } from 'node:path';
+import { datasetFiles, readTriageRecords } from '../../reporters/store.js';
+import { loadCorrections } from '../corrections.js';
+import { memories, recall, renderRecall } from './recall.js';
+/** Everything kept for this project: its failures, and the times a person overruled Tewip. */
+function kept(root, only) {
+    const dir = join(root, '.tewip', 'history');
+    if (!existsSync(dir))
+        return { records: [], corrections: [] };
+    const records = [];
+    const corrections = [];
+    for (const project of only ? [only] : readdirSync(dir)) {
+        const where = join(dir, project);
+        if (!existsSync(where))
+            continue;
+        records.push(...readTriageRecords(datasetFiles(where).triage));
+        corrections.push(...loadCorrections(where));
+    }
+    return { records, corrections };
+}
+export function runRecall(cmd) {
+    const { records, corrections } = kept(cmd.root, cmd.name);
+    if (records.length === 0) {
+        const empty = 'Nothing has been kept here yet, so there is no history to search. Run `tewip explain <report>` first.';
+        console.log(empty);
+        return empty;
+    }
+    const found = recall(cmd.question, memories(records, corrections), { testId: cmd.testId, limit: cmd.limit ?? 5 });
+    const report = renderRecall(cmd.question, found);
+    console.log(report);
+    console.error(`\nSearched ${records.length} kept failure(s) and ${corrections.length} correction(s). No model was called.`);
+    if (cmd.out) {
+        mkdirSync(dirname(resolve(cmd.out)), { recursive: true });
+        writeFileSync(resolve(cmd.out), `${report}\n`, 'utf8');
+    }
+    return report;
+}

package/dist/engine/recall/recall.js

@@ -0,0 +1,109 @@
+import { errorShape } from '../signature.js';
+/** Words worth matching on: rare, specific, not the furniture of every stack trace. */
+const COMMON = new Set([
+    'error', 'failed', 'failure', 'test', 'expected', 'received', 'actual', 'value', 'object', 'null', 'undefined', 'true', 'false', 'none', 'with', 'from', 'this', 'that', 'have', 'been', 'were', 'when', 'then', 'there', 'which', 'while', 'after', 'before', 'timeout', 'waiting', 'call', 'log', 'line', 'file', 'at', 'in', 'on', 'of', 'to', 'for', 'and', 'the', 'a', 'an', 'is', 'it', 'was', 'not', 'no', 'yes',
+]);
+const words = (text) => (errorShape(text).toLowerCase().match(/[a-z][a-z0-9_-]{2,}/g) ?? []).filter((w) => !COMMON.has(w));
+/** How rare each word is across the history: a word in every failure tells you nothing. */
+function rarity(all) {
+    const seen = new Map();
+    for (const doc of all)
+        for (const w of new Set(doc))
+            seen.set(w, (seen.get(w) ?? 0) + 1);
+    const total = Math.max(all.length, 1);
+    const out = new Map();
+    for (const [w, n] of seen)
+        out.set(w, Math.log(total / n) + 1);
+    return out;
+}
+/** Everything Tewip has kept about this project, as memories that can be cited. */
+export function memories(records, corrections = []) {
+    const byShape = new Map();
+    for (const c of corrections)
+        byShape.set(`${c.testId}::${c.errorShape}`, c);
+    const out = [];
+    for (const record of records) {
+        const diagnosis = record.diagnosis;
+        if (!diagnosis?.category)
+            continue;
+        const correction = byShape.get(`${record.testId}::${errorShape(record.errorMessage)}`);
+        out.push({
+            testId: record.testId,
+            category: correction?.category ?? diagnosis.category,
+            decidedBy: correction ? `${correction.by} (a person)` : String(diagnosis.source ?? 'unknown'),
+            when: record.timestamp,
+            runId: record.runId,
+            error: (record.errorMessage ?? '').replace(/\s+/g, ' ').slice(0, 220),
+            full: (record.errorMessage ?? '').replace(/\s+/g, ' ').slice(0, 4000),
+            ...(correction ? { correctedBy: correction.by, correctionReason: correction.reason } : {}),
+        });
+    }
+    return out;
+}
+/**
+ * The memories that answer a question, most useful first.
+ *
+ * Three things decide the order, and each is stated in `why` so a person can disagree with it: the
+ * same error shape is the strongest signal there is, the same test is next, and shared rare words
+ * carry the rest. A memory a person corrected is lifted above one only Tewip decided, because what
+ * the team concluded outranks what the tool concluded.
+ */
+export function recall(question, all, { testId, limit = 5 } = {}) {
+    if (all.length === 0)
+        return [];
+    const docs = all.map((m) => words(`${m.testId} ${m.full ?? m.error}`));
+    const weight = rarity(docs);
+    const asked = new Set(words(question));
+    const askedShape = errorShape(question);
+    const scored = all.map((memory, i) => {
+        const shared = docs[i].filter((w) => asked.has(w));
+        const mass = [...new Set(shared)].reduce((sum, w) => sum + (weight.get(w) ?? 1), 0);
+        const possible = [...asked].reduce((sum, w) => sum + (weight.get(w) ?? 1), 0) || 1;
+        let score = Math.min(mass / possible, 1);
+        const reasons = [];
+        if (askedShape && errorShape(memory.full ?? memory.error) === askedShape) {
+            score = Math.max(score, 0.95);
+            reasons.push('the same error, word for word once the changing parts are removed');
+        }
+        else if (shared.length > 0) {
+            reasons.push(`shares ${[...new Set(shared)].slice(0, 4).join(', ')}`);
+        }
+        if (testId && memory.testId === testId) {
+            score = Math.min(score + 0.25, 1);
+            reasons.push('the same test');
+        }
+        if (memory.correctedBy) {
+            score = Math.min(score + 0.1, 1);
+            reasons.push(`${memory.correctedBy} corrected this one`);
+        }
+        return { ...memory, score, sameTest: Boolean(testId && memory.testId === testId), why: reasons.join('; ') || 'nothing in common but the project' };
+    });
+    // Two memories can both reach the ceiling - an identical error recorded twice does - so the
+    // ordering needs a tie-break of its own. When the question is about one test, that test's own
+    // history comes first; otherwise the most recent, because the project changes underneath you.
+    return scored
+        .filter((m) => m.score > 0.15)
+        .sort((a, b) => b.score - a.score || Number(b.sameTest) - Number(a.sameTest) || (b.when > a.when ? 1 : -1))
+        .map(({ sameTest, ...rest }) => rest)
+        .slice(0, limit);
+}
+/** What a person reads. Every line names the run it came from, so nothing has to be believed. */
+export function renderRecall(question, found) {
+    const lines = [`# What this project already knows`, '', `Asked: ${question.replace(/\s+/g, ' ').slice(0, 160)}`, ''];
+    if (found.length === 0) {
+        lines.push('Nothing in this project\'s history resembles it. That is an answer, not a failure: it means', 'this is new here, and there is no earlier decision to follow.');
+        return lines.join('\n');
+    }
+    const corrected = found.filter((f) => f.correctedBy);
+    lines.push(`${found.length} earlier failure(s) look like this one.${corrected.length > 0 ? ` ${corrected.length} of them a person judged themselves.` : ''}`, '');
+    for (const memory of found) {
+        lines.push(`- **${memory.category}** — ${memory.testId}`);
+        lines.push(`  - Decided by ${memory.decidedBy}${memory.when ? ` on ${memory.when.slice(0, 10)}` : ''}, run \`${memory.runId}\``);
+        if (memory.correctionReason)
+            lines.push(`  - They said: ${memory.correctionReason}`);
+        lines.push(`  - Why it matched: ${memory.why}`);
+        lines.push(`  - The error then: ${memory.error.slice(0, 140)}`);
+    }
+    lines.push('', 'No model was called to answer this, and nothing above is a guess: each line is a record kept from a real run of this project.');
+    return lines.join('\n');
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "tewip-ai",
-  "version": "0.20.0",
+  "version": "0.21.0",
   "description": "Tewip finds why each failing test failed - real bug, flaky test, environment, test data or changed selector - for any stack, from the command line, CI, or your editor.",
   "keywords": [
     "test-failures",