tewip-ai 0.18.5 → 0.19.0

package/README.md

@@ -7,7 +7,7 @@ It works for every stack: Node.js (node:test, Jest, Vitest, Mocha), Python (pyte
 | Where | How |
 |---|---|
 | **Terminal** | `npm install -g tewip-ai`, then `tewip explain report.xml`, `tewip ci owner/repo --pr 12`, `tewip mcp` |
-| **VS Code, Cursor, Windsurf, VSCodium** | the Tewip extension: registers Tewip with the editor's AI assistant and adds *Explain a test report* and *Why did CI fail on this branch?* |
+| **VS Code, Cursor, Windsurf, VSCodium** | the Tewip extension ([download](https://github.com/hmamut39/tewip/releases/tag/vscode-0.5.0)): registers Tewip with the editor's AI assistant and adds six commands - explain a report, why CI failed on this branch, the risk review of your change, which flaky tests to quarantine, the security check, and what your corrections add up to |
 | **JetBrains, Claude, other AI assistants** | add the MCP server `tewip mcp` (see [editor setup](#use-it-from-your-editor-any-ide-that-speaks-mcp)) |
 | **Any model** | OpenAI, Claude, Gemini, Azure, or a model on your own machine with Ollama (no account, nothing leaves the computer): `TEWIP_PROVIDER` |
 | **GitHub CI** | the Tewip Action: a comment on every pull request, an optional merge gate, verified fix PRs ([QUICKSTART](QUICKSTART.md)) |
@@ -96,6 +96,20 @@ The report reads the history cache of the branch it runs on, usually the default
 
 The repository needs "Allow GitHub Actions to create and approve pull requests" enabled for fix PRs. Pull requests opened by the workflow token don't trigger CI themselves; the agent verified each change by re-running the test before opening the PR. See [triage-agent-sandbox](https://github.com/hmamut39/triage-agent-sandbox) for a working example. If this repository is private, other repositories can only use the action when Settings → Actions → Access allows it.
 
+### The editor extension (VS Code, Cursor, Windsurf, VSCodium)
+
+Download **[tewip-0.5.0.vsix](https://github.com/hmamut39/tewip/releases/tag/vscode-0.5.0)**, then in the
+editor press `Ctrl+Shift+P` and run **Extensions: Install from VSIX…**. It needs the command line:
+`npm install -g tewip-ai`.
+
+Press `Ctrl+Shift+P` and type "Tewip" for: *Explain a test report…*, *Why did CI fail on this
+branch?*, *Risk review of this change*, *Which flaky tests to quarantine*, *Check secrets and
+dependencies*, and *What our corrections add up to*. The editor's AI assistant also picks Tewip up
+as an MCP server with nothing to configure.
+
+It is not in the Visual Studio Marketplace: publishing there now requires an Azure subscription,
+which means a credit card for a free extension. The file above installs exactly the same thing.
+
 ### 2. From any AI assistant, through MCP (the whole IT team)
 
 `mcp/server.ts` is a read-only [Model Context Protocol](https://modelcontextprotocol.io) server. Anyone on the team can ask their own assistant about test failures, in the editor or chat they already use:

package/dist/cli/help.js

@@ -100,6 +100,11 @@ Usage:
                                     same way more than once, which rule decided it, and the failures
                                     it came from. Says "not yet" rather than inventing a pattern
                                     from two cases. No model, no cost.
+  tewip attest [--run <id>] [--out attestation.md] [--json bundle.json]
+                                    What Tewip checked for a build, as one artifact: the claims with
+                                    their evidence and which layer decided each, what it refused to
+                                    decide and why, and what it did not check at all. Nothing enters
+                                    it that Tewip did not verify itself. No model, no cost.
   tewip mcp                         Serve Tewip over MCP (stdio) for an editor or AI assistant.
   tewip --version                   Print the version.
 

package/dist/cli/tewip.js

@@ -177,6 +177,11 @@ async function main() {
             runLearn({ root: process.cwd(), name: flag(args, 'name'), labels: flag(args, 'labels'), out: flag(args, 'out') });
             return;
         }
+        case 'attest': {
+            const { runAttest } = await import('../engine/attest/attest-command.js');
+            runAttest({ root: process.cwd(), name: flag(args, 'name'), runId: flag(args, 'run'), out: flag(args, 'out'), json: flag(args, 'json') });
+            return;
+        }
         case 'mcp':
             // An editor starts this from any folder, and an installed tool's own folder may be read-only:
             // the AI answer cache goes to the user's home folder, which is always writable.

package/dist/engine/attest/attest-command.js

@@ -0,0 +1,63 @@
+// `tewip attest`: the evidence for one run, as an artifact a build can carry.
+import { existsSync, mkdirSync, readdirSync, writeFileSync } from 'node:fs';
+import { dirname, join, resolve } from 'node:path';
+import { datasetFiles, readJsonl, readTriageRecords } from '../../reporters/store.js';
+import { ENGINE_VERSION } from '../engine.js';
+import { attestRun, renderAttestation } from './attest.js';
+/** Everything Tewip kept for one project, newest run last. */
+function history(root, only) {
+    const dir = join(root, '.tewip', 'history');
+    const projects = existsSync(dir) ? (only ? [only] : readdirSync(dir)) : [];
+    for (const project of projects) {
+        const files = datasetFiles(join(dir, project));
+        const runs = readJsonl(files.runs);
+        if (runs.length > 0)
+            return { project, runs, records: readTriageRecords(files.triage) };
+    }
+    return { project: only ?? 'this project', runs: [], records: [] };
+}
+/**
+ * What ran, in the words of the thing that ran. Only what is recorded counts: an attestation that
+ * listed checks Tewip merely supports would be the fabrication this whole phase exists to prevent.
+ */
+function whatWasChecked(records) {
+    const checked = [];
+    const notChecked = [];
+    checked.push('every failing test in the run was read, and each was given a cause or left undecided');
+    const byRules = records.filter((r) => r.diagnosis?.source === 'rules').length;
+    const byModel = records.filter((r) => r.diagnosis?.source === 'llm').length;
+    if (byRules > 0)
+        checked.push(`${byRules} decided by deterministic rules (${ENGINE_VERSION}), which call no model and give the same answer every time`);
+    if (byModel > 0)
+        checked.push(`${byModel} sent to the model, because no rule could settle them`);
+    // What Tewip is silent about unless it was asked, and therefore must say it did not do.
+    notChecked.push('the application itself: no security scan, dependency audit or web check is part of this bundle unless `tewip secure` ran and is quoted here');
+    notChecked.push('tests that passed: a passing test is counted, not examined');
+    notChecked.push('anything no test covers, which no test report can show');
+    return { checked, notChecked };
+}
+export function runAttest(cmd) {
+    const { project, runs, records } = history(cmd.root, cmd.name);
+    if (runs.length === 0) {
+        throw new Error('nothing to attest: no run has been kept here yet. Run `tewip explain <report>` first, so there is evidence to attest to.');
+    }
+    const run = cmd.runId ? runs.find((r) => r.runId === cmd.runId) : runs[runs.length - 1];
+    if (!run)
+        throw new Error(`no run called ${cmd.runId} was kept here`);
+    const mine = records.filter((r) => r.runId === run.runId);
+    const { checked, notChecked } = whatWasChecked(mine);
+    const attestation = attestRun(project, run, mine, { checked, notChecked, tool: 'tewip', engine: ENGINE_VERSION });
+    const markdown = renderAttestation(attestation);
+    console.log(markdown);
+    if (cmd.out) {
+        mkdirSync(dirname(resolve(cmd.out)), { recursive: true });
+        writeFileSync(resolve(cmd.out), `${markdown}\n`, 'utf8');
+        console.error(`\nWritten to ${cmd.out}`);
+    }
+    if (cmd.json) {
+        mkdirSync(dirname(resolve(cmd.json)), { recursive: true });
+        writeFileSync(resolve(cmd.json), `${JSON.stringify(attestation, null, 2)}\n`, 'utf8');
+        console.error(`Machine-readable bundle written to ${cmd.json}`);
+    }
+    return attestation;
+}

package/dist/engine/attest/attest.js

@@ -0,0 +1,89 @@
+const sourceOf = (record) => String(record.diagnosis?.source ?? '');
+/** The evidence line a person reads: the first thing the deciding layer actually said. */
+function evidenceOf(record) {
+    const evidence = record.diagnosis?.evidence ?? [];
+    const first = evidence.find((e) => e.message)?.message ?? '';
+    return first.slice(0, 300);
+}
+/**
+ * The bundle for one run. `records` and `run` are what Tewip itself wrote when it read the report,
+ * so everything here is traceable; nothing is inferred and nothing is re-judged.
+ */
+export function attestRun(project, run, records, { checked, notChecked, tool, engine }) {
+    const claims = [];
+    const refusals = [];
+    for (const record of records) {
+        const diagnosis = record.diagnosis;
+        const category = diagnosis?.category ?? 'UNKNOWN';
+        const decidedBy = sourceOf(record);
+        // Undecided is not a claim. It is the refusal that makes the rest of the bundle believable.
+        if (category === 'UNKNOWN' || !decidedBy) {
+            refusals.push({
+                what: `${record.testId} failed, and Tewip did not classify it`,
+                why: evidenceOf(record) || 'no rule matched and the model was not asked or did not answer',
+                runId: record.runId,
+            });
+            continue;
+        }
+        claims.push({
+            claim: `${record.testId} failed because of ${category}`,
+            evidence: evidenceOf(record) || 'no evidence recorded',
+            runId: record.runId,
+            decidedBy,
+            confidence: Number(diagnosis?.confidence ?? 0),
+        });
+    }
+    return {
+        project,
+        commit: run.gitSha ?? '',
+        runId: run.runId,
+        startedAt: run.startedAt,
+        finishedAt: run.finishedAt,
+        testsAttempted: Object.keys(run.attempts ?? {}).length,
+        failures: records.length,
+        checked,
+        notChecked,
+        claims,
+        refusals,
+        producedBy: { tool, engine, at: new Date().toISOString() },
+    };
+}
+/** The bundle as a person reads it. The refusals are not hidden at the bottom; they are the point. */
+export function renderAttestation(a) {
+    const when = a.finishedAt ? a.finishedAt.slice(0, 19).replace('T', ' ') : 'unknown';
+    const lines = [
+        `# What Tewip checked: ${a.project}`,
+        '',
+        `Run \`${a.runId}\`${a.commit ? ` at commit \`${a.commit.slice(0, 8)}\`` : ''}, finished ${when} UTC.`,
+        `${a.testsAttempted} test(s) attempted, ${a.failures} failed.`,
+        '',
+        '## What was checked',
+        '',
+        ...(a.checked.length > 0 ? a.checked.map((c) => `- ${c}`) : ['- nothing: this bundle makes no claims']),
+        '',
+        '## What was NOT checked',
+        '',
+        ...(a.notChecked.length > 0
+            ? a.notChecked.map((c) => `- ${c}`)
+            : ['- nothing is recorded as unchecked, which is itself worth doubting: a check that was never run cannot report itself']),
+        '',
+    ];
+    lines.push(`## Claims (${a.claims.length})`, '');
+    if (a.claims.length === 0)
+        lines.push('None: nothing failed, so there is nothing to explain.', '');
+    for (const claim of a.claims) {
+        lines.push(`- **${claim.claim}** — ${claim.decidedBy}, confidence ${claim.confidence.toFixed(2)}`);
+        lines.push(`  - Evidence: ${claim.evidence}`);
+        lines.push(`  - From run \`${claim.runId}\``);
+    }
+    lines.push('', `## Refused (${a.refusals.length})`, '');
+    lines.push(a.refusals.length > 0
+        ? 'Tewip could not decide these. They are listed because a tool that never says so cannot be trusted about the rest.'
+        : 'Nothing was left undecided in this run.', '');
+    for (const refusal of a.refusals) {
+        lines.push(`- **${refusal.what}**`);
+        lines.push(`  - Why: ${refusal.why}`);
+    }
+    lines.push('', '---', '', `Produced by ${a.producedBy.tool} (${a.producedBy.engine}) at ${a.producedBy.at.slice(0, 19).replace('T', ' ')} UTC.`, 'Every claim above names the run it came from and the layer that decided it. This is a statement', 'about the checks listed under "what was checked" and nothing else.');
+    return lines.join('\n');
+}

package/dist/engine/secure/reconcile.js

@@ -0,0 +1,42 @@
+/** The path a web finding says is being served, e.g. "/.env" from "GET http://host/.env answered". */
+function servedPath(item) {
+    if (item.kind !== 'web')
+        return null;
+    // Take the whole address and ask the URL parser for the path. Pulling the path out with a regular
+    // expression captured "//127.0.0.1:3211/.env", host and all, and then matched nothing.
+    const found = /GET\s+(\S+)\s+answered/.exec(item.why);
+    if (!found)
+        return null;
+    try {
+        return new URL(found[1]).pathname;
+    }
+    catch {
+        return found[1].startsWith('/') ? found[1] : null;
+    }
+}
+/** The file a secret finding is in, from "a database URL with a password in it in .env:1". */
+const fileOf = (item) => (item.kind === 'secret' ? (item.where.split(':')[0] ?? '') : '');
+/**
+ * Where one check changes what another means. Today that is one case, and it is the one that
+ * matters most: a secret sitting in a file the application serves.
+ */
+export function reconcile(items) {
+    const served = new Set(items
+        .map(servedPath)
+        .filter((p) => Boolean(p))
+        .map((p) => p.replace(/^\//, '').toLowerCase()));
+    if (served.size === 0)
+        return items;
+    return items.map((item) => {
+        const file = fileOf(item).replace(/\\/g, '/').toLowerCase();
+        if (!file || !served.has(file) || item.verdict === 'accepted')
+            return item;
+        return {
+            ...item,
+            verdict: 'real-risk',
+            severity: 'critical',
+            why: `${item.why} - but the running application serves /${file} to anyone who asks, so this credential is public`,
+            next: `treat this credential as leaked and rotate it now, then stop serving /${file}`,
+        };
+    });
+}

package/dist/engine/secure/secure-command.js

@@ -9,6 +9,7 @@ import { auditDependencies } from './deps.js';
 import { scanSecrets, sourceFiles } from './secrets.js';
 import { readAccepted, triageDependency, triageSecret, triageWeb, worstFirst } from './triage.js';
 import { isLocal, scanWeb } from './web.js';
+import { reconcile } from './reconcile.js';
 import { loadVerdicts, saidByAPerson, verdictFor } from '../learn/verdicts.js';
 import { readZapReport, zapAvailable, zapInstructions } from './zap.js';
 const CODE = /\.(?:[cm]?[jt]sx?|py|rb|go|java|kt|cs|php|rs|vue|svelte)$/;
@@ -98,6 +99,11 @@ export async function runSecure(cmd) {
     else if (cmd.url) {
         notChecked.push(`a full web scan. Tewip's own check is passive; for the deep one: ${zapInstructions(cmd.url, zapAvailable(root))}`);
     }
+    // One check can change what another means: a secret in a file the application serves is not
+    // "never left the machine", whatever git says about it.
+    const joined = reconcile(items);
+    items.length = 0;
+    items.push(...joined);
     // A person's decision about a finding wins over Tewip's, and says whose it was.
     const corrections = loadVerdicts(root);
     const decided = items.map((item) => {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "tewip-ai",
-  "version": "0.18.5",
+  "version": "0.19.0",
   "description": "Tewip finds why each failing test failed - real bug, flaky test, environment, test data or changed selector - for any stack, from the command line, CI, or your editor.",
   "keywords": [
     "test-failures",