tewip-ai 0.76.0 → 0.77.0

package/README.md

@@ -1252,6 +1252,14 @@ offline, in CI, before anything runs.
   because that is the difference between a wrong answer and a tool call somebody else chose. On
   1,691 files of real agent code: **zero findings**, which is the precision claim; recall is
   measured against planted examples and said to be.
+- **Which of these tools anybody wrote a test about.** Not "does this project have tests" - every
+  project has tests - but whether *this* tool is mentioned in any of them, and especially the ones
+  that cannot be undone. A name too short or too common to look for is reported as undecidable
+  rather than counted as covered. And the statement is made precisely in both directions: a tool
+  missing from the list is not a tool whose behaviour is untested, it is a tool nothing tests **by
+  that name** - the wiring between the name the model sees and the code behind it. Tewip finds 8 of
+  its own 17 that way. Also names which eval framework the project uses, and which it looked for
+  when it finds none.
 - **What the coding agent installed here was permitted to do.** `.claude/settings.json` and
   `.mcp.json` are committed, anybody with write access can change them, and neither turns up in a
   review the way a source file does. Tewip names the approval gate switched off

package/dist/engine/agent/agent-command.js

@@ -1,56 +1,13 @@
 // `tewip agent`: what this agent is allowed to do, and what stands between it and the real world.
-import { readFileSync, writeFileSync } from 'node:fs';
-import { relative } from 'node:path';
-import { scanSourceFiles } from '../walk.js';
-import { CAPABILITY_MEANS, FRAMEWORKS_READ, IRREVERSIBLE, toolsIn, trifectaIn } from './tools.js';
-import { promptFlows } from './prompt-flow.js';
-import { configSection, scanConfig } from './claude-config.js';
-import { budgetSetToNothing, guardsIn, GUARD_MEANS, GUARD_PREVENTS } from './guards.js';
+import { writeFileSync } from 'node:fs';
+import { CAPABILITY_MEANS, FRAMEWORKS_READ, IRREVERSIBLE, trifectaIn } from './tools.js';
+import {} from './prompt-flow.js';
+import { configSection } from './claude-config.js';
+import { coverageSection } from './tool-tests.js';
+import { GUARD_MEANS, GUARD_PREVENTS } from './guards.js';
+import { scanAgent } from './scan.js';
+export { scanAgent };
 const NL = String.fromCharCode(10);
-const AGENT_FILE = /\.(?:py|[cm]?[jt]sx?)$/i;
-/**
- * What is not the shipped agent.
- *
- * The first version of this named the directory `tests` and the file `*.test.ts`, which is what the
- * documentation of every project says and not what projects do. Run against the OpenAI Agents SDK,
- * all 24 "tools" it reported were fixtures in `integration_tests/` and `test_*.py` - a directory
- * whose name ends in `tests` rather than being `tests`, and a Python convention that puts the word
- * at the front of the file. A list of an agent's tools that is actually a list of test doubles is
- * worse than no list.
- *
- * `[\w-]+[_-]tests?` and not `[\w-]*tests?`, because the second one also excludes `latest/`.
- */
-const NOT_SHIPPED = /(?:^|[\\/])(?:tests?|[\w-]+[_-]tests?|__tests__|spec|e2e|examples?|samples?|docs?|fixtures?|node_modules|\.venv|venv)[\\/]|\.(?:test|spec)\.[cm]?[jt]sx?$|_test\.py$|(?:^|[\\/])test_[^\\/]*\.py$/i;
-export function scanAgent(root) {
-    const scan = scanSourceFiles(root, 20_000, (f) => AGENT_FILE.test(f));
-    const tools = [];
-    const injections = [];
-    const guards = [];
-    let emptyBudget = null;
-    let filesRead = 0;
-    for (const path of scan.files) {
-        const where = relative(root, path).replace(/\\/g, '/');
-        if (NOT_SHIPPED.test(where))
-            continue;
-        let text;
-        try {
-            text = readFileSync(path, 'utf8');
-        }
-        catch {
-            continue;
-        }
-        filesRead += 1;
-        tools.push(...toolsIn(where, text));
-        injections.push(...promptFlows(where, text));
-        guards.push(...guardsIn(where, text));
-        if (!emptyBudget) {
-            const empty = budgetSetToNothing(text);
-            if (empty)
-                emptyBudget = { file: where, text: empty };
-        }
-    }
-    return { tools, injections, guards, filesRead, emptyBudget, config: scanConfig(root) };
-}
 const REACH_READS = {
     destructive: 'cannot be undone',
     deletes: 'deletes something',
@@ -62,7 +19,7 @@ const REACH_READS = {
     'not-sure': '**not decided**',
 };
 export function renderAgent(scan, root) {
-    const { tools, injections, guards, filesRead, emptyBudget, config } = scan;
+    const { tools, injections, guards, filesRead, emptyBudget, config, coverage, evals } = scan;
     const lines = ['# What this agent is allowed to do', ''];
     if (tools.length === 0) {
         return [
@@ -146,6 +103,7 @@ export function renderAgent(scan, root) {
         lines.push(`- **${capability}** (${CAPABILITY_MEANS[capability]}): ${legTools.length === 0 ? 'nothing found' : legTools.slice(0, 6).map((t) => `\`${t.name}\``).join(', ') + (legTools.length > 6 ? `, and ${legTools.length - 6} more` : '')}`);
     }
     lines.push('');
+    lines.push(...coverageSection(coverage, tools, evals));
     lines.push(...configSection(config));
     lines.push('## What stands between them and the real world', '');
     for (const kind of ['asks-a-person', 'dry-run', 'step-budget', 'tool-allow-list']) {

package/dist/engine/agent/scan.js

@@ -0,0 +1,84 @@
+// Reading an agent project once: its tools, its flows, its guards, its tests and its permissions.
+//
+// Split from agent-command.ts, which is the file in this project that keeps arriving at the
+// 300-line limit. The rendering belongs with the thing it renders; the walk belongs here.
+import { readFileSync } from 'node:fs';
+import { join, relative } from 'node:path';
+import { scanSourceFiles } from '../walk.js';
+import { toolsIn } from './tools.js';
+import { promptFlows } from './prompt-flow.js';
+import { budgetSetToNothing, guardsIn } from './guards.js';
+import { scanConfig } from './claude-config.js';
+import { coverageOf, evalEvidence } from './tool-tests.js';
+const NL = String.fromCharCode(10);
+const AGENT_FILE = /\.(?:py|[cm]?[jt]sx?)$/i;
+/**
+ * What is not the shipped agent.
+ *
+ * The first version of this named the directory `tests` and the file `*.test.ts`, which is what the
+ * documentation of every project says and not what projects do. Run against the OpenAI Agents SDK,
+ * all 24 "tools" it reported were fixtures in `integration_tests/` and `test_*.py` - a directory
+ * whose name ends in `tests` rather than being `tests`, and a Python convention that puts the word
+ * at the front of the file. A list of an agent's tools that is actually a list of test doubles is
+ * worse than no list.
+ *
+ * `[\w-]+[_-]tests?` and not `[\w-]*tests?`, because the second one also excludes `latest/`.
+ */
+const NOT_SHIPPED = /(?:^|[\\/])(?:tests?|[\w-]+[_-]tests?|__tests__|spec|e2e|examples?|samples?|docs?|fixtures?|node_modules|\.venv|venv)[\\/]|\.(?:test|spec)\.[cm]?[jt]sx?$|_test\.py$|(?:^|[\\/])test_[^\\/]*\.py$/i;
+export function scanAgent(root) {
+    const scan = scanSourceFiles(root, 20_000, (f) => AGENT_FILE.test(f));
+    const tools = [];
+    const tests = [];
+    const injections = [];
+    const guards = [];
+    let emptyBudget = null;
+    let filesRead = 0;
+    for (const path of scan.files) {
+        const where = relative(root, path).replace(/\\/g, '/');
+        // Excluded from the tool scan, but not thrown away: these files are the evidence for whether
+        // anybody ever wrote a test about a tool, which is a different question from what ships.
+        if (NOT_SHIPPED.test(where)) {
+            try {
+                tests.push({ file: where, text: readFileSync(path, 'utf8') });
+            }
+            catch {
+                // A file that cannot be read is not evidence of a test, and is not counted as one.
+            }
+            continue;
+        }
+        let text;
+        try {
+            text = readFileSync(path, 'utf8');
+        }
+        catch {
+            continue;
+        }
+        filesRead += 1;
+        tools.push(...toolsIn(where, text));
+        injections.push(...promptFlows(where, text));
+        guards.push(...guardsIn(where, text));
+        if (!emptyBudget) {
+            const empty = budgetSetToNothing(text);
+            if (empty)
+                emptyBudget = { file: where, text: empty };
+        }
+    }
+    const manifests = ['package.json', 'pyproject.toml', 'requirements.txt'].flatMap((m) => {
+        try {
+            return [readFileSync(join(root, m), 'utf8')];
+        }
+        catch {
+            return [];
+        }
+    });
+    return {
+        tools,
+        injections,
+        guards,
+        filesRead,
+        emptyBudget,
+        coverage: coverageOf(tools.map((t) => t.name), tests),
+        evals: evalEvidence(tests, manifests),
+        config: scanConfig(root),
+    };
+}

package/dist/engine/agent/tool-tests.js

@@ -0,0 +1,87 @@
+/**
+ * Eval frameworks, by the name they are imported or depended on under.
+ *
+ * Named rather than guessed at. The point of listing them is to be able to say "this project has
+ * one, here it is" or "none of these is here" - and the second sentence is only honest if the list
+ * is written down where a reader can see what was looked for.
+ */
+const FRAMEWORKS = [
+    { name: 'DeepEval', re: /\bdeepeval\b/i },
+    { name: 'promptfoo', re: /\bpromptfoo\b/i },
+    { name: 'Ragas', re: /\bragas\b/i },
+    { name: 'LangSmith', re: /\blangsmith\b/i },
+    { name: 'Braintrust', re: /\bbraintrust\b/i },
+    { name: 'Inspect AI', re: /\binspect_ai\b/i },
+    { name: 'Giskard', re: /\bgiskard\b/i },
+    { name: 'TruLens', re: /\btrulens\b/i },
+    { name: 'Langfuse', re: /\blangfuse\b/i },
+    { name: 'OpenAI Evals', re: /\bopenai[-_]?evals\b/i },
+];
+/** A path that holds an eval set rather than a unit test. */
+const EVAL_PATH = /(?:^|[\\/])(?:evals?|benchmarks?|golden|datasets?)[\\/]|\.eval\.[cm]?[jt]s$|_eval\.py$|eval_[^\\/]*\.(?:py|[cm]?[jt]s)$|\bevals?\.ya?ml$/i;
+/**
+ * Names too common to decide on.
+ *
+ * A tool called `get` or `run` appears in every file in the project, so finding it in a test file
+ * says nothing at all. Rather than counting those as covered - which is the comfortable direction
+ * and the wrong one - they are reported as undecidable, by name.
+ */
+const COMMON = new Set(['get', 'set', 'run', 'add', 'list', 'read', 'write', 'call', 'send', 'find', 'query', 'search', 'name', 'tool', 'test', 'echo', 'exec', 'open', 'close', 'load', 'save']);
+const tooCommonName = (name) => name.length < 5 || COMMON.has(name.toLowerCase());
+export function coverageOf(toolNames, tests) {
+    return toolNames.map((name) => {
+        if (tooCommonName(name))
+            return { name, mentionedIn: [], tooCommon: true };
+        // Word boundaries on both sides: `delete_user` must not be matched by `delete_user_profile`,
+        // which is a different tool and may well be the only one anybody tested.
+        const mention = new RegExp(String.raw `(?<![\w-])${name.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')}(?![\w-])`);
+        return { name, mentionedIn: tests.filter((t) => mention.test(t.text)).map((t) => t.file), tooCommon: false };
+    });
+}
+export function evalEvidence(files, manifests) {
+    const frameworks = new Set();
+    for (const { name, re } of FRAMEWORKS) {
+        if (manifests.some((m) => re.test(m)) || files.some((f) => re.test(f.text)))
+            frameworks.add(name);
+    }
+    const evalFiles = files.filter((f) => EVAL_PATH.test(f.file)).map((f) => f.file);
+    return { frameworks: [...frameworks], files: [...new Set(evalFiles)].slice(0, 12) };
+}
+/** The names of every eval framework looked for, so silence about them can be read correctly. */
+export const FRAMEWORKS_LOOKED_FOR = FRAMEWORKS.map((f) => f.name);
+/**
+ * Which tools anybody wrote a test about, rendered where the logic for deciding it lives.
+ *
+ * Split out for the same reason the config section was: agent-command.ts is the file that keeps
+ * reaching the 300-line limit, and a report section belongs beside the thing it reports on.
+ */
+export function coverageSection(coverage, tools, evals) {
+    const lines = [];
+    const IRREV = ['destructive', 'deletes', 'pays', 'sends', 'runs-code'];
+    const untested = coverage.filter((c) => !c.tooCommon && c.mentionedIn.length === 0);
+    const riskyUntested = untested.filter((u) => IRREV.includes(tools.find((t) => t.name === u.name)?.reach ?? 'reads'));
+    lines.push('## Which of these anybody wrote a test about', '');
+    lines.push(untested.length === 0
+        ? 'Every tool whose name is distinctive enough to look for appears in at least one test file.'
+        : `**${untested.length} of ${tools.length}** appear in no test file${riskyUntested.length > 0 ? `, and ${riskyUntested.length} of those cannot be undone` : ''}.`, '');
+    if (untested.length > 0) {
+        for (const u of untested.slice(0, 15)) {
+            const reach = tools.find((t) => t.name === u.name)?.reach;
+            lines.push(`- \`${u.name}\`${reach && IRREV.includes(reach) ? ' — **and it cannot be undone**' : ''}`);
+        }
+        if (untested.length > 15)
+            lines.push(`…and ${untested.length - 15} more`);
+        lines.push('');
+    }
+    const undecidable = coverage.filter((c) => c.tooCommon);
+    if (undecidable.length > 0) {
+        lines.push(`Not decided for ${undecidable.length}: ${undecidable.slice(0, 8).map((u) => `\`${u.name}\``).join(', ')}. A name this`, 'short or this common appears everywhere, so finding it in a test file would prove nothing.', '');
+    }
+    lines.push(evals.frameworks.length > 0
+        ? `Eval framework in this project: ${evals.frameworks.join(', ')}.`
+        : `No eval framework found. Looked for: ${FRAMEWORKS_LOOKED_FOR.join(', ')}.`, '');
+    if (evals.files.length > 0)
+        lines.push(`Eval sets: ${evals.files.map((f) => `\`${f}\``).join(', ')}.`, '');
+    lines.push('A name appearing in a test file means somebody wrote it down there. Whether that test exercises', 'the tool, mocks it, or only imports it is not decided here.', '', 'And the other way round, which matters more: a tool missing from this list is not a tool whose', 'behaviour is untested. It is a tool nothing tests **by that name** - the wiring between the name', 'the model sees and the code behind it. Tewip finds 8 of its own 17 this way, and the engine under', 'each one is covered; what nothing exercises is the tool surface itself.', '');
+    return lines;
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "tewip-ai",
-  "version": "0.76.0",
+  "version": "0.77.0",
   "mcpName": "io.github.hmamut39/tewip",
   "description": "Tewip finds why each failing test failed - real bug, flaky test, environment, test data or changed selector - for any stack, from the command line, CI, or your editor.",
   "keywords": [