tewip-ai 0.75.0 → 0.77.0

package/README.md

@@ -1252,6 +1252,22 @@ offline, in CI, before anything runs.
   because that is the difference between a wrong answer and a tool call somebody else chose. On
   1,691 files of real agent code: **zero findings**, which is the precision claim; recall is
   measured against planted examples and said to be.
+- **Which of these tools anybody wrote a test about.** Not "does this project have tests" - every
+  project has tests - but whether *this* tool is mentioned in any of them, and especially the ones
+  that cannot be undone. A name too short or too common to look for is reported as undecidable
+  rather than counted as covered. And the statement is made precisely in both directions: a tool
+  missing from the list is not a tool whose behaviour is untested, it is a tool nothing tests **by
+  that name** - the wiring between the name the model sees and the code behind it. Tewip finds 8 of
+  its own 17 that way. Also names which eval framework the project uses, and which it looked for
+  when it finds none.
+- **What the coding agent installed here was permitted to do.** `.claude/settings.json` and
+  `.mcp.json` are committed, anybody with write access can change them, and neither turns up in a
+  review the way a source file does. Tewip names the approval gate switched off
+  (`"defaultMode": "bypassPermissions"`), an allow entry that grants a shell with no constraint on
+  the command, every tool server trusted without approval, a credential written into a committed
+  file - located, never printed - and a boundary widened past the project. A `PreToolUse` hook
+  counts as a real gate, because the harness enforces it on every call; a rule in a markdown file
+  is a request to the model and does not.
 - **What stands in the way**: a human approval gate, a dry run, a step budget, a tool allow-list -
   and says plainly when nothing anywhere in the project has one. A ceiling written as
   `max_iterations=None` is reported apart from no ceiling at all, because it reads as deliberate.

package/dist/engine/agent/agent-command.js

@@ -1,55 +1,13 @@
 // `tewip agent`: what this agent is allowed to do, and what stands between it and the real world.
-import { readFileSync, writeFileSync } from 'node:fs';
-import { relative } from 'node:path';
-import { scanSourceFiles } from '../walk.js';
-import { CAPABILITY_MEANS, FRAMEWORKS_READ, IRREVERSIBLE, toolsIn, trifectaIn } from './tools.js';
-import { promptFlows } from './prompt-flow.js';
-import { budgetSetToNothing, guardsIn, GUARD_MEANS, GUARD_PREVENTS } from './guards.js';
+import { writeFileSync } from 'node:fs';
+import { CAPABILITY_MEANS, FRAMEWORKS_READ, IRREVERSIBLE, trifectaIn } from './tools.js';
+import {} from './prompt-flow.js';
+import { configSection } from './claude-config.js';
+import { coverageSection } from './tool-tests.js';
+import { GUARD_MEANS, GUARD_PREVENTS } from './guards.js';
+import { scanAgent } from './scan.js';
+export { scanAgent };
 const NL = String.fromCharCode(10);
-const AGENT_FILE = /\.(?:py|[cm]?[jt]sx?)$/i;
-/**
- * What is not the shipped agent.
- *
- * The first version of this named the directory `tests` and the file `*.test.ts`, which is what the
- * documentation of every project says and not what projects do. Run against the OpenAI Agents SDK,
- * all 24 "tools" it reported were fixtures in `integration_tests/` and `test_*.py` - a directory
- * whose name ends in `tests` rather than being `tests`, and a Python convention that puts the word
- * at the front of the file. A list of an agent's tools that is actually a list of test doubles is
- * worse than no list.
- *
- * `[\w-]+[_-]tests?` and not `[\w-]*tests?`, because the second one also excludes `latest/`.
- */
-const NOT_SHIPPED = /(?:^|[\\/])(?:tests?|[\w-]+[_-]tests?|__tests__|spec|e2e|examples?|samples?|docs?|fixtures?|node_modules|\.venv|venv)[\\/]|\.(?:test|spec)\.[cm]?[jt]sx?$|_test\.py$|(?:^|[\\/])test_[^\\/]*\.py$/i;
-export function scanAgent(root) {
-    const scan = scanSourceFiles(root, 20_000, (f) => AGENT_FILE.test(f));
-    const tools = [];
-    const injections = [];
-    const guards = [];
-    let emptyBudget = null;
-    let filesRead = 0;
-    for (const path of scan.files) {
-        const where = relative(root, path).replace(/\\/g, '/');
-        if (NOT_SHIPPED.test(where))
-            continue;
-        let text;
-        try {
-            text = readFileSync(path, 'utf8');
-        }
-        catch {
-            continue;
-        }
-        filesRead += 1;
-        tools.push(...toolsIn(where, text));
-        injections.push(...promptFlows(where, text));
-        guards.push(...guardsIn(where, text));
-        if (!emptyBudget) {
-            const empty = budgetSetToNothing(text);
-            if (empty)
-                emptyBudget = { file: where, text: empty };
-        }
-    }
-    return { tools, injections, guards, filesRead, emptyBudget };
-}
 const REACH_READS = {
     destructive: 'cannot be undone',
     deletes: 'deletes something',
@@ -61,12 +19,13 @@ const REACH_READS = {
     'not-sure': '**not decided**',
 };
 export function renderAgent(scan, root) {
-    const { tools, injections, guards, filesRead, emptyBudget } = scan;
+    const { tools, injections, guards, filesRead, emptyBudget, config, coverage, evals } = scan;
     const lines = ['# What this agent is allowed to do', ''];
     if (tools.length === 0) {
         return [
             '# What this agent is allowed to do',
             '',
+            ...configSection(config),
             `No tool definitions were found in ${filesRead} file(s).`,
             '',
             'That is a fact about what was looked for, not a finding that this agent has no tools. These',
@@ -80,6 +39,12 @@ export function renderAgent(scan, root) {
         ].join(NL);
     }
     const kinds = new Set(guards.map((g) => g.kind));
+    // A permission list is an allow-list, and a PreToolUse hook is enforced on every call - both are
+    // the real thing rather than a pattern someone wrote in a prompt.
+    if (config.hasAllowList)
+        kinds.add('tool-allow-list');
+    if (config.hasEnforcedHook)
+        kinds.add('asks-a-person');
     const risky = tools.filter((t) => IRREVERSIBLE.includes(t.reach));
     const undecided = tools.filter((t) => t.reach === 'not-sure');
     lines.push(`${tools.length} tool(s) in ${filesRead} file(s) read locally. No model was called and nothing left this machine.`, '');
@@ -138,6 +103,8 @@ export function renderAgent(scan, root) {
         lines.push(`- **${capability}** (${CAPABILITY_MEANS[capability]}): ${legTools.length === 0 ? 'nothing found' : legTools.slice(0, 6).map((t) => `\`${t.name}\``).join(', ') + (legTools.length > 6 ? `, and ${legTools.length - 6} more` : '')}`);
     }
     lines.push('');
+    lines.push(...coverageSection(coverage, tools, evals));
+    lines.push(...configSection(config));
     lines.push('## What stands between them and the real world', '');
     for (const kind of ['asks-a-person', 'dry-run', 'step-budget', 'tool-allow-list']) {
         const found = guards.find((g) => g.kind === kind);

package/dist/engine/agent/claude-config.js

@@ -0,0 +1,166 @@
+// The permissions a coding agent was given in this repository, read from the files that grant them.
+//
+// `tools.ts` reads what an agent you wrote can call. This reads what an agent you *installed* was
+// allowed to do here - Claude Code's `.claude/settings.json` and the project's `.mcp.json`, both of
+// which are committed, both of which anybody with write access can change, and neither of which
+// turns up in a code review the way a source file does.
+//
+// Worth saying plainly, because it is the whole reason this is a separate check: a rule written in
+// a markdown file is a request to the model. A permission in settings.json is enforced by the
+// harness on every tool call. They look equally like configuration and they are not the same kind
+// of thing at all, and only one of them is a control an auditor can rely on.
+import { existsSync, readFileSync } from 'node:fs';
+import { join } from 'node:path';
+// Spelled with forward slashes, which is how every other path in a Tewip report is spelled and
+// how a reader will search for it. `join` gives a backslash on Windows and the report then names a
+// file that does not look like the one in the repository.
+const FILES = ['.claude/settings.json', '.claude/settings.local.json', '.mcp.json'];
+/**
+ * An allow entry that grants a shell with no constraint on the command.
+ *
+ * `Bash(npm run test:*)` is a decision somebody made. `Bash`, `Bash(*)` and `Bash(*:*)` are the
+ * absence of one: they permit `rm -rf`, `curl | sh` and anything else, and they are what people
+ * write when the prompts became annoying.
+ */
+const UNCONSTRAINED = /^(?:Bash|Execute|Shell|Run)(?:\(\s*\*?\s*(?::\s*\*\s*)?\))?$/i;
+/** A value written into a committed file that looks like a credential rather than a reference. */
+const LOOKS_SECRET = /(?:KEY|TOKEN|SECRET|PASSWORD|PASS|CREDENTIAL|API)/i;
+const read = (root, name) => {
+    const path = join(root, ...name.split('/'));
+    if (!existsSync(path))
+        return null;
+    try {
+        const raw = readFileSync(path, 'utf8');
+        const json = JSON.parse(raw);
+        return json && typeof json === 'object' ? { json, raw } : null;
+    }
+    catch {
+        // A file that does not parse is not a file that grants nothing, and saying so beats guessing.
+        return null;
+    }
+};
+const asList = (v) => (Array.isArray(v) ? v.filter((x) => typeof x === 'string') : []);
+export function scanConfig(root) {
+    const out = { read: [], findings: [], servers: [], hasAllowList: false, hasEnforcedHook: false };
+    for (const name of FILES.slice(0, 2)) {
+        const file = read(root, name);
+        if (!file)
+            continue;
+        out.read.push(name);
+        const permissions = (file.json.permissions ?? {});
+        const allow = asList(permissions.allow);
+        const deny = asList(permissions.deny);
+        if (allow.length > 0 || deny.length > 0)
+            out.hasAllowList = true;
+        // The gate switched off outright. Nothing else in this file matters as much.
+        if (permissions.defaultMode === 'bypassPermissions') {
+            out.findings.push({
+                rule: 'permissions-bypassed',
+                file: name,
+                what: '"defaultMode": "bypassPermissions"',
+                why: 'every tool call here runs without asking, including the ones that delete, pay or send. This is the approval gate being switched off in a committed file.',
+            });
+        }
+        if (permissions.defaultMode === 'acceptEdits') {
+            out.findings.push({
+                rule: 'edits-auto-accepted',
+                file: name,
+                what: '"defaultMode": "acceptEdits"',
+                why: 'file edits are applied without asking. Weaker than bypassing everything, and still a person out of the loop on every change to the working tree.',
+            });
+        }
+        for (const entry of allow) {
+            if (!UNCONSTRAINED.test(entry.trim()))
+                continue;
+            out.findings.push({
+                rule: 'shell-unconstrained',
+                file: name,
+                what: `"allow": [… "${entry}" …]`,
+                why: 'this permits any shell command, not a named one. `Bash(npm run test:*)` is a decision; this is the absence of one.',
+            });
+        }
+        if (file.json.enableAllProjectMcpServers === true) {
+            out.findings.push({
+                rule: 'mcp-auto-trusted',
+                file: name,
+                what: '"enableAllProjectMcpServers": true',
+                why: 'every server in .mcp.json is trusted without being approved, including any added later by anyone who can commit to this repository.',
+            });
+        }
+        const env = (file.json.env ?? {});
+        for (const [key, value] of Object.entries(env)) {
+            // A reference to a variable is fine; a literal is the secret itself, in a committed file.
+            if (!LOOKS_SECRET.test(key) || typeof value !== 'string' || value === '' || value.startsWith('$'))
+                continue;
+            out.findings.push({
+                rule: 'secret-in-settings',
+                file: name,
+                what: `"env": { "${key}": … }`,
+                why: 'a credential written into a file that is committed. Tewip does not print the value; the point is that it is there.',
+            });
+        }
+        const extra = asList(permissions.additionalDirectories);
+        if (extra.length > 0) {
+            out.findings.push({
+                rule: 'boundary-widened',
+                file: name,
+                what: `"additionalDirectories": [${extra.slice(0, 3).map((d) => `"${d}"`).join(', ')}]`,
+                why: 'the agent may read and write outside this project. Whatever is on the other end is inside the blast radius.',
+            });
+        }
+        const hooks = (file.json.hooks ?? {});
+        // PreToolUse is the one that can refuse a call. The others observe it after the fact.
+        if (Array.isArray(hooks.PreToolUse) && hooks.PreToolUse.length > 0)
+            out.hasEnforcedHook = true;
+    }
+    const mcp = read(root, '.mcp.json');
+    if (mcp) {
+        out.read.push('.mcp.json');
+        const servers = (mcp.json.mcpServers ?? {});
+        for (const [name, spec] of Object.entries(servers)) {
+            const url = typeof spec?.url === 'string' ? spec.url : '';
+            const command = typeof spec?.command === 'string' ? spec.command : '';
+            const args = asList(spec?.args).join(' ');
+            out.servers.push(url
+                ? { name, where: 'somewhere else', detail: url.replace(/\/\/[^@]*@/, '//') }
+                : { name, where: 'this machine', detail: `${command} ${args}`.trim().slice(0, 90) });
+        }
+    }
+    return out;
+}
+/** What each rule stops, for a reader deciding which to act on first. */
+export const CONFIG_MEANS = {
+    'permissions-bypassed': 'nothing is asked before any tool call',
+    'edits-auto-accepted': 'file edits are applied without asking',
+    'shell-unconstrained': 'any shell command is permitted',
+    'mcp-auto-trusted': 'tool servers are trusted without approval',
+    'secret-in-settings': 'a credential is committed',
+    'boundary-widened': 'the agent may work outside this project',
+};
+/**
+ * What the coding agent installed here was permitted to do.
+ *
+ * Rendered whether or not any tools were found, because the two are independent: a repository with
+ * no agent code of its own can still hand Claude Code a shell with no constraint on the command,
+ * and that is a finding about this repository either way.
+ */
+export function configSection(config) {
+    if (config.read.length === 0)
+        return [];
+    const lines = ['## What the coding agent here is permitted to do', '', `Read: ${config.read.map((f) => `\`${f}\``).join(', ')}.`, ''];
+    if (config.servers.length > 0) {
+        lines.push('Tool servers wired into this project:', '');
+        for (const s of config.servers)
+            lines.push(`- \`${s.name}\` — runs ${s.where}: \`${s.detail}\``);
+        lines.push('');
+    }
+    if (config.findings.length === 0) {
+        lines.push('Nothing in these files switches off an approval, grants an unconstrained shell, trusts a', 'server without approval or commits a credential.', '');
+        return lines;
+    }
+    for (const f of config.findings) {
+        lines.push(`- **${f.rule}** — ${CONFIG_MEANS[f.rule] ?? ''}`, `  \`${f.what}\` in \`${f.file}\`. ${f.why}`);
+    }
+    lines.push('');
+    return lines;
+}

package/dist/engine/agent/scan.js

@@ -0,0 +1,84 @@
+// Reading an agent project once: its tools, its flows, its guards, its tests and its permissions.
+//
+// Split from agent-command.ts, which is the file in this project that keeps arriving at the
+// 300-line limit. The rendering belongs with the thing it renders; the walk belongs here.
+import { readFileSync } from 'node:fs';
+import { join, relative } from 'node:path';
+import { scanSourceFiles } from '../walk.js';
+import { toolsIn } from './tools.js';
+import { promptFlows } from './prompt-flow.js';
+import { budgetSetToNothing, guardsIn } from './guards.js';
+import { scanConfig } from './claude-config.js';
+import { coverageOf, evalEvidence } from './tool-tests.js';
+const NL = String.fromCharCode(10);
+const AGENT_FILE = /\.(?:py|[cm]?[jt]sx?)$/i;
+/**
+ * What is not the shipped agent.
+ *
+ * The first version of this named the directory `tests` and the file `*.test.ts`, which is what the
+ * documentation of every project says and not what projects do. Run against the OpenAI Agents SDK,
+ * all 24 "tools" it reported were fixtures in `integration_tests/` and `test_*.py` - a directory
+ * whose name ends in `tests` rather than being `tests`, and a Python convention that puts the word
+ * at the front of the file. A list of an agent's tools that is actually a list of test doubles is
+ * worse than no list.
+ *
+ * `[\w-]+[_-]tests?` and not `[\w-]*tests?`, because the second one also excludes `latest/`.
+ */
+const NOT_SHIPPED = /(?:^|[\\/])(?:tests?|[\w-]+[_-]tests?|__tests__|spec|e2e|examples?|samples?|docs?|fixtures?|node_modules|\.venv|venv)[\\/]|\.(?:test|spec)\.[cm]?[jt]sx?$|_test\.py$|(?:^|[\\/])test_[^\\/]*\.py$/i;
+export function scanAgent(root) {
+    const scan = scanSourceFiles(root, 20_000, (f) => AGENT_FILE.test(f));
+    const tools = [];
+    const tests = [];
+    const injections = [];
+    const guards = [];
+    let emptyBudget = null;
+    let filesRead = 0;
+    for (const path of scan.files) {
+        const where = relative(root, path).replace(/\\/g, '/');
+        // Excluded from the tool scan, but not thrown away: these files are the evidence for whether
+        // anybody ever wrote a test about a tool, which is a different question from what ships.
+        if (NOT_SHIPPED.test(where)) {
+            try {
+                tests.push({ file: where, text: readFileSync(path, 'utf8') });
+            }
+            catch {
+                // A file that cannot be read is not evidence of a test, and is not counted as one.
+            }
+            continue;
+        }
+        let text;
+        try {
+            text = readFileSync(path, 'utf8');
+        }
+        catch {
+            continue;
+        }
+        filesRead += 1;
+        tools.push(...toolsIn(where, text));
+        injections.push(...promptFlows(where, text));
+        guards.push(...guardsIn(where, text));
+        if (!emptyBudget) {
+            const empty = budgetSetToNothing(text);
+            if (empty)
+                emptyBudget = { file: where, text: empty };
+        }
+    }
+    const manifests = ['package.json', 'pyproject.toml', 'requirements.txt'].flatMap((m) => {
+        try {
+            return [readFileSync(join(root, m), 'utf8')];
+        }
+        catch {
+            return [];
+        }
+    });
+    return {
+        tools,
+        injections,
+        guards,
+        filesRead,
+        emptyBudget,
+        coverage: coverageOf(tools.map((t) => t.name), tests),
+        evals: evalEvidence(tests, manifests),
+        config: scanConfig(root),
+    };
+}

package/dist/engine/agent/tool-tests.js

@@ -0,0 +1,87 @@
+/**
+ * Eval frameworks, by the name they are imported or depended on under.
+ *
+ * Named rather than guessed at. The point of listing them is to be able to say "this project has
+ * one, here it is" or "none of these is here" - and the second sentence is only honest if the list
+ * is written down where a reader can see what was looked for.
+ */
+const FRAMEWORKS = [
+    { name: 'DeepEval', re: /\bdeepeval\b/i },
+    { name: 'promptfoo', re: /\bpromptfoo\b/i },
+    { name: 'Ragas', re: /\bragas\b/i },
+    { name: 'LangSmith', re: /\blangsmith\b/i },
+    { name: 'Braintrust', re: /\bbraintrust\b/i },
+    { name: 'Inspect AI', re: /\binspect_ai\b/i },
+    { name: 'Giskard', re: /\bgiskard\b/i },
+    { name: 'TruLens', re: /\btrulens\b/i },
+    { name: 'Langfuse', re: /\blangfuse\b/i },
+    { name: 'OpenAI Evals', re: /\bopenai[-_]?evals\b/i },
+];
+/** A path that holds an eval set rather than a unit test. */
+const EVAL_PATH = /(?:^|[\\/])(?:evals?|benchmarks?|golden|datasets?)[\\/]|\.eval\.[cm]?[jt]s$|_eval\.py$|eval_[^\\/]*\.(?:py|[cm]?[jt]s)$|\bevals?\.ya?ml$/i;
+/**
+ * Names too common to decide on.
+ *
+ * A tool called `get` or `run` appears in every file in the project, so finding it in a test file
+ * says nothing at all. Rather than counting those as covered - which is the comfortable direction
+ * and the wrong one - they are reported as undecidable, by name.
+ */
+const COMMON = new Set(['get', 'set', 'run', 'add', 'list', 'read', 'write', 'call', 'send', 'find', 'query', 'search', 'name', 'tool', 'test', 'echo', 'exec', 'open', 'close', 'load', 'save']);
+const tooCommonName = (name) => name.length < 5 || COMMON.has(name.toLowerCase());
+export function coverageOf(toolNames, tests) {
+    return toolNames.map((name) => {
+        if (tooCommonName(name))
+            return { name, mentionedIn: [], tooCommon: true };
+        // Word boundaries on both sides: `delete_user` must not be matched by `delete_user_profile`,
+        // which is a different tool and may well be the only one anybody tested.
+        const mention = new RegExp(String.raw `(?<![\w-])${name.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')}(?![\w-])`);
+        return { name, mentionedIn: tests.filter((t) => mention.test(t.text)).map((t) => t.file), tooCommon: false };
+    });
+}
+export function evalEvidence(files, manifests) {
+    const frameworks = new Set();
+    for (const { name, re } of FRAMEWORKS) {
+        if (manifests.some((m) => re.test(m)) || files.some((f) => re.test(f.text)))
+            frameworks.add(name);
+    }
+    const evalFiles = files.filter((f) => EVAL_PATH.test(f.file)).map((f) => f.file);
+    return { frameworks: [...frameworks], files: [...new Set(evalFiles)].slice(0, 12) };
+}
+/** The names of every eval framework looked for, so silence about them can be read correctly. */
+export const FRAMEWORKS_LOOKED_FOR = FRAMEWORKS.map((f) => f.name);
+/**
+ * Which tools anybody wrote a test about, rendered where the logic for deciding it lives.
+ *
+ * Split out for the same reason the config section was: agent-command.ts is the file that keeps
+ * reaching the 300-line limit, and a report section belongs beside the thing it reports on.
+ */
+export function coverageSection(coverage, tools, evals) {
+    const lines = [];
+    const IRREV = ['destructive', 'deletes', 'pays', 'sends', 'runs-code'];
+    const untested = coverage.filter((c) => !c.tooCommon && c.mentionedIn.length === 0);
+    const riskyUntested = untested.filter((u) => IRREV.includes(tools.find((t) => t.name === u.name)?.reach ?? 'reads'));
+    lines.push('## Which of these anybody wrote a test about', '');
+    lines.push(untested.length === 0
+        ? 'Every tool whose name is distinctive enough to look for appears in at least one test file.'
+        : `**${untested.length} of ${tools.length}** appear in no test file${riskyUntested.length > 0 ? `, and ${riskyUntested.length} of those cannot be undone` : ''}.`, '');
+    if (untested.length > 0) {
+        for (const u of untested.slice(0, 15)) {
+            const reach = tools.find((t) => t.name === u.name)?.reach;
+            lines.push(`- \`${u.name}\`${reach && IRREV.includes(reach) ? ' — **and it cannot be undone**' : ''}`);
+        }
+        if (untested.length > 15)
+            lines.push(`…and ${untested.length - 15} more`);
+        lines.push('');
+    }
+    const undecidable = coverage.filter((c) => c.tooCommon);
+    if (undecidable.length > 0) {
+        lines.push(`Not decided for ${undecidable.length}: ${undecidable.slice(0, 8).map((u) => `\`${u.name}\``).join(', ')}. A name this`, 'short or this common appears everywhere, so finding it in a test file would prove nothing.', '');
+    }
+    lines.push(evals.frameworks.length > 0
+        ? `Eval framework in this project: ${evals.frameworks.join(', ')}.`
+        : `No eval framework found. Looked for: ${FRAMEWORKS_LOOKED_FOR.join(', ')}.`, '');
+    if (evals.files.length > 0)
+        lines.push(`Eval sets: ${evals.files.map((f) => `\`${f}\``).join(', ')}.`, '');
+    lines.push('A name appearing in a test file means somebody wrote it down there. Whether that test exercises', 'the tool, mocks it, or only imports it is not decided here.', '', 'And the other way round, which matters more: a tool missing from this list is not a tool whose', 'behaviour is untested. It is a tool nothing tests **by that name** - the wiring between the name', 'the model sees and the code behind it. Tewip finds 8 of its own 17 this way, and the engine under', 'each one is covered; what nothing exercises is the tool surface itself.', '');
+    return lines;
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "tewip-ai",
-  "version": "0.75.0",
+  "version": "0.77.0",
   "mcpName": "io.github.hmamut39/tewip",
   "description": "Tewip finds why each failing test failed - real bug, flaky test, environment, test data or changed selector - for any stack, from the command line, CI, or your editor.",
   "keywords": [