tewip-ai 0.69.6 → 0.69.8
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/engine/access/access-command.js +2 -1
- package/dist/engine/secure/triage.js +2 -1
- package/dist/engine/standards/standards-command.js +18 -4
- package/dist/engine/standards/standards.js +23 -4
- package/dist/engine/trace/trace.js +26 -2
- package/dist/reporters/store.js +13 -0
- package/package.json +1 -1
|
@@ -12,6 +12,7 @@ import { existsSync, mkdirSync, readFileSync, writeFileSync } from 'node:fs';
|
|
|
12
12
|
import { normalise } from '../quality/source-text.js';
|
|
13
13
|
import { dirname, join, relative, resolve } from 'node:path';
|
|
14
14
|
import { sourceFiles } from '../secure/secrets.js';
|
|
15
|
+
import { readHandWritten } from '../../reporters/store.js';
|
|
15
16
|
import { JS_LIKE, MARKUP, tags } from './markup.js';
|
|
16
17
|
import { checkFile } from './checks.js';
|
|
17
18
|
import { readAxeReport } from './reports.js';
|
|
@@ -27,7 +28,7 @@ export function readAccepted(root) {
|
|
|
27
28
|
const file = join(root, '.tewip', 'access-accepted.txt');
|
|
28
29
|
if (!existsSync(file))
|
|
29
30
|
return new Set();
|
|
30
|
-
return new Set(
|
|
31
|
+
return new Set(readHandWritten(file)
|
|
31
32
|
.split('\n')
|
|
32
33
|
.map((l) => l.split(' - ')[0].trim())
|
|
33
34
|
.filter((l) => l !== '' && !l.startsWith('#')));
|
|
@@ -15,12 +15,13 @@
|
|
|
15
15
|
import { existsSync, readFileSync } from 'node:fs';
|
|
16
16
|
import { join } from 'node:path';
|
|
17
17
|
import { spawnSync } from 'node:child_process';
|
|
18
|
+
import { readHandWritten } from '../../reporters/store.js';
|
|
18
19
|
/** Findings a person has already judged, one per line: "<kind> <where> - reason". */
|
|
19
20
|
export function readAccepted(root) {
|
|
20
21
|
const file = join(root, '.tewip', 'security-accepted.txt');
|
|
21
22
|
if (!existsSync(file))
|
|
22
23
|
return new Set();
|
|
23
|
-
return new Set(
|
|
24
|
+
return new Set(readHandWritten(file)
|
|
24
25
|
.split('\n')
|
|
25
26
|
.map((l) => l.split(' - ')[0].trim())
|
|
26
27
|
.filter(Boolean));
|
|
@@ -78,11 +78,21 @@ export function renderStandards(standards, violations, checkedFiles) {
|
|
|
78
78
|
].join(String.fromCharCode(10));
|
|
79
79
|
}
|
|
80
80
|
const checkable = standards.filter((s) => s.rules.length > 0);
|
|
81
|
-
|
|
81
|
+
// Every standard that carries prose, not only the ones carrying nothing else.
|
|
82
|
+
//
|
|
83
|
+
// This was `s.rules.length === 0 && s.guidance.trim()`, so a standard written in the shape the
|
|
84
|
+
// feature is documented with - front matter rules, then prose below them - had its prose
|
|
85
|
+
// **invisible in this report**. Found by writing this project's own house rules in exactly that
|
|
86
|
+
// shape: four rules were listed as enforced and three paragraphs of guidance were never mentioned,
|
|
87
|
+
// under a closing line promising "everything else is prose Tewip carries for you". The other
|
|
88
|
+
// renderer, `renderViolations`, gets this right but only prints when something is broken - so a
|
|
89
|
+
// team doing it properly and keeping to its own rules is never told its guidance was read at all.
|
|
90
|
+
const carrying = standards.filter((s) => s.guidance.trim().length > 0);
|
|
91
|
+
const guidanceOnly = carrying.filter((s) => s.rules.length === 0);
|
|
82
92
|
const lines = [
|
|
83
93
|
'# What this project has taught Tewip',
|
|
84
94
|
'',
|
|
85
|
-
`${standards.length} standard(s) loaded: ${checkable.length} with rules Tewip can check, ${guidanceOnly.length} guidance only.`,
|
|
95
|
+
`${standards.length} standard(s) loaded: ${checkable.length} with rules Tewip can check, ${carrying.length} carrying guidance it cannot${guidanceOnly.length > 0 ? ` (${guidanceOnly.length} guidance only)` : ''}.`,
|
|
86
96
|
...(checkedFiles > 0 ? ['', `${checkedFiles} changed file(s) checked. **${violations.length}** broke a rule.`] : []),
|
|
87
97
|
'',
|
|
88
98
|
'## Checked and enforced',
|
|
@@ -97,8 +107,12 @@ export function renderStandards(standards, violations, checkedFiles) {
|
|
|
97
107
|
}
|
|
98
108
|
}
|
|
99
109
|
lines.push('');
|
|
100
|
-
if (
|
|
101
|
-
lines.push('## Carried, but not checked', '', 'Tewip passes these to the model when it writes or fixes something, and quotes them in reports. It', 'cannot verify them, and it will never report them as kept.', '', ...
|
|
110
|
+
if (carrying.length > 0) {
|
|
111
|
+
lines.push('## Carried, but not checked', '', 'Tewip passes these to the model when it writes or fixes something, and quotes them in reports. It', 'cannot verify them, and it will never report them as kept.', '', ...carrying.map((s) => {
|
|
112
|
+
const words = s.guidance.trim().split(/\s+/).length;
|
|
113
|
+
const also = s.rules.length > 0 ? ` (alongside its ${s.rules.length} enforced rule${s.rules.length === 1 ? '' : 's'})` : '';
|
|
114
|
+
return `- **${s.name}**${s.description ? ` — ${s.description}` : ''}: ${words} word(s) of guidance${also}`;
|
|
115
|
+
}), '');
|
|
102
116
|
}
|
|
103
117
|
const unusable = standards.filter((s) => s.unusable);
|
|
104
118
|
if (unusable.length > 0) {
|
|
@@ -6,8 +6,16 @@
|
|
|
6
6
|
* unusable rather than silently ignored - a rule nobody is enforcing is worse than no rule, because
|
|
7
7
|
* somebody believes in it.
|
|
8
8
|
*/
|
|
9
|
-
export function readStandard(file,
|
|
9
|
+
export function readStandard(file, rawText) {
|
|
10
10
|
const name = file.split(/[\\/]/).pop()?.replace(/\.md$/i, '') ?? file;
|
|
11
|
+
// A leading byte-order mark, which Notepad, Visual Studio and PowerShell's `Set-Content -Encoding
|
|
12
|
+
// utf8` all write by default on Windows. Three invisible bytes before `---` stop the front matter
|
|
13
|
+
// matching, so **every enforced rule in the file silently becomes prose** and the report says
|
|
14
|
+
// "0 with rules Tewip can check" without once mentioning why. Found by accident: this project's own
|
|
15
|
+
// standards file was rewritten by a PowerShell command and four working rules stopped being
|
|
16
|
+
// enforced with no error anywhere. For a team on Windows that is total, silent rule loss - the
|
|
17
|
+
// exact thing `unusable` exists to prevent, happening in the one place it could not report.
|
|
18
|
+
const text = rawText.replace(/^/, '');
|
|
11
19
|
const front = /^---\r?\n([\s\S]*?)\r?\n---/.exec(text);
|
|
12
20
|
const body = front ? text.slice(front[0].length).trim() : text.trim();
|
|
13
21
|
const standard = { name, description: '', appliesTo: [], rules: [], guidance: body, unusable: '' };
|
|
@@ -40,7 +48,12 @@ export function readStandard(file, text) {
|
|
|
40
48
|
standard.description = firstLine(body);
|
|
41
49
|
return standard;
|
|
42
50
|
}
|
|
43
|
-
/**
|
|
51
|
+
/**
|
|
52
|
+
* `forbid: /waitForTimeout/ in tests because a fixed wait hides a race`
|
|
53
|
+
*
|
|
54
|
+
* The `in` clause may carry an exception: `in engine/ not -command.ts`. Everything before `not` is
|
|
55
|
+
* where the rule applies, everything after it is where it does not.
|
|
56
|
+
*/
|
|
44
57
|
function parseRule(kind, value, scope) {
|
|
45
58
|
const shape = /^\/(.+?)\/([a-z]*)\s*(?:\bin\s+([^]*?))?\s*(?:\bbecause\s+([^]*))?$/i.exec(value.trim());
|
|
46
59
|
if (!shape)
|
|
@@ -51,10 +64,12 @@ function parseRule(kind, value, scope) {
|
|
|
51
64
|
catch {
|
|
52
65
|
return null;
|
|
53
66
|
}
|
|
67
|
+
const [inside, outside] = (shape[3] ?? '').split(/\bnot\b/);
|
|
54
68
|
return {
|
|
55
69
|
kind,
|
|
56
70
|
pattern: shape[2] ? `/${shape[1]}/${shape[2]}` : shape[1],
|
|
57
|
-
where:
|
|
71
|
+
where: inside && inside.trim() ? list(inside) : scope,
|
|
72
|
+
except: outside ? list(outside) : [],
|
|
58
73
|
because: (shape[4] ?? '').trim(),
|
|
59
74
|
};
|
|
60
75
|
}
|
|
@@ -92,7 +107,11 @@ export function check(standards, file, text) {
|
|
|
92
107
|
if (!covers(standard.appliesTo, file))
|
|
93
108
|
continue;
|
|
94
109
|
for (const rule of standard.rules) {
|
|
95
|
-
|
|
110
|
+
// `covers([])` is deliberately true - an empty scope means everything - so an exception has to
|
|
111
|
+
// be checked for existence first. Written the short way, `|| covers(rule.except, file)`, every
|
|
112
|
+
// rule with no exception is skipped and the report says nothing broke: a clean answer produced
|
|
113
|
+
// by a bug, which is the exact shape this project keeps finding.
|
|
114
|
+
if (!covers(rule.where, file) || (rule.except.length > 0 && covers(rule.except, file)))
|
|
96
115
|
continue;
|
|
97
116
|
let pattern;
|
|
98
117
|
try {
|
|
@@ -21,20 +21,44 @@ export function testFor(scenario, attempted) {
|
|
|
21
21
|
const id = clean(scenario.id);
|
|
22
22
|
return id.length >= 4 ? (attempted.find((t) => clean(t).includes(id)) ?? '') : '';
|
|
23
23
|
}
|
|
24
|
+
/**
|
|
25
|
+
* Why no test could be named - and what that does and does not establish.
|
|
26
|
+
*
|
|
27
|
+
* "no test in this project can be named for it, so nothing is checking it" was printed whatever the
|
|
28
|
+
* reason, including when **no run had been kept here at all**. Field-tested on this repository, which
|
|
29
|
+
* has 1,095 passing tests and a kept history belonging to a different project: all twelve scenarios
|
|
30
|
+
* of a real plan came back "nothing is checking it". Tewip was not looking at this project's tests,
|
|
31
|
+
* because it only ever sees the tests a kept run names, and it said so as though it had looked.
|
|
32
|
+
*
|
|
33
|
+
* The distinction matters because the two cases call for opposite actions: write a test, or keep
|
|
34
|
+
* your runs so the ones you have can be seen.
|
|
35
|
+
*/
|
|
36
|
+
function notCoveredBecause(runs, knownTests) {
|
|
37
|
+
if (runs.length === 0) {
|
|
38
|
+
return 'no run has been kept for this project, so Tewip has no list of its tests to match against - this is not a claim that nothing checks it. Run `tewip explain <report>` once and ask again';
|
|
39
|
+
}
|
|
40
|
+
return `no test among the ${knownTests} that the ${runs.length} kept run(s) name matches it, so nothing Tewip can see is checking it`;
|
|
41
|
+
}
|
|
24
42
|
/** Joins one plan to what the runs actually say. Nothing here is inferred from anything else. */
|
|
25
43
|
export function tracePlan(plan, runs, records) {
|
|
26
44
|
const newest = runs.at(-1);
|
|
27
45
|
const attempted = newest ? Object.keys(newest.attempts ?? {}) : [];
|
|
28
46
|
const everAttempted = new Set(runs.flatMap((r) => Object.keys(r.attempts ?? {})));
|
|
47
|
+
// A scenario's own id was matched against the **newest run only**, so a test that ran last week
|
|
48
|
+
// and not today could not be found at all - and the answer for it was "no test in this project can
|
|
49
|
+
// be named for it, so nothing is checking it", which is a different and much stronger claim than
|
|
50
|
+
// "it did not run today". The branch below that exists to say exactly that ("ran before but not in
|
|
51
|
+
// the last run") was unreachable for every scenario the plan did not name a test for.
|
|
52
|
+
const everyTest = [...everAttempted];
|
|
29
53
|
const failures = new Map();
|
|
30
54
|
for (const record of records)
|
|
31
55
|
if (record.runId === newest?.runId)
|
|
32
56
|
failures.set(record.testId, record);
|
|
33
57
|
const scenarios = plan.scenarios.map((scenario) => {
|
|
34
|
-
const test = testFor(scenario,
|
|
58
|
+
const test = testFor(scenario, everyTest);
|
|
35
59
|
const base = { id: scenario.id, title: scenario.title, priority: scenario.priority ?? '', test, runId: newest?.runId ?? '', when: newest?.finishedAt ?? '' };
|
|
36
60
|
if (!test) {
|
|
37
|
-
return { ...base, standing: 'not-covered', diagnosis: '', why:
|
|
61
|
+
return { ...base, standing: 'not-covered', diagnosis: '', why: notCoveredBecause(runs, everAttempted.size) };
|
|
38
62
|
}
|
|
39
63
|
if (!everAttempted.has(test)) {
|
|
40
64
|
return { ...base, standing: 'never-run', runId: '', when: '', diagnosis: '', why: `${test} is named as the cover, but no run kept here has attempted it` };
|
package/dist/reporters/store.js
CHANGED
|
@@ -7,6 +7,19 @@ export const DATA_DIR = join(PROJECT_ROOT, 'data');
|
|
|
7
7
|
const COLLECT_DIR = process.env.TRIAGE_DATA_DIR ? resolve(PROJECT_ROOT, process.env.TRIAGE_DATA_DIR) : DATA_DIR;
|
|
8
8
|
export const TRIAGE_FILE = join(COLLECT_DIR, 'triage.jsonl');
|
|
9
9
|
export const RUNS_FILE = join(COLLECT_DIR, 'runs.jsonl');
|
|
10
|
+
/**
|
|
11
|
+
* A file a person wrote, read as text, without the byte-order mark Windows editors put in front.
|
|
12
|
+
*
|
|
13
|
+
* Notepad, Visual Studio and PowerShell's `Set-Content -Encoding utf8` all write three invisible
|
|
14
|
+
* bytes before the first character. Nothing downstream expects them: a standards file stops being
|
|
15
|
+
* enforced because `/^---/` no longer matches, and the first line of an accepted-findings list stops
|
|
16
|
+
* matching the finding it was written to accept - so a judged finding comes back and keeps coming
|
|
17
|
+
* back, with nothing anywhere saying why. Every file this project reads **because a person wrote it
|
|
18
|
+
* by hand** goes through here.
|
|
19
|
+
*/
|
|
20
|
+
export function readHandWritten(file) {
|
|
21
|
+
return readFileSync(file, 'utf8').replace(/^\uFEFF/, '');
|
|
22
|
+
}
|
|
10
23
|
export function appendJsonl(file, rows) {
|
|
11
24
|
if (rows.length === 0)
|
|
12
25
|
return;
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "tewip-ai",
|
|
3
|
-
"version": "0.69.
|
|
3
|
+
"version": "0.69.8",
|
|
4
4
|
"mcpName": "io.github.hmamut39/tewip",
|
|
5
5
|
"description": "Tewip finds why each failing test failed - real bug, flaky test, environment, test data or changed selector - for any stack, from the command line, CI, or your editor.",
|
|
6
6
|
"keywords": [
|