tewip-ai 0.12.0 → 0.14.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +24 -0
- package/TEAM-GUIDE.md +1 -0
- package/dist/cli/tewip.js +61 -6
- package/dist/engine/build/build-command.js +55 -0
- package/dist/engine/build/propose-pr.js +88 -0
- package/dist/engine/design/design-command.js +17 -2
- package/dist/engine/flaky/flaky-command.js +9 -1
- package/dist/engine/learn/correct-command.js +88 -0
- package/dist/engine/learn/verdicts.js +71 -0
- package/dist/engine/secure/secure-command.js +9 -0
- package/package.json +1 -1
package/README.md
CHANGED
|
@@ -181,10 +181,28 @@ Each reader was checked against real reports published in public repositories, w
|
|
|
181
181
|
defects were found before release - a Robot timestamp that is not a date, k6's numbers living under
|
|
182
182
|
`values`, and a JMeter line that read "mostly 200" when an assertion had failed.
|
|
183
183
|
|
|
184
|
+
## Tell it when it is wrong (new)
|
|
185
|
+
|
|
186
|
+
```bash
|
|
187
|
+
tewip correct diagnosis "checkout charges twice" TEST_DATA "the fixture has stale prices"
|
|
188
|
+
tewip correct flaky "adds an item" fix-now "it is a real race, not flakiness"
|
|
189
|
+
tewip correct security "src/config.ts:12" noise "that is a sample key in a fixture"
|
|
190
|
+
tewip correct --list
|
|
191
|
+
```
|
|
192
|
+
|
|
193
|
+
Tewip answers with **your** decision from then on, and says it was yours: *"TEST_DATA, confidence
|
|
194
|
+
1.00, source: corrected by a person"*. Corrections live in your project, so they belong to the team;
|
|
195
|
+
the newest one wins, so changing your mind is just saying so again.
|
|
196
|
+
|
|
197
|
+
**A correction changes what Tewip says, never what it does on its own** — it will not change code,
|
|
198
|
+
quarantine a test or file a ticket because you corrected it.
|
|
199
|
+
|
|
184
200
|
## Build a feature, test first (new)
|
|
185
201
|
|
|
186
202
|
```bash
|
|
187
203
|
tewip build "naturalsize should take a sep argument that goes between the number and the unit"
|
|
204
|
+
tewip build --issue PROJ-123 # build from the ticket's own description and acceptance criteria
|
|
205
|
+
tewip build "..." --pr # commit the proved work on its own branch, open a draft PR
|
|
188
206
|
tewip build "..." --dry-run # prove it, show the patch, leave nothing on disk
|
|
189
207
|
```
|
|
190
208
|
|
|
@@ -199,6 +217,12 @@ TypeScript, $0.18) and a `sep` argument to **python-humanize** (2 lines, Python,
|
|
|
199
217
|
its own failing-then-passing test and the project's whole suite still green. Asked for something
|
|
200
218
|
that already worked, it stopped after one model call and kept nothing.
|
|
201
219
|
|
|
220
|
+
It also refuses a feature that does not belong here at all. Handed a real Apache ticket about a
|
|
221
|
+
Flink HTTP sink, a URL library got an "HTTP sink" built into it - test passing, suite green, every
|
|
222
|
+
proof rule held, because when Tewip writes *both* the test and the code it can always make a
|
|
223
|
+
self-consistent pair pass. So before writing anything it now asks whether the words of the feature
|
|
224
|
+
mean anything in this repository, and refuses for free when they do not.
|
|
225
|
+
|
|
202
226
|
The old promise was *"it can't touch your code"*. The new one is *"it can't change your code without
|
|
203
227
|
showing you it works"*.
|
|
204
228
|
|
package/TEAM-GUIDE.md
CHANGED
|
@@ -127,6 +127,7 @@ collects environment failures in one issue, and (optionally) opens a fix pull re
|
|
|
127
127
|
| `tewip spec "<feature>"` | An epic, its user stories and acceptance criteria, grounded in your code; files them in Jira with `--project PROJ --create` | Before the work is broken down |
|
|
128
128
|
| `tewip report report.xml --gate` | The CI command for GitLab, Jenkins, CircleCI, Azure, Bitbucket and the rest: diagnoses the run, comments on the GitLab merge request, and fails the build only on failures that need a person | Your CI is not GitHub Actions |
|
|
129
129
|
| `tewip watch --cmd "npm test"` | Runs your tests when you save a file and explains each failure straight away | While you are writing code |
|
|
130
|
+
| `tewip correct <kind> "<what>" <verdict> "why"` | Tell Tewip it was wrong about anything it decides; it answers with your decision from then on | Whenever you disagree with it |
|
|
130
131
|
| `tewip build "<what it should do>"` | Writes a feature test first, then the code, and keeps it only if the test failed without it and passes with it | You want a small feature built and proved |
|
|
131
132
|
| `tewip review --diff main...HEAD` | Which tests cover your change, which have flaked before, what nothing tests, and what to run first | Before you open a pull request |
|
|
132
133
|
| `tewip flaky --days 30` | Which tests to quarantine, which to fix, what they cost in CI minutes - and when a quarantined one can come back | Monday morning |
|
package/dist/cli/tewip.js
CHANGED
|
@@ -67,11 +67,14 @@ Usage:
|
|
|
67
67
|
the GitLab merge request, tells Slack or Teams only when a
|
|
68
68
|
failure needs a person, and with --gate fails the build only on
|
|
69
69
|
those failures.
|
|
70
|
-
tewip build "<what it should do>" [--dry-run] [--
|
|
70
|
+
tewip build "<what it should do>" | --issue PROJ-123 [--dry-run] [--pr] [--out feature.patch]
|
|
71
71
|
Writes a feature, test first: the test must fail without the
|
|
72
72
|
code and pass with it, and every test that was passing must
|
|
73
73
|
still pass - otherwise everything goes back byte for byte.
|
|
74
|
-
|
|
74
|
+
--issue PROJ-123 Build from the ticket itself: its description and acceptance
|
|
75
|
+
criteria. Works with a public tracker without an account.
|
|
76
|
+
--pr Commit the proved work on a branch of its own and open a draft
|
|
77
|
+
pull request. Never merges, never commits to your branch.
|
|
75
78
|
tewip review [--diff main...HEAD] [--out review.md]
|
|
76
79
|
Before anyone reads the diff: which tests cover this change,
|
|
77
80
|
which of them have flaked before, what has gone wrong in these
|
|
@@ -99,6 +102,11 @@ Usage:
|
|
|
99
102
|
Compare a Figma frame against the running page: the words the
|
|
100
103
|
design specifies and the controls its layer names imply. Says
|
|
101
104
|
"the design says X, the page says Y". No model, no cost.
|
|
105
|
+
tewip correct <kind> "<what>" <verdict> ["why"] | --list
|
|
106
|
+
Tell Tewip it was wrong, about anything it decides: a diagnosis,
|
|
107
|
+
a flaky verdict, a security finding, a design difference, a
|
|
108
|
+
refused build. It answers with your decision from then on, and
|
|
109
|
+
says it was yours. It never acts on a correction by itself.
|
|
102
110
|
tewip mcp Serve Tewip over MCP (stdio) for an editor or AI assistant.
|
|
103
111
|
tewip --version Print the version.
|
|
104
112
|
|
|
@@ -262,9 +270,24 @@ async function main() {
|
|
|
262
270
|
return;
|
|
263
271
|
}
|
|
264
272
|
case 'build': {
|
|
265
|
-
const
|
|
273
|
+
const typed = args.filter((a, i) => !a.startsWith('--') && !['--repairs', '--out', '--max-lines', '--issue'].includes(args[i - 1] ?? '')).join(' ').trim();
|
|
274
|
+
const issueKey = flag(args, 'issue');
|
|
275
|
+
let feature = typed;
|
|
276
|
+
let issueTitle = '';
|
|
277
|
+
if (issueKey) {
|
|
278
|
+
// A ticket says what was asked for better than a sentence typed twice.
|
|
279
|
+
const { Jira, jiraFromEnv, jiraHelp } = await import('../engine/work/jira.js');
|
|
280
|
+
const { featureFromIssue } = await import('../engine/work/jira-command.js');
|
|
281
|
+
const config = jiraFromEnv();
|
|
282
|
+
if (!config)
|
|
283
|
+
throw new Error(jiraHelp());
|
|
284
|
+
const issue = await new Jira(config).issue(issueKey);
|
|
285
|
+
console.error(`Read ${issue.key} (${issue.type}, ${issue.status})${issue.acceptanceCriteria ? ' with its acceptance criteria' : ''}.`);
|
|
286
|
+
issueTitle = `${issue.key}: ${issue.summary}`;
|
|
287
|
+
feature = [featureFromIssue(issue), typed].filter(Boolean).join('\n\n');
|
|
288
|
+
}
|
|
266
289
|
if (!feature)
|
|
267
|
-
throw new Error('usage: tewip build "<what the feature should do>" [--dry-run] [--
|
|
290
|
+
throw new Error('usage: tewip build "<what the feature should do>" | --issue PROJ-123 [--dry-run] [--pr] [--out feature.patch]');
|
|
268
291
|
const { runBuild, renderBuild } = await import('../engine/build/build-command.js');
|
|
269
292
|
const dryRun = args.includes('--dry-run');
|
|
270
293
|
const result = await runBuild({
|
|
@@ -276,9 +299,27 @@ async function main() {
|
|
|
276
299
|
out: flag(args, 'out'),
|
|
277
300
|
});
|
|
278
301
|
console.log(`
|
|
279
|
-
${renderBuild(result, feature, dryRun)}`);
|
|
280
|
-
if (result.status !== 'built')
|
|
302
|
+
${renderBuild(result, issueTitle || feature, dryRun)}`);
|
|
303
|
+
if (result.status !== 'built') {
|
|
281
304
|
process.exitCode = 1;
|
|
305
|
+
return;
|
|
306
|
+
}
|
|
307
|
+
if (args.includes('--pr')) {
|
|
308
|
+
if (dryRun)
|
|
309
|
+
throw new Error('--pr and --dry-run ask for opposite things: one keeps the work, the other puts it back.');
|
|
310
|
+
const { proposePullRequest } = await import('../engine/build/propose-pr.js');
|
|
311
|
+
const proposed = proposePullRequest({
|
|
312
|
+
root: process.cwd(),
|
|
313
|
+
files: result.files,
|
|
314
|
+
title: (issueTitle || feature.split('\n')[0]).slice(0, 72),
|
|
315
|
+
body: result.why,
|
|
316
|
+
issue: issueKey,
|
|
317
|
+
});
|
|
318
|
+
console.log(`
|
|
319
|
+
${proposed.status === 'opened' ? 'Opened a draft pull request.' : proposed.status === 'committed' ? `Committed on ${proposed.branch}.` : 'No pull request was opened.'}`);
|
|
320
|
+
for (const line of proposed.next)
|
|
321
|
+
console.log(` ${line}`);
|
|
322
|
+
}
|
|
282
323
|
return;
|
|
283
324
|
}
|
|
284
325
|
case 'review': {
|
|
@@ -361,6 +402,20 @@ ${renderBuild(result, feature, dryRun)}`);
|
|
|
361
402
|
// it in CI without the build going red over a design decision nobody has made yet.
|
|
362
403
|
return;
|
|
363
404
|
}
|
|
405
|
+
case 'correct': {
|
|
406
|
+
const { runCorrect } = await import('../engine/learn/correct-command.js');
|
|
407
|
+
const words = args.filter((a) => !a.startsWith('--'));
|
|
408
|
+
console.log(runCorrect({
|
|
409
|
+
root: process.cwd(),
|
|
410
|
+
kind: words[0] ?? '',
|
|
411
|
+
key: words[1],
|
|
412
|
+
verdict: words[2],
|
|
413
|
+
reason: words.slice(3).join(' ') || undefined,
|
|
414
|
+
list: args.includes('--list'),
|
|
415
|
+
by: flag(args, 'by'),
|
|
416
|
+
}));
|
|
417
|
+
return;
|
|
418
|
+
}
|
|
364
419
|
case 'mcp':
|
|
365
420
|
// An editor starts this from any folder, and an installed tool's own folder may be read-only:
|
|
366
421
|
// the AI answer cache goes to the user's home folder, which is always writable.
|
|
@@ -53,6 +53,57 @@ export function definingSources(root, feature, limit = 3) {
|
|
|
53
53
|
}
|
|
54
54
|
return scored.sort((a, b) => b.score - a.score).slice(0, limit).map((s) => s.file);
|
|
55
55
|
}
|
|
56
|
+
/**
|
|
57
|
+
* Does this feature belong in this project at all?
|
|
58
|
+
*
|
|
59
|
+
* Found by trying it: a real Apache ticket about a Flink HTTP sink was handed to a URL library, and
|
|
60
|
+
* Tewip built an "HTTP sink" into it - 28 lines, its own test passing, the suite green. Every proof
|
|
61
|
+
* rule held, because when Tewip writes both the test and the code it can always make a
|
|
62
|
+
* self-consistent pair pass. Nothing else referenced the invented API, so nothing else could break.
|
|
63
|
+
*
|
|
64
|
+
* The missing check is the obvious one: the words of the feature have to mean something here. A
|
|
65
|
+
* project's own vocabulary is its file names and what it exports, and a feature that shares almost
|
|
66
|
+
* none of it is a feature for some other repository.
|
|
67
|
+
*/
|
|
68
|
+
export function belongsHere(root, feature, sources, testNames) {
|
|
69
|
+
const stop = new Set(['should', 'when', 'with', 'that', 'this', 'from', 'into', 'have', 'been', 'will', 'must', 'also', 'able', 'given', 'then', 'there', 'their', 'which', 'while', 'where', 'after', 'before', 'value', 'values', 'return', 'returns', 'support', 'supports', 'accept', 'accepts', 'option', 'options', 'default', 'behaviour', 'behavior', 'feature', 'example', 'instead', 'without']);
|
|
70
|
+
const words = (text) => new Set((text.toLowerCase().match(/[a-z][a-z0-9]{3,}/g) ?? []).filter((w) => !stop.has(w)));
|
|
71
|
+
const asked = [...words(feature)];
|
|
72
|
+
if (asked.length < 3)
|
|
73
|
+
return { ok: true, why: '' };
|
|
74
|
+
const vocabulary = new Set();
|
|
75
|
+
for (const s of sources)
|
|
76
|
+
for (const w of words(`${s.file} ${s.content.slice(0, 20_000)}`))
|
|
77
|
+
vocabulary.add(w);
|
|
78
|
+
for (const name of testNames)
|
|
79
|
+
for (const w of words(name))
|
|
80
|
+
vocabulary.add(w);
|
|
81
|
+
// The names of a project's files are not its vocabulary: "naturalsize" lives inside filesize.py,
|
|
82
|
+
// not in its name, and judging by names alone refused a feature that really did belong here.
|
|
83
|
+
let read = 0;
|
|
84
|
+
for (const file of sourceFiles(root, 1200)) {
|
|
85
|
+
if (!CODE_FILE.test(file) || read >= 60)
|
|
86
|
+
continue;
|
|
87
|
+
read++;
|
|
88
|
+
try {
|
|
89
|
+
for (const w of words(`${basename(file)} ${readFileSync(file, 'utf8').slice(0, 20_000)}`))
|
|
90
|
+
vocabulary.add(w);
|
|
91
|
+
}
|
|
92
|
+
catch {
|
|
93
|
+
// unreadable here is simply one less file of vocabulary
|
|
94
|
+
}
|
|
95
|
+
}
|
|
96
|
+
if (vocabulary.size === 0)
|
|
97
|
+
return { ok: true, why: '' };
|
|
98
|
+
const shared = asked.filter((w) => vocabulary.has(w));
|
|
99
|
+
const share = shared.length / asked.length;
|
|
100
|
+
if (share >= 0.2)
|
|
101
|
+
return { ok: true, why: '' };
|
|
102
|
+
return {
|
|
103
|
+
ok: false,
|
|
104
|
+
why: `almost nothing in this project mentions what that describes (${shared.length} of ${asked.length} of its words appear here${shared.length ? `: ${shared.slice(0, 5).join(', ')}` : ''}). If this is the right repository, say the feature in its own words - the names of the functions and files it should change. Tewip can make a test of its own invention pass in any project, and that would prove nothing.`,
|
|
105
|
+
};
|
|
106
|
+
}
|
|
56
107
|
/** A test file name in the project's own shape, that does not collide with an existing one. */
|
|
57
108
|
function testPathFor(framework, feature) {
|
|
58
109
|
const stem = feature.toLowerCase().replace(/[^a-z0-9]+/g, '-').replace(/^-|-$/g, '').split('-').slice(0, 4).join('-') || 'feature';
|
|
@@ -88,6 +139,10 @@ export async function runBuild(cmd) {
|
|
|
88
139
|
testNames: tests.tests,
|
|
89
140
|
};
|
|
90
141
|
const log = [`framework: ${framework.label}`, `source files read: ${ctx.sources.map((s) => s.file).join(', ') || 'none found'}`];
|
|
142
|
+
// Before anything is written: is this even a feature for this repository?
|
|
143
|
+
const belongs = belongsHere(root, cmd.feature, ctx.sources, ctx.testNames);
|
|
144
|
+
if (!belongs.ok)
|
|
145
|
+
return { status: 'not-this-project', why: belongs.why, files: [], diff: '', log: [...log, belongs.why] };
|
|
91
146
|
const workspace = new Workspace(root);
|
|
92
147
|
const maxLines = cmd.maxLines ?? 120;
|
|
93
148
|
const stop = (status, why) => {
|
|
@@ -0,0 +1,88 @@
|
|
|
1
|
+
// Turning proved work into a pull request - the last step of the loop, and the one where Tewip
|
|
2
|
+
// touches somebody's repository, so it is the most careful.
|
|
3
|
+
//
|
|
4
|
+
// Three rules, enforced here rather than promised:
|
|
5
|
+
// it only ever pushes a new branch of its own, never yours;
|
|
6
|
+
// it never merges, and opens the request as a draft so nobody can merge it by reflex;
|
|
7
|
+
// it describes exactly what was proved and what was not, because a pull request nobody can check
|
|
8
|
+
// is worse than a patch file.
|
|
9
|
+
//
|
|
10
|
+
// The work must already be proved before anything reaches this file. Nothing here runs a test or
|
|
11
|
+
// decides anything; it commits what was verified and opens the request.
|
|
12
|
+
import { spawnSync } from 'node:child_process';
|
|
13
|
+
const run = (cwd, command, args) => {
|
|
14
|
+
const r = spawnSync(command, args, { cwd, encoding: 'utf8', timeout: 120_000, maxBuffer: 16 * 1024 * 1024 });
|
|
15
|
+
return { ok: r.status === 0, out: `${r.stdout ?? ''}${r.stderr ?? ''}`.trim() };
|
|
16
|
+
};
|
|
17
|
+
/** A branch name a person can read at a glance, and that cannot collide with an existing one. */
|
|
18
|
+
export function branchNameFor(title, now = new Date()) {
|
|
19
|
+
const stem = title.toLowerCase().replace(/[^a-z0-9]+/g, '-').replace(/^-|-$/g, '').split('-').slice(0, 6).join('-') || 'change';
|
|
20
|
+
return `tewip/${stem}-${now.toISOString().slice(2, 16).replace(/\D/g, '')}`;
|
|
21
|
+
}
|
|
22
|
+
/** The pull request body: what was built, and precisely how far the proof goes. */
|
|
23
|
+
export function requestBody(body, files, issue) {
|
|
24
|
+
return [
|
|
25
|
+
body,
|
|
26
|
+
'',
|
|
27
|
+
`Files: ${files.map((f) => `\`${f}\``).join(', ')}`,
|
|
28
|
+
issue ? `From: ${issue}` : '',
|
|
29
|
+
'',
|
|
30
|
+
'---',
|
|
31
|
+
'',
|
|
32
|
+
'Written by Tewip and kept only because it was proved: the test failed without this change and',
|
|
33
|
+
'passes with it, and every test that was passing still passes.',
|
|
34
|
+
'',
|
|
35
|
+
'What that does **not** say: whether this is the design you want, whether it is the right place',
|
|
36
|
+
'for it, or whether it is complete. That is what your review is for. This is a draft on a branch',
|
|
37
|
+
'of its own; Tewip never merges anything.',
|
|
38
|
+
]
|
|
39
|
+
.filter((line) => line !== '')
|
|
40
|
+
.join('\n');
|
|
41
|
+
}
|
|
42
|
+
/**
|
|
43
|
+
* Commits the proved files on a fresh branch and opens a draft pull request with the `gh` CLI the
|
|
44
|
+
* developer is already logged into. When there is no `gh`, the branch is still pushed and the
|
|
45
|
+
* command to open the request by hand is printed - half a job done is better than a silent failure.
|
|
46
|
+
*/
|
|
47
|
+
export function proposePullRequest(input) {
|
|
48
|
+
const { root, files } = input;
|
|
49
|
+
if (files.length === 0)
|
|
50
|
+
return { status: 'refused', next: ['there was nothing to commit'] };
|
|
51
|
+
if (!run(root, 'git', ['rev-parse', '--git-dir']).ok)
|
|
52
|
+
return { status: 'refused', next: ['this is not a git checkout, so there is no branch to open a request from'] };
|
|
53
|
+
const branch = branchNameFor(input.title);
|
|
54
|
+
const base = run(root, 'git', ['rev-parse', '--abbrev-ref', 'HEAD']).out.trim() || 'main';
|
|
55
|
+
// A branch of Tewip's own, from wherever the person is standing. Their branch is never committed to.
|
|
56
|
+
const made = run(root, 'git', ['checkout', '-b', branch]);
|
|
57
|
+
if (!made.ok)
|
|
58
|
+
return { status: 'refused', next: [`could not create the branch ${branch}: ${made.out.slice(0, 200)}`] };
|
|
59
|
+
const added = run(root, 'git', ['add', '--', ...files]);
|
|
60
|
+
if (!added.ok) {
|
|
61
|
+
run(root, 'git', ['checkout', base]);
|
|
62
|
+
return { status: 'refused', next: [`could not stage the files: ${added.out.slice(0, 200)}`] };
|
|
63
|
+
}
|
|
64
|
+
// Only Tewip's own files are committed: anything else the person had in progress stays theirs.
|
|
65
|
+
const committed = run(root, 'git', ['commit', '-m', input.title, '-m', requestBody(input.body, files, input.issue), '--only', '--', ...files]);
|
|
66
|
+
if (!committed.ok) {
|
|
67
|
+
run(root, 'git', ['checkout', base]);
|
|
68
|
+
return { status: 'refused', next: [`could not commit: ${committed.out.slice(0, 200)}`] };
|
|
69
|
+
}
|
|
70
|
+
const next = [];
|
|
71
|
+
const pushed = run(root, 'git', ['push', '-u', 'origin', branch]);
|
|
72
|
+
if (!pushed.ok) {
|
|
73
|
+
return {
|
|
74
|
+
status: 'committed',
|
|
75
|
+
branch,
|
|
76
|
+
next: [`the work is committed on ${branch}, but pushing failed: ${pushed.out.slice(0, 200)}`, `push it yourself with: git push -u origin ${branch}`],
|
|
77
|
+
};
|
|
78
|
+
}
|
|
79
|
+
const gh = run(root, 'gh', ['pr', 'create', '--draft', '--base', base, '--head', branch, '--title', input.title, '--body', requestBody(input.body, files, input.issue)]);
|
|
80
|
+
if (gh.ok) {
|
|
81
|
+
const url = (gh.out.match(/https:\/\/\S+/) ?? [])[0];
|
|
82
|
+
return { status: 'opened', branch, url, next: [`review it: ${url ?? branch}`, `back to where you were: git checkout ${base}`] };
|
|
83
|
+
}
|
|
84
|
+
next.push(`${branch} is pushed, but the pull request was not opened: ${gh.out.slice(0, 200)}`);
|
|
85
|
+
next.push(`open it with: gh pr create --draft --base ${base} --head ${branch}`);
|
|
86
|
+
next.push(`back to where you were: git checkout ${base}`);
|
|
87
|
+
return { status: 'committed', branch, next };
|
|
88
|
+
}
|
|
@@ -8,6 +8,7 @@ import { readPages } from '../quality/grounding.js';
|
|
|
8
8
|
import { Figma, figmaHelp, figmaTokenFromEnv, parseFigmaUrl, topFrames } from './figma.js';
|
|
9
9
|
import { frameChecks } from './frame.js';
|
|
10
10
|
import { compareDesign, renderDrift } from './drift.js';
|
|
11
|
+
import { loadVerdicts, saidByAPerson, verdictFor } from '../learn/verdicts.js';
|
|
11
12
|
/** A saved Figma response, in any of the shapes Figma's API returns. */
|
|
12
13
|
function fromFile(path) {
|
|
13
14
|
const raw = JSON.parse(readFileSync(path, 'utf8'));
|
|
@@ -70,8 +71,22 @@ export async function runDesign(cmd) {
|
|
|
70
71
|
console.log(`\nThe page was not read, so nothing was compared: ${pageNote ?? 'unknown reason'}`);
|
|
71
72
|
return [];
|
|
72
73
|
}
|
|
73
|
-
const
|
|
74
|
-
|
|
74
|
+
const found = compareDesign(checks, pages[0].aria);
|
|
75
|
+
// A difference a person has already looked at and accepted is not reported again as a surprise.
|
|
76
|
+
const corrections = loadVerdicts(cmd.root);
|
|
77
|
+
const settled = [];
|
|
78
|
+
const drifts = found.map((d) => {
|
|
79
|
+
const key = `${d.check.role ?? d.check.kind} ${d.check.name}`.trim();
|
|
80
|
+
const said = verdictFor('design', key, corrections) ?? verdictFor('design', d.check.name, corrections);
|
|
81
|
+
if (!said || said.verdict !== 'not-a-difference')
|
|
82
|
+
return d;
|
|
83
|
+
settled.push(`${key}: ${saidByAPerson(said)}`);
|
|
84
|
+
return { ...d, verdict: 'met' };
|
|
85
|
+
});
|
|
86
|
+
const alreadySettled = settled.length
|
|
87
|
+
? ['', `${settled.length} difference(s) your team has already settled, so they are not reported:`, ...settled.map((s) => ` - ${s}`)].join('\n')
|
|
88
|
+
: '';
|
|
89
|
+
const report = [renderDrift(drifts, design, cmd.url), alreadySettled].filter(Boolean).join('\n');
|
|
75
90
|
console.log(report);
|
|
76
91
|
if (note)
|
|
77
92
|
console.log(`\n${note}`);
|
|
@@ -6,6 +6,7 @@ import { join, resolve } from 'node:path';
|
|
|
6
6
|
import { datasetFiles, readRunSummaries, readTriageRecords } from '../../reporters/store.js';
|
|
7
7
|
import { flakyTests, readyToRelease, weeklyCost } from './flaky.js';
|
|
8
8
|
import { applyQuarantine, frameworkOf, quarantineFor } from './quarantine.js';
|
|
9
|
+
import { loadVerdicts, saidByAPerson, verdictFor } from '../learn/verdicts.js';
|
|
9
10
|
const DAY_MS = 24 * 60 * 60 * 1000;
|
|
10
11
|
const minutes = (ms) => `${(ms / 60_000).toFixed(1)} CI min`;
|
|
11
12
|
function projects(root, only) {
|
|
@@ -31,7 +32,14 @@ export async function runFlaky(cmd) {
|
|
|
31
32
|
const history = datasetFiles(join(cmd.root, '.tewip', 'history', name));
|
|
32
33
|
const records = readTriageRecords(history.triage);
|
|
33
34
|
const runs = readRunSummaries(history.runs);
|
|
34
|
-
const
|
|
35
|
+
const computed = flakyTests(records, runs, { since });
|
|
36
|
+
// What a person already decided about a test wins over what Tewip works out, and is shown as
|
|
37
|
+
// theirs: the argument is had once, not every Monday.
|
|
38
|
+
const corrections = loadVerdicts(cmd.root);
|
|
39
|
+
const tests = computed.map((test) => {
|
|
40
|
+
const said = verdictFor('flaky', test.title, corrections) ?? verdictFor('flaky', test.testId, corrections);
|
|
41
|
+
return said ? { ...test, verdict: said.verdict, why: `${saidByAPerson(said)} (Tewip had said ${test.verdict})` } : test;
|
|
42
|
+
});
|
|
35
43
|
if (tests.length === 0)
|
|
36
44
|
continue;
|
|
37
45
|
all.push(...tests);
|
|
@@ -0,0 +1,88 @@
|
|
|
1
|
+
// `tewip correct`: the one place a person tells Tewip it was wrong, about anything.
|
|
2
|
+
//
|
|
3
|
+
// A diagnosis can already be corrected where Tewip posts it in CI. This is the same thing for
|
|
4
|
+
// everything else it decides, and on a person's own machine, where most of the disagreeing happens.
|
|
5
|
+
import { existsSync, readdirSync } from 'node:fs';
|
|
6
|
+
import { join } from 'node:path';
|
|
7
|
+
import { ROOT_CAUSES } from '../../reporters/types.js';
|
|
8
|
+
import { datasetFiles, readTriageRecords } from '../../reporters/store.js';
|
|
9
|
+
import { loadCorrections, saveCorrections } from '../corrections.js';
|
|
10
|
+
import { errorShape } from '../signature.js';
|
|
11
|
+
import { isVerdictWord, loadVerdicts, saidByAPerson, saveVerdict, verdictHelp, VERDICT_WORDS } from './verdicts.js';
|
|
12
|
+
const KINDS = Object.keys(VERDICT_WORDS);
|
|
13
|
+
/**
|
|
14
|
+
* The failure a title names, and the history folder it lives in. The folder matters: corrections are
|
|
15
|
+
* kept beside the runs of that project, which is where the engine looks for them - an earlier
|
|
16
|
+
* version wrote them one directory up, where nothing ever read them.
|
|
17
|
+
*/
|
|
18
|
+
function findRecord(root, key) {
|
|
19
|
+
const dir = join(root, '.tewip', 'history');
|
|
20
|
+
if (!existsSync(dir))
|
|
21
|
+
return null;
|
|
22
|
+
for (const project of readdirSync(dir)) {
|
|
23
|
+
const historyDir = join(dir, project);
|
|
24
|
+
const records = readTriageRecords(datasetFiles(historyDir).triage);
|
|
25
|
+
const found = [...records].reverse().find((r) => r.testId.toLowerCase().includes(key.trim().toLowerCase()));
|
|
26
|
+
if (found)
|
|
27
|
+
return { record: found, historyDir };
|
|
28
|
+
}
|
|
29
|
+
return null;
|
|
30
|
+
}
|
|
31
|
+
export function runCorrect(cmd) {
|
|
32
|
+
const who = cmd.by ?? process.env.USER ?? process.env.USERNAME ?? 'someone on the team';
|
|
33
|
+
if (cmd.list) {
|
|
34
|
+
const verdicts = loadVerdicts(cmd.root);
|
|
35
|
+
const historyRoot = join(cmd.root, '.tewip', 'history');
|
|
36
|
+
const diagnoses = (existsSync(historyRoot) ? readdirSync(historyRoot) : []).flatMap((project) => loadCorrections(join(historyRoot, project)));
|
|
37
|
+
if (verdicts.length === 0 && diagnoses.length === 0)
|
|
38
|
+
return 'Nothing has been corrected yet.\n\n' + verdictHelp();
|
|
39
|
+
const lines = [`${verdicts.length + diagnoses.length} correction(s), newest last:`, ''];
|
|
40
|
+
for (const v of verdicts)
|
|
41
|
+
lines.push(` ${v.kind.padEnd(9)} ${v.key} -> ${v.verdict} (${saidByAPerson(v)})`);
|
|
42
|
+
for (const d of diagnoses)
|
|
43
|
+
lines.push(` diagnosis ${d.testId} -> ${d.category} (${d.by}${d.reason ? `: ${d.reason}` : ''})`);
|
|
44
|
+
return lines.join('\n');
|
|
45
|
+
}
|
|
46
|
+
const kind = cmd.kind?.trim().toLowerCase() ?? '';
|
|
47
|
+
if (!cmd.key || !cmd.verdict)
|
|
48
|
+
throw new Error(`usage: tewip correct <${[...KINDS, 'diagnosis'].join('|')}> "<what it is about>" <verdict> ["why"]\n\n${verdictHelp()}`);
|
|
49
|
+
// A diagnosis is corrected into the same store the CI path writes, so both ends agree.
|
|
50
|
+
if (kind === 'diagnosis') {
|
|
51
|
+
const category = cmd.verdict.trim().toUpperCase();
|
|
52
|
+
if (!ROOT_CAUSES.includes(category))
|
|
53
|
+
throw new Error(`"${cmd.verdict}" is not a category. They are: ${ROOT_CAUSES.join(', ')}`);
|
|
54
|
+
const found = findRecord(cmd.root, cmd.key);
|
|
55
|
+
if (!found)
|
|
56
|
+
throw new Error(`no failure of a test called "${cmd.key}" is in the history Tewip has kept here. Run \`tewip explain\` on the report first, so there is something to correct.`);
|
|
57
|
+
const { record, historyDir } = found;
|
|
58
|
+
const correction = {
|
|
59
|
+
schemaVersion: 1,
|
|
60
|
+
testId: record.testId,
|
|
61
|
+
// Stored exactly as the matcher compares it: a truncated shape can never equal a real one.
|
|
62
|
+
errorShape: errorShape(record.errorMessage),
|
|
63
|
+
category,
|
|
64
|
+
by: who,
|
|
65
|
+
at: new Date().toISOString(),
|
|
66
|
+
reason: cmd.reason,
|
|
67
|
+
source: 'tewip correct',
|
|
68
|
+
wasCategory: record.diagnosis?.category,
|
|
69
|
+
};
|
|
70
|
+
saveCorrections(historyDir, [correction]);
|
|
71
|
+
return [
|
|
72
|
+
`Noted: "${record.testId}" is ${category}${record.diagnosis?.category ? `, not ${record.diagnosis.category}` : ''}.`,
|
|
73
|
+
'Tewip will answer with your decision the next time that test fails the same way, and say it was yours.',
|
|
74
|
+
].join('\n');
|
|
75
|
+
}
|
|
76
|
+
if (!KINDS.includes(kind))
|
|
77
|
+
throw new Error(`"${cmd.kind}" is not something Tewip decides. It decides: ${[...KINDS, 'diagnosis'].join(', ')}.\n\n${verdictHelp()}`);
|
|
78
|
+
const verdictKind = kind;
|
|
79
|
+
if (!isVerdictWord(verdictKind, cmd.verdict)) {
|
|
80
|
+
throw new Error(`"${cmd.verdict}" is not one of the answers a ${verdictKind} verdict takes. They are: ${VERDICT_WORDS[verdictKind].join(', ')}.`);
|
|
81
|
+
}
|
|
82
|
+
const saved = saveVerdict(cmd.root, { kind: verdictKind, key: cmd.key, verdict: cmd.verdict.trim().toLowerCase(), reason: cmd.reason ?? '', by: who });
|
|
83
|
+
return [
|
|
84
|
+
`Noted: ${verdictKind} "${cmd.key}" is ${saved.verdict}.`,
|
|
85
|
+
'Tewip will say that from now on, and say it was your decision rather than its own.',
|
|
86
|
+
'It will not act on it by itself: a correction changes what it says, never what it does.',
|
|
87
|
+
].join('\n');
|
|
88
|
+
}
|
|
@@ -0,0 +1,71 @@
|
|
|
1
|
+
// The LEARN step, everywhere.
|
|
2
|
+
//
|
|
3
|
+
// A person could already correct a diagnosis by replying under Tewip's own comment in CI, and Tewip
|
|
4
|
+
// answers with their decision from then on. Every other verdict it gives - this test should be
|
|
5
|
+
// quarantined, this secret is a real risk, the page has drifted from the design, this feature
|
|
6
|
+
// cannot be built here - had no such path, so the same argument had to be had again every week.
|
|
7
|
+
//
|
|
8
|
+
// This is that path, for all of them, and it keeps the properties that made the first one worth
|
|
9
|
+
// having: a correction is written by a person, it is stored with the project rather than inside
|
|
10
|
+
// Tewip, it says who and why, and **it never authorises an action on its own**. Deciding that a
|
|
11
|
+
// failure is a real bug does not make Tewip change code; deciding a flaky test is really broken
|
|
12
|
+
// does not make it un-quarantine anything. It changes what Tewip *says*, which is what a person
|
|
13
|
+
// was disagreeing with.
|
|
14
|
+
import { existsSync } from 'node:fs';
|
|
15
|
+
import { join } from 'node:path';
|
|
16
|
+
import { appendJsonl, readJsonl } from '../../reporters/store.js';
|
|
17
|
+
/** The words each kind accepts, so a typo is caught rather than stored and silently ignored. */
|
|
18
|
+
export const VERDICT_WORDS = {
|
|
19
|
+
flaky: ['quarantine', 'fix-now', 'watch', 'release'],
|
|
20
|
+
security: ['real-risk', 'not-reachable', 'noise', 'accepted'],
|
|
21
|
+
design: ['real', 'not-a-difference'],
|
|
22
|
+
build: ['belongs-here', 'not-this-project'],
|
|
23
|
+
review: ['covers', 'does-not-cover'],
|
|
24
|
+
};
|
|
25
|
+
export const VERDICTS_FILE = 'verdicts.jsonl';
|
|
26
|
+
const fileIn = (root) => join(root, '.tewip', VERDICTS_FILE);
|
|
27
|
+
export function loadVerdicts(root) {
|
|
28
|
+
const path = fileIn(root);
|
|
29
|
+
return existsSync(path) ? readJsonl(path) : [];
|
|
30
|
+
}
|
|
31
|
+
/** Adds one correction, replacing an earlier one about the same thing. */
|
|
32
|
+
export function saveVerdict(root, correction) {
|
|
33
|
+
const entry = { schemaVersion: 1, at: correction.at ?? new Date().toISOString(), ...correction };
|
|
34
|
+
appendJsonl(fileIn(root), [entry]);
|
|
35
|
+
return entry;
|
|
36
|
+
}
|
|
37
|
+
const same = (a, b) => a.trim().toLowerCase() === b.trim().toLowerCase();
|
|
38
|
+
/**
|
|
39
|
+
* The person's answer about this thing, or null. The newest wins, so a team that changes its mind
|
|
40
|
+
* simply says so again rather than editing a file.
|
|
41
|
+
*/
|
|
42
|
+
export function verdictFor(kind, key, corrections) {
|
|
43
|
+
const matching = corrections.filter((c) => c.kind === kind && (same(c.key, key) || key.toLowerCase().endsWith(c.key.trim().toLowerCase())));
|
|
44
|
+
return matching.length > 0 ? matching[matching.length - 1] : null;
|
|
45
|
+
}
|
|
46
|
+
/** What a corrected verdict is shown as, so nobody mistakes a person's decision for Tewip's. */
|
|
47
|
+
export function saidByAPerson(correction) {
|
|
48
|
+
const when = correction.at.slice(0, 10);
|
|
49
|
+
return `${correction.by} decided this on ${when}${correction.reason ? `: ${correction.reason}` : ''}`;
|
|
50
|
+
}
|
|
51
|
+
/** Whether a word is one this kind of verdict uses. */
|
|
52
|
+
export function isVerdictWord(kind, word) {
|
|
53
|
+
return VERDICT_WORDS[kind].includes(word.trim().toLowerCase());
|
|
54
|
+
}
|
|
55
|
+
export function verdictHelp() {
|
|
56
|
+
return [
|
|
57
|
+
'Correct something Tewip said, and it will answer with your decision from now on:',
|
|
58
|
+
'',
|
|
59
|
+
' tewip correct flaky "adds an item to the cart" fix-now "it is a real race, not flakiness"',
|
|
60
|
+
' tewip correct security "src/config.ts:12" noise "that is a sample key in a fixture"',
|
|
61
|
+
' tewip correct design "button Sign in" not-a-difference "we renamed it on purpose"',
|
|
62
|
+
' tewip correct build "an HTTP sink" belongs-here "this really is the right repository"',
|
|
63
|
+
'',
|
|
64
|
+
'The words each kind takes:',
|
|
65
|
+
...Object.entries(VERDICT_WORDS).map(([kind, words]) => ` ${kind.padEnd(9)} ${words.join(', ')}`),
|
|
66
|
+
'',
|
|
67
|
+
'A correction changes what Tewip says, never what it does on its own: it will not change code,',
|
|
68
|
+
'quarantine a test or file a ticket because you corrected it.',
|
|
69
|
+
'They are kept in .tewip/verdicts.jsonl, with the project, so the whole team shares them.',
|
|
70
|
+
].join('\n');
|
|
71
|
+
}
|
|
@@ -9,6 +9,7 @@ import { auditDependencies } from './deps.js';
|
|
|
9
9
|
import { scanSecrets, sourceFiles } from './secrets.js';
|
|
10
10
|
import { readAccepted, triageDependency, triageSecret, triageWeb, worstFirst } from './triage.js';
|
|
11
11
|
import { isLocal, scanWeb } from './web.js';
|
|
12
|
+
import { loadVerdicts, saidByAPerson, verdictFor } from '../learn/verdicts.js';
|
|
12
13
|
import { readZapReport, zapAvailable, zapInstructions } from './zap.js';
|
|
13
14
|
const CODE = /\.(?:[cm]?[jt]sx?|py|rb|go|java|kt|cs|php|rs|vue|svelte)$/;
|
|
14
15
|
export function renderSecurity(items, checked, notChecked, root) {
|
|
@@ -97,6 +98,14 @@ export async function runSecure(cmd) {
|
|
|
97
98
|
else if (cmd.url) {
|
|
98
99
|
notChecked.push(`a full web scan. Tewip's own check is passive; for the deep one: ${zapInstructions(cmd.url, zapAvailable(root))}`);
|
|
99
100
|
}
|
|
101
|
+
// A person's decision about a finding wins over Tewip's, and says whose it was.
|
|
102
|
+
const corrections = loadVerdicts(root);
|
|
103
|
+
const decided = items.map((item) => {
|
|
104
|
+
const said = verdictFor('security', item.where, corrections) ?? verdictFor('security', item.title, corrections);
|
|
105
|
+
return said ? { ...item, verdict: said.verdict, why: `${saidByAPerson(said)} (Tewip had said ${item.verdict})` } : item;
|
|
106
|
+
});
|
|
107
|
+
items.length = 0;
|
|
108
|
+
items.push(...decided);
|
|
100
109
|
items.sort(worstFirst);
|
|
101
110
|
const report = renderSecurity(items, checked, notChecked, root);
|
|
102
111
|
console.log(report);
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "tewip-ai",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.14.0",
|
|
4
4
|
"description": "Tewip finds why each failing test failed - real bug, flaky test, environment, test data or changed selector - for any stack, from the command line, CI, or your editor.",
|
|
5
5
|
"keywords": [
|
|
6
6
|
"test-failures",
|