tewip-ai 0.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +202 -0
- package/QUICKSTART.md +178 -0
- package/README.md +308 -0
- package/TEAM-GUIDE.md +234 -0
- package/dist/cli/tewip.js +117 -0
- package/dist/engine/autofix/autofix.js +63 -0
- package/dist/engine/autofix/candidates.js +33 -0
- package/dist/engine/autofix/locate.js +78 -0
- package/dist/engine/corrections.js +66 -0
- package/dist/engine/dataset.js +41 -0
- package/dist/engine/decide.js +130 -0
- package/dist/engine/device-errors.js +45 -0
- package/dist/engine/drift-rules.js +98 -0
- package/dist/engine/driver-errors.js +103 -0
- package/dist/engine/engine.js +76 -0
- package/dist/engine/error-parse.js +150 -0
- package/dist/engine/history.js +105 -0
- package/dist/engine/import/blob.js +165 -0
- package/dist/engine/import/junit.js +235 -0
- package/dist/engine/import/xml.js +104 -0
- package/dist/engine/llm/anthropic.js +59 -0
- package/dist/engine/llm/client.js +139 -0
- package/dist/engine/llm/context.js +141 -0
- package/dist/engine/llm/llm-diagnose.js +48 -0
- package/dist/engine/llm/prompt.js +40 -0
- package/dist/engine/llm/providers.js +54 -0
- package/dist/engine/llm/usage.js +69 -0
- package/dist/engine/pipeline.js +33 -0
- package/dist/engine/quality/frameworks.js +181 -0
- package/dist/engine/quality/grounding.js +203 -0
- package/dist/engine/quality/model.js +17 -0
- package/dist/engine/quality/plan-command.js +34 -0
- package/dist/engine/quality/plan-render.js +38 -0
- package/dist/engine/quality/plan.js +119 -0
- package/dist/engine/quality/write-command.js +146 -0
- package/dist/engine/quality/write-verify.js +140 -0
- package/dist/engine/quality/write.js +144 -0
- package/dist/engine/rules.js +174 -0
- package/dist/engine/signature.js +23 -0
- package/dist/engine/similarity.js +76 -0
- package/dist/engine/snapshot.js +56 -0
- package/dist/engine/timing-rules.js +188 -0
- package/dist/engine/trace/dom.js +92 -0
- package/dist/engine/trace/trace-summary.js +87 -0
- package/dist/engine/trace/zip.js +53 -0
- package/dist/eval/harvest/gh.js +21 -0
- package/dist/eval/harvest/harvest.js +117 -0
- package/dist/mcp/github-report.js +50 -0
- package/dist/mcp/server.js +80 -0
- package/dist/mcp/tools.js +124 -0
- package/dist/outlets/design-debt.js +22 -0
- package/dist/outlets/items.js +94 -0
- package/dist/outlets/view.js +63 -0
- package/dist/outlets/views/architecture.js +77 -0
- package/dist/outlets/views/developer.js +59 -0
- package/dist/outlets/views/digest.js +66 -0
- package/dist/outlets/views/infra.js +49 -0
- package/dist/outlets/views/quality.js +108 -0
- package/dist/outlets/views/release.js +33 -0
- package/dist/outlets/views/sdet.js +29 -0
- package/dist/reporters/run-context.js +20 -0
- package/dist/reporters/store.js +46 -0
- package/dist/reporters/triage-reporter.js +101 -0
- package/dist/reporters/types.js +4 -0
- package/dist/scripts/explain.js +58 -0
- package/package.json +81 -0
|
@@ -0,0 +1,165 @@
|
|
|
1
|
+
// Imports Playwright blob reports as triage records in the same shape the reporter writes,
|
|
2
|
+
// plus one run summary with every attempt. Two sources: a CI artifact zip holding one
|
|
3
|
+
// report-N.zip per shard (harvesting public repos), or a local blob-report/ directory (the
|
|
4
|
+
// GitHub Action, right after the tests ran). Consumers only need Playwright's built-in
|
|
5
|
+
// `blob` reporter, no Tewip code in their config.
|
|
6
|
+
import { mkdirSync, readdirSync, readFileSync, writeFileSync } from 'node:fs';
|
|
7
|
+
import { basename, isAbsolute, join, relative, sep } from 'node:path';
|
|
8
|
+
import { stripVTControlCharacters } from 'node:util';
|
|
9
|
+
import { PROJECT_ROOT } from '../../reporters/store.js';
|
|
10
|
+
import { SCHEMA_VERSION } from '../../reporters/types.js';
|
|
11
|
+
import { readZipBuffer } from '../trace/zip.js';
|
|
12
|
+
// Screenshots and videos are skipped: the diagnosis layers read traces and error-context only.
|
|
13
|
+
const KEPT_ATTACHMENTS = new Set(['trace', 'error-context']);
|
|
14
|
+
const MAX_ERROR_CHARS = 8000;
|
|
15
|
+
const clip = (s) => (s.length <= MAX_ERROR_CHARS ? s : `${s.slice(0, MAX_ERROR_CHARS)}…`);
|
|
16
|
+
const slash = (p) => p.split(sep).join('/');
|
|
17
|
+
/** testId hash -> readable test id ("project > file > describe > title") and spec file. */
|
|
18
|
+
function titlesOf(events, workspace) {
|
|
19
|
+
const out = new Map();
|
|
20
|
+
const walk = (entries = [], path, file, testDir) => {
|
|
21
|
+
for (const e of entries) {
|
|
22
|
+
const f = e.location?.file ?? file;
|
|
23
|
+
if (e.testId) {
|
|
24
|
+
const abs = isAbsolute(f) ? f : join(testDir, f);
|
|
25
|
+
const shown = workspace && testDir ? slash(relative(workspace, abs)) : f;
|
|
26
|
+
out.set(e.testId, { id: [...path, e.title].filter(Boolean).join(' > '), file: shown });
|
|
27
|
+
}
|
|
28
|
+
else
|
|
29
|
+
walk(e.entries, [...path, e.title], f, testDir);
|
|
30
|
+
}
|
|
31
|
+
};
|
|
32
|
+
// Blob reports store suite paths relative to config.rootDir (absolute) and the project's testDir
|
|
33
|
+
// relative to that root, often empty.
|
|
34
|
+
const rootDir = events.find((x) => x.method === 'onConfigure')?.params.config.rootDir ?? '';
|
|
35
|
+
for (const e of events.filter((x) => x.method === 'onProject')) {
|
|
36
|
+
const testDir = e.params.project.testDir ?? '';
|
|
37
|
+
walk(e.params.project.suites, [e.params.project.name], '', rootDir ? join(rootDir, testDir) : testDir);
|
|
38
|
+
}
|
|
39
|
+
return out;
|
|
40
|
+
}
|
|
41
|
+
function parseEvents(buf) {
|
|
42
|
+
return buf
|
|
43
|
+
.toString('utf8')
|
|
44
|
+
.split(/\r?\n/)
|
|
45
|
+
.filter((l) => l.trim())
|
|
46
|
+
.map((l) => JSON.parse(l));
|
|
47
|
+
}
|
|
48
|
+
/** A CI artifact zip: one report-N.zip per shard, or a single shard's report.jsonl directly. */
|
|
49
|
+
export function importBlobArtifact(artifact, meta, filesDir) {
|
|
50
|
+
return importShards(shardsOf(artifact, 'artifact'), meta, filesDir);
|
|
51
|
+
}
|
|
52
|
+
/**
|
|
53
|
+
* Several CI artifacts that belong to one run, e.g. one blob-report-N per shard (Supabase,
|
|
54
|
+
* Penpot) instead of one merged artifact (Gutenberg): imported as a single run.
|
|
55
|
+
*/
|
|
56
|
+
export function importBlobArtifacts(artifacts, meta, filesDir) {
|
|
57
|
+
return importShards(artifacts.flatMap(([name, buf]) => shardsOf(buf, name)), meta, filesDir);
|
|
58
|
+
}
|
|
59
|
+
function shardsOf(artifact, name) {
|
|
60
|
+
const outer = readZipBuffer(artifact, (n) => n.endsWith('.zip') || n === 'report.jsonl');
|
|
61
|
+
return outer.has('report.jsonl') ? [[name, artifact]] : [...outer.entries()].filter(([n]) => n.endsWith('.zip')).map(([n, b]) => [`${name}/${n}`, b]);
|
|
62
|
+
}
|
|
63
|
+
/** A local blob-report/ directory as the blob reporter writes it (report*.zip per shard). */
|
|
64
|
+
export function importBlobReportDir(dir, meta, filesDir) {
|
|
65
|
+
const shards = readdirSync(dir)
|
|
66
|
+
.filter((n) => n.endsWith('.zip'))
|
|
67
|
+
.map((n) => [n, readFileSync(join(dir, n))]);
|
|
68
|
+
return importShards(shards, meta, filesDir);
|
|
69
|
+
}
|
|
70
|
+
function importShards(shards, meta, filesDir) {
|
|
71
|
+
const records = [];
|
|
72
|
+
const attempts = {};
|
|
73
|
+
const skippedShards = [];
|
|
74
|
+
let startedAt = Number.POSITIVE_INFINITY;
|
|
75
|
+
let finishedAt = 0;
|
|
76
|
+
let runStatus = 'passed';
|
|
77
|
+
mkdirSync(filesDir, { recursive: true });
|
|
78
|
+
for (const [shardName, shard] of shards) {
|
|
79
|
+
let events;
|
|
80
|
+
try {
|
|
81
|
+
events = parseEvents(readZipBuffer(shard, (n) => n === 'report.jsonl').get('report.jsonl') ?? Buffer.alloc(0));
|
|
82
|
+
}
|
|
83
|
+
catch {
|
|
84
|
+
// Seen in real artifacts: a shard blob truncated at upload. Keep the other shards.
|
|
85
|
+
skippedShards.push(shardName);
|
|
86
|
+
continue;
|
|
87
|
+
}
|
|
88
|
+
const titles = titlesOf(events, meta.workspace);
|
|
89
|
+
const begins = new Map();
|
|
90
|
+
const attachmentsByResult = new Map();
|
|
91
|
+
for (const e of events) {
|
|
92
|
+
if (e.method === 'onTestBegin')
|
|
93
|
+
begins.set(e.params.result.id, { retry: e.params.result.retry, startTime: e.params.result.startTime });
|
|
94
|
+
if (e.method === 'onAttach')
|
|
95
|
+
attachmentsByResult.set(e.params.resultId, [...(attachmentsByResult.get(e.params.resultId) ?? []), ...e.params.attachments]);
|
|
96
|
+
if (e.method === 'onEnd') {
|
|
97
|
+
startedAt = Math.min(startedAt, e.params.result.startTime);
|
|
98
|
+
finishedAt = Math.max(finishedAt, e.params.result.startTime + e.params.result.duration);
|
|
99
|
+
if (e.params.result.status !== 'passed')
|
|
100
|
+
runStatus = e.params.result.status;
|
|
101
|
+
}
|
|
102
|
+
}
|
|
103
|
+
const failed = [];
|
|
104
|
+
for (const e of events.filter((x) => x.method === 'onTestEnd')) {
|
|
105
|
+
const { test, result } = e.params;
|
|
106
|
+
const title = titles.get(test.testId) ?? { id: test.testId, file: '' };
|
|
107
|
+
if (result.status !== 'skipped' && result.status !== 'interrupted')
|
|
108
|
+
attempts[title.id] = (attempts[title.id] ?? 0) + 1;
|
|
109
|
+
if (result.status !== 'failed' && result.status !== 'timedOut')
|
|
110
|
+
continue;
|
|
111
|
+
const begin = begins.get(result.id) ?? { retry: 0, startTime: startedAt };
|
|
112
|
+
const error = result.errors?.[0] ?? {};
|
|
113
|
+
const all = (attachmentsByResult.get(result.id) ?? []).filter((a) => KEPT_ATTACHMENTS.has(a.name) && a.path);
|
|
114
|
+
failed.push({
|
|
115
|
+
wanted: all.filter((a, i) => all.findIndex((b) => b.path === a.path) === i),
|
|
116
|
+
record: {
|
|
117
|
+
schemaVersion: SCHEMA_VERSION,
|
|
118
|
+
testId: title.id,
|
|
119
|
+
file: title.file,
|
|
120
|
+
status: result.status,
|
|
121
|
+
errorMessage: clip(stripVTControlCharacters(error.message ?? '')),
|
|
122
|
+
errorStack: clip(stripVTControlCharacters(error.stack ?? '')),
|
|
123
|
+
duration: result.duration,
|
|
124
|
+
retry: begin.retry,
|
|
125
|
+
attachments: [],
|
|
126
|
+
runId: meta.runId,
|
|
127
|
+
gitSha: meta.gitSha,
|
|
128
|
+
timestamp: new Date(begin.startTime + result.duration).toISOString(),
|
|
129
|
+
},
|
|
130
|
+
});
|
|
131
|
+
}
|
|
132
|
+
// Only the resources failed attempts point at are extracted; the rest of the shard is skipped.
|
|
133
|
+
const needed = new Set(failed.flatMap((f) => f.wanted.map((a) => a.path)));
|
|
134
|
+
let resources = new Map();
|
|
135
|
+
try {
|
|
136
|
+
resources = readZipBuffer(shard, (n) => needed.has(n));
|
|
137
|
+
}
|
|
138
|
+
catch {
|
|
139
|
+
// A damaged resource leaves the records without attachments rather than dropping them.
|
|
140
|
+
}
|
|
141
|
+
for (const { record, wanted } of failed) {
|
|
142
|
+
for (const a of wanted) {
|
|
143
|
+
const data = resources.get(a.path);
|
|
144
|
+
if (!data)
|
|
145
|
+
continue;
|
|
146
|
+
const target = join(filesDir, basename(a.path));
|
|
147
|
+
writeFileSync(target, data);
|
|
148
|
+
const attachment = { name: a.name, contentType: a.contentType, path: slash(relative(PROJECT_ROOT, target)) };
|
|
149
|
+
record.attachments.push(attachment);
|
|
150
|
+
}
|
|
151
|
+
records.push(record);
|
|
152
|
+
}
|
|
153
|
+
}
|
|
154
|
+
const summary = {
|
|
155
|
+
schemaVersion: SCHEMA_VERSION,
|
|
156
|
+
runId: meta.runId,
|
|
157
|
+
gitSha: meta.gitSha,
|
|
158
|
+
...(meta.branch ? { branch: meta.branch } : {}),
|
|
159
|
+
startedAt: new Date(Number.isFinite(startedAt) ? startedAt : 0).toISOString(),
|
|
160
|
+
finishedAt: new Date(finishedAt).toISOString(),
|
|
161
|
+
status: runStatus,
|
|
162
|
+
attempts,
|
|
163
|
+
};
|
|
164
|
+
return { records, summary, skippedShards };
|
|
165
|
+
}
|
|
@@ -0,0 +1,235 @@
|
|
|
1
|
+
// Imports test reports from any language or framework into the agent's records: JUnit XML, which
|
|
2
|
+
// pytest, Jest, Vitest, Mocha, node:test, Maven Surefire, Gradle, gotestsum, PHPUnit, RSpec,
|
|
3
|
+
// cargo-nextest and Cypress write, and .NET's own TRX. Playwright keeps its own importer
|
|
4
|
+
// (blob.ts), which also reads traces.
|
|
5
|
+
//
|
|
6
|
+
// What these reports can and cannot say: each test's name, outcome, failure message and stack,
|
|
7
|
+
// and duration; retries when the runner records them. They carry no page snapshot or trace, so
|
|
8
|
+
// selector drift is not diagnosed from them. Every format below was checked against a report its
|
|
9
|
+
// real runner wrote (fixtures/polyglot, run by .github/workflows/polyglot-fixtures.yml).
|
|
10
|
+
import { readdirSync, readFileSync, statSync } from 'node:fs';
|
|
11
|
+
import { join } from 'node:path';
|
|
12
|
+
import { SCHEMA_VERSION } from '../../reporters/types.js';
|
|
13
|
+
import { descendants, parseXml } from './xml.js';
|
|
14
|
+
const PASSED = { failed: false, message: '', stack: '' };
|
|
15
|
+
const FAILURE = new Set(['failure', 'error']);
|
|
16
|
+
/** Maven Surefire's records of earlier attempts: flaky* when a later one passed, rerun* when not. */
|
|
17
|
+
const RETRY_FAILURE = new Set(['flakyFailure', 'flakyError', 'rerunFailure', 'rerunError']);
|
|
18
|
+
/** A failure's message and detail; runners disagree on which of the two holds what. */
|
|
19
|
+
function failure(rawMessage, rawDetail) {
|
|
20
|
+
// Terminal colour codes: XML cannot hold the escape character, so runners often write what is left
|
|
21
|
+
// of them ("[2mexpect([22m"); both forms are noise to the rules.
|
|
22
|
+
const plain = (s) => s.replace(/\u001b?\[(?:\d{1,3};)*\d{1,3}m/g, '');
|
|
23
|
+
const message = plain(rawMessage);
|
|
24
|
+
const detail = plain(rawDetail);
|
|
25
|
+
const head = message.trim() || detail.split('\n').find((l) => l.trim() !== '') || 'failed';
|
|
26
|
+
const stack = detail.trim();
|
|
27
|
+
// The full detail goes into the message too: outside Playwright the useful part (the assertion,
|
|
28
|
+
// the exception) is usually in the detail, and the rules read the message.
|
|
29
|
+
return { failed: true, message: stack && !stack.includes(head) ? `${head}\n${stack}` : stack || head, stack };
|
|
30
|
+
}
|
|
31
|
+
const fromElement = (e) => failure(e.attrs.message ?? e.attrs.type ?? '', e.children.find((c) => c.name === 'stackTrace')?.text ?? e.text);
|
|
32
|
+
/** The attempts one <testcase> records: earlier failed retries first, then the final outcome. */
|
|
33
|
+
function attemptsOf(testcase) {
|
|
34
|
+
const retries = testcase.children.filter((c) => RETRY_FAILURE.has(c.name)).map(fromElement);
|
|
35
|
+
const final = testcase.children.find((c) => FAILURE.has(c.name));
|
|
36
|
+
// No final failure means the last attempt passed: after flakyFailure retries, "failed, then passed".
|
|
37
|
+
return [...retries, final ? fromElement(final) : PASSED];
|
|
38
|
+
}
|
|
39
|
+
/** A file path relative to the checkout, so the same test has the same id on every machine. */
|
|
40
|
+
function relativeFile(file, workspace) {
|
|
41
|
+
const f = file.trim().replaceAll('\\', '/');
|
|
42
|
+
const ws = workspace?.replaceAll('\\', '/').replace(/\/?$/, '/');
|
|
43
|
+
if (ws && f.toLowerCase().startsWith(ws.toLowerCase()))
|
|
44
|
+
return f.slice(ws.length);
|
|
45
|
+
return f;
|
|
46
|
+
}
|
|
47
|
+
function junitCases(root, workspace) {
|
|
48
|
+
const out = [];
|
|
49
|
+
const walk = (e, suite) => {
|
|
50
|
+
for (const c of e.children) {
|
|
51
|
+
if (c.name === 'testcase')
|
|
52
|
+
add(c, suite);
|
|
53
|
+
else
|
|
54
|
+
walk(c, c.name === 'testsuite' ? c : suite);
|
|
55
|
+
}
|
|
56
|
+
};
|
|
57
|
+
const add = (testcase, suite) => {
|
|
58
|
+
if (testcase.children.some((c) => c.name === 'skipped'))
|
|
59
|
+
return;
|
|
60
|
+
const file = relativeFile(testcase.attrs.file ?? suite?.attrs.file ?? '', workspace);
|
|
61
|
+
const parts = [file, testcase.attrs.classname, testcase.attrs.name].map((p) => (p ?? '').trim()).filter(Boolean);
|
|
62
|
+
if (parts.length === 1 && suite?.attrs.name)
|
|
63
|
+
parts.unshift(suite.attrs.name.trim());
|
|
64
|
+
const testId = [...new Set(parts)].join(' > ');
|
|
65
|
+
const seconds = Number(testcase.attrs.time);
|
|
66
|
+
const attempts = attemptsOf(testcase);
|
|
67
|
+
const previous = out.at(-1);
|
|
68
|
+
if (previous?.testId === testId) {
|
|
69
|
+
// Rerun plugins (pytest-rerunfailures and others) write every attempt as its own entry and
|
|
70
|
+
// only re-run after a failure, so an earlier entry of the same test is a failed attempt even
|
|
71
|
+
// when the plugin left its error out. Only adjacent entries are merged.
|
|
72
|
+
const earlier = previous.attempts.at(-1);
|
|
73
|
+
if (!earlier.failed)
|
|
74
|
+
previous.attempts[previous.attempts.length - 1] = { failed: true, message: '', stack: '' };
|
|
75
|
+
previous.attempts.push(...attempts);
|
|
76
|
+
return;
|
|
77
|
+
}
|
|
78
|
+
out.push({ testId, file: file || (testcase.attrs.classname ?? '').trim(), duration: Number.isFinite(seconds) ? Math.round(seconds * 1000) : 0, attempts });
|
|
79
|
+
};
|
|
80
|
+
walk(root, root.name === 'testsuite' ? root : undefined);
|
|
81
|
+
// Attempts whose error the runner did not record borrow the final one's, when there is one.
|
|
82
|
+
for (const c of out) {
|
|
83
|
+
const known = [...c.attempts].reverse().find((a) => a.failed && a.message);
|
|
84
|
+
for (const [i, a] of c.attempts.entries()) {
|
|
85
|
+
if (a.failed && !a.message)
|
|
86
|
+
c.attempts[i] = known ?? failure('failed and was re-run; the runner did not record this attempt\'s error', '');
|
|
87
|
+
}
|
|
88
|
+
}
|
|
89
|
+
return out;
|
|
90
|
+
}
|
|
91
|
+
/** "00:00:01.2340000" -> 1234 ms. */
|
|
92
|
+
const trxDuration = (d) => {
|
|
93
|
+
const m = d?.match(/^(\d+):(\d+):(\d+(?:\.\d+)?)$/);
|
|
94
|
+
return m ? Math.round(((Number(m[1]) * 60 + Number(m[2])) * 60 + Number(m[3])) * 1000) : 0;
|
|
95
|
+
};
|
|
96
|
+
/** .NET's native TRX report (dotnet test --logger trx). */
|
|
97
|
+
function trxCases(root) {
|
|
98
|
+
const methods = new Map();
|
|
99
|
+
for (const unit of descendants(root, 'UnitTest')) {
|
|
100
|
+
const method = unit.children.find((c) => c.name === 'TestMethod');
|
|
101
|
+
if (unit.attrs.id && method)
|
|
102
|
+
methods.set(unit.attrs.id, { className: method.attrs.className ?? '', name: method.attrs.name ?? '' });
|
|
103
|
+
}
|
|
104
|
+
return descendants(root, 'UnitTestResult')
|
|
105
|
+
.filter((r) => r.attrs.outcome !== 'NotExecuted')
|
|
106
|
+
.map((r) => {
|
|
107
|
+
const method = methods.get(r.attrs.testId ?? '');
|
|
108
|
+
const testId = method?.className ? `${method.className} > ${method.name}` : (r.attrs.testName ?? '').trim();
|
|
109
|
+
const info = descendants(r, 'ErrorInfo')[0];
|
|
110
|
+
const text = (name) => info?.children.find((c) => c.name === name)?.text ?? '';
|
|
111
|
+
const failed = r.attrs.outcome === 'Failed' || r.attrs.outcome === 'Error' || r.attrs.outcome === 'Timeout';
|
|
112
|
+
return {
|
|
113
|
+
testId,
|
|
114
|
+
file: method?.className ?? testId,
|
|
115
|
+
duration: trxDuration(r.attrs.duration),
|
|
116
|
+
attempts: [failed ? failure(text('Message'), text('StackTrace')) : PASSED],
|
|
117
|
+
};
|
|
118
|
+
});
|
|
119
|
+
}
|
|
120
|
+
/**
|
|
121
|
+
* NUnit 3's own result file (`nunit3-console --result=x.xml`), whose root is <test-run>. NUnit does
|
|
122
|
+
* not write JUnit XML at all - its console offers only the nunit3 and nunit2 formats - so a .NET
|
|
123
|
+
* team on NUnit has nothing else to hand Tewip.
|
|
124
|
+
*/
|
|
125
|
+
function nunitCases(root) {
|
|
126
|
+
return descendants(root, 'test-case')
|
|
127
|
+
.filter((t) => t.attrs.result !== 'Skipped' && t.attrs.result !== 'Inconclusive')
|
|
128
|
+
.map((t) => {
|
|
129
|
+
const failureEl = t.children.find((c) => c.name === 'failure');
|
|
130
|
+
const child = (parent, name) => parent?.children.find((c) => c.name === name)?.text ?? '';
|
|
131
|
+
const testId = (t.attrs.fullname ?? '').trim() || [t.attrs.classname, t.attrs.methodname ?? t.attrs.name].filter(Boolean).join(' > ');
|
|
132
|
+
const seconds = Number(t.attrs.duration);
|
|
133
|
+
return {
|
|
134
|
+
testId,
|
|
135
|
+
file: (t.attrs.classname ?? '').trim(),
|
|
136
|
+
duration: Number.isFinite(seconds) ? Math.round(seconds * 1000) : 0,
|
|
137
|
+
attempts: [t.attrs.result === 'Failed' ? failure(child(failureEl, 'message'), child(failureEl, 'stack-trace')) : PASSED],
|
|
138
|
+
};
|
|
139
|
+
});
|
|
140
|
+
}
|
|
141
|
+
const REPORT_FILE = /\.(xml|trx)$/i;
|
|
142
|
+
/** Report files under `path` (a file, or a directory searched recursively). */
|
|
143
|
+
export function junitFiles(path) {
|
|
144
|
+
if (!statSync(path).isDirectory())
|
|
145
|
+
return [path];
|
|
146
|
+
return readdirSync(path, { recursive: true, withFileTypes: true })
|
|
147
|
+
.filter((d) => d.isFile() && REPORT_FILE.test(d.name))
|
|
148
|
+
.map((d) => join(d.parentPath, d.name))
|
|
149
|
+
.sort();
|
|
150
|
+
}
|
|
151
|
+
/**
|
|
152
|
+
* Imports one or more reports as a single run. Only failed attempts become records, as with
|
|
153
|
+
* Playwright; every test, passing or failing, is counted in the run summary so its history
|
|
154
|
+
* shows the passes too.
|
|
155
|
+
*/
|
|
156
|
+
export function importJunit(path, meta) {
|
|
157
|
+
const records = [];
|
|
158
|
+
const attempts = {};
|
|
159
|
+
const now = new Date().toISOString();
|
|
160
|
+
let startedAt = '';
|
|
161
|
+
let anyFailed = false;
|
|
162
|
+
const unread = [];
|
|
163
|
+
let parsedFiles = 0;
|
|
164
|
+
for (const file of junitFiles(path)) {
|
|
165
|
+
const root = parseXml(readFileSync(file, 'utf8'));
|
|
166
|
+
let cases;
|
|
167
|
+
if (root.name === 'TestRun') {
|
|
168
|
+
cases = trxCases(root);
|
|
169
|
+
const start = descendants(root, 'Times')[0]?.attrs.start;
|
|
170
|
+
if (start && (!startedAt || start < startedAt))
|
|
171
|
+
startedAt = start;
|
|
172
|
+
}
|
|
173
|
+
else if (root.name === 'testsuites' || root.name === 'testsuite') {
|
|
174
|
+
cases = junitCases(root, meta.workspace);
|
|
175
|
+
const stamp = root.attrs.timestamp ?? root.children.find((c) => c.name === 'testsuite')?.attrs.timestamp;
|
|
176
|
+
if (stamp && (!startedAt || stamp < startedAt))
|
|
177
|
+
startedAt = stamp;
|
|
178
|
+
}
|
|
179
|
+
else if (root.name === 'test-run') {
|
|
180
|
+
cases = nunitCases(root);
|
|
181
|
+
const start = root.attrs['start-time'];
|
|
182
|
+
if (start && (!startedAt || start < startedAt))
|
|
183
|
+
startedAt = start;
|
|
184
|
+
}
|
|
185
|
+
else {
|
|
186
|
+
// Not a report Tewip knows. Staying silent here would report a failing run as clean, so the
|
|
187
|
+
// file is remembered and, if nothing at all could be read, the caller is told.
|
|
188
|
+
unread.push(`${file} (root <${root.name}>)`);
|
|
189
|
+
continue;
|
|
190
|
+
}
|
|
191
|
+
parsedFiles++;
|
|
192
|
+
for (const c of cases) {
|
|
193
|
+
attempts[c.testId] = (attempts[c.testId] ?? 0) + c.attempts.length;
|
|
194
|
+
c.attempts.forEach((attempt, retry) => {
|
|
195
|
+
if (!attempt.failed)
|
|
196
|
+
return;
|
|
197
|
+
anyFailed = true;
|
|
198
|
+
records.push({
|
|
199
|
+
schemaVersion: SCHEMA_VERSION,
|
|
200
|
+
testId: c.testId,
|
|
201
|
+
file: c.file,
|
|
202
|
+
status: 'failed',
|
|
203
|
+
errorMessage: attempt.message,
|
|
204
|
+
errorStack: attempt.stack,
|
|
205
|
+
duration: c.duration,
|
|
206
|
+
retry,
|
|
207
|
+
attachments: [],
|
|
208
|
+
runId: meta.runId,
|
|
209
|
+
gitSha: meta.gitSha,
|
|
210
|
+
timestamp: now,
|
|
211
|
+
});
|
|
212
|
+
});
|
|
213
|
+
}
|
|
214
|
+
}
|
|
215
|
+
const summary = {
|
|
216
|
+
schemaVersion: SCHEMA_VERSION,
|
|
217
|
+
runId: meta.runId,
|
|
218
|
+
gitSha: meta.gitSha,
|
|
219
|
+
...(meta.branch ? { branch: meta.branch } : {}),
|
|
220
|
+
startedAt: startedAt ? new Date(startedAt).toISOString() : now,
|
|
221
|
+
finishedAt: now,
|
|
222
|
+
status: anyFailed ? 'failed' : 'passed',
|
|
223
|
+
attempts,
|
|
224
|
+
};
|
|
225
|
+
// Never let an unreadable report look like a clean run: a run that failed would pass the gate.
|
|
226
|
+
if (unread.length > 0) {
|
|
227
|
+
const list = unread.slice(0, 5).join(', ');
|
|
228
|
+
if (parsedFiles === 0) {
|
|
229
|
+
throw new Error(`no test report Tewip can read in ${path}: ${list}. It reads JUnit XML (<testsuites>/<testsuite>), NUnit 3 (<test-run>) and .NET TRX (<TestRun>). ` +
|
|
230
|
+
'Most runners can write JUnit XML; for TestNG use the Surefire report, and for NUnit use --result=x.xml.');
|
|
231
|
+
}
|
|
232
|
+
console.error(`[tewip] ignored ${unread.length} file(s) that are not a test report: ${list}`);
|
|
233
|
+
}
|
|
234
|
+
return { records, summary, skippedShards: [] };
|
|
235
|
+
}
|
|
@@ -0,0 +1,104 @@
|
|
|
1
|
+
// A small XML reader, enough for JUnit reports: elements, attributes, text, CDATA, comments and
|
|
2
|
+
// entities. Test reports are machine-written and well-formed, so this avoids a dependency; it is
|
|
3
|
+
// not a general-purpose parser (no DTDs, no namespaces beyond keeping prefixed names as written).
|
|
4
|
+
const ENTITIES = { amp: '&', lt: '<', gt: '>', quot: '"', apos: "'" };
|
|
5
|
+
export function decodeEntities(s) {
|
|
6
|
+
return s.replace(/&(#x[0-9a-fA-F]+|#\d+|[a-zA-Z]+);/g, (whole, body) => {
|
|
7
|
+
if (body.startsWith('#x'))
|
|
8
|
+
return String.fromCodePoint(Number.parseInt(body.slice(2), 16));
|
|
9
|
+
if (body.startsWith('#'))
|
|
10
|
+
return String.fromCodePoint(Number.parseInt(body.slice(1), 10));
|
|
11
|
+
return ENTITIES[body] ?? whole;
|
|
12
|
+
});
|
|
13
|
+
}
|
|
14
|
+
const ATTRIBUTE = /([^\s=/>]+)\s*=\s*(?:"([^"]*)"|'([^']*)')/g;
|
|
15
|
+
/** Parses a document and returns its root element; throws on input that is not XML. */
|
|
16
|
+
export function parseXml(source) {
|
|
17
|
+
const root = { name: '#document', attrs: {}, children: [], text: '' };
|
|
18
|
+
const stack = [root];
|
|
19
|
+
let i = 0;
|
|
20
|
+
while (i < source.length) {
|
|
21
|
+
const lt = source.indexOf('<', i);
|
|
22
|
+
const top = stack[stack.length - 1];
|
|
23
|
+
if (lt < 0) {
|
|
24
|
+
top.text += decodeEntities(source.slice(i));
|
|
25
|
+
break;
|
|
26
|
+
}
|
|
27
|
+
if (lt > i)
|
|
28
|
+
top.text += decodeEntities(source.slice(i, lt));
|
|
29
|
+
if (source.startsWith('<!--', lt)) {
|
|
30
|
+
i = end(source, '-->', lt) + 3;
|
|
31
|
+
}
|
|
32
|
+
else if (source.startsWith('<![CDATA[', lt)) {
|
|
33
|
+
const close = end(source, ']]>', lt);
|
|
34
|
+
top.text += source.slice(lt + 9, close);
|
|
35
|
+
i = close + 3;
|
|
36
|
+
}
|
|
37
|
+
else if (source.startsWith('<?', lt) || source.startsWith('<!', lt)) {
|
|
38
|
+
i = end(source, '>', lt) + 1;
|
|
39
|
+
}
|
|
40
|
+
else if (source.startsWith('</', lt)) {
|
|
41
|
+
const close = end(source, '>', lt);
|
|
42
|
+
const name = source.slice(lt + 2, close).trim();
|
|
43
|
+
// Tolerate a stray closing tag rather than corrupting the tree.
|
|
44
|
+
const at = stack.map((e) => e.name).lastIndexOf(name);
|
|
45
|
+
if (at > 0)
|
|
46
|
+
stack.length = at;
|
|
47
|
+
i = close + 1;
|
|
48
|
+
}
|
|
49
|
+
else {
|
|
50
|
+
const close = tagEnd(source, lt);
|
|
51
|
+
const raw = source.slice(lt + 1, close);
|
|
52
|
+
const selfClosing = raw.endsWith('/');
|
|
53
|
+
const body = selfClosing ? raw.slice(0, -1) : raw;
|
|
54
|
+
const name = body.match(/^[^\s/>]+/)?.[0] ?? '';
|
|
55
|
+
const attrs = {};
|
|
56
|
+
for (const [, key, dq, sq] of body.slice(name.length).matchAll(ATTRIBUTE))
|
|
57
|
+
attrs[key] = decodeEntities(dq ?? sq ?? '');
|
|
58
|
+
const element = { name, attrs, children: [], text: '' };
|
|
59
|
+
top.children.push(element);
|
|
60
|
+
if (!selfClosing)
|
|
61
|
+
stack.push(element);
|
|
62
|
+
i = close + 1;
|
|
63
|
+
}
|
|
64
|
+
}
|
|
65
|
+
const first = root.children[0];
|
|
66
|
+
if (!first)
|
|
67
|
+
throw new Error('not an XML document: no root element');
|
|
68
|
+
return first;
|
|
69
|
+
}
|
|
70
|
+
function end(source, token, from) {
|
|
71
|
+
const at = source.indexOf(token, from);
|
|
72
|
+
if (at < 0)
|
|
73
|
+
throw new Error(`unterminated XML near offset ${from}`);
|
|
74
|
+
return at;
|
|
75
|
+
}
|
|
76
|
+
/** The end of a start tag, skipping '>' characters inside quoted attribute values. */
|
|
77
|
+
function tagEnd(source, from) {
|
|
78
|
+
let quote = '';
|
|
79
|
+
for (let i = from + 1; i < source.length; i++) {
|
|
80
|
+
const c = source[i];
|
|
81
|
+
if (quote) {
|
|
82
|
+
if (c === quote)
|
|
83
|
+
quote = '';
|
|
84
|
+
}
|
|
85
|
+
else if (c === '"' || c === "'")
|
|
86
|
+
quote = c;
|
|
87
|
+
else if (c === '>')
|
|
88
|
+
return i;
|
|
89
|
+
}
|
|
90
|
+
throw new Error(`unterminated tag near offset ${from}`);
|
|
91
|
+
}
|
|
92
|
+
/** Every descendant element with this name, depth first. */
|
|
93
|
+
export function descendants(element, name) {
|
|
94
|
+
const out = [];
|
|
95
|
+
const walk = (e) => {
|
|
96
|
+
for (const c of e.children) {
|
|
97
|
+
if (c.name === name)
|
|
98
|
+
out.push(c);
|
|
99
|
+
walk(c);
|
|
100
|
+
}
|
|
101
|
+
};
|
|
102
|
+
walk(element);
|
|
103
|
+
return out;
|
|
104
|
+
}
|
|
@@ -0,0 +1,59 @@
|
|
|
1
|
+
// Anthropic's own API, for teams who have Claude rather than OpenAI. It has no "JSON schema
|
|
2
|
+
// response format", so the schema is given as a single tool the model must call: the arguments it
|
|
3
|
+
// passes back are the JSON Tewip asked for. Same LlmClient shape as every other provider, so the
|
|
4
|
+
// engine cannot tell the difference.
|
|
5
|
+
import { truncate } from '../signature.js';
|
|
6
|
+
import { recordCall } from './usage.js';
|
|
7
|
+
const VERSION = '2023-06-01';
|
|
8
|
+
const REQUEST_TIMEOUT_MS = 90_000;
|
|
9
|
+
const MAX_ATTEMPTS = 3;
|
|
10
|
+
const MAX_TOKENS = 4096;
|
|
11
|
+
const sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
|
|
12
|
+
export function anthropicClient({ apiKey, model, baseUrl = 'https://api.anthropic.com/v1' }) {
|
|
13
|
+
return {
|
|
14
|
+
model,
|
|
15
|
+
async complete({ system, user, schemaName, schema }) {
|
|
16
|
+
const body = JSON.stringify({
|
|
17
|
+
model,
|
|
18
|
+
max_tokens: MAX_TOKENS,
|
|
19
|
+
system,
|
|
20
|
+
messages: [{ role: 'user', content: user }],
|
|
21
|
+
// One tool, and the model must use it: its arguments are the answer in the shape asked for.
|
|
22
|
+
tools: [{ name: schemaName, description: 'Return the answer in this shape.', input_schema: schema }],
|
|
23
|
+
tool_choice: { type: 'tool', name: schemaName },
|
|
24
|
+
});
|
|
25
|
+
const startedAt = Date.now();
|
|
26
|
+
for (let attempt = 1;; attempt++) {
|
|
27
|
+
let res;
|
|
28
|
+
try {
|
|
29
|
+
res = await fetch(`${baseUrl}/messages`, {
|
|
30
|
+
method: 'POST',
|
|
31
|
+
headers: { 'x-api-key': apiKey, 'anthropic-version': VERSION, 'Content-Type': 'application/json' },
|
|
32
|
+
body,
|
|
33
|
+
signal: AbortSignal.timeout(REQUEST_TIMEOUT_MS),
|
|
34
|
+
});
|
|
35
|
+
}
|
|
36
|
+
catch (error) {
|
|
37
|
+
if (attempt >= MAX_ATTEMPTS)
|
|
38
|
+
throw new Error(`Anthropic request failed: ${String(error)}`);
|
|
39
|
+
await sleep(2000 * attempt);
|
|
40
|
+
continue;
|
|
41
|
+
}
|
|
42
|
+
if (res.ok) {
|
|
43
|
+
const json = (await res.json());
|
|
44
|
+
recordCall(Date.now() - startedAt, { prompt: json.usage?.input_tokens, completion: json.usage?.output_tokens, cached: json.usage?.cache_read_input_tokens }, model);
|
|
45
|
+
const call = json.content?.find((c) => c.type === 'tool_use' && c.input !== undefined);
|
|
46
|
+
if (!call) {
|
|
47
|
+
const refusal = json.content?.find((c) => c.type === 'text')?.text;
|
|
48
|
+
throw new Error(refusal ? `model refused: ${truncate(refusal, 200)}` : 'model returned no answer in the shape asked for');
|
|
49
|
+
}
|
|
50
|
+
return JSON.stringify(call.input);
|
|
51
|
+
}
|
|
52
|
+
const detail = truncate(await res.text(), 300);
|
|
53
|
+
if ((res.status !== 429 && res.status < 500) || attempt >= MAX_ATTEMPTS)
|
|
54
|
+
throw new Error(`Anthropic API ${res.status}: ${detail}`);
|
|
55
|
+
await sleep(2000 * attempt);
|
|
56
|
+
}
|
|
57
|
+
},
|
|
58
|
+
};
|
|
59
|
+
}
|