taphound 0.2.0-dev.11 → 0.2.0-dev.13
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/README.zh-CN.md +1 -1
- package/assets/skills/taphound-case-suite/SKILL.md +8 -6
- package/assets/skills/taphound-case-suite/scripts/ledger.mjs +84 -19
- package/assets/skills/taphound-journey-brief-author/SKILL.md +8 -3
- package/assets/skills/taphound-journey-brief-author/prompts/brief-author-role.md +3 -1
- package/assets/skills/taphound-journey-brief-author/prompts/brief-author-role.zh-CN.md +1 -1
- package/assets/skills/taphound-journey-generator/SKILL.md +21 -5
- package/assets/skills/taphound-journey-generator/prompts/consume-journey-brief.md +3 -1
- package/assets/skills/taphound-journey-generator/prompts/generate-step.md +9 -0
- package/assets/skills/taphound-journey-generator/scripts/envelope.mjs +80 -10
- package/assets/skills/taphound-verify-change/references/preserve.md +15 -7
- package/assets/skills/taphound-verify-change/scripts/ui-refactor.mjs +1 -1
- package/dist/adapters/appium/appium-ui-snapshot-provider.d.ts +8 -0
- package/dist/adapters/appium/appium-ui-snapshot-provider.js +123 -54
- package/dist/adapters/filesystem/diagnostics-journal.d.ts +17 -0
- package/dist/adapters/filesystem/diagnostics-journal.js +130 -0
- package/dist/application/diagnostics/diagnostics-exporter.d.ts +29 -0
- package/dist/application/diagnostics/diagnostics-exporter.js +283 -0
- package/dist/application/diagnostics/ui-capture-telemetry.d.ts +16 -0
- package/dist/application/diagnostics/ui-capture-telemetry.js +81 -0
- package/dist/application/generation/proposed-step-validator.js +5 -1
- package/dist/application/journey/journey-check-service.d.ts +7 -1
- package/dist/application/journey/journey-check-service.js +16 -1
- package/dist/application/report/report-writer.d.ts +5 -0
- package/dist/application/report/report-writer.js +20 -0
- package/dist/application/runtime/verify-runtime.js +32 -2
- package/dist/application/ui/observed-ui-snapshot-provider.d.ts +15 -0
- package/dist/application/ui/observed-ui-snapshot-provider.js +66 -0
- package/dist/application/ui/ui-stability-probe.d.ts +6 -0
- package/dist/application/ui/ui-stability-probe.js +21 -5
- package/dist/application/wait/idle-waiter.js +29 -6
- package/dist/cli/commands/diagnose.d.ts +3 -0
- package/dist/cli/commands/diagnose.js +84 -0
- package/dist/cli/commands/generation/session-commands.js +4 -1
- package/dist/cli/commands/journey.js +3 -1
- package/dist/cli/commands/verify.js +55 -3
- package/dist/cli/dependencies.d.ts +11 -0
- package/dist/cli/dependencies.js +58 -5
- package/dist/cli/diagnostics-recorder.d.ts +31 -0
- package/dist/cli/diagnostics-recorder.js +82 -0
- package/dist/cli/main.d.ts +1 -1
- package/dist/cli/main.js +58 -1
- package/dist/cli/program.js +3 -1
- package/dist/domain/contract.d.ts +6 -6
- package/dist/domain/diagnostics.d.ts +1396 -0
- package/dist/domain/diagnostics.js +193 -0
- package/dist/domain/failure.js +1 -1
- package/dist/domain/report.d.ts +11 -3
- package/dist/domain/report.js +1 -0
- package/dist/domain/verify-receipt.d.ts +17 -0
- package/dist/domain/verify-receipt.js +17 -0
- package/dist/domain/workspace.d.ts +11 -0
- package/dist/domain/workspace.js +14 -0
- package/dist/ports/diagnostics.d.ts +15 -0
- package/dist/ports/diagnostics.js +1 -0
- package/package.json +4 -3
- package/scripts/feedback-pack.mjs +399 -0
|
@@ -0,0 +1,283 @@
|
|
|
1
|
+
import { createHmac } from "node:crypto";
|
|
2
|
+
import { join, resolve } from "node:path";
|
|
3
|
+
import { TapHoundConfigSchema } from "../../domain/config.js";
|
|
4
|
+
import { CommandEventSchema, DiagnosticsBundleSchema } from "../../domain/diagnostics.js";
|
|
5
|
+
import { TapHoundReportSchema } from "../../domain/report.js";
|
|
6
|
+
import { CONFIG_PATH, DEFAULT_ARTIFACTS_DIR } from "../../domain/workspace.js";
|
|
7
|
+
const TOKEN = /^[a-zA-Z][a-zA-Z\d]{0,31}$/;
|
|
8
|
+
const VERSION = /^[\w.+-]{1,64}$/;
|
|
9
|
+
const TOOL_NAMES = ["node", "adb", "android"];
|
|
10
|
+
const LOCATOR_FIELDS = ["resourceId", "text", "contentDescription", "index", "within"];
|
|
11
|
+
/** Stable ordinal names for identifying strings, shared across one bundle. */
|
|
12
|
+
class Aliases {
|
|
13
|
+
prefix;
|
|
14
|
+
values = new Map();
|
|
15
|
+
constructor(prefix) {
|
|
16
|
+
this.prefix = prefix;
|
|
17
|
+
}
|
|
18
|
+
of(value) {
|
|
19
|
+
let alias = this.values.get(value);
|
|
20
|
+
if (alias === undefined) {
|
|
21
|
+
alias = `${this.prefix}#${String(this.values.size + 1)}`;
|
|
22
|
+
this.values.set(value, alias);
|
|
23
|
+
}
|
|
24
|
+
return alias;
|
|
25
|
+
}
|
|
26
|
+
}
|
|
27
|
+
function canonical(value) {
|
|
28
|
+
if (Array.isArray(value))
|
|
29
|
+
return value.map(canonical);
|
|
30
|
+
if (value !== null && typeof value === "object") {
|
|
31
|
+
return Object.fromEntries(Object.entries(value)
|
|
32
|
+
.sort(([left], [right]) => left.localeCompare(right))
|
|
33
|
+
.map(([key, entry]) => [key, canonical(entry)]));
|
|
34
|
+
}
|
|
35
|
+
return value;
|
|
36
|
+
}
|
|
37
|
+
function parseEvents(lines) {
|
|
38
|
+
const events = [];
|
|
39
|
+
let skipped = 0;
|
|
40
|
+
for (const line of lines) {
|
|
41
|
+
try {
|
|
42
|
+
const parsed = CommandEventSchema.safeParse(JSON.parse(line));
|
|
43
|
+
if (parsed.success) {
|
|
44
|
+
events.push(parsed.data);
|
|
45
|
+
continue;
|
|
46
|
+
}
|
|
47
|
+
}
|
|
48
|
+
catch {
|
|
49
|
+
// A torn or foreign line is counted, never exported.
|
|
50
|
+
}
|
|
51
|
+
skipped += 1;
|
|
52
|
+
}
|
|
53
|
+
return { events, skipped };
|
|
54
|
+
}
|
|
55
|
+
function configSummary(config) {
|
|
56
|
+
return {
|
|
57
|
+
idle: {
|
|
58
|
+
strategy: config.idle.strategy,
|
|
59
|
+
pollIntervalMs: config.idle.pollIntervalMs,
|
|
60
|
+
stablePolls: config.idle.stablePolls,
|
|
61
|
+
timeoutMs: config.idle.timeoutMs,
|
|
62
|
+
...(config.idle.ignoreCursorBlink === undefined
|
|
63
|
+
? {}
|
|
64
|
+
: { ignoreCursorBlink: config.idle.ignoreCursorBlink }),
|
|
65
|
+
...(config.idle.ignoreLayoutDrift === undefined
|
|
66
|
+
? {}
|
|
67
|
+
: { ignoreLayoutDrift: config.idle.ignoreLayoutDrift }),
|
|
68
|
+
deviceProfiles: config.idle.deviceProfiles?.length ?? 0
|
|
69
|
+
},
|
|
70
|
+
...(config.ui === undefined
|
|
71
|
+
? {}
|
|
72
|
+
: {
|
|
73
|
+
ui: {
|
|
74
|
+
backend: config.ui.backend,
|
|
75
|
+
...(config.ui.snapshotTimeoutMs === undefined
|
|
76
|
+
? {}
|
|
77
|
+
: { snapshotTimeoutMs: config.ui.snapshotTimeoutMs }),
|
|
78
|
+
...(config.ui.cacheEnabled === undefined
|
|
79
|
+
? {}
|
|
80
|
+
: { cacheEnabled: config.ui.cacheEnabled })
|
|
81
|
+
}
|
|
82
|
+
}),
|
|
83
|
+
...(config.runtime === undefined ? {} : { runtime: { backend: config.runtime.backend } })
|
|
84
|
+
};
|
|
85
|
+
}
|
|
86
|
+
/**
|
|
87
|
+
* Builds a feedback bundle from the local journal and the reports it
|
|
88
|
+
* references. Only allowlisted structural facts survive: identifying strings
|
|
89
|
+
* become ordinal aliases or salted digests, and everything else is dropped.
|
|
90
|
+
* The result is validated against the strict bundle schema before return.
|
|
91
|
+
*/
|
|
92
|
+
export class DiagnosticsExporter {
|
|
93
|
+
dependencies;
|
|
94
|
+
constructor(dependencies) {
|
|
95
|
+
this.dependencies = dependencies;
|
|
96
|
+
}
|
|
97
|
+
async export(input) {
|
|
98
|
+
const { events: allEvents, skipped } = parseEvents(await this.dependencies.journal.readLines(input.projectRoot));
|
|
99
|
+
const events = allEvents.slice(-Math.max(0, input.eventLimit));
|
|
100
|
+
const config = await this.readConfig(input.projectRoot);
|
|
101
|
+
const salt = await this.dependencies.journal.salt(input.projectRoot);
|
|
102
|
+
const runs = new Aliases("run");
|
|
103
|
+
const activities = new Aliases("activity");
|
|
104
|
+
const journeys = new Aliases("journey");
|
|
105
|
+
const devices = new Aliases("device");
|
|
106
|
+
const locatorId = (locator) => createHmac("sha256", salt)
|
|
107
|
+
.update(JSON.stringify(canonical(locator)))
|
|
108
|
+
.digest("hex")
|
|
109
|
+
.slice(0, 16);
|
|
110
|
+
const runIds = [...new Set(events.flatMap((event) => (event.runId === undefined ? [] : [event.runId])))];
|
|
111
|
+
// Aliases follow journal order, whichever runs end up summarized.
|
|
112
|
+
for (const runId of runIds)
|
|
113
|
+
runs.of(runId);
|
|
114
|
+
const artifactsDir = resolve(input.projectRoot, config?.artifactsDir ?? DEFAULT_ARTIFACTS_DIR);
|
|
115
|
+
const summaries = [];
|
|
116
|
+
for (const runId of runIds.slice(-Math.max(0, input.runLimit))) {
|
|
117
|
+
const report = await this.readReport(join(artifactsDir, runId, "report.json"));
|
|
118
|
+
if (report === undefined)
|
|
119
|
+
continue;
|
|
120
|
+
summaries.push(summarizeRun(report, runs.of(runId), {
|
|
121
|
+
activity: (value) => activities.of(value),
|
|
122
|
+
journey: (value) => journeys.of(value),
|
|
123
|
+
device: (value) => devices.of(value),
|
|
124
|
+
locatorId
|
|
125
|
+
}));
|
|
126
|
+
}
|
|
127
|
+
return DiagnosticsBundleSchema.parse({
|
|
128
|
+
version: 1,
|
|
129
|
+
generatedAt: this.dependencies.now().toISOString(),
|
|
130
|
+
taphoundVersion: this.dependencies.taphoundVersion,
|
|
131
|
+
host: this.dependencies.host,
|
|
132
|
+
...(config === undefined ? {} : { config: configSummary(config) }),
|
|
133
|
+
journal: {
|
|
134
|
+
events: events.map((event) => ({
|
|
135
|
+
...event,
|
|
136
|
+
...(event.runId === undefined ? {} : { runId: runs.of(event.runId) })
|
|
137
|
+
})),
|
|
138
|
+
skippedLines: skipped
|
|
139
|
+
},
|
|
140
|
+
runs: summaries
|
|
141
|
+
});
|
|
142
|
+
}
|
|
143
|
+
async readConfig(projectRoot) {
|
|
144
|
+
try {
|
|
145
|
+
const parsed = TapHoundConfigSchema.safeParse(await this.dependencies.readJson(join(projectRoot, CONFIG_PATH)));
|
|
146
|
+
return parsed.success ? parsed.data : undefined;
|
|
147
|
+
}
|
|
148
|
+
catch {
|
|
149
|
+
return undefined;
|
|
150
|
+
}
|
|
151
|
+
}
|
|
152
|
+
async readReport(path) {
|
|
153
|
+
try {
|
|
154
|
+
const parsed = TapHoundReportSchema.safeParse(await this.dependencies.readJson(path));
|
|
155
|
+
return parsed.success ? parsed.data : undefined;
|
|
156
|
+
}
|
|
157
|
+
catch {
|
|
158
|
+
return undefined;
|
|
159
|
+
}
|
|
160
|
+
}
|
|
161
|
+
}
|
|
162
|
+
function summarizeStep(step, redact) {
|
|
163
|
+
const locator = step.locator;
|
|
164
|
+
const requested = locator?.requested;
|
|
165
|
+
return {
|
|
166
|
+
index: step.index,
|
|
167
|
+
action: step.action,
|
|
168
|
+
status: step.status,
|
|
169
|
+
...(step.device === undefined ? {} : { device: redact.device(step.device) }),
|
|
170
|
+
durationMs: step.durationMs,
|
|
171
|
+
...(locator === undefined
|
|
172
|
+
? {}
|
|
173
|
+
: {
|
|
174
|
+
locator: {
|
|
175
|
+
status: locator.status,
|
|
176
|
+
...(locator.matchedBy === undefined ? {} : { matchedBy: locator.matchedBy }),
|
|
177
|
+
requestedFields: requested === undefined
|
|
178
|
+
? []
|
|
179
|
+
: LOCATOR_FIELDS.filter((field) => requested[field] !== undefined),
|
|
180
|
+
...(requested === undefined ? {} : { locatorId: redact.locatorId(requested) }),
|
|
181
|
+
...(locator.anchor === undefined ? {} : { anchorStatus: locator.anchor.status }),
|
|
182
|
+
fallbackUsed: locator.fallbackUsed
|
|
183
|
+
}
|
|
184
|
+
}),
|
|
185
|
+
...(step.idle === undefined
|
|
186
|
+
? {}
|
|
187
|
+
: {
|
|
188
|
+
idle: {
|
|
189
|
+
status: step.idle.status,
|
|
190
|
+
polls: step.idle.polls,
|
|
191
|
+
...(step.idle.durationMs === undefined ? {} : { durationMs: step.idle.durationMs }),
|
|
192
|
+
...(step.idle.samplingDurationMs === undefined
|
|
193
|
+
? {}
|
|
194
|
+
: { samplingDurationMs: step.idle.samplingDurationMs }),
|
|
195
|
+
...(step.idle.strategy === undefined ? {} : { strategy: step.idle.strategy }),
|
|
196
|
+
...(step.idle.backend === undefined ? {} : { backend: step.idle.backend }),
|
|
197
|
+
...(step.idle.fallbackUsed === undefined
|
|
198
|
+
? {}
|
|
199
|
+
: { fallbackUsed: step.idle.fallbackUsed }),
|
|
200
|
+
...(step.idle.frameActivityDetected === undefined
|
|
201
|
+
? {}
|
|
202
|
+
: { frameActivityDetected: step.idle.frameActivityDetected }),
|
|
203
|
+
...(step.idle.lastDiff === undefined
|
|
204
|
+
? {}
|
|
205
|
+
: { lastDiffCount: step.idle.lastDiff.length })
|
|
206
|
+
}
|
|
207
|
+
}),
|
|
208
|
+
...(step.activity === undefined
|
|
209
|
+
? {}
|
|
210
|
+
: {
|
|
211
|
+
activity: {
|
|
212
|
+
before: activityCheck(step.activity.before, redact),
|
|
213
|
+
after: activityCheck(step.activity.after, redact)
|
|
214
|
+
}
|
|
215
|
+
}),
|
|
216
|
+
...(step.expectation === undefined
|
|
217
|
+
? {}
|
|
218
|
+
: {
|
|
219
|
+
expectation: {
|
|
220
|
+
type: step.expectation.type,
|
|
221
|
+
status: step.expectation.status,
|
|
222
|
+
...(step.expectation.code === undefined ? {} : { code: step.expectation.code })
|
|
223
|
+
}
|
|
224
|
+
}),
|
|
225
|
+
...(step.scroll === undefined ? {} : { scroll: { ...step.scroll } })
|
|
226
|
+
};
|
|
227
|
+
}
|
|
228
|
+
function activityCheck(check, redact) {
|
|
229
|
+
return {
|
|
230
|
+
status: check.status,
|
|
231
|
+
expected: redact.activity(check.expected),
|
|
232
|
+
...(check.actual === undefined ? {} : { actual: redact.activity(check.actual) })
|
|
233
|
+
};
|
|
234
|
+
}
|
|
235
|
+
function summarizeRun(report, run, redact) {
|
|
236
|
+
const failure = report.primaryFailure;
|
|
237
|
+
return {
|
|
238
|
+
run,
|
|
239
|
+
status: report.status,
|
|
240
|
+
durationMs: report.durationMs,
|
|
241
|
+
journey: redact.journey(report.journey.name),
|
|
242
|
+
devices: report.environment.devices.map((device) => ({
|
|
243
|
+
device: redact.device(device.role),
|
|
244
|
+
...(device.uiBackend === undefined
|
|
245
|
+
? {}
|
|
246
|
+
: {
|
|
247
|
+
uiBackend: {
|
|
248
|
+
id: device.uiBackend.id,
|
|
249
|
+
adapterVersion: VERSION.test(device.uiBackend.adapterVersion)
|
|
250
|
+
? device.uiBackend.adapterVersion
|
|
251
|
+
: "unknown",
|
|
252
|
+
...(device.uiBackend.engineVersion !== undefined
|
|
253
|
+
&& VERSION.test(device.uiBackend.engineVersion)
|
|
254
|
+
? { engineVersion: device.uiBackend.engineVersion }
|
|
255
|
+
: {})
|
|
256
|
+
}
|
|
257
|
+
}),
|
|
258
|
+
...(device.uiCache === undefined ? {} : { uiCache: { ...device.uiCache } })
|
|
259
|
+
})),
|
|
260
|
+
tools: Object.fromEntries(TOOL_NAMES.flatMap((name) => {
|
|
261
|
+
const version = report.environment.tools[name];
|
|
262
|
+
return version !== undefined && VERSION.test(version) ? [[name, version]] : [];
|
|
263
|
+
})),
|
|
264
|
+
layers: { ...report.layers },
|
|
265
|
+
...(failure === undefined
|
|
266
|
+
? {}
|
|
267
|
+
: {
|
|
268
|
+
primaryFailure: {
|
|
269
|
+
code: failure.code,
|
|
270
|
+
phase: TOKEN.test(failure.phase) ? failure.phase : "other",
|
|
271
|
+
...(failure.stepIndex === undefined ? {} : { stepIndex: failure.stepIndex })
|
|
272
|
+
}
|
|
273
|
+
}),
|
|
274
|
+
secondaryErrorCodes: report.secondaryErrors.map((error) => error.code),
|
|
275
|
+
fallbackUsed: report.fallbackUsed,
|
|
276
|
+
logcatEvidence: (report.logcatEvidence ?? []).map((entry) => ({
|
|
277
|
+
device: redact.device(entry.role),
|
|
278
|
+
droppedLines: entry.droppedLines,
|
|
279
|
+
droppedBytes: entry.droppedBytes
|
|
280
|
+
})),
|
|
281
|
+
steps: report.steps.map((step) => summarizeStep(step, redact))
|
|
282
|
+
};
|
|
283
|
+
}
|
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
import { type UiBackendTelemetry, type UiCaptureFailureKind } from "../../domain/diagnostics.js";
|
|
2
|
+
import type { UiBackendId } from "../../domain/ui-backend.js";
|
|
3
|
+
import type { UiCaptureObserver } from "../../ports/diagnostics.js";
|
|
4
|
+
/**
|
|
5
|
+
* Classifies a capture failure from its type and status only, walking the
|
|
6
|
+
* cause chain. The message is read solely to spot a timeout and is never kept.
|
|
7
|
+
*/
|
|
8
|
+
export declare function classifyCaptureFailure(error: unknown): UiCaptureFailureKind;
|
|
9
|
+
/** Aggregates UI backend captures for one CLI invocation. */
|
|
10
|
+
export declare class UiCaptureTelemetry implements UiCaptureObserver {
|
|
11
|
+
private readonly backends;
|
|
12
|
+
captured(backend: UiBackendId, durationMs: number, error?: unknown): void;
|
|
13
|
+
sessionRecovered(backend: UiBackendId, succeeded: boolean): void;
|
|
14
|
+
summary(): UiBackendTelemetry[];
|
|
15
|
+
private entry;
|
|
16
|
+
}
|
|
@@ -0,0 +1,81 @@
|
|
|
1
|
+
import { UI_CAPTURE_LATENCY_BUCKETS_MS } from "../../domain/diagnostics.js";
|
|
2
|
+
/**
|
|
3
|
+
* Classifies a capture failure from its type and status only, walking the
|
|
4
|
+
* cause chain. The message is read solely to spot a timeout and is never kept.
|
|
5
|
+
*/
|
|
6
|
+
export function classifyCaptureFailure(error) {
|
|
7
|
+
let current = error;
|
|
8
|
+
for (let depth = 0; depth < 8 && current !== null && typeof current === "object"; depth += 1) {
|
|
9
|
+
const record = current;
|
|
10
|
+
if (record.name === "TimeoutError")
|
|
11
|
+
return "timeout";
|
|
12
|
+
if (record.name === "AbortError")
|
|
13
|
+
return "cancelled";
|
|
14
|
+
if (typeof record.status === "number") {
|
|
15
|
+
if (record.status >= 500)
|
|
16
|
+
return "http5xx";
|
|
17
|
+
if (record.status >= 400)
|
|
18
|
+
return "http4xx";
|
|
19
|
+
}
|
|
20
|
+
if (typeof record.message === "string" && /timed? ?out/i.test(record.message)) {
|
|
21
|
+
return "timeout";
|
|
22
|
+
}
|
|
23
|
+
current = record.cause;
|
|
24
|
+
}
|
|
25
|
+
return "error";
|
|
26
|
+
}
|
|
27
|
+
function emptyTelemetry(backend) {
|
|
28
|
+
return {
|
|
29
|
+
backend,
|
|
30
|
+
captures: 0,
|
|
31
|
+
failures: { timeout: 0, cancelled: 0, http4xx: 0, http5xx: 0, error: 0 },
|
|
32
|
+
totalMs: 0,
|
|
33
|
+
maxMs: 0,
|
|
34
|
+
latencyBuckets: new Array(UI_CAPTURE_LATENCY_BUCKETS_MS.length + 1).fill(0),
|
|
35
|
+
sessionRecoveries: 0,
|
|
36
|
+
sessionRecoveryFailures: 0
|
|
37
|
+
};
|
|
38
|
+
}
|
|
39
|
+
/** Aggregates UI backend captures for one CLI invocation. */
|
|
40
|
+
export class UiCaptureTelemetry {
|
|
41
|
+
backends = new Map();
|
|
42
|
+
captured(backend, durationMs, error) {
|
|
43
|
+
const entry = this.entry(backend);
|
|
44
|
+
const duration = Math.max(0, durationMs);
|
|
45
|
+
entry.captures += 1;
|
|
46
|
+
entry.totalMs += duration;
|
|
47
|
+
entry.maxMs = Math.max(entry.maxMs, duration);
|
|
48
|
+
const bucket = UI_CAPTURE_LATENCY_BUCKETS_MS.findIndex((limit) => duration < limit);
|
|
49
|
+
const index = bucket === -1 ? UI_CAPTURE_LATENCY_BUCKETS_MS.length : bucket;
|
|
50
|
+
entry.latencyBuckets[index] = (entry.latencyBuckets[index] ?? 0) + 1;
|
|
51
|
+
if (error !== undefined) {
|
|
52
|
+
entry.failures[classifyCaptureFailure(error)] += 1;
|
|
53
|
+
}
|
|
54
|
+
}
|
|
55
|
+
sessionRecovered(backend, succeeded) {
|
|
56
|
+
const entry = this.entry(backend);
|
|
57
|
+
if (succeeded) {
|
|
58
|
+
entry.sessionRecoveries += 1;
|
|
59
|
+
}
|
|
60
|
+
else {
|
|
61
|
+
entry.sessionRecoveryFailures += 1;
|
|
62
|
+
}
|
|
63
|
+
}
|
|
64
|
+
summary() {
|
|
65
|
+
return [...this.backends.values()].map((entry) => ({
|
|
66
|
+
...entry,
|
|
67
|
+
totalMs: Math.round(entry.totalMs),
|
|
68
|
+
maxMs: Math.round(entry.maxMs),
|
|
69
|
+
failures: { ...entry.failures },
|
|
70
|
+
latencyBuckets: [...entry.latencyBuckets]
|
|
71
|
+
}));
|
|
72
|
+
}
|
|
73
|
+
entry(backend) {
|
|
74
|
+
let entry = this.backends.get(backend);
|
|
75
|
+
if (entry === undefined) {
|
|
76
|
+
entry = emptyTelemetry(backend);
|
|
77
|
+
this.backends.set(backend, entry);
|
|
78
|
+
}
|
|
79
|
+
return entry;
|
|
80
|
+
}
|
|
81
|
+
}
|
|
@@ -51,7 +51,11 @@ function validateBinding(session, snapshot, proposal) {
|
|
|
51
51
|
throw new GenerationOperationError("PACKAGE_ESCAPE", "Foreground package escaped the generation target");
|
|
52
52
|
}
|
|
53
53
|
if (snapshot.activity !== proposal.activity.before) {
|
|
54
|
-
throw new GenerationOperationError("SNAPSHOT_STALE",
|
|
54
|
+
throw new GenerationOperationError("SNAPSHOT_STALE", `Proposal before Activity ${proposal.activity.before} does not match the current snapshot Activity ${snapshot.activity}`, {
|
|
55
|
+
field: "activity.before",
|
|
56
|
+
expected: snapshot.activity,
|
|
57
|
+
actual: proposal.activity.before
|
|
58
|
+
});
|
|
55
59
|
}
|
|
56
60
|
}
|
|
57
61
|
function validateAction(snapshot, proposal) {
|
|
@@ -30,7 +30,7 @@ export interface JourneyCheckResult {
|
|
|
30
30
|
entries: readonly JourneyCheckEntry[];
|
|
31
31
|
summary: JourneyCheckSummary;
|
|
32
32
|
}
|
|
33
|
-
export type JourneyCheckErrorCode = "CONFIG_INVALID";
|
|
33
|
+
export type JourneyCheckErrorCode = "CONFIG_INVALID" | "JOURNEY_NOT_FOUND";
|
|
34
34
|
export declare class JourneyCheckError extends Error {
|
|
35
35
|
readonly code: JourneyCheckErrorCode;
|
|
36
36
|
readonly name = "JourneyCheckError";
|
|
@@ -41,6 +41,12 @@ export interface JourneyCheckInput {
|
|
|
41
41
|
config: TapHoundConfig;
|
|
42
42
|
project: ProjectDescription;
|
|
43
43
|
bundle: ProjectContext;
|
|
44
|
+
/**
|
|
45
|
+
* Restrict the check to these Journeys. Each selector is a project-relative
|
|
46
|
+
* Journey path (`.taphound/journeys/<name>.json`) or a Journey name
|
|
47
|
+
* (`<name>`). A selector that matches no committed Journey fails.
|
|
48
|
+
*/
|
|
49
|
+
journeys?: readonly string[] | undefined;
|
|
44
50
|
}
|
|
45
51
|
export interface JourneyCheckDependencies {
|
|
46
52
|
store: Pick<JourneyCompositionStore, "read" | "listJourneyPaths" | "readJourneyMeta">;
|
|
@@ -21,6 +21,21 @@ function journeyName(journeyPath) {
|
|
|
21
21
|
: journeyPath;
|
|
22
22
|
return withoutPrefix.slice(0, -".json".length);
|
|
23
23
|
}
|
|
24
|
+
function selectJourneys(paths, selectors) {
|
|
25
|
+
if (selectors === undefined || selectors.length === 0) {
|
|
26
|
+
return paths;
|
|
27
|
+
}
|
|
28
|
+
const normalize = (value) => value
|
|
29
|
+
.replaceAll("\\", "/")
|
|
30
|
+
.replace(/^\.\//, "");
|
|
31
|
+
const wanted = new Set(selectors.map(normalize));
|
|
32
|
+
const selected = paths.filter((path) => (wanted.has(path) || wanted.has(journeyName(path))));
|
|
33
|
+
const missing = [...wanted].filter((selector) => !selected.some((path) => (path === selector || journeyName(path) === selector)));
|
|
34
|
+
if (missing.length > 0) {
|
|
35
|
+
throw new JourneyCheckError("JOURNEY_NOT_FOUND", `No committed Journey matches ${missing.join(", ")} under ${JOURNEYS_DIR}`);
|
|
36
|
+
}
|
|
37
|
+
return selected;
|
|
38
|
+
}
|
|
24
39
|
function summarize(entries) {
|
|
25
40
|
const summary = {
|
|
26
41
|
total: entries.length,
|
|
@@ -54,7 +69,7 @@ export class JourneyCheckService {
|
|
|
54
69
|
this.dependencies = dependencies;
|
|
55
70
|
}
|
|
56
71
|
check = async (input) => {
|
|
57
|
-
const paths = await this.dependencies.store.listJourneyPaths(input.projectRoot);
|
|
72
|
+
const paths = selectJourneys(await this.dependencies.store.listJourneyPaths(input.projectRoot), input.journeys);
|
|
58
73
|
const projectHash = hashGenerationBinding(input.project);
|
|
59
74
|
const configHash = hashGenerationBinding(input.config);
|
|
60
75
|
const modulesById = new Map(input.bundle.modules.map((module) => [module.id, module]));
|
|
@@ -5,6 +5,11 @@ export interface PublishedReport {
|
|
|
5
5
|
reportPath: string;
|
|
6
6
|
summaryPath: string;
|
|
7
7
|
}
|
|
8
|
+
/**
|
|
9
|
+
* One line naming dropped Logcat evidence. Logcat-based expectations already
|
|
10
|
+
* fail closed on a relevant drop; this keeps a passed run from hiding it.
|
|
11
|
+
*/
|
|
12
|
+
export declare function logcatEvidenceWarning(report: Pick<TapHoundReport, "logcatEvidence">): string | undefined;
|
|
8
13
|
export declare class ReportWriter {
|
|
9
14
|
writeAndPublish(session: ArtifactSession, input: TapHoundReport): Promise<PublishedReport>;
|
|
10
15
|
}
|
|
@@ -1,5 +1,21 @@
|
|
|
1
1
|
import { join } from "node:path";
|
|
2
2
|
import { TapHoundReportSchema } from "../../domain/report.js";
|
|
3
|
+
/**
|
|
4
|
+
* One line naming dropped Logcat evidence. Logcat-based expectations already
|
|
5
|
+
* fail closed on a relevant drop; this keeps a passed run from hiding it.
|
|
6
|
+
*/
|
|
7
|
+
export function logcatEvidenceWarning(report) {
|
|
8
|
+
const entries = report.logcatEvidence ?? [];
|
|
9
|
+
if (entries.length === 0)
|
|
10
|
+
return undefined;
|
|
11
|
+
const lines = entries.reduce((total, entry) => total + entry.droppedLines, 0);
|
|
12
|
+
if (entries.every((entry) => entry.expectationImpact === "none")) {
|
|
13
|
+
return `Warning (non-fatal): Logcat capture is partial (${String(lines)} line(s) dropped); `
|
|
14
|
+
+ "no Logcat expectation failed, and those expectations fail closed on drops in their window";
|
|
15
|
+
}
|
|
16
|
+
return `Warning: Logcat evidence is incomplete (${String(lines)} line(s) dropped) `
|
|
17
|
+
+ "and a Logcat expectation failed; the drops may be the cause";
|
|
18
|
+
}
|
|
3
19
|
function renderSummary(report) {
|
|
4
20
|
const lines = [
|
|
5
21
|
`TapHound run ${report.runId}: ${report.status.toUpperCase()}`,
|
|
@@ -10,6 +26,10 @@ function renderSummary(report) {
|
|
|
10
26
|
"Layers:",
|
|
11
27
|
...Object.entries(report.layers).map(([layer, status]) => `- ${layer}: ${status}`)
|
|
12
28
|
];
|
|
29
|
+
const logcatWarning = logcatEvidenceWarning(report);
|
|
30
|
+
if (logcatWarning !== undefined) {
|
|
31
|
+
lines.push("", logcatWarning);
|
|
32
|
+
}
|
|
13
33
|
if (report.primaryFailure !== undefined) {
|
|
14
34
|
lines.push("", `Primary failure: ${report.primaryFailure.code}: ${report.primaryFailure.message}`);
|
|
15
35
|
}
|
|
@@ -10,6 +10,20 @@ import { logcatStopFailed } from "../collector/logcat-stop.js";
|
|
|
10
10
|
import { ActivityWaiter } from "./activity-waiter.js";
|
|
11
11
|
import { coldLaunchApp } from "./cold-launch.js";
|
|
12
12
|
import { StepRunner } from "./step-runner.js";
|
|
13
|
+
/**
|
|
14
|
+
* The first step after a cold launch is the first to see the app's persisted
|
|
15
|
+
* state. TapHound never clears app data, so a target that is missing there
|
|
16
|
+
* most often means that state drifted since the Journey was generated.
|
|
17
|
+
*/
|
|
18
|
+
const INITIAL_STATE_FAILURE_CODES = new Set([
|
|
19
|
+
"LOCATOR_NOT_FOUND",
|
|
20
|
+
"ANCHOR_NOT_FOUND",
|
|
21
|
+
"SCROLL_TARGET_NOT_FOUND"
|
|
22
|
+
]);
|
|
23
|
+
const INITIAL_STATE_HINT = "this is the first step after cold launch: "
|
|
24
|
+
+ "TapHound does not reset app data, so check persisted app state such as "
|
|
25
|
+
+ "settings, login, or layout preferences against the state the Journey "
|
|
26
|
+
+ "was generated in before changing the Journey";
|
|
13
27
|
function commandFailed(result) {
|
|
14
28
|
return result.exitCode !== 0
|
|
15
29
|
|| result.timedOut
|
|
@@ -442,6 +456,7 @@ export class VerifyRuntime {
|
|
|
442
456
|
if (primaryFailure === undefined) {
|
|
443
457
|
const soleRole = soleRoleForJourney(input.journey);
|
|
444
458
|
const runtimeByRole = new Map(runtimes.map((runtime) => [runtime.role, runtime]));
|
|
459
|
+
const rolesStarted = new Set();
|
|
445
460
|
for (const [index, step] of input.journey.steps.entries()) {
|
|
446
461
|
input.progress?.({
|
|
447
462
|
stage: "replaying",
|
|
@@ -455,6 +470,8 @@ export class VerifyRuntime {
|
|
|
455
470
|
break;
|
|
456
471
|
}
|
|
457
472
|
const result = await runner.run(step, index, input.signal);
|
|
473
|
+
const firstStepOnDevice = !rolesStarted.has(role);
|
|
474
|
+
rolesStarted.add(role);
|
|
458
475
|
steps.push(result.report);
|
|
459
476
|
if (result.status === "manualRequired") {
|
|
460
477
|
setPrimary("MANUAL_STEP_REQUIRED", `Step ${String(index)} requires manual replay and the run is non-interactive`, "replay", index);
|
|
@@ -465,7 +482,10 @@ export class VerifyRuntime {
|
|
|
465
482
|
break;
|
|
466
483
|
}
|
|
467
484
|
if (result.status === "failed") {
|
|
468
|
-
setPrimary(result.failure.code,
|
|
485
|
+
setPrimary(result.failure.code, firstStepOnDevice
|
|
486
|
+
&& INITIAL_STATE_FAILURE_CODES.has(result.failure.code)
|
|
487
|
+
? `${result.failure.message} (${INITIAL_STATE_HINT})`
|
|
488
|
+
: result.failure.message, result.failure.phase, result.failure.stepIndex);
|
|
469
489
|
break;
|
|
470
490
|
}
|
|
471
491
|
const runtime = runtimeByRole.get(role);
|
|
@@ -632,10 +652,19 @@ export class VerifyRuntime {
|
|
|
632
652
|
"DEVICE_UNAVAILABLE",
|
|
633
653
|
"DEVICE_ROLE_UNMAPPED",
|
|
634
654
|
"APP_NOT_INSTALLED",
|
|
655
|
+
"UI_BACKEND_UNAVAILABLE",
|
|
656
|
+
"UI_SNAPSHOT_FAILED",
|
|
635
657
|
"INTERNAL_ERROR"
|
|
636
658
|
].includes(failure.code)
|
|
637
659
|
? "error"
|
|
638
660
|
: "failed";
|
|
661
|
+
// Logcat expectations fail closed on a relevant drop, so drops can only
|
|
662
|
+
// have changed the outcome of a Logcat expectation that failed.
|
|
663
|
+
const expectationImpact = steps.some((step) => (step.expectation?.status === "failed"
|
|
664
|
+
&& (step.expectation.type === "logcat"
|
|
665
|
+
|| step.expectation.type === "logcatEvent")))
|
|
666
|
+
? "possible"
|
|
667
|
+
: "none";
|
|
639
668
|
const report = {
|
|
640
669
|
schemaVersion: 4,
|
|
641
670
|
...(runtimes.some((runtime) => runtime.logcatStarted
|
|
@@ -653,7 +682,8 @@ export class VerifyRuntime {
|
|
|
653
682
|
...(metadata.lastDroppedAtMs === undefined ? {} : {
|
|
654
683
|
lastDroppedAtMs: metadata.lastDroppedAtMs
|
|
655
684
|
}),
|
|
656
|
-
status: "incomplete"
|
|
685
|
+
status: "incomplete",
|
|
686
|
+
expectationImpact
|
|
657
687
|
}];
|
|
658
688
|
})
|
|
659
689
|
}
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
import type { UiCaptureObserver } from "../../ports/diagnostics.js";
|
|
2
|
+
import type { OpenUiSnapshotProviderOptions, UiSnapshotProvider, UiSnapshotProviderFactory } from "../../ports/ui-snapshot.js";
|
|
3
|
+
/**
|
|
4
|
+
* Reports the duration and outcome of every UI backend read (captures and
|
|
5
|
+
* stability samples) to an observer. It sits under the snapshot cache, so a
|
|
6
|
+
* cache hit is not counted as a device read.
|
|
7
|
+
*/
|
|
8
|
+
export declare class ObservedUiSnapshotProviderFactory implements UiSnapshotProviderFactory {
|
|
9
|
+
private readonly source;
|
|
10
|
+
private readonly observer;
|
|
11
|
+
private readonly now;
|
|
12
|
+
constructor(source: UiSnapshotProviderFactory, observer: UiCaptureObserver, now: () => number);
|
|
13
|
+
open(options: OpenUiSnapshotProviderOptions): Promise<UiSnapshotProvider>;
|
|
14
|
+
}
|
|
15
|
+
export declare function observeProvider(provider: UiSnapshotProvider, observer: UiCaptureObserver, now: () => number): UiSnapshotProvider;
|
|
@@ -0,0 +1,66 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Reports the duration and outcome of every UI backend read (captures and
|
|
3
|
+
* stability samples) to an observer. It sits under the snapshot cache, so a
|
|
4
|
+
* cache hit is not counted as a device read.
|
|
5
|
+
*/
|
|
6
|
+
export class ObservedUiSnapshotProviderFactory {
|
|
7
|
+
source;
|
|
8
|
+
observer;
|
|
9
|
+
now;
|
|
10
|
+
constructor(source, observer, now) {
|
|
11
|
+
this.source = source;
|
|
12
|
+
this.observer = observer;
|
|
13
|
+
this.now = now;
|
|
14
|
+
}
|
|
15
|
+
async open(options) {
|
|
16
|
+
return observeProvider(await this.source.open(options), this.observer, this.now);
|
|
17
|
+
}
|
|
18
|
+
}
|
|
19
|
+
export function observeProvider(provider, observer, now) {
|
|
20
|
+
const source = provider;
|
|
21
|
+
const timed = async (read) => {
|
|
22
|
+
const startedAt = now();
|
|
23
|
+
try {
|
|
24
|
+
const value = await read();
|
|
25
|
+
observer.captured(source.descriptor.id, now() - startedAt);
|
|
26
|
+
return value;
|
|
27
|
+
}
|
|
28
|
+
catch (error) {
|
|
29
|
+
observer.captured(source.descriptor.id, now() - startedAt, error);
|
|
30
|
+
throw error;
|
|
31
|
+
}
|
|
32
|
+
};
|
|
33
|
+
const cacheTelemetry = source.cacheTelemetry;
|
|
34
|
+
const invalidate = source.invalidate;
|
|
35
|
+
const observed = {
|
|
36
|
+
descriptor: source.descriptor,
|
|
37
|
+
capture: (options) => timed(() => source.capture(options)),
|
|
38
|
+
close: () => source.close(),
|
|
39
|
+
...(cacheTelemetry === undefined
|
|
40
|
+
? {}
|
|
41
|
+
: { cacheTelemetry: () => cacheTelemetry.call(source) }),
|
|
42
|
+
...(invalidate === undefined
|
|
43
|
+
? {}
|
|
44
|
+
: {
|
|
45
|
+
invalidate: (reason) => {
|
|
46
|
+
invalidate.call(source, reason);
|
|
47
|
+
}
|
|
48
|
+
})
|
|
49
|
+
};
|
|
50
|
+
const sample = source.sample;
|
|
51
|
+
const reset = source.reset;
|
|
52
|
+
if (typeof sample === "function" && typeof reset === "function") {
|
|
53
|
+
observed.sample = (options) => timed(() => sample.call(source, options));
|
|
54
|
+
observed.reset = () => {
|
|
55
|
+
reset.call(source);
|
|
56
|
+
};
|
|
57
|
+
}
|
|
58
|
+
const withSupport = source;
|
|
59
|
+
if (withSupport.supportsStability !== undefined) {
|
|
60
|
+
Object.defineProperty(observed, "supportsStability", {
|
|
61
|
+
value: withSupport.supportsStability,
|
|
62
|
+
enumerable: true
|
|
63
|
+
});
|
|
64
|
+
}
|
|
65
|
+
return observed;
|
|
66
|
+
}
|
|
@@ -1,3 +1,9 @@
|
|
|
1
1
|
import type { UiSnapshotProvider } from "../../ports/ui-snapshot.js";
|
|
2
2
|
import type { UiStabilityProbe } from "../../ports/ui-stability.js";
|
|
3
|
+
/**
|
|
4
|
+
* A snapshot provider that samples stability itself (Appium) answers every
|
|
5
|
+
* structural sample, but frame-stat samples still go to the runtime probe:
|
|
6
|
+
* the provider only knows page sources, so it would answer a cheap
|
|
7
|
+
* `dumpsys gfxinfo` request with a full hierarchy capture.
|
|
8
|
+
*/
|
|
3
9
|
export declare function uiStabilityProbe(provider: UiSnapshotProvider, fallback: UiStabilityProbe): UiStabilityProbe;
|