taphound 0.2.0-dev.1 → 0.2.0-dev.12

package/dist/application/runtime/verify-runtime.js

@@ -1,12 +1,29 @@
 import { resolve } from "node:path";
 import { normalizeActivity } from "../../domain/activity.js";
-import { exitCodeForFailure } from "../../domain/failure.js";
+import { exitCodeForFailure, failureCodeFromUnknown } from "../../domain/failure.js";
+import { DEFAULT_DEVICE_ROLE } from "../../domain/journey.js";
 import { hashJourney } from "../../domain/report.js";
+import { uiStabilityProbe } from "../ui/ui-stability-probe.js";
 import { LogcatCollector } from "../collector/logcat-collector.js";
+import { CheckpointEvaluator } from "../checkpoint/checkpoint-evaluator.js";
+import { logcatStopFailed } from "../collector/logcat-stop.js";
 import { ActivityWaiter } from "./activity-waiter.js";
-import { launchFailure } from "./launch-failure.js";
-import { ProcessWaiter } from "./process-waiter.js";
+import { coldLaunchApp } from "./cold-launch.js";
 import { StepRunner } from "./step-runner.js";
+/**
+ * The first step after a cold launch is the first to see the app's persisted
+ * state. TapHound never clears app data, so a target that is missing there
+ * most often means that state drifted since the Journey was generated.
+ */
+const INITIAL_STATE_FAILURE_CODES = new Set([
+    "LOCATOR_NOT_FOUND",
+    "ANCHOR_NOT_FOUND",
+    "SCROLL_TARGET_NOT_FOUND"
+]);
+const INITIAL_STATE_HINT = "this is the first step after cold launch: "
+    + "TapHound does not reset app data, so check persisted app state such as "
+    + "settings, login, or layout preferences against the state the Journey "
+    + "was generated in before changing the Journey";
 function commandFailed(result) {
     return result.exitCode !== 0
         || result.timedOut
@@ -16,23 +33,14 @@ function commandFailed(result) {
 function commandMessage(result, fallback) {
     return result.stderr.trim() || result.spawnError || fallback;
 }
-function logcatStopFailed(result) {
-    if (result.timedOut
-        || result.cancelled
-        || result.spawnError !== undefined) {
-        return true;
-    }
-    return result.exitCode !== 0
-        && result.signal !== "SIGTERM"
-        && result.signal !== "SIGKILL";
-}
 function errorMessage(error) {
     return error instanceof Error ? error.message : String(error);
 }
 function layerForFailure(code) {
     if (code === "APP_NOT_INSTALLED"
         || code === "APP_LAUNCH_FAILED"
-        || code === "APP_CRASHED") {
+        || code === "APP_CRASHED"
+        || code === "RUNTIME_CAPABILITY_MISSING") {
         return "run";
     }
     if (code === "ACTIVITY_BEFORE_MISMATCH"
@@ -42,27 +50,133 @@ function layerForFailure(code) {
     if (code.startsWith("EXPECT_")) {
         return "explicitExpect";
     }
+    if (code === "CHECKPOINT_FAILED" || code === "CHECKPOINT_UNRESOLVED") {
+        return "explicitExpect";
+    }
     if (code === "COLLECTION_FAILED") {
         return "collection";
     }
     return "structural";
 }
+function soleRoleForJourney(journey) {
+    return journey.devices[0]?.role ?? DEFAULT_DEVICE_ROLE;
+}
+function taskHookOutcome(phase, runtime, result) {
+    return { ...result, phase, deviceRole: runtime.role };
+}
+async function runVerifyHook(phase, runtime, snapshot, hooks, clock) {
+    const hook = phase === "beforeSteps"
+        ? hooks.beforeSteps
+        : hooks.afterSteps;
+    if (hook === undefined) {
+        return taskHookOutcome(phase, runtime, {
+            status: "unresolved",
+            message: "Verify hook is not configured"
+        });
+    }
+    try {
+        const result = await hook({
+            deviceRole: runtime.role,
+            deviceSerial: runtime.deviceSerial,
+            adb: runtime.views.adb,
+            uiSnapshotProvider: runtime.provider,
+            logcat: runtime.logcat,
+            clock,
+            snapshot
+        });
+        return taskHookOutcome(phase, runtime, result);
+    }
+    catch (error) {
+        return taskHookOutcome(phase, runtime, {
+            status: "unresolved",
+            message: error instanceof Error ? error.message : String(error)
+        });
+    }
+}
+function stepDeviceRole(step, soleRole) {
+    return step.device ?? soleRole;
+}
+function deviceCoverageFailure(journey, devices) {
+    const duplicateRole = devices.find((assignment, index) => devices.some((other, otherIndex) => otherIndex > index && other.role === assignment.role));
+    if (duplicateRole !== undefined) {
+        return {
+            code: "CONFIG_INVALID",
+            message: `Duplicate device assignment for role: ${duplicateRole.role}`
+        };
+    }
+    const reusedSerial = devices.find((assignment, index) => devices.some((other, otherIndex) => otherIndex > index
+        && other.deviceSerial === assignment.deviceSerial));
+    if (reusedSerial !== undefined) {
+        return {
+            code: "CONFIG_INVALID",
+            message: `Device serial is mapped to multiple roles: ${reusedSerial.deviceSerial}`
+        };
+    }
+    const declaredRoles = journey.devices.map((device) => device.role);
+    const unknownRole = devices.find((assignment) => !declaredRoles.includes(assignment.role));
+    if (unknownRole !== undefined) {
+        return {
+            code: "CONFIG_INVALID",
+            message: `Device assignment references a role the Journey does not declare: ${unknownRole.role}`
+        };
+    }
+    const assignedRoles = new Set(devices.map((assignment) => assignment.role));
+    const missingRoles = declaredRoles.filter((role) => !assignedRoles.has(role));
+    if (missingRoles.length > 0) {
+        return {
+            code: "DEVICE_ROLE_UNMAPPED",
+            message: `No device mapping for journey role(s): ${missingRoles.join(", ")}`
+        };
+    }
+    return undefined;
+}
+function uniqueAssignmentsByRole(devices) {
+    const seen = new Set();
+    const unique = [];
+    for (const assignment of devices) {
+        if (seen.has(assignment.role)) {
+            continue;
+        }
+        seen.add(assignment.role);
+        unique.push(assignment);
+    }
+    return unique;
+}
 export class VerifyRuntime {
     dependencies;
     constructor(dependencies) {
         this.dependencies = dependencies;
     }
     async verify(input) {
+        if (input.devices.length === 0) {
+            throw new Error("VerifyInput.devices requires at least one device assignment");
+        }
+        input.progress?.({ stage: "preparing" });
         const startedAt = this.dependencies.now();
         const runId = this.dependencies.createRunId();
         const launchActivity = normalizeActivity(input.config.run.packageName, input.config.run.activity);
         const session = await this.dependencies.artifactStore.begin(resolve(input.projectRoot, input.config.artifactsDir), runId);
-        const logcat = new LogcatCollector(this.dependencies.adb, this.dependencies.clock);
-        let logcatStarted = false;
-        let primaryFailure;
+        const logcatRunStartedAt = this.dependencies.clock.now();
+        const markers = new Map();
+        const coverage = deviceCoverageFailure(input.journey, input.devices);
+        const assignmentByRole = new Map(input.devices.map((assignment) => [assignment.role, assignment]));
+        const runtimes = [];
+        let primaryFailure = coverage === undefined
+            ? undefined
+            : {
+                code: coverage.code,
+                message: coverage.message,
+                phase: "runtime"
+            };
         const secondaryErrors = [];
         const collectionErrors = [];
+        const hookOutcomes = [];
         const steps = [];
+        const checkpoints = [];
+        const checkpointScreens = [];
+        const checkpointEvaluator = new CheckpointEvaluator();
+        let knowledge;
+        let knowledgeError;
         const layers = {
             run: "notRun",
             structural: "notRun",
@@ -70,6 +184,9 @@ export class VerifyRuntime {
             explicitExpect: "notRun",
             collection: "passed"
         };
+        if (coverage !== undefined) {
+            layers[layerForFailure(coverage.code)] = "failed";
+        }
         const setPrimary = (code, message, phase, stepIndex) => {
             if (primaryFailure !== undefined) {
                 return;
@@ -92,197 +209,429 @@ export class VerifyRuntime {
             collectionErrors.push(failure);
         };
         try {
-            try {
-                const installed = await this.dependencies.adb.isInstalled({
+            if (input.journey.checkpoints?.some((checkpoint) => checkpoint.expect.allOf.some((condition) => condition.kind === "screen"))) {
+                if (this.dependencies.loadKnowledge === undefined) {
+                    knowledgeError = "Project Knowledge loader is not configured";
+                }
+                else {
+                    try {
+                        knowledge = await this.dependencies.loadKnowledge({
+                            projectRoot: input.projectRoot,
+                            packageName: input.config.run.packageName
+                        });
+                    }
+                    catch (error) {
+                        knowledgeError = `Project Knowledge unavailable: ${errorMessage(error)}`;
+                    }
+                }
+            }
+            const evaluateCheckpoint = async (checkpoint, runtime, stepStartedAt) => {
+                if (runtime.provider === undefined) {
+                    setPrimary("CHECKPOINT_UNRESOLVED", `Checkpoint ${checkpoint.id} has no UI snapshot provider`, "checkpoint", checkpoint.stepIndex);
+                    return false;
+                }
+                const evaluation = await checkpointEvaluator.evaluate({
+                    checkpoint,
+                    provider: runtime.provider,
+                    adb: runtime.views.adb,
                     packageName: input.config.run.packageName,
-                    deviceSerial: input.deviceSerial,
-                    ...(input.signal === undefined ? {} : { signal: input.signal }),
-                    timeoutMs: input.config.idle.timeoutMs
+                    deviceSerial: runtime.deviceSerial,
+                    timeoutMs: input.config.idle.timeoutMs,
+                    clock: this.dependencies.clock,
+                    logcat: runtime.logcat,
+                    runStartedAt: logcatRunStartedAt,
+                    markers,
+                    logcatEvidenceRef: `logcat-${runtime.role}.txt`,
+                    ...(stepStartedAt === undefined ? {} : { stepStartedAt }),
+                    writeSnapshot: (path, snapshot) => session.writeJson(path, snapshot),
+                    ...(knowledge === undefined ? {} : { knowledge }),
+                    ...(knowledgeError === undefined ? {} : { knowledgeError }),
+                    ...(input.signal === undefined ? {} : { signal: input.signal })
                 });
-                if (!installed) {
-                    setPrimary("APP_NOT_INSTALLED", `Package ${input.config.run.packageName} is not installed on ${input.deviceSerial}`, "install");
+                checkpoints.push(evaluation.report);
+                if (evaluation.matchedScreen !== undefined) {
+                    checkpointScreens.push({
+                        screen: evaluation.matchedScreen,
+                        status: "matched"
+                    });
+                }
+                if (evaluation.report.status !== "passed") {
+                    setPrimary(evaluation.report.status === "failed"
+                        ? "CHECKPOINT_FAILED"
+                        : "CHECKPOINT_UNRESOLVED", `Checkpoint ${checkpoint.id} ${evaluation.report.status}: ${evaluation.report.conditions
+                        .filter((condition) => condition.status !== "passed")
+                        .map((condition) => condition.message ?? condition.kind)
+                        .join("; ")}`, "checkpoint", checkpoint.stepIndex);
+                    return false;
+                }
+                return true;
+            };
+            const createStepRunner = this.dependencies.createStepRunner
+                ?? ((options) => new StepRunner(options));
+            // Fresh for every Replay, shared only across device runners in this run.
+            const bindings = new Map();
+            if (coverage === undefined) {
+                for (const declaration of input.journey.devices) {
+                    const assignment = assignmentByRole.get(declaration.role);
+                    if (assignment === undefined) {
+                        continue;
+                    }
+                    const deviceSession = await this.dependencies.sessions.openSession({
+                        deviceSerial: assignment.deviceSerial,
+                        ...(input.signal === undefined ? {} : { signal: input.signal })
+                    });
+                    const views = this.dependencies.sessionPorts(deviceSession);
+                    runtimes.push({
+                        role: declaration.role,
+                        deviceSerial: assignment.deviceSerial,
+                        session: deviceSession,
+                        views,
+                        provider: undefined,
+                        readySnapshot: undefined,
+                        logcat: new LogcatCollector(views.adb, this.dependencies.clock),
+                        logcatStarted: false,
+                        runner: undefined
+                    });
                 }
             }
-            catch (error) {
-                setPrimary("APP_NOT_INSTALLED", errorMessage(error), "install");
-            }
-            if (primaryFailure === undefined) {
+            for (const runtime of runtimes) {
+                if (primaryFailure !== undefined) {
+                    break;
+                }
+                const deviceSession = runtime.session;
+                const provider = await deviceSession.openUiSnapshots({
+                    timeoutMs: input.config.ui?.snapshotTimeoutMs
+                        ?? input.config.idle.timeoutMs,
+                    backend: input.config.ui?.backend ?? "auto",
+                    cacheEnabled: input.config.ui?.cacheEnabled ?? true,
+                    ...(input.signal === undefined ? {} : { signal: input.signal })
+                });
+                runtime.provider = provider;
+                let installFailed = false;
+                try {
+                    const installed = await deviceSession.isInstalled({
+                        packageName: input.config.run.packageName,
+                        ...(input.signal === undefined ? {} : { signal: input.signal }),
+                        timeoutMs: input.config.idle.timeoutMs
+                    });
+                    if (!installed) {
+                        setPrimary("APP_NOT_INSTALLED", `Package ${input.config.run.packageName} is not installed on ${runtime.deviceSerial}`, "install");
+                        installFailed = true;
+                    }
+                }
+                catch (error) {
+                    setPrimary("APP_NOT_INSTALLED", errorMessage(error), "install");
+                    installFailed = true;
+                }
+                if (installFailed) {
+                    break;
+                }
                 try {
-                    await logcat.start({
-                        deviceSerial: input.deviceSerial,
+                    await runtime.logcat.start({
+                        deviceSerial: runtime.deviceSerial,
                         ...(input.signal === undefined ? {} : { signal: input.signal })
                     });
-                    logcatStarted = true;
+                    runtime.logcatStarted = true;
                 }
                 catch (error) {
                     layers.collection = "failed";
-                    setPrimary("COLLECTION_FAILED", errorMessage(error), "collection");
+                    setPrimary(failureCodeFromUnknown(error) ?? "COLLECTION_FAILED", errorMessage(error), "collection");
                 }
-                if (logcatStarted) {
-                    const identity = {
+                if (!runtime.logcatStarted) {
+                    break;
+                }
+                try {
+                    const launched = await coldLaunchApp(runtime.views.adb, this.dependencies.clock, {
                         packageName: input.config.run.packageName,
-                        deviceSerial: input.deviceSerial,
-                        ...(input.signal === undefined ? {} : { signal: input.signal }),
-                        timeoutMs: input.config.idle.timeoutMs
-                    };
-                    const stopped = await this.dependencies.adb.forceStop(identity);
-                    const launched = commandFailed(stopped)
-                        ? undefined
-                        : await this.dependencies.adb.launchActivity({
-                            packageName: input.config.run.packageName,
-                            activity: launchActivity,
-                            deviceSerial: input.deviceSerial,
-                            ...(input.signal === undefined ? {} : { signal: input.signal }),
-                            timeoutMs: input.config.idle.timeoutMs
-                        });
-                    const launchError = launched === undefined
-                        ? commandMessage(stopped, "App reset failed")
-                        : launchFailure(launched);
-                    if (launchError !== undefined) {
-                        layers.run = "failed";
-                        setPrimary("APP_LAUNCH_FAILED", launchError, "run");
+                        activity: launchActivity,
+                        deviceSerial: runtime.deviceSerial,
+                        pollIntervalMs: input.config.idle.pollIntervalMs,
+                        timeoutMs: input.config.idle.timeoutMs,
+                        signal: input.signal
+                    });
+                    if (launched.status === "cancelled") {
+                        setPrimary("INTERNAL_ERROR", "Verification was cancelled", "readiness");
+                        break;
                     }
-                    else {
-                        try {
-                            const launchReadinessStartedAt = this.dependencies.clock.now();
-                            const processReadiness = await new ProcessWaiter(this.dependencies.adb, this.dependencies.clock).wait({
-                                packageName: input.config.run.packageName,
-                                deviceSerial: input.deviceSerial,
-                                pollIntervalMs: input.config.idle.pollIntervalMs,
-                                timeoutMs: input.config.idle.timeoutMs,
-                                ...(input.signal === undefined ? {} : { signal: input.signal })
-                            });
-                            if (processReadiness.status === "timeout") {
-                                setPrimary("APP_LAUNCH_FAILED", "App process was not found after launch", "readiness");
-                            }
-                            else if (processReadiness.status === "cancelled") {
-                                setPrimary("INTERNAL_ERROR", "Verification was cancelled", "readiness");
-                            }
-                            else {
-                                logcat.scopeToPids(processReadiness.pids);
-                                const firstStep = input.journey.steps[0];
-                                if (firstStep === undefined) {
-                                    throw new Error("Journey requires at least one step");
-                                }
-                                const remainingReadinessMs = input.config.idle.timeoutMs
-                                    - (this.dependencies.clock.now()
-                                        - launchReadinessStartedAt);
-                                if (remainingReadinessMs <= 0) {
-                                    setPrimary("APP_LAUNCH_FAILED", `Expected startup Activity ${firstStep.activity.before}, found none before timeout`, "readiness");
-                                }
-                                else {
-                                    const readiness = await new ActivityWaiter(this.dependencies.adb, this.dependencies.clock).wait({
-                                        packageName: input.config.run.packageName,
-                                        deviceSerial: input.deviceSerial,
-                                        expectedActivity: firstStep.activity.before,
-                                        pollIntervalMs: input.config.idle.pollIntervalMs,
-                                        timeoutMs: remainingReadinessMs,
-                                        ...(input.signal === undefined ? {} : { signal: input.signal })
-                                    });
-                                    if (readiness.status === "processMissing") {
-                                        setPrimary("APP_LAUNCH_FAILED", "App process exited before reaching the first Journey Activity", "readiness");
-                                    }
-                                    else if (readiness.status === "timeout") {
-                                        setPrimary("APP_LAUNCH_FAILED", `Expected startup Activity ${firstStep.activity.before}, found ${readiness.actual ?? "none"} before timeout`, "readiness");
-                                    }
-                                    else if (readiness.status === "cancelled") {
-                                        setPrimary("INTERNAL_ERROR", "Verification was cancelled", "readiness");
-                                    }
-                                    else {
-                                        await this.dependencies.androidCli.layout({
-                                            deviceSerial: input.deviceSerial,
-                                            ...(input.signal === undefined ? {} : { signal: input.signal }),
-                                            timeoutMs: input.config.idle.timeoutMs
-                                        });
-                                        layers.run = "passed";
-                                        layers.structural = "passed";
-                                        layers.activityCheckpoint = "passed";
-                                        layers.explicitExpect = "passed";
-                                    }
-                                }
-                            }
+                    if (launched.status === "failed") {
+                        if (launched.stage === "process") {
+                            setPrimary("APP_LAUNCH_FAILED", launched.message, "readiness");
                         }
-                        catch (error) {
-                            setPrimary("APP_LAUNCH_FAILED", errorMessage(error), "readiness");
+                        else {
+                            layers.run = "failed";
+                            setPrimary("APP_LAUNCH_FAILED", launched.message, "run");
                         }
+                        break;
+                    }
+                    runtime.logcat.scopeToPids(launched.pids);
+                    const soleRole = soleRoleForJourney(input.journey);
+                    const firstOwnStep = input.journey.steps.find((step) => stepDeviceRole(step, soleRole) === runtime.role);
+                    if (firstOwnStep === undefined) {
+                        throw new Error("Journey requires at least one step");
+                    }
+                    // The readiness budget starts once the launch command returned.
+                    const remainingReadinessMs = input.config.idle.timeoutMs
+                        - launched.processWaitMs;
+                    if (remainingReadinessMs <= 0) {
+                        setPrimary("APP_LAUNCH_FAILED", `Expected startup Activity ${firstOwnStep.activity.before}, found none before timeout`, "readiness");
+                        break;
                     }
+                    const readiness = await new ActivityWaiter(runtime.views.adb, this.dependencies.clock).wait({
+                        packageName: input.config.run.packageName,
+                        deviceSerial: runtime.deviceSerial,
+                        expectedActivity: firstOwnStep.activity.before,
+                        pollIntervalMs: input.config.idle.pollIntervalMs,
+                        timeoutMs: remainingReadinessMs,
+                        ...(input.signal === undefined ? {} : { signal: input.signal })
+                    });
+                    if (readiness.status === "processMissing") {
+                        setPrimary("APP_LAUNCH_FAILED", "App process exited before reaching the first Journey Activity", "readiness");
+                        break;
+                    }
+                    if (readiness.status === "timeout") {
+                        setPrimary("APP_LAUNCH_FAILED", `Expected startup Activity ${firstOwnStep.activity.before}, found ${readiness.actual ?? "none"} before timeout`, "readiness");
+                        break;
+                    }
+                    if (readiness.status === "cancelled") {
+                        setPrimary("INTERNAL_ERROR", "Verification was cancelled", "readiness");
+                        break;
+                    }
+                    const readySnapshot = await provider.capture({
+                        reason: "locate",
+                        ...(input.signal === undefined ? {} : { signal: input.signal }),
+                        timeoutMs: input.config.idle.timeoutMs
+                    });
+                    runtime.readySnapshot = readySnapshot;
+                    runtime.runner = createStepRunner({
+                        adb: runtime.views.adb,
+                        screenshots: runtime.views.screenshots,
+                        annotatedScreens: runtime.views.annotatedScreens,
+                        uiStability: uiStabilityProbe(provider, runtime.views.uiStability),
+                        uiSnapshotProvider: provider,
+                        clock: this.dependencies.clock,
+                        logcat: runtime.logcat,
+                        runStartedAt: logcatRunStartedAt,
+                        markers,
+                        bindings,
+                        artifacts: session,
+                        packageName: input.config.run.packageName,
+                        deviceSerial: runtime.deviceSerial,
+                        deviceRole: runtime.role,
+                        idle: input.config.idle,
+                        ...(this.dependencies.anchorResolverFor === undefined
+                            ? {}
+                            : { anchorResolver: this.dependencies.anchorResolverFor(input.projectRoot) }),
+                        ...(input.requireFocusedInput === undefined
+                            ? {}
+                            : { requireFocusedInput: input.requireFocusedInput }),
+                        ...(input.generatedReplayPolicy === undefined
+                            ? {}
+                            : { generatedReplayPolicy: input.generatedReplayPolicy }),
+                        ...(input.manualReplay === undefined
+                            ? {}
+                            : { manualReplay: input.manualReplay })
+                    });
+                }
+                catch (error) {
+                    setPrimary(failureCodeFromUnknown(error) ?? "APP_LAUNCH_FAILED", errorMessage(error), "readiness");
+                    break;
                 }
             }
             if (primaryFailure === undefined) {
-                const createStepRunner = this.dependencies.createStepRunner
-                    ?? ((options) => new StepRunner(options));
-                const runner = createStepRunner({
-                    adb: this.dependencies.adb,
-                    androidCli: this.dependencies.androidCli,
-                    clock: this.dependencies.clock,
-                    logcat,
-                    artifacts: session,
-                    packageName: input.config.run.packageName,
-                    deviceSerial: input.deviceSerial,
-                    idle: input.config.idle,
-                    ...(input.requireFocusedInput === undefined
-                        ? {}
-                        : { requireFocusedInput: input.requireFocusedInput }),
-                    ...(input.generatedReplayPolicy === undefined
-                        ? {}
-                        : { generatedReplayPolicy: input.generatedReplayPolicy })
-                });
+                layers.run = "passed";
+                layers.structural = "passed";
+                layers.activityCheckpoint = "passed";
+                layers.explicitExpect = "passed";
+            }
+            if (primaryFailure === undefined
+                && input.hooks?.beforeSteps !== undefined) {
+                for (const runtime of runtimes) {
+                    if (runtime.provider === undefined
+                        || runtime.readySnapshot === undefined) {
+                        continue;
+                    }
+                    hookOutcomes.push(await runVerifyHook("beforeSteps", runtime, runtime.readySnapshot, input.hooks, this.dependencies.clock));
+                }
+            }
+            if (primaryFailure === undefined) {
+                const soleRole = soleRoleForJourney(input.journey);
+                const runtimeByRole = new Map(runtimes.map((runtime) => [runtime.role, runtime]));
+                const rolesStarted = new Set();
                 for (const [index, step] of input.journey.steps.entries()) {
+                    input.progress?.({
+                        stage: "replaying",
+                        stepIndex: index,
+                        stepCount: input.journey.steps.length
+                    });
+                    const role = stepDeviceRole(step, soleRole);
+                    const runner = runtimeByRole.get(role)?.runner;
+                    if (runner === undefined) {
+                        setPrimary("CONFIG_INVALID", `Step references an unmapped device role: ${role}`, "replay", index);
+                        break;
+                    }
                     const result = await runner.run(step, index, input.signal);
+                    const firstStepOnDevice = !rolesStarted.has(role);
+                    rolesStarted.add(role);
                     steps.push(result.report);
+                    if (result.status === "manualRequired") {
+                        setPrimary("MANUAL_STEP_REQUIRED", `Step ${String(index)} requires manual replay and the run is non-interactive`, "replay", index);
+                        break;
+                    }
                     if (result.status === "cancelled") {
                         setPrimary("INTERNAL_ERROR", "Verification was cancelled", "replay", index);
                         break;
                     }
                     if (result.status === "failed") {
-                        setPrimary(result.failure.code, result.failure.message, result.failure.phase, result.failure.stepIndex);
+                        setPrimary(result.failure.code, firstStepOnDevice
+                            && INITIAL_STATE_FAILURE_CODES.has(result.failure.code)
+                            ? `${result.failure.message} (${INITIAL_STATE_HINT})`
+                            : result.failure.message, result.failure.phase, result.failure.stepIndex);
+                        break;
+                    }
+                    const runtime = runtimeByRole.get(role);
+                    if (runtime === undefined) {
+                        throw new Error("Checkpoint device runtime is unavailable");
+                    }
+                    let checkpointFailed = false;
+                    for (const checkpoint of input.journey.checkpoints ?? []) {
+                        if (checkpoint.stepIndex !== index) {
+                            continue;
+                        }
+                        if (!await evaluateCheckpoint(checkpoint, runtime, result.report.startedAtMs)) {
+                            checkpointFailed = true;
+                            break;
+                        }
+                    }
+                    if (checkpointFailed) {
                         break;
                     }
                 }
+                if (steps.length === input.journey.steps.length
+                    && steps.every((step) => step.status === "passed")
+                    && checkpoints.every((checkpoint) => checkpoint.status === "passed")) {
+                    const lastStep = input.journey.steps.at(-1);
+                    const lastRole = lastStep === undefined
+                        ? soleRole
+                        : stepDeviceRole(lastStep, soleRole);
+                    const runtime = runtimeByRole.get(lastRole);
+                    for (const checkpoint of input.journey.checkpoints ?? []) {
+                        if (checkpoint.stepIndex !== undefined) {
+                            continue;
+                        }
+                        if (runtime === undefined || !await evaluateCheckpoint(checkpoint, runtime, steps.at(-1)?.startedAtMs)) {
+                            break;
+                        }
+                    }
+                }
+                if (input.hooks?.afterSteps !== undefined) {
+                    for (const runtime of runtimes) {
+                        if (runtime.provider === undefined) {
+                            continue;
+                        }
+                        let snapshot;
+                        try {
+                            snapshot = await runtime.provider.capture({
+                                reason: "evidence",
+                                ...(input.signal === undefined
+                                    ? {}
+                                    : { signal: input.signal }),
+                                timeoutMs: input.config.idle.timeoutMs
+                            });
+                        }
+                        catch (error) {
+                            hookOutcomes.push({
+                                phase: "afterSteps",
+                                deviceRole: runtime.role,
+                                status: "unresolved",
+                                message: `Post-journey snapshot failed: ${error instanceof Error ? error.message : String(error)}`
+                            });
+                            continue;
+                        }
+                        hookOutcomes.push(await runVerifyHook("afterSteps", runtime, snapshot, input.hooks, this.dependencies.clock));
+                    }
+                }
             }
         }
         catch (error) {
+            const errorCode = failureCodeFromUnknown(error) ?? "INTERNAL_ERROR";
             if (primaryFailure === undefined) {
-                setPrimary("INTERNAL_ERROR", errorMessage(error), "runtime");
+                setPrimary(errorCode, errorMessage(error), "runtime");
             }
             else {
                 secondaryErrors.push({
-                    code: "INTERNAL_ERROR",
+                    code: errorCode,
                     message: errorMessage(error),
                     phase: "runtime"
                 });
             }
         }
-        const screenshotPath = "screenshot.png";
-        let screenshotCollected = false;
-        try {
-            const screenshot = await this.dependencies.androidCli.captureScreen({
-                outputPath: session.path(screenshotPath),
-                deviceSerial: input.deviceSerial,
-                ...(input.signal === undefined ? {} : { signal: input.signal })
-            });
-            if (commandFailed(screenshot)) {
-                collectionFailure(commandMessage(screenshot, "Screen capture failed"));
-            }
-            else {
-                screenshotCollected = true;
+        const screenshotEntries = [];
+        const uiHierarchyEntries = [];
+        const logcatEntries = [];
+        input.progress?.({ stage: "collecting" });
+        if (coverage === undefined) {
+            for (const runtime of runtimes) {
+                if (runtime.provider !== undefined
+                    && runtime.readySnapshot !== undefined) {
+                    const uiHierarchyPath = `ui-hierarchy-${runtime.role}.json`;
+                    try {
+                        const snapshot = await runtime.provider.capture({
+                            reason: "evidence",
+                            ...(input.signal === undefined ? {} : { signal: input.signal }),
+                            timeoutMs: input.config.idle.timeoutMs
+                        });
+                        await session.writeJson(uiHierarchyPath, snapshot);
+                        uiHierarchyEntries.push({
+                            role: runtime.role,
+                            path: uiHierarchyPath
+                        });
+                    }
+                    catch (error) {
+                        collectionFailure(errorMessage(error));
+                    }
+                }
+                const screenshotPath = `screenshot-${runtime.role}.png`;
+                try {
+                    const screenshot = await runtime.session.captureScreenshot({
+                        outputPath: session.path(screenshotPath),
+                        ...(input.signal === undefined ? {} : { signal: input.signal })
+                    });
+                    if (commandFailed(screenshot)) {
+                        collectionFailure(commandMessage(screenshot, "Screen capture failed"));
+                    }
+                    else {
+                        screenshotEntries.push({ role: runtime.role, path: screenshotPath });
+                    }
+                }
+                catch (error) {
+                    collectionFailure(errorMessage(error));
+                }
+                if (runtime.logcatStarted) {
+                    const logcatPath = `logcat-${runtime.role}.txt`;
+                    try {
+                        const stopped = await runtime.logcat.stop();
+                        if (logcatStopFailed(stopped)) {
+                            collectionFailure(commandMessage(stopped, "Logcat stop failed"));
+                        }
+                        await session.writeText(logcatPath, runtime.logcat.rawLines().map((line) => line.raw).join("\n"));
+                        logcatEntries.push({ role: runtime.role, path: logcatPath });
+                    }
+                    catch (error) {
+                        collectionFailure(errorMessage(error));
+                    }
+                }
             }
         }
-        catch (error) {
-            collectionFailure(errorMessage(error));
-        }
-        let logcatCollected = false;
-        if (logcatStarted) {
+        for (const runtime of runtimes) {
             try {
-                const stopped = await logcat.stop();
-                if (logcatStopFailed(stopped)) {
-                    collectionFailure(commandMessage(stopped, "Logcat stop failed"));
-                }
-                await session.writeText("logcat.txt", logcat.lines().map((line) => line.raw).join("\n"));
-                logcatCollected = true;
+                await runtime.session.close();
             }
             catch (error) {
-                collectionFailure(errorMessage(error));
+                secondaryErrors.push({
+                    code: "INTERNAL_ERROR",
+                    message: errorMessage(error),
+                    phase: "runtimeSessionClose"
+                });
             }
         }
         if (collectionErrors.length > 0) {
@@ -295,17 +644,42 @@ export class VerifyRuntime {
         const failure = primaryFailure;
         const status = failure === undefined
             ? "passed"
-            : [
-                "CONFIG_INVALID",
-                "ENVIRONMENT_MISSING_TOOL",
-                "DEVICE_UNAVAILABLE",
-                "APP_NOT_INSTALLED",
-                "INTERNAL_ERROR"
-            ].includes(failure.code)
-                ? "error"
-                : "failed";
+            : failure.code === "MANUAL_STEP_REQUIRED"
+                ? "manualRequired"
+                : [
+                    "CONFIG_INVALID",
+                    "ENVIRONMENT_MISSING_TOOL",
+                    "DEVICE_UNAVAILABLE",
+                    "DEVICE_ROLE_UNMAPPED",
+                    "APP_NOT_INSTALLED",
+                    "UI_BACKEND_UNAVAILABLE",
+                    "UI_SNAPSHOT_FAILED",
+                    "INTERNAL_ERROR"
+                ].includes(failure.code)
+                    ? "error"
+                    : "failed";
         const report = {
-            schemaVersion: 2,
+            schemaVersion: 4,
+            ...(runtimes.some((runtime) => runtime.logcatStarted
+                && (runtime.logcat.metadata().droppedLines ?? 0) > 0)
+                ? {
+                    logcatEvidence: runtimes.flatMap((runtime) => {
+                        if (!runtime.logcatStarted) {
+                            return [];
+                        }
+                        const metadata = runtime.logcat.metadata();
+                        return metadata.droppedLines === undefined ? [] : [{
+                                role: runtime.role,
+                                droppedLines: metadata.droppedLines,
+                                droppedBytes: metadata.droppedBytes ?? 0,
+                                ...(metadata.lastDroppedAtMs === undefined ? {} : {
+                                    lastDroppedAtMs: metadata.lastDroppedAtMs
+                                }),
+                                status: "incomplete"
+                            }];
+                    })
+                }
+                : {}),
             runId,
             status,
             startedAt: startedAt.toISOString(),
@@ -321,17 +695,44 @@ export class VerifyRuntime {
                 sha256: hashJourney(input.journey)
             },
             environment: {
-                deviceSerial: input.deviceSerial,
+                devices: coverage === undefined
+                    ? runtimes.map((runtime) => ({
+                        role: runtime.role,
+                        deviceSerial: runtime.deviceSerial,
+                        ...(runtime.provider === undefined
+                            ? {}
+                            : {
+                                uiBackend: runtime.provider.descriptor,
+                                ...(runtime.provider.cacheTelemetry === undefined
+                                    ? {}
+                                    : { uiCache: runtime.provider.cacheTelemetry() })
+                            })
+                    }))
+                    : uniqueAssignmentsByRole(input.devices).map((assignment) => ({
+                        role: assignment.role,
+                        deviceSerial: assignment.deviceSerial
+                    })),
                 tools: input.toolVersions
             },
             layers,
             steps,
+            ...(input.journey.checkpoints === undefined ? {} : { checkpoints }),
+            screens: Array.from(new Map([
+                ...hookOutcomes.flatMap((outcome) => (outcome.status === "passed" && outcome.screen !== undefined
+                    ? [[outcome.screen, {
+                                screen: outcome.screen,
+                                status: "matched"
+                            }]]
+                    : [])),
+                ...checkpointScreens.map((screen) => [screen.screen, screen])
+            ]).values()),
             artifacts: {
                 directory: session.finalDirectory,
                 report: "report.json",
                 summary: "summary.txt",
-                ...(screenshotCollected ? { screenshot: screenshotPath } : {}),
-                ...(logcatCollected ? { logcat: "logcat.txt" } : {}),
+                screenshots: screenshotEntries,
+                uiHierarchies: uiHierarchyEntries,
+                logcats: logcatEntries,
                 stepLogs: steps.flatMap((step) => (step.logcatPath === undefined ? [] : [step.logcatPath]))
             },
             ...(failure === undefined ? {} : { primaryFailure: failure }),
@@ -344,7 +745,8 @@ export class VerifyRuntime {
             exitCode: failure === undefined ? 0 : exitCodeForFailure(failure.code),
             report,
             reportPath: published.reportPath,
-            summaryPath: published.summaryPath
+            summaryPath: published.summaryPath,
+            ...(hookOutcomes.length === 0 ? {} : { hookOutcomes })
         };
     }
 }