verikun 0.18.0 → 0.18.1

package/README.md

@@ -144,11 +144,17 @@ skips the archive dump on green runs (failures still capture).
 - **`report.html`** — a self-contained report: every step, the identifiers used,
   any screenshots taken, the screenshot + hierarchy of any failed page, a link to
   the full device log, an app-scoped log accordion when the run launched an app,
-  and any per-step logs from `vk log`.
+  and any per-step logs from `vk log`. A run that did not pass says so in a banner
+  at the top.
 - **`artifacts/logcat.txt`** — full device log for the run window (default).
 - **`artifacts/logcat-app.txt`** — app-scoped log (when a package/bundle was launched).
 - **`run.json`** — the raw recording.
 
+A `vk ai` run can also fail where no single command did — a `repeat` that never
+sees its target, a cost/timeout abort. That verdict is recorded too (as
+`run.json`'s `failure`, plus a failed step carrying the reason and a screenshot),
+so a failed test is never reported as `failures="0"` in the JUnit that CI reads.
+
 `vk run archive` exits non-zero when the run contained failures, so the same
 command both produces the report and gates CI.
 

package/dist/cli.js

@@ -50,6 +50,7 @@ exports.chooseLogOpts = chooseLogOpts;
 exports.evalAssert = evalAssert;
 exports.tokenizeLine = tokenizeLine;
 exports.withBatchGlobals = withBatchGlobals;
+exports.terminalFailure = terminalFailure;
 exports.executeForServer = executeForServer;
 exports.run = run;
 const node_fs_1 = require("node:fs");
@@ -1322,6 +1323,24 @@ async function resolveBackend(platform, device, flags) {
                         driver.clearApp(appId);
                 },
                 preflight: () => driver.preflight(),
+                captureFailure: async () => {
+                    // Two independent tries: a screencap can succeed where a dump doesn't (and
+                    // vice versa), and neither is allowed to derail recording the failure.
+                    const out = {};
+                    try {
+                        out.png = driver.screenshot();
+                    }
+                    catch {
+                        /* device may be gone — that may be why we failed */
+                    }
+                    try {
+                        out.hierarchy = driver.getElements({ all: false });
+                    }
+                    catch {
+                        /* ditto */
+                    }
+                    return out;
+                },
             },
             platform,
             device,
@@ -1352,12 +1371,52 @@ async function resolveBackend(platform, device, flags) {
                 await (0, remote_1.pingServer)(opts);
                 await remote.getElements();
             },
+            // Hierarchy only: the server exposes no screenshot route, so a remote run's
+            // engine failure archives without a picture. Honest degrade over a protocol
+            // change here — tracked in #48.
+            captureFailure: async () => {
+                try {
+                    return { hierarchy: await remote.getElements() };
+                }
+                catch {
+                    return {};
+                }
+            },
         },
         platform: health.platform,
         device: health.serial,
         remote: { url: server, version: health.version },
     };
 }
+/**
+ * The one terminal-failure record for a non-ok engine result — `null` when the run
+ * passed. Exported for the unit suite.
+ *
+ * Budget and timeout aborts come back from the engine as a bare flag with NO `failure`
+ * object, so their reason is composed here; `where` is `run` because the abort is not
+ * attributable to one node. Both the recorded failure and the `[ai] …` status line are
+ * built from this, so the archive and the console cannot disagree about why a run died.
+ */
+function terminalFailure(r, opts) {
+    if (r.ok)
+        return null;
+    if (r.abortedForEnv) {
+        return { where: r.failure?.where ?? 'run', reason: r.failure?.reason ?? 'device unavailable', kind: 'env' };
+    }
+    if (r.abortedForBudget) {
+        return { where: r.failure?.where ?? 'run', reason: `cost ceiling $${opts.maxCostUsd} reached`, kind: 'budget' };
+    }
+    if (r.abortedForTimeout) {
+        return { where: 'run', reason: `run timeout (${Math.round(opts.timeoutMs / 1000)}s) reached`, kind: 'timeout' };
+    }
+    return { where: r.failure?.where ?? 'run', reason: r.failure?.reason ?? 'failed', kind: 'fail' };
+}
+/** The `[ai] …` console verdict for a terminal failure, phrased as it always was. */
+function terminalStatusLine(t) {
+    if (t.kind === 'fail')
+        return `FAIL at ${t.where}: ${t.reason}`;
+    return `ABORTED — ${t.kind === 'env' ? `environment: ${t.reason}` : t.reason}`;
+}
 /**
  * Best-effort archive-time device-log capture via an ExecBackend (local driver or
  * remote `/v1/logs`). Writes `artifacts/logcat.txt` onto the active run so a later
@@ -1470,6 +1529,13 @@ async function runAiTest(file, opts, backend, platform, device) {
         // seal the run so it is not left dangling in .verikun/run/ for the next command
         // to roll over. Then let the error map to an exit code as usual.
         run_1.Recorder.annotateRun({ ai: { ok: false, cost: cost.summaryLine(), modelRepairs: 0, improvements: [] } });
+        // No evidence capture here: a throw at this level usually IS the device dying, so
+        // the capture would fail the same way and only add noise to the error path.
+        run_1.Recorder.recordTerminalFailure({
+            where: 'run',
+            reason: e.message,
+            kind: (0, errors_1.isEnvError)(e) ? 'env' : 'fail',
+        });
         try {
             await prefetchArchiveLogs(backend);
             run_1.Recorder.archive();
@@ -1496,21 +1562,18 @@ async function runAiTest(file, opts, backend, platform, device) {
             (0, output_1.err)(`[ai] could not cache plan: ${e.message}`);
         }
     }
+    // A failure the engine produced (a control node giving up, a budget/timeout abort)
+    // never ran through a command, so nothing recorded it — without this the archive
+    // declares the failed test green. Must come BEFORE the archive that renders it.
+    const terminal = terminalFailure(result, opts);
+    if (terminal)
+        run_1.Recorder.recordTerminalFailure(terminal, await backend.captureFailure?.());
     run_1.Recorder.annotateRun({
         ai: { ok: result.ok, cost: costLine, modelRepairs: result.modelRepairs, improvements: result.improvements },
     });
     await prefetchArchiveLogs(backend);
     const { dir, xmlPath, htmlPath, state } = run_1.Recorder.archive();
-    const status = result.ok
-        ? 'PASS'
-        : result.abortedForBudget
-            ? `ABORTED — cost ceiling $${opts.maxCostUsd} reached`
-            : result.abortedForTimeout
-                ? `ABORTED — run timeout (${Math.round(opts.timeoutMs / 1000)}s) reached`
-                : result.abortedForEnv
-                    ? `ABORTED — environment: ${result.failure?.reason}`
-                    : `FAIL at ${result.failure?.where}: ${result.failure?.reason}`;
-    (0, output_1.err)(`[ai] ${status} · ${costLine}`);
+    (0, output_1.err)(`[ai] ${terminal ? terminalStatusLine(terminal) : 'PASS'} · ${costLine}`);
     (0, output_1.err)(`[ai] report: ${htmlPath}`);
     if (result.improvements.length) {
         (0, output_1.err)(`[ai] ${result.improvements.length} suggested improvement(s) (also in the report):`);

package/dist/report.js

@@ -5,6 +5,7 @@
 // recorder (run.ts) / suite runner (suite.ts) own all the I/O. The RunState data
 // model lives in run.ts; we import the types only.
 Object.defineProperty(exports, "__esModule", { value: true });
+exports.runFailure = runFailure;
 exports.toJUnitXml = toJUnitXml;
 exports.suiteTotals = suiteTotals;
 exports.toSuiteIndexJson = toSuiteIndexJson;
@@ -41,12 +42,40 @@ function resolvedLabel(s) {
 function fmtDuration(ms) {
     return ms < 1000 ? `${ms} ms` : `${(ms / 1000).toFixed(2)} s`;
 }
+/**
+ * The run's verdict, taken from the ENGINE rather than inferred from step statuses.
+ *
+ * A `vk ai` run can fail outside any command — a `repeat` that never sees its target,
+ * a budget/timeout abort — and those record no step, so a tally-only report declared
+ * the run green (issue #41). `Recorder.recordTerminalFailure` now writes `failure`, and
+ * `ai.ok` is the older, coarser signal we still fall back to; between them an
+ * unrecorded failure can no longer read as success.
+ */
+function runFailure(run) {
+    if (run.failure)
+        return run.failure;
+    if (run.ai && !run.ai.ok)
+        return { where: 'run', reason: 'the run did not pass (no step recorded the failure)' };
+    return null;
+}
 function counts(run) {
     const passed = run.steps.filter((s) => s.status === 'passed').length;
     const failures = run.steps.filter((s) => s.status === 'failed').length;
     const errors = run.steps.filter((s) => s.status === 'error').length;
     const timeMs = run.steps.reduce((a, s) => a + s.durationMs, 0);
-    return { tests: run.steps.length, passed, failures, errors, timeMs };
+    // Belt and braces. Normally the terminal failure IS a step by the time we render, so
+    // this stays false; it fires only if the failure never reached the recorder, and then
+    // both renderers emit one extra entry — hence `tests` grows too, so the tally never
+    // disagrees with the testcase list it is supposed to describe.
+    const unrecorded = failures + errors === 0 && runFailure(run) !== null;
+    return {
+        tests: run.steps.length + (unrecorded ? 1 : 0),
+        passed,
+        failures: failures + (unrecorded ? 1 : 0),
+        errors,
+        timeMs,
+        unrecorded,
+    };
 }
 // --- JUnit ----------------------------------------------------------------
 function toJUnitXml(run) {
@@ -93,6 +122,12 @@ function toJUnitXml(run) {
         return `  <testcase ${attrs}>${body}\n  </testcase>`;
     })
         .join('\n');
+    const f = runFailure(run);
+    const unrecordedCase = c.unrecorded && f
+        ? `  <testcase name="${xmlAttr(`run did not pass (${f.where})`)}" classname="verikun.run" time="0.000">` +
+            `\n    <failure message="${xmlAttr(f.reason)}" type="AssertionFailure">${xmlText(`${f.where}: ${f.reason}`)}</failure>\n  </testcase>`
+        : '';
+    const allCases = [cases, unrecordedCase].filter(Boolean).join('\n');
     const suiteAttrs = `name="${xmlAttr(run.name)}" tests="${c.tests}" failures="${c.failures}" ` +
         `errors="${c.errors}" time="${suiteTime}" timestamp="${xmlAttr(run.startedAt)}"`;
     const suiteExtras = [];
@@ -106,7 +141,7 @@ function toJUnitXml(run) {
     return (`<?xml version="1.0" encoding="UTF-8"?>\n` +
         `<testsuites name="verikun" tests="${c.tests}" failures="${c.failures}" errors="${c.errors}" time="${suiteTime}">\n` +
         `<testsuite ${suiteAttrs}>\n` +
-        `${cases}\n` +
+        `${allCases}\n` +
         (suiteExtras.length
             ? `  <system-out>${xmlText(suiteExtras.join('\n'))}</system-out>\n`
             : '') +
@@ -151,6 +186,9 @@ const STYLE = `
   .aibox { background:#fff; border:1px solid var(--line); border-radius:8px; padding:12px 14px; margin-bottom:20px; font-size:13px; }
   .aibox .cost { font-family:ui-monospace,SFMono-Regular,Menlo,monospace; color:var(--muted); margin-top:4px; }
   .aibox ul { margin:8px 0 0; padding-left:18px; }
+  .failbox { background:#fff; border:1px solid var(--fail); border-left-width:4px; border-radius:8px; padding:12px 14px; margin-bottom:20px; font-size:13px; }
+  .failbox .where { font-family:ui-monospace,SFMono-Regular,Menlo,monospace; color:var(--muted); }
+  .failbox .why { color:var(--fail); margin-top:4px; }
 `;
 function aiPanelHtml(ai) {
     const improvements = ai.improvements.length
@@ -164,6 +202,14 @@ function aiPanelHtml(ai) {
       ${improvements}
     </div>`;
 }
+/** The run-level failure, stated once at the top. This page is where a human looks
+ *  first, so the verdict has to be visible without reading 26 green rows. */
+function failBoxHtml(f) {
+    return `<div class="failbox">
+      <div><strong>This run did not pass.</strong> <span class="where">${htmlEsc(f.where)}</span></div>
+      <div class="why">${htmlEsc(f.reason)}</div>
+    </div>`;
+}
 function stepHtml(s) {
     const detail = [];
     if (selectorLabel(s))
@@ -333,6 +379,7 @@ ${suite.tests.map((x) => suiteTestRow(x, linkBase)).join('\n')}
  */
 function toHtml(run, opts = {}) {
     const c = counts(run);
+    const failure = runFailure(run);
     const chips = [
         `<span class="chip pass">${c.passed} passed</span>`,
         c.failures ? `<span class="chip fail">${c.failures} failed</span>` : '',
@@ -376,9 +423,27 @@ function toHtml(run, opts = {}) {
   <div class="summary">
       ${chips}
   </div>
+  ${failure ? failBoxHtml(failure) : ''}
   ${run.ai ? aiPanelHtml(run.ai) : ''}
   <ol class="steps">
-    ${run.steps.map(stepHtml).join('\n    ')}
+    ${[
+        ...run.steps.map(stepHtml),
+        // Only when the failure reached no step — otherwise it is already a red row.
+        ...(c.unrecorded && failure
+            ? [
+                stepHtml({
+                    index: run.steps.length,
+                    command: 'ai',
+                    name: `run did not pass (${failure.where})`,
+                    startedAt: run.startedAt,
+                    durationMs: 0,
+                    status: 'failed',
+                    exitCode: 1,
+                    message: failure.reason,
+                }),
+            ]
+            : []),
+    ].join('\n    ')}
   </ol>${appLogPanel}
 </div>
 </body>

package/dist/run.js

@@ -105,6 +105,19 @@ function loadState(dir) {
         return null;
     }
 }
+/** Relative path of a step's failure screenshot. One place, because a synthetic
+ *  terminal-failure step has to land on the same convention as a recorded one. */
+const failImagePath = (index) => `artifacts/step-${index}-fail.png`;
+function writeArtifactTo(dir, rel, buf) {
+    (0, node_fs_1.mkdirSync)((0, node_path_1.join)(dir, 'artifacts'), { recursive: true });
+    (0, node_fs_1.writeFileSync)((0, node_path_1.join)(dir, rel), buf);
+}
+/** Compact hierarchy text, HEAD-capped — a failure dump is read top-down, so the
+ *  first screenful is the useful part (unlike logs, which are kept tail-first). */
+function capHierarchy(els) {
+    const text = (0, format_1.formatCompact)(els);
+    return text.length > HIERARCHY_CAP ? text.slice(0, HIERARCHY_CAP) + '\n…(truncated)' : text;
+}
 function saveState(dir, state) {
     (0, node_fs_1.mkdirSync)(dir, { recursive: true });
     (0, node_fs_1.writeFileSync)(statePath(dir), JSON.stringify(state, null, 2));
@@ -427,8 +440,8 @@ class Recorder {
         if (!driver)
             return;
         try {
-            this.writeArtifact(`artifacts/step-${this.step.index}-fail.png`, driver.screenshot());
-            this.step.failImage = `artifacts/step-${this.step.index}-fail.png`;
+            this.writeArtifact(failImagePath(this.step.index), driver.screenshot());
+            this.step.failImage = failImagePath(this.step.index);
         }
         catch (e) {
             // Best-effort evidence: the device may be gone (often why the step failed). Surface
@@ -437,8 +450,7 @@ class Recorder {
                 (0, output_1.err)(`[verikun] could not capture failure screenshot (${e.message})`);
         }
         try {
-            const text = (0, format_1.formatCompact)(driver.getElements({ all: false }));
-            this.step.failHierarchy = text.length > HIERARCHY_CAP ? text.slice(0, HIERARCHY_CAP) + '\n…(truncated)' : text;
+            this.step.failHierarchy = capHierarchy(driver.getElements({ all: false }));
         }
         catch (e) {
             if (!quiet)
@@ -450,8 +462,7 @@ class Recorder {
             this.sink[rel] = buf;
             return;
         }
-        (0, node_fs_1.mkdirSync)((0, node_path_1.join)(this.dir, 'artifacts'), { recursive: true });
-        (0, node_fs_1.writeFileSync)((0, node_path_1.join)(this.dir, rel), buf);
+        writeArtifactTo(this.dir, rel, buf);
     }
     commit() {
         this.step.durationMs = Date.now() - this.startMs;
@@ -647,6 +658,56 @@ class Recorder {
             last.message = message;
         saveState(dir, state);
     }
+    /**
+     * Record a terminal failure the `vk ai` ENGINE produced rather than a command — a
+     * control node that gave up (`repeat` exhausted, `when` matched no branch), a
+     * budget/timeout abort, an engine-internal throw. None of those go through
+     * beginStep, so before this existed nothing marked the run red and the archived
+     * report declared a failed test fully green (issue #41).
+     *
+     * Always records the run-level verdict. Appends a synthetic failed step ONLY when
+     * no step is already red — a leaf failure carries its own step and evidence, and
+     * counting it twice would be its own kind of lie.
+     */
+    static recordTerminalFailure(failure, evidence) {
+        if (process.env.VERIKUN_NO_RUN)
+            return;
+        const dir = activeDir();
+        const state = loadState(dir);
+        if (!state)
+            return;
+        state.failure = { where: failure.where, reason: failure.reason };
+        if (state.steps.every((s) => s.status === 'passed')) {
+            const index = state.steps.length;
+            // An environment abort is exit 3 (the box is broken, not the app), matching
+            // what `vk ai` itself returns; everything else is an exit-1 assertion failure.
+            const env = failure.kind === 'env';
+            const step = {
+                index,
+                command: 'ai',
+                name: `ai ${env ? 'aborted' : 'failed'} at ${failure.where}`,
+                startedAt: nowIso(),
+                durationMs: 0,
+                status: env ? 'error' : 'failed',
+                exitCode: env ? 3 : 1,
+                message: failure.reason,
+            };
+            if (evidence?.png) {
+                try {
+                    writeArtifactTo(dir, failImagePath(index), evidence.png);
+                    step.failImage = failImagePath(index);
+                }
+                catch (e) {
+                    (0, output_1.err)(`[verikun] could not write the failure screenshot (${e.message})`);
+                }
+            }
+            if (evidence?.hierarchy)
+                step.failHierarchy = capHierarchy(evidence.hierarchy);
+            state.steps.push(step);
+        }
+        state.updatedAt = nowIso();
+        saveState(dir, state);
+    }
     static start(name, platform, device, force) {
         const dir = activeDir();
         const existing = loadState(dir);

package/dist/version.js

@@ -3,4 +3,4 @@ Object.defineProperty(exports, "__esModule", { value: true });
 exports.VERSION = void 0;
 // GENERATED by scripts/gen-version.mjs from package.json's "version" at build time
 // (the `prebuild` script). Do NOT edit by hand; bump package.json instead.
-exports.VERSION = '0.18.0';
+exports.VERSION = '0.18.1';

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "verikun",
-  "version": "0.18.0",
+  "version": "0.18.1",
   "description": "Drive Android emulators/devices and iOS simulators for AI agents: tap, type, swipe, screenshot, and inspect the UI hierarchy by semantic identifiers — like Puppeteer for native apps.",
   "keywords": [
     "android",