swipium 1.5.0 → 2.0.0

package/dist/report/toolHealth.js

@@ -0,0 +1,124 @@
+// Tool health for qa_report (real-device smoke finding: a run with WDA 404s and UNKNOWN tool errors
+// reported "Tool status: PASS"). Two pure pieces:
+//   - toolErrorFromResult: classify a tool call result > a ToolErrorRecord to store on the session
+//     (consent refusals / missing test data are user decisions, not tool errors);
+//   - toolVerdictFor: the report's TOOL verdict from mcp_limitation notes + recorded tool errors.
+// Tool health never changes the APP verdict. A Swipium/driver error is not an app defect.
+import { FAILURES } from '../oracle/failures.js';
+/** Buckets that are deliberate outcomes, not tool malfunctions. */
+const NOT_TOOL_ERRORS = new Set(['unsafe_refused', 'missing_data']);
+/** Codes an agent hits while PROBING the UI (a selector that does not match yet, a stale @eN ref, an
+ *  ambiguous match, a bad argument it then corrects). Recorded for visibility, but they never flip the
+ *  TOOL status to DEGRADED on their own; they are not Swipium/driver malfunctions. */
+export const AGENT_PROBING_CODES = new Set(['ELEMENT_NOT_FOUND', 'STALE_REF', 'AMBIGUOUS_SELECTOR', 'INVALID_ARGUMENT']);
+/** An error result without a typed failureCode (qaError defaults it to UNKNOWN). Most are deliberate
+ *  refusals / guard messages. Counted in a separate `uncoded` bucket that does NOT degrade by itself. */
+export const UNCODED_CODES = new Set(['UNKNOWN', 'UNCODED']);
+/** Does this recorded tool error, by itself, make the TOOL status DEGRADED? */
+export function isDegradingToolError(failureCode) {
+    if (failureCode === 'CANCELLED')
+        return false;
+    if (UNCODED_CODES.has(failureCode) || AGENT_PROBING_CODES.has(failureCode))
+        return false;
+    const info = FAILURES[failureCode];
+    return !(info && NOT_TOOL_ERRORS.has(info.bucket));
+}
+export function toolErrorFromResult(tool, result) {
+    const r = result;
+    if (!r || r.isError !== true)
+        return undefined;
+    const sc = r.structuredContent ?? {};
+    const failureCode = typeof sc.failureCode === 'string' ? sc.failureCode : 'UNKNOWN';
+    if (failureCode === 'CANCELLED')
+        return undefined; // cancelled work is not a tool error
+    const info = FAILURES[failureCode];
+    if (info && NOT_TOOL_ERRORS.has(info.bucket))
+        return undefined;
+    const what = typeof sc.what === 'string' ? sc.what : (r.content?.find((c) => c.type === 'text')?.text ?? '').split('\n')[0];
+    return { tool, failureCode, message: what.replace(/^❌\s*/, '') };
+}
+/** Record an error result against the calling session (best-effort, never throws). */
+export function recordToolErrorFromResult(sessions, tool, args, result, signal) {
+    try {
+        // A call whose request was aborted produced cancelled work, whatever error it surfaced
+        // (e.g. "uiautomator dump failed … AbortError" / "WDA … aborted"), never a tool error.
+        if (signal?.aborted)
+            return;
+        const sessionId = args?.sessionId;
+        if (typeof sessionId !== 'string')
+            return;
+        const rec = toolErrorFromResult(tool, result);
+        if (!rec)
+            return;
+        const session = sessions.get(sessionId);
+        if (session)
+            sessions.recordToolError(session, rec);
+    }
+    catch {
+        /* tool-health bookkeeping must never break a tool call */
+    }
+}
+/**
+ * TOOL verdict (never the app verdict):
+ *  - BLOCKED: a workflow was blocked/failed by a Swipium/MCP limitation (qa_note mcp_limitation);
+ *  - DEGRADED: tool calls returned TYPED tool-side errors (WDA 404, driver failures…). The run's
+ *    coverage may be incomplete, but these are tool-side, not app defects. Uncoded errors (no
+ *    failureCode, mostly deliberate refusals) and agent-probing codes (ELEMENT_NOT_FOUND, STALE_REF,
+ *    AMBIGUOUS_SELECTOR, INVALID_ARGUMENT) are counted and reported but never degrade on their own;
+ *  - PASS: neither.
+ */
+export function toolVerdictFor(toolLimitationNotes, toolErrors, redact = (s) => s) {
+    const errors = toolErrors ?? [];
+    const counts = new Map();
+    for (const e of errors)
+        counts.set(e.failureCode, (counts.get(e.failureCode) ?? 0) + 1);
+    const byCode = Object.fromEntries([...counts.entries()].sort((a, b) => b[1] - a[1]));
+    const codeText = Object.entries(byCode)
+        .map(([c, n]) => `${c}×${n}`)
+        .join(', ');
+    const degradingCount = errors.filter((e) => isDegradingToolError(e.failureCode)).length;
+    const uncodedCount = errors.filter((e) => UNCODED_CODES.has(e.failureCode)).length;
+    const probingCount = errors.filter((e) => AGENT_PROBING_CODES.has(e.failureCode)).length;
+    const breakdown = { degradingCount, uncodedCount, probingCount };
+    const errorText = errors.length
+        ? `${errors.length} tool error(s) recorded (${codeText}; e.g. ${errors[0].tool}: ${redact(errors[0].message) ?? errors[0].message})`
+        : '';
+    if (toolLimitationNotes.length) {
+        const first = toolLimitationNotes[0];
+        return {
+            status: 'BLOCKED',
+            summary: `${toolLimitationNotes.length} workflow(s) limited by Swipium/MCP capabilities (for example ${first.workflow}${first.reason ? `: ${redact(first.reason) ?? first.reason}` : ''}). These are tool limitations, not app defects.` +
+                (errorText ? ` Also ${errorText}.` : ''),
+            toolErrorCount: errors.length,
+            toolErrorsByCode: byCode,
+            ...breakdown,
+        };
+    }
+    if (degradingCount) {
+        return {
+            status: 'DEGRADED',
+            summary: `${errorText}. Tool-side errors, not app defects. Coverage may be incomplete; see toolErrors.`,
+            toolErrorCount: errors.length,
+            toolErrorsByCode: byCode,
+            ...breakdown,
+        };
+    }
+    if (errors.length) {
+        return {
+            status: 'PASS',
+            summary: `No Swipium/MCP limitations or tool-side failures recorded. ${errors.length} non-degrading error result(s) ` +
+                `(${codeText}: ${uncodedCount} uncoded refusal/guard message(s), ${probingCount} agent-probing miss(es)). See toolErrors.`,
+            toolErrorCount: errors.length,
+            toolErrorsByCode: byCode,
+            ...breakdown,
+        };
+    }
+    return {
+        status: 'PASS',
+        summary: 'No Swipium/MCP limitations or tool errors recorded in this run.',
+        toolErrorCount: 0,
+        toolErrorsByCode: {},
+        ...breakdown,
+    };
+}
+//# sourceMappingURL=toolHealth.js.map

package/dist/report/toolHealth.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"toolHealth.js","sourceRoot":"","sources":["../../src/report/toolHealth.ts"],"names":[],"mappings":"AAAA,oGAAoG;AACpG,kDAAkD;AAClD,mGAAmG;AACnG,kFAAkF;AAClF,kGAAkG;AAClG,0FAA0F;AAE1F,OAAO,EAAE,QAAQ,EAAE,MAAM,uBAAuB,CAAC;AAGjD,mEAAmE;AACnE,MAAM,eAAe,GAAG,IAAI,GAAG,CAAC,CAAC,gBAAgB,EAAE,cAAc,CAAC,CAAC,CAAC;AAEpE;;sFAEsF;AACtF,MAAM,CAAC,MAAM,mBAAmB,GAAG,IAAI,GAAG,CAAC,CAAC,mBAAmB,EAAE,WAAW,EAAE,oBAAoB,EAAE,kBAAkB,CAAC,CAAC,CAAC;AAEzH;yGACyG;AACzG,MAAM,CAAC,MAAM,aAAa,GAAG,IAAI,GAAG,CAAC,CAAC,SAAS,EAAE,SAAS,CAAC,CAAC,CAAC;AAE7D,+EAA+E;AAC/E,MAAM,UAAU,oBAAoB,CAAC,WAAmB;IACtD,IAAI,WAAW,KAAK,WAAW;QAAE,OAAO,KAAK,CAAC;IAC9C,IAAI,aAAa,CAAC,GAAG,CAAC,WAAW,CAAC,IAAI,mBAAmB,CAAC,GAAG,CAAC,WAAW,CAAC;QAAE,OAAO,KAAK,CAAC;IACzF,MAAM,IAAI,GAAI,QAA2D,CAAC,WAAW,CAAC,CAAC;IACvF,OAAO,CAAC,CAAC,IAAI,IAAI,eAAe,CAAC,GAAG,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC;AACrD,CAAC;AAED,MAAM,UAAU,mBAAmB,CAAC,IAAY,EAAE,MAAe;IAC/D,MAAM,CAAC,GAAG,MAA+H,CAAC;IAC1I,IAAI,CAAC,CAAC,IAAI,CAAC,CAAC,OAAO,KAAK,IAAI;QAAE,OAAO,SAAS,CAAC;IAC/C,MAAM,EAAE,GAAG,CAAC,CAAC,iBAAiB,IAAI,EAAE,CAAC;IACrC,MAAM,WAAW,GAAG,OAAO,EAAE,CAAC,WAAW,KAAK,QAAQ,CAAC,CAAC,CAAC,EAAE,CAAC,WAAW,CAAC,CAAC,CAAC,SAAS,CAAC;IACpF,IAAI,WAAW,KAAK,WAAW;QAAE,OAAO,SAAS,CAAC,CAAC,qCAAqC;IACxF,MAAM,IAAI,GAAI,QAA2D,CAAC,WAAW,CAAC,CAAC;IACvF,IAAI,IAAI,IAAI,eAAe,CAAC,GAAG,CAAC,IAAI,CAAC,MAAM,CAAC;QAAE,OAAO,SAAS,CAAC;IAC/D,MAAM,IAAI,GAAG,OAAO,EAAE,CAAC,IAAI,KAAK,QAAQ,CAAC,CAAC,CAAC,EAAE,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,OAAO,EAAE,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,MAAM,CAAC,EAAE,IAAI,IAAI,EAAE,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC;IAC5H,OAAO,EAAE,IAAI,EAAE,WAAW,EAAE,OAAO,EAAE,IAAI,CAAC,OAAO,CAAC,OAAO,EAAE,EAAE,CAAC,EAAE,CAAC;AACnE,CAAC;AAED,sFAAsF;AACtF,MAAM,UAAU,yBAAyB,CACvC,QAAsB,EACtB,IAAY,EACZ,IAAa,EACb,MAAe,EACf,MAAoB;IAEpB,IAAI,CAAC;QACH,uFAAuF;QACvF,uFAAuF;QACvF,IAAI,MAAM,EAAE,OAAO;YAAE,OAAO;QAC5B,MAAM,SAAS,GAAI,IAA4C,EAAE,SAAS,CAAC;QAC3E,IAAI,OAAO,SAAS,KAAK,QAAQ;YAAE,OAAO;QAC1C,MAAM,GAAG,GAAG,mBAAmB,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC;QAC9C,IAAI,CAAC,GAAG;YAAE,OAAO;QACjB,MAAM,OAAO,GAAG,QAAQ,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC;QACxC,IAAI,OAAO;YAAE,QAAQ,CAAC,eAAe,CAAC,OAAO,EAAE,GAAG,CAAC,CAAC;IACtD,CAAC;IAAC,MAAM,CAAC;QACP,0DAA0D;IAC5D,CAAC;AACH,CAAC;AAiBD;;;;;;;;GAQG;AACH,MAAM,UAAU,cAAc,CAC5B,mBAAiE,EACjE,UAAyC,EACzC,SAA6C,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;IAErD,MAAM,MAAM,GAAG,UAAU,IAAI,EAAE,CAAC;IAChC,MAAM,MAAM,GAAG,IAAI,GAAG,EAAkB,CAAC;IACzC,KAAK,MAAM,CAAC,IAAI,MAAM;QAAE,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,WAAW,EAAE,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,WAAW,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;IACxF,MAAM,MAAM,GAAG,MAAM,CAAC,WAAW,CAAC,CAAC,GAAG,MAAM,CAAC,OAAO,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IACrF,MAAM,QAAQ,GAAG,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC;SACpC,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC;SAC5B,IAAI,CAAC,IAAI,CAAC,CAAC;IACd,MAAM,cAAc,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,oBAAoB,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC,CAAC,MAAM,CAAC;IACxF,MAAM,YAAY,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,aAAa,CAAC,GAAG,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC,CAAC,MAAM,CAAC;IACnF,MAAM,YAAY,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,mBAAmB,CAAC,GAAG,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC,CAAC,MAAM,CAAC;IACzF,MAAM,SAAS,GAAG,EAAE,cAAc,EAAE,YAAY,EAAE,YAAY,EAAE,CAAC;IACjE,MAAM,SAAS,GAAG,MAAM,CAAC,MAAM;QAC7B,CAAC,CAAC,GAAG,MAAM,CAAC,MAAM,4BAA4B,QAAQ,UAAU,MAAM,CAAC,CAAC,CAAC,CAAC,IAAI,KAAK,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,IAAI,MAAM,CAAC,CAAC,CAAC,CAAC,OAAO,GAAG;QACpI,CAAC,CAAC,EAAE,CAAC;IACP,IAAI,mBAAmB,CAAC,MAAM,EAAE,CAAC;QAC/B,MAAM,KAAK,GAAG,mBAAmB,CAAC,CAAC,CAAC,CAAC;QACrC,OAAO;YACL,MAAM,EAAE,SAAS;YACjB,OAAO,EACL,GAAG,mBAAmB,CAAC,MAAM,iEAAiE,KAAK,CAAC,QAAQ,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,KAAK,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,IAAI,KAAK,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,EAAE,iDAAiD;gBAC/N,CAAC,SAAS,CAAC,CAAC,CAAC,SAAS,SAAS,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC;YAC1C,cAAc,EAAE,MAAM,CAAC,MAAM;YAC7B,gBAAgB,EAAE,MAAM;YACxB,GAAG,SAAS;SACb,CAAC;IACJ,CAAC;IACD,IAAI,cAAc,EAAE,CAAC;QACnB,OAAO;YACL,MAAM,EAAE,UAAU;YAClB,OAAO,EAAE,GAAG,SAAS,kFAAkF;YACvG,cAAc,EAAE,MAAM,CAAC,MAAM;YAC7B,gBAAgB,EAAE,MAAM;YACxB,GAAG,SAAS;SACb,CAAC;IACJ,CAAC;IACD,IAAI,MAAM,CAAC,MAAM,EAAE,CAAC;QAClB,OAAO;YACL,MAAM,EAAE,MAAM;YACd,OAAO,EACL,8DAA8D,MAAM,CAAC,MAAM,iCAAiC;gBAC5G,IAAI,QAAQ,KAAK,YAAY,sCAAsC,YAAY,2CAA2C;YAC5H,cAAc,EAAE,MAAM,CAAC,MAAM;YAC7B,gBAAgB,EAAE,MAAM;YACxB,GAAG,SAAS;SACb,CAAC;IACJ,CAAC;IACD,OAAO;QACL,MAAM,EAAE,MAAM;QACd,OAAO,EAAE,iEAAiE;QAC1E,cAAc,EAAE,CAAC;QACjB,gBAAgB,EAAE,EAAE;QACpB,GAAG,SAAS;KACb,CAAC;AACJ,CAAC"}