ravensight-playtest 0.2.9 → 0.3.0

package/src/run/drivers/webdriver-probe.js

@@ -0,0 +1,353 @@
+// The free capability probe for the native drivers. It asks the developer's own
+// local W3C automation server the three questions a persona run depends on, in
+// the order a run would hit them, and spends nothing: no job, no reservation, no
+// model call. `check` prints it as a row and `run` refuses to register a native
+// job when it fails.
+import fs from 'node:fs/promises';
+import path from 'node:path';
+import { readDriverConfig } from './webdriver.js';
+
+/**
+ * Capability keys that name the app a session opens, in the order they are
+ * reported. A path or URL (`app`) is the build; a bundle id or package names an
+ * app already installed on the device or simulator.
+ */
+export const APP_KEYS = Object.freeze([
+  'appium:app',
+  'app',
+  'appium:bundleId',
+  'bundleId',
+  'appium:appPackage',
+  'appPackage'
+]);
+
+const STATUS_TIMEOUT_MS = 10_000;
+// The first session on a fresh simulator builds and installs the automation
+// agent, which takes minutes, not seconds.
+const SESSION_TIMEOUT_MS = 180_000;
+const CLOSE_TIMEOUT_MS = 30_000;
+
+/**
+ * The app the capabilities name, or null.
+ * @param {Object} capabilities
+ * @returns {{key: string, value: string, kind: 'path'|'url'|'id'}|null}
+ */
+export function appReference(capabilities = {}) {
+  for (const key of APP_KEYS) {
+    const value = capabilities[key];
+    if (typeof value !== 'string' || !value.trim()) continue;
+    if (key === 'appium:app' || key === 'app') {
+      return { key, value, kind: /^[a-z][a-z0-9+.-]*:\/\//i.test(value) ? 'url' : 'path' };
+    }
+    return { key, value, kind: 'id' };
+  }
+  return null;
+}
+
+function message(error) {
+  if (!error) return 'unknown error';
+  if (error.name === 'TimeoutError') return 'timed out';
+  const cause = error.cause && (error.cause.code || error.cause.message);
+  return String(cause || error.message || error).split('\n')[0];
+}
+
+/** The same response cap the native driver applies. */
+export const MAX_RESPONSE_BYTES = 8 * 1024 * 1024;
+
+/** Read a response body, refusing anything past the cap. */
+async function boundedText(response) {
+  if (!response.body) return '';
+  const reader = response.body.getReader();
+  const chunks = [];
+  let size = 0;
+  try {
+    for (;;) {
+      const part = await reader.read();
+      if (part.done) break;
+      size += part.value.length;
+      if (size > MAX_RESPONSE_BYTES) {
+        await reader.cancel();
+        throw new Error('automation server response exceeded 8 MB');
+      }
+      chunks.push(Buffer.from(part.value));
+    }
+  } finally {
+    reader.releaseLock();
+  }
+  return Buffer.concat(chunks).toString('utf8');
+}
+
+/**
+ * One W3C request with a timeout. Answers `{ ok, status, value }` and never
+ * throws for an HTTP error, so the caller can say which step failed.
+ */
+async function call(fetchImpl, method, url, body, timeoutMs) {
+  const response = await fetchImpl(url, {
+    method,
+    redirect: 'error',
+    signal: AbortSignal.timeout(timeoutMs),
+    headers: { 'Content-Type': 'application/json' },
+    ...(body ? { body: JSON.stringify(body) } : {})
+  });
+  let data = null;
+  if (response.status !== 204) {
+    const text = await boundedText(response);
+    try {
+      data = text ? JSON.parse(text) : null;
+    } catch {
+      data = null;
+    }
+  }
+  const value = data && typeof data === 'object' ? data.value : undefined;
+  const error = value && typeof value === 'object' && value.error
+    ? `${value.error}${value.message ? `: ${String(value.message).split('\n')[0]}` : ''}`
+    : null;
+  return { ok: response.ok && !error, status: response.status, value, error };
+}
+
+/**
+ * The session ids the server lists, or null when it lists none. `GET /sessions`
+ * is not in the W3C standard; some servers answer it and many do not, so every
+ * failure here is null, never an error.
+ */
+async function listSessions(fetchImpl, endpoint, timeoutMs) {
+  try {
+    const answer = await call(fetchImpl, 'GET', `${endpoint}/sessions`, null, timeoutMs);
+    if (!answer.ok || !Array.isArray(answer.value)) return null;
+    return answer.value.map(entry => entry && (entry.id || entry.sessionId)).filter(id => typeof id === 'string');
+  } catch {
+    return null;
+  }
+}
+
+/**
+ * Which platformName each native driver accepts. Desktop, Unity and Unreal
+ * builds are desktop apps, so a mobile platform there is a config mistake.
+ */
+export function platformMismatch(driverName, platformName) {
+  const platform = String(platformName || '').toLowerCase();
+  if (driverName === 'ios_driver' && platform !== 'ios') return 'iOS';
+  if (driverName === 'android_driver' && platform !== 'android') return 'Android';
+  if (['desktop_driver', 'unity_driver', 'unreal_driver'].includes(driverName) && ['ios', 'android'].includes(platform)) {
+    return 'a desktop platform (for example mac or windows)';
+  }
+  return null;
+}
+
+/** The wording each driver family needs in a fix. */
+export function platformWords(driverName) {
+  if (driverName === 'ios_driver') {
+    return {
+      family: 'ios',
+      build: 'build the app for the simulator first, then point appium:app at the .app it produced',
+      noApp: 'add appium:app with the path to the built .app, or appium:bundleId for an installed one',
+      relative: 'use an absolute path to the built .app so check can confirm it exists',
+      installed: 'the automation server row confirms it; if that row was skipped, install the build on the simulator first',
+      session: 'boot the simulator and check the capabilities: platformName, deviceName, platformVersion and automationName must match a device the server can drive',
+      app: 'point appium:app at the built .app, or appium:bundleId at an app installed on the simulator'
+    };
+  }
+  if (driverName === 'android_driver') {
+    return {
+      family: 'android',
+      build: 'build the .apk first, then point appium:app at it',
+      noApp: 'add appium:app with the path to the built .apk, or appium:appPackage for an installed one',
+      relative: 'use an absolute path to the .apk so check can confirm it exists',
+      installed: 'the automation server row confirms it; if that row was skipped, install the .apk on the device or emulator first',
+      session: 'start the emulator or connect the device and check the capabilities: platformName, deviceName and automationName must match a device the server can drive',
+      app: 'point appium:app at the built .apk, or appium:appPackage at an app installed on the device or emulator'
+    };
+  }
+  return {
+    family: 'desktop',
+    build: 'build the game first, then point the app capability at what it produced',
+    noApp: 'add the capability your automation server uses to name the build (for example appium:app)',
+    relative: 'use an absolute path to the build so check can confirm it exists',
+    installed: 'the automation server row confirms it',
+    session: 'the automation server could not open the build; check the capabilities your server needs for this platform',
+    app: 'the automation server could not open the build; check the app capability points at it'
+  };
+}
+
+/**
+ * Probe the automation server a native run would use.
+ *
+ * Steps, each only after the previous one passed:
+ * 1. read the driver config (loopback endpoint, W3C capabilities);
+ * 2. `GET /status`: the server is reachable;
+ * 3. `POST /session` with the run's own capabilities: a session can be created;
+ * 4. `GET /session/:id/source`: the app the capabilities name opened and
+ *    answers, which is what "the app resolves" means to a persona;
+ * 5. `DELETE /session/:id`: the session is closed again.
+ *
+ * @param {Object} options
+ * @param {string} options.driverName
+ * @param {string|null} options.target the driver config file
+ * @param {typeof fetch} [options.fetch] test seam
+ * @param {{status?: number, session?: number, close?: number}} [options.timeouts]
+ * @param {(line: string) => void} [options.onProgress] told before the slow part starts
+ * @returns {Promise<{
+ *   endpoint: string|null, reachable: boolean|null, session: boolean|null,
+ *   closed: boolean|null, app: boolean|null, appRef: Object|null,
+ *   failed: 'config'|'reachable'|'session'|'session_timeout'|'app'|null, detail: string, leaked: boolean
+ * }>}
+ */
+export async function probeAutomation({ driverName, target, fetch: fetchImpl = globalThis.fetch, timeouts = {}, onProgress = () => {} } = {}) {
+  const result = { endpoint: null, reachable: null, session: null, closed: null, app: null, appRef: null, failed: null, detail: '', leaked: false };
+  const fail = (step, detail) => Object.assign(result, { failed: step, detail });
+
+  if (!target) {
+    return fail('config', `no driver config, so there is no automation server to probe for ${driverName}`);
+  }
+  let config;
+  try {
+    config = await readDriverConfig(target);
+  } catch (error) {
+    return fail('config', `driver config ${target}: ${message(error)}`);
+  }
+  result.endpoint = config.endpoint;
+  result.appRef = appReference(config.capabilities);
+  const wanted = platformMismatch(driverName, config.capabilities.platformName);
+  if (wanted) {
+    return fail('config', `${driverName} needs platformName ${wanted}, and the driver config says ${config.capabilities.platformName}`);
+  }
+  onProgress(`probing ${config.endpoint}, this can take up to 3 minutes on a fresh simulator`);
+
+  try {
+    const status = await call(fetchImpl, 'GET', `${config.endpoint}/status`, null, timeouts.status ?? STATUS_TIMEOUT_MS);
+    if (!status.ok) return Object.assign(fail('reachable', `${config.endpoint}/status answered ${status.error || `HTTP ${status.status}`}`), { reachable: false });
+    if (status.value && status.value.ready === false) {
+      return Object.assign(fail('reachable', `${config.endpoint} answered but says it is not ready${status.value.message ? `: ${status.value.message}` : ''}`), { reachable: false });
+    }
+    result.reachable = true;
+  } catch (error) {
+    result.reachable = false;
+    return fail('reachable', `nothing answered at ${config.endpoint} (${message(error)})`);
+  }
+
+  let sessionId = null;
+  // Taken only so a timed out create can find the session it may have left
+  // behind. Servers without `GET /sessions` answer null and nothing is closed.
+  const before = await listSessions(fetchImpl, config.endpoint, timeouts.status ?? STATUS_TIMEOUT_MS);
+  try {
+    const created = await call(
+      fetchImpl,
+      'POST',
+      `${config.endpoint}/session`,
+      { capabilities: { alwaysMatch: config.capabilities, firstMatch: [{}] } },
+      timeouts.session ?? SESSION_TIMEOUT_MS
+    );
+    sessionId = created.ok && created.value && typeof created.value.sessionId === 'string' ? created.value.sessionId : null;
+    if (!sessionId) {
+      result.session = false;
+      return fail('session', `the server did not create a session: ${created.error || `HTTP ${created.status}`}`);
+    }
+    result.session = true;
+  } catch (error) {
+    result.session = false;
+    if (error && error.name === 'TimeoutError') {
+      // The server may finish creating the session after the abort. Close any
+      // session that appeared since the list above, best effort.
+      const after = before ? await listSessions(fetchImpl, config.endpoint, timeouts.status ?? STATUS_TIMEOUT_MS) : null;
+      const strays = after ? after.filter(id => !before.includes(id)) : [];
+      for (const id of strays) {
+        try {
+          await call(fetchImpl, 'DELETE', `${config.endpoint}/session/${encodeURIComponent(id)}`, null, timeouts.close ?? CLOSE_TIMEOUT_MS);
+        } catch {
+          // Best effort: the detail already says a session may still be open.
+        }
+      }
+      result.leaked = true;
+      return fail('session_timeout', `creating a session timed out; a session may still be open on the server; restart it or wait before retrying${strays.length ? ` (asked the server to close ${strays.length} session${strays.length === 1 ? '' : 's'} that appeared meanwhile)` : ''}`);
+    }
+    return fail('session', `creating a session failed (${message(error)})`);
+  }
+
+  const sessionUrl = `${config.endpoint}/session/${encodeURIComponent(sessionId)}`;
+  try {
+    if (!result.appRef) {
+      result.app = false;
+      fail('app', `the capabilities name no app (one of ${APP_KEYS.join(', ')}), so a session opens nothing for a persona to play`);
+    } else {
+      const source = await call(fetchImpl, 'GET', `${sessionUrl}/source`, null, timeouts.status ?? STATUS_TIMEOUT_MS);
+      result.app = source.ok;
+      if (!source.ok) fail('app', `${result.appRef.value} did not answer in the session: ${source.error || `HTTP ${source.status}`}`);
+    }
+  } catch (error) {
+    result.app = false;
+    fail('app', `reading the app in the session failed (${message(error)})`);
+  } finally {
+    try {
+      const closed = await call(fetchImpl, 'DELETE', sessionUrl, null, timeouts.close ?? CLOSE_TIMEOUT_MS);
+      result.closed = closed.ok;
+    } catch {
+      result.closed = false;
+    }
+  }
+  if (result.failed) return result;
+
+  result.detail = `${config.endpoint} reachable, session created and ${result.closed ? 'closed' : 'NOT closed'}, app ${result.appRef.value} opened`;
+  return result;
+}
+
+/**
+ * What the native build is, from the driver config alone. Nothing is launched.
+ *
+ * @param {Object} options
+ * @param {string|null} options.target
+ * @param {string} [options.repoRoot]
+ * @returns {Promise<{status: 'pass'|'warn'|'fail', detail: string, fix: string|null, ref: string|null}>}
+ */
+export async function inspectNativeBuild({ target, repoRoot = process.cwd(), driverName = 'ios_driver' } = {}) {
+  const words = platformWords(driverName);
+  if (!target) {
+    return {
+      status: 'fail',
+      detail: 'no driver config: a native run has nothing to open',
+      fix: 'write a driver config (see docs/NATIVE_RUNNER.md) and pass --driver-config <file>, or set build to { "kind": "driver_config", "ref": "<file>" } in .ravensight/config.json',
+      ref: null
+    };
+  }
+  let config;
+  try {
+    config = await readDriverConfig(path.resolve(repoRoot, target));
+  } catch (error) {
+    return { status: 'fail', detail: `driver config ${target}: ${message(error)}`, fix: 'fix the driver config; see docs/NATIVE_RUNNER.md', ref: null };
+  }
+  const app = appReference(config.capabilities);
+  if (!app) {
+    return {
+      status: 'fail',
+      detail: `${target} names no app (one of ${APP_KEYS.join(', ')})`,
+      fix: words.noApp,
+      ref: null
+    };
+  }
+  if (app.kind === 'id') {
+    return {
+      status: 'warn',
+      detail: `${app.key} ${app.value}: an installed app, which only the automation server can confirm`,
+      fix: words.installed,
+      ref: app.value
+    };
+  }
+  if (app.kind === 'url' || !path.isAbsolute(app.value)) {
+    return {
+      status: 'warn',
+      detail: `${app.key} ${app.value}: resolved by the automation server, not checked on disk`,
+      fix: words.relative,
+      ref: app.value
+    };
+  }
+  try {
+    await fs.stat(app.value);
+  } catch {
+    return {
+      status: 'fail',
+      detail: `${app.key} ${app.value} does not exist`,
+      fix: words.build,
+      ref: app.value
+    };
+  }
+  return { status: 'pass', detail: `${app.key} ${app.value} (from ${target})`, fix: null, ref: app.value };
+}

package/src/run/dropped.js

@@ -108,8 +108,8 @@ const LIST_ITEM = /^\s*(?:[-*+]|\d+[.)])\s+/;
  * @param {{unidentified?: number}} [opts] a count the CLI could not attribute
  * @returns {string}
  */
-export function rewriteMarkdown(markdown, dropped, kept, { unidentified = 0 } = {}) {
-  const lines = withoutDroppedSection(String(markdown || '').replace(/\s+$/, '')).split('\n');
+export function rewriteMarkdown(markdown, dropped, kept, { unidentified = 0, heading = DROPPED_HEADING, explanation = null, emptyNote = null } = {}) {
+  const lines = withoutDroppedSection(String(markdown || '').replace(/\s+$/, ''), heading).split('\n');
   const droppedIds = dropped.map(finding => String(finding.id)).filter(Boolean);
   const keptIds = kept.map(finding => String(finding.id)).filter(Boolean);
   const names = (line, id) => new RegExp(`(^|[^A-Za-z0-9])${escapeRegExp(id)}(?![0-9])`).test(line);
@@ -126,7 +126,7 @@ export function rewriteMarkdown(markdown, dropped, kept, { unidentified = 0 } =
         out.pop();
         blanks += 1;
       }
-      out.push('- Nothing here that the server kept. See the dropped findings below.');
+      out.push(emptyNote || '- Nothing here that the server kept. See the dropped findings below.');
       for (let i = 0; i < blanks; i += 1) out.push('');
     }
   };
@@ -148,10 +148,10 @@ export function rewriteMarkdown(markdown, dropped, kept, { unidentified = 0 } =
   }
   closeSection();
 
-  const tail = ['', DROPPED_HEADING];
+  const tail = ['', heading];
   if (dropped.length > 0) {
     tail.push(
-      'The server keeps a finding only when at least one of its evidence references points at a file this run uploaded. These did not, so the dashboard does not show them and they are not counted. They are kept here so you can judge them yourself.',
+      explanation || 'The server keeps a finding only when at least one of its evidence references points at a file this run uploaded. These did not, so the dashboard does not show them and they are not counted. They are kept here so you can judge them yourself.',
       ''
     );
     for (const finding of dropped) {
@@ -176,9 +176,9 @@ export function rewriteMarkdown(markdown, dropped, kept, { unidentified = 0 } =
  * @param {string} markdown
  * @returns {string}
  */
-function withoutDroppedSection(markdown) {
+function withoutDroppedSection(markdown, heading = DROPPED_HEADING) {
   const lines = markdown.split('\n');
-  const start = lines.findIndex(line => line.trim() === DROPPED_HEADING);
+  const start = lines.findIndex(line => line.trim() === heading);
   if (start === -1) return markdown;
   let end = lines.findIndex((line, index) => index > start && SECTION.test(line));
   if (end === -1) end = lines.length;