ravensight-playtest 0.5.2 → 0.7.0

package/src/commands/validate.js

@@ -0,0 +1,414 @@
+import { randomUUID } from 'node:crypto';
+import { execFile } from 'node:child_process';
+import { mkdir, writeFile } from 'node:fs/promises';
+import path from 'node:path';
+import { promisify } from 'node:util';
+
+import { capabilityLive } from '../run/acceptance.js';
+import { finishJob } from '../run/settle.js';
+import { getDeps } from '../run/deps.js';
+import { DIMENSION_3D, dimensionGate } from '../run/dimension.js';
+import { GodotDriver } from '../run/drivers/godot.js';
+import EXIT from '../run/exit.js';
+import { jobDir, runDir as runDirFor } from '../run/paths.js';
+import { checkoutRef } from '../run/replay.js';
+import * as stateFile from '../run/state.js';
+import { seedFrom } from '../run/trace.js';
+import { playJob, explainRegisterRefusal, resolvePersonas, resolveSetup, formatCents, startBalanceLine, startBalanceShort } from './run.js';
+import { unknownPersonaMessage, unknownPersonas } from '../runDefaults.js';
+import { loadLocalBrief } from '../briefLocal.js';
+
+const execFileAsync = promisify(execFile);
+
+/**
+ * `ravensight-playtest validate <branch>`: N persona runs on the base and N on
+ * the head of one Godot project, and what changed between them.
+ *
+ * Every commit is extracted to a temp directory, so the developer's checkout is
+ * not touched, and both must import and boot BEFORE anything is priced: a head
+ * that does not run is a free answer. The whole declared plan is one charge at
+ * register, so a balance that runs out cannot leave one side short. A pair is the
+ * same persona on the same seed, one run on each side.
+ *
+ * The verdict is the server's, computed from what the runs reported. This command
+ * prints it and never words it more strongly than the server did.
+ */
+
+export const DEFAULT_RUNS_PER_SIDE = 3;
+export const MAX_RUNS_PER_SIDE = 10;
+
+const CAPABILITY = 'validate';
+
+/** The default base: what origin says its default branch is, else main. */
+export async function defaultBase(repoRoot) {
+  try {
+    const { stdout } = await execFileAsync('git', ['-C', repoRoot, 'symbolic-ref', '--short', 'refs/remotes/origin/HEAD']);
+    return stdout.trim().replace(/^origin\//, '') || 'main';
+  } catch {
+    return 'main';
+  }
+}
+
+/**
+ * The two commits import, boot and stop, free, before any price is shown.
+ *
+ * @returns {Promise<string|null>} why not, or null when both run
+ */
+export async function bootProblem(checked, { godotPath, makeDriver, headless = true }) {
+  const driver = makeDriver({ projectDir: checked.projectDir, godotPath, headless, lockstep: true, seed: 1, recordTrace: false });
+  try {
+    await driver.launch({});
+    return null;
+  } catch (error) {
+    return String(error && error.message).slice(0, 500);
+  } finally {
+    await driver.stop({ reason: 'preflight' }).catch(() => {});
+  }
+}
+
+function number(value) {
+  if (value === null || value === undefined) return 'n/a';
+  return Number.isInteger(value) ? String(value) : value.toFixed(2);
+}
+
+/** The server's comparison, as lines. Its notes are its words, not this command's. */
+export function reportLines(report) {
+  const lines = [
+    `Base ${report.base.ref} (${report.base.sha.slice(0, 12)}): ${report.base.usable} usable of ${report.base.planned} runs.`,
+    `Head ${report.head.ref} (${report.head.sha.slice(0, 12)}): ${report.head.usable} usable of ${report.head.planned} runs.`
+  ];
+  for (const metric of report.metrics) {
+    lines.push(`${metric.label}: base ${number(metric.base.value)} (n=${metric.base.n}), head ${number(metric.head.value)} (n=${metric.head.n}), change ${number(metric.delta)}. ${metric.note}`);
+  }
+  lines.push(report.verdict_note);
+  return lines;
+}
+
+/**
+ * @param {string} [head] the branch to validate, default HEAD
+ * @param {Object} flags
+ * @param {Object} [injected] test seam
+ * @returns {Promise<number>} an exit code
+ */
+export async function validateCommand(head, flags = {}, injected = {}) {
+  const deps = injected.deps || (await getDeps());
+  const log = injected.log || (deps.ui && deps.ui.info ? message => deps.ui.info(message) : message => console.log(message));
+  const options = {
+    apiUrl: flags.apiUrl,
+    repo: flags.repoRoot,
+    game: flags.game,
+    yes: flags.yes,
+    headless: flags.headless === true,
+    godotPath: flags.godotPath,
+    concurrency: flags.concurrency,
+    personas: flags.personas,
+    uploadTrace: true,
+    skipAggregate: true
+  };
+
+  const setup = await resolveSetup(deps, options, log);
+  if (setup.code !== undefined) return setup.code;
+  const { context, api, gameId, apiUrl, repoRoot, cliVersion } = setup;
+
+  // A resume replays the original register: same key, same plan, no second charge.
+  const resumed = flags.resume ? await stateFile.load(repoRoot, flags.resume) : null;
+  if (flags.resume && !resumed?.validate) {
+    log(`There is no validate journal for ${flags.resume} in this repository.`);
+    return EXIT.ENVIRONMENT;
+  }
+
+  const size = resumed ? resumed.validate.register.validate.runs_per_side : Number(flags.runs ?? DEFAULT_RUNS_PER_SIDE);
+  if (!Number.isInteger(size) || size < 1 || size > MAX_RUNS_PER_SIDE) {
+    log(`--runs must be a whole number from 1 to ${MAX_RUNS_PER_SIDE}. Nothing was charged.`);
+    return EXIT.ENVIRONMENT;
+  }
+
+  let labs = null;
+  try { labs = await api.labs.get(gameId); } catch { labs = null; }
+  if (!capabilityLive(labs, CAPABILITY)) {
+    log('Validate changes is part of Ravensight Labs (beta), and it is not on for this game. Turn Labs on from the Labs page in the dashboard. Nothing was charged.');
+    return EXIT.ENVIRONMENT;
+  }
+
+  const project = path.resolve(repoRoot, flags.godotProject || context.config?.godot_project || '.');
+  const subdir = path.relative(repoRoot, project);
+  if (subdir.startsWith('..')) {
+    log('The Godot project has to be inside this repository so its commits can be extracted. Nothing was charged.');
+    return EXIT.ENVIRONMENT;
+  }
+
+  const baseRef = resumed ? resumed.validate.register.validate.base_ref : (flags.base || await defaultBase(repoRoot));
+  const headRef = resumed ? resumed.validate.register.validate.head_ref : (head || 'HEAD');
+  const checkout = injected.checkoutRef || checkoutRef;
+  const makeProbe = injected.makeDriver || (driverOptions => new GodotDriver(driverOptions));
+
+  const pack = await deps.packs.get({ api, gameId, modules: ['validate', 'persona_playtest'] });
+  const personaSlugs = resumed
+    ? resumed.validate.register.personas
+    : resolvePersonas({ personas: flags.personas }, context.config, pack);
+  const unknown = unknownPersonas(personaSlugs, pack.personas || []);
+  if (unknown.length > 0) {
+    log(`${unknownPersonaMessage(unknown, pack.personas || [])} Nothing was registered or charged.`);
+    return EXIT.ENVIRONMENT;
+  }
+  if (personaSlugs.length > size) {
+    log(`${personaSlugs.length} personas need at least ${personaSlugs.length} runs a side. Nothing was charged.`);
+    return EXIT.ENVIRONMENT;
+  }
+  const personas = personaSlugs.map(slug => (pack.personas || []).find(persona => persona.slug === slug));
+
+  const brief = await deps.brief.get({ api, gameId });
+  if (!brief || brief.status !== 'complete') {
+    log('A persona run needs a complete expectations brief. Run "ravensight-playtest brief" first. Nothing was charged.');
+    return EXIT.ENVIRONMENT;
+  }
+
+  let baseCheckout = null;
+  let headCheckout = null;
+  try {
+    log(`Extracting ${baseRef} and ${headRef} and checking that both start. This costs nothing.`);
+    try {
+      baseCheckout = await checkout({ repoRoot, ref: baseRef, projectSubdir: subdir, godotPath: flags.godotPath });
+      headCheckout = await checkout({ repoRoot, ref: headRef, projectSubdir: subdir, godotPath: flags.godotPath });
+    } catch (error) {
+      log(`${String(error && error.message)} Nothing was charged.`);
+      return EXIT.ENVIRONMENT;
+    }
+    if (baseCheckout.sha === headCheckout.sha) {
+      log(`${headRef} and ${baseRef} are the same commit, so there is nothing to compare. Nothing was charged.`);
+      return EXIT.ENVIRONMENT;
+    }
+    // Free, before a price. A resumed job was already asked for and paid for.
+    if (!resumed) {
+      for (const [name, checked] of [[baseRef, baseCheckout], [headRef, headCheckout]]) {
+        const { refusal, dimension } = await dimensionGate({ projectDir: checked.projectDir, alpha3d: Boolean(flags.alpha3d), name: 'Validate changes' });
+        if (refusal) {
+          log(`${name}: ${refusal} Nothing was charged.`);
+          return EXIT.ENVIRONMENT;
+        }
+        if (dimension === DIMENSION_3D && name === headRef) log('3D Alpha: a proof of concept on one template. It is priced as any validation is, and a persona plays with keys and input actions, never mouse look.');
+      }
+    }
+    for (const [name, checked] of [[baseRef, baseCheckout], [headRef, headCheckout]]) {
+      const problem = await bootProblem(checked, { godotPath: flags.godotPath, makeDriver: makeProbe, headless: true });
+      if (problem) {
+        log(`${name} does not start: ${problem}`);
+        log('Nothing was charged.');
+        return EXIT.FAILED;
+      }
+    }
+
+    const body = resumed ? resumed.validate.register : {
+      modules: ['validate'],
+      personas: personaSlugs,
+      driver: 'godot_driver',
+      cli_version: cliVersion,
+      brief_version: brief.version ?? null,
+      build: { kind: 'godot_project', ref: subdir || '.' },
+      repo: { commit: headCheckout.sha, dirty: false, branch: headRef.slice(0, 200) },
+      pack_versions: { pack: pack.pack_version },
+      validate: { base_ref: baseRef.slice(0, 200), base_sha: baseCheckout.sha, head_ref: headRef.slice(0, 200), head_sha: headCheckout.sha, runs_per_side: size }
+    };
+
+    let idempotencyKey = resumed ? resumed.idempotency_key : randomUUID();
+    if (!resumed) {
+      let estimate;
+      try {
+        estimate = await api.jobs.estimate(gameId, { modules: ['validate'], personas: personaSlugs, validate: { runs_per_side: size } });
+      } catch (error) {
+        // Labs went off after the status read. A 403 is not a login problem, so it is
+        // said as the register says it, rather than falling through to the login advice.
+        if (error && error.code === 'labs_not_enabled') {
+          log(String(error.message));
+          return EXIT.ENVIRONMENT;
+        }
+        throw error;
+      }
+      const line = estimate.breakdown?.[0];
+      if (!line || line.unit !== 'playtest_run' || line.count !== size * 2) {
+        log('This server did not quote every run of the plan. Nothing was registered or charged. Update the server and CLI together.');
+        return EXIT.ENVIRONMENT;
+      }
+      log(`This validation: ${size} runs on ${baseRef} and ${size} on ${headRef}, ${formatCents(estimate.estimate_cents)} in total. Balance: ${formatCents(estimate.balance_cents)}.`);
+      if (size < MIN_RUNS_FOR_A_VERDICT) log(`With fewer than ${MIN_RUNS_FOR_A_VERDICT} runs a side the result is a direction, never a verdict. --runs ${MIN_RUNS_FOR_A_VERDICT} costs ${formatCents((estimate.estimate_cents / size) * MIN_RUNS_FOR_A_VERDICT)}.`);
+      log('A run that does not earn its unit is refunded and does not count. If one side ends with no usable run there is nothing to compare, and the runs on the other side that played and reported keep their units: those runs happened and their reports are real.');
+      const floorLine = startBalanceLine(estimate);
+      if (floorLine) log(floorLine);
+      if (estimate.ok === false && floorLine) {
+        log(startBalanceShort(estimate));
+        return EXIT.ENVIRONMENT;
+      }
+      if (estimate.ok === false) {
+        log(`That is more than the balance covers. Top up at ${estimate.topup_url || 'ravensight.io'} and run this again. Nothing was charged.`);
+        return EXIT.ENVIRONMENT;
+      }
+      if (estimate.wall_warning) log(estimate.wall_warning);
+      const confirm = injected.confirm || (deps.ui && deps.ui.confirm);
+      if (!flags.yes && confirm && !await confirm(`Validate for ${formatCents(estimate.estimate_cents)}?`, { yes: flags.yes })) {
+        log('Nothing was charged.');
+        return EXIT.CANCELED;
+      }
+      body.confirm_price_cents = estimate.estimate_cents;
+    }
+
+    let registered;
+    try {
+      registered = await api.jobs.register(gameId, body, { idempotencyKey });
+    } catch (error) {
+      if (error && error.code === 'labs_not_enabled') {
+        log(String(error.message));
+        return EXIT.ENVIRONMENT;
+      }
+      if (error && error.code === 'funding_required') {
+        log(`${String(error.message)} Top up at ${error.body?.topup_url || 'ravensight.io'}.`);
+        return EXIT.ENVIRONMENT;
+      }
+      const handled = explainRegisterRefusal(error, log);
+      if (handled !== null) return handled;
+      throw error;
+    }
+
+    const jobId = registered.job_id;
+    await mkdir(jobDir(repoRoot, jobId), { recursive: true });
+    const journal = resumed || stateFile.newState({
+      jobId,
+      gameId,
+      idempotencyKey,
+      cliVersion,
+      packVersion: pack.pack_version,
+      modules: ['validate'],
+      driver: 'godot_driver',
+      target: subdir || '.',
+      priceCents: registered.price && registered.price.price_cents,
+      options: { uploadTrace: true }
+    });
+    journal.validate = { register: body };
+    const addEntries = runs => {
+      for (const entry of runs) {
+        if (!journal.runs[entry.run_id]) journal.runs[entry.run_id] = stateFile.newRunEntry({ runId: entry.run_id, module: entry.module, persona: entry.persona });
+      }
+    };
+    addEntries(registered.runs || []);
+    await stateFile.save(repoRoot, journal);
+    log(`job ${jobId}${registered.replayed ? ' (replayed)' : ''}: ${formatCents(registered.price && registered.price.price_cents)} charged.`);
+
+    const prepare = runs => runs
+      .filter(entry => entry.module === 'persona_playtest')
+      .map(entry => {
+        if (!entry.validation) throw new Error('This server did not say which side a run plays. Update the server and CLI together.');
+        return {
+          ...entry,
+          target: entry.validation.side === 'base' ? baseCheckout.projectDir : headCheckout.projectDir,
+          trace_header: { commit: entry.validation.side === 'base' ? baseCheckout.sha : headCheckout.sha, side: entry.validation.side, pair: entry.validation.pair },
+          // The same seed for both sides of a pair, which is what makes the pair one.
+          seed: seedFrom(`${jobId}:${entry.validation.pair}`)
+        };
+      });
+
+    let localBrief = {};
+    try {
+      localBrief = await loadLocalBrief({ repoRoot });
+    } catch (error) {
+      log(`${String(error && error.message)} The runs go ahead without expected_noise and test_account.`);
+    }
+
+    const playAll = async runs => (injected.playJob || playJob)({
+      deps,
+      api,
+      apiUrl,
+      gameId,
+      jobId,
+      repoRoot,
+      runs,
+      pack,
+      personas,
+      driverName: 'godot_driver',
+      target: baseCheckout.projectDir,
+      options,
+      journal,
+      config: { ...context.config, game_version: context.config?.game_version },
+      brief,
+      localBrief,
+      injected,
+      log,
+      finish: false
+    });
+
+    let played = await playAll(prepare(registered.runs || []));
+    if (played === EXIT.CANCELED) {
+      log(`job ${jobId} was interrupted. "ravensight-playtest validate --resume ${jobId}" carries on with no second charge.`);
+      return played;
+    }
+
+    let report = await api.jobs.validation(gameId, jobId);
+
+    // MORE PAIRS ARE A SEPARATE PURCHASE, asked for up front with --extend-to and
+    // bought only if the first answer could not reach a conclusion.
+    const extendTo = Number(flags.extendTo);
+    if (report.too_few && Number.isInteger(extendTo) && extendTo > size) {
+      const pairs = Math.min(extendTo, MAX_RUNS_PER_SIDE) - size;
+      const price = formatCents(pairs * 2 * (registered.price.price_cents / (size * 2)));
+      const confirm = injected.confirm || (deps.ui && deps.ui.confirm);
+      if (flags.yes || !confirm || await confirm(`Add ${pairs} more run(s) a side for ${price}?`, { yes: flags.yes })) {
+        try {
+          const added = await api.runs.addValidatePairs(gameId, jobId, pairs);
+          addEntries(added.runs || []);
+          await stateFile.save(repoRoot, journal);
+          log(`Added ${pairs} run(s) a side: ${formatCents(added.price && added.price.price_cents)} charged.`);
+          played = await playAll(prepare(added.runs || []));
+          if (played === EXIT.CANCELED) return played;
+          report = await api.jobs.validation(gameId, jobId);
+        } catch (error) {
+          log(`More runs could not be added: ${String(error && error.message)}. The comparison stands on the runs it has.`);
+        }
+      }
+    }
+
+    await closeComparison({ api, deps, gameId, jobId, repoRoot, report, runs: registered.runs || [], log });
+    const finished = await finishJob({ api, gameId, jobId, body: { reason: '' }, log });
+    if (finished) {
+      journal.finished = true;
+      await stateFile.save(repoRoot, journal);
+      log(`job ${jobId} ${finished.job ? finished.job.state : 'finished'}: refunded ${formatCents(finished.refunded_cents)}.`);
+    }
+
+    if (flags.json) {
+      (deps.ui && deps.ui.json ? deps.ui.json : value => console.log(JSON.stringify(value)))({ job_id: jobId, ...report, refunded_cents: finished ? finished.refunded_cents : null });
+    } else {
+      for (const line of reportLines(report)) log(line);
+      log(`Local reports and traces: ${jobDir(repoRoot, jobId)}. "ravensight-playtest replay <runId> --ref <branch>" plays one again for free.`);
+    }
+    if (report.verdict === 'not_comparable') return EXIT.FAILED;
+    if (flags.failOnRegression && (report.verdict === 'regressed' || report.verdict === 'mixed')) return EXIT.FAILED;
+    return EXIT.OK;
+  } finally {
+    await baseCheckout?.cleanup();
+    await headCheckout?.cleanup();
+  }
+}
+
+/** A side needs this many usable runs before the server will call a difference. */
+const MIN_RUNS_FOR_A_VERDICT = 5;
+
+/**
+ * The job's own run: it holds the comparison as a file beside the persona runs.
+ * It costs nothing and calls nothing, and the answer the dashboard shows is the
+ * server's own computation rather than this upload, so a failure here is logged
+ * and does not change the result.
+ */
+async function closeComparison({ api, deps, gameId, jobId, repoRoot, report, runs, log }) {
+  const own = runs.find(entry => entry.module === 'validate');
+  if (!own) return;
+  const directory = runDirFor(repoRoot, jobId, own.run_id);
+  try {
+    await mkdir(directory, { recursive: true });
+    await writeFile(path.join(directory, 'validation-report.json'), `${JSON.stringify(report, null, 2)}\n`, { mode: 0o600 });
+    await writeFile(path.join(directory, 'validation-report.md'), String(await api.jobs.validationMarkdown(gameId, jobId)), { mode: 0o600 });
+    for (const state of ['launching', 'playing', 'reporting', 'uploading']) {
+      await api.runs.transition(gameId, jobId, own.run_id, state, { reason: '' });
+    }
+    await deps.finalizeRun(directory, { api, gameId, jobId, runId: own.run_id, body: { state: 'succeeded', actions_taken: 0 } });
+  } catch (error) {
+    log(`The comparison file was not stored (${String(error && error.message)}). The comparison itself is unaffected.`);
+  }
+}