ravensight-playtest 0.3.1 → 0.3.2

package/src/commands/run.js

@@ -14,6 +14,9 @@ import { readBridgeConfig } from "../run/drivers/stdio.js";
 import { WEBDRIVER_NAMES, readDriverConfig } from '../run/drivers/webdriver.js';
 import { automationChecks } from './check.js';
 import { runPersona } from '../run/personaLoop.js';
+import { loadLocalBrief, localBriefProblems } from '../briefLocal.js';
+import { DEFAULT_PERSONAS as RUN_DEFAULT_PERSONAS, RUN_MODULES, SIGN_UP_ACTIONS, defaultPersonasFor, describeBreakdown, personaCount, unknownPersonaMessage, unknownPersonas } from '../runDefaults.js';
+import { bold } from '../ui/index.js';
 import { reconcileDroppedFindings } from '../run/dropped.js';
 import * as stateFile from '../run/state.js';
 import { emitRunEvents } from '../run/synthetic.js';
@@ -44,8 +47,8 @@ import { emitRunEvents } from '../run/synthetic.js';
  * nothing to point at, 4 auth, 5 the proxy never answered, 10 cancelled.
  */
 
-/** Personas used when the developer names none. Spec 17 defaults to 2. */
-export const DEFAULT_PERSONAS = Object.freeze(['curious-kid', 'grumpy-veteran']);
+/** Personas used when the developer names none: one since 0.3.2 (see ../runDefaults.js). */
+export const DEFAULT_PERSONAS = RUN_DEFAULT_PERSONAS;
 
 /** Parallel runs. Each Chromium is 0.5 to 1.5 GB, so 2 is the honest default. */
 export const DEFAULT_CONCURRENCY = 2;
@@ -132,15 +135,26 @@ async function run(flags, injected, deps, log) {
       return EXIT.FAILED;
     }
   }
-  const modules = ['persona_playtest', 'aggregate_report'];
+  // The build's own health, before anything is priced or charged: the
+  // Snowball round paid for a run against an API that was down.
+  if (!options.skipPreflight) {
+    const health = await healthPreflight({ config: context.config, driverName, target, fetchImpl: injected.fetch, log });
+    if (health && !health.ok) {
+      log(`Preflight failed: ${health.detail}.`);
+      log('No job was registered and nothing was reserved. Start the build or its API and run this again, or pass --skip-preflight to register anyway.');
+      return EXIT.FAILED;
+    }
+  }
+  const modules = [...RUN_MODULES];
   const pack = await deps.packs.get({ api, gameId, modules });
   if (options.controller === 'jev' && !pack.routing?.action_controllers?.includes('jev')) { log('This server does not currently support the requested action controller. No job was registered.'); return EXIT.ENVIRONMENT; }
   const personaSlugs = resolvePersonas(options, context.config, pack);
-  const personas = personaSlugs.map(slug => {
-    const found = (pack.personas || []).find(persona => persona.slug === slug);
-    if (!found) throw new Error(`unknown persona "${slug}"; available: ${(pack.personas || []).map(entry => entry.slug).join(', ')}`);
-    return found;
-  });
+  const unknownSlugs = unknownPersonas(personaSlugs, pack.personas || []);
+  if (unknownSlugs.length > 0) {
+    log(`${unknownPersonaMessage(unknownSlugs, pack.personas || [])} Nothing was registered or charged.`);
+    return EXIT.ENVIRONMENT;
+  }
+  const personas = personaSlugs.map(slug => (pack.personas || []).find(persona => persona.slug === slug));
 
   // The same ceiling the server's `personaListRefusal` enforces, checked here
   // first so a list past it never reaches the network: no estimate, no brief
@@ -168,8 +182,43 @@ async function run(flags, injected, deps, log) {
     return EXIT.ENVIRONMENT;
   }
 
-  const estimate = await api.jobs.estimate(gameId, { modules, personas: personaSlugs });
-  log(`${personas.length} persona(s) on ${driverName}: ${formatCents(estimate.estimate_cents)}. Balance: ${formatCents(estimate.balance_cents)}.`);
+  let localBrief = {};
+  try {
+    localBrief = await loadLocalBrief({ repoRoot });
+  } catch (error) {
+    log(`${String(error && error.message)} The run goes ahead without expected_noise and test_account.`);
+  }
+
+  const runLength = options.runLength === 'long' ? 'long' : 'standard';
+  let estimate;
+  try {
+    estimate = await api.jobs.estimate(gameId, { modules, personas: personaSlugs, runLength });
+  } catch (error) {
+    if (error && error.code === 'invalid_estimate') {
+      log(`The server refused to price this job: ${String(error.message || 'invalid estimate')}${runLength === 'long' ? ' This server may not price long runs yet; run without --long.' : ''} Nothing was registered or charged.`);
+      return EXIT.ENVIRONMENT;
+    }
+    throw error;
+  }
+  // A long run is its own unit on the server. An older server that ignores
+  // run_length prices a standard run, and confirming that would buy a run
+  // the CLI then plays as long.
+  if (runLength === 'long' && !(Array.isArray(estimate.breakdown) && estimate.breakdown.some(line => line && line.unit === 'playtest_run_long'))) {
+    log('This server does not price long runs; nothing was registered or charged. Run without --long.');
+    return EXIT.ENVIRONMENT;
+  }
+  if (runLength === 'long') {
+    const factor = longRunFactor({ estimate, pack });
+    options.longRunFactor = factor.factor;
+    if (factor.note) log(factor.note);
+  }
+  // Said before any charge, even with --yes, in bold: how many personas and
+  // what they cost in total, each unit with its own price.
+  const units = describeBreakdown(estimate.breakdown, formatCents);
+  log(bold(`This run: ${personaCount(personas.length)} (${personaSlugs.join(', ')}) on ${driverName}, ${formatCents(estimate.estimate_cents)} in total${units ? `: ${units}` : ''}.`));
+  log(`Balance: ${formatCents(estimate.balance_cents)}.`);
+  if (runLength === 'long') log(longRunLine(options.longRunFactor));
+  for (const line of actionBudgetLines({ options, personas, pack, localBrief })) log(line);
   if (estimate.ok === false) {
     log(`That is more than the balance covers. Top up at ${estimate.topup_url || 'ravensight.io'} and run this again.`);
     // Nothing is wrong with the request; the account needs money. Spec 17's
@@ -203,6 +252,7 @@ async function run(flags, injected, deps, log) {
       build: { kind: driverName === 'playwright_web' ? 'url' : (WEBDRIVER_NAMES.includes(driverName) || driverName === 'cli_stdio') ? 'driver_config' : 'godot_project', ref: target },
       repo: { commit: (context.config && context.config.commit) || '', dirty: Boolean(context.config && context.config.dirty) },
       max_actions: Number.isInteger(options.maxActions) ? options.maxActions : null,
+      ...(runLength === 'long' ? { run_length: 'long' } : {}),
       pack_versions: { pack: pack.pack_version }
     }, { idempotencyKey });
   } catch (error) {
@@ -227,6 +277,8 @@ async function run(flags, injected, deps, log) {
     priceCents: registered.price && registered.price.price_cents,
     options: {
       maxActions: options.maxActions ?? null,
+      runLength,
+      ...(runLength === 'long' ? { longRunFactor: options.longRunFactor } : {}),
       controller: options.controller || 'standard',
       concurrency: options.concurrency ?? DEFAULT_CONCURRENCY,
       uploadVideo: Boolean(options.uploadVideo),
@@ -257,6 +309,7 @@ async function run(flags, injected, deps, log) {
     journal,
     config: context.config,
     brief,
+    localBrief,
     injected,
     log
   });
@@ -371,6 +424,7 @@ export async function playJob({
   journal,
   config,
   brief = null,
+  localBrief: givenLocalBrief = null,
   injected = {},
   log = () => {},
   finish = true
@@ -382,6 +436,17 @@ export async function playJob({
 
   const directory = jobDir(repoRoot, jobId);
   const personaRuns = runs.filter(entry => entry.module === 'persona_playtest');
+  // expected_noise and test_account, which never go to the server's brief.
+  let localBrief = givenLocalBrief || injected.localBrief || null;
+  if (!localBrief) {
+    try {
+      localBrief = await loadLocalBrief({ repoRoot });
+    } catch (error) {
+      log(`${String(error && error.message)} The run goes ahead without expected_noise and test_account.`);
+      localBrief = {};
+    }
+  }
+  for (const problem of localBriefProblems(localBrief)) log(`.ravensight/brief.local.json: ${problem}; that field is ignored.`);
   const aggregateRun = runs.find(entry => entry.module === 'aggregate_report') || null;
 
   const results = [];
@@ -423,6 +488,7 @@ export async function playJob({
           inFlight,
           makeDriver,
           brief,
+          localBrief,
           log
         }));
       }
@@ -517,6 +583,7 @@ export async function playPersonaRun({
   inFlight = null,
   makeDriver,
   brief = null,
+  localBrief = {},
   log
 }) {
   const runId = entry.run_id;
@@ -525,7 +592,7 @@ export async function playPersonaRun({
   if (inFlight) inFlight.set(runId, { state: 'queued' });
 
   const skill = skillFor(pack, 'persona_playtest');
-  const maxActions = resolveMaxActions(options, persona, pack);
+  const maxActions = resolveMaxActions(options, persona, pack, entry);
   // Headless is right for a browser and wrong for Godot: a headless Godot
   // routes no GUI input, so it is a smoke-test mode rather than a way to play.
   const headless = driverName === 'godot_driver' ? options.headless === true : options.headed !== true;
@@ -581,6 +648,7 @@ export async function playPersonaRun({
       // The brief this command fetched and checked is complete, not the
       // pack's copy, which a pack bundle does not always carry.
       brief: brief || pack.brief || null,
+      localBrief: usableLocalBrief(localBrief),
       intentNotes: pack.intent_notes || [],
       schemas: pack.schemas || [],
       maxActions,
@@ -1260,6 +1328,22 @@ export function capabilityNotesFor(pack, driverName) {
   return notes;
 }
 
+/**
+ * The local brief with any field that failed its check left out, so a
+ * malformed test account never reaches the prompt half filled.
+ *
+ * @param {Object} localBrief
+ * @returns {Object}
+ */
+export function usableLocalBrief(localBrief) {
+  const out = {};
+  for (const key of ['expected_noise', 'test_account']) {
+    if (!localBrief || localBrief[key] === undefined) continue;
+    if (localBriefProblems({ [key]: localBrief[key] }).length === 0) out[key] = localBrief[key];
+  }
+  return out;
+}
+
 export function skillFor(pack, moduleName) {
   const entry = (pack.skills || []).find(skill => skill.module === moduleName);
   if (!entry) throw new Error(`the content pack has no ${moduleName} skill; run a pack sync`);
@@ -1272,23 +1356,158 @@ export function resolvePersonas(options, config, pack) {
     return options.personas.split(',').map(slug => slug.trim()).filter(Boolean);
   }
   if (config && Array.isArray(config.personas) && config.personas.length > 0) return config.personas;
-  const available = (pack.personas || []).map(persona => persona.slug);
-  const defaults = DEFAULT_PERSONAS.filter(slug => available.includes(slug));
-  return defaults.length > 0 ? defaults : available.slice(0, 2);
+  return defaultPersonasFor(pack.personas || []);
+}
+
+/**
+ * What one persona run buys, said before the charge: the action budget each
+ * persona gets (its patience in the pack, lowered by --max-actions or the
+ * server's ceiling), and that a sign up form alone can take eight of them.
+ *
+ * @returns {Array<string>}
+ */
+export function actionBudgetLines({ options = {}, personas = [], pack = {}, localBrief = {} }) {
+  if (personas.length === 0) return [];
+  const budgets = personas.map(persona => ({ slug: persona.slug, actions: resolveMaxActions(options, persona, pack) }));
+  const same = budgets.every(entry => entry.actions === budgets[0].actions);
+  const typical = Math.min(...budgets.map(entry => entry.actions));
+  const lines = [
+    same
+      ? `Action budget: ${budgets[0].actions} actions per persona.`
+      : `Action budget: ${budgets.map(entry => `${entry.slug} ${entry.actions}`).join(', ')} actions.`,
+    `A persona typically reaches ${typical} actions; sign-up alone can take ${SIGN_UP_ACTIONS}.${localBrief && localBrief.test_account ? ' The brief\'s test account lets it sign in instead.' : ' A test_account in the brief lets it start past sign-up.'}`
+  ];
+  return lines;
+}
+
+/** How long the health check waits for an answer. */
+export const HEALTH_TIMEOUT_MS = 10000;
+
+/**
+ * Whether a status is what the health check expects: the configured
+ * `expect_status` (a number or a list of numbers), else any 2xx or 3xx.
+ *
+ * @param {number} status
+ * @param {number|Array<number>|undefined} expected
+ * @returns {boolean}
+ */
+export function statusExpected(status, expected) {
+  if (Array.isArray(expected) && expected.length > 0) return expected.map(Number).includes(status);
+  if (Number.isInteger(Number(expected)) && expected !== null && expected !== undefined && expected !== '') return status === Number(expected);
+  return status >= 200 && status < 400;
 }
 
+/**
+ * The free health check `run` makes before it prices or registers anything.
+ * `.ravensight/config.json` may name one: `preflight: { url, expect_status }`
+ * (a relative url is resolved against the build url). With none, a web
+ * build's own url has to answer 2xx or 3xx. Other drivers with no configured
+ * url are not checked. Redirects are not followed: a 3xx is an answer.
+ *
+ * @returns {Promise<{ok: boolean, url: string, status: number|null, detail: string}|null>} null when nothing was checked
+ */
+export async function healthPreflight({ config = null, driverName, target, fetchImpl = null, timeoutMs = HEALTH_TIMEOUT_MS, log = () => {} }) {
+  const configured = config && config.preflight && typeof config.preflight === 'object' ? config.preflight : null;
+  let url = configured && typeof configured.url === 'string' && configured.url.trim() ? configured.url.trim() : null;
+  const expected = configured ? configured.expect_status : undefined;
+  if (url && !/^https?:\/\//i.test(url)) {
+    if (driverName !== 'playwright_web' || !target) return { ok: false, url, status: null, detail: `preflight.url ${url} is not an http or https URL` };
+    url = new URL(url, target).toString();
+  }
+  if (!url) {
+    if (driverName !== 'playwright_web' || !/^https?:\/\//i.test(String(target || ''))) return null;
+    url = String(target);
+  }
+  const doFetch = fetchImpl || globalThis.fetch;
+  const wanted = Array.isArray(expected) && expected.length > 0 ? expected.join(' or ') : (expected !== undefined && expected !== null && expected !== '' ? String(expected) : '2xx or 3xx');
+  log(`preflight: GET ${url}, expecting ${wanted}`);
+  try {
+    const response = await doFetch(url, { method: 'GET', redirect: 'manual', signal: AbortSignal.timeout(timeoutMs) });
+    const status = Number(response.status);
+    const ok = statusExpected(status, expected);
+    return { ok, url, status, detail: ok ? `${url} answered ${status}` : `GET ${url} answered ${status}, expected ${wanted}` };
+  } catch (error) {
+    const reason = error && error.name === 'TimeoutError' ? `no answer in ${Math.round(timeoutMs / 1000)} seconds` : String((error && error.cause && error.cause.code) || (error && error.message) || error);
+    return { ok: false, url, status: null, detail: `GET ${url} failed: ${reason}` };
+  }
+}
+
+/** The factor a long run's limits use when the server names none. */
+export const FALLBACK_LONG_RUN_FACTOR = 2;
+
+/**
+ * How many times a standard run's limits a long run gets, from the server:
+ * the estimate's `long_run_factor`, a run's `limits.long_run_factor`, or the
+ * pack's client routing `long_run_factor`. Only when none of them says does it
+ * fall back to 2, with a note saying so.
+ *
+ * @returns {{factor: number, source: string, note: string|null}}
+ */
+export function longRunFactor({ estimate = null, run = null, pack = null } = {}) {
+  const sources = [
+    ['the estimate', estimate && estimate.long_run_factor],
+    ['the run limits', run && run.limits && run.limits.long_run_factor],
+    ['the content pack', pack && pack.routing && pack.routing.long_run_factor]
+  ];
+  for (const [source, value] of sources) {
+    const factor = Number(value);
+    if (Number.isInteger(factor) && factor >= 1) return { factor, source, note: null };
+  }
+  return { factor: FALLBACK_LONG_RUN_FACTOR, source: 'fallback', note: `The server did not say how much longer a long run is, so the local action budget assumes ${FALLBACK_LONG_RUN_FACTOR}x; the server's own limits still apply.` };
+}
+
+/** "twice", "3 times": a factor as the run line says it. */
+function times(factor) {
+  return factor === 2 ? 'twice' : `${factor} times`;
+}
+
+/**
+ * Said under the price when `--long` was passed; the price itself comes from the estimate.
+ *
+ * @param {number} [factor]
+ * @returns {string}
+ */
+export function longRunLine(factor = FALLBACK_LONG_RUN_FACTOR) {
+  return `Long run: each persona gets ${times(factor)} the actions, model budget and wall time of a standard run, and is priced as a long run in the total above.`;
+}
+
+/** The line for the default factor, kept for callers that print it as is. */
+export const LONG_RUN_LINE = longRunLine();
+
+/** A standard run's local action budget when nothing else sets one. */
+export const DEFAULT_MAX_ACTIONS = 25;
+
 /**
  * The action budget is the smallest of the three ceilings that exist: what the
- * developer asked for, the persona's own patience, and what routing.json allows
- * per run (`max_actions`, which arrives in the pack's client routing subset).
+ * developer asked for, the persona's own patience, and what the run is allowed
+ * (`max_actions`, which arrives in the pack's client routing subset).
+ *
+ * A long run (`options.runLength === 'long'`) uses the run's own action limit
+ * when the server's register answer carries one on the run (`max_actions`, or
+ * `limits.max_actions`), and otherwise the server's long run factor (see
+ * {@link longRunFactor}) times the routing ceiling. The persona's patience is
+ * multiplied too, or a long run would stop where a standard one does.
+ * `--max-actions` is the developer's own ceiling and is never doubled.
+ *
+ * @param {Object} options
+ * @param {Object} persona
+ * @param {Object} pack
+ * @param {Object|null} [run] the run as register answered it
+ * @returns {number}
  */
-export function resolveMaxActions(options, persona, pack) {
+export function resolveMaxActions(options, persona, pack, run = null) {
+  const factor = options && options.runLength === 'long'
+    ? (Number.isInteger(options.longRunFactor) && options.longRunFactor >= 1 ? options.longRunFactor : longRunFactor({ run, pack }).factor)
+    : 1;
+  const fromServer = Number(run && ((run.limits && run.limits.max_actions) ?? run.max_actions)) || null;
+  const routing = Number(pack && pack.routing && pack.routing.max_actions) || null;
+  const patience = Number(persona && persona.patience_actions) || null;
   const candidates = [
-    Number.isInteger(options.maxActions) ? options.maxActions : null,
-    Number(persona && persona.patience_actions) || null,
-    Number(pack && pack.routing && pack.routing.max_actions) || null
+    Number.isInteger(options && options.maxActions) ? options.maxActions : null,
+    patience === null ? null : patience * factor,
+    fromServer || (routing === null ? null : routing * factor)
   ].filter(value => Number.isFinite(value) && value > 0);
-  return candidates.length > 0 ? Math.min(...candidates) : 25;
+  return candidates.length > 0 ? Math.min(...candidates) : DEFAULT_MAX_ACTIONS * factor;
 }
 
 export function resolveTarget(options, config, driverName) {

package/src/run/args.js

@@ -35,6 +35,9 @@ const BOOLEAN = new Map([
   ['--headed', 'headed'],
   ['--headless', 'headless'],
   ['--verbose', 'verbose'],
+  // A long persona run: twice the actions, model budget and wall time of a
+  // standard run, priced by the server as its own unit.
+  ['--long', 'long'],
   // Native drivers only: register without the free automation probe first.
   ['--skip-preflight', 'skipPreflight'],
   // Resume only: replay an interrupted run from the start instead of writing a
@@ -106,6 +109,7 @@ export function optionsFrom(flags = {}) {
   return {
     options: {
       ...options,
+      runLength: runLengthFor(options),
       ...stripUndefined({
         repo: flags.repoRoot || flags.repo,
         apiUrl: flags.apiUrl,
@@ -128,8 +132,21 @@ export function optionsFrom(flags = {}) {
   };
 }
 
+/** The two run lengths the server prices: `standard`, and `long` at twice a standard run. */
+export const RUN_LENGTHS = Object.freeze(['standard', 'long']);
+
+/**
+ * `long` when `--long` was passed, otherwise `standard`.
+ *
+ * @param {{long?: boolean}} options
+ * @returns {'standard'|'long'}
+ */
+export function runLengthFor(options = {}) {
+  return options.long === true ? 'long' : 'standard';
+}
+
 function stripUndefined(value) {
   return Object.fromEntries(Object.entries(value).filter(([, entry]) => entry !== undefined));
 }
 
-export default { parseRunArgv, optionsFrom, unknownOptionNotes };
+export default { parseRunArgv, optionsFrom, unknownOptionNotes, runLengthFor };