ravensight-playtest 0.3.0 → 0.3.2

package/src/run/report.js

@@ -207,6 +207,191 @@ export function normaliseEvidence(report, runDirectory = null) {
   return { report: { ...report, findings }, changed };
 }
 
+/** Category words a model writes that are not in the schema, mapped to one that is, when the schema has it. */
+export const CATEGORY_SYNONYMS = Object.freeze({
+  ui: 'ux', usability: 'ux', onboarding: 'ux', clarity: 'ux', visual: 'ux', visuals: 'ux', feedback: 'ux', navigation: 'ux', polish: 'ux', design: 'ux',
+  crash: 'bug', error: 'bug', defect: 'bug', broken: 'bug', functional: 'bug', functionality: 'bug', network: 'bug',
+  perf: 'performance', lag: 'performance', fps: 'performance', loading: 'performance',
+  a11y: 'accessibility',
+  level: 'level_design', map: 'level_design', 'level design': 'level_design', 'level-design': 'level_design',
+  text: 'content', story: 'content', narrative: 'content', writing: 'content', copy: 'content', audio: 'content', art: 'content',
+  economy: 'balance', difficulty: 'balance', progression: 'balance'
+});
+
+/** The severity an out of range one becomes. */
+export const NORMALISED_SEVERITY = 'minor';
+
+/**
+ * The category an unknown one becomes: `other` when the schema has it, else a
+ * catch-all the schema names, else `ux`. The server's schema today has no
+ * catch-all (bug, ux, balance, content, performance, accessibility,
+ * level_design), so an unknown word that no synonym places becomes `ux`, the
+ * category most unplaced findings turn out to be.
+ *
+ * @param {Array<string>} categories the schema's enum
+ * @returns {string}
+ */
+export function catchAllCategory(categories) {
+  for (const candidate of ['other', 'misc', 'general', 'uncategorized']) if (categories.includes(candidate)) return candidate;
+  return categories.includes('ux') ? 'ux' : categories[0];
+}
+
+/**
+ * The enum lists and defaults of the report schema's root and of one finding,
+ * read from the schema itself so a pack update moves them.
+ *
+ * @param {Object} schema the playtest-report schema
+ * @returns {{severities: Array<string>, categories: Array<string>, root: Object, finding: Object}}
+ */
+export function enumFields(schema) {
+  const props = (schema && schema.properties) || {};
+  const item = (props.findings && props.findings.items && props.findings.items.properties) || {};
+  const enumsOf = properties => Object.fromEntries(Object.entries(properties)
+    .filter(([, spec]) => spec && Array.isArray(spec.enum))
+    .map(([key, spec]) => [key, { values: spec.enum, ...(spec.default !== undefined ? { default: spec.default } : {}) }]));
+  return {
+    severities: (item.severity && item.severity.enum) || [],
+    categories: (item.category && item.category.enum) || [],
+    root: enumsOf(props),
+    finding: enumsOf(item)
+  };
+}
+
+/**
+ * Bring known out of range values back into the schema before upload,
+ * instead of losing the run to them. The server's schema stays strict; this
+ * only rewrites what the model's own words say plainly:
+ *
+ * - a value that differs from an allowed one only in case or spacing takes
+ *   the allowed spelling (`Major` is `major`);
+ * - a severity that is a category word (`"severity": "ux"`) moves to
+ *   `category` when the category is missing or not allowed, and the severity
+ *   becomes `minor`; a severity that is anything else unknown becomes `minor`;
+ * - a category that is a severity word, with the severity out of range,
+ *   swaps back; any other unknown category becomes a synonym the schema has,
+ *   else `other` (or the catch-all, see {@link catchAllCategory});
+ * - any other enum field, at the root or on a finding, with a value the
+ *   schema does not allow takes the schema's `default` when it has one.
+ *
+ * Every change is returned, and recorded on the finding as a runner note at
+ * the end of its description. The Texas round lost a run to `"severity": "ux"`
+ * when the model's budget ran out before its repair turn.
+ *
+ * @param {Object} report
+ * @param {Object} schema the playtest-report schema
+ * @returns {{report: Object, changes: Array<{id: string|null, field: string, from: any, to: any, note: string}>}}
+ */
+export function normaliseEnums(report, schema) {
+  if (!report || typeof report !== 'object' || !schema) return { report, changes: [] };
+  const fields = enumFields(schema);
+  const changes = [];
+  const spelled = (value, allowed) => {
+    if (typeof value !== 'string') return null;
+    const key = value.trim().toLowerCase().replace(/[\s-]+/g, '_');
+    return allowed.find(entry => String(entry).toLowerCase() === key) ?? null;
+  };
+  const fixEnum = (target, key, spec, id) => {
+    if (!(key in target) || spec.values.includes(target[key])) return;
+    const respelled = spelled(target[key], spec.values);
+    const to = respelled !== null ? respelled : spec.default;
+    if (to === undefined) return;
+    changes.push({ id, field: key, from: target[key], to, note: `${key} ${JSON.stringify(target[key])} is not allowed, so it became ${JSON.stringify(to)}` });
+    target[key] = to;
+  };
+
+  const root = { ...report };
+  for (const [key, spec] of Object.entries(fields.root)) fixEnum(root, key, spec, null);
+
+  const { severities, categories } = fields;
+  if (Array.isArray(root.findings)) {
+    root.findings = root.findings.map(original => {
+      if (!original || typeof original !== 'object' || Array.isArray(original)) return original;
+      const finding = { ...original };
+      const id = typeof finding.id === 'string' ? finding.id : null;
+      const notes = [];
+      const record = (field, from, to, note) => {
+        changes.push({ id, field, from, to, note });
+        notes.push(note);
+      };
+      if (severities.length > 0 && categories.length > 0) {
+        const severityOk = severities.includes(finding.severity);
+        const categoryOk = categories.includes(finding.category);
+        const severityRespelled = severityOk ? finding.severity : spelled(finding.severity, severities);
+        const categoryRespelled = categoryOk ? finding.category : spelled(finding.category, categories);
+        const severityAsCategory = severityRespelled === null ? spelled(finding.severity, categories) : null;
+        const categoryAsSeverity = categoryRespelled === null ? spelled(finding.category, severities) : null;
+        if (severityRespelled !== null && !severityOk) record('severity', finding.severity, severityRespelled, `severity ${JSON.stringify(finding.severity)} became ${JSON.stringify(severityRespelled)}`);
+        if (categoryRespelled !== null && !categoryOk) record('category', finding.category, categoryRespelled, `category ${JSON.stringify(finding.category)} became ${JSON.stringify(categoryRespelled)}`);
+        let severity = severityRespelled;
+        let category = categoryRespelled;
+        if (severity === null && category === null && severityAsCategory !== null && categoryAsSeverity !== null) {
+          // The two were written in each other's slot.
+          record('severity', finding.severity, categoryAsSeverity, `severity ${JSON.stringify(finding.severity)} and category ${JSON.stringify(finding.category)} were swapped, so they were put back`);
+          record('category', finding.category, severityAsCategory, `category ${JSON.stringify(finding.category)} is a severity word, so it moved to severity`);
+          severity = categoryAsSeverity;
+          category = severityAsCategory;
+        }
+        if (severity === null && severityAsCategory !== null && category === null) {
+          record('category', finding.category === undefined ? null : finding.category, severityAsCategory, `severity ${JSON.stringify(finding.severity)} is a category, so it moved to category`);
+          category = severityAsCategory;
+        }
+        if (severity === null) {
+          record('severity', finding.severity === undefined ? null : finding.severity, NORMALISED_SEVERITY, `severity ${JSON.stringify(finding.severity === undefined ? null : finding.severity)} is not a severity, so it became ${NORMALISED_SEVERITY}`);
+          severity = NORMALISED_SEVERITY;
+        }
+        if (category === null) {
+          const word = typeof finding.category === 'string' ? finding.category.trim().toLowerCase() : '';
+          const synonym = CATEGORY_SYNONYMS[word] && categories.includes(CATEGORY_SYNONYMS[word]) ? CATEGORY_SYNONYMS[word] : null;
+          const to = synonym || catchAllCategory(categories);
+          record('category', finding.category === undefined ? null : finding.category, to, `category ${JSON.stringify(finding.category === undefined ? null : finding.category)} is not a category, so it became ${to}`);
+          category = to;
+        }
+        finding.severity = severity;
+        finding.category = category;
+      }
+      for (const [key, spec] of Object.entries(fields.finding)) {
+        if (key === 'severity' || key === 'category') continue;
+        const before = changes.length;
+        fixEnum(finding, key, spec, id);
+        if (changes.length > before) notes.push(changes[changes.length - 1].note);
+      }
+      if (notes.length > 0) {
+        const line = `Runner note: ${notes.join('; ')}.`;
+        const description = typeof finding.description === 'string' ? finding.description : '';
+        finding.description = description ? `${description}\n\n${line}` : line;
+      }
+      return finding;
+    });
+  }
+  return { report: changes.length > 0 ? root : report, changes };
+}
+
+/** How report.md lists what the runner normalised. */
+export const NORMALISED_HEADING = '## Normalised by the runner (values the report schema does not allow)';
+
+/**
+ * report.md agreeing with report.json about normalised values: each change is
+ * listed under {@link NORMALISED_HEADING}, replacing a list written before.
+ *
+ * @param {string} markdown
+ * @param {Array<{id: string|null, note: string}>} changes
+ * @returns {string}
+ */
+export function appendNormalisedMarkdown(markdown, changes) {
+  if (!Array.isArray(changes) || changes.length === 0) return markdown;
+  let body = String(markdown || '');
+  const at = body.indexOf(NORMALISED_HEADING);
+  if (at !== -1) {
+    // A list written before is replaced: from its heading to the next
+    // second level heading, or the end.
+    const next = body.indexOf('\n## ', at + NORMALISED_HEADING.length);
+    body = body.slice(0, at) + (next === -1 ? '' : body.slice(next + 1));
+  }
+  body = body.replace(/\s+$/, '');
+  const lines = changes.map(change => `- ${change.id ? `${change.id}: ` : ''}${change.note}.`);
+  return `${body}\n\n${NORMALISED_HEADING}\n${lines.join('\n')}\n`;
+}
+
 /**
  * Check every `screenshot` evidence ref against the files actually on disk.
  *
@@ -301,6 +486,8 @@ export function repairInstruction(details, evidenceProblems, missingSections = [
 
 export default {
   createReportValidator,
+  normaliseEnums,
+  appendNormalisedMarkdown,
   stampReport,
   checkEvidence,
   normaliseEvidence,

package/src/run/resume.js

@@ -149,6 +149,8 @@ export async function resumeJob({ api, gameId, jobId, job, runs, state, repoRoot
     build: { kind: driverName === 'playwright_web' ? 'url' : 'godot_project', ref: target },
     repo: { commit: '', dirty: false },
     max_actions: (journal.options && journal.options.maxActions) ?? null,
+    // The same body as the first register, so a long job replays as long.
+    ...(journal.options?.runLength === 'long' ? { run_length: 'long' } : {}),
     pack_versions: { pack: pack.pack_version }
   }, { idempotencyKey: journal.idempotency_key });
 
@@ -165,6 +167,9 @@ export async function resumeJob({ api, gameId, jobId, job, runs, state, repoRoot
     ...options,
     controller: journal.options?.controller || options.controller || 'standard',
     maxActions: (journal.options && journal.options.maxActions) ?? options.maxActions,
+    // The journal's answer, never the flag: a job is long or standard for good.
+    runLength: journal.options?.runLength === 'long' ? 'long' : 'standard',
+    ...(Number.isInteger(journal.options?.longRunFactor) ? { longRunFactor: journal.options.longRunFactor } : {}),
     concurrency: options.concurrency ?? (journal.options && journal.options.concurrency),
     uploadVideo: options.uploadVideo ?? (journal.options && journal.options.uploadVideo),
     uploadTranscript: options.uploadTranscript ?? (journal.options && journal.options.uploadTranscript)

package/src/run/tools.js

@@ -68,7 +68,7 @@ const note = {
 export const PLAY_TOOLS = Object.freeze([
   {
     name: 'snapshot',
-    description: 'Look at the current screen. Does not spend an action. Use it after anything that might have changed the game state.',
+    description: 'Look at the current screen. Does not spend an action. Use it after anything that might have changed the game state. The result also repeats this run\'s identity (the email and display name to use whenever a form asks for one).',
     input_schema: {
       type: 'object',
       additionalProperties: false,
@@ -83,7 +83,7 @@ export const PLAY_TOOLS = Object.freeze([
       additionalProperties: false,
       properties: {
         ref: { type: 'string', description: 'playwright_web only, the preferred target: the ref shown on the element, such as e12.' },
-        target: { type: 'string', description: 'web: button "Start" or Start Game. Godot: a node path such as /root/Main/UI/StartButton.' },
+        target: { type: 'string', description: 'web: button "Start" or Start Game; a whole line copied from the screen also works, and is clicked by its ref, else its @(x,y). Text that is not on the screen is refused at once. Godot: a node path such as /root/Main/UI/StartButton.' },
         x: { type: 'number' },
         y: { type: 'number' },
         thought_in_character: thought,
@@ -234,7 +234,7 @@ export function actionFor(toolUse) {
  *   screen, so a ref in this call was read off a screen that is gone
  * @returns {Promise<{result: Object, observation: Object|null, spentAction: boolean, changed: boolean, error: string|null}>}
  */
-export async function executeTool({ driver, toolUse, onNote, onScreenshot, staleRefs = false }) {
+export async function executeTool({ driver, toolUse, onNote, onScreenshot, staleRefs = false, identity = null }) {
   const input = toolUse.input || {};
   if (toolUse.malformed) {
     return {
@@ -272,8 +272,13 @@ export async function executeTool({ driver, toolUse, onNote, onScreenshot, stale
 
   if (toolUse.name === 'snapshot') {
     const observation = await driver.observe();
+    const result = observationResult(toolUse.id, observation);
+    // The run's identity rides on the one tool a persona calls to look
+    // around: the act step's tool allowlist has no room for a tool of its own.
+    const line = identityLine(identity);
+    if (line) result.content.push({ type: 'text', text: line });
     return {
-      result: observationResult(toolUse.id, observation),
+      result,
       observation,
       spentAction: false,
       changed: false,
@@ -351,6 +356,17 @@ export function observationResult(id, observation, prefix = '', isError = false)
   return { type: 'tool_result', tool_use_id: id, content, ...(isError ? { is_error: true } : {}) };
 }
 
+/**
+ * The run's identity as one line of a snapshot result, or '' with none.
+ *
+ * @param {{email?: string, displayName?: string}|null} identity
+ * @returns {string}
+ */
+export function identityLine(identity) {
+  if (!identity || !identity.email) return '';
+  return `run identity: email ${identity.email}, display name ${identity.displayName}. Use these whenever a form asks for an email or a name.`;
+}
+
 /**
  * @param {string} id
  * @param {string} text

package/src/runDefaults.js

@@ -0,0 +1,90 @@
+/**
+ * What a plain `run` asks for, in one place, so `brief`, `check` and `run`
+ * quote the same job. Kept free of runner imports: `check` and `brief` load
+ * this without pulling in the drivers.
+ */
+
+/**
+ * Personas used when the developer names none. One since 0.3.2: a first run
+ * should cost one persona run, and a second persona is one `--personas` away.
+ * Spec 17 defaulted to two, which made the default run cost twice what most
+ * customers expected to pay for a first look.
+ */
+export const DEFAULT_PERSONAS = Object.freeze(['curious-kid']);
+
+/** The modules `run` registers. `aggregate_report` is free and buys the aggregate's steps. */
+export const RUN_MODULES = Object.freeze(['persona_playtest', 'aggregate_report']);
+
+/**
+ * Actions a sign up form alone took in customer runs (an email, a password, a
+ * confirm, a name, the submit, and the clicks between them), which is why a
+ * brief's test account matters on a short action budget.
+ */
+export const SIGN_UP_ACTIONS = 8;
+
+/**
+ * The default persona set as far as this pack has them: the defaults it
+ * carries, else its first persona.
+ *
+ * @param {Array<{slug: string}>} [available]
+ * @returns {Array<string>}
+ */
+export function defaultPersonasFor(available = null) {
+  if (!Array.isArray(available)) return [...DEFAULT_PERSONAS];
+  const slugs = available.map(persona => persona && persona.slug).filter(Boolean);
+  const defaults = DEFAULT_PERSONAS.filter(slug => slugs.includes(slug));
+  return defaults.length > 0 ? defaults : slugs.slice(0, 1);
+}
+
+/** "1 persona", "3 personas". */
+export function personaCount(count) {
+  return `${count} persona${count === 1 ? '' : 's'}`;
+}
+
+/**
+ * An estimate's breakdown in words, every unit named with its own price:
+ * "profile job $4.00, a persona run $1.00 each (x2)".
+ *
+ * @param {Array<{unit: string, count: number, unit_cents: number}>} breakdown
+ * @param {(cents: number) => string} money
+ * @returns {string}
+ */
+export function describeBreakdown(breakdown, money) {
+  const parts = [];
+  for (const line of Array.isArray(breakdown) ? breakdown : []) {
+    if (!line || !(Number(line.count) > 0)) continue;
+    const price = money(line.unit_cents);
+    const times = Number(line.count) > 1 ? ` (x${line.count})` : '';
+    if (line.unit === 'playtest_run') parts.push(`a persona run ${price} each${times}`);
+    else if (line.unit === 'playtest_run_long') parts.push(`a long persona run ${price} each${times}`);
+    else if (line.unit === 'playtest_profile') parts.push(`profile job ${price}${times}`);
+    else if (line.unit === 'playtest_fix') parts.push(`a fix attempt ${price} each${times}`);
+    else parts.push(`${line.unit} ${price}${times}`);
+  }
+  return parts.join(', ');
+}
+
+/**
+ * The persona slugs in a list that the pack does not have.
+ *
+ * @param {Array<string>} slugs
+ * @param {Array<{slug: string}>} available
+ * @returns {Array<string>}
+ */
+export function unknownPersonas(slugs, available) {
+  const known = new Set((available || []).map(persona => persona && persona.slug));
+  return (slugs || []).filter(slug => !known.has(slug));
+}
+
+/**
+ * "unknown persona "x"; the valid slugs are a, b and c. "ravensight-playtest personas" lists them."
+ *
+ * @param {Array<string>} unknown
+ * @param {Array<{slug: string}>} available
+ * @returns {string}
+ */
+export function unknownPersonaMessage(unknown, available) {
+  const slugs = (available || []).map(persona => persona.slug);
+  const listed = slugs.length > 1 ? `${slugs.slice(0, -1).join(', ')} and ${slugs[slugs.length - 1]}` : slugs.join('');
+  return `unknown persona${unknown.length === 1 ? '' : 's'} ${unknown.map(slug => JSON.stringify(slug)).join(', ')}; the valid slugs are ${listed || '(none in this pack)'}. "ravensight-playtest personas" lists them.`;
+}

package/src/state/index.js

@@ -100,6 +100,9 @@ export async function listStates(options = {}) {
       state
     });
   }
-  found.sort((a, b) => String(b.updated_at).localeCompare(String(a.updated_at)));
+  // Two journals written in the same millisecond tie on updated_at; the job
+  // id breaks the tie (descending, like the time), so the order never
+  // depends on how the filesystem lists the directory.
+  found.sort((a, b) => String(b.updated_at).localeCompare(String(a.updated_at)) || String(b.job_id).localeCompare(String(a.job_id)));
   return found;
 }

package/src/ui/index.js

@@ -184,6 +184,19 @@ export async function choose(question, labels, options = {}) {
   }
 }
 
+/**
+ * Bold on a terminal that shows it, plain everywhere else (a pipe, CI, a
+ * log file, NO_COLOR), so a saved log never carries escape codes.
+ *
+ * @param {string} text
+ * @param {{isTTY?: boolean}} [target] the stream it will be written to
+ * @returns {string}
+ */
+export function bold(text, target = stream) {
+  if (process.env.NO_COLOR || !(target && target.isTTY)) return String(text);
+  return `\u001b[1m${text}\u001b[22m`;
+}
+
 /** Cents as dollars, the one place that formatting exists. */
 export function money(cents) {
   if (cents === null || cents === undefined) return 'unknown';
@@ -219,5 +232,6 @@ export const ui = {
   confirm,
   choose,
   money,
-  bytes
+  bytes,
+  bold
 };