ravensight-playtest 0.3.1 → 0.3.2

package/src/run/report.js

@@ -207,6 +207,191 @@ export function normaliseEvidence(report, runDirectory = null) {
   return { report: { ...report, findings }, changed };
 }
 
+/** Category words a model writes that are not in the schema, mapped to one that is, when the schema has it. */
+export const CATEGORY_SYNONYMS = Object.freeze({
+  ui: 'ux', usability: 'ux', onboarding: 'ux', clarity: 'ux', visual: 'ux', visuals: 'ux', feedback: 'ux', navigation: 'ux', polish: 'ux', design: 'ux',
+  crash: 'bug', error: 'bug', defect: 'bug', broken: 'bug', functional: 'bug', functionality: 'bug', network: 'bug',
+  perf: 'performance', lag: 'performance', fps: 'performance', loading: 'performance',
+  a11y: 'accessibility',
+  level: 'level_design', map: 'level_design', 'level design': 'level_design', 'level-design': 'level_design',
+  text: 'content', story: 'content', narrative: 'content', writing: 'content', copy: 'content', audio: 'content', art: 'content',
+  economy: 'balance', difficulty: 'balance', progression: 'balance'
+});
+
+/** The severity an out of range one becomes. */
+export const NORMALISED_SEVERITY = 'minor';
+
+/**
+ * The category an unknown one becomes: `other` when the schema has it, else a
+ * catch-all the schema names, else `ux`. The server's schema today has no
+ * catch-all (bug, ux, balance, content, performance, accessibility,
+ * level_design), so an unknown word that no synonym places becomes `ux`, the
+ * category most unplaced findings turn out to be.
+ *
+ * @param {Array<string>} categories the schema's enum
+ * @returns {string}
+ */
+export function catchAllCategory(categories) {
+  for (const candidate of ['other', 'misc', 'general', 'uncategorized']) if (categories.includes(candidate)) return candidate;
+  return categories.includes('ux') ? 'ux' : categories[0];
+}
+
+/**
+ * The enum lists and defaults of the report schema's root and of one finding,
+ * read from the schema itself so a pack update moves them.
+ *
+ * @param {Object} schema the playtest-report schema
+ * @returns {{severities: Array<string>, categories: Array<string>, root: Object, finding: Object}}
+ */
+export function enumFields(schema) {
+  const props = (schema && schema.properties) || {};
+  const item = (props.findings && props.findings.items && props.findings.items.properties) || {};
+  const enumsOf = properties => Object.fromEntries(Object.entries(properties)
+    .filter(([, spec]) => spec && Array.isArray(spec.enum))
+    .map(([key, spec]) => [key, { values: spec.enum, ...(spec.default !== undefined ? { default: spec.default } : {}) }]));
+  return {
+    severities: (item.severity && item.severity.enum) || [],
+    categories: (item.category && item.category.enum) || [],
+    root: enumsOf(props),
+    finding: enumsOf(item)
+  };
+}
+
+/**
+ * Bring known out of range values back into the schema before upload,
+ * instead of losing the run to them. The server's schema stays strict; this
+ * only rewrites what the model's own words say plainly:
+ *
+ * - a value that differs from an allowed one only in case or spacing takes
+ *   the allowed spelling (`Major` is `major`);
+ * - a severity that is a category word (`"severity": "ux"`) moves to
+ *   `category` when the category is missing or not allowed, and the severity
+ *   becomes `minor`; a severity that is anything else unknown becomes `minor`;
+ * - a category that is a severity word, with the severity out of range,
+ *   swaps back; any other unknown category becomes a synonym the schema has,
+ *   else `other` (or the catch-all, see {@link catchAllCategory});
+ * - any other enum field, at the root or on a finding, with a value the
+ *   schema does not allow takes the schema's `default` when it has one.
+ *
+ * Every change is returned, and recorded on the finding as a runner note at
+ * the end of its description. The Texas round lost a run to `"severity": "ux"`
+ * when the model's budget ran out before its repair turn.
+ *
+ * @param {Object} report
+ * @param {Object} schema the playtest-report schema
+ * @returns {{report: Object, changes: Array<{id: string|null, field: string, from: any, to: any, note: string}>}}
+ */
+export function normaliseEnums(report, schema) {
+  if (!report || typeof report !== 'object' || !schema) return { report, changes: [] };
+  const fields = enumFields(schema);
+  const changes = [];
+  const spelled = (value, allowed) => {
+    if (typeof value !== 'string') return null;
+    const key = value.trim().toLowerCase().replace(/[\s-]+/g, '_');
+    return allowed.find(entry => String(entry).toLowerCase() === key) ?? null;
+  };
+  const fixEnum = (target, key, spec, id) => {
+    if (!(key in target) || spec.values.includes(target[key])) return;
+    const respelled = spelled(target[key], spec.values);
+    const to = respelled !== null ? respelled : spec.default;
+    if (to === undefined) return;
+    changes.push({ id, field: key, from: target[key], to, note: `${key} ${JSON.stringify(target[key])} is not allowed, so it became ${JSON.stringify(to)}` });
+    target[key] = to;
+  };
+
+  const root = { ...report };
+  for (const [key, spec] of Object.entries(fields.root)) fixEnum(root, key, spec, null);
+
+  const { severities, categories } = fields;
+  if (Array.isArray(root.findings)) {
+    root.findings = root.findings.map(original => {
+      if (!original || typeof original !== 'object' || Array.isArray(original)) return original;
+      const finding = { ...original };
+      const id = typeof finding.id === 'string' ? finding.id : null;
+      const notes = [];
+      const record = (field, from, to, note) => {
+        changes.push({ id, field, from, to, note });
+        notes.push(note);
+      };
+      if (severities.length > 0 && categories.length > 0) {
+        const severityOk = severities.includes(finding.severity);
+        const categoryOk = categories.includes(finding.category);
+        const severityRespelled = severityOk ? finding.severity : spelled(finding.severity, severities);
+        const categoryRespelled = categoryOk ? finding.category : spelled(finding.category, categories);
+        const severityAsCategory = severityRespelled === null ? spelled(finding.severity, categories) : null;
+        const categoryAsSeverity = categoryRespelled === null ? spelled(finding.category, severities) : null;
+        if (severityRespelled !== null && !severityOk) record('severity', finding.severity, severityRespelled, `severity ${JSON.stringify(finding.severity)} became ${JSON.stringify(severityRespelled)}`);
+        if (categoryRespelled !== null && !categoryOk) record('category', finding.category, categoryRespelled, `category ${JSON.stringify(finding.category)} became ${JSON.stringify(categoryRespelled)}`);
+        let severity = severityRespelled;
+        let category = categoryRespelled;
+        if (severity === null && category === null && severityAsCategory !== null && categoryAsSeverity !== null) {
+          // The two were written in each other's slot.
+          record('severity', finding.severity, categoryAsSeverity, `severity ${JSON.stringify(finding.severity)} and category ${JSON.stringify(finding.category)} were swapped, so they were put back`);
+          record('category', finding.category, severityAsCategory, `category ${JSON.stringify(finding.category)} is a severity word, so it moved to severity`);
+          severity = categoryAsSeverity;
+          category = severityAsCategory;
+        }
+        if (severity === null && severityAsCategory !== null && category === null) {
+          record('category', finding.category === undefined ? null : finding.category, severityAsCategory, `severity ${JSON.stringify(finding.severity)} is a category, so it moved to category`);
+          category = severityAsCategory;
+        }
+        if (severity === null) {
+          record('severity', finding.severity === undefined ? null : finding.severity, NORMALISED_SEVERITY, `severity ${JSON.stringify(finding.severity === undefined ? null : finding.severity)} is not a severity, so it became ${NORMALISED_SEVERITY}`);
+          severity = NORMALISED_SEVERITY;
+        }
+        if (category === null) {
+          const word = typeof finding.category === 'string' ? finding.category.trim().toLowerCase() : '';
+          const synonym = CATEGORY_SYNONYMS[word] && categories.includes(CATEGORY_SYNONYMS[word]) ? CATEGORY_SYNONYMS[word] : null;
+          const to = synonym || catchAllCategory(categories);
+          record('category', finding.category === undefined ? null : finding.category, to, `category ${JSON.stringify(finding.category === undefined ? null : finding.category)} is not a category, so it became ${to}`);
+          category = to;
+        }
+        finding.severity = severity;
+        finding.category = category;
+      }
+      for (const [key, spec] of Object.entries(fields.finding)) {
+        if (key === 'severity' || key === 'category') continue;
+        const before = changes.length;
+        fixEnum(finding, key, spec, id);
+        if (changes.length > before) notes.push(changes[changes.length - 1].note);
+      }
+      if (notes.length > 0) {
+        const line = `Runner note: ${notes.join('; ')}.`;
+        const description = typeof finding.description === 'string' ? finding.description : '';
+        finding.description = description ? `${description}\n\n${line}` : line;
+      }
+      return finding;
+    });
+  }
+  return { report: changes.length > 0 ? root : report, changes };
+}
+
+/** How report.md lists what the runner normalised. */
+export const NORMALISED_HEADING = '## Normalised by the runner (values the report schema does not allow)';
+
+/**
+ * report.md agreeing with report.json about normalised values: each change is
+ * listed under {@link NORMALISED_HEADING}, replacing a list written before.
+ *
+ * @param {string} markdown
+ * @param {Array<{id: string|null, note: string}>} changes
+ * @returns {string}
+ */
+export function appendNormalisedMarkdown(markdown, changes) {
+  if (!Array.isArray(changes) || changes.length === 0) return markdown;
+  let body = String(markdown || '');
+  const at = body.indexOf(NORMALISED_HEADING);
+  if (at !== -1) {
+    // A list written before is replaced: from its heading to the next
+    // second level heading, or the end.
+    const next = body.indexOf('\n## ', at + NORMALISED_HEADING.length);
+    body = body.slice(0, at) + (next === -1 ? '' : body.slice(next + 1));
+  }
+  body = body.replace(/\s+$/, '');
+  const lines = changes.map(change => `- ${change.id ? `${change.id}: ` : ''}${change.note}.`);
+  return `${body}\n\n${NORMALISED_HEADING}\n${lines.join('\n')}\n`;
+}
+
 /**
  * Check every `screenshot` evidence ref against the files actually on disk.
  *
@@ -301,6 +486,8 @@ export function repairInstruction(details, evidenceProblems, missingSections = [
 
 export default {
   createReportValidator,
+  normaliseEnums,
+  appendNormalisedMarkdown,
   stampReport,
   checkEvidence,
   normaliseEvidence,

package/src/run/resume.js

@@ -149,6 +149,8 @@ export async function resumeJob({ api, gameId, jobId, job, runs, state, repoRoot
     build: { kind: driverName === 'playwright_web' ? 'url' : 'godot_project', ref: target },
     repo: { commit: '', dirty: false },
     max_actions: (journal.options && journal.options.maxActions) ?? null,
+    // The same body as the first register, so a long job replays as long.
+    ...(journal.options?.runLength === 'long' ? { run_length: 'long' } : {}),
     pack_versions: { pack: pack.pack_version }
   }, { idempotencyKey: journal.idempotency_key });
 
@@ -165,6 +167,9 @@ export async function resumeJob({ api, gameId, jobId, job, runs, state, repoRoot
     ...options,
     controller: journal.options?.controller || options.controller || 'standard',
     maxActions: (journal.options && journal.options.maxActions) ?? options.maxActions,
+    // The journal's answer, never the flag: a job is long or standard for good.
+    runLength: journal.options?.runLength === 'long' ? 'long' : 'standard',
+    ...(Number.isInteger(journal.options?.longRunFactor) ? { longRunFactor: journal.options.longRunFactor } : {}),
     concurrency: options.concurrency ?? (journal.options && journal.options.concurrency),
     uploadVideo: options.uploadVideo ?? (journal.options && journal.options.uploadVideo),
     uploadTranscript: options.uploadTranscript ?? (journal.options && journal.options.uploadTranscript)

package/src/run/tools.js

@@ -83,7 +83,7 @@ export const PLAY_TOOLS = Object.freeze([
       additionalProperties: false,
       properties: {
         ref: { type: 'string', description: 'playwright_web only, the preferred target: the ref shown on the element, such as e12.' },
-        target: { type: 'string', description: 'web: button "Start" or Start Game. Godot: a node path such as /root/Main/UI/StartButton.' },
+        target: { type: 'string', description: 'web: button "Start" or Start Game; a whole line copied from the screen also works, and is clicked by its ref, else its @(x,y). Text that is not on the screen is refused at once. Godot: a node path such as /root/Main/UI/StartButton.' },
         x: { type: 'number' },
         y: { type: 'number' },
         thought_in_character: thought,

package/src/runDefaults.js

@@ -0,0 +1,90 @@
+/**
+ * What a plain `run` asks for, in one place, so `brief`, `check` and `run`
+ * quote the same job. Kept free of runner imports: `check` and `brief` load
+ * this without pulling in the drivers.
+ */
+
+/**
+ * Personas used when the developer names none. One since 0.3.2: a first run
+ * should cost one persona run, and a second persona is one `--personas` away.
+ * Spec 17 defaulted to two, which made the default run cost twice what most
+ * customers expected to pay for a first look.
+ */
+export const DEFAULT_PERSONAS = Object.freeze(['curious-kid']);
+
+/** The modules `run` registers. `aggregate_report` is free and buys the aggregate's steps. */
+export const RUN_MODULES = Object.freeze(['persona_playtest', 'aggregate_report']);
+
+/**
+ * Actions a sign up form alone took in customer runs (an email, a password, a
+ * confirm, a name, the submit, and the clicks between them), which is why a
+ * brief's test account matters on a short action budget.
+ */
+export const SIGN_UP_ACTIONS = 8;
+
+/**
+ * The default persona set as far as this pack has them: the defaults it
+ * carries, else its first persona.
+ *
+ * @param {Array<{slug: string}>} [available]
+ * @returns {Array<string>}
+ */
+export function defaultPersonasFor(available = null) {
+  if (!Array.isArray(available)) return [...DEFAULT_PERSONAS];
+  const slugs = available.map(persona => persona && persona.slug).filter(Boolean);
+  const defaults = DEFAULT_PERSONAS.filter(slug => slugs.includes(slug));
+  return defaults.length > 0 ? defaults : slugs.slice(0, 1);
+}
+
+/** "1 persona", "3 personas". */
+export function personaCount(count) {
+  return `${count} persona${count === 1 ? '' : 's'}`;
+}
+
+/**
+ * An estimate's breakdown in words, every unit named with its own price:
+ * "profile job $4.00, a persona run $1.00 each (x2)".
+ *
+ * @param {Array<{unit: string, count: number, unit_cents: number}>} breakdown
+ * @param {(cents: number) => string} money
+ * @returns {string}
+ */
+export function describeBreakdown(breakdown, money) {
+  const parts = [];
+  for (const line of Array.isArray(breakdown) ? breakdown : []) {
+    if (!line || !(Number(line.count) > 0)) continue;
+    const price = money(line.unit_cents);
+    const times = Number(line.count) > 1 ? ` (x${line.count})` : '';
+    if (line.unit === 'playtest_run') parts.push(`a persona run ${price} each${times}`);
+    else if (line.unit === 'playtest_run_long') parts.push(`a long persona run ${price} each${times}`);
+    else if (line.unit === 'playtest_profile') parts.push(`profile job ${price}${times}`);
+    else if (line.unit === 'playtest_fix') parts.push(`a fix attempt ${price} each${times}`);
+    else parts.push(`${line.unit} ${price}${times}`);
+  }
+  return parts.join(', ');
+}
+
+/**
+ * The persona slugs in a list that the pack does not have.
+ *
+ * @param {Array<string>} slugs
+ * @param {Array<{slug: string}>} available
+ * @returns {Array<string>}
+ */
+export function unknownPersonas(slugs, available) {
+  const known = new Set((available || []).map(persona => persona && persona.slug));
+  return (slugs || []).filter(slug => !known.has(slug));
+}
+
+/**
+ * "unknown persona "x"; the valid slugs are a, b and c. "ravensight-playtest personas" lists them."
+ *
+ * @param {Array<string>} unknown
+ * @param {Array<{slug: string}>} available
+ * @returns {string}
+ */
+export function unknownPersonaMessage(unknown, available) {
+  const slugs = (available || []).map(persona => persona.slug);
+  const listed = slugs.length > 1 ? `${slugs.slice(0, -1).join(', ')} and ${slugs[slugs.length - 1]}` : slugs.join('');
+  return `unknown persona${unknown.length === 1 ? '' : 's'} ${unknown.map(slug => JSON.stringify(slug)).join(', ')}; the valid slugs are ${listed || '(none in this pack)'}. "ravensight-playtest personas" lists them.`;
+}

package/src/state/index.js

@@ -100,6 +100,9 @@ export async function listStates(options = {}) {
       state
     });
   }
-  found.sort((a, b) => String(b.updated_at).localeCompare(String(a.updated_at)));
+  // Two journals written in the same millisecond tie on updated_at; the job
+  // id breaks the tie (descending, like the time), so the order never
+  // depends on how the filesystem lists the directory.
+  found.sort((a, b) => String(b.updated_at).localeCompare(String(a.updated_at)) || String(b.job_id).localeCompare(String(a.job_id)));
   return found;
 }

package/src/ui/index.js

@@ -184,6 +184,19 @@ export async function choose(question, labels, options = {}) {
   }
 }
 
+/**
+ * Bold on a terminal that shows it, plain everywhere else (a pipe, CI, a
+ * log file, NO_COLOR), so a saved log never carries escape codes.
+ *
+ * @param {string} text
+ * @param {{isTTY?: boolean}} [target] the stream it will be written to
+ * @returns {string}
+ */
+export function bold(text, target = stream) {
+  if (process.env.NO_COLOR || !(target && target.isTTY)) return String(text);
+  return `\u001b[1m${text}\u001b[22m`;
+}
+
 /** Cents as dollars, the one place that formatting exists. */
 export function money(cents) {
   if (cents === null || cents === undefined) return 'unknown';
@@ -219,5 +232,6 @@ export const ui = {
   confirm,
   choose,
   money,
-  bytes
+  bytes,
+  bold
 };