ravensight-playtest 0.2.5 → 0.2.7

package/README.md

@@ -223,11 +223,41 @@ from the page: the browser driver sets `window.__RAVENSIGHT_PLAYTEST__` and
 `localStorage['ravensight.playtest']` before the game's scripts run. It is never
 sent as a request header, which would reach your game's own API too.
 
+With no ingest key configured, a Godot run says "no ingest key configured, so
+no synthetic telemetry was sent". A web run says "synthetic telemetry comes from
+the game's own SDK on the page, sent with the run's playtest token" instead,
+because the page's SDK already sends the game's events marked synthetic.
+
 When a run reaches its model budget before the report step, nothing more is
 asked of the model. The persona's notes are written to a partial `report.json`
 (`partial: true`, one unreviewed finding per note with the nearest screenshot as
 evidence) and uploaded, so the job's aggregate still counts them.
 
+When the budget runs out during the report step itself, the same partial report
+is uploaded and the run is completed as `budget_exceeded`. The job does not exit 2 for it, and the CLI
+prints "Run ran out of budget; the partial report was uploaded and the refund
+decision is the server's". When the server's answer says a refund is
+expected, the line ends "a refund of $X.XX is expected when the job settles"
+instead, and "the run was refunded" once the job has settled and the run was
+refunded. If the server refused the partial report, it says so and where the
+notes are instead. An older API that refuses that completion prints its own
+message, the run is completed as `failed`, and the job exits 1.
+
+How the report step reads what happened:
+
+- Console errors are recorded as automatic notes. A 4xx on a request (`Failed
+  to load resource`, `HTTP 404`, `status: 401`, `401 Unauthorized` and similar)
+  is a network note: context, not a bug. A finding whose only evidence is
+  automatic network notes is capped at `minor` by the CLI, and its description
+  says so.
+- Every note carries `after`, the URL and title of the screen that followed
+  it. A blocker or major finding has to cite the observation where the failure
+  is visible and must not be contradicted by that next screen or by a later
+  persona note.
+- The model is told never to write calendar dates. `report.md` carries the
+  CLI's own `Generated at` timestamp under its title, and the aggregate report's
+  header uses its `generated_at`.
+
 ### fix
 
 Takes one finding, attempts the change on a branch of its own, replays the
@@ -383,7 +413,7 @@ start avoids that.
 |---|---|
 | 0 | Success |
 | 1 | Failure |
-| 2 | Budget exceeded. Partial results were written and uploaded. |
+| 2 | Budget exceeded during play. Partial results were written and uploaded. A run that ran out during the report step and was completed does not make the job exit 2. |
 | 3 | Environment check failed |
 | 4 | Authentication failed |
 | 5 | Model API error after retries |
@@ -395,7 +425,7 @@ CI can treat 2 as a success with a warning.
 
 | Variable | Effect |
 |---|---|
-| `RAVENSIGHT_TOKEN` | Use this token instead of the keychain. The CI path. |
+| `RAVENSIGHT_TOKEN` | Use this token instead of the keychain. The CI path. It must be a `gt_cli_` token, the kind `login` stores. A `gt_mcp_` token is an MCP personal access token and a `gt_live_` token is a game's ingest key; `check` refuses both and says which one it saw. |
 | `RAVENSIGHT_API_URL` | The API base URL. Defaults to `https://api.ravensight.io`. |
 | `RAVENSIGHT_DASHBOARD_URL` | Where the dashboard is, if it is not derivable from the API URL. |
 | `RAVENSIGHT_GAME_ID` | The game, when there is no linked repo. |

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "ravensight-playtest",
-  "version": "0.2.5",
+  "version": "0.2.7",
   "description": "AI personas play your game and file reports to Ravensight.",
   "license": "MIT",
   "homepage": "https://ravensight.io/docs/#playtest",

package/src/auth/session.js

@@ -68,6 +68,10 @@ export function looksLikeIngestKey(token) {
   return typeof token === 'string' && token.startsWith('gt_live_');
 }
 
+export function looksLikeMcpToken(token) {
+  return typeof token === 'string' && token.startsWith('gt_mcp_');
+}
+
 /**
  * What to do when the server refuses the credential.
  *

package/src/commands/check.js

@@ -1,7 +1,7 @@
 import { createClient } from '../api/client.js';
 import { ApiError } from '../api/errors.js';
 import { resolveApiUrl, paths } from '../paths.js';
-import { resolveToken, looksLikeIngestKey } from '../auth/session.js';
+import { resolveToken, looksLikeIngestKey, looksLikeMcpToken } from '../auth/session.js';
 import { backendName } from '../auth/keychain.js';
 import { loadConfig } from '../config.js';
 import { packs } from '../packs/index.js';
@@ -183,12 +183,19 @@ export async function runChecks(flags = {}, deps = {}) {
       'that is an ingest key, not a CLI token',
       'run login, or set RAVENSIGHT_TOKEN to a gt_cli_ token'
     ));
+  } else if (looksLikeMcpToken(token)) {
+    rows.push(row(
+      'token valid',
+      'fail',
+      'This is an MCP personal access token (gt_mcp_). The CLI needs its own token (gt_cli_): run `ravensight-playtest login`, or set RAVENSIGHT_TOKEN to a gt_cli_ token in CI.',
+      'ravensight-playtest login'
+    ));
   } else if (!token) {
     rows.push(row('token valid', 'fail', `not logged in to ${host}`, 'ravensight-playtest login'));
   }
 
   let whoami = null;
-  if (token && !looksLikeIngestKey(token)) {
+  if (token && !looksLikeIngestKey(token) && !looksLikeMcpToken(token)) {
     const authed = deps.authedApi || createClient({ apiUrl, token });
     try {
       whoami = await authed.cli.whoami();

package/src/commands/fakeCore.js

@@ -18,13 +18,17 @@ const SKILL = ['---', 'name: persona-playtest', 'prompt_version: persona-playtes
 const PROFILE_SKILL = ['---', 'name: game-profile', 'prompt_version: game-profile@0.1.0', '---', '# Game Profile'].join('\n');
 const FIX_SKILL = ['---', 'name: fix', 'prompt_version: fix@0.1.0', '---', '# Fix'].join('\n');
 
-/** The server's table, verbatim. */
+/**
+ * The server's table, verbatim, including the change that lets a run out of
+ * money in its report step end as budget_exceeded from `reporting` or
+ * `uploading`.
+ */
 export const RUN_TRANSITIONS = Object.freeze({
   queued: ['launching', 'canceled', 'skipped', 'failed', 'interrupted'],
   launching: ['playing', 'failed', 'canceled', 'interrupted'],
   playing: ['reporting', 'failed', 'canceled', 'budget_exceeded', 'interrupted'],
-  reporting: ['uploading', 'failed', 'interrupted'],
-  uploading: ['succeeded', 'failed', 'interrupted'],
+  reporting: ['uploading', 'failed', 'budget_exceeded', 'interrupted'],
+  uploading: ['succeeded', 'failed', 'budget_exceeded', 'interrupted'],
   interrupted: ['launching', 'playing', 'reporting', 'uploading', 'failed', 'canceled'],
   succeeded: [],
   failed: [],
@@ -33,6 +37,13 @@ export const RUN_TRANSITIONS = Object.freeze({
   skipped: []
 });
 
+/** An API from before that change, which refuses budget_exceeded after `playing`. */
+export const OLDER_RUN_TRANSITIONS = Object.freeze({
+  ...RUN_TRANSITIONS,
+  reporting: ['uploading', 'failed', 'interrupted'],
+  uploading: ['succeeded', 'failed', 'interrupted']
+});
+
 export const RUN_TERMINAL_STATES = Object.freeze(['succeeded', 'failed', 'canceled', 'budget_exceeded', 'skipped']);
 
 /** Shaped like cli-core's `ApiError`: `status`, `code`, `body`. */
@@ -234,7 +245,10 @@ export function fakeCore({
   findings = [fakeFinding()],
   triage = { verdict: 'bug', confidence: 0.9, source: 'brief' },
   graphContext = { levels: [{ name: 'title', exits: ['level_1'] }] },
-  verifyVerdict = { reproduced: 0.1, source: 'judge' }
+  verifyVerdict = { reproduced: 0.1, source: 'judge' },
+  transitions = RUN_TRANSITIONS,
+  completeExtras = {},
+  runRefund = {}
 } = {}) {
   const calls = [];
   const record = (name, payload) => calls.push({ name, ...payload });
@@ -250,7 +264,7 @@ export function fakeCore({
 
   const move = (runId, to) => {
     const from = runStates.get(runId) || 'queued';
-    if (!(RUN_TRANSITIONS[from] || []).includes(to)) {
+    if (!(transitions[from] || []).includes(to)) {
       throw new FakeApiError({
         status: 409,
         code: 'invalid_transition',
@@ -347,8 +361,23 @@ export function fakeCore({
         if (RUN_TERMINAL_STATES.includes(from)) {
           throw new FakeApiError({ status: 409, code: 'run_already_complete', message: 'That run has already finished.', body: { state: from } });
         }
-        runStates.set(runId, body.state);
-        return { run: { run_id: runId, state: body.state }, dropped_findings: 0, report_lint: [] };
+        // The server's complete reads the table too, after advancing a
+        // `reporting` run that is succeeding to `uploading` itself.
+        const state = body.state || 'succeeded';
+        const effective = from === 'reporting' && state === 'succeeded' ? 'uploading' : from;
+        if (!(transitions[effective] || []).includes(state)) {
+          throw new FakeApiError({ status: 409, code: 'invalid_transition', message: `A run cannot go from "${effective}" to "${state}".`, body: { from: effective, to: state } });
+        }
+        runStates.set(runId, state);
+        // The server's refund fields ride on `run`. Before the job settles
+        // `refunded` is false and `refund_expected` says whether settling
+        // would give the run's unit back.
+        return {
+          run: { run_id: runId, state, refunded: false, refunded_cents: 0, refund_expected: false, refund_expected_cents: 0, ...runRefund },
+          dropped_findings: 0,
+          report_lint: [],
+          ...completeExtras
+        };
       },
       async list(gameId, jobId) {
         record('list', { jobId });

package/src/commands/run.js

@@ -433,7 +433,14 @@ export async function playJob({
     log
   });
 
+  // A run that ran out of money in the report step, after its partial report
+  // was uploaded and the run completed, delivered what it bought: it does not
+  // make the job exit 2. One the server refused to complete exits 1, never 2,
+  // because 2 promises the partial results were shipped.
+  const playBudgetHit = results.some(result => result.state === 'budget_exceeded' && !result.reportStepBudget);
   const budgetHit = results.some(result => result.state === 'budget_exceeded');
+  const reportBudgetRefused = results.some(result => result.reportStepBudget === 'refused');
+  const reportBudgetDelivered = results.some(result => result.reportStepBudget === 'delivered' || result.reportStepBudget === 'dry_upload');
   const canceled = results.some(result => result.state === 'canceled');
   const anySucceeded = results.some(result => result.state === 'succeeded');
 
@@ -449,8 +456,9 @@ export async function playJob({
   log(`Reports: ${directory}`);
 
   if (canceled) return EXIT.CANCELED;
-  if (budgetHit) return EXIT.BUDGET;
-  return anySucceeded ? EXIT.OK : EXIT.FAILED;
+  if (reportBudgetRefused) return EXIT.FAILED;
+  if (playBudgetHit) return EXIT.BUDGET;
+  return anySucceeded || reportBudgetDelivered ? EXIT.OK : EXIT.FAILED;
 }
 
 /**
@@ -554,7 +562,47 @@ export async function playPersonaRun({
     if (inFlight) inFlight.delete(runId);
     const failed = error.outcome || { state: 'failed', quitReason: 'error', actionsTaken: 0, report: null, reportLint: [], reachedState: 'playing' };
     await stateFile.patchRun(repoRoot, journal, runId, { state: failed.state, error: String(error && error.message), quit_reason: failed.quitReason });
-    await deliver({ deps, api, gameId, jobId, runId, directory, outcome: failed, options, log });
+    const delivery = {};
+    const answer = await deliver({ deps, api, gameId, jobId, runId, directory, outcome: failed, options, log, delivery });
+    if (isReportStepBudget(failed)) {
+      // The money ran out while the report was being written. The partial
+      // report went up and the run is completed as budget_exceeded from
+      // `uploading`; what happens to the charge is the server's decision.
+      if (answer) {
+        await stateFile.patchRun(repoRoot, journal, runId, { uploaded: true, completed: true });
+        // "Uploaded" only when it was: the server can refuse the partial
+        // report (and the run is completed without it), and the report may
+        // never have been written.
+        const message = delivery.reportRefused || !failed.report
+          ? `Run ran out of budget; the server refused the partial report; the persona's notes are in ${directory}`
+          : reportBudgetMessage(answer);
+        log(`run ${runId} (${persona.slug}): ${message}`);
+        return { runId, persona: persona.slug, state: failed.state, quitReason: failed.quitReason, reportStepBudget: 'delivered' };
+      }
+      if (options.dryUpload) {
+        // Nothing was sent, so the server still holds the run where it was.
+        // Not a refusal, so the job does not exit 1 for it.
+        log(`run ${runId} (${persona.slug}) ran out of budget while writing its report: dry upload, run not completed on the server.`);
+        return { runId, persona: persona.slug, state: failed.state, quitReason: failed.quitReason, reportStepBudget: 'dry_upload' };
+      }
+      log(`run ${runId} (${persona.slug}) ran out of budget while writing its report, and the server would not complete it: ${delivery.message || 'no reason given'}`);
+      // An older API refuses budget_exceeded after `playing`. The CLI moved
+      // the run to `uploading` itself, so it does not leave it there: `failed`
+      // is allowed from `uploading` on every table.
+      try {
+        await api.runs.complete(gameId, jobId, runId, {
+          state: 'failed',
+          reason: `ran out of budget while writing the report; the server refused budget_exceeded (${delivery.message || 'no reason given'})`,
+          actions_taken: failed.actionsTaken,
+          quit_reason: failed.quitReason
+        });
+        await stateFile.patchRun(repoRoot, journal, runId, { completed: true, state: 'failed' });
+        log(`run ${runId} (${persona.slug}): completed as failed instead.`);
+      } catch (fallbackError) {
+        log(`run ${runId} (${persona.slug}): could not be completed as failed either (${String(fallbackError && fallbackError.message)}). "ravensight-playtest resume" can retry it.`);
+      }
+      return { runId, persona: persona.slug, state: failed.state, quitReason: failed.quitReason, reportStepBudget: 'refused', error };
+    }
     log(`run ${runId} (${persona.slug}) failed: ${String(error && error.message)}`);
     return { runId, persona: persona.slug, state: failed.state, quitReason: failed.quitReason, error };
   }
@@ -576,6 +624,9 @@ export async function playPersonaRun({
       apiUrl,
       ingestKey: (config && config.ingest_key) || process.env.RAVENSIGHT_INGEST_KEY || null,
       playtestToken: entry.playtest_token,
+      // The web driver injects the playtest token into the page (see
+      // driverOptions), so the game's own SDK sends with it.
+      pageSdk: driverName === 'playwright_web' && Boolean(entry.playtest_token),
       context: { jobId, runId, persona: persona.slug, gameVersion: (config && config.game_version) || 'unknown' },
       report: outcome.report,
       driver: driverName,
@@ -604,7 +655,7 @@ export async function playPersonaRun({
  * uploaded and completed in exactly this order, including the `uploading`
  * transition the server's table allows from `reporting` and nowhere else.
  */
-export async function deliver({ deps, api, gameId, jobId, runId, directory, outcome, options, log, journal, repoRoot }) {
+export async function deliver({ deps, api, gameId, jobId, runId, directory, outcome, options, log, journal, repoRoot, delivery = null }) {
   const body = {
     state: outcome.state,
     reason: outcome.quitDetail || '',
@@ -655,6 +706,7 @@ export async function deliver({ deps, api, gameId, jobId, runId, directory, outc
       // report.json either way.
       if (!(body.report && body.report.partial && error && error.status === 422 && error.code === 'invalid_report')) throw error;
       log(`run ${runId}: the server refused the partial report (${String(error.message)}), so the run is completed without it. The notes are in ${directory}.`);
+      if (delivery) delivery.reportRefused = true;
       const { report: ignored, report_md: alsoIgnored, ...withoutReport } = body;
       answer = await api.runs.complete(gameId, jobId, runId, withoutReport);
     }
@@ -667,11 +719,54 @@ export async function deliver({ deps, api, gameId, jobId, runId, directory, outc
     }
     return answer;
   } catch (error) {
+    if (delivery) {
+      delivery.message = String(error && error.message);
+      delivery.status = error && error.status;
+      delivery.code = error && error.code;
+    }
     log(`run ${runId}: upload or complete was refused (${String(error && error.message)}). The artifacts are in the run directory and "ravensight-playtest resume" can retry it.`);
     return null;
   }
 }
 
+/**
+ * Whether a run ran out of money in the report step: it is budget_exceeded
+ * and the server had already accepted `reporting` (or `uploading`) for it.
+ *
+ * @param {Object} outcome
+ * @returns {boolean}
+ */
+export function isReportStepBudget(outcome) {
+  return Boolean(outcome) && outcome.state === 'budget_exceeded' && ['reporting', 'uploading'].includes(outcome.reachedState);
+}
+
+/**
+ * What the developer is told when a report-step budget run was completed.
+ * The refund is the server's decision; the CLI only repeats what the
+ * complete answer says about it. Before the job settles the answer carries
+ * `refund_expected` and `refund_expected_cents`; `refunded` is true only once
+ * settlement gave the unit back. Each may be on the answer or on its `run`.
+ *
+ * @param {Object} answer the complete response
+ * @returns {string}
+ */
+export function reportBudgetMessage(answer) {
+  const read = name => {
+    if (!answer) return undefined;
+    if (answer[name] !== undefined) return answer[name];
+    return answer.run ? answer.run[name] : undefined;
+  };
+  const prefix = 'Run ran out of budget; the partial report was uploaded and';
+  if (read('refunded') === true) return `${prefix} the run was refunded`;
+  if (read('refund_expected') === true) {
+    const cents = read('refund_expected_cents');
+    return Number.isFinite(Number(cents)) && cents !== null && Number(cents) > 0
+      ? `${prefix} a refund of ${formatCents(cents)} is expected when the job settles`
+      : `${prefix} a refund is expected when the job settles`;
+  }
+  return `${prefix} the refund decision is the server's`;
+}
+
 /**
  * Build the aggregate, and drive the `aggregate_report` run's own state machine
  * while doing it.

package/src/run/aggregate.js

@@ -447,6 +447,7 @@ export async function judgeAggregate({ client, jobId, runId, ranked, inputs, bri
     ' "known":["iss_..."], "excluded_by_intent":[{"issue_id":"iss_...","intent":"<the stated intent it matches>"}],',
     ' "tldr":["up to five short bullets: the most important fixes and the strongest positive signal"]}',
     '',
+    'Never write calendar dates or times in the tldr or anywhere else; the runner stamps generated_at on the report.',
     'Rules. A criterion no run touched is "not testable"; never invent evidence for one. An issue is "known" only when the brief already describes it. An issue is excluded by intent only when the persona experience is CONSISTENT with the stated intent; when it CONTRADICTS the intent, leave it in the list, because that is the valuable case.',
     '',
     `success criteria:\n${criteria.length > 0 ? criteria.map(row => `  ${row.criterion_id}: ${row.text}`).join('\n') : '  (none stated)'}`,

package/src/run/drivers/web.js

@@ -361,6 +361,16 @@ export function createWebDriver({ importer, headless = true, playtest = null } =
         url: current.url(),
         elapsedMs: Date.now() - started
       };
+      // The page title is what a note's "after" state is checked against in
+      // the report step. A page that cannot answer has no title, not an error.
+      try {
+        if (typeof current.title === 'function') {
+          const title = await current.title();
+          if (title) observation.title = String(title);
+        }
+      } catch {
+        // No title is still an observation.
+      }
       // Canvas builds (a Godot or Unity web export is one <canvas> and
       // nothing else) have no accessibility tree worth reading, so vision
       // is the only way to play them. Spend an image only then.

package/src/run/personaLoop.js

@@ -51,8 +51,22 @@ const nextOrder = () => {
   return ORDER;
 };
 
-/** `Failed to load resource: ... status of 4xx`: a missing asset or a refused call, not a crash. */
-const RESOURCE_4XX = /Failed to load resource: the server responded with a status of 4\d\d/;
+/**
+ * A 4xx on a request, in the shapes a browser console and a game's own logging
+ * print it: `Failed to load resource: ... status of 4xx`, `HTTP 404`,
+ * `status: 401`, `statusCode=403`, `401 Unauthorized`. A missing asset or a
+ * refused call, not a crash.
+ */
+export const RESOURCE_4XX = /Failed to load resource: the server responded with a status of 4\d\d|\bHTTP 4\d\d\b|status(?:Code)?\s*[:=]?\s*4\d\d\b|\b4\d\d (?:Unauthorized|Forbidden|Not Found|Bad Request)\b/i;
+
+/** The screenshots the runner takes on its own, which no persona described. */
+const RUNNER_SCREENSHOTS = Object.freeze(['landing', 'console-error', 'final']);
+
+/** Severities above `minor`, which network notes alone cannot earn. */
+const ABOVE_MINOR = Object.freeze(['blocker', 'major']);
+
+/** What a capped finding's description gains, so the reader knows why. */
+export const NETWORK_CAP_NOTE = 'Severity capped at minor by the runner: the only evidence is automatic network notes, and a failed request is a finding above minor only when the persona saw the game break because of it.';
 
 /** A persona note kind as a report finding category. */
 export const NOTE_CATEGORY = Object.freeze({ bug: 'bug', confusing: 'ux', suggestion: 'ux', good: 'ux', reaction: 'ux' });
@@ -127,6 +141,10 @@ export async function runPersona(args) {
   const startedAt = new Date().toISOString();
   const notes = [];
   const screenshots = [];
+  // Every observation the persona was shown, by index, so a note can carry
+  // what the screen looked like next and a finding can cite where a failure
+  // is visible.
+  const observations = [];
   let imageBudget = maxImages;
 
   await mkdir(path.join(runDirectory, 'screenshots'), { recursive: true });
@@ -186,6 +204,7 @@ export async function runPersona(args) {
       ledger,
       notes,
       screenshots,
+      observations,
       imageBudget,
       keepObservations,
       onImageBudget: next => {
@@ -242,6 +261,7 @@ export async function runPersona(args) {
       ledger,
       notes,
       screenshots,
+      observations,
       play,
       startedAt,
       endedAt,
@@ -305,6 +325,7 @@ async function playLoop({
   ledger,
   notes,
   screenshots,
+  observations = [],
   imageBudget,
   keepObservations,
   heartbeat,
@@ -343,7 +364,26 @@ async function playLoop({
   let wrapUpLeft = Infinity;
   const maxTurns = maxActions * 3 + 20;
 
+  // Notes waiting for the next observation, which is the screen they are
+  // checked against: a failure the next screen does not show is not a
+  // blocker.
+  let awaitingAfter = [];
+  const observed = observation => {
+    const entry = { index: observations.length + 1, step: observation.step, url: observation.url || null, title: observationTitle(observation) };
+    observations.push(entry);
+    for (const note of awaitingAfter) note.after = { url: entry.url, title: entry.title };
+    awaitingAfter = [];
+    return entry;
+  };
+  const noted = note => {
+    const entry = { ...note, at_action: actions, observation: observations.length, order: nextOrder() };
+    notes.push(entry);
+    awaitingAfter.push(entry);
+    return entry;
+  };
+
   const first = await driver.observe();
+  observed(first);
   latestObservation = first.text;
   visualObservation = Boolean(first.screenshotBase64);
   transcript.write({ kind: 'observation', step: first.step, text: first.text, url: first.url });
@@ -458,7 +498,7 @@ async function playLoop({
           // (bug, confusing, good, suggestion, reaction) would otherwise
           // overwrite the transcript line's `kind`, and every note would
           // read as a different sort of line.
-          notes.push({ ...note, at_action: actions, order: nextOrder() });
+          noted(note);
           transcript.write({ kind: 'note', tool: toolUse.name, note });
         },
         onScreenshot: shot => {
@@ -479,6 +519,7 @@ async function playLoop({
         actions
       });
       if (executed.observation) {
+        observed(executed.observation);
         latestObservation = executed.observation.text;
         visualObservation = Boolean(executed.observation.screenshotBase64);
         recentActions.push(`${toolUse.name}: ${executed.changed ? 'screen changed' : 'no observed change'}`);
@@ -513,8 +554,8 @@ async function playLoop({
       const shot = await safeScreenshot(driver, 'console-error', screenshots, transcript);
       for (const entry of consoleErrors) {
         const kind = RESOURCE_4XX.test(String(entry.text)) ? 'network' : 'bug';
-        const note = { kind, text: `${entry.type}: ${entry.text}`, automatic: true };
-        notes.push({ ...note, at_action: actions, order: nextOrder() });
+        const note = { kind, text: `${entry.type}: ${entry.text}`, automatic: true, ...(shot ? { screenshot: shot.relativePath } : {}) };
+        noted(note);
         transcript.write({ kind: 'note', note });
       }
       asides.push(`The game logged ${consoleErrors.length} error(s): ${consoleErrors.map(entry => entry.text).join(' | ').slice(0, 800)}${shot ? ` A screenshot is saved at ${shot.relativePath}.` : ''}`);
@@ -584,6 +625,7 @@ async function reportPhase({
   ledger,
   notes,
   screenshots,
+  observations = [],
   play,
   startedAt,
   endedAt,
@@ -600,9 +642,16 @@ async function reportPhase({
         `actions taken: ${play.actionsTaken}   stop reason: ${play.quitReason}${play.quitDetail ? ` (${play.quitDetail})` : ''}`,
         '',
         'Notes you logged, in order:',
-        notes.length > 0 ? notes.map((note, i) => `${i + 1}. [${note.kind}${note.severity ? `/${note.severity}` : ''}] ${note.text}`).join('\n') : '(none)',
+        notes.length > 0 ? notes.map((note, i) => noteLine(note, i)).join('\n') : '(none)',
+        '',
+        'Notes marked automatic were recorded by the runner from the console, with no severity: decide one from what the persona saw. Network notes are context, not bugs: a failed request is a finding only when the persona saw a broken screen because of it. A finding whose only evidence is automatic network notes is capped at minor by the runner.',
+        '',
+        'Observations the persona was shown, by index:',
+        observationLines(observations),
         '',
-        'Notes marked automatic were recorded by the runner from the console, with no severity: decide one from what the persona saw. Network notes are context, not bugs: a failed request is a finding only when the persona saw a broken screen because of it.',
+        'Each note carries "after": the URL and title of the observation that followed it. A blocker or major finding must cite the observation index where the failure is visible, as evidence {"type": "transcript_step", "ref": "observation N"}, and must not be contradicted by its note\'s "after" state or by a later persona note: if the next screen or a later note shows the game working, it is not a blocker or a major. Cite a note as {"type": "transcript_step", "ref": "note N"}.',
+        '',
+        'Never write calendar dates or times in the prose of report.md or report.json. The runner stamps when the report was generated.',
         '',
         'Screenshots available as evidence (use these exact paths in evidence refs):',
         screenshots.length > 0 ? screenshots.map(shot => `- ${shot.relativePath}${shot.reason ? ` (${shot.reason})` : ''}`).join('\n') : '(none)',
@@ -669,7 +718,7 @@ async function reportPhase({
       quitReason: play.quitReason,
       quitDetail: play.quitDetail
     };
-    const candidate = parsed ? stampReport(parsed, facts) : null;
+    const candidate = parsed ? capNetworkOnlyFindings(stampReport(parsed, facts), { notes, screenshots }) : null;
     const validation = candidate ? validator.validate(candidate) : { ok: false, details: [{ instancePath: '/', message: parseError || 'report.json was not written' }] };
     const evidenceProblems = candidate ? await checkEvidence(candidate, runDirectory) : [];
     // The server lints the same four sections at `complete` and records the
@@ -715,6 +764,7 @@ async function reportPhase({
   if (!markdown) {
     markdown = renderFallbackMarkdown({ persona, driverName, target, notes, screenshots, play, reason: 'The report step wrote report.json but no report.md.' });
   }
+  markdown = stampGeneratedAt(markdown, new Date().toISOString());
 
   await writeReportFiles({ runDirectory, markdown, report });
   return { report, markdown, lint };
@@ -1033,17 +1083,148 @@ async function writePartialReport({ outcome, runDirectory, persona, driverName,
     play,
     reason: 'The run reached its model budget before the report step. What the persona saw is below, and report.json carries the notes as unreviewed findings.'
   });
-  await writeReportFiles({ runDirectory, markdown, report });
+  const stamped = stampGeneratedAt(markdown, new Date().toISOString());
+  await writeReportFiles({ runDirectory, markdown: stamped, report });
   outcome.report = report;
-  outcome.reportMarkdown = markdown;
+  outcome.reportMarkdown = stamped;
 }
 
 async function writePartialMarkdown({ runDirectory, persona, driverName, target, notes, screenshots, play, reason }) {
-  const markdown = renderFallbackMarkdown({ persona, driverName, target, notes, screenshots, play, reason });
+  const markdown = stampGeneratedAt(renderFallbackMarkdown({ persona, driverName, target, notes, screenshots, play, reason }), new Date().toISOString());
   assertNoSecrets('report.md', markdown);
   await writeFile(path.join(runDirectory, RUN_FILES.reportMd), markdown, 'utf8');
 }
 
+/**
+ * A note as one numbered line of the report prompt: its kind, whether the
+ * runner recorded it, the observation it was made on and the screen that
+ * followed it.
+ *
+ * @param {Object} note
+ * @param {number} i
+ * @returns {string}
+ */
+export function noteLine(note, i) {
+  const tags = [
+    note.automatic ? 'automatic' : null,
+    Number.isInteger(note.observation) && note.observation > 0 ? `seen at observation ${note.observation}` : null,
+    note.screenshot ? `screenshot ${note.screenshot}` : null
+  ].filter(Boolean);
+  const after = note.after
+    ? ` after: ${JSON.stringify({ url: note.after.url ?? null, title: note.after.title ?? null })}`
+    : ' after: (no later observation)';
+  return `${i + 1}. [${note.kind}${note.severity ? `/${note.severity}` : ''}] ${note.text}${tags.length > 0 ? ` (${tags.join(', ')})` : ''}${after}`;
+}
+
+/** The most observations listed in the report prompt; the newest are kept. */
+export const MAX_OBSERVATIONS_IN_REPORT = 200;
+
+function observationLines(observations) {
+  if (!observations || observations.length === 0) return '(none)';
+  const shown = observations.slice(-MAX_OBSERVATIONS_IN_REPORT);
+  const lines = shown.map(entry => `observation ${entry.index}: step ${entry.step}${entry.url ? ` ${entry.url}` : ''}${entry.title ? ` "${entry.title}"` : ''}`);
+  if (shown.length < observations.length) lines.unshift(`(the first ${observations.length - shown.length} are left out)`);
+  return lines.join('\n');
+}
+
+/**
+ * A short title for an observation: the page title when the driver has one,
+ * otherwise the first line of what was observed.
+ *
+ * @param {Object} observation
+ * @returns {string|null}
+ */
+export function observationTitle(observation) {
+  if (observation && typeof observation.title === 'string' && observation.title.trim()) return observation.title.trim().slice(0, 120);
+  const line = String((observation && observation.text) || '').split('\n').map(part => part.trim()).find(Boolean);
+  return line ? line.slice(0, 80) : null;
+}
+
+/**
+ * Cap at `minor` every finding whose evidence is only automatic network
+ * notes, in code rather than by asking.
+ *
+ * A finding rests only on network notes when it cites at least one (a
+ * `note N` ref to an automatic network note, or the runner's console-error
+ * screenshot whose notes were all network notes) and nothing else: no persona
+ * note, no automatic non-network note, no observation ref, no screenshot the
+ * persona took, no other evidence type. Only the runner's own landing and
+ * final screenshots neither count for nor against it.
+ *
+ * @param {Object} report a stamped report
+ * @param {{notes: Array<Object>, screenshots: Array<Object>}} context
+ * @returns {Object} the report, with capped findings rewritten
+ */
+export function capNetworkOnlyFindings(report, { notes = [], screenshots = [] } = {}) {
+  if (!report || !Array.isArray(report.findings)) return report;
+  const byShot = new Map();
+  for (const note of notes) {
+    if (!note || !note.automatic || !note.screenshot) continue;
+    if (!byShot.has(note.screenshot)) byShot.set(note.screenshot, []);
+    byShot.get(note.screenshot).push(note);
+  }
+  const reasons = new Map(screenshots.filter(shot => shot && shot.relativePath).map(shot => [shot.relativePath, String(shot.reason || '')]));
+
+  const verdict = finding => {
+    let network = 0;
+    for (const item of Array.isArray(finding.evidence) ? finding.evidence : []) {
+      if (!item) continue;
+      const ref = String(item.ref || '');
+      if (item.type === 'transcript_step') {
+        // An observation ref, or anything that is not a network note, is
+        // other evidence.
+        const match = ref.match(/^\s*note\s*#?\s*(\d+)\s*$/i);
+        const note = match ? notes[Number(match[1]) - 1] : null;
+        if (note && note.automatic && note.kind === 'network') network += 1;
+        else return false;
+      } else if (item.type === 'screenshot') {
+        const attached = byShot.get(ref);
+        if (attached && attached.length > 0) {
+          if (attached.every(note => note.kind === 'network')) network += 1;
+          else return false;
+          continue;
+        }
+        // The runner's own landing and final screenshots say nothing either
+        // way; any screenshot the persona took is other evidence.
+        const reason = reasons.has(ref) ? reasons.get(ref) : null;
+        if (reason !== null && RUNNER_SCREENSHOTS.includes(reason)) continue;
+        return false;
+      } else {
+        return false;
+      }
+    }
+    return network > 0;
+  };
+
+  return {
+    ...report,
+    findings: report.findings.map(finding => {
+      if (!finding || !ABOVE_MINOR.includes(finding.severity) || !verdict(finding)) return finding;
+      const description = String(finding.description || '');
+      return { ...finding, severity: 'minor', description: description ? `${description}\n\n${NETWORK_CAP_NOTE}` : NETWORK_CAP_NOTE };
+    })
+  };
+}
+
+/**
+ * Put the runner's own timestamp at the head of a report.md, right under
+ * its first heading, replacing one stamped before. Whatever the model wrote
+ * in its prose is left alone; the header is what says when the report was
+ * generated.
+ *
+ * @param {string} markdown
+ * @param {string} generatedAt an ISO timestamp
+ * @returns {string}
+ */
+export function stampGeneratedAt(markdown, generatedAt) {
+  const line = `**Generated at:** ${generatedAt}`;
+  const lines = String(markdown || '').split('\n').filter(entry => !/^\*\*Generated at:\*\* /.test(entry));
+  const heading = lines.findIndex(entry => /^# /.test(entry));
+  if (heading === -1) return [line, '', ...lines].join('\n');
+  lines.splice(heading + 1, 0, line);
+  return lines.join('\n');
+}
+
 /**
  * The `prompt_version` the report has to carry, read out of the skill's own
  * frontmatter so a content pack update moves it without a CLI release.
@@ -1056,4 +1237,4 @@ export function promptVersionOf(skill) {
   return match ? match[1] : null;
 }
 
-export default { runPersona, buildSystem, promptVersionOf, shouldEscalate, renderFallbackMarkdown, notesToFindings, nearestScreenshot };
+export default { runPersona, buildSystem, promptVersionOf, shouldEscalate, renderFallbackMarkdown, notesToFindings, nearestScreenshot, capNetworkOnlyFindings, stampGeneratedAt, noteLine, observationTitle };

package/src/run/synthetic.js

@@ -154,6 +154,9 @@ export class SyntheticSession {
   }
 }
 
+/** The run summary's line when the page's own SDK sends the run's telemetry. */
+export const PAGE_SDK_REASON = "synthetic telemetry comes from the game's own SDK on the page, sent with the run's playtest token";
+
 /**
  * Emit the three events a run produces, best effort.
  *
@@ -161,6 +164,8 @@ export class SyntheticSession {
  * @param {string} args.apiUrl
  * @param {string|null} args.ingestKey
  * @param {string|null} args.playtestToken
+ * @param {boolean} [args.pageSdk] the driver injected the playtest token into
+ *   the page (the web driver), so the game's own SDK sends with it
  * @param {Object} args.context
  * @param {Object} args.report the run's report.json
  * @param {string} args.driver
@@ -168,8 +173,14 @@ export class SyntheticSession {
  * @param {(message: string) => void} [args.log]
  * @returns {Promise<{sent: boolean, accepted: number, reason?: string}>}
  */
-export async function emitRunEvents({ apiUrl, ingestKey, playtestToken, context, report, driver, fetchImpl, log }) {
+export async function emitRunEvents({ apiUrl, ingestKey, playtestToken, pageSdk = false, context, report, driver, fetchImpl, log }) {
   if (!ingestKey) {
+    // A web build carries its own SDK, and the web driver injected the run's
+    // playtest token into the page, so the game's own events already arrive
+    // marked synthetic. Saying "nothing was sent" there would be wrong.
+    if (pageSdk && playtestToken) {
+      return { sent: false, accepted: 0, pageSdk: true, reason: PAGE_SDK_REASON };
+    }
     return { sent: false, accepted: 0, reason: 'no ingest key configured, so no synthetic telemetry was sent' };
   }
   if (!playtestToken) {

package/src/states.js

@@ -90,8 +90,10 @@ export const RUN_TRANSITIONS = Object.freeze({
   queued: ['launching', 'canceled', 'skipped', 'failed', 'interrupted'],
   launching: ['playing', 'failed', 'canceled', 'interrupted'],
   playing: ['reporting', 'failed', 'canceled', 'budget_exceeded', 'interrupted'],
-  reporting: ['uploading', 'failed', 'interrupted'],
-  uploading: ['succeeded', 'failed', 'interrupted'],
+  // A run out of money in its report step uploads its partial report and
+  // ends here; the API allows it from 0.2.7's companion server change.
+  reporting: ['uploading', 'failed', 'budget_exceeded', 'interrupted'],
+  uploading: ['succeeded', 'failed', 'budget_exceeded', 'interrupted'],
   interrupted: ['launching', 'playing', 'reporting', 'uploading', 'failed', 'canceled'],
   succeeded: [],
   failed: [],