ravensight-playtest 0.1.0

package/src/upload/index.js

@@ -0,0 +1,467 @@
+import { readdir, readFile, stat } from 'node:fs/promises';
+import { join, sep } from 'node:path';
+import { ApiError, isRetryable } from '../api/errors.js';
+import { backoffDelay } from '../api/http.js';
+import { sha256File } from '../fsutil.js';
+import { CliError } from '../errors.js';
+import {
+  MAX_FILES_PER_RUN,
+  MAX_RUN_BYTES,
+  MAX_VIDEO_BYTES,
+  VIDEO_PATHS,
+  contentTypeFor,
+  skipReason
+} from './allowlist.js';
+import { STATUS, record, uploadedKeys, keyFor } from './queue.js';
+
+const PUT_RETRIES = 4;
+
+/**
+ * Walk a directory into run relative, forward slashed paths. Symlinks are not
+ * followed: a run directory is written by this CLI, and a symlink in it is
+ * either a mistake or a way to get a file from outside the run into the bucket.
+ * @param {string} root
+ * @param {string} [prefix]
+ * @returns {Promise<string[]>}
+ */
+export async function walkRunDir(root, prefix = '') {
+  let entries;
+  try {
+    entries = await readdir(join(root, prefix), { withFileTypes: true });
+  } catch (error) {
+    if (error.code === 'ENOENT') return [];
+    throw error;
+  }
+  const found = [];
+  for (const entry of entries) {
+    const relPath = prefix ? `${prefix}/${entry.name}` : entry.name;
+    if (entry.isDirectory()) {
+      found.push(...await walkRunDir(root, relPath));
+    } else if (entry.isFile()) {
+      found.push(relPath);
+    }
+  }
+  return found.sort();
+}
+
+/**
+ * Turn a directory into the `files` array the presign route wants, plus the
+ * list of everything deliberately left out and why.
+ *
+ * Reports come first in the returned order. That matters when a run is near the
+ * per run byte ceiling: the server plans uploads in the order it is given, so
+ * sending the report before the video is what makes a refusal cost the video
+ * rather than the findings.
+ *
+ * `allowedPaths`, when given, narrows the allowlist to exactly those run
+ * relative names. The job level upload uses it (see JOB_UPLOAD_PATHS) so that a
+ * file sitting in the job directory for some other reason is not swept into the
+ * bucket under a job key.
+ *
+ * @param {string} runDir
+ * @param {{video?: boolean, transcript?: boolean, allowedPaths?: string[]}} [opts]
+ * @returns {Promise<{files: Array, skipped: Array, bytes: number}>}
+ */
+export async function planRunDir(runDir, opts = {}) {
+  const relPaths = await walkRunDir(runDir);
+  const files = [];
+  const skipped = [];
+  const narrowed = Array.isArray(opts.allowedPaths) ? new Set(opts.allowedPaths) : null;
+
+  for (const relPath of relPaths) {
+    const contentType = contentTypeFor(relPath, opts);
+    if (!contentType) {
+      skipped.push({ path: relPath, reason: skipReason(relPath, opts) });
+      continue;
+    }
+    if (narrowed && !narrowed.has(relPath)) {
+      skipped.push({ path: relPath, reason: 'not a job level artifact' });
+      continue;
+    }
+    const info = await stat(join(runDir, relPath.split('/').join(sep)));
+    if (VIDEO_PATHS.includes(relPath) && info.size > MAX_VIDEO_BYTES) {
+      skipped.push({ path: relPath, reason: `video is larger than ${MAX_VIDEO_BYTES} bytes` });
+      continue;
+    }
+    files.push({
+      path: relPath,
+      size: info.size,
+      sha256: await sha256File(join(runDir, relPath.split('/').join(sep))),
+      content_type: contentType
+    });
+  }
+
+  const priority = path => {
+    if (path === 'report.json' || path === 'report.md') return 0;
+    if (path === 'usage.json') return 1;
+    if (path.startsWith('screenshots/')) return 2;
+    return 3;
+  };
+  files.sort((a, b) => priority(a.path) - priority(b.path) || a.path.localeCompare(b.path));
+
+  const bytes = files.reduce((sum, file) => sum + file.size, 0);
+  if (files.length > MAX_FILES_PER_RUN) {
+    throw new CliError(
+      `That run has ${files.length} uploadable files, over the limit of ${MAX_FILES_PER_RUN}.`,
+      1,
+      { hint: 'Drop some screenshots, or raise the screenshot budget in the run config.' }
+    );
+  }
+  if (bytes > MAX_RUN_BYTES) {
+    throw new CliError(
+      `That run is ${bytes} bytes, over the per run limit of ${MAX_RUN_BYTES}.`,
+      1,
+      { hint: 'Turn off the video upload, or re-encode it smaller.' }
+    );
+  }
+  return { files, skipped, bytes };
+}
+
+/**
+ * The verbatim `x-amz-tagging` value for a presigned entry.
+ *
+ * The presign response is the authority: it carries back exactly what was
+ * signed, so the CLI never computes the tag itself. Two shapes are accepted
+ * because the two server tasks that meet here spell it differently: W8's
+ * `presignPut` returns the encoded string, and W4b's upload route currently
+ * passes the tag object through. Either is normalized to the string S3 wants;
+ * a mismatch would be a SignatureDoesNotMatch rather than an untagged object,
+ * so guessing is not an option and accepting both is.
+ *
+ * @param {unknown} tagging
+ * @returns {string|null}
+ */
+export function taggingHeader(tagging) {
+  if (tagging === null || tagging === undefined || tagging === '') return null;
+  if (typeof tagging === 'string') return tagging;
+  if (typeof tagging === 'object' && !Array.isArray(tagging)) {
+    const params = new URLSearchParams();
+    for (const [key, value] of Object.entries(tagging)) params.set(key, String(value));
+    const encoded = params.toString();
+    return encoded || null;
+  }
+  return null;
+}
+
+/**
+ * PUT one file at a presigned URL.
+ *
+ * The headers are exactly the ones the URL was signed with: the content type,
+ * `x-amz-tagging` when the presign said so, and `x-amz-checksum-sha256` with
+ * the base64 digest the presign echoed back. The checksum header is what
+ * makes S3 hash the body it receives and refuse one that differs, and what
+ * lets the server's `complete` read the checksum back; a presign that names
+ * none is an older server, and the header is simply not sent. An extra
+ * signed header that is missing, or a value that differs by one character,
+ * is a SignatureDoesNotMatch from S3 and not a retryable failure, so nothing
+ * is added here on a guess.
+ *
+ * @param {Object} options
+ * @param {{path: string, put_url: string, tagging?: unknown, checksum_sha256?: string}} options.entry
+ * @param {string} options.contentType
+ * @param {Buffer} options.body
+ * @param {typeof fetch} [options.fetchImpl]
+ * @param {number} [options.retries]
+ * @param {Function} [options.sleep]
+ * @param {() => number} [options.random]
+ * @returns {Promise<void>}
+ */
+export async function putFile(options) {
+  const {
+    entry,
+    contentType,
+    body,
+    fetchImpl = globalThis.fetch,
+    retries = PUT_RETRIES,
+    sleep = ms => new Promise(resolve => setTimeout(resolve, ms)),
+    random = Math.random
+  } = options;
+
+  const headers = { 'content-type': contentType };
+  const tagging = taggingHeader(entry.tagging);
+  if (tagging) headers['x-amz-tagging'] = tagging;
+  if (typeof entry.checksum_sha256 === 'string' && entry.checksum_sha256.length > 0) {
+    headers['x-amz-checksum-sha256'] = entry.checksum_sha256;
+  }
+
+  let lastError = null;
+  for (let attempt = 1; attempt <= retries + 1; attempt += 1) {
+    let response = null;
+    try {
+      response = await fetchImpl(entry.put_url, { method: 'PUT', headers, body });
+    } catch (cause) {
+      lastError = new ApiError({
+        status: 0,
+        code: 'network_error',
+        message: `PUT ${entry.path} failed: ${cause.message}`,
+        method: 'PUT',
+        path: entry.path,
+        cause
+      });
+    }
+
+    if (response && response.ok) return;
+    if (response) {
+      const text = await response.text().catch(() => '');
+      lastError = new ApiError({
+        status: response.status,
+        code: response.status === 403 ? 'presign_expired' : `s3_${response.status}`,
+        message: `PUT ${entry.path} answered ${response.status}: ${text.slice(0, 300)}`,
+        method: 'PUT',
+        path: entry.path
+      });
+      // A 403 is the expired or otherwise invalid URL, which a retry against
+      // the same URL can never fix. The caller re-presigns and calls again.
+      if (response.status === 403) throw lastError;
+    }
+
+    if (attempt > retries || !isRetryable(lastError)) throw lastError;
+    await sleep(backoffDelay(attempt, lastError.retryAfter, random));
+  }
+  throw lastError;
+}
+
+/**
+ * Ask the right presign route for a run or a job.
+ *
+ * `include_video` is ONE flag on the server and it gates BOTH opt in artifacts:
+ * `limits.contentTypeFor` skips every `optIn` entry when it is false, and the
+ * transcript is an `optIn` entry. So a transcript only upload sent with
+ * `include_video: false` is refused, and the refusal is a 400 on the whole
+ * batch rather than on the one path. Until the server grows a second flag, the
+ * honest reading of the one it has is "include the opt in artifacts", and that
+ * is what is sent.
+ */
+function presignWith({ api, gameId, jobId, runId, includeVideo, includeTranscript }) {
+  const includeOptIn = Boolean(includeVideo) || Boolean(includeTranscript);
+  return files => (runId
+    ? api.runs.uploads(gameId, jobId, runId, { files, includeVideo: includeOptIn })
+    : api.jobs.uploads(gameId, jobId, { files }));
+}
+
+/**
+ * Upload one run directory.
+ *
+ * Presign every file in one request, then PUT each. A 403 on a PUT means the
+ * URL expired (they last 15 minutes, and a big video upload can outlive one):
+ * that file is presigned again, once, and retried. Everything else retries with
+ * backoff inside `putFile`.
+ *
+ * Every attempt is journalled to the job's `upload-queue.jsonl`, and a file
+ * already recorded as uploaded at the same sha256 is skipped rather than sent
+ * twice. That is what makes this safe to call again after a crash, which is
+ * what `resume` and `upload <job>` do.
+ *
+ * @param {string} runDir
+ * @param {Object} options
+ * @param {Object} options.api
+ * @param {string} options.gameId
+ * @param {string} options.jobId
+ * @param {string} [options.runId] - omit for a job level upload
+ * @param {boolean} [options.includeVideo]
+ * @param {boolean} [options.includeTranscript]
+ * @param {boolean} [options.dryRun]
+ * @param {boolean} [options.force] - re-send files the journal says are done
+ * @param {string[]} [options.only] - restrict to these run relative paths
+ * @param {string[]} [options.allowedPaths] - narrow the allowlist to these names
+ * @param {(info: Object) => void} [options.onFile]
+ * @param {{repoRoot?: string}} [options.repoRoot]
+ * @param {typeof fetch} [options.fetch]
+ * @param {Function} [options.sleep]
+ * @returns {Promise<{uploaded: Array, skipped: Array, bytes: number, dryRun: boolean}>}
+ */
+export async function uploadRunDir(runDir, options) {
+  const {
+    api, gameId, jobId, runId = null,
+    includeVideo = false, includeTranscript = false,
+    dryRun = false, force = false, only = null, allowedPaths = null,
+    onFile, repoRoot, fetch: fetchImpl, sleep
+  } = options;
+
+  const journalOptions = { repoRoot };
+  const plan = await planRunDir(runDir, {
+    video: includeVideo,
+    transcript: includeTranscript,
+    allowedPaths
+  });
+  let files = plan.files;
+  const skipped = [...plan.skipped];
+
+  if (Array.isArray(only)) {
+    const wanted = new Set(only);
+    for (const file of files) {
+      if (!wanted.has(file.path)) skipped.push({ path: file.path, reason: 'not in the requested set' });
+    }
+    files = files.filter(file => wanted.has(file.path));
+  }
+
+  if (!force) {
+    const done = await uploadedKeys(jobId, journalOptions);
+    const remaining = [];
+    for (const file of files) {
+      if (done.has(keyFor(runId, file.path, file.sha256))) {
+        skipped.push({ path: file.path, reason: 'already uploaded' });
+      } else {
+        remaining.push(file);
+      }
+    }
+    files = remaining;
+  }
+
+  if (dryRun) {
+    for (const file of files) {
+      if (onFile) onFile({ ...file, status: 'dry-run' });
+    }
+    return { uploaded: files, skipped, bytes: files.reduce((sum, f) => sum + f.size, 0), dryRun: true };
+  }
+
+  if (files.length === 0) {
+    return { uploaded: [], skipped, bytes: 0, dryRun: false };
+  }
+
+  const presign = presignWith({ api, gameId, jobId, runId, includeVideo, includeTranscript });
+  const request = files.map(({ path, size, sha256, content_type: contentType }) => ({
+    path, size, sha256, content_type: contentType
+  }));
+
+  // The presign comes first, and the journal records `queued` only once it
+  // succeeds. Journalling first would leave a queue full of entries for files
+  // the server refused to sign at all, and `upload <job>` would then report
+  // them as owed forever, retrying a request that cannot start working. A
+  // refusal is recorded as `failed` with the server's own message, because
+  // "that path is not one a playtest run may upload" is the whole diagnosis.
+  let answer;
+  try {
+    answer = await presign(request);
+  } catch (error) {
+    const detail = error instanceof ApiError
+      ? [error.message, error.formatDetails()].filter(Boolean).join('\n')
+      : String(error && error.message);
+    for (const file of files) {
+      await record(jobId, {
+        kind: runId ? 'run' : 'job',
+        run_id: runId,
+        path: file.path,
+        sha256: file.sha256,
+        size: file.size,
+        status: STATUS.FAILED,
+        detail
+      }, journalOptions);
+    }
+    throw error;
+  }
+
+  for (const file of files) {
+    await record(jobId, {
+      kind: runId ? 'run' : 'job',
+      run_id: runId,
+      path: file.path,
+      sha256: file.sha256,
+      size: file.size,
+      status: STATUS.QUEUED
+    }, journalOptions);
+  }
+
+  const byPath = new Map((answer.urls || []).map(entry => [entry.path, entry]));
+  const uploaded = [];
+
+  for (const file of files) {
+    const entry = byPath.get(file.path);
+    if (!entry) {
+      throw new CliError(`The server presigned nothing for ${file.path}.`);
+    }
+    const body = await readFile(join(runDir, file.path.split('/').join(sep)));
+
+    try {
+      await putFile({ entry, contentType: file.content_type, body, fetchImpl, sleep });
+    } catch (error) {
+      if (error instanceof ApiError && error.code === 'presign_expired') {
+        // One retry with a fresh URL. A second 403 is not an expiry, it is a
+        // policy or a clock problem, and hiding that behind a loop would turn a
+        // configuration error into a hang.
+        const refreshed = await presign([{
+          path: file.path, size: file.size, sha256: file.sha256, content_type: file.content_type
+        }]);
+        const fresh = (refreshed.urls || []).find(url => url.path === file.path);
+        if (!fresh) throw error;
+        await putFile({ entry: fresh, contentType: file.content_type, body, fetchImpl, sleep });
+      } else {
+        await record(jobId, {
+          kind: runId ? 'run' : 'job',
+          run_id: runId,
+          path: file.path,
+          sha256: file.sha256,
+          size: file.size,
+          status: STATUS.FAILED,
+          detail: error.message
+        }, journalOptions);
+        throw error;
+      }
+    }
+
+    await record(jobId, {
+      kind: runId ? 'run' : 'job',
+      run_id: runId,
+      path: file.path,
+      sha256: file.sha256,
+      size: file.size,
+      status: STATUS.UPLOADED
+    }, journalOptions);
+    uploaded.push(file);
+    if (onFile) onFile({ ...file, status: 'uploaded' });
+  }
+
+  return { uploaded, skipped, bytes: uploaded.reduce((sum, f) => sum + f.size, 0), dryRun: false };
+}
+
+/**
+ * The paths a 422 `upload_unverified` named. The server writes them as
+ * `instancePath: '/manifest/<run relative path>'`, so this is the one place
+ * that spelling is parsed.
+ * @param {ApiError} error
+ * @returns {string[]}
+ */
+export function unverifiedPaths(error) {
+  return (error.details || [])
+    .map(detail => String(detail.instancePath || ''))
+    .filter(pointer => pointer.startsWith('/manifest/'))
+    .map(pointer => pointer.slice('/manifest/'.length));
+}
+
+/**
+ * Upload a run directory and complete the run.
+ *
+ * A 422 `upload_unverified` is the one refusal worth repairing automatically:
+ * it means an object in the bucket does not match what was declared, the run is
+ * deliberately left non terminal so a corrected upload is possible, and the
+ * correction is simply to send those exact files again. So the named paths are
+ * re-uploaded (forced, ignoring the journal, since the journal is what was
+ * wrong) and `complete` is called once more. A second failure throws.
+ *
+ * 422 `invalid_report` and 422 `secrets_detected` are NOT repaired. A report the
+ * schema refuses needs regenerating, and a secret needs rotating; retrying
+ * either would just fail again more slowly and, in the secret case, would be
+ * the CLI trying to talk the server into storing a live key.
+ *
+ * @param {string} runDir
+ * @param {Object} options - as uploadRunDir, plus `body` for the complete call
+ * @returns {Promise<Object>} the `complete` response
+ */
+export async function finalizeRun(runDir, options) {
+  const { api, gameId, jobId, runId, body = {} } = options;
+  if (!runId) throw new TypeError('finalizeRun needs a runId');
+
+  await uploadRunDir(runDir, options);
+
+  try {
+    return await api.runs.complete(gameId, jobId, runId, body);
+  } catch (error) {
+    if (!(error instanceof ApiError) || error.status !== 422 || error.code !== 'upload_unverified') {
+      throw error;
+    }
+    const paths = unverifiedPaths(error);
+    if (paths.length === 0) throw error;
+    await uploadRunDir(runDir, { ...options, only: paths, force: true });
+    return api.runs.complete(gameId, jobId, runId, body);
+  }
+}

package/src/upload/queue.js

@@ -0,0 +1,114 @@
+import { appendFile, mkdir, readFile } from 'node:fs/promises';
+import { dirname } from 'node:path';
+import { paths } from '../paths.js';
+
+/**
+ * `<repo>/.ravensight/jobs/<jobId>/upload-queue.jsonl`: an append only journal
+ * of every upload this CLI has attempted for a job.
+ *
+ * Append only, one JSON object per line, rather than a rewritten state file,
+ * for two reasons. An append is a single write that either lands or does not,
+ * so a kill mid upload cannot corrupt the history of the uploads that already
+ * finished. And the file doubles as a log: when a pilot user says "it uploaded
+ * nothing", this is the artifact that says what was tried and what the answer
+ * was.
+ *
+ * Each line is `{ at, kind, run_id, path, sha256, size, status, detail }`.
+ * `status` is one of `queued`, `uploaded`, `failed`, `skipped`.
+ *
+ * Idempotency is by sha256: a path recorded `uploaded` with the same hash is
+ * not sent again, which is what makes `resume` and the manual `upload <job>`
+ * safe to run repeatedly.
+ */
+
+export const STATUS = Object.freeze({
+  QUEUED: 'queued',
+  UPLOADED: 'uploaded',
+  FAILED: 'failed',
+  SKIPPED: 'skipped'
+});
+
+/**
+ * Append one entry.
+ * @param {string} jobId
+ * @param {Object} entry
+ * @param {{repoRoot?: string}} [options]
+ */
+export async function record(jobId, entry, options = {}) {
+  const file = paths.queueFile(jobId, options);
+  await mkdir(dirname(file), { recursive: true });
+  await appendFile(file, `${JSON.stringify({ at: new Date().toISOString(), ...entry })}\n`);
+}
+
+/**
+ * Read the journal. A line that does not parse is skipped rather than fatal: a
+ * torn final line from a kill is exactly the case this file exists to survive,
+ * and refusing to read the other 200 good lines because of it would be
+ * backwards.
+ * @param {string} jobId
+ * @param {{repoRoot?: string}} [options]
+ * @returns {Promise<Object[]>}
+ */
+export async function readQueue(jobId, options = {}) {
+  let text;
+  try {
+    text = await readFile(paths.queueFile(jobId, options), 'utf8');
+  } catch (error) {
+    if (error.code === 'ENOENT') return [];
+    throw error;
+  }
+  const entries = [];
+  for (const line of text.split('\n')) {
+    if (!line.trim()) continue;
+    try {
+      entries.push(JSON.parse(line));
+    } catch {
+      // torn or hand edited line
+    }
+  }
+  return entries;
+}
+
+/**
+ * The set of `<runId>|<path>|<sha256>` triples already recorded as uploaded.
+ * The sha256 is part of the key on purpose: a corrected file at the same path
+ * has a different hash and must be sent again.
+ * @param {string} jobId
+ * @param {{repoRoot?: string}} [options]
+ * @returns {Promise<Set<string>>}
+ */
+export async function uploadedKeys(jobId, options = {}) {
+  const done = new Set();
+  for (const entry of await readQueue(jobId, options)) {
+    if (entry.status === STATUS.UPLOADED) {
+      done.add(keyFor(entry.run_id || null, entry.path, entry.sha256));
+    }
+  }
+  return done;
+}
+
+export function keyFor(runId, path, sha256) {
+  return `${runId || ''}|${path}|${sha256}`;
+}
+
+/**
+ * Everything still owed: a path whose last entry is `queued` or `failed` and
+ * which was never recorded `uploaded` at that hash. What `upload <job>` drains.
+ * @param {string} jobId
+ * @param {{repoRoot?: string}} [options]
+ * @returns {Promise<Object[]>}
+ */
+export async function pending(jobId, options = {}) {
+  const entries = await readQueue(jobId, options);
+  const done = new Set();
+  const latest = new Map();
+  for (const entry of entries) {
+    if (!entry.path) continue;
+    const key = keyFor(entry.run_id || null, entry.path, entry.sha256);
+    if (entry.status === STATUS.UPLOADED) done.add(key);
+    latest.set(key, entry);
+  }
+  return [...latest.entries()]
+    .filter(([key, entry]) => !done.has(key) && entry.status !== STATUS.SKIPPED)
+    .map(([, entry]) => entry);
+}

package/src/version.js

@@ -0,0 +1,63 @@
+import { readFileSync } from 'node:fs';
+import { fileURLToPath } from 'node:url';
+import { dirname, join } from 'node:path';
+
+const here = dirname(fileURLToPath(import.meta.url));
+
+/**
+ * This package's version, read from package.json rather than duplicated as a
+ * literal. It is sent as `cli_version` on every job register, and the server
+ * refuses a job from a CLI older than its `min_supported`, so a stale copy of
+ * this string would be a refusal nobody could explain.
+ */
+export const CLI_VERSION = JSON.parse(
+  readFileSync(join(here, '..', 'package.json'), 'utf8')
+).version;
+
+/** The `client` name sent with a device code request, shown on the approval page. */
+export const CLIENT_NAME = 'ravensight-playtest';
+
+/**
+ * The User-Agent a request carries. Takes the version rather than reading the
+ * constant, so a client built with an explicit `cliVersion` says so on the wire
+ * as well as in the `cli_version` field: the two disagreeing is exactly the kind
+ * of thing this header exists to let the server notice.
+ * @param {string} [version]
+ * @returns {string}
+ */
+export function userAgentFor(version = CLI_VERSION) {
+  return `${CLIENT_NAME}/${version} node/${process.versions.node}`;
+}
+
+/** The User-Agent this build sends by default. */
+export const USER_AGENT = userAgentFor();
+
+/**
+ * Compare two dotted version strings. Returns a negative number when `a` is
+ * older than `b`, 0 when they are equal, positive when `a` is newer. Only the
+ * numeric prefix of each segment is compared, so `1.2.0-beta.3` sorts as
+ * `1.2.0`: a prerelease of a supported version is treated as that version,
+ * which is the forgiving direction for a version gate.
+ * @param {string} a
+ * @param {string} b
+ * @returns {number}
+ */
+export function compareVersions(a, b) {
+  // The prerelease suffix is cut before splitting, not after: splitting first
+  // turns `1.2.0-beta.3` into four segments and the stray `3` would make it
+  // sort as NEWER than `1.2.0`, which is the wrong direction for a gate that
+  // decides whether this CLI is too old to run.
+  const parse = value => String(value || '0')
+    .split('-')[0]
+    .split('+')[0]
+    .split('.')
+    .map(part => parseInt(part, 10) || 0);
+  const left = parse(a);
+  const right = parse(b);
+  const length = Math.max(left.length, right.length);
+  for (let i = 0; i < length; i += 1) {
+    const diff = (left[i] || 0) - (right[i] || 0);
+    if (diff !== 0) return diff < 0 ? -1 : 1;
+  }
+  return 0;
+}