software-defence-factory 0.15.3 → 0.15.4

package/docs/usage.md

@@ -46,3 +46,60 @@ Historical recovery is optional and budgeted to eight lookups per second and
 512 cached terminal attempts per controller lifetime. Further attempts remain
 unknown; new persisted measurements bypass these legacy read limits. This
 prevents large old queues from rereading all logs on every status poll.
+
+## Pi stdout measurements (0.15.4)
+
+New Pi attempts use `source: "pi_jsonl"`. Factory collects stdout before bounded
+log truncation; stderr, tool results and streaming updates cannot supply usage.
+The audited official `@earendil-works/pi-coding-agent` **0.87.1** OpenAI
+completions adapter reports uncached `input`, `cacheRead`, `cacheWrite`, `output`
+and their sum in `totalTokens`. Factory maps them to the existing contract:
+
+- `input_tokens = input + cacheRead + cacheWrite` (inclusive input).
+- `cached_input_tokens = cacheRead`, and `cache_write_input_tokens = cacheWrite`.
+- `token_usage = input_tokens + output_tokens`; neither cache subset is added again.
+
+Cache-write attribution is optional for historical/Codex records, not assumed
+zero when absent. Pi requires all five counters, validates their sum, and
+preserves exact decimal strings. Unsafe numeric values are rejected. An all-zero
+Pi usage object remains unknown: the installed provider also emits this initial
+placeholder when no usage arrived, including failed requests. Provider cost
+fields and configured zero prices do not establish monetary consumption; local
+hardware/electricity and unpriced provider cost remain unknown.
+
+The pinned agent loop emits completed `message_end` assistant messages and
+repeats them, in order, in that agent turn's `agent_end.messages`. Factory
+reconciles positions within each `agent_start`/`agent_end` interval. Equal token
+counts and equal timestamps are not unique IDs: distinct positions and explicit
+new agent turns count separately. Repeated snapshots do not add tokens; a final
+aggregate alone can supply counts. Contradictory or ambiguous snapshots never
+add unmatched counts. Missing, malformed and oversized evidence cannot turn an
+observation into complete coverage or a measured zero.
+
+All Pi observations currently have **partial** coverage, even after a successful
+run. Pi 0.87.1 performs compaction summary calls outside the visible assistant
+aggregate and can retry provider requests without independent failed-request
+usage accounting. A recovered job can succeed while these totals remain
+incomplete. Absence of a retry/compaction event is not proof of full coverage.
+No scorer, cost estimate, quality claim or acceptance authority follows from
+these measurements.
+
+Parsing is bounded to an 8 MiB line buffer, 262,144 framed lines, 2,048 observed
+assistant messages and at most 1 MiB of retained counter signatures per turn.
+Message content is discarded after each line. Counts and aggregate totals are
+bounded to 128 decimal digits. Beyond a limit, already observed totals remain
+partial; with no usable evidence the result is unknown.
+
+A private, atomic `jobs/JOB/RUN/usage.json` checkpoint contains only usage and
+job/run/phase/frozen execution identity. It is controller-owned, outside the
+worker output mount, and preserves observations when cancellation kills the
+executor before its final result. Normal completion stores the same normalized
+usage in SQLite and measurement artifacts. CLI `status`, the status API and
+existing Analytics/issue detail consume that shared contract after restart.
+Model/provider/role attribution still comes from the frozen execution profile,
+never from usage event fields. Failed/cancelled runs keep their outcome.
+
+There is **no Pi historical log backfill**. Existing records and mixed or
+truncated legacy logs are not rewritten. This bounded measurement slice leaves
+#51's scoring/benchmark engine, #69's model qualification and #70's improvement
+proposal workflow open.

package/factory/execution-profile.mjs

@@ -27,16 +27,18 @@ import { expectedWebStories, webPolicyHash } from './web-verification.mjs';
 // 0.15.3 isolates local Pi mutable state and denies project resource trust.
 // Evidence schemas, candidate reconstruction and frozen profile guards are unchanged;
 // compatibility does not qualify an old local startup or accept a failed attempt.
+// 0.15.4 adds Pi usage metadata only; protected evidence schemas, frozen
+// execution selection, isolation and acceptance policy remain unchanged.
 // Older writers still require exact patch/tree reconstruction before publication.
 // Deliberately independent of VERSION: a release bump is not
 // evidence compatibility. Re-audit this list for every trust-relevant writer,
 // isolation or validation change; remove versions whose guarantees no longer
 // satisfy current policy. See docs/npm.md. This predicate alone grants no trust.
-const SUPPORTED_EXECUTION_RUNTIMES_V1 = new Set(['0.8.0', '0.9.0', '0.9.1', '0.10.0', '0.11.0', '0.11.1', '0.11.2', '0.12.0', '0.13.0', '0.13.1', '0.14.0', '0.15.0', '0.15.1', '0.15.2', '0.15.3']);
+const SUPPORTED_EXECUTION_RUNTIMES_V1 = new Set(['0.8.0', '0.9.0', '0.9.1', '0.10.0', '0.11.0', '0.11.1', '0.11.2', '0.12.0', '0.13.0', '0.13.1', '0.14.0', '0.15.0', '0.15.1', '0.15.2', '0.15.3', '0.15.4']);
 
 export function isSupportedExecutionProfile(profile) {
-  if ([2, 3].includes(profile?.version)) return (profile.version === 3 ? ['0.15.0', '0.15.1', '0.15.2', '0.15.3'].includes(profile.runtimeVersion) && Object.hasOwn(profile, 'localBinding')
-    : ['0.14.0', '0.15.0', '0.15.1', '0.15.2', '0.15.3'].includes(profile.runtimeVersion) && !Object.hasOwn(profile, 'localBinding') && profile.inferenceProvider !== 'factory-local')
+  if ([2, 3].includes(profile?.version)) return (profile.version === 3 ? ['0.15.0', '0.15.1', '0.15.2', '0.15.3', '0.15.4'].includes(profile.runtimeVersion) && Object.hasOwn(profile, 'localBinding')
+    : ['0.14.0', '0.15.0', '0.15.1', '0.15.2', '0.15.3', '0.15.4'].includes(profile.runtimeVersion) && !Object.hasOwn(profile, 'localBinding') && profile.inferenceProvider !== 'factory-local')
     && profile.role === (Object.keys(ROLE_PHASES).find(role => ROLE_PHASES[role] === profile.phase) || null)
     && (profile.reasoningEffort === null || ['low', 'medium', 'high'].includes(profile.reasoningEffort))
     && /^[a-f0-9]{64}$/.test(profile.selectionHash || '');

package/factory/executor.mjs

@@ -9,6 +9,7 @@ import { ROOT, run, save, json, digest, instanceLabel, stopContainers } from './
 import { incidentFor, validateReport } from './incident.mjs';
 import { BoundedLog } from './bounded-log.mjs';
 import { CodexUsageParser, emptyUsage, usageFields } from './usage.mjs';
+import { PiUsageParser } from './pi-usage.mjs';
 import { assertRetainedSource, publicSourceAdmission, publicContinuation, restoreBuildCheckout } from './source-admission.mjs';
 import { runCandidateGit, runCandidateGitRaw } from './git-environment.mjs';
 import { selectedInferenceSecrets, writeSelectedModelEnvironment } from './model-environment.mjs';
@@ -103,7 +104,9 @@ async function container(mode, input, command, options = {}) {
   if (scratch) args.push('--mount',`type=bind,source=${scratch},target=/scratch`);
   if (mode === 'verify') args.push('--env',`FACTORY_BASE_REVISION=${git('rev-parse',`${metadata().base}^{commit}`)}`);
   const logPath = join(folder, attempt, `${mode}.log`);
-  const log = new BoundedLog(), usageParser = execution.executor === 'codex' ? new CodexUsageParser() : null;
+  const log = new BoundedLog(), usageParser = mode === 'verify' ? null
+    : execution.executor === 'codex' ? new CodexUsageParser() : execution.executor === 'pi' ? new PiUsageParser() : null;
+  let savedObservations = 0;
   let exitSignal, selectedModelEnvironment = false, inferenceSecrets = [], code, cleanupError, reportError;
   try {
     if (localDirectory) {
@@ -127,7 +130,17 @@ async function container(mode, input, command, options = {}) {
     console.log(JSON.stringify({ phase: mode, event: 'started', synthetic: harnessOf(config) === 'mock' }));
     code = await new Promise((ok, fail) => {
       const child = spawn('docker', args, { stdio: ['pipe','pipe','pipe'] });
-      child.stdout.on('data', bytes => { log.write('stdout', bytes); usageParser?.write(bytes); });
+      child.stdout.on('data', bytes => {
+        usageParser?.write(bytes);
+        // Controller-owned metadata, outside the worker's /output mount. Atomic
+        // checkpoints survive cancellation without consulting mixed log tails.
+        if (usageParser instanceof PiUsageParser && usageParser.observed !== savedObservations) {
+          observedUsage = usageParser.snapshot();
+          save(join(folder, attempt, 'usage.json'), { version: 1, job, attempt, phase, execution, usage: observedUsage });
+          savedObservations = usageParser.observed;
+        }
+        log.write('stdout', bytes);
+      });
       child.stderr.on('data', bytes => log.write('stderr', bytes));
       child.stdin.on('error',error => { if (error.code !== 'EPIPE') fail(error); });
       child.on('error',fail); child.on('close',(code,signal) => { exitSignal=signal; ok(code); }); child.stdin.end(input);

package/factory/pi-usage.mjs

@@ -0,0 +1,106 @@
+import { UsageLineParser, tokenCount, MAX_USAGE_EVENTS, MAX_USAGE_COUNT_DIGITS } from './usage.mjs';
+
+// Pi's final aggregate includes message content and can be much larger than a
+// Codex usage line. Discard content after parsing; retain only bounded counters.
+export const MAX_PI_USAGE_LINE_BYTES = 8 * 1024 * 1024;
+export const MAX_PI_USAGE_METADATA_BYTES = 1024 * 1024;
+export const MAX_PI_USAGE_EVENTS = 262144;
+const object = value => value !== null && typeof value === 'object' && !Array.isArray(value);
+
+function counts(message) {
+  if (!object(message.usage) || !['stop', 'toolUse', 'length', 'error', 'aborted'].includes(message.stopReason)) return null;
+  try {
+    const { input, output, cacheRead, cacheWrite, totalTokens } = message.usage;
+    const uncached = tokenCount(input), out = tokenCount(output), read = tokenCount(cacheRead), write = tokenCount(cacheWrite);
+    const inclusive = uncached + read + write;
+    if (inclusive + out !== tokenCount(totalTokens)) return null;
+    if (Object.hasOwn(message.usage, 'reasoning') && tokenCount(message.usage.reasoning) > out) return null;
+    // The provider initializes missing/failed-request usage to zero. Even a
+    // successful stop with all zeros does not establish a measured zero call.
+    if (inclusive + out === 0n) return null;
+    return [inclusive, out, read, write].map(value => value.toString());
+  } catch { return null; }
+}
+
+export class PiUsageParser extends UsageLineParser {
+  constructor() {
+    super(MAX_PI_USAGE_LINE_BYTES);
+    this.totals = [0n, 0n, 0n, 0n];
+    this.observed = 0;
+    this.events = 0;
+    this.metadataBytes = 0;
+    this.messages = [];
+    this.ended = false;
+    this.ambiguous = false;
+    this.exhausted = false;
+  }
+
+  lineComplete() {
+    if (!this.exhausted && ++this.events > MAX_PI_USAGE_EVENTS) this.exhausted = true;
+    super.lineComplete();
+  }
+
+  parseLine() {
+    if (!this.length || this.exhausted) return;
+    let event;
+    try { event = JSON.parse(this.line.toString('utf8', 0, this.length)); }
+    catch { this.ambiguous = true; return; }
+    if (!object(event)) return;
+    // The pinned agent loop's newMessages starts afresh at each agent_start.
+    // Positions within that turn, not timestamps or equal token counts, match
+    // message_end to the repeated agent_end snapshot. New turns may be identical.
+    if (event.type === 'agent_start') {
+      this.messages = []; this.metadataBytes = 0;
+      this.ended = false; this.ambiguous = false;
+    } else if (event.type === 'message_end') {
+      if (!object(event.message)) { this.ambiguous = true; return; }
+      if (event.message.role !== 'assistant') return;
+      if (this.ended) { this.ambiguous = true; return; }
+      this.append(counts(event.message));
+    } else if (event.type === 'agent_end') {
+      if (!Array.isArray(event.messages) || event.messages.length > MAX_USAGE_EVENTS) { this.ambiguous = true; return; }
+      let index = 0;
+      for (const message of event.messages) {
+        if (!object(message)) { this.ambiguous = true; break; }
+        if (message.role !== 'assistant') continue;
+        const value = counts(message), signature = JSON.stringify(value);
+        if (index < this.messages.length) {
+          if (signature !== this.messages[index]) this.ambiguous = true;
+        } else if (!this.ended && !this.ambiguous && !this.incomplete) this.append(value);
+        else this.ambiguous = true;
+        index++;
+      }
+      if (index !== this.messages.length) this.ambiguous = true;
+      this.ended = true;
+    }
+  }
+
+  append(value) {
+    if (this.exhausted) return;
+    const signature = JSON.stringify(value);
+    if (this.observed >= MAX_USAGE_EVENTS || this.messages.length >= MAX_USAGE_EVENTS || this.metadataBytes + signature.length > MAX_PI_USAGE_METADATA_BYTES) {
+      this.exhausted = true; return;
+    }
+    this.messages.push(signature); this.metadataBytes += signature.length;
+    if (!value) return;
+    const next = this.totals.map((total, index) => total + BigInt(value[index]));
+    // Keep aggregate counts and their shared total representable on readback.
+    if ([...next, next[0] + next[1]].some(total => total.toString().length > MAX_USAGE_COUNT_DIGITS)) {
+      this.exhausted = true; return;
+    }
+    this.totals = next; this.observed++;
+  }
+
+  snapshot() {
+    if (!this.observed) return null;
+    const [input, output, cached, writes] = this.totals.map(String);
+    return { input_tokens: input, output_tokens: output, cached_input_tokens: cached,
+      cache_write_input_tokens: writes, source: 'pi_jsonl', coverage: 'partial' };
+  }
+
+  finish() {
+    // JSONL requires a newline. A trailing fragment is not a completed event.
+    if (this.length || this.overflow) this.incomplete = true;
+    return this.snapshot();
+  }
+}

package/factory/processes.mjs

@@ -84,10 +84,35 @@ export function retainedCodexUsage(state, job, attempt) {
   } catch { return null; }
 }
 
+// New Pi measurements are controller-owned metadata, never historical log
+// backfill. A stopped executor may have checkpointed before producing a result.
+export function retainedPiUsage(state, job, attempt) {
+  try {
+    if (!/^job_[a-z0-9]+$/.test(job?.id || '') || !/^run_[a-z0-9]+$/.test(attempt?.id || '')
+      || !['build', 'review', 'defence'].includes(attempt.command) || attempt.execution?.executor !== 'pi') return null;
+    const folder = join(state, 'jobs', job.id), runFolder = join(folder, attempt.id);
+    for (const path of [join(state, 'jobs'), folder, runFolder]) {
+      const stat = lstatSync(path);
+      if (!stat.isDirectory() || stat.isSymbolicLink() || (stat.mode & 0o077)) return null;
+    }
+    const path = join(runFolder, 'usage.json'), stat = lstatSync(path);
+    if (!stat.isFile() || stat.isSymbolicLink() || (stat.mode & 0o077) || stat.size > 16384) return null;
+    const saved = JSON.parse(readFileSync(path, 'utf8'));
+    if (saved.version !== 1 || saved.job !== job.id || saved.attempt !== attempt.id || saved.phase !== attempt.command
+      || !isDeepStrictEqual(saved.execution, attempt.execution) || saved.usage?.source !== 'pi_jsonl') return null;
+    const normalized = usageFields(saved.usage, attempt.execution, attempt.command).usage;
+    return normalized.status ? null : normalized;
+  } catch { return null; }
+}
+
+function retainedUsage(state, job, attempt) {
+  return retainedPiUsage(state, job, attempt) || retainedCodexUsage(state, job, attempt);
+}
+
 function usageForAttempt(state, job, attempt) {
   const stored = usageFields(attempt.usage, attempt.execution, attempt.command);
   if (stored.usage.status !== 'unknown') return stored;
-  const recovered = retainedCodexUsage(state, job, attempt);
+  const recovered = retainedUsage(state, job, attempt);
   return recovered ? usageFields(recovered, attempt.execution, attempt.command) : stored;
 }
 
@@ -186,7 +211,7 @@ export function executors(state, recovery = {}) {
     try { exit = await done; }
     finally { clearTimeout(deadline); children.delete(job.id); }
     if (exit.error) throw exit.error;
-    const recovered = retainedCodexUsage(state, job, attempt);
+    const recovered = retainedUsage(state, job, attempt);
     if (!existsSync(resultPath)) return { outcome: 'blocked', ...usageFields(recovered || emptyUsage(attempt.execution, attempt.command), attempt.execution, attempt.command), summary: `Executor exited ${exit.code} without a result` };
     let outcome;
     try { outcome = JSON.parse(readFileSync(resultPath, 'utf8')); }