skillfid 0.1.0

package/src/journal.js

@@ -0,0 +1,193 @@
+import { createHash, randomUUID } from 'node:crypto';
+import { mkdir } from 'node:fs/promises';
+import path from 'node:path';
+import { DatabaseSync } from 'node:sqlite';
+
+import { stableStringify } from './json.js';
+
+export function operationId(kind, inputs) {
+  const hash = createHash('sha256').update(stableStringify({ kind, inputs }), 'utf8').digest('hex');
+  return `${kind}_${hash.slice(0, 16)}`;
+}
+
+export function valueHash(value) {
+  return createHash('sha256').update(stableStringify(value), 'utf8').digest('hex');
+}
+
+export class OperationJournal {
+  static async open(filePath, options = {}) {
+    if (!options.readOnly) await mkdir(path.dirname(path.resolve(filePath)), { recursive: true });
+    return new OperationJournal(filePath, options);
+  }
+
+  constructor(filePath, { clock = () => Date.now(), leaseMs = 15 * 60 * 1000, readOnly = false } = {}) {
+    this.clock = clock;
+    this.leaseMs = leaseMs;
+    this.database = new DatabaseSync(path.resolve(filePath), { readOnly });
+    if (readOnly) {
+      this.database.exec('PRAGMA busy_timeout = 5000; PRAGMA foreign_keys = ON;');
+      return;
+    }
+    this.database.exec('PRAGMA journal_mode = WAL; PRAGMA busy_timeout = 5000; PRAGMA foreign_keys = ON;');
+    this.database.exec(`
+      CREATE TABLE IF NOT EXISTS operations (
+        operation_id TEXT PRIMARY KEY,
+        kind TEXT NOT NULL,
+        input_hash TEXT NOT NULL,
+        config_json TEXT NOT NULL,
+        status TEXT NOT NULL CHECK(status IN ('running', 'completed', 'failed')),
+        publication_path TEXT,
+        created_at INTEGER NOT NULL,
+        updated_at INTEGER NOT NULL
+      );
+      CREATE TABLE IF NOT EXISTS jobs (
+        job_id TEXT PRIMARY KEY,
+        operation_id TEXT NOT NULL REFERENCES operations(operation_id),
+        stage TEXT NOT NULL,
+        entity_id TEXT NOT NULL,
+        input_hash TEXT NOT NULL,
+        status TEXT NOT NULL CHECK(status IN ('pending', 'running', 'completed', 'failed')),
+        attempts INTEGER NOT NULL DEFAULT 0,
+        lease_owner TEXT,
+        lease_expires_at INTEGER,
+        output_json TEXT,
+        output_hash TEXT,
+        error TEXT,
+        created_at INTEGER NOT NULL,
+        updated_at INTEGER NOT NULL,
+        UNIQUE(operation_id, stage, entity_id, input_hash)
+      );
+      CREATE INDEX IF NOT EXISTS jobs_claim_idx ON jobs(operation_id, stage, status, lease_expires_at, created_at);
+    `);
+  }
+
+  close() {
+    this.database.close();
+  }
+
+  startOperation({ operationId: id, kind, inputs, config = {} }) {
+    const now = this.clock();
+    const inputHash = valueHash(inputs);
+    this.database.prepare(`
+      INSERT INTO operations (operation_id, kind, input_hash, config_json, status, created_at, updated_at)
+      VALUES (?, ?, ?, ?, 'running', ?, ?)
+      ON CONFLICT(operation_id) DO UPDATE SET status = 'running', updated_at = excluded.updated_at
+      WHERE operations.input_hash = excluded.input_hash
+    `).run(id, kind, inputHash, stableStringify(config), now, now);
+    const operation = this.getOperation(id);
+    if (!operation || operation.inputHash !== inputHash) throw new Error(`Operation identity collision for ${id}`);
+    return operation;
+  }
+
+  getOperation(id) {
+    const row = this.database.prepare('SELECT * FROM operations WHERE operation_id = ?').get(id);
+    return row ? mapOperation(row) : undefined;
+  }
+
+  listOperations() {
+    return this.database.prepare('SELECT * FROM operations ORDER BY updated_at DESC, operation_id').all().map(mapOperation);
+  }
+
+  findResumableOperation(kind, inputs) {
+    const row = this.database.prepare(`
+      SELECT * FROM operations
+      WHERE kind = ? AND input_hash = ? AND status != 'completed'
+      ORDER BY updated_at DESC, created_at DESC, operation_id
+      LIMIT 1
+    `).get(kind, valueHash(inputs));
+    return row ? mapOperation(row) : undefined;
+  }
+
+  ensureJob({ operationId: id, stage, entityId, inputs }) {
+    const now = this.clock();
+    const inputHash = valueHash(inputs);
+    const jobId = `job_${valueHash({ id, stage, entityId, inputHash }).slice(0, 20)}`;
+    this.database.prepare(`
+      INSERT OR IGNORE INTO jobs (job_id, operation_id, stage, entity_id, input_hash, status, created_at, updated_at)
+      VALUES (?, ?, ?, ?, ?, 'pending', ?, ?)
+    `).run(jobId, id, stage, entityId, inputHash, now, now);
+    return this.getJob(jobId);
+  }
+
+  claimJob(jobId, workerId = randomUUID()) {
+    const now = this.clock();
+    const expires = now + this.leaseMs;
+    this.database.exec('BEGIN IMMEDIATE');
+    try {
+      const result = this.database.prepare(`
+        UPDATE jobs SET status = 'running', attempts = attempts + 1, lease_owner = ?, lease_expires_at = ?, error = NULL, updated_at = ?
+        WHERE job_id = ? AND (status IN ('pending', 'failed') OR (status = 'running' AND lease_expires_at <= ?))
+      `).run(workerId, expires, now, jobId, now);
+      this.database.exec('COMMIT');
+      return result.changes ? this.getJob(jobId) : undefined;
+    } catch (error) {
+      this.database.exec('ROLLBACK');
+      throw error;
+    }
+  }
+
+  renewLease(jobId, workerId) {
+    const now = this.clock();
+    const result = this.database.prepare(`
+      UPDATE jobs SET lease_expires_at = ?, updated_at = ?
+      WHERE job_id = ? AND status = 'running' AND lease_owner = ?
+    `).run(now + this.leaseMs, now, jobId, workerId);
+    return result.changes === 1;
+  }
+
+  completeJob(jobId, workerId, output) {
+    const now = this.clock();
+    const outputJson = stableStringify(output);
+    const result = this.database.prepare(`
+      UPDATE jobs SET status = 'completed', output_json = ?, output_hash = ?, lease_owner = NULL, lease_expires_at = NULL, error = NULL, updated_at = ?
+      WHERE job_id = ? AND status = 'running' AND lease_owner = ?
+    `).run(outputJson, valueHash(output), now, jobId, workerId);
+    if (result.changes !== 1) throw new Error(`Worker ${workerId} does not own running job ${jobId}`);
+    return this.getJob(jobId);
+  }
+
+  failJob(jobId, workerId, error) {
+    const now = this.clock();
+    const result = this.database.prepare(`
+      UPDATE jobs SET status = 'failed', lease_owner = NULL, lease_expires_at = NULL, error = ?, updated_at = ?
+      WHERE job_id = ? AND status = 'running' AND lease_owner = ?
+    `).run(error instanceof Error ? error.message : String(error), now, jobId, workerId);
+    if (result.changes !== 1) throw new Error(`Worker ${workerId} does not own running job ${jobId}`);
+    return this.getJob(jobId);
+  }
+
+  getJob(jobId) {
+    const row = this.database.prepare('SELECT * FROM jobs WHERE job_id = ?').get(jobId);
+    return row ? mapJob(row) : undefined;
+  }
+
+  listJobs(operationIdValue) {
+    return this.database.prepare('SELECT * FROM jobs WHERE operation_id = ? ORDER BY stage, entity_id, created_at').all(operationIdValue).map(mapJob);
+  }
+
+  jobCounts(operationIdValue) {
+    const counts = { pending: 0, running: 0, completed: 0, failed: 0 };
+    for (const row of this.database.prepare('SELECT status, COUNT(*) AS count FROM jobs WHERE operation_id = ? GROUP BY status').all(operationIdValue)) counts[row.status] = Number(row.count);
+    return counts;
+  }
+
+  completeOperation(id, publicationPath) {
+    this.assertAllJobsCompleted(id);
+    const now = this.clock();
+    this.database.prepare("UPDATE operations SET status = 'completed', publication_path = ?, updated_at = ? WHERE operation_id = ?").run(publicationPath, now, id);
+    return this.getOperation(id);
+  }
+
+  assertAllJobsCompleted(id) {
+    const incomplete = this.database.prepare("SELECT COUNT(*) AS count FROM jobs WHERE operation_id = ? AND status != 'completed'").get(id).count;
+    if (incomplete) throw new Error(`Operation ${id} has ${incomplete} incomplete job(s)`);
+  }
+}
+
+function mapOperation(row) {
+  return { operationId: row.operation_id, kind: row.kind, inputHash: row.input_hash, config: JSON.parse(row.config_json), status: row.status, publicationPath: row.publication_path, createdAt: row.created_at, updatedAt: row.updated_at };
+}
+
+function mapJob(row) {
+  return { jobId: row.job_id, operationId: row.operation_id, stage: row.stage, entityId: row.entity_id, inputHash: row.input_hash, status: row.status, attempts: row.attempts, leaseOwner: row.lease_owner, leaseExpiresAt: row.lease_expires_at, output: row.output_json ? JSON.parse(row.output_json) : undefined, outputHash: row.output_hash, error: row.error, createdAt: row.created_at, updatedAt: row.updated_at };
+}

package/src/json.js

@@ -0,0 +1,26 @@
+export function stableStringify(value, space) {
+  return JSON.stringify(sortValue(value), null, space);
+}
+
+export function parseJsonObject(response, label = 'Response') {
+  let value;
+  try {
+    value = JSON.parse(response);
+  } catch (error) {
+    throw new InvalidStructuredResponse(`${label} is not valid JSON: ${error.message}`);
+  }
+  if (value === null || Array.isArray(value) || typeof value !== 'object') {
+    throw new InvalidStructuredResponse(`${label} must be a JSON object`);
+  }
+  return value;
+}
+
+export class InvalidStructuredResponse extends Error {}
+
+function sortValue(value) {
+  if (Array.isArray(value)) return value.map(sortValue);
+  if (value !== null && typeof value === 'object') {
+    return Object.fromEntries(Object.keys(value).sort().map((key) => [key, sortValue(value[key])]));
+  }
+  return value;
+}

package/src/judge.js

@@ -0,0 +1,39 @@
+import { stableStringify } from './json.js';
+import { InvalidStructuredResponse, parseJsonObject } from './structured.js';
+
+export class InvalidJudgeResponse extends InvalidStructuredResponse {}
+
+export function buildJudgePrompt({ question, source, candidateAnswer, rubric }) {
+  return [
+    'Judge the candidate answer against each rubric criterion using only the complete supplied source as ground truth. Return only valid JSON with this shape:',
+    '{"criterionResults":[{"criterionIndex":0,"score":0,"rationale":"..."}],"unsupportedClaims":[]}',
+    'Scores must be numbers from 0 to 1. Include every criterion exactly once.',
+    'INPUT:',
+    stableStringify({ question, source, candidateAnswer, rubric }),
+  ].join('\n');
+}
+
+export function parseJudgeResponse(response, criterionCount) {
+  let data;
+  try {
+    data = parseJsonObject(response, 'Judge response');
+  } catch (error) {
+    throw new InvalidJudgeResponse(error.message);
+  }
+  if (!Array.isArray(data.criterionResults)) throw new InvalidJudgeResponse('criterionResults must be an array');
+  if (data.criterionResults.length !== criterionCount) throw new InvalidJudgeResponse(`Expected ${criterionCount} criterion results, got ${data.criterionResults.length}`);
+  const seen = new Set();
+  const criterionResults = data.criterionResults.map((item) => {
+    if (item === null || Array.isArray(item) || typeof item !== 'object') throw new InvalidJudgeResponse('Each criterion result must be an object');
+    const { criterionIndex, score, rationale } = item;
+    if (!Number.isInteger(criterionIndex)) throw new InvalidJudgeResponse('criterionIndex must be an integer');
+    if (criterionIndex < 0 || criterionIndex >= criterionCount || seen.has(criterionIndex)) throw new InvalidJudgeResponse(`Invalid or duplicate criterionIndex: ${criterionIndex}`);
+    if (typeof score !== 'number' || !Number.isFinite(score)) throw new InvalidJudgeResponse(`Criterion ${criterionIndex} score must be a number`);
+    if (score < 0 || score > 1) throw new InvalidJudgeResponse(`Criterion ${criterionIndex} score must be from 0 to 1`);
+    if (typeof rationale !== 'string' || !rationale.trim()) throw new InvalidJudgeResponse(`Criterion ${criterionIndex} rationale must not be empty`);
+    seen.add(criterionIndex);
+    return { criterionIndex, score, rationale: rationale.trim() };
+  }).sort((left, right) => left.criterionIndex - right.criterionIndex);
+  if (!Array.isArray(data.unsupportedClaims) || data.unsupportedClaims.some((claim) => typeof claim !== 'string')) throw new InvalidJudgeResponse('unsupportedClaims must be an array of strings');
+  return { criterionResults, unsupportedClaims: data.unsupportedClaims.map((claim) => claim.trim()).filter(Boolean) };
+}

package/src/progress.js

@@ -0,0 +1,223 @@
+import { performance } from 'node:perf_hooks';
+
+const SPINNER = ['⠋', '⠙', '⠹', '⠸', '⠼', '⠴', '⠦', '⠧', '⠇', '⠏'];
+const IMPORTANT_PATTERN = /(?:operation\s|jobs:|retrying|dataset ready:|evaluation complete:|dataset verification complete:|failed\b|error\b|interrupted)/i;
+const IMPORTANT_TYPES = new Set(['checkpoint', 'complete', 'error', 'operation', 'retry']);
+
+export function createProgressReporter({ mode = 'auto', intervalSeconds = 30, stream = process.stderr, startedAt = performance.now(), clock = () => performance.now() } = {}) {
+  const resolvedMode = mode === 'auto' ? (stream.isTTY ? 'human' : 'agent') : mode;
+  const options = { stream, startedAt, clock, intervalSeconds };
+  if (resolvedMode === 'quiet') return new QuietProgressReporter();
+  if (resolvedMode === 'human') return new HumanProgressReporter(options);
+  if (resolvedMode === 'agent') return new AgentProgressReporter(options);
+  if (resolvedMode === 'json') return new JsonProgressReporter(options);
+  throw new Error(`Unknown progress mode: ${mode}`);
+}
+
+export class ProgressReporter {
+  constructor(options = {}) {
+    return createProgressReporter({ ...options, mode: options.mode ?? 'agent' });
+  }
+}
+
+class BaseProgressReporter {
+  constructor({ stream, startedAt, clock }) {
+    this.stream = stream;
+    this.startedAt = startedAt;
+    this.clock = clock;
+  }
+
+  elapsedSeconds() {
+    return (this.clock() - this.startedAt) / 1000;
+  }
+
+  close() {}
+}
+
+class HumanProgressReporter extends BaseProgressReporter {
+  constructor(options) {
+    super(options);
+    this.state = { title: 'Working', current: 'Starting' };
+    this.estimate = { samples: [] };
+    this.frame = 0;
+    this.renderedLines = 0;
+    this.timer = setInterval(() => this.render(), 80);
+    this.timer.unref();
+  }
+
+  report(input) {
+    const event = normalizeEvent(input);
+    if (!event.phase && !event.current && event.message && !IMPORTANT_TYPES.has(event.type)) event.current = event.message;
+    this.state = {
+      ...this.state,
+      ...pickDefined(event, ['workflow', 'title', 'phase', 'current', 'output', 'progress']),
+      metrics: event.metrics,
+    };
+    if (event.progress) this.recordProgress(event.progress);
+    if (event.type === 'complete') this.complete(event);
+    else if (event.type === 'error') this.notice('error', event.message ?? event.current ?? 'Failed');
+    else if (event.type === 'retry') this.notice('retry', event.message ?? 'Retrying');
+    else this.render();
+  }
+
+  close() {
+    clearInterval(this.timer);
+    this.clear();
+  }
+
+  render() {
+    if (this.finished) return;
+    const elapsed = this.elapsedSeconds();
+    const lines = [`\x1b[36m${SPINNER[this.frame++ % SPINNER.length]}\x1b[0m  \x1b[1m${this.state.title}\x1b[0m`];
+    if (this.state.progress) {
+      const { done, total, label } = this.state.progress;
+      const ratio = total > 0 ? Math.min(1, done / total) : 0;
+      const percentageValue = done >= total ? 100 : Math.floor(ratio * 100);
+      const percentage = total > 0 ? `  \x1b[2m${percentageValue}%\x1b[0m` : '';
+      lines.push(`   \x1b[36m${progressBar(ratio)}\x1b[0m  \x1b[1m${done}/${total}\x1b[0m ${label}${percentage}`);
+    } else if (this.state.metrics) {
+      lines.push(...Object.entries(this.state.metrics).map(([label, value]) => `   ${label.padEnd(11)} ${value}`));
+    }
+    lines.push(`   \x1b[2m${this.state.current ?? this.state.phase ?? 'Working'}\x1b[0m`);
+    lines.push(`   \x1b[2m${formatElapsed(elapsed)} elapsed${formatRemaining(this.remainingSeconds(elapsed))}\x1b[0m`);
+    this.clear();
+    const width = Number.isInteger(this.stream.columns) ? Math.max(1, this.stream.columns - 1) : Infinity;
+    this.stream.write(lines.map((line) => fitTerminalLine(line, width)).join('\n'));
+    this.renderedLines = lines.length;
+  }
+
+  complete(event) {
+    this.clear();
+    this.finished = true;
+    const lines = [`\x1b[32m[ok]\x1b[0m ${event.title ?? this.state.title}  \x1b[2m${formatElapsed(this.elapsedSeconds())}\x1b[0m`];
+    for (const line of event.summary ?? []) lines.push(`     ${line}`);
+    if (event.output) lines.push(`     Output: ${event.output}`);
+    this.stream.write(`${lines.join('\n')}\n`);
+    this.state.current = undefined;
+  }
+
+  notice(kind, message) {
+    this.clear();
+    const marker = kind === 'error' ? '\x1b[31m[error]\x1b[0m' : '\x1b[33m[retry]\x1b[0m';
+    this.stream.write(`${marker} ${message}\n`);
+    this.render();
+  }
+
+  recordProgress(progress) {
+    if (!Number.isFinite(progress.done) || !Number.isFinite(progress.total)) return;
+    const elapsed = this.elapsedSeconds();
+    const estimateDone = progress.etaDone ?? progress.done;
+    const last = this.estimate.samples.at(-1);
+    if (!last || estimateDone > last.done) {
+      this.estimate.samples.push({ done: estimateDone, elapsed });
+      this.estimate.samples = this.estimate.samples.slice(-8);
+    }
+  }
+
+  remainingSeconds(elapsed) {
+    const progress = this.state.progress;
+    if (!progress || progress.done >= progress.total || progress.total <= 0) return undefined;
+    const samples = this.estimate.samples;
+    if (samples.length < 2 || elapsed < 5) return null;
+    const first = samples[0];
+    const last = samples.at(-1);
+    const rate = (last.done - first.done) / (last.elapsed - first.elapsed);
+    if (rate <= 0) return null;
+    const remainingAtLastSample = (progress.total - progress.done) / rate;
+    const remaining = remainingAtLastSample - (elapsed - last.elapsed);
+    return remaining > 0 ? remaining : Number.NaN;
+  }
+
+  clear() {
+    if (!this.renderedLines) return;
+    this.stream.write('\r\x1b[2K');
+    for (let index = 1; index < this.renderedLines; index += 1) this.stream.write('\x1b[1A\r\x1b[2K');
+    this.renderedLines = 0;
+  }
+}
+
+class AgentProgressReporter extends BaseProgressReporter {
+  constructor(options) {
+    super(options);
+    this.intervalSeconds = options.intervalSeconds;
+    this.lastWrittenAt = -Infinity;
+    this.suppressed = 0;
+  }
+
+  report(input) {
+    const event = normalizeEvent(input);
+    const elapsed = this.elapsedSeconds();
+    const important = IMPORTANT_TYPES.has(event.type);
+    if (!important && elapsed - this.lastWrittenAt < this.intervalSeconds) { this.suppressed += 1; return; }
+    const suppressed = this.suppressed ? ` suppressed=${this.suppressed}` : '';
+    const phase = event.phase ? ` phase=${JSON.stringify(event.phase)}` : '';
+    const metrics = event.metrics && Object.keys(event.metrics).length ? ` metrics=${JSON.stringify(event.metrics)}` : '';
+    const message = event.message ?? event.current ?? event.title ?? event.type;
+    this.stream.write(`[progress] elapsed=${Math.round(elapsed)}s${suppressed}${phase}${metrics} message=${JSON.stringify(message)}\n`);
+    this.lastWrittenAt = elapsed;
+    this.suppressed = 0;
+  }
+}
+
+class JsonProgressReporter extends BaseProgressReporter {
+  report(input) {
+    const event = normalizeEvent(input);
+    this.stream.write(`${JSON.stringify({ ...event, elapsedSeconds: Number(this.elapsedSeconds().toFixed(1)) })}\n`);
+  }
+}
+
+class QuietProgressReporter {
+  report() {}
+  close() {}
+}
+
+function eventType(message) {
+  if (/retrying/i.test(message)) return 'retry';
+  if (/jobs:/i.test(message)) return 'checkpoint';
+  if (/failed|error|interrupted/i.test(message)) return 'error';
+  if (/dataset ready:|evaluation complete:|dataset verification complete:/i.test(message)) return 'complete';
+  if (/operation\s/i.test(message)) return 'operation';
+  return 'progress';
+}
+
+function normalizeEvent(input) {
+  if (typeof input === 'string') return { type: eventType(input), message: input };
+  if (!input || typeof input !== 'object') throw new TypeError('Progress events must be strings or objects');
+  return { type: 'update', ...input };
+}
+
+function pickDefined(source, keys) {
+  return Object.fromEntries(keys.filter((key) => source[key] !== undefined).map((key) => [key, source[key]]));
+}
+
+function formatElapsed(seconds) {
+  const rounded = Math.max(0, Math.round(seconds));
+  const minutes = Math.floor(rounded / 60);
+  const remainder = rounded % 60;
+  return minutes ? `${minutes}m ${String(remainder).padStart(2, '0')}s` : `${remainder}s`;
+}
+
+function formatRemaining(seconds) {
+  if (seconds === undefined) return '';
+  if (seconds === null) return ' · estimating remaining time';
+  if (!Number.isFinite(seconds)) return ' · remaining time uncertain';
+  return ` · about ${formatElapsed(seconds)} remaining`;
+}
+
+function progressBar(ratio, width = 16) {
+  const filled = ratio >= 1 ? width : Math.floor(ratio * width);
+  return `${'━'.repeat(filled)}${'─'.repeat(width - filled)}`;
+}
+
+function fitTerminalLine(line, width) {
+  if (!Number.isFinite(width)) return line;
+  const tokens = line.match(/\x1b\[[0-?]*[ -/]*[@-~]|./gu) ?? [];
+  let visible = 0;
+  let output = '';
+  for (const token of tokens) {
+    if (token.startsWith('\x1b[')) output += token;
+    else if (visible < width) { output += token; visible += 1; }
+    else return `${output}\x1b[0m`;
+  }
+  return output;
+}

package/src/prompts.js

@@ -0,0 +1,42 @@
+import { stableStringify } from './json.js';
+
+export const ORACLE_PROMPT_VERSION = '3';
+
+export function subjectPrompt(question) {
+  return ['Answer the user\'s question. Use an available skill when relevant.', 'Do not search external sources or use shell commands.', 'If the answer is not available in the provided context or an available skill, say that you do not know; do not infer or invent specific details.', 'Return only a concise final answer.', '', `User question: ${question}`].join('\n');
+}
+
+export function skillPrompt(question, invocationMode, skillName) {
+  const prompt = subjectPrompt(question);
+  return invocationMode === 'explicit' ? [`/${skillName}`, '', prompt].join('\n') : prompt;
+}
+
+export function oraclePrompt(question, source, rubric) {
+  const lines = [
+    'Answer the user\'s question using only the complete source below.',
+    'Answer every part of the question with the source\'s stated detail.',
+    'Preserve numbers, counts, qualifiers, causal claims, and scope exactly; do not round, broaden, combine, or strengthen them.',
+    'When the source gives apparently conflicting measurements from different contexts, distinguish those contexts explicitly.',
+    'If the source does not support a claim, omit it or state that it is not specified.',
+    'Do not use skills, external sources, or shell commands. Return only the final answer.',
+    '',
+    `User question: ${question}`,
+  ];
+  if (rubric) lines.push(
+    '',
+    'Calibration coverage requirements:',
+    stableStringify(rubric.map(({ criterion }) => criterion)),
+    'Treat these requirements as minimum coverage, not an exhaustive answer boundary.',
+    'Before answering, scan the complete source independently of the requirements for every passage that addresses the question. Account for all applicable prerequisites, exceptions, and repeated measurements, including relevant passages outside the sections represented by the requirements.',
+    'Answer every requirement directly and answer the user\'s question fully, including relevant source facts and qualifications needed for accuracy and completeness.',
+    'If a requirement conflicts with the source, the source wins: report every distinct source-supported value or outcome with its location or context, and never repeat the requirement unqualified.',
+    'Treat the source as authoritative. Omit unrelated background, examples, and adjacent source facts.',
+    'Add no claim merely because it appears in a requirement; every claim must be supported by the source.',
+  );
+  lines.push(
+    '',
+    'Use only this complete source:',
+    stableStringify(source),
+  );
+  return lines.join('\n');
+}

package/src/questions.js

@@ -0,0 +1,80 @@
+import { createHash } from 'node:crypto';
+
+import { stableStringify } from './json.js';
+import { InvalidStructuredResponse, parseJsonObject } from './structured.js';
+import { requiredString } from './inventory.js';
+
+export const MAX_ITEMS_PER_QUESTION = 6;
+const QUESTION_TYPES = new Set(['fact', 'procedure', 'application', 'synthesis', 'conflict']);
+const DIFFICULTIES = new Set(['easy', 'medium', 'hard']);
+
+export function buildQuestionPrompt(items, evidence) {
+  const evidenceById = new Map(evidence.map((record) => [record.evidenceId, record.quote]));
+  const payload = items.map((item) => ({
+    knowledgeId: item.knowledgeId,
+    kind: item.kind,
+    statement: item.statement,
+    importance: item.importance,
+    evidence: item.evidenceIds.map((id) => evidenceById.get(id)),
+  }));
+  return [
+    'Generate focused, independently scorable, user-realistic questions that test every supplied knowledge item. Each item must be required by at least one rubric criterion. Each question must cover one coherent operational concern. Before writing questions, group items into the complete local concerns expressed by the source: a storage policy includes both placement and read behavior; a tier policy includes eligibility and backing media; a retention policy includes its rationale; an admission policy includes all paired limits, overload behavior, and stated risks. Preserve a source paragraph\'s coherent concern when it fits within the item limit instead of splitting its facts across questions.',
+    'Knowledge items may appear in more than one question when overlap is necessary to make related questions complete. Combine configuration with backing, policy with rationale, trigger with response, paired limits, and mechanism with purpose. Prefer one complete question over atomizing those relationships. Do not combine unrelated operational concerns merely to reduce the number of questions.',
+    `A question may cover at most ${MAX_ITEMS_PER_QUESTION} knowledge items.`,
+    'Do not include tested values, limits, outcomes, or other answer-bearing facts in the question. A model without the supplied knowledge must not be able to derive the answer from the wording or generic domain conventions. Every claim in each rubric criterion must be explicitly supported by the supplied evidence.',
+    '',
+    'Return only one JSON object with this shape:',
+    '{"questions":[{"question":"...","type":"fact","difficulty":"medium","rubric":[{"criterion":"...","weight":1,"knowledgeItemIds":["ki_..."]}]}]}',
+    'Allowed types: fact, procedure, application, synthesis, conflict. Allowed difficulties: easy, medium, hard. Rubric weights for each question must be positive and sum to 1. Use only supplied knowledge IDs.',
+    '',
+    'KNOWLEDGE INVENTORY:',
+    stableStringify(payload),
+  ].join('\n');
+}
+
+export function parseQuestionResponse(response, items) {
+  const data = parseJsonObject(response);
+  if (!Array.isArray(data.questions) || !data.questions.length) throw new InvalidStructuredResponse('questions must be a non-empty array');
+  const itemById = new Map(items.map((item) => [item.knowledgeId, item]));
+  const coveredIds = new Set();
+  const seen = new Set();
+  const questions = data.questions.map((rawQuestion) => {
+    if (rawQuestion === null || Array.isArray(rawQuestion) || typeof rawQuestion !== 'object') throw new InvalidStructuredResponse('Each question must be an object');
+    if ('referenceAnswer' in rawQuestion || 'reference_answer' in rawQuestion) throw new InvalidStructuredResponse('Questions must not include a reference answer; the source is ground truth');
+    const question = requiredString(rawQuestion, 'question');
+    const questionType = requiredString(rawQuestion, 'type');
+    const difficulty = requiredString(rawQuestion, 'difficulty');
+    if (!QUESTION_TYPES.has(questionType)) throw new InvalidStructuredResponse(`Unsupported question type: ${questionType}`);
+    if (!DIFFICULTIES.has(difficulty)) throw new InvalidStructuredResponse(`Unsupported difficulty: ${difficulty}`);
+    if (seen.has(question)) throw new InvalidStructuredResponse(`Duplicate question: ${question}`);
+    if (!Array.isArray(rawQuestion.rubric) || !rawQuestion.rubric.length) throw new InvalidStructuredResponse('rubric must be a non-empty array');
+    let totalWeight = 0;
+    const questionItemIds = new Set();
+    const rubric = rawQuestion.rubric.map((rawCriterion) => {
+      if (rawCriterion === null || Array.isArray(rawCriterion) || typeof rawCriterion !== 'object') throw new InvalidStructuredResponse('Each rubric criterion must be an object');
+      const criterion = requiredString(rawCriterion, 'criterion');
+      const { weight, knowledgeItemIds } = rawCriterion;
+      if (typeof weight !== 'number' || !Number.isFinite(weight) || weight <= 0) throw new InvalidStructuredResponse('Rubric weight must be a positive number');
+      if (!Array.isArray(knowledgeItemIds) || !knowledgeItemIds.length || knowledgeItemIds.some((id) => typeof id !== 'string')) throw new InvalidStructuredResponse('knowledgeItemIds must be a non-empty string array');
+      const unknown = [...new Set(knowledgeItemIds)].filter((id) => !itemById.has(id));
+      if (unknown.length) throw new InvalidStructuredResponse(`Rubric references unknown knowledge IDs: ${unknown.sort().join(', ')}`);
+      totalWeight += weight;
+      knowledgeItemIds.forEach((id) => questionItemIds.add(id));
+      return { criterion, weight, knowledgeItemIds: [...new Set(knowledgeItemIds)].sort() };
+    });
+    if (Math.abs(totalWeight - 1) > 1e-6) throw new InvalidStructuredResponse(`Rubric weights must sum to 1, got ${totalWeight}`);
+    if (questionItemIds.size > MAX_ITEMS_PER_QUESTION) throw new InvalidStructuredResponse(`A question may cover at most ${MAX_ITEMS_PER_QUESTION} knowledge items, got ${questionItemIds.size}`);
+    const evidenceIds = [...new Set([...questionItemIds].flatMap((id) => itemById.get(id).evidenceIds))].sort();
+    const testId = `ke_${digest(`${question}:${[...questionItemIds].sort().join(',')}`).slice(0, 16)}`;
+    questionItemIds.forEach((id) => coveredIds.add(id));
+    seen.add(question);
+    return { testId, question, questionType, difficulty, rubric, evidenceIds };
+  });
+  const missing = [...itemById.keys()].filter((id) => !coveredIds.has(id));
+  if (missing.length) throw new InvalidStructuredResponse(`Questions do not cover knowledge IDs: ${missing.sort().join(', ')}`);
+  return questions;
+}
+
+function digest(value) {
+  return createHash('sha256').update(value, 'utf8').digest('hex');
+}