pi-pier 0.2.6 → 0.3.0

package/src/jev-client.ts

@@ -0,0 +1,146 @@
+/**
+ * Jev decision layer — direct HTTP client (RFC docs/rfc-jev-integration.md §5).
+ *
+ * Why not the official SDK (@typesafe-ai/sdk v0.6.0): its retry machinery has
+ * no total-latency budget (DEFAULT_TIMEOUT_MS is per attempt, apiTimeoutError
+ * retries timeouts by default, Retry-After honored up to 60s — worst case
+ * >80s), which is incompatible with pier's fail-open budgets. The call surface
+ * here is one endpoint and three question types, and pi-pier ships as an npm
+ * package where a zero-dependency transport wins. Payload is pinned to the
+ * versioned model id; see the RFC for re-evaluation triggers.
+ *
+ * Every ask() resolves (never throws): errors, timeouts, 429s and low
+ * confidence at the call site all mean " unanswered" -> caller fails open.
+ */
+import { sha256Hex } from './observation-core.ts';
+import { appendEfficiencyLog, efficiencyLogPath } from './efficiency-store.ts';
+import { parseJevAnswers, type JevAnswer, type JevQuestion, type JevUsage } from './jev-core.ts';
+import type { JevConfig } from './efficiency-config-core.ts';
+
+export const JEV_DEFAULT_BASE_URL = 'https://api.typesafe.ai';
+
+export interface JevAskMeta {
+  /** Stable question-set id for telemetry rows (e.g. 'epr-diagnostic-gate'). */
+  questionId: string;
+  /** Total-budget override in ms; defaults to the config timeoutMs. */
+  timeoutMs?: number;
+  sessionId?: string;
+  /** Site-specific static fields for telemetry; must never contain request bodies. */
+  extra?: Record<string, unknown>;
+  /**
+   * Post-answer decision fields for telemetry (choices/scores/final verdicts —
+   * the data later threshold tuning needs; still never request bodies).
+   * Called exactly once per ask, success or failure.
+   */
+  enrich?: (outcome: { ok: boolean; answers: Record<string, JevAnswer> | null }) => Record<string, unknown>;
+}
+
+export type JevAskResult =
+  | { readonly ok: true; readonly answers: Record<string, JevAnswer>; readonly model: string; readonly usage: JevUsage; readonly latencyMs: number }
+  | { readonly ok: false; readonly reason: string; readonly latencyMs: number };
+
+export interface JevRuntimeDeps {
+  fetchImpl?: typeof fetch;
+  now?: () => number;
+  /** Session root for jev.jsonl; telemetry is skipped when absent. */
+  getSessionRoot?: () => string | null;
+}
+
+export interface JevRuntime {
+  readonly available: boolean;
+  ask(request: { state: unknown; questions: Record<string, JevQuestion> }, meta: JevAskMeta): Promise<JevAskResult>;
+}
+
+export function createJevRuntime(getConfig: () => JevConfig, deps: JevRuntimeDeps = {}): JevRuntime {
+  const doFetch = deps.fetchImpl ?? ((input, init) => globalThis.fetch(input, init));
+  const now = deps.now ?? Date.now;
+
+  async function logAsk(
+    config: JevConfig,
+    meta: JevAskMeta,
+    outcome: { ok: true; model: string; usage: JevUsage } | { ok: false; reason: string },
+    latencyMs: number,
+    state: unknown,
+    enriched: Record<string, unknown>,
+  ): Promise<void> {
+    if (!config.logEnabled) return;
+    const sessionRoot = deps.getSessionRoot?.() ?? null;
+    if (!sessionRoot) return;
+    const serialized = JSON.stringify(state) ?? '';
+    const record: Record<string, unknown> = {
+      schema: 'pier-efficiency/1',
+      mechanism: 'jev',
+      ts: new Date(now()).toISOString(),
+      sessionId: meta.sessionId ?? 'unknown',
+      questionId: meta.questionId,
+      model: outcome.ok ? outcome.model : config.model,
+      latencyMs,
+      verdict: outcome.ok ? 'answered' : 'fallback',
+      fallback: !outcome.ok,
+      reason: outcome.ok ? null : outcome.reason,
+      stateBytes: Buffer.byteLength(serialized, 'utf8'),
+      stateHash: sha256Hex(serialized),
+      ...(outcome.ok ? { usage: { input_tokens: outcome.usage.inputTokens, output_tokens: outcome.usage.outputTokens } } : {}),
+      ...(meta.extra ?? {}),
+      ...enriched,
+    };
+    try {
+      await appendEfficiencyLog(efficiencyLogPath(sessionRoot, 'jev'), record);
+    } catch {
+      // Telemetry is best effort; never surface logging failures.
+    }
+  }
+
+  return {
+    // Getter: config reloads (session_start re-resolves from disk) propagate.
+    get available(): boolean {
+      const config = getConfig();
+      return config.enabled && Boolean(config.apiKey);
+    },
+    async ask(request, meta): Promise<JevAskResult> {
+      const startedAt = now();
+      const config = getConfig();
+      const fail = async (reason: string): Promise<JevAskResult> => {
+        const latencyMs = now() - startedAt;
+        const enriched = meta.enrich?.({ ok: false, answers: null }) ?? {};
+        await logAsk(config, meta, { ok: false, reason }, latencyMs, request.state, enriched);
+        return { ok: false, reason, latencyMs };
+      };
+      if (!config.enabled) return fail('disabled');
+      if (!config.apiKey) return fail('no-api-key');
+
+      const timeoutMs = meta.timeoutMs ?? config.timeoutMs;
+      const controller = new AbortController();
+      const timer = setTimeout(() => controller.abort(), timeoutMs);
+      try {
+        const base = (config.baseUrl ?? JEV_DEFAULT_BASE_URL).replace(/\/+$/, '');
+        const response = await doFetch(`${base}/v1/systemone`, {
+          method: 'POST',
+          headers: {
+            Authorization: `Bearer ${config.apiKey}`,
+            'Content-Type': 'application/json',
+          },
+          body: JSON.stringify({ model: config.model, state: request.state, questions: request.questions }),
+          signal: controller.signal,
+        });
+        if (!response.ok) {
+          // 429 (and everything else) fails open immediately: limits are
+          // dynamically adjusted upstream and every pier call is droppable.
+          return fail(response.status === 429 ? 'rate-limited' : `http-${response.status}`);
+        }
+        const raw: unknown = await response.json();
+        const parsed = parseJevAnswers(raw, Object.keys(request.questions));
+        if (!parsed.ok) return fail(parsed.reason);
+        const latencyMs = now() - startedAt;
+        const enriched = meta.enrich?.({ ok: true, answers: parsed.answers }) ?? {};
+        await logAsk(config, meta, { ok: true, model: parsed.model, usage: parsed.usage }, latencyMs, request.state, enriched);
+        return { ok: true, answers: parsed.answers, model: parsed.model, usage: parsed.usage, latencyMs };
+      } catch (error) {
+        const aborted = error instanceof Error && error.name === 'AbortError';
+        return fail(aborted ? 'timeout' : 'network-error');
+      } finally {
+        clearTimeout(timer);
+      }
+    },
+  };
+}

package/src/jev-core.ts

@@ -0,0 +1,381 @@
+/**
+ * Jev decision layer — pure core (RFC docs/rfc-jev-integration.md).
+ *
+ * Question/answer shapes mirror the closed set of POST /v1/systemone
+ * (https://docs.typesafe.ai/api): noul / choice / score. No I/O here; the HTTP
+ * client + jev.jsonl telemetry live in jev-client.ts.
+ *
+ * Design rules from the RFC:
+ *  - arithmetic, counting and thresholds stay in code (jaggedness #2/#3);
+ *  - state is minimal — only the fields each question needs (jaggedness #5);
+ *  - instructions are English (models page: English is the primary training
+ *    language; CJK is handled but weaker, so P0-2 runs a stricter gate);
+ *  - any answer below its confidence gate counts as unanswered -> caller
+ *    fails open to the existing heuristic.
+ */
+import { FAILURE_SIGNAL } from './reducer-core.ts';
+
+// ---------------------------------------------------------------------------
+// Request/response shapes (closed set)
+// ---------------------------------------------------------------------------
+
+export type JevQuestion =
+  | { readonly type: 'noul'; readonly instructions: string; readonly criteria?: { readonly true?: string; readonly false?: string } }
+  | { readonly type: 'choice'; readonly instructions: string; readonly criteria: Record<string, string> }
+  | { readonly type: 'score'; readonly instructions: string; readonly criteria: readonly string[] };
+
+export interface JevUsage {
+  readonly inputTokens: number;
+  readonly outputTokens: number;
+}
+
+export type JevAnswer =
+  | { readonly type: 'noul'; readonly noul: number }
+  | { readonly type: 'choice'; readonly choice: string; readonly probabilities: Record<string, number>; readonly confidence: number }
+  | { readonly type: 'score'; readonly score: number; readonly probabilities: Record<string, number>; readonly confidence: number };
+
+export interface JevRequest {
+  readonly state: unknown;
+  readonly questions: Record<string, JevQuestion>;
+}
+
+export type JevParsed =
+  | { readonly ok: true; readonly answers: Record<string, JevAnswer>; readonly model: string; readonly usage: JevUsage }
+  | { readonly ok: false; readonly reason: string };
+
+function isPlainObject(v: unknown): v is Record<string, unknown> {
+  return typeof v === 'object' && v !== null && !Array.isArray(v);
+}
+
+function finiteNumber(v: unknown): number | undefined {
+  return typeof v === 'number' && Number.isFinite(v) ? v : undefined;
+}
+
+function unitNumber(v: unknown): number | undefined {
+  const n = finiteNumber(v);
+  return n !== undefined && n >= 0 && n <= 1 ? n : undefined;
+}
+
+function parseProbabilities(v: unknown): Record<string, number> | undefined {
+  if (!isPlainObject(v)) return undefined;
+  const out: Record<string, number> = {};
+  for (const [k, raw] of Object.entries(v)) {
+    const n = finiteNumber(raw);
+    if (n === undefined) return undefined;
+    out[k] = n;
+  }
+  return out;
+}
+
+/**
+ * Strict validation of one response against the question ids we sent.
+ * Extra answer ids are ignored; every expected id must be present and
+ * well-formed or the whole call counts as unanswered (fail-open).
+ */
+export function parseJevAnswers(raw: unknown, expectedIds: readonly string[]): JevParsed {
+  if (!isPlainObject(raw)) return { ok: false, reason: 'response-not-object' };
+  const model = typeof raw.model === 'string' && raw.model.trim() ? raw.model : undefined;
+  if (model === undefined) return { ok: false, reason: 'missing-model' };
+  const rawUsage = isPlainObject(raw.usage) ? raw.usage : undefined;
+  const inputTokens = finiteNumber(rawUsage?.input_tokens);
+  const outputTokens = finiteNumber(rawUsage?.output_tokens);
+  if (inputTokens === undefined || inputTokens < 0 || outputTokens === undefined || outputTokens < 0) {
+    return { ok: false, reason: 'bad-usage' };
+  }
+  const rawAnswers = isPlainObject(raw.answers) ? raw.answers : undefined;
+  if (rawAnswers === undefined) return { ok: false, reason: 'missing-answers' };
+
+  const answers: Record<string, JevAnswer> = {};
+  for (const id of expectedIds) {
+    const a = isPlainObject(rawAnswers[id]) ? rawAnswers[id] : undefined;
+    if (a === undefined) return { ok: false, reason: `missing-answer:${id}` };
+    if (a.type === 'noul') {
+      const noul = unitNumber(a.noul);
+      if (noul === undefined) return { ok: false, reason: `bad-noul:${id}` };
+      answers[id] = { type: 'noul', noul };
+      continue;
+    }
+    if (a.type === 'choice') {
+      const choice = typeof a.choice === 'string' ? a.choice : undefined;
+      const probabilities = parseProbabilities(a.probabilities);
+      const confidence = unitNumber(a.confidence);
+      if (choice === undefined || probabilities === undefined || confidence === undefined) {
+        return { ok: false, reason: `bad-choice:${id}` };
+      }
+      answers[id] = { type: 'choice', choice, probabilities, confidence };
+      continue;
+    }
+    if (a.type === 'score') {
+      const score = finiteNumber(a.score);
+      const probabilities = parseProbabilities(a.probabilities);
+      const confidence = unitNumber(a.confidence);
+      if (score === undefined || score < 0 || probabilities === undefined || confidence === undefined) {
+        return { ok: false, reason: `bad-score:${id}` };
+      }
+      answers[id] = { type: 'score', score, probabilities, confidence };
+      continue;
+    }
+    return { ok: false, reason: `unknown-answer-type:${id}` };
+  }
+  return { ok: true, answers, model, usage: { inputTokens, outputTokens } };
+}
+
+// ---------------------------------------------------------------------------
+// P0-1: EPR diagnostic-command gate
+// ---------------------------------------------------------------------------
+
+export const DIAGNOSTIC_OUTPUT_MIN_NOUL = 0.6;
+
+const DIAGNOSTIC_GATE_QUESTIONS: Record<string, JevQuestion> = {
+  cmd_kind: {
+    type: 'choice',
+    instructions: 'What kind of shell command is this?',
+    criteria: {
+      build_test_run: 'Runs a build, test, lint, or typecheck pipeline whose output is repetitive diagnostic log lines (pass/fail, errors, warnings)',
+      install_deps: 'Installs or updates dependencies (npm/pip/cargo fetch); output is progress bars and version lists',
+      server_watch: 'Starts a long-running server, watcher, or REPL that stays in the foreground',
+      source_inspection: 'Prints or searches source/config (cat, ls, grep, git log, diff)',
+      other: 'None of the above',
+    },
+  },
+  diagnostic_output: {
+    type: 'noul',
+    instructions: 'Will this command produce repetitive test/build diagnostic log output (rather than conversational or interactive output)?',
+  },
+};
+
+export function diagnosticGateRequest(command: string): JevRequest {
+  return { state: { command }, questions: DIAGNOSTIC_GATE_QUESTIONS };
+}
+
+export interface DiagnosticGateVerdict {
+  readonly hit: boolean;
+  readonly reason: string;
+  readonly confidence: number | null;
+  /** Selected cmd_kind option (telemetry); null when the answer was malformed. */
+  readonly choice: string | null;
+  /** diagnostic_output noul value (telemetry); null when the answer was malformed. */
+  readonly noul: number | null;
+}
+export function evaluateDiagnosticGate(
+  answers: Record<string, JevAnswer>,
+  minConfidence: number,
+): DiagnosticGateVerdict {
+  const kind = answers.cmd_kind;
+  const output = answers.diagnostic_output;
+  if (kind === undefined || output === undefined || kind.type !== 'choice' || output.type !== 'noul') {
+    return { hit: false, reason: 'missing-answer', confidence: null, choice: null, noul: null };
+  }
+  if (kind.choice !== 'build_test_run') {
+    return { hit: false, reason: `not-build-test:${kind.choice}`, confidence: kind.confidence, choice: kind.choice, noul: output.noul };
+  }
+  if (kind.confidence < minConfidence) {
+    return { hit: false, reason: 'low-confidence', confidence: kind.confidence, choice: kind.choice, noul: output.noul };
+  }
+  if (output.noul < DIAGNOSTIC_OUTPUT_MIN_NOUL) {
+    return { hit: false, reason: 'non-diagnostic-output', confidence: kind.confidence, choice: kind.choice, noul: output.noul };
+  }
+  return { hit: true, reason: 'hit', confidence: kind.confidence, choice: kind.choice, noul: output.noul };
+}
+
+// ---------------------------------------------------------------------------
+// P0-2: settlement-notice relevance ranking
+// ---------------------------------------------------------------------------
+
+/**
+ * Stricter than the global minConfidence default: settlement text is mostly
+ * Chinese while English is jev's primary training language (models page), so
+ * the initial gate starts at 0.7 and calibrates from jev.jsonl.
+ */
+export const NOTICE_RANK_MIN_CONFIDENCE = 0.7;
+export const RANK_SCORE_WEIGHT = 0.6;
+
+const RANK_LEVELS = [
+  'Routine completion; the master agent can check it later',
+  'The master agent should be aware; it affects planning',
+  'Reports a failure, error, or blocked state needing immediate attention',
+] as const;
+
+export function noticeRankRequest(input: {
+  inProgressTodos: readonly string[];
+  notices: readonly string[];
+}): JevRequest {
+  const questions: Record<string, JevQuestion> = {};
+  for (let i = 0; i < input.notices.length; i++) {
+    // Question keys are NOT sent to the model, so identical instructions would
+    // collapse every notice into the same question (observed live: 5x "1.98:0.97").
+    // Each instruction must name its settlement by state index (cookbook pattern).
+    questions[`notice_${i}_rank`] = {
+      type: 'score',
+      instructions: `How urgently does the master agent need to see settlements[${i}] right now, given in_progress_todos?`,
+      criteria: RANK_LEVELS,
+    };
+    questions[`notice_${i}_fail`] = {
+      type: 'noul',
+      instructions: `Does settlements[${i}] report a failure, error, or blocked state?`,
+    };
+  }
+  return {
+    state: {
+      in_progress_todos: input.inProgressTodos,
+      settlements: input.notices,
+    },
+    questions,
+  };
+}
+
+/**
+ * Failure-pin threshold: a settlement whose fail-noul clears this is forced to
+ * the front regardless of its relevance score. Live calibration (2026-09-18):
+ * two runs over the same Chinese batch reordered a failure notice from #2 to
+ * #4 — beyond the 3-shown cap. Score noise on CJK must not be able to bury a
+ * failure; the invariant lives in code, not in the model.
+ */
+export const NOTICE_FAIL_PIN_THRESHOLD = 0.7;
+
+/**
+ * Compose the display order: pinned failures first, then descending composed
+ * relevance (stable on ties -> arrival order). Per-item confidence gate: an
+ * item whose score answer is below the gate sinks to the routine bucket
+ * instead of voiding the batch (one unsure routine notice must not reorder
+ * everything back to arrival). All items gated -> null (nothing was answered).
+ * The fail pin itself ignores the score gate: noul answers carry no confidence
+ * field, and failure visibility is the safety property.
+ */
+export function composeNoticeRanking(
+  noticeCount: number,
+  answers: Record<string, JevAnswer>,
+  minConfidence: number,
+): number[] | null {
+  const entries: Array<{ pin: number; value: number; index: number }> = [];
+  let answered = 0;
+  for (let i = 0; i < noticeCount; i++) {
+    const rank = answers[`notice_${i}_rank`];
+    const fail = answers[`notice_${i}_fail`];
+    if (rank === undefined || fail === undefined || rank.type !== 'score' || fail.type !== 'noul') {
+      return null;
+    }
+    const gated = rank.confidence < minConfidence;
+    if (!gated) answered++;
+    const normalizedScore = rank.score / Math.max(1, RANK_LEVELS.length - 1);
+    entries.push({
+      pin: fail.noul >= NOTICE_FAIL_PIN_THRESHOLD ? 1 : 0,
+      value: gated ? 0 : RANK_SCORE_WEIGHT * normalizedScore + (1 - RANK_SCORE_WEIGHT) * fail.noul,
+      index: i,
+    });
+  }
+  if (answered === 0) return null;
+  entries.sort((a, b) => (b.pin - a.pin) || (b.value - a.value) || (a.index - b.index));
+  return entries.map((entry) => entry.index);
+}
+
+// ---------------------------------------------------------------------------
+// P0-3: OBS excerpt window candidates (head/tail halves miss the middle)
+// ---------------------------------------------------------------------------
+
+export type ExcerptWindowId = 'first_signal' | 'densest';
+
+export interface ExcerptWindow {
+  readonly id: ExcerptWindowId;
+  readonly label: string;
+  readonly text: string;
+}
+
+interface LineWindow {
+  readonly text: string;
+  readonly bytes: number;
+  readonly signalCount: number;
+}
+
+function buildLineWindow(lines: readonly string[], start: number, budgetBytes: number, signalLines: ReadonlySet<number>): LineWindow | null {
+  const selected: string[] = [];
+  let bytes = 0;
+  let signalCount = 0;
+  for (let i = start; i < lines.length; i++) {
+    const line = lines[i]!;
+    const lineBytes = Buffer.byteLength(line, 'utf8') + 1; // +1 newline
+    if (bytes + lineBytes > budgetBytes) {
+      if (selected.length === 0) return null; // anchor line alone exceeds the budget
+      break;
+    }
+    selected.push(line);
+    bytes += lineBytes;
+    if (signalLines.has(i)) signalCount++;
+  }
+  if (selected.length === 0) return null;
+  return { text: selected.join('\n'), bytes, signalCount };
+}
+
+/**
+ * Generate code-owned middle-window candidates for a packed observation:
+ * the region around the first failure-signal line, and the densest cluster
+ * of failure-signal lines within one half-budget. Deterministic, no model.
+ */
+export function buildExcerptWindows(text: string, halfBudgetBytes: number): ExcerptWindow[] {
+  if (halfBudgetBytes <= 0) return [];
+  const lines = text.split('\n');
+  const signalLines = new Set<number>();
+  for (let i = 0; i < lines.length; i++) {
+    if (FAILURE_SIGNAL.test(lines[i]!)) signalLines.add(i);
+  }
+  if (signalLines.size === 0) return [];
+
+  const out: ExcerptWindow[] = [];
+  const seen = new Set<string>();
+
+  const first = Math.min(...signalLines);
+  const firstWindow = buildLineWindow(lines, Math.max(0, first - 2), halfBudgetBytes, signalLines);
+  if (firstWindow !== null && !seen.has(firstWindow.text)) {
+    seen.add(firstWindow.text);
+    out.push({ id: 'first_signal', label: 'first failure-signal region', text: firstWindow.text });
+  }
+
+  // Densest cluster: anchor at every signal line, extend greedily while in budget.
+  let best: LineWindow | null = null;
+  for (const anchor of signalLines) {
+    const window = buildLineWindow(lines, anchor, halfBudgetBytes, signalLines);
+    if (window === null) continue;
+    if (best === null || window.signalCount > best.signalCount) best = window;
+  }
+  if (best !== null && !seen.has(best.text)) {
+    seen.add(best.text);
+    out.push({ id: 'densest', label: 'densest failure-signal region', text: best.text });
+  }
+  return out;
+}
+
+export function excerptPickRequest(
+  middleWindows: readonly ExcerptWindow[],
+  headExcerpt: string,
+  tailExcerpt: string,
+): JevRequest | null {
+  if (middleWindows.length === 0) return null;
+  const criteria: Record<string, string> = {
+    keep_head_tail: 'The existing head and tail excerpts are already the most informative; keep them',
+  };
+  const excerpts: Record<string, string> = { head_excerpt: headExcerpt, tail_excerpt: tailExcerpt };
+  for (const window of middleWindows) {
+    criteria[window.id] = `${window.label}: excerpt from the middle of the output`;
+    excerpts[window.id] = window.text;
+  }
+  return {
+    state: { excerpts },
+    questions: {
+      window: {
+        type: 'choice',
+        instructions: 'A large tool output was replaced by a placeholder keeping two short excerpts. Which excerpt set would most help a coding agent locate the problem without recalling the full text?',
+        criteria,
+      },
+    },
+  };
+}
+
+export function evaluateExcerptPick(
+  answers: Record<string, JevAnswer>,
+  minConfidence: number,
+): ExcerptWindowId | null {
+  const picked = answers.window;
+  if (picked === undefined || picked.type !== 'choice') return null;
+  if (picked.confidence < minConfidence) return null;
+  return picked.choice === 'first_signal' || picked.choice === 'densest' ? picked.choice : null;
+}