pi-jev-auto-mode 0.1.0

package/src/intent.ts

@@ -0,0 +1,68 @@
+/**
+ * Recover what the user actually asked for.
+ *
+ * Only user-authored text is used. Assistant messages and tool output are
+ * attacker-influenced in general (they contain file contents and command output),
+ * so letting them shape "the user intent" would let repository content argue for
+ * its own approval.
+ */
+
+export interface IntentOptions {
+  /** How many of the most recent user messages to consider. */
+  readonly maxMessages: number;
+  readonly maxMessageChars: number;
+  readonly maxTotalChars: number;
+}
+
+export const DEFAULT_INTENT_OPTIONS: IntentOptions = {
+  maxMessages: 8,
+  maxMessageChars: 1200,
+  maxTotalChars: 4000,
+};
+
+export const NO_INTENT_TEXT = "";
+
+function truncate(value: string, maxLength: number): string {
+  return value.length > maxLength ? `${value.slice(0, maxLength)}...` : value;
+}
+
+/** Flatten a Pi message content value into plain text. */
+export function messageText(content: unknown): string {
+  if (typeof content === "string") return content;
+  if (!Array.isArray(content)) return "";
+
+  return content
+    .filter((part): part is { type?: unknown; text?: unknown } => Boolean(part) && typeof part === "object")
+    .filter((part) => part.type === "text" && typeof part.text === "string")
+    .map((part) => part.text as string)
+    .join("\n")
+    .trim();
+}
+
+/**
+ * Recent user turns, oldest first.
+ *
+ * Messages carrying a `customType` are extension-injected context (plan mode and
+ * similar), not user speech, so they are skipped.
+ */
+export function extractRecentIntent(
+  branch: readonly unknown[],
+  options: IntentOptions = DEFAULT_INTENT_OPTIONS,
+): string {
+  const collected: string[] = [];
+
+  for (let index = branch.length - 1; index >= 0 && collected.length < options.maxMessages; index -= 1) {
+    const entry = branch[index];
+    if (!entry || typeof entry !== "object") continue;
+    if ((entry as { type?: unknown }).type !== "message") continue;
+
+    const message = (entry as { message?: { role?: unknown; content?: unknown; customType?: unknown } }).message;
+    if (!message || message.role !== "user") continue;
+    if (typeof message.customType === "string" && message.customType.length > 0) continue;
+
+    const text = truncate(messageText(message.content), options.maxMessageChars);
+    if (text) collected.push(text);
+  }
+
+  return truncate(collected.reverse().join("\n\n"), options.maxTotalChars);
+}

package/src/jev/availability.ts

@@ -0,0 +1,51 @@
+/**
+ * Is the semantic layer usable right now, and where did the key come from?
+ *
+ * A missing key must not silently degrade into "allow everything": the engine
+ * falls back to the ask-only engine, which confirms in a UI and blocks without
+ * one. The reason and the key's origin are surfaced in `/jev-auto-mode status` so
+ * the degradation and the credential path are visible rather than mysterious.
+ *
+ * `TYPESAFE_API_KEY` wins over the stored secret, so a one-off or CI override does
+ * not require touching the stored credential.
+ */
+
+export type JevKeySource = "env" | "stored" | "none";
+
+export interface JevAvailability {
+  readonly available: boolean;
+  readonly apiKey?: string;
+  readonly model: string;
+  readonly source: JevKeySource;
+  readonly reason?: string;
+}
+
+export function describeJevAvailability(
+  env: NodeJS.ProcessEnv = process.env,
+  storedApiKey?: string,
+): JevAvailability {
+  const model = env.TYPESAFE_DEFAULT_MODEL?.trim() || "jev-latest";
+
+  const environmentKey = env.TYPESAFE_API_KEY?.trim();
+  if (environmentKey) {
+    return { available: true, apiKey: environmentKey, model, source: "env" };
+  }
+
+  const stored = storedApiKey?.trim();
+  if (stored) {
+    return { available: true, apiKey: stored, model, source: "stored" };
+  }
+
+  return {
+    available: false,
+    model,
+    source: "none",
+    reason: "no TypeSafe API key is available (run /jev-auto-mode login, or set TYPESAFE_API_KEY)",
+  };
+}
+
+export function describeKeySource(source: JevKeySource): string {
+  if (source === "env") return "TYPESAFE_API_KEY";
+  if (source === "stored") return "stored secret (/jev-auto-mode login)";
+  return "none";
+}

package/src/jev/criteria.ts

@@ -0,0 +1,19 @@
+/**
+ * Shared noul criteria.
+ *
+ * The default criteria matter more than they look. JEV is calibrated, and
+ * `noul` has no `confidence` field: the only signal is the probability. If the
+ * criteria leave the middle open, an ambiguous condition lands somewhere in the
+ * middle and the two-sided thresholds in `decide.ts` can route it to a human.
+ * If they force every answer to an extreme, "uncertain" stops existing and the
+ * gate has to guess.
+ */
+
+import type { JevEntry } from "./types.ts";
+
+export const DEFAULT_CRITERIA: { readonly true: JevEntry; readonly false: JevEntry } = {
+  true: "The condition clearly holds for the item under validation.",
+  false:
+    "The condition clearly does not hold for the item under validation. " +
+    "An item the state says nothing about, or that is too ambiguous to decide, is neither clearly true nor clearly false.",
+};

package/src/jev/decide.ts

@@ -0,0 +1,187 @@
+/**
+ * Probability to decision.
+ *
+ * Two symmetric thresholds per condition:
+ *
+ *   p >= t          → satisfied
+ *   p <= 1 - t      → rejected ("the opposite is as certain as a pass would be")
+ *   in between      → the middle band
+ *
+ * The middle band is not a bug to be squeezed out. Measured JEV answers sit at
+ * 0.98/0.02 for clear cases but also at 0.85–0.95 for conditions that are clear
+ * to a human and merely not certain to the model, so a single high bar would
+ * report almost everything as uncertain. What the middle band *means* depends on
+ * the rule:
+ *
+ *   required → the middle band is escalated (ask the user)
+ *   hazard   → the middle band is ignored, because "no hazard is evident" is not
+ *              the same as "a hazard is present"
+ *
+ * Composition is done here, in code, so the model never has to weigh concerns
+ * against each other. A single un-cleared hazard decides the call.
+ */
+
+import type { JevRule } from "./questions.ts";
+
+export type ConditionVerdict = "satisfied" | "rejected" | "uncertain";
+
+/**
+ * Tolerance for the two boundaries.
+ *
+ * `1 - 0.9` is `0.09999999999999998` in IEEE 754, so an exact comparison would put
+ * `p = 0.1, t = 0.9` in the middle band even though the rule says a rejection.
+ * The widened side is the reject side, which is the direction a gate should err in.
+ */
+const BOUNDARY_EPSILON = 1e-9;
+
+export interface Observation {
+  readonly ruleId: string;
+  readonly probability: number;
+  readonly threshold: number;
+  /** The band the probability falls into. */
+  readonly verdict: ConditionVerdict;
+  /**
+   * What the rule's mode makes of that band. A `hazard` rule that lands in the
+   * middle band is treated as satisfied, because "no hazard is evident" is not
+   * the same as "a hazard is present". Kept separate from `verdict` so the
+   * distinction stays visible in records.
+   */
+  readonly effective: ConditionVerdict;
+}
+
+export function classifyCondition(probability: number, threshold: number): ConditionVerdict {
+  if (probability >= threshold - BOUNDARY_EPSILON) return "satisfied";
+  if (probability <= 1 - threshold + BOUNDARY_EPSILON) return "rejected";
+  return "uncertain";
+}
+
+export function observe(rules: readonly JevRule[], answers: Readonly<Record<string, number>>): Observation[] {
+  return rules.map((rule) => {
+    // A missing answer must not become an approval. The engine rejects a response
+    // with missing keys before reaching here; defaulting to 0 means "rejected".
+    const probability = answers[rule.id] ?? 0;
+    const verdict = classifyCondition(probability, rule.threshold);
+    const effective = rule.mode === "hazard" && verdict === "uncertain" ? "satisfied" : verdict;
+    return { ruleId: rule.id, probability, threshold: rule.threshold, verdict, effective };
+  });
+}
+
+export interface CombinedDecision {
+  readonly verdict: "allow" | "deny" | "uncertain";
+  readonly rationale: string;
+  readonly probabilities: Readonly<Record<string, number>>;
+  /** The condition that decided the call, when one did. */
+  readonly decidingRule: string | undefined;
+  /** Rules whose clear rejection was cleared by the user's own request. */
+  readonly clearedByIntent: readonly string[];
+}
+
+const INTENT_RULE_ID = "intent_coverage";
+
+function formatProbability(value: number): string {
+  return value.toFixed(2);
+}
+
+/**
+ * Format a threshold without losing it to rounding, and keep plain values aligned.
+ *
+ * `0.995.toFixed(2)` is `"0.99"`, which would display a threshold that is not the one
+ * in effect. Tuning reads these numbers, so they have to be the real ones: two decimals
+ * when that is exact, more when it is not.
+ */
+export function formatThreshold(value: number): string {
+  const twoDecimals = value.toFixed(2);
+  return Number(twoDecimals) === Number(value.toFixed(4)) ? twoDecimals : Number(value.toFixed(4)).toString();
+}
+
+function describe(rules: readonly JevRule[], observation: Observation): string {
+  const rule = rules.find((candidate) => candidate.id === observation.ruleId);
+  return `${rule?.label ?? observation.ruleId} (p=${formatProbability(observation.probability)})`;
+}
+
+/**
+ * Combine conditions:
+ *
+ * 1. A rejected `hazard`-severity rule blocks, whatever the user asked for.
+ * 2. A rejected `soft` rule is cleared when the intent condition is satisfied —
+ *    the user's own words are the authority for actions they are entitled to
+ *    request, and the deterministic layer already holds the non-negotiable line.
+ * 3. An unclear `required` condition escalates to a confirmation.
+ * 4. Otherwise the call is approved.
+ */
+export function combine(rules: readonly JevRule[], observations: readonly Observation[]): CombinedDecision {
+  const probabilities: Record<string, number> = {};
+  for (const observation of observations) probabilities[observation.ruleId] = observation.probability;
+
+  const severityOf = (observation: Observation): JevRule["severity"] =>
+    rules.find((rule) => rule.id === observation.ruleId)?.severity ?? "hazard";
+
+  const rejected = observations.filter((observation) => observation.effective === "rejected");
+  const blocking = rejected.filter((observation) => severityOf(observation) === "hazard");
+  if (blocking.length > 0) {
+    const first = blocking[0] as Observation;
+    const rule = rules.find((candidate) => candidate.id === first.ruleId);
+    return {
+      verdict: "deny",
+      rationale: `${rule?.denyMessage ?? "A safety condition was clearly violated."} ${describe(rules, first)}`,
+      probabilities,
+      decidingRule: first.ruleId,
+      clearedByIntent: [],
+    };
+  }
+
+  const intentSatisfied = observations.some(
+    (observation) => observation.ruleId === INTENT_RULE_ID && observation.effective === "satisfied",
+  );
+
+  const soft = rejected.filter((observation) => severityOf(observation) === "soft");
+  if (soft.length > 0) {
+    const first = soft[0] as Observation;
+    const rule = rules.find((candidate) => candidate.id === first.ruleId);
+    if (!intentSatisfied) {
+      return {
+        verdict: "deny",
+        rationale: `${rule?.denyMessage ?? "A safety condition was clearly violated."} ${describe(rules, first)}`,
+        probabilities,
+        decidingRule: first.ruleId,
+        clearedByIntent: [],
+      };
+    }
+    return {
+      verdict: "allow",
+      rationale: `The user's request covers this call, clearing ${describe(rules, first)}.`,
+      probabilities,
+      decidingRule: first.ruleId,
+      clearedByIntent: soft.map((observation) => observation.ruleId),
+    };
+  }
+
+  const uncertain = observations.filter((observation) => observation.effective === "uncertain");
+  const firstUncertain = uncertain[0];
+  if (firstUncertain) {
+    const rule = rules.find((candidate) => candidate.id === firstUncertain.ruleId);
+    return {
+      verdict: "uncertain",
+      rationale: `${rule?.uncertainMessage ?? "A safety condition could not be decided."} ${describe(rules, firstUncertain)}`,
+      probabilities,
+      decidingRule: firstUncertain.ruleId,
+      clearedByIntent: [],
+    };
+  }
+
+  const lowest = observations.reduce<Observation | undefined>(
+    (current, observation) =>
+      current === undefined || observation.probability < current.probability ? observation : current,
+    undefined,
+  );
+  return {
+    verdict: "allow",
+    rationale:
+      lowest === undefined
+        ? "No safety conditions applied."
+        : `No hazard was evident across ${observations.length} conditions (lowest p=${formatProbability(lowest.probability)} on ${lowest.ruleId}).`,
+    probabilities,
+    decidingRule: undefined,
+    clearedByIntent: [],
+  };
+}

package/src/jev/engine.ts

@@ -0,0 +1,163 @@
+/**
+ * The JEV decision engine.
+ *
+ * One tool call in, one request out. All conditions for the call travel in the
+ * same request because JEV answers them in parallel and independently, so the
+ * marginal cost of an extra condition is a few tokens rather than a round trip.
+ *
+ * Everything that can go wrong resolves to `unavailable`, and the caller turns
+ * that into a block. The engine never invents an approval.
+ */
+
+import { toJevState, NO_POLICY_PLACEHOLDER } from "../call.ts";
+import type { CandidateInput, ConditionReport, DecisionEngine, EngineVerdict, JudgeOptions } from "../decide.ts";
+import { combine, observe, type Observation } from "./decide.ts";
+import { DEFAULT_RULES, applyThresholdOverrides, buildQuestions, rulesForTool, type JevRule } from "./questions.ts";
+import { parseAnswers } from "./response.ts";
+import type { JevTransport } from "./types.ts";
+
+export interface ObservationMeta {
+  readonly model: string;
+  readonly latencyMs: number;
+  readonly inputTokens: number;
+  readonly outputTokens: number;
+}
+
+export interface JevEngineOptions {
+  readonly transport: JevTransport;
+  readonly rules?: readonly JevRule[];
+  readonly model?: string;
+  /** Per-rule threshold overrides from settings. */
+  readonly thresholds?: Readonly<Record<string, number>>;
+  /** Shared state + questions budget, in characters. */
+  readonly maxStateCharacters?: number;
+  readonly now?: () => number;
+  /**
+   * Called for every condition of every judgment, including the ones that
+   * passed. This is the calibration channel: without the passing probabilities
+   * there is no way to choose a threshold that is not a guess.
+   */
+  readonly onObservation?: (observations: readonly Observation[], meta: ObservationMeta) => void;
+}
+
+export const DEFAULT_MAX_STATE_CHARACTERS = 120_000;
+
+const UNAVAILABLE_TEXT: Record<string, string> = {
+  timeout: "the JEV request timed out",
+  network: "the JEV request could not reach the API",
+  http: "the JEV API returned an error status",
+  malformed_response: "the JEV response did not match the questions that were asked",
+  state_too_large: "the call description exceeded the request budget",
+  unknown: "the JEV request failed for an unknown reason",
+};
+
+export function createJevEngine(options: JevEngineOptions): DecisionEngine {
+  const rules = options.rules ?? DEFAULT_RULES;
+  const maxStateCharacters = options.maxStateCharacters ?? DEFAULT_MAX_STATE_CHARACTERS;
+  const now = options.now ?? (() => Date.now());
+
+  return {
+    id: "jev",
+
+    async judge(input: CandidateInput, judgeOptions: JudgeOptions): Promise<EngineVerdict> {
+      // The policy condition is skipped when no policy is configured: asking
+      // "does this violate the policy" with an empty policy produced 0.66-0.85 on
+      // every fixture, which would have escalated every call.
+      const hasPolicy = input.policy.trim().length > 0 && input.policy !== NO_POLICY_PLACEHOLDER;
+      const applicable = applyThresholdOverrides(
+        rulesForTool(input.call.tool, rules, {
+          hasPolicy,
+          hasProtectedTarget: input.call.protectedReason !== undefined,
+        }),
+        options.thresholds,
+      );
+      const state = toJevState(input);
+      const questions = buildQuestions(applicable);
+
+      // The API budget is shared between state and questions, and a request that
+      // is too large is rejected before it is sent: cheaper, and it keeps a huge
+      // diff or command from being truncated on the remote side.
+      const characters = JSON.stringify(state).length + JSON.stringify(questions).length;
+      if (characters > maxStateCharacters) {
+        return {
+          verdict: "unavailable",
+          reason: "state_too_large",
+          rationale: `The call description was ${characters} characters, over the ${maxStateCharacters} character budget.`,
+        };
+      }
+
+      const startedAt = now();
+      const result = await options.transport.systemOne({
+        state,
+        questions,
+        ...(options.model === undefined ? {} : { model: options.model }),
+        ...(judgeOptions.signal === undefined ? {} : { signal: judgeOptions.signal }),
+      });
+      const latencyMs = now() - startedAt;
+
+      if (!result.ok) {
+        return {
+          verdict: "unavailable",
+          reason: result.reason,
+          rationale: UNAVAILABLE_TEXT[result.reason] ?? UNAVAILABLE_TEXT.unknown ?? "JEV could not decide.",
+          latencyMs,
+        };
+      }
+
+      const parsed = parseAnswers(result.response, Object.keys(questions));
+      if (!parsed.ok) {
+        return {
+          verdict: "unavailable",
+          reason: parsed.reason,
+          rationale: UNAVAILABLE_TEXT[parsed.reason] ?? "The JEV response could not be used.",
+          latencyMs,
+        };
+      }
+
+      const observations = observe(applicable, parsed.parsed.answers);
+      options.onObservation?.(observations, {
+        model: parsed.parsed.model,
+        latencyMs,
+        inputTokens: parsed.parsed.inputTokens,
+        outputTokens: parsed.parsed.outputTokens,
+      });
+
+      const combined = combine(applicable, observations);
+      const thresholds: Record<string, number> = {};
+      for (const rule of applicable) thresholds[rule.id] = rule.threshold;
+
+      const cleared = new Set(combined.clearedByIntent);
+      const conditions: ConditionReport[] = applicable.map((rule, index) => {
+        const observation = observations[index];
+        const verdict = observation?.verdict ?? "uncertain";
+        return {
+          ruleId: rule.id,
+          label: rule.label,
+          probability: observation?.probability ?? 0,
+          threshold: rule.threshold,
+          verdict: verdict === "uncertain" && rule.mode === "hazard" ? "ignored" : verdict,
+          clearedByIntent: cleared.has(rule.id),
+        };
+      });
+
+      const evidence = {
+        probabilities: combined.probabilities,
+        thresholds,
+        conditions,
+        model: parsed.parsed.model,
+        latencyMs,
+        ...(combined.decidingRule === undefined ? {} : { decidingRule: combined.decidingRule }),
+        ...(combined.clearedByIntent.length === 0 ? {} : { clearedByIntent: combined.clearedByIntent }),
+      };
+
+      switch (combined.verdict) {
+        case "allow":
+          return { verdict: "allow", rationale: combined.rationale, ...evidence };
+        case "deny":
+          return { verdict: "deny", rationale: combined.rationale, ...evidence };
+        case "uncertain":
+          return { verdict: "uncertain", rationale: combined.rationale, ...evidence };
+      }
+    },
+  };
+}

package/src/jev/index.ts

@@ -0,0 +1,20 @@
+/**
+ * JEV layer: transport, question set, response validation, and the engine.
+ */
+
+export { createJevEngine, DEFAULT_MAX_STATE_CHARACTERS, type JevEngineOptions, type ObservationMeta } from "./engine.ts";
+export { describeJevAvailability, describeKeySource, type JevAvailability, type JevKeySource } from "./availability.ts";
+export { createSdkTransport, isAbortError, verifyApiKey, type ApiKeyVerification, type SdkTransportOptions } from "./transport.ts";
+export { DEFAULT_RULES, applyThresholdOverrides, buildQuestions, ruleById, rulesForTool, type JevRule, type JevRuleMode, type JevSeverity } from "./questions.ts";
+export { DEFAULT_CRITERIA } from "./criteria.ts";
+export { classifyCondition, combine, formatThreshold, observe, type CombinedDecision, type ConditionVerdict, type Observation } from "./decide.ts";
+export { parseAnswers, type ParsedAnswers } from "./response.ts";
+export type {
+  JevEntry,
+  JevNoulQuestion,
+  JevRequest,
+  JevTransport,
+  JevTransportResult,
+  JevUnavailableReason,
+} from "./types.ts";
+export type { JevJson, JevState } from "./state.ts";