pi-jev-guard 0.1.0

package/src/ask.ts

@@ -0,0 +1,189 @@
+/**
+ * jev_ask: tool generico per decisioni tipate del modello.
+ *
+ * A differenza di jev_validate (3 regole fisse di policy), qui il modello
+ * definisce le sue domande (noul/choice/score) su un testo e riceve risposte
+ * calibrate invece di prosa. Istruzioni di lettura allineate alla nostra
+ * calibrazione (soglie 0.20/0.90, confidence floor 0.6).
+ */
+import type { ExtensionAPI } from "@earendil-works/pi-coding-agent";
+import { Type } from "typebox";
+import type { JevConfig } from "./config.ts";
+import type {
+  TypedAnswer,
+  TypedAskFn,
+  TypedAskResult,
+  TypedQuestion,
+} from "./reviewer.ts";
+
+export const ASK_MAX_STATE_CHARS = 20000;
+export const ASK_MAX_QUESTIONS = 8;
+
+const QuestionParam = Type.Object({
+  id: Type.String({
+    minLength: 1,
+    maxLength: 64,
+    description: "Short key for this question. The answer comes back under it.",
+  }),
+  type: Type.Union([Type.Literal("noul"), Type.Literal("choice"), Type.Literal("score")], {
+    description: "noul = yes/no probability, choice = pick one option, score = value on a rubric",
+  }),
+  instructions: Type.String({
+    minLength: 1,
+    maxLength: 2000,
+    description: "The one thing to judge. One specific, well-scoped gut-check per question.",
+  }),
+  options: Type.Optional(
+    Type.Array(
+      Type.Object({
+        name: Type.String({ minLength: 1, maxLength: 64, description: "Option key" }),
+        description: Type.Optional(
+          Type.String({ maxLength: 500, description: "When this option applies" }),
+        ),
+      }),
+      { minItems: 1, maxItems: 12, description: "choice only: the options to choose between" },
+    ),
+  ),
+  levels: Type.Optional(
+    Type.Array(Type.String({ minLength: 1, maxLength: 200 }), {
+      minItems: 2,
+      maxItems: 8,
+      description: "score only: ordered rubric levels, lowest first, at least two",
+    }),
+  ),
+});
+
+export const AskParams = Type.Object({
+  state: Type.String({
+    minLength: 1,
+    maxLength: ASK_MAX_STATE_CHARS,
+    description: "The text to judge: tool output, a diff, a message, a document excerpt.",
+  }),
+  questions: Type.Array(QuestionParam, {
+    minItems: 1,
+    maxItems: ASK_MAX_QUESTIONS,
+    description: "One or more questions. All are evaluated in parallel against the same state.",
+  }),
+});
+
+export interface AskQuestionParam {
+  id: string;
+  type: "noul" | "choice" | "score";
+  instructions: string;
+  options?: { name: string; description?: string }[];
+  levels?: string[];
+}
+
+/** Costruisce la domanda Jev, oppure il motivo per cui la forma non vale. */
+export function toTypedQuestion(question: AskQuestionParam): TypedQuestion | string {
+  const { id, instructions } = question;
+  if (question.type === "choice") {
+    if (!question.options || question.options.length === 0) {
+      return `question "${id}": choice needs at least one option`;
+    }
+    const options: Record<string, string | null> = {};
+    for (const option of question.options) {
+      options[option.name] = option.description ?? null;
+    }
+    return { kind: "choice", id, instructions, options };
+  }
+  if (question.type === "score") {
+    if (!question.levels || question.levels.length < 2) {
+      return `question "${id}": score needs at least two levels`;
+    }
+    return { kind: "score", id, instructions, levels: [...question.levels] };
+  }
+  return { kind: "noul", id, instructions };
+}
+
+function describeAnswer(answer: TypedAnswer): string {
+  if (answer.type === "noul") return `yes ${answer.p.toFixed(2)}`;
+  if (answer.type === "choice") {
+    const ranked = Object.entries(answer.probabilities)
+      .sort(([, a], [, b]) => b - a)
+      .map(([option, p]) => `${option} ${p.toFixed(2)}`)
+      .join(", ");
+    return `${answer.choice} (conf ${answer.confidence?.toFixed(2) ?? "n/a"})${ranked ? ` [${ranked}]` : ""}`;
+  }
+  return `${answer.score.toFixed(2)} (conf ${answer.confidence?.toFixed(2) ?? "n/a"})`;
+}
+
+export function renderAskResult(
+  result: TypedAskResult,
+  questions: TypedQuestion[],
+): string {
+  const lines = [`model ${result.model ?? "?"}`];
+  const byId = new Map(questions.map((q) => [q.id, q]));
+  for (const answer of result.answers) {
+    const question = byId.get(answer.id);
+    const tail = question ? `  <- ${question.instructions}` : "";
+    lines.push(`${answer.id}: ${describeAnswer(answer)}${tail}`);
+  }
+  if (result.usage) {
+    lines.push(`tokens ${result.usage.input} in / ${result.usage.output} out`);
+  }
+  return lines.join("\n");
+}
+
+export interface JevAskDeps {
+  ask: () => TypedAskFn;
+  config: () => JevConfig;
+  keyPresent: () => boolean;
+}
+
+export function registerJevAskTool(pi: ExtensionAPI, deps: JevAskDeps) {
+  pi.registerTool({
+    name: "jev_ask",
+    label: "Jev Ask",
+    description:
+      "Ask TypeSafe Jev typed questions about a piece of text and get calibrated answers " +
+      "(probabilities, a chosen option, a rubric score) instead of prose.",
+    promptSnippet: "Ask Jev typed questions (yes/no, choice, rubric) about text and get calibrated answers.",
+    promptGuidelines: [
+      "Use jev_ask when a judgement must be typed and calibrated rather than written: classification, relevance, yes/no checks, rubric scores.",
+      "Ask one specific question per entry in jev_ask; split multi-factor judgements into separate questions and combine the answers yourself.",
+      "Read noul answers as: p >= 0.90 means yes, p <= 0.20 means no, anything between is uncertain — say so explicitly.",
+      "Read choice/score answers as uncertain when confidence is below 0.60.",
+      "jev_ask answers are advisory input for your decision, not policy verdicts; jev_validate remains the tool for code flaw checks.",
+    ],
+    parameters: AskParams,
+    async execute(_toolCallId, params, signal, _onUpdate, _ctx) {
+      if (!deps.keyPresent()) {
+        return {
+          content: [{ type: "text", text: "jev_ask: no Jev API key configured (see /jev status)." }],
+          details: { ok: false },
+        };
+      }
+      const questions: TypedQuestion[] = [];
+      for (const question of params.questions as AskQuestionParam[]) {
+        const built = toTypedQuestion(question);
+        if (typeof built === "string") {
+          return {
+            content: [{ type: "text", text: `jev_ask: ${built}` }],
+            details: { ok: false },
+          };
+        }
+        questions.push(built);
+      }
+      try {
+        const result = await deps.ask()(params.state, questions, signal ?? undefined);
+        return {
+          content: [{ type: "text", text: renderAskResult(result, questions) }],
+          details: {
+            ok: true,
+            model: result.model,
+            backend: (result as { backend?: string }).backend,
+            usage: result.usage,
+            answers: result.answers,
+          },
+        };
+      } catch (error) {
+        if (signal?.aborted) throw error;
+        return {
+          content: [{ type: "text", text: `jev_ask: ${error instanceof Error ? error.message : String(error)}` }],
+          details: { ok: false },
+        };
+      }
+    },
+  });
+}

package/src/automatic/gate.ts

@@ -0,0 +1,45 @@
+/**
+ * Gate logico puro, separato dall'API provider specifica.
+ * Il provider-adapter deve: generare in privato, validare, pubblicare solo approvato.
+ */
+import type { ReviewResult } from "../reviewer.ts";
+
+export interface GateCandidate {
+  /** Testo destinato alla pubblicazione (già serializzato dal provider). */
+  text: string;
+  /** Metadati opachi del provider, preservati in pubblicazione. */
+  meta?: unknown;
+}
+
+export interface GateVerdict extends ReviewResult {}
+
+export type GateDecision =
+  | { action: "publish"; candidate: GateCandidate; verdict: GateVerdict }
+  | { action: "regenerate"; candidate: GateCandidate; verdict: GateVerdict; attempt: number }
+  | { action: "hold"; verdict: GateVerdict; reason: string };
+
+export function decideGate(args: {
+  candidate: GateCandidate;
+  verdict: GateVerdict;
+  attempt: number;
+  maxRegenerations: number;
+}): GateDecision {
+  const { candidate, verdict, attempt, maxRegenerations } = args;
+
+  if (verdict.status === "pass") {
+    return { action: "publish", candidate, verdict };
+  }
+
+  if (verdict.status === "block" && attempt < maxRegenerations) {
+    return { action: "regenerate", candidate, verdict, attempt: attempt + 1 };
+  }
+
+  const reason =
+    verdict.status === "block"
+      ? `blocked after ${attempt + 1} attempt(s)`
+      : verdict.status === "review"
+        ? "uncertain: hold for user"
+        : `unavailable (${verdict.errorCode ?? "unknown"}): hold for user`;
+
+  return { action: "hold", verdict, reason };
+}

package/src/automatic/guardian.ts

@@ -0,0 +1,320 @@
+import type {
+  AssistantMessage,
+  AssistantMessageEventStream,
+  Context,
+  SimpleStreamOptions,
+  Usage,
+} from "@earendil-works/pi-ai";
+import { createAssistantMessageEventStream } from "@earendil-works/pi-ai";
+import type { JevConfig } from "../config.ts";
+import type { ReviewFn, ReviewResult } from "../reviewer.ts";
+import { reviewWithCoverage } from "../reviewer.ts";
+import {
+  aggregateUsage,
+  buildRepairContext,
+  buildSafeFailureText,
+  byteLengthUtf8,
+  emptyUsage,
+  extractCandidateText,
+  extractRequirements,
+  replayApproved,
+} from "./serialize.ts";
+
+export type UpstreamStreamFn = (
+  context: Context,
+  options?: SimpleStreamOptions,
+) => AssistantMessageEventStream;
+
+export interface GuardianDeps {
+  upstream: UpstreamStreamFn;
+  review: ReviewFn;
+  config: JevConfig;
+  /** Identità pubblica del messaggio ripubblicato. */
+  publishedProvider: string;
+  publishedModel: string;
+  /** Chiamato una volta per esito finale con verdetto Jev (pass e hold). */
+  onVerdict?: (report: GateVerdictReport) => void;
+}
+
+/** Esito finale del gate con verdetto, per `/jev last` e diagnostica. */
+export interface GateVerdictReport {
+  verdict: ReviewResult;
+  /** Tentativi upstream totali (1 + rigenerazioni). */
+  attempts: number;
+  /** Come si è chiuso: pubblicato dopo ok, o trattenuto. */
+  outcome: "published" | "held";
+}
+
+export class GuardianLimitError extends Error {
+  code = "BUFFER_LIMIT" as const;
+}
+
+export class GuardianUpstreamError extends Error {
+  code = "UPSTREAM_ERROR" as const;
+  upstreamMessage: AssistantMessage;
+  constructor(message: AssistantMessage) {
+    super(message.errorMessage || "Upstream error");
+    this.upstreamMessage = message;
+  }
+}
+
+/**
+ * Consuma privatamente lo stream upstream fino al messaggio finale.
+ * Non inoltra alcun evento al chiamante. Applica limite byte e abort.
+ */
+export async function consumePrivate(
+  stream: AssistantMessageEventStream,
+  args: { maxBufferedBytes: number; signal?: AbortSignal },
+): Promise<AssistantMessage> {
+  let bufferedBytes = 0;
+  const count = (text: string) => {
+    bufferedBytes += byteLengthUtf8(text);
+    if (bufferedBytes > args.maxBufferedBytes) {
+      throw new GuardianLimitError(
+        `Buffered ${bufferedBytes} bytes over limit ${args.maxBufferedBytes}`,
+      );
+    }
+  };
+
+  // Consuma eventi per enforcement limite; risultato finale via result().
+  // Nota: non conservare riferimenti partial per replay (sono live).
+  // Race con abort: for-await da solo resterebbe appeso se upstream non chiude.
+  let onAbort: (() => void) | undefined;
+  const abortPromise = new Promise<never>((_resolve, reject) => {
+    if (!args.signal) return;
+    if (args.signal.aborted) {
+      reject(args.signal.reason ?? new DOMException("Aborted", "AbortError"));
+      return;
+    }
+    onAbort = () => {
+      reject(args.signal?.reason ?? new DOMException("Aborted", "AbortError"));
+    };
+    args.signal.addEventListener("abort", onAbort, { once: true });
+  });
+
+  const consume = (async () => {
+    for await (const event of stream) {
+      args.signal?.throwIfAborted();
+      if (event.type === "text_delta") count(event.delta);
+      else if (event.type === "thinking_delta") count(event.delta);
+      else if (event.type === "toolcall_delta") count(event.delta);
+      else if (event.type === "error") {
+        throw new GuardianUpstreamError(event.error);
+      }
+    }
+  })();
+
+  try {
+    await Promise.race([consume, abortPromise]);
+  } finally {
+    if (args.signal && onAbort) {
+      args.signal.removeEventListener("abort", onAbort);
+    }
+  }
+  // Se abort è scattato, consume resta appeso su upstream non collaborativo:
+  // solleva subito senza attendere result().
+  args.signal?.throwIfAborted();
+  const final = await stream.result();
+  args.signal?.throwIfAborted();
+
+  if (final.stopReason === "error" || final.stopReason === "aborted") {
+    throw new GuardianUpstreamError(final);
+  }
+  if (final.stopReason === "pending") {
+    throw new GuardianUpstreamError({
+      ...final,
+      stopReason: "error",
+      errorMessage: "Upstream ended without stop reason",
+    });
+  }
+  return final;
+}
+
+function assertCompleteCandidate(message: AssistantMessage): void {
+  // Rifiuta terminali non supportati per pubblicazione.
+  if (
+    message.stopReason !== "stop" &&
+    message.stopReason !== "length" &&
+    message.stopReason !== "toolUse" &&
+    message.stopReason !== "deferred"
+  ) {
+    throw new GuardianUpstreamError({
+      ...message,
+      stopReason: "error",
+      errorMessage: `Unsupported stop reason: ${message.stopReason}`,
+    });
+  }
+}
+
+function toSafeErrorText(error: unknown, aborted: boolean): string {
+  if (aborted) return "Richiesta annullata.";
+  if (error instanceof GuardianLimitError) {
+    return `[JEV — OUTPUT NON PUBBLICATO]\n\nLimite buffer superato (${error.message}).`;
+  }
+  if (error instanceof GuardianUpstreamError) {
+    // Non trapelare contenuto candidato, solo motivo.
+    return `[JEV — ERRORE UPSTREAM]\n\n${error.message.slice(0, 500)}`;
+  }
+  if (error instanceof Error && error.name === "AbortError") {
+    return "Richiesta annullata.";
+  }
+  return "[JEV — ERRORE INTERNO]\n\nErrore durante la verifica. Output non pubblicato.";
+}
+
+/**
+ * Esegue il gate e ritorna lo stream pubblico.
+ * Nessun evento upstream viene inoltrato prima dell'approvazione.
+ */
+export function runGuardedStream(
+  originalContext: Context,
+  options: SimpleStreamOptions | undefined,
+  deps: GuardianDeps,
+): AssistantMessageEventStream {
+  const out = createAssistantMessageEventStream();
+  const signal = options?.signal;
+
+  void (async () => {
+    const usages: Usage[] = [];
+    let attemptContext = originalContext;
+    const maxRegenerations = deps.config.automatic.maxRegenerations;
+
+    try {
+      for (let attempt = 0; attempt <= maxRegenerations; attempt++) {
+        signal?.throwIfAborted();
+
+        const upstreamStream = deps.upstream(attemptContext, options);
+        let candidate: AssistantMessage;
+        try {
+          candidate = await consumePrivate(upstreamStream, {
+            maxBufferedBytes: deps.config.limits.maxBufferedBytes,
+            signal,
+          });
+        } catch (error) {
+          if (error instanceof GuardianUpstreamError) {
+            usages.push(error.upstreamMessage.usage ?? emptyUsage());
+          }
+          throw error;
+        }
+        usages.push(candidate.usage ?? emptyUsage());
+
+        assertCompleteCandidate(candidate);
+        signal?.throwIfAborted();
+
+        // Copertura intera: chunk sul candidato, requisiti dichiarati.
+        // Regole gate = difetti concreti: troncamento requisiti tollerato.
+        const rawCandidate = extractCandidateText(candidate);
+        const rawRequirements = extractRequirements(originalContext);
+
+        let verdict: ReviewResult;
+        try {
+          verdict = await reviewWithCoverage(
+            deps.review,
+            { requirements: rawRequirements, candidate: rawCandidate },
+            { limits: deps.config.limits, requirementsSensitive: false },
+            signal,
+          );
+        } catch (error) {
+          if (signal?.aborted) throw error;
+          verdict = {
+            status: "unavailable",
+            checks: [],
+            elapsedMs: 0,
+            errorCode: "JEV_THROWN",
+          };
+        }
+        signal?.throwIfAborted();
+
+        if (verdict.status === "pass") {
+          deps.onVerdict?.({ verdict, attempts: attempt + 1, outcome: "published" });
+          const aggregated = aggregateUsage(usages);
+          const replayed = replayApproved(candidate, {
+            provider: deps.publishedProvider,
+            model: deps.publishedModel,
+            usage: aggregated,
+          });
+          // Inoltra SOLO eventi approvati.
+          for await (const event of replayed) {
+            signal?.throwIfAborted();
+            out.push(event as never);
+          }
+          const final = await replayed.result();
+          out.end(final);
+          return;
+        }
+
+        if (verdict.status === "block" && attempt < maxRegenerations) {
+          attemptContext = buildRepairContext({
+            originalContext,
+            rejectedText: rawCandidate,
+            verdict,
+            attempt: attempt + 1,
+          });
+          continue;
+        }
+
+        // Hold: pubblica failure sicura, mai il candidato bocciato.
+        const attempts = attempt + 1;
+        deps.onVerdict?.({ verdict, attempts, outcome: "held" });
+        const safeText = buildSafeFailureText({
+          verdict,
+          attempts,
+          notice: verdict.downgradedForTruncation
+            ? "Input troncato per la verifica: Jev ha visto solo una parte (vedi truncated)."
+            : undefined,
+        });
+        const aggregated = aggregateUsage(usages);
+        const safeMessage: AssistantMessage = {
+          role: "assistant",
+          content: [{ type: "text", text: "" }],
+          api: candidate.api,
+          provider: deps.publishedProvider as AssistantMessage["provider"],
+          model: deps.publishedModel,
+          responseModel: candidate.model,
+          usage: aggregated,
+          stopReason: "stop",
+          timestamp: Date.now(),
+        };
+        out.push({ type: "start", partial: safeMessage });
+        out.push({ type: "text_start", contentIndex: 0, partial: safeMessage });
+        const block = safeMessage.content[0];
+        if (block && block.type === "text") block.text = safeText;
+        out.push({ type: "text_delta", contentIndex: 0, delta: safeText, partial: safeMessage });
+        out.push({ type: "text_end", contentIndex: 0, content: safeText, partial: safeMessage });
+        out.push({ type: "done", reason: "stop", message: safeMessage });
+        out.end(safeMessage);
+        return;
+      }
+    } catch (error) {
+      const aborted = signal?.aborted === true;
+      // Su abort, termina come error/aborted senza contenuto candidato.
+      const usage = usages.length > 0 ? aggregateUsage(usages) : emptyUsage();
+      const errorMessage: AssistantMessage = {
+        role: "assistant",
+        content: [{ type: "text", text: toSafeErrorText(error, aborted) }],
+        api: "openai-completions",
+        provider: deps.publishedProvider as AssistantMessage["provider"],
+        model: deps.publishedModel,
+        usage,
+        stopReason: aborted ? "aborted" : "error",
+        errorMessage: aborted
+          ? "aborted"
+          : error instanceof Error
+            ? error.message.slice(0, 300)
+            : String(error).slice(0, 300),
+        timestamp: Date.now(),
+      };
+      try {
+        out.push({
+          type: "error",
+          reason: aborted ? "aborted" : "error",
+          error: errorMessage,
+        });
+      } catch {
+        // stream già chiuso
+      }
+      out.end(errorMessage);
+    }
+  })();
+
+  return out;
+}