pi-jev-guard 0.1.0

package/package.json

@@ -0,0 +1,39 @@
+{
+  "name": "pi-jev-guard",
+  "version": "0.1.0",
+  "type": "module",
+  "description": "Jev validation guard for pi: on-demand tool + automatic provider gate. Dual backend: TypeSafe direct + OpenRouter.",
+  "keywords": ["pi-package"],
+  "files": ["extensions", "src", "skills", "config"],
+  "pi": {
+    "extensions": ["./extensions/index.ts"],
+    "skills": ["./skills"]
+  },
+  "scripts": {
+    "typecheck": "tsc --noEmit",
+    "test:unit": "vitest run test/unit",
+    "test:integration": "vitest run test/integration",
+    "bench:jev": "tsx scripts/bench-jev.ts",
+    "jev-check": "tsx scripts/jev-check.ts"
+  },
+  "dependencies": {
+    "@openrouter/sdk": "1.3.1",
+    "@typesafe-ai/sdk": "0.6.0",
+    "mdast-util-from-markdown": "2.0.2",
+    "unist-util-visit": "5.0.0"
+  },
+  "peerDependencies": {
+    "@earendil-works/pi-coding-agent": "*",
+    "@earendil-works/pi-ai": "*",
+    "typebox": "*"
+  },
+  "devDependencies": {
+    "@earendil-works/pi-ai": "0.85.1",
+    "@earendil-works/pi-coding-agent": "0.85.1",
+    "@types/node": "22.19.0",
+    "tsx": "4.23.13",
+    "typebox": "1.3.34",
+    "typescript": "5.9.3",
+    "vitest": "5.0.1"
+  }
+}

package/skills/jev-review/SKILL.md

@@ -0,0 +1,25 @@
+---
+name: jev-review
+description: Validate a code fragment or technical decision with Jev when explicitly requested or when an independent targeted check is useful.
+---
+
+# Jev review
+
+Identify the candidate and its explicit requirements.
+
+Read the necessary surrounding code before validating an isolated fragment.
+
+Call `jev_validate` with the candidate and relevant requirements.
+
+For `block`, correct the flagged issue and validate the changed candidate.
+
+For `review`, collect missing context or explain the uncertainty.
+
+For `unavailable`, state that validation was not completed.
+
+Never describe Jev approval as proof of correctness.
+Run the appropriate compiler, static checks, or targeted tests separately.
+
+For ad-hoc typed judgements (classification, relevance, rubric scores) use
+`jev_ask` with model-defined questions instead; read p >= 0.90 as yes,
+p <= 0.20 as no, and choice/score below 0.60 confidence as uncertain.

package/src/adapters/openrouter.ts

@@ -0,0 +1,289 @@
+import { performance } from "node:perf_hooks";
+import { OpenRouter } from "@openrouter/sdk";
+import { extractCodeBlocks } from "../markdown.ts";
+import { applyPolicy } from "../policy.ts";
+import type {
+  CheckResult,
+  JevBackend,
+  ReviewInput,
+  ReviewResult,
+  RuleDefinition,
+  TypedAnswer,
+  TypedAskFn,
+  TypedQuestion,
+} from "../reviewer.ts";
+import { DEFAULT_RULES } from "../reviewer.ts";
+
+export interface TransientRetry {
+  enabled: boolean;
+  maxRetries: number;
+}
+
+export interface OpenRouterAdapterConfig {
+  model: string;
+  timeoutMs: number;
+  retryTransients: TransientRetry;
+  maxPayloadBytes: number;
+  maxCodeBlocks: number;
+  passMax: number;
+  blockMin: number;
+  rules?: RuleDefinition[];
+  /** Chiave risolta (env vince sul file). Fallback env per compatibilità. */
+  apiKey?: string;
+  /** Opzionali per ranking OpenRouter, mai segreti. */
+  httpReferer?: string;
+  appTitle?: string;
+}
+
+/**
+ * Retry solo su transienti: backoff SDK su 429/5xx (statusCodes del
+ * router) ed errori di connessione. Timeout = abort, non ritentato.
+ * Nota: l'SDK OpenRouter non espone un conteggio tentativi, quindi qui
+ * `enabled` accende il backoff di default e `maxRetries` non si applica;
+ * il conteggio vale per il backend diretto. Spento → "none".
+ */
+export function openRouterRetryOption(retry: TransientRetry): { strategy: "none" } | { strategy: "backoff"; retryConnectionErrors: true } {
+  return retry.enabled && Math.floor(retry.maxRetries) > 0
+    ? { strategy: "backoff", retryConnectionErrors: true }
+    : { strategy: "none" };
+}
+
+export const OPENROUTER_BACKEND: JevBackend = "openrouter";
+
+function buildState(input: ReviewInput, maxCodeBlocks: number): { state: string; dropped: number } {
+  const allBlocks = extractCodeBlocks(input.candidate);
+  const codeBlocks = allBlocks.slice(0, maxCodeBlocks);
+  const dropped = allBlocks.length - codeBlocks.length;
+  return {
+    state: JSON.stringify({
+      requirements: input.requirements,
+      candidate: input.candidate,
+      codeBlocks,
+      truncatedBlocks: dropped,
+    }),
+    dropped,
+  };
+}
+
+export function createOpenRouterReviewer(adapterConfig: OpenRouterAdapterConfig) {
+  let client: OpenRouter | undefined;
+  const rules = adapterConfig.rules ?? DEFAULT_RULES;
+
+  return async function reviewOpenRouter(
+    input: ReviewInput,
+    signal?: AbortSignal,
+  ): Promise<ReviewResult> {
+    const started = performance.now();
+
+    try {
+      signal?.throwIfAborted();
+
+      const apiKey = adapterConfig.apiKey ?? process.env.OPENROUTER_API_KEY;
+      if (!apiKey) {
+        return {
+          status: "unavailable",
+          checks: [],
+          elapsedMs: performance.now() - started,
+          backend: OPENROUTER_BACKEND,
+          errorCode: "MISSING_API_KEY",
+        };
+      }
+
+      const { state, dropped } = buildState(input, adapterConfig.maxCodeBlocks);
+      const droppedInfo = dropped > 0 ? { truncated: { codeBlocksDropped: dropped } } : {};
+
+      if (Buffer.byteLength(state, "utf8") > adapterConfig.maxPayloadBytes) {
+        return {
+          status: "unavailable",
+          checks: [],
+          elapsedMs: performance.now() - started,
+          backend: OPENROUTER_BACKEND,
+          errorCode: "PAYLOAD_LIMIT",
+        };
+      }
+
+      client ??= new OpenRouter({
+        apiKey,
+        ...(adapterConfig.httpReferer
+          ? { httpReferer: adapterConfig.httpReferer }
+          : {}),
+        ...(adapterConfig.appTitle ? { appTitle: adapterConfig.appTitle } : {}),
+      });
+
+      const questions: Record<
+        string,
+        { type: "noul"; instructions: string; criteria?: { true: string; false: string } }
+      > = {};
+      for (const rule of rules) {
+        questions[rule.ruleId] = {
+          type: "noul",
+          instructions: rule.question,
+          ...(rule.criteria ? { criteria: { ...rule.criteria } } : {}),
+        };
+      }
+
+      const decision = await client.alpha.decisions.create(
+        {
+          decisionsRequest: {
+            model: adapterConfig.model,
+            state,
+            questions,
+          },
+        },
+        {
+          signal,
+          timeoutMs: adapterConfig.timeoutMs,
+          retries: openRouterRetryOption(adapterConfig.retryTransients),
+        } as any,
+      );
+
+      const checks: CheckResult[] = rules.map((rule) => {
+        const answer = (decision.answers as Record<string, any>)[rule.ruleId];
+        if (!answer || answer.type !== "noul") {
+          throw new Error(`Invalid OpenRouter answer for rule ${rule.ruleId}`);
+        }
+        const pFlaw = answer.noul as unknown;
+        if (
+          typeof pFlaw !== "number" ||
+          !Number.isFinite(pFlaw) ||
+          pFlaw < 0 ||
+          pFlaw > 1
+        ) {
+          throw new Error(`Invalid pFlaw for rule ${rule.ruleId}`);
+        }
+        return { ruleId: rule.ruleId, pFlaw };
+      });
+
+      const status = applyPolicy(checks, {
+        passMax: adapterConfig.passMax,
+        blockMin: adapterConfig.blockMin,
+      });
+
+      return {
+        status,
+        checks,
+        model: decision.model,
+        backend: OPENROUTER_BACKEND,
+        elapsedMs: performance.now() - started,
+        ...droppedInfo,
+      };
+    } catch (error) {
+      if (signal?.aborted) throw error;
+      return {
+        status: "unavailable",
+        checks: [],
+        elapsedMs: performance.now() - started,
+        backend: OPENROUTER_BACKEND,
+        errorCode: classifyOpenRouterError(error),
+      };
+    }
+  };
+}
+
+function classifyOpenRouterError(error: unknown): string {
+  const name =
+    typeof error === "object" && error !== null && "name" in error
+      ? String((error as { name: unknown }).name)
+      : "";
+  const message =
+    typeof error === "object" && error !== null && "message" in error
+      ? String((error as { message: unknown }).message)
+      : "";
+  const haystack = `${name} ${message}`.toLowerCase();
+  if (haystack.includes("timeout")) return "JEV_TIMEOUT";
+  if (haystack.includes("auth") || haystack.includes("401") || haystack.includes("403"))
+    return "JEV_AUTH";
+  if (haystack.includes("429") || haystack.includes("rate")) return "JEV_RATE_LIMIT";
+  if (haystack.includes("abort")) return "JEV_ABORTED";
+  return "JEV_REQUEST_FAILED";
+}
+
+export interface OpenRouterTypedConfig {
+  model: string;
+  timeoutMs: number;
+  retryTransients: TransientRetry;
+  apiKey?: string;
+  httpReferer?: string;
+  appTitle?: string;
+}
+
+/** Client tipato generico (noul + choice). Solleva su errore (fail-open a carico del chiamante). */
+export function createOpenRouterTypedClient(tcfg: OpenRouterTypedConfig): TypedAskFn {
+  let client: OpenRouter | undefined;
+  return async function askTyped(state, questions, signal) {
+    const started = performance.now();
+    signal?.throwIfAborted();
+    if (questions.length === 0) throw new Error("askTyped: no questions");
+    const apiKey = tcfg.apiKey ?? process.env.OPENROUTER_API_KEY;
+    if (!apiKey) throw new Error("askTyped: missing OPENROUTER_API_KEY");
+    client ??= new OpenRouter({
+      apiKey,
+      ...(tcfg.httpReferer ? { httpReferer: tcfg.httpReferer } : {}),
+      ...(tcfg.appTitle ? { appTitle: tcfg.appTitle } : {}),
+    });
+    const built: Record<string, any> = {};
+    for (const q of questions) {
+      if (q.kind === "noul") {
+        built[q.id] = {
+          type: "noul",
+          instructions: q.instructions,
+          ...(q.criteria ? { criteria: { ...q.criteria } } : {}),
+        };
+      } else if (q.kind === "choice") {
+        built[q.id] = { type: "choice", instructions: q.instructions, criteria: { ...q.options } };
+      } else {
+        built[q.id] = { type: "score", instructions: q.instructions, criteria: [...q.levels] };
+      }
+    }
+    const decision = await client.alpha.decisions.create(
+      { decisionsRequest: { model: tcfg.model, state: state as string, questions: built } },
+      {
+        signal,
+        timeoutMs: tcfg.timeoutMs,
+        retries: openRouterRetryOption(tcfg.retryTransients),
+      } as any,
+    );
+    const answers: TypedAnswer[] = questions.map((q: TypedQuestion): TypedAnswer => {
+      const a = (decision.answers as Record<string, any>)[q.id];
+      if (q.kind === "noul") {
+        if (!a || a.type !== "noul" || typeof a.noul !== "number") {
+          throw new Error(`Invalid noul answer for ${q.id}`);
+        }
+        return { id: q.id, type: "noul", p: a.noul };
+      }
+      if (q.kind === "choice") {
+        if (!a || a.type !== "choice" || typeof a.choice !== "string") {
+          throw new Error(`Invalid choice answer for ${q.id}`);
+        }
+        return {
+          id: q.id,
+          type: "choice",
+          choice: a.choice,
+          confidence: typeof a.confidence === "number" ? a.confidence : undefined,
+          probabilities: { ...(a.probabilities ?? {}) },
+        };
+      }
+      if (!a || a.type !== "score" || typeof a.score !== "number") {
+        throw new Error(`Invalid score answer for ${q.id}`);
+      }
+      return {
+        id: q.id,
+        type: "score",
+        score: a.score,
+        confidence: typeof a.confidence === "number" ? a.confidence : undefined,
+        probabilities: { ...(a.probabilities ?? {}) },
+      };
+    });
+    const usage = decision.usage as { inputTokens?: unknown; outputTokens?: unknown } | undefined;
+    return {
+      answers,
+      elapsedMs: performance.now() - started,
+      model: decision.model,
+      backend: OPENROUTER_BACKEND,
+      usage:
+        usage && typeof usage.inputTokens === "number" && typeof usage.outputTokens === "number"
+          ? { input: usage.inputTokens, output: usage.outputTokens }
+          : undefined,
+    };
+  };
+}

package/src/adapters/typesafe.ts

@@ -0,0 +1,244 @@
+import { performance } from "node:perf_hooks";
+import { choice, noul, score as scoreQ, TypeSafeClient } from "@typesafe-ai/sdk";
+import { extractCodeBlocks } from "../markdown.ts";
+import { applyPolicy } from "../policy.ts";
+import type {
+  CheckResult,
+  JevBackend,
+  ReviewInput,
+  ReviewResult,
+  RuleDefinition,
+  TypedAnswer,
+  TypedAskFn,
+  TypedQuestion,
+} from "../reviewer.ts";
+import { DEFAULT_RULES } from "../reviewer.ts";
+
+export interface TransientRetry {
+  enabled: boolean;
+  maxRetries: number;
+}
+
+export interface TypesafeAdapterConfig {
+  model: string;
+  timeoutMs: number;
+  retryTransients: TransientRetry;
+  maxPayloadBytes: number;
+  maxCodeBlocks: number;
+  passMax: number;
+  blockMin: number;
+  rules?: RuleDefinition[];
+  /** Chiave risolta (env vince sul file). Fallback env dell'SDK. */
+  apiKey?: string;
+}
+
+export const TYPESAFE_BACKEND: JevBackend = "typesafe";
+
+function buildState(input: ReviewInput, maxCodeBlocks: number): { state: string; dropped: number } {
+  const allBlocks = extractCodeBlocks(input.candidate);
+  const codeBlocks = allBlocks.slice(0, maxCodeBlocks);
+  const dropped = allBlocks.length - codeBlocks.length;
+  return {
+    state: JSON.stringify({
+      requirements: input.requirements,
+      candidate: input.candidate,
+      codeBlocks,
+      truncatedBlocks: dropped,
+    }),
+    dropped,
+  };
+}
+
+/**
+ * Retry solo su transienti: 429/5xx ed errori di connessione secondo i
+ * default dell'SDK. Timeout mai ritentati (`apiTimeoutError: false`);
+ * auth (401/403) fuori dai retryable di default. Spento → zero retry.
+ */
+export function typesafeRetryOption(retry: TransientRetry): { maxRetries: number; apiTimeoutError: boolean } {
+  return retry.enabled
+    ? { maxRetries: Math.max(0, Math.floor(retry.maxRetries)), apiTimeoutError: false }
+    : { maxRetries: 0, apiTimeoutError: false };
+}
+
+export function createTypesafeReviewer(adapterConfig: TypesafeAdapterConfig) {
+  let client: TypeSafeClient | undefined;
+  const rules = adapterConfig.rules ?? DEFAULT_RULES;
+
+  return async function reviewTypesafe(
+    input: ReviewInput,
+    signal?: AbortSignal,
+  ): Promise<ReviewResult> {
+    const started = performance.now();
+
+    try {
+      signal?.throwIfAborted();
+
+      const { state, dropped } = buildState(input, adapterConfig.maxCodeBlocks);
+      const droppedInfo = dropped > 0 ? { truncated: { codeBlocksDropped: dropped } } : {};
+
+      if (Buffer.byteLength(state, "utf8") > adapterConfig.maxPayloadBytes) {
+        return {
+          status: "unavailable",
+          checks: [],
+          elapsedMs: performance.now() - started,
+          backend: TYPESAFE_BACKEND,
+          errorCode: "PAYLOAD_LIMIT",
+        };
+      }
+
+      client ??= new TypeSafeClient({
+        ...(adapterConfig.apiKey ? { apiKey: adapterConfig.apiKey } : {}),
+        timeout: adapterConfig.timeoutMs,
+        retry: typesafeRetryOption(adapterConfig.retryTransients),
+        logLevel: "off",
+      });
+
+      const questions: Record<string, ReturnType<typeof noul>> = {};
+      for (const rule of rules) {
+        questions[rule.ruleId] = noul(rule.question, rule.criteria ?? undefined);
+      }
+
+      const result = await client.systemOne(
+        {
+          model: adapterConfig.model,
+          state,
+          questions,
+        },
+        { signal },
+      );
+
+      const checks: CheckResult[] = rules.map((rule) => {
+        const answer = (result.answers as Record<string, { noul?: unknown }>)[
+          rule.ruleId
+        ];
+        const pFlaw = answer?.noul;
+        if (
+          typeof pFlaw !== "number" ||
+          !Number.isFinite(pFlaw) ||
+          pFlaw < 0 ||
+          pFlaw > 1
+        ) {
+          throw new Error(`Invalid Jev response for rule ${rule.ruleId}`);
+        }
+        return { ruleId: rule.ruleId, pFlaw };
+      });
+
+      const status = applyPolicy(checks, {
+        passMax: adapterConfig.passMax,
+        blockMin: adapterConfig.blockMin,
+      });
+
+      return {
+        status,
+        checks,
+        model: result.model,
+        backend: TYPESAFE_BACKEND,
+        elapsedMs: performance.now() - started,
+        ...droppedInfo,
+      };
+    } catch (error) {
+      if (signal?.aborted) throw error;
+      return {
+        status: "unavailable",
+        checks: [],
+        elapsedMs: performance.now() - started,
+        backend: TYPESAFE_BACKEND,
+        errorCode: classifyTypesafeError(error),
+      };
+    }
+  };
+}
+
+function classifyTypesafeError(error: unknown): string {
+  const name =
+    typeof error === "object" && error !== null && "name" in error
+      ? String((error as { name: unknown }).name)
+      : "";
+  if (name.includes("Timeout")) return "JEV_TIMEOUT";
+  if (name.includes("Auth")) return "JEV_AUTH";
+  if (name.includes("RateLimit")) return "JEV_RATE_LIMIT";
+  if (name.includes("Abort") || name.includes("UserAbort")) return "JEV_ABORTED";
+  return "JEV_REQUEST_FAILED";
+}
+
+export interface TypesafeTypedConfig {
+  model: string;
+  timeoutMs: number;
+  retryTransients: TransientRetry;
+  apiKey?: string;
+}
+
+/**
+ * Client tipato generico (noul + choice). A differenza di review(), solleva
+ * su errore: il chiamante decide fail-open/closed.
+ */
+export function createTypesafeTypedClient(tcfg: TypesafeTypedConfig): TypedAskFn {
+  let client: TypeSafeClient | undefined;
+  return async function askTyped(state, questions, signal) {
+    const started = performance.now();
+    signal?.throwIfAborted();
+    if (questions.length === 0) throw new Error("askTyped: no questions");
+    client ??= new TypeSafeClient({
+      ...(tcfg.apiKey ? { apiKey: tcfg.apiKey } : {}),
+      timeout: tcfg.timeoutMs,
+      retry: typesafeRetryOption(tcfg.retryTransients),
+      logLevel: "off",
+    });
+    const built: Record<string, unknown> = {};
+    for (const q of questions) {
+      if (q.kind === "noul") {
+        built[q.id] = noul(q.instructions, q.criteria ?? undefined);
+      } else if (q.kind === "choice") {
+        built[q.id] = choice(q.instructions, q.options as Record<string, string | null>);
+      } else {
+        built[q.id] = scoreQ(q.instructions, q.levels as [string, string, ...string[]]);
+      }
+    }
+    const result = await client.systemOne(
+      { model: tcfg.model, state: state as string, questions: built as never },
+      { signal },
+    );
+    const answers: TypedAnswer[] = questions.map((q: TypedQuestion): TypedAnswer => {
+      const a = (result.answers as Record<string, any>)[q.id];
+      if (q.kind === "noul") {
+        if (!a || a.type !== "noul" || typeof a.noul !== "number") {
+          throw new Error(`Invalid noul answer for ${q.id}`);
+        }
+        return { id: q.id, type: "noul", p: a.noul };
+      }
+      if (q.kind === "choice") {
+        if (!a || a.type !== "choice" || typeof a.choice !== "string") {
+          throw new Error(`Invalid choice answer for ${q.id}`);
+        }
+        return {
+          id: q.id,
+          type: "choice",
+          choice: a.choice,
+          confidence: typeof a.confidence === "number" ? a.confidence : undefined,
+          probabilities: { ...(a.probabilities ?? {}) },
+        };
+      }
+      if (!a || a.type !== "score" || typeof a.score !== "number") {
+        throw new Error(`Invalid score answer for ${q.id}`);
+      }
+      return {
+        id: q.id,
+        type: "score",
+        score: a.score,
+        confidence: typeof a.confidence === "number" ? a.confidence : undefined,
+        probabilities: { ...(a.probabilities ?? {}) },
+      };
+    });
+    const usage = result.usage as { input_tokens?: unknown; output_tokens?: unknown } | undefined;
+    return {
+      answers,
+      elapsedMs: performance.now() - started,
+      model: result.model,
+      backend: TYPESAFE_BACKEND,
+      usage:
+        usage && typeof usage.input_tokens === "number" && typeof usage.output_tokens === "number"
+          ? { input: usage.input_tokens, output: usage.output_tokens }
+          : undefined,
+    };
+  };
+}