pi-pair-programmer 0.1.5 → 0.1.6

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "pi-pair-programmer",
-  "version": "0.1.5",
+  "version": "0.1.6",
   "description": "A TypeScript extension for Pi and OMP",
   "repository": {
     "type": "git",

package/src/finding-admission.ts

@@ -1,21 +1,41 @@
 import { createHash } from "node:crypto";
 import { noul, TypeSafeClient } from "@typesafe-ai/sdk";
-import { observeJudgment, type ModelCallObserver } from "./model-usage.js";
+import { z } from "zod";
+import {
+  notify,
+  observeJudgment,
+  type ModelCallObserver,
+} from "./model-usage.js";
+import type { DedupJudgment, JudgmentObserver } from "./pair-stats.js";
 import type { ProposedFinding } from "./review-runner.js";
 import type { ReviewStore, Finding, StoredFinding } from "./review-store.js";
 
-interface AdmissionRequest {
+interface FindingScope {
   file: string;
   reviewer: string;
   revision: string;
+}
+
+interface AdmissionRequest extends FindingScope {
   findings: readonly ProposedFinding[];
   signal: AbortSignal;
   onModelCall?: ModelCallObserver;
+  onJudgment?: JudgmentObserver;
   isCurrent: () => Promise<boolean>;
 }
 
+type Observers = Pick<
+  AdmissionRequest,
+  "signal" | "onModelCall" | "onJudgment"
+>;
 type Candidate = Finding & { duplicateKey: string };
 
+const DUPLICATE_THRESHOLD = 0.5;
+const dedupSchema = z.object({
+  answers: z.object({
+    duplicate: z.object({ noul: z.number().min(0).max(1) }),
+  }),
+});
 let jevClient: TypeSafeClient | undefined;
 
 function hash(parts: readonly string[]): string {
@@ -25,8 +45,15 @@ function hash(parts: readonly string[]): string {
     .slice(0, 16);
 }
 
+export function findingId(
+  scope: FindingScope,
+  proposed: ProposedFinding,
+): string {
+  return candidateOf(scope, proposed).id;
+}
+
 function candidateOf(
-  request: AdmissionRequest,
+  request: FindingScope,
   proposed: ProposedFinding,
 ): Candidate {
   const duplicateKey = hash([
@@ -52,27 +79,38 @@ function sameProblem(candidate: Candidate, finding: Finding): boolean {
   return candidate.duplicateKey === (finding.duplicateKey ?? finding.id);
 }
 
-async function isNovel(
+async function judgeNovelty(
   candidate: Candidate,
   history: readonly StoredFinding[],
   earlier: readonly Candidate[],
-  signal: AbortSignal,
-  onModelCall: ModelCallObserver | undefined,
-): Promise<boolean> {
-  const matches = (finding: Finding): boolean =>
-    sameProblem(candidate, finding);
-  const unchanged = (finding: Finding): boolean =>
-    matches(finding) && candidate.evidence === finding.evidence;
-  if (
-    history.some(({ finding }) => unchanged(finding)) ||
-    earlier.some(unchanged)
-  )
-    return false;
-  if (history.length === 0 && earlier.length === 0) return true;
+  { signal, onModelCall }: Observers,
+): Promise<DedupJudgment> {
+  const priors = [...history.map(({ finding }) => finding), ...earlier];
+  const matching = priors.filter((finding) => sameProblem(candidate, finding));
+  const judgment = (
+    decision: DedupJudgment["decision"],
+    reason: DedupJudgment["reason"],
+  ): DedupJudgment => ({
+    stage: "dedup",
+    decision,
+    reason,
+    findingId: candidate.id,
+    duplicateKey: candidate.duplicateKey,
+    line: candidate.line,
+    title: candidate.title,
+    history: history.map(({ finding }) => finding.id),
+    candidates: earlier.map(({ id }) => id),
+    sameKey: matching.map(({ id }) => id),
+  });
+  if (matching.some(({ evidence }) => evidence === candidate.evidence))
+    return judgment("drop", "unchanged");
+  if (priors.length === 0) return judgment("keep", "first");
+  const fallback = matching.length === 0 ? "keep" : "drop";
+  let response: unknown;
   try {
     jevClient ??= new TypeSafeClient({ logLevel: "off" });
     const client = jevClient;
-    const response = await observeJudgment("dedup", signal, onModelCall, () =>
+    response = await observeJudgment("dedup", signal, onModelCall, () =>
       client.systemOne(
         {
           model: "jev-latest",
@@ -99,28 +137,32 @@ async function isNovel(
         { signal, timeout: 30_000, retry: { maxRetries: 0 } },
       ),
     );
-    return response.answers.duplicate.noul < 0.5;
   } catch {
-    return (
-      history.every(({ finding }) => !matches(finding)) &&
-      earlier.every((finding) => !matches(finding))
-    );
+    return judgment(fallback, signal.aborted ? "cancelled" : "request_failed");
   }
+  const parsed = dedupSchema.safeParse(response);
+  if (!parsed.success) return judgment(fallback, "response_invalid");
+  const score = parsed.data.answers.duplicate.noul;
+  return {
+    ...judgment(score < DUPLICATE_THRESHOLD ? "keep" : "drop", "judged"),
+    score,
+    threshold: DUPLICATE_THRESHOLD,
+  };
 }
 
 async function novelFindings(
   candidates: readonly Candidate[],
   history: readonly StoredFinding[],
   compareCandidates: boolean,
-  signal: AbortSignal,
-  onModelCall: ModelCallObserver | undefined,
+  observers: Observers,
 ): Promise<Candidate[]> {
   const novel: Candidate[] = [];
   for (const [index, candidate] of candidates.entries()) {
-    if (signal.aborted) break;
+    if (observers.signal.aborted) break;
     const earlier = compareCandidates ? candidates.slice(0, index) : [];
-    if (await isNovel(candidate, history, earlier, signal, onModelCall))
-      novel.push(candidate);
+    const judgment = await judgeNovelty(candidate, history, earlier, observers);
+    notify(observers.onJudgment, judgment);
+    if (judgment.decision === "keep") novel.push(candidate);
   }
   return novel;
 }
@@ -141,8 +183,7 @@ export class FindingAdmission {
       request.findings.map((finding) => candidateOf(request, finding)),
       history,
       true,
-      signal,
-      request.onModelCall,
+      request,
     );
     for (;;) {
       if (!(await request.isCurrent()) || signal.aborted) return "obsolete";
@@ -150,13 +191,7 @@ export class FindingAdmission {
       const added = latest.slice(history.length);
       if (novel.length === 0 || added.length === 0) break;
       history = latest;
-      novel = await novelFindings(
-        novel,
-        added,
-        false,
-        signal,
-        request.onModelCall,
-      );
+      novel = await novelFindings(novel, added, false, request);
     }
     let result: "unchanged" | "added" = "unchanged";
     for (const finding of novel) {

package/src/index.ts

@@ -12,12 +12,13 @@ import {
   captureBaseline,
   type TaskBaseline,
 } from "./change-evidence.js";
-import { FindingAdmission } from "./finding-admission.js";
+import { FindingAdmission, findingId } from "./finding-admission.js";
 import { createPairLogger, type PairLogger } from "./logger.js";
 import type { ModelCallObserver } from "./model-usage.js";
 import {
   SessionAccounting,
   STATS_ENTRY,
+  type JudgmentObserver,
   type PairStats,
   type ReviewOutcome,
 } from "./pair-stats.js";
@@ -373,6 +374,22 @@ export default function pairProgrammer(pi: ExtensionAPI): void {
         });
       }
     };
+    const onJudgment: JudgmentObserver = (judgment) => {
+      job.stats.judge(job.id, judgment);
+      logger?.log("finding.judged", {
+        ...fields,
+        stage: judgment.stage,
+        outcome: judgment.decision === "fail" ? "invalid" : judgment.decision,
+        reasonCode: judgment.reason,
+        confidence:
+          judgment.stage === "attribution" ? judgment.confidence : undefined,
+        probability:
+          judgment.stage === "attribution"
+            ? judgment.probabilities?.inherited
+            : undefined,
+        score: judgment.stage === "dedup" ? judgment.score : undefined,
+      });
+    };
     controllers.set(controller, job.key);
     cancelJobs.set(controller, () => {
       finish("cancelled", false);
@@ -381,7 +398,12 @@ export default function pairProgrammer(pi: ExtensionAPI): void {
     logger?.log("review.started", fields);
     void (async () => {
       try {
-        const result = await runJob(job, controller.signal, onModelCall);
+        const result = await runJob(
+          job,
+          controller.signal,
+          onModelCall,
+          onJudgment,
+        );
         const settledOutcome =
           reviewState.outcome === "success" ? result : reviewState.outcome;
         finish(controller.signal.aborted ? "cancelled" : settledOutcome);
@@ -424,6 +446,7 @@ export default function pairProgrammer(pi: ExtensionAPI): void {
     job: ReviewJob,
     signal: AbortSignal,
     onModelCall: ModelCallObserver,
+    onJudgment: JudgmentObserver,
   ): Promise<ReviewOutcome> {
     const proposed = await reviewFile({
       host: job.host,
@@ -434,6 +457,7 @@ export default function pairProgrammer(pi: ExtensionAPI): void {
       source: job.source,
       signal,
       onModelCall,
+      onJudgment,
     });
     if (!(await current(job))) {
       logger?.log("review.skipped", {
@@ -448,6 +472,7 @@ export default function pairProgrammer(pi: ExtensionAPI): void {
       reviewed.set(`${job.key}:${reviewer}:${job.model}`, job.revision);
       return "success";
     }
+    const scope = { file: job.file, reviewer, revision: job.revision };
     const judged = await Promise.all(
       proposed.map(async (finding) => ({
         finding,
@@ -463,6 +488,12 @@ export default function pairProgrammer(pi: ExtensionAPI): void {
           ),
           signal,
           onModelCall,
+          onJudgment: (attribution) => {
+            onJudgment({
+              ...attribution,
+              findingId: findingId(scope, finding),
+            });
+          },
         }),
       })),
     );
@@ -474,15 +505,14 @@ export default function pairProgrammer(pi: ExtensionAPI): void {
     });
     if (signal.aborted || !(await current(job))) return "obsolete";
     const result = await admission.admit({
-      file: job.file,
-      reviewer,
-      revision: job.revision,
+      ...scope,
       findings: judged.flatMap(({ finding, inherited }) =>
         inherited ? [] : [finding],
       ),
       signal,
       isCurrent: () => current(job),
       onModelCall,
+      onJudgment,
     });
     logger?.log("finding.admission", {
       sessionId: job.stats.sessionId,

package/src/logger.ts

@@ -16,6 +16,7 @@ export type PairLogEvent =
   | "review.skipped"
   | "model.finished"
   | "finding.filtered"
+  | "finding.judged"
   | "finding.admission"
   | "finding.verdict"
   | "delivery.sent"
@@ -37,6 +38,9 @@ export interface PairLogFields {
   cacheWriteTokens?: number;
   totalTokens?: number;
   costUsd?: number;
+  confidence?: number | undefined;
+  probability?: number | undefined;
+  score?: number | undefined;
 }
 
 export interface PairLogger {
@@ -56,6 +60,7 @@ const events: Record<PairLogEvent, true> = {
   "review.skipped": true,
   "model.finished": true,
   "finding.filtered": true,
+  "finding.judged": true,
   "finding.admission": true,
   "finding.verdict": true,
   "delivery.sent": true,
@@ -70,6 +75,9 @@ const numericFields: Record<string, true> = {
   cacheWriteTokens: true,
   totalTokens: true,
   costUsd: true,
+  confidence: true,
+  probability: true,
+  score: true,
 };
 const identifierFields: Record<string, true> = {
   sessionId: true,
@@ -93,6 +101,8 @@ const outcomes = {
   accept: true,
   reject: true,
   invalid: true,
+  keep: true,
+  drop: true,
   on: true,
   off: true,
 };
@@ -123,6 +133,20 @@ const reasonCodes = {
   baseline_unavailable: true,
   configuration_failed: true,
   ui_unavailable: true,
+  entry_invalid: true,
+  line_out_of_range: true,
+  quote_mismatch: true,
+  output_not_json: true,
+  envelope_invalid: true,
+  too_many_findings: true,
+  judged: true,
+  evidence_unavailable: true,
+  no_diff: true,
+  no_origin: true,
+  response_invalid: true,
+  request_failed: true,
+  cancelled: true,
+  first: true,
 };
 const maxFileBytes = 1024 * 1024;
 const maxBufferedBytes = 64 * 1024;
@@ -163,6 +187,7 @@ const warningReasons: Record<string, true> = {
   missing_model: true,
   baseline_unavailable: true,
   ui_unavailable: true,
+  response_invalid: true,
 };
 
 function severity(
@@ -192,6 +217,7 @@ function severity(
     outcome === "obsolete" ||
     event === "review.skipped" ||
     event === "finding.filtered" ||
+    event === "finding.judged" ||
     (event === "delivery.wake" && reason !== "ready")
     ? "debug"
     : "info";

package/src/model-usage.ts

@@ -69,9 +69,9 @@ function metadata(value: unknown, judge = false): Metadata {
   return result;
 }
 
-function notify(
-  observer: ModelCallObserver | undefined,
-  event: ModelCallObservation,
+export function notify<T>(
+  observer: ((event: T) => void) | undefined,
+  event: T,
 ): void {
   try {
     observer?.(event);

package/src/pair-stats.ts

@@ -34,6 +34,79 @@ const Observation = z.object({
     })
     .optional(),
 });
+const Probability = z.number().min(0).max(1);
+const Decision = z.enum(["keep", "drop"]);
+const ReviewDrop = z.object({
+  stage: z.literal("review"),
+  decision: z.literal("drop"),
+  reason: z.enum(["entry_invalid", "line_out_of_range", "quote_mismatch"]),
+  line: z.number().nullable(),
+  title: z.string().nullable(),
+  quote: z.string().nullable(),
+  lineText: z.string().nullable(),
+  quoteLines: z.array(z.number()),
+  invalidFields: z.array(z.string()),
+});
+const ReviewFailure = z.object({
+  stage: z.literal("review"),
+  decision: z.literal("fail"),
+  reason: z.enum(["output_not_json", "envelope_invalid", "too_many_findings"]),
+  count: Count.nullable(),
+});
+const Attribution = z.object({
+  stage: z.literal("attribution"),
+  decision: Decision,
+  reason: z.enum([
+    "judged",
+    "evidence_unavailable",
+    "no_diff",
+    "no_origin",
+    "response_invalid",
+    "request_failed",
+    "cancelled",
+  ]),
+  findingId: z.string(),
+  line: z.number(),
+  title: z.string(),
+  quote: z.string(),
+  evidenceStatus: z.enum(["available", "unavailable"]),
+  evidenceReason: z.string().nullable(),
+  choice: z.enum(["inherited", "introduced", "unknown"]).optional(),
+  confidence: Probability.optional(),
+  probabilities: z
+    .object({
+      inherited: Probability,
+      introduced: Probability,
+      unknown: Probability,
+    })
+    .optional(),
+  thresholds: z
+    .object({ confidence: Probability, inherited: Probability })
+    .optional(),
+  issues: z.array(z.string()).optional(),
+});
+const Dedup = z.object({
+  stage: z.literal("dedup"),
+  decision: Decision,
+  reason: z.enum([
+    "first",
+    "unchanged",
+    "judged",
+    "response_invalid",
+    "request_failed",
+    "cancelled",
+  ]),
+  findingId: z.string(),
+  duplicateKey: z.string(),
+  line: z.number(),
+  title: z.string(),
+  history: z.array(z.string()),
+  candidates: z.array(z.string()),
+  sameKey: z.array(z.string()),
+  score: Probability.optional(),
+  threshold: Probability.optional(),
+});
+const Judgment = z.union([ReviewDrop, ReviewFailure, Attribution, Dedup]);
 const Event = z.discriminatedUnion("action", [
   z.object({
     action: z.literal("start"),
@@ -60,6 +133,13 @@ const Event = z.discriminatedUnion("action", [
     jobId: z.string(),
     observation: Observation,
   }),
+  z.object({
+    action: z.literal("judgment"),
+    id: z.string(),
+    sessionId: z.string(),
+    jobId: z.string(),
+    judgment: Judgment,
+  }),
 ]);
 const Entry = z.object({
   type: z.literal("custom"),
@@ -69,6 +149,12 @@ const Entry = z.object({
 
 type StatsEvent = z.infer<typeof Event>;
 export type ReviewOutcome = z.infer<typeof Outcome>;
+export type ReviewDropJudgment = z.infer<typeof ReviewDrop>;
+export type ReviewFailureJudgment = z.infer<typeof ReviewFailure>;
+export type AttributionJudgment = z.infer<typeof Attribution>;
+export type DedupJudgment = z.infer<typeof Dedup>;
+export type Judgment = z.infer<typeof Judgment>;
+export type JudgmentObserver = (judgment: Judgment) => void;
 export interface MeasuredTotal {
   value: number;
   measured: number;
@@ -175,6 +261,16 @@ export class PairStats {
     });
   }
 
+  judge(jobId: string, judgment: Judgment): void {
+    this.record({
+      action: "judgment",
+      id: randomUUID(),
+      sessionId: this.sessionId,
+      jobId,
+      judgment,
+    });
+  }
+
   snapshot(): StatsSnapshot {
     const result: StatsSnapshot = {
       reviews: {
@@ -225,6 +321,7 @@ export class PairStats {
           break;
         }
         case "settled":
+        case "judgment":
           break;
       }
     }

package/src/review-runner.ts

@@ -3,19 +3,34 @@ import { choice, TypeSafeClient } from "@typesafe-ai/sdk";
 import { z } from "zod";
 import type { ChangeEvidence } from "./change-evidence.js";
 import {
+  notify,
   observeJudgment,
   ReviewEventStream,
   type ModelCallObserver,
 } from "./model-usage.js";
+import type {
+  AttributionJudgment,
+  JudgmentObserver,
+  ReviewDropJudgment,
+  ReviewFailureJudgment,
+} from "./pair-stats.js";
 
 const MAX_SOURCE_CHARACTERS = 60_000;
 const MAX_FINDINGS = 5;
+const MAX_RECORDED_TEXT = 500;
+const MAX_QUOTE_LINES = 5;
+const ATTRIBUTION_THRESHOLDS = { confidence: 0.95, inherited: 0.95 } as const;
 const proposedFindingSchema = z.looseObject({
   line: z.number().int().positive(),
   title: z.string().min(1).max(160).regex(/\S/u),
   quote: z.string().min(1).max(240).regex(/\S/u),
   evidence: z.string().min(1).max(500).regex(/\S/u),
 });
+const rawFindingSchema = z.looseObject({
+  line: z.unknown(),
+  title: z.unknown(),
+  quote: z.unknown(),
+});
 const reviewResultSchema = z.object({
   findings: z.array(z.unknown()).max(MAX_FINDINGS),
 });
@@ -71,6 +86,7 @@ interface ReviewRequest {
   source: string;
   signal: AbortSignal;
   onModelCall?: ModelCallObserver;
+  onJudgment?: JudgmentObserver;
 }
 
 function invoke(
@@ -152,7 +168,7 @@ async function complete(
   prompt: string,
   signal: AbortSignal,
   onModelCall: ModelCallObserver | undefined,
-): Promise<unknown> {
+): Promise<string> {
   const args = [
     "--no-session",
     "--no-tools",
@@ -177,11 +193,7 @@ async function complete(
     args.unshift(script);
   }
 
-  const stdout = (
-    await invoke(args, cwd, prompt, signal, model, onModelCall)
-  ).trim();
-  const fenced = /^```(?:json)?[ \t]*\r?\n([\s\S]*?)\r?\n```$/iu.exec(stdout);
-  return JSON.parse(fenced?.[1] ?? stdout) as unknown;
+  return (await invoke(args, cwd, prompt, signal, model, onModelCall)).trim();
 }
 
 export async function reviewFile(
@@ -228,8 +240,36 @@ Reply with the JSON object alone, without any other text.`,
     request.onModelCall,
   );
 
-  const envelope = reviewResultSchema.safeParse(result);
+  return validFindings(result, lines, request.onJudgment);
+}
+
+function validFindings(
+  answer: string,
+  lines: readonly string[],
+  onJudgment: JudgmentObserver | undefined,
+): ProposedFinding[] {
+  const fail = (
+    reason: ReviewFailureJudgment["reason"],
+    count: number | null,
+  ): void => {
+    notify(onJudgment, { stage: "review", decision: "fail", reason, count });
+  };
+  const fenced = /^```(?:json)?[ \t]*\r?\n([\s\S]*?)\r?\n```$/iu.exec(answer);
+  let output: unknown;
+  try {
+    output = JSON.parse(fenced?.[1] ?? answer);
+  } catch (error) {
+    fail("output_not_json", null);
+    throw error;
+  }
+  const envelope = reviewResultSchema.safeParse(output);
   if (!envelope.success) {
+    const entries = z
+      .object({ findings: z.array(z.unknown()) })
+      .safeParse(output);
+    if (entries.success)
+      fail("too_many_findings", entries.data.findings.length);
+    else fail("envelope_invalid", null);
     throw new Error("Reviewer returned an invalid findings array");
   }
   const findings: ProposedFinding[] = [];
@@ -240,28 +280,110 @@ Reply with the JSON object alone, without any other text.`,
       lines[parsed.data.line - 1]?.includes(parsed.data.quote) === true
     ) {
       findings.push(parsed.data);
+    } else {
+      notify(onJudgment, rejectedEntry(entry, parsed, lines));
     }
   }
   return findings;
 }
 
-export async function isInherited(request: {
+function rejectedEntry(
+  entry: unknown,
+  parsed: z.ZodSafeParseResult<z.infer<typeof proposedFindingSchema>>,
+  lines: readonly string[],
+): ReviewDropJudgment {
+  if (parsed.success) {
+    const { line, title, quote } = parsed.data;
+    const lineText = lines[line - 1];
+    return {
+      stage: "review",
+      decision: "drop",
+      reason: lineText === undefined ? "line_out_of_range" : "quote_mismatch",
+      line,
+      title,
+      quote,
+      lineText: recordedText(lineText),
+      quoteLines: linesContaining(lines, quote),
+      invalidFields: [],
+    };
+  }
+  const raw = rawFindingSchema.safeParse(entry);
+  const fields: { line?: unknown; title?: unknown; quote?: unknown } =
+    raw.success ? raw.data : {};
+  const line = Number.isSafeInteger(fields.line) ? Number(fields.line) : null;
+  const quote = typeof fields.quote === "string" ? fields.quote : null;
+  return {
+    stage: "review",
+    decision: "drop",
+    reason: "entry_invalid",
+    line,
+    title: recordedText(fields.title),
+    quote: recordedText(quote),
+    lineText: line === null ? null : recordedText(lines[line - 1]),
+    quoteLines:
+      quote === null || !/\S/u.test(quote) ? [] : linesContaining(lines, quote),
+    invalidFields: [
+      ...new Set(
+        parsed.error.issues.map((issue) =>
+          issue.path.length === 0 ? "entry" : String(issue.path[0]),
+        ),
+      ),
+    ],
+  };
+}
+
+function recordedText(value: unknown): string | null {
+  return typeof value === "string" ? value.slice(0, MAX_RECORDED_TEXT) : null;
+}
+
+function linesContaining(lines: readonly string[], quote: string): number[] {
+  const found: number[] = [];
+  for (const [index, line] of lines.entries()) {
+    if (found.length === MAX_QUOTE_LINES) break;
+    if (line.includes(quote)) found.push(index + 1);
+  }
+  return found;
+}
+
+type Attribution = Omit<AttributionJudgment, "findingId">;
+
+interface AttributionRequest {
   finding: ProposedFinding;
   evidence: ChangeEvidence;
   signal: AbortSignal;
   onModelCall?: ModelCallObserver;
-}): Promise<boolean> {
-  const evidence = request.evidence;
-  if (
-    evidence.status !== "available" ||
-    evidence.diff === null ||
-    (evidence.before === null && evidence.origins.length === 0)
-  ) {
-    return false;
+  onJudgment?: (judgment: Attribution) => void;
+}
+
+export async function isInherited(
+  request: AttributionRequest,
+): Promise<boolean> {
+  const judgment = await attribute(request);
+  notify(request.onJudgment, judgment);
+  return judgment.decision === "drop";
+}
+
+async function attribute(request: AttributionRequest): Promise<Attribution> {
+  const { evidence, finding } = request;
+  const keep = (reason: Attribution["reason"]): Attribution => ({
+    stage: "attribution",
+    decision: "keep",
+    reason,
+    line: finding.line,
+    title: finding.title,
+    quote: finding.quote,
+    evidenceStatus: evidence.status,
+    evidenceReason: evidence.reason,
+  });
+  if (evidence.status !== "available") return keep("evidence_unavailable");
+  if (evidence.diff === null) return keep("no_diff");
+  if (evidence.before === null && evidence.origins.length === 0) {
+    return keep("no_origin");
   }
+  let response: unknown;
   try {
     const client = jev();
-    const response = await observeJudgment(
+    response = await observeJudgment(
       "attribution",
       request.signal,
       request.onModelCall,
@@ -295,14 +417,30 @@ export async function isInherited(request: {
           { signal: request.signal, timeout: 30_000, retry: { maxRetries: 0 } },
         ),
     );
-    const parsed = attributionSchema.safeParse(response);
-    return (
-      parsed.success &&
-      parsed.data.answers.category.choice === "inherited" &&
-      parsed.data.answers.category.confidence >= 0.95 &&
-      parsed.data.answers.category.probabilities.inherited >= 0.95
-    );
   } catch {
-    return false;
+    return keep(request.signal.aborted ? "cancelled" : "request_failed");
+  }
+  const parsed = attributionSchema.safeParse(response);
+  if (!parsed.success) {
+    return {
+      ...keep("response_invalid"),
+      issues: [
+        ...new Set(parsed.error.issues.map((issue) => issue.path.join("."))),
+      ],
+    };
   }
+  const category = parsed.data.answers.category;
+  const { confidence, probabilities } = category;
+  const inherited =
+    category.choice === "inherited" &&
+    confidence >= ATTRIBUTION_THRESHOLDS.confidence &&
+    probabilities.inherited >= ATTRIBUTION_THRESHOLDS.inherited;
+  return {
+    ...keep("judged"),
+    decision: inherited ? "drop" : "keep",
+    choice: category.choice,
+    confidence,
+    probabilities: { ...probabilities },
+    thresholds: { ...ATTRIBUTION_THRESHOLDS },
+  };
 }