pi-workflow-engine 0.10.3 → 0.12.0

package/.pi/extensions/pi-workflow-engine/src/provider-usage-limit.ts

@@ -0,0 +1,189 @@
+import { WorkflowPauseError } from "./cancellation.ts";
+import {
+  WORKFLOW_USAGE_LIMIT_DELAY_MIN_MS,
+  type ResolvedWorkflowRunOptions,
+} from "./options.ts";
+import type { WorkflowProviderUsageLimitPause } from "./workflow-run-record.ts";
+
+export const WORKFLOW_PROVIDER_USAGE_LIMIT_CODE = "WORKFLOW_PROVIDER_USAGE_LIMIT";
+export const WORKFLOW_USAGE_LIMIT_FALLBACK_DELAY_MS = 60_000;
+const MAX_RESET_HINT_LENGTH = 512;
+
+export interface ProviderUsageLimitDetails {
+  readonly stopReason: "error";
+  readonly providerMessage: string;
+  readonly provider?: string;
+  readonly model?: string;
+  readonly api?: string;
+  readonly resetHint?: string;
+  readonly resetAt?: number;
+}
+
+export interface ProviderUsageLimitPauseRecord {
+  readonly message: string;
+  readonly pause: WorkflowProviderUsageLimitPause;
+}
+
+/** Fatal-to-the-current-run provider limit that remains resumable from its journal boundary. */
+export class WorkflowProviderUsageLimitError extends WorkflowPauseError {
+  override readonly name = "WorkflowProviderUsageLimitError";
+  readonly code = WORKFLOW_PROVIDER_USAGE_LIMIT_CODE;
+
+  constructor(readonly details: ProviderUsageLimitDetails) {
+    super(details.providerMessage);
+  }
+
+  toJSON(): {
+    readonly name: string;
+    readonly message: string;
+    readonly code: string;
+    readonly details: ProviderUsageLimitDetails;
+  } {
+    return { name: this.name, message: this.message, code: this.code, details: this.details };
+  }
+}
+
+export function providerUsageLimitFromMessages(
+  messages: readonly unknown[],
+  now = Date.now(),
+): WorkflowProviderUsageLimitError | undefined {
+  const message = messages.findLast(isAssistantMessage);
+  if (!message || message.stopReason !== "error") return undefined;
+  const providerMessage = typeof message.errorMessage === "string" ? message.errorMessage.trim() : "";
+  if (!providerMessage || !isUsageLimitMessage(providerMessage)) return undefined;
+  const reset = parseProviderResetHint(providerMessage, now);
+  const provider = stringDetail(message.provider);
+  const model = stringDetail(message.model);
+  const api = stringDetail(message.api);
+  return new WorkflowProviderUsageLimitError({
+    stopReason: "error",
+    providerMessage,
+    ...(provider ? { provider } : {}),
+    ...(model ? { model } : {}),
+    ...(api ? { api } : {}),
+    ...reset,
+  });
+}
+
+export function createProviderUsageLimitPauseRecord(
+  error: WorkflowProviderUsageLimitError,
+  options: ResolvedWorkflowRunOptions,
+  replayableInvocation: boolean,
+  now = Date.now(),
+): ProviderUsageLimitPauseRecord {
+  const attempt = options.usageLimitAttempt + 1;
+  const hintedDelay = error.details.resetAt === undefined
+    ? WORKFLOW_USAGE_LIMIT_FALLBACK_DELAY_MS
+    : error.details.resetAt - now;
+  const delayMs = Math.min(
+    options.usageLimitMaxDelayMs,
+    Math.max(
+      WORKFLOW_USAGE_LIMIT_DELAY_MIN_MS,
+      hintedDelay > 0 ? hintedDelay : WORKFLOW_USAGE_LIMIT_FALLBACK_DELAY_MS,
+    ),
+  );
+  const autoResume = options.autoResumeOnUsageLimit
+    && attempt < options.usageLimitMaxAttempts
+    && replayableInvocation;
+  return {
+    message: autoResume
+      ? `Provider usage limit paused this run; automatic attempt ${attempt + 1}/${options.usageLimitMaxAttempts} is scheduled.`
+      : `Provider usage limit paused this run at attempt ${attempt}/${options.usageLimitMaxAttempts}.`,
+    pause: {
+      kind: "provider_usage_limit",
+      reason: "provider_usage_limit",
+      providerMessage: error.details.providerMessage,
+      provider: error.details.provider,
+      model: error.details.model,
+      api: error.details.api,
+      resetHint: error.details.resetHint,
+      attempt,
+      nextEligibleAt: now + delayMs,
+      autoResume,
+      maxAttempts: options.usageLimitMaxAttempts,
+    },
+  };
+}
+
+export function parseProviderResetHint(
+  message: string,
+  now = Date.now(),
+): { readonly resetHint?: string; readonly resetAt?: number } {
+  const iso = message.match(
+    /(?:[a-z-]*reset[a-z-]*|resets?\s+at|available\s+again\s+at)\s*[:=]?\s*(\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(?:\.\d+)?(?:Z|[+-]\d{2}:\d{2}))/i,
+  )?.[1];
+  if (iso) {
+    const parsed = Date.parse(iso);
+    return Number.isFinite(parsed) && parsed > now
+      ? { resetHint: boundedHint(iso), resetAt: parsed }
+      : { resetHint: boundedHint(iso) };
+  }
+
+  const labelled = message.match(
+    /(?:retry[- ]after|try again in|resets? in|available again in|x-ratelimit-reset(?:-[a-z-]+)?)\s*[:=]?\s*((?:\d+(?:\.\d+)?\s*(?:milliseconds?|ms|seconds?|secs?|s|minutes?|mins?|m|hours?|hrs?|h|days?|d)\s*)+)/i,
+  );
+  if (labelled?.[1]) {
+    const delayMs = parseDurationMs(labelled[1]);
+    return delayMs === undefined
+      ? { resetHint: boundedHint(labelled[0]) }
+      : { resetHint: boundedHint(labelled[0]), resetAt: now + delayMs };
+  }
+
+  const retryAfter = message.match(/retry[- ]after\s*[:=]\s*([^\s,;]+)/i);
+  if (retryAfter?.[1]) {
+    const seconds = Number(retryAfter[1]);
+    return Number.isFinite(seconds) && seconds >= 0
+      ? { resetHint: boundedHint(retryAfter[0]), resetAt: now + seconds * 1_000 }
+      : { resetHint: boundedHint(retryAfter[0]) };
+  }
+  return {};
+}
+
+function isUsageLimitMessage(message: string): boolean {
+  return /(?:GoUsageLimitError|FreeUsageLimitError|insufficient_quota|rate_limit_error|ResourceExhausted)/i.test(message)
+    || /\btoo many requests\b/i.test(message)
+    || /\b(?:http|status(?: code)?|error)\s*[:=]?\s*429\b/i.test(message)
+    || /\b429\b[^\n]*(?:rate|requests?|quota|limit)/i.test(message)
+    || /\b(?:usage|quota|monthly|weekly|daily)\s+(?:window\s+)?limit\b/i.test(message)
+    || /\bquota\s+(?:has\s+been\s+)?exceeded\b/i.test(message)
+    || /\bexceeded\b[^\n]*\bquota\b/i.test(message)
+    || /\b(?:usage|rate)\s+limit\s+(?:(?:has\s+been|was)\s+)?(?:reached|exceeded)\b/i.test(message);
+}
+
+function parseDurationMs(value: string): number | undefined {
+  const unitPattern = /(\d+(?:\.\d+)?)\s*(milliseconds?|ms|seconds?|secs?|s|minutes?|mins?|m|hours?|hrs?|h|days?|d)/gi;
+  let total = 0;
+  let matched = "";
+  for (const part of value.matchAll(unitPattern)) {
+    const amount = Number(part[1]);
+    const unit = part[2]?.toLowerCase();
+    if (!Number.isFinite(amount) || !unit) return undefined;
+    matched += part[0];
+    if (unit === "ms" || unit.startsWith("millisecond")) total += amount;
+    else if (unit === "s" || unit.startsWith("sec")) total += amount * 1_000;
+    else if (unit === "m" || unit.startsWith("min")) total += amount * 60_000;
+    else if (unit === "h" || unit.startsWith("hr") || unit.startsWith("hour")) total += amount * 3_600_000;
+    else total += amount * 86_400_000;
+  }
+  if (!matched || value.replace(/\s+/g, "") !== matched.replace(/\s+/g, "")) return undefined;
+  return Number.isFinite(total) && total >= 0 ? total : undefined;
+}
+
+function boundedHint(value: string): string {
+  return value.length <= MAX_RESET_HINT_LENGTH ? value : `${value.slice(0, MAX_RESET_HINT_LENGTH - 1)}…`;
+}
+
+function isAssistantMessage(value: unknown): value is {
+  readonly role: "assistant";
+  readonly stopReason?: unknown;
+  readonly errorMessage?: unknown;
+  readonly provider?: unknown;
+  readonly model?: unknown;
+  readonly api?: unknown;
+} {
+  return typeof value === "object" && value !== null && "role" in value && value.role === "assistant";
+}
+
+function stringDetail(value: unknown): string | undefined {
+  return typeof value === "string" && value.length > 0 ? value : undefined;
+}

package/.pi/extensions/pi-workflow-engine/src/replay-path-identity.ts

@@ -0,0 +1,48 @@
+import { relative, resolve, sep } from "node:path";
+import { isPathWithin } from "./tree-fingerprint.ts";
+
+export interface ReplayWorkspaceRoots {
+  readonly sessionCwd: string;
+  readonly workspaceRoot: string;
+}
+
+/** Map a path in either the disposable session or main workspace to one stable namespace. */
+export function logicalWorkspacePath(path: string, roots: ReplayWorkspaceRoots): string | undefined {
+  for (const root of resolvedWorkspaceRoots(roots)) {
+    if (isPathWithin(root, path)) return `workspace:${portableRelativePath(root, path)}`;
+  }
+  return undefined;
+}
+
+/** Normalize absolute workspace paths embedded in generated system prompts. */
+export function normalizeWorkspaceReferences(value: string, roots: ReplayWorkspaceRoots): string {
+  const paths = [...resolvedWorkspaceRoots(roots)].sort((left, right) => right.length - left.length);
+  return paths.reduce((normalized, path) => replacePathPrefix(normalized, path, "<workspace>"), value);
+}
+
+export function portableRelativePath(root: string, path: string): string {
+  const value = relative(root, path).split(sep).join("/");
+  return value.length === 0 ? "." : value;
+}
+
+function resolvedWorkspaceRoots(roots: ReplayWorkspaceRoots): readonly string[] {
+  return [...new Set([resolve(roots.sessionCwd), resolve(roots.workspaceRoot)])];
+}
+
+function replacePathPrefix(value: string, prefix: string, replacement: string): string {
+  let result = "";
+  let cursor = 0;
+  while (cursor < value.length) {
+    const index = value.indexOf(prefix, cursor);
+    if (index === -1) return result + value.slice(cursor);
+    const next = value[index + prefix.length];
+    if (next !== undefined && next !== "/" && next !== "\\" && /[\p{L}\p{N}._-]/u.test(next)) {
+      result += value.slice(cursor, index + prefix.length);
+      cursor = index + prefix.length;
+      continue;
+    }
+    result += value.slice(cursor, index) + replacement;
+    cursor = index + prefix.length;
+  }
+  return result;
+}

package/.pi/extensions/pi-workflow-engine/src/research-contract.ts

@@ -0,0 +1,89 @@
+import { Type, type Static } from "typebox";
+
+export const MAX_RESEARCH_LANES = 4;
+export const MAX_LANE_EVIDENCE = 6;
+export const MAX_VERIFICATION_CLAIMS = 12;
+
+export const ResearchSourceSchema = Type.Object({
+  title: Type.String({ minLength: 1, description: "Human-readable title of the supporting page." }),
+  url: Type.String({ minLength: 1, description: "Direct HTTP(S) URL of the supporting page, never a search-results URL." }),
+  publishedAt: Type.Optional(Type.String({ description: "Publication or update date when the page states one." })),
+});
+
+export type ResearchSource = Static<typeof ResearchSourceSchema>;
+
+export const ResearchPlanSchema = Type.Object({
+  scopeConstraints: Type.Array(Type.String()),
+  lanes: Type.Array(
+    Type.Object({
+      id: Type.String({ minLength: 1 }),
+      title: Type.String({ minLength: 1 }),
+      objective: Type.String({ minLength: 1 }),
+      queries: Type.Array(Type.String({ minLength: 1 }), { minItems: 1, maxItems: 4 }),
+    }),
+    { minItems: 1, maxItems: MAX_RESEARCH_LANES },
+  ),
+});
+
+export type ResearchPlan = Static<typeof ResearchPlanSchema>;
+export type ResearchLane = ResearchPlan["lanes"][number];
+
+export const ResearchEvidenceSchema = Type.Object({
+  claim: Type.String({ minLength: 1 }),
+  importance: Type.Union([Type.Literal("high"), Type.Literal("medium"), Type.Literal("low")]),
+  stance: Type.Union([Type.Literal("supports"), Type.Literal("conflicts")]),
+  evidence: Type.String({ minLength: 1, description: "A short supporting passage or precise paraphrase from the page." }),
+  source: ResearchSourceSchema,
+});
+
+export type ResearchEvidence = Static<typeof ResearchEvidenceSchema>;
+
+export const ResearchLaneResultSchema = Type.Object({
+  laneId: Type.String({ minLength: 1 }),
+  evidence: Type.Array(ResearchEvidenceSchema, { maxItems: MAX_LANE_EVIDENCE }),
+  gaps: Type.Array(Type.String()),
+});
+
+export type ResearchLaneResult = Static<typeof ResearchLaneResultSchema>;
+
+export interface ResearchClaimCandidate {
+  readonly claim: string;
+  readonly importance: ResearchEvidence["importance"];
+  readonly evidence: readonly ResearchEvidence[];
+}
+
+export const ResearchVerificationSchema = Type.Object({
+  claim: Type.String({ minLength: 1 }),
+  verdict: Type.Union([
+    Type.Literal("SUPPORTED"),
+    Type.Literal("CONFLICTED"),
+    Type.Literal("UNCERTAIN"),
+    Type.Literal("INFERENCE"),
+    Type.Literal("REJECTED"),
+  ]),
+  explanation: Type.String({ minLength: 1 }),
+  sources: Type.Array(ResearchSourceSchema, { maxItems: 8 }),
+});
+
+export type ResearchVerification = Static<typeof ResearchVerificationSchema>;
+
+export const ResearchReportEntrySchema = Type.Object({
+  claim: Type.String({ minLength: 1 }),
+  explanation: Type.String({ minLength: 1 }),
+  citations: Type.Array(ResearchSourceSchema, { maxItems: 8 }),
+});
+
+export type ResearchReportEntry = Static<typeof ResearchReportEntrySchema>;
+
+export const ResearchReportSchema = Type.Object({
+  answer: Type.String(),
+  supportedClaims: Type.Array(ResearchReportEntrySchema),
+  conflictingEvidence: Type.Array(ResearchReportEntrySchema),
+  uncertainties: Type.Array(ResearchReportEntrySchema),
+  inferences: Type.Array(ResearchReportEntrySchema),
+  sources: Type.Array(ResearchSourceSchema),
+  limitations: Type.Array(Type.String()),
+  nextSteps: Type.Array(Type.String()),
+});
+
+export type ResearchReport = Static<typeof ResearchReportSchema>;

package/.pi/extensions/pi-workflow-engine/src/research-evidence.ts

@@ -0,0 +1,289 @@
+import {
+  MAX_RESEARCH_LANES,
+  MAX_VERIFICATION_CLAIMS,
+  type ResearchClaimCandidate,
+  type ResearchEvidence,
+  type ResearchLane,
+  type ResearchLaneResult,
+  type ResearchPlan,
+  type ResearchReport,
+  type ResearchReportEntry,
+  type ResearchSource,
+  type ResearchVerification,
+} from "./research-contract.ts";
+
+const TRACKING_PARAMETERS = new Set(["fbclid", "gclid", "mc_cid", "mc_eid", "ref_src"]);
+const IMPORTANCE_RANK: Readonly<Record<ResearchEvidence["importance"], number>> = { high: 0, medium: 1, low: 2 };
+
+export function normalizeResearchLanes(plan: ResearchPlan): ResearchLane[] {
+  const seen = new Set<string>();
+  const lanes: ResearchLane[] = [];
+  for (const lane of plan.lanes) {
+    const id = normalizedText(lane.id);
+    const title = normalizedText(lane.title);
+    const objective = normalizedText(lane.objective);
+    const queries = dedupeText(lane.queries).slice(0, 4);
+    if (!id || !title || !objective || queries.length === 0 || seen.has(id.toLowerCase())) continue;
+    seen.add(id.toLowerCase());
+    lanes.push({ id, title, objective, queries });
+    if (lanes.length >= MAX_RESEARCH_LANES) break;
+  }
+  return lanes;
+}
+
+export function sanitizeLaneResults(results: readonly ResearchLaneResult[]): ResearchLaneResult[] {
+  return results.map((result) => ({
+    laneId: normalizedText(result.laneId),
+    gaps: dedupeText(result.gaps),
+    evidence: dedupeEvidence(result.evidence.map(sanitizeEvidence).filter(isDefined)),
+  }));
+}
+
+export function buildClaimCandidates(results: readonly ResearchLaneResult[]): ResearchClaimCandidate[] {
+  const byClaim = new Map<string, { claim: string; importance: ResearchEvidence["importance"]; evidence: ResearchEvidence[] }>();
+  for (const item of results.flatMap((result) => result.evidence)) {
+    const key = normalizedText(item.claim).toLowerCase();
+    if (!key) continue;
+    const current = byClaim.get(key);
+    if (!current) {
+      byClaim.set(key, { claim: normalizedText(item.claim), importance: item.importance, evidence: [item] });
+      continue;
+    }
+    if (IMPORTANCE_RANK[item.importance] < IMPORTANCE_RANK[current.importance]) current.importance = item.importance;
+    current.evidence = dedupeEvidence([...current.evidence, item]);
+  }
+  return [...byClaim.values()]
+    .sort((left, right) => IMPORTANCE_RANK[left.importance] - IMPORTANCE_RANK[right.importance])
+    .slice(0, MAX_VERIFICATION_CLAIMS);
+}
+
+export function sanitizeVerification(
+  verification: ResearchVerification,
+  candidate: ResearchClaimCandidate,
+): ResearchVerification {
+  const sources = dedupeSources(verification.sources);
+  const verdict = (verification.verdict === "SUPPORTED" || verification.verdict === "CONFLICTED") && sources.length === 0
+    ? "UNCERTAIN"
+    : verification.verdict;
+  return {
+    claim: candidate.claim,
+    verdict,
+    explanation: normalizedText(verification.explanation) || "Independent verification returned no explanation.",
+    sources,
+  };
+}
+
+export function unavailableVerification(candidate: ResearchClaimCandidate): ResearchVerification {
+  return {
+    claim: candidate.claim,
+    verdict: "UNCERTAIN",
+    explanation: "Independent verification did not complete; gathered evidence remains unverified.",
+    sources: dedupeSources(candidate.evidence.map((item) => item.source)),
+  };
+}
+
+export function sanitizeResearchReport(
+  report: ResearchReport,
+  verifications: readonly ResearchVerification[],
+): ResearchReport {
+  const sources = dedupeSources(
+    verifications
+      .filter((verification) => verification.verdict !== "REJECTED")
+      .flatMap((verification) => verification.sources),
+  );
+  const uncertainties = sanitizeEntries(
+    report.uncertainties,
+    verificationSources(verifications, "UNCERTAIN"),
+    false,
+  );
+  const supportedClaims = sanitizeEntries(
+    report.supportedClaims,
+    verificationSources(verifications, "SUPPORTED"),
+    true,
+    uncertainties,
+  );
+  const conflictingEvidence = sanitizeEntries(
+    report.conflictingEvidence,
+    verificationSources(verifications, "CONFLICTED"),
+    true,
+    uncertainties,
+  );
+  const inferences = sanitizeEntries(
+    report.inferences,
+    verificationSources(verifications, "INFERENCE"),
+    false,
+  );
+  return {
+    answer: normalizedText(report.answer),
+    supportedClaims,
+    conflictingEvidence,
+    uncertainties,
+    inferences,
+    sources,
+    limitations: dedupeText(report.limitations),
+    nextSteps: dedupeText(report.nextSteps),
+  };
+}
+
+function verificationSources(
+  verifications: readonly ResearchVerification[],
+  verdict: ResearchVerification["verdict"],
+): ReadonlyMap<string, ReadonlyMap<string, ResearchSource>> {
+  const byClaim = new Map<string, Map<string, ResearchSource>>();
+  for (const verification of verifications) {
+    if (verification.verdict !== verdict) continue;
+    const key = normalizedText(verification.claim).toLowerCase();
+    const claimSources = byClaim.get(key) ?? new Map<string, ResearchSource>();
+    dedupeSources(
+      verification.sources,
+    ).forEach((source) => claimSources.set(source.url, source));
+    byClaim.set(key, claimSources);
+  }
+  return byClaim;
+}
+
+export function fallbackResearchReport(
+  verifications: readonly ResearchVerification[],
+  limitation: string,
+): ResearchReport {
+  const entry = (verification: ResearchVerification): ResearchReportEntry => ({
+    claim: verification.claim,
+    explanation: verification.explanation,
+    citations: verification.sources,
+  });
+  return {
+    answer: "The synthesis stage did not complete; independently verified claims are grouped below without additional model interpretation.",
+    supportedClaims: verifications.filter((item) => item.verdict === "SUPPORTED").map(entry),
+    conflictingEvidence: verifications.filter((item) => item.verdict === "CONFLICTED").map(entry),
+    uncertainties: verifications.filter((item) => item.verdict === "UNCERTAIN").map(entry),
+    inferences: verifications.filter((item) => item.verdict === "INFERENCE").map(entry),
+    sources: dedupeSources(verifications.flatMap((verification) => verification.sources)),
+    limitations: [limitation],
+    nextSteps: ["Review the verified claim groups and rerun synthesis if a narrative answer is required."],
+  };
+}
+
+export function unavailableResearchReport(reason: "empty-question" | "missing-capability" | "no-evidence"): ResearchReport {
+  const message = reason === "empty-question"
+    ? "No research question was provided."
+    : reason === "missing-capability"
+      ? "Research could not start because pi exposed no installed external web-search or URL-extraction tool."
+      : "Research completed without enough direct-page evidence to support an answer.";
+  const nextStep = reason === "empty-question"
+    ? "Run `/workflow research <question>` and include any source, date, or geography constraints in the arguments."
+    : reason === "missing-capability"
+      ? "Install or enable a pi tool that can search the web or extract HTTP(S) pages, then rerun the workflow."
+      : "Check the installed external-search tool, narrow the question, or provide preferred source domains.";
+  return {
+    answer: message,
+    supportedClaims: [],
+    conflictingEvidence: [],
+    uncertainties: [],
+    inferences: [],
+    sources: [],
+    limitations: [message],
+    nextSteps: [nextStep],
+  };
+}
+
+export function canonicalSourceUrl(raw: string): string | undefined {
+  try {
+    const url = new URL(raw.trim());
+    if ((url.protocol !== "http:" && url.protocol !== "https:") || url.username || url.password) return undefined;
+    if (isSearchResultsUrl(url)) return undefined;
+    url.hash = "";
+    for (const key of [...url.searchParams.keys()]) {
+      if (key.toLowerCase().startsWith("utm_") || TRACKING_PARAMETERS.has(key.toLowerCase())) url.searchParams.delete(key);
+    }
+    url.searchParams.sort();
+    if (url.pathname.length > 1 && url.pathname.endsWith("/")) url.pathname = url.pathname.slice(0, -1);
+    return url.toString();
+  } catch {
+    return undefined;
+  }
+}
+
+export function dedupeSources(sources: readonly ResearchSource[]): ResearchSource[] {
+  const byUrl = new Map<string, ResearchSource>();
+  for (const source of sources) {
+    const url = canonicalSourceUrl(source.url);
+    const title = normalizedText(source.title);
+    if (!url || !title || byUrl.has(url)) continue;
+    const publishedAt = source.publishedAt === undefined ? undefined : normalizedText(source.publishedAt);
+    byUrl.set(url, { title, url, ...(publishedAt ? { publishedAt } : {}) });
+  }
+  return [...byUrl.values()];
+}
+
+function sanitizeEvidence(item: ResearchEvidence): ResearchEvidence | undefined {
+  const source = dedupeSources([item.source])[0];
+  const claim = normalizedText(item.claim);
+  const evidence = normalizedText(item.evidence);
+  if (!source || !claim || !evidence) return undefined;
+  return { claim, importance: item.importance, stance: item.stance, evidence, source };
+}
+
+function dedupeEvidence(evidence: readonly ResearchEvidence[]): ResearchEvidence[] {
+  const seen = new Set<string>();
+  return evidence.filter((item) => {
+    const key = `${item.claim.toLowerCase()}\0${item.stance}\0${item.source.url}`;
+    if (seen.has(key)) return false;
+    seen.add(key);
+    return true;
+  });
+}
+
+function sanitizeEntries(
+  entries: readonly ResearchReportEntry[],
+  allowedByClaim: ReadonlyMap<string, ReadonlyMap<string, ResearchSource>>,
+  requireCitation: boolean,
+  demoted: ResearchReportEntry[] = [],
+): ResearchReportEntry[] {
+  const sanitized: ResearchReportEntry[] = [];
+  for (const entry of entries) {
+    const claim = normalizedText(entry.claim);
+    const explanation = normalizedText(entry.explanation);
+    if (!claim || !explanation) continue;
+    const allowed = allowedByClaim.get(claim.toLowerCase()) ?? new Map<string, ResearchSource>();
+    const citations = dedupeSources(entry.citations)
+      .map((source) => allowed.get(source.url))
+      .filter(isDefined);
+    if (requireCitation && citations.length === 0) {
+      demoted.push({ claim, explanation: `${explanation} No verified direct-page citation survived validation.`, citations: [] });
+      continue;
+    }
+    sanitized.push({ claim, explanation, citations });
+  }
+  return sanitized;
+}
+
+function isSearchResultsUrl(url: URL): boolean {
+  const host = url.hostname.toLowerCase().replace(/^www\./, "");
+  const path = url.pathname.replace(/\/+$/, "") || "/";
+  if (/^(?:google\.[a-z.]+|bing\.com|search\.yahoo\.com|search\.brave\.com|yandex\.[a-z.]+)$/.test(host)) {
+    return path === "/search" || path === "/results" || path === "/yandsearch";
+  }
+  if (host === "duckduckgo.com") return (path === "/" || path === "/html" || path === "/lite") && url.searchParams.has("q");
+  return false;
+}
+
+function dedupeText(values: readonly string[]): string[] {
+  const seen = new Set<string>();
+  const result: string[] = [];
+  for (const value of values) {
+    const normalized = normalizedText(value);
+    const key = normalized.toLowerCase();
+    if (!normalized || seen.has(key)) continue;
+    seen.add(key);
+    result.push(normalized);
+  }
+  return result;
+}
+
+function normalizedText(value: string): string {
+  return value.trim().replace(/\s+/g, " ");
+}
+
+function isDefined<T>(value: T | undefined): value is T {
+  return value !== undefined;
+}