specvector 0.3.3 → 0.6.1

package/src/pipeline/merger.ts

@@ -0,0 +1,329 @@
+/**
+ * Finding Merger & Deduplication for the Scalable Review Pipeline.
+ *
+ * Takes raw findings from BatchResult, deduplicates semantically similar
+ * findings, generalizes patterns that appear in 3+ files, sorts by severity,
+ * and produces a ReviewResult for the formatter.
+ *
+ * This is a pure function — no LLM calls, no IO.
+ */
+
+import type { BatchResult, BatchError } from "./batcher";
+import type {
+  ReviewFinding,
+  ReviewResult,
+  ReviewStats,
+  Severity,
+} from "../types/review";
+import { calculateStats, determineRecommendation } from "../types/review";
+
+// ---------------------------------------------------------------------------
+// Types
+// ---------------------------------------------------------------------------
+
+/** Configuration for the merger. */
+export interface MergerConfig {
+  /** Jaccard similarity threshold for title deduplication (default: 0.7) */
+  similarityThreshold: number;
+  /** Minimum files for pattern generalization (default: 3) */
+  patternThreshold: number;
+}
+
+const DEFAULT_MERGER_CONFIG: MergerConfig = {
+  similarityThreshold: 0.7,
+  patternThreshold: 3,
+};
+
+/** Severity sort order (lower = higher priority). */
+const SEVERITY_ORDER: Record<Severity, number> = {
+  CRITICAL: 0,
+  HIGH: 1,
+  MEDIUM: 2,
+  LOW: 3,
+};
+
+// ---------------------------------------------------------------------------
+// Main Entry Point
+// ---------------------------------------------------------------------------
+
+/**
+ * Merge findings from a BatchResult into a single ReviewResult.
+ *
+ * Pipeline: deduplicate → generalize patterns → sort → build ReviewResult.
+ */
+export function mergeFindings(
+  batchResult: BatchResult,
+  filesReviewed: number,
+  config?: Partial<MergerConfig>,
+): ReviewResult {
+  const cfg: MergerConfig = { ...DEFAULT_MERGER_CONFIG, ...config };
+
+  // Pipeline
+  const deduplicated = deduplicateFindings(batchResult.findings, cfg.similarityThreshold);
+  const generalized = generalizePatterns(deduplicated, cfg.patternThreshold);
+  const sorted = sortFindings(generalized);
+
+  // Compute stats, recommendation, and summary
+  const stats = calculateStats(sorted);
+  const recommendation = determineRecommendation(stats);
+  const summary = buildSummary(sorted, batchResult.errors, stats);
+
+  return {
+    findings: sorted,
+    summary,
+    recommendation,
+    stats,
+    filesReviewed,
+    contextSources:
+      batchResult.contextSources.length > 0
+        ? batchResult.contextSources
+        : undefined,
+  };
+}
+
+// ---------------------------------------------------------------------------
+// Deduplication
+// ---------------------------------------------------------------------------
+
+/**
+ * Internal representation of a finding with collected file references.
+ */
+interface FindingCluster {
+  /** Representative finding (longest description) */
+  representative: ReviewFinding;
+  /** All files from merged findings */
+  affectedFiles: string[];
+}
+
+/**
+ * Deduplicate semantically similar findings.
+ *
+ * Groups findings into clusters by similarity, keeps the most detailed
+ * representative for each cluster, and collects all affected file paths.
+ */
+export function deduplicateFindings(
+  findings: ReviewFinding[],
+  similarityThreshold: number = DEFAULT_MERGER_CONFIG.similarityThreshold,
+): ReviewFinding[] {
+  if (findings.length === 0) return [];
+
+  // Sort deterministically before clustering so that input order
+  // (which depends on non-deterministic Promise.allSettled resolution)
+  // does not affect which clusters form.
+  const sorted = [...findings].sort((a, b) => {
+    const sevDiff = SEVERITY_ORDER[a.severity] - SEVERITY_ORDER[b.severity];
+    if (sevDiff !== 0) return sevDiff;
+    const titleDiff = a.title.localeCompare(b.title);
+    if (titleDiff !== 0) return titleDiff;
+    return (a.file ?? "").localeCompare(b.file ?? "");
+  });
+
+  const clusters: FindingCluster[] = [];
+
+  for (const finding of sorted) {
+    let merged = false;
+
+    for (const cluster of clusters) {
+      if (areSimilarFindings(finding, cluster.representative, similarityThreshold)) {
+        // Merge into existing cluster
+        if (finding.file && !cluster.affectedFiles.includes(finding.file)) {
+          cluster.affectedFiles.push(finding.file);
+        }
+        // Keep the longer description as representative
+        if (finding.description.length > cluster.representative.description.length) {
+          const prev = cluster.representative;
+          const files = cluster.affectedFiles;
+          cluster.representative = { ...finding };
+          cluster.affectedFiles = files;
+          // Carry forward suggestion from previous representative if new one lacks it
+          if (!cluster.representative.suggestion && prev.suggestion) {
+            cluster.representative.suggestion = prev.suggestion;
+          }
+        }
+        merged = true;
+        break;
+      }
+    }
+
+    if (!merged) {
+      clusters.push({
+        representative: { ...finding },
+        affectedFiles: finding.file ? [finding.file] : [],
+      });
+    }
+  }
+
+  // Convert clusters back to findings, attaching affectedFiles metadata
+  return clusters.map((cluster) => {
+    const finding = { ...cluster.representative };
+    if (cluster.affectedFiles.length > 1) {
+      // Store affected files for pattern generalization
+      (finding as FindingWithFiles)._affectedFiles = cluster.affectedFiles;
+    }
+    return finding;
+  });
+}
+
+/** Internal extension to carry affected files through the pipeline. */
+interface FindingWithFiles extends ReviewFinding {
+  _affectedFiles?: string[];
+}
+
+// ---------------------------------------------------------------------------
+// Similarity
+// ---------------------------------------------------------------------------
+
+/**
+ * Check if two findings are semantically similar (candidates for deduplication).
+ *
+ * Criteria:
+ * - Same severity
+ * - Same category (both null or both equal)
+ * - Title Jaccard similarity >= threshold
+ * - Different files (don't merge findings pointing to the same file)
+ */
+export function areSimilarFindings(
+  a: ReviewFinding,
+  b: ReviewFinding,
+  threshold: number = DEFAULT_MERGER_CONFIG.similarityThreshold,
+): boolean {
+  // Severity must match
+  if (a.severity !== b.severity) return false;
+
+  // Category must match (both undefined or both equal)
+  if ((a.category ?? null) !== (b.category ?? null)) return false;
+
+  // Don't merge findings about the same file (they're likely different issues)
+  if (a.file && b.file && a.file === b.file) return false;
+
+  // Title similarity via Jaccard
+  return jaccardSimilarity(a.title, b.title) >= threshold;
+}
+
+/**
+ * Compute Jaccard similarity between two strings based on word tokens.
+ * Returns a value between 0 (no overlap) and 1 (identical).
+ */
+export function jaccardSimilarity(a: string, b: string): number {
+  const wordsA = tokenize(a);
+  const wordsB = tokenize(b);
+
+  if (wordsA.size === 0 && wordsB.size === 0) return 1;
+  if (wordsA.size === 0 || wordsB.size === 0) return 0;
+
+  let intersectionSize = 0;
+  for (const word of wordsA) {
+    if (wordsB.has(word)) intersectionSize++;
+  }
+
+  const unionSize = new Set([...wordsA, ...wordsB]).size;
+  return intersectionSize / unionSize;
+}
+
+/**
+ * Tokenize a string into a set of lowercase words.
+ */
+function tokenize(text: string): Set<string> {
+  return new Set(
+    text
+      .toLowerCase()
+      .split(/\s+/)
+      .filter((w) => w.length > 0),
+  );
+}
+
+// ---------------------------------------------------------------------------
+// Pattern Generalization
+// ---------------------------------------------------------------------------
+
+/**
+ * Generalize findings that appear in many files into pattern comments.
+ *
+ * If a finding has been deduplicated across >= threshold files, it becomes
+ * a repo-wide pattern comment without a specific file reference.
+ */
+export function generalizePatterns(
+  findings: ReviewFinding[],
+  threshold: number = DEFAULT_MERGER_CONFIG.patternThreshold,
+): ReviewFinding[] {
+  return findings.map((finding) => {
+    const files = (finding as FindingWithFiles)._affectedFiles;
+    if (files && files.length >= threshold) {
+      // Generalize to pattern comment
+      const fileList = files.join(", ");
+      const result: ReviewFinding = {
+        ...finding,
+        title: `${finding.title} (pattern)`,
+        description: `${finding.description}\n\nFound in ${files.length} files: ${fileList}`,
+        file: undefined,
+        line: undefined,
+      };
+      // Clean internal metadata
+      delete (result as FindingWithFiles)._affectedFiles;
+      return result;
+    }
+
+    // Below threshold — keep file reference, clean metadata
+    const result = { ...finding };
+    delete (result as FindingWithFiles)._affectedFiles;
+
+    // If deduplicated across 2 files, note the other file in description
+    if (files && files.length === 2) {
+      const otherFile = files.find((f) => f !== finding.file);
+      if (otherFile) {
+        result.description = `${finding.description}\n\nAlso found in: ${otherFile}`;
+        // Clear line — it may reference the wrong file after representative swap
+        result.line = undefined;
+      }
+    }
+
+    return result;
+  });
+}
+
+// ---------------------------------------------------------------------------
+// Sorting
+// ---------------------------------------------------------------------------
+
+/**
+ * Sort findings by severity (CRITICAL > HIGH > MEDIUM > LOW),
+ * then alphabetically by title within the same severity.
+ */
+export function sortFindings(findings: ReviewFinding[]): ReviewFinding[] {
+  return [...findings].sort((a, b) => {
+    const severityDiff = SEVERITY_ORDER[a.severity] - SEVERITY_ORDER[b.severity];
+    if (severityDiff !== 0) return severityDiff;
+    return a.title.localeCompare(b.title);
+  });
+}
+
+// ---------------------------------------------------------------------------
+// Summary Builder
+// ---------------------------------------------------------------------------
+
+/**
+ * Build a human-readable summary from merged findings and batch errors.
+ */
+function buildSummary(findings: ReviewFinding[], errors: BatchError[], stats: ReviewStats): string {
+  const parts: string[] = [];
+
+  if (findings.length === 0) {
+    parts.push("No issues found. Code looks good to merge.");
+  } else {
+    const counts: string[] = [];
+    if (stats.critical > 0) counts.push(`${stats.critical} critical`);
+    if (stats.high > 0) counts.push(`${stats.high} high`);
+    if (stats.medium > 0) counts.push(`${stats.medium} medium`);
+    if (stats.low > 0) counts.push(`${stats.low} low`);
+    parts.push(`Found ${findings.length} issue${findings.length === 1 ? "" : "s"}: ${counts.join(", ")}.`);
+  }
+
+  if (errors.length > 0) {
+    const totalAffected = errors.reduce((sum, e) => sum + e.filesAffected.length, 0);
+    parts.push(
+      `Note: ${errors.length} review batch${errors.length === 1 ? "" : "es"} failed (${totalAffected} file${totalAffected === 1 ? "" : "s"} not reviewed). Findings may be incomplete.`,
+    );
+  }
+
+  return parts.join(" ");
+}

package/src/review/engine.ts

@@ -17,6 +17,7 @@ import { createOutlineTool } from "../agent/tools/outline";
 import { createFindSymbolTool } from "../agent/tools/find-symbol";
 import { calculateStats, determineRecommendation } from "../types/review";
 import type { ReviewResult, ReviewFinding, Severity, ContextSource } from "../types/review";
+import { parseReviewResponseWithFallback, REVIEW_JSON_INSTRUCTION } from "./json-parser";
 import type { Result } from "../types/result";
 import { ok, err } from "../types/result";
 import { loadConfig, getStrictnessModifier } from "../config";
@@ -96,9 +97,9 @@ export async function runReview(
     createFindSymbolTool({ workingDir: config.workingDir }),
   ];
 
-  // Build system prompt with strictness modifier
+  // Build system prompt with strictness modifier and JSON instruction
   const strictnessGuidance = getStrictnessModifier(strictness);
-  let systemPrompt = REVIEW_SYSTEM_PROMPT + `\n\n## Strictness Setting: ${strictness.toUpperCase()}\n${strictnessGuidance}`;
+  let systemPrompt = REVIEW_SYSTEM_PROMPT + REVIEW_JSON_INSTRUCTION + `\n\n## Strictness Setting: ${strictness.toUpperCase()}\n${strictnessGuidance}`;
 
   // Track context sources for citation
   const contextSources: ContextSource[] = [];
@@ -163,8 +164,8 @@ export async function runReview(
     });
   }
 
-  // Parse the response into structured findings
-  const reviewResult = parseReviewResponse(agentResult.value, diffSummary, contextSources);
+  // Parse the response into structured findings (JSON first, regex fallback)
+  const reviewResult = parseReviewResponseWithFallback(agentResult.value, diffSummary, contextSources);
 
   return ok(reviewResult);
 }

package/src/review/json-parser.ts

@@ -0,0 +1,283 @@
+/**
+ * Structured JSON Response Parser for LLM Review Output.
+ *
+ * Parses LLM responses that contain chain-of-thought reasoning followed
+ * by a JSON block conforming to the ReviewFinding schema.
+ *
+ * Falls back to the existing regex parser (parseReviewResponse) when
+ * JSON extraction or validation fails.
+ */
+
+import { parseReviewResponse } from "./engine";
+import { calculateStats, determineRecommendation } from "../types/review";
+import type {
+  ReviewFinding,
+  ReviewResult,
+  Severity,
+  ContextSource,
+} from "../types/review";
+
+// ---------------------------------------------------------------------------
+// Types
+// ---------------------------------------------------------------------------
+
+/** Shape of the JSON block the LLM is expected to produce. */
+export interface StructuredReviewResponse {
+  summary: string;
+  findings: StructuredFinding[];
+}
+
+/** A single finding in the JSON output. Mirrors ReviewFinding. */
+interface StructuredFinding {
+  severity: string;
+  title: string;
+  description: string;
+  file?: string;
+  line?: number;
+  suggestion?: string;
+  category?: string;
+}
+
+/** Valid severity values (upper-case). */
+const VALID_SEVERITIES: ReadonlySet<string> = new Set([
+  "CRITICAL",
+  "HIGH",
+  "MEDIUM",
+  "LOW",
+]);
+
+// ---------------------------------------------------------------------------
+// Prompt Instruction
+// ---------------------------------------------------------------------------
+
+/**
+ * Prompt suffix that instructs the LLM to output chain-of-thought
+ * reasoning followed by a JSON block.
+ *
+ * Append this to both REVIEW_SYSTEM_PROMPT and FAST_PASS_SYSTEM_PROMPT.
+ */
+export const REVIEW_JSON_INSTRUCTION = `
+
+## Output Format — IMPORTANT
+IGNORE the "Response Format" section above. Instead, use this JSON format:
+
+After your reasoning, output your review as a single JSON code block.
+The JSON must conform to this schema:
+
+    {
+      "summary": "1-2 sentence summary of the review",
+      "findings": [
+        {
+          "severity": "CRITICAL | HIGH | MEDIUM | LOW",
+          "category": "Security | Performance | Code Quality | ...",
+          "title": "Short title of the finding",
+          "description": "Detailed description and how to fix",
+          "file": "path/to/file.ts",
+          "line": 42,
+          "suggestion": "Suggested fix (optional)"
+        }
+      ]
+    }
+
+If the code is clean, use an empty findings array:
+
+    { "summary": "Code looks good to merge.", "findings": [] }
+
+Important:
+- Think step by step FIRST (chain of thought), then output the JSON block
+- severity must be one of: CRITICAL, HIGH, MEDIUM, LOW
+- file, line, suggestion, and category are optional
+- The JSON block MUST be wrapped in \`\`\`json fences`;
+
+// ---------------------------------------------------------------------------
+// JSON Extraction
+// ---------------------------------------------------------------------------
+
+/**
+ * Extract a JSON string from mixed text+JSON content.
+ *
+ * Priority:
+ * 1. Fenced ```json ... ``` block (most explicit)
+ * 2. Fenced ``` ... ``` block (no language tag, if content looks like JSON)
+ *
+ * Returns null if no JSON-like content is found.
+ * Note: bare JSON extraction (no fences) is intentionally omitted — the prompt
+ * explicitly instructs the LLM to use fenced blocks, and bare brace scanning
+ * is fragile (cannot distinguish braces inside JSON strings from structural braces).
+ */
+export function extractJSON(text: string): string | null {
+  // 1. Fenced ```json block — take the last one, which is likely
+  //    the final output after CoT reasoning
+  const jsonFenced = findLastFencedBlock(text, "json");
+  if (jsonFenced !== null) return jsonFenced;
+
+  // 2. Fenced ``` block (no language tag)
+  const plainFenced = findLastFencedBlock(text, "");
+  if (plainFenced !== null && looksLikeJSON(plainFenced)) return plainFenced;
+
+  return null;
+}
+
+/**
+ * Find the last fenced code block with the given language tag.
+ * Empty string matches blocks with no language tag.
+ */
+function findLastFencedBlock(text: string, lang: string): string | null {
+  // Build pattern: ```lang\n...\n``` with closing ``` anchored to line start
+  // to avoid matching inline triple-backtick snippets in CoT reasoning.
+  const escapedLang = lang.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
+  const pattern = lang
+    ? new RegExp("```" + escapedLang + "\\s*\\n([\\s\\S]*?)^```", "gm")
+    : new RegExp("```\\s*\\n([\\s\\S]*?)^```", "gm");
+
+  let lastMatch: string | null = null;
+  let match: RegExpExecArray | null;
+  while ((match = pattern.exec(text)) !== null) {
+    if (match[1] !== undefined) {
+      lastMatch = match[1].trim();
+    }
+  }
+  return lastMatch;
+}
+
+/** Quick heuristic: does the string start with `{`? */
+function looksLikeJSON(text: string): boolean {
+  return text.trim().startsWith("{");
+}
+
+// ---------------------------------------------------------------------------
+// JSON Response Parser
+// ---------------------------------------------------------------------------
+
+/**
+ * Parse a JSON-formatted review response into a ReviewResult.
+ *
+ * Returns null if the response doesn't contain valid JSON or if
+ * required fields are missing / invalid. The caller should fall back
+ * to the regex parser in that case.
+ */
+export function parseJSONReviewResponse(
+  response: string,
+  diffSummary: string,
+  contextSources: ContextSource[] = [],
+): ReviewResult | null {
+  // Step 1: Extract JSON
+  const jsonStr = extractJSON(response);
+  if (jsonStr === null) return null;
+
+  // Step 2: Parse JSON
+  let parsed: unknown;
+  try {
+    parsed = JSON.parse(jsonStr);
+  } catch {
+    return null;
+  }
+
+  // Step 3: Validate top-level shape
+  if (!isObject(parsed)) return null;
+  const obj = parsed as Record<string, unknown>;
+
+  const summary =
+    typeof obj.summary === "string" ? obj.summary : "";
+  const rawFindings = Array.isArray(obj.findings) ? obj.findings : null;
+  if (rawFindings === null) return null;
+
+  // Step 4: Validate and convert each finding (skip invalid ones)
+  const findings: ReviewFinding[] = [];
+  for (const raw of rawFindings) {
+    const finding = validateFinding(raw);
+    if (finding !== null) {
+      findings.push(finding);
+    }
+  }
+
+  // Step 5: Build ReviewResult
+  const filesReviewed = extractFilesReviewed(diffSummary);
+  const stats = calculateStats(findings);
+  const recommendation = determineRecommendation(stats);
+
+  return {
+    findings,
+    summary: summary || (findings.length === 0 ? "No issues found." : `Found ${findings.length} issue${findings.length === 1 ? "" : "s"}.`),
+    recommendation,
+    stats,
+    filesReviewed,
+    contextSources: contextSources.length > 0 ? contextSources : undefined,
+  };
+}
+
+/**
+ * Validate a single finding from the JSON response.
+ * Returns null if required fields are missing or severity is invalid.
+ */
+function validateFinding(raw: unknown): ReviewFinding | null {
+  if (!isObject(raw)) return null;
+  const obj = raw as Record<string, unknown>;
+
+  // Required fields — trim before checking to reject whitespace-only values
+  const severity = typeof obj.severity === "string" ? obj.severity.trim().toUpperCase() : null;
+  const title = typeof obj.title === "string" ? obj.title.trim() : null;
+  const description = typeof obj.description === "string" ? obj.description.trim() : null;
+
+  if (!severity || !title || !description) return null;
+  if (!VALID_SEVERITIES.has(severity)) return null;
+
+  const finding: ReviewFinding = {
+    severity: severity as Severity,
+    title,
+    description,
+  };
+
+  // Optional fields
+  if (typeof obj.file === "string" && obj.file.trim().length > 0) {
+    finding.file = obj.file.trim();
+  }
+  if (typeof obj.line === "number" && Number.isInteger(obj.line) && obj.line > 0) {
+    finding.line = obj.line;
+  }
+  if (typeof obj.suggestion === "string" && obj.suggestion.length > 0) {
+    finding.suggestion = obj.suggestion.trim();
+  }
+  if (typeof obj.category === "string" && obj.category.length > 0) {
+    finding.category = obj.category.trim();
+  }
+
+  return finding;
+}
+
+function isObject(val: unknown): val is Record<string, unknown> {
+  return typeof val === "object" && val !== null && !Array.isArray(val);
+}
+
+function extractFilesReviewed(diffSummary: string): number {
+  const match = diffSummary.match(/(\d+)\s*files?\s*changed/i);
+  if (match?.[1]) return parseInt(match[1], 10);
+  // Fallback: look for just a number at the start (e.g., "3 files changed")
+  const numMatch = diffSummary.match(/^(\d+)/);
+  return numMatch?.[1] ? parseInt(numMatch[1], 10) : 0;
+}
+
+// ---------------------------------------------------------------------------
+// Parse-with-Fallback Wrapper
+// ---------------------------------------------------------------------------
+
+/**
+ * Parse an LLM review response, trying JSON first, then regex fallback.
+ *
+ * This is the drop-in replacement for `parseReviewResponse()` at both
+ * integration points (engine.ts agent loop, batcher.ts FAST_PASS).
+ */
+export function parseReviewResponseWithFallback(
+  response: string,
+  diffSummary: string,
+  contextSources: ContextSource[] = [],
+): ReviewResult {
+  // Try JSON parser first
+  const jsonResult = parseJSONReviewResponse(response, diffSummary, contextSources);
+  if (jsonResult !== null) {
+    return jsonResult;
+  }
+
+  // Fall back to existing regex parser
+  return parseReviewResponse(response, diffSummary, contextSources);
+}