specvector 0.3.1 → 0.6.1

package/src/pipeline/classifier.ts

@@ -0,0 +1,361 @@
+/**
+ * File Risk Classifier for the Scalable Review Pipeline.
+ *
+ * Classifies each file in a PR diff as SKIP, FAST_PASS, or DEEP_DIVE
+ * using heuristics only (no LLM calls). This determines how each file
+ * is reviewed: skipped entirely, single-pass LLM review, or full
+ * agent-loop deep review.
+ */
+
+import type { DiffFile, ParsedDiff } from "../types/diff";
+
+/** Risk level assigned to each file in a PR diff. */
+export type RiskLevel = "SKIP" | "FAST_PASS" | "DEEP_DIVE";
+
+/** A file with its assigned risk classification and reasons. */
+export interface ClassifiedFile {
+  /** File path (uses newPath, falls back to oldPath) */
+  path: string;
+  /** Assigned risk level */
+  risk: RiskLevel;
+  /** Human-readable reasons for the classification */
+  reasons: string[];
+  /** Original diff file data */
+  diffFile: DiffFile;
+}
+
+/** Classification result with summary counts. */
+export interface ClassificationResult {
+  files: ClassifiedFile[];
+  counts: { skip: number; fastPass: number; deepDive: number };
+}
+
+// --- SKIP patterns ---
+
+const LOCKFILE_NAMES = new Set([
+  "bun.lock",
+  "bun.lockb",
+  "package-lock.json",
+  "yarn.lock",
+  "pnpm-lock.yaml",
+  "composer.lock",
+  "Gemfile.lock",
+  "Cargo.lock",
+  "poetry.lock",
+]);
+
+const SKIP_EXTENSIONS = new Set([
+  ".md",
+  ".mdx",
+  ".txt",
+  ".toml",
+  ".ini",
+  ".csv",
+  ".svg",
+  ".ico",
+]);
+
+const SKIP_FILENAME_PREFIXES = [
+  "LICENSE",
+  "CHANGELOG",
+  "CHANGES",
+  "HISTORY",
+  ".gitignore",
+  ".gitattributes",
+  ".editorconfig",
+  ".prettierrc",
+  ".eslintignore",
+  ".dockerignore",
+];
+
+const SKIP_PATH_SEGMENTS = ["dist/", "build/", "out/", "coverage/", ".next/", "node_modules/"];
+
+const GENERATED_PATTERNS = [
+  /\.generated\.\w+$/,
+  /\.min\.(js|css)$/,
+  /\.d\.ts$/,
+  /\.map$/,
+  /\.snap$/,
+];
+
+const SKIP_CONFIG_EXTENSIONS = new Set([".json"]);
+const CONFIG_JSON_EXCEPTIONS = new Set(["package.json", "tsconfig.json"]);
+
+const SAFE_ENV_NAMES = new Set([".env.example", ".env.sample", ".env.template"]);
+
+// --- DEEP_DIVE patterns ---
+
+const SECURITY_PATH_SEGMENTS = [
+  "auth",
+  "crypto",
+  "payment",
+  "secret",
+  "token",
+  "session",
+  "permission",
+  "middleware",
+  "security",
+  "credential",
+  "sql",
+  "migration",
+  "database",
+  "db",
+];
+
+const SECURITY_FILENAME_PATTERNS = [
+  /password/i,
+  /credential/i,
+  /oauth/i,
+  /jwt/i,
+  /encrypt/i,
+  /decrypt/i,
+  /apikey/i,
+  /api-key/i,
+  /query/i,
+];
+
+const CORE_ARCHITECTURE_PATHS = [
+  "src/index.ts",
+  "src/agent/",
+  "src/llm/",
+  "src/pipeline/",
+];
+
+/** High complexity thresholds */
+const HIGH_CHURN_THRESHOLD = 50;
+const HIGH_HUNK_THRESHOLD = 3;
+
+/**
+ * Classify all files in a parsed diff by risk level.
+ */
+export function classifyFiles(diff: ParsedDiff): ClassificationResult {
+  const files: ClassifiedFile[] = [];
+  const counts = { skip: 0, fastPass: 0, deepDive: 0 };
+
+  for (const diffFile of diff.files) {
+    const path = diffFile.newPath ?? diffFile.oldPath ?? "unknown";
+    const classified = classifyFile(path, diffFile);
+    files.push(classified);
+
+    switch (classified.risk) {
+      case "SKIP":
+        counts.skip++;
+        break;
+      case "FAST_PASS":
+        counts.fastPass++;
+        break;
+      case "DEEP_DIVE":
+        counts.deepDive++;
+        break;
+    }
+  }
+
+  console.log(
+    `📊 Classified ${files.length} files: ${counts.skip} SKIP, ${counts.fastPass} FAST_PASS, ${counts.deepDive} DEEP_DIVE`
+  );
+
+  return { files, counts };
+}
+
+/**
+ * Classify a single file. DEEP_DIVE is checked first so security-sensitive
+ * files are never silently skipped. Then SKIP, then FAST_PASS as default.
+ */
+function classifyFile(path: string, diffFile: DiffFile): ClassifiedFile {
+  // Check DEEP_DIVE first — security-sensitive files must never be silently skipped.
+  // For renamed files, check both old and new paths.
+  const deepDivePath = path;
+  const deepDiveAltPath =
+    diffFile.status === "renamed"
+      ? (diffFile.oldPath ?? undefined)
+      : undefined;
+
+  const deepDiveResult = checkDeepDive(deepDivePath, diffFile, deepDiveAltPath);
+  if (deepDiveResult) {
+    return { path, risk: "DEEP_DIVE", reasons: deepDiveResult, diffFile };
+  }
+
+  const skipResult = checkSkip(path, diffFile);
+  if (skipResult) {
+    return { path, risk: "SKIP", reasons: skipResult, diffFile };
+  }
+
+  return {
+    path,
+    risk: "FAST_PASS",
+    reasons: ["Standard change, single-pass review"],
+    diffFile,
+  };
+}
+
+/**
+ * Check if a file should be SKIPped. Returns reasons if yes, null if no.
+ */
+function checkSkip(path: string, diffFile: DiffFile): string[] | null {
+  const reasons: string[] = [];
+  const filename = path.split("/").pop() ?? "";
+  const ext = getExtension(filename);
+
+  // Binary files
+  if (diffFile.binary) {
+    reasons.push("Binary file");
+  }
+
+  // Lockfiles
+  if (LOCKFILE_NAMES.has(filename)) {
+    reasons.push(`Lockfile: ${filename}`);
+  }
+
+  // Generated code patterns
+  for (const pattern of GENERATED_PATTERNS) {
+    if (pattern.test(filename)) {
+      reasons.push(`Generated file: matches ${pattern}`);
+      break;
+    }
+  }
+
+  // Files in output directories — boundary-aware matching
+  for (const segment of SKIP_PATH_SEGMENTS) {
+    if (pathContainsSegment(path, segment)) {
+      reasons.push(`Output directory: ${segment}`);
+      break;
+    }
+  }
+
+  // Documentation and markup
+  if (SKIP_EXTENSIONS.has(ext)) {
+    reasons.push(`Documentation/config extension: ${ext}`);
+  }
+
+  // YAML/YML config files (skip unless in security path — handled by DEEP_DIVE-first)
+  if (ext === ".yaml" || ext === ".yml") {
+    reasons.push(`Config file: ${ext}`);
+  }
+
+  // Config JSON files (except package.json, tsconfig.json)
+  if (SKIP_CONFIG_EXTENSIONS.has(ext) && !CONFIG_JSON_EXCEPTIONS.has(filename)) {
+    reasons.push(`Config JSON file: ${filename}`);
+  }
+
+  // Known skip filenames
+  for (const prefix of SKIP_FILENAME_PREFIXES) {
+    if (filename.startsWith(prefix)) {
+      reasons.push(`Known skip file: ${prefix}`);
+      break;
+    }
+  }
+
+  // Safe .env template files only
+  if (SAFE_ENV_NAMES.has(filename)) {
+    reasons.push(`Environment template: ${filename}`);
+  }
+
+  // Deleted files
+  if (diffFile.status === "deleted") {
+    reasons.push("Deleted file");
+  }
+
+  return reasons.length > 0 ? reasons : null;
+}
+
+/**
+ * Check if a file needs DEEP_DIVE review. Returns reasons if yes, null if no.
+ * Optionally checks an alternate path (for renamed files).
+ */
+function checkDeepDive(path: string, diffFile: DiffFile, altPath?: string): string[] | null {
+  const reasons: string[] = [];
+
+  // Check security paths on both primary and alternate paths
+  const pathsToCheck = altPath ? [path, altPath] : [path];
+  for (const p of pathsToCheck) {
+    const securityReason = checkSecurityPath(p);
+    if (securityReason) {
+      reasons.push(securityReason);
+      break;
+    }
+  }
+
+  // Security-sensitive filename patterns
+  const filename = path.split("/").pop() ?? "";
+  for (const pattern of SECURITY_FILENAME_PATTERNS) {
+    if (pattern.test(filename)) {
+      reasons.push(`Security-sensitive filename: matches ${pattern}`);
+      break;
+    }
+  }
+
+  // Also check alt path filename for renames
+  if (altPath) {
+    const altFilename = altPath.split("/").pop() ?? "";
+    if (altFilename !== filename) {
+      for (const pattern of SECURITY_FILENAME_PATTERNS) {
+        if (pattern.test(altFilename)) {
+          reasons.push(`Security-sensitive renamed from: ${altFilename}`);
+          break;
+        }
+      }
+    }
+  }
+
+  // .env files with potential secrets (not safe templates)
+  if (/^\.env/.test(filename) && !SAFE_ENV_NAMES.has(filename)) {
+    reasons.push(`Environment file with potential secrets: ${filename}`);
+  }
+
+  // High complexity delta
+  const totalChurn = diffFile.additions + diffFile.deletions;
+  if (totalChurn > HIGH_CHURN_THRESHOLD) {
+    reasons.push(`High churn: ${totalChurn} lines changed (threshold: ${HIGH_CHURN_THRESHOLD})`);
+  }
+  if (diffFile.hunks.length > HIGH_HUNK_THRESHOLD) {
+    reasons.push(`Many hunks: ${diffFile.hunks.length} (threshold: ${HIGH_HUNK_THRESHOLD})`);
+  }
+
+  // Core architecture files
+  for (const corePath of CORE_ARCHITECTURE_PATHS) {
+    if (path === corePath || path.startsWith(corePath)) {
+      reasons.push(`Core architecture file: ${corePath}`);
+      break;
+    }
+  }
+
+  return reasons.length > 0 ? reasons : null;
+}
+
+/**
+ * Check if a path contains a security-sensitive segment.
+ * Handles top-level dirs (no leading slash) and mid-path segments.
+ */
+function checkSecurityPath(path: string): string | null {
+  const lowerPath = path.toLowerCase();
+  for (const segment of SECURITY_PATH_SEGMENTS) {
+    if (
+      lowerPath.startsWith(`${segment}/`) ||
+      lowerPath.startsWith(`${segment}.`) ||
+      lowerPath.includes(`/${segment}/`) ||
+      lowerPath.includes(`/${segment}.`)
+    ) {
+      return `Security-sensitive path: contains "${segment}"`;
+    }
+  }
+  return null;
+}
+
+/**
+ * Check if a path contains an output directory segment at a proper boundary.
+ * Prevents false positives like "src/layout/" matching "out/".
+ */
+function pathContainsSegment(path: string, segment: string): boolean {
+  return path.startsWith(segment) || path.includes(`/${segment}`);
+}
+
+/**
+ * Extract file extension including the dot.
+ * Returns empty string for dotfiles with no extension (e.g., ".gitignore").
+ */
+function getExtension(filename: string): string {
+  const lastDot = filename.lastIndexOf(".");
+  // Dotfiles with no extension: the dot is at position 0 and there's no other dot
+  if (lastDot <= 0) return "";
+  return filename.slice(lastDot);
+}

package/src/pipeline/index.ts

@@ -0,0 +1,34 @@
+/**
+ * Scalable Review Pipeline - barrel exports.
+ */
+
+export {
+  classifyFiles,
+  type RiskLevel,
+  type ClassifiedFile,
+  type ClassificationResult,
+} from "./classifier";
+
+export {
+  runBatchedReviews,
+  reviewFastPassBatch,
+  reviewDeepDiveFile,
+  buildFastPassTask,
+  reconstructFileDiff,
+  splitIntoBatches,
+  runWithConcurrencyLimit,
+  FAST_PASS_SYSTEM_PROMPT,
+  type BatchConfig,
+  type BatchError,
+  type BatchResult,
+} from "./batcher";
+
+export {
+  mergeFindings,
+  deduplicateFindings,
+  generalizePatterns,
+  sortFindings,
+  areSimilarFindings,
+  jaccardSimilarity,
+  type MergerConfig,
+} from "./merger";