weavatrix 0.3.14 → 1.0.0

package/src/analysis/duplicates.compute.js

@@ -1,269 +0,0 @@
-// duplicates.compute.js — fragment extraction, inverted-index pairing, and the computeDuplicates
-// pipeline (split from duplicates.js; see the facade there for the full algorithm overview).
-import { readFileSync } from "node:fs";
-import { createRepoBoundary } from "../repo-path.js";
-import { createPathClassifier, hasPathClass } from "../path-classification.js";
-import { listRepoFiles } from "./internal-audit.collect.js";
-import { stripNonCode, bodyEndLineCount, tokenize, fingerprints, extractLargeStrings } from "./duplicates.tokenize.js";
-
-const FLOOR_TOKENS = 12;   // absolute safety floor; normal scans still default to 30+ tokens
-const FLOOR_SIM = 0.5;     // pairs below this are not reported (UI slider min)
-const MAX_BODY_LINES = 400;
-// File types with no code SYMBOLS to fragment (stylesheets, markup, docs, single-file components). The
-// user still wants them clone-checked ("check the frontend / md too"), so each is sliced into fixed line
-// WINDOWS and each window is fingerprinted like a function body — a duplicated CSS rule block / HTML
-// section / doc passage then clones the same way.
-const WINDOW_EXTS = /\.(css|scss|sass|less|styl|html?|md|markdown|mdx|vue|svelte|astro)$/i;
-const WINDOW_LINES = 24;   // non-overlapping block size for windowed files
-// Asset files are DEDUP-ONLY — they are NOT put in the code graph (that would score md/css as fake
-// "methods" in Health and clutter the GUI board), so the clone scanner finds them by walking the repo.
-const MAX_ASSET_FILES = 4000;
-// Fingerprints shared by MORE than this many fragments are ubiquitous boilerplate and are excluded
-// from BOTH the shared count AND the union (so jaccard stays honest — see pairsForMode). Set well
-// above realistic clone multiplicity: N byte-identical copies each put all their fingerprints in
-// buckets of size N, so the cap must exceed N or the whole clone family is invisible. 120 covers all
-// but pathological mass-duplication; the O(n²) pair work per bucket stays bounded by it.
-const BUCKET_CAP = 120;
-// A clone must share at least this many DISTINCT (non-ubiquitous) fingerprints. Winnowing collapses a
-// UNIFORM repetitive body (a constant table where every line is `KEY: 'VALUE',` → renamed `I: <str>`) to a
-// handful of fingerprints, so two UNRELATED such tables hit 100% renamed Jaccard on ~3-6 shared fingerprints
-// — "same shape, zero shared content". Requiring real shared evidence drops those false positives while a
-// genuine clone (which shares dozens of fingerprints) is untouched.
-const MIN_SHARED_FP = 8;
-
-// Same-name symbols across DIFFERENT files — the semantic complement to token clones. A token clone
-// says "same body"; a name twin with LOW similarity says "same name, different behavior" — the
-// drift-hazard class of duplicate (three divergent uniqueStrings() implementations) that jaccard
-// ranks too low to surface. OOP-generic lifecycle names are stoplisted; interface implementations
-// still appear, so the caller's report keeps neutral wording.
-const TWIN_STOP = new Set(["constructor", "tostring", "valueof", "dispose", "close", "render", "setup", "teardown", "initialize", "destroy", "connect", "disconnect", "update", "reset", "clear", "create", "handle", "execute", "invoke", "main", "start", "stop", "build", "parse", "value", "index"]);
-function computeNameTwins(frags) {
-  const byName = new Map();
-  frags.forEach((f, i) => {
-    if (f.kind === "string") return;
-    const name = String(f.label || "").trim().replace(/\(\)$/, "");
-    if (name.length < 5 || !/^[A-Za-z_$][\w$]*$/.test(name) || TWIN_STOP.has(name.toLowerCase()) || /^do_(?:get|post|put|patch|delete|head|options)$/i.test(name)) return;
-    const key = name.toLowerCase();
-    let a = byName.get(key); if (!a) byName.set(key, (a = []));
-    a.push(i);
-  });
-  const jac = (A, B) => { let inter = 0; for (const h of A) if (B.has(h)) inter++; const u = A.size + B.size - inter; return u ? inter / u : 0; };
-  const out = [];
-  for (const idxs of byName.values()) {
-    const files = new Set(idxs.map((i) => frags[i].file));
-    if (files.size < 2 || idxs.length > 12) continue; // single-file overloads / framework-name explosions
-    let simMin = 1, simMax = 0;
-    const pairs = [];
-    for (let a = 0; a < idxs.length; a++) for (let b = a + 1; b < idxs.length; b++) {
-      if (frags[idxs[a]].file === frags[idxs[b]].file) continue;
-      const s = jac(frags[idxs[a]].fp.renamed, frags[idxs[b]].fp.renamed);
-      if (s < simMin) simMin = s;
-      if (s > simMax) simMax = s;
-      pairs.push({ a: idxs[a], b: idxs[b], similarity: Math.round(s * 100) });
-    }
-    out.push({
-      label: String(frags[idxs[0]].label || "").replace(/\(\)$/, ""), members: idxs, files: files.size,
-      simMin: Math.round(simMin * 100), simMax: Math.round(simMax * 100),
-      pairs,
-      tokens: idxs.reduce((n, i) => n + frags[i].n, 0),
-    });
-  }
-  return out.sort((x, y) => y.tokens - x.tokens).slice(0, 200);
-}
-
-function pairsForMode(frags, mode) {
-  const index = new Map();
-  frags.forEach((f, i) => { for (const h of f.fp[mode]) { let a = index.get(h); if (!a) index.set(h, (a = [])); a.push(i); } });
-  // eff[i] = count of i's fingerprints that sit in NON-ubiquitous buckets. Jaccard's numerator (shared)
-  // and denominator (union) are BOTH computed over this same restricted set, so excluding a boilerplate
-  // fingerprint never deflates similarity — two real clones sharing a widely-repeated preamble still
-  // pair at 100% on their unique bodies, and the sim is a true jaccard, not an under-count.
-  const eff = new Array(frags.length).fill(0);
-  const shared = new Map();
-  for (const arr of index.values()) {
-    if (arr.length > BUCKET_CAP) continue; // ubiquitous k-gram → excluded from count AND size (both sides)
-    for (const idx of arr) eff[idx]++;
-    for (let a = 0; a < arr.length; a++) for (let b = a + 1; b < arr.length; b++) {
-      const key = arr[a] * 1000000 + arr[b]; // frag count is bounded way below 1e6
-      shared.set(key, (shared.get(key) || 0) + 1);
-    }
-  }
-  const pairs = [];
-  for (const [key, count] of shared) {
-    const i = Math.floor(key / 1000000), j = key % 1000000;
-    const union = eff[i] + eff[j] - count;
-    const sim = union > 0 ? count / union : 0;
-    const smallFragment = Math.min(frags[i].n, frags[j].n) < 30;
-    const enoughEvidence = smallFragment
-      ? sim >= 0.95 && count >= 2
-      : count >= MIN_SHARED_FP;
-    if (sim >= FLOOR_SIM && enoughEvidence) pairs.push([i, j, Math.round(sim * 100)]);
-  }
-  return pairs;
-}
-
-// Parse "L<line>" graph symbol locations into per-file ordered symbol lists.
-function symbolRanges(graph) {
-  const byFile = new Map();
-  for (const n of graph.nodes || []) {
-    if (!n.source_file || !String(n.id || "").includes("#")) continue;
-    const line = Number((String(n.source_location || "").match(/L(\d+)/) || [])[1] || 0);
-    if (!line) continue;
-    let arr = byFile.get(n.source_file);
-    if (!arr) byFile.set(n.source_file, (arr = []));
-    arr.push({ id: n.id, label: n.label || n.id, line, symbolKind: n.symbol_kind || "", testSurface: n.test_surface === true });
-  }
-  for (const arr of byFile.values()) arr.sort((a, b) => a.line - b.line);
-  return byFile;
-}
-
-function loadGraph(graphInput) {
-  if (graphInput && typeof graphInput === "object" && !Array.isArray(graphInput)) {
-    return { nodes: Array.isArray(graphInput.nodes) ? graphInput.nodes : [] };
-  }
-  if (typeof graphInput !== "string" || !graphInput) throw new TypeError("graph path or graph object is required");
-  const parsed = JSON.parse(readFileSync(graphInput, "utf8"));
-  if (!parsed || typeof parsed !== "object" || Array.isArray(parsed)) throw new TypeError("graph must be an object");
-  return { nodes: Array.isArray(parsed.nodes) ? parsed.nodes : [] };
-}
-
-export function computeDuplicates(repoPath, graphJsonPath, opts = {}) {
-  const includeStrings = !!opts.includeStrings;
-  const scanTokenFloor = Math.max(FLOOR_TOKENS, Math.min(400, Number(opts.minTokens) || 30));
-  const graph = loadGraph(graphJsonPath);
-  const byFile = symbolRanges(graph);
-  // total symbol nodes in the graph — 0 means a file-only graph (built by an older builder before
-  // symbol extraction, or a stale graph): clone detection has nothing to work with, and the UI must
-  // say "rebuild the graph" instead of the misleading "no clones at these thresholds".
-  const repoFiles = listRepoFiles(repoPath);
-  const allowedFiles = new Set(repoFiles);
-  let graphSymbols = 0;
-  for (const [file, arr] of byFile) if (allowedFiles.has(file)) graphSymbols += arr.length;
-  const frags = [];
-  const boundary = createRepoBoundary(repoPath);
-  const classifier = createPathClassifier(repoPath);
-  const classificationByFile = new Map();
-  const classify = (file, content) => {
-    if (!classificationByFile.has(file)) classificationByFile.set(file, classifier.explain(file, { content }));
-    return classificationByFile.get(file);
-  };
-  const classificationFields = (info) => ({
-    test: hasPathClass(info, "test", "e2e"),
-    classes: info.classes,
-    excluded: info.excluded,
-    matchedRule: info.matchedRule,
-  });
-  for (const [file, syms] of byFile) {
-    if (!allowedFiles.has(file)) continue;
-    const resolved = boundary.resolve(file);
-    if (!resolved.ok) continue;
-    const full = resolved.path;
-    let lines;
-    try { lines = readFileSync(full, "utf8").split(/\r?\n/); } catch { continue; }
-    const py = /\.py$/i.test(file);
-    const pathInfo = classify(file, lines.join("\n"));
-    for (let i = 0; i < syms.length; i++) {
-      const start = syms[i].line;
-      // hard cap first (next symbol, or EOF, whichever comes first, ≤ MAX_BODY_LINES) …
-      const hardEnd = Math.min(i + 1 < syms.length ? syms[i + 1].line - 1 : lines.length, start + MAX_BODY_LINES);
-      if (hardEnd - start < 2) continue; // one-liners can't be meaningful clones
-      // … then shrink to where the construct actually closes, so the last symbol doesn't absorb
-      // trailing module-level code (module.exports tables, main() calls) up to EOF.
-      const bodyLines = lines.slice(start - 1, hardEnd);
-      const body = bodyLines.slice(0, bodyEndLineCount(bodyLines, py));
-      const end = start + body.length - 1;
-      if (end - start < 2) continue;
-      const toks = tokenize(stripNonCode(body.join("\n"), py));
-      if (toks.strict.length < scanTokenFloor) continue;
-      const strippedBody = stripNonCode(body.join("\n"), py);
-      const declarativeValue = /^(?:export\s+)?(?:const|let|var)\s+[\w$]+(?:\s*:[^=]+)?\s*=\s*(?:Object\.(?:freeze|seal)\s*\(\s*)?[\[{]/s.test(strippedBody.trim())
-        && !/=>|\b(?:function|if|for|while|switch|try|await|yield|return|throw|new)\b/.test(strippedBody);
-      const declarativeFactory = /^(?:export\s+)?const\s+[\w$]+(?:\s*:[^=]+)?\s*=\s*(?:sqliteTable|pgTable|mysqlTable)\s*\(/s.test(strippedBody.trim());
-      const declarative = declarativeValue || declarativeFactory || /^(?:interface|type|enum)$/i.test(syms[i].symbolKind);
-      frags.push({
-        id: syms[i].id, label: syms[i].label, file, start, end,
-        n: toks.strict.length, declarative,
-        ...classificationFields(pathInfo),
-        // Inline test surfaces (Rust `#[cfg(test)] mod tests`, `#[test]` fns) live in production files, so a
-        // path check alone misses them; the graph already flags the symbol. OR it in so skip-tests suppresses
-        // them by default and two #[test] fns are never reported as a production clone.
-        test: hasPathClass(pathInfo, "test", "e2e") || syms[i].testSurface === true,
-        fp: { strict: fingerprints(toks.strict), renamed: fingerprints(toks.renamed) },
-      });
-    }
-    // ---- opt-in: large multi-line string literals as their own fragments. The code pass above strips
-    // string bodies (correctly — content isn't code), which makes embedded DSL templates (inline
-    // C#/SQL/PowerShell) invisible to clone detection. Tokenized RAW: the content IS the payload.
-    if (includeStrings) {
-      const base = file.split("/").pop();
-      const pushStr = (startLine, endLine, content) => {
-        const toks = tokenize(content);
-        if (toks.strict.length < scanTokenFloor) return;
-        frags.push({
-          id: `${file}#str@${startLine}`, label: `${base}:${startLine} ~${endLine - startLine + 1}-line string`,
-          file, start: startLine, end: endLine, n: toks.strict.length, ...classificationFields(pathInfo), kind: "string",
-          fp: { strict: fingerprints(toks.strict), renamed: fingerprints(toks.renamed) },
-        });
-      };
-      for (const str of extractLargeStrings(lines.join("\n"), { py, cs: /\.cs$/i.test(file) })) {
-        const strLines = str.content.split(/\r?\n/).slice(0, MAX_BODY_LINES);
-        // A SECTION shared between two big templates dilutes below the similarity floor when the whole
-        // literals are compared (70 shared lines inside 220- and 107-line strings ≈ 27% jaccard). Big
-        // literals therefore get the same WINDOW treatment as CSS/HTML files; small ones stay whole.
-        if (strLines.length <= WINDOW_LINES * 2) {
-          pushStr(str.start, str.start + strLines.length - 1, strLines.join("\n"));
-        } else {
-          for (let off = 0; off < strLines.length; off += WINDOW_LINES) {
-            const chunk = strLines.slice(off, off + WINDOW_LINES);
-            if (chunk.length < 3) break;
-            pushStr(str.start + off, str.start + off + chunk.length - 1, chunk.join("\n"));
-          }
-        }
-      }
-    }
-  }
-  // ---- windowed fragments for symbol-less file types (CSS/HTML/MD/…), found by walking the repo (NOT the
-  // graph — assets are dedup-only). Each file is sliced into WINDOW_LINES blocks and fingerprinted.
-  const symFiles = new Set(byFile.keys());
-  const allAssetFiles = repoFiles.filter((file) => WINDOW_EXTS.test(file));
-  const assetFiles = allAssetFiles.slice(0, MAX_ASSET_FILES);
-  for (const file of assetFiles) {
-    if (symFiles.has(file)) continue;
-    const resolved = boundary.resolve(file);
-    if (!resolved.ok) continue;
-    let lines;
-    try { lines = readFileSync(resolved.path, "utf8").split(/\r?\n/); } catch { continue; }
-    const pathInfo = classify(file, lines.join("\n"));
-    for (let start = 1; start <= lines.length; start += WINDOW_LINES) {
-      const end = Math.min(start + WINDOW_LINES - 1, lines.length);
-      if (end - start < 2) continue;
-      const toks = tokenize(stripNonCode(lines.slice(start - 1, end).join("\n"), false));
-      if (toks.strict.length < scanTokenFloor) continue;
-      frags.push({
-        id: `${file}#win@${start}`, label: `${file.split("/").pop()}:${start}-${end}`, file, start, end,
-        n: toks.strict.length, ...classificationFields(pathInfo),
-        fp: { strict: fingerprints(toks.strict), renamed: fingerprints(toks.renamed) },
-      });
-    }
-  }
-  const modes = { strict: pairsForMode(frags, "strict"), renamed: pairsForMode(frags, "renamed") };
-  const nameTwins = opts.nameTwins ? computeNameTwins(frags) : null;
-  // fp sets are worker-internal — strip them from the payload that crosses the thread boundary
-  const slim = frags.map(({ fp, ...rest }) => rest);
-  return {
-    ok: true,
-    frags: slim,
-    modes,
-    ...(nameTwins ? { nameTwins } : {}),
-    graphSymbols,
-    completeness: {
-      assetFiles: {
-        total: allAssetFiles.length,
-        scanned: assetFiles.length,
-        truncated: allAssetFiles.length > assetFiles.length,
-      },
-      nameTwinsTruncated: !!nameTwins && nameTwins.length >= 200,
-    },
-    floors: { tokens: scanTokenFloor, absoluteTokens: FLOOR_TOKENS, sim: FLOOR_SIM * 100 },
-  };
-}

package/src/analysis/duplicates.js

@@ -1,15 +0,0 @@
-// duplicates.js — content-based clone detection over the repo's OWN graph.json symbols (the Health
-// tab's engine). MOSS-style pipeline: symbol bodies (line ranges from the graph) → strip comments &
-// string bodies → tokenize → k-gram rolling hashes → winnowing fingerprints → inverted index (no
-// O(n²) all-pairs) → jaccard similarity. BOTH normalization modes are computed in one pass so every
-// UI knob (similarity %, min size, strict/renamed) filters instantly on the renderer side:
-//   strict  — identifiers kept: only literal copy-paste (Type-1 clones)
-//   renamed — identifiers canonicalized to "I": catches copy-paste-then-rename (Type-2 clones)
-// Pairs are reported down to the FLOOR values; the renderer slices from there upward.
-//
-// Facade: the implementation lives in the sibling modules —
-//   duplicates.tokenize.js — stripping, body-end detection, tokenization, winnowing fingerprints
-//   duplicates.compute.js  — fragment extraction, inverted-index pairing, computeDuplicates
-//   duplicates.run.js      — worker offload + mtime cache (runDuplicates entry)
-export { computeDuplicates } from "./duplicates.compute.js";
-export { runDuplicates } from "./duplicates.run.js";

package/src/analysis/duplicates.run.js

@@ -1,52 +0,0 @@
-// duplicates.run.js — the repos:duplicates entry point (split from duplicates.js): worker-thread
-// offload with in-process fallback, cached per (repo, graph.json mtime).
-import { existsSync, statSync } from "node:fs";
-import { join } from "node:path";
-import { Worker } from "node:worker_threads";
-import { childProcessEnv } from "../child-env.js";
-import { graphOutDirForRepo } from "../graph/layout.js";
-import { computeDuplicates } from "./duplicates.compute.js";
-
-function computeInWorker(repoPath, graphJsonPath) {
-  return new Promise((resolve, reject) => {
-    let worker;
-    try {
-      worker = new Worker(new URL("./duplicates-worker.js", import.meta.url), { workerData: { repoPath, graphJsonPath }, env: childProcessEnv() });
-    } catch (e) {
-      reject(Object.assign(e, { workerStartFailed: true }));
-      return;
-    }
-    let settled = false;
-    const done = (fn, v) => { if (!settled) { settled = true; fn(v); } };
-    worker.once("message", (msg) => done(resolve, msg));
-    worker.once("error", (e) => done(reject, Object.assign(e, { workerStartFailed: true })));
-    worker.once("exit", (code) => { if (code !== 0) done(reject, Object.assign(new Error(`duplicates worker exited with code ${code}`), { workerStartFailed: true })); });
-  });
-}
-
-// repos:duplicates entry — cached per (repo, graph.json mtime): re-running with an unchanged graph is
-// free. `force` (the UI's ↻ rescan) bypasses the cache: fragment BODIES are read from live source, so a
-// source edit that didn't rebuild the graph (same mtime) must still be re-scanned on demand.
-const _cache = new Map();
-export async function runDuplicates(repoPath, force = false) {
-  const repo = String(repoPath || "");
-  if (!repo || !existsSync(repo)) return { ok: false, error: "Repo path not found" };
-  const graphJsonPath = join(graphOutDirForRepo(repo), "graph.json");
-  if (!existsSync(graphJsonPath)) {
-    return { ok: false, needsGraph: true, error: "No graph yet — build the Relations graph first (↻ on the Relations tab)" };
-  }
-  let mtime = 0;
-  try { mtime = statSync(graphJsonPath).mtimeMs; } catch { /* treat as uncached */ }
-  const cached = _cache.get(repo);
-  if (!force && cached && cached.mtime === mtime) return cached.result;
-  let result;
-  try {
-    result = await computeInWorker(repo, graphJsonPath);
-  } catch (e) {
-    if (!e || !e.workerStartFailed) return { ok: false, error: e.message || String(e) };
-    try { result = computeDuplicates(repo, graphJsonPath); } // in-process fallback (exotic packaging)
-    catch (e2) { return { ok: false, error: e2.message || String(e2) }; }
-  }
-  if (result && result.ok) _cache.set(repo, { mtime, result });
-  return result;
-}

package/src/analysis/duplicates.tokenize.js

@@ -1,199 +0,0 @@
-// duplicates.tokenize.js — lexical layer of the clone detector (split from duplicates.js): comment &
-// string stripping, body-end detection, strict/renamed tokenization, and winnowing fingerprints.
-
-const K = 8;               // k-gram length (tokens)
-const W = 4;               // winnowing window → guaranteed detection of matches ≥ K+W-1 tokens
-const STRLIT = String.fromCharCode(0); // opaque-literal sentinel for string bodies: a NUL
-                           // its own class (never an identifier "I" nor a number "N"), so a literal can
-                           // never masquerade as code in either mode
-const REGEXLIT = String.fromCharCode(1); // regex bodies are executable behavior, so retain an equality anchor
-const literalHash = (value) => {
-  let hash = 2166136261;
-  for (const char of String(value || "")) hash = Math.imul(hash ^ char.charCodeAt(0), 16777619);
-  return (hash >>> 0).toString(16);
-};
-const regexMarker = (value) => ` ${REGEXLIT} 0x${literalHash(value)} `;
-
-const KEYWORDS = new Set(("if else for while do switch case break continue return function const let var new class extends async await try catch finally throw import from export default typeof instanceof in of delete void yield static get set this super null undefined true false def elif except lambda pass raise with as is not and or None True False func go defer chan map range struct interface type package nil err string int bool byte float64 public private protected final void long double boolean").split(" "));
-
-// A leading `/` is a REGEX literal (not division) when the previous significant char is one of these
-// or the fragment start; after a word char / ) / ] it is division. Covers .match(/…/), = /…/, (/…/),
-// and arrow `x => /…/`. NOTE: `+` and `-` are deliberately EXCLUDED — a regex never legitimately
-// follows them, but `i++ / 2` and `a - b / c` do, and including them mis-scanned the division as a
-// regex and ate the rest of the line.
-const REGEX_PREV = new Set("(,=:[!&|?{};~*%^<>".split(""));
-
-// Replace comments with nothing and string bodies with an opaque sentinel. Regex bodies retain only
-// a one-way equality anchor: their source is executable behavior, and collapsing every pattern to the
-// same token creates false clone groups for unrelated replace/validate pipelines.
-export function stripNonCode(text, py) {
-  let out = "", i = 0, prevSig = "";
-  const s = String(text || "");
-  const push = (c) => { out += c; const t = c.trim(); if (t) prevSig = t[t.length - 1]; };
-  while (i < s.length) {
-    const ch = s[i], two = s.slice(i, i + 2), three = s.slice(i, i + 3);
-    if (py && ch === "#") { while (i < s.length && s[i] !== "\n") i++; continue; }
-    if (py && (three === "'''" || three === '"""')) {
-      i += 3;
-      while (i < s.length && s.slice(i, i + 3) !== three) i++;
-      i += 3; push(` ${STRLIT} `); continue;
-    }
-    if (!py && two === "//") { while (i < s.length && s[i] !== "\n") i++; continue; }
-    if (!py && two === "/*") { i += 2; while (i < s.length && s.slice(i, i + 2) !== "*/") i++; i += 2; continue; }
-    if (!py && ch === "/" && REGEX_PREV.has(prevSig)) {
-      i++; const regexStart = i; let inClass = false; // '/' inside a [...] char class is literal, not the terminator
-      while (i < s.length && s[i] !== "\n") {
-        const c = s[i];
-        if (c === "\\") { i += 2; continue; }
-        if (c === "[") inClass = true;
-        else if (c === "]") inClass = false;
-        else if (c === "/" && !inClass) { i++; break; }
-        i++;
-      }
-      while (i < s.length && /[a-z]/i.test(s[i])) i++; // regex flags (gimsuy)
-      push(regexMarker(s.slice(regexStart, i))); continue;
-    }
-    if (ch === '"' || ch === "'" || ch === "`") {
-      const q = ch; i++;
-      while (i < s.length && s[i] !== q) { if (s[i] === "\\") i++; i++; }
-      i++; push(` ${STRLIT} `); continue;
-    }
-    push(ch); i++;
-  }
-  return out;
-}
-
-// Where a symbol's body ends — so the LAST symbol in a file doesn't swallow trailing module-level code
-// (module.exports tables, main() calls, handler registrations) up to EOF and dilute/misattribute
-// clones. Brace/bracket depth for brace langs (body ends when {}()[] balance after first opening);
-// indentation for Python. Returns a 1-based line count within bodyLines (≤ its length; = its length
-// when the construct never closes, so the caller's cap stands).
-export function bodyEndLineCount(bodyLines, py) {
-  if (py) {
-    const base = bodyLines[0].match(/^[ \t]*/)[0].length;
-    let last = 1;
-    for (let k = 1; k < bodyLines.length; k++) {
-      const ln = bodyLines[k];
-      if (!ln.trim()) continue; // blank lines never end a block
-      if (ln.match(/^[ \t]*/)[0].length <= base) return last; // dedent to ≤ the def → body ended above
-      last = k + 1;
-    }
-    return bodyLines.length;
-  }
-  const text = bodyLines.join("\n");
-  let depth = 0, opened = false, i = 0, line = 1, prevSig = "";
-  while (i < text.length) {
-    const ch = text[i], two = text.slice(i, i + 2);
-    if (ch === "\n") { line++; i++; continue; }
-    if (two === "//") { while (i < text.length && text[i] !== "\n") i++; continue; }
-    if (two === "/*") { i += 2; while (i < text.length && text.slice(i, i + 2) !== "*/") { if (text[i] === "\n") line++; i++; } i += 2; continue; }
-    if (ch === "/" && REGEX_PREV.has(prevSig)) {
-      i++; let inClass = false;
-      while (i < text.length && text[i] !== "\n") { const c = text[i]; if (c === "\\") { i += 2; continue; } if (c === "[") inClass = true; else if (c === "]") inClass = false; else if (c === "/" && !inClass) { i++; break; } i++; }
-      prevSig = "/"; continue;
-    }
-    if (ch === '"' || ch === "'" || ch === "`") { const q = ch; i++; while (i < text.length && text[i] !== q) { if (text[i] === "\\") i++; else if (text[i] === "\n") line++; i++; } i++; prevSig = q; continue; }
-    // depth tracks all brackets, but only a { or [ opens the BODY — a bare (param list) returning to
-    // depth 0 must not be read as the construct closing on line 1
-    if (ch === "{" || ch === "(" || ch === "[") { depth++; if (ch === "{" || ch === "[") opened = true; }
-    else if (ch === "}" || ch === ")" || ch === "]") { depth--; if (opened && depth <= 0) return line; }
-    if (ch.trim()) prevSig = ch;
-    i++;
-  }
-  return bodyLines.length;
-}
-
-// Large multi-line string literals — the OPPOSITE selection to stripNonCode: everything the code
-// scan throws away. Embedded DSLs (inline C#/SQL/PowerShell templates) are invisible to the normal
-// clone pass because their bodies are stripped; this extractor feeds them back in as their own
-// fragments when the caller opts in (find_duplicates include_strings). Comment/regex handling
-// mirrors stripNonCode so a quote inside a comment never opens a phantom literal.
-export function extractLargeStrings(text, { py = false, cs = false, minLines = 6 } = {}) {
-  const s = String(text || "");
-  const out = [];
-  let i = 0, line = 1, prevSig = "";
-  const capture = (from, to, startLine) => {
-    const content = s.slice(from, to);
-    const nl = (content.match(/\n/g) || []).length;
-    if (nl + 1 >= minLines) out.push({ start: startLine, end: startLine + nl, content });
-  };
-  while (i < s.length) {
-    const ch = s[i], two = s.slice(i, i + 2), three = s.slice(i, i + 3);
-    if (ch === "\n") { line++; i++; continue; }
-    if (py && ch === "#") { while (i < s.length && s[i] !== "\n") i++; continue; }
-    if (py && (three === "'''" || three === '"""')) {
-      const startLine = line; i += 3; const from = i;
-      while (i < s.length && s.slice(i, i + 3) !== three) { if (s[i] === "\n") line++; i++; }
-      capture(from, i, startLine); i += 3; continue;
-    }
-    if (!py && two === "//") { while (i < s.length && s[i] !== "\n") i++; continue; }
-    if (!py && two === "/*") { i += 2; while (i < s.length && s.slice(i, i + 2) !== "*/") { if (s[i] === "\n") line++; i++; } i += 2; continue; }
-    if (!py && ch === "/" && REGEX_PREV.has(prevSig)) {
-      i++; let inClass = false;
-      while (i < s.length && s[i] !== "\n") { const c = s[i]; if (c === "\\") { i += 2; continue; } if (c === "[") inClass = true; else if (c === "]") inClass = false; else if (c === "/" && !inClass) { i++; break; } i++; }
-      prevSig = "/"; continue;
-    }
-    if (cs && two === '@"') { // C# verbatim string; "" is the escaped quote
-      const startLine = line; i += 2; const from = i;
-      while (i < s.length) { if (s.slice(i, i + 2) === '""') { i += 2; continue; } if (s[i] === '"') break; if (s[i] === "\n") line++; i++; }
-      capture(from, i, startLine); i++; continue;
-    }
-    if (!py && ch === "`") { // JS/TS template literal — the main multi-line carrier
-      const startLine = line; i++; const from = i;
-      while (i < s.length && s[i] !== "`") { if (s[i] === "\\") { i++; if (s[i] === "\n") line++; i++; continue; } if (s[i] === "\n") line++; i++; }
-      capture(from, i, startLine); i++; continue;
-    }
-    if (ch === '"' || ch === "'") { // single-line strings can't span minLines — just skip past
-      const q = ch; i++;
-      while (i < s.length && s[i] !== q && s[i] !== "\n") { if (s[i] === "\\") i++; i++; }
-      if (s[i] === q) i++; continue;
-    }
-    if (ch.trim()) prevSig = ch;
-    i++;
-  }
-  return out;
-}
-
-// one raw token stream; the two modes differ only in identifier canonicalization
-export function tokenize(text) {
-  const raw = text.match(/[A-Za-z_$][\w$]*|(?:0[xX][\dA-Fa-f_]+|0[bB][01_]+|0[oO][0-7_]+|\d[\d_]*(?:\.\d[\d_]*)?(?:[eE][+-]?\d[\d_]*)?)n?|[^\s\w]/g) || [];
-  const strict = [];
-  const renamed = [];
-  for (const t of raw) {
-    if (/^\d/.test(t)) {
-      // Numeric constants are behavior, not renameable identifiers. Preserve
-      // their value so same-shape limit/config tables with different numbers do
-      // not become perfect clones; a small constant edit still leaves the rest
-      // of a genuine copied body highly similar.
-      strict.push(`N:${t}`); renamed.push(`N:${t}`); continue;
-    }
-    if (/^[A-Za-z_$]/.test(t) && !KEYWORDS.has(t)) { strict.push(t); renamed.push("I"); continue; }
-    strict.push(t); renamed.push(t);
-  }
-  return { strict, renamed };
-}
-
-export function fingerprints(toks) {
-  if (toks.length < K + W - 1) return new Set();
-  const hashes = [];
-  for (let i = 0; i + K <= toks.length; i++) {
-    let h = 0;
-    for (let j = i; j < i + K; j++) {
-      const s = toks[j];
-      for (let c = 0; c < s.length; c++) h = (h * 31 + s.charCodeAt(c)) | 0;
-      h = (h * 131) | 0;
-    }
-    hashes.push(h >>> 0);
-  }
-  const fp = new Set();
-  for (let i = 0; i + W <= hashes.length + 1; i++) {
-    let min = Infinity;
-    for (let j = i; j < Math.min(i + W, hashes.length); j++) if (hashes[j] < min) min = hashes[j];
-    fp.add(min);
-  }
-  // Winnowing can skip every changed literal in a repetitive policy table.
-  // Exact numeric anchors keep behavior-defining limits in the similarity
-  // denominator while leaving ordinary copied bodies tolerant of a small edit.
-  for (const token of toks) if (token.startsWith("N:")) fp.add(`numeric:${token}`);
-  return fp;
-}

package/src/analysis/endpoints/common.js

@@ -1,79 +0,0 @@
-export const MAX_ENDPOINT_FILES = 3_000;
-export const MAX_ENDPOINTS = 2_000;
-export const HTTP_METHODS = new Set(["GET", "POST", "PUT", "PATCH", "DELETE", "HEAD", "OPTIONS", "TRACE", "CONNECT", "ALL", "ANY"]);
-
-export function lineAt(text, index) {
-  let line = 1;
-  for (let i = 0; i < index && i < text.length; i++) if (text[i] === "\n") line++;
-  return line;
-}
-
-// Preserve offsets and literals while hiding comments from regex extractors.
-export function maskComments(text, { hashComments = false } = {}) {
-  const chars = String(text || "").split("");
-  let quote = "", escaped = false, lineComment = false, blockComment = false, regex = false, regexClass = false;
-  const startsRegex = (index) => {
-    let cursor = index - 1;
-    while (cursor >= 0 && /\s/.test(chars[cursor])) cursor--;
-    if (cursor < 0) return true;
-    if (/[=(:,!&|?{}[\];]/.test(chars[cursor])) return true;
-    const prefix = chars.slice(Math.max(0, cursor - 12), cursor + 1).join("");
-    return /(?:^|\b)(?:return|throw|case|yield|await)\s*$/i.test(prefix) || /=>\s*$/.test(prefix);
-  };
-  for (let i = 0; i < chars.length; i++) {
-    const ch = chars[i], next = chars[i + 1];
-    if (lineComment) {
-      if (ch === "\n" || ch === "\r") lineComment = false;
-      else chars[i] = " ";
-      continue;
-    }
-    if (blockComment) {
-      if (ch === "*" && next === "/") { chars[i] = chars[i + 1] = " "; i++; blockComment = false; }
-      else if (ch !== "\n" && ch !== "\r") chars[i] = " ";
-      continue;
-    }
-    if (regex) {
-      if (escaped) escaped = false;
-      else if (ch === "\\") escaped = true;
-      else if (ch === "[") regexClass = true;
-      else if (ch === "]") regexClass = false;
-      else if (ch === "/" && !regexClass) regex = false;
-      continue;
-    }
-    if (quote) {
-      if (escaped) escaped = false;
-      else if (ch === "\\") escaped = true;
-      else if (ch === quote) quote = "";
-      continue;
-    }
-    if (ch === '"' || ch === "'" || ch === "`") { quote = ch; continue; }
-    if (ch === "/" && next === "/") { chars[i] = chars[i + 1] = " "; i++; lineComment = true; continue; }
-    if (ch === "/" && next === "*") { chars[i] = chars[i + 1] = " "; i++; blockComment = true; continue; }
-    if (!hashComments && ch === "/" && startsRegex(i)) { regex = true; regexClass = false; escaped = false; continue; }
-    if (hashComments && ch === "#") { chars[i] = " "; lineComment = true; }
-  }
-  return chars.join("");
-}
-
-export function handlerName(expr) {
-  const source = String(expr || "").trim();
-  if (!source || /=>/.test(source) || /^\s*(async\s+)?function\b/.test(source) || /^\s*(?:async\s+)?(?:move\s+)?\|[^|]*\|/.test(source)) return "";
-  const turbofish = /(?:^|::)([A-Za-z_][\w]*)\s*::<[\s\S]*>\s*$/.exec(source);
-  if (turbofish) return turbofish[1];
-  const identifiers = source.match(/[A-Za-z_$][\w$]*/g);
-  if (!identifiers) return "";
-  const skipped = new Set(["async", "function", "await", "req", "res", "ctx", "request", "response", "next", "return"]);
-  for (let i = identifiers.length - 1; i >= 0; i--) if (!skipped.has(identifiers[i])) return identifiers[i];
-  return "";
-}
-
-export function handlerReference(expr) {
-  const source = String(expr || "").trim();
-  if (!source || /=>/.test(source) || /^\s*(async\s+)?function\b/.test(source)) return "";
-  const refs = [...source.matchAll(/([A-Za-z_$][\w$]*(?:\s*\.\s*[A-Za-z_$][\w$]*)+)/g)];
-  return refs.length ? refs.at(-1)[1].replace(/\s+/g, "") : "";
-}
-
-export const looksLikePath = (path) => typeof path === "string" && /^\/[\w\-./:{}*$?]*$/.test(path) && !path.includes("://");
-export const cleanPath = (path) => String(path || "").replace(/\/+$/, "") || "/";
-export const normalizedFile = (file) => String(file || "").replace(/\\/g, "/").replace(/^\.\//, "");