weavatrix 0.1.0 → 0.1.2

package/src/security/malware-heuristics.scan.js

@@ -1,87 +1,87 @@
-// scanMalware orchestration (split out of malware-heuristics.js): phase A install scripts +
-// lockfile origins, phase B content sweep, path-aware file heuristics, then scoring.
-import { readFileSync, existsSync } from "node:fs";
-import { join } from "node:path";
-import { resolveRg } from "../scan/search.js";
-import { classifyInstallScript, classifyResolvedUrl, isCloudSdkMetadataUse } from "./registry-sig.js";
-import { fileHeuristicSweep } from "./malware-file-heuristics.js";
-import { buildMalwareFindings } from "./malware-scoring.js";
-import { parseMalwareExclusions, isManifestUrlNoise, isDocUrlNoise, isMalwareSignalExcluded, lifecycleScriptSnippet } from "./malware-heuristics.exclusions.js";
-import { buildContentRoots, listPkgDirs } from "./malware-heuristics.roots.js";
-import { rgSweep, nodeSweep } from "./malware-heuristics.sweep.js";
-
-// → { findings, scanMode } ; importedPkgs = Set of packages the repo's own code imports (evidence note).
-export async function scanMalware(repoPath, { installed = [], importedPkgs = new Set(), rgPath = "", forceNode = false, timeoutMs = 60000, malwareExclusions = {} } = {}) {
-  const nmDir = join(repoPath, "node_modules");
-  const contentRoots = buildContentRoots(repoPath, installed);
-  const byPkg = new Map(); // pkg -> [{rule signal, file, line, snippet}]
-  const exclusions = parseMalwareExclusions(malwareExclusions);
-  let excludedSignals = 0;
-  const add = (pkg, sig, file = "", line = 0, snippet = "") => {
-    if (!pkg) return;
-    const signal = { ...sig, file, line, snippet: snippet || sig.snippet || "" };
-    if (isManifestUrlNoise(file, signal) || isDocUrlNoise(file, signal) || isCloudSdkMetadataUse(pkg, signal.key) || isMalwareSignalExcluded(pkg, signal, exclusions)) {
-      excludedSignals++;
-      return;
-    }
-    (byPkg.get(pkg) || byPkg.set(pkg, []).get(pkg)).push(signal);
-  };
-
-  // ---- phase A: install scripts (from node_modules package.jsons) + lockfile resolved origins ----
-  const pkgDirs = existsSync(nmDir) ? listPkgDirs(nmDir) : [];
-  const lockScriptMeta = new Map(
-    (installed || [])
-      .filter((p) => p.ecosystem === "npm" && p.source === "package-lock" && p.hasInstallScript !== undefined)
-      .map((p) => [`${p.name}@${p.version}`, !!p.hasInstallScript])
-  );
-  for (const { pkg, dir } of pkgDirs) {
-    try {
-      const pj = JSON.parse(readFileSync(join(dir, "package.json"), "utf8"));
-      const scriptSnippet = lifecycleScriptSnippet(pj.scripts);
-      if (scriptSnippet && lockScriptMeta.get(`${pkg}@${pj.version || ""}`) === false) {
-        add(pkg, {
-          key: "install-script-drift",
-          severity: "high",
-          nearZeroFp: false,
-          what: "installed package has lifecycle scripts absent from package-lock metadata",
-          snippet: scriptSnippet,
-        }, join(dir, "package.json"));
-      }
-      for (const sig of classifyInstallScript(pj.scripts)) add(pkg, sig, join(dir, "package.json"));
-    } catch { /* not a package */ }
-  }
-  for (const p of installed) {
-    const sig = p.resolved ? classifyResolvedUrl(p.resolved) : null;
-    if (sig) add(p.name, sig, "package-lock.json");
-  }
-
-  // ---- phase B: content sweep ----
-  let scanMode = "none";
-  if (contentRoots.length) {
-    const rg = forceNode ? null : await resolveRg("auto", rgPath);
-    let swept = false;
-    if (rg) {
-      const { hits, errored, total } = await rgSweep(rg, contentRoots, timeoutMs);
-      if (errored < total) {
-        // honest label: full when every rule ran, partial when some rules failed (don't claim
-        // full coverage we didn't get); every-rule-failed falls through to the Node sweep below.
-        const kinds = [...new Set(contentRoots.map((r) => r.kind))].join("+");
-        scanMode = errored ? `ripgrep-partial (${total - errored}/${total} sweeps; ${kinds})` : `ripgrep-full (${kinds})`;
-        swept = true;
-        for (const h of hits) add(h.pkg, h.rule, h.file, h.line, h.text);
-      }
-    }
-    if (!swept) {
-      scanMode = rg ? "node-limited (ripgrep failed)" : "node-limited";
-      for (const h of nodeSweep(nmDir, pkgDirs)) add(h.pkg, h.rule, h.file, h.line, h.text);
-    }
-  } else {
-    scanMode = "skipped";
-  }
-
-  // ---- path-aware cheap checks: things that need file names or magic bytes, not just regex text.
-  // These run regardless of rg availability and stay bounded; content regex still does the broad sweep.
-  for (const h of fileHeuristicSweep(contentRoots, pkgDirs)) add(h.pkg, h.rule, h.file, h.line, h.text);
-
+// scanMalware orchestration (split out of malware-heuristics.js): phase A install scripts +
+// lockfile origins, phase B content sweep, path-aware file heuristics, then scoring.
+import { readFileSync, existsSync } from "node:fs";
+import { join } from "node:path";
+import { resolveRg } from "../scan/search.js";
+import { classifyInstallScript, classifyResolvedUrl, isCloudSdkMetadataUse } from "./registry-sig.js";
+import { fileHeuristicSweep } from "./malware-file-heuristics.js";
+import { buildMalwareFindings } from "./malware-scoring.js";
+import { parseMalwareExclusions, isManifestUrlNoise, isDocUrlNoise, isMalwareSignalExcluded, lifecycleScriptSnippet } from "./malware-heuristics.exclusions.js";
+import { buildContentRoots, listPkgDirs } from "./malware-heuristics.roots.js";
+import { rgSweep, nodeSweep } from "./malware-heuristics.sweep.js";
+
+// → { findings, scanMode } ; importedPkgs = Set of packages the repo's own code imports (evidence note).
+export async function scanMalware(repoPath, { installed = [], importedPkgs = new Set(), rgPath = "", forceNode = false, timeoutMs = 60000, malwareExclusions = {} } = {}) {
+  const nmDir = join(repoPath, "node_modules");
+  const contentRoots = buildContentRoots(repoPath, installed);
+  const byPkg = new Map(); // pkg -> [{rule signal, file, line, snippet}]
+  const exclusions = parseMalwareExclusions(malwareExclusions);
+  let excludedSignals = 0;
+  const add = (pkg, sig, file = "", line = 0, snippet = "") => {
+    if (!pkg) return;
+    const signal = { ...sig, file, line, snippet: snippet || sig.snippet || "" };
+    if (isManifestUrlNoise(file, signal) || isDocUrlNoise(file, signal) || isCloudSdkMetadataUse(pkg, signal.key) || isMalwareSignalExcluded(pkg, signal, exclusions)) {
+      excludedSignals++;
+      return;
+    }
+    (byPkg.get(pkg) || byPkg.set(pkg, []).get(pkg)).push(signal);
+  };
+
+  // ---- phase A: install scripts (from node_modules package.jsons) + lockfile resolved origins ----
+  const pkgDirs = existsSync(nmDir) ? listPkgDirs(nmDir) : [];
+  const lockScriptMeta = new Map(
+    (installed || [])
+      .filter((p) => p.ecosystem === "npm" && p.source === "package-lock" && p.hasInstallScript !== undefined)
+      .map((p) => [`${p.name}@${p.version}`, !!p.hasInstallScript])
+  );
+  for (const { pkg, dir } of pkgDirs) {
+    try {
+      const pj = JSON.parse(readFileSync(join(dir, "package.json"), "utf8"));
+      const scriptSnippet = lifecycleScriptSnippet(pj.scripts);
+      if (scriptSnippet && lockScriptMeta.get(`${pkg}@${pj.version || ""}`) === false) {
+        add(pkg, {
+          key: "install-script-drift",
+          severity: "high",
+          nearZeroFp: false,
+          what: "installed package has lifecycle scripts absent from package-lock metadata",
+          snippet: scriptSnippet,
+        }, join(dir, "package.json"));
+      }
+      for (const sig of classifyInstallScript(pj.scripts)) add(pkg, sig, join(dir, "package.json"));
+    } catch { /* not a package */ }
+  }
+  for (const p of installed) {
+    const sig = p.resolved ? classifyResolvedUrl(p.resolved) : null;
+    if (sig) add(p.name, sig, "package-lock.json");
+  }
+
+  // ---- phase B: content sweep ----
+  let scanMode = "none";
+  if (contentRoots.length) {
+    const rg = forceNode ? null : await resolveRg("auto", rgPath);
+    let swept = false;
+    if (rg) {
+      const { hits, errored, total } = await rgSweep(rg, contentRoots, timeoutMs);
+      if (errored < total) {
+        // honest label: full when every rule ran, partial when some rules failed (don't claim
+        // full coverage we didn't get); every-rule-failed falls through to the Node sweep below.
+        const kinds = [...new Set(contentRoots.map((r) => r.kind))].join("+");
+        scanMode = errored ? `ripgrep-partial (${total - errored}/${total} sweeps; ${kinds})` : `ripgrep-full (${kinds})`;
+        swept = true;
+        for (const h of hits) add(h.pkg, h.rule, h.file, h.line, h.text);
+      }
+    }
+    if (!swept) {
+      scanMode = rg ? "node-limited (ripgrep failed)" : "node-limited";
+      for (const h of nodeSweep(nmDir, pkgDirs)) add(h.pkg, h.rule, h.file, h.line, h.text);
+    }
+  } else {
+    scanMode = "skipped";
+  }
+
+  // ---- path-aware cheap checks: things that need file names or magic bytes, not just regex text.
+  // These run regardless of rg availability and stay bounded; content regex still does the broad sweep.
+  for (const h of fileHeuristicSweep(contentRoots, pkgDirs)) add(h.pkg, h.rule, h.file, h.line, h.text);
+
   return buildMalwareFindings({ byPkg, importedPkgs, contentRoots, scanMode, excludedSignals });
 }

package/src/security/malware-heuristics.sweep.js

@@ -1,122 +1,122 @@
-// Content sweeps for the malware scanner (split out of malware-heuristics.js): a ripgrep sweep
-// over all content roots when rg is available, else a bounded Node fallback over entry files.
-import { readFileSync, readdirSync, existsSync, statSync } from "node:fs";
-import { join } from "node:path";
-import { runCommand } from "../process.js";
-import { CONTENT_RULES, classifyContent } from "./registry-sig.js";
-import { normPath } from "./malware-heuristics.exclusions.js";
-
-const SKIP_GLOBS = [
-  "!**/*.map", "!**/*.md", "!**/*.mdx", "!**/*.markdown", "!**/*.mdown", "!**/*.mkd", "!**/*.rst", "!**/*.d.ts",
-  "!**/doc/**", "!**/docs/**", "!**/documentation/**",
-  "!**/test/**", "!**/tests/**", "!**/__tests__/**", "!**/fixtures/**", "!**/example/**", "!**/examples/**"
-];
-
-function globalRuleRegex(rule) {
-  const flags = rule.re.flags.includes("g") ? rule.re.flags : `${rule.re.flags}g`;
-  return new RegExp(rule.re.source, flags);
-}
-
-function matchRuleOffsets(rule, lineText, subText = "") {
-  const text = String(lineText || "");
-  if (text) {
-    const offsets = [];
-    const re = globalRuleRegex(rule);
-    let guard = 0;
-    let m;
-    while ((m = re.exec(text)) && guard++ < 24) {
-      offsets.push(m.index);
-      if (!m[0].length) re.lastIndex++;
-    }
-    if (!offsets.length && rule.key === "obfuscated-code") {
-      const at = subText ? text.indexOf(subText) : 0;
-      if (at >= 0) offsets.push(at);
-    }
-    return offsets;
-  }
-  if (subText && (rule.key === "obfuscated-code" || rule.re.test(subText))) return [0];
-  return [];
-}
-
-function rootOfFile(file, roots) {
-  const f = normPath(file).toLowerCase();
-  return roots.find((r) => {
-    const root = normPath(r.root).toLowerCase();
-    return f === root || f.startsWith(`${root}/`);
-  });
-}
-
-function chunks(list, size) {
-  const out = [];
-  for (let i = 0; i < list.length; i += size) out.push(list.slice(i, i + size));
-  return out;
-}
-
-export async function rgSweep(rg, contentRoots, timeoutMs) {
-  const hits = []; // {pkg, file, line, text, rule}
-  let errored = 0; // a failed/broken rule must not masquerade as "scanned, 0 hits"
-  let total = 0;
-  for (const rule of CONTENT_RULES) {
-    for (const group of chunks(contentRoots, 40)) {
-      total++;
-      const args = ["--json", "-i", "--no-ignore", "--hidden", "--follow", "--max-filesize", "2M", "--max-count", "20", ...SKIP_GLOBS.flatMap((g) => ["-g", g]), "-e", rule.pattern, "--", ...group.map((r) => r.root)];
-      let r;
-      try { r = await runCommand(rg, args, { timeoutMs }); } catch { errored++; continue; }
-      if (r.exitCode === 2 && !String(r.stdout || "").trim()) { errored++; continue; }
-      for (const line of String(r.stdout || "").split(/\r?\n/)) {
-        if (!line) continue;
-        let obj; try { obj = JSON.parse(line); } catch { continue; }
-        if (obj.type !== "match") continue;
-        const file = obj.data?.path?.text || "";
-        const root = rootOfFile(file, group);
-        const pkg = root?.pathToPkg(file) || "";
-        if (!pkg) continue;
-        const lineText = String(obj.data?.lines?.text || "");
-        const subText = String(obj.data?.submatches?.[0]?.match?.text || "");
-        for (const at of matchRuleOffsets(rule, lineText, subText)) {
-          const text = (lineText ? lineText.slice(Math.max(0, at - 60), at + 180) : subText).trim().slice(0, 240);
-          hits.push({ pkg, file, line: obj.data?.line_number || 0, text, rule });
-        }
-      }
-    }
-  }
-  return { hits, errored, total };
-}
-
-// no-rg fallback: each package's package.json main/bin entries + root JS files, bounded
-export function nodeSweep(nmDir, pkgDirs, capFiles = 3000) {
-  const hits = [];
-  let seenFiles = 0;
-  for (const { pkg, dir } of pkgDirs) {
-    if (seenFiles >= capFiles) break;
-    let pj = null;
-    try { pj = JSON.parse(readFileSync(join(dir, "package.json"), "utf8")); } catch { continue; }
-    const cands = new Set();
-    for (const f of [pj.main, pj.module, pj.browser]) if (typeof f === "string") cands.add(f);
-    if (pj.bin) for (const b of typeof pj.bin === "string" ? [pj.bin] : Object.values(pj.bin)) cands.add(b);
-    let rootJs = [];
-    try { rootJs = readdirSync(dir).filter((f) => /\.(c|m)?js$/.test(f)).slice(0, 6); } catch { /* gone */ }
-    for (const f of rootJs) cands.add(f);
-    if (!cands.size) cands.add("index.js");
-    for (const rel of cands) {
-      if (seenFiles >= capFiles) break;
-      const full = join(dir, String(rel));
-      try {
-        if (!existsSync(full) || statSync(full).size > 2_000_000) continue;
-        seenFiles++;
-        const text = readFileSync(full, "utf8");
-        for (const rule of classifyContent(text)) {
-          for (const at of matchRuleOffsets(rule, text)) {
-            // snippet keeps up to 60 chars of same-line PRE-context (like the rg path) so the
-            // comment/doc-context noise filter can see "// see" markers before the match
-            const lineStart = text.lastIndexOf("\n", at) + 1;
-            const nl = text.indexOf("\n", at);
-            const snippet = text.slice(Math.max(lineStart, at - 60), Math.min(nl < 0 ? text.length : nl, at + 200)).trim().slice(0, 240);
-            hits.push({ pkg, file: full, line: at >= 0 ? text.slice(0, at).split("\n").length : 0, text: snippet, rule });
-          }
-        }
-      } catch { /* unreadable */ }
-    }
-  }
-  return hits;
-}
+// Content sweeps for the malware scanner (split out of malware-heuristics.js): a ripgrep sweep
+// over all content roots when rg is available, else a bounded Node fallback over entry files.
+import { readFileSync, readdirSync, existsSync, statSync } from "node:fs";
+import { join } from "node:path";
+import { runCommand } from "../process.js";
+import { CONTENT_RULES, classifyContent } from "./registry-sig.js";
+import { normPath } from "./malware-heuristics.exclusions.js";
+
+const SKIP_GLOBS = [
+  "!**/*.map", "!**/*.md", "!**/*.mdx", "!**/*.markdown", "!**/*.mdown", "!**/*.mkd", "!**/*.rst", "!**/*.d.ts",
+  "!**/doc/**", "!**/docs/**", "!**/documentation/**",
+  "!**/test/**", "!**/tests/**", "!**/__tests__/**", "!**/fixtures/**", "!**/example/**", "!**/examples/**"
+];
+
+function globalRuleRegex(rule) {
+  const flags = rule.re.flags.includes("g") ? rule.re.flags : `${rule.re.flags}g`;
+  return new RegExp(rule.re.source, flags);
+}
+
+function matchRuleOffsets(rule, lineText, subText = "") {
+  const text = String(lineText || "");
+  if (text) {
+    const offsets = [];
+    const re = globalRuleRegex(rule);
+    let guard = 0;
+    let m;
+    while ((m = re.exec(text)) && guard++ < 24) {
+      offsets.push(m.index);
+      if (!m[0].length) re.lastIndex++;
+    }
+    if (!offsets.length && rule.key === "obfuscated-code") {
+      const at = subText ? text.indexOf(subText) : 0;
+      if (at >= 0) offsets.push(at);
+    }
+    return offsets;
+  }
+  if (subText && (rule.key === "obfuscated-code" || rule.re.test(subText))) return [0];
+  return [];
+}
+
+function rootOfFile(file, roots) {
+  const f = normPath(file).toLowerCase();
+  return roots.find((r) => {
+    const root = normPath(r.root).toLowerCase();
+    return f === root || f.startsWith(`${root}/`);
+  });
+}
+
+function chunks(list, size) {
+  const out = [];
+  for (let i = 0; i < list.length; i += size) out.push(list.slice(i, i + size));
+  return out;
+}
+
+export async function rgSweep(rg, contentRoots, timeoutMs) {
+  const hits = []; // {pkg, file, line, text, rule}
+  let errored = 0; // a failed/broken rule must not masquerade as "scanned, 0 hits"
+  let total = 0;
+  for (const rule of CONTENT_RULES) {
+    for (const group of chunks(contentRoots, 40)) {
+      total++;
+      const args = ["--json", "-i", "--no-ignore", "--hidden", "--follow", "--max-filesize", "2M", "--max-count", "20", ...SKIP_GLOBS.flatMap((g) => ["-g", g]), "-e", rule.pattern, "--", ...group.map((r) => r.root)];
+      let r;
+      try { r = await runCommand(rg, args, { timeoutMs }); } catch { errored++; continue; }
+      if (r.exitCode === 2 && !String(r.stdout || "").trim()) { errored++; continue; }
+      for (const line of String(r.stdout || "").split(/\r?\n/)) {
+        if (!line) continue;
+        let obj; try { obj = JSON.parse(line); } catch { continue; }
+        if (obj.type !== "match") continue;
+        const file = obj.data?.path?.text || "";
+        const root = rootOfFile(file, group);
+        const pkg = root?.pathToPkg(file) || "";
+        if (!pkg) continue;
+        const lineText = String(obj.data?.lines?.text || "");
+        const subText = String(obj.data?.submatches?.[0]?.match?.text || "");
+        for (const at of matchRuleOffsets(rule, lineText, subText)) {
+          const text = (lineText ? lineText.slice(Math.max(0, at - 60), at + 180) : subText).trim().slice(0, 240);
+          hits.push({ pkg, file, line: obj.data?.line_number || 0, text, rule });
+        }
+      }
+    }
+  }
+  return { hits, errored, total };
+}
+
+// no-rg fallback: each package's package.json main/bin entries + root JS files, bounded
+export function nodeSweep(nmDir, pkgDirs, capFiles = 3000) {
+  const hits = [];
+  let seenFiles = 0;
+  for (const { pkg, dir } of pkgDirs) {
+    if (seenFiles >= capFiles) break;
+    let pj = null;
+    try { pj = JSON.parse(readFileSync(join(dir, "package.json"), "utf8")); } catch { continue; }
+    const cands = new Set();
+    for (const f of [pj.main, pj.module, pj.browser]) if (typeof f === "string") cands.add(f);
+    if (pj.bin) for (const b of typeof pj.bin === "string" ? [pj.bin] : Object.values(pj.bin)) cands.add(b);
+    let rootJs = [];
+    try { rootJs = readdirSync(dir).filter((f) => /\.(c|m)?js$/.test(f)).slice(0, 6); } catch { /* gone */ }
+    for (const f of rootJs) cands.add(f);
+    if (!cands.size) cands.add("index.js");
+    for (const rel of cands) {
+      if (seenFiles >= capFiles) break;
+      const full = join(dir, String(rel));
+      try {
+        if (!existsSync(full) || statSync(full).size > 2_000_000) continue;
+        seenFiles++;
+        const text = readFileSync(full, "utf8");
+        for (const rule of classifyContent(text)) {
+          for (const at of matchRuleOffsets(rule, text)) {
+            // snippet keeps up to 60 chars of same-line PRE-context (like the rg path) so the
+            // comment/doc-context noise filter can see "// see" markers before the match
+            const lineStart = text.lastIndexOf("\n", at) + 1;
+            const nl = text.indexOf("\n", at);
+            const snippet = text.slice(Math.max(lineStart, at - 60), Math.min(nl < 0 ? text.length : nl, at + 200)).trim().slice(0, 240);
+            hits.push({ pkg, file: full, line: at >= 0 ? text.slice(0, at).split("\n").length : 0, text: snippet, rule });
+          }
+        }
+      } catch { /* unreadable */ }
+    }
+  }
+  return hits;
+}