squadrant 0.27.0 → 0.27.1

package/dist/index.js

@@ -10341,16 +10341,275 @@ var init_paths = __esm({
   }
 });
 
-// packages/core/dist/knowledge/store.js
+// packages/core/dist/knowledge/ingest.js
+import crypto3 from "crypto";
 import fs15 from "fs";
 import os7 from "os";
 import path16 from "path";
+import { execFile as execFile2 } from "child_process";
+import { promisify as promisify2 } from "util";
+function refFor(kbRoot, file) {
+  const rel = path16.relative(kbRoot, file);
+  if (rel && !rel.startsWith("..") && !path16.isAbsolute(rel))
+    return rel.split(path16.sep).join("/");
+  const home = os7.homedir();
+  return file.startsWith(home + path16.sep) ? `~/${path16.relative(home, file).split(path16.sep).join("/")}` : file;
+}
+function readState2(kbRoot, ref) {
+  try {
+    return JSON.parse(fs15.readFileSync(stampFile(kbRoot, ref), "utf8"));
+  } catch {
+    return null;
+  }
+}
+function writeState(kbRoot, st) {
+  fs15.mkdirSync(path16.join(kbRoot, CONVERTED_DIR), { recursive: true });
+  fs15.writeFileSync(stampFile(kbRoot, st.ref), JSON.stringify(st) + "\n");
+}
+function globToRegExp(glob) {
+  let re = "";
+  for (let i = 0; i < glob.length; i++) {
+    const c = glob[i];
+    if (c === "*") {
+      if (glob[i + 1] === "*") {
+        re += glob[i + 2] === "/" ? "(?:.*/)?" : ".*";
+        i += glob[i + 2] === "/" ? 2 : 1;
+      } else
+        re += "[^/]*";
+    } else if (c === "?")
+      re += "[^/]";
+    else
+      re += c.replace(/[.+^${}()|[\]\\]/g, "\\$&");
+  }
+  return new RegExp(`^${re}$`);
+}
+function walkFiles(dir, out = []) {
+  if (!fs15.existsSync(dir))
+    return out;
+  for (const ent of fs15.readdirSync(dir, { withFileTypes: true })) {
+    if (ent.name === ".git" || ent.name === CONVERTED_DIR || ent.name === "node_modules")
+      continue;
+    const full = path16.join(dir, ent.name);
+    if (ent.isDirectory())
+      walkFiles(full, out);
+    else if (ent.isFile())
+      out.push(full);
+  }
+  return out;
+}
+function expandSource(kbRoot, entry) {
+  const abs = path16.resolve(kbRoot, expandHome(entry.path));
+  if (!/[*?]/.test(entry.path))
+    return fs15.existsSync(abs) && fs15.statSync(abs).isFile() ? [abs] : [];
+  const base = abs.split(/[*?]/)[0];
+  const root = base.endsWith(path16.sep) ? base : path16.dirname(base);
+  const re = globToRegExp(abs.split(path16.sep).join("/"));
+  return walkFiles(root).filter((f) => re.test(f.split(path16.sep).join("/"))).sort();
+}
+function detectChange(kbRoot, file) {
+  const ref = refFor(kbRoot, file);
+  const st = fs15.statSync(file);
+  const prior = readState2(kbRoot, ref);
+  if (prior && prior.size === st.size && prior.mtimeMs === st.mtimeMs) {
+    return { ref, changed: false, stamp: { sha: prior.sha, size: st.size, mtimeMs: st.mtimeMs }, prior };
+  }
+  const sha = sha256(file);
+  return { ref, changed: !prior || prior.sha !== sha, stamp: { sha, size: st.size, mtimeMs: st.mtimeMs }, prior };
+}
+function splitSections(text) {
+  const cuts = [];
+  const re = /^#{1,6}[ \t]+(.+)$/gm;
+  for (let m = re.exec(text); m; m = re.exec(text))
+    cuts.push({ at: m.index, heading: m[1].trim() });
+  if (!cuts.length || cuts[0].at > 0)
+    cuts.unshift({ at: 0 });
+  const spans = [];
+  cuts.forEach((c, i) => {
+    const end = i + 1 < cuts.length ? cuts[i + 1].at : text.length;
+    let start = c.at;
+    while (end - start > SECTION_TARGET_CHARS * 1.5) {
+      const brk = text.indexOf("\n\n", start + SECTION_TARGET_CHARS);
+      if (brk < 0 || brk >= end)
+        break;
+      spans.push({ index: 0, start, end: brk, heading: c.heading });
+      start = brk + 2;
+    }
+    if (text.slice(start, end).trim())
+      spans.push({ index: 0, start, end, heading: c.heading });
+  });
+  return spans.map((s, i) => ({ ...s, index: i }));
+}
+async function planIngest(kbRoot, opts) {
+  const converter = opts.converter ?? markitdownConverter;
+  const cap = opts.maxSectionsPerPass ?? DEFAULT_MAX_SECTIONS_PER_PASS;
+  const plan = { sources: [], skipped: [], failed: [], carriedSections: 0, sectionCount: 0, estimatedTokens: 0, existing: opts.existing ?? [] };
+  let budget = cap;
+  const seen = /* @__PURE__ */ new Set();
+  for (const entry of opts.sources) {
+    for (const file of expandSource(kbRoot, entry)) {
+      const { ref, changed, stamp, prior } = detectChange(kbRoot, file);
+      if (seen.has(ref))
+        continue;
+      seen.add(ref);
+      const pending = !!prior && prior.cursor < prior.sections;
+      if (!changed && !pending)
+        continue;
+      if (entry.sensitivity === "local-only" && !opts.localModel) {
+        plan.skipped.push({ ref, reason: "sensitivity: local-only and no local model available" });
+        continue;
+      }
+      if (budget <= 0) {
+        plan.carriedSections += changed ? 1 : prior.sections - prior.cursor;
+        continue;
+      }
+      let text;
+      const conv = convertedFile(kbRoot, ref);
+      try {
+        if (!changed && fs15.existsSync(conv))
+          text = fs15.readFileSync(conv, "utf8");
+        else {
+          text = PASS_THROUGH.has(path16.extname(file).toLowerCase()) ? fs15.readFileSync(file, "utf8") : await converter(file);
+          if (!opts.dryRun) {
+            fs15.mkdirSync(path16.dirname(conv), { recursive: true });
+            fs15.writeFileSync(conv, text);
+          }
+        }
+      } catch (e) {
+        plan.failed.push({ ref, error: e.message.split("\n")[0] });
+        continue;
+      }
+      const all = splitSections(text);
+      const from = changed ? 0 : prior.cursor;
+      const take = all.slice(from, from + budget);
+      budget -= take.length;
+      plan.carriedSections += all.length - from - take.length;
+      plan.sectionCount += take.length;
+      plan.estimatedTokens += Math.ceil(take.reduce((n, s) => n + (s.end - s.start), 0) / CHARS_PER_TOKEN);
+      plan.sources.push({
+        ref,
+        file,
+        converted: conv,
+        sha: stamp.sha,
+        priority: entry.priority,
+        domain: entry.domain,
+        sensitivity: entry.sensitivity,
+        sections: take,
+        totalSections: all.length
+      });
+      if (!opts.dryRun)
+        writeState(kbRoot, { ref, ...stamp, sections: all.length, cursor: from + take.length, priority: entry.priority });
+    }
+  }
+  return plan;
+}
+var execFileAsync, CONVERTED_DIR, DEFAULT_MAX_SECTIONS_PER_PASS, CHARS_PER_TOKEN, PASS_THROUGH, SECTION_TARGET_CHARS, markitdownConverter, expandHome, fileForRef, cacheKey, convertedFile, stampFile, sha256;
+var init_ingest = __esm({
+  "packages/core/dist/knowledge/ingest.js"() {
+    execFileAsync = promisify2(execFile2);
+    CONVERTED_DIR = ".converted";
+    DEFAULT_MAX_SECTIONS_PER_PASS = 40;
+    CHARS_PER_TOKEN = 4;
+    PASS_THROUGH = /* @__PURE__ */ new Set([".md", ".markdown", ".txt"]);
+    SECTION_TARGET_CHARS = 4e3;
+    markitdownConverter = async (file) => {
+      const { stdout } = await execFileAsync("markitdown", [file], { maxBuffer: 64 * 1024 * 1024 });
+      return stdout;
+    };
+    expandHome = (p) => p === "~" || p.startsWith("~/") ? path16.join(os7.homedir(), p.slice(1)) : p;
+    fileForRef = (kbRoot, ref) => path16.resolve(kbRoot, expandHome(ref));
+    cacheKey = (ref) => `${ref.replace(/[^A-Za-z0-9._-]+/g, "_")}.${crypto3.createHash("sha1").update(ref).digest("hex").slice(0, 6)}`;
+    convertedFile = (kbRoot, ref) => path16.join(kbRoot, CONVERTED_DIR, `${cacheKey(ref)}.md`);
+    stampFile = (kbRoot, ref) => path16.join(kbRoot, CONVERTED_DIR, `${cacheKey(ref)}.sha`);
+    sha256 = (file) => crypto3.createHash("sha256").update(fs15.readFileSync(file)).digest("hex");
+  }
+});
+
+// packages/core/dist/knowledge/verify.js
+function normalizeWithMap(src, keepBreakHyphen = false) {
+  const out = [];
+  const map2 = [];
+  const push = (s, at) => {
+    for (const ch of s) {
+      out.push(ch);
+      map2.push(at);
+    }
+  };
+  let i = 0;
+  while (i < src.length) {
+    const ch = src[i];
+    if (ch === "-" && /^-[ \t]*\r?\n[ \t]*[a-z]/.test(src.slice(i, i + 40)) && i > 0 && /[A-Za-z]/.test(src[i - 1])) {
+      if (keepBreakHyphen)
+        push("-", i);
+      i++;
+      while (/\s/.test(src[i]))
+        i++;
+      continue;
+    }
+    if (/\s/.test(ch)) {
+      const start = i;
+      while (i < src.length && /\s/.test(src[i]))
+        i++;
+      if (out.length && out[out.length - 1] !== " ")
+        push(" ", start);
+      continue;
+    }
+    if (MARKUP.has(ch)) {
+      i++;
+      continue;
+    }
+    if (LIGATURES[ch])
+      push(LIGATURES[ch], i);
+    else if (QUOTES[ch])
+      push(QUOTES[ch], i);
+    else if (DASHES.has(ch))
+      push("-", i);
+    else
+      push(ch, i);
+    i++;
+  }
+  while (out.length && out[out.length - 1] === " ") {
+    out.pop();
+    map2.pop();
+  }
+  return { text: out.join(""), map: map2 };
+}
+function findQuote(source, quote) {
+  const needle = normalizeText(quote);
+  if (!needle)
+    return null;
+  const views = typeof source === "string" ? normalizedViews(source) : source;
+  for (const hay of views) {
+    const at = hay.text.indexOf(needle);
+    if (at >= 0)
+      return [hay.map[at], hay.map[at + needle.length - 1] + 1];
+  }
+  return null;
+}
+var LIGATURES, QUOTES, DASHES, MARKUP, normalizeText, normalizedViews, DOCLING_DROP_THRESHOLD, needsDocling, spansOverlap;
+var init_verify = __esm({
+  "packages/core/dist/knowledge/verify.js"() {
+    LIGATURES = { "\uFB01": "fi", "\uFB02": "fl", "\uFB00": "ff", "\uFB03": "ffi", "\uFB04": "ffl", "\uFB05": "st", "\uFB06": "st" };
+    QUOTES = { "\u2018": "'", "\u2019": "'", "\u201A": "'", "\u201B": "'", "\u201C": '"', "\u201D": '"', "\u201E": '"', "\u201F": '"' };
+    DASHES = /* @__PURE__ */ new Set(["\u2010", "\u2011", "\u2012", "\u2013", "\u2014", "\u2015", "\u2212"]);
+    MARKUP = /* @__PURE__ */ new Set(["*", "_", "`", "\\"]);
+    normalizeText = (s) => normalizeWithMap(s).text;
+    normalizedViews = (src) => [normalizeWithMap(src), normalizeWithMap(src, true)];
+    DOCLING_DROP_THRESHOLD = 0.2;
+    needsDocling = (dropped, total) => total > 0 && dropped / total > DOCLING_DROP_THRESHOLD;
+    spansOverlap = (a, b) => a[0] < b[1] && b[0] < a[1];
+  }
+});
+
+// packages/core/dist/knowledge/store.js
+import fs16 from "fs";
+import os8 from "os";
+import path17 from "path";
 import matter2 from "gray-matter";
 function duplicateIdProblem(id, ignored, used) {
   return `duplicate id ${id}: ${ignored} is ignored, ${used} is used (live files win over _proposed, then path order). Fix: rename the id in one file, or delete or move one of them.`;
 }
 function isPendingProposalDup(e, rulesDir) {
-  return e.problems.every((p) => p.startsWith("duplicate id")) && path16.relative(rulesDir, e.file).split(path16.sep).includes(PROPOSED_DIR);
+  return e.problems.every((p) => p.startsWith("duplicate id")) && path17.relative(rulesDir, e.file).split(path17.sep).includes(PROPOSED_DIR);
 }
 function splitStatement(body) {
   const first = body.trim().split(/\n\s*\n/)[0] ?? "";
@@ -10359,8 +10618,8 @@ function splitStatement(body) {
 function listMarkdown(dir, includeProposed) {
   const out = [];
   const walk = (d) => {
-    for (const ent of fs15.readdirSync(d, { withFileTypes: true })) {
-      const full = path16.join(d, ent.name);
+    for (const ent of fs16.readdirSync(d, { withFileTypes: true })) {
+      const full = path17.join(d, ent.name);
       if (ent.isDirectory()) {
         if (ent.name === PROPOSED_DIR && !includeProposed)
           continue;
@@ -10371,11 +10630,11 @@ function listMarkdown(dir, includeProposed) {
     }
   };
   walk(dir);
-  const isProposed = (f) => path16.relative(dir, f).split(path16.sep).includes(PROPOSED_DIR);
+  const isProposed = (f) => path17.relative(dir, f).split(path17.sep).includes(PROPOSED_DIR);
   return out.sort((a, b) => Number(isProposed(a)) - Number(isProposed(b)) || (a < b ? -1 : a > b ? 1 : 0));
 }
 function loadRulesDir(dir, layer, opts = {}) {
-  if (!fs15.existsSync(dir))
+  if (!fs16.existsSync(dir))
     return { rules: [], errors: [] };
   const rules = [];
   const errors = [];
@@ -10383,7 +10642,7 @@ function loadRulesDir(dir, layer, opts = {}) {
   for (const file of listMarkdown(dir, opts.includeProposed ?? false)) {
     let parsed;
     try {
-      parsed = matter2(fs15.readFileSync(file, "utf8"), NO_JS_FRONTMATTER);
+      parsed = matter2(fs16.readFileSync(file, "utf8"), NO_JS_FRONTMATTER);
     } catch (e) {
       errors.push({ file, problems: [`frontmatter parse error: ${e.message}`] });
       continue;
@@ -10407,14 +10666,14 @@ function loadRulesDir(dir, layer, opts = {}) {
 function resolveSourceRef(ref, kbRoot, sources) {
   const bases = /* @__PURE__ */ new Set([kbRoot]);
   for (const s of sources) {
-    let d = path16.resolve(kbRoot, expandHome(s.path.split(/[*?[{]/)[0]));
-    for (let i = 0; i < 6 && d !== path16.dirname(d); i++) {
-      d = path16.dirname(d);
+    let d = path17.resolve(kbRoot, expandHome2(s.path.split(/[*?[{]/)[0]));
+    for (let i = 0; i < 6 && d !== path17.dirname(d); i++) {
+      d = path17.dirname(d);
       bases.add(d);
     }
   }
-  const cands = [path16.resolve(kbRoot, expandHome(ref)), ...[...bases].map((b) => path16.join(b, ref))];
-  return cands.find((c) => fs15.existsSync(c) && fs15.statSync(c).isFile());
+  const cands = [path17.resolve(kbRoot, expandHome2(ref)), ...[...bases].map((b) => path17.join(b, ref))];
+  return cands.find((c) => fs16.existsSync(c) && fs16.statSync(c).isFile());
 }
 function validateKb(opts) {
   const { rules, errors } = loadRulesDir(opts.rulesDir, "kb", { includeProposed: true });
@@ -10422,7 +10681,7 @@ function validateKb(opts) {
   const dups = errors.filter((e) => e.problems.some((p) => p.startsWith("duplicate id")) && !isPendingProposalDup(e, opts.rulesDir));
   const pendingDups = errors.filter((e) => isPendingProposalDup(e, opts.rulesDir)).length;
   const invalid = errors.filter((e) => !dups.includes(e) && !isPendingProposalDup(e, opts.rulesDir));
-  const proposed = rules.filter((r) => path16.relative(opts.rulesDir, r.file).split(path16.sep).includes(PROPOSED_DIR)).length;
+  const proposed = rules.filter((r) => path17.relative(opts.rulesDir, r.file).split(path17.sep).includes(PROPOSED_DIR)).length;
   checks.push({ level: "ok", message: `${rules.length + errors.length} rule files read (${rules.length - proposed} live, ${proposed + pendingDups} proposed parsed OK)` });
   if (invalid.length)
     for (const e of invalid)
@@ -10435,7 +10694,7 @@ function validateKb(opts) {
   else
     checks.push({ level: "ok", message: "no duplicate ids" });
   const cache = /* @__PURE__ */ new Map();
-  let verified = 0, unresolved = 0;
+  let verified = 0, unresolved = 0, notIngested = 0;
   for (const r of rules) {
     r.sources.forEach((src, i) => {
       const file = resolveSourceRef(src.ref, opts.kbRoot, opts.sources);
@@ -10443,16 +10702,28 @@ function validateKb(opts) {
         unresolved++;
         return;
       }
-      if (!cache.has(file))
-        cache.set(file, fs15.readFileSync(file, "utf8"));
-      if (cache.get(file).includes(src.quote))
+      const key = `${src.ref}\0${file}`;
+      if (!cache.has(key)) {
+        const conv = readState2(opts.kbRoot, src.ref) ? convertedFile(opts.kbRoot, src.ref) : null;
+        const cached = !!conv && fs16.existsSync(conv);
+        cache.set(key, {
+          hay: normalizedViews(fs16.readFileSync(cached ? conv : file, "utf8")),
+          convertible: !cached && !PASS_THROUGH.has(path17.extname(file).toLowerCase())
+        });
+      }
+      const { hay, convertible } = cache.get(key);
+      if (findQuote(hay, src.quote))
         verified++;
+      else if (convertible)
+        notIngested++;
       else
         checks.push({ level: "fail", message: `quote not found in ${file}: ${r.id} sources[${i}] (${r.file})` });
     });
   }
   if (!checks.some((c) => c.message.startsWith("quote not found")))
     checks.push({ level: "ok", message: `${verified} source quotes found in their source files` });
+  if (notIngested)
+    checks.push({ level: "skip", message: `${notIngested} source refs need conversion but have no .converted cache, quote not checked (run \`squadrant knowledge ingest <kb>\`)` });
   if (unresolved)
     checks.push({ level: "skip", message: `${unresolved} source refs not resolvable on this machine, quote not checked` });
   if (opts.domains) {
@@ -10463,22 +10734,24 @@ function validateKb(opts) {
   }
   return { checks, ok: !checks.some((c) => c.level === "fail") };
 }
-var NO_JS_FRONTMATTER, expandHome;
+var NO_JS_FRONTMATTER, expandHome2;
 var init_store2 = __esm({
   "packages/core/dist/knowledge/store.js"() {
     init_dist();
+    init_ingest();
     init_paths();
+    init_verify();
     NO_JS_FRONTMATTER = {
       engines: { javascript: { parse: () => {
         throw new Error("only YAML front matter is allowed");
       } } }
     };
-    expandHome = (p) => p === "~" || p.startsWith("~/") ? path16.join(os7.homedir(), p.slice(1)) : p;
+    expandHome2 = (p) => p === "~" || p.startsWith("~/") ? path17.join(os8.homedir(), p.slice(1)) : p;
   }
 });
 
 // packages/core/dist/knowledge/layers.js
-import fs16 from "fs";
+import fs17 from "fs";
 function loadKbRules(cfg, kb, opts = {}) {
   return loadRulesDir(kbRulesDir(cfg, kb), `kb:${kb}`, opts);
 }
@@ -10490,7 +10763,7 @@ function resolveProjectRules(cfg, project, opts = {}) {
   const errors = [];
   const warnings = [];
   for (const kb of subscribedKbs(cfg, project)) {
-    if (!fs16.existsSync(kbDir(cfg, kb))) {
+    if (!fs17.existsSync(kbDir(cfg, kb))) {
       warnings.push(`knowledge base '${kb}' not found (run: squadrant knowledge init ${kb})`);
       continue;
     }
@@ -10528,8 +10801,8 @@ var init_layers = __esm({
 });
 
 // packages/core/dist/knowledge/index-file.js
-import fs17 from "fs";
-import path17 from "path";
+import fs18 from "fs";
+import path18 from "path";
 function compileIndex(kb, rules, now = /* @__PURE__ */ new Date()) {
   return {
     kb,
@@ -10538,18 +10811,18 @@ function compileIndex(kb, rules, now = /* @__PURE__ */ new Date()) {
   };
 }
 function writeIndex(kbRoot, index) {
-  const file = path17.join(kbRoot, "index.json");
+  const file = path18.join(kbRoot, "index.json");
   const tmp = `${file}.tmp-${process.pid}`;
-  fs17.writeFileSync(tmp, JSON.stringify(index, null, 2) + "\n");
-  fs17.renameSync(tmp, file);
+  fs18.writeFileSync(tmp, JSON.stringify(index, null, 2) + "\n");
+  fs18.renameSync(tmp, file);
   return file;
 }
 function readIndex(kbRoot, kb) {
-  const file = path17.join(kbRoot, "index.json");
-  if (!fs17.existsSync(file))
+  const file = path18.join(kbRoot, "index.json");
+  if (!fs18.existsSync(file))
     return { index: null, problem: `index.json missing for kb '${kb}'` };
   try {
-    const index = JSON.parse(fs17.readFileSync(file, "utf8"));
+    const index = JSON.parse(fs18.readFileSync(file, "utf8"));
     if (!Array.isArray(index.rules))
       return { index: null, problem: `index.json corrupt for kb '${kb}' (no rules array)` };
     return { index };
@@ -13644,15 +13917,15 @@ var init_js_yaml = __esm({
 });
 
 // packages/core/dist/knowledge/sources.js
-import fs18 from "fs";
-import path18 from "path";
+import fs19 from "fs";
+import path19 from "path";
 function loadSources(kbRoot) {
-  const file = path18.join(kbRoot, "sources.yaml");
-  if (!fs18.existsSync(file))
+  const file = path19.join(kbRoot, "sources.yaml");
+  if (!fs19.existsSync(file))
     return { sources: [], errors: [] };
   let data;
   try {
-    data = yaml.load(fs18.readFileSync(file, "utf8"));
+    data = yaml.load(fs19.readFileSync(file, "utf8"));
   } catch (e) {
     return { sources: [], errors: [`YAML parse error: ${e.message.split("\n")[0]}`] };
   }
@@ -13690,189 +13963,6 @@ var init_sources = __esm({
   }
 });
 
-// packages/core/dist/knowledge/ingest.js
-import crypto3 from "crypto";
-import fs19 from "fs";
-import os8 from "os";
-import path19 from "path";
-import { execFile as execFile2 } from "child_process";
-import { promisify as promisify2 } from "util";
-function refFor(kbRoot, file) {
-  const rel = path19.relative(kbRoot, file);
-  if (rel && !rel.startsWith("..") && !path19.isAbsolute(rel))
-    return rel.split(path19.sep).join("/");
-  const home = os8.homedir();
-  return file.startsWith(home + path19.sep) ? `~/${path19.relative(home, file).split(path19.sep).join("/")}` : file;
-}
-function readState2(kbRoot, ref) {
-  try {
-    return JSON.parse(fs19.readFileSync(stampFile(kbRoot, ref), "utf8"));
-  } catch {
-    return null;
-  }
-}
-function writeState(kbRoot, st) {
-  fs19.mkdirSync(path19.join(kbRoot, CONVERTED_DIR), { recursive: true });
-  fs19.writeFileSync(stampFile(kbRoot, st.ref), JSON.stringify(st) + "\n");
-}
-function globToRegExp(glob) {
-  let re = "";
-  for (let i = 0; i < glob.length; i++) {
-    const c = glob[i];
-    if (c === "*") {
-      if (glob[i + 1] === "*") {
-        re += glob[i + 2] === "/" ? "(?:.*/)?" : ".*";
-        i += glob[i + 2] === "/" ? 2 : 1;
-      } else
-        re += "[^/]*";
-    } else if (c === "?")
-      re += "[^/]";
-    else
-      re += c.replace(/[.+^${}()|[\]\\]/g, "\\$&");
-  }
-  return new RegExp(`^${re}$`);
-}
-function walkFiles(dir, out = []) {
-  if (!fs19.existsSync(dir))
-    return out;
-  for (const ent of fs19.readdirSync(dir, { withFileTypes: true })) {
-    if (ent.name === ".git" || ent.name === CONVERTED_DIR || ent.name === "node_modules")
-      continue;
-    const full = path19.join(dir, ent.name);
-    if (ent.isDirectory())
-      walkFiles(full, out);
-    else if (ent.isFile())
-      out.push(full);
-  }
-  return out;
-}
-function expandSource(kbRoot, entry) {
-  const abs = path19.resolve(kbRoot, expandHome2(entry.path));
-  if (!/[*?]/.test(entry.path))
-    return fs19.existsSync(abs) && fs19.statSync(abs).isFile() ? [abs] : [];
-  const base = abs.split(/[*?]/)[0];
-  const root = base.endsWith(path19.sep) ? base : path19.dirname(base);
-  const re = globToRegExp(abs.split(path19.sep).join("/"));
-  return walkFiles(root).filter((f) => re.test(f.split(path19.sep).join("/"))).sort();
-}
-function detectChange(kbRoot, file) {
-  const ref = refFor(kbRoot, file);
-  const st = fs19.statSync(file);
-  const prior = readState2(kbRoot, ref);
-  if (prior && prior.size === st.size && prior.mtimeMs === st.mtimeMs) {
-    return { ref, changed: false, stamp: { sha: prior.sha, size: st.size, mtimeMs: st.mtimeMs }, prior };
-  }
-  const sha = sha256(file);
-  return { ref, changed: !prior || prior.sha !== sha, stamp: { sha, size: st.size, mtimeMs: st.mtimeMs }, prior };
-}
-function splitSections(text) {
-  const cuts = [];
-  const re = /^#{1,6}[ \t]+(.+)$/gm;
-  for (let m = re.exec(text); m; m = re.exec(text))
-    cuts.push({ at: m.index, heading: m[1].trim() });
-  if (!cuts.length || cuts[0].at > 0)
-    cuts.unshift({ at: 0 });
-  const spans = [];
-  cuts.forEach((c, i) => {
-    const end = i + 1 < cuts.length ? cuts[i + 1].at : text.length;
-    let start = c.at;
-    while (end - start > SECTION_TARGET_CHARS * 1.5) {
-      const brk = text.indexOf("\n\n", start + SECTION_TARGET_CHARS);
-      if (brk < 0 || brk >= end)
-        break;
-      spans.push({ index: 0, start, end: brk, heading: c.heading });
-      start = brk + 2;
-    }
-    if (text.slice(start, end).trim())
-      spans.push({ index: 0, start, end, heading: c.heading });
-  });
-  return spans.map((s, i) => ({ ...s, index: i }));
-}
-async function planIngest(kbRoot, opts) {
-  const converter = opts.converter ?? markitdownConverter;
-  const cap = opts.maxSectionsPerPass ?? DEFAULT_MAX_SECTIONS_PER_PASS;
-  const plan = { sources: [], skipped: [], failed: [], carriedSections: 0, sectionCount: 0, estimatedTokens: 0, existing: opts.existing ?? [] };
-  let budget = cap;
-  const seen = /* @__PURE__ */ new Set();
-  for (const entry of opts.sources) {
-    for (const file of expandSource(kbRoot, entry)) {
-      const { ref, changed, stamp, prior } = detectChange(kbRoot, file);
-      if (seen.has(ref))
-        continue;
-      seen.add(ref);
-      const pending = !!prior && prior.cursor < prior.sections;
-      if (!changed && !pending)
-        continue;
-      if (entry.sensitivity === "local-only" && !opts.localModel) {
-        plan.skipped.push({ ref, reason: "sensitivity: local-only and no local model available" });
-        continue;
-      }
-      if (budget <= 0) {
-        plan.carriedSections += changed ? 1 : prior.sections - prior.cursor;
-        continue;
-      }
-      let text;
-      const conv = convertedFile(kbRoot, ref);
-      try {
-        if (!changed && fs19.existsSync(conv))
-          text = fs19.readFileSync(conv, "utf8");
-        else {
-          text = PASS_THROUGH.has(path19.extname(file).toLowerCase()) ? fs19.readFileSync(file, "utf8") : await converter(file);
-          if (!opts.dryRun) {
-            fs19.mkdirSync(path19.dirname(conv), { recursive: true });
-            fs19.writeFileSync(conv, text);
-          }
-        }
-      } catch (e) {
-        plan.failed.push({ ref, error: e.message.split("\n")[0] });
-        continue;
-      }
-      const all = splitSections(text);
-      const from = changed ? 0 : prior.cursor;
-      const take = all.slice(from, from + budget);
-      budget -= take.length;
-      plan.carriedSections += all.length - from - take.length;
-      plan.sectionCount += take.length;
-      plan.estimatedTokens += Math.ceil(take.reduce((n, s) => n + (s.end - s.start), 0) / CHARS_PER_TOKEN);
-      plan.sources.push({
-        ref,
-        file,
-        converted: conv,
-        sha: stamp.sha,
-        priority: entry.priority,
-        domain: entry.domain,
-        sensitivity: entry.sensitivity,
-        sections: take,
-        totalSections: all.length
-      });
-      if (!opts.dryRun)
-        writeState(kbRoot, { ref, ...stamp, sections: all.length, cursor: from + take.length, priority: entry.priority });
-    }
-  }
-  return plan;
-}
-var execFileAsync, CONVERTED_DIR, DEFAULT_MAX_SECTIONS_PER_PASS, CHARS_PER_TOKEN, PASS_THROUGH, SECTION_TARGET_CHARS, markitdownConverter, expandHome2, fileForRef, cacheKey, convertedFile, stampFile, sha256;
-var init_ingest = __esm({
-  "packages/core/dist/knowledge/ingest.js"() {
-    execFileAsync = promisify2(execFile2);
-    CONVERTED_DIR = ".converted";
-    DEFAULT_MAX_SECTIONS_PER_PASS = 40;
-    CHARS_PER_TOKEN = 4;
-    PASS_THROUGH = /* @__PURE__ */ new Set([".md", ".markdown", ".txt"]);
-    SECTION_TARGET_CHARS = 4e3;
-    markitdownConverter = async (file) => {
-      const { stdout } = await execFileAsync("markitdown", [file], { maxBuffer: 64 * 1024 * 1024 });
-      return stdout;
-    };
-    expandHome2 = (p) => p === "~" || p.startsWith("~/") ? path19.join(os8.homedir(), p.slice(1)) : p;
-    fileForRef = (kbRoot, ref) => path19.resolve(kbRoot, expandHome2(ref));
-    cacheKey = (ref) => `${ref.replace(/[^A-Za-z0-9._-]+/g, "_")}.${crypto3.createHash("sha1").update(ref).digest("hex").slice(0, 6)}`;
-    convertedFile = (kbRoot, ref) => path19.join(kbRoot, CONVERTED_DIR, `${cacheKey(ref)}.md`);
-    stampFile = (kbRoot, ref) => path19.join(kbRoot, CONVERTED_DIR, `${cacheKey(ref)}.sha`);
-    sha256 = (file) => crypto3.createHash("sha256").update(fs19.readFileSync(file)).digest("hex");
-  }
-});
-
 // packages/core/dist/knowledge/match.js
 import path20 from "path";
 function globMatches(glob, relPath) {
@@ -14290,6 +14380,13 @@ function ensureGitRepo(dir) {
     return false;
   }
 }
+function ensureKbGitignore(dir) {
+  const file = path23.join(dir, ".gitignore");
+  if (fs22.existsSync(file))
+    return false;
+  fs22.writeFileSync(file, KB_GITIGNORE);
+  return true;
+}
 function findLegacyOverlays(cfg) {
   return Object.keys(cfg.projects).filter((p) => fs22.existsSync(legacyOverlay(cfg, p))).sort();
 }
@@ -14344,6 +14441,8 @@ function moveLegacyKbs(cfg, opts = {}) {
     }
     if (ensureGitRepo(to))
       notes.push("git init");
+    if (ensureKbGitignore(to))
+      notes.push("wrote .gitignore (raw/, .converted/)");
     if (fs22.existsSync(path23.join(to, "index.json")))
       notes.push(`index.json holds old paths: run squadrant knowledge reindex ${kb}`);
     if (action === "copied")
@@ -14351,87 +14450,17 @@ function moveLegacyKbs(cfg, opts = {}) {
     return { kb, from, to, action, ...notes.length ? { note: notes.join("; ") } : {} };
   });
 }
-var legacyRoot, legacyOverlay;
+var legacyRoot, KB_GITIGNORE, legacyOverlay;
 var init_migrate = __esm({
   "packages/core/dist/knowledge/migrate.js"() {
     init_dist();
     init_paths();
     legacyRoot = (cfg) => path23.join(resolveHome(cfg.hubVault), "knowledge");
+    KB_GITIGNORE = "# squadrant KB defaults: source documents stay local\nraw/\n.converted/\n";
     legacyOverlay = (cfg, project) => path23.join(resolveHome(cfg.projects[project].spokeVault), "knowledge", "rules");
   }
 });
 
-// packages/core/dist/knowledge/verify.js
-function normalizeWithMap(src, keepBreakHyphen = false) {
-  const out = [];
-  const map2 = [];
-  const push = (s, at) => {
-    for (const ch of s) {
-      out.push(ch);
-      map2.push(at);
-    }
-  };
-  let i = 0;
-  while (i < src.length) {
-    const ch = src[i];
-    if (ch === "-" && /^-[ \t]*\r?\n[ \t]*[a-z]/.test(src.slice(i, i + 40)) && i > 0 && /[A-Za-z]/.test(src[i - 1])) {
-      if (keepBreakHyphen)
-        push("-", i);
-      i++;
-      while (/\s/.test(src[i]))
-        i++;
-      continue;
-    }
-    if (/\s/.test(ch)) {
-      const start = i;
-      while (i < src.length && /\s/.test(src[i]))
-        i++;
-      if (out.length && out[out.length - 1] !== " ")
-        push(" ", start);
-      continue;
-    }
-    if (LIGATURES[ch])
-      push(LIGATURES[ch], i);
-    else if (QUOTES[ch])
-      push(QUOTES[ch], i);
-    else if (DASHES.has(ch))
-      push("-", i);
-    else
-      push(ch, i);
-    i++;
-  }
-  while (out.length && out[out.length - 1] === " ") {
-    out.pop();
-    map2.pop();
-  }
-  return { text: out.join(""), map: map2 };
-}
-function findQuote(source, quote) {
-  const needle = normalizeText(quote);
-  if (!needle)
-    return null;
-  const views = typeof source === "string" ? normalizedViews(source) : source;
-  for (const hay of views) {
-    const at = hay.text.indexOf(needle);
-    if (at >= 0)
-      return [hay.map[at], hay.map[at + needle.length - 1] + 1];
-  }
-  return null;
-}
-var LIGATURES, QUOTES, DASHES, normalizeText, normalizedViews, DOCLING_DROP_THRESHOLD, needsDocling, spansOverlap;
-var init_verify = __esm({
-  "packages/core/dist/knowledge/verify.js"() {
-    LIGATURES = { "\uFB01": "fi", "\uFB02": "fl", "\uFB00": "ff", "\uFB03": "ffi", "\uFB04": "ffl", "\uFB05": "st", "\uFB06": "st" };
-    QUOTES = { "\u2018": "'", "\u2019": "'", "\u201A": "'", "\u201B": "'", "\u201C": '"', "\u201D": '"', "\u201E": '"', "\u201F": '"' };
-    DASHES = /* @__PURE__ */ new Set(["\u2010", "\u2011", "\u2012", "\u2013", "\u2014", "\u2015", "\u2212"]);
-    normalizeText = (s) => normalizeWithMap(s).text;
-    normalizedViews = (src) => [normalizeWithMap(src), normalizeWithMap(src, true)];
-    DOCLING_DROP_THRESHOLD = 0.2;
-    needsDocling = (dropped, total) => total > 0 && dropped / total > DOCLING_DROP_THRESHOLD;
-    spansOverlap = (a, b) => a[0] < b[1] && b[0] < a[1];
-  }
-});
-
 // packages/core/dist/knowledge/anchor.js
 import fs23 from "fs";
 import path24 from "path";
@@ -15957,6 +15986,7 @@ __export(dist_exports2, {
   NOISY_RATE: () => NOISY_RATE,
   OPERATOR_INITIATED_COMMANDS: () => OPERATOR_INITIATED_COMMANDS,
   PANE_IDLE_AFTER_MS: () => PANE_IDLE_AFTER_MS,
+  PASS_THROUGH: () => PASS_THROUGH,
   PROBE_QUIET_MS: () => PROBE_QUIET_MS,
   PROMPT_MIN_SCORE: () => PROMPT_MIN_SCORE,
   PROMPT_RULE_CAP: () => PROMPT_RULE_CAP,
@@ -16096,6 +16126,7 @@ __export(dist_exports2, {
   encodeMsg: () => encodeMsg,
   ensureDaemon: () => ensureDaemon,
   ensureGitRepo: () => ensureGitRepo,
+  ensureKbGitignore: () => ensureKbGitignore,
   ensureSocksDir: () => ensureSocksDir,
   escalationsFile: () => escalationsFile,
   evaluateCodeConflict: () => evaluateCodeConflict,
@@ -30670,6 +30701,7 @@ function runKnowledgeInit(kb, configPath = DEFAULT_CONFIG_PATH) {
     created.push("index.json");
   }
   if (ensureGitRepo(dir)) created.push(".git");
+  if (ensureKbGitignore(dir)) created.push(".gitignore");
   if (!kbConfigs(cfg)[kb]) {
     cfg.knowledgeBases = { ...cfg.knowledgeBases ?? {}, [kb]: {} };
     saveConfig(cfg, configPath);
@@ -30700,6 +30732,7 @@ var knowledgeCommand = new Command39("knowledge").description("Manage rules know
 knowledgeCommand.command("init <kb>").description("Create a knowledge base (its own git repo) under ~/squadrant/kb/<kb>/").action((kb) => {
   const { dir, created } = runKnowledgeInit(kb);
   console.log(created.length ? chalk38.green(`Initialised ${dir} (${created.join(", ")})`) : `Already initialised: ${dir}`);
+  console.log(chalk38.dim("raw/ and .converted/ are git-ignored by default (edit .gitignore to opt in)."));
   console.log(chalk38.dim(KNOWLEDGE_PRIVACY_NOTICE));
 });
 knowledgeCommand.command("subscribe <kb>").description("Subscribe a project to a knowledge base").requiredOption("--project <name>", "project to subscribe").action((kb, opts) => {