rigjs 4.0.12 → 4.0.14

package/lib/wiki/gitignore.ts

@@ -0,0 +1,90 @@
+// Multi-repo-aware .gitignore checker.
+//
+// Plain `git check-ignore --stdin` only knows the rules of one repo —
+// the one its cwd lives in. When the wiki walker traverses a parent
+// project that contains git submodules, each submodule has its OWN
+// .gitignore (e.g. `node_modules/`) that the parent repo does NOT
+// inherit. Running check-ignore from the parent's root therefore lets
+// the submodule's node_modules / build/ / etc. slip through as wiki
+// candidates — exactly the bug the user reported.
+//
+// Fix: for every candidate path, find its owning git repo (the nearest
+// ancestor that has `.git` as a dir or file — submodules use a `.git`
+// file pointing into the parent's `modules/` dir), bucket the
+// candidates by owning repo, then run a single `git check-ignore` per
+// bucket with cwd set to that repo's root. Each repo's own
+// .gitignore(s) get correctly applied to the paths inside it.
+//
+// Paths outside any git repo are treated as "not ignored" (no rules to
+// apply). Files whose owning repo can't process them (e.g. git binary
+// missing) are also treated as "not ignored" — conservative, since we
+// already have the walk-time hidden / binary-extension filters as a
+// hard floor.
+
+import fs from 'fs';
+import path from 'path';
+import { spawnSync } from 'child_process';
+
+/**
+ * Returns a Set of the *absolute* paths that any owning git repo's
+ * .gitignore rules consider ignored.
+ */
+export function batchGitignored(absPaths: string[]): Set<string> {
+  const ignored = new Set<string>();
+  if (absPaths.length === 0) return ignored;
+
+  // Bucket by owning git repo. Memoize directory → repo so a deep walk
+  // doesn't re-traverse parents for every file in the same dir.
+  const dirToRepo = new Map<string, string | null>();
+  const byRepo = new Map<string, string[]>();
+  for (const abs of absPaths) {
+    const dir = path.dirname(abs);
+    let repo = dirToRepo.get(dir);
+    if (repo === undefined) {
+      repo = findGitRoot(dir);
+      dirToRepo.set(dir, repo);
+    }
+    if (!repo) continue; // outside any git repo → not ignored
+    const list = byRepo.get(repo);
+    if (list) list.push(abs);
+    else byRepo.set(repo, [abs]);
+  }
+
+  for (const [repo, paths] of byRepo) {
+    // Pass paths as relative to the repo's root (git check-ignore
+    // accepts both, but relatives are friendlier with --stdin).
+    const rels = paths.map(p => path.relative(repo, p));
+    const r = spawnSync('git', ['check-ignore', '--stdin', '-z'], {
+      cwd: repo,
+      input: Buffer.from(rels.join('\0') + '\0'),
+    });
+    if (r.status === 128) continue;          // not a git repo (shouldn't happen here)
+    if (!r.stdout || r.stdout.length === 0) continue;
+    const lines = Buffer.isBuffer(r.stdout)
+      ? r.stdout.toString('utf8').split('\0')
+      : String(r.stdout).split('\0');
+    for (const line of lines) {
+      if (!line) continue;
+      ignored.add(path.resolve(repo, line));
+    }
+  }
+
+  return ignored;
+}
+
+/**
+ * Walk up from `start` looking for `.git` (either a directory for a
+ * regular repo OR a file for a submodule pointing into its parent's
+ * `modules/` storage). Returns the directory containing `.git`, or
+ * null if no ancestor has one.
+ */
+export function findGitRoot(start: string): string | null {
+  let dir = path.resolve(start);
+  while (true) {
+    const dotgit = path.join(dir, '.git');
+    if (fs.existsSync(dotgit)) return dir;
+    const parent = path.dirname(dir);
+    if (parent === dir) return null;
+    dir = parent;
+  }
+}

package/lib/wiki/index.ts

@@ -1,6 +1,7 @@
 import wikiInit from './init';
 import wikiScan from './scan';
 import wikiSurvey from './survey';
+import wikiSync from './sync';
 import wikiFetch from './fetch';
 import wikiIngest from './ingest';
 import wikiQuery from './query';
@@ -20,8 +21,8 @@ import { registerDaemonCommands } from './daemon';
 export function registerWikiCommands(program: any): void {
   const wiki = program.command('wiki').description('Karpathy-style LLM Wiki ops (macOS only in v1)');
 
-  wiki.command('init <scope>')
-    .description('bootstrap a vault scoped to <scope>/ — an existing data subdir of the project. Metadata is auto-created at ./rig-wiki/.')
+  wiki.command('init [scope]')
+    .description('bootstrap a vault under ./rig-wiki/. With no <scope>, the vault covers the whole CWD; with <scope>, only that subdir is scanned. Hidden/.gitignored/binary files are filtered at walk time regardless.')
     .action(wikiInit);
 
   wiki.command('scan')
@@ -38,6 +39,13 @@ export function registerWikiCommands(program: any): void {
     .option('--json', 'machine-readable output')
     .action(wikiSurvey);
 
+  wiki.command('sync')
+    .description('one-shot wiki update: scan → ingest NEW + MODIFIED → prune wiki pages for DELETED sources')
+    .option('--dry-run', 'print scan diff and what would change without writing anything')
+    .option('--no-prune', 'skip the delete step (only ingest NEW + MODIFIED)')
+    .option('--json', 'machine-readable output')
+    .action(wikiSync);
+
   wiki.command('fetch <url>')
     .description('verbatim download URL into raw/YYYY-MM-DD-<slug>.md')
     .option('--slug <slug>', 'override the auto-derived slug')

package/lib/wiki/init.ts

@@ -154,28 +154,30 @@ function defaultVaultConfig(scope: string, rootRel: string): VaultConfig {
 }
 
 export default function wikiInit(scope?: string): void {
-  if (!scope || !scope.trim()) {
-    print.error('rig wiki init requires a scope.');
-    print.info('usage: rig wiki init <scope>     (e.g. `rig wiki init personal` to ingest from ./personal/)');
-    print.info(`<scope> is an existing data subdir of the project. Vault metadata is auto-created at ./${VAULT_DIRNAME}/.`);
-    process.exit(1);
-  }
-
   const cwd = process.cwd();
   const vaultDir = path.join(cwd, VAULT_DIRNAME);
-  const scopeAbs = path.resolve(cwd, scope);
 
-  // The scope must already exist — pointing the wiki at a missing dir would
-  // hide what is almost certainly a typo.
-  if (!fs.existsSync(scopeAbs) || !fs.statSync(scopeAbs).isDirectory()) {
-    print.error(`scope dir not found: ${scope}`);
-    print.info(`expected an existing data subdir at ${shortPath(scopeAbs)}`);
-    process.exit(1);
-  }
-  // The scope can't be (or contain) the vault dir itself.
-  if (scopeAbs === vaultDir || vaultDir.startsWith(scopeAbs + path.sep)) {
-    print.error(`scope cannot be or contain the vault dir (${VAULT_DIRNAME}/).`);
-    process.exit(1);
+  // No-arg form: scope = the whole project (CWD). Vault scans up one level
+  // from `<CWD>/rig-wiki/` to reach CWD. Wiki name defaults to basename(CWD).
+  // Hidden segments, binary extensions, and .gitignored files are filtered
+  // by scan/survey at walk time — no scope restriction needed.
+  const hasScope = !!(scope && scope.trim());
+  const scopeAbs = hasScope ? path.resolve(cwd, scope!) : cwd;
+  const scopeName = hasScope ? scope!.trim() : path.basename(cwd);
+
+  if (hasScope) {
+    // The explicit scope must already exist — pointing the wiki at a missing
+    // dir would hide what is almost certainly a typo.
+    if (!fs.existsSync(scopeAbs) || !fs.statSync(scopeAbs).isDirectory()) {
+      print.error(`scope dir not found: ${scope}`);
+      print.info(`expected an existing data subdir at ${shortPath(scopeAbs)}`);
+      process.exit(1);
+    }
+    // The scope can't be (or contain) the vault dir itself.
+    if (scopeAbs === vaultDir || vaultDir.startsWith(scopeAbs + path.sep)) {
+      print.error(`scope cannot be or contain the vault dir (${VAULT_DIRNAME}/).`);
+      process.exit(1);
+    }
   }
 
   const guard = guardPath(vaultDir, cwd);
@@ -189,6 +191,7 @@ export default function wikiInit(scope?: string): void {
   // If the vault already has a config, it must already be scoped to the
   // same data dir — otherwise the user is trying to re-target an existing
   // vault, which we won't do silently. Manual config edit only.
+  // No-arg init resolves to the CWD itself, which equals `path.dirname(vaultDir)`.
   const cfgFile = vaultConfigPath(vaultDir);
   if (fs.existsSync(cfgFile)) {
     const existing = loadVaultConfig(vaultDir);
@@ -222,12 +225,13 @@ export default function wikiInit(scope?: string): void {
 
   if (!fs.existsSync(cfgFile)) {
     const rootRel = path.relative(vaultDir, scopeAbs);
-    saveVaultConfig(vaultDir, defaultVaultConfig(scope, rootRel));
+    saveVaultConfig(vaultDir, defaultVaultConfig(scopeName, rootRel));
   }
 
-  print.succeed(`vault initialized at ${shortPath(vaultDir)} (scope: ${scope})`);
+  const scopeLabel = hasScope ? `scope: ${scope}` : `scope: <project-wide, name "${scopeName}">`;
+  print.succeed(`vault initialized at ${shortPath(vaultDir)} (${scopeLabel})`);
   print.info(`next: edit ${shortPath(path.join(vaultDir, 'purpose.md'))} to describe what this wiki is for.`);
-  print.info(`then run \`rig wiki scan\` from anywhere inside ${shortPath(cwd)} to see what will be ingested.`);
+  print.info(`then run \`rig wiki sync\` from anywhere inside ${shortPath(cwd)} to ingest, update, and prune in one shot.`);
 }
 
 function writeIfMissing(file: string, content: string) {

package/lib/wiki/prune.ts

@@ -0,0 +1,145 @@
+// Prune wiki pages whose underlying source file has been deleted from the
+// scan root. Triggered by `rig wiki sync` (and exposed as a helper for
+// `rig wiki prune` if/when we add a standalone command).
+//
+// Three actions per deleted source path:
+//
+//   1. Delete `sources/<slug>.md` whose `source-path:` frontmatter resolves
+//      to the deleted file. The slug = basename of that wiki source page.
+//
+//   2. Scrub the deleted slug from every derived page's `sources: [...]`
+//      frontmatter array (entities/concepts/synthesis/queries). Pages that
+//      end up with an empty `sources: []` are left in place for `rig wiki
+//      lint` to surface as orphans — we don't auto-delete derived pages
+//      since they may carry the user's intellectual content distinct from
+//      any single source.
+//
+//   3. Drop the row from `state.db.source_sha` so future scans don't keep
+//      reporting the file as DELETED.
+
+import fs from 'fs';
+import path from 'path';
+import yaml from 'js-yaml';
+import { WikiEntry } from './config';
+import { deleteSourceSha } from './db';
+
+const DERIVED_DIRS = ['entities', 'concepts', 'synthesis', 'queries'];
+
+export interface PruneReport {
+  /** wiki-relative paths of source pages deleted */
+  deletedSourcePages: string[];
+  /** wiki-relative paths of derived pages whose sources[] lost the dropped slug */
+  scrubbedDerivedPages: string[];
+  /** root-relative paths of source_sha rows dropped */
+  shaRowsDropped: string[];
+}
+
+export function pruneDeletedSources(target: WikiEntry, deletedRelPaths: string[]): PruneReport {
+  const report: PruneReport = { deletedSourcePages: [], scrubbedDerivedPages: [], shaRowsDropped: [] };
+  if (deletedRelPaths.length === 0) return report;
+
+  const deletedSet = new Set(deletedRelPaths.map(normalizePath));
+  const sourcesDir = path.join(target.path, 'sources');
+  const droppedSlugs = new Set<string>();
+
+  // 1. sources/<slug>.md whose `source-path:` points at a deleted file
+  if (fs.existsSync(sourcesDir)) {
+    for (const file of fs.readdirSync(sourcesDir)) {
+      if (!file.endsWith('.md') || file === '.gitkeep') continue;
+      const abs = path.join(sourcesDir, file);
+      const fm = readFrontmatter(abs);
+      if (!fm) continue;
+      const rel = extractSourcePathRel(String(fm['source-path'] || ''));
+      if (rel && deletedSet.has(rel)) {
+        fs.rmSync(abs, { force: true });
+        report.deletedSourcePages.push(path.relative(target.path, abs));
+        droppedSlugs.add(path.basename(file, '.md'));
+      }
+    }
+  }
+
+  // 2. derived pages — strip dropped slugs from `sources: [...]`
+  if (droppedSlugs.size > 0) {
+    for (const sub of DERIVED_DIRS) {
+      const dir = path.join(target.path, sub);
+      if (!fs.existsSync(dir)) continue;
+      for (const file of fs.readdirSync(dir)) {
+        if (!file.endsWith('.md') || file === '.gitkeep') continue;
+        const abs = path.join(dir, file);
+        const content = fs.readFileSync(abs, 'utf8');
+        const next = scrubSlugsFromSourcesArray(content, droppedSlugs);
+        if (next !== content) {
+          fs.writeFileSync(abs, next, 'utf8');
+          report.scrubbedDerivedPages.push(path.relative(target.path, abs));
+        }
+      }
+    }
+  }
+
+  // 3. state.db cleanup
+  for (const p of deletedRelPaths) {
+    deleteSourceSha(target.name, p);
+    report.shaRowsDropped.push(p);
+  }
+
+  return report;
+}
+
+// ─── helpers ────────────────────────────────────────────────────────────
+
+function normalizePath(p: string): string {
+  return p.replace(/\\/g, '/');
+}
+
+function readFrontmatter(file: string): Record<string, unknown> | null {
+  let content: string;
+  try { content = fs.readFileSync(file, 'utf8'); } catch { return null; }
+  if (!content.startsWith('---\n')) return null;
+  const end = content.indexOf('\n---', 4);
+  if (end < 0) return null;
+  try {
+    const parsed = yaml.load(content.slice(4, end));
+    return parsed && typeof parsed === 'object' ? parsed as Record<string, unknown> : null;
+  } catch { return null; }
+}
+
+/**
+ * `source-path` can be:
+ *   - an obsidian:// URL:  `obsidian://open?vault=name&file=<encoded-path>`
+ *   - a plain root-relative path: `personal/work/foo.md`
+ * Returns the underlying root-relative path, or null if we can't extract one.
+ */
+function extractSourcePathRel(srcPath: string): string | null {
+  if (!srcPath) return null;
+  const m = srcPath.match(/[?&]file=([^&]+)/);
+  if (m) {
+    try { return normalizePath(decodeURIComponent(m[1])); }
+    catch { return normalizePath(m[1]); }
+  }
+  return normalizePath(srcPath);
+}
+
+/**
+ * Surgically rewrites the `sources: [a, b, c]` frontmatter line, removing
+ * any entries that appear in `slugs`. Leaves the rest of the frontmatter
+ * and body untouched. If no match, returns the content unchanged.
+ */
+function scrubSlugsFromSourcesArray(content: string, slugs: Set<string>): string {
+  if (!content.startsWith('---\n')) return content;
+  const end = content.indexOf('\n---', 4);
+  if (end < 0) return content;
+  const fm = content.slice(4, end);
+  const after = content.slice(end);
+
+  const m = fm.match(/^([ \t]*sources[ \t]*:[ \t]*)\[(.*?)\]/m);
+  if (!m) return content;
+
+  const items = m[2]
+    .split(',')
+    .map(s => s.trim().replace(/^['"]|['"]$/g, ''))
+    .filter(s => s && !slugs.has(s));
+
+  const newSources = `${m[1]}[${items.map(s => `'${s}'`).join(', ')}]`;
+  const newFm = fm.replace(m[0], newSources);
+  return '---\n' + newFm + after;
+}

package/lib/wiki/survey.ts

@@ -14,10 +14,10 @@
 
 import fs from 'fs';
 import path from 'path';
-import { spawnSync } from 'child_process';
 import print from '../print';
 import { requireVault, loadRigConfig, WikiEntry } from './config';
 import { isBinaryExtension } from './fileTypes';
+import { batchGitignored } from './gitignore';
 import { adapters } from './agent/registry';
 import { default as wikiIngest } from './ingest';
 
@@ -107,25 +107,10 @@ function collectCandidates(entry: WikiEntry): Candidate[] {
   }
   // Gitignore filter via batch `git check-ignore --stdin -z` (best-effort,
   // silent fallback outside a git repo)
-  const ignored = batchGitignored(root, out.map(c => c.abs));
+  const ignored = batchGitignored(out.map(c => c.abs));
   return out.filter(c => !ignored.has(c.abs));
 }
 
-function batchGitignored(root: string, abs: string[]): Set<string> {
-  const ignored = new Set<string>();
-  if (abs.length === 0) return ignored;
-  const r = spawnSync('git', ['check-ignore', '--stdin', '-z'], {
-    cwd: root,
-    input: Buffer.from(abs.join('\0') + '\0'),
-  });
-  if (r.status === 128 || !r.stdout || r.stdout.length === 0) return ignored;
-  const lines = Buffer.isBuffer(r.stdout)
-    ? r.stdout.toString('utf8').split('\0')
-    : String(r.stdout).split('\0');
-  for (const line of lines) if (line) ignored.add(path.resolve(root, line));
-  return ignored;
-}
-
 async function classifyWithAgent(target: WikiEntry, candidates: Candidate[]): Promise<SurveyRow[]> {
   const rig = loadRigConfig();
   const which = rig.wiki?.defaultAgent || 'claude';

package/lib/wiki/sync.ts

@@ -0,0 +1,111 @@
+// `rig wiki sync` — one-shot wiki update.
+//
+//   1. scan: compute NEW / MODIFIED / DELETED against state.db.source_sha
+//      (hidden + .gitignored + binary-extension filters apply at walk time,
+//      and the gitignore check is multi-repo-aware so git submodules are
+//      respected — see `lib/wiki/gitignore.ts`).
+//   2. ingest each NEW + MODIFIED source via the existing `wikiIngest`
+//      code path. Sequential so Claude doesn't double-write the same wiki
+//      page from concurrent runs, and so token spend is bounded.
+//   3. prune the wiki pages whose underlying source files have been
+//      deleted: removes `sources/<slug>.md`, strips that slug from derived
+//      pages' `sources: [...]` frontmatter, drops the source_sha row.
+//
+// RAW DRIFT (a `raw/` file's bytes changed in place) is surfaced as an
+// error — we do NOT try to "fix" it via re-ingest. raw/ is immutable by
+// design; drift means something else (a copy-on-write filesystem,
+// accidental edit, etc.) and the user needs to look.
+
+import path from 'path';
+import print from '../print';
+import { requireVault } from './config';
+import { recordLastRun } from './db';
+import { scanOne, ScanReport } from './scan';
+import { pruneDeletedSources, PruneReport } from './prune';
+import { default as wikiIngest } from './ingest';
+
+interface SyncOpts {
+  json?: boolean;
+  dryRun?: boolean;
+  noPrune?: boolean;
+}
+
+export default async function wikiSync(opts: SyncOpts): Promise<void> {
+  const target = requireVault();
+
+  // Step 1: scan. We don't baseline yet — ingest+prune are the ground truth
+  // for what gets recorded, and baselining before would mark unprocessed
+  // files as "known" prematurely.
+  const scan: ScanReport = scanOne(target, false);
+
+  if (scan.rawDrift.length > 0) {
+    print.error(`RAW DRIFT detected — ${scan.rawDrift.length} file(s) in raw/ changed. raw/ is immutable; resolve manually before syncing.`);
+    for (const p of scan.rawDrift) {
+      // eslint-disable-next-line no-console
+      console.log(`  ${p}`);
+    }
+    recordLastRun(target.name, 'sync', 10);
+    process.exit(10);
+  }
+
+  const toIngest = [...scan.new, ...scan.modified];
+  print.info(`sync: ${target.name}  NEW ${scan.new.length}  MODIFIED ${scan.modified.length}  DELETED ${scan.deleted.length}  UNCHANGED ${scan.unchanged}`);
+
+  // Step 2: ingest NEW + MODIFIED (one Claude call per file).
+  let ingestOk = 0, ingestFail = 0;
+  for (const rel of toIngest) {
+    const abs = path.resolve(target.root, rel);
+    print.start(`ingest ${rel}`);
+    try {
+      await wikiIngest(abs, { dryRun: !!opts.dryRun });
+      ingestOk++;
+    } catch (e) {
+      ingestFail++;
+      print.error(`ingest ${rel} failed: ${(e as Error).message}`);
+    }
+  }
+
+  // Step 3: prune DELETED unless --no-prune / --dry-run.
+  let prune: PruneReport = { deletedSourcePages: [], scrubbedDerivedPages: [], shaRowsDropped: [] };
+  if (scan.deleted.length > 0 && !opts.noPrune && !opts.dryRun) {
+    prune = pruneDeletedSources(target, scan.deleted);
+  }
+
+  // Step 4: report.
+  if (opts.json) {
+    // eslint-disable-next-line no-console
+    console.log(JSON.stringify({
+      ok: ingestFail === 0,
+      code: ingestFail === 0 ? 0 : 1,
+      data: {
+        wiki: target.name,
+        scan,
+        ingested: { ok: ingestOk, failed: ingestFail },
+        pruned: prune,
+        dryRun: !!opts.dryRun,
+      },
+    }, null, 2));
+  } else {
+    print.succeed(`sync: ingested ${ingestOk} ok, ${ingestFail} failed.`);
+    if (prune.deletedSourcePages.length > 0) {
+      print.info(`pruned ${prune.deletedSourcePages.length} source page(s):`);
+      for (const p of prune.deletedSourcePages) {
+        // eslint-disable-next-line no-console
+        console.log(`  − ${p}`);
+      }
+    }
+    if (prune.scrubbedDerivedPages.length > 0) {
+      print.info(`scrubbed deleted-source refs from ${prune.scrubbedDerivedPages.length} derived page(s):`);
+      for (const p of prune.scrubbedDerivedPages) {
+        // eslint-disable-next-line no-console
+        console.log(`  ~ ${p}`);
+      }
+    }
+    if (opts.dryRun && scan.deleted.length > 0) {
+      print.info(`dry-run: ${scan.deleted.length} delete(s) NOT applied. Re-run without --dry-run to prune.`);
+    }
+  }
+
+  recordLastRun(target.name, 'sync', ingestFail === 0 ? 0 : 1);
+  if (ingestFail > 0) process.exit(1);
+}

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "rigjs",
-  "version": "4.0.12",
-  "versionCode": 26052418,
+  "version": "4.0.14",
+  "versionCode": 26052420,
   "description": "A multi-repos dev tool based on yarn and git.Rigjs is intended to be the simplest way to develop,share and deliver codes between different developers or different projects.",
   "keywords": [
     "modular",