sdocs-dev 1.6.2 → 1.12.0

package/lib/library-ephemeral.js

@@ -0,0 +1,111 @@
+// Hardcoded list of OS paths that auto-delete files. Two categories:
+//
+//   ephemeralRoots() - paths a user might legitimately put work in
+//   (`/tmp`, `~/.Trash`). Files here get a *rescue copy* taken into
+//   ~/.sdocs/library/rescued/ at index time, because the original is
+//   going to vanish on the OS's schedule.
+//
+//   throwawayRoots() - OS-managed scratch directories like
+//   `os.tmpdir()` on macOS (which resolves to a per-user folder under
+//   /var/folders/...). Nothing meaningful lives here long enough to
+//   index; it is where Playwright sandboxes, build tools, and other
+//   transient processes write their working files. Anything under
+//   these roots is skipped entirely at index time - no rescue, no
+//   library entry. This is what stops test-run pollution from
+//   accumulating in the user's library.
+//
+// Both checks realpath() the input so macOS's /private/var/folders/...
+// alias does not slip through.
+
+const os   = require('os');
+const fs   = require('fs');
+const path = require('path');
+
+function homedir() { return os.homedir(); }
+
+// Resolve all known forms of a path so the `/private/...` macOS alias
+// is matched the same as the bare form.
+function expandAliases(p) {
+  if (!p) return [];
+  const out = new Set();
+  const resolved = path.resolve(p);
+  out.add(resolved);
+  try {
+    const real = fs.realpathSync(resolved);
+    if (real) out.add(real);
+  } catch (_) { /* path may not exist; resolved form is enough */ }
+  return [...out];
+}
+
+function ephemeralRoots() {
+  const roots = new Set();
+  for (const r of expandAliases(os.tmpdir())) roots.add(r);
+
+  if (process.platform === 'darwin') {
+    roots.add('/tmp');
+    roots.add('/private/tmp');
+    roots.add('/var/tmp');
+    roots.add('/private/var/tmp');
+    roots.add(path.join(homedir(), '.Trash'));
+  } else if (process.platform === 'linux') {
+    roots.add('/tmp');
+    roots.add('/var/tmp');
+    roots.add('/run');
+    roots.add('/dev/shm');
+    roots.add(path.join(homedir(), '.cache'));
+  } else if (process.platform === 'win32') {
+    if (process.env.TEMP) roots.add(path.resolve(process.env.TEMP));
+    if (process.env.TMP)  roots.add(path.resolve(process.env.TMP));
+    roots.add('C:\\Windows\\Temp');
+  }
+
+  return [...roots].filter(Boolean);
+}
+
+// OS scratch directories - never index files from here, even with
+// rescue. These are where test runners, build tools, and short-lived
+// processes write their working files; the user does not put meaningful
+// long-lived documents under them.
+function throwawayRoots() {
+  const roots = new Set();
+  for (const r of expandAliases(os.tmpdir())) roots.add(r);
+  if (process.platform === 'darwin') {
+    // /var/folders/<u>/<gid>/T is per-user macOS tmp; the /private alias
+    // is the realpath, the un-prefixed form is the symlink. Cover both.
+    roots.add('/var/folders');
+    roots.add('/private/var/folders');
+  } else if (process.platform === 'linux') {
+    roots.add('/run');
+    roots.add('/dev/shm');
+  } else if (process.platform === 'win32') {
+    if (process.env.TEMP) roots.add(path.resolve(process.env.TEMP));
+    if (process.env.TMP)  roots.add(path.resolve(process.env.TMP));
+    roots.add('C:\\Windows\\Temp');
+  }
+  return [...roots].filter(Boolean);
+}
+
+function pathUnder(absPath, roots) {
+  if (!absPath) return false;
+  for (const candidate of expandAliases(absPath)) {
+    for (const root of roots) {
+      if (candidate === root || candidate.startsWith(root + path.sep)) return true;
+    }
+  }
+  return false;
+}
+
+function isEphemeralPath(absPath) {
+  return pathUnder(absPath, ephemeralRoots());
+}
+
+function isThrowawayPath(absPath) {
+  return pathUnder(absPath, throwawayRoots());
+}
+
+module.exports = {
+  ephemeralRoots,
+  throwawayRoots,
+  isEphemeralPath,
+  isThrowawayPath,
+};

package/lib/library-index.js

@@ -0,0 +1,280 @@
+// Top-level library operations. Brings the modules together:
+//   - read a markdown file
+//   - extract title, body excerpt, tags, agent metadata
+//   - if the file is in an ephemeral location, take a rescue copy
+//   - upsert into the index
+//
+// Also handles the YAML front-matter edit when the CLI passes tags.
+
+const fs   = require('fs');
+const path = require('path');
+const crypto = require('crypto');
+
+const SDocYaml      = require('../shared/sdocs-yaml.js');
+const SDocLibTags   = require('../shared/sdocs-library-tags.js');
+
+const store        = require('./library-store');
+const scanner      = require('./library-scan');
+const ephemeral    = require('./library-ephemeral');
+const paths        = require('./library-paths');
+
+const MAX_EXCERPT = 400;
+
+function deriveTitle(meta, body) {
+  if (meta && typeof meta.title === 'string' && meta.title.trim()) return meta.title.trim();
+  const m = (body || '').match(/^#\s+(.+?)\s*$/m);
+  if (m) return m[1];
+  return null;
+}
+
+function bodyExcerpt(body) {
+  if (!body) return '';
+  const cleaned = body
+    .replace(/```[\s\S]*?```/g, '')
+    .replace(/`[^`\n]+`/g, '')
+    .replace(/^#{1,6} .*$/gm, '')
+    .replace(/\s+/g, ' ')
+    .trim();
+  if (cleaned.length <= MAX_EXCERPT) return cleaned;
+  return cleaned.slice(0, MAX_EXCERPT) + '...';
+}
+
+function readFileSafe(absPath) {
+  try { return fs.readFileSync(absPath, 'utf8'); } catch (_) { return null; }
+}
+
+function detectGitProject(absPath) {
+  let dir = path.dirname(absPath);
+  for (let i = 0; i < 30; i++) {
+    if (fs.existsSync(path.join(dir, '.git'))) {
+      let branch = null;
+      try {
+        const head = fs.readFileSync(path.join(dir, '.git', 'HEAD'), 'utf8').trim();
+        const m = head.match(/^ref:\s+refs\/heads\/(.+)$/);
+        if (m) branch = m[1];
+      } catch (_) {}
+      return { project: path.basename(dir), root: dir, branch };
+    }
+    const parent = path.dirname(dir);
+    if (parent === dir) break;
+    dir = parent;
+  }
+  return { project: null, root: null, branch: null };
+}
+
+function ensureRescueCopy(absPath, content) {
+  const dest = path.join(paths.rescuedDir(),
+    crypto.createHash('sha1').update(absPath).digest('hex').slice(0, 12) + '-' +
+    path.basename(absPath));
+  store.ensureDir(path.dirname(dest));
+  fs.writeFileSync(dest, content);
+  return dest;
+}
+
+// Parse, merge tags, and produce a normalised entry from a file's content
+// plus metadata. Pure (no fs writes), so tests can drive it directly.
+function buildEntry({ absPath, content, addTags, stats, rescuedFrom = null }) {
+  const parsed = SDocYaml.parseFrontMatter(content || '');
+  const meta = parsed.meta || {};
+  const body = parsed.body || '';
+
+  const fmTags = Array.isArray(meta.tags) ? meta.tags : [];
+  const tags = SDocLibTags.mergeTags(fmTags, addTags || []);
+
+  const git = detectGitProject(absPath);
+  const sdocsMeta = meta['sdocs-library'] && typeof meta['sdocs-library'] === 'object'
+    ? meta['sdocs-library'] : {};
+
+  return {
+    id:         store.idForPath(absPath),
+    path:       absPath,
+    rescued:    !!rescuedFrom,
+    rescuedFrom,
+    title:      deriveTitle(meta, body) || path.basename(absPath, path.extname(absPath)),
+    bodyExcerpt: bodyExcerpt(body),
+    body:       body,
+    tags:       tags,
+    mtime:      stats ? new Date(stats.mtimeMs).toISOString() : new Date().toISOString(),
+    size:       stats ? stats.size : (content ? Buffer.byteLength(content) : 0),
+    gitProject: git.project,
+    gitBranch:  git.branch,
+    agent:      typeof meta.agent === 'string' ? meta.agent : sdocsMeta['agent'] || null,
+    sessionId:  sdocsMeta['session-id'] || null,
+    resumeCmd:  sdocsMeta['resume-cmd'] || null,
+  };
+}
+
+// Add tags to a file's `tags:` front matter, write the file back.
+// Returns the new content. If no tags would change the file, returns
+// null (caller can skip the write).
+function injectTagsIntoFile(absPath, addTags) {
+  if (!addTags || !addTags.length) return null;
+  const raw = readFileSafe(absPath);
+  if (raw == null) return null;
+  const parsed = SDocYaml.parseFrontMatter(raw);
+  const meta = parsed.meta || {};
+  const existing = Array.isArray(meta.tags) ? meta.tags : [];
+  const merged = SDocLibTags.mergeTags(existing, addTags);
+  const same = existing.length === merged.length && existing.every((t, i) => merged[i] === t);
+  if (same) return null;
+  meta.tags = merged;
+  const out = SDocYaml.serializeFrontMatter(meta) + '\n' + (parsed.body || '');
+  fs.writeFileSync(absPath, out);
+  return out;
+}
+
+// Remove tags from a file's front matter. Returns the new tag list or
+// null if the file was missing / had no tags to remove.
+function removeTagsFromFile(absPath, removeTags) {
+  if (!removeTags || !removeTags.length) return null;
+  const raw = readFileSafe(absPath);
+  if (raw == null) return null;
+  const parsed = SDocYaml.parseFrontMatter(raw);
+  const meta = parsed.meta || {};
+  const existing = Array.isArray(meta.tags) ? meta.tags : [];
+  if (!existing.length) return null;
+  const drop = new Set(removeTags.map(t => String(t).toLowerCase()));
+  const next = existing.filter(t => !drop.has(String(t).toLowerCase()));
+  if (next.length === existing.length) return null;
+  if (next.length) meta.tags = next; else delete meta.tags;
+  const out = SDocYaml.serializeFrontMatter(meta) + '\n' + (parsed.body || '');
+  fs.writeFileSync(absPath, out);
+  return next;
+}
+
+// Per-file opt-out: front matter `sdocs-library: false` skips indexing.
+function isOptedOut(content) {
+  if (!content) return false;
+  const parsed = SDocYaml.parseFrontMatter(content);
+  const meta = parsed.meta || {};
+  const v = meta['sdocs-library'];
+  return v === false || v === 'false';
+}
+
+// Add or update a single file. Returns the upserted entry, or null if
+// the file was skipped.
+function indexFile(absPath, { addTags } = {}) {
+  const resolved = path.resolve(absPath);
+  if (!fs.existsSync(resolved)) return null;
+
+  // OS scratch directories (macOS /var/folders, Linux /run, Windows
+  // %TEMP%) never make it into the library. Test runners and build
+  // tools write here constantly; rescue copies would just accumulate.
+  // The escape hatch is for the test suite, which legitimately uses
+  // mkdtemp under os.tmpdir() to isolate fixtures and needs indexing
+  // to work against those paths.
+  if (ephemeral.isThrowawayPath(resolved) && !process.env.SDOCS_ALLOW_THROWAWAY_INDEXING) {
+    return null;
+  }
+
+  if (addTags && addTags.length) {
+    injectTagsIntoFile(resolved, addTags);
+  }
+
+  let content = readFileSafe(resolved);
+  if (content == null) return null;
+  if (isOptedOut(content)) return null;
+
+  let rescuedFrom = null;
+  let entryPath = resolved;
+  if (ephemeral.isEphemeralPath(resolved)) {
+    const rescued = ensureRescueCopy(resolved, content);
+    rescuedFrom = resolved;
+    entryPath = rescued;
+  }
+
+  let stats;
+  try { stats = fs.statSync(entryPath); } catch (_) { stats = null; }
+
+  const entry = buildEntry({
+    absPath: entryPath, content, addTags, stats, rescuedFrom,
+  });
+  return store.upsertEntry(entry);
+}
+
+// Drop entries whose underlying file is gone. For rescued entries the
+// rescue copy under ~/.sdocs/library/rescued/ is the source of truth -
+// if that's missing the entry can't be opened, so it's pruned even if
+// rescuedFrom happens to still exist. Returns the count removed.
+function pruneMissing() {
+  const idx = store.loadIndex();
+  let removed = 0;
+  for (const e of idx.entries.slice()) {
+    if (!e.path || !fs.existsSync(e.path)) {
+      store.removeEntry(e.id);
+      removed++;
+    }
+  }
+  return removed;
+}
+
+// Bulk scan: walk roots, index each .md found, then prune entries
+// whose files have disappeared since the last scan. Returns counts.
+function scanAndIndex({ roots, excludes, maxFileSize } = {}) {
+  const found = scanner.scan({ roots, excludes, maxFileSize });
+  let added = 0, updated = 0;
+  for (const f of found) {
+    const before = store.getEntry(store.idForPath(f.path));
+    indexFile(f.path);
+    if (before) updated++; else added++;
+  }
+  const removed = pruneMissing();
+  const state = store.loadState();
+  state.lastScanAt = Date.now();
+  store.saveState(state);
+  return { scanned: found.length, added, updated, removed };
+}
+
+function rebuild() {
+  store.clearIndex();
+  return scanAndIndex();
+}
+
+// Tags used by entries whose path is under a given prefix (the project
+// path, typically). For the file-info-card autocomplete and `sdoc
+// library ls --tags`. Both the prefix and each entry path are tested
+// against their realpaths too so a symlinked /var on macOS doesn't
+// hide entries from the tag bag.
+function tagsUnderPrefix(prefix) {
+  const root = path.resolve(prefix);
+  let rootReal = root;
+  try { rootReal = fs.realpathSync(root); } catch (_) {}
+  // Special case: the filesystem root ('/' on posix, 'C:\' on win) is
+  // already its own separator, so `root + path.sep` becomes '//' which
+  // matches nothing. Treat it as "everything".
+  const rootIsFsRoot = root === path.sep || /^[A-Za-z]:[\\/]$/.test(root);
+  const sep = path.sep;
+  function under(p) {
+    if (rootIsFsRoot) return true;
+    if (!p) return false;
+    if (p === root || p.startsWith(root + sep)) return true;
+    if (rootReal !== root && (p === rootReal || p.startsWith(rootReal + sep))) return true;
+    return false;
+  }
+  const counts = {};
+  for (const e of store.loadIndex().entries) {
+    const p = e.rescued && e.rescuedFrom ? e.rescuedFrom : e.path;
+    let pReal = p;
+    try { pReal = fs.realpathSync(p); } catch (_) {}
+    if (under(p) || under(pReal)) {
+      for (const t of e.tags || []) counts[t] = (counts[t] || 0) + 1;
+    }
+  }
+  return Object.entries(counts)
+    .sort((a, b) => b[1] - a[1] || a[0].localeCompare(b[0]))
+    .map(([tag, count]) => ({ tag, count }));
+}
+
+module.exports = {
+  indexFile,
+  pruneMissing,
+  scanAndIndex,
+  rebuild,
+  buildEntry,
+  injectTagsIntoFile,
+  removeTagsFromFile,
+  tagsUnderPrefix,
+  isOptedOut,
+  bodyExcerpt,
+  deriveTitle,
+};

package/lib/library-paths.js

@@ -0,0 +1,20 @@
+// Canonical paths the library subsystem uses. One module so tests can
+// rebind for sandboxing (set SDOCS_HOME to redirect everything into a
+// temp directory).
+
+const path = require('path');
+const os   = require('os');
+
+function root() {
+  if (process.env.SDOCS_HOME) return process.env.SDOCS_HOME;
+  return path.join(os.homedir(), '.sdocs');
+}
+
+module.exports = {
+  root,
+  libraryDir:    () => path.join(root(), 'library'),
+  rescuedDir:    () => path.join(root(), 'library', 'rescued'),
+  indexFile:     () => path.join(root(), 'library-index.json'),
+  stateFile:     () => path.join(root(), 'library-state.json'),
+  configFile:    () => path.join(root(), 'library.yaml'),
+};

package/lib/library-scan.js

@@ -0,0 +1,258 @@
+// Filesystem walker for the library. Walks one or more roots, returning
+// the absolute paths of .md files that pass the size cap and ignore
+// rules. Pure: takes config and returns files; doesn't touch the index.
+//
+// For v1 we walk everything. Mtime-based shortcuts can come once it
+// proves slow in practice.
+
+const fs   = require('fs');
+const path = require('path');
+const os   = require('os');
+
+const ephemeral = require('./library-ephemeral');
+const paths     = require('./library-paths');
+
+const DEFAULT_MAX_SIZE = 1 * 1024 * 1024;
+
+const DIRNAME_BLOCKLIST = new Set([
+  'node_modules',
+  '.git',
+  '.svn',
+  '.hg',
+  'dist',
+  'build',
+  'vendor',
+  '.venv',
+  '.next',
+  '.cache',
+  '__pycache__',
+  'target',
+  '.gradle',
+  '.idea',
+  '.vscode',
+  '.DS_Store',
+  '.Trash',
+  // Directories whose contents are sensitive even when the user
+  // happens to have them outside a hidden-dir ancestor. Belt-and-
+  // suspenders for the hidden-dir rule above.
+  '.ssh',
+  '.aws',
+  '.gnupg',
+  '.docker',
+  '.kube',
+  '.gcloud',
+  '.azure',
+  '.bitwarden',
+  '.password-store',
+]);
+
+// File basenames that should never make it into the library, regardless
+// of which directory they live in or what extension they carry. Most of
+// these don't have a markdown extension and so the existing extension
+// filter already drops them - but the deny list is the right place for
+// the rule, and is in position for when more extensions get indexed.
+//
+// Crucially, the credentials/secrets patterns require a config-file
+// extension (json/yaml/env/...) - they will NOT match `.md` because
+// markdown is a notes format and "company-secrets.md" or
+// "credentials-handling.md" are legitimate user notes about secrets,
+// not the secrets themselves.
+const DENY_BASENAME_PATTERNS = [
+  // SSH private/public keys
+  /^id_rsa(\.pub)?$/i,
+  /^id_ed25519(\.pub)?$/i,
+  /^id_ecdsa(\.pub)?$/i,
+  /^id_dsa(\.pub)?$/i,
+  /\.ppk$/i,
+  // Environment files (.env, .env.local, .env.production, ...)
+  /^\.env(\..+)?$/i,
+  // Cryptographic material
+  /\.(key|pem|p12|pfx|cer|crt|jks|keystore)$/i,
+  // PGP / GPG
+  /\.(gpg|pgp|asc)$/i,
+  // Password databases / wallets
+  /\.(kdbx|kdb|agilekeychain|1pif)$/i,
+  /^wallet\.dat$/i,
+  // Credential files: literal name (no extension - this is how
+  // git/aws/gh store them) or with a config / data extension.
+  /(^|[._-])credentials$/i,
+  /(^|[._-])credentials\.(json|yaml|yml|toml|ini|env|conf|cfg|key|txt|sh|properties)$/i,
+  // Files literally named secret(s).<config-ext>. Does NOT match
+  // `secret.md` or other markdown notes ABOUT secrets.
+  /^secrets?\.(json|yaml|yml|toml|ini|env|conf|cfg|key|txt|sh|properties)$/i,
+  // api_secret.json, api-secret.yaml, apisecret.txt, ...
+  /^api[_-]?secret\.(json|yaml|yml|toml|ini|env|conf|cfg|key|txt)$/i,
+  // Common single-file secrets in $HOME
+  /^\.netrc$/i,
+  /^\.pgpass$/i,
+  /^\.htpasswd$/i,
+];
+
+function systemSkipPaths() {
+  const home = os.homedir();
+  const out = new Set();
+  if (process.platform === 'darwin') {
+    out.add('/Applications');
+    out.add('/System');
+    out.add('/Library');
+    out.add('/usr');
+    out.add('/private');
+    out.add(path.join(home, 'Library'));
+  } else if (process.platform === 'linux') {
+    out.add('/usr');
+    out.add('/var');
+    out.add('/etc');
+    out.add('/proc');
+    out.add('/sys');
+    out.add('/boot');
+    out.add(path.join(home, '.local'));
+    out.add(path.join(home, '.config'));
+  } else if (process.platform === 'win32') {
+    out.add('C:\\Program Files');
+    out.add('C:\\Program Files (x86)');
+    out.add('C:\\Windows');
+    out.add(path.join(home, 'AppData'));
+  }
+  return out;
+}
+
+// Exported so the agent's file-read and bridge-spawn endpoints can
+// apply the same deny rule on the path they're handed.
+function deniedByPattern(absPath) {
+  const base = path.basename(absPath);
+  if (DENY_BASENAME_PATTERNS.some(re => re.test(base))) return true;
+  const segs = absPath.split(path.sep);
+  for (const s of segs) {
+    if (DIRNAME_BLOCKLIST.has(s)) return true;
+  }
+  return false;
+}
+
+function shouldSkipDir(absDir, base, skipSet, exemptRoots) {
+  if (base.startsWith('.') && base !== '.' && base !== '..') return true;
+  if (DIRNAME_BLOCKLIST.has(base)) return true;
+  if (skipSet.has(absDir)) return true;
+  // Skip ephemeral paths during descent unless we're inside a root that
+  // the caller explicitly named (in which case they want it scanned).
+  if (ephemeral.isEphemeralPath(absDir) && !insideAnyRoot(absDir, exemptRoots)) return true;
+  if (absDir === paths.root() || absDir.startsWith(paths.root() + path.sep)) return true;
+  return false;
+}
+
+function insideAnyRoot(absDir, roots) {
+  if (!roots || !roots.length) return false;
+  for (const r of roots) {
+    if (absDir === r || absDir.startsWith(r + path.sep)) return true;
+  }
+  return false;
+}
+
+function defaultRoots() {
+  return [os.homedir()];
+}
+
+// .sdocsignore: per-directory dotfile that excludes files / directories
+// from indexing. Subset of gitignore syntax:
+//   - blank lines and lines starting with # are ignored
+//   - trailing / means "directory only"
+//   - * matches any run of non-/ characters
+//   - ** matches across directory boundaries
+//   - patterns without a / match against basename only
+//   - patterns with a / match against the path relative to the .sdocsignore
+// Negation (!) and other gitignore niceties are not supported in v1.
+function parseSdocsignore(dir) {
+  let raw;
+  try { raw = fs.readFileSync(path.join(dir, '.sdocsignore'), 'utf8'); }
+  catch (_) { return null; }
+  const out = [];
+  for (let line of raw.split(/\r?\n/)) {
+    line = line.trim();
+    if (!line || line.startsWith('#')) continue;
+    let dirOnly = false;
+    if (line.endsWith('/')) { dirOnly = true; line = line.slice(0, -1); }
+    const hasSlash = line.includes('/');
+    const re = new RegExp('^' + line
+      .replace(/[.+^${}()|[\]\\]/g, '\\$&')
+      .replace(/\*\*/g, '__GLOBSTAR__')
+      .replace(/\*/g, '[^/]*')
+      .replace(/\?/g, '[^/]')
+      .replace(/__GLOBSTAR__/g, '.*') + '$');
+    out.push({ re, dirOnly, anchored: hasSlash, dir });
+  }
+  return out.length ? out : null;
+}
+
+function matchesIgnoreStack(stack, absPath, isDir) {
+  for (const layer of stack) {
+    const rel = path.relative(layer.dir, absPath);
+    if (!rel || rel.startsWith('..')) continue; // outside this .sdocsignore's scope
+    const base = path.basename(absPath);
+    for (const p of layer.patterns) {
+      if (p.dirOnly && !isDir) continue;
+      const target = p.anchored ? rel : base;
+      if (p.re.test(target)) return true;
+    }
+  }
+  return false;
+}
+
+// Walk synchronously and accumulate matches. Cheap enough for personal
+// libraries; we'll revisit if a user reports a slow scan.
+function scan({ roots, excludes, maxFileSize } = {}) {
+  const rootsToWalk = (roots && roots.length ? roots : defaultRoots()).map(p => path.resolve(p));
+  const exSet = new Set((excludes || []).map(p => path.resolve(p)));
+  for (const sys of systemSkipPaths()) exSet.add(sys);
+  const limit = maxFileSize || DEFAULT_MAX_SIZE;
+
+  const found = [];
+
+  function walk(dir, ignoreStack) {
+    let entries;
+    try {
+      entries = fs.readdirSync(dir, { withFileTypes: true });
+    } catch (_) {
+      return;
+    }
+    // Pick up .sdocsignore in this directory (cumulative with ancestors).
+    const here = parseSdocsignore(dir);
+    const stack = here ? ignoreStack.concat([{ dir, patterns: here }]) : ignoreStack;
+
+    for (const ent of entries) {
+      const full = path.join(dir, ent.name);
+      if (ent.isSymbolicLink()) continue;
+      if (matchesIgnoreStack(stack, full, ent.isDirectory())) continue;
+      if (ent.isDirectory()) {
+        if (!shouldSkipDir(full, ent.name, exSet, rootsToWalk)) walk(full, stack);
+        continue;
+      }
+      if (!ent.isFile()) continue;
+      if (deniedByPattern(full)) continue;
+      if (!/\.(md|mdx|markdown)$/i.test(ent.name)) continue;
+      let st;
+      try { st = fs.statSync(full); } catch (_) { continue; }
+      if (st.size > limit) continue;
+      found.push({ path: full, mtime: st.mtimeMs, size: st.size });
+    }
+  }
+
+  for (const r of rootsToWalk) {
+    let st;
+    try { st = fs.statSync(r); } catch (_) { continue; }
+    if (!st.isDirectory()) continue;
+    walk(r, []);
+  }
+  return found;
+}
+
+module.exports = {
+  scan,
+  defaultRoots,
+  systemSkipPaths,
+  DIRNAME_BLOCKLIST,
+  DENY_BASENAME_PATTERNS,
+  DEFAULT_MAX_SIZE,
+  shouldSkipDir,
+  deniedByPattern,
+  parseSdocsignore,
+  matchesIgnoreStack,
+};