sdtk-brain-kit 0.1.0

package/src/lib/wiki-paths.js

@@ -0,0 +1,256 @@
+"use strict";
+
+const path = require("path");
+
+const WIKI_WORKSPACE_RELATIVE = ".brain";
+const WIKI_GRAPH_RELATIVE = path.join(".brain", "graph");
+const WIKI_MANIFEST_RELATIVE = path.join(".brain", "manifest.json");
+const WIKI_PAGES_RELATIVE = path.join(".brain", "pages");
+const WIKI_RAW_RELATIVE = path.join(".brain", "raw");
+const WIKI_RAW_DESCRIPTORS_RELATIVE = path.join(".brain", "raw", "descriptors");
+const WIKI_RAW_SOURCES_RELATIVE = path.join(".brain", "raw", "sources.json");
+const WIKI_PROVENANCE_RELATIVE = path.join(".brain", "provenance");
+const WIKI_PROVENANCE_INGEST_EVENTS_RELATIVE = path.join(".brain", "provenance", "ingest-events.json");
+const WIKI_PROVENANCE_SOURCES_RELATIVE = path.join(".brain", "provenance", "sources.json");
+const WIKI_QUERIES_RELATIVE = path.join(".brain", "queries");
+const WIKI_REPORTS_RELATIVE = path.join(".brain", "reports");
+const WIKI_LOGS_RELATIVE = path.join(".brain", "logs");
+const CANONICAL_WIKI_RELATIVE = "wiki";
+const LEGACY_PERSONAL_BRAIN_RELATIVE = path.join(".brain", "personal-brain");
+const LEGACY_ATLAS_RELATIVE = path.join(".sdtk", "atlas");
+
+function resolveProjectPath(projectPath) {
+  return path.resolve(projectPath || process.cwd());
+}
+
+function normalizeComparablePath(targetPath) {
+  const resolved = path.resolve(targetPath);
+  return process.platform === "win32" ? resolved.toLowerCase() : resolved;
+}
+
+function isPathInsideOrEqual(targetPath, rootPath) {
+  const comparableTarget = normalizeComparablePath(targetPath);
+  const comparableRoot = normalizeComparablePath(rootPath);
+  return (
+    comparableTarget === comparableRoot ||
+    comparableTarget.startsWith(comparableRoot + path.sep)
+  );
+}
+
+function assertPathInsideOrEqual(
+  targetPath,
+  rootPath,
+  message = "Refusing to access a path outside the allowed root"
+) {
+  const resolvedTarget = path.resolve(targetPath);
+  if (!isPathInsideOrEqual(resolvedTarget, rootPath)) {
+    throw new Error(`${message}: ${resolvedTarget}`);
+  }
+  return resolvedTarget;
+}
+
+function getWikiWorkspacePath(projectPath) {
+  return path.join(resolveProjectPath(projectPath), WIKI_WORKSPACE_RELATIVE);
+}
+
+function getCanonicalWikiPath(projectPath) {
+  return path.join(resolveProjectPath(projectPath), CANONICAL_WIKI_RELATIVE);
+}
+
+function getLegacyPersonalBrainPath(projectPath) {
+  return path.join(resolveProjectPath(projectPath), LEGACY_PERSONAL_BRAIN_RELATIVE);
+}
+
+function getPreferredWikiContentPath(projectPath) {
+  const canonical = getCanonicalWikiPath(projectPath);
+  const legacy = getLegacyPersonalBrainPath(projectPath);
+  try {
+    const fs = require("fs");
+    if (fs.existsSync(canonical) && fs.statSync(canonical).isDirectory()) {
+      return {
+        path: canonical,
+        mode: "canonical_project_wiki",
+        relative: CANONICAL_WIKI_RELATIVE,
+      };
+    }
+    if (fs.existsSync(legacy) && fs.statSync(legacy).isDirectory()) {
+      return {
+        path: legacy,
+        mode: "legacy_personal_brain_fallback",
+        relative: LEGACY_PERSONAL_BRAIN_RELATIVE,
+      };
+    }
+  } catch (_) {
+    // Callers perform their own existence validation and error reporting.
+  }
+  return {
+    path: canonical,
+    mode: "canonical_project_wiki",
+    relative: CANONICAL_WIKI_RELATIVE,
+  };
+}
+
+// BK-318: ordered content roots for the unified query surface.
+// The atlas pages mirror (full original content, built by `atlas build`) comes
+// first; the compiled canonical wiki (or its legacy personal-brain fallback)
+// second. Union of both is the one logical corpus query/search/ask ground on.
+function getWikiContentRoots(projectPath) {
+  const fs = require("fs");
+  const roots = [];
+  const atlasPages = getWikiPagesPath(projectPath);
+  try {
+    if (
+      fs.existsSync(atlasPages) &&
+      fs.statSync(atlasPages).isDirectory() &&
+      fs.readdirSync(atlasPages).length > 0
+    ) {
+      // An empty pages dir (scaffolded by `init --no-build`) is not a store yet.
+      roots.push({ path: atlasPages, store: "atlas", mode: "atlas_pages" });
+    }
+  } catch (_) {
+    // fall through — callers validate emptiness themselves
+  }
+  const preferred = getPreferredWikiContentPath(projectPath);
+  try {
+    if (fs.existsSync(preferred.path) && fs.statSync(preferred.path).isDirectory()) {
+      roots.push({ path: preferred.path, store: "wiki", mode: preferred.mode });
+    }
+  } catch (_) {
+    // fall through
+  }
+  return roots;
+}
+
+function getWikiGraphPath(projectPath) {
+  return path.join(resolveProjectPath(projectPath), WIKI_GRAPH_RELATIVE);
+}
+
+function getWikiManifestPath(projectPath) {
+  return path.join(resolveProjectPath(projectPath), WIKI_MANIFEST_RELATIVE);
+}
+
+function getWikiPagesPath(projectPath) {
+  return path.join(resolveProjectPath(projectPath), WIKI_PAGES_RELATIVE);
+}
+
+function getWikiRawPath(projectPath) {
+  return path.join(resolveProjectPath(projectPath), WIKI_RAW_RELATIVE);
+}
+
+function getWikiRawDescriptorsPath(projectPath) {
+  return path.join(resolveProjectPath(projectPath), WIKI_RAW_DESCRIPTORS_RELATIVE);
+}
+
+function getWikiRawSourcesPath(projectPath) {
+  return path.join(resolveProjectPath(projectPath), WIKI_RAW_SOURCES_RELATIVE);
+}
+
+function getWikiProvenancePath(projectPath) {
+  return path.join(resolveProjectPath(projectPath), WIKI_PROVENANCE_RELATIVE);
+}
+
+function getWikiProvenanceIngestEventsPath(projectPath) {
+  return path.join(resolveProjectPath(projectPath), WIKI_PROVENANCE_INGEST_EVENTS_RELATIVE);
+}
+
+function getWikiProvenanceSourcesPath(projectPath) {
+  return path.join(resolveProjectPath(projectPath), WIKI_PROVENANCE_SOURCES_RELATIVE);
+}
+
+function getWikiQueriesPath(projectPath) {
+  return path.join(resolveProjectPath(projectPath), WIKI_QUERIES_RELATIVE);
+}
+
+function getWikiReportsPath(projectPath) {
+  return path.join(resolveProjectPath(projectPath), WIKI_REPORTS_RELATIVE);
+}
+
+function getWikiLogsPath(projectPath) {
+  return path.join(resolveProjectPath(projectPath), WIKI_LOGS_RELATIVE);
+}
+
+function getLegacyAtlasPath(projectPath) {
+  return path.join(resolveProjectPath(projectPath), LEGACY_ATLAS_RELATIVE);
+}
+
+function assertWikiWorkspaceWritePath(targetPath, projectPath) {
+  return assertPathInsideOrEqual(
+    targetPath,
+    getWikiWorkspacePath(projectPath),
+    "Refusing to write outside project-local .brain workspace"
+  );
+}
+
+function assertCanonicalWikiWritePath(targetPath, projectPath) {
+  return assertPathInsideOrEqual(
+    targetPath,
+    getCanonicalWikiPath(projectPath),
+    "Refusing to write outside project-local wiki output"
+  );
+}
+
+function describeWikiPaths(projectPath) {
+  return {
+    projectPath: resolveProjectPath(projectPath),
+    canonicalWikiPath: getCanonicalWikiPath(projectPath),
+    legacyPersonalBrainPath: getLegacyPersonalBrainPath(projectPath),
+    wikiWorkspacePath: getWikiWorkspacePath(projectPath),
+    wikiGraphPath: getWikiGraphPath(projectPath),
+    wikiManifestPath: getWikiManifestPath(projectPath),
+    wikiPagesPath: getWikiPagesPath(projectPath),
+    wikiRawPath: getWikiRawPath(projectPath),
+    wikiRawDescriptorsPath: getWikiRawDescriptorsPath(projectPath),
+    wikiRawSourcesPath: getWikiRawSourcesPath(projectPath),
+    wikiProvenancePath: getWikiProvenancePath(projectPath),
+    wikiProvenanceIngestEventsPath: getWikiProvenanceIngestEventsPath(projectPath),
+    wikiProvenanceSourcesPath: getWikiProvenanceSourcesPath(projectPath),
+    wikiQueriesPath: getWikiQueriesPath(projectPath),
+    wikiReportsPath: getWikiReportsPath(projectPath),
+    wikiLogsPath: getWikiLogsPath(projectPath),
+    legacyAtlasPath: getLegacyAtlasPath(projectPath),
+  };
+}
+
+module.exports = {
+  CANONICAL_WIKI_RELATIVE,
+  LEGACY_ATLAS_RELATIVE,
+  LEGACY_PERSONAL_BRAIN_RELATIVE,
+  WIKI_GRAPH_RELATIVE,
+  WIKI_LOGS_RELATIVE,
+  WIKI_MANIFEST_RELATIVE,
+  WIKI_PAGES_RELATIVE,
+  WIKI_RAW_DESCRIPTORS_RELATIVE,
+  WIKI_RAW_RELATIVE,
+  WIKI_RAW_SOURCES_RELATIVE,
+  WIKI_PROVENANCE_INGEST_EVENTS_RELATIVE,
+  WIKI_PROVENANCE_RELATIVE,
+  WIKI_PROVENANCE_SOURCES_RELATIVE,
+  WIKI_QUERIES_RELATIVE,
+  WIKI_REPORTS_RELATIVE,
+  WIKI_WORKSPACE_RELATIVE,
+  assertPathInsideOrEqual,
+  assertCanonicalWikiWritePath,
+  assertWikiWorkspaceWritePath,
+  describeWikiPaths,
+  getCanonicalWikiPath,
+  getLegacyPersonalBrainPath,
+  getLegacyAtlasPath,
+  getPreferredWikiContentPath,
+  getWikiContentRoots,
+  getWikiGraphPath,
+  getWikiLogsPath,
+  getWikiManifestPath,
+  getWikiPagesPath,
+  getWikiProvenanceIngestEventsPath,
+  getWikiProvenancePath,
+  getWikiProvenanceSourcesPath,
+  getWikiQueriesPath,
+  getWikiRawDescriptorsPath,
+  getWikiRawPath,
+  getWikiRawSourcesPath,
+  getWikiReportsPath,
+  getWikiWorkspacePath,
+  isPathInsideOrEqual,
+  normalizeComparablePath,
+  resolveProjectPath,
+};

package/src/lib/wiki-score.js

@@ -0,0 +1,64 @@
+"use strict";
+
+// Shared deterministic relevance scorer (BK-318).
+//
+// One implementation used by BOTH surfaces that rank content against a query:
+//   - `sdtk-wiki search` / `query` (lib/wiki-search.js) over markdown files
+//   - `sdtk-wiki ask` source selection (lib/wiki-ask.js) over atlas index docs
+// Extracted from wiki-search.js so the two can never drift apart.
+// Exact-phrase + token-overlap scoring; no LLM, no network, fully offline.
+
+function normalizeText(value) {
+  return String(value || "").toLowerCase();
+}
+
+function tokenize(query) {
+  return normalizeText(query)
+    .split(/[^a-z0-9À-ỹ_]+/i)
+    .map((part) => part.trim())
+    .filter((part) => part.length >= 2);
+}
+
+function scoreFile({ text, title, relativePath, query, tokens }) {
+  const lowerText = normalizeText(text);
+  const lowerTitle = normalizeText(title);
+  const lowerPath = normalizeText(relativePath);
+  const phrase = normalizeText(query);
+  const reasons = [];
+  let score = 0;
+
+  if (phrase && lowerText.includes(phrase)) {
+    score += 50;
+    reasons.push("exact phrase match in page content");
+  }
+  if (phrase && lowerTitle.includes(phrase)) {
+    score += 30;
+    reasons.push("exact phrase match in title");
+  }
+  if (phrase && lowerPath.includes(phrase)) {
+    score += 20;
+    reasons.push("exact phrase match in path");
+  }
+
+  let matchedTokens = 0;
+  for (const token of tokens) {
+    const inText = lowerText.includes(token);
+    const inTitle = lowerTitle.includes(token);
+    const inPath = lowerPath.includes(token);
+    if (inText || inTitle || inPath) {
+      matchedTokens += 1;
+      score += inTitle ? 12 : inPath ? 8 : 5;
+    }
+  }
+  if (matchedTokens > 0) {
+    reasons.push(`matched ${matchedTokens}/${tokens.length} query token(s)`);
+  }
+
+  return { score, reasons };
+}
+
+module.exports = {
+  normalizeText,
+  tokenize,
+  scoreFile,
+};

package/src/lib/wiki-search.js

@@ -0,0 +1,213 @@
+"use strict";
+
+const fs = require("fs");
+const path = require("path");
+const { ValidationError } = require("./errors");
+const {
+  getWikiContentRoots,
+  isPathInsideOrEqual,
+  resolveProjectPath,
+} = require("./wiki-paths");
+const { normalizeText, tokenize, scoreFile } = require("./wiki-score");
+
+const DEFAULT_LIMIT = 10;
+const CANONICAL_WIKI_RELATIVE = "wiki";
+const LEGACY_PERSONAL_BRAIN_RELATIVE = path.join(".sdtk", "wiki", "personal-brain");
+
+function toPosix(value) {
+  return String(value || "").replace(/\\/g, "/");
+}
+
+function collectMarkdownFiles(rootPath) {
+  const files = [];
+  function visit(current) {
+    const stat = fs.statSync(current);
+    if (stat.isDirectory()) {
+      for (const child of fs.readdirSync(current).sort()) {
+        visit(path.join(current, child));
+      }
+      return;
+    }
+    if (stat.isFile() && /\.md(?:arkdown)?$/i.test(current)) {
+      files.push(current);
+    }
+  }
+  visit(rootPath);
+  return files.sort((a, b) => toPosix(a).localeCompare(toPosix(b)));
+}
+
+function extractTitle(text, filePath) {
+  const heading = text.match(/^#\s+(.+?)\s*$/m);
+  if (heading) return heading[1].trim();
+  return path.basename(filePath, path.extname(filePath)).replace(/[-_]+/g, " ").trim();
+}
+
+function snippetFor(text, query, tokens) {
+  const lower = normalizeText(text);
+  const phrase = normalizeText(query);
+  let index = phrase ? lower.indexOf(phrase) : -1;
+  if (index < 0) {
+    for (const token of tokens) {
+      index = lower.indexOf(token);
+      if (index >= 0) break;
+    }
+  }
+  if (index < 0) {
+    return text.replace(/\s+/g, " ").trim().slice(0, 180);
+  }
+  const start = Math.max(0, index - 80);
+  const end = Math.min(text.length, index + 180);
+  return text.slice(start, end).replace(/\s+/g, " ").trim();
+}
+
+// BK-318 union de-dup: identify the underlying source a page describes.
+// Atlas pages carry `source_path:` (repo-relative). Compiled wiki source pages
+// carry `aliases: ["<logical path>"]` (relative to their ingest root). A
+// compiled page is a duplicate of an atlas page when the atlas source_path
+// equals — or path-suffix-matches — one of its aliases (atlas wins).
+function extractFrontmatterBlock(text) {
+  const match = String(text || "").match(/^---\n([\s\S]*?)\n---/);
+  return match ? match[1] : "";
+}
+
+function extractSourceKey(text) {
+  const fm = extractFrontmatterBlock(text);
+  const sourcePath = fm.match(/^source_path:\s*"?([^"\n]+)"?\s*$/m);
+  if (sourcePath) {
+    return { kind: "source_path", value: toPosix(sourcePath[1]).toLowerCase() };
+  }
+  const aliases = fm.match(/^aliases:\s*\[\s*"([^"\n]+)"/m);
+  if (aliases) {
+    return { kind: "alias", value: toPosix(aliases[1]).toLowerCase() };
+  }
+  return null;
+}
+
+function isDuplicateOfAtlas(aliasValue, atlasSourcePaths) {
+  if (atlasSourcePaths.has(aliasValue)) return true;
+  for (const sourcePath of atlasSourcePaths) {
+    if (sourcePath.endsWith(`/${aliasValue}`)) return true;
+  }
+  return false;
+}
+
+function runWikiSearch({ projectPath, query, limit = DEFAULT_LIMIT }) {
+  const resolvedProjectPath = resolveProjectPath(projectPath || process.cwd());
+  if (!fs.existsSync(resolvedProjectPath) || !fs.statSync(resolvedProjectPath).isDirectory()) {
+    throw new ValidationError(`--project-path is not a valid directory: ${resolvedProjectPath}`);
+  }
+
+  const normalizedQuery = String(query || "").trim();
+  if (!normalizedQuery) {
+    throw new ValidationError('sdtk-brain search requires a query, for example: sdtk-brain search --project-path <path> "multi-agent".');
+  }
+
+  const parsedLimit = Number.parseInt(limit, 10);
+  const safeLimit = Number.isFinite(parsedLimit) && parsedLimit > 0 ? Math.min(parsedLimit, 50) : DEFAULT_LIMIT;
+
+  const contentRoots = getWikiContentRoots(resolvedProjectPath);
+  for (const root of contentRoots) {
+    if (!isPathInsideOrEqual(root.path, resolvedProjectPath)) {
+      throw new ValidationError("Refusing to search outside the project root.");
+    }
+  }
+  if (contentRoots.length === 0) {
+    throw new ValidationError(
+      `No SDTK-BRAIN local wiki found under ${resolvedProjectPath}. ` +
+        'Run "sdtk-brain init" / "sdtk-brain atlas build" to index your project docs (searched at .brain/pages), ' +
+        'or "sdtk-brain ingest <source-root>" and "sdtk-brain compile --mode safe --apply" for the compiled wiki. ' +
+        "Legacy .brain/personal-brain workspaces are still readable when present."
+    );
+  }
+
+  const tokens = tokenize(normalizedQuery);
+
+  // Pass 1 — read every candidate once, keyed by root; collect atlas source paths.
+  const candidates = [];
+  const atlasSourcePaths = new Set();
+  let scannedFiles = 0;
+  for (const root of contentRoots) {
+    for (const filePath of collectMarkdownFiles(root.path)) {
+      scannedFiles += 1;
+      const text = fs.readFileSync(filePath, "utf-8");
+      const sourceKey = extractSourceKey(text);
+      if (root.store === "atlas" && sourceKey && sourceKey.kind === "source_path") {
+        atlasSourcePaths.add(sourceKey.value);
+      }
+      candidates.push({ filePath, text, store: root.store, sourceKey });
+    }
+  }
+
+  // Pass 2 — score, suppressing compiled-wiki duplicates of atlas sources.
+  const matches = [];
+  for (const candidate of candidates) {
+    if (
+      candidate.store === "wiki" &&
+      candidate.sourceKey &&
+      isDuplicateOfAtlas(candidate.sourceKey.value, atlasSourcePaths)
+    ) {
+      continue;
+    }
+    const relativePath = toPosix(path.relative(resolvedProjectPath, candidate.filePath));
+    const title = extractTitle(candidate.text, candidate.filePath);
+    const scored = scoreFile({
+      text: candidate.text,
+      title,
+      relativePath,
+      query: normalizedQuery,
+      tokens,
+    });
+    if (scored.score <= 0) continue;
+    matches.push({
+      path: relativePath,
+      title,
+      store: candidate.store,
+      score: scored.score,
+      why: scored.reasons.join("; "),
+      snippet: snippetFor(candidate.text, normalizedQuery, tokens),
+    });
+  }
+
+  matches.sort((a, b) => {
+    if (b.score !== a.score) return b.score - a.score;
+    if (a.store !== b.store) return a.store === "atlas" ? -1 : 1;
+    return a.path.localeCompare(b.path);
+  });
+
+  const hasAtlas = contentRoots.some((root) => root.store === "atlas");
+  const wikiRoot = contentRoots.find((root) => root.store === "wiki");
+  const primaryRoot = contentRoots[0];
+
+  return {
+    query: normalizedQuery,
+    projectPath: resolvedProjectPath,
+    wikiContentPath: primaryRoot.path,
+    wikiContentMode: primaryRoot.mode,
+    contentRoots: contentRoots.map((root) => ({
+      path: root.path,
+      store: root.store,
+      mode: root.mode,
+    })),
+    personalBrainPath: primaryRoot.path,
+    scannedFiles,
+    matches: matches.slice(0, safeLimit),
+    totalMatches: matches.length,
+    limit: safeLimit,
+    searchMode: hasAtlas
+      ? (wikiRoot
+        ? "local_deterministic_union_atlas_and_wiki_markdown"
+        : "local_deterministic_atlas_pages_markdown")
+      : (primaryRoot.mode === "canonical_project_wiki"
+        ? "local_deterministic_project_wiki_markdown"
+        : "local_deterministic_legacy_personal_brain_markdown"),
+    premiumRequired: false,
+    mutated: false,
+  };
+}
+
+module.exports = {
+  CANONICAL_WIKI_RELATIVE,
+  LEGACY_PERSONAL_BRAIN_RELATIVE,
+  runWikiSearch,
+  tokenize,
+};

package/templates/VAULT_CLAUDE.md

@@ -0,0 +1,39 @@
+# CLAUDE.md — Vault Operating Contract
+
+This folder is a **second-brain vault** managed with `sdtk-brain` (rails) and an
+AI agent (the knowledge engine — you). Read this contract before real work.
+
+## Two layers that never blur
+
+- **`raw/`** — immutable source documents (articles, papers, repos, notes,
+  meeting transcripts). You read from here; you never edit meaning here.
+- **`wiki/`** — the compiled knowledge layer (sources, concepts, entities,
+  comparisons, syntheses, decisions, queries, dashboards). You maintain this
+  freely, in markdown, with relative links (Obsidian-compatible).
+- **`workspace/`** — scratch space; nothing here is permanent knowledge.
+- **`.brain/`** — machine state written by `sdtk-brain` (reports, graph,
+  provenance). Never edit it by hand.
+
+## Core workflow
+
+```
+add source        →  raw/inbox/<file>
+ingest            →  sdtk-brain ingest raw/inbox
+compile           →  sdtk-brain compile --mode safe --apply
+ask a question    →  YOU read wiki/, synthesize the answer, cite pages
+file the answer   →  wiki/queries/<page> or wiki/syntheses/<page>
+lint periodically →  sdtk-brain lint  (fix orphans, broken links, stale claims)
+view the graph    →  sdtk-brain open
+```
+
+## Rules for the agent
+
+1. Answers must be grounded in `wiki/` (and `raw/` when needed) with page
+   citations — label anything unverified as unverified.
+2. New knowledge goes into `wiki/` as pages with frontmatter; keep reciprocal
+   `related_pages` links so the graph stays connected.
+3. Never mutate `raw/` content; corrections live in `wiki/` with a note.
+4. External claims (stars, licenses, activity of repos) stay unverified until
+   an explicit enrichment step confirms them.
+5. `sdtk-brain` commands are the only writers of `.brain/`; you are the only
+   writer of prose in `wiki/`.

package/templates/VAULT_README.md

@@ -0,0 +1,18 @@
+# Second-Brain Vault
+
+A local-first, markdown-first knowledge vault: `raw/` holds immutable sources,
+`wiki/` holds the compiled knowledge layer, and your AI agent (reading
+`CLAUDE.md`) is the knowledge engine. `sdtk-brain` provides the deterministic
+rails: ingest bookkeeping, compile, search, lint, and a docs/graph viewer.
+
+Quick start:
+
+```
+sdtk-brain ingest raw/inbox
+sdtk-brain compile --mode safe --apply
+sdtk-brain search "<topic>"
+sdtk-brain open
+```
+
+Open this folder in your agent (or Obsidian) and ask questions — everything is
+local: no LLM in the CLI, no network, no telemetry.