sdtk-brain-kit 0.1.0

package/src/lib/wiki-discover.js

@@ -0,0 +1,271 @@
+"use strict";
+
+const fs = require("fs");
+const path = require("path");
+const { CliError, ValidationError } = require("./errors");
+const { analyzePages } = require("./wiki-lint");
+const {
+  assertWikiWorkspaceWritePath,
+  getWikiProvenanceSourcesPath,
+  getWikiRawSourcesPath,
+  getWikiReportsPath,
+  getWikiWorkspacePath,
+  resolveProjectPath,
+} = require("./wiki-paths");
+
+const REPORT_PREFIX = "discover-plan";
+
+const CATEGORY_LABELS = {
+  missing_source_coverage: "Missing source coverage",
+  broken_internal_link_target: "Broken internal link target",
+  todo_open_questions_gaps: "TODO / Open Questions / Gaps",
+  weak_integration: "Weak integration",
+  stale_candidate_human_decision: "Stale candidate requiring human decision",
+  contradiction_candidate_source_verification: "Contradiction candidate requiring source verification",
+  raw_source_not_represented: "Raw source not represented in managed pages",
+};
+
+function todayStamp(date = new Date()) {
+  return date.toISOString().slice(0, 10);
+}
+
+function readJsonIfPresent(filePath, fallback = null) {
+  if (!fs.existsSync(filePath)) return fallback;
+  try {
+    return JSON.parse(fs.readFileSync(filePath, "utf-8"));
+  } catch (error) {
+    return { __error: `Could not parse JSON at ${filePath}: ${error.message}` };
+  }
+}
+
+function asArray(value) {
+  return Array.isArray(value) ? value : [];
+}
+
+function toPosix(value) {
+  return String(value || "").replace(/\\/g, "/");
+}
+
+function priorityForCategory(category) {
+  if (
+    category === "missing_source_coverage" ||
+    category === "stale_candidate_human_decision" ||
+    category === "contradiction_candidate_source_verification"
+  ) {
+    return "high";
+  }
+  if (category === "broken_internal_link_target" || category === "raw_source_not_represented") {
+    return "medium";
+  }
+  return "normal";
+}
+
+function sourceTypeForCategory(category) {
+  if (category === "broken_internal_link_target") {
+    return "missing internal page/link";
+  }
+  if (category === "raw_source_not_represented") {
+    return "local document to inspect";
+  }
+  if (category === "missing_source_coverage") {
+    return "local source coverage review";
+  }
+  return "stakeholder/manual review need";
+}
+
+function nextActionForCategory(category) {
+  switch (category) {
+    case "missing_source_coverage":
+      return "Review whether the referenced local source should be restored, re-ingested, or intentionally retired.";
+    case "broken_internal_link_target":
+      return "Decide whether to create the missing managed page or update the local link target.";
+    case "todo_open_questions_gaps":
+      return "Resolve the marker through local source review before compile/apply work.";
+    case "weak_integration":
+      return "Add local context, source references, or relationships in a later compile/apply workflow.";
+    case "stale_candidate_human_decision":
+      return "Review stale evidence before any future prune/archive issue is considered.";
+    case "contradiction_candidate_source_verification":
+      return "Verify the conflicting statement against local authoritative sources.";
+    case "raw_source_not_represented":
+      return "Consider whether this raw source should feed a later compile/additive update plan.";
+    default:
+      return "Review locally.";
+  }
+}
+
+function addPlanItems(items, category, evidenceItems, evidenceSource) {
+  for (const evidence of evidenceItems) {
+    items.push({
+      category,
+      evidence,
+      evidenceSource,
+      suggestedSourceType: sourceTypeForCategory(category),
+      priority: priorityForCategory(category),
+      confidence: "heuristic",
+      safetyMode: "local-only; manual-review",
+      nextAction: nextActionForCategory(category),
+    });
+  }
+}
+
+function collectRawSourceGaps(projectPath) {
+  const rawPath = getWikiRawSourcesPath(projectPath);
+  const provenancePath = getWikiProvenanceSourcesPath(projectPath);
+  const raw = readJsonIfPresent(rawPath, { sources: [] });
+  const provenance = readJsonIfPresent(provenancePath, { sources: [] });
+  const represented = new Set(
+    asArray(provenance.sources)
+      .filter((record) => record && record.sourcePath)
+      .map((record) => toPosix(record.sourcePath))
+  );
+
+  return asArray(raw.sources)
+    .filter((record) => record && record.sourcePath)
+    .filter((record) => !represented.has(toPosix(record.sourcePath)))
+    .map((record) => `Raw source \`${toPosix(record.sourcePath)}\` is registered but not represented in active provenance.`);
+}
+
+function buildPlanItems(projectPath) {
+  const analysis = analyzePages(projectPath);
+  const findings = analysis.findings;
+  const items = [];
+
+  addPlanItems(items, "missing_source_coverage", findings.provenance || [], "lint.provenance");
+  addPlanItems(items, "broken_internal_link_target", findings.brokenLinks || [], "lint.brokenLinks");
+  addPlanItems(items, "todo_open_questions_gaps", findings.markers || [], "lint.markers");
+  addPlanItems(items, "weak_integration", [...(findings.orphans || []), ...(findings.downstream || []), ...(findings.thin || [])], "lint.integration");
+  addPlanItems(items, "stale_candidate_human_decision", findings.stale || [], "lint.stale");
+  addPlanItems(items, "contradiction_candidate_source_verification", findings.contradictions || [], "lint.contradictions");
+  addPlanItems(items, "raw_source_not_represented", collectRawSourceGaps(projectPath), "raw.sources");
+
+  return {
+    pageCount: analysis.pageCount,
+    items: items.map((item, index) => ({
+      ...item,
+      planItemId: `discover-${String(index + 1).padStart(3, "0")}`,
+    })),
+  };
+}
+
+function summarizeByCategory(items) {
+  const summary = Object.fromEntries(Object.keys(CATEGORY_LABELS).map((key) => [key, 0]));
+  for (const item of items) {
+    summary[item.category] = (summary[item.category] || 0) + 1;
+  }
+  return summary;
+}
+
+function renderReport({ projectPath, workspacePath, generatedAt, pageCount, items }) {
+  const summary = summarizeByCategory(items);
+  const lines = [
+    "# SDTK-BRAIN Discovery Plan",
+    "",
+    `Generated: ${generatedAt}`,
+    `Project path: ${projectPath}`,
+    `Workspace path: ${workspacePath}`,
+    `Pages scanned: ${pageCount}`,
+    `Plan items: ${items.length}`,
+    "",
+    "Plan-only and local-only: No source was fetched, ingested, compiled, applied, pruned, deleted, archived, or persisted as query history.",
+    "No web or network discovery is executed. Future web research requires a separate controller-approved opt-in issue.",
+    "",
+    "## Input Evidence References",
+    "",
+    "- Direct lint/gap analysis of `.brain/pages`",
+    "- `.brain/provenance/sources.json` when present",
+    "- `.brain/raw/sources.json` when present",
+    "- BK-119 gap taxonomy",
+    "",
+    "## Gap Category Summary",
+    "",
+    "| Gap category | Count |",
+    "|---|---:|",
+    ...Object.entries(CATEGORY_LABELS).map(([key, label]) => `| \`${key}\` - ${label} | ${summary[key] || 0} |`),
+    "",
+    "## Plan Items",
+    "",
+  ];
+
+  if (items.length === 0) {
+    lines.push("No discovery plan items were generated from current local evidence.");
+  } else {
+    for (const item of items) {
+      lines.push(`### ${item.planItemId}`);
+      lines.push("");
+      lines.push(`- Gap category: \`${item.category}\``);
+      lines.push(`- Evidence source: \`${item.evidenceSource}\``);
+      lines.push(`- Evidence: ${item.evidence}`);
+      lines.push(`- Suggested source type: ${item.suggestedSourceType}`);
+      lines.push(`- Priority: ${item.priority}`);
+      lines.push(`- Confidence: ${item.confidence}`);
+      lines.push(`- Safety mode: ${item.safetyMode}`);
+      lines.push(`- Next action: ${item.nextAction}`);
+      lines.push("");
+    }
+  }
+
+  lines.push("");
+  lines.push("## Deferred Actions");
+  lines.push("");
+  lines.push("Fetch, ingest, compile, apply, prune, delete, archive, Ask changes, query persistence, and release work are outside BK-125.");
+  lines.push("");
+  return `${lines.join("\n").trimEnd()}\n`;
+}
+
+function ensureExistingWorkspace(projectPath) {
+  const workspacePath = getWikiWorkspacePath(projectPath);
+  if (!fs.existsSync(workspacePath) || !fs.statSync(workspacePath).isDirectory()) {
+    throw new ValidationError(
+      `No SDTK-BRAIN workspace found at ${workspacePath}. Run "sdtk-brain init" or "sdtk-brain atlas build" first. No project files were changed.`
+    );
+  }
+  return workspacePath;
+}
+
+function runWikiDiscoverPlan({ projectPath }) {
+  const resolvedProjectPath = resolveProjectPath(projectPath || process.cwd());
+  if (!fs.existsSync(resolvedProjectPath) || !fs.statSync(resolvedProjectPath).isDirectory()) {
+    throw new ValidationError(`--project-path is not a valid directory: ${resolvedProjectPath}`);
+  }
+
+  try {
+    const workspacePath = ensureExistingWorkspace(resolvedProjectPath);
+    const reportsPath = getWikiReportsPath(resolvedProjectPath);
+    assertWikiWorkspaceWritePath(reportsPath, resolvedProjectPath);
+
+    const generatedAt = new Date().toISOString();
+    const result = buildPlanItems(resolvedProjectPath);
+    const reportPath = path.join(reportsPath, `${REPORT_PREFIX}-${todayStamp(new Date(generatedAt))}.md`);
+    assertWikiWorkspaceWritePath(reportPath, resolvedProjectPath);
+    fs.mkdirSync(reportsPath, { recursive: true });
+    fs.writeFileSync(
+      reportPath,
+      renderReport({
+        projectPath: resolvedProjectPath,
+        workspacePath,
+        generatedAt,
+        pageCount: result.pageCount,
+        items: result.items,
+      }),
+      "utf-8"
+    );
+
+    return {
+      reportPath,
+      pageCount: result.pageCount,
+      items: result.items,
+      summary: summarizeByCategory(result.items),
+    };
+  } catch (error) {
+    if (error instanceof CliError) throw error;
+    throw new CliError(`Failed to write SDTK-BRAIN discovery plan report: ${error.message}`);
+  }
+}
+
+module.exports = {
+  REPORT_PREFIX,
+  buildPlanItems,
+  renderReport,
+  runWikiDiscoverPlan,
+};

package/src/lib/wiki-enrich.js

@@ -0,0 +1,264 @@
+"use strict";
+
+const fs = require("fs");
+const path = require("path");
+const { ValidationError } = require("./errors");
+const {
+  assertWikiWorkspaceWritePath,
+  getPreferredWikiContentPath,
+  getWikiReportsPath,
+  isPathInsideOrEqual,
+  resolveProjectPath,
+} = require("./wiki-paths");
+
+const REPORT_PREFIX = "github-enrichment-review";
+
+function toPosix(value) {
+  return String(value || "").replace(/\\/g, "/");
+}
+
+function todayStamp() {
+  return new Date().toISOString().slice(0, 10);
+}
+
+function parseFrontmatter(text) {
+  const source = String(text || "");
+  const match = source.match(/^---\r?\n([\s\S]*?)\r?\n---\r?\n?/);
+  if (!match) return { fields: {}, body: source };
+  const fields = {};
+  for (const line of match[1].split(/\r?\n/)) {
+    const separator = line.indexOf(":");
+    if (separator === -1) continue;
+    const key = line.slice(0, separator).trim();
+    const value = line.slice(separator + 1).trim();
+    fields[key] = value.replace(/^["']|["']$/g, "");
+  }
+  return { fields, body: source.slice(match[0].length) };
+}
+
+function parseInlineList(value) {
+  const text = String(value || "").trim();
+  if (!text || text === "[]") return [];
+  const inner = text.startsWith("[") && text.endsWith("]") ? text.slice(1, -1) : text;
+  return inner
+    .split(",")
+    .map((item) => item.trim().replace(/^["']|["']$/g, ""))
+    .filter(Boolean);
+}
+
+function collectMarkdownFiles(rootPath) {
+  const files = [];
+  if (!fs.existsSync(rootPath)) return files;
+  function visit(current) {
+    const stat = fs.statSync(current);
+    if (stat.isDirectory()) {
+      for (const child of fs.readdirSync(current).sort()) {
+        visit(path.join(current, child));
+      }
+      return;
+    }
+    if (stat.isFile() && current.toLowerCase().endsWith(".md")) {
+      files.push(current);
+    }
+  }
+  visit(rootPath);
+  return files.sort((a, b) => toPosix(a).localeCompare(toPosix(b)));
+}
+
+function extractGithubRepos(text) {
+  const repos = [];
+  const seen = new Set();
+  const matcher = /https?:\/\/github\.com\/([A-Za-z0-9](?:[A-Za-z0-9-]{0,38}))\/([A-Za-z0-9._-]+)/gi;
+  let match;
+  while ((match = matcher.exec(String(text || ""))) !== null) {
+    const owner = match[1];
+    const repo = match[2].replace(/[).,;:]+$/g, "").replace(/\.git$/i, "");
+    if (!repo || repo.includes("...")) continue;
+    const url = `https://github.com/${owner}/${repo}`;
+    const key = url.toLowerCase();
+    if (seen.has(key)) continue;
+    seen.add(key);
+    repos.push({ owner, repo, url });
+  }
+  return repos;
+}
+
+function firstSection(body, heading) {
+  const escaped = String(heading).replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
+  const matcher = new RegExp(`^##\\s+${escaped}\\s*$([\\s\\S]*?)(?=^##\\s+|$)`, "im");
+  const match = String(body || "").match(matcher);
+  if (!match) return "";
+  return match[1].replace(/\s+/g, " ").trim();
+}
+
+function extractProvenanceRefs(body) {
+  const section = firstSection(body, "Provenance");
+  if (!section) return [];
+  return section
+    .split(/\s*-\s+/)
+    .map((item) => item.trim())
+    .filter((item) => item.startsWith("prov_"))
+    .slice(0, 12);
+}
+
+function recordFromPage(filePath, contentRoot) {
+  const text = fs.readFileSync(filePath, "utf-8");
+  const parsed = parseFrontmatter(text);
+  const fields = parsed.fields;
+  const repos = extractGithubRepos(text);
+  if (repos.length === 0) return [];
+
+  const relPath = toPosix(path.relative(contentRoot.path, filePath));
+  const type = String(fields.type || "");
+  const title = String(fields.title || path.basename(filePath, ".md"));
+  const tags = parseInlineList(fields.tags);
+  const sourceRefs = parseInlineList(fields.source_refs);
+  const summary = firstSection(parsed.body, "Summary");
+  const provenanceRefs = extractProvenanceRefs(parsed.body);
+
+  return repos.map((repo) => ({
+    record_type: "sdtk_wiki_github_enrichment_candidate",
+    status: "review_only_local_identity",
+    page_path: `${toPosix(contentRoot.relative)}/${relPath}`,
+    page_type: type || "unknown",
+    page_title: title,
+    repo_url: repo.url,
+    source_url: repo.url,
+    owner: repo.owner,
+    repo: repo.repo,
+    local_summary: summary || null,
+    local_topics: tags,
+    source_refs: sourceRefs,
+    provenance_refs: provenanceRefs,
+    confidence: fields.confidence || "unknown",
+    metadata: {
+      stars: null,
+      license: null,
+      description: null,
+      current_owner: null,
+      default_branch: null,
+      last_verified_at: null,
+    },
+    fetch_timestamp: null,
+    fetch_status: "not_fetched_local_review_mode",
+    failure_status: "not_fetched_local_review_mode",
+    failure_reason: "Network metadata was not requested in review-only local mode.",
+    enrichment_mode: "review",
+    mutation: "none",
+  }));
+}
+
+function uniqueRecords(records) {
+  const byRepo = new Map();
+  for (const record of records) {
+    const key = record.repo_url.toLowerCase();
+    const existing = byRepo.get(key);
+    if (!existing) {
+      byRepo.set(key, record);
+      continue;
+    }
+    existing.page_path = existing.page_path || record.page_path;
+    existing.local_topics = Array.from(new Set([...existing.local_topics, ...record.local_topics])).sort();
+    existing.source_refs = Array.from(new Set([...existing.source_refs, ...record.source_refs])).sort();
+    existing.provenance_refs = Array.from(new Set([...existing.provenance_refs, ...record.provenance_refs])).sort();
+  }
+  return [...byRepo.values()].sort((a, b) => a.repo_url.localeCompare(b.repo_url));
+}
+
+function renderMarkdownReport({ projectPath, reportJsonPath, records, contentRoot }) {
+  const lines = [
+    "# SDTK-BRAIN GitHub Enrichment Review",
+    "",
+    `Date: ${todayStamp()}`,
+    `Project root: \`${projectPath}\``,
+    `Wiki content root: \`${contentRoot.path}\``,
+    `Wiki content mode: \`${contentRoot.mode}\``,
+    `JSON report: \`${reportJsonPath}\``,
+    "",
+    "Report-only and non-destructive: this command does not fetch network metadata, rewrite pages, mutate sources, or touch `.brain-legacy-unused`.",
+    "Network-backed GitHub verification is deferred to a later controller-approved slice.",
+    "",
+    "## Summary",
+    "",
+    `- candidates: ${records.length}`,
+    "- source: github",
+    "- mode: review",
+    "- fetch status: not_fetched_local_review_mode",
+    "",
+    "## Candidates",
+    "",
+    "| Repo | Page | Confidence | Source refs | Fetch status | Pending metadata |",
+    "|---|---|---|---:|---|---|",
+  ];
+
+  for (const record of records) {
+    lines.push(
+      `| [${record.owner}/${record.repo}](${record.repo_url}) | \`${record.page_path}\` | ${record.confidence} | ${record.source_refs.length} | ${record.fetch_status} | stars, license, description, current owner |`
+    );
+  }
+  if (records.length === 0) {
+    lines.push("| none | - | - | 0 | - | - |");
+  }
+
+  lines.push(
+    "",
+    "## Review Notes",
+    "",
+    "- Treat every candidate as locally identified only until a future explicit network/enrichment apply issue verifies it.",
+    "- Do not copy stars, license, or activity claims into generated pages from this report unless those fields are later verified.",
+    ""
+  );
+  return lines.join("\n");
+}
+
+function runWikiGithubEnrichmentReview(options = {}) {
+  const projectPath = resolveProjectPath(options.projectPath || process.cwd());
+  if (!fs.existsSync(projectPath) || !fs.statSync(projectPath).isDirectory()) {
+    throw new ValidationError(`--project-path is not a valid directory: ${projectPath}`);
+  }
+
+  const contentRoot = getPreferredWikiContentPath(projectPath);
+  if (!isPathInsideOrEqual(contentRoot.path, projectPath)) {
+    throw new ValidationError("Refusing to read local wiki pages outside the project root.");
+  }
+  if (!fs.existsSync(contentRoot.path) || !fs.statSync(contentRoot.path).isDirectory()) {
+    throw new ValidationError(
+      `No SDTK-BRAIN local wiki found at ${contentRoot.path}. Run "sdtk-brain ingest <source-root>" and "sdtk-brain compile --mode safe --apply" first.`
+    );
+  }
+
+  const records = uniqueRecords(collectMarkdownFiles(contentRoot.path).flatMap((filePath) =>
+    recordFromPage(filePath, contentRoot)
+  ));
+
+  const reportsRoot = getWikiReportsPath(projectPath);
+  assertWikiWorkspaceWritePath(reportsRoot, projectPath);
+  fs.mkdirSync(reportsRoot, { recursive: true });
+  const jsonPath = path.join(reportsRoot, `${REPORT_PREFIX}-${todayStamp()}.json`);
+  const markdownPath = path.join(reportsRoot, `${REPORT_PREFIX}-${todayStamp()}.md`);
+  assertWikiWorkspaceWritePath(jsonPath, projectPath);
+  assertWikiWorkspaceWritePath(markdownPath, projectPath);
+
+  const payload = {
+    schema_version: 1,
+    record_type: "sdtk_wiki_github_enrichment_review",
+    generated_at: new Date().toISOString(),
+    project_path: projectPath,
+    wiki_content_path: contentRoot.path,
+    wiki_content_mode: contentRoot.mode,
+    source: "github",
+    mode: "review",
+    network_fetch: false,
+    mutation: "none",
+    candidate_count: records.length,
+    records,
+  };
+  fs.writeFileSync(jsonPath, JSON.stringify(payload, null, 2) + "\n", "utf-8");
+  fs.writeFileSync(markdownPath, renderMarkdownReport({ projectPath, reportJsonPath: jsonPath, records, contentRoot }), "utf-8");
+
+  return { markdownPath, jsonPath, records, projectPath };
+}
+
+module.exports = {
+  runWikiGithubEnrichmentReview,
+};