sdtk-brain-kit 0.1.0

package/src/lib/wiki-extract.js

@@ -0,0 +1,1313 @@
+"use strict";
+
+const crypto = require("crypto");
+const fs = require("fs");
+const path = require("path");
+const { CliError, ValidationError } = require("./errors");
+const {
+  assertWikiWorkspaceWritePath,
+  getWikiReportsPath,
+  resolveProjectPath,
+} = require("./wiki-paths");
+
+const REPORT_PREFIX = "semantic-extraction-dry-run";
+const SCHEMA_VERSION = 1;
+const EXCLUDE_FRAGS = [
+  ".git",
+  ".brain",
+  ".brain-legacy-unused",
+  "node_modules",
+  ".venv",
+  "venv",
+  "dist",
+  "build",
+  "coverage",
+  ".next",
+  ".turbo",
+  ".cache",
+  "__pycache__",
+];
+
+const CONCEPT_RULES = [
+  {
+    concept_id: "concept_self_hosted_project_management",
+    name: "self-hosted project management",
+    aliases: ["Trello alternative", "realtime collaboration", "project management"],
+    keywords: ["trello", "jira", "project management", "realtime collaboration", "collaboration"],
+    category: "project_management",
+  },
+  {
+    concept_id: "concept_agent_skills",
+    name: "agent skills",
+    aliases: ["AI agent skills", "skill pack", "coding agent skills"],
+    keywords: ["skill", "skills", "agent", "claude code", "codex", "cursor", "gemini"],
+    category: "agent_skills",
+  },
+  {
+    concept_id: "concept_secret_scanning",
+    name: "secret scanning",
+    aliases: ["API key detection", "token detection", "credential scanning"],
+    keywords: ["gitleaks", "secret", "token", "password", "api key"],
+    category: "secret_scanning",
+  },
+  {
+    concept_id: "concept_self_hosted_local_first_tools",
+    name: "self-hosted local-first tools",
+    aliases: ["local-first", "privacy", "self-hosted"],
+    keywords: ["self-hosted", "local", "privacy", "offline"],
+    category: "local_first_privacy",
+  },
+  {
+    concept_id: "concept_ai_media_generation",
+    name: "AI media generation",
+    aliases: ["AI video", "AI music", "generative media"],
+    keywords: ["video", "music", "suno", "lipdub", "comfyui", "ace-step"],
+    category: "ai_media",
+  },
+  {
+    concept_id: "concept_document_management",
+    name: "document management",
+    aliases: ["paperless", "PDF workflow", "document archive"],
+    keywords: ["paperless", "document", "pdf", "ocr", "archive"],
+    category: "document_management",
+  },
+  {
+    concept_id: "concept_developer_tooling",
+    name: "developer tooling",
+    aliases: ["open source developer tool", "CLI", "framework"],
+    keywords: ["github", "repo", "open-source", "mã nguồn", "cli", "framework", "developer"],
+    category: "developer_tooling",
+  },
+];
+
+function toPosix(value) {
+  return String(value || "").replace(/\\/g, "/");
+}
+
+function isRemoteUrl(value) {
+  return /^(?:https?|ftp):\/\//i.test(String(value || ""));
+}
+
+function sha256(value) {
+  return crypto.createHash("sha256").update(String(value)).digest("hex");
+}
+
+function timestampStamp(date = new Date()) {
+  return date.toISOString().replace(/[-:]/g, "").replace(/\.\d{3}Z$/, "Z").replace("T", "-");
+}
+
+function safeSlug(value, fallback = "item") {
+  const slug = String(value || "")
+    .normalize("NFKD")
+    .replace(/[\u0300-\u036f]/g, "")
+    .toLowerCase()
+    .replace(/https?:\/\/\S+/g, "")
+    .replace(/[^a-z0-9]+/g, "-")
+    .replace(/^-+|-+$/g, "")
+    .replace(/-+/g, "-")
+    .slice(0, 64)
+    .replace(/^-+|-+$/g, "");
+  return slug || fallback;
+}
+
+function normaliseExcludeFragment(frag) {
+  return toPosix(frag).replace(/^\/+|\/+$/g, "").toLowerCase().split("/").filter(Boolean);
+}
+
+function isExcluded(targetPath, sourceRoot) {
+  const relative = toPosix(path.relative(sourceRoot, targetPath)).toLowerCase();
+  const parts = relative.split("/").filter((part) => part && part !== ".");
+  for (const frag of EXCLUDE_FRAGS) {
+    const fragParts = normaliseExcludeFragment(frag);
+    if (fragParts.length === 0) continue;
+    if (fragParts.length === 1) {
+      if (parts.includes(fragParts[0])) return `exclude:${frag}`;
+      continue;
+    }
+    for (let idx = 0; idx <= parts.length - fragParts.length; idx += 1) {
+      if (parts.slice(idx, idx + fragParts.length).join("/") === fragParts.join("/")) {
+        return `exclude:${frag}`;
+      }
+    }
+  }
+  return null;
+}
+
+function collectMarkdownFiles(sourceRoot) {
+  const files = [];
+  const skipped = [];
+  let scanned = 0;
+
+  function visit(current) {
+    const stat = fs.statSync(current);
+    if (stat.isDirectory()) {
+      const excluded = isExcluded(current, sourceRoot);
+      if (excluded) return;
+      for (const child of fs.readdirSync(current).sort()) {
+        visit(path.join(current, child));
+      }
+      return;
+    }
+    if (!stat.isFile()) return;
+    if (!/\.md(?:arkdown)?$/i.test(current)) return;
+    scanned += 1;
+    const excluded = isExcluded(current, sourceRoot);
+    if (excluded) {
+      skipped.push({
+        path: toPosix(path.relative(sourceRoot, current)),
+        reason: excluded,
+      });
+      return;
+    }
+    files.push(current);
+  }
+
+  const stat = fs.statSync(sourceRoot);
+  if (stat.isFile()) {
+    scanned += /\.md(?:arkdown)?$/i.test(sourceRoot) ? 1 : 0;
+    if (/\.md(?:arkdown)?$/i.test(sourceRoot)) files.push(sourceRoot);
+  } else {
+    visit(sourceRoot);
+  }
+
+  return { files: files.sort((a, b) => toPosix(a).localeCompare(toPosix(b))), skipped, scanned };
+}
+
+function collectJsonFiles(sourceRoot) {
+  const files = [];
+  let scanned = 0;
+
+  function visit(current) {
+    const stat = fs.statSync(current);
+    if (stat.isDirectory()) {
+      const excluded = isExcluded(current, sourceRoot);
+      if (excluded) return;
+      for (const child of fs.readdirSync(current).sort()) {
+        visit(path.join(current, child));
+      }
+      return;
+    }
+    if (!stat.isFile()) return;
+    if (!/\.json$/i.test(current)) return;
+    scanned += 1;
+    const excluded = isExcluded(current, sourceRoot);
+    if (excluded) return;
+    files.push(current);
+  }
+
+  const stat = fs.statSync(sourceRoot);
+  if (stat.isFile()) {
+    scanned += /\.json$/i.test(sourceRoot) ? 1 : 0;
+    if (/\.json$/i.test(sourceRoot)) files.push(sourceRoot);
+  } else {
+    visit(sourceRoot);
+  }
+
+  return { files: files.sort((a, b) => toPosix(a).localeCompare(toPosix(b))), scanned };
+}
+
+function parseFrontmatterTitle(text) {
+  const lines = text.split(/\r?\n/);
+  if (!lines.length || lines[0].trim() !== "---") return "";
+  for (let idx = 1; idx < lines.length; idx += 1) {
+    const line = lines[idx];
+    if (line.trim() === "---") break;
+    const match = line.match(/^title:\s*(.+?)\s*$/i);
+    if (match) return match[1].trim().replace(/^["']|["']$/g, "");
+  }
+  return "";
+}
+
+function extractTitle(text, filePath) {
+  const frontmatterTitle = parseFrontmatterTitle(text);
+  if (frontmatterTitle) return frontmatterTitle;
+  const heading = text.match(/^#\s+(.+?)\s*$/m);
+  if (heading) return heading[1].trim();
+  return path.basename(filePath, path.extname(filePath)).replace(/[_-]+/g, " ").trim();
+}
+
+function detectMojibake(text) {
+  const matches = text.match(/�|Ã.|Â.|â€|ðŸ/gi) || [];
+  return {
+    hasMojibake: matches.length > 0,
+    score: Math.min(1, matches.length / Math.max(1, text.length / 500)),
+  };
+}
+
+function extractGithubRepos(text) {
+  const repos = [];
+  const seen = new Set();
+  const githubRegex = /(?:https?:\/\/)?(?:www\.)?github\.com\/([A-Za-z0-9](?:[A-Za-z0-9-]{0,38}))\/([A-Za-z0-9._-]+)/gi;
+  let match;
+  while ((match = githubRegex.exec(text)) !== null) {
+    const owner = match[1];
+    const rawRepo = match[2].replace(/[).,;:]+$/g, "");
+    if (!rawRepo || rawRepo === "..." || rawRepo.includes("...")) continue;
+    const repo = rawRepo.replace(/\.git$/i, "");
+    const key = `${owner.toLowerCase()}/${repo.toLowerCase()}`;
+    if (seen.has(key)) continue;
+    seen.add(key);
+    repos.push({
+      owner,
+      repo,
+      github_url: `https://github.com/${owner}/${repo}`,
+      key,
+    });
+  }
+  return repos;
+}
+
+function parseGithubRepoUrl(value) {
+  const match = String(value || "").match(/^(?:https?:\/\/)?(?:www\.)?github\.com\/([A-Za-z0-9](?:[A-Za-z0-9-]{0,38}))\/([A-Za-z0-9._-]+)(?:\.git)?(?:[/?#].*)?$/i);
+  if (!match) return null;
+  const owner = match[1];
+  const repo = match[2].replace(/[).,;:]+$/g, "").replace(/\.git$/i, "");
+  if (!repo || repo === "..." || repo.includes("...")) return null;
+  return {
+    owner,
+    repo,
+    github_url: `https://github.com/${owner}/${repo}`,
+    key: `${owner.toLowerCase()}/${repo.toLowerCase()}`,
+  };
+}
+
+function extractUnsupportedGithubItems(text) {
+  const items = [];
+  const invalidRegex = /github\.com\/(?:\.\.\.|[^\s)]+\.{3}[^\s)]*)/gi;
+  let match;
+  while ((match = invalidRegex.exec(text)) !== null) {
+    items.push(match[0]);
+  }
+  return [...new Set(items)];
+}
+
+function normalizeTopic(value) {
+  return safeSlug(String(value || "").replace(/_/g, " "), "topic");
+}
+
+function conceptFromTopic(topic) {
+  const name = String(topic || "").trim();
+  const slug = normalizeTopic(name);
+  if (!slug || slug === "topic") return null;
+  return {
+    concept_id: `concept_topic_${slug.replace(/-/g, "_")}`,
+    name: name.replace(/[_-]+/g, " "),
+    aliases: [name],
+    definition: `Local structured sources include topic evidence for ${name.replace(/[_-]+/g, " ")}.`,
+    related_entities: [],
+    source_refs: [],
+    provenance_refs: [],
+    confidence: 0.6,
+    confidence_tier: "medium",
+    target_page_path: `wiki/concepts/${slug}.md`,
+  };
+}
+
+function inferConcepts(text) {
+  const lower = text.toLowerCase();
+  return CONCEPT_RULES.filter((rule) => rule.keywords.some((keyword) => lower.includes(keyword.toLowerCase())));
+}
+
+function inferConceptsFromTopics(topics) {
+  return (Array.isArray(topics) ? topics : [])
+    .map(conceptFromTopic)
+    .filter(Boolean);
+}
+
+function categoryForSource(text, concepts) {
+  if (concepts.length > 0) return concepts[0].category;
+  const lower = text.toLowerCase();
+  if (lower.includes("github") || lower.includes("repo")) return "developer_tooling";
+  return "uncategorized";
+}
+
+function confidenceNumber(value) {
+  if (typeof value === "number" && Number.isFinite(value)) return Math.max(0, Math.min(1, value));
+  const text = String(value || "").trim().toLowerCase();
+  if (text === "high") return 0.85;
+  if (text === "medium") return 0.65;
+  if (text === "low") return 0.35;
+  if (text === "unsupported") return 0.1;
+  return 0.5;
+}
+
+function firstArray(value) {
+  if (Array.isArray(value)) return value;
+  if (!value || typeof value !== "object") return [];
+  for (const key of ["records", "repos", "repositories", "items", "data"]) {
+    if (Array.isArray(value[key])) return value[key];
+  }
+  return [value];
+}
+
+function normalizeJsonRepoRecord(raw) {
+  if (!raw || typeof raw !== "object" || Array.isArray(raw)) return null;
+  const repoUrl = String(raw.repo_url || raw.github_url || raw.url || "").trim();
+  const parsedRepo = parseGithubRepoUrl(repoUrl);
+  const owner = String(raw.owner || parsedRepo?.owner || "").trim();
+  const repoName = String(raw.repo_name || raw.name || raw.repo || parsedRepo?.repo || "").trim();
+  const topics = Array.isArray(raw.topics) ? raw.topics.map((topic) => String(topic).trim()).filter(Boolean) : [];
+  const snippet = String(raw.message_text_snippet || raw.snippet || raw.description || raw.summary || "").trim();
+  const sourceLink = String(raw.source_link || raw.source_url || raw.link || "").trim();
+  if (!repoUrl && !owner && !repoName && !snippet && topics.length === 0 && !sourceLink) return null;
+  return {
+    repo_url: parsedRepo ? parsedRepo.github_url : repoUrl || null,
+    owner,
+    repo_name: repoName,
+    message_text_snippet: snippet,
+    source_link: sourceLink || null,
+    topics,
+    confidence_raw: raw.confidence ?? null,
+    confidence: confidenceNumber(raw.confidence),
+    parsed_repo: parsedRepo || (owner && repoName ? {
+      owner,
+      repo: repoName,
+      github_url: repoUrl || `https://github.com/${owner}/${repoName}`,
+      key: `${owner.toLowerCase()}/${repoName.toLowerCase()}`,
+    } : null),
+    raw,
+  };
+}
+
+function confidenceTier(confidence) {
+  if (confidence >= 0.8) return "high";
+  if (confidence >= 0.5) return "medium";
+  if (confidence >= 0.2) return "low";
+  return "unsupported";
+}
+
+function makeSourceRootRef(sourceRoot) {
+  const display = toPosix(sourceRoot);
+  const label = safeSlug(path.basename(sourceRoot), "source-root");
+  return {
+    source_root_id: `root_${sha256(`source-root:v1:${display.toLowerCase()}`).slice(0, 12)}`,
+    source_root_label: label,
+    source_root_type: "external_local",
+    source_root_display_path: display,
+    normalization_version: "source-root:v1",
+  };
+}
+
+function buildProvenance({ sourceId, sourceHash, sourceRelativePath, sourceLogicalPath, line, generatedAt, confidence }) {
+  return {
+    provenance_id: `prov_${sourceId}_${String(line || 1).padStart(3, "0")}`,
+    source_id: sourceId,
+    source_hash: sourceHash,
+    source_relative_path: sourceRelativePath,
+    source_logical_path: sourceLogicalPath,
+    locator: {
+      type: "line_range",
+      start_line: line || 1,
+      end_line: line || 1,
+      heading: null,
+    },
+    evidence_quote_hash: sha256(`${sourceId}:${line || 1}`),
+    extractor: "sdtk-wiki.semantic-extract",
+    extractor_version: "bk132-dry-run",
+    generated_at: generatedAt,
+    confidence,
+  };
+}
+
+function lineOf(text, needle) {
+  const idx = text.indexOf(needle);
+  if (idx < 0) return 1;
+  return text.slice(0, idx).split(/\r?\n/).length;
+}
+
+function boundedText(value, maxLength = 700) {
+  const text = String(value || "").replace(/\s+/g, " ").trim();
+  if (text.length <= maxLength) return text;
+  return `${text.slice(0, maxLength - 1).trim()}...`;
+}
+
+function asArray(value) {
+  return Array.isArray(value) ? value : [];
+}
+
+function localRepoSnippet(text, repoUrl) {
+  const lines = String(text || "").split(/\r?\n/);
+  const index = lines.findIndex((line) => line.includes(repoUrl));
+  if (index < 0) return boundedText(text, 500);
+  const start = Math.max(0, index - 2);
+  const end = Math.min(lines.length, index + 4);
+  return boundedText(lines.slice(start, end).join(" "));
+}
+
+function pushUnique(array, value, keyFn = (item) => item) {
+  if (!value) return;
+  const key = keyFn(value);
+  if (!array.some((item) => keyFn(item) === key)) array.push(value);
+}
+
+function sourceRefsForEntity(entity) {
+  return [...new Set(asArray(entity && entity.source_refs).filter(Boolean).map(String))];
+}
+
+function decisionStrengthsForEntity(entity, conceptName) {
+  const strengths = [];
+  if (sourceRefsForEntity(entity).length > 1) strengths.push("mentioned by multiple local source records");
+  if (asArray(entity.topics).length > 0) strengths.push(`topic fit: ${asArray(entity.topics).slice(0, 3).join(", ")}`);
+  if (entity.category && entity.category !== "uncategorized") strengths.push(`category fit: ${entity.category}`);
+  if (entity.summary) strengths.push("has local snippet evidence");
+  return strengths.length > 0 ? strengths : [`local evidence connects this repository to ${conceptName}`];
+}
+
+function decisionCaveatsForEntity(entity) {
+  const caveats = ["local evidence only; verify externally before adoption"];
+  if (!entity.github_url) caveats.push("missing canonical repository URL");
+  if (["low", "unsupported"].includes(entity.confidence_tier)) caveats.push("low extraction confidence");
+  if (sourceRefsForEntity(entity).length <= 1) caveats.push("single local source reference");
+  return caveats;
+}
+
+function confidenceSummary(records) {
+  const counts = { high: 0, medium: 0, low: 0, unsupported: 0, unknown: 0 };
+  for (const record of asArray(records)) {
+    const tier = String(record.confidence_tier || confidenceTier(record.confidence) || "unknown").toLowerCase();
+    if (Object.prototype.hasOwnProperty.call(counts, tier)) counts[tier] += 1;
+    else counts.unknown += 1;
+  }
+  return counts;
+}
+
+function buildExtraction({ projectPath, sourceRoot }) {
+  const generatedAt = new Date().toISOString();
+  const sourceRootRef = makeSourceRootRef(sourceRoot);
+  const collected = collectMarkdownFiles(sourceRoot);
+  const collectedJson = collectJsonFiles(sourceRoot);
+  const sources = [];
+  const toolEntitiesById = new Map();
+  const conceptsById = new Map();
+  const claims = [];
+  const relations = [];
+  const provenance = [];
+  const sourceQualityFindings = [];
+  const unsupportedItems = [];
+  const sourceUrlUsage = new Map();
+
+  for (const filePath of collected.files) {
+    const text = fs.readFileSync(filePath, "utf-8");
+    const sourceHash = sha256(fs.readFileSync(filePath));
+    const stats = fs.statSync(filePath);
+    const sourceRelativePath = toPosix(path.relative(sourceRoot, filePath));
+    const sourceLogicalPath = `${sourceRootRef.source_root_label}/${sourceRelativePath}`;
+    const sourceDisplayPath = toPosix(filePath);
+    const sourceId = `src_${sha256(`local-md:v1:${sourceRootRef.source_root_id}:${sourceRelativePath.toLowerCase()}`).slice(0, 16)}`;
+    const title = extractTitle(text, filePath);
+    const repos = extractGithubRepos(text);
+    const unsupportedGithub = extractUnsupportedGithubItems(text);
+    const concepts = inferConcepts(`${title}\n${text}`);
+    const category = categoryForSource(`${title}\n${text}`, concepts);
+    const mojibake = detectMojibake(text);
+    const weakTitle = title.length < 6 || /^untitled|note|readme$/i.test(title);
+    const sourceUrl = repos.length > 0 ? repos[0].github_url : null;
+    const sourceSlug = `${safeSlug(title || sourceRelativePath, "source")}--${sourceId.slice(0, 8)}`;
+    const qualityFlags = [];
+    const qualityNotes = [];
+
+    if (mojibake.hasMojibake) {
+      qualityFlags.push("mojibake_detected");
+      qualityNotes.push("Potential mojibake or replacement characters detected.");
+    }
+    if (!sourceUrl) {
+      qualityFlags.push("missing_source_url");
+      qualityNotes.push("No valid GitHub/source URL was extracted.");
+    }
+    if (weakTitle) {
+      qualityFlags.push("weak_title");
+      qualityNotes.push("Title is missing, very short, or generic.");
+    }
+    if (repos.length === 0) {
+      qualityFlags.push("low_confidence_extraction");
+    }
+
+    const sourceRecord = {
+      source_id: sourceId,
+      source_root_id: sourceRootRef.source_root_id,
+      source_relative_path: sourceRelativePath,
+      source_logical_path: sourceLogicalPath,
+      source_display_path: sourceDisplayPath,
+      source_type: "markdown",
+      title,
+      source_url: sourceUrl,
+      source_hash: sourceHash,
+      size_bytes: stats.size,
+      modified_time: stats.mtime.toISOString(),
+      encoding_quality: mojibake.hasMojibake ? "suspect" : "clean",
+      source_quality: {
+        has_mojibake: mojibake.hasMojibake,
+        mojibake_score: Number(mojibake.score.toFixed(3)),
+        has_source_url: Boolean(sourceUrl),
+        weak_title: weakTitle,
+        duplicate_candidate: false,
+        duplicate_group_id: null,
+        low_confidence_extraction: repos.length === 0,
+        quality_flags: qualityFlags,
+        notes: qualityNotes,
+      },
+      provenance_refs: [],
+      target_page_path: `wiki/sources/${sourceSlug}.md`,
+    };
+
+    sources.push(sourceRecord);
+
+    if (sourceUrl) {
+      const existing = sourceUrlUsage.get(sourceUrl) || [];
+      existing.push(sourceRecord);
+      sourceUrlUsage.set(sourceUrl, existing);
+    }
+
+    for (const rawUnsupported of unsupportedGithub) {
+      unsupportedItems.push({
+        record_type: "unsupported_item",
+        item_id: `unsupported_${sourceId}_${String(unsupportedItems.length + 1).padStart(3, "0")}`,
+        source_id: sourceId,
+        reason: "unsupported_url_format",
+        raw_observation_summary: rawUnsupported,
+        confidence: 0.1,
+        confidence_tier: "unsupported",
+        provenance_refs: [],
+      });
+      if (!qualityFlags.includes("unsupported_url_format")) {
+        sourceRecord.source_quality.quality_flags.push("unsupported_url_format");
+      }
+    }
+
+    if (qualityFlags.length > 0) {
+      sourceQualityFindings.push({
+        finding_id: `sq_${sourceId}`,
+        source_id: sourceId,
+        source_relative_path: sourceRelativePath,
+        source_logical_path: sourceLogicalPath,
+        quality_flags: [...sourceRecord.source_quality.quality_flags],
+        confidence: repos.length === 0 ? 0.3 : 0.7,
+        confidence_tier: repos.length === 0 ? "low" : "medium",
+        notes: qualityNotes,
+      });
+    }
+
+    for (const repo of repos) {
+      const entityId = `tool_github_${safeSlug(repo.owner, "owner")}_${safeSlug(repo.repo, "repo")}`;
+      const prov = buildProvenance({
+        sourceId,
+        sourceHash,
+        sourceRelativePath,
+        sourceLogicalPath,
+        line: lineOf(text, repo.github_url),
+        generatedAt,
+        confidence: 0.9,
+      });
+      provenance.push(prov);
+      sourceRecord.provenance_refs.push(prov.provenance_id);
+
+      if (!toolEntitiesById.has(entityId)) {
+        toolEntitiesById.set(entityId, {
+          entity_id: entityId,
+          entity_type: "tool_entity",
+          name: repo.repo,
+          repo_owner: repo.owner,
+          repo_name: repo.repo,
+          github_url: repo.github_url,
+          category,
+          summary: `${repo.repo} is a locally sourced GitHub tool candidate in category ${category}.`,
+          confidence: 0.9,
+          confidence_tier: "high",
+          source_refs: [],
+          provenance_refs: [],
+          topics: concepts.map((concept) => concept.name),
+          source_links: [],
+          evidence_snippets: [],
+          discovery_sources: [],
+          evidence_records: [],
+          target_page_path: `wiki/entities/tools/${safeSlug(repo.repo, "tool")}--${entityId}.md`,
+        });
+      }
+      const entity = toolEntitiesById.get(entityId);
+      if (!entity.source_refs.includes(sourceId)) entity.source_refs.push(sourceId);
+      if (!entity.provenance_refs.includes(prov.provenance_id)) entity.provenance_refs.push(prov.provenance_id);
+      for (const concept of concepts) {
+        if (!entity.topics) entity.topics = [];
+        pushUnique(entity.topics, concept.name);
+      }
+      const snippet = localRepoSnippet(text, repo.github_url);
+      if (!entity.evidence_snippets) entity.evidence_snippets = [];
+      pushUnique(entity.evidence_snippets, snippet);
+      if (!entity.discovery_sources) entity.discovery_sources = [];
+      pushUnique(entity.discovery_sources, sourceLogicalPath);
+      if (!entity.evidence_records) entity.evidence_records = [];
+      pushUnique(entity.evidence_records, {
+        source_id: sourceId,
+        source_logical_path: sourceLogicalPath,
+        source_link: sourceUrl || null,
+        snippet,
+        topics: concepts.map((concept) => concept.name),
+        provenance_refs: [prov.provenance_id],
+        confidence: 0.9,
+        confidence_tier: "high",
+      }, (record) => `${record.source_id}:${record.source_link || ""}:${record.snippet}`);
+
+      claims.push({
+        claim_id: `claim_${sourceId}_${String(claims.length + 1).padStart(3, "0")}`,
+        text: `The local source presents ${repo.repo} as a ${category} tool or project.`,
+        subject_entity_id: entityId,
+        source_refs: [sourceId],
+        provenance_refs: [prov.provenance_id],
+        confidence: 0.75,
+        confidence_tier: "medium",
+        contested: false,
+      });
+
+      relations.push({
+        relation_id: `rel_${sourceId}_${String(relations.length + 1).padStart(3, "0")}`,
+        source_id: sourceId,
+        target_id: entityId,
+        relation_type: "source_mentions_entity",
+        evidence: "The local Markdown source includes a GitHub repository URL.",
+        source_refs: [sourceId],
+        provenance_refs: [prov.provenance_id],
+        confidence: 0.9,
+        confidence_tier: "high",
+      });
+    }
+
+    for (const conceptRule of concepts) {
+      if (!conceptsById.has(conceptRule.concept_id)) {
+        conceptsById.set(conceptRule.concept_id, {
+          concept_id: conceptRule.concept_id,
+          name: conceptRule.name,
+          aliases: conceptRule.aliases,
+          definition: `Local sources contain evidence related to ${conceptRule.name}.`,
+          related_entities: [],
+          source_refs: [],
+          provenance_refs: [],
+          confidence: 0.65,
+          confidence_tier: "medium",
+          target_page_path: `wiki/concepts/${safeSlug(conceptRule.name, "concept")}.md`,
+        });
+      }
+      const concept = conceptsById.get(conceptRule.concept_id);
+      if (!concept.source_refs.includes(sourceId)) concept.source_refs.push(sourceId);
+
+      for (const repo of repos) {
+        const entityId = `tool_github_${safeSlug(repo.owner, "owner")}_${safeSlug(repo.repo, "repo")}`;
+        if (!concept.related_entities.includes(entityId)) concept.related_entities.push(entityId);
+        relations.push({
+          relation_id: `rel_${sourceId}_${String(relations.length + 1).padStart(3, "0")}`,
+          source_id: entityId,
+          target_id: conceptRule.concept_id,
+          relation_type: "entity_implements_concept",
+          evidence: `The local source text matches ${conceptRule.name} keywords.`,
+          source_refs: [sourceId],
+          provenance_refs: [...sourceRecord.provenance_refs],
+          confidence: 0.7,
+          confidence_tier: "medium",
+        });
+      }
+    }
+  }
+
+  for (const filePath of collectedJson.files) {
+    const sourceRelativePath = toPosix(path.relative(sourceRoot, filePath));
+    const sourceDisplayPath = toPosix(filePath);
+    const sourceLogicalFilePath = `${sourceRootRef.source_root_label}/${sourceRelativePath}`;
+    const fileBytes = fs.readFileSync(filePath);
+    const fileHash = sha256(fileBytes);
+    const stats = fs.statSync(filePath);
+    let parsed;
+
+    try {
+      parsed = JSON.parse(fileBytes.toString("utf-8"));
+    } catch (error) {
+      const sourceId = `src_${sha256(`local-json-file:v1:${sourceRootRef.source_root_id}:${sourceRelativePath.toLowerCase()}`).slice(0, 16)}`;
+      const sourceRecord = {
+        source_id: sourceId,
+        source_root_id: sourceRootRef.source_root_id,
+        source_relative_path: sourceRelativePath,
+        source_logical_path: sourceLogicalFilePath,
+        source_display_path: sourceDisplayPath,
+        source_type: "json",
+        title: path.basename(filePath),
+        source_url: null,
+        source_hash: fileHash,
+        size_bytes: stats.size,
+        modified_time: stats.mtime.toISOString(),
+        encoding_quality: "unknown",
+        source_quality: {
+          has_mojibake: false,
+          mojibake_score: 0,
+          has_source_url: false,
+          weak_title: false,
+          duplicate_candidate: false,
+          duplicate_group_id: null,
+          low_confidence_extraction: true,
+          quality_flags: ["invalid_json"],
+          notes: [`Invalid JSON could not be parsed: ${error.message}`],
+        },
+        provenance_refs: [],
+        target_page_path: `wiki/sources/${safeSlug(path.basename(filePath), "json-source")}--${sourceId.slice(0, 8)}.md`,
+      };
+      sources.push(sourceRecord);
+      sourceQualityFindings.push({
+        finding_id: `sq_${sourceId}`,
+        source_id: sourceId,
+        source_relative_path: sourceRelativePath,
+        source_logical_path: sourceLogicalFilePath,
+        quality_flags: ["invalid_json"],
+        confidence: 0.1,
+        confidence_tier: "unsupported",
+        notes: sourceRecord.source_quality.notes,
+      });
+      unsupportedItems.push({
+        record_type: "unsupported_item",
+        item_id: `unsupported_${sourceId}_001`,
+        source_id: sourceId,
+        reason: "invalid_json",
+        raw_observation_summary: `Invalid JSON file: ${sourceRelativePath}`,
+        confidence: 0.1,
+        confidence_tier: "unsupported",
+        provenance_refs: [],
+      });
+      continue;
+    }
+
+    const rawRecords = firstArray(parsed);
+    const normalizedRecords = rawRecords.map(normalizeJsonRepoRecord).filter(Boolean);
+    if (normalizedRecords.length === 0) {
+      const sourceId = `src_${sha256(`local-json-empty:v1:${sourceRootRef.source_root_id}:${sourceRelativePath.toLowerCase()}`).slice(0, 16)}`;
+      const sourceRecord = {
+        source_id: sourceId,
+        source_root_id: sourceRootRef.source_root_id,
+        source_relative_path: sourceRelativePath,
+        source_logical_path: sourceLogicalFilePath,
+        source_display_path: sourceDisplayPath,
+        source_type: "json",
+        title: path.basename(filePath),
+        source_url: null,
+        source_hash: fileHash,
+        size_bytes: stats.size,
+        modified_time: stats.mtime.toISOString(),
+        encoding_quality: "clean",
+        source_quality: {
+          has_mojibake: false,
+          mojibake_score: 0,
+          has_source_url: false,
+          weak_title: false,
+          duplicate_candidate: false,
+          duplicate_group_id: null,
+          low_confidence_extraction: true,
+          quality_flags: ["empty_json_records"],
+          notes: ["JSON parsed successfully but contained no supported repository records."],
+        },
+        provenance_refs: [],
+        target_page_path: `wiki/sources/${safeSlug(path.basename(filePath), "json-source")}--${sourceId.slice(0, 8)}.md`,
+      };
+      sources.push(sourceRecord);
+      sourceQualityFindings.push({
+        finding_id: `sq_${sourceId}`,
+        source_id: sourceId,
+        source_relative_path: sourceRelativePath,
+        source_logical_path: sourceLogicalFilePath,
+        quality_flags: ["empty_json_records"],
+        confidence: 0.2,
+        confidence_tier: "unsupported",
+        notes: sourceRecord.source_quality.notes,
+      });
+      continue;
+    }
+
+    normalizedRecords.forEach((record, index) => {
+      const recordRef = `record-${String(index + 1).padStart(3, "0")}`;
+      const repoKey = record.repo_url || `${record.owner}/${record.repo_name}` || sha256(JSON.stringify(record.raw));
+      const sourceId = `src_${sha256(`local-json:v1:${sourceRootRef.source_root_id}:${sourceRelativePath.toLowerCase()}:${repoKey.toLowerCase()}`).slice(0, 16)}`;
+      const sourceHash = sha256(`${fileHash}:${sha256(JSON.stringify(record.raw))}`);
+      const title = record.repo_name || record.owner || `${path.basename(filePath)} ${recordRef}`;
+      const sourceLogicalPath = `${sourceLogicalFilePath}#${recordRef}`;
+      const sourceDisplayRecordPath = `${sourceDisplayPath}#${recordRef}`;
+      const mojibake = detectMojibake(record.message_text_snippet);
+      const weakTitle = title.length < 3;
+      const qualityFlags = [];
+      const qualityNotes = [];
+      const sourceUrl = record.repo_url || record.source_link || null;
+      const confidence = record.confidence;
+      const confidenceBand = confidenceTier(confidence);
+
+      if (mojibake.hasMojibake) {
+        qualityFlags.push("mojibake_detected");
+        qualityNotes.push("Potential mojibake or replacement characters detected in JSON snippet.");
+      }
+      if (!record.parsed_repo) {
+        qualityFlags.push("missing_repo_url");
+        qualityNotes.push("JSON record does not include a supported GitHub repository URL.");
+      }
+      if (!record.source_link) {
+        qualityFlags.push("missing_source_link");
+        qualityNotes.push("JSON record does not include a source_link.");
+      }
+      if (weakTitle) {
+        qualityFlags.push("weak_title");
+        qualityNotes.push("Repository title is missing or very short.");
+      }
+      if (confidence < 0.5) {
+        qualityFlags.push("low_confidence_extraction");
+      }
+
+      const sourceRecord = {
+        source_id: sourceId,
+        source_root_id: sourceRootRef.source_root_id,
+        source_relative_path: sourceRelativePath,
+        source_logical_path: sourceLogicalPath,
+        source_display_path: sourceDisplayRecordPath,
+        source_type: "json_record",
+        title,
+        source_url: sourceUrl,
+        source_hash: sourceHash,
+        size_bytes: stats.size,
+        modified_time: stats.mtime.toISOString(),
+        encoding_quality: mojibake.hasMojibake ? "suspect" : "clean",
+        source_record_locator: {
+          type: "json_record",
+          record_index: index,
+          record_ref: recordRef,
+          record_pointer: `/${index}`,
+        },
+        structured_fields: {
+          repo_url: record.repo_url,
+          owner: record.owner,
+          repo_name: record.repo_name,
+          message_text_snippet: record.message_text_snippet,
+          source_link: record.source_link,
+          topics: record.topics,
+          confidence: record.confidence_raw,
+        },
+        source_quality: {
+          has_mojibake: mojibake.hasMojibake,
+          mojibake_score: Number(mojibake.score.toFixed(3)),
+          has_source_url: Boolean(sourceUrl),
+          weak_title: weakTitle,
+          duplicate_candidate: false,
+          duplicate_group_id: null,
+          low_confidence_extraction: confidence < 0.5,
+          quality_flags: qualityFlags,
+          notes: qualityNotes,
+        },
+        provenance_refs: [],
+        target_page_path: `wiki/sources/${safeSlug(title || sourceRelativePath, "source")}--${sourceId.slice(0, 8)}.md`,
+      };
+
+      sources.push(sourceRecord);
+      if (record.repo_url) {
+        const existing = sourceUrlUsage.get(record.repo_url) || [];
+        existing.push(sourceRecord);
+        sourceUrlUsage.set(record.repo_url, existing);
+      }
+
+      if (qualityFlags.length > 0) {
+        sourceQualityFindings.push({
+          finding_id: `sq_${sourceId}`,
+          source_id: sourceId,
+          source_relative_path: sourceRelativePath,
+          source_logical_path: sourceLogicalPath,
+          quality_flags: [...qualityFlags],
+          confidence,
+          confidence_tier: confidenceBand,
+          notes: qualityNotes,
+        });
+      }
+
+      const concepts = [
+        ...inferConcepts(`${title}\n${record.message_text_snippet}\n${record.topics.join("\n")}`),
+        ...inferConceptsFromTopics(record.topics),
+      ];
+      const category = record.topics[0] ? normalizeTopic(record.topics[0]) : categoryForSource(record.message_text_snippet, concepts);
+
+      if (!record.parsed_repo) {
+        unsupportedItems.push({
+          record_type: "unsupported_item",
+          item_id: `unsupported_${sourceId}_001`,
+          source_id: sourceId,
+          reason: "missing_repo_url",
+          raw_observation_summary: `${sourceRelativePath}#${recordRef}`,
+          confidence: 0.2,
+          confidence_tier: "unsupported",
+          provenance_refs: [],
+        });
+      }
+
+      if (record.parsed_repo) {
+        const repo = record.parsed_repo;
+        const entityId = `tool_github_${safeSlug(repo.owner, "owner")}_${safeSlug(repo.repo, "repo")}`;
+        const prov = {
+          provenance_id: `prov_${sourceId}_${recordRef}`,
+          source_id: sourceId,
+          source_hash: sourceHash,
+          source_relative_path: sourceRelativePath,
+          source_logical_path: sourceLogicalPath,
+          locator: {
+            type: "json_record",
+            record_index: index,
+            record_ref: recordRef,
+            record_pointer: `/${index}`,
+            field: "repo_url",
+          },
+          evidence_quote_hash: sha256(`${sourceId}:${recordRef}:${repo.github_url}`),
+          extractor: "sdtk-wiki.semantic-extract",
+          extractor_version: "bk140-json-records",
+          generated_at: generatedAt,
+          confidence,
+        };
+        provenance.push(prov);
+        sourceRecord.provenance_refs.push(prov.provenance_id);
+
+        if (!toolEntitiesById.has(entityId)) {
+          toolEntitiesById.set(entityId, {
+            entity_id: entityId,
+            entity_type: "tool_entity",
+            name: repo.repo,
+            repo_owner: repo.owner,
+            repo_name: repo.repo,
+            github_url: repo.github_url,
+            category,
+            summary: record.message_text_snippet || `${repo.repo} is a locally sourced GitHub tool candidate in category ${category}.`,
+            confidence,
+            confidence_tier: confidenceBand,
+            source_refs: [],
+            provenance_refs: [],
+            topics: [...record.topics],
+            source_links: record.source_link ? [record.source_link] : [],
+            evidence_snippets: record.message_text_snippet ? [boundedText(record.message_text_snippet)] : [],
+            discovery_sources: [record.source_link, sourceLogicalPath].filter(Boolean),
+            evidence_records: [{
+              source_id: sourceId,
+              source_logical_path: sourceLogicalPath,
+              source_link: record.source_link || null,
+              snippet: boundedText(record.message_text_snippet),
+              topics: [...record.topics],
+              provenance_refs: [prov.provenance_id],
+              confidence,
+              confidence_tier: confidenceBand,
+            }],
+            target_page_path: `wiki/entities/tools/${safeSlug(repo.repo, "tool")}--${entityId}.md`,
+          });
+        }
+        const entity = toolEntitiesById.get(entityId);
+        if (!entity.source_refs.includes(sourceId)) entity.source_refs.push(sourceId);
+        if (!entity.provenance_refs.includes(prov.provenance_id)) entity.provenance_refs.push(prov.provenance_id);
+        for (const topic of record.topics) {
+          if (!entity.topics) entity.topics = [];
+          if (!entity.topics.includes(topic)) entity.topics.push(topic);
+        }
+        if (record.source_link) {
+          if (!entity.source_links) entity.source_links = [];
+          if (!entity.source_links.includes(record.source_link)) entity.source_links.push(record.source_link);
+        }
+        if (record.message_text_snippet) {
+          if (!entity.evidence_snippets) entity.evidence_snippets = [];
+          pushUnique(entity.evidence_snippets, boundedText(record.message_text_snippet));
+        }
+        if (!entity.discovery_sources) entity.discovery_sources = [];
+        pushUnique(entity.discovery_sources, record.source_link);
+        pushUnique(entity.discovery_sources, sourceLogicalPath);
+        if (!entity.evidence_records) entity.evidence_records = [];
+        pushUnique(entity.evidence_records, {
+          source_id: sourceId,
+          source_logical_path: sourceLogicalPath,
+          source_link: record.source_link || null,
+          snippet: boundedText(record.message_text_snippet),
+          topics: [...record.topics],
+          provenance_refs: [prov.provenance_id],
+          confidence,
+          confidence_tier: confidenceBand,
+        }, (item) => `${item.source_id}:${item.source_link || ""}:${item.snippet || ""}`);
+
+        claims.push({
+          claim_id: `claim_${sourceId}_${String(claims.length + 1).padStart(3, "0")}`,
+          text: `The local JSON record presents ${repo.repo} as a ${category} repository candidate.`,
+          subject_entity_id: entityId,
+          source_refs: [sourceId],
+          provenance_refs: [prov.provenance_id],
+          confidence,
+          confidence_tier: confidenceBand,
+          contested: false,
+        });
+
+        relations.push({
+          relation_id: `rel_${sourceId}_${String(relations.length + 1).padStart(3, "0")}`,
+          source_id: sourceId,
+          target_id: entityId,
+          relation_type: "source_mentions_entity",
+          evidence: "The local JSON record includes a GitHub repository URL.",
+          source_refs: [sourceId],
+          provenance_refs: [prov.provenance_id],
+          confidence,
+          confidence_tier: confidenceBand,
+        });
+
+        for (const conceptRule of concepts) {
+          if (!conceptsById.has(conceptRule.concept_id)) {
+            conceptsById.set(conceptRule.concept_id, {
+              ...conceptRule,
+              related_entities: conceptRule.related_entities || [],
+              source_refs: conceptRule.source_refs || [],
+              provenance_refs: conceptRule.provenance_refs || [],
+              confidence: conceptRule.confidence || 0.6,
+              confidence_tier: conceptRule.confidence_tier || "medium",
+              target_page_path: conceptRule.target_page_path || `wiki/concepts/${safeSlug(conceptRule.name, "concept")}.md`,
+            });
+          }
+          const concept = conceptsById.get(conceptRule.concept_id);
+          if (!concept.source_refs.includes(sourceId)) concept.source_refs.push(sourceId);
+          if (!concept.provenance_refs.includes(prov.provenance_id)) concept.provenance_refs.push(prov.provenance_id);
+          if (!concept.related_entities.includes(entityId)) concept.related_entities.push(entityId);
+          relations.push({
+            relation_id: `rel_${sourceId}_${String(relations.length + 1).padStart(3, "0")}`,
+            source_id: entityId,
+            target_id: conceptRule.concept_id,
+            relation_type: "entity_implements_concept",
+            evidence: "The local JSON record includes matching topics or semantic keywords.",
+            source_refs: [sourceId],
+            provenance_refs: [prov.provenance_id],
+            confidence: Math.min(0.8, confidence + 0.05),
+            confidence_tier: confidenceTier(Math.min(0.8, confidence + 0.05)),
+          });
+        }
+      }
+    });
+  }
+
+  for (const [sourceUrl, sourceRecords] of sourceUrlUsage.entries()) {
+    if (sourceRecords.length < 2) continue;
+    const duplicateGroupId = `dup_${sha256(sourceUrl).slice(0, 12)}`;
+    for (const record of sourceRecords) {
+      record.source_quality.duplicate_candidate = true;
+      record.source_quality.duplicate_group_id = duplicateGroupId;
+      if (!record.source_quality.quality_flags.includes("duplicate_source_candidate")) {
+        record.source_quality.quality_flags.push("duplicate_source_candidate");
+      }
+      sourceQualityFindings.push({
+        finding_id: `sq_duplicate_${record.source_id}`,
+        source_id: record.source_id,
+        source_relative_path: record.source_relative_path,
+        source_logical_path: record.source_logical_path,
+        quality_flags: ["duplicate_source_candidate"],
+        duplicate_group_id: duplicateGroupId,
+        confidence: 0.8,
+        confidence_tier: "high",
+        notes: [`Duplicate source URL candidate: ${sourceUrl}`],
+      });
+    }
+  }
+
+  const toolEntityValues = [...toolEntitiesById.values()];
+  for (const entity of toolEntityValues) {
+    const entityTopics = new Set(asArray(entity.topics).map((topic) => String(topic).toLowerCase()));
+    const related = [];
+    for (const candidate of toolEntityValues) {
+      if (candidate.entity_id === entity.entity_id) continue;
+      const sharedTopics = asArray(candidate.topics).filter((topic) => entityTopics.has(String(topic).toLowerCase()));
+      const sameCategory = entity.category && candidate.category && entity.category === candidate.category;
+      if (sharedTopics.length === 0 && !sameCategory) continue;
+      related.push({
+        entity_id: candidate.entity_id,
+        name: candidate.name || candidate.repo_name || candidate.entity_id,
+        github_url: candidate.github_url || null,
+        shared_topics: sharedTopics,
+        relation_hint: sharedTopics.length > 0 ? "shared_topic" : "same_category",
+      });
+    }
+    entity.related_repos = related
+      .sort((a, b) => b.shared_topics.length - a.shared_topics.length || a.entity_id.localeCompare(b.entity_id))
+      .slice(0, 8);
+  }
+
+  for (const concept of conceptsById.values()) {
+    const relatedDetails = [];
+    const axisCounts = new Map();
+    for (const entityId of asArray(concept.related_entities)) {
+      const entity = toolEntitiesById.get(entityId);
+      if (!entity) continue;
+      for (const topic of asArray(entity.topics)) {
+        const key = String(topic || "").trim();
+        if (key) axisCounts.set(key, (axisCounts.get(key) || 0) + 1);
+      }
+      if (entity.category) axisCounts.set(entity.category, (axisCounts.get(entity.category) || 0) + 1);
+      relatedDetails.push({
+        entity_id: entity.entity_id,
+        name: entity.name || entity.repo_name || entity.entity_id,
+        repo_owner: entity.repo_owner || null,
+        repo_name: entity.repo_name || null,
+        github_url: entity.github_url || null,
+        category: entity.category || "uncategorized",
+        topics: asArray(entity.topics).slice(0, 8),
+        summary: entity.summary || "",
+        target_page_path: entity.target_page_path || null,
+        source_refs: sourceRefsForEntity(entity),
+        confidence: entity.confidence,
+        confidence_tier: entity.confidence_tier || confidenceTier(entity.confidence),
+      });
+    }
+    concept.related_entity_details = relatedDetails
+      .sort((a, b) => (b.source_refs.length - a.source_refs.length) || a.name.localeCompare(b.name))
+      .slice(0, 12);
+    concept.key_axes = [...axisCounts.entries()]
+      .sort((a, b) => b[1] - a[1] || a[0].localeCompare(b[0]))
+      .slice(0, 8)
+      .map(([name, count]) => ({ name, evidence_count: count }));
+    concept.patterns = concept.key_axes.slice(0, 5).map((axis) => ({
+      pattern: axis.name,
+      evidence: `${axis.evidence_count} locally extracted tool candidate(s) are associated with this axis.`,
+    }));
+  }
+
+  const toolEntities = [...toolEntitiesById.values()].sort((a, b) => a.entity_id.localeCompare(b.entity_id));
+  const concepts = [...conceptsById.values()].sort((a, b) => a.concept_id.localeCompare(b.concept_id));
+  const comparisons = [];
+  const syntheses = [];
+
+  for (const concept of concepts) {
+    if (concept.related_entities.length < 2) continue;
+    const topicSlug = safeSlug(concept.name, "topic");
+    const comparedDetails = asArray(concept.related_entity_details).slice(0, 8);
+    const decisionAxes = asArray(concept.key_axes).slice(0, 6);
+    const matrixRows = comparedDetails.map((entity) => ({
+      entity_id: entity.entity_id,
+      name: entity.repo_owner && entity.repo_name ? `${entity.repo_owner}/${entity.repo_name}` : entity.name,
+      github_url: entity.github_url || null,
+      category: entity.category || "uncategorized",
+      topics: asArray(entity.topics).slice(0, 6),
+      strengths: decisionStrengthsForEntity(entity, concept.name),
+      caveats: decisionCaveatsForEntity(entity),
+      source_ref_count: sourceRefsForEntity(entity).length,
+      source_confidence: entity.confidence_tier || confidenceTier(entity.confidence),
+      local_recommendation: sourceRefsForEntity(entity).length > 1
+        ? "shortlist for human review"
+        : "keep as a candidate until more local evidence is available",
+    }));
+    const summary = `Local sources mention ${concept.related_entities.length} tool candidate(s) related to ${concept.name}. This page compares candidates for review, not verified ranking.`;
+    const recommendations = matrixRows.slice(0, 3).map((row) => `${row.name}: ${row.local_recommendation}; strengths: ${row.strengths.slice(0, 2).join("; ")}.`);
+    comparisons.push({
+      comparison_id: `comparison_${topicSlug}_${sha256(concept.related_entities.join("|")).slice(0, 8)}`,
+      topic: concept.name,
+      compared_entities: concept.related_entities.slice(0, 8),
+      compared_entity_details: comparedDetails,
+      decision_axes: decisionAxes,
+      matrix_rows: matrixRows,
+      criteria: ["local evidence", "category/topic fit", "source confidence", "review caveats"],
+      summary,
+      recommendations,
+      caveats: [
+        "The comparison uses local source evidence only.",
+        "Do not treat ordering as an external quality ranking.",
+        "Verify license, maintenance, security, and ecosystem fit before adoption.",
+      ],
+      source_confidence_summary: confidenceSummary(matrixRows),
+      source_refs: concept.source_refs,
+      provenance_refs: concept.provenance_refs,
+      confidence: 0.55,
+      confidence_tier: "medium",
+      target_page_path: `wiki/comparisons/${topicSlug}.md`,
+    });
+    syntheses.push({
+      synthesis_id: `synthesis_${topicSlug}_${sha256(concept.source_refs.join("|")).slice(0, 8)}`,
+      topic: concept.name,
+      summary,
+      landscape_axes: decisionAxes,
+      candidate_tools: matrixRows,
+      patterns: asArray(concept.patterns),
+      related_comparison_path: `wiki/comparisons/${topicSlug}.md`,
+      source_confidence_summary: confidenceSummary(matrixRows),
+      recommendations: [
+        ...recommendations,
+        "Use this synthesis to select review candidates; defer adoption until external verification is complete.",
+      ],
+      caveats: [
+        "Local extraction can include stale or incomplete source snippets.",
+        "No web verification, GitHub API data, stars, licenses, or release cadence are claimed here.",
+        "Human review should resolve topic fit and source-quality warnings before product decisions.",
+      ],
+      source_refs: concept.source_refs,
+      provenance_refs: concept.provenance_refs,
+      confidence: 0.55,
+      confidence_tier: "medium",
+      target_page_path: `wiki/syntheses/${topicSlug}.md`,
+    });
+  }
+
+  const lowConfidence = sourceQualityFindings.filter((finding) => ["low", "unsupported"].includes(finding.confidence_tier)).length;
+
+  return {
+    schema_version: SCHEMA_VERSION,
+    record_type: "sdtk_wiki_semantic_extraction",
+    generated_at: generatedAt,
+    project_path: projectPath,
+    source_root_refs: [sourceRootRef],
+    source_counts: {
+      scanned: collected.scanned + collectedJson.scanned,
+      indexed: sources.length,
+      extracted: toolEntities.length,
+      skipped: collected.skipped.length,
+      low_confidence: lowConfidence,
+      unsupported: unsupportedItems.length,
+    },
+    skipped_sources: collected.skipped,
+    sources,
+    tool_entities: toolEntities,
+    concepts,
+    claims,
+    relations,
+    comparisons,
+    syntheses,
+    source_quality_findings: sourceQualityFindings,
+    unsupported_items: unsupportedItems,
+    provenance,
+  };
+}
+
+function resolveSourceRoot(sourceRootArg) {
+  if (!sourceRootArg) {
+    throw new ValidationError("sdtk-brain wiki extract requires --source-root <path>. No project files were changed.");
+  }
+  if (isRemoteUrl(sourceRootArg)) {
+    throw new ValidationError("Remote URL source roots are not supported for semantic extraction dry-run. No project files were changed.");
+  }
+  const resolved = path.resolve(sourceRootArg);
+  if (!fs.existsSync(resolved)) {
+    throw new ValidationError(`--source-root does not exist: ${resolved}. No project files were changed.`);
+  }
+  const stat = fs.statSync(resolved);
+  if (!stat.isDirectory() && !stat.isFile()) {
+    throw new ValidationError(`--source-root is not a file or directory: ${resolved}. No project files were changed.`);
+  }
+  return resolved;
+}
+
+function runWikiExtractDryRun({ projectPath, sourceRootArg }) {
+  const resolvedProjectPath = resolveProjectPath(projectPath || process.cwd());
+  if (!fs.existsSync(resolvedProjectPath) || !fs.statSync(resolvedProjectPath).isDirectory()) {
+    throw new ValidationError(`--project-path is not a valid directory: ${resolvedProjectPath}`);
+  }
+  const sourceRoot = resolveSourceRoot(sourceRootArg);
+
+  try {
+    const reportsPath = getWikiReportsPath(resolvedProjectPath);
+    assertWikiWorkspaceWritePath(reportsPath, resolvedProjectPath);
+    const extraction = buildExtraction({ projectPath: resolvedProjectPath, sourceRoot });
+    const reportPath = path.join(reportsPath, `${REPORT_PREFIX}-${timestampStamp(new Date(extraction.generated_at))}.json`);
+    assertWikiWorkspaceWritePath(reportPath, resolvedProjectPath);
+    fs.mkdirSync(reportsPath, { recursive: true });
+    fs.writeFileSync(reportPath, JSON.stringify(extraction, null, 2) + "\n", "utf-8");
+    return {
+      reportPath,
+      extraction,
+    };
+  } catch (error) {
+    if (error instanceof CliError) throw error;
+    throw new CliError(`Failed to write SDTK-BRAIN semantic extraction dry-run report: ${error.message}`);
+  }
+}
+
+module.exports = {
+  REPORT_PREFIX,
+  buildExtraction,
+  runWikiExtractDryRun,
+};