sdtk-wiki-kit 0.1.2 → 0.1.4

package/src/lib/wiki-extract.js

@@ -173,6 +173,39 @@ function collectMarkdownFiles(sourceRoot) {
   return { files: files.sort((a, b) => toPosix(a).localeCompare(toPosix(b))), skipped, scanned };
 }
 
+function collectJsonFiles(sourceRoot) {
+  const files = [];
+  let scanned = 0;
+
+  function visit(current) {
+    const stat = fs.statSync(current);
+    if (stat.isDirectory()) {
+      const excluded = isExcluded(current, sourceRoot);
+      if (excluded) return;
+      for (const child of fs.readdirSync(current).sort()) {
+        visit(path.join(current, child));
+      }
+      return;
+    }
+    if (!stat.isFile()) return;
+    if (!/\.json$/i.test(current)) return;
+    scanned += 1;
+    const excluded = isExcluded(current, sourceRoot);
+    if (excluded) return;
+    files.push(current);
+  }
+
+  const stat = fs.statSync(sourceRoot);
+  if (stat.isFile()) {
+    scanned += /\.json$/i.test(sourceRoot) ? 1 : 0;
+    if (/\.json$/i.test(sourceRoot)) files.push(sourceRoot);
+  } else {
+    visit(sourceRoot);
+  }
+
+  return { files: files.sort((a, b) => toPosix(a).localeCompare(toPosix(b))), scanned };
+}
+
 function parseFrontmatterTitle(text) {
   const lines = text.split(/\r?\n/);
   if (!lines.length || lines[0].trim() !== "---") return "";
@@ -224,6 +257,20 @@ function extractGithubRepos(text) {
   return repos;
 }
 
+function parseGithubRepoUrl(value) {
+  const match = String(value || "").match(/^(?:https?:\/\/)?(?:www\.)?github\.com\/([A-Za-z0-9](?:[A-Za-z0-9-]{0,38}))\/([A-Za-z0-9._-]+)(?:\.git)?(?:[/?#].*)?$/i);
+  if (!match) return null;
+  const owner = match[1];
+  const repo = match[2].replace(/[).,;:]+$/g, "").replace(/\.git$/i, "");
+  if (!repo || repo === "..." || repo.includes("...")) return null;
+  return {
+    owner,
+    repo,
+    github_url: `https://github.com/${owner}/${repo}`,
+    key: `${owner.toLowerCase()}/${repo.toLowerCase()}`,
+  };
+}
+
 function extractUnsupportedGithubItems(text) {
   const items = [];
   const invalidRegex = /github\.com\/(?:\.\.\.|[^\s)]+\.{3}[^\s)]*)/gi;
@@ -234,11 +281,39 @@ function extractUnsupportedGithubItems(text) {
   return [...new Set(items)];
 }
 
+function normalizeTopic(value) {
+  return safeSlug(String(value || "").replace(/_/g, " "), "topic");
+}
+
+function conceptFromTopic(topic) {
+  const name = String(topic || "").trim();
+  const slug = normalizeTopic(name);
+  if (!slug || slug === "topic") return null;
+  return {
+    concept_id: `concept_topic_${slug.replace(/-/g, "_")}`,
+    name: name.replace(/[_-]+/g, " "),
+    aliases: [name],
+    definition: `Local structured sources include topic evidence for ${name.replace(/[_-]+/g, " ")}.`,
+    related_entities: [],
+    source_refs: [],
+    provenance_refs: [],
+    confidence: 0.6,
+    confidence_tier: "medium",
+    target_page_path: `wiki/concepts/${slug}.md`,
+  };
+}
+
 function inferConcepts(text) {
   const lower = text.toLowerCase();
   return CONCEPT_RULES.filter((rule) => rule.keywords.some((keyword) => lower.includes(keyword.toLowerCase())));
 }
 
+function inferConceptsFromTopics(topics) {
+  return (Array.isArray(topics) ? topics : [])
+    .map(conceptFromTopic)
+    .filter(Boolean);
+}
+
 function categoryForSource(text, concepts) {
   if (concepts.length > 0) return concepts[0].category;
   const lower = text.toLowerCase();
@@ -246,6 +321,54 @@ function categoryForSource(text, concepts) {
   return "uncategorized";
 }
 
+function confidenceNumber(value) {
+  if (typeof value === "number" && Number.isFinite(value)) return Math.max(0, Math.min(1, value));
+  const text = String(value || "").trim().toLowerCase();
+  if (text === "high") return 0.85;
+  if (text === "medium") return 0.65;
+  if (text === "low") return 0.35;
+  if (text === "unsupported") return 0.1;
+  return 0.5;
+}
+
+function firstArray(value) {
+  if (Array.isArray(value)) return value;
+  if (!value || typeof value !== "object") return [];
+  for (const key of ["records", "repos", "repositories", "items", "data"]) {
+    if (Array.isArray(value[key])) return value[key];
+  }
+  return [value];
+}
+
+function normalizeJsonRepoRecord(raw) {
+  if (!raw || typeof raw !== "object" || Array.isArray(raw)) return null;
+  const repoUrl = String(raw.repo_url || raw.github_url || raw.url || "").trim();
+  const parsedRepo = parseGithubRepoUrl(repoUrl);
+  const owner = String(raw.owner || parsedRepo?.owner || "").trim();
+  const repoName = String(raw.repo_name || raw.name || raw.repo || parsedRepo?.repo || "").trim();
+  const topics = Array.isArray(raw.topics) ? raw.topics.map((topic) => String(topic).trim()).filter(Boolean) : [];
+  const snippet = String(raw.message_text_snippet || raw.snippet || raw.description || raw.summary || "").trim();
+  const sourceLink = String(raw.source_link || raw.source_url || raw.link || "").trim();
+  if (!repoUrl && !owner && !repoName && !snippet && topics.length === 0 && !sourceLink) return null;
+  return {
+    repo_url: parsedRepo ? parsedRepo.github_url : repoUrl || null,
+    owner,
+    repo_name: repoName,
+    message_text_snippet: snippet,
+    source_link: sourceLink || null,
+    topics,
+    confidence_raw: raw.confidence ?? null,
+    confidence: confidenceNumber(raw.confidence),
+    parsed_repo: parsedRepo || (owner && repoName ? {
+      owner,
+      repo: repoName,
+      github_url: repoUrl || `https://github.com/${owner}/${repoName}`,
+      key: `${owner.toLowerCase()}/${repoName.toLowerCase()}`,
+    } : null),
+    raw,
+  };
+}
+
 function confidenceTier(confidence) {
   if (confidence >= 0.8) return "high";
   if (confidence >= 0.5) return "medium";
@@ -292,10 +415,67 @@ function lineOf(text, needle) {
   return text.slice(0, idx).split(/\r?\n/).length;
 }
 
+function boundedText(value, maxLength = 700) {
+  const text = String(value || "").replace(/\s+/g, " ").trim();
+  if (text.length <= maxLength) return text;
+  return `${text.slice(0, maxLength - 1).trim()}...`;
+}
+
+function asArray(value) {
+  return Array.isArray(value) ? value : [];
+}
+
+function localRepoSnippet(text, repoUrl) {
+  const lines = String(text || "").split(/\r?\n/);
+  const index = lines.findIndex((line) => line.includes(repoUrl));
+  if (index < 0) return boundedText(text, 500);
+  const start = Math.max(0, index - 2);
+  const end = Math.min(lines.length, index + 4);
+  return boundedText(lines.slice(start, end).join(" "));
+}
+
+function pushUnique(array, value, keyFn = (item) => item) {
+  if (!value) return;
+  const key = keyFn(value);
+  if (!array.some((item) => keyFn(item) === key)) array.push(value);
+}
+
+function sourceRefsForEntity(entity) {
+  return [...new Set(asArray(entity && entity.source_refs).filter(Boolean).map(String))];
+}
+
+function decisionStrengthsForEntity(entity, conceptName) {
+  const strengths = [];
+  if (sourceRefsForEntity(entity).length > 1) strengths.push("mentioned by multiple local source records");
+  if (asArray(entity.topics).length > 0) strengths.push(`topic fit: ${asArray(entity.topics).slice(0, 3).join(", ")}`);
+  if (entity.category && entity.category !== "uncategorized") strengths.push(`category fit: ${entity.category}`);
+  if (entity.summary) strengths.push("has local snippet evidence");
+  return strengths.length > 0 ? strengths : [`local evidence connects this repository to ${conceptName}`];
+}
+
+function decisionCaveatsForEntity(entity) {
+  const caveats = ["local evidence only; verify externally before adoption"];
+  if (!entity.github_url) caveats.push("missing canonical repository URL");
+  if (["low", "unsupported"].includes(entity.confidence_tier)) caveats.push("low extraction confidence");
+  if (sourceRefsForEntity(entity).length <= 1) caveats.push("single local source reference");
+  return caveats;
+}
+
+function confidenceSummary(records) {
+  const counts = { high: 0, medium: 0, low: 0, unsupported: 0, unknown: 0 };
+  for (const record of asArray(records)) {
+    const tier = String(record.confidence_tier || confidenceTier(record.confidence) || "unknown").toLowerCase();
+    if (Object.prototype.hasOwnProperty.call(counts, tier)) counts[tier] += 1;
+    else counts.unknown += 1;
+  }
+  return counts;
+}
+
 function buildExtraction({ projectPath, sourceRoot }) {
   const generatedAt = new Date().toISOString();
   const sourceRootRef = makeSourceRootRef(sourceRoot);
   const collected = collectMarkdownFiles(sourceRoot);
+  const collectedJson = collectJsonFiles(sourceRoot);
   const sources = [];
   const toolEntitiesById = new Map();
   const conceptsById = new Map();
@@ -367,7 +547,7 @@ function buildExtraction({ projectPath, sourceRoot }) {
         notes: qualityNotes,
       },
       provenance_refs: [],
-      target_page_path: `.sdtk/wiki/personal-brain/sources/${sourceSlug}.md`,
+      target_page_path: `wiki/sources/${sourceSlug}.md`,
     };
 
     sources.push(sourceRecord);
@@ -435,12 +615,37 @@ function buildExtraction({ projectPath, sourceRoot }) {
           confidence_tier: "high",
           source_refs: [],
           provenance_refs: [],
-          target_page_path: `.sdtk/wiki/personal-brain/entities/tools/${safeSlug(repo.repo, "tool")}--${entityId}.md`,
+          topics: concepts.map((concept) => concept.name),
+          source_links: [],
+          evidence_snippets: [],
+          discovery_sources: [],
+          evidence_records: [],
+          target_page_path: `wiki/entities/tools/${safeSlug(repo.repo, "tool")}--${entityId}.md`,
         });
       }
       const entity = toolEntitiesById.get(entityId);
       if (!entity.source_refs.includes(sourceId)) entity.source_refs.push(sourceId);
       if (!entity.provenance_refs.includes(prov.provenance_id)) entity.provenance_refs.push(prov.provenance_id);
+      for (const concept of concepts) {
+        if (!entity.topics) entity.topics = [];
+        pushUnique(entity.topics, concept.name);
+      }
+      const snippet = localRepoSnippet(text, repo.github_url);
+      if (!entity.evidence_snippets) entity.evidence_snippets = [];
+      pushUnique(entity.evidence_snippets, snippet);
+      if (!entity.discovery_sources) entity.discovery_sources = [];
+      pushUnique(entity.discovery_sources, sourceLogicalPath);
+      if (!entity.evidence_records) entity.evidence_records = [];
+      pushUnique(entity.evidence_records, {
+        source_id: sourceId,
+        source_logical_path: sourceLogicalPath,
+        source_link: sourceUrl || null,
+        snippet,
+        topics: concepts.map((concept) => concept.name),
+        provenance_refs: [prov.provenance_id],
+        confidence: 0.9,
+        confidence_tier: "high",
+      }, (record) => `${record.source_id}:${record.source_link || ""}:${record.snippet}`);
 
       claims.push({
         claim_id: `claim_${sourceId}_${String(claims.length + 1).padStart(3, "0")}`,
@@ -478,7 +683,7 @@ function buildExtraction({ projectPath, sourceRoot }) {
           provenance_refs: [],
           confidence: 0.65,
           confidence_tier: "medium",
-          target_page_path: `.sdtk/wiki/personal-brain/concepts/${safeSlug(conceptRule.name, "concept")}.md`,
+          target_page_path: `wiki/concepts/${safeSlug(conceptRule.name, "concept")}.md`,
         });
       }
       const concept = conceptsById.get(conceptRule.concept_id);
@@ -502,6 +707,374 @@ function buildExtraction({ projectPath, sourceRoot }) {
     }
   }
 
+  for (const filePath of collectedJson.files) {
+    const sourceRelativePath = toPosix(path.relative(sourceRoot, filePath));
+    const sourceDisplayPath = toPosix(filePath);
+    const sourceLogicalFilePath = `${sourceRootRef.source_root_label}/${sourceRelativePath}`;
+    const fileBytes = fs.readFileSync(filePath);
+    const fileHash = sha256(fileBytes);
+    const stats = fs.statSync(filePath);
+    let parsed;
+
+    try {
+      parsed = JSON.parse(fileBytes.toString("utf-8"));
+    } catch (error) {
+      const sourceId = `src_${sha256(`local-json-file:v1:${sourceRootRef.source_root_id}:${sourceRelativePath.toLowerCase()}`).slice(0, 16)}`;
+      const sourceRecord = {
+        source_id: sourceId,
+        source_root_id: sourceRootRef.source_root_id,
+        source_relative_path: sourceRelativePath,
+        source_logical_path: sourceLogicalFilePath,
+        source_display_path: sourceDisplayPath,
+        source_type: "json",
+        title: path.basename(filePath),
+        source_url: null,
+        source_hash: fileHash,
+        size_bytes: stats.size,
+        modified_time: stats.mtime.toISOString(),
+        encoding_quality: "unknown",
+        source_quality: {
+          has_mojibake: false,
+          mojibake_score: 0,
+          has_source_url: false,
+          weak_title: false,
+          duplicate_candidate: false,
+          duplicate_group_id: null,
+          low_confidence_extraction: true,
+          quality_flags: ["invalid_json"],
+          notes: [`Invalid JSON could not be parsed: ${error.message}`],
+        },
+        provenance_refs: [],
+        target_page_path: `wiki/sources/${safeSlug(path.basename(filePath), "json-source")}--${sourceId.slice(0, 8)}.md`,
+      };
+      sources.push(sourceRecord);
+      sourceQualityFindings.push({
+        finding_id: `sq_${sourceId}`,
+        source_id: sourceId,
+        source_relative_path: sourceRelativePath,
+        source_logical_path: sourceLogicalFilePath,
+        quality_flags: ["invalid_json"],
+        confidence: 0.1,
+        confidence_tier: "unsupported",
+        notes: sourceRecord.source_quality.notes,
+      });
+      unsupportedItems.push({
+        record_type: "unsupported_item",
+        item_id: `unsupported_${sourceId}_001`,
+        source_id: sourceId,
+        reason: "invalid_json",
+        raw_observation_summary: `Invalid JSON file: ${sourceRelativePath}`,
+        confidence: 0.1,
+        confidence_tier: "unsupported",
+        provenance_refs: [],
+      });
+      continue;
+    }
+
+    const rawRecords = firstArray(parsed);
+    const normalizedRecords = rawRecords.map(normalizeJsonRepoRecord).filter(Boolean);
+    if (normalizedRecords.length === 0) {
+      const sourceId = `src_${sha256(`local-json-empty:v1:${sourceRootRef.source_root_id}:${sourceRelativePath.toLowerCase()}`).slice(0, 16)}`;
+      const sourceRecord = {
+        source_id: sourceId,
+        source_root_id: sourceRootRef.source_root_id,
+        source_relative_path: sourceRelativePath,
+        source_logical_path: sourceLogicalFilePath,
+        source_display_path: sourceDisplayPath,
+        source_type: "json",
+        title: path.basename(filePath),
+        source_url: null,
+        source_hash: fileHash,
+        size_bytes: stats.size,
+        modified_time: stats.mtime.toISOString(),
+        encoding_quality: "clean",
+        source_quality: {
+          has_mojibake: false,
+          mojibake_score: 0,
+          has_source_url: false,
+          weak_title: false,
+          duplicate_candidate: false,
+          duplicate_group_id: null,
+          low_confidence_extraction: true,
+          quality_flags: ["empty_json_records"],
+          notes: ["JSON parsed successfully but contained no supported repository records."],
+        },
+        provenance_refs: [],
+        target_page_path: `wiki/sources/${safeSlug(path.basename(filePath), "json-source")}--${sourceId.slice(0, 8)}.md`,
+      };
+      sources.push(sourceRecord);
+      sourceQualityFindings.push({
+        finding_id: `sq_${sourceId}`,
+        source_id: sourceId,
+        source_relative_path: sourceRelativePath,
+        source_logical_path: sourceLogicalFilePath,
+        quality_flags: ["empty_json_records"],
+        confidence: 0.2,
+        confidence_tier: "unsupported",
+        notes: sourceRecord.source_quality.notes,
+      });
+      continue;
+    }
+
+    normalizedRecords.forEach((record, index) => {
+      const recordRef = `record-${String(index + 1).padStart(3, "0")}`;
+      const repoKey = record.repo_url || `${record.owner}/${record.repo_name}` || sha256(JSON.stringify(record.raw));
+      const sourceId = `src_${sha256(`local-json:v1:${sourceRootRef.source_root_id}:${sourceRelativePath.toLowerCase()}:${repoKey.toLowerCase()}`).slice(0, 16)}`;
+      const sourceHash = sha256(`${fileHash}:${sha256(JSON.stringify(record.raw))}`);
+      const title = record.repo_name || record.owner || `${path.basename(filePath)} ${recordRef}`;
+      const sourceLogicalPath = `${sourceLogicalFilePath}#${recordRef}`;
+      const sourceDisplayRecordPath = `${sourceDisplayPath}#${recordRef}`;
+      const mojibake = detectMojibake(record.message_text_snippet);
+      const weakTitle = title.length < 3;
+      const qualityFlags = [];
+      const qualityNotes = [];
+      const sourceUrl = record.repo_url || record.source_link || null;
+      const confidence = record.confidence;
+      const confidenceBand = confidenceTier(confidence);
+
+      if (mojibake.hasMojibake) {
+        qualityFlags.push("mojibake_detected");
+        qualityNotes.push("Potential mojibake or replacement characters detected in JSON snippet.");
+      }
+      if (!record.parsed_repo) {
+        qualityFlags.push("missing_repo_url");
+        qualityNotes.push("JSON record does not include a supported GitHub repository URL.");
+      }
+      if (!record.source_link) {
+        qualityFlags.push("missing_source_link");
+        qualityNotes.push("JSON record does not include a source_link.");
+      }
+      if (weakTitle) {
+        qualityFlags.push("weak_title");
+        qualityNotes.push("Repository title is missing or very short.");
+      }
+      if (confidence < 0.5) {
+        qualityFlags.push("low_confidence_extraction");
+      }
+
+      const sourceRecord = {
+        source_id: sourceId,
+        source_root_id: sourceRootRef.source_root_id,
+        source_relative_path: sourceRelativePath,
+        source_logical_path: sourceLogicalPath,
+        source_display_path: sourceDisplayRecordPath,
+        source_type: "json_record",
+        title,
+        source_url: sourceUrl,
+        source_hash: sourceHash,
+        size_bytes: stats.size,
+        modified_time: stats.mtime.toISOString(),
+        encoding_quality: mojibake.hasMojibake ? "suspect" : "clean",
+        source_record_locator: {
+          type: "json_record",
+          record_index: index,
+          record_ref: recordRef,
+          record_pointer: `/${index}`,
+        },
+        structured_fields: {
+          repo_url: record.repo_url,
+          owner: record.owner,
+          repo_name: record.repo_name,
+          message_text_snippet: record.message_text_snippet,
+          source_link: record.source_link,
+          topics: record.topics,
+          confidence: record.confidence_raw,
+        },
+        source_quality: {
+          has_mojibake: mojibake.hasMojibake,
+          mojibake_score: Number(mojibake.score.toFixed(3)),
+          has_source_url: Boolean(sourceUrl),
+          weak_title: weakTitle,
+          duplicate_candidate: false,
+          duplicate_group_id: null,
+          low_confidence_extraction: confidence < 0.5,
+          quality_flags: qualityFlags,
+          notes: qualityNotes,
+        },
+        provenance_refs: [],
+        target_page_path: `wiki/sources/${safeSlug(title || sourceRelativePath, "source")}--${sourceId.slice(0, 8)}.md`,
+      };
+
+      sources.push(sourceRecord);
+      if (record.repo_url) {
+        const existing = sourceUrlUsage.get(record.repo_url) || [];
+        existing.push(sourceRecord);
+        sourceUrlUsage.set(record.repo_url, existing);
+      }
+
+      if (qualityFlags.length > 0) {
+        sourceQualityFindings.push({
+          finding_id: `sq_${sourceId}`,
+          source_id: sourceId,
+          source_relative_path: sourceRelativePath,
+          source_logical_path: sourceLogicalPath,
+          quality_flags: [...qualityFlags],
+          confidence,
+          confidence_tier: confidenceBand,
+          notes: qualityNotes,
+        });
+      }
+
+      const concepts = [
+        ...inferConcepts(`${title}\n${record.message_text_snippet}\n${record.topics.join("\n")}`),
+        ...inferConceptsFromTopics(record.topics),
+      ];
+      const category = record.topics[0] ? normalizeTopic(record.topics[0]) : categoryForSource(record.message_text_snippet, concepts);
+
+      if (!record.parsed_repo) {
+        unsupportedItems.push({
+          record_type: "unsupported_item",
+          item_id: `unsupported_${sourceId}_001`,
+          source_id: sourceId,
+          reason: "missing_repo_url",
+          raw_observation_summary: `${sourceRelativePath}#${recordRef}`,
+          confidence: 0.2,
+          confidence_tier: "unsupported",
+          provenance_refs: [],
+        });
+      }
+
+      if (record.parsed_repo) {
+        const repo = record.parsed_repo;
+        const entityId = `tool_github_${safeSlug(repo.owner, "owner")}_${safeSlug(repo.repo, "repo")}`;
+        const prov = {
+          provenance_id: `prov_${sourceId}_${recordRef}`,
+          source_id: sourceId,
+          source_hash: sourceHash,
+          source_relative_path: sourceRelativePath,
+          source_logical_path: sourceLogicalPath,
+          locator: {
+            type: "json_record",
+            record_index: index,
+            record_ref: recordRef,
+            record_pointer: `/${index}`,
+            field: "repo_url",
+          },
+          evidence_quote_hash: sha256(`${sourceId}:${recordRef}:${repo.github_url}`),
+          extractor: "sdtk-wiki.semantic-extract",
+          extractor_version: "bk140-json-records",
+          generated_at: generatedAt,
+          confidence,
+        };
+        provenance.push(prov);
+        sourceRecord.provenance_refs.push(prov.provenance_id);
+
+        if (!toolEntitiesById.has(entityId)) {
+          toolEntitiesById.set(entityId, {
+            entity_id: entityId,
+            entity_type: "tool_entity",
+            name: repo.repo,
+            repo_owner: repo.owner,
+            repo_name: repo.repo,
+            github_url: repo.github_url,
+            category,
+            summary: record.message_text_snippet || `${repo.repo} is a locally sourced GitHub tool candidate in category ${category}.`,
+            confidence,
+            confidence_tier: confidenceBand,
+            source_refs: [],
+            provenance_refs: [],
+            topics: [...record.topics],
+            source_links: record.source_link ? [record.source_link] : [],
+            evidence_snippets: record.message_text_snippet ? [boundedText(record.message_text_snippet)] : [],
+            discovery_sources: [record.source_link, sourceLogicalPath].filter(Boolean),
+            evidence_records: [{
+              source_id: sourceId,
+              source_logical_path: sourceLogicalPath,
+              source_link: record.source_link || null,
+              snippet: boundedText(record.message_text_snippet),
+              topics: [...record.topics],
+              provenance_refs: [prov.provenance_id],
+              confidence,
+              confidence_tier: confidenceBand,
+            }],
+            target_page_path: `wiki/entities/tools/${safeSlug(repo.repo, "tool")}--${entityId}.md`,
+          });
+        }
+        const entity = toolEntitiesById.get(entityId);
+        if (!entity.source_refs.includes(sourceId)) entity.source_refs.push(sourceId);
+        if (!entity.provenance_refs.includes(prov.provenance_id)) entity.provenance_refs.push(prov.provenance_id);
+        for (const topic of record.topics) {
+          if (!entity.topics) entity.topics = [];
+          if (!entity.topics.includes(topic)) entity.topics.push(topic);
+        }
+        if (record.source_link) {
+          if (!entity.source_links) entity.source_links = [];
+          if (!entity.source_links.includes(record.source_link)) entity.source_links.push(record.source_link);
+        }
+        if (record.message_text_snippet) {
+          if (!entity.evidence_snippets) entity.evidence_snippets = [];
+          pushUnique(entity.evidence_snippets, boundedText(record.message_text_snippet));
+        }
+        if (!entity.discovery_sources) entity.discovery_sources = [];
+        pushUnique(entity.discovery_sources, record.source_link);
+        pushUnique(entity.discovery_sources, sourceLogicalPath);
+        if (!entity.evidence_records) entity.evidence_records = [];
+        pushUnique(entity.evidence_records, {
+          source_id: sourceId,
+          source_logical_path: sourceLogicalPath,
+          source_link: record.source_link || null,
+          snippet: boundedText(record.message_text_snippet),
+          topics: [...record.topics],
+          provenance_refs: [prov.provenance_id],
+          confidence,
+          confidence_tier: confidenceBand,
+        }, (item) => `${item.source_id}:${item.source_link || ""}:${item.snippet || ""}`);
+
+        claims.push({
+          claim_id: `claim_${sourceId}_${String(claims.length + 1).padStart(3, "0")}`,
+          text: `The local JSON record presents ${repo.repo} as a ${category} repository candidate.`,
+          subject_entity_id: entityId,
+          source_refs: [sourceId],
+          provenance_refs: [prov.provenance_id],
+          confidence,
+          confidence_tier: confidenceBand,
+          contested: false,
+        });
+
+        relations.push({
+          relation_id: `rel_${sourceId}_${String(relations.length + 1).padStart(3, "0")}`,
+          source_id: sourceId,
+          target_id: entityId,
+          relation_type: "source_mentions_entity",
+          evidence: "The local JSON record includes a GitHub repository URL.",
+          source_refs: [sourceId],
+          provenance_refs: [prov.provenance_id],
+          confidence,
+          confidence_tier: confidenceBand,
+        });
+
+        for (const conceptRule of concepts) {
+          if (!conceptsById.has(conceptRule.concept_id)) {
+            conceptsById.set(conceptRule.concept_id, {
+              ...conceptRule,
+              related_entities: conceptRule.related_entities || [],
+              source_refs: conceptRule.source_refs || [],
+              provenance_refs: conceptRule.provenance_refs || [],
+              confidence: conceptRule.confidence || 0.6,
+              confidence_tier: conceptRule.confidence_tier || "medium",
+              target_page_path: conceptRule.target_page_path || `wiki/concepts/${safeSlug(conceptRule.name, "concept")}.md`,
+            });
+          }
+          const concept = conceptsById.get(conceptRule.concept_id);
+          if (!concept.source_refs.includes(sourceId)) concept.source_refs.push(sourceId);
+          if (!concept.provenance_refs.includes(prov.provenance_id)) concept.provenance_refs.push(prov.provenance_id);
+          if (!concept.related_entities.includes(entityId)) concept.related_entities.push(entityId);
+          relations.push({
+            relation_id: `rel_${sourceId}_${String(relations.length + 1).padStart(3, "0")}`,
+            source_id: entityId,
+            target_id: conceptRule.concept_id,
+            relation_type: "entity_implements_concept",
+            evidence: "The local JSON record includes matching topics or semantic keywords.",
+            source_refs: [sourceId],
+            provenance_refs: [prov.provenance_id],
+            confidence: Math.min(0.8, confidence + 0.05),
+            confidence_tier: confidenceTier(Math.min(0.8, confidence + 0.05)),
+          });
+        }
+      }
+    });
+  }
+
   for (const [sourceUrl, sourceRecords] of sourceUrlUsage.entries()) {
     if (sourceRecords.length < 2) continue;
     const duplicateGroupId = `dup_${sha256(sourceUrl).slice(0, 12)}`;
@@ -525,6 +1098,67 @@ function buildExtraction({ projectPath, sourceRoot }) {
     }
   }
 
+  const toolEntityValues = [...toolEntitiesById.values()];
+  for (const entity of toolEntityValues) {
+    const entityTopics = new Set(asArray(entity.topics).map((topic) => String(topic).toLowerCase()));
+    const related = [];
+    for (const candidate of toolEntityValues) {
+      if (candidate.entity_id === entity.entity_id) continue;
+      const sharedTopics = asArray(candidate.topics).filter((topic) => entityTopics.has(String(topic).toLowerCase()));
+      const sameCategory = entity.category && candidate.category && entity.category === candidate.category;
+      if (sharedTopics.length === 0 && !sameCategory) continue;
+      related.push({
+        entity_id: candidate.entity_id,
+        name: candidate.name || candidate.repo_name || candidate.entity_id,
+        github_url: candidate.github_url || null,
+        shared_topics: sharedTopics,
+        relation_hint: sharedTopics.length > 0 ? "shared_topic" : "same_category",
+      });
+    }
+    entity.related_repos = related
+      .sort((a, b) => b.shared_topics.length - a.shared_topics.length || a.entity_id.localeCompare(b.entity_id))
+      .slice(0, 8);
+  }
+
+  for (const concept of conceptsById.values()) {
+    const relatedDetails = [];
+    const axisCounts = new Map();
+    for (const entityId of asArray(concept.related_entities)) {
+      const entity = toolEntitiesById.get(entityId);
+      if (!entity) continue;
+      for (const topic of asArray(entity.topics)) {
+        const key = String(topic || "").trim();
+        if (key) axisCounts.set(key, (axisCounts.get(key) || 0) + 1);
+      }
+      if (entity.category) axisCounts.set(entity.category, (axisCounts.get(entity.category) || 0) + 1);
+      relatedDetails.push({
+        entity_id: entity.entity_id,
+        name: entity.name || entity.repo_name || entity.entity_id,
+        repo_owner: entity.repo_owner || null,
+        repo_name: entity.repo_name || null,
+        github_url: entity.github_url || null,
+        category: entity.category || "uncategorized",
+        topics: asArray(entity.topics).slice(0, 8),
+        summary: entity.summary || "",
+        target_page_path: entity.target_page_path || null,
+        source_refs: sourceRefsForEntity(entity),
+        confidence: entity.confidence,
+        confidence_tier: entity.confidence_tier || confidenceTier(entity.confidence),
+      });
+    }
+    concept.related_entity_details = relatedDetails
+      .sort((a, b) => (b.source_refs.length - a.source_refs.length) || a.name.localeCompare(b.name))
+      .slice(0, 12);
+    concept.key_axes = [...axisCounts.entries()]
+      .sort((a, b) => b[1] - a[1] || a[0].localeCompare(b[0]))
+      .slice(0, 8)
+      .map(([name, count]) => ({ name, evidence_count: count }));
+    concept.patterns = concept.key_axes.slice(0, 5).map((axis) => ({
+      pattern: axis.name,
+      evidence: `${axis.evidence_count} locally extracted tool candidate(s) are associated with this axis.`,
+    }));
+  }
+
   const toolEntities = [...toolEntitiesById.values()].sort((a, b) => a.entity_id.localeCompare(b.entity_id));
   const concepts = [...conceptsById.values()].sort((a, b) => a.concept_id.localeCompare(b.concept_id));
   const comparisons = [];
@@ -533,27 +1167,69 @@ function buildExtraction({ projectPath, sourceRoot }) {
   for (const concept of concepts) {
     if (concept.related_entities.length < 2) continue;
     const topicSlug = safeSlug(concept.name, "topic");
+    const comparedDetails = asArray(concept.related_entity_details).slice(0, 8);
+    const decisionAxes = asArray(concept.key_axes).slice(0, 6);
+    const matrixRows = comparedDetails.map((entity) => ({
+      entity_id: entity.entity_id,
+      name: entity.repo_owner && entity.repo_name ? `${entity.repo_owner}/${entity.repo_name}` : entity.name,
+      github_url: entity.github_url || null,
+      category: entity.category || "uncategorized",
+      topics: asArray(entity.topics).slice(0, 6),
+      strengths: decisionStrengthsForEntity(entity, concept.name),
+      caveats: decisionCaveatsForEntity(entity),
+      source_ref_count: sourceRefsForEntity(entity).length,
+      source_confidence: entity.confidence_tier || confidenceTier(entity.confidence),
+      local_recommendation: sourceRefsForEntity(entity).length > 1
+        ? "shortlist for human review"
+        : "keep as a candidate until more local evidence is available",
+    }));
+    const summary = `Local sources mention ${concept.related_entities.length} tool candidate(s) related to ${concept.name}. This page compares candidates for review, not verified ranking.`;
+    const recommendations = matrixRows.slice(0, 3).map((row) => `${row.name}: ${row.local_recommendation}; strengths: ${row.strengths.slice(0, 2).join("; ")}.`);
     comparisons.push({
       comparison_id: `comparison_${topicSlug}_${sha256(concept.related_entities.join("|")).slice(0, 8)}`,
       topic: concept.name,
       compared_entities: concept.related_entities.slice(0, 8),
-      criteria: ["local evidence", "category fit", "source confidence"],
+      compared_entity_details: comparedDetails,
+      decision_axes: decisionAxes,
+      matrix_rows: matrixRows,
+      criteria: ["local evidence", "category/topic fit", "source confidence", "review caveats"],
+      summary,
+      recommendations,
+      caveats: [
+        "The comparison uses local source evidence only.",
+        "Do not treat ordering as an external quality ranking.",
+        "Verify license, maintenance, security, and ecosystem fit before adoption.",
+      ],
+      source_confidence_summary: confidenceSummary(matrixRows),
       source_refs: concept.source_refs,
       provenance_refs: concept.provenance_refs,
       confidence: 0.55,
       confidence_tier: "medium",
-      target_page_path: `.sdtk/wiki/personal-brain/comparisons/${topicSlug}.md`,
+      target_page_path: `wiki/comparisons/${topicSlug}.md`,
     });
     syntheses.push({
       synthesis_id: `synthesis_${topicSlug}_${sha256(concept.source_refs.join("|")).slice(0, 8)}`,
       topic: concept.name,
-      summary: `Local sources mention ${concept.related_entities.length} tool candidate(s) related to ${concept.name}.`,
-      recommendations: ["Review extracted entities and source quality findings before compile/apply work."],
+      summary,
+      landscape_axes: decisionAxes,
+      candidate_tools: matrixRows,
+      patterns: asArray(concept.patterns),
+      related_comparison_path: `wiki/comparisons/${topicSlug}.md`,
+      source_confidence_summary: confidenceSummary(matrixRows),
+      recommendations: [
+        ...recommendations,
+        "Use this synthesis to select review candidates; defer adoption until external verification is complete.",
+      ],
+      caveats: [
+        "Local extraction can include stale or incomplete source snippets.",
+        "No web verification, GitHub API data, stars, licenses, or release cadence are claimed here.",
+        "Human review should resolve topic fit and source-quality warnings before product decisions.",
+      ],
       source_refs: concept.source_refs,
       provenance_refs: concept.provenance_refs,
       confidence: 0.55,
       confidence_tier: "medium",
-      target_page_path: `.sdtk/wiki/personal-brain/syntheses/${topicSlug}.md`,
+      target_page_path: `wiki/syntheses/${topicSlug}.md`,
     });
   }
 
@@ -566,7 +1242,7 @@ function buildExtraction({ projectPath, sourceRoot }) {
     project_path: projectPath,
     source_root_refs: [sourceRootRef],
     source_counts: {
-      scanned: collected.scanned,
+      scanned: collected.scanned + collectedJson.scanned,
       indexed: sources.length,
       extracted: toolEntities.length,
       skipped: collected.skipped.length,