sdtk-wiki-kit 0.1.2 → 0.1.4

package/src/lib/wiki-enrich.js

@@ -0,0 +1,264 @@
+"use strict";
+
+const fs = require("fs");
+const path = require("path");
+const { ValidationError } = require("./errors");
+const {
+  assertWikiWorkspaceWritePath,
+  getPreferredWikiContentPath,
+  getWikiReportsPath,
+  isPathInsideOrEqual,
+  resolveProjectPath,
+} = require("./wiki-paths");
+
+const REPORT_PREFIX = "github-enrichment-review";
+
+function toPosix(value) {
+  return String(value || "").replace(/\\/g, "/");
+}
+
+function todayStamp() {
+  return new Date().toISOString().slice(0, 10);
+}
+
+function parseFrontmatter(text) {
+  const source = String(text || "");
+  const match = source.match(/^---\r?\n([\s\S]*?)\r?\n---\r?\n?/);
+  if (!match) return { fields: {}, body: source };
+  const fields = {};
+  for (const line of match[1].split(/\r?\n/)) {
+    const separator = line.indexOf(":");
+    if (separator === -1) continue;
+    const key = line.slice(0, separator).trim();
+    const value = line.slice(separator + 1).trim();
+    fields[key] = value.replace(/^["']|["']$/g, "");
+  }
+  return { fields, body: source.slice(match[0].length) };
+}
+
+function parseInlineList(value) {
+  const text = String(value || "").trim();
+  if (!text || text === "[]") return [];
+  const inner = text.startsWith("[") && text.endsWith("]") ? text.slice(1, -1) : text;
+  return inner
+    .split(",")
+    .map((item) => item.trim().replace(/^["']|["']$/g, ""))
+    .filter(Boolean);
+}
+
+function collectMarkdownFiles(rootPath) {
+  const files = [];
+  if (!fs.existsSync(rootPath)) return files;
+  function visit(current) {
+    const stat = fs.statSync(current);
+    if (stat.isDirectory()) {
+      for (const child of fs.readdirSync(current).sort()) {
+        visit(path.join(current, child));
+      }
+      return;
+    }
+    if (stat.isFile() && current.toLowerCase().endsWith(".md")) {
+      files.push(current);
+    }
+  }
+  visit(rootPath);
+  return files.sort((a, b) => toPosix(a).localeCompare(toPosix(b)));
+}
+
+function extractGithubRepos(text) {
+  const repos = [];
+  const seen = new Set();
+  const matcher = /https?:\/\/github\.com\/([A-Za-z0-9](?:[A-Za-z0-9-]{0,38}))\/([A-Za-z0-9._-]+)/gi;
+  let match;
+  while ((match = matcher.exec(String(text || ""))) !== null) {
+    const owner = match[1];
+    const repo = match[2].replace(/[).,;:]+$/g, "").replace(/\.git$/i, "");
+    if (!repo || repo.includes("...")) continue;
+    const url = `https://github.com/${owner}/${repo}`;
+    const key = url.toLowerCase();
+    if (seen.has(key)) continue;
+    seen.add(key);
+    repos.push({ owner, repo, url });
+  }
+  return repos;
+}
+
+function firstSection(body, heading) {
+  const escaped = String(heading).replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
+  const matcher = new RegExp(`^##\\s+${escaped}\\s*$([\\s\\S]*?)(?=^##\\s+|$)`, "im");
+  const match = String(body || "").match(matcher);
+  if (!match) return "";
+  return match[1].replace(/\s+/g, " ").trim();
+}
+
+function extractProvenanceRefs(body) {
+  const section = firstSection(body, "Provenance");
+  if (!section) return [];
+  return section
+    .split(/\s*-\s+/)
+    .map((item) => item.trim())
+    .filter((item) => item.startsWith("prov_"))
+    .slice(0, 12);
+}
+
+function recordFromPage(filePath, contentRoot) {
+  const text = fs.readFileSync(filePath, "utf-8");
+  const parsed = parseFrontmatter(text);
+  const fields = parsed.fields;
+  const repos = extractGithubRepos(text);
+  if (repos.length === 0) return [];
+
+  const relPath = toPosix(path.relative(contentRoot.path, filePath));
+  const type = String(fields.type || "");
+  const title = String(fields.title || path.basename(filePath, ".md"));
+  const tags = parseInlineList(fields.tags);
+  const sourceRefs = parseInlineList(fields.source_refs);
+  const summary = firstSection(parsed.body, "Summary");
+  const provenanceRefs = extractProvenanceRefs(parsed.body);
+
+  return repos.map((repo) => ({
+    record_type: "sdtk_wiki_github_enrichment_candidate",
+    status: "review_only_local_identity",
+    page_path: `${toPosix(contentRoot.relative)}/${relPath}`,
+    page_type: type || "unknown",
+    page_title: title,
+    repo_url: repo.url,
+    source_url: repo.url,
+    owner: repo.owner,
+    repo: repo.repo,
+    local_summary: summary || null,
+    local_topics: tags,
+    source_refs: sourceRefs,
+    provenance_refs: provenanceRefs,
+    confidence: fields.confidence || "unknown",
+    metadata: {
+      stars: null,
+      license: null,
+      description: null,
+      current_owner: null,
+      default_branch: null,
+      last_verified_at: null,
+    },
+    fetch_timestamp: null,
+    fetch_status: "not_fetched_local_review_mode",
+    failure_status: "not_fetched_local_review_mode",
+    failure_reason: "Network metadata was not requested in review-only local mode.",
+    enrichment_mode: "review",
+    mutation: "none",
+  }));
+}
+
+function uniqueRecords(records) {
+  const byRepo = new Map();
+  for (const record of records) {
+    const key = record.repo_url.toLowerCase();
+    const existing = byRepo.get(key);
+    if (!existing) {
+      byRepo.set(key, record);
+      continue;
+    }
+    existing.page_path = existing.page_path || record.page_path;
+    existing.local_topics = Array.from(new Set([...existing.local_topics, ...record.local_topics])).sort();
+    existing.source_refs = Array.from(new Set([...existing.source_refs, ...record.source_refs])).sort();
+    existing.provenance_refs = Array.from(new Set([...existing.provenance_refs, ...record.provenance_refs])).sort();
+  }
+  return [...byRepo.values()].sort((a, b) => a.repo_url.localeCompare(b.repo_url));
+}
+
+function renderMarkdownReport({ projectPath, reportJsonPath, records, contentRoot }) {
+  const lines = [
+    "# SDTK-WIKI GitHub Enrichment Review",
+    "",
+    `Date: ${todayStamp()}`,
+    `Project root: \`${projectPath}\``,
+    `Wiki content root: \`${contentRoot.path}\``,
+    `Wiki content mode: \`${contentRoot.mode}\``,
+    `JSON report: \`${reportJsonPath}\``,
+    "",
+    "Report-only and non-destructive: this command does not fetch network metadata, rewrite pages, mutate sources, or touch `.sdtk/atlas`.",
+    "Network-backed GitHub verification is deferred to a later controller-approved slice.",
+    "",
+    "## Summary",
+    "",
+    `- candidates: ${records.length}`,
+    "- source: github",
+    "- mode: review",
+    "- fetch status: not_fetched_local_review_mode",
+    "",
+    "## Candidates",
+    "",
+    "| Repo | Page | Confidence | Source refs | Fetch status | Pending metadata |",
+    "|---|---|---|---:|---|---|",
+  ];
+
+  for (const record of records) {
+    lines.push(
+      `| [${record.owner}/${record.repo}](${record.repo_url}) | \`${record.page_path}\` | ${record.confidence} | ${record.source_refs.length} | ${record.fetch_status} | stars, license, description, current owner |`
+    );
+  }
+  if (records.length === 0) {
+    lines.push("| none | - | - | 0 | - | - |");
+  }
+
+  lines.push(
+    "",
+    "## Review Notes",
+    "",
+    "- Treat every candidate as locally identified only until a future explicit network/enrichment apply issue verifies it.",
+    "- Do not copy stars, license, or activity claims into generated pages from this report unless those fields are later verified.",
+    ""
+  );
+  return lines.join("\n");
+}
+
+function runWikiGithubEnrichmentReview(options = {}) {
+  const projectPath = resolveProjectPath(options.projectPath || process.cwd());
+  if (!fs.existsSync(projectPath) || !fs.statSync(projectPath).isDirectory()) {
+    throw new ValidationError(`--project-path is not a valid directory: ${projectPath}`);
+  }
+
+  const contentRoot = getPreferredWikiContentPath(projectPath);
+  if (!isPathInsideOrEqual(contentRoot.path, projectPath)) {
+    throw new ValidationError("Refusing to read local wiki pages outside the project root.");
+  }
+  if (!fs.existsSync(contentRoot.path) || !fs.statSync(contentRoot.path).isDirectory()) {
+    throw new ValidationError(
+      `No SDTK-WIKI local wiki found at ${contentRoot.path}. Run "sdtk-wiki ingest <source-root>" and "sdtk-wiki compile --mode safe --apply" first.`
+    );
+  }
+
+  const records = uniqueRecords(collectMarkdownFiles(contentRoot.path).flatMap((filePath) =>
+    recordFromPage(filePath, contentRoot)
+  ));
+
+  const reportsRoot = getWikiReportsPath(projectPath);
+  assertWikiWorkspaceWritePath(reportsRoot, projectPath);
+  fs.mkdirSync(reportsRoot, { recursive: true });
+  const jsonPath = path.join(reportsRoot, `${REPORT_PREFIX}-${todayStamp()}.json`);
+  const markdownPath = path.join(reportsRoot, `${REPORT_PREFIX}-${todayStamp()}.md`);
+  assertWikiWorkspaceWritePath(jsonPath, projectPath);
+  assertWikiWorkspaceWritePath(markdownPath, projectPath);
+
+  const payload = {
+    schema_version: 1,
+    record_type: "sdtk_wiki_github_enrichment_review",
+    generated_at: new Date().toISOString(),
+    project_path: projectPath,
+    wiki_content_path: contentRoot.path,
+    wiki_content_mode: contentRoot.mode,
+    source: "github",
+    mode: "review",
+    network_fetch: false,
+    mutation: "none",
+    candidate_count: records.length,
+    records,
+  };
+  fs.writeFileSync(jsonPath, JSON.stringify(payload, null, 2) + "\n", "utf-8");
+  fs.writeFileSync(markdownPath, renderMarkdownReport({ projectPath, reportJsonPath: jsonPath, records, contentRoot }), "utf-8");
+
+  return { markdownPath, jsonPath, records, projectPath };
+}
+
+module.exports = {
+  runWikiGithubEnrichmentReview,
+};