sdtk-wiki-kit 0.1.0

package/src/lib/wiki-lint.js

@@ -0,0 +1,468 @@
+"use strict";
+
+const fs = require("fs");
+const path = require("path");
+const { CliError, ValidationError } = require("./errors");
+const {
+  assertWikiWorkspaceWritePath,
+  getWikiGraphPath,
+  getWikiPagesPath,
+  getWikiProvenanceSourcesPath,
+  getWikiReportsPath,
+  getWikiWorkspacePath,
+  isPathInsideOrEqual,
+  resolveProjectPath,
+} = require("./wiki-paths");
+
+const REQUIRED_PAGE_FIELDS = [
+  "schema_version",
+  "product",
+  "managed_by",
+  "page_id",
+  "source_path",
+  "source_hash",
+  "title",
+  "family",
+  "role",
+];
+
+const CATEGORY_DEFS = [
+  ["schema", "Frontmatter and schema"],
+  ["duplicates", "Duplicate page IDs"],
+  ["brokenLinks", "Broken internal links"],
+  ["orphans", "Orphan pages"],
+  ["missingReciprocal", "Missing reciprocal related_pages"],
+  ["provenance", "Provenance gaps"],
+  ["downstream", "Downstream integration gaps"],
+  ["thin", "Thin pages"],
+  ["stale", "Stale pages"],
+  ["markers", "TODO/Open Questions/Gaps"],
+  ["contradictions", "Candidate contradictions"],
+];
+
+function toPosix(value) {
+  return value.split(path.sep).join("/");
+}
+
+function stripQuotes(value) {
+  const text = String(value || "").trim();
+  if (
+    (text.startsWith('"') && text.endsWith('"')) ||
+    (text.startsWith("'") && text.endsWith("'"))
+  ) {
+    return text.slice(1, -1);
+  }
+  return text;
+}
+
+function parseInlineList(value) {
+  const text = String(value || "").trim();
+  if (!text) return [];
+  if (text.startsWith("[") && text.endsWith("]")) {
+    const inner = text.slice(1, -1).trim();
+    if (!inner) return [];
+    return inner
+      .split(",")
+      .map((item) => stripQuotes(item.trim()))
+      .filter(Boolean);
+  }
+  return text
+    .split(",")
+    .map((item) => stripQuotes(item.trim()))
+    .filter(Boolean);
+}
+
+function parseFrontmatter(text) {
+  const source = String(text || "");
+  const match = source.match(/^---\r?\n([\s\S]*?)\r?\n---\r?\n?/);
+  if (!match) {
+    return { fields: {}, body: source, hasFrontmatter: false };
+  }
+
+  const block = match[1].split(/\r?\n/);
+  const fields = {};
+  for (const line of block) {
+    const separator = line.indexOf(":");
+    if (separator === -1) continue;
+    const key = line.slice(0, separator).trim();
+    const rawValue = line.slice(separator + 1).trim();
+    fields[key] = key === "related_pages" ? parseInlineList(rawValue) : stripQuotes(rawValue);
+  }
+
+  return {
+    fields,
+    body: source.slice(match[0].length),
+    hasFrontmatter: true,
+  };
+}
+
+function readJsonIfPresent(filePath, fallback = null) {
+  if (!fs.existsSync(filePath)) return fallback;
+  try {
+    return JSON.parse(fs.readFileSync(filePath, "utf-8"));
+  } catch (error) {
+    return { __lintError: `Could not parse JSON at ${filePath}: ${error.message}` };
+  }
+}
+
+function listMarkdownPages(rootPath) {
+  if (!fs.existsSync(rootPath)) return [];
+  const files = [];
+
+  function walk(current) {
+    const entries = fs.readdirSync(current, { withFileTypes: true }).sort((a, b) =>
+      a.name.localeCompare(b.name)
+    );
+    for (const entry of entries) {
+      const absolute = path.join(current, entry.name);
+      if (entry.isDirectory()) {
+        walk(absolute);
+      } else if (entry.isFile() && entry.name.toLowerCase().endsWith(".md")) {
+        if (entry.name === "_index.md") continue;
+        files.push(absolute);
+      }
+    }
+  }
+
+  walk(rootPath);
+  return files;
+}
+
+function extractMarkdownLinks(body) {
+  const links = [];
+  const matcher = /\[[^\]]+\]\(([^)]+)\)/g;
+  let match;
+  while ((match = matcher.exec(body || "")) !== null) {
+    const rawTarget = String(match[1] || "").trim();
+    if (!rawTarget) continue;
+    if (
+      rawTarget.startsWith("#") ||
+      /^(?:https?:|mailto:|tel:)/i.test(rawTarget)
+    ) {
+      continue;
+    }
+    links.push(rawTarget);
+  }
+  return links;
+}
+
+function createFindings() {
+  return Object.fromEntries(CATEGORY_DEFS.map(([key]) => [key, []]));
+}
+
+function appendFinding(findings, category, text) {
+  findings[category].push(text);
+}
+
+function graphDegreeMap(graphPayload) {
+  const degrees = new Map();
+  if (!graphPayload || graphPayload.__lintError) return degrees;
+  const edges = Array.isArray(graphPayload.edges) ? graphPayload.edges : [];
+  for (const edge of edges) {
+    const source = typeof edge.source === "string" ? edge.source : null;
+    const target = typeof edge.target === "string" ? edge.target : null;
+    if (source) degrees.set(source, (degrees.get(source) || 0) + 1);
+    if (target) degrees.set(target, (degrees.get(target) || 0) + 1);
+  }
+  return degrees;
+}
+
+function readLintInputs(projectPath, findings) {
+  const pagesRoot = getWikiPagesPath(projectPath);
+  const pageFiles = listMarkdownPages(pagesRoot);
+  const provenance = readJsonIfPresent(getWikiProvenanceSourcesPath(projectPath), { sources: [] });
+  const graph = readJsonIfPresent(path.join(getWikiGraphPath(projectPath), "SDTK_DOC_GRAPH.json"), {
+    edges: [],
+  });
+
+  if (provenance && provenance.__lintError) {
+    appendFinding(findings, "provenance", provenance.__lintError);
+  }
+  if (graph && graph.__lintError) {
+    appendFinding(findings, "downstream", graph.__lintError);
+  }
+
+  const sources = provenance && Array.isArray(provenance.sources) ? provenance.sources : [];
+  return { graph, pageFiles, pagesRoot, sources };
+}
+
+function analyzePages(projectPath) {
+  const findings = createFindings();
+  const inputs = readLintInputs(projectPath, findings);
+  const pages = [];
+  const pagesById = new Map();
+  const inboundMarkdown = new Map();
+  const inboundRelated = new Map();
+
+  for (const filePath of inputs.pageFiles) {
+    const relPath = toPosix(path.relative(inputs.pagesRoot, filePath));
+    const parsed = parseFrontmatter(fs.readFileSync(filePath, "utf-8"));
+    const fields = parsed.fields;
+    const page = {
+      filePath,
+      relPath,
+      body: parsed.body,
+      fields,
+      pageId: fields.page_id || "",
+      sourcePath: fields.source_path || "",
+      sourceHash: fields.source_hash || "",
+      relatedPages: Array.isArray(fields.related_pages) ? fields.related_pages : [],
+      links: extractMarkdownLinks(parsed.body),
+    };
+    pages.push(page);
+
+    if (!parsed.hasFrontmatter) {
+      appendFinding(findings, "schema", `\`${relPath}\` is missing parseable frontmatter.`);
+    }
+
+    const missingFields = REQUIRED_PAGE_FIELDS.filter((field) => !fields[field]);
+    if (missingFields.length > 0) {
+      appendFinding(
+        findings,
+        "schema",
+        `\`${relPath}\` is missing required fields: ${missingFields.join(", ")}.`
+      );
+    }
+    if (fields.product && fields.product !== "SDTK-WIKI") {
+      appendFinding(findings, "schema", `\`${relPath}\` has unexpected product \`${fields.product}\`.`);
+    }
+    if (fields.managed_by && fields.managed_by !== "sdtk-wiki") {
+      appendFinding(findings, "schema", `\`${relPath}\` has unexpected managed_by \`${fields.managed_by}\`.`);
+    }
+
+    if (page.pageId) {
+      if (!pagesById.has(page.pageId)) {
+        pagesById.set(page.pageId, []);
+      }
+      pagesById.get(page.pageId).push(page);
+    }
+  }
+
+  for (const [pageId, records] of pagesById.entries()) {
+    if (records.length > 1) {
+      appendFinding(
+        findings,
+        "duplicates",
+        `\`${pageId}\` appears in ${records.map((record) => `\`${record.relPath}\``).join(", ")}.`
+      );
+    }
+  }
+
+  const pagesByRelPath = new Map(pages.map((page) => [toPosix(path.normalize(page.relPath)), page]));
+  for (const page of pages) {
+    for (const link of page.links) {
+      const rawPath = link.split("#")[0].trim();
+      if (!rawPath) continue;
+      const resolved = path.resolve(path.dirname(page.filePath), rawPath);
+      if (!isPathInsideOrEqual(resolved, inputs.pagesRoot) || !fs.existsSync(resolved)) {
+        appendFinding(findings, "brokenLinks", `\`${page.relPath}\` links to missing \`${link}\`.`);
+        continue;
+      }
+      const targetRel = toPosix(path.relative(inputs.pagesRoot, resolved));
+      if (pagesByRelPath.has(targetRel)) {
+        inboundMarkdown.set(targetRel, (inboundMarkdown.get(targetRel) || 0) + 1);
+      }
+    }
+
+    for (const targetId of page.relatedPages) {
+      const targetRecords = pagesById.get(targetId) || [];
+      if (targetRecords.length === 0) {
+        appendFinding(
+          findings,
+          "missingReciprocal",
+          `\`${page.relPath}\` declares related page \`${targetId}\`, but no target page exists.`
+        );
+        continue;
+      }
+      for (const target of targetRecords) {
+        inboundRelated.set(target.pageId, (inboundRelated.get(target.pageId) || 0) + 1);
+        if (!target.relatedPages.includes(page.pageId)) {
+          appendFinding(
+            findings,
+            "missingReciprocal",
+            `\`${page.pageId}\` references \`${targetId}\`, but the target does not reference \`${page.pageId}\`.`
+          );
+        }
+      }
+    }
+  }
+
+  const provenanceBySource = new Map(
+    inputs.sources
+      .filter((record) => record && typeof record.sourcePath === "string")
+      .map((record) => [record.sourcePath, record])
+  );
+  const degrees = graphDegreeMap(inputs.graph);
+
+  for (const page of pages) {
+    const sourceRecord = page.sourcePath ? provenanceBySource.get(page.sourcePath) : null;
+    if (page.sourcePath && !sourceRecord) {
+      appendFinding(
+        findings,
+        "provenance",
+        `\`${page.relPath}\` references source \`${page.sourcePath}\` missing from provenance sources.`
+      );
+      appendFinding(
+        findings,
+        "stale",
+        `\`${page.relPath}\` may be stale because source \`${page.sourcePath}\` is absent from provenance.`
+      );
+    } else if (
+      page.sourcePath &&
+      sourceRecord &&
+      typeof sourceRecord.sourceHash === "string" &&
+      page.sourceHash &&
+      sourceRecord.sourceHash !== page.sourceHash
+    ) {
+      appendFinding(
+        findings,
+        "provenance",
+        `\`${page.relPath}\` source hash differs from provenance for \`${page.sourcePath}\`.`
+      );
+      appendFinding(
+        findings,
+        "stale",
+        `\`${page.relPath}\` may be stale because its source hash differs from provenance.`
+      );
+    }
+
+    const inboundLinks = inboundMarkdown.get(page.relPath) || 0;
+    const relatedInboundCount = inboundRelated.get(page.pageId) || 0;
+    if (inboundLinks === 0 && relatedInboundCount === 0) {
+      appendFinding(findings, "orphans", `\`${page.relPath}\` has no inbound wiki links or reciprocal relationships.`);
+    }
+
+    const graphDegree = page.sourcePath ? degrees.get(page.sourcePath) || 0 : 0;
+    if (
+      page.sourcePath &&
+      graphDegree === 0 &&
+      inboundLinks === 0 &&
+      relatedInboundCount === 0 &&
+      page.relatedPages.length === 0
+    ) {
+      appendFinding(
+        findings,
+        "downstream",
+        `\`${page.relPath}\` has no downstream integration signal for source \`${page.sourcePath}\`.`
+      );
+    }
+
+    const compactBody = page.body.replace(/\s+/g, " ").trim();
+    if (compactBody.length < 120) {
+      appendFinding(findings, "thin", `\`${page.relPath}\` is thin (${compactBody.length} normalized characters).`);
+    }
+
+    const markerLabels = [];
+    if (/\bTODO\b/i.test(page.body)) markerLabels.push("TODO");
+    if (/open questions/i.test(page.body)) markerLabels.push("Open Questions");
+    if (/\bgaps?\b/i.test(page.body)) markerLabels.push("Gaps");
+    if (markerLabels.length > 0) {
+      appendFinding(findings, "markers", `\`${page.relPath}\` contains ${markerLabels.join(", ")} markers.`);
+    }
+
+    if (
+      /\b(?:conflict|contradict|inconsistent)\b/i.test(page.body) ||
+      /\bmust\b[\s\S]{0,160}\bmust not\b/i.test(page.body) ||
+      /\bmust not\b[\s\S]{0,160}\bmust\b/i.test(page.body)
+    ) {
+      appendFinding(
+        findings,
+        "contradictions",
+        `\`${page.relPath}\` contains heuristic contradiction language and should be reviewed manually.`
+      );
+    }
+  }
+
+  return { findings, pageCount: pages.length };
+}
+
+function todayStamp() {
+  return new Date().toISOString().slice(0, 10);
+}
+
+function totalFindings(findings) {
+  return CATEGORY_DEFS.reduce((sum, [key]) => sum + findings[key].length, 0);
+}
+
+function renderReport({ projectPath, workspaceRoot, findings, pageCount }) {
+  const summaryRows = CATEGORY_DEFS.map(
+    ([key, label]) => `| ${label} | ${findings[key].length} |`
+  );
+  const detailSections = CATEGORY_DEFS.flatMap(([key, label]) => {
+    const items = findings[key];
+    return [
+      `## ${label}`,
+      "",
+      ...(items.length > 0 ? items.map((item) => `- ${item}`) : ["- None."]),
+      "",
+    ];
+  });
+
+  return [
+    "# SDTK-WIKI Lint Report",
+    "",
+    `Date: ${todayStamp()}`,
+    `Project root: \`${projectPath}\``,
+    `Workspace root: \`${workspaceRoot}\``,
+    `Pages scanned: ${pageCount}`,
+    `Total findings: ${totalFindings(findings)}`,
+    "",
+    "Report-only and non-destructive: lint writes this report and does not auto-modify wiki or source content.",
+    "Candidate contradictions are heuristic only and require manual review.",
+    "",
+    "## Summary",
+    "",
+    "| Category | Count |",
+    "|---|---:|",
+    ...summaryRows,
+    "",
+    ...detailSections,
+  ].join("\n");
+}
+
+function runWikiLint(options = {}) {
+  const projectPath = resolveProjectPath(options.projectPath);
+  if (!fs.existsSync(projectPath) || !fs.statSync(projectPath).isDirectory()) {
+    throw new ValidationError(`--project-path is not a valid directory: ${projectPath}`);
+  }
+
+  const workspaceRoot = getWikiWorkspacePath(projectPath);
+  if (!fs.existsSync(workspaceRoot) || !fs.statSync(workspaceRoot).isDirectory()) {
+    throw new ValidationError(
+      `No SDTK-WIKI workspace found at ${workspaceRoot}. Run "sdtk-wiki init" first.`
+    );
+  }
+
+  const analysis = analyzePages(projectPath);
+  const reportsRoot = getWikiReportsPath(projectPath);
+  try {
+    assertWikiWorkspaceWritePath(reportsRoot, projectPath);
+    fs.mkdirSync(reportsRoot, { recursive: true });
+    const reportPath = path.join(reportsRoot, `lint-report-${todayStamp()}.md`);
+    assertWikiWorkspaceWritePath(reportPath, projectPath);
+    const report = renderReport({
+      projectPath,
+      workspaceRoot,
+      findings: analysis.findings,
+      pageCount: analysis.pageCount,
+    });
+    fs.writeFileSync(reportPath, report + "\n", "utf-8");
+    return {
+      reportPath,
+      totalFindings: totalFindings(analysis.findings),
+      findings: analysis.findings,
+    };
+  } catch (error) {
+    if (error instanceof CliError) throw error;
+    throw new CliError(`Failed to write SDTK-WIKI lint report: ${error.message}`);
+  }
+}
+
+module.exports = {
+  CATEGORY_DEFS,
+  REQUIRED_PAGE_FIELDS,
+  analyzePages,
+  parseFrontmatter,
+  renderReport,
+  runWikiLint,
+};

package/src/lib/wiki-paths.js

@@ -0,0 +1,169 @@
+"use strict";
+
+const path = require("path");
+
+const WIKI_WORKSPACE_RELATIVE = path.join(".sdtk", "wiki");
+const WIKI_GRAPH_RELATIVE = path.join(".sdtk", "wiki", "graph");
+const WIKI_MANIFEST_RELATIVE = path.join(".sdtk", "wiki", "manifest.json");
+const WIKI_PAGES_RELATIVE = path.join(".sdtk", "wiki", "pages");
+const WIKI_RAW_RELATIVE = path.join(".sdtk", "wiki", "raw");
+const WIKI_RAW_DESCRIPTORS_RELATIVE = path.join(".sdtk", "wiki", "raw", "descriptors");
+const WIKI_RAW_SOURCES_RELATIVE = path.join(".sdtk", "wiki", "raw", "sources.json");
+const WIKI_PROVENANCE_RELATIVE = path.join(".sdtk", "wiki", "provenance");
+const WIKI_PROVENANCE_INGEST_EVENTS_RELATIVE = path.join(".sdtk", "wiki", "provenance", "ingest-events.json");
+const WIKI_PROVENANCE_SOURCES_RELATIVE = path.join(".sdtk", "wiki", "provenance", "sources.json");
+const WIKI_QUERIES_RELATIVE = path.join(".sdtk", "wiki", "queries");
+const WIKI_REPORTS_RELATIVE = path.join(".sdtk", "wiki", "reports");
+const WIKI_LOGS_RELATIVE = path.join(".sdtk", "wiki", "logs");
+const LEGACY_ATLAS_RELATIVE = path.join(".sdtk", "atlas");
+
+function resolveProjectPath(projectPath) {
+  return path.resolve(projectPath || process.cwd());
+}
+
+function normalizeComparablePath(targetPath) {
+  const resolved = path.resolve(targetPath);
+  return process.platform === "win32" ? resolved.toLowerCase() : resolved;
+}
+
+function isPathInsideOrEqual(targetPath, rootPath) {
+  const comparableTarget = normalizeComparablePath(targetPath);
+  const comparableRoot = normalizeComparablePath(rootPath);
+  return (
+    comparableTarget === comparableRoot ||
+    comparableTarget.startsWith(comparableRoot + path.sep)
+  );
+}
+
+function assertPathInsideOrEqual(
+  targetPath,
+  rootPath,
+  message = "Refusing to access a path outside the allowed root"
+) {
+  const resolvedTarget = path.resolve(targetPath);
+  if (!isPathInsideOrEqual(resolvedTarget, rootPath)) {
+    throw new Error(`${message}: ${resolvedTarget}`);
+  }
+  return resolvedTarget;
+}
+
+function getWikiWorkspacePath(projectPath) {
+  return path.join(resolveProjectPath(projectPath), WIKI_WORKSPACE_RELATIVE);
+}
+
+function getWikiGraphPath(projectPath) {
+  return path.join(resolveProjectPath(projectPath), WIKI_GRAPH_RELATIVE);
+}
+
+function getWikiManifestPath(projectPath) {
+  return path.join(resolveProjectPath(projectPath), WIKI_MANIFEST_RELATIVE);
+}
+
+function getWikiPagesPath(projectPath) {
+  return path.join(resolveProjectPath(projectPath), WIKI_PAGES_RELATIVE);
+}
+
+function getWikiRawPath(projectPath) {
+  return path.join(resolveProjectPath(projectPath), WIKI_RAW_RELATIVE);
+}
+
+function getWikiRawDescriptorsPath(projectPath) {
+  return path.join(resolveProjectPath(projectPath), WIKI_RAW_DESCRIPTORS_RELATIVE);
+}
+
+function getWikiRawSourcesPath(projectPath) {
+  return path.join(resolveProjectPath(projectPath), WIKI_RAW_SOURCES_RELATIVE);
+}
+
+function getWikiProvenancePath(projectPath) {
+  return path.join(resolveProjectPath(projectPath), WIKI_PROVENANCE_RELATIVE);
+}
+
+function getWikiProvenanceIngestEventsPath(projectPath) {
+  return path.join(resolveProjectPath(projectPath), WIKI_PROVENANCE_INGEST_EVENTS_RELATIVE);
+}
+
+function getWikiProvenanceSourcesPath(projectPath) {
+  return path.join(resolveProjectPath(projectPath), WIKI_PROVENANCE_SOURCES_RELATIVE);
+}
+
+function getWikiQueriesPath(projectPath) {
+  return path.join(resolveProjectPath(projectPath), WIKI_QUERIES_RELATIVE);
+}
+
+function getWikiReportsPath(projectPath) {
+  return path.join(resolveProjectPath(projectPath), WIKI_REPORTS_RELATIVE);
+}
+
+function getWikiLogsPath(projectPath) {
+  return path.join(resolveProjectPath(projectPath), WIKI_LOGS_RELATIVE);
+}
+
+function getLegacyAtlasPath(projectPath) {
+  return path.join(resolveProjectPath(projectPath), LEGACY_ATLAS_RELATIVE);
+}
+
+function assertWikiWorkspaceWritePath(targetPath, projectPath) {
+  return assertPathInsideOrEqual(
+    targetPath,
+    getWikiWorkspacePath(projectPath),
+    "Refusing to write outside project-local .sdtk/wiki workspace"
+  );
+}
+
+function describeWikiPaths(projectPath) {
+  return {
+    projectPath: resolveProjectPath(projectPath),
+    wikiWorkspacePath: getWikiWorkspacePath(projectPath),
+    wikiGraphPath: getWikiGraphPath(projectPath),
+    wikiManifestPath: getWikiManifestPath(projectPath),
+    wikiPagesPath: getWikiPagesPath(projectPath),
+    wikiRawPath: getWikiRawPath(projectPath),
+    wikiRawDescriptorsPath: getWikiRawDescriptorsPath(projectPath),
+    wikiRawSourcesPath: getWikiRawSourcesPath(projectPath),
+    wikiProvenancePath: getWikiProvenancePath(projectPath),
+    wikiProvenanceIngestEventsPath: getWikiProvenanceIngestEventsPath(projectPath),
+    wikiProvenanceSourcesPath: getWikiProvenanceSourcesPath(projectPath),
+    wikiQueriesPath: getWikiQueriesPath(projectPath),
+    wikiReportsPath: getWikiReportsPath(projectPath),
+    wikiLogsPath: getWikiLogsPath(projectPath),
+    legacyAtlasPath: getLegacyAtlasPath(projectPath),
+  };
+}
+
+module.exports = {
+  LEGACY_ATLAS_RELATIVE,
+  WIKI_GRAPH_RELATIVE,
+  WIKI_LOGS_RELATIVE,
+  WIKI_MANIFEST_RELATIVE,
+  WIKI_PAGES_RELATIVE,
+  WIKI_RAW_DESCRIPTORS_RELATIVE,
+  WIKI_RAW_RELATIVE,
+  WIKI_RAW_SOURCES_RELATIVE,
+  WIKI_PROVENANCE_INGEST_EVENTS_RELATIVE,
+  WIKI_PROVENANCE_RELATIVE,
+  WIKI_PROVENANCE_SOURCES_RELATIVE,
+  WIKI_QUERIES_RELATIVE,
+  WIKI_REPORTS_RELATIVE,
+  WIKI_WORKSPACE_RELATIVE,
+  assertPathInsideOrEqual,
+  assertWikiWorkspaceWritePath,
+  describeWikiPaths,
+  getLegacyAtlasPath,
+  getWikiGraphPath,
+  getWikiLogsPath,
+  getWikiManifestPath,
+  getWikiPagesPath,
+  getWikiProvenanceIngestEventsPath,
+  getWikiProvenancePath,
+  getWikiProvenanceSourcesPath,
+  getWikiQueriesPath,
+  getWikiRawDescriptorsPath,
+  getWikiRawPath,
+  getWikiRawSourcesPath,
+  getWikiReportsPath,
+  getWikiWorkspacePath,
+  isPathInsideOrEqual,
+  normalizeComparablePath,
+  resolveProjectPath,
+};