sdtk-wiki-kit 0.1.0

package/src/lib/wiki-discover.js

@@ -0,0 +1,271 @@
+"use strict";
+
+const fs = require("fs");
+const path = require("path");
+const { CliError, ValidationError } = require("./errors");
+const { analyzePages } = require("./wiki-lint");
+const {
+  assertWikiWorkspaceWritePath,
+  getWikiProvenanceSourcesPath,
+  getWikiRawSourcesPath,
+  getWikiReportsPath,
+  getWikiWorkspacePath,
+  resolveProjectPath,
+} = require("./wiki-paths");
+
+const REPORT_PREFIX = "discover-plan";
+
+const CATEGORY_LABELS = {
+  missing_source_coverage: "Missing source coverage",
+  broken_internal_link_target: "Broken internal link target",
+  todo_open_questions_gaps: "TODO / Open Questions / Gaps",
+  weak_integration: "Weak integration",
+  stale_candidate_human_decision: "Stale candidate requiring human decision",
+  contradiction_candidate_source_verification: "Contradiction candidate requiring source verification",
+  raw_source_not_represented: "Raw source not represented in managed pages",
+};
+
+function todayStamp(date = new Date()) {
+  return date.toISOString().slice(0, 10);
+}
+
+function readJsonIfPresent(filePath, fallback = null) {
+  if (!fs.existsSync(filePath)) return fallback;
+  try {
+    return JSON.parse(fs.readFileSync(filePath, "utf-8"));
+  } catch (error) {
+    return { __error: `Could not parse JSON at ${filePath}: ${error.message}` };
+  }
+}
+
+function asArray(value) {
+  return Array.isArray(value) ? value : [];
+}
+
+function toPosix(value) {
+  return String(value || "").replace(/\\/g, "/");
+}
+
+function priorityForCategory(category) {
+  if (
+    category === "missing_source_coverage" ||
+    category === "stale_candidate_human_decision" ||
+    category === "contradiction_candidate_source_verification"
+  ) {
+    return "high";
+  }
+  if (category === "broken_internal_link_target" || category === "raw_source_not_represented") {
+    return "medium";
+  }
+  return "normal";
+}
+
+function sourceTypeForCategory(category) {
+  if (category === "broken_internal_link_target") {
+    return "missing internal page/link";
+  }
+  if (category === "raw_source_not_represented") {
+    return "local document to inspect";
+  }
+  if (category === "missing_source_coverage") {
+    return "local source coverage review";
+  }
+  return "stakeholder/manual review need";
+}
+
+function nextActionForCategory(category) {
+  switch (category) {
+    case "missing_source_coverage":
+      return "Review whether the referenced local source should be restored, re-ingested, or intentionally retired.";
+    case "broken_internal_link_target":
+      return "Decide whether to create the missing managed page or update the local link target.";
+    case "todo_open_questions_gaps":
+      return "Resolve the marker through local source review before compile/apply work.";
+    case "weak_integration":
+      return "Add local context, source references, or relationships in a later compile/apply workflow.";
+    case "stale_candidate_human_decision":
+      return "Review stale evidence before any future prune/archive issue is considered.";
+    case "contradiction_candidate_source_verification":
+      return "Verify the conflicting statement against local authoritative sources.";
+    case "raw_source_not_represented":
+      return "Consider whether this raw source should feed a later compile/additive update plan.";
+    default:
+      return "Review locally.";
+  }
+}
+
+function addPlanItems(items, category, evidenceItems, evidenceSource) {
+  for (const evidence of evidenceItems) {
+    items.push({
+      category,
+      evidence,
+      evidenceSource,
+      suggestedSourceType: sourceTypeForCategory(category),
+      priority: priorityForCategory(category),
+      confidence: "heuristic",
+      safetyMode: "local-only; manual-review",
+      nextAction: nextActionForCategory(category),
+    });
+  }
+}
+
+function collectRawSourceGaps(projectPath) {
+  const rawPath = getWikiRawSourcesPath(projectPath);
+  const provenancePath = getWikiProvenanceSourcesPath(projectPath);
+  const raw = readJsonIfPresent(rawPath, { sources: [] });
+  const provenance = readJsonIfPresent(provenancePath, { sources: [] });
+  const represented = new Set(
+    asArray(provenance.sources)
+      .filter((record) => record && record.sourcePath)
+      .map((record) => toPosix(record.sourcePath))
+  );
+
+  return asArray(raw.sources)
+    .filter((record) => record && record.sourcePath)
+    .filter((record) => !represented.has(toPosix(record.sourcePath)))
+    .map((record) => `Raw source \`${toPosix(record.sourcePath)}\` is registered but not represented in active provenance.`);
+}
+
+function buildPlanItems(projectPath) {
+  const analysis = analyzePages(projectPath);
+  const findings = analysis.findings;
+  const items = [];
+
+  addPlanItems(items, "missing_source_coverage", findings.provenance || [], "lint.provenance");
+  addPlanItems(items, "broken_internal_link_target", findings.brokenLinks || [], "lint.brokenLinks");
+  addPlanItems(items, "todo_open_questions_gaps", findings.markers || [], "lint.markers");
+  addPlanItems(items, "weak_integration", [...(findings.orphans || []), ...(findings.downstream || []), ...(findings.thin || [])], "lint.integration");
+  addPlanItems(items, "stale_candidate_human_decision", findings.stale || [], "lint.stale");
+  addPlanItems(items, "contradiction_candidate_source_verification", findings.contradictions || [], "lint.contradictions");
+  addPlanItems(items, "raw_source_not_represented", collectRawSourceGaps(projectPath), "raw.sources");
+
+  return {
+    pageCount: analysis.pageCount,
+    items: items.map((item, index) => ({
+      ...item,
+      planItemId: `discover-${String(index + 1).padStart(3, "0")}`,
+    })),
+  };
+}
+
+function summarizeByCategory(items) {
+  const summary = Object.fromEntries(Object.keys(CATEGORY_LABELS).map((key) => [key, 0]));
+  for (const item of items) {
+    summary[item.category] = (summary[item.category] || 0) + 1;
+  }
+  return summary;
+}
+
+function renderReport({ projectPath, workspacePath, generatedAt, pageCount, items }) {
+  const summary = summarizeByCategory(items);
+  const lines = [
+    "# SDTK-WIKI Discovery Plan",
+    "",
+    `Generated: ${generatedAt}`,
+    `Project path: ${projectPath}`,
+    `Workspace path: ${workspacePath}`,
+    `Pages scanned: ${pageCount}`,
+    `Plan items: ${items.length}`,
+    "",
+    "Plan-only and local-only: No source was fetched, ingested, compiled, applied, pruned, deleted, archived, or persisted as query history.",
+    "No web or network discovery is executed. Future web research requires a separate controller-approved opt-in issue.",
+    "",
+    "## Input Evidence References",
+    "",
+    "- Direct lint/gap analysis of `.sdtk/wiki/pages`",
+    "- `.sdtk/wiki/provenance/sources.json` when present",
+    "- `.sdtk/wiki/raw/sources.json` when present",
+    "- BK-119 gap taxonomy",
+    "",
+    "## Gap Category Summary",
+    "",
+    "| Gap category | Count |",
+    "|---|---:|",
+    ...Object.entries(CATEGORY_LABELS).map(([key, label]) => `| \`${key}\` - ${label} | ${summary[key] || 0} |`),
+    "",
+    "## Plan Items",
+    "",
+  ];
+
+  if (items.length === 0) {
+    lines.push("No discovery plan items were generated from current local evidence.");
+  } else {
+    for (const item of items) {
+      lines.push(`### ${item.planItemId}`);
+      lines.push("");
+      lines.push(`- Gap category: \`${item.category}\``);
+      lines.push(`- Evidence source: \`${item.evidenceSource}\``);
+      lines.push(`- Evidence: ${item.evidence}`);
+      lines.push(`- Suggested source type: ${item.suggestedSourceType}`);
+      lines.push(`- Priority: ${item.priority}`);
+      lines.push(`- Confidence: ${item.confidence}`);
+      lines.push(`- Safety mode: ${item.safetyMode}`);
+      lines.push(`- Next action: ${item.nextAction}`);
+      lines.push("");
+    }
+  }
+
+  lines.push("");
+  lines.push("## Deferred Actions");
+  lines.push("");
+  lines.push("Fetch, ingest, compile, apply, prune, delete, archive, Ask changes, query persistence, and release work are outside BK-125.");
+  lines.push("");
+  return `${lines.join("\n").trimEnd()}\n`;
+}
+
+function ensureExistingWorkspace(projectPath) {
+  const workspacePath = getWikiWorkspacePath(projectPath);
+  if (!fs.existsSync(workspacePath) || !fs.statSync(workspacePath).isDirectory()) {
+    throw new ValidationError(
+      `No SDTK-WIKI workspace found at ${workspacePath}. Run "sdtk-wiki init" or "sdtk-wiki atlas build" first. No project files were changed.`
+    );
+  }
+  return workspacePath;
+}
+
+function runWikiDiscoverPlan({ projectPath }) {
+  const resolvedProjectPath = resolveProjectPath(projectPath || process.cwd());
+  if (!fs.existsSync(resolvedProjectPath) || !fs.statSync(resolvedProjectPath).isDirectory()) {
+    throw new ValidationError(`--project-path is not a valid directory: ${resolvedProjectPath}`);
+  }
+
+  try {
+    const workspacePath = ensureExistingWorkspace(resolvedProjectPath);
+    const reportsPath = getWikiReportsPath(resolvedProjectPath);
+    assertWikiWorkspaceWritePath(reportsPath, resolvedProjectPath);
+
+    const generatedAt = new Date().toISOString();
+    const result = buildPlanItems(resolvedProjectPath);
+    const reportPath = path.join(reportsPath, `${REPORT_PREFIX}-${todayStamp(new Date(generatedAt))}.md`);
+    assertWikiWorkspaceWritePath(reportPath, resolvedProjectPath);
+    fs.mkdirSync(reportsPath, { recursive: true });
+    fs.writeFileSync(
+      reportPath,
+      renderReport({
+        projectPath: resolvedProjectPath,
+        workspacePath,
+        generatedAt,
+        pageCount: result.pageCount,
+        items: result.items,
+      }),
+      "utf-8"
+    );
+
+    return {
+      reportPath,
+      pageCount: result.pageCount,
+      items: result.items,
+      summary: summarizeByCategory(result.items),
+    };
+  } catch (error) {
+    if (error instanceof CliError) throw error;
+    throw new CliError(`Failed to write SDTK-WIKI discovery plan report: ${error.message}`);
+  }
+}
+
+module.exports = {
+  REPORT_PREFIX,
+  buildPlanItems,
+  renderReport,
+  runWikiDiscoverPlan,
+};

package/src/lib/wiki-flags.js

@@ -0,0 +1,89 @@
+"use strict";
+
+const { parseFlags } = require("./args");
+
+const BASE_FLAG_DEFS = {
+  "project-path": { type: "string" },
+  "output-dir": { type: "string" },
+  "scan-root": { type: "string" },
+  verbose: { type: "boolean" },
+};
+
+const OPEN_FLAG_DEFS = {
+  ...BASE_FLAG_DEFS,
+  host: { type: "string" },
+  port: { type: "string" },
+  "no-open": { type: "boolean" },
+};
+
+const INIT_FLAG_DEFS = {
+  ...OPEN_FLAG_DEFS,
+  force: { type: "boolean" },
+  "no-build": { type: "boolean" },
+};
+
+const WATCH_FLAG_DEFS = {
+  ...BASE_FLAG_DEFS,
+  port: { type: "string" },
+  "no-open": { type: "boolean" },
+};
+
+const LINT_FLAG_DEFS = {
+  "project-path": { type: "string" },
+};
+
+const PRUNE_FLAG_DEFS = {
+  "project-path": { type: "string" },
+  "dry-run": { type: "boolean" },
+};
+
+const DISCOVER_FLAG_DEFS = {
+  "project-path": { type: "string" },
+  plan: { type: "boolean" },
+};
+
+const COMPILE_FLAG_DEFS = {
+  "project-path": { type: "string" },
+  plan: { type: "string" },
+  "dry-run": { type: "boolean" },
+  apply: { type: "boolean" },
+};
+
+function parseWikiFlags(args, defs) {
+  const scanRoots = [];
+  const filteredArgs = [];
+  let i = 0;
+
+  while (i < args.length) {
+    const arg = args[i];
+    if (arg === "--scan-root") {
+      i++;
+      if (i < args.length) {
+        scanRoots.push(args[i]);
+        i++;
+      }
+    } else if (arg.startsWith("--scan-root=")) {
+      scanRoots.push(arg.slice("--scan-root=".length));
+      i++;
+    } else {
+      filteredArgs.push(arg);
+      i++;
+    }
+  }
+
+  const { flags, positional } = parseFlags(filteredArgs, defs);
+  flags["scan-root"] = scanRoots.length > 0 ? scanRoots : undefined;
+  return { flags, positional };
+}
+
+module.exports = {
+  BASE_FLAG_DEFS,
+  COMPILE_FLAG_DEFS,
+  DISCOVER_FLAG_DEFS,
+  INIT_FLAG_DEFS,
+  LINT_FLAG_DEFS,
+  OPEN_FLAG_DEFS,
+  PRUNE_FLAG_DEFS,
+  WATCH_FLAG_DEFS,
+  parseWikiFlags,
+};

package/src/lib/wiki-ingest.js

@@ -0,0 +1,198 @@
+"use strict";
+
+const crypto = require("crypto");
+const fs = require("fs");
+const path = require("path");
+const { ValidationError } = require("./errors");
+const {
+  assertWikiWorkspaceWritePath,
+  getWikiProvenanceIngestEventsPath,
+  getWikiProvenancePath,
+  getWikiRawDescriptorsPath,
+  getWikiRawPath,
+  getWikiRawSourcesPath,
+  isPathInsideOrEqual,
+  resolveProjectPath,
+} = require("./wiki-paths");
+
+const INGEST_SCHEMA_VERSION = 1;
+
+function nowIso() {
+  return new Date().toISOString();
+}
+
+function readJsonIfPresent(filePath, fallback) {
+  if (!fs.existsSync(filePath)) return fallback;
+  try {
+    return JSON.parse(fs.readFileSync(filePath, "utf-8"));
+  } catch (_) {
+    return fallback;
+  }
+}
+
+function writeJson(filePath, payload, projectPath) {
+  assertWikiWorkspaceWritePath(filePath, projectPath);
+  fs.mkdirSync(path.dirname(filePath), { recursive: true });
+  fs.writeFileSync(filePath, JSON.stringify(payload, null, 2) + "\n", "utf-8");
+}
+
+function rejectRemoteSource(sourceArg) {
+  if (/^[a-z][a-z0-9+.-]*:\/\//i.test(sourceArg)) {
+    throw new ValidationError("Local project sources only. Remote URLs are not supported in BK-113.");
+  }
+}
+
+function toProjectRelative(projectPath, sourcePath) {
+  return path.relative(projectPath, sourcePath).split(path.sep).join("/");
+}
+
+function detectSourceType(sourcePath) {
+  const ext = path.extname(sourcePath).toLowerCase();
+  if (ext === ".md" || ext === ".markdown" || ext === ".mdx") return "markdown";
+  if (ext === ".txt") return "text";
+  return ext ? ext.slice(1) : "file";
+}
+
+function fingerprintSource(sourcePath) {
+  const bytes = fs.readFileSync(sourcePath);
+  return crypto.createHash("sha256").update(bytes).digest("hex");
+}
+
+function stableSourceId(relativePath) {
+  return `raw:${crypto.createHash("sha256").update(relativePath).digest("hex").slice(0, 24)}`;
+}
+
+function ensureApprovedIngestDirs(projectPath) {
+  const rawRoot = getWikiRawPath(projectPath);
+  const rawDescriptorsRoot = getWikiRawDescriptorsPath(projectPath);
+  const provenanceRoot = getWikiProvenancePath(projectPath);
+  for (const target of [rawRoot, rawDescriptorsRoot, provenanceRoot]) {
+    assertWikiWorkspaceWritePath(target, projectPath);
+    fs.mkdirSync(target, { recursive: true });
+  }
+}
+
+function resolveLocalSource(projectPath, sourceArg) {
+  if (!sourceArg || !String(sourceArg).trim()) {
+    throw new ValidationError("Missing required --source <path>.");
+  }
+
+  rejectRemoteSource(String(sourceArg));
+  const projectRoot = resolveProjectPath(projectPath);
+  const candidate = path.resolve(projectRoot, sourceArg);
+
+  if (!isPathInsideOrEqual(candidate, projectRoot)) {
+    throw new ValidationError(`Source must stay inside the project root: ${candidate}`);
+  }
+  if (!fs.existsSync(candidate)) {
+    throw new ValidationError(`Source does not exist: ${candidate}`);
+  }
+  if (!fs.statSync(candidate).isFile()) {
+    throw new ValidationError(`Source must be a file in BK-113: ${candidate}`);
+  }
+  return candidate;
+}
+
+function upsertRegistry(registry, record) {
+  const existing = Array.isArray(registry.sources) ? registry.sources : [];
+  const next = existing.filter((item) => item.sourceId !== record.sourceId);
+  next.push(record);
+  next.sort((a, b) => a.sourcePath.localeCompare(b.sourcePath));
+  return {
+    schemaVersion: INGEST_SCHEMA_VERSION,
+    product: "SDTK-WIKI",
+    sources: next,
+  };
+}
+
+function appendIngestEvent(eventsPayload, event) {
+  const events = Array.isArray(eventsPayload.events) ? eventsPayload.events.slice() : [];
+  events.push(event);
+  return {
+    schemaVersion: INGEST_SCHEMA_VERSION,
+    product: "SDTK-WIKI",
+    events,
+  };
+}
+
+function ingestLocalSource({ projectPath, sourceArg }) {
+  const projectRoot = resolveProjectPath(projectPath);
+  const sourcePath = resolveLocalSource(projectRoot, sourceArg);
+  const sourceStat = fs.statSync(sourcePath);
+  const relativeSourcePath = toProjectRelative(projectRoot, sourcePath);
+  const sourceId = stableSourceId(relativeSourcePath);
+  const sourceHash = fingerprintSource(sourcePath);
+  const ingestedAt = nowIso();
+  const descriptorPath = path.join(getWikiRawDescriptorsPath(projectRoot), `${sourceId.replace(/[:/]/g, "-")}.json`);
+  const registryPath = getWikiRawSourcesPath(projectRoot);
+  const eventsPath = getWikiProvenanceIngestEventsPath(projectRoot);
+  const descriptorRelative = toProjectRelative(projectRoot, descriptorPath);
+  const provenanceRelative = toProjectRelative(projectRoot, eventsPath);
+
+  ensureApprovedIngestDirs(projectRoot);
+
+  const descriptor = {
+    schemaVersion: INGEST_SCHEMA_VERSION,
+    product: "SDTK-WIKI",
+    sourceId,
+    sourcePath: relativeSourcePath,
+    sourceType: detectSourceType(sourcePath),
+    sourceHash,
+    sizeBytes: sourceStat.size,
+    mtimeMs: Math.trunc(sourceStat.mtimeMs),
+    ingestedAt,
+    revision: sourceHash.slice(0, 16),
+    status: "ingested",
+    provenancePath: provenanceRelative,
+  };
+  writeJson(descriptorPath, descriptor, projectRoot);
+
+  const record = {
+    sourceId,
+    sourcePath: relativeSourcePath,
+    sourceType: descriptor.sourceType,
+    sourceHash,
+    sizeBytes: descriptor.sizeBytes,
+    mtimeMs: descriptor.mtimeMs,
+    ingestedAt,
+    revision: descriptor.revision,
+    status: descriptor.status,
+    descriptorPath: descriptorRelative,
+    provenancePath: provenanceRelative,
+  };
+
+  const registry = upsertRegistry(
+    readJsonIfPresent(registryPath, { schemaVersion: INGEST_SCHEMA_VERSION, product: "SDTK-WIKI", sources: [] }),
+    record
+  );
+  writeJson(registryPath, registry, projectRoot);
+
+  const events = appendIngestEvent(
+    readJsonIfPresent(eventsPath, { schemaVersion: INGEST_SCHEMA_VERSION, product: "SDTK-WIKI", events: [] }),
+    {
+      event: "ingest.recorded",
+      sourceId,
+      sourcePath: relativeSourcePath,
+      sourceHash,
+      descriptorPath: descriptorRelative,
+      recordedAt: ingestedAt,
+      status: "ingested",
+    }
+  );
+  writeJson(eventsPath, events, projectRoot);
+
+  return {
+    sourceId,
+    sourcePath: relativeSourcePath,
+    registryPath,
+    descriptorPath,
+    eventsPath,
+  };
+}
+
+module.exports = {
+  INGEST_SCHEMA_VERSION,
+  ingestLocalSource,
+  resolveLocalSource,
+  stableSourceId,
+};