sdtk-wiki-kit 0.1.0

package/src/lib/wiki-query-history.js

@@ -0,0 +1,111 @@
+"use strict";
+
+const crypto = require("crypto");
+const fs = require("fs");
+const path = require("path");
+const {
+  assertWikiWorkspaceWritePath,
+  getWikiQueriesPath,
+  resolveProjectPath,
+} = require("./wiki-paths");
+
+function toPosix(value) {
+  return String(value || "").replace(/\\/g, "/");
+}
+
+function shortHash(value, length = 12) {
+  return crypto.createHash("sha256").update(String(value || "")).digest("hex").slice(0, length);
+}
+
+function timestampParts(date = new Date()) {
+  const iso = date.toISOString();
+  const stamp = iso.slice(0, 19).replace(/[-:]/g, "").replace("T", "-");
+  return {
+    iso,
+    stamp,
+  };
+}
+
+function projectFingerprint(projectPath) {
+  return shortHash(path.resolve(projectPath), 16);
+}
+
+function summarizeAnswer(answer, citations) {
+  const answerLength = String(answer || "").length;
+  const citationCount = Array.isArray(citations) ? citations.length : 0;
+  return `Redacted answer summary; answer length ${answerLength} characters with ${citationCount} source reference(s).`;
+}
+
+function sourceRefs(citations) {
+  if (!Array.isArray(citations)) return [];
+  return citations
+    .map((citation) => {
+      if (typeof citation === "string") return citation;
+      if (citation && typeof citation === "object") {
+        return citation.path || citation.sourcePath || citation.id || citation.title || "";
+      }
+      return "";
+    })
+    .filter(Boolean)
+    .map(toPosix);
+}
+
+function relativeGraphPath(result, projectPath) {
+  const graphPath = result && result.graphPath ? String(result.graphPath) : "";
+  if (!graphPath) return ".sdtk/wiki/graph";
+  const relative = path.relative(resolveProjectPath(projectPath), graphPath);
+  if (!relative || relative.startsWith("..") || path.isAbsolute(relative)) {
+    return ".sdtk/wiki/graph";
+  }
+  return toPosix(relative);
+}
+
+function buildQueryRecord({ projectPath, question, result, createdAt = new Date() }) {
+  const time = timestampParts(createdAt);
+  const refs = sourceRefs(result.citations);
+  const queryId = `query-${shortHash(`${time.iso}:${question}:${result.graphPath || ""}`)}`;
+  return {
+    schema_version: 1,
+    product: "SDTK-WIKI",
+    record_type: "sdtk_wiki_query",
+    query_id: queryId,
+    created_at: time.iso,
+    command_surface: "sdtk-wiki ask",
+    project_path_fingerprint: projectFingerprint(projectPath),
+    question_redacted: "[redacted]",
+    answer_summary: summarizeAnswer(result.answer, refs),
+    source_refs: refs,
+    graph_path: relativeGraphPath(result, projectPath),
+    runtime: {
+      capability: "wiki.ask",
+      confidence: result.confidence || null,
+    },
+    model: null,
+    privacy_mode: "redacted",
+    retention_policy: "Project-local redacted query record. Full question and full answer are not stored by default.",
+    status: "answered",
+    review_status: "unreviewed",
+  };
+}
+
+function saveQueryHistoryRecord({ projectPath, question, result }) {
+  const resolvedProjectPath = resolveProjectPath(projectPath);
+  const queriesPath = getWikiQueriesPath(resolvedProjectPath);
+  assertWikiWorkspaceWritePath(queriesPath, resolvedProjectPath);
+  const record = buildQueryRecord({ projectPath: resolvedProjectPath, question, result });
+  const time = timestampParts(new Date(record.created_at));
+  const fileName = `${time.stamp}-${shortHash(record.query_id)}.json`;
+  const recordPath = path.join(queriesPath, fileName);
+  assertWikiWorkspaceWritePath(recordPath, resolvedProjectPath);
+  fs.mkdirSync(queriesPath, { recursive: true });
+  fs.writeFileSync(recordPath, `${JSON.stringify(record, null, 2)}\n`, "utf-8");
+  return {
+    record,
+    recordPath,
+  };
+}
+
+module.exports = {
+  buildQueryRecord,
+  saveQueryHistoryRecord,
+};

package/src/lib/wiki-runner.js

@@ -0,0 +1,373 @@
+"use strict";
+
+const fs = require("fs");
+const http = require("http");
+const https = require("https");
+const net = require("net");
+const path = require("path");
+const { spawn, execFile } = require("child_process");
+const { CliError, DependencyError } = require("./errors");
+const { resolveBuilderPath } = require("./wiki-config");
+const { openBrowser } = require("./browser-open");
+
+const HEALTH_CHECK_RETRIES = 20;
+const HEALTH_CHECK_INTERVAL_MS = 300;
+
+function findPython() {
+  return new Promise((resolve, reject) => {
+    const candidates = process.platform === "win32"
+      ? ["python", "python3"]
+      : ["python3", "python"];
+
+    let idx = 0;
+    function tryNext() {
+      if (idx >= candidates.length) {
+        reject(
+          new DependencyError(
+            "Python not found. Install Python 3.8+ and ensure it is in PATH.\n" +
+              "SDTK-WIKI graph build requires Python to run the local builder."
+          )
+        );
+        return;
+      }
+
+      const candidate = candidates[idx++];
+      execFile(candidate, ["--version"], { timeout: 5000 }, (err) => {
+        if (!err) resolve(candidate);
+        else tryNext();
+      });
+    }
+    tryNext();
+  });
+}
+
+async function runBuild(config) {
+  const builderPath = resolveBuilderPath();
+  if (!fs.existsSync(builderPath)) {
+    throw new DependencyError(
+      `SDTK-WIKI builder not found: ${builderPath}\n` +
+        "This is a packaging error. Reinstall sdtk-wiki-kit."
+    );
+  }
+
+  const pythonExe = await findPython();
+  const args = [
+    builderPath,
+    "--project-root",
+    config.projectPath,
+    "--output-dir",
+    config.outputDir,
+  ];
+
+  for (const sr of config.scanRoots) {
+    args.push("--scan-root", sr);
+  }
+  for (const ex of config.excludes) {
+    args.push("--exclude", ex);
+  }
+  if (config.verbose) {
+    args.push("--verbose");
+  }
+
+  return new Promise((resolve, reject) => {
+    const child = spawn(pythonExe, args, {
+      stdio: ["ignore", "pipe", "pipe"],
+    });
+
+    let stdout = "";
+    let stderr = "";
+
+    child.stdout.on("data", (chunk) => {
+      const text = chunk.toString();
+      stdout += text;
+      process.stdout.write(text);
+    });
+
+    child.stderr.on("data", (chunk) => {
+      const text = chunk.toString();
+      stderr += text;
+      process.stderr.write(text);
+    });
+
+    child.on("error", (err) => {
+      if (err.code === "ENOENT") {
+        reject(
+          new DependencyError(
+            `Python executable not found: ${pythonExe}\n` +
+              "Install Python 3.8+ and ensure it is in PATH."
+          )
+        );
+      } else {
+        reject(new CliError(`Failed to start SDTK-WIKI builder: ${err.message}`));
+      }
+    });
+
+    child.on("close", (code) => {
+      if (code !== 0) {
+        reject(
+          new CliError(
+            `SDTK-WIKI build failed (exit code ${code}).\n` +
+              (stderr ? `Builder error output:\n${stderr}` : "See output above for details.")
+          )
+        );
+        return;
+      }
+
+      const resultLine = stdout.split("\n").find((l) => l.startsWith("[atlas:result] "));
+      if (resultLine) {
+        try {
+          const result = JSON.parse(resultLine.replace("[atlas:result] ", ""));
+          resolve({
+            docCount: result.doc_count || 0,
+            nodeCount: result.node_count || 0,
+            edgeCount: result.edge_count || 0,
+            generated: result.generated || "",
+            pageCount: result.page_count || 0,
+            pagesRoot: result.pages_root || "",
+            pageIndexPath: result.page_index_path || "",
+            provenancePath: result.provenance_path || "",
+            changesPath: result.changes_path || "",
+          });
+          return;
+        } catch (_) {
+          // fall through
+        }
+      }
+
+      resolve({
+        docCount: 0,
+        nodeCount: 0,
+        edgeCount: 0,
+        generated: "",
+        pageCount: 0,
+        pagesRoot: "",
+        pageIndexPath: "",
+        provenancePath: "",
+        changesPath: "",
+      });
+    });
+  });
+}
+
+function isPortOpen(host, port) {
+  return new Promise((resolve) => {
+    const socket = net.createConnection({ host, port, timeout: 500 });
+    socket.once("connect", () => {
+      socket.destroy();
+      resolve(true);
+    });
+    socket.once("error", () => resolve(false));
+    socket.once("timeout", () => {
+      socket.destroy();
+      resolve(false);
+    });
+  });
+}
+
+async function waitForServer(host, port) {
+  for (let i = 0; i < HEALTH_CHECK_RETRIES; i++) {
+    const ok = await isPortOpen(host, port);
+    if (ok) return;
+    await new Promise((r) => setTimeout(r, HEALTH_CHECK_INTERVAL_MS));
+  }
+  throw new CliError(
+    `SDTK-WIKI viewer server did not start on http://${host}:${port}\n` +
+      "Try passing a different --port if the port is occupied."
+  );
+}
+
+function probeUrl(url) {
+  return new Promise((resolve) => {
+    let target;
+    try {
+      target = new URL(url);
+    } catch (_) {
+      resolve({ ok: false, statusCode: 0, body: "" });
+      return;
+    }
+
+    const transport = target.protocol === "https:" ? https : http;
+    const req = transport.request(
+      target,
+      { method: "GET", timeout: 1200 },
+      (res) => {
+        let body = "";
+        res.setEncoding("utf8");
+        res.on("data", (chunk) => {
+          if (body.length < 512) {
+            body += chunk.slice(0, 512 - body.length);
+          }
+        });
+        res.on("end", () => {
+          const statusCode = res.statusCode || 0;
+          resolve({
+            ok: statusCode >= 200 && statusCode < 300,
+            statusCode,
+            body,
+          });
+        });
+      }
+    );
+
+    req.on("timeout", () => {
+      req.destroy();
+      resolve({ ok: false, statusCode: 0, body: "" });
+    });
+    req.on("error", () => resolve({ ok: false, statusCode: 0, body: "" }));
+    req.end();
+  });
+}
+
+async function probeExistingWikiServer(host, port, viewerUrl) {
+  const health = await probeUrl(`http://${host}:${port}/api/health`);
+  const viewer = await probeUrl(`${viewerUrl}?embedded=1&probe=1`);
+  return {
+    reusable: health.ok && viewer.ok,
+    health: { ok: health.ok, statusCode: health.statusCode },
+    viewer: { ok: viewer.ok, statusCode: viewer.statusCode },
+  };
+}
+
+function startWikiServer(host, port, outputDir, projectPath) {
+  return new Promise((resolve, reject) => {
+    const MIME_TYPES = {
+      ".html": "text/html; charset=utf-8",
+      ".js": "application/javascript; charset=utf-8",
+      ".json": "application/json; charset=utf-8",
+      ".css": "text/css; charset=utf-8",
+      ".md": "text/plain; charset=utf-8",
+      ".txt": "text/plain; charset=utf-8",
+    };
+
+    const server = http.createServer((req, res) => {
+      const url = req.url || "/";
+
+      if (url === "/api/health" || url === "/api/health/") {
+        res.writeHead(200, { "Content-Type": "application/json" });
+        res.end(JSON.stringify({ ok: true, product: "SDTK-WIKI" }));
+        return;
+      }
+
+      if (url.startsWith("/api/note")) {
+        const qs = new URL(url, `http://${host}:${port}`).searchParams;
+        const notePath = qs.get("path");
+        if (!notePath) {
+          res.writeHead(400, { "Content-Type": "application/json" });
+          res.end(JSON.stringify({ error: "Missing path parameter" }));
+          return;
+        }
+
+        const projectRoot = projectPath || outputDir;
+        const resolved = path.resolve(projectRoot, notePath);
+        if (!resolved.startsWith(projectRoot + path.sep) && resolved !== projectRoot) {
+          res.writeHead(403, { "Content-Type": "application/json" });
+          res.end(JSON.stringify({ error: "Access denied" }));
+          return;
+        }
+
+        if (!fs.existsSync(resolved)) {
+          res.writeHead(404, { "Content-Type": "application/json" });
+          res.end(JSON.stringify({ error: "Not found" }));
+          return;
+        }
+
+        try {
+          const content = fs.readFileSync(resolved, "utf-8");
+          res.writeHead(200, { "Content-Type": "text/plain; charset=utf-8" });
+          res.end(content);
+        } catch (_) {
+          res.writeHead(500, { "Content-Type": "application/json" });
+          res.end(JSON.stringify({ error: "Read error" }));
+        }
+        return;
+      }
+
+      let filePath = url.split("?")[0];
+      if (filePath === "/" || filePath === "") {
+        filePath = "/viewer.html";
+      }
+
+      const resolved = path.resolve(outputDir, filePath.replace(/^\/+/, ""));
+      if (!resolved.startsWith(outputDir + path.sep) && resolved !== outputDir) {
+        res.writeHead(403, { "Content-Type": "text/plain" });
+        res.end("403 Forbidden");
+        return;
+      }
+
+      if (!fs.existsSync(resolved) || fs.statSync(resolved).isDirectory()) {
+        res.writeHead(404, { "Content-Type": "text/plain" });
+        res.end("404 Not Found");
+        return;
+      }
+
+      const ext = path.extname(resolved).toLowerCase();
+      const contentType = MIME_TYPES[ext] || "application/octet-stream";
+
+      try {
+        const data = fs.readFileSync(resolved);
+        res.writeHead(200, { "Content-Type": contentType });
+        res.end(data);
+      } catch (_) {
+        res.writeHead(500, { "Content-Type": "text/plain" });
+        res.end("500 Internal Server Error");
+      }
+    });
+
+    server.once("error", (err) => {
+      if (err.code === "EADDRINUSE") {
+        reject(
+          new CliError(
+            `Port ${port} is already in use on ${host}.\n` +
+              "Pass --port <number> to use a different port, or stop the process using that port."
+          )
+        );
+      } else {
+        reject(new CliError(`SDTK-WIKI server error: ${err.message}`));
+      }
+    });
+
+    server.listen(port, host, () => resolve(server));
+  });
+}
+
+async function openViewer(config, noOpen = false) {
+  const { host, port, outputDir } = config;
+  const viewerUrl = `http://${host}:${port}/viewer.html`;
+  const alreadyRunning = await isPortOpen(host, port);
+
+  let server = null;
+  if (!alreadyRunning) {
+    console.log(`[wiki] Starting local SDTK-WIKI viewer on http://${host}:${port} ...`);
+    server = await startWikiServer(host, port, outputDir, config.projectPath);
+    await waitForServer(host, port);
+    console.log(`[wiki] Viewer server ready: ${viewerUrl}`);
+  } else {
+    const probe = await probeExistingWikiServer(host, port, viewerUrl);
+    if (!probe.reusable) {
+      throw new CliError(
+        `Port ${port} is already occupied by an incompatible viewer server.\n` +
+          `  Health endpoint status: ${probe.health.statusCode || "unreachable"}\n` +
+          `  Viewer endpoint status: ${probe.viewer.statusCode || "unreachable"}\n` +
+          "Stop the existing process on that port, then rerun the command, or pass --port <number>."
+      );
+    }
+    console.log(`[wiki] Reusing existing server at http://${host}:${port}`);
+  }
+
+  if (!noOpen) {
+    console.log("[wiki] Opening viewer in default browser...");
+    await openBrowser(viewerUrl);
+  } else {
+    console.log(`[wiki] Viewer URL: ${viewerUrl}`);
+    console.log("[wiki] --no-open specified; skipping browser launch.");
+  }
+
+  return { url: viewerUrl, server };
+}
+
+module.exports = {
+  findPython,
+  openViewer,
+  runBuild,
+  startWikiServer,
+};

package/src/lib/wiki-workspace.js

@@ -0,0 +1,144 @@
+"use strict";
+
+const fs = require("fs");
+const path = require("path");
+const {
+  assertPathInsideOrEqual,
+  getLegacyAtlasPath,
+  getWikiGraphPath,
+  getWikiLogsPath,
+  getWikiManifestPath,
+  getWikiPagesPath,
+  getWikiRawDescriptorsPath,
+  getWikiRawPath,
+  getWikiRawSourcesPath,
+  getWikiProvenanceIngestEventsPath,
+  getWikiProvenancePath,
+  getWikiProvenanceSourcesPath,
+  getWikiQueriesPath,
+  getWikiReportsPath,
+  getWikiWorkspacePath,
+} = require("./wiki-paths");
+
+const WIKI_WORKSPACE_SCHEMA_VERSION = 1;
+
+const RESERVED_DIRECTORIES = [
+  "graph",
+  "pages",
+  "raw",
+  "provenance",
+  "queries",
+  "reports",
+  "logs",
+];
+
+function nowIso() {
+  return new Date().toISOString();
+}
+
+function describeWorkspace(projectPath, graphRoot) {
+  return {
+    workspaceRoot: getWikiWorkspacePath(projectPath),
+    manifestPath: getWikiManifestPath(projectPath),
+    graphRoot: graphRoot || getWikiGraphPath(projectPath),
+    defaultGraphRoot: getWikiGraphPath(projectPath),
+    pagesRoot: getWikiPagesPath(projectPath),
+    rawRoot: getWikiRawPath(projectPath),
+    rawDescriptorsRoot: getWikiRawDescriptorsPath(projectPath),
+    rawSourcesPath: getWikiRawSourcesPath(projectPath),
+    provenanceRoot: getWikiProvenancePath(projectPath),
+    provenanceIngestEventsPath: getWikiProvenanceIngestEventsPath(projectPath),
+    provenanceSourcesPath: getWikiProvenanceSourcesPath(projectPath),
+    queriesRoot: getWikiQueriesPath(projectPath),
+    reportsRoot: getWikiReportsPath(projectPath),
+    logsRoot: getWikiLogsPath(projectPath),
+    legacyAtlasRoot: getLegacyAtlasPath(projectPath),
+  };
+}
+
+function assertInsideWorkspace(targetPath, workspaceRoot) {
+  return assertPathInsideOrEqual(
+    targetPath,
+    workspaceRoot,
+    "Refusing to write outside SDTK-WIKI workspace"
+  );
+}
+
+function readWorkspaceManifest(projectPath) {
+  const manifestPath = getWikiManifestPath(projectPath);
+  if (!fs.existsSync(manifestPath)) return null;
+  try {
+    return JSON.parse(fs.readFileSync(manifestPath, "utf-8"));
+  } catch (_) {
+    return null;
+  }
+}
+
+function buildManifest(projectPath, graphRoot, existingManifest) {
+  const workspace = describeWorkspace(projectPath, graphRoot);
+  const timestamp = nowIso();
+  const createdAt =
+    existingManifest && typeof existingManifest.createdAt === "string"
+      ? existingManifest.createdAt
+      : timestamp;
+
+  return {
+    schemaVersion: WIKI_WORKSPACE_SCHEMA_VERSION,
+    product: "SDTK-WIKI",
+    workspaceRoot: workspace.workspaceRoot,
+    graphRoot: workspace.graphRoot,
+    legacyAtlasRoot: workspace.legacyAtlasRoot,
+    createdAt,
+    updatedAt: timestamp,
+    features: {
+      graph: true,
+      pages: false,
+      raw: true,
+      queries: false,
+      reports: false,
+    },
+  };
+}
+
+function ensureWorkspace(projectPath, graphRoot) {
+  const workspace = describeWorkspace(projectPath, graphRoot);
+  fs.mkdirSync(workspace.workspaceRoot, { recursive: true });
+
+  for (const dirName of RESERVED_DIRECTORIES) {
+    const target = path.join(workspace.workspaceRoot, dirName);
+    assertInsideWorkspace(target, workspace.workspaceRoot);
+    fs.mkdirSync(target, { recursive: true });
+  }
+
+  const existingManifest = readWorkspaceManifest(projectPath);
+  const manifest = buildManifest(projectPath, graphRoot, existingManifest);
+  assertInsideWorkspace(workspace.manifestPath, workspace.workspaceRoot);
+  fs.writeFileSync(workspace.manifestPath, JSON.stringify(manifest, null, 2) + "\n", "utf-8");
+
+  return {
+    manifest,
+    workspace,
+  };
+}
+
+function getWorkspaceStatus(projectPath) {
+  const workspace = describeWorkspace(projectPath);
+  const manifest = readWorkspaceManifest(projectPath);
+  return {
+    exists: fs.existsSync(workspace.workspaceRoot),
+    manifestExists: fs.existsSync(workspace.manifestPath),
+    schemaVersion: manifest ? manifest.schemaVersion : null,
+    manifest,
+    workspace,
+  };
+}
+
+module.exports = {
+  RESERVED_DIRECTORIES,
+  WIKI_WORKSPACE_SCHEMA_VERSION,
+  buildManifest,
+  describeWorkspace,
+  ensureWorkspace,
+  getWorkspaceStatus,
+  readWorkspaceManifest,
+};