sdtk-brain-kit 0.1.0

package/src/lib/wiki-flags.js

@@ -0,0 +1,97 @@
+"use strict";
+
+const { parseFlags } = require("./args");
+
+const BASE_FLAG_DEFS = {
+  "project-path": { type: "string" },
+  "output-dir": { type: "string" },
+  "scan-root": { type: "string" },
+  verbose: { type: "boolean" },
+};
+
+const OPEN_FLAG_DEFS = {
+  ...BASE_FLAG_DEFS,
+  host: { type: "string" },
+  port: { type: "string" },
+  "no-open": { type: "boolean" },
+};
+
+const INIT_FLAG_DEFS = {
+  ...OPEN_FLAG_DEFS,
+  force: { type: "boolean" },
+  "no-build": { type: "boolean" },
+};
+
+const WATCH_FLAG_DEFS = {
+  ...BASE_FLAG_DEFS,
+  port: { type: "string" },
+  "no-open": { type: "boolean" },
+};
+
+const LINT_FLAG_DEFS = {
+  "project-path": { type: "string" },
+};
+
+const PRUNE_FLAG_DEFS = {
+  "project-path": { type: "string" },
+  "dry-run": { type: "boolean" },
+};
+
+const DISCOVER_FLAG_DEFS = {
+  "project-path": { type: "string" },
+  plan: { type: "boolean" },
+};
+
+const EXTRACT_FLAG_DEFS = {
+  "project-path": { type: "string" },
+  "source-root": { type: "string" },
+  "dry-run": { type: "boolean" },
+};
+
+const COMPILE_FLAG_DEFS = {
+  "project-path": { type: "string" },
+  plan: { type: "string" },
+  "dry-run": { type: "boolean" },
+  apply: { type: "boolean" },
+  yes: { type: "boolean" },
+};
+
+function parseWikiFlags(args, defs) {
+  const scanRoots = [];
+  const filteredArgs = [];
+  let i = 0;
+
+  while (i < args.length) {
+    const arg = args[i];
+    if (arg === "--scan-root") {
+      i++;
+      if (i < args.length) {
+        scanRoots.push(args[i]);
+        i++;
+      }
+    } else if (arg.startsWith("--scan-root=")) {
+      scanRoots.push(arg.slice("--scan-root=".length));
+      i++;
+    } else {
+      filteredArgs.push(arg);
+      i++;
+    }
+  }
+
+  const { flags, positional } = parseFlags(filteredArgs, defs);
+  flags["scan-root"] = scanRoots.length > 0 ? scanRoots : undefined;
+  return { flags, positional };
+}
+
+module.exports = {
+  BASE_FLAG_DEFS,
+  COMPILE_FLAG_DEFS,
+  DISCOVER_FLAG_DEFS,
+  EXTRACT_FLAG_DEFS,
+  INIT_FLAG_DEFS,
+  LINT_FLAG_DEFS,
+  OPEN_FLAG_DEFS,
+  PRUNE_FLAG_DEFS,
+  WATCH_FLAG_DEFS,
+  parseWikiFlags,
+};

package/src/lib/wiki-ingest.js

@@ -0,0 +1,198 @@
+"use strict";
+
+const crypto = require("crypto");
+const fs = require("fs");
+const path = require("path");
+const { ValidationError } = require("./errors");
+const {
+  assertWikiWorkspaceWritePath,
+  getWikiProvenanceIngestEventsPath,
+  getWikiProvenancePath,
+  getWikiRawDescriptorsPath,
+  getWikiRawPath,
+  getWikiRawSourcesPath,
+  isPathInsideOrEqual,
+  resolveProjectPath,
+} = require("./wiki-paths");
+
+const INGEST_SCHEMA_VERSION = 1;
+
+function nowIso() {
+  return new Date().toISOString();
+}
+
+function readJsonIfPresent(filePath, fallback) {
+  if (!fs.existsSync(filePath)) return fallback;
+  try {
+    return JSON.parse(fs.readFileSync(filePath, "utf-8"));
+  } catch (_) {
+    return fallback;
+  }
+}
+
+function writeJson(filePath, payload, projectPath) {
+  assertWikiWorkspaceWritePath(filePath, projectPath);
+  fs.mkdirSync(path.dirname(filePath), { recursive: true });
+  fs.writeFileSync(filePath, JSON.stringify(payload, null, 2) + "\n", "utf-8");
+}
+
+function rejectRemoteSource(sourceArg) {
+  if (/^[a-z][a-z0-9+.-]*:\/\//i.test(sourceArg)) {
+    throw new ValidationError("Local project sources only. Remote URLs are not supported in BK-113.");
+  }
+}
+
+function toProjectRelative(projectPath, sourcePath) {
+  return path.relative(projectPath, sourcePath).split(path.sep).join("/");
+}
+
+function detectSourceType(sourcePath) {
+  const ext = path.extname(sourcePath).toLowerCase();
+  if (ext === ".md" || ext === ".markdown" || ext === ".mdx") return "markdown";
+  if (ext === ".txt") return "text";
+  return ext ? ext.slice(1) : "file";
+}
+
+function fingerprintSource(sourcePath) {
+  const bytes = fs.readFileSync(sourcePath);
+  return crypto.createHash("sha256").update(bytes).digest("hex");
+}
+
+function stableSourceId(relativePath) {
+  return `raw:${crypto.createHash("sha256").update(relativePath).digest("hex").slice(0, 24)}`;
+}
+
+function ensureApprovedIngestDirs(projectPath) {
+  const rawRoot = getWikiRawPath(projectPath);
+  const rawDescriptorsRoot = getWikiRawDescriptorsPath(projectPath);
+  const provenanceRoot = getWikiProvenancePath(projectPath);
+  for (const target of [rawRoot, rawDescriptorsRoot, provenanceRoot]) {
+    assertWikiWorkspaceWritePath(target, projectPath);
+    fs.mkdirSync(target, { recursive: true });
+  }
+}
+
+function resolveLocalSource(projectPath, sourceArg) {
+  if (!sourceArg || !String(sourceArg).trim()) {
+    throw new ValidationError("Missing required --source <path>.");
+  }
+
+  rejectRemoteSource(String(sourceArg));
+  const projectRoot = resolveProjectPath(projectPath);
+  const candidate = path.resolve(projectRoot, sourceArg);
+
+  if (!isPathInsideOrEqual(candidate, projectRoot)) {
+    throw new ValidationError(`Source must stay inside the project root: ${candidate}`);
+  }
+  if (!fs.existsSync(candidate)) {
+    throw new ValidationError(`Source does not exist: ${candidate}`);
+  }
+  if (!fs.statSync(candidate).isFile()) {
+    throw new ValidationError(`Source must be a file in BK-113: ${candidate}`);
+  }
+  return candidate;
+}
+
+function upsertRegistry(registry, record) {
+  const existing = Array.isArray(registry.sources) ? registry.sources : [];
+  const next = existing.filter((item) => item.sourceId !== record.sourceId);
+  next.push(record);
+  next.sort((a, b) => a.sourcePath.localeCompare(b.sourcePath));
+  return {
+    schemaVersion: INGEST_SCHEMA_VERSION,
+    product: "SDTK-BRAIN",
+    sources: next,
+  };
+}
+
+function appendIngestEvent(eventsPayload, event) {
+  const events = Array.isArray(eventsPayload.events) ? eventsPayload.events.slice() : [];
+  events.push(event);
+  return {
+    schemaVersion: INGEST_SCHEMA_VERSION,
+    product: "SDTK-BRAIN",
+    events,
+  };
+}
+
+function ingestLocalSource({ projectPath, sourceArg }) {
+  const projectRoot = resolveProjectPath(projectPath);
+  const sourcePath = resolveLocalSource(projectRoot, sourceArg);
+  const sourceStat = fs.statSync(sourcePath);
+  const relativeSourcePath = toProjectRelative(projectRoot, sourcePath);
+  const sourceId = stableSourceId(relativeSourcePath);
+  const sourceHash = fingerprintSource(sourcePath);
+  const ingestedAt = nowIso();
+  const descriptorPath = path.join(getWikiRawDescriptorsPath(projectRoot), `${sourceId.replace(/[:/]/g, "-")}.json`);
+  const registryPath = getWikiRawSourcesPath(projectRoot);
+  const eventsPath = getWikiProvenanceIngestEventsPath(projectRoot);
+  const descriptorRelative = toProjectRelative(projectRoot, descriptorPath);
+  const provenanceRelative = toProjectRelative(projectRoot, eventsPath);
+
+  ensureApprovedIngestDirs(projectRoot);
+
+  const descriptor = {
+    schemaVersion: INGEST_SCHEMA_VERSION,
+    product: "SDTK-BRAIN",
+    sourceId,
+    sourcePath: relativeSourcePath,
+    sourceType: detectSourceType(sourcePath),
+    sourceHash,
+    sizeBytes: sourceStat.size,
+    mtimeMs: Math.trunc(sourceStat.mtimeMs),
+    ingestedAt,
+    revision: sourceHash.slice(0, 16),
+    status: "ingested",
+    provenancePath: provenanceRelative,
+  };
+  writeJson(descriptorPath, descriptor, projectRoot);
+
+  const record = {
+    sourceId,
+    sourcePath: relativeSourcePath,
+    sourceType: descriptor.sourceType,
+    sourceHash,
+    sizeBytes: descriptor.sizeBytes,
+    mtimeMs: descriptor.mtimeMs,
+    ingestedAt,
+    revision: descriptor.revision,
+    status: descriptor.status,
+    descriptorPath: descriptorRelative,
+    provenancePath: provenanceRelative,
+  };
+
+  const registry = upsertRegistry(
+    readJsonIfPresent(registryPath, { schemaVersion: INGEST_SCHEMA_VERSION, product: "SDTK-BRAIN", sources: [] }),
+    record
+  );
+  writeJson(registryPath, registry, projectRoot);
+
+  const events = appendIngestEvent(
+    readJsonIfPresent(eventsPath, { schemaVersion: INGEST_SCHEMA_VERSION, product: "SDTK-BRAIN", events: [] }),
+    {
+      event: "ingest.recorded",
+      sourceId,
+      sourcePath: relativeSourcePath,
+      sourceHash,
+      descriptorPath: descriptorRelative,
+      recordedAt: ingestedAt,
+      status: "ingested",
+    }
+  );
+  writeJson(eventsPath, events, projectRoot);
+
+  return {
+    sourceId,
+    sourcePath: relativeSourcePath,
+    registryPath,
+    descriptorPath,
+    eventsPath,
+  };
+}
+
+module.exports = {
+  INGEST_SCHEMA_VERSION,
+  ingestLocalSource,
+  resolveLocalSource,
+  stableSourceId,
+};