wiki-graph-sdk 0.7.1 → 0.8.0

package/dist/default-worker.js

@@ -8,14 +8,14 @@ import {
   getChapterDetails,
   openWikimediaResolver,
   recordBuildJobInputRevision,
-  WikiGraphArchiveFile as WikiGraphArchiveFile2,
+  WikiGraphArchiveFile as WikiGraphArchiveFile3,
   WikiGraphScope,
   withLoggingContext
 } from "wiki-graph-core";
 import {
   applyChapterJobArtifactFile as applyChapterJobArtifactFile3,
   createLocalChapterJobFileExecutor as createLocalChapterJobFileExecutor2,
-  writeChapterJobInputFile as writeChapterJobInputFile2
+  writeChapterJobInputFile as writeChapterJobInputFile3
 } from "wiki-graph-core/worker";
 
 // src/embedding.ts
@@ -289,6 +289,14 @@ function decodeNodeResourceIdentity(identity, kind) {
     return void 0;
   }
 }
+function getNodeResourcePath(resource) {
+  const brandedPath = getBrandedNodeResourcePath(resource, resource.kind);
+  if (brandedPath !== void 0) return brandedPath;
+  if (typeof resource === "object" && resource !== null && "path" in resource && typeof resource.path === "string") {
+    return resource.path;
+  }
+  throw new TypeError("Expected a Node File or Directory resource");
+}
 function assertChildName(name) {
   if (name.length === 0 || name === "." || name === ".." || pathModule.isAbsolute(name) || name.includes("/") || name.includes("\\")) {
     throw new TypeError(`Directory child must be a relative name: ${name}`);
@@ -300,42 +308,42 @@ var NodeDatabaseConnection = class {
   }
   database;
   async close() {
-    await new Promise((resolve2, reject) => {
+    await new Promise((resolve4, reject) => {
       this.database.close((error) => {
         if (error) reject(error);
-        else resolve2();
+        else resolve4();
       });
     });
   }
   async execute(sql) {
-    await new Promise((resolve2, reject) => {
+    await new Promise((resolve4, reject) => {
       this.database.exec(sql, (error) => {
         if (error) reject(error);
-        else resolve2();
+        else resolve4();
       });
     });
   }
   async queryAll(sql, params = []) {
-    return await new Promise((resolve2, reject) => {
+    return await new Promise((resolve4, reject) => {
       this.database.all(sql, [...params], (error, rows) => {
         if (error) reject(error);
-        else resolve2(rows);
+        else resolve4(rows);
       });
     });
   }
   async queryOne(sql, params = []) {
-    return await new Promise((resolve2, reject) => {
+    return await new Promise((resolve4, reject) => {
       this.database.get(sql, [...params], (error, row) => {
         if (error) reject(error);
-        else resolve2(row);
+        else resolve4(row);
       });
     });
   }
   async run(sql, params = []) {
-    await new Promise((resolve2, reject) => {
+    await new Promise((resolve4, reject) => {
       this.database.run(sql, [...params], (error) => {
         if (error) reject(error);
-        else resolve2();
+        else resolve4();
       });
     });
   }
@@ -349,10 +357,10 @@ async function openNativeNodeDatabase(file, flags) {
   if (path === void 0) {
     throw new TypeError("The Node database adapter requires a NodeFile");
   }
-  return await new Promise((resolve2, reject) => {
+  return await new Promise((resolve4, reject) => {
     const database = new nodeSqlite3.Database(path, flags, (error) => {
       if (error) reject(error);
-      else resolve2(database);
+      else resolve4(database);
     });
   });
 }
@@ -361,19 +369,19 @@ async function openNodeZip(file) {
   if (path === void 0) {
     throw new TypeError("The Node ZIP adapter requires a NodeFile");
   }
-  const zipFile = await new Promise((resolve2, reject) => {
+  const zipFile = await new Promise((resolve4, reject) => {
     yauzl.open(
       path,
       { autoClose: false, lazyEntries: true },
       (error, opened) => {
         if (error || opened === void 0)
           reject(error ?? new Error("Cannot open ZIP"));
-        else resolve2(opened);
+        else resolve4(opened);
       }
     );
   });
   const entries = await new Promise(
-    (resolve2, reject) => {
+    (resolve4, reject) => {
       const discovered = /* @__PURE__ */ new Map();
       const rejectAndClose = (error) => {
         zipFile.close();
@@ -383,7 +391,7 @@ async function openNodeZip(file) {
       };
       const resolveEntries = () => {
         zipFile.off("error", rejectAndClose);
-        resolve2(discovered);
+        resolve4(discovered);
       };
       zipFile.on("entry", (entry) => {
         if (!entry.fileName.endsWith("/")) {
@@ -493,12 +501,12 @@ async function collectDecodedNodeStreamRange(input, offset, length) {
   return Buffer.concat(chunks, collected);
 }
 async function openNodeZipEntryStream(zipFile, entry, options) {
-  return await new Promise((resolve2, reject) => {
+  return await new Promise((resolve4, reject) => {
     const callback = (error, opened) => {
       if (error || opened === void 0) {
         reject(error ?? new Error(`Cannot read ZIP entry: ${entry.fileName}`));
       } else {
-        resolve2(opened);
+        resolve4(opened);
       }
     };
     if (options === void 0) zipFile.openReadStream(entry, callback);
@@ -629,11 +637,11 @@ async function writeNodeStream(input, writer) {
   );
 }
 async function collectNodeStream(input) {
-  return await new Promise((resolve2, reject) => {
+  return await new Promise((resolve4, reject) => {
     const chunks = [];
     input.on("data", (chunk) => chunks.push(chunk));
     input.once("error", reject);
-    input.once("end", () => resolve2(Buffer.concat(chunks)));
+    input.once("end", () => resolve4(Buffer.concat(chunks)));
   });
 }
 var nodeWikiGraphPlatform = {
@@ -1380,12 +1388,13 @@ var DEFAULT_GENERATION_JOB_CONCURRENCY = 3;
 // src/wikispine.ts
 import { spawn } from "child_process";
 var nodeWikispineCommandRunner = {
-  run: async ({ args, command, input, onStdout, signal }) => await new Promise((resolve2, reject) => {
+  run: async ({ args, command, input, onStdout, signal }) => await new Promise((resolve4, reject) => {
     signal?.throwIfAborted();
     const child = spawn(command, [...args], {
       stdio: ["pipe", "pipe", "pipe"]
     });
     const stderr = [];
+    let stdoutQueue = Promise.resolve();
     let settled = false;
     const settle = (operation) => {
       if (settled) return;
@@ -1401,14 +1410,15 @@ var nodeWikispineCommandRunner = {
     child.stdout.setEncoding("utf8");
     child.stdout.on("data", (chunk) => {
       if (settled) return;
-      try {
-        onStdout(chunk);
-      } catch (error) {
+      child.stdout.pause();
+      stdoutQueue = stdoutQueue.then(async () => await onStdout(chunk)).then(() => {
+        child.stdout.resume();
+      }).catch((error) => {
         child.kill();
         settle(
           () => reject(error instanceof Error ? error : new Error(String(error)))
         );
-      }
+      });
     });
     child.stderr.on("data", (chunk) => stderr.push(chunk));
     child.on("error", (error) => {
@@ -1419,12 +1429,14 @@ var nodeWikispineCommandRunner = {
       );
     });
     child.on("close", (exitCode) => {
-      settle(
-        () => resolve2({
-          exitCode,
-          stderr: Buffer.concat(stderr).toString("utf8")
-        })
-      );
+      void stdoutQueue.then(() => {
+        settle(
+          () => resolve4({
+            exitCode,
+            stderr: Buffer.concat(stderr).toString("utf8")
+          })
+        );
+      });
     });
     child.stdin.end(input);
   })
@@ -1436,13 +1448,243 @@ function abortError(signal) {
 // src/worker.ts
 import {
   applyChapterJobArtifactFile,
-  runBuildJobWorker as runCoreBuildJobWorker
+  runBuildJobWorker as runCoreBuildJobWorker,
+  writeChapterJobInputFile
 } from "wiki-graph-core/worker";
-import { WikiGraphArchiveFile } from "wiki-graph-core";
+import {
+  resolveChapterPathReadonly,
+  WikiGraphArchiveFile as WikiGraphArchiveFile2
+} from "wiki-graph-core";
+
+// src/archive/target.ts
+import { realpath, stat as stat2 } from "fs/promises";
+import { basename as basename2, dirname as dirname2, isAbsolute as isAbsolute3, join as join3, relative, resolve as resolve3 } from "path";
+import {
+  formatLocatedWikiGraphUri,
+  formatWikiGraphLibraryUri,
+  listWikiGraphLibraries,
+  listWikiGraphLibraryArchives,
+  parseLocatedWikiGraphUri,
+  parseWikiGraphLibraryUri,
+  resolveWikiGraphLibrary,
+  resolveWikiGraphLibraryArchiveFile
+} from "wiki-graph-core";
+
+// src/runtime-path.ts
+import { homedir as homedir3 } from "os";
+import { isAbsolute as isAbsolute2, resolve as resolve2 } from "path";
+function resolveWikiGraphRuntimePath(path) {
+  const context = getWikiGraphSDKRuntimeContext();
+  const environmentHome = context.env.HOME?.trim();
+  const home = environmentHome === void 0 || environmentHome === "" ? homedir3() : environmentHome;
+  if (path === "~") return home;
+  if (path.startsWith("~/")) return resolve2(home, path.slice(2));
+  return isAbsolute2(path) ? path : resolve2(context.cwd, path);
+}
+
+// src/archive/target.ts
+var WikiGraphArchiveOwnershipError = class extends Error {
+  code = "WIKI_GRAPH_ARCHIVE_OWNERSHIP_MISMATCH";
+  constructor(message) {
+    super(message);
+    this.name = "WikiGraphArchiveOwnershipError";
+  }
+};
+async function resolveWikiGraphArchiveLocation(target) {
+  if (target.kind === "standalone") {
+    return await resolveStandaloneLocation(target);
+  }
+  return await resolveLibraryLocation(target);
+}
+async function assertStandaloneWikiGraphArchivePath(inputPath) {
+  const path = resolveWikiGraphRuntimePath(inputPath);
+  const candidate = await canonicalPath(path);
+  const candidateIdentity = await readFileIdentity(path);
+  const libraries = await listWikiGraphLibraries();
+  const matches = await findMatchingMembers(
+    libraries,
+    candidate,
+    candidateIdentity
+  );
+  if (matches.length > 0) {
+    const match = requireUnambiguousManagedMatch(path, matches);
+    throw new WikiGraphArchiveOwnershipError(
+      `Standalone archive path belongs to registered library ${formatWikiGraphLibraryUri(
+        match.library.isDefault ? void 0 : match.library.publicId
+      )}: ${path}. Use the library UUID URI ${match.archive.uri} instead.`
+    );
+  }
+  for (const library of libraries) {
+    const rootPath = getNodeResourcePath(library.folder);
+    const root = await canonicalPath(rootPath);
+    if (!isWithin(root, candidate)) continue;
+    throw new WikiGraphArchiveOwnershipError(
+      `Standalone archive path belongs to registered library ${formatWikiGraphLibraryUri(
+        library.isDefault ? void 0 : library.publicId
+      )}: ${path}. Scan the library, then address the archive by its library UUID URI.`
+    );
+  }
+  return path;
+}
+async function resolveWikiGraphArchiveTarget(file) {
+  const path = getNodeResourcePath(file);
+  const candidate = await canonicalPath(path);
+  const candidateIdentity = await readFileIdentity(path);
+  const matches = await findMatchingMembers(
+    await listWikiGraphLibraries(),
+    candidate,
+    candidateIdentity
+  );
+  if (matches.length > 0) {
+    const match = requireUnambiguousManagedMatch(path, matches);
+    return { kind: "library", uri: match.archive.uri };
+  }
+  return { kind: "standalone", path: getNodeResourcePath(file) };
+}
+async function resolveStandaloneLocation(target) {
+  if (target.path.startsWith("wikg://lib/")) {
+    throw new WikiGraphArchiveOwnershipError(
+      `A library archive URI cannot be used as a standalone path: ${target.path}`
+    );
+  }
+  const path = await assertStandaloneWikiGraphArchivePath(target.path);
+  const normalized = {
+    kind: "standalone",
+    path,
+    ...target.objectUri === void 0 ? {} : { objectUri: target.objectUri }
+  };
+  return {
+    archiveFile: new NodeFile(path),
+    archiveKey: path,
+    archivePath: path,
+    indexScope: { archiveKey: path, archivePath: path, kind: "archive-index" },
+    locatedUri: formatLocatedWikiGraphUri(path, target.objectUri),
+    target: normalized
+  };
+}
+async function resolveLibraryLocation(target) {
+  const parsedLocated = parseLocatedWikiGraphUri(target.uri);
+  const archiveUri = parsedLocated.archivePath ?? target.uri;
+  const parsed = parseWikiGraphLibraryUri(archiveUri);
+  if (parsed?.kind !== "archive") {
+    throw new WikiGraphArchiveOwnershipError(
+      `Library archive target must be a UUID archive URI: ${target.uri}`
+    );
+  }
+  await resolveWikiGraphLibrary(parsed);
+  const archiveFile = await resolveWikiGraphLibraryArchiveFile(archiveUri);
+  const normalized = {
+    kind: "library",
+    uri: target.uri
+  };
+  return {
+    archiveFile,
+    archiveKey: archiveUri,
+    archivePath: archiveUri,
+    indexScope: {
+      archiveKey: archiveUri,
+      archivePath: archiveUri,
+      kind: "archive-index"
+    },
+    libraryArchiveTarget: parsed,
+    locatedUri: target.uri,
+    publicArchiveUri: archiveUri,
+    target: normalized
+  };
+}
+async function canonicalPath(path) {
+  try {
+    return await realpath(path);
+  } catch {
+    const absolute = resolve3(path);
+    const parent = dirname2(absolute);
+    return parent === absolute ? absolute : join3(await canonicalPath(parent), basename2(absolute));
+  }
+}
+function isWithin(root, candidate) {
+  const child = relative(root, candidate);
+  return child === "" || !child.startsWith("..") && !isAbsolute3(child);
+}
+async function findMatchingMembers(libraries, candidate, candidateIdentity) {
+  const members = [];
+  for (const library of libraries) {
+    const archives = await listWikiGraphLibraryArchives({
+      isDefault: library.isDefault,
+      kind: "scope",
+      ...library.isDefault ? {} : { publicId: library.publicId }
+    });
+    for (const archive of archives) {
+      if (archive.file === void 0) continue;
+      members.push({
+        archive,
+        library,
+        memberPath: getNodeResourcePath(archive.file)
+      });
+    }
+  }
+  const exact = [];
+  for (const member of members) {
+    if (await canonicalPath(member.memberPath) === candidate) {
+      exact.push(member);
+    }
+  }
+  if (exact.length > 0 || candidateIdentity === void 0) return exact;
+  const identityMatches = [];
+  for (const member of members) {
+    if (await readFileIdentity(member.memberPath) === candidateIdentity) {
+      identityMatches.push(member);
+    }
+  }
+  return identityMatches;
+}
+function requireUnambiguousManagedMatch(path, matches) {
+  if (matches.length === 1) return matches[0];
+  throw new WikiGraphArchiveOwnershipError(
+    `Archive file identity is ambiguous across managed archives: ${path}. Use an explicit library UUID URI (${matches.map((match) => match.archive.uri).join(", ")}).`
+  );
+}
+async function readFileIdentity(path) {
+  try {
+    const value = await stat2(path);
+    return `${value.dev}:${value.ino}`;
+  } catch {
+    return void 0;
+  }
+}
+
+// src/archive/write.ts
+import {
+  WikiGraphArchiveFile,
+  writeWikiGraphLibraryArchive
+} from "wiki-graph-core";
+async function writeWikiGraphArchiveLocation(location, operation, options = {}) {
+  const cacheOnly = options.searchIndexWritebackPolicy === "cache";
+  const write = async (archiveFile = location.archiveFile) => await new WikiGraphArchiveFile(archiveFile).write(operation, {
+    ...cacheOnly ? {} : { derivedStateKeys: [location.archiveKey, location.archivePath] },
+    ...options.invalidateContinuationCursors === void 0 ? {} : {
+      invalidateContinuationCursors: options.invalidateContinuationCursors
+    },
+    ...options.searchIndexWritebackPolicy === void 0 ? {} : { searchIndexWritebackPolicy: options.searchIndexWritebackPolicy }
+  });
+  if (location.libraryArchiveTarget === void 0 || cacheOnly || options.refreshLibraryIndex === false) {
+    return await write();
+  }
+  return await writeWikiGraphLibraryArchive({
+    additionalDerivedStateKeys: [location.archiveKey, location.archivePath],
+    ...options.invalidateContinuationCursors === void 0 ? {} : {
+      invalidateContinuationCursors: options.invalidateContinuationCursors
+    },
+    ...options.onIndexSyncError === void 0 ? {} : { onIndexSyncError: options.onIndexSyncError },
+    operation: async (archiveFile) => await write(archiveFile),
+    target: location.libraryArchiveTarget
+  });
+}
+
+// src/worker.ts
 import {
   applyChapterJobArtifactFile as applyChapterJobArtifactFile2,
   createLocalChapterJobFileExecutor,
-  writeChapterJobInputFile
+  writeChapterJobInputFile as writeChapterJobInputFile2
 } from "wiki-graph-core/worker";
 async function runBuildJobWorker(options) {
   installNodeWikiGraphPlatform();
@@ -1531,10 +1773,10 @@ async function executeStep(job, kind, execution, reporter, context, inputOptions
   const stepWorkspace = await resetStepWorkspace(job.workspace, kind);
   const inputFile = await stepWorkspace.createFile("input.jsonl");
   const outputWorkspace = await stepWorkspace.createDirectory("output");
-  const revision = await new WikiGraphArchiveFile2(job.archive).readDocument(
+  const revision = await new WikiGraphArchiveFile3(job.archive).readDocument(
     async (document) => {
       await getChapterDetails(document, job.chapterId);
-      return await writeChapterJobInputFile2(
+      return await writeChapterJobInputFile3(
         document,
         job.chapterId,
         kind,
@@ -1573,16 +1815,23 @@ async function executeStep(job, kind, execution, reporter, context, inputOptions
     total: 1,
     unit: "item"
   });
-  await new WikiGraphArchiveFile2(job.archive).write(async (document) => {
-    assertJobStillRunning(await getBuildJob(job.jobId));
-    await applyChapterJobArtifactFile3(
-      document,
-      job.chapterId,
-      kind,
-      revision,
-      result.artifactFile
-    );
-  });
+  const target = await resolveWikiGraphArchiveTarget(job.archive);
+  const resolvedLocation = await resolveWikiGraphArchiveLocation(target);
+  const location = target.kind === "standalone" ? { ...resolvedLocation, archiveFile: job.archive } : resolvedLocation;
+  await writeWikiGraphArchiveLocation(
+    location,
+    async (document) => {
+      assertJobStillRunning(await getBuildJob(job.jobId));
+      await applyChapterJobArtifactFile3(
+        document,
+        job.chapterId,
+        kind,
+        revision,
+        result.artifactFile
+      );
+    },
+    { invalidateContinuationCursors: true }
+  );
   await reporter.updatePhase({
     done: 1,
     phase: "committing",
@@ -1621,6 +1870,10 @@ async function openJobExecutor(job, config, signal, reporter) {
       llmRequest: requireValue(
         request,
         "Local Wikimedia resolution requires an LLM."
+      ),
+      llmModelId: requireValue(
+        stageConfig?.llm?.model,
+        "Local Wikimedia resolution requires an LLM model."
       )
     } : {
       endpoint: config.wikimedia.endpoint,
@@ -1658,12 +1911,12 @@ async function resetStepWorkspace(workspace, kind) {
   return await workspace.createDirectory(name);
 }
 async function readChapterStage(job) {
-  return await new WikiGraphArchiveFile2(job.archive).readDocument(
+  return await new WikiGraphArchiveFile3(job.archive).readDocument(
     async (document) => (await getChapterDetails(document, job.chapterId)).stage
   );
 }
 async function assertCurrentFtsArtifact(job) {
-  await new WikiGraphArchiveFile2(job.archive).readDocument(async (document) => {
+  await new WikiGraphArchiveFile3(job.archive).readDocument(async (document) => {
     const revision = await document.serials.getRevision(job.chapterId);
     const artifact = await document.indexArtifacts.get(job.chapterId, "fts");
     if (artifact?.sourceRevision !== revision) {