wiki-graph-sdk 0.7.0 → 0.8.0

package/dist/default-worker.cjs

@@ -35,7 +35,7 @@ __export(default_worker_exports, {
   runWikiGraphQueueWorker: () => runWikiGraphQueueWorker
 });
 module.exports = __toCommonJS(default_worker_exports);
-var import_wiki_graph_core5 = require("wiki-graph-core");
+var import_wiki_graph_core8 = require("wiki-graph-core");
 var import_worker3 = require("wiki-graph-core/worker");
 
 // src/embedding.ts
@@ -303,6 +303,14 @@ function decodeNodeResourceIdentity(identity, kind) {
     return void 0;
   }
 }
+function getNodeResourcePath(resource) {
+  const brandedPath = getBrandedNodeResourcePath(resource, resource.kind);
+  if (brandedPath !== void 0) return brandedPath;
+  if (typeof resource === "object" && resource !== null && "path" in resource && typeof resource.path === "string") {
+    return resource.path;
+  }
+  throw new TypeError("Expected a Node File or Directory resource");
+}
 function assertChildName(name) {
   if (name.length === 0 || name === "." || name === ".." || pathModule.isAbsolute(name) || name.includes("/") || name.includes("\\")) {
     throw new TypeError(`Directory child must be a relative name: ${name}`);
@@ -314,42 +322,42 @@ var NodeDatabaseConnection = class {
   }
   database;
   async close() {
-    await new Promise((resolve2, reject) => {
+    await new Promise((resolve4, reject) => {
       this.database.close((error) => {
         if (error) reject(error);
-        else resolve2();
+        else resolve4();
       });
     });
   }
   async execute(sql) {
-    await new Promise((resolve2, reject) => {
+    await new Promise((resolve4, reject) => {
       this.database.exec(sql, (error) => {
         if (error) reject(error);
-        else resolve2();
+        else resolve4();
       });
     });
   }
   async queryAll(sql, params = []) {
-    return await new Promise((resolve2, reject) => {
+    return await new Promise((resolve4, reject) => {
       this.database.all(sql, [...params], (error, rows) => {
         if (error) reject(error);
-        else resolve2(rows);
+        else resolve4(rows);
       });
     });
   }
   async queryOne(sql, params = []) {
-    return await new Promise((resolve2, reject) => {
+    return await new Promise((resolve4, reject) => {
       this.database.get(sql, [...params], (error, row) => {
         if (error) reject(error);
-        else resolve2(row);
+        else resolve4(row);
       });
     });
   }
   async run(sql, params = []) {
-    await new Promise((resolve2, reject) => {
+    await new Promise((resolve4, reject) => {
       this.database.run(sql, [...params], (error) => {
         if (error) reject(error);
-        else resolve2();
+        else resolve4();
       });
     });
   }
@@ -363,10 +371,10 @@ async function openNativeNodeDatabase(file, flags) {
   if (path === void 0) {
     throw new TypeError("The Node database adapter requires a NodeFile");
   }
-  return await new Promise((resolve2, reject) => {
+  return await new Promise((resolve4, reject) => {
     const database = new nodeSqlite3.Database(path, flags, (error) => {
       if (error) reject(error);
-      else resolve2(database);
+      else resolve4(database);
     });
   });
 }
@@ -375,19 +383,19 @@ async function openNodeZip(file) {
   if (path === void 0) {
     throw new TypeError("The Node ZIP adapter requires a NodeFile");
   }
-  const zipFile = await new Promise((resolve2, reject) => {
+  const zipFile = await new Promise((resolve4, reject) => {
     yauzl.open(
       path,
       { autoClose: false, lazyEntries: true },
       (error, opened) => {
         if (error || opened === void 0)
           reject(error ?? new Error("Cannot open ZIP"));
-        else resolve2(opened);
+        else resolve4(opened);
       }
     );
   });
   const entries = await new Promise(
-    (resolve2, reject) => {
+    (resolve4, reject) => {
       const discovered = /* @__PURE__ */ new Map();
       const rejectAndClose = (error) => {
         zipFile.close();
@@ -397,7 +405,7 @@ async function openNodeZip(file) {
       };
       const resolveEntries = () => {
         zipFile.off("error", rejectAndClose);
-        resolve2(discovered);
+        resolve4(discovered);
       };
       zipFile.on("entry", (entry) => {
         if (!entry.fileName.endsWith("/")) {
@@ -507,12 +515,12 @@ async function collectDecodedNodeStreamRange(input, offset, length) {
   return Buffer.concat(chunks, collected);
 }
 async function openNodeZipEntryStream(zipFile, entry, options) {
-  return await new Promise((resolve2, reject) => {
+  return await new Promise((resolve4, reject) => {
     const callback = (error, opened) => {
       if (error || opened === void 0) {
         reject(error ?? new Error(`Cannot read ZIP entry: ${entry.fileName}`));
       } else {
-        resolve2(opened);
+        resolve4(opened);
       }
     };
     if (options === void 0) zipFile.openReadStream(entry, callback);
@@ -643,11 +651,11 @@ async function writeNodeStream(input, writer) {
   );
 }
 async function collectNodeStream(input) {
-  return await new Promise((resolve2, reject) => {
+  return await new Promise((resolve4, reject) => {
     const chunks = [];
     input.on("data", (chunk) => chunks.push(chunk));
     input.once("error", reject);
-    input.once("end", () => resolve2(Buffer.concat(chunks)));
+    input.once("end", () => resolve4(Buffer.concat(chunks)));
   });
 }
 var nodeWikiGraphPlatform = {
@@ -1389,12 +1397,13 @@ var DEFAULT_GENERATION_JOB_CONCURRENCY = 3;
 // src/wikispine.ts
 var import_child_process = require("child_process");
 var nodeWikispineCommandRunner = {
-  run: async ({ args, command, input, onStdout, signal }) => await new Promise((resolve2, reject) => {
+  run: async ({ args, command, input, onStdout, signal }) => await new Promise((resolve4, reject) => {
     signal?.throwIfAborted();
     const child = (0, import_child_process.spawn)(command, [...args], {
       stdio: ["pipe", "pipe", "pipe"]
     });
     const stderr = [];
+    let stdoutQueue = Promise.resolve();
     let settled = false;
     const settle = (operation) => {
       if (settled) return;
@@ -1410,14 +1419,15 @@ var nodeWikispineCommandRunner = {
     child.stdout.setEncoding("utf8");
     child.stdout.on("data", (chunk) => {
       if (settled) return;
-      try {
-        onStdout(chunk);
-      } catch (error) {
+      child.stdout.pause();
+      stdoutQueue = stdoutQueue.then(async () => await onStdout(chunk)).then(() => {
+        child.stdout.resume();
+      }).catch((error) => {
         child.kill();
         settle(
           () => reject(error instanceof Error ? error : new Error(String(error)))
         );
-      }
+      });
     });
     child.stderr.on("data", (chunk) => stderr.push(chunk));
     child.on("error", (error) => {
@@ -1428,12 +1438,14 @@ var nodeWikispineCommandRunner = {
       );
     });
     child.on("close", (exitCode) => {
-      settle(
-        () => resolve2({
-          exitCode,
-          stderr: Buffer.concat(stderr).toString("utf8")
-        })
-      );
+      void stdoutQueue.then(() => {
+        settle(
+          () => resolve4({
+            exitCode,
+            stderr: Buffer.concat(stderr).toString("utf8")
+          })
+        );
+      });
     });
     child.stdin.end(input);
   })
@@ -1444,6 +1456,221 @@ function abortError(signal) {
 
 // src/worker.ts
 var import_worker = require("wiki-graph-core/worker");
+var import_wiki_graph_core7 = require("wiki-graph-core");
+
+// src/archive/target.ts
+var import_promises2 = require("fs/promises");
+var import_path3 = require("path");
+var import_wiki_graph_core5 = require("wiki-graph-core");
+
+// src/runtime-path.ts
+var import_os2 = require("os");
+var import_path2 = require("path");
+function resolveWikiGraphRuntimePath(path) {
+  const context = getWikiGraphSDKRuntimeContext();
+  const environmentHome = context.env.HOME?.trim();
+  const home = environmentHome === void 0 || environmentHome === "" ? (0, import_os2.homedir)() : environmentHome;
+  if (path === "~") return home;
+  if (path.startsWith("~/")) return (0, import_path2.resolve)(home, path.slice(2));
+  return (0, import_path2.isAbsolute)(path) ? path : (0, import_path2.resolve)(context.cwd, path);
+}
+
+// src/archive/target.ts
+var WikiGraphArchiveOwnershipError = class extends Error {
+  code = "WIKI_GRAPH_ARCHIVE_OWNERSHIP_MISMATCH";
+  constructor(message) {
+    super(message);
+    this.name = "WikiGraphArchiveOwnershipError";
+  }
+};
+async function resolveWikiGraphArchiveLocation(target) {
+  if (target.kind === "standalone") {
+    return await resolveStandaloneLocation(target);
+  }
+  return await resolveLibraryLocation(target);
+}
+async function assertStandaloneWikiGraphArchivePath(inputPath) {
+  const path = resolveWikiGraphRuntimePath(inputPath);
+  const candidate = await canonicalPath(path);
+  const candidateIdentity = await readFileIdentity(path);
+  const libraries = await (0, import_wiki_graph_core5.listWikiGraphLibraries)();
+  const matches = await findMatchingMembers(
+    libraries,
+    candidate,
+    candidateIdentity
+  );
+  if (matches.length > 0) {
+    const match = requireUnambiguousManagedMatch(path, matches);
+    throw new WikiGraphArchiveOwnershipError(
+      `Standalone archive path belongs to registered library ${(0, import_wiki_graph_core5.formatWikiGraphLibraryUri)(
+        match.library.isDefault ? void 0 : match.library.publicId
+      )}: ${path}. Use the library UUID URI ${match.archive.uri} instead.`
+    );
+  }
+  for (const library of libraries) {
+    const rootPath = getNodeResourcePath(library.folder);
+    const root = await canonicalPath(rootPath);
+    if (!isWithin(root, candidate)) continue;
+    throw new WikiGraphArchiveOwnershipError(
+      `Standalone archive path belongs to registered library ${(0, import_wiki_graph_core5.formatWikiGraphLibraryUri)(
+        library.isDefault ? void 0 : library.publicId
+      )}: ${path}. Scan the library, then address the archive by its library UUID URI.`
+    );
+  }
+  return path;
+}
+async function resolveWikiGraphArchiveTarget(file) {
+  const path = getNodeResourcePath(file);
+  const candidate = await canonicalPath(path);
+  const candidateIdentity = await readFileIdentity(path);
+  const matches = await findMatchingMembers(
+    await (0, import_wiki_graph_core5.listWikiGraphLibraries)(),
+    candidate,
+    candidateIdentity
+  );
+  if (matches.length > 0) {
+    const match = requireUnambiguousManagedMatch(path, matches);
+    return { kind: "library", uri: match.archive.uri };
+  }
+  return { kind: "standalone", path: getNodeResourcePath(file) };
+}
+async function resolveStandaloneLocation(target) {
+  if (target.path.startsWith("wikg://lib/")) {
+    throw new WikiGraphArchiveOwnershipError(
+      `A library archive URI cannot be used as a standalone path: ${target.path}`
+    );
+  }
+  const path = await assertStandaloneWikiGraphArchivePath(target.path);
+  const normalized = {
+    kind: "standalone",
+    path,
+    ...target.objectUri === void 0 ? {} : { objectUri: target.objectUri }
+  };
+  return {
+    archiveFile: new NodeFile(path),
+    archiveKey: path,
+    archivePath: path,
+    indexScope: { archiveKey: path, archivePath: path, kind: "archive-index" },
+    locatedUri: (0, import_wiki_graph_core5.formatLocatedWikiGraphUri)(path, target.objectUri),
+    target: normalized
+  };
+}
+async function resolveLibraryLocation(target) {
+  const parsedLocated = (0, import_wiki_graph_core5.parseLocatedWikiGraphUri)(target.uri);
+  const archiveUri = parsedLocated.archivePath ?? target.uri;
+  const parsed = (0, import_wiki_graph_core5.parseWikiGraphLibraryUri)(archiveUri);
+  if (parsed?.kind !== "archive") {
+    throw new WikiGraphArchiveOwnershipError(
+      `Library archive target must be a UUID archive URI: ${target.uri}`
+    );
+  }
+  await (0, import_wiki_graph_core5.resolveWikiGraphLibrary)(parsed);
+  const archiveFile = await (0, import_wiki_graph_core5.resolveWikiGraphLibraryArchiveFile)(archiveUri);
+  const normalized = {
+    kind: "library",
+    uri: target.uri
+  };
+  return {
+    archiveFile,
+    archiveKey: archiveUri,
+    archivePath: archiveUri,
+    indexScope: {
+      archiveKey: archiveUri,
+      archivePath: archiveUri,
+      kind: "archive-index"
+    },
+    libraryArchiveTarget: parsed,
+    locatedUri: target.uri,
+    publicArchiveUri: archiveUri,
+    target: normalized
+  };
+}
+async function canonicalPath(path) {
+  try {
+    return await (0, import_promises2.realpath)(path);
+  } catch {
+    const absolute = (0, import_path3.resolve)(path);
+    const parent = (0, import_path3.dirname)(absolute);
+    return parent === absolute ? absolute : (0, import_path3.join)(await canonicalPath(parent), (0, import_path3.basename)(absolute));
+  }
+}
+function isWithin(root, candidate) {
+  const child = (0, import_path3.relative)(root, candidate);
+  return child === "" || !child.startsWith("..") && !(0, import_path3.isAbsolute)(child);
+}
+async function findMatchingMembers(libraries, candidate, candidateIdentity) {
+  const members = [];
+  for (const library of libraries) {
+    const archives = await (0, import_wiki_graph_core5.listWikiGraphLibraryArchives)({
+      isDefault: library.isDefault,
+      kind: "scope",
+      ...library.isDefault ? {} : { publicId: library.publicId }
+    });
+    for (const archive of archives) {
+      if (archive.file === void 0) continue;
+      members.push({
+        archive,
+        library,
+        memberPath: getNodeResourcePath(archive.file)
+      });
+    }
+  }
+  const exact = [];
+  for (const member of members) {
+    if (await canonicalPath(member.memberPath) === candidate) {
+      exact.push(member);
+    }
+  }
+  if (exact.length > 0 || candidateIdentity === void 0) return exact;
+  const identityMatches = [];
+  for (const member of members) {
+    if (await readFileIdentity(member.memberPath) === candidateIdentity) {
+      identityMatches.push(member);
+    }
+  }
+  return identityMatches;
+}
+function requireUnambiguousManagedMatch(path, matches) {
+  if (matches.length === 1) return matches[0];
+  throw new WikiGraphArchiveOwnershipError(
+    `Archive file identity is ambiguous across managed archives: ${path}. Use an explicit library UUID URI (${matches.map((match) => match.archive.uri).join(", ")}).`
+  );
+}
+async function readFileIdentity(path) {
+  try {
+    const value = await (0, import_promises2.stat)(path);
+    return `${value.dev}:${value.ino}`;
+  } catch {
+    return void 0;
+  }
+}
+
+// src/archive/write.ts
+var import_wiki_graph_core6 = require("wiki-graph-core");
+async function writeWikiGraphArchiveLocation(location, operation, options = {}) {
+  const cacheOnly = options.searchIndexWritebackPolicy === "cache";
+  const write = async (archiveFile = location.archiveFile) => await new import_wiki_graph_core6.WikiGraphArchiveFile(archiveFile).write(operation, {
+    ...cacheOnly ? {} : { derivedStateKeys: [location.archiveKey, location.archivePath] },
+    ...options.invalidateContinuationCursors === void 0 ? {} : {
+      invalidateContinuationCursors: options.invalidateContinuationCursors
+    },
+    ...options.searchIndexWritebackPolicy === void 0 ? {} : { searchIndexWritebackPolicy: options.searchIndexWritebackPolicy }
+  });
+  if (location.libraryArchiveTarget === void 0 || cacheOnly || options.refreshLibraryIndex === false) {
+    return await write();
+  }
+  return await (0, import_wiki_graph_core6.writeWikiGraphLibraryArchive)({
+    additionalDerivedStateKeys: [location.archiveKey, location.archivePath],
+    ...options.invalidateContinuationCursors === void 0 ? {} : {
+      invalidateContinuationCursors: options.invalidateContinuationCursors
+    },
+    ...options.onIndexSyncError === void 0 ? {} : { onIndexSyncError: options.onIndexSyncError },
+    operation: async (archiveFile) => await write(archiveFile),
+    target: location.libraryArchiveTarget
+  });
+}
+
+// src/worker.ts
 var import_worker2 = require("wiki-graph-core/worker");
 async function runBuildJobWorker(options) {
   installNodeWikiGraphPlatform();
@@ -1460,7 +1687,7 @@ async function runWikiGraphQueueWorker(options = {}) {
   await runBuildJobWorker({
     concurrency: config.concurrent?.job ?? DEFAULT_GENERATION_JOB_CONCURRENCY,
     executeJob: async (job, reporter, context) => {
-      await (0, import_wiki_graph_core5.withLoggingContext)(
+      await (0, import_wiki_graph_core8.withLoggingContext)(
         { logDirectory: job.log, operation: "build-job" },
         async () => await executeBuildJob(job, reporter, context)
       );
@@ -1483,7 +1710,7 @@ async function executeBuildJob(job, reporter, context) {
       let stage = await readChapterStage(job);
       if (stage === "sourced") {
         await executeStep(job, "reading-graph", execution, reporter, context, {
-          ...execution.extractionPrompt === void 0 ? {} : { extractionPrompt: execution.extractionPrompt }
+          ...execution.extractionPrompt === void 0 ? {} : { prompt: execution.extractionPrompt }
         });
         stage = await readChapterStage(job);
       }
@@ -1516,8 +1743,8 @@ async function executeBuildJob(job, reporter, context) {
       }
     }
     await executeStep(job, job.target, execution, reporter, context, {
-      ...job.target === "reading-graph" ? { extractionPrompt: execution.extractionPrompt } : {},
-      ...job.target === "knowledge-graph" ? { policyPrompt: execution.knowledgeGraphPrompt } : {}
+      ...job.target === "reading-graph" ? { prompt: execution.extractionPrompt } : {},
+      ...job.target === "knowledge-graph" ? { prompt: execution.knowledgeGraphPrompt } : {}
     });
   } finally {
     await execution.close();
@@ -1532,9 +1759,9 @@ async function executeStep(job, kind, execution, reporter, context, inputOptions
   const stepWorkspace = await resetStepWorkspace(job.workspace, kind);
   const inputFile = await stepWorkspace.createFile("input.jsonl");
   const outputWorkspace = await stepWorkspace.createDirectory("output");
-  const revision = await new import_wiki_graph_core5.WikiGraphArchiveFile(job.archive).readDocument(
+  const revision = await new import_wiki_graph_core8.WikiGraphArchiveFile(job.archive).readDocument(
     async (document) => {
-      await (0, import_wiki_graph_core5.getChapterDetails)(document, job.chapterId);
+      await (0, import_wiki_graph_core8.getChapterDetails)(document, job.chapterId);
       return await (0, import_worker3.writeChapterJobInputFile)(
         document,
         job.chapterId,
@@ -1544,7 +1771,7 @@ async function executeStep(job, kind, execution, reporter, context, inputOptions
       );
     }
   );
-  await (0, import_wiki_graph_core5.recordBuildJobInputRevision)({
+  await (0, import_wiki_graph_core8.recordBuildJobInputRevision)({
     currentRevision: revision,
     jobId: job.jobId,
     ownerId: requireRunningJobOwnerId(job)
@@ -1574,16 +1801,23 @@ async function executeStep(job, kind, execution, reporter, context, inputOptions
     total: 1,
     unit: "item"
   });
-  await new import_wiki_graph_core5.WikiGraphArchiveFile(job.archive).write(async (document) => {
-    assertJobStillRunning(await (0, import_wiki_graph_core5.getBuildJob)(job.jobId));
-    await (0, import_worker3.applyChapterJobArtifactFile)(
-      document,
-      job.chapterId,
-      kind,
-      revision,
-      result.artifactFile
-    );
-  });
+  const target = await resolveWikiGraphArchiveTarget(job.archive);
+  const resolvedLocation = await resolveWikiGraphArchiveLocation(target);
+  const location = target.kind === "standalone" ? { ...resolvedLocation, archiveFile: job.archive } : resolvedLocation;
+  await writeWikiGraphArchiveLocation(
+    location,
+    async (document) => {
+      assertJobStillRunning(await (0, import_wiki_graph_core8.getBuildJob)(job.jobId));
+      await (0, import_worker3.applyChapterJobArtifactFile)(
+        document,
+        job.chapterId,
+        kind,
+        revision,
+        result.artifactFile
+      );
+    },
+    { invalidateContinuationCursors: true }
+  );
   await reporter.updatePhase({
     done: 1,
     phase: "committing",
@@ -1591,7 +1825,7 @@ async function executeStep(job, kind, execution, reporter, context, inputOptions
     unit: "item"
   });
   await reporter.stepCompleted(kind);
-  assertJobStillRunning(await (0, import_wiki_graph_core5.getBuildJob)(job.jobId));
+  assertJobStillRunning(await (0, import_wiki_graph_core8.getBuildJob)(job.jobId));
 }
 async function openJobExecutor(job, config, signal, reporter) {
   const generation = !job.target.startsWith("index-");
@@ -1610,18 +1844,22 @@ async function openJobExecutor(job, config, signal, reporter) {
   const request = llm === void 0 ? void 0 : async (messages, index, maxRetries) => await llm.request(messages, {
     retryIndex: index,
     retryMax: maxRetries,
-    scope: import_wiki_graph_core5.WikiGraphScope.ReaderExtraction,
+    scope: import_wiki_graph_core8.WikiGraphScope.ReaderExtraction,
     signal
   });
   if (request !== void 0 && llm !== void 0) {
     request.lazy = async (operation) => await llm.request(async () => await operation(request));
   }
-  const wikimediaResolver = job.target !== "knowledge-graph" ? void 0 : await (0, import_wiki_graph_core5.openWikimediaResolver)(
+  const wikimediaResolver = job.target !== "knowledge-graph" ? void 0 : await (0, import_wiki_graph_core8.openWikimediaResolver)(
     config.wikimedia === void 0 ? {
       kind: "local",
       llmRequest: requireValue(
         request,
         "Local Wikimedia resolution requires an LLM."
+      ),
+      llmModelId: requireValue(
+        stageConfig?.llm?.model,
+        "Local Wikimedia resolution requires an LLM model."
       )
     } : {
       endpoint: config.wikimedia.endpoint,
@@ -1659,12 +1897,12 @@ async function resetStepWorkspace(workspace, kind) {
   return await workspace.createDirectory(name);
 }
 async function readChapterStage(job) {
-  return await new import_wiki_graph_core5.WikiGraphArchiveFile(job.archive).readDocument(
-    async (document) => (await (0, import_wiki_graph_core5.getChapterDetails)(document, job.chapterId)).stage
+  return await new import_wiki_graph_core8.WikiGraphArchiveFile(job.archive).readDocument(
+    async (document) => (await (0, import_wiki_graph_core8.getChapterDetails)(document, job.chapterId)).stage
   );
 }
 async function assertCurrentFtsArtifact(job) {
-  await new import_wiki_graph_core5.WikiGraphArchiveFile(job.archive).readDocument(async (document) => {
+  await new import_wiki_graph_core8.WikiGraphArchiveFile(job.archive).readDocument(async (document) => {
     const revision = await document.serials.getRevision(job.chapterId);
     const artifact = await document.indexArtifacts.get(job.chapterId, "fts");
     if (artifact?.sourceRevision !== revision) {