wiki-graph-sdk 0.8.0 → 0.8.2

package/dist/default-worker.d.ts

@@ -2,6 +2,7 @@ import { WikiGraphRuntimeConfig } from './runtime-config.js';
 import 'zod';
 
 interface WikiGraphQueueWorkerOptions {
+    readonly idleTimeoutMs?: number;
     readonly signal?: AbortSignal;
 }
 declare function runWikiGraphQueueWorker(options?: WikiGraphQueueWorkerOptions): Promise<void>;

package/dist/default-worker.js

@@ -49,12 +49,16 @@ function getWikiGraphSDKRuntimeContext() {
   return runtimeContext.getStore() ?? {
     cwd: process.cwd(),
     env: process.env,
-    envPolicy: "production"
+    envPolicy: "production",
+    providers: {}
   };
 }
 function getWikiGraphSDKStateDir() {
   return getWikiGraphSDKRuntimeContext().stateDir;
 }
+function getWikiGraphSDKProviders() {
+  return getWikiGraphSDKRuntimeContext().providers ?? {};
+}
 
 // src/node-platform.ts
 import * as asyncHooks from "async_hooks";
@@ -71,6 +75,7 @@ import * as yauzl from "yauzl";
 import * as yazl from "yazl";
 import { Environment, Loader } from "nunjucks";
 import {
+  getWikiGraphPlatform,
   getWikiGraphStorage,
   installWikiGraphPlatform,
   installWikiGraphStorage,
@@ -732,15 +737,28 @@ function resolveNodeDataDirectory() {
     directory = parent;
   }
 }
-function createNodeWikiGraphStorage(stateRoot = pathModule.join(os.homedir(), ".wikigraph")) {
+function createNodeWikiGraphStorage(stateRootOrOptions = pathModule.join(
+  os.homedir(),
+  ".wikigraph"
+)) {
+  const libraryRoot = typeof stateRootOrOptions === "string" ? stateRootOrOptions : stateRootOrOptions.libraryRoot;
+  const documentStoreRoot = typeof stateRootOrOptions === "string" ? pathModule.join(stateRootOrOptions, "documents") : stateRootOrOptions.documentStoreRoot;
   return {
-    library: new NodeDirectory(stateRoot),
-    documentStore: new NodeDirectory(pathModule.join(stateRoot, "documents"))
+    library: new NodeDirectory(libraryRoot),
+    documentStore: new NodeDirectory(documentStoreRoot)
   };
 }
-function installNodeWikiGraphPlatform(stateRoot) {
-  installWikiGraphPlatform(nodeWikiGraphPlatform);
-  installWikiGraphStorage(createNodeWikiGraphStorage(stateRoot));
+function ensureNodeWikiGraphPlatform() {
+  try {
+    getWikiGraphPlatform();
+  } catch {
+    installWikiGraphPlatform(nodeWikiGraphPlatform);
+  }
+  try {
+    getWikiGraphStorage();
+  } catch {
+    installWikiGraphStorage(createNodeWikiGraphStorage());
+  }
 }
 async function withNodeWikiGraphStorage(stateRoot, operation) {
   return await withWikiGraphStorage(
@@ -1076,8 +1094,8 @@ var inlineLLMConfigSchema = z.object({
 });
 async function loadWikiGraphRuntimeConfig(options) {
   const [embedding, localLLM, concurrent, wikimedia, wikispine] = await Promise.all([
-    readLocalConfigSection("embeddings"),
-    readLocalConfigSection("llm"),
+    options?.skipEmbeddingConfig === true ? Promise.resolve({}) : readLocalConfigSection("embeddings"),
+    options?.skipLLMConfig === true ? Promise.resolve({}) : readLocalConfigSection("llm"),
     readLocalConfigSection("concurrent"),
     readLocalConfigSection("wikimedia"),
     readLocalConfigSection("wikispine")
@@ -1357,7 +1375,10 @@ function createOpenAICompatibleName2(baseURL) {
 
 // src/stage.ts
 function createStageLLM(config, options) {
-  const llmOptions = buildWikiGraphLLMOptions(config);
+  const llmOptions = options?.provider === void 0 ? buildWikiGraphLLMOptions(config) : {
+    ...config.concurrent?.request === void 0 ? {} : { concurrent: config.concurrent.request },
+    streamProvider: options.provider
+  };
   return new LLM({
     sampling: createDefaultWikiGraphSampling({
       ...llmOptions.temperature === void 0 ? {} : { temperature: llmOptions.temperature },
@@ -1371,10 +1392,13 @@ function createStageLLM(config, options) {
   });
 }
 async function loadRequiredStageConfig(options) {
+  const injected = options.llmJSON === void 0 ? getWikiGraphSDKProviders().llm : void 0;
   const config = await loadWikiGraphRuntimeConfig({
-    ...options.llmJSON === void 0 ? {} : { llmJSON: options.llmJSON }
+    ...options.llmJSON === void 0 ? {} : { llmJSON: options.llmJSON },
+    ...options.llmJSON === void 0 && injected !== void 0 ? { skipLLMConfig: true } : {},
+    ...getWikiGraphSDKProviders().embedding === void 0 ? {} : { skipEmbeddingConfig: true }
   });
-  if (config.llm?.provider === void 0 || config.llm.model === void 0) {
+  if (injected === void 0 && (config.llm?.provider === void 0 || config.llm.model === void 0)) {
     throw new Error(
       "Missing LLM configuration. Set --llm for one run, or configure `wikg://local/config/llm` with provider and model."
     );
@@ -1456,6 +1480,34 @@ import {
   WikiGraphArchiveFile as WikiGraphArchiveFile2
 } from "wiki-graph-core";
 
+// src/host.ts
+import {
+  installWikiGraphPlatform as installWikiGraphPlatform2,
+  withWikiGraphStorage as withWikiGraphStorage2
+} from "wiki-graph-core/platform";
+function prepareWikiGraphSDKHost(host) {
+  if (host?.platform === void 0) {
+    ensureNodeWikiGraphPlatform();
+    return;
+  }
+  installWikiGraphPlatform2(host.platform);
+}
+function assertWikiGraphSDKHostOptions(options) {
+  if (options.host?.storage !== void 0 && options.stateDir !== void 0) {
+    throw new TypeError(
+      "Wiki Graph SDK host.storage and stateDir cannot be used together."
+    );
+  }
+}
+async function withWikiGraphSDKHost(options, operation) {
+  assertWikiGraphSDKHostOptions(options);
+  prepareWikiGraphSDKHost(options.host);
+  if (options.host?.storage !== void 0) {
+    return await withWikiGraphStorage2(options.host.storage, operation);
+  }
+  return await withNodeWikiGraphStorage(options.stateDir, operation);
+}
+
 // src/archive/target.ts
 import { realpath, stat as stat2 } from "fs/promises";
 import { basename as basename2, dirname as dirname2, isAbsolute as isAbsolute3, join as join3, relative, resolve as resolve3 } from "path";
@@ -1687,17 +1739,23 @@ import {
   writeChapterJobInputFile as writeChapterJobInputFile2
 } from "wiki-graph-core/worker";
 async function runBuildJobWorker(options) {
-  installNodeWikiGraphPlatform();
-  const { stateDir, ...coreOptions } = options;
-  await withNodeWikiGraphStorage(
-    stateDir,
+  const { host, stateDir, ...coreOptions } = options;
+  await withWikiGraphSDKHost(
+    {
+      ...host === void 0 ? {} : { host },
+      ...stateDir === void 0 ? {} : { stateDir }
+    },
     async () => await runCoreBuildJobWorker(coreOptions)
   );
 }
 
 // src/default-worker.ts
 async function runWikiGraphQueueWorker(options = {}) {
-  const config = await loadWikiGraphRuntimeConfig();
+  const providers = getWikiGraphSDKProviders();
+  const config = await loadWikiGraphRuntimeConfig({
+    ...providers.embedding === void 0 ? {} : { skipEmbeddingConfig: true },
+    ...providers.llm === void 0 ? {} : { skipLLMConfig: true }
+  });
   await runBuildJobWorker({
     concurrency: config.concurrent?.job ?? DEFAULT_GENERATION_JOB_CONCURRENCY,
     executeJob: async (job, reporter, context) => {
@@ -1706,12 +1764,16 @@ async function runWikiGraphQueueWorker(options = {}) {
         async () => await executeBuildJob(job, reporter, context)
       );
     },
+    ...options.idleTimeoutMs === void 0 ? {} : { idleTimeoutMs: options.idleTimeoutMs },
     ...options.signal === void 0 ? {} : { signal: options.signal }
   });
 }
 async function executeBuildJob(job, reporter, context) {
+  const providers = getWikiGraphSDKProviders();
   const config = await loadWikiGraphRuntimeConfig({
-    ...job.llmJSON === void 0 ? {} : { llmJSON: job.llmJSON }
+    ...job.llmJSON === void 0 ? {} : { llmJSON: job.llmJSON },
+    ...providers.embedding === void 0 ? {} : { skipEmbeddingConfig: true },
+    ...job.llmJSON === void 0 && providers.llm !== void 0 ? { skipLLMConfig: true } : {}
   });
   const execution = await openJobExecutor(
     job,
@@ -1846,12 +1908,17 @@ async function openJobExecutor(job, config, signal, reporter) {
   const stageConfig = generation ? await loadRequiredStageConfig({
     ...job.llmJSON === void 0 ? {} : { llmJSON: job.llmJSON }
   }) : void 0;
+  const injectedLLM = job.llmJSON === void 0 ? getWikiGraphSDKProviders().llm : void 0;
   const llm = stageConfig === void 0 ? void 0 : createStageLLM(stageConfig, {
     cacheDirectory: job.cache,
     logDirectory: job.log,
     onTokenUsage: async (usage) => {
       await reporter.addTokenUsage(usage);
-    }
+    },
+    onStreamProgress: async ({ outputCharacters }) => {
+      await reporter.addOutputCharacters(outputCharacters);
+    },
+    ...injectedLLM === void 0 ? {} : { provider: injectedLLM }
   });
   const promptSource = job.prompt ?? stageConfig?.prompt;
   const wikispine = job.target === "knowledge-graph" ? requireKnowledgeGraphWikispineConfig(config) : void 0;
@@ -1872,7 +1939,7 @@ async function openJobExecutor(job, config, signal, reporter) {
         "Local Wikimedia resolution requires an LLM."
       ),
       llmModelId: requireValue(
-        stageConfig?.llm?.model,
+        stageConfig?.llm?.model ?? injectedLLM?.model,
         "Local Wikimedia resolution requires an LLM model."
       )
     } : {
@@ -1881,7 +1948,7 @@ async function openJobExecutor(job, config, signal, reporter) {
       token: config.wikimedia.token
     }
   );
-  const embeddingProvider = job.target === "index-embedding-source" || job.target === "index-embedding-summary" ? buildSearchIndexEmbeddingProvider(requireEmbeddingConfig(config)) : void 0;
+  const embeddingProvider = job.target === "index-embedding-source" || job.target === "index-embedding-summary" ? getWikiGraphSDKProviders().embedding ?? buildSearchIndexEmbeddingProvider(requireEmbeddingConfig(config)) : void 0;
   return {
     async close() {
       await wikimediaResolver?.close();