wiki-graph-sdk 0.8.0 → 0.8.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +46 -0
- package/dist/archives.cjs +98 -17
- package/dist/archives.cjs.map +1 -1
- package/dist/archives.d.ts +10 -2
- package/dist/archives.js +100 -17
- package/dist/archives.js.map +1 -1
- package/dist/conversions.cjs +23 -5
- package/dist/conversions.cjs.map +1 -1
- package/dist/conversions.d.ts +6 -1
- package/dist/conversions.js +24 -5
- package/dist/conversions.js.map +1 -1
- package/dist/default-worker.cjs +84 -21
- package/dist/default-worker.cjs.map +1 -1
- package/dist/default-worker.d.ts +1 -0
- package/dist/default-worker.js +88 -21
- package/dist/default-worker.js.map +1 -1
- package/dist/embedding.cjs +2 -1
- package/dist/embedding.cjs.map +1 -1
- package/dist/embedding.d.ts +4 -4
- package/dist/embedding.js +3 -1
- package/dist/embedding.js.map +1 -1
- package/dist/gc.cjs +52 -10
- package/dist/gc.cjs.map +1 -1
- package/dist/gc.d.ts +3 -0
- package/dist/gc.js +56 -10
- package/dist/gc.js.map +1 -1
- package/dist/host-pQiVAKEY.d.ts +10 -0
- package/dist/index.cjs +422 -57
- package/dist/index.cjs.map +1 -1
- package/dist/index.d.ts +33 -12
- package/dist/index.js +427 -59
- package/dist/index.js.map +1 -1
- package/dist/{jobs-xIm_kshM.d.ts → jobs-CRK-DZ40.d.ts} +5 -0
- package/dist/local-config.cjs +2 -1
- package/dist/local-config.cjs.map +1 -1
- package/dist/local-config.js +3 -1
- package/dist/local-config.js.map +1 -1
- package/dist/maintenance.cjs +2 -1
- package/dist/maintenance.cjs.map +1 -1
- package/dist/maintenance.d.ts +4 -1
- package/dist/maintenance.js +3 -1
- package/dist/maintenance.js.map +1 -1
- package/dist/node-platform.cjs +18 -4
- package/dist/node-platform.cjs.map +1 -1
- package/dist/node-platform.d.ts +12 -3
- package/dist/node-platform.js +18 -4
- package/dist/node-platform.js.map +1 -1
- package/dist/runtime-config.cjs +4 -3
- package/dist/runtime-config.cjs.map +1 -1
- package/dist/runtime-config.d.ts +4 -1
- package/dist/runtime-config.js +5 -3
- package/dist/runtime-config.js.map +1 -1
- package/dist/stage-BnWI3Rqt.d.ts +33 -0
- package/dist/stage.cjs +22 -12
- package/dist/stage.cjs.map +1 -1
- package/dist/stage.d.ts +2 -14
- package/dist/stage.js +23 -12
- package/dist/stage.js.map +1 -1
- package/dist/worker.cjs +83 -43
- package/dist/worker.cjs.map +1 -1
- package/dist/worker.d.ts +7 -2
- package/dist/worker.js +87 -43
- package/dist/worker.js.map +1 -1
- package/package.json +2 -2
package/dist/conversions.js
CHANGED
|
@@ -98,6 +98,7 @@ import * as yauzl from "yauzl";
|
|
|
98
98
|
import * as yazl from "yazl";
|
|
99
99
|
import { Environment, Loader } from "nunjucks";
|
|
100
100
|
import {
|
|
101
|
+
getWikiGraphPlatform,
|
|
101
102
|
getWikiGraphStorage,
|
|
102
103
|
installWikiGraphPlatform,
|
|
103
104
|
installWikiGraphStorage,
|
|
@@ -789,12 +790,16 @@ function getWikiGraphSDKRuntimeContext() {
|
|
|
789
790
|
return runtimeContext.getStore() ?? {
|
|
790
791
|
cwd: process.cwd(),
|
|
791
792
|
env: process.env,
|
|
792
|
-
envPolicy: "production"
|
|
793
|
+
envPolicy: "production",
|
|
794
|
+
providers: {}
|
|
793
795
|
};
|
|
794
796
|
}
|
|
795
797
|
function getWikiGraphSDKStateDir() {
|
|
796
798
|
return getWikiGraphSDKRuntimeContext().stateDir;
|
|
797
799
|
}
|
|
800
|
+
function getWikiGraphSDKProviders() {
|
|
801
|
+
return getWikiGraphSDKRuntimeContext().providers ?? {};
|
|
802
|
+
}
|
|
798
803
|
|
|
799
804
|
// src/local-config.ts
|
|
800
805
|
import { join as join2 } from "path";
|
|
@@ -1012,8 +1017,8 @@ var inlineLLMConfigSchema = z.object({
|
|
|
1012
1017
|
});
|
|
1013
1018
|
async function loadWikiGraphRuntimeConfig(options) {
|
|
1014
1019
|
const [embedding, localLLM, concurrent, wikimedia, wikispine] = await Promise.all([
|
|
1015
|
-
readLocalConfigSection("embeddings"),
|
|
1016
|
-
readLocalConfigSection("llm"),
|
|
1020
|
+
options?.skipEmbeddingConfig === true ? Promise.resolve({}) : readLocalConfigSection("embeddings"),
|
|
1021
|
+
options?.skipLLMConfig === true ? Promise.resolve({}) : readLocalConfigSection("llm"),
|
|
1017
1022
|
readLocalConfigSection("concurrent"),
|
|
1018
1023
|
readLocalConfigSection("wikimedia"),
|
|
1019
1024
|
readLocalConfigSection("wikispine")
|
|
@@ -1209,6 +1214,14 @@ function formatError2(error) {
|
|
|
1209
1214
|
return error instanceof Error ? error.message : String(error);
|
|
1210
1215
|
}
|
|
1211
1216
|
|
|
1217
|
+
// src/providers.ts
|
|
1218
|
+
function resolveWikiGraphLLMJSON(options) {
|
|
1219
|
+
if (options.llm !== void 0 && options.llmJSON !== void 0) {
|
|
1220
|
+
throw new Error("Pass either llm or llmJSON, not both.");
|
|
1221
|
+
}
|
|
1222
|
+
return options.llm === void 0 ? options.llmJSON : JSON.stringify(options.llm);
|
|
1223
|
+
}
|
|
1224
|
+
|
|
1212
1225
|
// src/runtime-path.ts
|
|
1213
1226
|
import { homedir as homedir3 } from "os";
|
|
1214
1227
|
import { isAbsolute as isAbsolute2, resolve as resolve2 } from "path";
|
|
@@ -1355,12 +1368,18 @@ var WikiGraphConversionManager = class {
|
|
|
1355
1368
|
}
|
|
1356
1369
|
const requiresDigest = options.input.format !== "wikg";
|
|
1357
1370
|
const requiresLLM = requiresDigest && targetStage !== "planned" && targetStage !== "sourced";
|
|
1371
|
+
const llmJSON = resolveWikiGraphLLMJSON(options);
|
|
1372
|
+
const injectedLLM = llmJSON === void 0 ? getWikiGraphSDKProviders().llm : void 0;
|
|
1358
1373
|
const config = await loadWikiGraphRuntimeConfig({
|
|
1359
|
-
...
|
|
1374
|
+
...llmJSON === void 0 ? {} : { llmJSON },
|
|
1375
|
+
...injectedLLM === void 0 ? {} : { skipLLMConfig: true },
|
|
1376
|
+
...getWikiGraphSDKProviders().embedding === void 0 ? {} : { skipEmbeddingConfig: true }
|
|
1360
1377
|
});
|
|
1361
1378
|
const app = new WikiGraph({
|
|
1362
1379
|
...options.verbose === true ? { verbose: true } : {},
|
|
1363
|
-
...requiresLLM ? {
|
|
1380
|
+
...requiresLLM ? {
|
|
1381
|
+
llm: injectedLLM === void 0 ? buildWikiGraphLLMOptions(config) : { streamProvider: injectedLLM }
|
|
1382
|
+
} : {}
|
|
1364
1383
|
});
|
|
1365
1384
|
const documentDirectory = await prepareDigestDirectory(
|
|
1366
1385
|
options.digestDirectory,
|