wiki-graph-sdk 0.8.0 → 0.8.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +46 -0
- package/dist/archives.cjs +98 -17
- package/dist/archives.cjs.map +1 -1
- package/dist/archives.d.ts +10 -2
- package/dist/archives.js +100 -17
- package/dist/archives.js.map +1 -1
- package/dist/conversions.cjs +23 -5
- package/dist/conversions.cjs.map +1 -1
- package/dist/conversions.d.ts +6 -1
- package/dist/conversions.js +24 -5
- package/dist/conversions.js.map +1 -1
- package/dist/default-worker.cjs +84 -21
- package/dist/default-worker.cjs.map +1 -1
- package/dist/default-worker.d.ts +1 -0
- package/dist/default-worker.js +88 -21
- package/dist/default-worker.js.map +1 -1
- package/dist/embedding.cjs +2 -1
- package/dist/embedding.cjs.map +1 -1
- package/dist/embedding.d.ts +4 -4
- package/dist/embedding.js +3 -1
- package/dist/embedding.js.map +1 -1
- package/dist/gc.cjs +52 -10
- package/dist/gc.cjs.map +1 -1
- package/dist/gc.d.ts +3 -0
- package/dist/gc.js +56 -10
- package/dist/gc.js.map +1 -1
- package/dist/host-pQiVAKEY.d.ts +10 -0
- package/dist/index.cjs +422 -57
- package/dist/index.cjs.map +1 -1
- package/dist/index.d.ts +33 -12
- package/dist/index.js +427 -59
- package/dist/index.js.map +1 -1
- package/dist/{jobs-xIm_kshM.d.ts → jobs-CRK-DZ40.d.ts} +5 -0
- package/dist/local-config.cjs +2 -1
- package/dist/local-config.cjs.map +1 -1
- package/dist/local-config.js +3 -1
- package/dist/local-config.js.map +1 -1
- package/dist/maintenance.cjs +2 -1
- package/dist/maintenance.cjs.map +1 -1
- package/dist/maintenance.d.ts +4 -1
- package/dist/maintenance.js +3 -1
- package/dist/maintenance.js.map +1 -1
- package/dist/node-platform.cjs +18 -4
- package/dist/node-platform.cjs.map +1 -1
- package/dist/node-platform.d.ts +12 -3
- package/dist/node-platform.js +18 -4
- package/dist/node-platform.js.map +1 -1
- package/dist/runtime-config.cjs +4 -3
- package/dist/runtime-config.cjs.map +1 -1
- package/dist/runtime-config.d.ts +4 -1
- package/dist/runtime-config.js +5 -3
- package/dist/runtime-config.js.map +1 -1
- package/dist/stage-BnWI3Rqt.d.ts +33 -0
- package/dist/stage.cjs +22 -12
- package/dist/stage.cjs.map +1 -1
- package/dist/stage.d.ts +2 -14
- package/dist/stage.js +23 -12
- package/dist/stage.js.map +1 -1
- package/dist/worker.cjs +83 -43
- package/dist/worker.cjs.map +1 -1
- package/dist/worker.d.ts +7 -2
- package/dist/worker.js +87 -43
- package/dist/worker.js.map +1 -1
- package/package.json +2 -2
package/dist/index.cjs
CHANGED
|
@@ -61,6 +61,7 @@ __export(src_exports, {
|
|
|
61
61
|
createEntryRuntimeContext: () => createEntryRuntimeContext,
|
|
62
62
|
createGenerationPerformanceHints: () => createGenerationPerformanceHints,
|
|
63
63
|
createNodeTemplateEnvironment: () => createNodeTemplateEnvironment,
|
|
64
|
+
createNodeWikiGraphPlatform: () => createNodeWikiGraphPlatform,
|
|
64
65
|
createNodeWikiGraphStorage: () => createNodeWikiGraphStorage,
|
|
65
66
|
createStageLLM: () => createStageLLM,
|
|
66
67
|
createWikiGraphSDK: () => createWikiGraphSDK,
|
|
@@ -75,15 +76,20 @@ __export(src_exports, {
|
|
|
75
76
|
formatLocatedChapterSourceCollectionUri: () => import_wiki_graph_core19.formatLocatedChapterSourceCollectionUri,
|
|
76
77
|
formatLocatedChapterUri: () => import_wiki_graph_core19.formatLocatedChapterUri,
|
|
77
78
|
formatLocatedWikiGraphUri: () => import_wiki_graph_core19.formatLocatedWikiGraphUri,
|
|
79
|
+
formatSourceArtifactUri: () => import_wiki_graph_core19.formatSourceArtifactUri,
|
|
80
|
+
formatSourceLocatorFragment: () => import_wiki_graph_core19.formatSourceLocatorFragment,
|
|
78
81
|
formatWikiGraphCommandUri: () => import_wiki_graph_core19.formatWikiGraphCommandUri,
|
|
79
82
|
formatWikiGraphLibraryUri: () => import_wiki_graph_core19.formatWikiGraphLibraryUri,
|
|
80
83
|
getNodeResourcePath: () => getNodeResourcePath,
|
|
84
|
+
getWikiGraphSDKProviders: () => getWikiGraphSDKProviders,
|
|
81
85
|
getWikiGraphSDKRuntimeContext: () => getWikiGraphSDKRuntimeContext,
|
|
82
86
|
getWikiGraphSDKStateDir: () => getWikiGraphSDKStateDir,
|
|
83
87
|
inspectWikiGraphArchive: () => inspectWikiGraphArchive,
|
|
84
88
|
installNodeWikiGraphPlatform: () => installNodeWikiGraphPlatform,
|
|
85
89
|
isSourceLocatorScopeUri: () => import_wiki_graph_core19.isSourceLocatorScopeUri,
|
|
90
|
+
isWikiGraphJobUri: () => import_wiki_graph_core19.isWikiGraphJobUri,
|
|
86
91
|
isWikiGraphLibraryUri: () => import_wiki_graph_core19.isWikiGraphLibraryUri,
|
|
92
|
+
isWikiGraphUri: () => import_wiki_graph_core19.isWikiGraphUri,
|
|
87
93
|
loadRequiredStageConfig: () => loadRequiredStageConfig,
|
|
88
94
|
loadWikiGraphRuntimeConfig: () => loadWikiGraphRuntimeConfig,
|
|
89
95
|
maskLocalConfigSection: () => maskLocalConfigSection,
|
|
@@ -95,6 +101,7 @@ __export(src_exports, {
|
|
|
95
101
|
parseChapterUriPath: () => import_wiki_graph_core19.parseChapterUriPath,
|
|
96
102
|
parseLocalConfigSection: () => parseLocalConfigSection,
|
|
97
103
|
parseLocatedWikiGraphUri: () => import_wiki_graph_core19.parseLocatedWikiGraphUri,
|
|
104
|
+
parseSourceLocatorFragment: () => import_wiki_graph_core19.parseSourceLocatorFragment,
|
|
98
105
|
parseSourceTextJsonl: () => import_wiki_graph_core19.parseSourceTextJsonl,
|
|
99
106
|
parseWikiGraphLibraryUri: () => import_wiki_graph_core19.parseWikiGraphLibraryUri,
|
|
100
107
|
parseWikiGraphUriSyntax: () => import_wiki_graph_core19.parseWikiGraphUriSyntax,
|
|
@@ -103,10 +110,14 @@ __export(src_exports, {
|
|
|
103
110
|
readLocalConfigSection: () => readLocalConfigSection,
|
|
104
111
|
readWikiGraphEmbeddingConfig: () => readWikiGraphEmbeddingConfig,
|
|
105
112
|
replaceLocalConfigSection: () => replaceLocalConfigSection,
|
|
113
|
+
requireArchiveUri: () => import_wiki_graph_core19.requireArchiveUri,
|
|
106
114
|
requireKnowledgeGraphWikispineConfig: () => requireKnowledgeGraphWikispineConfig,
|
|
115
|
+
requireLocatedObjectOrArchiveUri: () => import_wiki_graph_core19.requireLocatedObjectOrArchiveUri,
|
|
116
|
+
requireLocatedObjectUri: () => import_wiki_graph_core19.requireLocatedObjectUri,
|
|
107
117
|
resolveExtractionPrompt: () => import_wiki_graph_core10.resolveExtractionPrompt,
|
|
108
118
|
resolveHostedProviderEndpoint: () => resolveHostedProviderEndpoint,
|
|
109
119
|
resolveKnowledgeGraphRecallPrompt: () => import_wiki_graph_core10.resolveKnowledgeGraphRecallPrompt,
|
|
120
|
+
resolveWikiGraphLLMJSON: () => resolveWikiGraphLLMJSON,
|
|
110
121
|
resolveWikispineConfig: () => resolveWikispineConfig,
|
|
111
122
|
runWikiGraphQueueWorker: () => runWikiGraphQueueWorker,
|
|
112
123
|
validateLocalConfigSection: () => validateLocalConfigSection,
|
|
@@ -834,6 +845,9 @@ var nodeWikiGraphPlatform = {
|
|
|
834
845
|
write: writeNodeZip
|
|
835
846
|
}
|
|
836
847
|
};
|
|
848
|
+
function createNodeWikiGraphPlatform(options = {}) {
|
|
849
|
+
return options.lifecycle === void 0 ? nodeWikiGraphPlatform : { ...nodeWikiGraphPlatform, lifecycle: options.lifecycle };
|
|
850
|
+
}
|
|
837
851
|
var NodeTemplateLoader = class extends import_nunjucks.Loader {
|
|
838
852
|
#dataDirectory;
|
|
839
853
|
constructor(dataDirectory) {
|
|
@@ -879,10 +893,15 @@ function resolveNodeDataDirectory() {
|
|
|
879
893
|
directory = parent;
|
|
880
894
|
}
|
|
881
895
|
}
|
|
882
|
-
function createNodeWikiGraphStorage(
|
|
896
|
+
function createNodeWikiGraphStorage(stateRootOrOptions = pathModule.join(
|
|
897
|
+
os.homedir(),
|
|
898
|
+
".wikigraph"
|
|
899
|
+
)) {
|
|
900
|
+
const libraryRoot = typeof stateRootOrOptions === "string" ? stateRootOrOptions : stateRootOrOptions.libraryRoot;
|
|
901
|
+
const documentStoreRoot = typeof stateRootOrOptions === "string" ? pathModule.join(stateRootOrOptions, "documents") : stateRootOrOptions.documentStoreRoot;
|
|
883
902
|
return {
|
|
884
|
-
library: new NodeDirectory(
|
|
885
|
-
documentStore: new NodeDirectory(
|
|
903
|
+
library: new NodeDirectory(libraryRoot),
|
|
904
|
+
documentStore: new NodeDirectory(documentStoreRoot)
|
|
886
905
|
};
|
|
887
906
|
}
|
|
888
907
|
function installNodeWikiGraphPlatform(stateRoot) {
|
|
@@ -890,7 +909,11 @@ function installNodeWikiGraphPlatform(stateRoot) {
|
|
|
890
909
|
(0, import_platform.installWikiGraphStorage)(createNodeWikiGraphStorage(stateRoot));
|
|
891
910
|
}
|
|
892
911
|
function ensureNodeWikiGraphPlatform() {
|
|
893
|
-
|
|
912
|
+
try {
|
|
913
|
+
(0, import_platform.getWikiGraphPlatform)();
|
|
914
|
+
} catch {
|
|
915
|
+
(0, import_platform.installWikiGraphPlatform)(nodeWikiGraphPlatform);
|
|
916
|
+
}
|
|
894
917
|
try {
|
|
895
918
|
(0, import_platform.getWikiGraphStorage)();
|
|
896
919
|
} catch {
|
|
@@ -936,12 +959,16 @@ function getWikiGraphSDKRuntimeContext() {
|
|
|
936
959
|
return runtimeContext.getStore() ?? {
|
|
937
960
|
cwd: process.cwd(),
|
|
938
961
|
env: process.env,
|
|
939
|
-
envPolicy: "production"
|
|
962
|
+
envPolicy: "production",
|
|
963
|
+
providers: {}
|
|
940
964
|
};
|
|
941
965
|
}
|
|
942
966
|
function getWikiGraphSDKStateDir() {
|
|
943
967
|
return getWikiGraphSDKRuntimeContext().stateDir;
|
|
944
968
|
}
|
|
969
|
+
function getWikiGraphSDKProviders() {
|
|
970
|
+
return getWikiGraphSDKRuntimeContext().providers ?? {};
|
|
971
|
+
}
|
|
945
972
|
|
|
946
973
|
// src/local-config.ts
|
|
947
974
|
var import_path = require("path");
|
|
@@ -1242,8 +1269,8 @@ var inlineLLMConfigSchema = import_zod.z.object({
|
|
|
1242
1269
|
});
|
|
1243
1270
|
async function loadWikiGraphRuntimeConfig(options) {
|
|
1244
1271
|
const [embedding, localLLM, concurrent, wikimedia, wikispine] = await Promise.all([
|
|
1245
|
-
readLocalConfigSection("embeddings"),
|
|
1246
|
-
readLocalConfigSection("llm"),
|
|
1272
|
+
options?.skipEmbeddingConfig === true ? Promise.resolve({}) : readLocalConfigSection("embeddings"),
|
|
1273
|
+
options?.skipLLMConfig === true ? Promise.resolve({}) : readLocalConfigSection("llm"),
|
|
1247
1274
|
readLocalConfigSection("concurrent"),
|
|
1248
1275
|
readLocalConfigSection("wikimedia"),
|
|
1249
1276
|
readLocalConfigSection("wikispine")
|
|
@@ -1439,6 +1466,14 @@ function formatError2(error) {
|
|
|
1439
1466
|
return error instanceof Error ? error.message : String(error);
|
|
1440
1467
|
}
|
|
1441
1468
|
|
|
1469
|
+
// src/providers.ts
|
|
1470
|
+
function resolveWikiGraphLLMJSON(options) {
|
|
1471
|
+
if (options.llm !== void 0 && options.llmJSON !== void 0) {
|
|
1472
|
+
throw new Error("Pass either llm or llmJSON, not both.");
|
|
1473
|
+
}
|
|
1474
|
+
return options.llm === void 0 ? options.llmJSON : JSON.stringify(options.llm);
|
|
1475
|
+
}
|
|
1476
|
+
|
|
1442
1477
|
// src/runtime-path.ts
|
|
1443
1478
|
var import_os2 = require("os");
|
|
1444
1479
|
var import_path2 = require("path");
|
|
@@ -1663,12 +1698,18 @@ var WikiGraphConversionManager = class {
|
|
|
1663
1698
|
}
|
|
1664
1699
|
const requiresDigest = options.input.format !== "wikg";
|
|
1665
1700
|
const requiresLLM = requiresDigest && targetStage !== "planned" && targetStage !== "sourced";
|
|
1701
|
+
const llmJSON = resolveWikiGraphLLMJSON(options);
|
|
1702
|
+
const injectedLLM = llmJSON === void 0 ? getWikiGraphSDKProviders().llm : void 0;
|
|
1666
1703
|
const config = await loadWikiGraphRuntimeConfig({
|
|
1667
|
-
...
|
|
1704
|
+
...llmJSON === void 0 ? {} : { llmJSON },
|
|
1705
|
+
...injectedLLM === void 0 ? {} : { skipLLMConfig: true },
|
|
1706
|
+
...getWikiGraphSDKProviders().embedding === void 0 ? {} : { skipEmbeddingConfig: true }
|
|
1668
1707
|
});
|
|
1669
1708
|
const app = new import_wiki_graph_core3.WikiGraph({
|
|
1670
1709
|
...options.verbose === true ? { verbose: true } : {},
|
|
1671
|
-
...requiresLLM ? {
|
|
1710
|
+
...requiresLLM ? {
|
|
1711
|
+
llm: injectedLLM === void 0 ? buildWikiGraphLLMOptions(config) : { streamProvider: injectedLLM }
|
|
1712
|
+
} : {}
|
|
1672
1713
|
});
|
|
1673
1714
|
const documentDirectory = await prepareDigestDirectory(
|
|
1674
1715
|
options.digestDirectory,
|
|
@@ -1981,6 +2022,8 @@ function createOpenAICompatibleName2(baseURL) {
|
|
|
1981
2022
|
|
|
1982
2023
|
// src/query-runtime.ts
|
|
1983
2024
|
async function createConfiguredEmbeddingProvider() {
|
|
2025
|
+
const injected = getWikiGraphSDKProviders().embedding;
|
|
2026
|
+
if (injected !== void 0) return injected;
|
|
1984
2027
|
const config = await readWikiGraphEmbeddingConfig();
|
|
1985
2028
|
if (config.provider === void 0 || config.model === void 0 || config.provider === "openai-compatible" && config.baseURL === void 0 || config.provider === "openai" && config.baseURL !== void 0) {
|
|
1986
2029
|
return void 0;
|
|
@@ -2226,13 +2269,15 @@ var WikiGraphArchiveHandle = class {
|
|
|
2226
2269
|
);
|
|
2227
2270
|
}
|
|
2228
2271
|
async search(query, options = {}) {
|
|
2229
|
-
const embeddingProvider = await this.#
|
|
2230
|
-
|
|
2272
|
+
const embeddingProvider = await this.#resolveQueryEmbeddingProvider(
|
|
2273
|
+
options.queryMode,
|
|
2231
2274
|
options.signal
|
|
2232
2275
|
);
|
|
2233
2276
|
return await this.#writeDocument(
|
|
2234
2277
|
async (document) => {
|
|
2235
|
-
const chapters = await this.#resolveQueryChapters(document, options
|
|
2278
|
+
const chapters = await this.#resolveQueryChapters(document, options, {
|
|
2279
|
+
embeddingProvider
|
|
2280
|
+
});
|
|
2236
2281
|
await ensureArchiveSearchIndex(document, {
|
|
2237
2282
|
...chapters === void 0 ? {} : { chapters },
|
|
2238
2283
|
...embeddingProvider === void 0 ? {} : { embeddingProvider }
|
|
@@ -2340,6 +2385,13 @@ var WikiGraphArchiveHandle = class {
|
|
|
2340
2385
|
);
|
|
2341
2386
|
return status;
|
|
2342
2387
|
}
|
|
2388
|
+
/** Clear cached search sessions without changing the archive index. */
|
|
2389
|
+
async clearSearchSessions(options = {}) {
|
|
2390
|
+
await this.#runtime.run(
|
|
2391
|
+
async () => await (0, import_wiki_graph_core7.deleteArchiveSearchSessions)(this.path),
|
|
2392
|
+
options.signal
|
|
2393
|
+
);
|
|
2394
|
+
}
|
|
2343
2395
|
async listChapters(options = {}) {
|
|
2344
2396
|
return await this.#readDocument(
|
|
2345
2397
|
async (document) => await (0, import_wiki_graph_core7.listChapters)(document),
|
|
@@ -2530,6 +2582,9 @@ var WikiGraphArchiveHandle = class {
|
|
|
2530
2582
|
}
|
|
2531
2583
|
async related(objectUri = this.objectUri, options = {}) {
|
|
2532
2584
|
if (options.query === void 0) {
|
|
2585
|
+
if (options.queryMode !== void 0) {
|
|
2586
|
+
throw new Error("`queryMode` requires `query`.");
|
|
2587
|
+
}
|
|
2533
2588
|
return await this.#readDocument(async (document) => {
|
|
2534
2589
|
const chapters = await this.#resolveScope(document, {});
|
|
2535
2590
|
return await (0, import_wiki_graph_core7.listRelatedArchiveObjects)(document, objectUri, {
|
|
@@ -2538,19 +2593,22 @@ var WikiGraphArchiveHandle = class {
|
|
|
2538
2593
|
});
|
|
2539
2594
|
}, options);
|
|
2540
2595
|
}
|
|
2541
|
-
const embeddingProvider = await this.#
|
|
2542
|
-
|
|
2596
|
+
const embeddingProvider = await this.#resolveQueryEmbeddingProvider(
|
|
2597
|
+
options.queryMode,
|
|
2543
2598
|
options.signal
|
|
2544
2599
|
);
|
|
2545
2600
|
return await this.#writeDocument(
|
|
2546
2601
|
async (document) => {
|
|
2547
|
-
const chapters = await this.#resolveQueryChapters(document, options
|
|
2602
|
+
const chapters = await this.#resolveQueryChapters(document, options, {
|
|
2603
|
+
embeddingProvider
|
|
2604
|
+
});
|
|
2548
2605
|
await ensureArchiveSearchIndex(document, {
|
|
2549
2606
|
...chapters === void 0 ? {} : { chapters },
|
|
2550
2607
|
...embeddingProvider === void 0 ? {} : { embeddingProvider }
|
|
2551
2608
|
});
|
|
2552
2609
|
return await (0, import_wiki_graph_core7.listRelatedArchiveObjects)(document, objectUri, {
|
|
2553
2610
|
...withoutOperation(options),
|
|
2611
|
+
...embeddingProvider === void 0 ? {} : { embeddingProvider },
|
|
2554
2612
|
...chapters === void 0 ? {} : { chapters }
|
|
2555
2613
|
});
|
|
2556
2614
|
},
|
|
@@ -2562,6 +2620,9 @@ var WikiGraphArchiveHandle = class {
|
|
|
2562
2620
|
}
|
|
2563
2621
|
async evidence(objectUri = this.objectUri, options = {}) {
|
|
2564
2622
|
if (options.query === void 0) {
|
|
2623
|
+
if (options.queryMode !== void 0) {
|
|
2624
|
+
throw new Error("`queryMode` requires `query`.");
|
|
2625
|
+
}
|
|
2565
2626
|
return await this.#readDocument(async (document) => {
|
|
2566
2627
|
const chapters = await this.#resolveScope(document, {});
|
|
2567
2628
|
return await (0, import_wiki_graph_core7.listArchiveEvidence)(document, objectUri, {
|
|
@@ -2570,19 +2631,22 @@ var WikiGraphArchiveHandle = class {
|
|
|
2570
2631
|
});
|
|
2571
2632
|
}, options);
|
|
2572
2633
|
}
|
|
2573
|
-
const embeddingProvider = await this.#
|
|
2574
|
-
|
|
2634
|
+
const embeddingProvider = await this.#resolveQueryEmbeddingProvider(
|
|
2635
|
+
options.queryMode,
|
|
2575
2636
|
options.signal
|
|
2576
2637
|
);
|
|
2577
2638
|
return await this.#writeDocument(
|
|
2578
2639
|
async (document) => {
|
|
2579
|
-
const chapters = await this.#resolveQueryChapters(document, options
|
|
2640
|
+
const chapters = await this.#resolveQueryChapters(document, options, {
|
|
2641
|
+
embeddingProvider
|
|
2642
|
+
});
|
|
2580
2643
|
await ensureArchiveSearchIndex(document, {
|
|
2581
2644
|
...chapters === void 0 ? {} : { chapters },
|
|
2582
2645
|
...embeddingProvider === void 0 ? {} : { embeddingProvider }
|
|
2583
2646
|
});
|
|
2584
2647
|
return await (0, import_wiki_graph_core7.listArchiveEvidence)(document, objectUri, {
|
|
2585
2648
|
...withoutOperation(options),
|
|
2649
|
+
...embeddingProvider === void 0 ? {} : { embeddingProvider },
|
|
2586
2650
|
...chapters === void 0 ? {} : { chapters }
|
|
2587
2651
|
});
|
|
2588
2652
|
},
|
|
@@ -2616,6 +2680,24 @@ var WikiGraphArchiveHandle = class {
|
|
|
2616
2680
|
return meta;
|
|
2617
2681
|
}, options);
|
|
2618
2682
|
}
|
|
2683
|
+
async setArchiveTitle(title, options = {}) {
|
|
2684
|
+
const normalized = title?.trim() ?? null;
|
|
2685
|
+
if (title !== null && normalized === "") {
|
|
2686
|
+
throw new Error("Archive title cannot be empty.");
|
|
2687
|
+
}
|
|
2688
|
+
return await this.#writeDocument(async (document) => {
|
|
2689
|
+
if (normalized === null) {
|
|
2690
|
+
await document.metadata.deleteKey("", "title");
|
|
2691
|
+
} else {
|
|
2692
|
+
await document.metadata.put(
|
|
2693
|
+
{ kind: import_wiki_graph_core7.ObjectMetadataKind.Archive, objectPath: "" },
|
|
2694
|
+
"title",
|
|
2695
|
+
normalized
|
|
2696
|
+
);
|
|
2697
|
+
}
|
|
2698
|
+
return normalized;
|
|
2699
|
+
}, options);
|
|
2700
|
+
}
|
|
2619
2701
|
async readCover(options = {}) {
|
|
2620
2702
|
const app = new import_wiki_graph_core7.WikiGraph({});
|
|
2621
2703
|
return await this.#runtime.run(
|
|
@@ -2686,11 +2768,20 @@ var WikiGraphArchiveHandle = class {
|
|
|
2686
2768
|
if (options.chapters !== void 0) return options.chapters;
|
|
2687
2769
|
return (await resolveArchiveScope(document, this.objectUri, options.depth))?.chapterIds;
|
|
2688
2770
|
}
|
|
2689
|
-
async #resolveQueryChapters(document, options) {
|
|
2771
|
+
async #resolveQueryChapters(document, options, capabilities) {
|
|
2690
2772
|
const chapters = await this.#resolveScope(document, options);
|
|
2691
|
-
|
|
2773
|
+
const coverageOptions = {
|
|
2774
|
+
...chapters === void 0 ? {} : { chapters },
|
|
2775
|
+
...capabilities.embeddingProvider === void 0 ? {} : { embeddingProvider: capabilities.embeddingProvider },
|
|
2776
|
+
requireEmbeddingProvider: true,
|
|
2777
|
+
...options.queryMode === void 0 ? {} : { queryMode: options.queryMode }
|
|
2778
|
+
};
|
|
2779
|
+
if (options.skipUnindexed !== true) {
|
|
2780
|
+
await (0, import_wiki_graph_core7.assertArchiveIndexArtifactsReady)(document, coverageOptions);
|
|
2781
|
+
return chapters;
|
|
2782
|
+
}
|
|
2692
2783
|
const queryable = await (0, import_wiki_graph_core7.listArchiveQueryableChapterIds)(document, {
|
|
2693
|
-
...
|
|
2784
|
+
...coverageOptions
|
|
2694
2785
|
});
|
|
2695
2786
|
if (queryable.length === 0) {
|
|
2696
2787
|
throw new Error(
|
|
@@ -2699,6 +2790,19 @@ var WikiGraphArchiveHandle = class {
|
|
|
2699
2790
|
}
|
|
2700
2791
|
return queryable;
|
|
2701
2792
|
}
|
|
2793
|
+
async #resolveQueryEmbeddingProvider(queryMode, signal) {
|
|
2794
|
+
if (queryMode === "fts") return void 0;
|
|
2795
|
+
const provider = await this.#runtime.run(
|
|
2796
|
+
createConfiguredEmbeddingProvider,
|
|
2797
|
+
signal
|
|
2798
|
+
);
|
|
2799
|
+
if (queryMode === "embedding" && provider === void 0) {
|
|
2800
|
+
throw new Error(
|
|
2801
|
+
"Embedding query mode requires embeddings configuration at `wikg://local/config/embeddings`."
|
|
2802
|
+
);
|
|
2803
|
+
}
|
|
2804
|
+
return provider;
|
|
2805
|
+
}
|
|
2702
2806
|
};
|
|
2703
2807
|
async function ensureArchiveSearchIndex(document, options = {}) {
|
|
2704
2808
|
const scope = options.chapters === void 0 ? {} : { chapters: options.chapters };
|
|
@@ -3078,6 +3182,7 @@ var WikiGraphContinuationManager = class {
|
|
|
3078
3182
|
}
|
|
3079
3183
|
};
|
|
3080
3184
|
async function continueArchiveCursor(cursor, limit) {
|
|
3185
|
+
const embeddingProvider = cursor.kind === "evidence" || cursor.kind === "related" ? await resolveCursorEmbeddingProvider(cursor) : void 0;
|
|
3081
3186
|
const location = await resolveWikiGraphArchiveLocation(
|
|
3082
3187
|
archiveTargetFromLogicalLocator(getCursorArchivePath(cursor))
|
|
3083
3188
|
);
|
|
@@ -3107,6 +3212,8 @@ async function continueArchiveCursor(cursor, limit) {
|
|
|
3107
3212
|
limit,
|
|
3108
3213
|
order: cursor.order,
|
|
3109
3214
|
...cursor.query === void 0 ? {} : { query: cursor.query },
|
|
3215
|
+
...cursor.queryMode === void 0 ? {} : { queryMode: cursor.queryMode },
|
|
3216
|
+
...embeddingProvider === void 0 ? {} : { embeddingProvider },
|
|
3110
3217
|
...cursor.skipUnindexed === void 0 ? {} : { skipUnindexed: cursor.skipUnindexed },
|
|
3111
3218
|
...cursor.sourceContext === void 0 ? {} : { sourceContext: cursor.sourceContext }
|
|
3112
3219
|
});
|
|
@@ -3118,6 +3225,8 @@ async function continueArchiveCursor(cursor, limit) {
|
|
|
3118
3225
|
limit,
|
|
3119
3226
|
order: cursor.order,
|
|
3120
3227
|
...cursor.query === void 0 ? {} : { query: cursor.query },
|
|
3228
|
+
...cursor.queryMode === void 0 ? {} : { queryMode: cursor.queryMode },
|
|
3229
|
+
...embeddingProvider === void 0 ? {} : { embeddingProvider },
|
|
3121
3230
|
...cursor.role === void 0 ? {} : { role: cursor.role },
|
|
3122
3231
|
...cursor.skipUnindexed === void 0 ? {} : { skipUnindexed: cursor.skipUnindexed },
|
|
3123
3232
|
...cursor.sourceContext === void 0 ? {} : { sourceContext: cursor.sourceContext }
|
|
@@ -3127,6 +3236,7 @@ async function continueArchiveCursor(cursor, limit) {
|
|
|
3127
3236
|
);
|
|
3128
3237
|
}
|
|
3129
3238
|
async function continueLibraryCursor(cursor, limit) {
|
|
3239
|
+
const embeddingProvider = cursor.kind === "evidence" || cursor.kind === "related" ? await resolveCursorEmbeddingProvider(cursor) : void 0;
|
|
3130
3240
|
if (cursor.kind === "source-locators") {
|
|
3131
3241
|
throw new Error(
|
|
3132
3242
|
"Source locator cursors do not use the aggregate library index."
|
|
@@ -3163,6 +3273,8 @@ async function continueLibraryCursor(cursor, limit) {
|
|
|
3163
3273
|
limit,
|
|
3164
3274
|
order: cursor.order,
|
|
3165
3275
|
...cursor.query === void 0 ? {} : { query: cursor.query },
|
|
3276
|
+
...cursor.queryMode === void 0 ? {} : { queryMode: cursor.queryMode },
|
|
3277
|
+
...embeddingProvider === void 0 ? {} : { embeddingProvider },
|
|
3166
3278
|
...cursor.skipUnindexed === void 0 ? {} : { skipUnindexed: cursor.skipUnindexed },
|
|
3167
3279
|
...cursor.sourceContext === void 0 ? {} : { sourceContext: cursor.sourceContext }
|
|
3168
3280
|
});
|
|
@@ -3176,6 +3288,8 @@ async function continueLibraryCursor(cursor, limit) {
|
|
|
3176
3288
|
limit,
|
|
3177
3289
|
order: cursor.order,
|
|
3178
3290
|
...cursor.query === void 0 ? {} : { query: cursor.query },
|
|
3291
|
+
...cursor.queryMode === void 0 ? {} : { queryMode: cursor.queryMode },
|
|
3292
|
+
...embeddingProvider === void 0 ? {} : { embeddingProvider },
|
|
3179
3293
|
...cursor.role === void 0 ? {} : { role: cursor.role },
|
|
3180
3294
|
...cursor.skipUnindexed === void 0 ? {} : { skipUnindexed: cursor.skipUnindexed },
|
|
3181
3295
|
...cursor.sourceContext === void 0 ? {} : { sourceContext: cursor.sourceContext }
|
|
@@ -3200,7 +3314,7 @@ function createCollectionOptions(cursor, limit) {
|
|
|
3200
3314
|
};
|
|
3201
3315
|
}
|
|
3202
3316
|
async function createFindOptions(cursor, limit) {
|
|
3203
|
-
const embeddingProvider = await
|
|
3317
|
+
const embeddingProvider = await resolveCursorEmbeddingProvider(cursor);
|
|
3204
3318
|
return {
|
|
3205
3319
|
archiveKey: cursor.archiveKey,
|
|
3206
3320
|
...cursor.backlinks === void 0 ? {} : { backlinks: cursor.backlinks },
|
|
@@ -3208,6 +3322,9 @@ async function createFindOptions(cursor, limit) {
|
|
|
3208
3322
|
cursor: cursor.cursor,
|
|
3209
3323
|
...cursor.evidenceLimit === void 0 ? {} : { evidenceLimit: cursor.evidenceLimit },
|
|
3210
3324
|
limit,
|
|
3325
|
+
...cursor.match === void 0 ? {} : { match: cursor.match },
|
|
3326
|
+
...cursor.order === void 0 ? {} : { order: cursor.order },
|
|
3327
|
+
...cursor.queryMode === void 0 ? {} : { queryMode: cursor.queryMode },
|
|
3211
3328
|
...cursor.skipUnindexed === void 0 ? {} : { skipUnindexed: cursor.skipUnindexed },
|
|
3212
3329
|
...cursor.sourceContext === void 0 ? {} : { sourceContext: cursor.sourceContext },
|
|
3213
3330
|
...cursor.triplePattern === void 0 ? {} : { triplePattern: cursor.triplePattern },
|
|
@@ -3257,6 +3374,7 @@ function createContinuationPayload(context, cursor) {
|
|
|
3257
3374
|
kind: "evidence",
|
|
3258
3375
|
order: context.order ?? "doc-asc",
|
|
3259
3376
|
...context.query === void 0 ? {} : { query: context.query },
|
|
3377
|
+
...context.queryMode === void 0 ? {} : { queryMode: context.queryMode },
|
|
3260
3378
|
...context.skipUnindexed === void 0 ? {} : { skipUnindexed: context.skipUnindexed },
|
|
3261
3379
|
...context.sourceContext === void 0 ? {} : { sourceContext: context.sourceContext },
|
|
3262
3380
|
targetUri: requireTargetUri(context, "Evidence")
|
|
@@ -3273,6 +3391,7 @@ function createContinuationPayload(context, cursor) {
|
|
|
3273
3391
|
kind: "related",
|
|
3274
3392
|
order: context.order ?? "doc-asc",
|
|
3275
3393
|
...context.query === void 0 ? {} : { query: context.query },
|
|
3394
|
+
...context.queryMode === void 0 ? {} : { queryMode: context.queryMode },
|
|
3276
3395
|
...context.role === void 0 ? {} : { role: context.role },
|
|
3277
3396
|
...context.skipUnindexed === void 0 ? {} : { skipUnindexed: context.skipUnindexed },
|
|
3278
3397
|
...context.sourceContext === void 0 ? {} : { sourceContext: context.sourceContext },
|
|
@@ -3308,7 +3427,10 @@ function createContinuationPayload(context, cursor) {
|
|
|
3308
3427
|
indexScope: context.indexScope,
|
|
3309
3428
|
kind: "search",
|
|
3310
3429
|
...createLibraryQueryPayload(context),
|
|
3430
|
+
...context.match === void 0 ? {} : { match: context.match },
|
|
3431
|
+
...context.order === void 0 ? {} : { order: context.order },
|
|
3311
3432
|
...context.query === void 0 ? {} : { query: context.query },
|
|
3433
|
+
...context.queryMode === void 0 ? {} : { queryMode: context.queryMode },
|
|
3312
3434
|
...context.skipUnindexed === void 0 ? {} : { skipUnindexed: context.skipUnindexed },
|
|
3313
3435
|
...context.sourceContext === void 0 ? {} : { sourceContext: context.sourceContext },
|
|
3314
3436
|
...context.triplePattern === void 0 ? {} : { triplePattern: context.triplePattern },
|
|
@@ -3316,6 +3438,17 @@ function createContinuationPayload(context, cursor) {
|
|
|
3316
3438
|
};
|
|
3317
3439
|
}
|
|
3318
3440
|
}
|
|
3441
|
+
async function resolveCursorEmbeddingProvider(cursor) {
|
|
3442
|
+
if (cursor.kind !== "search" && cursor.query === void 0) return void 0;
|
|
3443
|
+
if (cursor.queryMode === "fts") return void 0;
|
|
3444
|
+
const provider = await createConfiguredEmbeddingProvider();
|
|
3445
|
+
if (cursor.queryMode === "embedding" && provider === void 0) {
|
|
3446
|
+
throw new Error(
|
|
3447
|
+
"Embedding query mode requires embeddings configuration at `wikg://local/config/embeddings`."
|
|
3448
|
+
);
|
|
3449
|
+
}
|
|
3450
|
+
return provider;
|
|
3451
|
+
}
|
|
3319
3452
|
function requireTargetUri(context, label) {
|
|
3320
3453
|
if (context.targetUri !== void 0) return context.targetUri;
|
|
3321
3454
|
throw new Error(`${label} continuation cursors require a target URI.`);
|
|
@@ -3339,7 +3472,10 @@ var import_wiki_graph_core10 = require("wiki-graph-core");
|
|
|
3339
3472
|
var import_wiki_graph_core11 = require("wiki-graph-core");
|
|
3340
3473
|
var import_wiki_graph_core12 = require("wiki-graph-core");
|
|
3341
3474
|
function createStageLLM(config, options) {
|
|
3342
|
-
const llmOptions = buildWikiGraphLLMOptions(config)
|
|
3475
|
+
const llmOptions = options?.provider === void 0 ? buildWikiGraphLLMOptions(config) : {
|
|
3476
|
+
...config.concurrent?.request === void 0 ? {} : { concurrent: config.concurrent.request },
|
|
3477
|
+
streamProvider: options.provider
|
|
3478
|
+
};
|
|
3343
3479
|
return new import_wiki_graph_core12.LLM({
|
|
3344
3480
|
sampling: (0, import_wiki_graph_core11.createDefaultWikiGraphSampling)({
|
|
3345
3481
|
...llmOptions.temperature === void 0 ? {} : { temperature: llmOptions.temperature },
|
|
@@ -3353,10 +3489,13 @@ function createStageLLM(config, options) {
|
|
|
3353
3489
|
});
|
|
3354
3490
|
}
|
|
3355
3491
|
async function loadRequiredStageConfig(options) {
|
|
3492
|
+
const injected = options.llmJSON === void 0 ? getWikiGraphSDKProviders().llm : void 0;
|
|
3356
3493
|
const config = await loadWikiGraphRuntimeConfig({
|
|
3357
|
-
...options.llmJSON === void 0 ? {} : { llmJSON: options.llmJSON }
|
|
3494
|
+
...options.llmJSON === void 0 ? {} : { llmJSON: options.llmJSON },
|
|
3495
|
+
...options.llmJSON === void 0 && injected !== void 0 ? { skipLLMConfig: true } : {},
|
|
3496
|
+
...getWikiGraphSDKProviders().embedding === void 0 ? {} : { skipEmbeddingConfig: true }
|
|
3358
3497
|
});
|
|
3359
|
-
if (config.llm?.provider === void 0 || config.llm.model === void 0) {
|
|
3498
|
+
if (injected === void 0 && (config.llm?.provider === void 0 || config.llm.model === void 0)) {
|
|
3360
3499
|
throw new Error(
|
|
3361
3500
|
"Missing LLM configuration. Set --llm for one run, or configure `wikg://local/config/llm` with provider and model."
|
|
3362
3501
|
);
|
|
@@ -3492,19 +3631,52 @@ function getGenerationPlanningProfile(task) {
|
|
|
3492
3631
|
// src/worker.ts
|
|
3493
3632
|
var import_worker = require("wiki-graph-core/worker");
|
|
3494
3633
|
var import_wiki_graph_core13 = require("wiki-graph-core");
|
|
3634
|
+
|
|
3635
|
+
// src/host.ts
|
|
3636
|
+
var import_platform2 = require("wiki-graph-core/platform");
|
|
3637
|
+
function prepareWikiGraphSDKHost(host) {
|
|
3638
|
+
if (host?.platform === void 0) {
|
|
3639
|
+
ensureNodeWikiGraphPlatform();
|
|
3640
|
+
return;
|
|
3641
|
+
}
|
|
3642
|
+
(0, import_platform2.installWikiGraphPlatform)(host.platform);
|
|
3643
|
+
}
|
|
3644
|
+
function assertWikiGraphSDKHostOptions(options) {
|
|
3645
|
+
if (options.host?.storage !== void 0 && options.stateDir !== void 0) {
|
|
3646
|
+
throw new TypeError(
|
|
3647
|
+
"Wiki Graph SDK host.storage and stateDir cannot be used together."
|
|
3648
|
+
);
|
|
3649
|
+
}
|
|
3650
|
+
}
|
|
3651
|
+
async function withWikiGraphSDKHost(options, operation) {
|
|
3652
|
+
assertWikiGraphSDKHostOptions(options);
|
|
3653
|
+
prepareWikiGraphSDKHost(options.host);
|
|
3654
|
+
if (options.host?.storage !== void 0) {
|
|
3655
|
+
return await (0, import_platform2.withWikiGraphStorage)(options.host.storage, operation);
|
|
3656
|
+
}
|
|
3657
|
+
return await withNodeWikiGraphStorage(options.stateDir, operation);
|
|
3658
|
+
}
|
|
3659
|
+
|
|
3660
|
+
// src/worker.ts
|
|
3495
3661
|
var import_worker2 = require("wiki-graph-core/worker");
|
|
3496
3662
|
async function runBuildJobWorker(options) {
|
|
3497
|
-
|
|
3498
|
-
|
|
3499
|
-
|
|
3500
|
-
|
|
3663
|
+
const { host, stateDir, ...coreOptions } = options;
|
|
3664
|
+
await withWikiGraphSDKHost(
|
|
3665
|
+
{
|
|
3666
|
+
...host === void 0 ? {} : { host },
|
|
3667
|
+
...stateDir === void 0 ? {} : { stateDir }
|
|
3668
|
+
},
|
|
3501
3669
|
async () => await (0, import_worker.runBuildJobWorker)(coreOptions)
|
|
3502
3670
|
);
|
|
3503
3671
|
}
|
|
3504
3672
|
|
|
3505
3673
|
// src/default-worker.ts
|
|
3506
3674
|
async function runWikiGraphQueueWorker(options = {}) {
|
|
3507
|
-
const
|
|
3675
|
+
const providers = getWikiGraphSDKProviders();
|
|
3676
|
+
const config = await loadWikiGraphRuntimeConfig({
|
|
3677
|
+
...providers.embedding === void 0 ? {} : { skipEmbeddingConfig: true },
|
|
3678
|
+
...providers.llm === void 0 ? {} : { skipLLMConfig: true }
|
|
3679
|
+
});
|
|
3508
3680
|
await runBuildJobWorker({
|
|
3509
3681
|
concurrency: config.concurrent?.job ?? DEFAULT_GENERATION_JOB_CONCURRENCY,
|
|
3510
3682
|
executeJob: async (job, reporter, context) => {
|
|
@@ -3513,12 +3685,16 @@ async function runWikiGraphQueueWorker(options = {}) {
|
|
|
3513
3685
|
async () => await executeBuildJob(job, reporter, context)
|
|
3514
3686
|
);
|
|
3515
3687
|
},
|
|
3688
|
+
...options.idleTimeoutMs === void 0 ? {} : { idleTimeoutMs: options.idleTimeoutMs },
|
|
3516
3689
|
...options.signal === void 0 ? {} : { signal: options.signal }
|
|
3517
3690
|
});
|
|
3518
3691
|
}
|
|
3519
3692
|
async function executeBuildJob(job, reporter, context) {
|
|
3693
|
+
const providers = getWikiGraphSDKProviders();
|
|
3520
3694
|
const config = await loadWikiGraphRuntimeConfig({
|
|
3521
|
-
...job.llmJSON === void 0 ? {} : { llmJSON: job.llmJSON }
|
|
3695
|
+
...job.llmJSON === void 0 ? {} : { llmJSON: job.llmJSON },
|
|
3696
|
+
...providers.embedding === void 0 ? {} : { skipEmbeddingConfig: true },
|
|
3697
|
+
...job.llmJSON === void 0 && providers.llm !== void 0 ? { skipLLMConfig: true } : {}
|
|
3522
3698
|
});
|
|
3523
3699
|
const execution = await openJobExecutor(
|
|
3524
3700
|
job,
|
|
@@ -3653,12 +3829,17 @@ async function openJobExecutor(job, config, signal, reporter) {
|
|
|
3653
3829
|
const stageConfig = generation ? await loadRequiredStageConfig({
|
|
3654
3830
|
...job.llmJSON === void 0 ? {} : { llmJSON: job.llmJSON }
|
|
3655
3831
|
}) : void 0;
|
|
3832
|
+
const injectedLLM = job.llmJSON === void 0 ? getWikiGraphSDKProviders().llm : void 0;
|
|
3656
3833
|
const llm = stageConfig === void 0 ? void 0 : createStageLLM(stageConfig, {
|
|
3657
3834
|
cacheDirectory: job.cache,
|
|
3658
3835
|
logDirectory: job.log,
|
|
3659
3836
|
onTokenUsage: async (usage) => {
|
|
3660
3837
|
await reporter.addTokenUsage(usage);
|
|
3661
|
-
}
|
|
3838
|
+
},
|
|
3839
|
+
onStreamProgress: async ({ outputCharacters }) => {
|
|
3840
|
+
await reporter.addOutputCharacters(outputCharacters);
|
|
3841
|
+
},
|
|
3842
|
+
...injectedLLM === void 0 ? {} : { provider: injectedLLM }
|
|
3662
3843
|
});
|
|
3663
3844
|
const promptSource = job.prompt ?? stageConfig?.prompt;
|
|
3664
3845
|
const wikispine = job.target === "knowledge-graph" ? requireKnowledgeGraphWikispineConfig(config) : void 0;
|
|
@@ -3679,7 +3860,7 @@ async function openJobExecutor(job, config, signal, reporter) {
|
|
|
3679
3860
|
"Local Wikimedia resolution requires an LLM."
|
|
3680
3861
|
),
|
|
3681
3862
|
llmModelId: requireValue(
|
|
3682
|
-
stageConfig?.llm?.model,
|
|
3863
|
+
stageConfig?.llm?.model ?? injectedLLM?.model,
|
|
3683
3864
|
"Local Wikimedia resolution requires an LLM model."
|
|
3684
3865
|
)
|
|
3685
3866
|
} : {
|
|
@@ -3688,7 +3869,7 @@ async function openJobExecutor(job, config, signal, reporter) {
|
|
|
3688
3869
|
token: config.wikimedia.token
|
|
3689
3870
|
}
|
|
3690
3871
|
);
|
|
3691
|
-
const embeddingProvider = job.target === "index-embedding-source" || job.target === "index-embedding-summary" ? buildSearchIndexEmbeddingProvider(requireEmbeddingConfig(config)) : void 0;
|
|
3872
|
+
const embeddingProvider = job.target === "index-embedding-source" || job.target === "index-embedding-summary" ? getWikiGraphSDKProviders().embedding ?? buildSearchIndexEmbeddingProvider(requireEmbeddingConfig(config)) : void 0;
|
|
3692
3873
|
return {
|
|
3693
3874
|
async close() {
|
|
3694
3875
|
await wikimediaResolver?.close();
|
|
@@ -3881,6 +4062,12 @@ var WikiGraphJobManager = class {
|
|
|
3881
4062
|
async clean() {
|
|
3882
4063
|
return await this.#runtime.run(async () => await (0, import_wiki_graph_core15.cleanBuildJobs)());
|
|
3883
4064
|
}
|
|
4065
|
+
async runWorker(options = {}) {
|
|
4066
|
+
await this.#runtime.run(
|
|
4067
|
+
async () => await runWikiGraphQueueWorker(options),
|
|
4068
|
+
options.signal
|
|
4069
|
+
);
|
|
4070
|
+
}
|
|
3884
4071
|
async get(jobId) {
|
|
3885
4072
|
return await this.#runtime.run(
|
|
3886
4073
|
async () => await this.#createHandle(await this.#backend.get(jobId))
|
|
@@ -3956,12 +4143,13 @@ var WikiGraphJobManager = class {
|
|
|
3956
4143
|
}
|
|
3957
4144
|
async enqueue(options) {
|
|
3958
4145
|
return await this.#runtime.run(async () => {
|
|
4146
|
+
const llmJSON = resolveWikiGraphLLMJSON(options);
|
|
3959
4147
|
const location = await resolveWikiGraphArchiveLocation(options.archive);
|
|
3960
4148
|
return await (0, import_wiki_graph_core15.withArchiveBuildJobCreationLock)(
|
|
3961
4149
|
location.archiveFile,
|
|
3962
4150
|
async () => {
|
|
3963
4151
|
const target = options.target ?? "reading-summary";
|
|
3964
|
-
await validateQueueTargetConfig(target,
|
|
4152
|
+
await validateQueueTargetConfig(target, llmJSON);
|
|
3965
4153
|
const selectedExplicitly = options.chapterId !== void 0 || options.chapterIds !== void 0 || options.chapterPath !== void 0 || options.depth !== void 0;
|
|
3966
4154
|
const candidates = await new import_wiki_graph_core15.WikiGraphArchiveFile(
|
|
3967
4155
|
location.archiveFile
|
|
@@ -4011,7 +4199,7 @@ var WikiGraphJobManager = class {
|
|
|
4011
4199
|
archive: location.archiveFile,
|
|
4012
4200
|
boost: options.boost ?? false,
|
|
4013
4201
|
chapterId: candidate.chapter.chapterId,
|
|
4014
|
-
...
|
|
4202
|
+
...llmJSON === void 0 ? {} : { llmJSON },
|
|
4015
4203
|
...options.prompt === void 0 ? {} : { prompt: options.prompt },
|
|
4016
4204
|
target
|
|
4017
4205
|
}),
|
|
@@ -4033,9 +4221,10 @@ var WikiGraphJobManager = class {
|
|
|
4033
4221
|
}
|
|
4034
4222
|
async planEnqueue(options) {
|
|
4035
4223
|
return await this.#runtime.run(async () => {
|
|
4224
|
+
const llmJSON = resolveWikiGraphLLMJSON(options);
|
|
4036
4225
|
const location = await resolveWikiGraphArchiveLocation(options.archive);
|
|
4037
4226
|
const target = options.target ?? "reading-summary";
|
|
4038
|
-
await validateQueueTargetConfig(target,
|
|
4227
|
+
await validateQueueTargetConfig(target, llmJSON);
|
|
4039
4228
|
const selectedExplicitly = options.chapterId !== void 0 || options.chapterIds !== void 0 || options.chapterPath !== void 0 || options.depth !== void 0;
|
|
4040
4229
|
const candidates = await new import_wiki_graph_core15.WikiGraphArchiveFile(
|
|
4041
4230
|
location.archiveFile
|
|
@@ -4280,12 +4469,15 @@ async function selectQueueChapters(document, chapters, options) {
|
|
|
4280
4469
|
);
|
|
4281
4470
|
}
|
|
4282
4471
|
async function validateQueueTargetConfig(target, llmJSON) {
|
|
4472
|
+
const providers = getWikiGraphSDKProviders();
|
|
4283
4473
|
const config = target === "knowledge-graph" || target === "reading-graph" || target === "reading-summary" ? await loadRequiredStageConfig({
|
|
4284
4474
|
...llmJSON === void 0 ? {} : { llmJSON }
|
|
4285
4475
|
}) : await loadWikiGraphRuntimeConfig({
|
|
4286
|
-
...llmJSON === void 0 ? {} : { llmJSON }
|
|
4476
|
+
...llmJSON === void 0 ? {} : { llmJSON },
|
|
4477
|
+
...providers.embedding === void 0 ? {} : { skipEmbeddingConfig: true },
|
|
4478
|
+
...llmJSON === void 0 && providers.llm !== void 0 ? { skipLLMConfig: true } : {}
|
|
4287
4479
|
});
|
|
4288
|
-
if ((target === "index-embedding-source" || target === "index-embedding-summary") && config.embedding === void 0) {
|
|
4480
|
+
if ((target === "index-embedding-source" || target === "index-embedding-summary") && config.embedding === void 0 && providers.embedding === void 0) {
|
|
4289
4481
|
throw new Error(
|
|
4290
4482
|
"Missing embeddings configuration. Configure `wikg://local/config/embeddings` before queueing embedding index artifact jobs."
|
|
4291
4483
|
);
|
|
@@ -4365,6 +4557,16 @@ function forwardAbort(source, target) {
|
|
|
4365
4557
|
// src/libraries.ts
|
|
4366
4558
|
var import_wiki_graph_core16 = require("wiki-graph-core");
|
|
4367
4559
|
var import_promises5 = require("fs/promises");
|
|
4560
|
+
async function resolveQueryEmbeddingProvider(queryMode) {
|
|
4561
|
+
if (queryMode === "fts") return void 0;
|
|
4562
|
+
const provider = await createConfiguredEmbeddingProvider();
|
|
4563
|
+
if (queryMode === "embedding" && provider === void 0) {
|
|
4564
|
+
throw new Error(
|
|
4565
|
+
"Embedding query mode requires embeddings configuration at `wikg://local/config/embeddings`."
|
|
4566
|
+
);
|
|
4567
|
+
}
|
|
4568
|
+
return provider;
|
|
4569
|
+
}
|
|
4368
4570
|
var WikiGraphLibraryManager = class {
|
|
4369
4571
|
#runtime;
|
|
4370
4572
|
constructor(runtime) {
|
|
@@ -4434,6 +4636,43 @@ var WikiGraphLibraryManager = class {
|
|
|
4434
4636
|
)
|
|
4435
4637
|
);
|
|
4436
4638
|
}
|
|
4639
|
+
async readArchive(target, consume, options = {}) {
|
|
4640
|
+
return await this.#runtime.run(async () => {
|
|
4641
|
+
const parsedTarget = requireLibraryArchiveTarget(target);
|
|
4642
|
+
const library = await (0, import_wiki_graph_core16.resolveWikiGraphLibrary)(parsedTarget);
|
|
4643
|
+
return await (0, import_wiki_graph_core16.withWikiGraphLibraryLock)(
|
|
4644
|
+
library.id,
|
|
4645
|
+
"read",
|
|
4646
|
+
async () => {
|
|
4647
|
+
throwIfAborted(options.signal);
|
|
4648
|
+
const archive = await (0, import_wiki_graph_core16.getWikiGraphLibraryArchive)(parsedTarget);
|
|
4649
|
+
const file = requireReadableLibraryArchiveFile(archive);
|
|
4650
|
+
const reader = await file.openReader();
|
|
4651
|
+
let active = true;
|
|
4652
|
+
try {
|
|
4653
|
+
const content = {
|
|
4654
|
+
mediaType: "application/vnd.wiki-graph.archive",
|
|
4655
|
+
size: reader.size,
|
|
4656
|
+
stream: createCallbackScopedArchiveStream(
|
|
4657
|
+
reader,
|
|
4658
|
+
() => active,
|
|
4659
|
+
options.signal
|
|
4660
|
+
),
|
|
4661
|
+
uri: archive.uri
|
|
4662
|
+
};
|
|
4663
|
+
return await runAbortableCallback(
|
|
4664
|
+
async () => await consume(content),
|
|
4665
|
+
options.signal
|
|
4666
|
+
);
|
|
4667
|
+
} finally {
|
|
4668
|
+
active = false;
|
|
4669
|
+
await reader.close();
|
|
4670
|
+
}
|
|
4671
|
+
},
|
|
4672
|
+
options.signal === void 0 ? {} : { signal: options.signal }
|
|
4673
|
+
);
|
|
4674
|
+
}, options.signal);
|
|
4675
|
+
}
|
|
4437
4676
|
async moveArchive(options) {
|
|
4438
4677
|
return toArchiveSnapshot(
|
|
4439
4678
|
await this.#runtime.run(
|
|
@@ -4523,7 +4762,9 @@ var WikiGraphLibraryManager = class {
|
|
|
4523
4762
|
}
|
|
4524
4763
|
async search(target, query, options = {}) {
|
|
4525
4764
|
return await this.#runtime.run(async () => {
|
|
4526
|
-
const embeddingProvider = await
|
|
4765
|
+
const embeddingProvider = await resolveQueryEmbeddingProvider(
|
|
4766
|
+
options.queryMode
|
|
4767
|
+
);
|
|
4527
4768
|
return await (0, import_wiki_graph_core16.findWikiGraphLibraryObjects)(
|
|
4528
4769
|
requireLibraryTarget(target),
|
|
4529
4770
|
query,
|
|
@@ -4536,7 +4777,9 @@ var WikiGraphLibraryManager = class {
|
|
|
4536
4777
|
}
|
|
4537
4778
|
async searchArchiveMembers(target, query, options = {}) {
|
|
4538
4779
|
return await this.#runtime.run(async () => {
|
|
4539
|
-
const embeddingProvider = await
|
|
4780
|
+
const embeddingProvider = await resolveQueryEmbeddingProvider(
|
|
4781
|
+
options.queryMode
|
|
4782
|
+
);
|
|
4540
4783
|
return await (0, import_wiki_graph_core16.findWikiGraphLibraryArchiveMembers)(
|
|
4541
4784
|
requireLibraryTarget(target),
|
|
4542
4785
|
query,
|
|
@@ -4577,22 +4820,36 @@ var WikiGraphLibraryManager = class {
|
|
|
4577
4820
|
);
|
|
4578
4821
|
}
|
|
4579
4822
|
async related(target, objectUri, options = {}) {
|
|
4580
|
-
return await this.#runtime.run(
|
|
4581
|
-
|
|
4823
|
+
return await this.#runtime.run(async () => {
|
|
4824
|
+
if (options.query === void 0 && options.queryMode !== void 0) {
|
|
4825
|
+
throw new Error("`queryMode` requires `query`.");
|
|
4826
|
+
}
|
|
4827
|
+
const embeddingProvider = options.query === void 0 ? void 0 : await resolveQueryEmbeddingProvider(options.queryMode);
|
|
4828
|
+
return await (0, import_wiki_graph_core16.listRelatedWikiGraphLibraryObjects)(
|
|
4582
4829
|
requireLibraryTarget(target),
|
|
4583
4830
|
objectUri,
|
|
4584
|
-
|
|
4585
|
-
|
|
4586
|
-
|
|
4831
|
+
{
|
|
4832
|
+
...options,
|
|
4833
|
+
...embeddingProvider === void 0 ? {} : { embeddingProvider }
|
|
4834
|
+
}
|
|
4835
|
+
);
|
|
4836
|
+
});
|
|
4587
4837
|
}
|
|
4588
4838
|
async evidence(target, objectUri, options = {}) {
|
|
4589
|
-
return await this.#runtime.run(
|
|
4590
|
-
|
|
4839
|
+
return await this.#runtime.run(async () => {
|
|
4840
|
+
if (options.query === void 0 && options.queryMode !== void 0) {
|
|
4841
|
+
throw new Error("`queryMode` requires `query`.");
|
|
4842
|
+
}
|
|
4843
|
+
const embeddingProvider = options.query === void 0 ? void 0 : await resolveQueryEmbeddingProvider(options.queryMode);
|
|
4844
|
+
return await (0, import_wiki_graph_core16.listWikiGraphLibraryEvidence)(
|
|
4591
4845
|
requireLibraryTarget(target),
|
|
4592
4846
|
objectUri,
|
|
4593
|
-
|
|
4594
|
-
|
|
4595
|
-
|
|
4847
|
+
{
|
|
4848
|
+
...options,
|
|
4849
|
+
...embeddingProvider === void 0 ? {} : { embeddingProvider }
|
|
4850
|
+
}
|
|
4851
|
+
);
|
|
4852
|
+
});
|
|
4596
4853
|
}
|
|
4597
4854
|
async pack(target, objectUri, budget = 5e3) {
|
|
4598
4855
|
return await this.#runtime.run(
|
|
@@ -4674,6 +4931,97 @@ function requireLibraryTarget(value) {
|
|
|
4674
4931
|
}
|
|
4675
4932
|
return target;
|
|
4676
4933
|
}
|
|
4934
|
+
function requireLibraryArchiveTarget(value) {
|
|
4935
|
+
const target = requireLibraryTarget(value);
|
|
4936
|
+
if (target.kind !== "archive" || target.archivePublicId === void 0 || target.objectUri !== void 0) {
|
|
4937
|
+
throw new TypeError("Expected a Wiki Graph library archive URI.");
|
|
4938
|
+
}
|
|
4939
|
+
return target;
|
|
4940
|
+
}
|
|
4941
|
+
function requireReadableLibraryArchiveFile(archive) {
|
|
4942
|
+
if (!archive.exists || archive.status === "missing") {
|
|
4943
|
+
throw new Error(`Wiki Graph library archive is missing: ${archive.uri}`);
|
|
4944
|
+
}
|
|
4945
|
+
if (archive.status === "conflict") {
|
|
4946
|
+
throw new Error(
|
|
4947
|
+
`Wiki Graph library archive has a conflict and cannot be read: ${archive.uri}`
|
|
4948
|
+
);
|
|
4949
|
+
}
|
|
4950
|
+
if (archive.file === void 0) {
|
|
4951
|
+
throw new Error(
|
|
4952
|
+
`Wiki Graph library archive is unavailable: ${archive.uri}`
|
|
4953
|
+
);
|
|
4954
|
+
}
|
|
4955
|
+
return archive.file;
|
|
4956
|
+
}
|
|
4957
|
+
var ARCHIVE_STREAM_CHUNK_SIZE = 64 * 1024;
|
|
4958
|
+
function createCallbackScopedArchiveStream(reader, isActive, signal) {
|
|
4959
|
+
let consumed = false;
|
|
4960
|
+
return {
|
|
4961
|
+
[Symbol.asyncIterator]() {
|
|
4962
|
+
assertArchiveReadSessionActive(isActive, signal);
|
|
4963
|
+
if (consumed) {
|
|
4964
|
+
throw new Error(
|
|
4965
|
+
"The library archive stream has already been consumed."
|
|
4966
|
+
);
|
|
4967
|
+
}
|
|
4968
|
+
consumed = true;
|
|
4969
|
+
let offset = 0;
|
|
4970
|
+
let finished = false;
|
|
4971
|
+
return {
|
|
4972
|
+
async next() {
|
|
4973
|
+
if (finished) return { done: true, value: void 0 };
|
|
4974
|
+
assertArchiveReadSessionActive(isActive, signal);
|
|
4975
|
+
if (offset >= reader.size) {
|
|
4976
|
+
finished = true;
|
|
4977
|
+
return { done: true, value: void 0 };
|
|
4978
|
+
}
|
|
4979
|
+
const length = Math.min(
|
|
4980
|
+
ARCHIVE_STREAM_CHUNK_SIZE,
|
|
4981
|
+
reader.size - offset
|
|
4982
|
+
);
|
|
4983
|
+
const chunk = await reader.read(offset, length);
|
|
4984
|
+
assertArchiveReadSessionActive(isActive, signal);
|
|
4985
|
+
offset += chunk.byteLength;
|
|
4986
|
+
return { done: false, value: chunk };
|
|
4987
|
+
},
|
|
4988
|
+
return() {
|
|
4989
|
+
finished = true;
|
|
4990
|
+
return Promise.resolve({ done: true, value: void 0 });
|
|
4991
|
+
}
|
|
4992
|
+
};
|
|
4993
|
+
}
|
|
4994
|
+
};
|
|
4995
|
+
}
|
|
4996
|
+
function assertArchiveReadSessionActive(isActive, signal) {
|
|
4997
|
+
if (!isActive()) {
|
|
4998
|
+
throw new Error(
|
|
4999
|
+
"The library archive stream cannot be read after its callback has finished."
|
|
5000
|
+
);
|
|
5001
|
+
}
|
|
5002
|
+
throwIfAborted(signal);
|
|
5003
|
+
}
|
|
5004
|
+
async function runAbortableCallback(operation, signal) {
|
|
5005
|
+
if (signal === void 0) return await operation();
|
|
5006
|
+
throwIfAborted(signal);
|
|
5007
|
+
let rejectAbort;
|
|
5008
|
+
const aborted = new Promise((_resolve, reject) => {
|
|
5009
|
+
rejectAbort = reject;
|
|
5010
|
+
});
|
|
5011
|
+
const onAbort = () => rejectAbort(abortReason(signal));
|
|
5012
|
+
signal.addEventListener("abort", onAbort, { once: true });
|
|
5013
|
+
try {
|
|
5014
|
+
return await Promise.race([operation(), aborted]);
|
|
5015
|
+
} finally {
|
|
5016
|
+
signal.removeEventListener("abort", onAbort);
|
|
5017
|
+
}
|
|
5018
|
+
}
|
|
5019
|
+
function throwIfAborted(signal) {
|
|
5020
|
+
if (signal?.aborted === true) throw abortReason(signal);
|
|
5021
|
+
}
|
|
5022
|
+
function abortReason(signal) {
|
|
5023
|
+
return signal.reason ?? new DOMException("The operation was aborted.", "AbortError");
|
|
5024
|
+
}
|
|
4677
5025
|
|
|
4678
5026
|
// src/maintenance.ts
|
|
4679
5027
|
var import_wiki_graph_core17 = require("wiki-graph-core");
|
|
@@ -4743,11 +5091,14 @@ var WikiGraphSDK = class {
|
|
|
4743
5091
|
libraries;
|
|
4744
5092
|
maintenance;
|
|
4745
5093
|
constructor(options = {}) {
|
|
4746
|
-
|
|
5094
|
+
assertWikiGraphSDKHostOptions(options);
|
|
5095
|
+
prepareWikiGraphSDKHost(options.host);
|
|
4747
5096
|
this.#context = {
|
|
4748
5097
|
cwd: options.cwd ?? process.cwd(),
|
|
4749
5098
|
env: { ...process.env, ...options.env },
|
|
4750
5099
|
envPolicy: options.envPolicy ?? "production",
|
|
5100
|
+
...options.host === void 0 ? {} : { host: options.host },
|
|
5101
|
+
providers: { ...options.providers },
|
|
4751
5102
|
...options.stateDir === void 0 ? {} : { stateDir: options.stateDir }
|
|
4752
5103
|
};
|
|
4753
5104
|
this.archives = new WikiGraphArchiveManager(this);
|
|
@@ -4759,7 +5110,7 @@ var WikiGraphSDK = class {
|
|
|
4759
5110
|
this.maintenance = new WikiGraphMaintenanceManager(this);
|
|
4760
5111
|
}
|
|
4761
5112
|
async run(operation, signal, options = {}) {
|
|
4762
|
-
|
|
5113
|
+
throwIfAborted2(signal);
|
|
4763
5114
|
const runWithContext = async () => await withWikiGraphSDKRuntimeContext(this.#context, async () => {
|
|
4764
5115
|
if (options.skipHomeBootstrap !== true) {
|
|
4765
5116
|
this.#homeReady ??= (0, import_wiki_graph_core18.ensureWikiGraphHomeSchemaCurrent)().catch(
|
|
@@ -4772,18 +5123,21 @@ var WikiGraphSDK = class {
|
|
|
4772
5123
|
}
|
|
4773
5124
|
return await operation();
|
|
4774
5125
|
});
|
|
4775
|
-
const result =
|
|
4776
|
-
|
|
5126
|
+
const result = await withWikiGraphSDKHost(
|
|
5127
|
+
{
|
|
5128
|
+
...this.#context.host === void 0 ? {} : { host: this.#context.host },
|
|
5129
|
+
...this.#context.stateDir === void 0 ? {} : { stateDir: this.#context.stateDir }
|
|
5130
|
+
},
|
|
4777
5131
|
runWithContext
|
|
4778
5132
|
);
|
|
4779
|
-
|
|
5133
|
+
throwIfAborted2(signal);
|
|
4780
5134
|
return result;
|
|
4781
5135
|
}
|
|
4782
5136
|
close() {
|
|
4783
5137
|
this.jobs.close();
|
|
4784
5138
|
}
|
|
4785
5139
|
};
|
|
4786
|
-
function
|
|
5140
|
+
function throwIfAborted2(signal) {
|
|
4787
5141
|
if (signal?.aborted !== true) return;
|
|
4788
5142
|
throw signal.reason instanceof Error ? signal.reason : new Error("The Wiki Graph SDK operation was aborted.");
|
|
4789
5143
|
}
|
|
@@ -4825,6 +5179,7 @@ var import_wiki_graph_core19 = require("wiki-graph-core");
|
|
|
4825
5179
|
createEntryRuntimeContext,
|
|
4826
5180
|
createGenerationPerformanceHints,
|
|
4827
5181
|
createNodeTemplateEnvironment,
|
|
5182
|
+
createNodeWikiGraphPlatform,
|
|
4828
5183
|
createNodeWikiGraphStorage,
|
|
4829
5184
|
createStageLLM,
|
|
4830
5185
|
createWikiGraphSDK,
|
|
@@ -4839,15 +5194,20 @@ var import_wiki_graph_core19 = require("wiki-graph-core");
|
|
|
4839
5194
|
formatLocatedChapterSourceCollectionUri,
|
|
4840
5195
|
formatLocatedChapterUri,
|
|
4841
5196
|
formatLocatedWikiGraphUri,
|
|
5197
|
+
formatSourceArtifactUri,
|
|
5198
|
+
formatSourceLocatorFragment,
|
|
4842
5199
|
formatWikiGraphCommandUri,
|
|
4843
5200
|
formatWikiGraphLibraryUri,
|
|
4844
5201
|
getNodeResourcePath,
|
|
5202
|
+
getWikiGraphSDKProviders,
|
|
4845
5203
|
getWikiGraphSDKRuntimeContext,
|
|
4846
5204
|
getWikiGraphSDKStateDir,
|
|
4847
5205
|
inspectWikiGraphArchive,
|
|
4848
5206
|
installNodeWikiGraphPlatform,
|
|
4849
5207
|
isSourceLocatorScopeUri,
|
|
5208
|
+
isWikiGraphJobUri,
|
|
4850
5209
|
isWikiGraphLibraryUri,
|
|
5210
|
+
isWikiGraphUri,
|
|
4851
5211
|
loadRequiredStageConfig,
|
|
4852
5212
|
loadWikiGraphRuntimeConfig,
|
|
4853
5213
|
maskLocalConfigSection,
|
|
@@ -4859,6 +5219,7 @@ var import_wiki_graph_core19 = require("wiki-graph-core");
|
|
|
4859
5219
|
parseChapterUriPath,
|
|
4860
5220
|
parseLocalConfigSection,
|
|
4861
5221
|
parseLocatedWikiGraphUri,
|
|
5222
|
+
parseSourceLocatorFragment,
|
|
4862
5223
|
parseSourceTextJsonl,
|
|
4863
5224
|
parseWikiGraphLibraryUri,
|
|
4864
5225
|
parseWikiGraphUriSyntax,
|
|
@@ -4867,10 +5228,14 @@ var import_wiki_graph_core19 = require("wiki-graph-core");
|
|
|
4867
5228
|
readLocalConfigSection,
|
|
4868
5229
|
readWikiGraphEmbeddingConfig,
|
|
4869
5230
|
replaceLocalConfigSection,
|
|
5231
|
+
requireArchiveUri,
|
|
4870
5232
|
requireKnowledgeGraphWikispineConfig,
|
|
5233
|
+
requireLocatedObjectOrArchiveUri,
|
|
5234
|
+
requireLocatedObjectUri,
|
|
4871
5235
|
resolveExtractionPrompt,
|
|
4872
5236
|
resolveHostedProviderEndpoint,
|
|
4873
5237
|
resolveKnowledgeGraphRecallPrompt,
|
|
5238
|
+
resolveWikiGraphLLMJSON,
|
|
4874
5239
|
resolveWikispineConfig,
|
|
4875
5240
|
runWikiGraphQueueWorker,
|
|
4876
5241
|
validateLocalConfigSection,
|