wiki-graph-sdk 0.7.0 → 0.7.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +16 -0
- package/dist/archives.cjs +71 -7
- package/dist/archives.cjs.map +1 -1
- package/dist/archives.js +67 -3
- package/dist/archives.js.map +1 -1
- package/dist/conversions.cjs +65 -1
- package/dist/conversions.cjs.map +1 -1
- package/dist/conversions.d.ts +13 -3
- package/dist/conversions.js +65 -1
- package/dist/conversions.js.map +1 -1
- package/dist/default-worker.cjs +17 -16
- package/dist/default-worker.cjs.map +1 -1
- package/dist/default-worker.js +13 -11
- package/dist/default-worker.js.map +1 -1
- package/dist/index.cjs +173 -108
- package/dist/index.cjs.map +1 -1
- package/dist/index.d.ts +1 -1
- package/dist/index.js +86 -20
- package/dist/index.js.map +1 -1
- package/dist/worker.cjs +37 -0
- package/dist/worker.cjs.map +1 -1
- package/dist/worker.d.ts +17 -1
- package/dist/worker.js +39 -2
- package/dist/worker.js.map +1 -1
- package/package.json +2 -2
package/dist/default-worker.cjs
CHANGED
|
@@ -35,7 +35,7 @@ __export(default_worker_exports, {
|
|
|
35
35
|
runWikiGraphQueueWorker: () => runWikiGraphQueueWorker
|
|
36
36
|
});
|
|
37
37
|
module.exports = __toCommonJS(default_worker_exports);
|
|
38
|
-
var
|
|
38
|
+
var import_wiki_graph_core6 = require("wiki-graph-core");
|
|
39
39
|
var import_worker3 = require("wiki-graph-core/worker");
|
|
40
40
|
|
|
41
41
|
// src/embedding.ts
|
|
@@ -1444,6 +1444,7 @@ function abortError(signal) {
|
|
|
1444
1444
|
|
|
1445
1445
|
// src/worker.ts
|
|
1446
1446
|
var import_worker = require("wiki-graph-core/worker");
|
|
1447
|
+
var import_wiki_graph_core5 = require("wiki-graph-core");
|
|
1447
1448
|
var import_worker2 = require("wiki-graph-core/worker");
|
|
1448
1449
|
async function runBuildJobWorker(options) {
|
|
1449
1450
|
installNodeWikiGraphPlatform();
|
|
@@ -1460,7 +1461,7 @@ async function runWikiGraphQueueWorker(options = {}) {
|
|
|
1460
1461
|
await runBuildJobWorker({
|
|
1461
1462
|
concurrency: config.concurrent?.job ?? DEFAULT_GENERATION_JOB_CONCURRENCY,
|
|
1462
1463
|
executeJob: async (job, reporter, context) => {
|
|
1463
|
-
await (0,
|
|
1464
|
+
await (0, import_wiki_graph_core6.withLoggingContext)(
|
|
1464
1465
|
{ logDirectory: job.log, operation: "build-job" },
|
|
1465
1466
|
async () => await executeBuildJob(job, reporter, context)
|
|
1466
1467
|
);
|
|
@@ -1483,7 +1484,7 @@ async function executeBuildJob(job, reporter, context) {
|
|
|
1483
1484
|
let stage = await readChapterStage(job);
|
|
1484
1485
|
if (stage === "sourced") {
|
|
1485
1486
|
await executeStep(job, "reading-graph", execution, reporter, context, {
|
|
1486
|
-
...execution.extractionPrompt === void 0 ? {} : {
|
|
1487
|
+
...execution.extractionPrompt === void 0 ? {} : { prompt: execution.extractionPrompt }
|
|
1487
1488
|
});
|
|
1488
1489
|
stage = await readChapterStage(job);
|
|
1489
1490
|
}
|
|
@@ -1516,8 +1517,8 @@ async function executeBuildJob(job, reporter, context) {
|
|
|
1516
1517
|
}
|
|
1517
1518
|
}
|
|
1518
1519
|
await executeStep(job, job.target, execution, reporter, context, {
|
|
1519
|
-
...job.target === "reading-graph" ? {
|
|
1520
|
-
...job.target === "knowledge-graph" ? {
|
|
1520
|
+
...job.target === "reading-graph" ? { prompt: execution.extractionPrompt } : {},
|
|
1521
|
+
...job.target === "knowledge-graph" ? { prompt: execution.knowledgeGraphPrompt } : {}
|
|
1521
1522
|
});
|
|
1522
1523
|
} finally {
|
|
1523
1524
|
await execution.close();
|
|
@@ -1532,9 +1533,9 @@ async function executeStep(job, kind, execution, reporter, context, inputOptions
|
|
|
1532
1533
|
const stepWorkspace = await resetStepWorkspace(job.workspace, kind);
|
|
1533
1534
|
const inputFile = await stepWorkspace.createFile("input.jsonl");
|
|
1534
1535
|
const outputWorkspace = await stepWorkspace.createDirectory("output");
|
|
1535
|
-
const revision = await new
|
|
1536
|
+
const revision = await new import_wiki_graph_core6.WikiGraphArchiveFile(job.archive).readDocument(
|
|
1536
1537
|
async (document) => {
|
|
1537
|
-
await (0,
|
|
1538
|
+
await (0, import_wiki_graph_core6.getChapterDetails)(document, job.chapterId);
|
|
1538
1539
|
return await (0, import_worker3.writeChapterJobInputFile)(
|
|
1539
1540
|
document,
|
|
1540
1541
|
job.chapterId,
|
|
@@ -1544,7 +1545,7 @@ async function executeStep(job, kind, execution, reporter, context, inputOptions
|
|
|
1544
1545
|
);
|
|
1545
1546
|
}
|
|
1546
1547
|
);
|
|
1547
|
-
await (0,
|
|
1548
|
+
await (0, import_wiki_graph_core6.recordBuildJobInputRevision)({
|
|
1548
1549
|
currentRevision: revision,
|
|
1549
1550
|
jobId: job.jobId,
|
|
1550
1551
|
ownerId: requireRunningJobOwnerId(job)
|
|
@@ -1574,8 +1575,8 @@ async function executeStep(job, kind, execution, reporter, context, inputOptions
|
|
|
1574
1575
|
total: 1,
|
|
1575
1576
|
unit: "item"
|
|
1576
1577
|
});
|
|
1577
|
-
await new
|
|
1578
|
-
assertJobStillRunning(await (0,
|
|
1578
|
+
await new import_wiki_graph_core6.WikiGraphArchiveFile(job.archive).write(async (document) => {
|
|
1579
|
+
assertJobStillRunning(await (0, import_wiki_graph_core6.getBuildJob)(job.jobId));
|
|
1579
1580
|
await (0, import_worker3.applyChapterJobArtifactFile)(
|
|
1580
1581
|
document,
|
|
1581
1582
|
job.chapterId,
|
|
@@ -1591,7 +1592,7 @@ async function executeStep(job, kind, execution, reporter, context, inputOptions
|
|
|
1591
1592
|
unit: "item"
|
|
1592
1593
|
});
|
|
1593
1594
|
await reporter.stepCompleted(kind);
|
|
1594
|
-
assertJobStillRunning(await (0,
|
|
1595
|
+
assertJobStillRunning(await (0, import_wiki_graph_core6.getBuildJob)(job.jobId));
|
|
1595
1596
|
}
|
|
1596
1597
|
async function openJobExecutor(job, config, signal, reporter) {
|
|
1597
1598
|
const generation = !job.target.startsWith("index-");
|
|
@@ -1610,13 +1611,13 @@ async function openJobExecutor(job, config, signal, reporter) {
|
|
|
1610
1611
|
const request = llm === void 0 ? void 0 : async (messages, index, maxRetries) => await llm.request(messages, {
|
|
1611
1612
|
retryIndex: index,
|
|
1612
1613
|
retryMax: maxRetries,
|
|
1613
|
-
scope:
|
|
1614
|
+
scope: import_wiki_graph_core6.WikiGraphScope.ReaderExtraction,
|
|
1614
1615
|
signal
|
|
1615
1616
|
});
|
|
1616
1617
|
if (request !== void 0 && llm !== void 0) {
|
|
1617
1618
|
request.lazy = async (operation) => await llm.request(async () => await operation(request));
|
|
1618
1619
|
}
|
|
1619
|
-
const wikimediaResolver = job.target !== "knowledge-graph" ? void 0 : await (0,
|
|
1620
|
+
const wikimediaResolver = job.target !== "knowledge-graph" ? void 0 : await (0, import_wiki_graph_core6.openWikimediaResolver)(
|
|
1620
1621
|
config.wikimedia === void 0 ? {
|
|
1621
1622
|
kind: "local",
|
|
1622
1623
|
llmRequest: requireValue(
|
|
@@ -1659,12 +1660,12 @@ async function resetStepWorkspace(workspace, kind) {
|
|
|
1659
1660
|
return await workspace.createDirectory(name);
|
|
1660
1661
|
}
|
|
1661
1662
|
async function readChapterStage(job) {
|
|
1662
|
-
return await new
|
|
1663
|
-
async (document) => (await (0,
|
|
1663
|
+
return await new import_wiki_graph_core6.WikiGraphArchiveFile(job.archive).readDocument(
|
|
1664
|
+
async (document) => (await (0, import_wiki_graph_core6.getChapterDetails)(document, job.chapterId)).stage
|
|
1664
1665
|
);
|
|
1665
1666
|
}
|
|
1666
1667
|
async function assertCurrentFtsArtifact(job) {
|
|
1667
|
-
await new
|
|
1668
|
+
await new import_wiki_graph_core6.WikiGraphArchiveFile(job.archive).readDocument(async (document) => {
|
|
1668
1669
|
const revision = await document.serials.getRevision(job.chapterId);
|
|
1669
1670
|
const artifact = await document.indexArtifacts.get(job.chapterId, "fts");
|
|
1670
1671
|
if (artifact?.sourceRevision !== revision) {
|