wiki-graph-sdk 0.7.1 → 0.8.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/dist/archives.cjs +406 -642
- package/dist/archives.cjs.map +1 -1
- package/dist/archives.d.ts +42 -80
- package/dist/archives.js +342 -571
- package/dist/archives.js.map +1 -1
- package/dist/conversions.cjs +160 -28
- package/dist/conversions.cjs.map +1 -1
- package/dist/conversions.d.ts +3 -1
- package/dist/conversions.js +165 -24
- package/dist/conversions.js.map +1 -1
- package/dist/default-worker.cjs +289 -52
- package/dist/default-worker.cjs.map +1 -1
- package/dist/default-worker.js +303 -50
- package/dist/default-worker.js.map +1 -1
- package/dist/index.cjs +967 -911
- package/dist/index.cjs.map +1 -1
- package/dist/index.d.ts +36 -19
- package/dist/index.js +836 -767
- package/dist/index.js.map +1 -1
- package/dist/{jobs-Dd5PMSId.d.ts → jobs-xIm_kshM.d.ts} +22 -14
- package/dist/maintenance.cjs +138 -29
- package/dist/maintenance.cjs.map +1 -1
- package/dist/maintenance.d.ts +2 -1
- package/dist/maintenance.js +143 -25
- package/dist/maintenance.js.map +1 -1
- package/dist/target-CZ7elwmj.d.ts +14 -0
- package/dist/wikispine.cjs +14 -10
- package/dist/wikispine.cjs.map +1 -1
- package/dist/wikispine.js +14 -10
- package/dist/wikispine.js.map +1 -1
- package/dist/worker.cjs +294 -25
- package/dist/worker.cjs.map +1 -1
- package/dist/worker.d.ts +28 -5
- package/dist/worker.js +312 -28
- package/dist/worker.js.map +1 -1
- package/package.json +2 -2
package/dist/index.js
CHANGED
|
@@ -4,9 +4,9 @@ globalThis.__WIKIGRAPH_DATA_DIR__ ??= __WIKIGRAPH_RESOLVE__(__WIKIGRAPH_FILE_URL
|
|
|
4
4
|
|
|
5
5
|
// src/archive/archive.ts
|
|
6
6
|
import { randomUUID as randomUUID2 } from "crypto";
|
|
7
|
-
import { link, mkdtemp, rename as rename2, rm as rm3, stat as
|
|
7
|
+
import { link, mkdtemp, rename as rename2, rm as rm3, stat as stat3 } from "fs/promises";
|
|
8
8
|
import { tmpdir } from "os";
|
|
9
|
-
import { basename as
|
|
9
|
+
import { basename as basename4, dirname as dirname3, join as join4 } from "path";
|
|
10
10
|
import { Readable as Readable2 } from "stream";
|
|
11
11
|
import {
|
|
12
12
|
addChapter,
|
|
@@ -15,9 +15,8 @@ import {
|
|
|
15
15
|
assertNoActiveBuildJobs,
|
|
16
16
|
deleteArchiveSearchSessions,
|
|
17
17
|
DirectoryDocument,
|
|
18
|
-
finalizeWikiGraphLibraryArchiveWrite,
|
|
19
18
|
findArchiveObjects,
|
|
20
|
-
formatLocatedWikiGraphUri,
|
|
19
|
+
formatLocatedWikiGraphUri as formatLocatedWikiGraphUri2,
|
|
21
20
|
isArchiveSearchIndexCurrent as isArchiveSearchIndexCurrent2,
|
|
22
21
|
isSourceLocatorScopeUri,
|
|
23
22
|
listArchiveCollection,
|
|
@@ -28,23 +27,19 @@ import {
|
|
|
28
27
|
listRelatedArchiveObjects,
|
|
29
28
|
moveChapter,
|
|
30
29
|
packArchiveContext,
|
|
31
|
-
parseLocatedWikiGraphUri,
|
|
32
|
-
parseWikiGraphLibraryUri,
|
|
30
|
+
parseLocatedWikiGraphUri as parseLocatedWikiGraphUri2,
|
|
33
31
|
readArchivePage,
|
|
34
32
|
readSearchIndexCapabilityStatus,
|
|
35
|
-
readWikiGraphLibraryIndexState,
|
|
36
33
|
rebuildArchiveSearchIndex,
|
|
37
|
-
rebuildWikiGraphLibraryIndex,
|
|
38
34
|
removeChapter,
|
|
39
35
|
resetChapter,
|
|
40
36
|
resolveChapterPathReadonly,
|
|
41
|
-
resolveWikiGraphLibraryArchiveFile,
|
|
42
37
|
setChapterSource,
|
|
43
38
|
setChapterSummary,
|
|
44
39
|
setChapterTitle,
|
|
45
40
|
getChapterTree,
|
|
46
41
|
WikiGraph as WikiGraph2,
|
|
47
|
-
WikiGraphArchiveFile,
|
|
42
|
+
WikiGraphArchiveFile as WikiGraphArchiveFile2,
|
|
48
43
|
TOC_FILE_VERSION,
|
|
49
44
|
writeWikgArchive,
|
|
50
45
|
ObjectMetadataKind
|
|
@@ -54,7 +49,7 @@ import {
|
|
|
54
49
|
import { createHash } from "crypto";
|
|
55
50
|
import { createReadStream } from "fs";
|
|
56
51
|
import { rm as rm2 } from "fs/promises";
|
|
57
|
-
import { basename as
|
|
52
|
+
import { basename as basename3 } from "path";
|
|
58
53
|
import {
|
|
59
54
|
WikiGraph
|
|
60
55
|
} from "wiki-graph-core";
|
|
@@ -383,42 +378,42 @@ var NodeDatabaseConnection = class {
|
|
|
383
378
|
}
|
|
384
379
|
database;
|
|
385
380
|
async close() {
|
|
386
|
-
await new Promise((
|
|
381
|
+
await new Promise((resolve4, reject) => {
|
|
387
382
|
this.database.close((error) => {
|
|
388
383
|
if (error) reject(error);
|
|
389
|
-
else
|
|
384
|
+
else resolve4();
|
|
390
385
|
});
|
|
391
386
|
});
|
|
392
387
|
}
|
|
393
388
|
async execute(sql) {
|
|
394
|
-
await new Promise((
|
|
389
|
+
await new Promise((resolve4, reject) => {
|
|
395
390
|
this.database.exec(sql, (error) => {
|
|
396
391
|
if (error) reject(error);
|
|
397
|
-
else
|
|
392
|
+
else resolve4();
|
|
398
393
|
});
|
|
399
394
|
});
|
|
400
395
|
}
|
|
401
396
|
async queryAll(sql, params = []) {
|
|
402
|
-
return await new Promise((
|
|
397
|
+
return await new Promise((resolve4, reject) => {
|
|
403
398
|
this.database.all(sql, [...params], (error, rows) => {
|
|
404
399
|
if (error) reject(error);
|
|
405
|
-
else
|
|
400
|
+
else resolve4(rows);
|
|
406
401
|
});
|
|
407
402
|
});
|
|
408
403
|
}
|
|
409
404
|
async queryOne(sql, params = []) {
|
|
410
|
-
return await new Promise((
|
|
405
|
+
return await new Promise((resolve4, reject) => {
|
|
411
406
|
this.database.get(sql, [...params], (error, row) => {
|
|
412
407
|
if (error) reject(error);
|
|
413
|
-
else
|
|
408
|
+
else resolve4(row);
|
|
414
409
|
});
|
|
415
410
|
});
|
|
416
411
|
}
|
|
417
412
|
async run(sql, params = []) {
|
|
418
|
-
await new Promise((
|
|
413
|
+
await new Promise((resolve4, reject) => {
|
|
419
414
|
this.database.run(sql, [...params], (error) => {
|
|
420
415
|
if (error) reject(error);
|
|
421
|
-
else
|
|
416
|
+
else resolve4();
|
|
422
417
|
});
|
|
423
418
|
});
|
|
424
419
|
}
|
|
@@ -432,10 +427,10 @@ async function openNativeNodeDatabase(file, flags) {
|
|
|
432
427
|
if (path === void 0) {
|
|
433
428
|
throw new TypeError("The Node database adapter requires a NodeFile");
|
|
434
429
|
}
|
|
435
|
-
return await new Promise((
|
|
430
|
+
return await new Promise((resolve4, reject) => {
|
|
436
431
|
const database = new nodeSqlite3.Database(path, flags, (error) => {
|
|
437
432
|
if (error) reject(error);
|
|
438
|
-
else
|
|
433
|
+
else resolve4(database);
|
|
439
434
|
});
|
|
440
435
|
});
|
|
441
436
|
}
|
|
@@ -444,19 +439,19 @@ async function openNodeZip(file) {
|
|
|
444
439
|
if (path === void 0) {
|
|
445
440
|
throw new TypeError("The Node ZIP adapter requires a NodeFile");
|
|
446
441
|
}
|
|
447
|
-
const zipFile = await new Promise((
|
|
442
|
+
const zipFile = await new Promise((resolve4, reject) => {
|
|
448
443
|
yauzl.open(
|
|
449
444
|
path,
|
|
450
445
|
{ autoClose: false, lazyEntries: true },
|
|
451
446
|
(error, opened) => {
|
|
452
447
|
if (error || opened === void 0)
|
|
453
448
|
reject(error ?? new Error("Cannot open ZIP"));
|
|
454
|
-
else
|
|
449
|
+
else resolve4(opened);
|
|
455
450
|
}
|
|
456
451
|
);
|
|
457
452
|
});
|
|
458
453
|
const entries = await new Promise(
|
|
459
|
-
(
|
|
454
|
+
(resolve4, reject) => {
|
|
460
455
|
const discovered = /* @__PURE__ */ new Map();
|
|
461
456
|
const rejectAndClose = (error) => {
|
|
462
457
|
zipFile.close();
|
|
@@ -466,7 +461,7 @@ async function openNodeZip(file) {
|
|
|
466
461
|
};
|
|
467
462
|
const resolveEntries = () => {
|
|
468
463
|
zipFile.off("error", rejectAndClose);
|
|
469
|
-
|
|
464
|
+
resolve4(discovered);
|
|
470
465
|
};
|
|
471
466
|
zipFile.on("entry", (entry) => {
|
|
472
467
|
if (!entry.fileName.endsWith("/")) {
|
|
@@ -576,12 +571,12 @@ async function collectDecodedNodeStreamRange(input, offset, length) {
|
|
|
576
571
|
return Buffer.concat(chunks, collected);
|
|
577
572
|
}
|
|
578
573
|
async function openNodeZipEntryStream(zipFile, entry, options) {
|
|
579
|
-
return await new Promise((
|
|
574
|
+
return await new Promise((resolve4, reject) => {
|
|
580
575
|
const callback = (error, opened) => {
|
|
581
576
|
if (error || opened === void 0) {
|
|
582
577
|
reject(error ?? new Error(`Cannot read ZIP entry: ${entry.fileName}`));
|
|
583
578
|
} else {
|
|
584
|
-
|
|
579
|
+
resolve4(opened);
|
|
585
580
|
}
|
|
586
581
|
};
|
|
587
582
|
if (options === void 0) zipFile.openReadStream(entry, callback);
|
|
@@ -712,11 +707,11 @@ async function writeNodeStream(input, writer) {
|
|
|
712
707
|
);
|
|
713
708
|
}
|
|
714
709
|
async function collectNodeStream(input) {
|
|
715
|
-
return await new Promise((
|
|
710
|
+
return await new Promise((resolve4, reject) => {
|
|
716
711
|
const chunks = [];
|
|
717
712
|
input.on("data", (chunk) => chunks.push(chunk));
|
|
718
713
|
input.once("error", reject);
|
|
719
|
-
input.once("end", () =>
|
|
714
|
+
input.once("end", () => resolve4(Buffer.concat(chunks)));
|
|
720
715
|
});
|
|
721
716
|
}
|
|
722
717
|
var nodeWikiGraphPlatform = {
|
|
@@ -1385,6 +1380,203 @@ function resolveWikiGraphRuntimePath(path) {
|
|
|
1385
1380
|
return isAbsolute2(path) ? path : resolve2(context.cwd, path);
|
|
1386
1381
|
}
|
|
1387
1382
|
|
|
1383
|
+
// src/archive/target.ts
|
|
1384
|
+
import { realpath, stat as stat2 } from "fs/promises";
|
|
1385
|
+
import { basename as basename2, dirname as dirname2, isAbsolute as isAbsolute3, join as join3, relative, resolve as resolve3 } from "path";
|
|
1386
|
+
import {
|
|
1387
|
+
formatLocatedWikiGraphUri,
|
|
1388
|
+
formatWikiGraphLibraryUri,
|
|
1389
|
+
listWikiGraphLibraries,
|
|
1390
|
+
listWikiGraphLibraryArchives,
|
|
1391
|
+
parseLocatedWikiGraphUri,
|
|
1392
|
+
parseWikiGraphLibraryUri,
|
|
1393
|
+
resolveWikiGraphLibrary,
|
|
1394
|
+
resolveWikiGraphLibraryArchiveFile
|
|
1395
|
+
} from "wiki-graph-core";
|
|
1396
|
+
var WikiGraphArchiveOwnershipError = class extends Error {
|
|
1397
|
+
code = "WIKI_GRAPH_ARCHIVE_OWNERSHIP_MISMATCH";
|
|
1398
|
+
constructor(message) {
|
|
1399
|
+
super(message);
|
|
1400
|
+
this.name = "WikiGraphArchiveOwnershipError";
|
|
1401
|
+
}
|
|
1402
|
+
};
|
|
1403
|
+
async function resolveWikiGraphArchiveLocation(target) {
|
|
1404
|
+
if (target.kind === "standalone") {
|
|
1405
|
+
return await resolveStandaloneLocation(target);
|
|
1406
|
+
}
|
|
1407
|
+
return await resolveLibraryLocation(target);
|
|
1408
|
+
}
|
|
1409
|
+
async function assertStandaloneWikiGraphArchivePath(inputPath) {
|
|
1410
|
+
const path = resolveWikiGraphRuntimePath(inputPath);
|
|
1411
|
+
const candidate = await canonicalPath(path);
|
|
1412
|
+
const candidateIdentity = await readFileIdentity(path);
|
|
1413
|
+
const libraries = await listWikiGraphLibraries();
|
|
1414
|
+
const matches = await findMatchingMembers(
|
|
1415
|
+
libraries,
|
|
1416
|
+
candidate,
|
|
1417
|
+
candidateIdentity
|
|
1418
|
+
);
|
|
1419
|
+
if (matches.length > 0) {
|
|
1420
|
+
const match = requireUnambiguousManagedMatch(path, matches);
|
|
1421
|
+
throw new WikiGraphArchiveOwnershipError(
|
|
1422
|
+
`Standalone archive path belongs to registered library ${formatWikiGraphLibraryUri(
|
|
1423
|
+
match.library.isDefault ? void 0 : match.library.publicId
|
|
1424
|
+
)}: ${path}. Use the library UUID URI ${match.archive.uri} instead.`
|
|
1425
|
+
);
|
|
1426
|
+
}
|
|
1427
|
+
for (const library of libraries) {
|
|
1428
|
+
const rootPath = getNodeResourcePath(library.folder);
|
|
1429
|
+
const root = await canonicalPath(rootPath);
|
|
1430
|
+
if (!isWithin(root, candidate)) continue;
|
|
1431
|
+
throw new WikiGraphArchiveOwnershipError(
|
|
1432
|
+
`Standalone archive path belongs to registered library ${formatWikiGraphLibraryUri(
|
|
1433
|
+
library.isDefault ? void 0 : library.publicId
|
|
1434
|
+
)}: ${path}. Scan the library, then address the archive by its library UUID URI.`
|
|
1435
|
+
);
|
|
1436
|
+
}
|
|
1437
|
+
return path;
|
|
1438
|
+
}
|
|
1439
|
+
function archiveTargetFromLogicalLocator(locator, objectUri) {
|
|
1440
|
+
if (locator.startsWith("wikg://lib/")) {
|
|
1441
|
+
return { kind: "library", uri: withObjectUri(locator, objectUri) };
|
|
1442
|
+
}
|
|
1443
|
+
return {
|
|
1444
|
+
kind: "standalone",
|
|
1445
|
+
path: locator,
|
|
1446
|
+
...objectUri === void 0 ? {} : { objectUri }
|
|
1447
|
+
};
|
|
1448
|
+
}
|
|
1449
|
+
async function resolveWikiGraphArchiveTarget(file) {
|
|
1450
|
+
const path = getNodeResourcePath(file);
|
|
1451
|
+
const candidate = await canonicalPath(path);
|
|
1452
|
+
const candidateIdentity = await readFileIdentity(path);
|
|
1453
|
+
const matches = await findMatchingMembers(
|
|
1454
|
+
await listWikiGraphLibraries(),
|
|
1455
|
+
candidate,
|
|
1456
|
+
candidateIdentity
|
|
1457
|
+
);
|
|
1458
|
+
if (matches.length > 0) {
|
|
1459
|
+
const match = requireUnambiguousManagedMatch(path, matches);
|
|
1460
|
+
return { kind: "library", uri: match.archive.uri };
|
|
1461
|
+
}
|
|
1462
|
+
return { kind: "standalone", path: getNodeResourcePath(file) };
|
|
1463
|
+
}
|
|
1464
|
+
async function resolveStandaloneLocation(target) {
|
|
1465
|
+
if (target.path.startsWith("wikg://lib/")) {
|
|
1466
|
+
throw new WikiGraphArchiveOwnershipError(
|
|
1467
|
+
`A library archive URI cannot be used as a standalone path: ${target.path}`
|
|
1468
|
+
);
|
|
1469
|
+
}
|
|
1470
|
+
const path = await assertStandaloneWikiGraphArchivePath(target.path);
|
|
1471
|
+
const normalized = {
|
|
1472
|
+
kind: "standalone",
|
|
1473
|
+
path,
|
|
1474
|
+
...target.objectUri === void 0 ? {} : { objectUri: target.objectUri }
|
|
1475
|
+
};
|
|
1476
|
+
return {
|
|
1477
|
+
archiveFile: new NodeFile(path),
|
|
1478
|
+
archiveKey: path,
|
|
1479
|
+
archivePath: path,
|
|
1480
|
+
indexScope: { archiveKey: path, archivePath: path, kind: "archive-index" },
|
|
1481
|
+
locatedUri: formatLocatedWikiGraphUri(path, target.objectUri),
|
|
1482
|
+
target: normalized
|
|
1483
|
+
};
|
|
1484
|
+
}
|
|
1485
|
+
async function resolveLibraryLocation(target) {
|
|
1486
|
+
const parsedLocated = parseLocatedWikiGraphUri(target.uri);
|
|
1487
|
+
const archiveUri = parsedLocated.archivePath ?? target.uri;
|
|
1488
|
+
const parsed = parseWikiGraphLibraryUri(archiveUri);
|
|
1489
|
+
if (parsed?.kind !== "archive") {
|
|
1490
|
+
throw new WikiGraphArchiveOwnershipError(
|
|
1491
|
+
`Library archive target must be a UUID archive URI: ${target.uri}`
|
|
1492
|
+
);
|
|
1493
|
+
}
|
|
1494
|
+
await resolveWikiGraphLibrary(parsed);
|
|
1495
|
+
const archiveFile = await resolveWikiGraphLibraryArchiveFile(archiveUri);
|
|
1496
|
+
const normalized = {
|
|
1497
|
+
kind: "library",
|
|
1498
|
+
uri: target.uri
|
|
1499
|
+
};
|
|
1500
|
+
return {
|
|
1501
|
+
archiveFile,
|
|
1502
|
+
archiveKey: archiveUri,
|
|
1503
|
+
archivePath: archiveUri,
|
|
1504
|
+
indexScope: {
|
|
1505
|
+
archiveKey: archiveUri,
|
|
1506
|
+
archivePath: archiveUri,
|
|
1507
|
+
kind: "archive-index"
|
|
1508
|
+
},
|
|
1509
|
+
libraryArchiveTarget: parsed,
|
|
1510
|
+
locatedUri: target.uri,
|
|
1511
|
+
publicArchiveUri: archiveUri,
|
|
1512
|
+
target: normalized
|
|
1513
|
+
};
|
|
1514
|
+
}
|
|
1515
|
+
function withObjectUri(archiveLocator, objectUri) {
|
|
1516
|
+
if (objectUri === void 0) return archiveLocator;
|
|
1517
|
+
const path = objectUri.replace(/^wikg:\/\//u, "");
|
|
1518
|
+
return `${archiveLocator.replace(/\/$/u, "")}/${path}`;
|
|
1519
|
+
}
|
|
1520
|
+
async function canonicalPath(path) {
|
|
1521
|
+
try {
|
|
1522
|
+
return await realpath(path);
|
|
1523
|
+
} catch {
|
|
1524
|
+
const absolute = resolve3(path);
|
|
1525
|
+
const parent = dirname2(absolute);
|
|
1526
|
+
return parent === absolute ? absolute : join3(await canonicalPath(parent), basename2(absolute));
|
|
1527
|
+
}
|
|
1528
|
+
}
|
|
1529
|
+
function isWithin(root, candidate) {
|
|
1530
|
+
const child = relative(root, candidate);
|
|
1531
|
+
return child === "" || !child.startsWith("..") && !isAbsolute3(child);
|
|
1532
|
+
}
|
|
1533
|
+
async function findMatchingMembers(libraries, candidate, candidateIdentity) {
|
|
1534
|
+
const members = [];
|
|
1535
|
+
for (const library of libraries) {
|
|
1536
|
+
const archives = await listWikiGraphLibraryArchives({
|
|
1537
|
+
isDefault: library.isDefault,
|
|
1538
|
+
kind: "scope",
|
|
1539
|
+
...library.isDefault ? {} : { publicId: library.publicId }
|
|
1540
|
+
});
|
|
1541
|
+
for (const archive of archives) {
|
|
1542
|
+
if (archive.file === void 0) continue;
|
|
1543
|
+
members.push({
|
|
1544
|
+
archive,
|
|
1545
|
+
library,
|
|
1546
|
+
memberPath: getNodeResourcePath(archive.file)
|
|
1547
|
+
});
|
|
1548
|
+
}
|
|
1549
|
+
}
|
|
1550
|
+
const exact = [];
|
|
1551
|
+
for (const member of members) {
|
|
1552
|
+
if (await canonicalPath(member.memberPath) === candidate) {
|
|
1553
|
+
exact.push(member);
|
|
1554
|
+
}
|
|
1555
|
+
}
|
|
1556
|
+
if (exact.length > 0 || candidateIdentity === void 0) return exact;
|
|
1557
|
+
const identityMatches = [];
|
|
1558
|
+
for (const member of members) {
|
|
1559
|
+
if (await readFileIdentity(member.memberPath) === candidateIdentity) {
|
|
1560
|
+
identityMatches.push(member);
|
|
1561
|
+
}
|
|
1562
|
+
}
|
|
1563
|
+
return identityMatches;
|
|
1564
|
+
}
|
|
1565
|
+
function requireUnambiguousManagedMatch(path, matches) {
|
|
1566
|
+
if (matches.length === 1) return matches[0];
|
|
1567
|
+
throw new WikiGraphArchiveOwnershipError(
|
|
1568
|
+
`Archive file identity is ambiguous across managed archives: ${path}. Use an explicit library UUID URI (${matches.map((match) => match.archive.uri).join(", ")}).`
|
|
1569
|
+
);
|
|
1570
|
+
}
|
|
1571
|
+
async function readFileIdentity(path) {
|
|
1572
|
+
try {
|
|
1573
|
+
const value = await stat2(path);
|
|
1574
|
+
return `${value.dev}:${value.ino}`;
|
|
1575
|
+
} catch {
|
|
1576
|
+
return void 0;
|
|
1577
|
+
}
|
|
1578
|
+
}
|
|
1579
|
+
|
|
1388
1580
|
// src/conversions.ts
|
|
1389
1581
|
var WikiGraphConversionManager = class {
|
|
1390
1582
|
#runtime;
|
|
@@ -1421,11 +1613,15 @@ var WikiGraphConversionManager = class {
|
|
|
1421
1613
|
requiresDigest
|
|
1422
1614
|
);
|
|
1423
1615
|
let chapterCount;
|
|
1616
|
+
let chapterPaths;
|
|
1424
1617
|
const write = async (archive) => {
|
|
1425
1618
|
if (requiresDigest) await options.onSourceImported?.();
|
|
1426
1619
|
if (typeof archive.readToc === "function") {
|
|
1427
1620
|
const toc = await archive.readToc();
|
|
1428
1621
|
chapterCount = toc === void 0 ? void 0 : countTocChapters(toc.items);
|
|
1622
|
+
if (options.output.format === "wikg") {
|
|
1623
|
+
chapterPaths = toc === void 0 ? [] : collectTocChapterPaths(toc.items);
|
|
1624
|
+
}
|
|
1429
1625
|
}
|
|
1430
1626
|
await writeArchive(archive, options.output.path, options.output.format);
|
|
1431
1627
|
};
|
|
@@ -1434,7 +1630,9 @@ var WikiGraphConversionManager = class {
|
|
|
1434
1630
|
throw new Error("wikg input requires a file path.");
|
|
1435
1631
|
}
|
|
1436
1632
|
await app.openSession(
|
|
1437
|
-
new NodeFile(
|
|
1633
|
+
new NodeFile(
|
|
1634
|
+
await assertStandaloneWikiGraphArchivePath(options.input.path)
|
|
1635
|
+
),
|
|
1438
1636
|
write
|
|
1439
1637
|
);
|
|
1440
1638
|
} else if ("stream" in options.input) {
|
|
@@ -1477,6 +1675,7 @@ var WikiGraphConversionManager = class {
|
|
|
1477
1675
|
const sourceArtifact = await resolveSourceArtifact(options.input);
|
|
1478
1676
|
return {
|
|
1479
1677
|
...chapterCount === void 0 ? {} : { chapterCount },
|
|
1678
|
+
...chapterPaths === void 0 ? {} : { chapterPaths },
|
|
1480
1679
|
inputFormat: options.input.format,
|
|
1481
1680
|
outputFormat: options.output.format,
|
|
1482
1681
|
outputPath: options.output.path,
|
|
@@ -1493,7 +1692,8 @@ async function prepareDigestDirectory(path, required) {
|
|
|
1493
1692
|
return new NodeDirectory(resolved);
|
|
1494
1693
|
}
|
|
1495
1694
|
async function writeArchive(archive, path, format) {
|
|
1496
|
-
const
|
|
1695
|
+
const outputPath = format === "wikg" ? await assertStandaloneWikiGraphArchivePath(path) : resolveWikiGraphRuntimePath(path);
|
|
1696
|
+
const file = new NodeFile(outputPath);
|
|
1497
1697
|
if (format === "epub") await archive.exportEpub(file);
|
|
1498
1698
|
else if (format === "wikg") await archive.saveAs(file);
|
|
1499
1699
|
else if (format === "pcex") throw new Error("pcex output is not supported.");
|
|
@@ -1522,7 +1722,7 @@ async function resolveSourceArtifact(input) {
|
|
|
1522
1722
|
return {
|
|
1523
1723
|
digest: hash.digest("hex"),
|
|
1524
1724
|
mediaType: "application/epub+zip",
|
|
1525
|
-
name:
|
|
1725
|
+
name: basename3(path)
|
|
1526
1726
|
};
|
|
1527
1727
|
}
|
|
1528
1728
|
function isNodeErrorCode(error, code) {
|
|
@@ -1538,6 +1738,21 @@ function countTocChapters(items) {
|
|
|
1538
1738
|
}
|
|
1539
1739
|
return count;
|
|
1540
1740
|
}
|
|
1741
|
+
function collectTocChapterPaths(items, ancestorKeys = []) {
|
|
1742
|
+
const paths = [];
|
|
1743
|
+
for (const item of items) {
|
|
1744
|
+
const key = item.key ?? `chapter-${item.serialId ?? "group"}`;
|
|
1745
|
+
const chapterPath = [...ancestorKeys, key];
|
|
1746
|
+
if (item.serialId !== void 0) paths.push(chapterPath.join("/"));
|
|
1747
|
+
paths.push(
|
|
1748
|
+
...collectTocChapterPaths(
|
|
1749
|
+
item.children,
|
|
1750
|
+
chapterPath
|
|
1751
|
+
)
|
|
1752
|
+
);
|
|
1753
|
+
}
|
|
1754
|
+
return paths;
|
|
1755
|
+
}
|
|
1541
1756
|
|
|
1542
1757
|
// src/query-runtime.ts
|
|
1543
1758
|
import {
|
|
@@ -1734,499 +1949,130 @@ import {
|
|
|
1734
1949
|
isArchiveSearchIndexCurrent,
|
|
1735
1950
|
listChapters
|
|
1736
1951
|
} from "wiki-graph-core";
|
|
1737
|
-
|
|
1738
|
-
|
|
1739
|
-
|
|
1740
|
-
var DEFAULT_GENERATION_REQUEST_CONCURRENCY = 6;
|
|
1741
|
-
function planGenerationTask(task, words, chapters, concurrent, model) {
|
|
1742
|
-
const profile = getGenerationPlanningProfile(task);
|
|
1743
|
-
const calls = Math.max(chapters, Math.ceil(words / profile.wordsPerCall));
|
|
1744
|
-
const effectiveConcurrency = Math.max(
|
|
1745
|
-
1,
|
|
1746
|
-
task === "reading-graph" ? 1 : concurrent.request
|
|
1747
|
-
);
|
|
1748
|
-
const callBatches = Math.ceil(calls / effectiveConcurrency);
|
|
1749
|
-
const wordSeconds = words * profile.secondsPerWord;
|
|
1750
|
-
return {
|
|
1751
|
-
model,
|
|
1752
|
-
timeSeconds: {
|
|
1753
|
-
max: Math.ceil(callBatches * profile.maxSecondsPerCall + wordSeconds),
|
|
1754
|
-
min: Math.ceil(callBatches * profile.minSecondsPerCall + wordSeconds)
|
|
1755
|
-
},
|
|
1756
|
-
tokens: {
|
|
1757
|
-
cacheableInput: Math.ceil(words * profile.cacheableInputTokenPerWord),
|
|
1758
|
-
input: Math.ceil(words * profile.inputTokenPerWord),
|
|
1759
|
-
output: Math.ceil(words * profile.outputTokenPerWord)
|
|
1760
|
-
}
|
|
1761
|
-
};
|
|
1762
|
-
}
|
|
1763
|
-
function createGenerationPerformanceHints(input) {
|
|
1764
|
-
if (!input.hasJobWork) {
|
|
1765
|
-
return [];
|
|
1766
|
-
}
|
|
1767
|
-
const hints = [];
|
|
1768
|
-
const recommendedRequest = input.concurrent.request < 4 ? DEFAULT_GENERATION_REQUEST_CONCURRENCY : input.concurrent.request < 8 ? 8 : void 0;
|
|
1769
|
-
if (input.hasRequestWork && recommendedRequest !== void 0) {
|
|
1770
|
-
hints.push({
|
|
1771
|
-
configKey: "request",
|
|
1772
|
-
configValue: recommendedRequest,
|
|
1773
|
-
current: input.concurrent.request,
|
|
1774
|
-
kind: "request",
|
|
1775
|
-
message: "LLM request concurrency can often be higher. Use at least 4; 6-8 is usually faster when the provider allows it.",
|
|
1776
|
-
recommended: recommendedRequest
|
|
1777
|
-
});
|
|
1778
|
-
}
|
|
1779
|
-
if (input.concurrent.job < 4 && input.chapters > 1) {
|
|
1780
|
-
hints.push({
|
|
1781
|
-
configKey: "job",
|
|
1782
|
-
configValue: 4,
|
|
1783
|
-
current: input.concurrent.job,
|
|
1784
|
-
kind: "job",
|
|
1785
|
-
message: "Multiple chapters need work and job concurrency is below 4. Raising it lets chapter jobs run in parallel.",
|
|
1786
|
-
recommended: 4
|
|
1787
|
-
});
|
|
1788
|
-
}
|
|
1789
|
-
return hints;
|
|
1790
|
-
}
|
|
1791
|
-
function formatGenerationPlanningModel(llm) {
|
|
1792
|
-
if (llm?.model === void 0) {
|
|
1793
|
-
return "not configured";
|
|
1794
|
-
}
|
|
1795
|
-
return llm.provider === void 0 ? llm.model : `${llm.provider}/${llm.model}`;
|
|
1796
|
-
}
|
|
1797
|
-
function formatGenerationPlanningDuration(seconds) {
|
|
1798
|
-
if (seconds < 60) {
|
|
1799
|
-
return `${seconds}s`;
|
|
1800
|
-
}
|
|
1801
|
-
const minutes = Math.round(seconds / 60);
|
|
1802
|
-
if (minutes < 60) {
|
|
1803
|
-
return `${minutes}m`;
|
|
1804
|
-
}
|
|
1805
|
-
return `${Math.round(minutes / 60)}h`;
|
|
1806
|
-
}
|
|
1807
|
-
function getGenerationPlanningProfile(task) {
|
|
1808
|
-
switch (task) {
|
|
1809
|
-
case "index-fts":
|
|
1810
|
-
return {
|
|
1811
|
-
cacheableInputTokenPerWord: 0,
|
|
1812
|
-
inputTokenPerWord: 0,
|
|
1813
|
-
maxSecondsPerCall: 1,
|
|
1814
|
-
minSecondsPerCall: 0,
|
|
1815
|
-
outputTokenPerWord: 0,
|
|
1816
|
-
secondsPerWord: 5e-4,
|
|
1817
|
-
wordsPerCall: 1e5
|
|
1818
|
-
};
|
|
1819
|
-
case "index-embedding-source":
|
|
1820
|
-
case "index-embedding-summary":
|
|
1821
|
-
return {
|
|
1822
|
-
cacheableInputTokenPerWord: 0,
|
|
1823
|
-
inputTokenPerWord: 0,
|
|
1824
|
-
maxSecondsPerCall: 10,
|
|
1825
|
-
minSecondsPerCall: 1,
|
|
1826
|
-
outputTokenPerWord: 0,
|
|
1827
|
-
secondsPerWord: 2e-3,
|
|
1828
|
-
wordsPerCall: 1e4
|
|
1829
|
-
};
|
|
1830
|
-
case "knowledge-graph":
|
|
1831
|
-
return {
|
|
1832
|
-
cacheableInputTokenPerWord: 60,
|
|
1833
|
-
inputTokenPerWord: 80,
|
|
1834
|
-
maxSecondsPerCall: 35,
|
|
1835
|
-
minSecondsPerCall: 15,
|
|
1836
|
-
outputTokenPerWord: 25,
|
|
1837
|
-
secondsPerWord: 0.02,
|
|
1838
|
-
wordsPerCall: 35
|
|
1839
|
-
};
|
|
1840
|
-
case "reading-graph":
|
|
1841
|
-
return {
|
|
1842
|
-
cacheableInputTokenPerWord: 20,
|
|
1843
|
-
inputTokenPerWord: 25,
|
|
1844
|
-
maxSecondsPerCall: 45,
|
|
1845
|
-
minSecondsPerCall: 15,
|
|
1846
|
-
outputTokenPerWord: 4,
|
|
1847
|
-
secondsPerWord: 0.01,
|
|
1848
|
-
wordsPerCall: 160
|
|
1849
|
-
};
|
|
1850
|
-
case "reading-summary":
|
|
1851
|
-
return {
|
|
1852
|
-
cacheableInputTokenPerWord: 45,
|
|
1853
|
-
inputTokenPerWord: 55,
|
|
1854
|
-
maxSecondsPerCall: 25,
|
|
1855
|
-
minSecondsPerCall: 10,
|
|
1856
|
-
outputTokenPerWord: 4,
|
|
1857
|
-
secondsPerWord: 0.01,
|
|
1858
|
-
wordsPerCall: 110
|
|
1859
|
-
};
|
|
1860
|
-
}
|
|
1861
|
-
}
|
|
1862
|
-
|
|
1863
|
-
// src/archive/inspect.ts
|
|
1864
|
-
async function createWikiGraphArchiveInspectReport(document, options) {
|
|
1865
|
-
const { archiveUri } = options;
|
|
1866
|
-
const [
|
|
1867
|
-
chapters,
|
|
1868
|
-
summaryWords,
|
|
1869
|
-
indexCacheCurrent,
|
|
1870
|
-
config,
|
|
1871
|
-
ftsArtifactCoverage,
|
|
1872
|
-
sourceEmbeddingArtifactCoverage,
|
|
1873
|
-
summaryEmbeddingArtifactCoverage
|
|
1874
|
-
] = await Promise.all([
|
|
1875
|
-
readInspectChapters(document, options.chapterId),
|
|
1876
|
-
readSummaryWords(document, options.chapterId),
|
|
1952
|
+
async function inspectWikiGraphArchive(document, options) {
|
|
1953
|
+
const [allChapters, indexCacheCurrent, summaryWords, ...coverage] = await Promise.all([
|
|
1954
|
+
listChapters(document),
|
|
1877
1955
|
isArchiveSearchIndexCurrent(document),
|
|
1878
|
-
|
|
1879
|
-
|
|
1880
|
-
|
|
1881
|
-
|
|
1956
|
+
readSummaryWords(document),
|
|
1957
|
+
document.indexArtifacts.listCoverage("fts"),
|
|
1958
|
+
document.indexArtifacts.listCoverage("embedding-source"),
|
|
1959
|
+
document.indexArtifacts.listCoverage("embedding-summary")
|
|
1882
1960
|
]);
|
|
1883
|
-
const
|
|
1884
|
-
const
|
|
1885
|
-
|
|
1886
|
-
|
|
1887
|
-
|
|
1961
|
+
const chapters = selectChapters(allChapters, options.chapterId);
|
|
1962
|
+
const coverageByKind = {
|
|
1963
|
+
fts: byChapterId(coverage[0]),
|
|
1964
|
+
sourceEmbedding: byChapterId(coverage[1]),
|
|
1965
|
+
summaryEmbedding: byChapterId(coverage[2])
|
|
1888
1966
|
};
|
|
1889
|
-
const
|
|
1890
|
-
|
|
1891
|
-
|
|
1892
|
-
|
|
1893
|
-
|
|
1894
|
-
|
|
1895
|
-
|
|
1896
|
-
|
|
1897
|
-
|
|
1898
|
-
|
|
1899
|
-
|
|
1900
|
-
|
|
1901
|
-
|
|
1902
|
-
|
|
1903
|
-
|
|
1904
|
-
|
|
1905
|
-
|
|
1906
|
-
|
|
1907
|
-
|
|
1908
|
-
|
|
1909
|
-
|
|
1910
|
-
|
|
1911
|
-
|
|
1912
|
-
|
|
1913
|
-
|
|
1914
|
-
|
|
1915
|
-
|
|
1916
|
-
|
|
1917
|
-
|
|
1918
|
-
|
|
1919
|
-
(chapter) => createInspectChapterReference(archiveUri, chapter)
|
|
1967
|
+
const inspections = await Promise.all(
|
|
1968
|
+
chapters.map(async (chapter) => {
|
|
1969
|
+
const serial = await document.serials.getById(chapter.chapterId);
|
|
1970
|
+
return {
|
|
1971
|
+
capabilities: {
|
|
1972
|
+
knowledgeGraph: {
|
|
1973
|
+
completed: serial?.knowledgeGraphReady === true
|
|
1974
|
+
},
|
|
1975
|
+
readingGraph: { completed: serial?.topologyReady === true },
|
|
1976
|
+
readingSummary: { completed: chapter.stage === "summarized" }
|
|
1977
|
+
},
|
|
1978
|
+
chapter,
|
|
1979
|
+
indexes: {
|
|
1980
|
+
fts: inspectArtifact(
|
|
1981
|
+
"fts",
|
|
1982
|
+
coverageByKind.fts.get(chapter.chapterId)
|
|
1983
|
+
),
|
|
1984
|
+
sourceEmbedding: inspectArtifact(
|
|
1985
|
+
"embedding-source",
|
|
1986
|
+
coverageByKind.sourceEmbedding.get(chapter.chapterId)
|
|
1987
|
+
),
|
|
1988
|
+
summaryEmbedding: inspectArtifact(
|
|
1989
|
+
"embedding-summary",
|
|
1990
|
+
coverageByKind.summaryEmbedding.get(chapter.chapterId)
|
|
1991
|
+
)
|
|
1992
|
+
},
|
|
1993
|
+
revision: serial?.revision ?? 0,
|
|
1994
|
+
summaryWords: summaryWords.get(chapter.chapterId) ?? 0
|
|
1995
|
+
};
|
|
1996
|
+
})
|
|
1920
1997
|
);
|
|
1921
|
-
const
|
|
1922
|
-
const improvements = createInspectImprovements({
|
|
1923
|
-
concurrent,
|
|
1924
|
-
contentChapters,
|
|
1925
|
-
indexCacheCurrent,
|
|
1926
|
-
planningModel,
|
|
1927
|
-
queryBlockedChapters: queryBlockedContentChapters,
|
|
1928
|
-
summaryCovered,
|
|
1929
|
-
knowledgeGraphCovered,
|
|
1930
|
-
readingGraphCovered
|
|
1931
|
-
});
|
|
1932
|
-
const performanceHints = createGenerationPerformanceHints({
|
|
1933
|
-
chapters: Math.max(
|
|
1934
|
-
0,
|
|
1935
|
-
...improvements.map((improvement) => improvement.missingChapters ?? 0)
|
|
1936
|
-
),
|
|
1937
|
-
concurrent,
|
|
1938
|
-
hasJobWork: improvements.some(
|
|
1939
|
-
(improvement) => improvement.planning !== void 0
|
|
1940
|
-
),
|
|
1941
|
-
hasRequestWork: improvements.some(
|
|
1942
|
-
(improvement) => improvement.planning !== void 0
|
|
1943
|
-
)
|
|
1944
|
-
});
|
|
1998
|
+
const selected = inspections[0];
|
|
1945
1999
|
return {
|
|
1946
|
-
archiveUri,
|
|
1947
|
-
|
|
2000
|
+
archiveUri: options.archiveUri,
|
|
2001
|
+
chapters: inspections,
|
|
2002
|
+
localIndexCache: { current: indexCacheCurrent },
|
|
1948
2003
|
scope: options.chapterId === void 0 ? { type: "archive" } : { chapterId: options.chapterId, type: "chapter" },
|
|
1949
|
-
|
|
1950
|
-
chapters: {
|
|
1951
|
-
content: contentChapters.length,
|
|
1952
|
-
planned: chapters.length - contentChapters.length,
|
|
1953
|
-
total: chapters.length
|
|
1954
|
-
},
|
|
1955
|
-
sourceWords,
|
|
1956
|
-
summaryWords
|
|
1957
|
-
},
|
|
1958
|
-
localIndexCache: {
|
|
1959
|
-
current: indexCacheCurrent,
|
|
1960
|
-
...!queryReady ? {
|
|
1961
|
-
blockedBy: "missing-search-artifacts",
|
|
1962
|
-
impact: "The cache cannot be built until every content chapter has a current FTS or source embedding artifact."
|
|
1963
|
-
} : indexCacheCurrent ? {} : {
|
|
1964
|
-
impact: "The next archive query may need to sync index cache first.",
|
|
1965
|
-
resource: "local CPU/disk time only; no provider calls."
|
|
1966
|
-
},
|
|
1967
|
-
status: !queryReady ? "blocked" : indexCacheCurrent ? "current" : "missing-or-outdated",
|
|
1968
|
-
syncAvailable: queryReady && !indexCacheCurrent
|
|
1969
|
-
},
|
|
1970
|
-
coverage: {
|
|
1971
|
-
ftsIndexArtifact: createInspectCoverage(
|
|
1972
|
-
ftsArtifactCovered,
|
|
1973
|
-
contentChapters
|
|
1974
|
-
),
|
|
1975
|
-
knowledgeGraph: createInspectCoverage(
|
|
1976
|
-
knowledgeGraphCovered,
|
|
1977
|
-
contentChapters
|
|
1978
|
-
),
|
|
1979
|
-
readingGraph: createInspectCoverage(readingGraphCovered, contentChapters),
|
|
1980
|
-
summary: createInspectCoverage(summaryCovered, contentChapters),
|
|
1981
|
-
sourceEmbeddingIndexArtifact: createInspectCoverage(
|
|
1982
|
-
sourceEmbeddingArtifactCovered,
|
|
1983
|
-
contentChapters
|
|
1984
|
-
),
|
|
1985
|
-
summaryEmbeddingIndexArtifact: createInspectCoverage(
|
|
1986
|
-
summaryEmbeddingArtifactCovered,
|
|
1987
|
-
contentChapters
|
|
1988
|
-
)
|
|
1989
|
-
},
|
|
1990
|
-
query: {
|
|
1991
|
-
ready: queryReady
|
|
1992
|
-
},
|
|
1993
|
-
searchArtifacts: {
|
|
1994
|
-
blockedChapters: queryBlockedChapters,
|
|
1995
|
-
...!queryReady ? {
|
|
1996
|
-
resource: "local CPU/disk time only; no provider calls."
|
|
1997
|
-
} : {},
|
|
1998
|
-
ready: queryReady,
|
|
1999
|
-
status: queryReady ? "ready" : "incomplete"
|
|
2000
|
-
},
|
|
2001
|
-
retrievalGuidance: formatRetrievalGuidance({
|
|
2002
|
-
indexCacheCurrent,
|
|
2003
|
-
knowledgeGraphCovered,
|
|
2004
|
-
readingGraphCovered,
|
|
2005
|
-
contentChapters,
|
|
2006
|
-
queryReady,
|
|
2007
|
-
sourceWords
|
|
2008
|
-
}),
|
|
2009
|
-
improvements,
|
|
2010
|
-
performanceHints,
|
|
2011
|
-
summaryCoverageNote: "Summary coverage means a current summary artifact exists for the chapter; it does not guarantee that every source topic is represented in the summary text."
|
|
2012
|
-
};
|
|
2013
|
-
}
|
|
2014
|
-
function createInspectChapterReference(archiveUri, chapter) {
|
|
2015
|
-
return {
|
|
2016
|
-
chapterId: chapter.chapterId,
|
|
2017
|
-
locatedUri: formatInspectLocatedChapterUri(archiveUri, chapter),
|
|
2018
|
-
title: chapter.title,
|
|
2019
|
-
uri: chapter.uri
|
|
2004
|
+
uri: selected === void 0 || options.chapterId === void 0 ? options.archiveUri : formatLocatedChapterUri(options.archiveUri, selected.chapter)
|
|
2020
2005
|
};
|
|
2021
2006
|
}
|
|
2022
|
-
function
|
|
2023
|
-
|
|
2024
|
-
|
|
2025
|
-
async function readInspectChapters(document, chapterId) {
|
|
2026
|
-
const chapters = chapterId === void 0 ? await listChapters(document) : (await listChapters(document)).filter(
|
|
2007
|
+
function selectChapters(chapters, chapterId) {
|
|
2008
|
+
if (chapterId === void 0) return chapters;
|
|
2009
|
+
const selected = chapters.filter(
|
|
2027
2010
|
(chapter) => chapter.chapterId === chapterId
|
|
2028
2011
|
);
|
|
2029
|
-
if (
|
|
2012
|
+
if (selected.length === 0) {
|
|
2030
2013
|
throw new Error(`Chapter ${chapterId} does not exist.`);
|
|
2031
2014
|
}
|
|
2032
|
-
return
|
|
2033
|
-
chapters.map(async (chapter) => {
|
|
2034
|
-
const serial = await document.serials.getById(chapter.chapterId);
|
|
2035
|
-
return {
|
|
2036
|
-
...chapter,
|
|
2037
|
-
knowledgeGraphReady: serial?.knowledgeGraphReady === true,
|
|
2038
|
-
readingGraphReady: serial?.topologyReady === true,
|
|
2039
|
-
summaryReady: chapter.stage === "summarized"
|
|
2040
|
-
};
|
|
2041
|
-
})
|
|
2042
|
-
);
|
|
2043
|
-
}
|
|
2044
|
-
async function readSummaryWords(document, chapterId) {
|
|
2045
|
-
return await document.readDatabase(
|
|
2046
|
-
async (database) => await database.queryOne(
|
|
2047
|
-
`
|
|
2048
|
-
SELECT COALESCE(SUM(words_count), 0) AS words
|
|
2049
|
-
FROM text_sentence_records
|
|
2050
|
-
WHERE kind = 2
|
|
2051
|
-
${chapterId === void 0 ? "" : "AND chapter_id = ?"}
|
|
2052
|
-
`,
|
|
2053
|
-
chapterId === void 0 ? void 0 : [chapterId],
|
|
2054
|
-
(row) => Number(row.words)
|
|
2055
|
-
) ?? 0
|
|
2056
|
-
);
|
|
2015
|
+
return selected;
|
|
2057
2016
|
}
|
|
2058
|
-
|
|
2059
|
-
|
|
2060
|
-
return chapterId === void 0 ? coverage : coverage.filter((item) => item.serialId === chapterId);
|
|
2017
|
+
function byChapterId(coverage) {
|
|
2018
|
+
return new Map(coverage.map((item) => [item.serialId, item]));
|
|
2061
2019
|
}
|
|
2062
|
-
function
|
|
2063
|
-
const currentChapterIds = new Set(
|
|
2064
|
-
coverage.filter((item) => item.current).map((item) => item.serialId)
|
|
2065
|
-
);
|
|
2066
|
-
return chapters.filter((chapter) => currentChapterIds.has(chapter.chapterId));
|
|
2067
|
-
}
|
|
2068
|
-
function createInspectCoverage(covered, total) {
|
|
2069
|
-
const coveredWords = sumWords(covered);
|
|
2070
|
-
const totalWords = sumWords(total);
|
|
2020
|
+
function inspectArtifact(kind, coverage) {
|
|
2071
2021
|
return {
|
|
2072
|
-
|
|
2073
|
-
|
|
2074
|
-
|
|
2075
|
-
|
|
2076
|
-
totalWords
|
|
2022
|
+
current: coverage?.current === true,
|
|
2023
|
+
exists: coverage?.sourceRevision !== void 0,
|
|
2024
|
+
kind,
|
|
2025
|
+
...coverage?.sourceRevision === void 0 ? {} : { sourceRevision: coverage.sourceRevision }
|
|
2077
2026
|
};
|
|
2078
2027
|
}
|
|
2079
|
-
function
|
|
2080
|
-
|
|
2081
|
-
|
|
2082
|
-
|
|
2083
|
-
|
|
2084
|
-
|
|
2085
|
-
|
|
2086
|
-
|
|
2087
|
-
|
|
2088
|
-
|
|
2089
|
-
|
|
2090
|
-
|
|
2091
|
-
|
|
2092
|
-
|
|
2093
|
-
input.contentChapters
|
|
2094
|
-
)
|
|
2095
|
-
);
|
|
2096
|
-
lines.push(
|
|
2097
|
-
formatObjectSearchGuidance(
|
|
2098
|
-
"Entity/triple retrieval",
|
|
2099
|
-
input.knowledgeGraphCovered,
|
|
2100
|
-
input.contentChapters
|
|
2028
|
+
async function readSummaryWords(document) {
|
|
2029
|
+
const rows = await document.readDatabase(
|
|
2030
|
+
async (database) => await database.queryAll(
|
|
2031
|
+
`
|
|
2032
|
+
SELECT chapter_id, COALESCE(SUM(words_count), 0) AS words
|
|
2033
|
+
FROM text_sentence_records
|
|
2034
|
+
WHERE kind = 2
|
|
2035
|
+
GROUP BY chapter_id
|
|
2036
|
+
`,
|
|
2037
|
+
void 0,
|
|
2038
|
+
(row) => ({
|
|
2039
|
+
chapterId: Number(row.chapter_id),
|
|
2040
|
+
words: Number(row.words)
|
|
2041
|
+
})
|
|
2101
2042
|
)
|
|
2102
2043
|
);
|
|
2103
|
-
return
|
|
2104
|
-
}
|
|
2105
|
-
function formatObjectSearchGuidance(label, covered, total) {
|
|
2106
|
-
const coveredWords = sumWords(covered);
|
|
2107
|
-
const totalWords = sumWords(total);
|
|
2108
|
-
const ratio = totalWords === 0 ? 0 : coveredWords / totalWords;
|
|
2109
|
-
const coverage = formatPercent(coveredWords, totalWords);
|
|
2110
|
-
if (ratio >= 1) {
|
|
2111
|
-
return `${label}: covers all source content; it can represent the full scope for object retrieval.`;
|
|
2112
|
-
}
|
|
2113
|
-
if (ratio >= 0.9) {
|
|
2114
|
-
return `${label}: covers ${coverage}; use it as the main path, but source --query is needed for uncovered content.`;
|
|
2115
|
-
}
|
|
2116
|
-
if (ratio >= 0.5) {
|
|
2117
|
-
return `${label}: covers ${coverage}; use it as leads, not as a full-scope substitute for source --query.`;
|
|
2118
|
-
}
|
|
2119
|
-
return `${label}: covers ${coverage}; build missing graph coverage before relying on object retrieval.`;
|
|
2120
|
-
}
|
|
2121
|
-
function createInspectImprovements(input) {
|
|
2122
|
-
const improvements = [];
|
|
2123
|
-
if (input.contentChapters.length === 0) {
|
|
2124
|
-
improvements.push({
|
|
2125
|
-
kind: "add-source",
|
|
2126
|
-
recommendation: "No source content is available yet; add or import source text before graph or summary generation.",
|
|
2127
|
-
title: "Add source content"
|
|
2128
|
-
});
|
|
2129
|
-
return improvements;
|
|
2130
|
-
}
|
|
2131
|
-
if (input.queryBlockedChapters.length > 0) {
|
|
2132
|
-
improvements.push({
|
|
2133
|
-
kind: "build",
|
|
2134
|
-
missingChapters: input.queryBlockedChapters.length,
|
|
2135
|
-
missingWords: sumWords(input.queryBlockedChapters),
|
|
2136
|
-
recommendation: "Build local FTS artifacts to make this scope queryable. This does not call a provider.",
|
|
2137
|
-
task: "index-fts",
|
|
2138
|
-
title: "Enable query with local FTS"
|
|
2139
|
-
});
|
|
2140
|
-
} else if (!input.indexCacheCurrent) {
|
|
2141
|
-
improvements.push({
|
|
2142
|
-
kind: "sync-index",
|
|
2143
|
-
recommendation: "Sync the index cache from chapter index artifacts so query starts without a lazy cache rebuild.",
|
|
2144
|
-
title: "Sync index cache"
|
|
2145
|
-
});
|
|
2146
|
-
}
|
|
2147
|
-
improvements.push(
|
|
2148
|
-
...createGraphImprovement({
|
|
2149
|
-
concurrent: input.concurrent,
|
|
2150
|
-
covered: input.readingGraphCovered,
|
|
2151
|
-
planningModel: input.planningModel,
|
|
2152
|
-
task: "reading-graph",
|
|
2153
|
-
title: "Complete Reading Graph coverage",
|
|
2154
|
-
total: input.contentChapters
|
|
2155
|
-
})
|
|
2156
|
-
);
|
|
2157
|
-
improvements.push(
|
|
2158
|
-
...createGraphImprovement({
|
|
2159
|
-
concurrent: input.concurrent,
|
|
2160
|
-
covered: input.knowledgeGraphCovered,
|
|
2161
|
-
planningModel: input.planningModel,
|
|
2162
|
-
task: "knowledge-graph",
|
|
2163
|
-
title: "Complete Knowledge Graph coverage",
|
|
2164
|
-
total: input.contentChapters
|
|
2165
|
-
})
|
|
2166
|
-
);
|
|
2167
|
-
improvements.push(
|
|
2168
|
-
...createGraphImprovement({
|
|
2169
|
-
concurrent: input.concurrent,
|
|
2170
|
-
covered: input.summaryCovered,
|
|
2171
|
-
planningModel: input.planningModel,
|
|
2172
|
-
task: "reading-summary",
|
|
2173
|
-
title: "Complete Summary coverage",
|
|
2174
|
-
total: input.contentChapters
|
|
2175
|
-
})
|
|
2176
|
-
);
|
|
2177
|
-
return improvements;
|
|
2044
|
+
return new Map(rows.map((row) => [row.chapterId, row.words]));
|
|
2178
2045
|
}
|
|
2179
|
-
function
|
|
2180
|
-
|
|
2181
|
-
const missing = input.total.filter(
|
|
2182
|
-
(chapter) => !coveredIds.has(chapter.chapterId)
|
|
2183
|
-
);
|
|
2184
|
-
if (missing.length === 0) {
|
|
2185
|
-
return [];
|
|
2186
|
-
}
|
|
2187
|
-
const missingWords = sumWords(missing);
|
|
2188
|
-
return [
|
|
2189
|
-
{
|
|
2190
|
-
kind: "build",
|
|
2191
|
-
missingChapters: missing.length,
|
|
2192
|
-
missingWords,
|
|
2193
|
-
planning: planGenerationTask(
|
|
2194
|
-
input.task,
|
|
2195
|
-
missingWords,
|
|
2196
|
-
missing.length,
|
|
2197
|
-
input.concurrent,
|
|
2198
|
-
input.planningModel
|
|
2199
|
-
),
|
|
2200
|
-
recommendation: formatImprovementRecommendation(
|
|
2201
|
-
input.covered,
|
|
2202
|
-
input.total,
|
|
2203
|
-
missingWords
|
|
2204
|
-
),
|
|
2205
|
-
task: input.task,
|
|
2206
|
-
title: input.title
|
|
2207
|
-
}
|
|
2208
|
-
];
|
|
2209
|
-
}
|
|
2210
|
-
function formatImprovementRecommendation(covered, total, missingWords) {
|
|
2211
|
-
const totalWords = sumWords(total);
|
|
2212
|
-
const coveredWords = sumWords(covered);
|
|
2213
|
-
if (totalWords === 0) {
|
|
2214
|
-
return "No source content is available.";
|
|
2215
|
-
}
|
|
2216
|
-
if (total.length === 1 || coveredWords / totalWords >= 0.9 || missingWords <= 1e3) {
|
|
2217
|
-
return "Queue the full scope to finish the remaining gap.";
|
|
2218
|
-
}
|
|
2219
|
-
return "Queue selected chapters first if only part of the scope matters.";
|
|
2220
|
-
}
|
|
2221
|
-
function sumWords(chapters) {
|
|
2222
|
-
return chapters.reduce((total, chapter) => total + chapter.words, 0);
|
|
2046
|
+
function formatLocatedChapterUri(archiveUri, chapter) {
|
|
2047
|
+
return `${archiveUri.replace(/\/+$/u, "")}/${chapter.uri.replace(/^wikg:\/\//u, "")}`;
|
|
2223
2048
|
}
|
|
2224
|
-
|
|
2225
|
-
|
|
2226
|
-
|
|
2049
|
+
|
|
2050
|
+
// src/archive/write.ts
|
|
2051
|
+
import {
|
|
2052
|
+
WikiGraphArchiveFile,
|
|
2053
|
+
writeWikiGraphLibraryArchive
|
|
2054
|
+
} from "wiki-graph-core";
|
|
2055
|
+
async function writeWikiGraphArchiveLocation(location, operation, options = {}) {
|
|
2056
|
+
const cacheOnly = options.searchIndexWritebackPolicy === "cache";
|
|
2057
|
+
const write = async (archiveFile = location.archiveFile) => await new WikiGraphArchiveFile(archiveFile).write(operation, {
|
|
2058
|
+
...cacheOnly ? {} : { derivedStateKeys: [location.archiveKey, location.archivePath] },
|
|
2059
|
+
...options.invalidateContinuationCursors === void 0 ? {} : {
|
|
2060
|
+
invalidateContinuationCursors: options.invalidateContinuationCursors
|
|
2061
|
+
},
|
|
2062
|
+
...options.searchIndexWritebackPolicy === void 0 ? {} : { searchIndexWritebackPolicy: options.searchIndexWritebackPolicy }
|
|
2063
|
+
});
|
|
2064
|
+
if (location.libraryArchiveTarget === void 0 || cacheOnly || options.refreshLibraryIndex === false) {
|
|
2065
|
+
return await write();
|
|
2227
2066
|
}
|
|
2228
|
-
|
|
2229
|
-
|
|
2067
|
+
return await writeWikiGraphLibraryArchive({
|
|
2068
|
+
additionalDerivedStateKeys: [location.archiveKey, location.archivePath],
|
|
2069
|
+
...options.invalidateContinuationCursors === void 0 ? {} : {
|
|
2070
|
+
invalidateContinuationCursors: options.invalidateContinuationCursors
|
|
2071
|
+
},
|
|
2072
|
+
...options.onIndexSyncError === void 0 ? {} : { onIndexSyncError: options.onIndexSyncError },
|
|
2073
|
+
operation: async (archiveFile) => await write(archiveFile),
|
|
2074
|
+
target: location.libraryArchiveTarget
|
|
2075
|
+
});
|
|
2230
2076
|
}
|
|
2231
2077
|
|
|
2232
2078
|
// src/archive/archive.ts
|
|
@@ -2246,13 +2092,13 @@ var WikiGraphArchiveManager = class {
|
|
|
2246
2092
|
}
|
|
2247
2093
|
async create(options) {
|
|
2248
2094
|
return await this.#runtime.run(async () => {
|
|
2249
|
-
const path =
|
|
2095
|
+
const path = await assertStandaloneWikiGraphArchivePath(options.path);
|
|
2250
2096
|
if (options.replace !== true && await nodePathExists(path)) {
|
|
2251
2097
|
throw new WikiGraphArchiveExistsError(path);
|
|
2252
2098
|
}
|
|
2253
|
-
const outputPath =
|
|
2254
|
-
|
|
2255
|
-
`.${
|
|
2099
|
+
const outputPath = join4(
|
|
2100
|
+
dirname3(path),
|
|
2101
|
+
`.${basename4(path)}.${randomUUID2()}.tmp.wikg`
|
|
2256
2102
|
);
|
|
2257
2103
|
try {
|
|
2258
2104
|
if (options.importPath === void 0) {
|
|
@@ -2277,15 +2123,15 @@ var WikiGraphArchiveManager = class {
|
|
|
2277
2123
|
throw error;
|
|
2278
2124
|
}
|
|
2279
2125
|
}
|
|
2280
|
-
return { locatedUri:
|
|
2126
|
+
return { locatedUri: formatLocatedWikiGraphUri2(path), path };
|
|
2281
2127
|
} finally {
|
|
2282
2128
|
await rm3(outputPath, { force: true, recursive: true });
|
|
2283
2129
|
}
|
|
2284
2130
|
}, options.signal);
|
|
2285
2131
|
}
|
|
2286
|
-
async open(
|
|
2132
|
+
async open(target, options = {}) {
|
|
2287
2133
|
const location = await this.#runtime.run(
|
|
2288
|
-
async () => await resolveWikiGraphArchiveLocation(
|
|
2134
|
+
async () => await resolveWikiGraphArchiveLocation(target),
|
|
2289
2135
|
options.signal
|
|
2290
2136
|
);
|
|
2291
2137
|
return new WikiGraphArchiveHandle(this.#runtime, location);
|
|
@@ -2308,14 +2154,17 @@ var WikiGraphArchiveHandle = class {
|
|
|
2308
2154
|
return this.#location.locatedUri;
|
|
2309
2155
|
}
|
|
2310
2156
|
get objectUri() {
|
|
2311
|
-
return
|
|
2157
|
+
return parseLocatedWikiGraphUri2(this.#location.locatedUri).objectUri ?? "wikg://";
|
|
2312
2158
|
}
|
|
2313
2159
|
get path() {
|
|
2314
2160
|
return this.#location.archivePath;
|
|
2315
2161
|
}
|
|
2162
|
+
get target() {
|
|
2163
|
+
return this.#location.target;
|
|
2164
|
+
}
|
|
2316
2165
|
async inspect(options = {}) {
|
|
2317
2166
|
return await this.#readDocument(
|
|
2318
|
-
async (document) => await
|
|
2167
|
+
async (document) => await inspectWikiGraphArchive(document, {
|
|
2319
2168
|
archiveUri: this.#location.publicArchiveUri ?? this.locatedUri,
|
|
2320
2169
|
...options.chapterId === void 0 ? {} : { chapterId: options.chapterId }
|
|
2321
2170
|
}),
|
|
@@ -2616,7 +2465,7 @@ var WikiGraphArchiveHandle = class {
|
|
|
2616
2465
|
objectUri,
|
|
2617
2466
|
async (wikimediaOptions) => await this.#readDocument(
|
|
2618
2467
|
async (document) => await readArchivePage(document, objectUri, {
|
|
2619
|
-
...
|
|
2468
|
+
...options,
|
|
2620
2469
|
...wikimediaOptions
|
|
2621
2470
|
}),
|
|
2622
2471
|
options
|
|
@@ -2762,7 +2611,7 @@ var WikiGraphArchiveHandle = class {
|
|
|
2762
2611
|
}
|
|
2763
2612
|
async #readDocument(operation, options = {}) {
|
|
2764
2613
|
return await this.#runtime.run(
|
|
2765
|
-
async () => await new
|
|
2614
|
+
async () => await new WikiGraphArchiveFile2(this.#location.archiveFile).readDocument(
|
|
2766
2615
|
operation
|
|
2767
2616
|
),
|
|
2768
2617
|
options.signal
|
|
@@ -2770,17 +2619,13 @@ var WikiGraphArchiveHandle = class {
|
|
|
2770
2619
|
}
|
|
2771
2620
|
async #writeDocument(operation, options = {}) {
|
|
2772
2621
|
return await this.#runtime.run(async () => {
|
|
2773
|
-
|
|
2774
|
-
|
|
2775
|
-
|
|
2622
|
+
return await writeWikiGraphArchiveLocation(this.#location, operation, {
|
|
2623
|
+
...options.onIndexSyncError === void 0 ? {} : { onIndexSyncError: options.onIndexSyncError },
|
|
2624
|
+
...options.refreshLibraryIndex === void 0 ? {} : { refreshLibraryIndex: options.refreshLibraryIndex },
|
|
2776
2625
|
...options.searchIndexWritebackPolicy === void 0 ? {} : {
|
|
2777
2626
|
searchIndexWritebackPolicy: options.searchIndexWritebackPolicy
|
|
2778
2627
|
}
|
|
2779
2628
|
});
|
|
2780
|
-
if (options.refreshLibraryIndex !== false) {
|
|
2781
|
-
await this.#refreshLibraryIndex(options.onIndexSyncError);
|
|
2782
|
-
}
|
|
2783
|
-
return result;
|
|
2784
2629
|
}, options.signal);
|
|
2785
2630
|
}
|
|
2786
2631
|
async #resolveScope(document, options) {
|
|
@@ -2800,52 +2645,7 @@ var WikiGraphArchiveHandle = class {
|
|
|
2800
2645
|
}
|
|
2801
2646
|
return queryable;
|
|
2802
2647
|
}
|
|
2803
|
-
async #refreshLibraryIndex(onError) {
|
|
2804
|
-
if (this.#location.libraryDirtyTarget === void 0) return;
|
|
2805
|
-
if (this.#location.libraryArchiveTarget !== void 0) {
|
|
2806
|
-
await finalizeWikiGraphLibraryArchiveWrite({
|
|
2807
|
-
target: this.#location.libraryArchiveTarget
|
|
2808
|
-
});
|
|
2809
|
-
}
|
|
2810
|
-
try {
|
|
2811
|
-
const state = await readWikiGraphLibraryIndexState(
|
|
2812
|
-
this.#location.libraryDirtyTarget
|
|
2813
|
-
);
|
|
2814
|
-
if (state.status !== "missing") {
|
|
2815
|
-
await rebuildWikiGraphLibraryIndex(this.#location.libraryDirtyTarget);
|
|
2816
|
-
}
|
|
2817
|
-
} catch (error) {
|
|
2818
|
-
onError?.(error);
|
|
2819
|
-
}
|
|
2820
|
-
}
|
|
2821
2648
|
};
|
|
2822
|
-
async function resolveWikiGraphArchiveLocation(uriOrPath) {
|
|
2823
|
-
if (!uriOrPath.startsWith("wikg://")) {
|
|
2824
|
-
return createPathLocation(resolveWikiGraphRuntimePath(uriOrPath));
|
|
2825
|
-
}
|
|
2826
|
-
const parsed = parseLocatedWikiGraphUri(uriOrPath);
|
|
2827
|
-
const portableArchiveLocator = parsed.archivePath ?? uriOrPath;
|
|
2828
|
-
const archiveLocator = portableArchiveLocator.startsWith("wikg://lib/") ? portableArchiveLocator : resolveWikiGraphRuntimePath(portableArchiveLocator);
|
|
2829
|
-
const libraryArchiveTarget = archiveLocator.startsWith("wikg://lib/") ? parseWikiGraphLibraryUri(archiveLocator) : void 0;
|
|
2830
|
-
const archiveFile = libraryArchiveTarget?.kind === "archive" ? await resolveWikiGraphLibraryArchiveFile(archiveLocator) : new NodeFile(archiveLocator);
|
|
2831
|
-
const archivePath = getNodeResourcePath(archiveFile);
|
|
2832
|
-
return {
|
|
2833
|
-
archiveFile,
|
|
2834
|
-
archiveKey: archivePath,
|
|
2835
|
-
archivePath,
|
|
2836
|
-
indexScope: { archiveKey: archivePath, archivePath, kind: "archive-index" },
|
|
2837
|
-
...libraryArchiveTarget?.kind === "archive" ? {
|
|
2838
|
-
libraryArchiveTarget,
|
|
2839
|
-
libraryDirtyTarget: {
|
|
2840
|
-
isDefault: libraryArchiveTarget.isDefault,
|
|
2841
|
-
kind: "scope",
|
|
2842
|
-
...libraryArchiveTarget.publicId === void 0 ? {} : { publicId: libraryArchiveTarget.publicId }
|
|
2843
|
-
},
|
|
2844
|
-
publicArchiveUri: portableArchiveLocator
|
|
2845
|
-
} : {},
|
|
2846
|
-
locatedUri: formatLocatedWikiGraphUri(archivePath, parsed.objectUri)
|
|
2847
|
-
};
|
|
2848
|
-
}
|
|
2849
2649
|
async function ensureArchiveSearchIndex(document, options = {}) {
|
|
2850
2650
|
const scope = options.chapters === void 0 ? {} : { chapters: options.chapters };
|
|
2851
2651
|
if (await isArchiveSearchIndexCurrent2(document, scope)) return;
|
|
@@ -2880,22 +2680,13 @@ function selectChapterSubtree(chapters, rootChapterId, depth) {
|
|
|
2880
2680
|
throw new Error(`Chapter ${rootChapterId} does not exist.`);
|
|
2881
2681
|
}
|
|
2882
2682
|
const prefix = `${root.path}/`;
|
|
2883
|
-
return chapters.filter(
|
|
2884
|
-
(chapter) => chapter.chapterId === root.chapterId || chapter.path.startsWith(prefix) && (depth === void 0 || chapter.depth - root.depth <= depth)
|
|
2885
|
-
);
|
|
2886
|
-
}
|
|
2887
|
-
function createPathLocation(path) {
|
|
2888
|
-
return {
|
|
2889
|
-
archiveFile: new NodeFile(path),
|
|
2890
|
-
archiveKey: path,
|
|
2891
|
-
archivePath: path,
|
|
2892
|
-
indexScope: { archiveKey: path, archivePath: path, kind: "archive-index" },
|
|
2893
|
-
locatedUri: formatLocatedWikiGraphUri(path)
|
|
2894
|
-
};
|
|
2683
|
+
return chapters.filter(
|
|
2684
|
+
(chapter) => chapter.chapterId === root.chapterId || chapter.path.startsWith(prefix) && (depth === void 0 || chapter.depth - root.depth <= depth)
|
|
2685
|
+
);
|
|
2895
2686
|
}
|
|
2896
2687
|
async function createEmptyArchiveFile(path) {
|
|
2897
2688
|
const directoryPath = await mkdtemp(
|
|
2898
|
-
|
|
2689
|
+
join4(tmpdir(), "wikigraph-archive-write-")
|
|
2899
2690
|
);
|
|
2900
2691
|
try {
|
|
2901
2692
|
const directory = new NodeDirectory(directoryPath);
|
|
@@ -2917,7 +2708,7 @@ function isNodeEEXISTError(error) {
|
|
|
2917
2708
|
}
|
|
2918
2709
|
async function nodePathExists(path) {
|
|
2919
2710
|
try {
|
|
2920
|
-
await
|
|
2711
|
+
await stat3(path);
|
|
2921
2712
|
return true;
|
|
2922
2713
|
} catch (error) {
|
|
2923
2714
|
if (isNodeENOENTError(error)) return false;
|
|
@@ -3023,12 +2814,13 @@ async function assertChapterResetAllowed(archive, chapterId, stage) {
|
|
|
3023
2814
|
// src/wikispine.ts
|
|
3024
2815
|
import { spawn } from "child_process";
|
|
3025
2816
|
var nodeWikispineCommandRunner = {
|
|
3026
|
-
run: async ({ args, command, input, onStdout, signal }) => await new Promise((
|
|
2817
|
+
run: async ({ args, command, input, onStdout, signal }) => await new Promise((resolve4, reject) => {
|
|
3027
2818
|
signal?.throwIfAborted();
|
|
3028
2819
|
const child = spawn(command, [...args], {
|
|
3029
2820
|
stdio: ["pipe", "pipe", "pipe"]
|
|
3030
2821
|
});
|
|
3031
2822
|
const stderr = [];
|
|
2823
|
+
let stdoutQueue = Promise.resolve();
|
|
3032
2824
|
let settled = false;
|
|
3033
2825
|
const settle = (operation) => {
|
|
3034
2826
|
if (settled) return;
|
|
@@ -3044,14 +2836,15 @@ var nodeWikispineCommandRunner = {
|
|
|
3044
2836
|
child.stdout.setEncoding("utf8");
|
|
3045
2837
|
child.stdout.on("data", (chunk) => {
|
|
3046
2838
|
if (settled) return;
|
|
3047
|
-
|
|
3048
|
-
|
|
3049
|
-
|
|
2839
|
+
child.stdout.pause();
|
|
2840
|
+
stdoutQueue = stdoutQueue.then(async () => await onStdout(chunk)).then(() => {
|
|
2841
|
+
child.stdout.resume();
|
|
2842
|
+
}).catch((error) => {
|
|
3050
2843
|
child.kill();
|
|
3051
2844
|
settle(
|
|
3052
2845
|
() => reject(error instanceof Error ? error : new Error(String(error)))
|
|
3053
2846
|
);
|
|
3054
|
-
}
|
|
2847
|
+
});
|
|
3055
2848
|
});
|
|
3056
2849
|
child.stderr.on("data", (chunk) => stderr.push(chunk));
|
|
3057
2850
|
child.on("error", (error) => {
|
|
@@ -3062,12 +2855,14 @@ var nodeWikispineCommandRunner = {
|
|
|
3062
2855
|
);
|
|
3063
2856
|
});
|
|
3064
2857
|
child.on("close", (exitCode) => {
|
|
3065
|
-
|
|
3066
|
-
(
|
|
3067
|
-
|
|
3068
|
-
|
|
3069
|
-
|
|
3070
|
-
|
|
2858
|
+
void stdoutQueue.then(() => {
|
|
2859
|
+
settle(
|
|
2860
|
+
() => resolve4({
|
|
2861
|
+
exitCode,
|
|
2862
|
+
stderr: Buffer.concat(stderr).toString("utf8")
|
|
2863
|
+
})
|
|
2864
|
+
);
|
|
2865
|
+
});
|
|
3071
2866
|
});
|
|
3072
2867
|
child.stdin.end(input);
|
|
3073
2868
|
})
|
|
@@ -3199,7 +2994,7 @@ import {
|
|
|
3199
2994
|
listWikiGraphLibraryObjects,
|
|
3200
2995
|
readContinuationCursor,
|
|
3201
2996
|
resolveWikiGraphLibraryQueryTargetById,
|
|
3202
|
-
WikiGraphArchiveFile as
|
|
2997
|
+
WikiGraphArchiveFile as WikiGraphArchiveFile3
|
|
3203
2998
|
} from "wiki-graph-core";
|
|
3204
2999
|
var WikiGraphContinuationManager = class {
|
|
3205
3000
|
#runtime;
|
|
@@ -3246,9 +3041,9 @@ var WikiGraphContinuationManager = class {
|
|
|
3246
3041
|
};
|
|
3247
3042
|
async function continueArchiveCursor(cursor, limit) {
|
|
3248
3043
|
const location = await resolveWikiGraphArchiveLocation(
|
|
3249
|
-
getCursorArchivePath(cursor)
|
|
3044
|
+
archiveTargetFromLogicalLocator(getCursorArchivePath(cursor))
|
|
3250
3045
|
);
|
|
3251
|
-
return await new
|
|
3046
|
+
return await new WikiGraphArchiveFile3(location.archiveFile).readDocument(
|
|
3252
3047
|
async (document) => {
|
|
3253
3048
|
switch (cursor.kind) {
|
|
3254
3049
|
case "source-locators":
|
|
@@ -3503,14 +3298,14 @@ import {
|
|
|
3503
3298
|
getChapterDetails,
|
|
3504
3299
|
openWikimediaResolver as openWikimediaResolver2,
|
|
3505
3300
|
recordBuildJobInputRevision,
|
|
3506
|
-
WikiGraphArchiveFile as
|
|
3301
|
+
WikiGraphArchiveFile as WikiGraphArchiveFile5,
|
|
3507
3302
|
WikiGraphScope,
|
|
3508
3303
|
withLoggingContext
|
|
3509
3304
|
} from "wiki-graph-core";
|
|
3510
3305
|
import {
|
|
3511
3306
|
applyChapterJobArtifactFile as applyChapterJobArtifactFile3,
|
|
3512
3307
|
createLocalChapterJobFileExecutor as createLocalChapterJobFileExecutor2,
|
|
3513
|
-
writeChapterJobInputFile as
|
|
3308
|
+
writeChapterJobInputFile as writeChapterJobInputFile3
|
|
3514
3309
|
} from "wiki-graph-core/worker";
|
|
3515
3310
|
|
|
3516
3311
|
// src/stage.ts
|
|
@@ -3548,16 +3343,145 @@ async function loadRequiredStageConfig(options) {
|
|
|
3548
3343
|
return config;
|
|
3549
3344
|
}
|
|
3550
3345
|
|
|
3346
|
+
// src/planning.ts
|
|
3347
|
+
var DEFAULT_GENERATION_JOB_CONCURRENCY = 3;
|
|
3348
|
+
var DEFAULT_GENERATION_REQUEST_CONCURRENCY = 6;
|
|
3349
|
+
function planGenerationTask(task, words, chapters, concurrent, model) {
|
|
3350
|
+
const profile = getGenerationPlanningProfile(task);
|
|
3351
|
+
const calls = Math.max(chapters, Math.ceil(words / profile.wordsPerCall));
|
|
3352
|
+
const effectiveConcurrency = Math.max(
|
|
3353
|
+
1,
|
|
3354
|
+
task === "reading-graph" ? 1 : concurrent.request
|
|
3355
|
+
);
|
|
3356
|
+
const callBatches = Math.ceil(calls / effectiveConcurrency);
|
|
3357
|
+
const wordSeconds = words * profile.secondsPerWord;
|
|
3358
|
+
return {
|
|
3359
|
+
model,
|
|
3360
|
+
timeSeconds: {
|
|
3361
|
+
max: Math.ceil(callBatches * profile.maxSecondsPerCall + wordSeconds),
|
|
3362
|
+
min: Math.ceil(callBatches * profile.minSecondsPerCall + wordSeconds)
|
|
3363
|
+
},
|
|
3364
|
+
tokens: {
|
|
3365
|
+
cacheableInput: Math.ceil(words * profile.cacheableInputTokenPerWord),
|
|
3366
|
+
input: Math.ceil(words * profile.inputTokenPerWord),
|
|
3367
|
+
output: Math.ceil(words * profile.outputTokenPerWord)
|
|
3368
|
+
}
|
|
3369
|
+
};
|
|
3370
|
+
}
|
|
3371
|
+
function createGenerationPerformanceHints(input) {
|
|
3372
|
+
if (!input.hasJobWork) {
|
|
3373
|
+
return [];
|
|
3374
|
+
}
|
|
3375
|
+
const hints = [];
|
|
3376
|
+
const recommendedRequest = input.concurrent.request < 4 ? DEFAULT_GENERATION_REQUEST_CONCURRENCY : input.concurrent.request < 8 ? 8 : void 0;
|
|
3377
|
+
if (input.hasRequestWork && recommendedRequest !== void 0) {
|
|
3378
|
+
hints.push({
|
|
3379
|
+
configKey: "request",
|
|
3380
|
+
configValue: recommendedRequest,
|
|
3381
|
+
current: input.concurrent.request,
|
|
3382
|
+
kind: "request",
|
|
3383
|
+
message: "LLM request concurrency can often be higher. Use at least 4; 6-8 is usually faster when the provider allows it.",
|
|
3384
|
+
recommended: recommendedRequest
|
|
3385
|
+
});
|
|
3386
|
+
}
|
|
3387
|
+
if (input.concurrent.job < 4 && input.chapters > 1) {
|
|
3388
|
+
hints.push({
|
|
3389
|
+
configKey: "job",
|
|
3390
|
+
configValue: 4,
|
|
3391
|
+
current: input.concurrent.job,
|
|
3392
|
+
kind: "job",
|
|
3393
|
+
message: "Multiple chapters need work and job concurrency is below 4. Raising it lets chapter jobs run in parallel.",
|
|
3394
|
+
recommended: 4
|
|
3395
|
+
});
|
|
3396
|
+
}
|
|
3397
|
+
return hints;
|
|
3398
|
+
}
|
|
3399
|
+
function formatGenerationPlanningModel(llm) {
|
|
3400
|
+
if (llm?.model === void 0) {
|
|
3401
|
+
return "not configured";
|
|
3402
|
+
}
|
|
3403
|
+
return llm.provider === void 0 ? llm.model : `${llm.provider}/${llm.model}`;
|
|
3404
|
+
}
|
|
3405
|
+
function formatGenerationPlanningDuration(seconds) {
|
|
3406
|
+
if (seconds < 60) {
|
|
3407
|
+
return `${seconds}s`;
|
|
3408
|
+
}
|
|
3409
|
+
const minutes = Math.round(seconds / 60);
|
|
3410
|
+
if (minutes < 60) {
|
|
3411
|
+
return `${minutes}m`;
|
|
3412
|
+
}
|
|
3413
|
+
return `${Math.round(minutes / 60)}h`;
|
|
3414
|
+
}
|
|
3415
|
+
function getGenerationPlanningProfile(task) {
|
|
3416
|
+
switch (task) {
|
|
3417
|
+
case "index-fts":
|
|
3418
|
+
return {
|
|
3419
|
+
cacheableInputTokenPerWord: 0,
|
|
3420
|
+
inputTokenPerWord: 0,
|
|
3421
|
+
maxSecondsPerCall: 1,
|
|
3422
|
+
minSecondsPerCall: 0,
|
|
3423
|
+
outputTokenPerWord: 0,
|
|
3424
|
+
secondsPerWord: 5e-4,
|
|
3425
|
+
wordsPerCall: 1e5
|
|
3426
|
+
};
|
|
3427
|
+
case "index-embedding-source":
|
|
3428
|
+
case "index-embedding-summary":
|
|
3429
|
+
return {
|
|
3430
|
+
cacheableInputTokenPerWord: 0,
|
|
3431
|
+
inputTokenPerWord: 0,
|
|
3432
|
+
maxSecondsPerCall: 10,
|
|
3433
|
+
minSecondsPerCall: 1,
|
|
3434
|
+
outputTokenPerWord: 0,
|
|
3435
|
+
secondsPerWord: 2e-3,
|
|
3436
|
+
wordsPerCall: 1e4
|
|
3437
|
+
};
|
|
3438
|
+
case "knowledge-graph":
|
|
3439
|
+
return {
|
|
3440
|
+
cacheableInputTokenPerWord: 60,
|
|
3441
|
+
inputTokenPerWord: 80,
|
|
3442
|
+
maxSecondsPerCall: 35,
|
|
3443
|
+
minSecondsPerCall: 15,
|
|
3444
|
+
outputTokenPerWord: 25,
|
|
3445
|
+
secondsPerWord: 0.02,
|
|
3446
|
+
wordsPerCall: 35
|
|
3447
|
+
};
|
|
3448
|
+
case "reading-graph":
|
|
3449
|
+
return {
|
|
3450
|
+
cacheableInputTokenPerWord: 20,
|
|
3451
|
+
inputTokenPerWord: 25,
|
|
3452
|
+
maxSecondsPerCall: 45,
|
|
3453
|
+
minSecondsPerCall: 15,
|
|
3454
|
+
outputTokenPerWord: 4,
|
|
3455
|
+
secondsPerWord: 0.01,
|
|
3456
|
+
wordsPerCall: 160
|
|
3457
|
+
};
|
|
3458
|
+
case "reading-summary":
|
|
3459
|
+
return {
|
|
3460
|
+
cacheableInputTokenPerWord: 45,
|
|
3461
|
+
inputTokenPerWord: 55,
|
|
3462
|
+
maxSecondsPerCall: 25,
|
|
3463
|
+
minSecondsPerCall: 10,
|
|
3464
|
+
outputTokenPerWord: 4,
|
|
3465
|
+
secondsPerWord: 0.01,
|
|
3466
|
+
wordsPerCall: 110
|
|
3467
|
+
};
|
|
3468
|
+
}
|
|
3469
|
+
}
|
|
3470
|
+
|
|
3551
3471
|
// src/worker.ts
|
|
3552
3472
|
import {
|
|
3553
3473
|
applyChapterJobArtifactFile,
|
|
3554
|
-
runBuildJobWorker as runCoreBuildJobWorker
|
|
3474
|
+
runBuildJobWorker as runCoreBuildJobWorker,
|
|
3475
|
+
writeChapterJobInputFile
|
|
3555
3476
|
} from "wiki-graph-core/worker";
|
|
3556
|
-
import {
|
|
3477
|
+
import {
|
|
3478
|
+
resolveChapterPathReadonly as resolveChapterPathReadonly2,
|
|
3479
|
+
WikiGraphArchiveFile as WikiGraphArchiveFile4
|
|
3480
|
+
} from "wiki-graph-core";
|
|
3557
3481
|
import {
|
|
3558
3482
|
applyChapterJobArtifactFile as applyChapterJobArtifactFile2,
|
|
3559
3483
|
createLocalChapterJobFileExecutor,
|
|
3560
|
-
writeChapterJobInputFile
|
|
3484
|
+
writeChapterJobInputFile as writeChapterJobInputFile2
|
|
3561
3485
|
} from "wiki-graph-core/worker";
|
|
3562
3486
|
async function runBuildJobWorker(options) {
|
|
3563
3487
|
installNodeWikiGraphPlatform();
|
|
@@ -3646,10 +3570,10 @@ async function executeStep(job, kind, execution, reporter, context, inputOptions
|
|
|
3646
3570
|
const stepWorkspace = await resetStepWorkspace(job.workspace, kind);
|
|
3647
3571
|
const inputFile = await stepWorkspace.createFile("input.jsonl");
|
|
3648
3572
|
const outputWorkspace = await stepWorkspace.createDirectory("output");
|
|
3649
|
-
const revision = await new
|
|
3573
|
+
const revision = await new WikiGraphArchiveFile5(job.archive).readDocument(
|
|
3650
3574
|
async (document) => {
|
|
3651
3575
|
await getChapterDetails(document, job.chapterId);
|
|
3652
|
-
return await
|
|
3576
|
+
return await writeChapterJobInputFile3(
|
|
3653
3577
|
document,
|
|
3654
3578
|
job.chapterId,
|
|
3655
3579
|
kind,
|
|
@@ -3688,16 +3612,23 @@ async function executeStep(job, kind, execution, reporter, context, inputOptions
|
|
|
3688
3612
|
total: 1,
|
|
3689
3613
|
unit: "item"
|
|
3690
3614
|
});
|
|
3691
|
-
await
|
|
3692
|
-
|
|
3693
|
-
|
|
3694
|
-
|
|
3695
|
-
|
|
3696
|
-
|
|
3697
|
-
|
|
3698
|
-
|
|
3699
|
-
|
|
3700
|
-
|
|
3615
|
+
const target = await resolveWikiGraphArchiveTarget(job.archive);
|
|
3616
|
+
const resolvedLocation = await resolveWikiGraphArchiveLocation(target);
|
|
3617
|
+
const location = target.kind === "standalone" ? { ...resolvedLocation, archiveFile: job.archive } : resolvedLocation;
|
|
3618
|
+
await writeWikiGraphArchiveLocation(
|
|
3619
|
+
location,
|
|
3620
|
+
async (document) => {
|
|
3621
|
+
assertJobStillRunning(await getBuildJob(job.jobId));
|
|
3622
|
+
await applyChapterJobArtifactFile3(
|
|
3623
|
+
document,
|
|
3624
|
+
job.chapterId,
|
|
3625
|
+
kind,
|
|
3626
|
+
revision,
|
|
3627
|
+
result.artifactFile
|
|
3628
|
+
);
|
|
3629
|
+
},
|
|
3630
|
+
{ invalidateContinuationCursors: true }
|
|
3631
|
+
);
|
|
3701
3632
|
await reporter.updatePhase({
|
|
3702
3633
|
done: 1,
|
|
3703
3634
|
phase: "committing",
|
|
@@ -3736,6 +3667,10 @@ async function openJobExecutor(job, config, signal, reporter) {
|
|
|
3736
3667
|
llmRequest: requireValue(
|
|
3737
3668
|
request,
|
|
3738
3669
|
"Local Wikimedia resolution requires an LLM."
|
|
3670
|
+
),
|
|
3671
|
+
llmModelId: requireValue(
|
|
3672
|
+
stageConfig?.llm?.model,
|
|
3673
|
+
"Local Wikimedia resolution requires an LLM model."
|
|
3739
3674
|
)
|
|
3740
3675
|
} : {
|
|
3741
3676
|
endpoint: config.wikimedia.endpoint,
|
|
@@ -3773,12 +3708,12 @@ async function resetStepWorkspace(workspace, kind) {
|
|
|
3773
3708
|
return await workspace.createDirectory(name);
|
|
3774
3709
|
}
|
|
3775
3710
|
async function readChapterStage(job) {
|
|
3776
|
-
return await new
|
|
3711
|
+
return await new WikiGraphArchiveFile5(job.archive).readDocument(
|
|
3777
3712
|
async (document) => (await getChapterDetails(document, job.chapterId)).stage
|
|
3778
3713
|
);
|
|
3779
3714
|
}
|
|
3780
3715
|
async function assertCurrentFtsArtifact(job) {
|
|
3781
|
-
await new
|
|
3716
|
+
await new WikiGraphArchiveFile5(job.archive).readDocument(async (document) => {
|
|
3782
3717
|
const revision = await document.serials.getRevision(job.chapterId);
|
|
3783
3718
|
const artifact = await document.indexArtifacts.get(job.chapterId, "fts");
|
|
3784
3719
|
if (artifact?.sourceRevision !== revision) {
|
|
@@ -3882,7 +3817,7 @@ import {
|
|
|
3882
3817
|
boostBuildJob,
|
|
3883
3818
|
cancelBuildJob,
|
|
3884
3819
|
cleanBuildJobs,
|
|
3885
|
-
formatLocatedChapterUri,
|
|
3820
|
+
formatLocatedChapterUri as formatLocatedChapterUri2,
|
|
3886
3821
|
getBuildJob as getBuildJob2,
|
|
3887
3822
|
listBuildJobs,
|
|
3888
3823
|
listChapters as listChapters3,
|
|
@@ -3891,8 +3826,9 @@ import {
|
|
|
3891
3826
|
resolveBuildJobId,
|
|
3892
3827
|
resumeBuildJob,
|
|
3893
3828
|
updateBuildJobTarget,
|
|
3894
|
-
|
|
3895
|
-
|
|
3829
|
+
withArchiveBuildJobCreationLock,
|
|
3830
|
+
resolveChapterPathReadonly as resolveChapterPathReadonly3,
|
|
3831
|
+
WikiGraphArchiveFile as WikiGraphArchiveFile6
|
|
3896
3832
|
} from "wiki-graph-core";
|
|
3897
3833
|
var TERMINAL_STATES = /* @__PURE__ */ new Set(["canceled", "failed", "succeeded"]);
|
|
3898
3834
|
var TERMINAL_EVENT_GRACE_READS = 3;
|
|
@@ -3937,20 +3873,24 @@ var WikiGraphJobManager = class {
|
|
|
3937
3873
|
this.#backend = backend;
|
|
3938
3874
|
}
|
|
3939
3875
|
async create(options) {
|
|
3940
|
-
const
|
|
3941
|
-
|
|
3942
|
-
|
|
3943
|
-
|
|
3944
|
-
|
|
3945
|
-
|
|
3946
|
-
|
|
3876
|
+
const resolved = await this.#runtime.run(async () => {
|
|
3877
|
+
const location = await resolveWikiGraphArchiveLocation(options.archive);
|
|
3878
|
+
return {
|
|
3879
|
+
job: await this.#backend.add({
|
|
3880
|
+
...options,
|
|
3881
|
+
archive: location.archiveFile
|
|
3882
|
+
}),
|
|
3883
|
+
target: location.target
|
|
3884
|
+
};
|
|
3885
|
+
});
|
|
3886
|
+
return await this.#createHandle(resolved.job, resolved.target);
|
|
3947
3887
|
}
|
|
3948
3888
|
async clean() {
|
|
3949
3889
|
return await this.#runtime.run(async () => await cleanBuildJobs());
|
|
3950
3890
|
}
|
|
3951
3891
|
async get(jobId) {
|
|
3952
|
-
return this.#
|
|
3953
|
-
|
|
3892
|
+
return await this.#runtime.run(
|
|
3893
|
+
async () => await this.#createHandle(await this.#backend.get(jobId))
|
|
3954
3894
|
);
|
|
3955
3895
|
}
|
|
3956
3896
|
async resolveId(reference) {
|
|
@@ -3959,23 +3899,25 @@ var WikiGraphJobManager = class {
|
|
|
3959
3899
|
);
|
|
3960
3900
|
}
|
|
3961
3901
|
async list(options = {}) {
|
|
3962
|
-
|
|
3902
|
+
return await this.#runtime.run(async () => {
|
|
3963
3903
|
const normalized = {
|
|
3964
3904
|
...options.activeOnly === void 0 ? {} : { activeOnly: options.activeOnly },
|
|
3965
3905
|
...options.all === void 0 ? {} : { all: options.all },
|
|
3966
3906
|
...options.archive === void 0 ? {} : {
|
|
3967
|
-
archive:
|
|
3907
|
+
archive: (await resolveWikiGraphArchiveLocation(options.archive)).archiveFile
|
|
3968
3908
|
}
|
|
3969
3909
|
};
|
|
3970
|
-
|
|
3910
|
+
const jobs = await this.#backend.list(normalized);
|
|
3911
|
+
return await Promise.all(
|
|
3912
|
+
jobs.map(async (job) => await this.#createHandle(job))
|
|
3913
|
+
);
|
|
3971
3914
|
});
|
|
3972
|
-
return jobs.map((job) => this.#createHandle(job));
|
|
3973
3915
|
}
|
|
3974
3916
|
async resolveChapters(jobs) {
|
|
3975
3917
|
return await this.#runtime.run(async () => {
|
|
3976
3918
|
const jobsByArchive = /* @__PURE__ */ new Map();
|
|
3977
3919
|
for (const job of jobs) {
|
|
3978
|
-
const archivePath =
|
|
3920
|
+
const archivePath = getJobArchiveLocator(job);
|
|
3979
3921
|
const grouped = jobsByArchive.get(archivePath) ?? [];
|
|
3980
3922
|
grouped.push(job);
|
|
3981
3923
|
jobsByArchive.set(archivePath, grouped);
|
|
@@ -3983,8 +3925,11 @@ var WikiGraphJobManager = class {
|
|
|
3983
3925
|
const entries = (await Promise.all(
|
|
3984
3926
|
[...jobsByArchive].map(async ([archivePath, archiveJobs]) => {
|
|
3985
3927
|
try {
|
|
3986
|
-
const
|
|
3987
|
-
|
|
3928
|
+
const location = await resolveWikiGraphArchiveLocation(
|
|
3929
|
+
archiveJobs[0].archive
|
|
3930
|
+
);
|
|
3931
|
+
const chapters = await new WikiGraphArchiveFile6(
|
|
3932
|
+
location.archiveFile
|
|
3988
3933
|
).readDocument(async (document) => await listChapters3(document));
|
|
3989
3934
|
const chaptersById = new Map(
|
|
3990
3935
|
chapters.map((chapter) => [chapter.chapterId, chapter])
|
|
@@ -3995,7 +3940,7 @@ var WikiGraphJobManager = class {
|
|
|
3995
3940
|
[
|
|
3996
3941
|
job.jobId,
|
|
3997
3942
|
{
|
|
3998
|
-
locatedUri:
|
|
3943
|
+
locatedUri: formatLocatedChapterUri2(
|
|
3999
3944
|
archivePath,
|
|
4000
3945
|
chapter.path
|
|
4001
3946
|
),
|
|
@@ -4019,62 +3964,78 @@ var WikiGraphJobManager = class {
|
|
|
4019
3964
|
async enqueue(options) {
|
|
4020
3965
|
return await this.#runtime.run(async () => {
|
|
4021
3966
|
const location = await resolveWikiGraphArchiveLocation(options.archive);
|
|
4022
|
-
|
|
4023
|
-
|
|
4024
|
-
|
|
4025
|
-
|
|
4026
|
-
|
|
4027
|
-
|
|
4028
|
-
|
|
4029
|
-
|
|
4030
|
-
|
|
4031
|
-
|
|
4032
|
-
|
|
4033
|
-
|
|
4034
|
-
|
|
4035
|
-
|
|
4036
|
-
return checked;
|
|
4037
|
-
});
|
|
4038
|
-
const created = [];
|
|
4039
|
-
const skipped = [];
|
|
4040
|
-
for (const candidate of candidates) {
|
|
4041
|
-
if (candidate.reason !== void 0) {
|
|
4042
|
-
if (selectedExplicitly && candidates.length === 1) {
|
|
4043
|
-
throw new Error(
|
|
4044
|
-
formatQueueReadinessError(
|
|
4045
|
-
candidate.chapter,
|
|
4046
|
-
target,
|
|
4047
|
-
candidate.reason
|
|
4048
|
-
)
|
|
3967
|
+
return await withArchiveBuildJobCreationLock(
|
|
3968
|
+
location.archiveFile,
|
|
3969
|
+
async () => {
|
|
3970
|
+
const target = options.target ?? "reading-summary";
|
|
3971
|
+
await validateQueueTargetConfig(target, options.llmJSON);
|
|
3972
|
+
const selectedExplicitly = options.chapterId !== void 0 || options.chapterIds !== void 0 || options.chapterPath !== void 0 || options.depth !== void 0;
|
|
3973
|
+
const candidates = await new WikiGraphArchiveFile6(
|
|
3974
|
+
location.archiveFile
|
|
3975
|
+
).readDocument(async (document) => {
|
|
3976
|
+
const chapters = await listChapters3(document);
|
|
3977
|
+
const selected = await selectQueueChapters(
|
|
3978
|
+
document,
|
|
3979
|
+
chapters,
|
|
3980
|
+
options
|
|
4049
3981
|
);
|
|
4050
|
-
|
|
4051
|
-
|
|
4052
|
-
|
|
4053
|
-
|
|
4054
|
-
|
|
4055
|
-
|
|
4056
|
-
|
|
4057
|
-
|
|
4058
|
-
|
|
4059
|
-
|
|
4060
|
-
|
|
4061
|
-
archive: location.archiveFile,
|
|
4062
|
-
boost: options.boost ?? false,
|
|
4063
|
-
chapterId: candidate.chapter.chapterId,
|
|
4064
|
-
...options.llmJSON === void 0 ? {} : { llmJSON: options.llmJSON },
|
|
4065
|
-
...options.prompt === void 0 ? {} : { prompt: options.prompt },
|
|
4066
|
-
target
|
|
4067
|
-
})
|
|
4068
|
-
});
|
|
4069
|
-
} catch (error) {
|
|
4070
|
-
if (selectedExplicitly && candidates.length === 1) throw error;
|
|
4071
|
-
skipped.push({
|
|
4072
|
-
chapter: candidate.chapter,
|
|
4073
|
-
reason: error instanceof Error ? error.message : String(error)
|
|
3982
|
+
const checked = await Promise.all(
|
|
3983
|
+
selected.map(async (chapter) => ({
|
|
3984
|
+
chapter,
|
|
3985
|
+
reason: await readQueueReadinessReason(
|
|
3986
|
+
document,
|
|
3987
|
+
chapter,
|
|
3988
|
+
target
|
|
3989
|
+
)
|
|
3990
|
+
}))
|
|
3991
|
+
);
|
|
3992
|
+
return checked;
|
|
4074
3993
|
});
|
|
3994
|
+
const created = [];
|
|
3995
|
+
const skipped = [];
|
|
3996
|
+
for (const candidate of candidates) {
|
|
3997
|
+
if (candidate.reason !== void 0) {
|
|
3998
|
+
if (selectedExplicitly && candidates.length === 1) {
|
|
3999
|
+
throw new Error(
|
|
4000
|
+
formatQueueReadinessError(
|
|
4001
|
+
candidate.chapter,
|
|
4002
|
+
target,
|
|
4003
|
+
candidate.reason
|
|
4004
|
+
)
|
|
4005
|
+
);
|
|
4006
|
+
}
|
|
4007
|
+
skipped.push({
|
|
4008
|
+
chapter: candidate.chapter,
|
|
4009
|
+
reason: candidate.reason
|
|
4010
|
+
});
|
|
4011
|
+
continue;
|
|
4012
|
+
}
|
|
4013
|
+
try {
|
|
4014
|
+
created.push({
|
|
4015
|
+
chapter: candidate.chapter,
|
|
4016
|
+
job: await this.#createHandle(
|
|
4017
|
+
await this.#backend.add({
|
|
4018
|
+
archive: location.archiveFile,
|
|
4019
|
+
boost: options.boost ?? false,
|
|
4020
|
+
chapterId: candidate.chapter.chapterId,
|
|
4021
|
+
...options.llmJSON === void 0 ? {} : { llmJSON: options.llmJSON },
|
|
4022
|
+
...options.prompt === void 0 ? {} : { prompt: options.prompt },
|
|
4023
|
+
target
|
|
4024
|
+
}),
|
|
4025
|
+
location.target
|
|
4026
|
+
)
|
|
4027
|
+
});
|
|
4028
|
+
} catch (error) {
|
|
4029
|
+
if (selectedExplicitly && candidates.length === 1) throw error;
|
|
4030
|
+
skipped.push({
|
|
4031
|
+
chapter: candidate.chapter,
|
|
4032
|
+
reason: error instanceof Error ? error.message : String(error)
|
|
4033
|
+
});
|
|
4034
|
+
}
|
|
4035
|
+
}
|
|
4036
|
+
return { created, skipped };
|
|
4075
4037
|
}
|
|
4076
|
-
|
|
4077
|
-
return { created, skipped };
|
|
4038
|
+
);
|
|
4078
4039
|
});
|
|
4079
4040
|
}
|
|
4080
4041
|
async planEnqueue(options) {
|
|
@@ -4083,7 +4044,7 @@ var WikiGraphJobManager = class {
|
|
|
4083
4044
|
const target = options.target ?? "reading-summary";
|
|
4084
4045
|
await validateQueueTargetConfig(target, options.llmJSON);
|
|
4085
4046
|
const selectedExplicitly = options.chapterId !== void 0 || options.chapterIds !== void 0 || options.chapterPath !== void 0 || options.depth !== void 0;
|
|
4086
|
-
const candidates = await new
|
|
4047
|
+
const candidates = await new WikiGraphArchiveFile6(
|
|
4087
4048
|
location.archiveFile
|
|
4088
4049
|
).readDocument(async (document) => {
|
|
4089
4050
|
const chapters = await listChapters3(document);
|
|
@@ -4120,34 +4081,39 @@ var WikiGraphJobManager = class {
|
|
|
4120
4081
|
this.#subscriptions.add(controller);
|
|
4121
4082
|
return () => this.#subscriptions.delete(controller);
|
|
4122
4083
|
}
|
|
4123
|
-
#createHandle(snapshot) {
|
|
4124
|
-
|
|
4084
|
+
async #createHandle(snapshot, target) {
|
|
4085
|
+
const legacy = snapshot;
|
|
4086
|
+
const resolvedTarget = target ?? (snapshot.archive === void 0 ? archiveTargetFromLogicalLocator(legacy.archivePath ?? "unknown.wikg") : await resolveWikiGraphArchiveTarget(snapshot.archive));
|
|
4087
|
+
return new WikiGraphJob(
|
|
4088
|
+
this,
|
|
4089
|
+
this.#runtime,
|
|
4090
|
+
this.#backend,
|
|
4091
|
+
snapshot,
|
|
4092
|
+
cloneWikiGraphArchiveTarget(resolvedTarget)
|
|
4093
|
+
);
|
|
4125
4094
|
}
|
|
4126
4095
|
};
|
|
4127
|
-
function
|
|
4128
|
-
|
|
4129
|
-
if (record.archive !== void 0) {
|
|
4130
|
-
return getNodeResourcePath(record.archive);
|
|
4131
|
-
}
|
|
4132
|
-
if (typeof record.archivePath === "string") return record.archivePath;
|
|
4133
|
-
throw new TypeError("Build job is missing archive");
|
|
4096
|
+
function getJobArchiveLocator(job) {
|
|
4097
|
+
return job.archive.kind === "library" ? job.archive.uri : job.archive.path;
|
|
4134
4098
|
}
|
|
4135
4099
|
var WikiGraphJob = class {
|
|
4136
4100
|
#manager;
|
|
4137
4101
|
#runtime;
|
|
4138
4102
|
#backend;
|
|
4103
|
+
#archiveTarget;
|
|
4139
4104
|
#snapshot;
|
|
4140
|
-
constructor(manager, runtime, backend, snapshot) {
|
|
4105
|
+
constructor(manager, runtime, backend, snapshot, archiveTarget) {
|
|
4141
4106
|
this.#manager = manager;
|
|
4142
4107
|
this.#runtime = runtime;
|
|
4143
4108
|
this.#backend = backend;
|
|
4144
4109
|
this.#snapshot = snapshot;
|
|
4110
|
+
this.#archiveTarget = archiveTarget;
|
|
4145
4111
|
}
|
|
4146
4112
|
get id() {
|
|
4147
4113
|
return this.#snapshot.jobId;
|
|
4148
4114
|
}
|
|
4149
4115
|
get snapshot() {
|
|
4150
|
-
return this.#snapshot;
|
|
4116
|
+
return toPublicJobSnapshot(this.#snapshot, this.#archiveTarget);
|
|
4151
4117
|
}
|
|
4152
4118
|
async status() {
|
|
4153
4119
|
return await this.#update(async () => await this.#backend.get(this.id));
|
|
@@ -4174,7 +4140,8 @@ var WikiGraphJob = class {
|
|
|
4174
4140
|
let cursor = 0;
|
|
4175
4141
|
let job = this.#snapshot;
|
|
4176
4142
|
if (options.from === "now") {
|
|
4177
|
-
|
|
4143
|
+
await this.status();
|
|
4144
|
+
job = this.#snapshot;
|
|
4178
4145
|
const chunk = await this.#readEventChunkSafely(job, cursor);
|
|
4179
4146
|
const events = chunk.events;
|
|
4180
4147
|
cursor = chunk.cursor;
|
|
@@ -4194,7 +4161,8 @@ var WikiGraphJob = class {
|
|
|
4194
4161
|
yield event;
|
|
4195
4162
|
}
|
|
4196
4163
|
if (sawTerminalEvent) return;
|
|
4197
|
-
|
|
4164
|
+
await this.status();
|
|
4165
|
+
job = this.#snapshot;
|
|
4198
4166
|
if (isTerminalState(job.state)) {
|
|
4199
4167
|
terminalReadsWithoutEvent += 1;
|
|
4200
4168
|
if (terminalReadsWithoutEvent >= TERMINAL_EVENT_GRACE_READS) return;
|
|
@@ -4231,7 +4199,7 @@ var WikiGraphJob = class {
|
|
|
4231
4199
|
}
|
|
4232
4200
|
async #update(operation) {
|
|
4233
4201
|
this.#snapshot = await this.#runtime.run(operation);
|
|
4234
|
-
return this
|
|
4202
|
+
return this.snapshot;
|
|
4235
4203
|
}
|
|
4236
4204
|
async #readEventChunk(job, cursor) {
|
|
4237
4205
|
return await this.#runtime.run(async () => {
|
|
@@ -4250,6 +4218,54 @@ var WikiGraphJob = class {
|
|
|
4250
4218
|
}
|
|
4251
4219
|
}
|
|
4252
4220
|
};
|
|
4221
|
+
function toPublicJobSnapshot(job, archive) {
|
|
4222
|
+
const legacy = job;
|
|
4223
|
+
return {
|
|
4224
|
+
archive: cloneWikiGraphArchiveTarget(archive),
|
|
4225
|
+
archiveKey: archive.kind === "library" ? archive.uri : job.archiveKey,
|
|
4226
|
+
...getJobResourcePath(job.cache, legacy.cachePath) === void 0 ? {} : { cachePath: getJobResourcePath(job.cache, legacy.cachePath) },
|
|
4227
|
+
chapterId: job.chapterId,
|
|
4228
|
+
createdAt: job.createdAt,
|
|
4229
|
+
...job.currentStep === void 0 ? {} : { currentStep: job.currentStep },
|
|
4230
|
+
...job.errorJSON === void 0 ? {} : { errorJSON: job.errorJSON },
|
|
4231
|
+
...getJobResourcePath(job.events, legacy.eventsPath) === void 0 ? {} : { eventsPath: getJobResourcePath(job.events, legacy.eventsPath) },
|
|
4232
|
+
...job.finishedAt === void 0 ? {} : { finishedAt: job.finishedAt },
|
|
4233
|
+
jobId: job.jobId,
|
|
4234
|
+
...job.inputRevision === void 0 ? {} : { inputRevision: job.inputRevision },
|
|
4235
|
+
...getJobResourcePath(job.log, legacy.logPath) === void 0 ? {} : { logPath: getJobResourcePath(job.log, legacy.logPath) },
|
|
4236
|
+
...job.llmJSON === void 0 ? {} : { llmJSON: job.llmJSON },
|
|
4237
|
+
...job.ownerId === void 0 ? {} : { ownerId: job.ownerId },
|
|
4238
|
+
...job.prompt === void 0 ? {} : { prompt: job.prompt },
|
|
4239
|
+
queueRank: job.queueRank,
|
|
4240
|
+
...job.readingSummaryStartedAt === void 0 ? {} : { readingSummaryStartedAt: job.readingSummaryStartedAt },
|
|
4241
|
+
state: job.state,
|
|
4242
|
+
target: job.target,
|
|
4243
|
+
updatedAt: job.updatedAt,
|
|
4244
|
+
...getJobResourcePath(job.workspace, legacy.workspacePath) === void 0 ? {} : {
|
|
4245
|
+
workspacePath: getJobResourcePath(
|
|
4246
|
+
job.workspace,
|
|
4247
|
+
legacy.workspacePath
|
|
4248
|
+
)
|
|
4249
|
+
}
|
|
4250
|
+
};
|
|
4251
|
+
}
|
|
4252
|
+
function cloneWikiGraphArchiveTarget(target) {
|
|
4253
|
+
return target.kind === "library" ? Object.freeze({ kind: "library", uri: target.uri }) : Object.freeze({
|
|
4254
|
+
kind: "standalone",
|
|
4255
|
+
path: target.path,
|
|
4256
|
+
...target.objectUri === void 0 ? {} : { objectUri: target.objectUri }
|
|
4257
|
+
});
|
|
4258
|
+
}
|
|
4259
|
+
function getJobResourcePath(resource, legacy) {
|
|
4260
|
+
if (resource === void 0) return legacy;
|
|
4261
|
+
try {
|
|
4262
|
+
return getNodeResourcePath(resource);
|
|
4263
|
+
} catch {
|
|
4264
|
+
const identity = resource.identity;
|
|
4265
|
+
if (typeof identity === "string") return identity;
|
|
4266
|
+
throw new TypeError("Expected a Node File or Directory resource");
|
|
4267
|
+
}
|
|
4268
|
+
}
|
|
4253
4269
|
async function selectQueueChapters(document, chapters, options) {
|
|
4254
4270
|
const ids = options.chapterIds ?? (options.chapterId === void 0 ? void 0 : [options.chapterId]);
|
|
4255
4271
|
if (ids !== void 0) {
|
|
@@ -4260,7 +4276,7 @@ async function selectQueueChapters(document, chapters, options) {
|
|
|
4260
4276
|
if (options.chapterPath === void 0) {
|
|
4261
4277
|
return options.depth === void 0 ? chapters : chapters.filter((chapter) => chapter.depth <= options.depth);
|
|
4262
4278
|
}
|
|
4263
|
-
const rootId = await
|
|
4279
|
+
const rootId = await resolveChapterPathReadonly3(
|
|
4264
4280
|
document,
|
|
4265
4281
|
options.chapterPath
|
|
4266
4282
|
);
|
|
@@ -4325,14 +4341,14 @@ function formatQueueReadinessError(chapter, target, reason) {
|
|
|
4325
4341
|
}
|
|
4326
4342
|
async function wait(milliseconds, signal) {
|
|
4327
4343
|
if (signal?.aborted === true) return;
|
|
4328
|
-
await new Promise((
|
|
4344
|
+
await new Promise((resolve4) => {
|
|
4329
4345
|
let settled = false;
|
|
4330
4346
|
const complete = () => {
|
|
4331
4347
|
if (settled) return;
|
|
4332
4348
|
settled = true;
|
|
4333
4349
|
clearTimeout(timeout);
|
|
4334
4350
|
signal?.removeEventListener("abort", complete);
|
|
4335
|
-
|
|
4351
|
+
resolve4();
|
|
4336
4352
|
};
|
|
4337
4353
|
const timeout = setTimeout(complete, milliseconds);
|
|
4338
4354
|
signal?.addEventListener("abort", complete, { once: true });
|
|
@@ -4363,12 +4379,12 @@ import {
|
|
|
4363
4379
|
deleteWikiGraphLibraryMetadataKey,
|
|
4364
4380
|
findWikiGraphLibraryObjects as findWikiGraphLibraryObjects2,
|
|
4365
4381
|
findWikiGraphLibraryArchiveMembers as findWikiGraphLibraryArchiveMembers2,
|
|
4366
|
-
formatWikiGraphLibraryUri,
|
|
4382
|
+
formatWikiGraphLibraryUri as formatWikiGraphLibraryUri2,
|
|
4367
4383
|
getWikiGraphLibraryArchive,
|
|
4368
4384
|
getWikiGraphLibraryMetadata,
|
|
4369
4385
|
listRelatedWikiGraphLibraryObjects as listRelatedWikiGraphLibraryObjects2,
|
|
4370
|
-
listWikiGraphLibraries,
|
|
4371
|
-
listWikiGraphLibraryArchives,
|
|
4386
|
+
listWikiGraphLibraries as listWikiGraphLibraries2,
|
|
4387
|
+
listWikiGraphLibraryArchives as listWikiGraphLibraryArchives2,
|
|
4372
4388
|
listWikiGraphLibraryArchiveMembers as listWikiGraphLibraryArchiveMembers2,
|
|
4373
4389
|
listWikiGraphLibraryEvidence as listWikiGraphLibraryEvidence2,
|
|
4374
4390
|
listWikiGraphLibraryObjects as listWikiGraphLibraryObjects2,
|
|
@@ -4376,14 +4392,15 @@ import {
|
|
|
4376
4392
|
packWikiGraphLibraryContext,
|
|
4377
4393
|
parseWikiGraphLibraryUri as parseWikiGraphLibraryUri2,
|
|
4378
4394
|
putWikiGraphLibraryMetadata,
|
|
4379
|
-
readWikiGraphLibraryIndexState
|
|
4395
|
+
readWikiGraphLibraryIndexState,
|
|
4380
4396
|
readWikiGraphLibraryPage,
|
|
4381
|
-
rebuildWikiGraphLibraryIndex
|
|
4397
|
+
rebuildWikiGraphLibraryIndex,
|
|
4382
4398
|
rebindWikiGraphLibrary,
|
|
4383
4399
|
removeWikiGraphLibrary,
|
|
4384
4400
|
removeWikiGraphLibraryArchive,
|
|
4401
|
+
replaceWikiGraphLibraryArchive,
|
|
4385
4402
|
replaceWikiGraphLibraryMetadata,
|
|
4386
|
-
resolveWikiGraphLibrary,
|
|
4403
|
+
resolveWikiGraphLibrary as resolveWikiGraphLibrary2,
|
|
4387
4404
|
scanWikiGraphLibrary
|
|
4388
4405
|
} from "wiki-graph-core";
|
|
4389
4406
|
import { mkdir as mkdir2 } from "fs/promises";
|
|
@@ -4400,22 +4417,24 @@ var WikiGraphLibraryManager = class {
|
|
|
4400
4417
|
});
|
|
4401
4418
|
return new WikiGraphLibrary(this.#runtime, record);
|
|
4402
4419
|
}
|
|
4403
|
-
async get(uri =
|
|
4420
|
+
async get(uri = formatWikiGraphLibraryUri2()) {
|
|
4404
4421
|
const target = requireLibraryTarget(uri);
|
|
4405
4422
|
const record = await this.#runtime.run(
|
|
4406
|
-
async () => await
|
|
4423
|
+
async () => await resolveWikiGraphLibrary2(target)
|
|
4407
4424
|
);
|
|
4408
4425
|
return new WikiGraphLibrary(this.#runtime, record);
|
|
4409
4426
|
}
|
|
4410
4427
|
async addArchive(options) {
|
|
4411
|
-
return
|
|
4412
|
-
|
|
4413
|
-
|
|
4414
|
-
|
|
4415
|
-
|
|
4416
|
-
|
|
4417
|
-
|
|
4418
|
-
|
|
4428
|
+
return toArchiveSnapshot(
|
|
4429
|
+
await this.#runtime.run(
|
|
4430
|
+
async () => await addWikiGraphLibraryArchive({
|
|
4431
|
+
inputFile: new NodeFile(
|
|
4432
|
+
await assertStandaloneWikiGraphArchivePath(options.inputPath)
|
|
4433
|
+
),
|
|
4434
|
+
target: requireLibraryTarget(options.target),
|
|
4435
|
+
...options.to === void 0 ? {} : { to: options.to }
|
|
4436
|
+
})
|
|
4437
|
+
)
|
|
4419
4438
|
);
|
|
4420
4439
|
}
|
|
4421
4440
|
async assertCurrent(target) {
|
|
@@ -4424,51 +4443,78 @@ var WikiGraphLibraryManager = class {
|
|
|
4424
4443
|
);
|
|
4425
4444
|
}
|
|
4426
4445
|
async rebind(target, folder) {
|
|
4427
|
-
return
|
|
4428
|
-
|
|
4429
|
-
|
|
4430
|
-
|
|
4431
|
-
|
|
4446
|
+
return toScanSnapshot(
|
|
4447
|
+
await this.#runtime.run(
|
|
4448
|
+
async () => await rebindWikiGraphLibrary({
|
|
4449
|
+
folder: new NodeDirectory(resolveWikiGraphRuntimePath(folder)),
|
|
4450
|
+
target: requireLibraryTarget(target)
|
|
4451
|
+
})
|
|
4452
|
+
)
|
|
4432
4453
|
);
|
|
4433
4454
|
}
|
|
4434
4455
|
async remove(target) {
|
|
4435
|
-
return
|
|
4436
|
-
|
|
4456
|
+
return toLibrarySnapshot(
|
|
4457
|
+
await this.#runtime.run(
|
|
4458
|
+
async () => await removeWikiGraphLibrary(requireLibraryTarget(target))
|
|
4459
|
+
)
|
|
4437
4460
|
);
|
|
4438
4461
|
}
|
|
4439
4462
|
async scan(target) {
|
|
4440
|
-
return
|
|
4441
|
-
|
|
4463
|
+
return toScanSnapshot(
|
|
4464
|
+
await this.#runtime.run(
|
|
4465
|
+
async () => await scanWikiGraphLibrary(requireLibraryTarget(target))
|
|
4466
|
+
)
|
|
4442
4467
|
);
|
|
4443
4468
|
}
|
|
4444
4469
|
async getArchive(target) {
|
|
4445
|
-
return
|
|
4446
|
-
|
|
4470
|
+
return toArchiveSnapshot(
|
|
4471
|
+
await this.#runtime.run(
|
|
4472
|
+
async () => await getWikiGraphLibraryArchive(requireLibraryTarget(target))
|
|
4473
|
+
)
|
|
4447
4474
|
);
|
|
4448
4475
|
}
|
|
4449
4476
|
async moveArchive(options) {
|
|
4450
|
-
return
|
|
4451
|
-
|
|
4452
|
-
|
|
4453
|
-
|
|
4454
|
-
|
|
4477
|
+
return toArchiveSnapshot(
|
|
4478
|
+
await this.#runtime.run(
|
|
4479
|
+
async () => await moveWikiGraphLibraryArchive({
|
|
4480
|
+
target: requireLibraryTarget(options.target),
|
|
4481
|
+
to: options.to
|
|
4482
|
+
})
|
|
4483
|
+
)
|
|
4455
4484
|
);
|
|
4456
4485
|
}
|
|
4457
4486
|
async removeArchive(target) {
|
|
4458
|
-
return
|
|
4459
|
-
|
|
4460
|
-
|
|
4487
|
+
return toArchiveSnapshot(
|
|
4488
|
+
await this.#runtime.run(
|
|
4489
|
+
async () => await removeWikiGraphLibraryArchive({
|
|
4490
|
+
target: requireLibraryTarget(target)
|
|
4491
|
+
})
|
|
4492
|
+
)
|
|
4493
|
+
);
|
|
4494
|
+
}
|
|
4495
|
+
async replaceArchive(options) {
|
|
4496
|
+
return toArchiveSnapshot(
|
|
4497
|
+
await this.#runtime.run(async () => {
|
|
4498
|
+
const target = requireLibraryTarget(options.target);
|
|
4499
|
+
const current = await getWikiGraphLibraryArchive(target);
|
|
4500
|
+
return await replaceWikiGraphLibraryArchive({
|
|
4501
|
+
additionalDerivedStateKeys: current.file === void 0 ? [] : [getNodeResourcePath(current.file)],
|
|
4502
|
+
inputFile: new NodeFile(
|
|
4503
|
+
await assertStandaloneWikiGraphArchivePath(options.inputPath)
|
|
4504
|
+
),
|
|
4505
|
+
target
|
|
4506
|
+
});
|
|
4461
4507
|
})
|
|
4462
4508
|
);
|
|
4463
4509
|
}
|
|
4464
4510
|
async indexState(target) {
|
|
4465
4511
|
return await this.#runtime.run(
|
|
4466
|
-
async () => await
|
|
4512
|
+
async () => await readWikiGraphLibraryIndexState(requireLibraryTarget(target))
|
|
4467
4513
|
);
|
|
4468
4514
|
}
|
|
4469
4515
|
async rebuildIndex(target, onProgress) {
|
|
4470
4516
|
return await this.#runtime.run(
|
|
4471
|
-
async () => await
|
|
4517
|
+
async () => await rebuildWikiGraphLibraryIndex(
|
|
4472
4518
|
requireLibraryTarget(target),
|
|
4473
4519
|
onProgress
|
|
4474
4520
|
)
|
|
@@ -4565,7 +4611,8 @@ var WikiGraphLibraryManager = class {
|
|
|
4565
4611
|
objectUri,
|
|
4566
4612
|
{ ...options, ...wikimediaOptions }
|
|
4567
4613
|
)
|
|
4568
|
-
)
|
|
4614
|
+
),
|
|
4615
|
+
options.signal
|
|
4569
4616
|
);
|
|
4570
4617
|
}
|
|
4571
4618
|
async related(target, objectUri, options = {}) {
|
|
@@ -4597,7 +4644,7 @@ var WikiGraphLibraryManager = class {
|
|
|
4597
4644
|
}
|
|
4598
4645
|
async list() {
|
|
4599
4646
|
const records = await this.#runtime.run(
|
|
4600
|
-
async () => await
|
|
4647
|
+
async () => await listWikiGraphLibraries2()
|
|
4601
4648
|
);
|
|
4602
4649
|
return records.map((record) => new WikiGraphLibrary(this.#runtime, record));
|
|
4603
4650
|
}
|
|
@@ -4610,32 +4657,54 @@ var WikiGraphLibrary = class {
|
|
|
4610
4657
|
this.#record = record;
|
|
4611
4658
|
}
|
|
4612
4659
|
get snapshot() {
|
|
4613
|
-
return this.#record;
|
|
4660
|
+
return toLibrarySnapshot(this.#record);
|
|
4614
4661
|
}
|
|
4615
4662
|
get uri() {
|
|
4616
|
-
return
|
|
4663
|
+
return formatWikiGraphLibraryUri2(
|
|
4617
4664
|
this.#record.isDefault ? void 0 : this.#record.publicId
|
|
4618
4665
|
);
|
|
4619
4666
|
}
|
|
4620
4667
|
async archives() {
|
|
4621
4668
|
const target = requireLibraryTarget(this.uri);
|
|
4622
|
-
return await this.#runtime.run(
|
|
4623
|
-
async () => await
|
|
4624
|
-
);
|
|
4669
|
+
return (await this.#runtime.run(
|
|
4670
|
+
async () => await listWikiGraphLibraryArchives2(target)
|
|
4671
|
+
)).map(toArchiveSnapshot);
|
|
4625
4672
|
}
|
|
4626
4673
|
async scan() {
|
|
4627
4674
|
const target = requireLibraryTarget(this.uri);
|
|
4628
|
-
return
|
|
4629
|
-
async () => await scanWikiGraphLibrary(target)
|
|
4675
|
+
return toScanSnapshot(
|
|
4676
|
+
await this.#runtime.run(async () => await scanWikiGraphLibrary(target))
|
|
4630
4677
|
);
|
|
4631
4678
|
}
|
|
4632
4679
|
async remove() {
|
|
4633
4680
|
const target = requireLibraryTarget(this.uri);
|
|
4634
|
-
return
|
|
4635
|
-
async () => await removeWikiGraphLibrary(target)
|
|
4681
|
+
return toLibrarySnapshot(
|
|
4682
|
+
await this.#runtime.run(async () => await removeWikiGraphLibrary(target))
|
|
4636
4683
|
);
|
|
4637
4684
|
}
|
|
4638
4685
|
};
|
|
4686
|
+
function toLibrarySnapshot(record) {
|
|
4687
|
+
const compatible = record;
|
|
4688
|
+
return {
|
|
4689
|
+
id: record.id,
|
|
4690
|
+
publicId: record.publicId,
|
|
4691
|
+
uri: record.uri,
|
|
4692
|
+
folderPath: compatible.folderPath ?? (record.folder === void 0 ? "" : getNodeResourcePath(record.folder)),
|
|
4693
|
+
isDefault: record.isDefault,
|
|
4694
|
+
createdAt: record.createdAt,
|
|
4695
|
+
updatedAt: record.updatedAt
|
|
4696
|
+
};
|
|
4697
|
+
}
|
|
4698
|
+
function toArchiveSnapshot(record) {
|
|
4699
|
+
const { file: _file, ...snapshot } = record;
|
|
4700
|
+
return snapshot;
|
|
4701
|
+
}
|
|
4702
|
+
function toScanSnapshot(result) {
|
|
4703
|
+
return {
|
|
4704
|
+
...result.library === void 0 ? {} : { library: toLibrarySnapshot(result.library) },
|
|
4705
|
+
archives: result.archives.map(toArchiveSnapshot)
|
|
4706
|
+
};
|
|
4707
|
+
}
|
|
4639
4708
|
function requireLibraryTarget(value) {
|
|
4640
4709
|
if (typeof value !== "string") return value;
|
|
4641
4710
|
const target = parseWikiGraphLibraryUri2(value);
|
|
@@ -4651,7 +4720,7 @@ import {
|
|
|
4651
4720
|
parseWikiGraphLibraryUri as parseWikiGraphLibraryUri3,
|
|
4652
4721
|
upgradeWikiGraphMaintenanceTarget
|
|
4653
4722
|
} from "wiki-graph-core";
|
|
4654
|
-
import { basename as
|
|
4723
|
+
import { basename as basename5 } from "path";
|
|
4655
4724
|
var WikiGraphMaintenanceManager = class {
|
|
4656
4725
|
#runtime;
|
|
4657
4726
|
constructor(runtime) {
|
|
@@ -4675,7 +4744,7 @@ var WikiGraphMaintenanceManager = class {
|
|
|
4675
4744
|
target: parsed
|
|
4676
4745
|
});
|
|
4677
4746
|
}
|
|
4678
|
-
const path =
|
|
4747
|
+
const path = await assertStandaloneWikiGraphArchivePath(target.path);
|
|
4679
4748
|
return await upgradeWikiGraphMaintenanceTarget({
|
|
4680
4749
|
file: new NodeFile(path),
|
|
4681
4750
|
kind: "archive"
|
|
@@ -4691,7 +4760,7 @@ var WikiGraphMaintenanceManager = class {
|
|
|
4691
4760
|
const resolvedOutputPath = resolveWikiGraphRuntimePath(outputPath);
|
|
4692
4761
|
await migrateLegacySdpubToWikg(
|
|
4693
4762
|
new NodeFile(resolvedInputPath),
|
|
4694
|
-
new NodeFile(resolvedOutputPath,
|
|
4763
|
+
new NodeFile(resolvedOutputPath, basename5(resolvedOutputPath))
|
|
4695
4764
|
);
|
|
4696
4765
|
return {
|
|
4697
4766
|
inputPath: resolvedInputPath,
|
|
@@ -4773,16 +4842,16 @@ import {
|
|
|
4773
4842
|
formatError as formatError3,
|
|
4774
4843
|
formatLocatedChapterResourceUri,
|
|
4775
4844
|
formatLocatedChapterSourceCollectionUri,
|
|
4776
|
-
formatLocatedChapterUri as
|
|
4777
|
-
formatLocatedWikiGraphUri as
|
|
4845
|
+
formatLocatedChapterUri as formatLocatedChapterUri3,
|
|
4846
|
+
formatLocatedWikiGraphUri as formatLocatedWikiGraphUri3,
|
|
4778
4847
|
formatWikiGraphCommandUri,
|
|
4779
|
-
formatWikiGraphLibraryUri as
|
|
4848
|
+
formatWikiGraphLibraryUri as formatWikiGraphLibraryUri3,
|
|
4780
4849
|
isSourceLocatorScopeUri as isSourceLocatorScopeUri2,
|
|
4781
4850
|
isWikiGraphLibraryUri,
|
|
4782
4851
|
parseChapterPath,
|
|
4783
4852
|
parseChapterTreeInput,
|
|
4784
4853
|
parseChapterUriPath,
|
|
4785
|
-
parseLocatedWikiGraphUri as
|
|
4854
|
+
parseLocatedWikiGraphUri as parseLocatedWikiGraphUri3,
|
|
4786
4855
|
parseSourceTextJsonl,
|
|
4787
4856
|
parseWikiGraphLibraryUri as parseWikiGraphLibraryUri4,
|
|
4788
4857
|
parseWikiGraphUriSyntax
|
|
@@ -4801,6 +4870,7 @@ export {
|
|
|
4801
4870
|
WikiGraphArchiveExistsError,
|
|
4802
4871
|
WikiGraphArchiveHandle,
|
|
4803
4872
|
WikiGraphArchiveManager,
|
|
4873
|
+
WikiGraphArchiveOwnershipError,
|
|
4804
4874
|
WikiGraphConfigManager,
|
|
4805
4875
|
WikiGraphContinuationManager,
|
|
4806
4876
|
WikiGraphConversionManager,
|
|
@@ -4819,7 +4889,6 @@ export {
|
|
|
4819
4889
|
createNodeTemplateEnvironment,
|
|
4820
4890
|
createNodeWikiGraphStorage,
|
|
4821
4891
|
createStageLLM,
|
|
4822
|
-
createWikiGraphArchiveInspectReport,
|
|
4823
4892
|
createWikiGraphSDK,
|
|
4824
4893
|
deleteLocalConfigValue,
|
|
4825
4894
|
embedQueryText,
|
|
@@ -4830,13 +4899,14 @@ export {
|
|
|
4830
4899
|
formatGenerationPlanningModel,
|
|
4831
4900
|
formatLocatedChapterResourceUri,
|
|
4832
4901
|
formatLocatedChapterSourceCollectionUri,
|
|
4833
|
-
|
|
4834
|
-
|
|
4902
|
+
formatLocatedChapterUri3 as formatLocatedChapterUri,
|
|
4903
|
+
formatLocatedWikiGraphUri3 as formatLocatedWikiGraphUri,
|
|
4835
4904
|
formatWikiGraphCommandUri,
|
|
4836
|
-
|
|
4905
|
+
formatWikiGraphLibraryUri3 as formatWikiGraphLibraryUri,
|
|
4837
4906
|
getNodeResourcePath,
|
|
4838
4907
|
getWikiGraphSDKRuntimeContext,
|
|
4839
4908
|
getWikiGraphSDKStateDir,
|
|
4909
|
+
inspectWikiGraphArchive,
|
|
4840
4910
|
installNodeWikiGraphPlatform,
|
|
4841
4911
|
isSourceLocatorScopeUri2 as isSourceLocatorScopeUri,
|
|
4842
4912
|
isWikiGraphLibraryUri,
|
|
@@ -4850,7 +4920,7 @@ export {
|
|
|
4850
4920
|
parseChapterTreeInput,
|
|
4851
4921
|
parseChapterUriPath,
|
|
4852
4922
|
parseLocalConfigSection,
|
|
4853
|
-
|
|
4923
|
+
parseLocatedWikiGraphUri3 as parseLocatedWikiGraphUri,
|
|
4854
4924
|
parseSourceTextJsonl,
|
|
4855
4925
|
parseWikiGraphLibraryUri4 as parseWikiGraphLibraryUri,
|
|
4856
4926
|
parseWikiGraphUriSyntax,
|
|
@@ -4863,7 +4933,6 @@ export {
|
|
|
4863
4933
|
resolveExtractionPrompt,
|
|
4864
4934
|
resolveHostedProviderEndpoint,
|
|
4865
4935
|
resolveKnowledgeGraphRecallPrompt,
|
|
4866
|
-
resolveWikiGraphArchiveLocation,
|
|
4867
4936
|
resolveWikispineConfig,
|
|
4868
4937
|
runWikiGraphQueueWorker,
|
|
4869
4938
|
validateLocalConfigSection,
|