wiki-graph-sdk 0.7.0 → 0.8.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +16 -0
- package/dist/archives.cjs +470 -642
- package/dist/archives.cjs.map +1 -1
- package/dist/archives.d.ts +42 -80
- package/dist/archives.js +405 -570
- package/dist/archives.js.map +1 -1
- package/dist/conversions.cjs +223 -27
- package/dist/conversions.cjs.map +1 -1
- package/dist/conversions.d.ts +16 -4
- package/dist/conversions.js +228 -23
- package/dist/conversions.js.map +1 -1
- package/dist/default-worker.cjs +293 -55
- package/dist/default-worker.cjs.map +1 -1
- package/dist/default-worker.js +309 -54
- package/dist/default-worker.js.map +1 -1
- package/dist/index.cjs +1044 -923
- package/dist/index.cjs.map +1 -1
- package/dist/index.d.ts +36 -19
- package/dist/index.js +915 -780
- package/dist/index.js.map +1 -1
- package/dist/{jobs-Dd5PMSId.d.ts → jobs-xIm_kshM.d.ts} +22 -14
- package/dist/maintenance.cjs +138 -29
- package/dist/maintenance.cjs.map +1 -1
- package/dist/maintenance.d.ts +2 -1
- package/dist/maintenance.js +143 -25
- package/dist/maintenance.js.map +1 -1
- package/dist/target-CZ7elwmj.d.ts +14 -0
- package/dist/wikispine.cjs +14 -10
- package/dist/wikispine.cjs.map +1 -1
- package/dist/wikispine.js +14 -10
- package/dist/wikispine.js.map +1 -1
- package/dist/worker.cjs +328 -22
- package/dist/worker.cjs.map +1 -1
- package/dist/worker.d.ts +43 -4
- package/dist/worker.js +348 -27
- package/dist/worker.js.map +1 -1
- package/package.json +2 -2
package/dist/index.js
CHANGED
|
@@ -4,9 +4,9 @@ globalThis.__WIKIGRAPH_DATA_DIR__ ??= __WIKIGRAPH_RESOLVE__(__WIKIGRAPH_FILE_URL
|
|
|
4
4
|
|
|
5
5
|
// src/archive/archive.ts
|
|
6
6
|
import { randomUUID as randomUUID2 } from "crypto";
|
|
7
|
-
import { link, mkdtemp, rename as rename2, rm as rm3, stat as
|
|
7
|
+
import { link, mkdtemp, rename as rename2, rm as rm3, stat as stat3 } from "fs/promises";
|
|
8
8
|
import { tmpdir } from "os";
|
|
9
|
-
import { basename as
|
|
9
|
+
import { basename as basename4, dirname as dirname3, join as join4 } from "path";
|
|
10
10
|
import { Readable as Readable2 } from "stream";
|
|
11
11
|
import {
|
|
12
12
|
addChapter,
|
|
@@ -15,9 +15,8 @@ import {
|
|
|
15
15
|
assertNoActiveBuildJobs,
|
|
16
16
|
deleteArchiveSearchSessions,
|
|
17
17
|
DirectoryDocument,
|
|
18
|
-
finalizeWikiGraphLibraryArchiveWrite,
|
|
19
18
|
findArchiveObjects,
|
|
20
|
-
formatLocatedWikiGraphUri,
|
|
19
|
+
formatLocatedWikiGraphUri as formatLocatedWikiGraphUri2,
|
|
21
20
|
isArchiveSearchIndexCurrent as isArchiveSearchIndexCurrent2,
|
|
22
21
|
isSourceLocatorScopeUri,
|
|
23
22
|
listArchiveCollection,
|
|
@@ -28,30 +27,29 @@ import {
|
|
|
28
27
|
listRelatedArchiveObjects,
|
|
29
28
|
moveChapter,
|
|
30
29
|
packArchiveContext,
|
|
31
|
-
parseLocatedWikiGraphUri,
|
|
32
|
-
parseWikiGraphLibraryUri,
|
|
30
|
+
parseLocatedWikiGraphUri as parseLocatedWikiGraphUri2,
|
|
33
31
|
readArchivePage,
|
|
34
32
|
readSearchIndexCapabilityStatus,
|
|
35
|
-
readWikiGraphLibraryIndexState,
|
|
36
33
|
rebuildArchiveSearchIndex,
|
|
37
|
-
rebuildWikiGraphLibraryIndex,
|
|
38
34
|
removeChapter,
|
|
39
35
|
resetChapter,
|
|
40
36
|
resolveChapterPathReadonly,
|
|
41
|
-
resolveWikiGraphLibraryArchiveFile,
|
|
42
37
|
setChapterSource,
|
|
43
38
|
setChapterSummary,
|
|
44
39
|
setChapterTitle,
|
|
45
40
|
getChapterTree,
|
|
46
41
|
WikiGraph as WikiGraph2,
|
|
47
|
-
WikiGraphArchiveFile,
|
|
42
|
+
WikiGraphArchiveFile as WikiGraphArchiveFile2,
|
|
48
43
|
TOC_FILE_VERSION,
|
|
49
44
|
writeWikgArchive,
|
|
50
45
|
ObjectMetadataKind
|
|
51
46
|
} from "wiki-graph-core";
|
|
52
47
|
|
|
53
48
|
// src/conversions.ts
|
|
49
|
+
import { createHash } from "crypto";
|
|
50
|
+
import { createReadStream } from "fs";
|
|
54
51
|
import { rm as rm2 } from "fs/promises";
|
|
52
|
+
import { basename as basename3 } from "path";
|
|
55
53
|
import {
|
|
56
54
|
WikiGraph
|
|
57
55
|
} from "wiki-graph-core";
|
|
@@ -380,42 +378,42 @@ var NodeDatabaseConnection = class {
|
|
|
380
378
|
}
|
|
381
379
|
database;
|
|
382
380
|
async close() {
|
|
383
|
-
await new Promise((
|
|
381
|
+
await new Promise((resolve4, reject) => {
|
|
384
382
|
this.database.close((error) => {
|
|
385
383
|
if (error) reject(error);
|
|
386
|
-
else
|
|
384
|
+
else resolve4();
|
|
387
385
|
});
|
|
388
386
|
});
|
|
389
387
|
}
|
|
390
388
|
async execute(sql) {
|
|
391
|
-
await new Promise((
|
|
389
|
+
await new Promise((resolve4, reject) => {
|
|
392
390
|
this.database.exec(sql, (error) => {
|
|
393
391
|
if (error) reject(error);
|
|
394
|
-
else
|
|
392
|
+
else resolve4();
|
|
395
393
|
});
|
|
396
394
|
});
|
|
397
395
|
}
|
|
398
396
|
async queryAll(sql, params = []) {
|
|
399
|
-
return await new Promise((
|
|
397
|
+
return await new Promise((resolve4, reject) => {
|
|
400
398
|
this.database.all(sql, [...params], (error, rows) => {
|
|
401
399
|
if (error) reject(error);
|
|
402
|
-
else
|
|
400
|
+
else resolve4(rows);
|
|
403
401
|
});
|
|
404
402
|
});
|
|
405
403
|
}
|
|
406
404
|
async queryOne(sql, params = []) {
|
|
407
|
-
return await new Promise((
|
|
405
|
+
return await new Promise((resolve4, reject) => {
|
|
408
406
|
this.database.get(sql, [...params], (error, row) => {
|
|
409
407
|
if (error) reject(error);
|
|
410
|
-
else
|
|
408
|
+
else resolve4(row);
|
|
411
409
|
});
|
|
412
410
|
});
|
|
413
411
|
}
|
|
414
412
|
async run(sql, params = []) {
|
|
415
|
-
await new Promise((
|
|
413
|
+
await new Promise((resolve4, reject) => {
|
|
416
414
|
this.database.run(sql, [...params], (error) => {
|
|
417
415
|
if (error) reject(error);
|
|
418
|
-
else
|
|
416
|
+
else resolve4();
|
|
419
417
|
});
|
|
420
418
|
});
|
|
421
419
|
}
|
|
@@ -429,10 +427,10 @@ async function openNativeNodeDatabase(file, flags) {
|
|
|
429
427
|
if (path === void 0) {
|
|
430
428
|
throw new TypeError("The Node database adapter requires a NodeFile");
|
|
431
429
|
}
|
|
432
|
-
return await new Promise((
|
|
430
|
+
return await new Promise((resolve4, reject) => {
|
|
433
431
|
const database = new nodeSqlite3.Database(path, flags, (error) => {
|
|
434
432
|
if (error) reject(error);
|
|
435
|
-
else
|
|
433
|
+
else resolve4(database);
|
|
436
434
|
});
|
|
437
435
|
});
|
|
438
436
|
}
|
|
@@ -441,19 +439,19 @@ async function openNodeZip(file) {
|
|
|
441
439
|
if (path === void 0) {
|
|
442
440
|
throw new TypeError("The Node ZIP adapter requires a NodeFile");
|
|
443
441
|
}
|
|
444
|
-
const zipFile = await new Promise((
|
|
442
|
+
const zipFile = await new Promise((resolve4, reject) => {
|
|
445
443
|
yauzl.open(
|
|
446
444
|
path,
|
|
447
445
|
{ autoClose: false, lazyEntries: true },
|
|
448
446
|
(error, opened) => {
|
|
449
447
|
if (error || opened === void 0)
|
|
450
448
|
reject(error ?? new Error("Cannot open ZIP"));
|
|
451
|
-
else
|
|
449
|
+
else resolve4(opened);
|
|
452
450
|
}
|
|
453
451
|
);
|
|
454
452
|
});
|
|
455
453
|
const entries = await new Promise(
|
|
456
|
-
(
|
|
454
|
+
(resolve4, reject) => {
|
|
457
455
|
const discovered = /* @__PURE__ */ new Map();
|
|
458
456
|
const rejectAndClose = (error) => {
|
|
459
457
|
zipFile.close();
|
|
@@ -463,7 +461,7 @@ async function openNodeZip(file) {
|
|
|
463
461
|
};
|
|
464
462
|
const resolveEntries = () => {
|
|
465
463
|
zipFile.off("error", rejectAndClose);
|
|
466
|
-
|
|
464
|
+
resolve4(discovered);
|
|
467
465
|
};
|
|
468
466
|
zipFile.on("entry", (entry) => {
|
|
469
467
|
if (!entry.fileName.endsWith("/")) {
|
|
@@ -573,12 +571,12 @@ async function collectDecodedNodeStreamRange(input, offset, length) {
|
|
|
573
571
|
return Buffer.concat(chunks, collected);
|
|
574
572
|
}
|
|
575
573
|
async function openNodeZipEntryStream(zipFile, entry, options) {
|
|
576
|
-
return await new Promise((
|
|
574
|
+
return await new Promise((resolve4, reject) => {
|
|
577
575
|
const callback = (error, opened) => {
|
|
578
576
|
if (error || opened === void 0) {
|
|
579
577
|
reject(error ?? new Error(`Cannot read ZIP entry: ${entry.fileName}`));
|
|
580
578
|
} else {
|
|
581
|
-
|
|
579
|
+
resolve4(opened);
|
|
582
580
|
}
|
|
583
581
|
};
|
|
584
582
|
if (options === void 0) zipFile.openReadStream(entry, callback);
|
|
@@ -709,11 +707,11 @@ async function writeNodeStream(input, writer) {
|
|
|
709
707
|
);
|
|
710
708
|
}
|
|
711
709
|
async function collectNodeStream(input) {
|
|
712
|
-
return await new Promise((
|
|
710
|
+
return await new Promise((resolve4, reject) => {
|
|
713
711
|
const chunks = [];
|
|
714
712
|
input.on("data", (chunk) => chunks.push(chunk));
|
|
715
713
|
input.once("error", reject);
|
|
716
|
-
input.once("end", () =>
|
|
714
|
+
input.once("end", () => resolve4(Buffer.concat(chunks)));
|
|
717
715
|
});
|
|
718
716
|
}
|
|
719
717
|
var nodeWikiGraphPlatform = {
|
|
@@ -1382,6 +1380,203 @@ function resolveWikiGraphRuntimePath(path) {
|
|
|
1382
1380
|
return isAbsolute2(path) ? path : resolve2(context.cwd, path);
|
|
1383
1381
|
}
|
|
1384
1382
|
|
|
1383
|
+
// src/archive/target.ts
|
|
1384
|
+
import { realpath, stat as stat2 } from "fs/promises";
|
|
1385
|
+
import { basename as basename2, dirname as dirname2, isAbsolute as isAbsolute3, join as join3, relative, resolve as resolve3 } from "path";
|
|
1386
|
+
import {
|
|
1387
|
+
formatLocatedWikiGraphUri,
|
|
1388
|
+
formatWikiGraphLibraryUri,
|
|
1389
|
+
listWikiGraphLibraries,
|
|
1390
|
+
listWikiGraphLibraryArchives,
|
|
1391
|
+
parseLocatedWikiGraphUri,
|
|
1392
|
+
parseWikiGraphLibraryUri,
|
|
1393
|
+
resolveWikiGraphLibrary,
|
|
1394
|
+
resolveWikiGraphLibraryArchiveFile
|
|
1395
|
+
} from "wiki-graph-core";
|
|
1396
|
+
var WikiGraphArchiveOwnershipError = class extends Error {
|
|
1397
|
+
code = "WIKI_GRAPH_ARCHIVE_OWNERSHIP_MISMATCH";
|
|
1398
|
+
constructor(message) {
|
|
1399
|
+
super(message);
|
|
1400
|
+
this.name = "WikiGraphArchiveOwnershipError";
|
|
1401
|
+
}
|
|
1402
|
+
};
|
|
1403
|
+
async function resolveWikiGraphArchiveLocation(target) {
|
|
1404
|
+
if (target.kind === "standalone") {
|
|
1405
|
+
return await resolveStandaloneLocation(target);
|
|
1406
|
+
}
|
|
1407
|
+
return await resolveLibraryLocation(target);
|
|
1408
|
+
}
|
|
1409
|
+
async function assertStandaloneWikiGraphArchivePath(inputPath) {
|
|
1410
|
+
const path = resolveWikiGraphRuntimePath(inputPath);
|
|
1411
|
+
const candidate = await canonicalPath(path);
|
|
1412
|
+
const candidateIdentity = await readFileIdentity(path);
|
|
1413
|
+
const libraries = await listWikiGraphLibraries();
|
|
1414
|
+
const matches = await findMatchingMembers(
|
|
1415
|
+
libraries,
|
|
1416
|
+
candidate,
|
|
1417
|
+
candidateIdentity
|
|
1418
|
+
);
|
|
1419
|
+
if (matches.length > 0) {
|
|
1420
|
+
const match = requireUnambiguousManagedMatch(path, matches);
|
|
1421
|
+
throw new WikiGraphArchiveOwnershipError(
|
|
1422
|
+
`Standalone archive path belongs to registered library ${formatWikiGraphLibraryUri(
|
|
1423
|
+
match.library.isDefault ? void 0 : match.library.publicId
|
|
1424
|
+
)}: ${path}. Use the library UUID URI ${match.archive.uri} instead.`
|
|
1425
|
+
);
|
|
1426
|
+
}
|
|
1427
|
+
for (const library of libraries) {
|
|
1428
|
+
const rootPath = getNodeResourcePath(library.folder);
|
|
1429
|
+
const root = await canonicalPath(rootPath);
|
|
1430
|
+
if (!isWithin(root, candidate)) continue;
|
|
1431
|
+
throw new WikiGraphArchiveOwnershipError(
|
|
1432
|
+
`Standalone archive path belongs to registered library ${formatWikiGraphLibraryUri(
|
|
1433
|
+
library.isDefault ? void 0 : library.publicId
|
|
1434
|
+
)}: ${path}. Scan the library, then address the archive by its library UUID URI.`
|
|
1435
|
+
);
|
|
1436
|
+
}
|
|
1437
|
+
return path;
|
|
1438
|
+
}
|
|
1439
|
+
function archiveTargetFromLogicalLocator(locator, objectUri) {
|
|
1440
|
+
if (locator.startsWith("wikg://lib/")) {
|
|
1441
|
+
return { kind: "library", uri: withObjectUri(locator, objectUri) };
|
|
1442
|
+
}
|
|
1443
|
+
return {
|
|
1444
|
+
kind: "standalone",
|
|
1445
|
+
path: locator,
|
|
1446
|
+
...objectUri === void 0 ? {} : { objectUri }
|
|
1447
|
+
};
|
|
1448
|
+
}
|
|
1449
|
+
async function resolveWikiGraphArchiveTarget(file) {
|
|
1450
|
+
const path = getNodeResourcePath(file);
|
|
1451
|
+
const candidate = await canonicalPath(path);
|
|
1452
|
+
const candidateIdentity = await readFileIdentity(path);
|
|
1453
|
+
const matches = await findMatchingMembers(
|
|
1454
|
+
await listWikiGraphLibraries(),
|
|
1455
|
+
candidate,
|
|
1456
|
+
candidateIdentity
|
|
1457
|
+
);
|
|
1458
|
+
if (matches.length > 0) {
|
|
1459
|
+
const match = requireUnambiguousManagedMatch(path, matches);
|
|
1460
|
+
return { kind: "library", uri: match.archive.uri };
|
|
1461
|
+
}
|
|
1462
|
+
return { kind: "standalone", path: getNodeResourcePath(file) };
|
|
1463
|
+
}
|
|
1464
|
+
async function resolveStandaloneLocation(target) {
|
|
1465
|
+
if (target.path.startsWith("wikg://lib/")) {
|
|
1466
|
+
throw new WikiGraphArchiveOwnershipError(
|
|
1467
|
+
`A library archive URI cannot be used as a standalone path: ${target.path}`
|
|
1468
|
+
);
|
|
1469
|
+
}
|
|
1470
|
+
const path = await assertStandaloneWikiGraphArchivePath(target.path);
|
|
1471
|
+
const normalized = {
|
|
1472
|
+
kind: "standalone",
|
|
1473
|
+
path,
|
|
1474
|
+
...target.objectUri === void 0 ? {} : { objectUri: target.objectUri }
|
|
1475
|
+
};
|
|
1476
|
+
return {
|
|
1477
|
+
archiveFile: new NodeFile(path),
|
|
1478
|
+
archiveKey: path,
|
|
1479
|
+
archivePath: path,
|
|
1480
|
+
indexScope: { archiveKey: path, archivePath: path, kind: "archive-index" },
|
|
1481
|
+
locatedUri: formatLocatedWikiGraphUri(path, target.objectUri),
|
|
1482
|
+
target: normalized
|
|
1483
|
+
};
|
|
1484
|
+
}
|
|
1485
|
+
async function resolveLibraryLocation(target) {
|
|
1486
|
+
const parsedLocated = parseLocatedWikiGraphUri(target.uri);
|
|
1487
|
+
const archiveUri = parsedLocated.archivePath ?? target.uri;
|
|
1488
|
+
const parsed = parseWikiGraphLibraryUri(archiveUri);
|
|
1489
|
+
if (parsed?.kind !== "archive") {
|
|
1490
|
+
throw new WikiGraphArchiveOwnershipError(
|
|
1491
|
+
`Library archive target must be a UUID archive URI: ${target.uri}`
|
|
1492
|
+
);
|
|
1493
|
+
}
|
|
1494
|
+
await resolveWikiGraphLibrary(parsed);
|
|
1495
|
+
const archiveFile = await resolveWikiGraphLibraryArchiveFile(archiveUri);
|
|
1496
|
+
const normalized = {
|
|
1497
|
+
kind: "library",
|
|
1498
|
+
uri: target.uri
|
|
1499
|
+
};
|
|
1500
|
+
return {
|
|
1501
|
+
archiveFile,
|
|
1502
|
+
archiveKey: archiveUri,
|
|
1503
|
+
archivePath: archiveUri,
|
|
1504
|
+
indexScope: {
|
|
1505
|
+
archiveKey: archiveUri,
|
|
1506
|
+
archivePath: archiveUri,
|
|
1507
|
+
kind: "archive-index"
|
|
1508
|
+
},
|
|
1509
|
+
libraryArchiveTarget: parsed,
|
|
1510
|
+
locatedUri: target.uri,
|
|
1511
|
+
publicArchiveUri: archiveUri,
|
|
1512
|
+
target: normalized
|
|
1513
|
+
};
|
|
1514
|
+
}
|
|
1515
|
+
function withObjectUri(archiveLocator, objectUri) {
|
|
1516
|
+
if (objectUri === void 0) return archiveLocator;
|
|
1517
|
+
const path = objectUri.replace(/^wikg:\/\//u, "");
|
|
1518
|
+
return `${archiveLocator.replace(/\/$/u, "")}/${path}`;
|
|
1519
|
+
}
|
|
1520
|
+
async function canonicalPath(path) {
|
|
1521
|
+
try {
|
|
1522
|
+
return await realpath(path);
|
|
1523
|
+
} catch {
|
|
1524
|
+
const absolute = resolve3(path);
|
|
1525
|
+
const parent = dirname2(absolute);
|
|
1526
|
+
return parent === absolute ? absolute : join3(await canonicalPath(parent), basename2(absolute));
|
|
1527
|
+
}
|
|
1528
|
+
}
|
|
1529
|
+
function isWithin(root, candidate) {
|
|
1530
|
+
const child = relative(root, candidate);
|
|
1531
|
+
return child === "" || !child.startsWith("..") && !isAbsolute3(child);
|
|
1532
|
+
}
|
|
1533
|
+
async function findMatchingMembers(libraries, candidate, candidateIdentity) {
|
|
1534
|
+
const members = [];
|
|
1535
|
+
for (const library of libraries) {
|
|
1536
|
+
const archives = await listWikiGraphLibraryArchives({
|
|
1537
|
+
isDefault: library.isDefault,
|
|
1538
|
+
kind: "scope",
|
|
1539
|
+
...library.isDefault ? {} : { publicId: library.publicId }
|
|
1540
|
+
});
|
|
1541
|
+
for (const archive of archives) {
|
|
1542
|
+
if (archive.file === void 0) continue;
|
|
1543
|
+
members.push({
|
|
1544
|
+
archive,
|
|
1545
|
+
library,
|
|
1546
|
+
memberPath: getNodeResourcePath(archive.file)
|
|
1547
|
+
});
|
|
1548
|
+
}
|
|
1549
|
+
}
|
|
1550
|
+
const exact = [];
|
|
1551
|
+
for (const member of members) {
|
|
1552
|
+
if (await canonicalPath(member.memberPath) === candidate) {
|
|
1553
|
+
exact.push(member);
|
|
1554
|
+
}
|
|
1555
|
+
}
|
|
1556
|
+
if (exact.length > 0 || candidateIdentity === void 0) return exact;
|
|
1557
|
+
const identityMatches = [];
|
|
1558
|
+
for (const member of members) {
|
|
1559
|
+
if (await readFileIdentity(member.memberPath) === candidateIdentity) {
|
|
1560
|
+
identityMatches.push(member);
|
|
1561
|
+
}
|
|
1562
|
+
}
|
|
1563
|
+
return identityMatches;
|
|
1564
|
+
}
|
|
1565
|
+
function requireUnambiguousManagedMatch(path, matches) {
|
|
1566
|
+
if (matches.length === 1) return matches[0];
|
|
1567
|
+
throw new WikiGraphArchiveOwnershipError(
|
|
1568
|
+
`Archive file identity is ambiguous across managed archives: ${path}. Use an explicit library UUID URI (${matches.map((match) => match.archive.uri).join(", ")}).`
|
|
1569
|
+
);
|
|
1570
|
+
}
|
|
1571
|
+
async function readFileIdentity(path) {
|
|
1572
|
+
try {
|
|
1573
|
+
const value = await stat2(path);
|
|
1574
|
+
return `${value.dev}:${value.ino}`;
|
|
1575
|
+
} catch {
|
|
1576
|
+
return void 0;
|
|
1577
|
+
}
|
|
1578
|
+
}
|
|
1579
|
+
|
|
1385
1580
|
// src/conversions.ts
|
|
1386
1581
|
var WikiGraphConversionManager = class {
|
|
1387
1582
|
#runtime;
|
|
@@ -1396,6 +1591,9 @@ var WikiGraphConversionManager = class {
|
|
|
1396
1591
|
if (options.targetStage !== void 0 && options.output.format !== "wikg") {
|
|
1397
1592
|
throw new Error("targetStage is only supported for wikg output.");
|
|
1398
1593
|
}
|
|
1594
|
+
if (options.output.format === "pcex") {
|
|
1595
|
+
throw new Error("pcex is supported only as a conversion input.");
|
|
1596
|
+
}
|
|
1399
1597
|
if (options.targetStage !== void 0 && options.input.format === "wikg") {
|
|
1400
1598
|
throw new Error(
|
|
1401
1599
|
"targetStage is only supported when creating a wikg archive."
|
|
@@ -1414,7 +1612,17 @@ var WikiGraphConversionManager = class {
|
|
|
1414
1612
|
options.digestDirectory,
|
|
1415
1613
|
requiresDigest
|
|
1416
1614
|
);
|
|
1615
|
+
let chapterCount;
|
|
1616
|
+
let chapterPaths;
|
|
1417
1617
|
const write = async (archive) => {
|
|
1618
|
+
if (requiresDigest) await options.onSourceImported?.();
|
|
1619
|
+
if (typeof archive.readToc === "function") {
|
|
1620
|
+
const toc = await archive.readToc();
|
|
1621
|
+
chapterCount = toc === void 0 ? void 0 : countTocChapters(toc.items);
|
|
1622
|
+
if (options.output.format === "wikg") {
|
|
1623
|
+
chapterPaths = toc === void 0 ? [] : collectTocChapterPaths(toc.items);
|
|
1624
|
+
}
|
|
1625
|
+
}
|
|
1418
1626
|
await writeArchive(archive, options.output.path, options.output.format);
|
|
1419
1627
|
};
|
|
1420
1628
|
if (options.input.format === "wikg") {
|
|
@@ -1422,7 +1630,9 @@ var WikiGraphConversionManager = class {
|
|
|
1422
1630
|
throw new Error("wikg input requires a file path.");
|
|
1423
1631
|
}
|
|
1424
1632
|
await app.openSession(
|
|
1425
|
-
new NodeFile(
|
|
1633
|
+
new NodeFile(
|
|
1634
|
+
await assertStandaloneWikiGraphArchivePath(options.input.path)
|
|
1635
|
+
),
|
|
1426
1636
|
write
|
|
1427
1637
|
);
|
|
1428
1638
|
} else if ("stream" in options.input) {
|
|
@@ -1447,16 +1657,29 @@ var WikiGraphConversionManager = class {
|
|
|
1447
1657
|
};
|
|
1448
1658
|
if (options.input.format === "epub") {
|
|
1449
1659
|
await app.digestEpubSession(digestOptions, write);
|
|
1660
|
+
} else if (options.input.format === "pcex") {
|
|
1661
|
+
await app.digestPcexSession(
|
|
1662
|
+
{
|
|
1663
|
+
...digestOptions,
|
|
1664
|
+
pdfDigest: options.input.sourcePdf.digest,
|
|
1665
|
+
...options.input.sourcePdf.name === void 0 ? {} : { pdfName: options.input.sourcePdf.name }
|
|
1666
|
+
},
|
|
1667
|
+
write
|
|
1668
|
+
);
|
|
1450
1669
|
} else if (options.input.format === "markdown") {
|
|
1451
1670
|
await app.digestMarkdownSession(digestOptions, write);
|
|
1452
1671
|
} else {
|
|
1453
1672
|
await app.digestTxtSession(digestOptions, write);
|
|
1454
1673
|
}
|
|
1455
1674
|
}
|
|
1675
|
+
const sourceArtifact = await resolveSourceArtifact(options.input);
|
|
1456
1676
|
return {
|
|
1677
|
+
...chapterCount === void 0 ? {} : { chapterCount },
|
|
1678
|
+
...chapterPaths === void 0 ? {} : { chapterPaths },
|
|
1457
1679
|
inputFormat: options.input.format,
|
|
1458
1680
|
outputFormat: options.output.format,
|
|
1459
|
-
outputPath: options.output.path
|
|
1681
|
+
outputPath: options.output.path,
|
|
1682
|
+
...sourceArtifact === void 0 ? {} : { sourceArtifact }
|
|
1460
1683
|
};
|
|
1461
1684
|
}
|
|
1462
1685
|
};
|
|
@@ -1469,11 +1692,67 @@ async function prepareDigestDirectory(path, required) {
|
|
|
1469
1692
|
return new NodeDirectory(resolved);
|
|
1470
1693
|
}
|
|
1471
1694
|
async function writeArchive(archive, path, format) {
|
|
1472
|
-
const
|
|
1695
|
+
const outputPath = format === "wikg" ? await assertStandaloneWikiGraphArchivePath(path) : resolveWikiGraphRuntimePath(path);
|
|
1696
|
+
const file = new NodeFile(outputPath);
|
|
1473
1697
|
if (format === "epub") await archive.exportEpub(file);
|
|
1474
1698
|
else if (format === "wikg") await archive.saveAs(file);
|
|
1699
|
+
else if (format === "pcex") throw new Error("pcex output is not supported.");
|
|
1475
1700
|
else await archive.exportText(file);
|
|
1476
1701
|
}
|
|
1702
|
+
async function resolveSourceArtifact(input) {
|
|
1703
|
+
if (!("path" in input) || input.format === "wikg") return void 0;
|
|
1704
|
+
if (input.format === "pcex") {
|
|
1705
|
+
return {
|
|
1706
|
+
digest: input.sourcePdf.digest.trim().toLowerCase(),
|
|
1707
|
+
mediaType: "application/pdf",
|
|
1708
|
+
...input.sourcePdf.name === void 0 ? {} : { name: input.sourcePdf.name }
|
|
1709
|
+
};
|
|
1710
|
+
}
|
|
1711
|
+
if (input.format !== "epub") return void 0;
|
|
1712
|
+
const path = resolveWikiGraphRuntimePath(input.path);
|
|
1713
|
+
const hash = createHash("sha256");
|
|
1714
|
+
try {
|
|
1715
|
+
for await (const chunk of createReadStream(path)) {
|
|
1716
|
+
hash.update(chunk);
|
|
1717
|
+
}
|
|
1718
|
+
} catch (error) {
|
|
1719
|
+
if (isNodeErrorCode(error, "ENOENT")) return void 0;
|
|
1720
|
+
throw error;
|
|
1721
|
+
}
|
|
1722
|
+
return {
|
|
1723
|
+
digest: hash.digest("hex"),
|
|
1724
|
+
mediaType: "application/epub+zip",
|
|
1725
|
+
name: basename3(path)
|
|
1726
|
+
};
|
|
1727
|
+
}
|
|
1728
|
+
function isNodeErrorCode(error, code) {
|
|
1729
|
+
return typeof error === "object" && error !== null && "code" in error && error.code === code;
|
|
1730
|
+
}
|
|
1731
|
+
function countTocChapters(items) {
|
|
1732
|
+
let count = 0;
|
|
1733
|
+
for (const item of items) {
|
|
1734
|
+
if (item.serialId !== void 0) count += 1;
|
|
1735
|
+
count += countTocChapters(
|
|
1736
|
+
item.children
|
|
1737
|
+
);
|
|
1738
|
+
}
|
|
1739
|
+
return count;
|
|
1740
|
+
}
|
|
1741
|
+
function collectTocChapterPaths(items, ancestorKeys = []) {
|
|
1742
|
+
const paths = [];
|
|
1743
|
+
for (const item of items) {
|
|
1744
|
+
const key = item.key ?? `chapter-${item.serialId ?? "group"}`;
|
|
1745
|
+
const chapterPath = [...ancestorKeys, key];
|
|
1746
|
+
if (item.serialId !== void 0) paths.push(chapterPath.join("/"));
|
|
1747
|
+
paths.push(
|
|
1748
|
+
...collectTocChapterPaths(
|
|
1749
|
+
item.children,
|
|
1750
|
+
chapterPath
|
|
1751
|
+
)
|
|
1752
|
+
);
|
|
1753
|
+
}
|
|
1754
|
+
return paths;
|
|
1755
|
+
}
|
|
1477
1756
|
|
|
1478
1757
|
// src/query-runtime.ts
|
|
1479
1758
|
import {
|
|
@@ -1670,499 +1949,130 @@ import {
|
|
|
1670
1949
|
isArchiveSearchIndexCurrent,
|
|
1671
1950
|
listChapters
|
|
1672
1951
|
} from "wiki-graph-core";
|
|
1673
|
-
|
|
1674
|
-
|
|
1675
|
-
|
|
1676
|
-
|
|
1677
|
-
|
|
1678
|
-
|
|
1679
|
-
|
|
1680
|
-
|
|
1681
|
-
|
|
1682
|
-
|
|
1952
|
+
async function inspectWikiGraphArchive(document, options) {
|
|
1953
|
+
const [allChapters, indexCacheCurrent, summaryWords, ...coverage] = await Promise.all([
|
|
1954
|
+
listChapters(document),
|
|
1955
|
+
isArchiveSearchIndexCurrent(document),
|
|
1956
|
+
readSummaryWords(document),
|
|
1957
|
+
document.indexArtifacts.listCoverage("fts"),
|
|
1958
|
+
document.indexArtifacts.listCoverage("embedding-source"),
|
|
1959
|
+
document.indexArtifacts.listCoverage("embedding-summary")
|
|
1960
|
+
]);
|
|
1961
|
+
const chapters = selectChapters(allChapters, options.chapterId);
|
|
1962
|
+
const coverageByKind = {
|
|
1963
|
+
fts: byChapterId(coverage[0]),
|
|
1964
|
+
sourceEmbedding: byChapterId(coverage[1]),
|
|
1965
|
+
summaryEmbedding: byChapterId(coverage[2])
|
|
1966
|
+
};
|
|
1967
|
+
const inspections = await Promise.all(
|
|
1968
|
+
chapters.map(async (chapter) => {
|
|
1969
|
+
const serial = await document.serials.getById(chapter.chapterId);
|
|
1970
|
+
return {
|
|
1971
|
+
capabilities: {
|
|
1972
|
+
knowledgeGraph: {
|
|
1973
|
+
completed: serial?.knowledgeGraphReady === true
|
|
1974
|
+
},
|
|
1975
|
+
readingGraph: { completed: serial?.topologyReady === true },
|
|
1976
|
+
readingSummary: { completed: chapter.stage === "summarized" }
|
|
1977
|
+
},
|
|
1978
|
+
chapter,
|
|
1979
|
+
indexes: {
|
|
1980
|
+
fts: inspectArtifact(
|
|
1981
|
+
"fts",
|
|
1982
|
+
coverageByKind.fts.get(chapter.chapterId)
|
|
1983
|
+
),
|
|
1984
|
+
sourceEmbedding: inspectArtifact(
|
|
1985
|
+
"embedding-source",
|
|
1986
|
+
coverageByKind.sourceEmbedding.get(chapter.chapterId)
|
|
1987
|
+
),
|
|
1988
|
+
summaryEmbedding: inspectArtifact(
|
|
1989
|
+
"embedding-summary",
|
|
1990
|
+
coverageByKind.summaryEmbedding.get(chapter.chapterId)
|
|
1991
|
+
)
|
|
1992
|
+
},
|
|
1993
|
+
revision: serial?.revision ?? 0,
|
|
1994
|
+
summaryWords: summaryWords.get(chapter.chapterId) ?? 0
|
|
1995
|
+
};
|
|
1996
|
+
})
|
|
1683
1997
|
);
|
|
1684
|
-
const
|
|
1685
|
-
const wordSeconds = words * profile.secondsPerWord;
|
|
1998
|
+
const selected = inspections[0];
|
|
1686
1999
|
return {
|
|
1687
|
-
|
|
1688
|
-
|
|
1689
|
-
|
|
1690
|
-
|
|
1691
|
-
|
|
1692
|
-
tokens: {
|
|
1693
|
-
cacheableInput: Math.ceil(words * profile.cacheableInputTokenPerWord),
|
|
1694
|
-
input: Math.ceil(words * profile.inputTokenPerWord),
|
|
1695
|
-
output: Math.ceil(words * profile.outputTokenPerWord)
|
|
1696
|
-
}
|
|
2000
|
+
archiveUri: options.archiveUri,
|
|
2001
|
+
chapters: inspections,
|
|
2002
|
+
localIndexCache: { current: indexCacheCurrent },
|
|
2003
|
+
scope: options.chapterId === void 0 ? { type: "archive" } : { chapterId: options.chapterId, type: "chapter" },
|
|
2004
|
+
uri: selected === void 0 || options.chapterId === void 0 ? options.archiveUri : formatLocatedChapterUri(options.archiveUri, selected.chapter)
|
|
1697
2005
|
};
|
|
1698
2006
|
}
|
|
1699
|
-
function
|
|
1700
|
-
if (
|
|
1701
|
-
|
|
1702
|
-
|
|
1703
|
-
|
|
1704
|
-
|
|
1705
|
-
|
|
1706
|
-
hints.push({
|
|
1707
|
-
configKey: "request",
|
|
1708
|
-
configValue: recommendedRequest,
|
|
1709
|
-
current: input.concurrent.request,
|
|
1710
|
-
kind: "request",
|
|
1711
|
-
message: "LLM request concurrency can often be higher. Use at least 4; 6-8 is usually faster when the provider allows it.",
|
|
1712
|
-
recommended: recommendedRequest
|
|
1713
|
-
});
|
|
1714
|
-
}
|
|
1715
|
-
if (input.concurrent.job < 4 && input.chapters > 1) {
|
|
1716
|
-
hints.push({
|
|
1717
|
-
configKey: "job",
|
|
1718
|
-
configValue: 4,
|
|
1719
|
-
current: input.concurrent.job,
|
|
1720
|
-
kind: "job",
|
|
1721
|
-
message: "Multiple chapters need work and job concurrency is below 4. Raising it lets chapter jobs run in parallel.",
|
|
1722
|
-
recommended: 4
|
|
1723
|
-
});
|
|
2007
|
+
function selectChapters(chapters, chapterId) {
|
|
2008
|
+
if (chapterId === void 0) return chapters;
|
|
2009
|
+
const selected = chapters.filter(
|
|
2010
|
+
(chapter) => chapter.chapterId === chapterId
|
|
2011
|
+
);
|
|
2012
|
+
if (selected.length === 0) {
|
|
2013
|
+
throw new Error(`Chapter ${chapterId} does not exist.`);
|
|
1724
2014
|
}
|
|
1725
|
-
return
|
|
2015
|
+
return selected;
|
|
1726
2016
|
}
|
|
1727
|
-
function
|
|
1728
|
-
|
|
1729
|
-
return "not configured";
|
|
1730
|
-
}
|
|
1731
|
-
return llm.provider === void 0 ? llm.model : `${llm.provider}/${llm.model}`;
|
|
2017
|
+
function byChapterId(coverage) {
|
|
2018
|
+
return new Map(coverage.map((item) => [item.serialId, item]));
|
|
1732
2019
|
}
|
|
1733
|
-
function
|
|
1734
|
-
|
|
1735
|
-
|
|
1736
|
-
|
|
1737
|
-
|
|
1738
|
-
|
|
1739
|
-
|
|
1740
|
-
}
|
|
1741
|
-
return `${Math.round(minutes / 60)}h`;
|
|
2020
|
+
function inspectArtifact(kind, coverage) {
|
|
2021
|
+
return {
|
|
2022
|
+
current: coverage?.current === true,
|
|
2023
|
+
exists: coverage?.sourceRevision !== void 0,
|
|
2024
|
+
kind,
|
|
2025
|
+
...coverage?.sourceRevision === void 0 ? {} : { sourceRevision: coverage.sourceRevision }
|
|
2026
|
+
};
|
|
1742
2027
|
}
|
|
1743
|
-
function
|
|
1744
|
-
|
|
1745
|
-
|
|
1746
|
-
|
|
1747
|
-
|
|
1748
|
-
|
|
1749
|
-
|
|
1750
|
-
|
|
1751
|
-
|
|
1752
|
-
|
|
1753
|
-
|
|
1754
|
-
|
|
1755
|
-
|
|
1756
|
-
|
|
1757
|
-
|
|
1758
|
-
|
|
1759
|
-
|
|
1760
|
-
|
|
1761
|
-
|
|
1762
|
-
|
|
1763
|
-
|
|
1764
|
-
|
|
1765
|
-
|
|
1766
|
-
|
|
1767
|
-
|
|
1768
|
-
|
|
1769
|
-
|
|
1770
|
-
|
|
1771
|
-
|
|
1772
|
-
|
|
1773
|
-
|
|
1774
|
-
|
|
1775
|
-
|
|
1776
|
-
|
|
1777
|
-
|
|
1778
|
-
|
|
1779
|
-
|
|
1780
|
-
|
|
1781
|
-
minSecondsPerCall: 15,
|
|
1782
|
-
outputTokenPerWord: 4,
|
|
1783
|
-
secondsPerWord: 0.01,
|
|
1784
|
-
wordsPerCall: 160
|
|
1785
|
-
};
|
|
1786
|
-
case "reading-summary":
|
|
1787
|
-
return {
|
|
1788
|
-
cacheableInputTokenPerWord: 45,
|
|
1789
|
-
inputTokenPerWord: 55,
|
|
1790
|
-
maxSecondsPerCall: 25,
|
|
1791
|
-
minSecondsPerCall: 10,
|
|
1792
|
-
outputTokenPerWord: 4,
|
|
1793
|
-
secondsPerWord: 0.01,
|
|
1794
|
-
wordsPerCall: 110
|
|
1795
|
-
};
|
|
2028
|
+
async function readSummaryWords(document) {
|
|
2029
|
+
const rows = await document.readDatabase(
|
|
2030
|
+
async (database) => await database.queryAll(
|
|
2031
|
+
`
|
|
2032
|
+
SELECT chapter_id, COALESCE(SUM(words_count), 0) AS words
|
|
2033
|
+
FROM text_sentence_records
|
|
2034
|
+
WHERE kind = 2
|
|
2035
|
+
GROUP BY chapter_id
|
|
2036
|
+
`,
|
|
2037
|
+
void 0,
|
|
2038
|
+
(row) => ({
|
|
2039
|
+
chapterId: Number(row.chapter_id),
|
|
2040
|
+
words: Number(row.words)
|
|
2041
|
+
})
|
|
2042
|
+
)
|
|
2043
|
+
);
|
|
2044
|
+
return new Map(rows.map((row) => [row.chapterId, row.words]));
|
|
2045
|
+
}
|
|
2046
|
+
function formatLocatedChapterUri(archiveUri, chapter) {
|
|
2047
|
+
return `${archiveUri.replace(/\/+$/u, "")}/${chapter.uri.replace(/^wikg:\/\//u, "")}`;
|
|
2048
|
+
}
|
|
2049
|
+
|
|
2050
|
+
// src/archive/write.ts
|
|
2051
|
+
import {
|
|
2052
|
+
WikiGraphArchiveFile,
|
|
2053
|
+
writeWikiGraphLibraryArchive
|
|
2054
|
+
} from "wiki-graph-core";
|
|
2055
|
+
async function writeWikiGraphArchiveLocation(location, operation, options = {}) {
|
|
2056
|
+
const cacheOnly = options.searchIndexWritebackPolicy === "cache";
|
|
2057
|
+
const write = async (archiveFile = location.archiveFile) => await new WikiGraphArchiveFile(archiveFile).write(operation, {
|
|
2058
|
+
...cacheOnly ? {} : { derivedStateKeys: [location.archiveKey, location.archivePath] },
|
|
2059
|
+
...options.invalidateContinuationCursors === void 0 ? {} : {
|
|
2060
|
+
invalidateContinuationCursors: options.invalidateContinuationCursors
|
|
2061
|
+
},
|
|
2062
|
+
...options.searchIndexWritebackPolicy === void 0 ? {} : { searchIndexWritebackPolicy: options.searchIndexWritebackPolicy }
|
|
2063
|
+
});
|
|
2064
|
+
if (location.libraryArchiveTarget === void 0 || cacheOnly || options.refreshLibraryIndex === false) {
|
|
2065
|
+
return await write();
|
|
1796
2066
|
}
|
|
1797
|
-
|
|
1798
|
-
|
|
1799
|
-
|
|
1800
|
-
|
|
1801
|
-
|
|
1802
|
-
|
|
1803
|
-
|
|
1804
|
-
|
|
1805
|
-
|
|
1806
|
-
config,
|
|
1807
|
-
ftsArtifactCoverage,
|
|
1808
|
-
sourceEmbeddingArtifactCoverage,
|
|
1809
|
-
summaryEmbeddingArtifactCoverage
|
|
1810
|
-
] = await Promise.all([
|
|
1811
|
-
readInspectChapters(document, options.chapterId),
|
|
1812
|
-
readSummaryWords(document, options.chapterId),
|
|
1813
|
-
isArchiveSearchIndexCurrent(document),
|
|
1814
|
-
loadWikiGraphRuntimeConfig(),
|
|
1815
|
-
readIndexArtifactCoverage(document, "fts", options.chapterId),
|
|
1816
|
-
readIndexArtifactCoverage(document, "embedding-source", options.chapterId),
|
|
1817
|
-
readIndexArtifactCoverage(document, "embedding-summary", options.chapterId)
|
|
1818
|
-
]);
|
|
1819
|
-
const selectedChapter = options.chapterId === void 0 ? void 0 : chapters[0];
|
|
1820
|
-
const scopeUri = selectedChapter === void 0 ? archiveUri : formatInspectLocatedChapterUri(archiveUri, selectedChapter);
|
|
1821
|
-
const concurrent = {
|
|
1822
|
-
job: config.concurrent?.job ?? DEFAULT_GENERATION_JOB_CONCURRENCY,
|
|
1823
|
-
request: config.concurrent?.request ?? DEFAULT_GENERATION_REQUEST_CONCURRENCY
|
|
1824
|
-
};
|
|
1825
|
-
const planningModel = formatGenerationPlanningModel(config.llm);
|
|
1826
|
-
const contentChapters = chapters.filter(
|
|
1827
|
-
(chapter) => chapter.stage !== "planned"
|
|
1828
|
-
);
|
|
1829
|
-
const sourceWords = sumWords(contentChapters);
|
|
1830
|
-
const readingGraphCovered = contentChapters.filter(
|
|
1831
|
-
(chapter) => chapter.readingGraphReady
|
|
1832
|
-
);
|
|
1833
|
-
const knowledgeGraphCovered = contentChapters.filter(
|
|
1834
|
-
(chapter) => chapter.knowledgeGraphReady
|
|
1835
|
-
);
|
|
1836
|
-
const summaryCovered = contentChapters.filter(
|
|
1837
|
-
(chapter) => chapter.summaryReady
|
|
1838
|
-
);
|
|
1839
|
-
const ftsArtifactCovered = filterChaptersByCurrentIndexArtifact(
|
|
1840
|
-
contentChapters,
|
|
1841
|
-
ftsArtifactCoverage
|
|
1842
|
-
);
|
|
1843
|
-
const sourceEmbeddingArtifactCovered = filterChaptersByCurrentIndexArtifact(
|
|
1844
|
-
contentChapters,
|
|
1845
|
-
sourceEmbeddingArtifactCoverage
|
|
1846
|
-
);
|
|
1847
|
-
const summaryEmbeddingArtifactCovered = filterChaptersByCurrentIndexArtifact(
|
|
1848
|
-
contentChapters,
|
|
1849
|
-
summaryEmbeddingArtifactCoverage
|
|
1850
|
-
);
|
|
1851
|
-
const queryBlockedContentChapters = contentChapters.filter(
|
|
1852
|
-
(chapter) => !ftsArtifactCovered.includes(chapter) && !sourceEmbeddingArtifactCovered.includes(chapter)
|
|
1853
|
-
);
|
|
1854
|
-
const queryBlockedChapters = queryBlockedContentChapters.map(
|
|
1855
|
-
(chapter) => createInspectChapterReference(archiveUri, chapter)
|
|
1856
|
-
);
|
|
1857
|
-
const queryReady = queryBlockedChapters.length === 0;
|
|
1858
|
-
const improvements = createInspectImprovements({
|
|
1859
|
-
concurrent,
|
|
1860
|
-
contentChapters,
|
|
1861
|
-
indexCacheCurrent,
|
|
1862
|
-
planningModel,
|
|
1863
|
-
queryBlockedChapters: queryBlockedContentChapters,
|
|
1864
|
-
summaryCovered,
|
|
1865
|
-
knowledgeGraphCovered,
|
|
1866
|
-
readingGraphCovered
|
|
1867
|
-
});
|
|
1868
|
-
const performanceHints = createGenerationPerformanceHints({
|
|
1869
|
-
chapters: Math.max(
|
|
1870
|
-
0,
|
|
1871
|
-
...improvements.map((improvement) => improvement.missingChapters ?? 0)
|
|
1872
|
-
),
|
|
1873
|
-
concurrent,
|
|
1874
|
-
hasJobWork: improvements.some(
|
|
1875
|
-
(improvement) => improvement.planning !== void 0
|
|
1876
|
-
),
|
|
1877
|
-
hasRequestWork: improvements.some(
|
|
1878
|
-
(improvement) => improvement.planning !== void 0
|
|
1879
|
-
)
|
|
1880
|
-
});
|
|
1881
|
-
return {
|
|
1882
|
-
archiveUri,
|
|
1883
|
-
uri: scopeUri,
|
|
1884
|
-
scope: options.chapterId === void 0 ? { type: "archive" } : { chapterId: options.chapterId, type: "chapter" },
|
|
1885
|
-
content: {
|
|
1886
|
-
chapters: {
|
|
1887
|
-
content: contentChapters.length,
|
|
1888
|
-
planned: chapters.length - contentChapters.length,
|
|
1889
|
-
total: chapters.length
|
|
1890
|
-
},
|
|
1891
|
-
sourceWords,
|
|
1892
|
-
summaryWords
|
|
1893
|
-
},
|
|
1894
|
-
localIndexCache: {
|
|
1895
|
-
current: indexCacheCurrent,
|
|
1896
|
-
...!queryReady ? {
|
|
1897
|
-
blockedBy: "missing-search-artifacts",
|
|
1898
|
-
impact: "The cache cannot be built until every content chapter has a current FTS or source embedding artifact."
|
|
1899
|
-
} : indexCacheCurrent ? {} : {
|
|
1900
|
-
impact: "The next archive query may need to sync index cache first.",
|
|
1901
|
-
resource: "local CPU/disk time only; no provider calls."
|
|
1902
|
-
},
|
|
1903
|
-
status: !queryReady ? "blocked" : indexCacheCurrent ? "current" : "missing-or-outdated",
|
|
1904
|
-
syncAvailable: queryReady && !indexCacheCurrent
|
|
1905
|
-
},
|
|
1906
|
-
coverage: {
|
|
1907
|
-
ftsIndexArtifact: createInspectCoverage(
|
|
1908
|
-
ftsArtifactCovered,
|
|
1909
|
-
contentChapters
|
|
1910
|
-
),
|
|
1911
|
-
knowledgeGraph: createInspectCoverage(
|
|
1912
|
-
knowledgeGraphCovered,
|
|
1913
|
-
contentChapters
|
|
1914
|
-
),
|
|
1915
|
-
readingGraph: createInspectCoverage(readingGraphCovered, contentChapters),
|
|
1916
|
-
summary: createInspectCoverage(summaryCovered, contentChapters),
|
|
1917
|
-
sourceEmbeddingIndexArtifact: createInspectCoverage(
|
|
1918
|
-
sourceEmbeddingArtifactCovered,
|
|
1919
|
-
contentChapters
|
|
1920
|
-
),
|
|
1921
|
-
summaryEmbeddingIndexArtifact: createInspectCoverage(
|
|
1922
|
-
summaryEmbeddingArtifactCovered,
|
|
1923
|
-
contentChapters
|
|
1924
|
-
)
|
|
1925
|
-
},
|
|
1926
|
-
query: {
|
|
1927
|
-
ready: queryReady
|
|
1928
|
-
},
|
|
1929
|
-
searchArtifacts: {
|
|
1930
|
-
blockedChapters: queryBlockedChapters,
|
|
1931
|
-
...!queryReady ? {
|
|
1932
|
-
resource: "local CPU/disk time only; no provider calls."
|
|
1933
|
-
} : {},
|
|
1934
|
-
ready: queryReady,
|
|
1935
|
-
status: queryReady ? "ready" : "incomplete"
|
|
1936
|
-
},
|
|
1937
|
-
retrievalGuidance: formatRetrievalGuidance({
|
|
1938
|
-
indexCacheCurrent,
|
|
1939
|
-
knowledgeGraphCovered,
|
|
1940
|
-
readingGraphCovered,
|
|
1941
|
-
contentChapters,
|
|
1942
|
-
queryReady,
|
|
1943
|
-
sourceWords
|
|
1944
|
-
}),
|
|
1945
|
-
improvements,
|
|
1946
|
-
performanceHints,
|
|
1947
|
-
summaryCoverageNote: "Summary coverage means a current summary artifact exists for the chapter; it does not guarantee that every source topic is represented in the summary text."
|
|
1948
|
-
};
|
|
1949
|
-
}
|
|
1950
|
-
function createInspectChapterReference(archiveUri, chapter) {
|
|
1951
|
-
return {
|
|
1952
|
-
chapterId: chapter.chapterId,
|
|
1953
|
-
locatedUri: formatInspectLocatedChapterUri(archiveUri, chapter),
|
|
1954
|
-
title: chapter.title,
|
|
1955
|
-
uri: chapter.uri
|
|
1956
|
-
};
|
|
1957
|
-
}
|
|
1958
|
-
function formatInspectLocatedChapterUri(archiveUri, chapter) {
|
|
1959
|
-
return `${archiveUri.replace(/\/+$/u, "")}/${chapter.uri.replace(/^wikg:\/\//u, "")}`;
|
|
1960
|
-
}
|
|
1961
|
-
async function readInspectChapters(document, chapterId) {
|
|
1962
|
-
const chapters = chapterId === void 0 ? await listChapters(document) : (await listChapters(document)).filter(
|
|
1963
|
-
(chapter) => chapter.chapterId === chapterId
|
|
1964
|
-
);
|
|
1965
|
-
if (chapterId !== void 0 && chapters.length === 0) {
|
|
1966
|
-
throw new Error(`Chapter ${chapterId} does not exist.`);
|
|
1967
|
-
}
|
|
1968
|
-
return await Promise.all(
|
|
1969
|
-
chapters.map(async (chapter) => {
|
|
1970
|
-
const serial = await document.serials.getById(chapter.chapterId);
|
|
1971
|
-
return {
|
|
1972
|
-
...chapter,
|
|
1973
|
-
knowledgeGraphReady: serial?.knowledgeGraphReady === true,
|
|
1974
|
-
readingGraphReady: serial?.topologyReady === true,
|
|
1975
|
-
summaryReady: chapter.stage === "summarized"
|
|
1976
|
-
};
|
|
1977
|
-
})
|
|
1978
|
-
);
|
|
1979
|
-
}
|
|
1980
|
-
async function readSummaryWords(document, chapterId) {
|
|
1981
|
-
return await document.readDatabase(
|
|
1982
|
-
async (database) => await database.queryOne(
|
|
1983
|
-
`
|
|
1984
|
-
SELECT COALESCE(SUM(words_count), 0) AS words
|
|
1985
|
-
FROM text_sentence_records
|
|
1986
|
-
WHERE kind = 2
|
|
1987
|
-
${chapterId === void 0 ? "" : "AND chapter_id = ?"}
|
|
1988
|
-
`,
|
|
1989
|
-
chapterId === void 0 ? void 0 : [chapterId],
|
|
1990
|
-
(row) => Number(row.words)
|
|
1991
|
-
) ?? 0
|
|
1992
|
-
);
|
|
1993
|
-
}
|
|
1994
|
-
async function readIndexArtifactCoverage(document, kind, chapterId) {
|
|
1995
|
-
const coverage = await document.indexArtifacts.listCoverage(kind);
|
|
1996
|
-
return chapterId === void 0 ? coverage : coverage.filter((item) => item.serialId === chapterId);
|
|
1997
|
-
}
|
|
1998
|
-
function filterChaptersByCurrentIndexArtifact(chapters, coverage) {
|
|
1999
|
-
const currentChapterIds = new Set(
|
|
2000
|
-
coverage.filter((item) => item.current).map((item) => item.serialId)
|
|
2001
|
-
);
|
|
2002
|
-
return chapters.filter((chapter) => currentChapterIds.has(chapter.chapterId));
|
|
2003
|
-
}
|
|
2004
|
-
function createInspectCoverage(covered, total) {
|
|
2005
|
-
const coveredWords = sumWords(covered);
|
|
2006
|
-
const totalWords = sumWords(total);
|
|
2007
|
-
return {
|
|
2008
|
-
coveredChapters: covered.length,
|
|
2009
|
-
coveredWords,
|
|
2010
|
-
percent: formatPercent(coveredWords, totalWords),
|
|
2011
|
-
totalChapters: total.length,
|
|
2012
|
-
totalWords
|
|
2013
|
-
};
|
|
2014
|
-
}
|
|
2015
|
-
function formatRetrievalGuidance(input) {
|
|
2016
|
-
if (input.sourceWords === 0 || input.contentChapters.length === 0) {
|
|
2017
|
-
return [
|
|
2018
|
-
"Source content: empty.",
|
|
2019
|
-
"Add source text before using query coverage or graph-based retrieval."
|
|
2020
|
-
];
|
|
2021
|
-
}
|
|
2022
|
-
const lines = [
|
|
2023
|
-
input.queryReady ? `Query: available; local index cache is ${input.indexCacheCurrent ? "current" : "missing or outdated and can be synced from artifacts"}.` : "Query: unavailable until every content chapter has a current FTS or source embedding artifact."
|
|
2024
|
-
];
|
|
2025
|
-
lines.push(
|
|
2026
|
-
formatObjectSearchGuidance(
|
|
2027
|
-
"Reading Graph object retrieval",
|
|
2028
|
-
input.readingGraphCovered,
|
|
2029
|
-
input.contentChapters
|
|
2030
|
-
)
|
|
2031
|
-
);
|
|
2032
|
-
lines.push(
|
|
2033
|
-
formatObjectSearchGuidance(
|
|
2034
|
-
"Entity/triple retrieval",
|
|
2035
|
-
input.knowledgeGraphCovered,
|
|
2036
|
-
input.contentChapters
|
|
2037
|
-
)
|
|
2038
|
-
);
|
|
2039
|
-
return lines;
|
|
2040
|
-
}
|
|
2041
|
-
function formatObjectSearchGuidance(label, covered, total) {
|
|
2042
|
-
const coveredWords = sumWords(covered);
|
|
2043
|
-
const totalWords = sumWords(total);
|
|
2044
|
-
const ratio = totalWords === 0 ? 0 : coveredWords / totalWords;
|
|
2045
|
-
const coverage = formatPercent(coveredWords, totalWords);
|
|
2046
|
-
if (ratio >= 1) {
|
|
2047
|
-
return `${label}: covers all source content; it can represent the full scope for object retrieval.`;
|
|
2048
|
-
}
|
|
2049
|
-
if (ratio >= 0.9) {
|
|
2050
|
-
return `${label}: covers ${coverage}; use it as the main path, but source --query is needed for uncovered content.`;
|
|
2051
|
-
}
|
|
2052
|
-
if (ratio >= 0.5) {
|
|
2053
|
-
return `${label}: covers ${coverage}; use it as leads, not as a full-scope substitute for source --query.`;
|
|
2054
|
-
}
|
|
2055
|
-
return `${label}: covers ${coverage}; build missing graph coverage before relying on object retrieval.`;
|
|
2056
|
-
}
|
|
2057
|
-
function createInspectImprovements(input) {
|
|
2058
|
-
const improvements = [];
|
|
2059
|
-
if (input.contentChapters.length === 0) {
|
|
2060
|
-
improvements.push({
|
|
2061
|
-
kind: "add-source",
|
|
2062
|
-
recommendation: "No source content is available yet; add or import source text before graph or summary generation.",
|
|
2063
|
-
title: "Add source content"
|
|
2064
|
-
});
|
|
2065
|
-
return improvements;
|
|
2066
|
-
}
|
|
2067
|
-
if (input.queryBlockedChapters.length > 0) {
|
|
2068
|
-
improvements.push({
|
|
2069
|
-
kind: "build",
|
|
2070
|
-
missingChapters: input.queryBlockedChapters.length,
|
|
2071
|
-
missingWords: sumWords(input.queryBlockedChapters),
|
|
2072
|
-
recommendation: "Build local FTS artifacts to make this scope queryable. This does not call a provider.",
|
|
2073
|
-
task: "index-fts",
|
|
2074
|
-
title: "Enable query with local FTS"
|
|
2075
|
-
});
|
|
2076
|
-
} else if (!input.indexCacheCurrent) {
|
|
2077
|
-
improvements.push({
|
|
2078
|
-
kind: "sync-index",
|
|
2079
|
-
recommendation: "Sync the index cache from chapter index artifacts so query starts without a lazy cache rebuild.",
|
|
2080
|
-
title: "Sync index cache"
|
|
2081
|
-
});
|
|
2082
|
-
}
|
|
2083
|
-
improvements.push(
|
|
2084
|
-
...createGraphImprovement({
|
|
2085
|
-
concurrent: input.concurrent,
|
|
2086
|
-
covered: input.readingGraphCovered,
|
|
2087
|
-
planningModel: input.planningModel,
|
|
2088
|
-
task: "reading-graph",
|
|
2089
|
-
title: "Complete Reading Graph coverage",
|
|
2090
|
-
total: input.contentChapters
|
|
2091
|
-
})
|
|
2092
|
-
);
|
|
2093
|
-
improvements.push(
|
|
2094
|
-
...createGraphImprovement({
|
|
2095
|
-
concurrent: input.concurrent,
|
|
2096
|
-
covered: input.knowledgeGraphCovered,
|
|
2097
|
-
planningModel: input.planningModel,
|
|
2098
|
-
task: "knowledge-graph",
|
|
2099
|
-
title: "Complete Knowledge Graph coverage",
|
|
2100
|
-
total: input.contentChapters
|
|
2101
|
-
})
|
|
2102
|
-
);
|
|
2103
|
-
improvements.push(
|
|
2104
|
-
...createGraphImprovement({
|
|
2105
|
-
concurrent: input.concurrent,
|
|
2106
|
-
covered: input.summaryCovered,
|
|
2107
|
-
planningModel: input.planningModel,
|
|
2108
|
-
task: "reading-summary",
|
|
2109
|
-
title: "Complete Summary coverage",
|
|
2110
|
-
total: input.contentChapters
|
|
2111
|
-
})
|
|
2112
|
-
);
|
|
2113
|
-
return improvements;
|
|
2114
|
-
}
|
|
2115
|
-
function createGraphImprovement(input) {
|
|
2116
|
-
const coveredIds = new Set(input.covered.map((chapter) => chapter.chapterId));
|
|
2117
|
-
const missing = input.total.filter(
|
|
2118
|
-
(chapter) => !coveredIds.has(chapter.chapterId)
|
|
2119
|
-
);
|
|
2120
|
-
if (missing.length === 0) {
|
|
2121
|
-
return [];
|
|
2122
|
-
}
|
|
2123
|
-
const missingWords = sumWords(missing);
|
|
2124
|
-
return [
|
|
2125
|
-
{
|
|
2126
|
-
kind: "build",
|
|
2127
|
-
missingChapters: missing.length,
|
|
2128
|
-
missingWords,
|
|
2129
|
-
planning: planGenerationTask(
|
|
2130
|
-
input.task,
|
|
2131
|
-
missingWords,
|
|
2132
|
-
missing.length,
|
|
2133
|
-
input.concurrent,
|
|
2134
|
-
input.planningModel
|
|
2135
|
-
),
|
|
2136
|
-
recommendation: formatImprovementRecommendation(
|
|
2137
|
-
input.covered,
|
|
2138
|
-
input.total,
|
|
2139
|
-
missingWords
|
|
2140
|
-
),
|
|
2141
|
-
task: input.task,
|
|
2142
|
-
title: input.title
|
|
2143
|
-
}
|
|
2144
|
-
];
|
|
2145
|
-
}
|
|
2146
|
-
function formatImprovementRecommendation(covered, total, missingWords) {
|
|
2147
|
-
const totalWords = sumWords(total);
|
|
2148
|
-
const coveredWords = sumWords(covered);
|
|
2149
|
-
if (totalWords === 0) {
|
|
2150
|
-
return "No source content is available.";
|
|
2151
|
-
}
|
|
2152
|
-
if (total.length === 1 || coveredWords / totalWords >= 0.9 || missingWords <= 1e3) {
|
|
2153
|
-
return "Queue the full scope to finish the remaining gap.";
|
|
2154
|
-
}
|
|
2155
|
-
return "Queue selected chapters first if only part of the scope matters.";
|
|
2156
|
-
}
|
|
2157
|
-
function sumWords(chapters) {
|
|
2158
|
-
return chapters.reduce((total, chapter) => total + chapter.words, 0);
|
|
2159
|
-
}
|
|
2160
|
-
function formatPercent(numerator, denominator) {
|
|
2161
|
-
if (denominator === 0) {
|
|
2162
|
-
return "n/a";
|
|
2163
|
-
}
|
|
2164
|
-
const percent = numerator / denominator * 100;
|
|
2165
|
-
return `${Number.isInteger(percent) ? percent.toFixed(0) : percent.toFixed(1)}%`;
|
|
2067
|
+
return await writeWikiGraphLibraryArchive({
|
|
2068
|
+
additionalDerivedStateKeys: [location.archiveKey, location.archivePath],
|
|
2069
|
+
...options.invalidateContinuationCursors === void 0 ? {} : {
|
|
2070
|
+
invalidateContinuationCursors: options.invalidateContinuationCursors
|
|
2071
|
+
},
|
|
2072
|
+
...options.onIndexSyncError === void 0 ? {} : { onIndexSyncError: options.onIndexSyncError },
|
|
2073
|
+
operation: async (archiveFile) => await write(archiveFile),
|
|
2074
|
+
target: location.libraryArchiveTarget
|
|
2075
|
+
});
|
|
2166
2076
|
}
|
|
2167
2077
|
|
|
2168
2078
|
// src/archive/archive.ts
|
|
@@ -2182,13 +2092,13 @@ var WikiGraphArchiveManager = class {
|
|
|
2182
2092
|
}
|
|
2183
2093
|
async create(options) {
|
|
2184
2094
|
return await this.#runtime.run(async () => {
|
|
2185
|
-
const path =
|
|
2095
|
+
const path = await assertStandaloneWikiGraphArchivePath(options.path);
|
|
2186
2096
|
if (options.replace !== true && await nodePathExists(path)) {
|
|
2187
2097
|
throw new WikiGraphArchiveExistsError(path);
|
|
2188
2098
|
}
|
|
2189
|
-
const outputPath =
|
|
2190
|
-
|
|
2191
|
-
`.${
|
|
2099
|
+
const outputPath = join4(
|
|
2100
|
+
dirname3(path),
|
|
2101
|
+
`.${basename4(path)}.${randomUUID2()}.tmp.wikg`
|
|
2192
2102
|
);
|
|
2193
2103
|
try {
|
|
2194
2104
|
if (options.importPath === void 0) {
|
|
@@ -2213,15 +2123,15 @@ var WikiGraphArchiveManager = class {
|
|
|
2213
2123
|
throw error;
|
|
2214
2124
|
}
|
|
2215
2125
|
}
|
|
2216
|
-
return { locatedUri:
|
|
2126
|
+
return { locatedUri: formatLocatedWikiGraphUri2(path), path };
|
|
2217
2127
|
} finally {
|
|
2218
2128
|
await rm3(outputPath, { force: true, recursive: true });
|
|
2219
2129
|
}
|
|
2220
2130
|
}, options.signal);
|
|
2221
2131
|
}
|
|
2222
|
-
async open(
|
|
2132
|
+
async open(target, options = {}) {
|
|
2223
2133
|
const location = await this.#runtime.run(
|
|
2224
|
-
async () => await resolveWikiGraphArchiveLocation(
|
|
2134
|
+
async () => await resolveWikiGraphArchiveLocation(target),
|
|
2225
2135
|
options.signal
|
|
2226
2136
|
);
|
|
2227
2137
|
return new WikiGraphArchiveHandle(this.#runtime, location);
|
|
@@ -2244,14 +2154,17 @@ var WikiGraphArchiveHandle = class {
|
|
|
2244
2154
|
return this.#location.locatedUri;
|
|
2245
2155
|
}
|
|
2246
2156
|
get objectUri() {
|
|
2247
|
-
return
|
|
2157
|
+
return parseLocatedWikiGraphUri2(this.#location.locatedUri).objectUri ?? "wikg://";
|
|
2248
2158
|
}
|
|
2249
2159
|
get path() {
|
|
2250
2160
|
return this.#location.archivePath;
|
|
2251
2161
|
}
|
|
2162
|
+
get target() {
|
|
2163
|
+
return this.#location.target;
|
|
2164
|
+
}
|
|
2252
2165
|
async inspect(options = {}) {
|
|
2253
2166
|
return await this.#readDocument(
|
|
2254
|
-
async (document) => await
|
|
2167
|
+
async (document) => await inspectWikiGraphArchive(document, {
|
|
2255
2168
|
archiveUri: this.#location.publicArchiveUri ?? this.locatedUri,
|
|
2256
2169
|
...options.chapterId === void 0 ? {} : { chapterId: options.chapterId }
|
|
2257
2170
|
}),
|
|
@@ -2552,7 +2465,7 @@ var WikiGraphArchiveHandle = class {
|
|
|
2552
2465
|
objectUri,
|
|
2553
2466
|
async (wikimediaOptions) => await this.#readDocument(
|
|
2554
2467
|
async (document) => await readArchivePage(document, objectUri, {
|
|
2555
|
-
...
|
|
2468
|
+
...options,
|
|
2556
2469
|
...wikimediaOptions
|
|
2557
2470
|
}),
|
|
2558
2471
|
options
|
|
@@ -2698,7 +2611,7 @@ var WikiGraphArchiveHandle = class {
|
|
|
2698
2611
|
}
|
|
2699
2612
|
async #readDocument(operation, options = {}) {
|
|
2700
2613
|
return await this.#runtime.run(
|
|
2701
|
-
async () => await new
|
|
2614
|
+
async () => await new WikiGraphArchiveFile2(this.#location.archiveFile).readDocument(
|
|
2702
2615
|
operation
|
|
2703
2616
|
),
|
|
2704
2617
|
options.signal
|
|
@@ -2706,17 +2619,13 @@ var WikiGraphArchiveHandle = class {
|
|
|
2706
2619
|
}
|
|
2707
2620
|
async #writeDocument(operation, options = {}) {
|
|
2708
2621
|
return await this.#runtime.run(async () => {
|
|
2709
|
-
|
|
2710
|
-
|
|
2711
|
-
|
|
2622
|
+
return await writeWikiGraphArchiveLocation(this.#location, operation, {
|
|
2623
|
+
...options.onIndexSyncError === void 0 ? {} : { onIndexSyncError: options.onIndexSyncError },
|
|
2624
|
+
...options.refreshLibraryIndex === void 0 ? {} : { refreshLibraryIndex: options.refreshLibraryIndex },
|
|
2712
2625
|
...options.searchIndexWritebackPolicy === void 0 ? {} : {
|
|
2713
2626
|
searchIndexWritebackPolicy: options.searchIndexWritebackPolicy
|
|
2714
2627
|
}
|
|
2715
2628
|
});
|
|
2716
|
-
if (options.refreshLibraryIndex !== false) {
|
|
2717
|
-
await this.#refreshLibraryIndex(options.onIndexSyncError);
|
|
2718
|
-
}
|
|
2719
|
-
return result;
|
|
2720
2629
|
}, options.signal);
|
|
2721
2630
|
}
|
|
2722
2631
|
async #resolveScope(document, options) {
|
|
@@ -2736,52 +2645,7 @@ var WikiGraphArchiveHandle = class {
|
|
|
2736
2645
|
}
|
|
2737
2646
|
return queryable;
|
|
2738
2647
|
}
|
|
2739
|
-
async #refreshLibraryIndex(onError) {
|
|
2740
|
-
if (this.#location.libraryDirtyTarget === void 0) return;
|
|
2741
|
-
if (this.#location.libraryArchiveTarget !== void 0) {
|
|
2742
|
-
await finalizeWikiGraphLibraryArchiveWrite({
|
|
2743
|
-
target: this.#location.libraryArchiveTarget
|
|
2744
|
-
});
|
|
2745
|
-
}
|
|
2746
|
-
try {
|
|
2747
|
-
const state = await readWikiGraphLibraryIndexState(
|
|
2748
|
-
this.#location.libraryDirtyTarget
|
|
2749
|
-
);
|
|
2750
|
-
if (state.status !== "missing") {
|
|
2751
|
-
await rebuildWikiGraphLibraryIndex(this.#location.libraryDirtyTarget);
|
|
2752
|
-
}
|
|
2753
|
-
} catch (error) {
|
|
2754
|
-
onError?.(error);
|
|
2755
|
-
}
|
|
2756
|
-
}
|
|
2757
2648
|
};
|
|
2758
|
-
async function resolveWikiGraphArchiveLocation(uriOrPath) {
|
|
2759
|
-
if (!uriOrPath.startsWith("wikg://")) {
|
|
2760
|
-
return createPathLocation(resolveWikiGraphRuntimePath(uriOrPath));
|
|
2761
|
-
}
|
|
2762
|
-
const parsed = parseLocatedWikiGraphUri(uriOrPath);
|
|
2763
|
-
const portableArchiveLocator = parsed.archivePath ?? uriOrPath;
|
|
2764
|
-
const archiveLocator = portableArchiveLocator.startsWith("wikg://lib/") ? portableArchiveLocator : resolveWikiGraphRuntimePath(portableArchiveLocator);
|
|
2765
|
-
const libraryArchiveTarget = archiveLocator.startsWith("wikg://lib/") ? parseWikiGraphLibraryUri(archiveLocator) : void 0;
|
|
2766
|
-
const archiveFile = libraryArchiveTarget?.kind === "archive" ? await resolveWikiGraphLibraryArchiveFile(archiveLocator) : new NodeFile(archiveLocator);
|
|
2767
|
-
const archivePath = getNodeResourcePath(archiveFile);
|
|
2768
|
-
return {
|
|
2769
|
-
archiveFile,
|
|
2770
|
-
archiveKey: archivePath,
|
|
2771
|
-
archivePath,
|
|
2772
|
-
indexScope: { archiveKey: archivePath, archivePath, kind: "archive-index" },
|
|
2773
|
-
...libraryArchiveTarget?.kind === "archive" ? {
|
|
2774
|
-
libraryArchiveTarget,
|
|
2775
|
-
libraryDirtyTarget: {
|
|
2776
|
-
isDefault: libraryArchiveTarget.isDefault,
|
|
2777
|
-
kind: "scope",
|
|
2778
|
-
...libraryArchiveTarget.publicId === void 0 ? {} : { publicId: libraryArchiveTarget.publicId }
|
|
2779
|
-
},
|
|
2780
|
-
publicArchiveUri: portableArchiveLocator
|
|
2781
|
-
} : {},
|
|
2782
|
-
locatedUri: formatLocatedWikiGraphUri(archivePath, parsed.objectUri)
|
|
2783
|
-
};
|
|
2784
|
-
}
|
|
2785
2649
|
async function ensureArchiveSearchIndex(document, options = {}) {
|
|
2786
2650
|
const scope = options.chapters === void 0 ? {} : { chapters: options.chapters };
|
|
2787
2651
|
if (await isArchiveSearchIndexCurrent2(document, scope)) return;
|
|
@@ -2820,18 +2684,9 @@ function selectChapterSubtree(chapters, rootChapterId, depth) {
|
|
|
2820
2684
|
(chapter) => chapter.chapterId === root.chapterId || chapter.path.startsWith(prefix) && (depth === void 0 || chapter.depth - root.depth <= depth)
|
|
2821
2685
|
);
|
|
2822
2686
|
}
|
|
2823
|
-
function createPathLocation(path) {
|
|
2824
|
-
return {
|
|
2825
|
-
archiveFile: new NodeFile(path),
|
|
2826
|
-
archiveKey: path,
|
|
2827
|
-
archivePath: path,
|
|
2828
|
-
indexScope: { archiveKey: path, archivePath: path, kind: "archive-index" },
|
|
2829
|
-
locatedUri: formatLocatedWikiGraphUri(path)
|
|
2830
|
-
};
|
|
2831
|
-
}
|
|
2832
2687
|
async function createEmptyArchiveFile(path) {
|
|
2833
2688
|
const directoryPath = await mkdtemp(
|
|
2834
|
-
|
|
2689
|
+
join4(tmpdir(), "wikigraph-archive-write-")
|
|
2835
2690
|
);
|
|
2836
2691
|
try {
|
|
2837
2692
|
const directory = new NodeDirectory(directoryPath);
|
|
@@ -2853,7 +2708,7 @@ function isNodeEEXISTError(error) {
|
|
|
2853
2708
|
}
|
|
2854
2709
|
async function nodePathExists(path) {
|
|
2855
2710
|
try {
|
|
2856
|
-
await
|
|
2711
|
+
await stat3(path);
|
|
2857
2712
|
return true;
|
|
2858
2713
|
} catch (error) {
|
|
2859
2714
|
if (isNodeENOENTError(error)) return false;
|
|
@@ -2959,12 +2814,13 @@ async function assertChapterResetAllowed(archive, chapterId, stage) {
|
|
|
2959
2814
|
// src/wikispine.ts
|
|
2960
2815
|
import { spawn } from "child_process";
|
|
2961
2816
|
var nodeWikispineCommandRunner = {
|
|
2962
|
-
run: async ({ args, command, input, onStdout, signal }) => await new Promise((
|
|
2817
|
+
run: async ({ args, command, input, onStdout, signal }) => await new Promise((resolve4, reject) => {
|
|
2963
2818
|
signal?.throwIfAborted();
|
|
2964
2819
|
const child = spawn(command, [...args], {
|
|
2965
2820
|
stdio: ["pipe", "pipe", "pipe"]
|
|
2966
2821
|
});
|
|
2967
2822
|
const stderr = [];
|
|
2823
|
+
let stdoutQueue = Promise.resolve();
|
|
2968
2824
|
let settled = false;
|
|
2969
2825
|
const settle = (operation) => {
|
|
2970
2826
|
if (settled) return;
|
|
@@ -2980,14 +2836,15 @@ var nodeWikispineCommandRunner = {
|
|
|
2980
2836
|
child.stdout.setEncoding("utf8");
|
|
2981
2837
|
child.stdout.on("data", (chunk) => {
|
|
2982
2838
|
if (settled) return;
|
|
2983
|
-
|
|
2984
|
-
|
|
2985
|
-
|
|
2839
|
+
child.stdout.pause();
|
|
2840
|
+
stdoutQueue = stdoutQueue.then(async () => await onStdout(chunk)).then(() => {
|
|
2841
|
+
child.stdout.resume();
|
|
2842
|
+
}).catch((error) => {
|
|
2986
2843
|
child.kill();
|
|
2987
2844
|
settle(
|
|
2988
2845
|
() => reject(error instanceof Error ? error : new Error(String(error)))
|
|
2989
2846
|
);
|
|
2990
|
-
}
|
|
2847
|
+
});
|
|
2991
2848
|
});
|
|
2992
2849
|
child.stderr.on("data", (chunk) => stderr.push(chunk));
|
|
2993
2850
|
child.on("error", (error) => {
|
|
@@ -2998,12 +2855,14 @@ var nodeWikispineCommandRunner = {
|
|
|
2998
2855
|
);
|
|
2999
2856
|
});
|
|
3000
2857
|
child.on("close", (exitCode) => {
|
|
3001
|
-
|
|
3002
|
-
(
|
|
3003
|
-
|
|
3004
|
-
|
|
3005
|
-
|
|
3006
|
-
|
|
2858
|
+
void stdoutQueue.then(() => {
|
|
2859
|
+
settle(
|
|
2860
|
+
() => resolve4({
|
|
2861
|
+
exitCode,
|
|
2862
|
+
stderr: Buffer.concat(stderr).toString("utf8")
|
|
2863
|
+
})
|
|
2864
|
+
);
|
|
2865
|
+
});
|
|
3007
2866
|
});
|
|
3008
2867
|
child.stdin.end(input);
|
|
3009
2868
|
})
|
|
@@ -3135,7 +2994,7 @@ import {
|
|
|
3135
2994
|
listWikiGraphLibraryObjects,
|
|
3136
2995
|
readContinuationCursor,
|
|
3137
2996
|
resolveWikiGraphLibraryQueryTargetById,
|
|
3138
|
-
WikiGraphArchiveFile as
|
|
2997
|
+
WikiGraphArchiveFile as WikiGraphArchiveFile3
|
|
3139
2998
|
} from "wiki-graph-core";
|
|
3140
2999
|
var WikiGraphContinuationManager = class {
|
|
3141
3000
|
#runtime;
|
|
@@ -3182,9 +3041,9 @@ var WikiGraphContinuationManager = class {
|
|
|
3182
3041
|
};
|
|
3183
3042
|
async function continueArchiveCursor(cursor, limit) {
|
|
3184
3043
|
const location = await resolveWikiGraphArchiveLocation(
|
|
3185
|
-
getCursorArchivePath(cursor)
|
|
3044
|
+
archiveTargetFromLogicalLocator(getCursorArchivePath(cursor))
|
|
3186
3045
|
);
|
|
3187
|
-
return await new
|
|
3046
|
+
return await new WikiGraphArchiveFile3(location.archiveFile).readDocument(
|
|
3188
3047
|
async (document) => {
|
|
3189
3048
|
switch (cursor.kind) {
|
|
3190
3049
|
case "source-locators":
|
|
@@ -3439,14 +3298,14 @@ import {
|
|
|
3439
3298
|
getChapterDetails,
|
|
3440
3299
|
openWikimediaResolver as openWikimediaResolver2,
|
|
3441
3300
|
recordBuildJobInputRevision,
|
|
3442
|
-
WikiGraphArchiveFile as
|
|
3301
|
+
WikiGraphArchiveFile as WikiGraphArchiveFile5,
|
|
3443
3302
|
WikiGraphScope,
|
|
3444
3303
|
withLoggingContext
|
|
3445
3304
|
} from "wiki-graph-core";
|
|
3446
3305
|
import {
|
|
3447
|
-
applyChapterJobArtifactFile as
|
|
3306
|
+
applyChapterJobArtifactFile as applyChapterJobArtifactFile3,
|
|
3448
3307
|
createLocalChapterJobFileExecutor as createLocalChapterJobFileExecutor2,
|
|
3449
|
-
writeChapterJobInputFile as
|
|
3308
|
+
writeChapterJobInputFile as writeChapterJobInputFile3
|
|
3450
3309
|
} from "wiki-graph-core/worker";
|
|
3451
3310
|
|
|
3452
3311
|
// src/stage.ts
|
|
@@ -3484,14 +3343,145 @@ async function loadRequiredStageConfig(options) {
|
|
|
3484
3343
|
return config;
|
|
3485
3344
|
}
|
|
3486
3345
|
|
|
3346
|
+
// src/planning.ts
|
|
3347
|
+
var DEFAULT_GENERATION_JOB_CONCURRENCY = 3;
|
|
3348
|
+
var DEFAULT_GENERATION_REQUEST_CONCURRENCY = 6;
|
|
3349
|
+
function planGenerationTask(task, words, chapters, concurrent, model) {
|
|
3350
|
+
const profile = getGenerationPlanningProfile(task);
|
|
3351
|
+
const calls = Math.max(chapters, Math.ceil(words / profile.wordsPerCall));
|
|
3352
|
+
const effectiveConcurrency = Math.max(
|
|
3353
|
+
1,
|
|
3354
|
+
task === "reading-graph" ? 1 : concurrent.request
|
|
3355
|
+
);
|
|
3356
|
+
const callBatches = Math.ceil(calls / effectiveConcurrency);
|
|
3357
|
+
const wordSeconds = words * profile.secondsPerWord;
|
|
3358
|
+
return {
|
|
3359
|
+
model,
|
|
3360
|
+
timeSeconds: {
|
|
3361
|
+
max: Math.ceil(callBatches * profile.maxSecondsPerCall + wordSeconds),
|
|
3362
|
+
min: Math.ceil(callBatches * profile.minSecondsPerCall + wordSeconds)
|
|
3363
|
+
},
|
|
3364
|
+
tokens: {
|
|
3365
|
+
cacheableInput: Math.ceil(words * profile.cacheableInputTokenPerWord),
|
|
3366
|
+
input: Math.ceil(words * profile.inputTokenPerWord),
|
|
3367
|
+
output: Math.ceil(words * profile.outputTokenPerWord)
|
|
3368
|
+
}
|
|
3369
|
+
};
|
|
3370
|
+
}
|
|
3371
|
+
function createGenerationPerformanceHints(input) {
|
|
3372
|
+
if (!input.hasJobWork) {
|
|
3373
|
+
return [];
|
|
3374
|
+
}
|
|
3375
|
+
const hints = [];
|
|
3376
|
+
const recommendedRequest = input.concurrent.request < 4 ? DEFAULT_GENERATION_REQUEST_CONCURRENCY : input.concurrent.request < 8 ? 8 : void 0;
|
|
3377
|
+
if (input.hasRequestWork && recommendedRequest !== void 0) {
|
|
3378
|
+
hints.push({
|
|
3379
|
+
configKey: "request",
|
|
3380
|
+
configValue: recommendedRequest,
|
|
3381
|
+
current: input.concurrent.request,
|
|
3382
|
+
kind: "request",
|
|
3383
|
+
message: "LLM request concurrency can often be higher. Use at least 4; 6-8 is usually faster when the provider allows it.",
|
|
3384
|
+
recommended: recommendedRequest
|
|
3385
|
+
});
|
|
3386
|
+
}
|
|
3387
|
+
if (input.concurrent.job < 4 && input.chapters > 1) {
|
|
3388
|
+
hints.push({
|
|
3389
|
+
configKey: "job",
|
|
3390
|
+
configValue: 4,
|
|
3391
|
+
current: input.concurrent.job,
|
|
3392
|
+
kind: "job",
|
|
3393
|
+
message: "Multiple chapters need work and job concurrency is below 4. Raising it lets chapter jobs run in parallel.",
|
|
3394
|
+
recommended: 4
|
|
3395
|
+
});
|
|
3396
|
+
}
|
|
3397
|
+
return hints;
|
|
3398
|
+
}
|
|
3399
|
+
function formatGenerationPlanningModel(llm) {
|
|
3400
|
+
if (llm?.model === void 0) {
|
|
3401
|
+
return "not configured";
|
|
3402
|
+
}
|
|
3403
|
+
return llm.provider === void 0 ? llm.model : `${llm.provider}/${llm.model}`;
|
|
3404
|
+
}
|
|
3405
|
+
function formatGenerationPlanningDuration(seconds) {
|
|
3406
|
+
if (seconds < 60) {
|
|
3407
|
+
return `${seconds}s`;
|
|
3408
|
+
}
|
|
3409
|
+
const minutes = Math.round(seconds / 60);
|
|
3410
|
+
if (minutes < 60) {
|
|
3411
|
+
return `${minutes}m`;
|
|
3412
|
+
}
|
|
3413
|
+
return `${Math.round(minutes / 60)}h`;
|
|
3414
|
+
}
|
|
3415
|
+
function getGenerationPlanningProfile(task) {
|
|
3416
|
+
switch (task) {
|
|
3417
|
+
case "index-fts":
|
|
3418
|
+
return {
|
|
3419
|
+
cacheableInputTokenPerWord: 0,
|
|
3420
|
+
inputTokenPerWord: 0,
|
|
3421
|
+
maxSecondsPerCall: 1,
|
|
3422
|
+
minSecondsPerCall: 0,
|
|
3423
|
+
outputTokenPerWord: 0,
|
|
3424
|
+
secondsPerWord: 5e-4,
|
|
3425
|
+
wordsPerCall: 1e5
|
|
3426
|
+
};
|
|
3427
|
+
case "index-embedding-source":
|
|
3428
|
+
case "index-embedding-summary":
|
|
3429
|
+
return {
|
|
3430
|
+
cacheableInputTokenPerWord: 0,
|
|
3431
|
+
inputTokenPerWord: 0,
|
|
3432
|
+
maxSecondsPerCall: 10,
|
|
3433
|
+
minSecondsPerCall: 1,
|
|
3434
|
+
outputTokenPerWord: 0,
|
|
3435
|
+
secondsPerWord: 2e-3,
|
|
3436
|
+
wordsPerCall: 1e4
|
|
3437
|
+
};
|
|
3438
|
+
case "knowledge-graph":
|
|
3439
|
+
return {
|
|
3440
|
+
cacheableInputTokenPerWord: 60,
|
|
3441
|
+
inputTokenPerWord: 80,
|
|
3442
|
+
maxSecondsPerCall: 35,
|
|
3443
|
+
minSecondsPerCall: 15,
|
|
3444
|
+
outputTokenPerWord: 25,
|
|
3445
|
+
secondsPerWord: 0.02,
|
|
3446
|
+
wordsPerCall: 35
|
|
3447
|
+
};
|
|
3448
|
+
case "reading-graph":
|
|
3449
|
+
return {
|
|
3450
|
+
cacheableInputTokenPerWord: 20,
|
|
3451
|
+
inputTokenPerWord: 25,
|
|
3452
|
+
maxSecondsPerCall: 45,
|
|
3453
|
+
minSecondsPerCall: 15,
|
|
3454
|
+
outputTokenPerWord: 4,
|
|
3455
|
+
secondsPerWord: 0.01,
|
|
3456
|
+
wordsPerCall: 160
|
|
3457
|
+
};
|
|
3458
|
+
case "reading-summary":
|
|
3459
|
+
return {
|
|
3460
|
+
cacheableInputTokenPerWord: 45,
|
|
3461
|
+
inputTokenPerWord: 55,
|
|
3462
|
+
maxSecondsPerCall: 25,
|
|
3463
|
+
minSecondsPerCall: 10,
|
|
3464
|
+
outputTokenPerWord: 4,
|
|
3465
|
+
secondsPerWord: 0.01,
|
|
3466
|
+
wordsPerCall: 110
|
|
3467
|
+
};
|
|
3468
|
+
}
|
|
3469
|
+
}
|
|
3470
|
+
|
|
3487
3471
|
// src/worker.ts
|
|
3488
3472
|
import {
|
|
3489
|
-
|
|
3473
|
+
applyChapterJobArtifactFile,
|
|
3474
|
+
runBuildJobWorker as runCoreBuildJobWorker,
|
|
3475
|
+
writeChapterJobInputFile
|
|
3490
3476
|
} from "wiki-graph-core/worker";
|
|
3491
3477
|
import {
|
|
3492
|
-
|
|
3478
|
+
resolveChapterPathReadonly as resolveChapterPathReadonly2,
|
|
3479
|
+
WikiGraphArchiveFile as WikiGraphArchiveFile4
|
|
3480
|
+
} from "wiki-graph-core";
|
|
3481
|
+
import {
|
|
3482
|
+
applyChapterJobArtifactFile as applyChapterJobArtifactFile2,
|
|
3493
3483
|
createLocalChapterJobFileExecutor,
|
|
3494
|
-
writeChapterJobInputFile
|
|
3484
|
+
writeChapterJobInputFile as writeChapterJobInputFile2
|
|
3495
3485
|
} from "wiki-graph-core/worker";
|
|
3496
3486
|
async function runBuildJobWorker(options) {
|
|
3497
3487
|
installNodeWikiGraphPlatform();
|
|
@@ -3531,7 +3521,7 @@ async function executeBuildJob(job, reporter, context) {
|
|
|
3531
3521
|
let stage = await readChapterStage(job);
|
|
3532
3522
|
if (stage === "sourced") {
|
|
3533
3523
|
await executeStep(job, "reading-graph", execution, reporter, context, {
|
|
3534
|
-
...execution.extractionPrompt === void 0 ? {} : {
|
|
3524
|
+
...execution.extractionPrompt === void 0 ? {} : { prompt: execution.extractionPrompt }
|
|
3535
3525
|
});
|
|
3536
3526
|
stage = await readChapterStage(job);
|
|
3537
3527
|
}
|
|
@@ -3564,8 +3554,8 @@ async function executeBuildJob(job, reporter, context) {
|
|
|
3564
3554
|
}
|
|
3565
3555
|
}
|
|
3566
3556
|
await executeStep(job, job.target, execution, reporter, context, {
|
|
3567
|
-
...job.target === "reading-graph" ? {
|
|
3568
|
-
...job.target === "knowledge-graph" ? {
|
|
3557
|
+
...job.target === "reading-graph" ? { prompt: execution.extractionPrompt } : {},
|
|
3558
|
+
...job.target === "knowledge-graph" ? { prompt: execution.knowledgeGraphPrompt } : {}
|
|
3569
3559
|
});
|
|
3570
3560
|
} finally {
|
|
3571
3561
|
await execution.close();
|
|
@@ -3580,10 +3570,10 @@ async function executeStep(job, kind, execution, reporter, context, inputOptions
|
|
|
3580
3570
|
const stepWorkspace = await resetStepWorkspace(job.workspace, kind);
|
|
3581
3571
|
const inputFile = await stepWorkspace.createFile("input.jsonl");
|
|
3582
3572
|
const outputWorkspace = await stepWorkspace.createDirectory("output");
|
|
3583
|
-
const revision = await new
|
|
3573
|
+
const revision = await new WikiGraphArchiveFile5(job.archive).readDocument(
|
|
3584
3574
|
async (document) => {
|
|
3585
3575
|
await getChapterDetails(document, job.chapterId);
|
|
3586
|
-
return await
|
|
3576
|
+
return await writeChapterJobInputFile3(
|
|
3587
3577
|
document,
|
|
3588
3578
|
job.chapterId,
|
|
3589
3579
|
kind,
|
|
@@ -3622,16 +3612,23 @@ async function executeStep(job, kind, execution, reporter, context, inputOptions
|
|
|
3622
3612
|
total: 1,
|
|
3623
3613
|
unit: "item"
|
|
3624
3614
|
});
|
|
3625
|
-
await
|
|
3626
|
-
|
|
3627
|
-
|
|
3628
|
-
|
|
3629
|
-
|
|
3630
|
-
|
|
3631
|
-
|
|
3632
|
-
|
|
3633
|
-
|
|
3634
|
-
|
|
3615
|
+
const target = await resolveWikiGraphArchiveTarget(job.archive);
|
|
3616
|
+
const resolvedLocation = await resolveWikiGraphArchiveLocation(target);
|
|
3617
|
+
const location = target.kind === "standalone" ? { ...resolvedLocation, archiveFile: job.archive } : resolvedLocation;
|
|
3618
|
+
await writeWikiGraphArchiveLocation(
|
|
3619
|
+
location,
|
|
3620
|
+
async (document) => {
|
|
3621
|
+
assertJobStillRunning(await getBuildJob(job.jobId));
|
|
3622
|
+
await applyChapterJobArtifactFile3(
|
|
3623
|
+
document,
|
|
3624
|
+
job.chapterId,
|
|
3625
|
+
kind,
|
|
3626
|
+
revision,
|
|
3627
|
+
result.artifactFile
|
|
3628
|
+
);
|
|
3629
|
+
},
|
|
3630
|
+
{ invalidateContinuationCursors: true }
|
|
3631
|
+
);
|
|
3635
3632
|
await reporter.updatePhase({
|
|
3636
3633
|
done: 1,
|
|
3637
3634
|
phase: "committing",
|
|
@@ -3670,6 +3667,10 @@ async function openJobExecutor(job, config, signal, reporter) {
|
|
|
3670
3667
|
llmRequest: requireValue(
|
|
3671
3668
|
request,
|
|
3672
3669
|
"Local Wikimedia resolution requires an LLM."
|
|
3670
|
+
),
|
|
3671
|
+
llmModelId: requireValue(
|
|
3672
|
+
stageConfig?.llm?.model,
|
|
3673
|
+
"Local Wikimedia resolution requires an LLM model."
|
|
3673
3674
|
)
|
|
3674
3675
|
} : {
|
|
3675
3676
|
endpoint: config.wikimedia.endpoint,
|
|
@@ -3707,12 +3708,12 @@ async function resetStepWorkspace(workspace, kind) {
|
|
|
3707
3708
|
return await workspace.createDirectory(name);
|
|
3708
3709
|
}
|
|
3709
3710
|
async function readChapterStage(job) {
|
|
3710
|
-
return await new
|
|
3711
|
+
return await new WikiGraphArchiveFile5(job.archive).readDocument(
|
|
3711
3712
|
async (document) => (await getChapterDetails(document, job.chapterId)).stage
|
|
3712
3713
|
);
|
|
3713
3714
|
}
|
|
3714
3715
|
async function assertCurrentFtsArtifact(job) {
|
|
3715
|
-
await new
|
|
3716
|
+
await new WikiGraphArchiveFile5(job.archive).readDocument(async (document) => {
|
|
3716
3717
|
const revision = await document.serials.getRevision(job.chapterId);
|
|
3717
3718
|
const artifact = await document.indexArtifacts.get(job.chapterId, "fts");
|
|
3718
3719
|
if (artifact?.sourceRevision !== revision) {
|
|
@@ -3816,7 +3817,7 @@ import {
|
|
|
3816
3817
|
boostBuildJob,
|
|
3817
3818
|
cancelBuildJob,
|
|
3818
3819
|
cleanBuildJobs,
|
|
3819
|
-
formatLocatedChapterUri,
|
|
3820
|
+
formatLocatedChapterUri as formatLocatedChapterUri2,
|
|
3820
3821
|
getBuildJob as getBuildJob2,
|
|
3821
3822
|
listBuildJobs,
|
|
3822
3823
|
listChapters as listChapters3,
|
|
@@ -3825,8 +3826,9 @@ import {
|
|
|
3825
3826
|
resolveBuildJobId,
|
|
3826
3827
|
resumeBuildJob,
|
|
3827
3828
|
updateBuildJobTarget,
|
|
3828
|
-
|
|
3829
|
-
|
|
3829
|
+
withArchiveBuildJobCreationLock,
|
|
3830
|
+
resolveChapterPathReadonly as resolveChapterPathReadonly3,
|
|
3831
|
+
WikiGraphArchiveFile as WikiGraphArchiveFile6
|
|
3830
3832
|
} from "wiki-graph-core";
|
|
3831
3833
|
var TERMINAL_STATES = /* @__PURE__ */ new Set(["canceled", "failed", "succeeded"]);
|
|
3832
3834
|
var TERMINAL_EVENT_GRACE_READS = 3;
|
|
@@ -3871,20 +3873,24 @@ var WikiGraphJobManager = class {
|
|
|
3871
3873
|
this.#backend = backend;
|
|
3872
3874
|
}
|
|
3873
3875
|
async create(options) {
|
|
3874
|
-
const
|
|
3875
|
-
|
|
3876
|
-
|
|
3877
|
-
|
|
3878
|
-
|
|
3879
|
-
|
|
3880
|
-
|
|
3876
|
+
const resolved = await this.#runtime.run(async () => {
|
|
3877
|
+
const location = await resolveWikiGraphArchiveLocation(options.archive);
|
|
3878
|
+
return {
|
|
3879
|
+
job: await this.#backend.add({
|
|
3880
|
+
...options,
|
|
3881
|
+
archive: location.archiveFile
|
|
3882
|
+
}),
|
|
3883
|
+
target: location.target
|
|
3884
|
+
};
|
|
3885
|
+
});
|
|
3886
|
+
return await this.#createHandle(resolved.job, resolved.target);
|
|
3881
3887
|
}
|
|
3882
3888
|
async clean() {
|
|
3883
3889
|
return await this.#runtime.run(async () => await cleanBuildJobs());
|
|
3884
3890
|
}
|
|
3885
3891
|
async get(jobId) {
|
|
3886
|
-
return this.#
|
|
3887
|
-
|
|
3892
|
+
return await this.#runtime.run(
|
|
3893
|
+
async () => await this.#createHandle(await this.#backend.get(jobId))
|
|
3888
3894
|
);
|
|
3889
3895
|
}
|
|
3890
3896
|
async resolveId(reference) {
|
|
@@ -3893,23 +3899,25 @@ var WikiGraphJobManager = class {
|
|
|
3893
3899
|
);
|
|
3894
3900
|
}
|
|
3895
3901
|
async list(options = {}) {
|
|
3896
|
-
|
|
3902
|
+
return await this.#runtime.run(async () => {
|
|
3897
3903
|
const normalized = {
|
|
3898
3904
|
...options.activeOnly === void 0 ? {} : { activeOnly: options.activeOnly },
|
|
3899
3905
|
...options.all === void 0 ? {} : { all: options.all },
|
|
3900
3906
|
...options.archive === void 0 ? {} : {
|
|
3901
|
-
archive:
|
|
3907
|
+
archive: (await resolveWikiGraphArchiveLocation(options.archive)).archiveFile
|
|
3902
3908
|
}
|
|
3903
3909
|
};
|
|
3904
|
-
|
|
3910
|
+
const jobs = await this.#backend.list(normalized);
|
|
3911
|
+
return await Promise.all(
|
|
3912
|
+
jobs.map(async (job) => await this.#createHandle(job))
|
|
3913
|
+
);
|
|
3905
3914
|
});
|
|
3906
|
-
return jobs.map((job) => this.#createHandle(job));
|
|
3907
3915
|
}
|
|
3908
3916
|
async resolveChapters(jobs) {
|
|
3909
3917
|
return await this.#runtime.run(async () => {
|
|
3910
3918
|
const jobsByArchive = /* @__PURE__ */ new Map();
|
|
3911
3919
|
for (const job of jobs) {
|
|
3912
|
-
const archivePath =
|
|
3920
|
+
const archivePath = getJobArchiveLocator(job);
|
|
3913
3921
|
const grouped = jobsByArchive.get(archivePath) ?? [];
|
|
3914
3922
|
grouped.push(job);
|
|
3915
3923
|
jobsByArchive.set(archivePath, grouped);
|
|
@@ -3917,8 +3925,11 @@ var WikiGraphJobManager = class {
|
|
|
3917
3925
|
const entries = (await Promise.all(
|
|
3918
3926
|
[...jobsByArchive].map(async ([archivePath, archiveJobs]) => {
|
|
3919
3927
|
try {
|
|
3920
|
-
const
|
|
3921
|
-
|
|
3928
|
+
const location = await resolveWikiGraphArchiveLocation(
|
|
3929
|
+
archiveJobs[0].archive
|
|
3930
|
+
);
|
|
3931
|
+
const chapters = await new WikiGraphArchiveFile6(
|
|
3932
|
+
location.archiveFile
|
|
3922
3933
|
).readDocument(async (document) => await listChapters3(document));
|
|
3923
3934
|
const chaptersById = new Map(
|
|
3924
3935
|
chapters.map((chapter) => [chapter.chapterId, chapter])
|
|
@@ -3929,7 +3940,7 @@ var WikiGraphJobManager = class {
|
|
|
3929
3940
|
[
|
|
3930
3941
|
job.jobId,
|
|
3931
3942
|
{
|
|
3932
|
-
locatedUri:
|
|
3943
|
+
locatedUri: formatLocatedChapterUri2(
|
|
3933
3944
|
archivePath,
|
|
3934
3945
|
chapter.path
|
|
3935
3946
|
),
|
|
@@ -3953,62 +3964,78 @@ var WikiGraphJobManager = class {
|
|
|
3953
3964
|
async enqueue(options) {
|
|
3954
3965
|
return await this.#runtime.run(async () => {
|
|
3955
3966
|
const location = await resolveWikiGraphArchiveLocation(options.archive);
|
|
3956
|
-
|
|
3957
|
-
|
|
3958
|
-
|
|
3959
|
-
|
|
3960
|
-
|
|
3961
|
-
|
|
3962
|
-
|
|
3963
|
-
|
|
3964
|
-
|
|
3965
|
-
|
|
3966
|
-
|
|
3967
|
-
|
|
3968
|
-
|
|
3969
|
-
|
|
3970
|
-
return checked;
|
|
3971
|
-
});
|
|
3972
|
-
const created = [];
|
|
3973
|
-
const skipped = [];
|
|
3974
|
-
for (const candidate of candidates) {
|
|
3975
|
-
if (candidate.reason !== void 0) {
|
|
3976
|
-
if (selectedExplicitly && candidates.length === 1) {
|
|
3977
|
-
throw new Error(
|
|
3978
|
-
formatQueueReadinessError(
|
|
3979
|
-
candidate.chapter,
|
|
3980
|
-
target,
|
|
3981
|
-
candidate.reason
|
|
3982
|
-
)
|
|
3967
|
+
return await withArchiveBuildJobCreationLock(
|
|
3968
|
+
location.archiveFile,
|
|
3969
|
+
async () => {
|
|
3970
|
+
const target = options.target ?? "reading-summary";
|
|
3971
|
+
await validateQueueTargetConfig(target, options.llmJSON);
|
|
3972
|
+
const selectedExplicitly = options.chapterId !== void 0 || options.chapterIds !== void 0 || options.chapterPath !== void 0 || options.depth !== void 0;
|
|
3973
|
+
const candidates = await new WikiGraphArchiveFile6(
|
|
3974
|
+
location.archiveFile
|
|
3975
|
+
).readDocument(async (document) => {
|
|
3976
|
+
const chapters = await listChapters3(document);
|
|
3977
|
+
const selected = await selectQueueChapters(
|
|
3978
|
+
document,
|
|
3979
|
+
chapters,
|
|
3980
|
+
options
|
|
3983
3981
|
);
|
|
3984
|
-
|
|
3985
|
-
|
|
3986
|
-
|
|
3987
|
-
|
|
3988
|
-
|
|
3989
|
-
|
|
3990
|
-
|
|
3991
|
-
|
|
3992
|
-
|
|
3993
|
-
|
|
3994
|
-
|
|
3995
|
-
archive: location.archiveFile,
|
|
3996
|
-
boost: options.boost ?? false,
|
|
3997
|
-
chapterId: candidate.chapter.chapterId,
|
|
3998
|
-
...options.llmJSON === void 0 ? {} : { llmJSON: options.llmJSON },
|
|
3999
|
-
...options.prompt === void 0 ? {} : { prompt: options.prompt },
|
|
4000
|
-
target
|
|
4001
|
-
})
|
|
4002
|
-
});
|
|
4003
|
-
} catch (error) {
|
|
4004
|
-
if (selectedExplicitly && candidates.length === 1) throw error;
|
|
4005
|
-
skipped.push({
|
|
4006
|
-
chapter: candidate.chapter,
|
|
4007
|
-
reason: error instanceof Error ? error.message : String(error)
|
|
3982
|
+
const checked = await Promise.all(
|
|
3983
|
+
selected.map(async (chapter) => ({
|
|
3984
|
+
chapter,
|
|
3985
|
+
reason: await readQueueReadinessReason(
|
|
3986
|
+
document,
|
|
3987
|
+
chapter,
|
|
3988
|
+
target
|
|
3989
|
+
)
|
|
3990
|
+
}))
|
|
3991
|
+
);
|
|
3992
|
+
return checked;
|
|
4008
3993
|
});
|
|
3994
|
+
const created = [];
|
|
3995
|
+
const skipped = [];
|
|
3996
|
+
for (const candidate of candidates) {
|
|
3997
|
+
if (candidate.reason !== void 0) {
|
|
3998
|
+
if (selectedExplicitly && candidates.length === 1) {
|
|
3999
|
+
throw new Error(
|
|
4000
|
+
formatQueueReadinessError(
|
|
4001
|
+
candidate.chapter,
|
|
4002
|
+
target,
|
|
4003
|
+
candidate.reason
|
|
4004
|
+
)
|
|
4005
|
+
);
|
|
4006
|
+
}
|
|
4007
|
+
skipped.push({
|
|
4008
|
+
chapter: candidate.chapter,
|
|
4009
|
+
reason: candidate.reason
|
|
4010
|
+
});
|
|
4011
|
+
continue;
|
|
4012
|
+
}
|
|
4013
|
+
try {
|
|
4014
|
+
created.push({
|
|
4015
|
+
chapter: candidate.chapter,
|
|
4016
|
+
job: await this.#createHandle(
|
|
4017
|
+
await this.#backend.add({
|
|
4018
|
+
archive: location.archiveFile,
|
|
4019
|
+
boost: options.boost ?? false,
|
|
4020
|
+
chapterId: candidate.chapter.chapterId,
|
|
4021
|
+
...options.llmJSON === void 0 ? {} : { llmJSON: options.llmJSON },
|
|
4022
|
+
...options.prompt === void 0 ? {} : { prompt: options.prompt },
|
|
4023
|
+
target
|
|
4024
|
+
}),
|
|
4025
|
+
location.target
|
|
4026
|
+
)
|
|
4027
|
+
});
|
|
4028
|
+
} catch (error) {
|
|
4029
|
+
if (selectedExplicitly && candidates.length === 1) throw error;
|
|
4030
|
+
skipped.push({
|
|
4031
|
+
chapter: candidate.chapter,
|
|
4032
|
+
reason: error instanceof Error ? error.message : String(error)
|
|
4033
|
+
});
|
|
4034
|
+
}
|
|
4035
|
+
}
|
|
4036
|
+
return { created, skipped };
|
|
4009
4037
|
}
|
|
4010
|
-
|
|
4011
|
-
return { created, skipped };
|
|
4038
|
+
);
|
|
4012
4039
|
});
|
|
4013
4040
|
}
|
|
4014
4041
|
async planEnqueue(options) {
|
|
@@ -4017,7 +4044,7 @@ var WikiGraphJobManager = class {
|
|
|
4017
4044
|
const target = options.target ?? "reading-summary";
|
|
4018
4045
|
await validateQueueTargetConfig(target, options.llmJSON);
|
|
4019
4046
|
const selectedExplicitly = options.chapterId !== void 0 || options.chapterIds !== void 0 || options.chapterPath !== void 0 || options.depth !== void 0;
|
|
4020
|
-
const candidates = await new
|
|
4047
|
+
const candidates = await new WikiGraphArchiveFile6(
|
|
4021
4048
|
location.archiveFile
|
|
4022
4049
|
).readDocument(async (document) => {
|
|
4023
4050
|
const chapters = await listChapters3(document);
|
|
@@ -4054,34 +4081,39 @@ var WikiGraphJobManager = class {
|
|
|
4054
4081
|
this.#subscriptions.add(controller);
|
|
4055
4082
|
return () => this.#subscriptions.delete(controller);
|
|
4056
4083
|
}
|
|
4057
|
-
#createHandle(snapshot) {
|
|
4058
|
-
|
|
4084
|
+
async #createHandle(snapshot, target) {
|
|
4085
|
+
const legacy = snapshot;
|
|
4086
|
+
const resolvedTarget = target ?? (snapshot.archive === void 0 ? archiveTargetFromLogicalLocator(legacy.archivePath ?? "unknown.wikg") : await resolveWikiGraphArchiveTarget(snapshot.archive));
|
|
4087
|
+
return new WikiGraphJob(
|
|
4088
|
+
this,
|
|
4089
|
+
this.#runtime,
|
|
4090
|
+
this.#backend,
|
|
4091
|
+
snapshot,
|
|
4092
|
+
cloneWikiGraphArchiveTarget(resolvedTarget)
|
|
4093
|
+
);
|
|
4059
4094
|
}
|
|
4060
4095
|
};
|
|
4061
|
-
function
|
|
4062
|
-
|
|
4063
|
-
if (record.archive !== void 0) {
|
|
4064
|
-
return getNodeResourcePath(record.archive);
|
|
4065
|
-
}
|
|
4066
|
-
if (typeof record.archivePath === "string") return record.archivePath;
|
|
4067
|
-
throw new TypeError("Build job is missing archive");
|
|
4096
|
+
function getJobArchiveLocator(job) {
|
|
4097
|
+
return job.archive.kind === "library" ? job.archive.uri : job.archive.path;
|
|
4068
4098
|
}
|
|
4069
4099
|
var WikiGraphJob = class {
|
|
4070
4100
|
#manager;
|
|
4071
4101
|
#runtime;
|
|
4072
4102
|
#backend;
|
|
4103
|
+
#archiveTarget;
|
|
4073
4104
|
#snapshot;
|
|
4074
|
-
constructor(manager, runtime, backend, snapshot) {
|
|
4105
|
+
constructor(manager, runtime, backend, snapshot, archiveTarget) {
|
|
4075
4106
|
this.#manager = manager;
|
|
4076
4107
|
this.#runtime = runtime;
|
|
4077
4108
|
this.#backend = backend;
|
|
4078
4109
|
this.#snapshot = snapshot;
|
|
4110
|
+
this.#archiveTarget = archiveTarget;
|
|
4079
4111
|
}
|
|
4080
4112
|
get id() {
|
|
4081
4113
|
return this.#snapshot.jobId;
|
|
4082
4114
|
}
|
|
4083
4115
|
get snapshot() {
|
|
4084
|
-
return this.#snapshot;
|
|
4116
|
+
return toPublicJobSnapshot(this.#snapshot, this.#archiveTarget);
|
|
4085
4117
|
}
|
|
4086
4118
|
async status() {
|
|
4087
4119
|
return await this.#update(async () => await this.#backend.get(this.id));
|
|
@@ -4108,7 +4140,8 @@ var WikiGraphJob = class {
|
|
|
4108
4140
|
let cursor = 0;
|
|
4109
4141
|
let job = this.#snapshot;
|
|
4110
4142
|
if (options.from === "now") {
|
|
4111
|
-
|
|
4143
|
+
await this.status();
|
|
4144
|
+
job = this.#snapshot;
|
|
4112
4145
|
const chunk = await this.#readEventChunkSafely(job, cursor);
|
|
4113
4146
|
const events = chunk.events;
|
|
4114
4147
|
cursor = chunk.cursor;
|
|
@@ -4128,7 +4161,8 @@ var WikiGraphJob = class {
|
|
|
4128
4161
|
yield event;
|
|
4129
4162
|
}
|
|
4130
4163
|
if (sawTerminalEvent) return;
|
|
4131
|
-
|
|
4164
|
+
await this.status();
|
|
4165
|
+
job = this.#snapshot;
|
|
4132
4166
|
if (isTerminalState(job.state)) {
|
|
4133
4167
|
terminalReadsWithoutEvent += 1;
|
|
4134
4168
|
if (terminalReadsWithoutEvent >= TERMINAL_EVENT_GRACE_READS) return;
|
|
@@ -4165,7 +4199,7 @@ var WikiGraphJob = class {
|
|
|
4165
4199
|
}
|
|
4166
4200
|
async #update(operation) {
|
|
4167
4201
|
this.#snapshot = await this.#runtime.run(operation);
|
|
4168
|
-
return this
|
|
4202
|
+
return this.snapshot;
|
|
4169
4203
|
}
|
|
4170
4204
|
async #readEventChunk(job, cursor) {
|
|
4171
4205
|
return await this.#runtime.run(async () => {
|
|
@@ -4184,6 +4218,54 @@ var WikiGraphJob = class {
|
|
|
4184
4218
|
}
|
|
4185
4219
|
}
|
|
4186
4220
|
};
|
|
4221
|
+
function toPublicJobSnapshot(job, archive) {
|
|
4222
|
+
const legacy = job;
|
|
4223
|
+
return {
|
|
4224
|
+
archive: cloneWikiGraphArchiveTarget(archive),
|
|
4225
|
+
archiveKey: archive.kind === "library" ? archive.uri : job.archiveKey,
|
|
4226
|
+
...getJobResourcePath(job.cache, legacy.cachePath) === void 0 ? {} : { cachePath: getJobResourcePath(job.cache, legacy.cachePath) },
|
|
4227
|
+
chapterId: job.chapterId,
|
|
4228
|
+
createdAt: job.createdAt,
|
|
4229
|
+
...job.currentStep === void 0 ? {} : { currentStep: job.currentStep },
|
|
4230
|
+
...job.errorJSON === void 0 ? {} : { errorJSON: job.errorJSON },
|
|
4231
|
+
...getJobResourcePath(job.events, legacy.eventsPath) === void 0 ? {} : { eventsPath: getJobResourcePath(job.events, legacy.eventsPath) },
|
|
4232
|
+
...job.finishedAt === void 0 ? {} : { finishedAt: job.finishedAt },
|
|
4233
|
+
jobId: job.jobId,
|
|
4234
|
+
...job.inputRevision === void 0 ? {} : { inputRevision: job.inputRevision },
|
|
4235
|
+
...getJobResourcePath(job.log, legacy.logPath) === void 0 ? {} : { logPath: getJobResourcePath(job.log, legacy.logPath) },
|
|
4236
|
+
...job.llmJSON === void 0 ? {} : { llmJSON: job.llmJSON },
|
|
4237
|
+
...job.ownerId === void 0 ? {} : { ownerId: job.ownerId },
|
|
4238
|
+
...job.prompt === void 0 ? {} : { prompt: job.prompt },
|
|
4239
|
+
queueRank: job.queueRank,
|
|
4240
|
+
...job.readingSummaryStartedAt === void 0 ? {} : { readingSummaryStartedAt: job.readingSummaryStartedAt },
|
|
4241
|
+
state: job.state,
|
|
4242
|
+
target: job.target,
|
|
4243
|
+
updatedAt: job.updatedAt,
|
|
4244
|
+
...getJobResourcePath(job.workspace, legacy.workspacePath) === void 0 ? {} : {
|
|
4245
|
+
workspacePath: getJobResourcePath(
|
|
4246
|
+
job.workspace,
|
|
4247
|
+
legacy.workspacePath
|
|
4248
|
+
)
|
|
4249
|
+
}
|
|
4250
|
+
};
|
|
4251
|
+
}
|
|
4252
|
+
function cloneWikiGraphArchiveTarget(target) {
|
|
4253
|
+
return target.kind === "library" ? Object.freeze({ kind: "library", uri: target.uri }) : Object.freeze({
|
|
4254
|
+
kind: "standalone",
|
|
4255
|
+
path: target.path,
|
|
4256
|
+
...target.objectUri === void 0 ? {} : { objectUri: target.objectUri }
|
|
4257
|
+
});
|
|
4258
|
+
}
|
|
4259
|
+
function getJobResourcePath(resource, legacy) {
|
|
4260
|
+
if (resource === void 0) return legacy;
|
|
4261
|
+
try {
|
|
4262
|
+
return getNodeResourcePath(resource);
|
|
4263
|
+
} catch {
|
|
4264
|
+
const identity = resource.identity;
|
|
4265
|
+
if (typeof identity === "string") return identity;
|
|
4266
|
+
throw new TypeError("Expected a Node File or Directory resource");
|
|
4267
|
+
}
|
|
4268
|
+
}
|
|
4187
4269
|
async function selectQueueChapters(document, chapters, options) {
|
|
4188
4270
|
const ids = options.chapterIds ?? (options.chapterId === void 0 ? void 0 : [options.chapterId]);
|
|
4189
4271
|
if (ids !== void 0) {
|
|
@@ -4194,7 +4276,7 @@ async function selectQueueChapters(document, chapters, options) {
|
|
|
4194
4276
|
if (options.chapterPath === void 0) {
|
|
4195
4277
|
return options.depth === void 0 ? chapters : chapters.filter((chapter) => chapter.depth <= options.depth);
|
|
4196
4278
|
}
|
|
4197
|
-
const rootId = await
|
|
4279
|
+
const rootId = await resolveChapterPathReadonly3(
|
|
4198
4280
|
document,
|
|
4199
4281
|
options.chapterPath
|
|
4200
4282
|
);
|
|
@@ -4259,14 +4341,14 @@ function formatQueueReadinessError(chapter, target, reason) {
|
|
|
4259
4341
|
}
|
|
4260
4342
|
async function wait(milliseconds, signal) {
|
|
4261
4343
|
if (signal?.aborted === true) return;
|
|
4262
|
-
await new Promise((
|
|
4344
|
+
await new Promise((resolve4) => {
|
|
4263
4345
|
let settled = false;
|
|
4264
4346
|
const complete = () => {
|
|
4265
4347
|
if (settled) return;
|
|
4266
4348
|
settled = true;
|
|
4267
4349
|
clearTimeout(timeout);
|
|
4268
4350
|
signal?.removeEventListener("abort", complete);
|
|
4269
|
-
|
|
4351
|
+
resolve4();
|
|
4270
4352
|
};
|
|
4271
4353
|
const timeout = setTimeout(complete, milliseconds);
|
|
4272
4354
|
signal?.addEventListener("abort", complete, { once: true });
|
|
@@ -4297,12 +4379,12 @@ import {
|
|
|
4297
4379
|
deleteWikiGraphLibraryMetadataKey,
|
|
4298
4380
|
findWikiGraphLibraryObjects as findWikiGraphLibraryObjects2,
|
|
4299
4381
|
findWikiGraphLibraryArchiveMembers as findWikiGraphLibraryArchiveMembers2,
|
|
4300
|
-
formatWikiGraphLibraryUri,
|
|
4382
|
+
formatWikiGraphLibraryUri as formatWikiGraphLibraryUri2,
|
|
4301
4383
|
getWikiGraphLibraryArchive,
|
|
4302
4384
|
getWikiGraphLibraryMetadata,
|
|
4303
4385
|
listRelatedWikiGraphLibraryObjects as listRelatedWikiGraphLibraryObjects2,
|
|
4304
|
-
listWikiGraphLibraries,
|
|
4305
|
-
listWikiGraphLibraryArchives,
|
|
4386
|
+
listWikiGraphLibraries as listWikiGraphLibraries2,
|
|
4387
|
+
listWikiGraphLibraryArchives as listWikiGraphLibraryArchives2,
|
|
4306
4388
|
listWikiGraphLibraryArchiveMembers as listWikiGraphLibraryArchiveMembers2,
|
|
4307
4389
|
listWikiGraphLibraryEvidence as listWikiGraphLibraryEvidence2,
|
|
4308
4390
|
listWikiGraphLibraryObjects as listWikiGraphLibraryObjects2,
|
|
@@ -4310,14 +4392,15 @@ import {
|
|
|
4310
4392
|
packWikiGraphLibraryContext,
|
|
4311
4393
|
parseWikiGraphLibraryUri as parseWikiGraphLibraryUri2,
|
|
4312
4394
|
putWikiGraphLibraryMetadata,
|
|
4313
|
-
readWikiGraphLibraryIndexState
|
|
4395
|
+
readWikiGraphLibraryIndexState,
|
|
4314
4396
|
readWikiGraphLibraryPage,
|
|
4315
|
-
rebuildWikiGraphLibraryIndex
|
|
4397
|
+
rebuildWikiGraphLibraryIndex,
|
|
4316
4398
|
rebindWikiGraphLibrary,
|
|
4317
4399
|
removeWikiGraphLibrary,
|
|
4318
4400
|
removeWikiGraphLibraryArchive,
|
|
4401
|
+
replaceWikiGraphLibraryArchive,
|
|
4319
4402
|
replaceWikiGraphLibraryMetadata,
|
|
4320
|
-
resolveWikiGraphLibrary,
|
|
4403
|
+
resolveWikiGraphLibrary as resolveWikiGraphLibrary2,
|
|
4321
4404
|
scanWikiGraphLibrary
|
|
4322
4405
|
} from "wiki-graph-core";
|
|
4323
4406
|
import { mkdir as mkdir2 } from "fs/promises";
|
|
@@ -4334,22 +4417,24 @@ var WikiGraphLibraryManager = class {
|
|
|
4334
4417
|
});
|
|
4335
4418
|
return new WikiGraphLibrary(this.#runtime, record);
|
|
4336
4419
|
}
|
|
4337
|
-
async get(uri =
|
|
4420
|
+
async get(uri = formatWikiGraphLibraryUri2()) {
|
|
4338
4421
|
const target = requireLibraryTarget(uri);
|
|
4339
4422
|
const record = await this.#runtime.run(
|
|
4340
|
-
async () => await
|
|
4423
|
+
async () => await resolveWikiGraphLibrary2(target)
|
|
4341
4424
|
);
|
|
4342
4425
|
return new WikiGraphLibrary(this.#runtime, record);
|
|
4343
4426
|
}
|
|
4344
4427
|
async addArchive(options) {
|
|
4345
|
-
return
|
|
4346
|
-
|
|
4347
|
-
|
|
4348
|
-
|
|
4349
|
-
|
|
4350
|
-
|
|
4351
|
-
|
|
4352
|
-
|
|
4428
|
+
return toArchiveSnapshot(
|
|
4429
|
+
await this.#runtime.run(
|
|
4430
|
+
async () => await addWikiGraphLibraryArchive({
|
|
4431
|
+
inputFile: new NodeFile(
|
|
4432
|
+
await assertStandaloneWikiGraphArchivePath(options.inputPath)
|
|
4433
|
+
),
|
|
4434
|
+
target: requireLibraryTarget(options.target),
|
|
4435
|
+
...options.to === void 0 ? {} : { to: options.to }
|
|
4436
|
+
})
|
|
4437
|
+
)
|
|
4353
4438
|
);
|
|
4354
4439
|
}
|
|
4355
4440
|
async assertCurrent(target) {
|
|
@@ -4358,51 +4443,78 @@ var WikiGraphLibraryManager = class {
|
|
|
4358
4443
|
);
|
|
4359
4444
|
}
|
|
4360
4445
|
async rebind(target, folder) {
|
|
4361
|
-
return
|
|
4362
|
-
|
|
4363
|
-
|
|
4364
|
-
|
|
4365
|
-
|
|
4446
|
+
return toScanSnapshot(
|
|
4447
|
+
await this.#runtime.run(
|
|
4448
|
+
async () => await rebindWikiGraphLibrary({
|
|
4449
|
+
folder: new NodeDirectory(resolveWikiGraphRuntimePath(folder)),
|
|
4450
|
+
target: requireLibraryTarget(target)
|
|
4451
|
+
})
|
|
4452
|
+
)
|
|
4366
4453
|
);
|
|
4367
4454
|
}
|
|
4368
4455
|
async remove(target) {
|
|
4369
|
-
return
|
|
4370
|
-
|
|
4456
|
+
return toLibrarySnapshot(
|
|
4457
|
+
await this.#runtime.run(
|
|
4458
|
+
async () => await removeWikiGraphLibrary(requireLibraryTarget(target))
|
|
4459
|
+
)
|
|
4371
4460
|
);
|
|
4372
4461
|
}
|
|
4373
4462
|
async scan(target) {
|
|
4374
|
-
return
|
|
4375
|
-
|
|
4463
|
+
return toScanSnapshot(
|
|
4464
|
+
await this.#runtime.run(
|
|
4465
|
+
async () => await scanWikiGraphLibrary(requireLibraryTarget(target))
|
|
4466
|
+
)
|
|
4376
4467
|
);
|
|
4377
4468
|
}
|
|
4378
4469
|
async getArchive(target) {
|
|
4379
|
-
return
|
|
4380
|
-
|
|
4470
|
+
return toArchiveSnapshot(
|
|
4471
|
+
await this.#runtime.run(
|
|
4472
|
+
async () => await getWikiGraphLibraryArchive(requireLibraryTarget(target))
|
|
4473
|
+
)
|
|
4381
4474
|
);
|
|
4382
4475
|
}
|
|
4383
4476
|
async moveArchive(options) {
|
|
4384
|
-
return
|
|
4385
|
-
|
|
4386
|
-
|
|
4387
|
-
|
|
4388
|
-
|
|
4477
|
+
return toArchiveSnapshot(
|
|
4478
|
+
await this.#runtime.run(
|
|
4479
|
+
async () => await moveWikiGraphLibraryArchive({
|
|
4480
|
+
target: requireLibraryTarget(options.target),
|
|
4481
|
+
to: options.to
|
|
4482
|
+
})
|
|
4483
|
+
)
|
|
4389
4484
|
);
|
|
4390
4485
|
}
|
|
4391
4486
|
async removeArchive(target) {
|
|
4392
|
-
return
|
|
4393
|
-
|
|
4394
|
-
|
|
4487
|
+
return toArchiveSnapshot(
|
|
4488
|
+
await this.#runtime.run(
|
|
4489
|
+
async () => await removeWikiGraphLibraryArchive({
|
|
4490
|
+
target: requireLibraryTarget(target)
|
|
4491
|
+
})
|
|
4492
|
+
)
|
|
4493
|
+
);
|
|
4494
|
+
}
|
|
4495
|
+
async replaceArchive(options) {
|
|
4496
|
+
return toArchiveSnapshot(
|
|
4497
|
+
await this.#runtime.run(async () => {
|
|
4498
|
+
const target = requireLibraryTarget(options.target);
|
|
4499
|
+
const current = await getWikiGraphLibraryArchive(target);
|
|
4500
|
+
return await replaceWikiGraphLibraryArchive({
|
|
4501
|
+
additionalDerivedStateKeys: current.file === void 0 ? [] : [getNodeResourcePath(current.file)],
|
|
4502
|
+
inputFile: new NodeFile(
|
|
4503
|
+
await assertStandaloneWikiGraphArchivePath(options.inputPath)
|
|
4504
|
+
),
|
|
4505
|
+
target
|
|
4506
|
+
});
|
|
4395
4507
|
})
|
|
4396
4508
|
);
|
|
4397
4509
|
}
|
|
4398
4510
|
async indexState(target) {
|
|
4399
4511
|
return await this.#runtime.run(
|
|
4400
|
-
async () => await
|
|
4512
|
+
async () => await readWikiGraphLibraryIndexState(requireLibraryTarget(target))
|
|
4401
4513
|
);
|
|
4402
4514
|
}
|
|
4403
4515
|
async rebuildIndex(target, onProgress) {
|
|
4404
4516
|
return await this.#runtime.run(
|
|
4405
|
-
async () => await
|
|
4517
|
+
async () => await rebuildWikiGraphLibraryIndex(
|
|
4406
4518
|
requireLibraryTarget(target),
|
|
4407
4519
|
onProgress
|
|
4408
4520
|
)
|
|
@@ -4499,7 +4611,8 @@ var WikiGraphLibraryManager = class {
|
|
|
4499
4611
|
objectUri,
|
|
4500
4612
|
{ ...options, ...wikimediaOptions }
|
|
4501
4613
|
)
|
|
4502
|
-
)
|
|
4614
|
+
),
|
|
4615
|
+
options.signal
|
|
4503
4616
|
);
|
|
4504
4617
|
}
|
|
4505
4618
|
async related(target, objectUri, options = {}) {
|
|
@@ -4531,7 +4644,7 @@ var WikiGraphLibraryManager = class {
|
|
|
4531
4644
|
}
|
|
4532
4645
|
async list() {
|
|
4533
4646
|
const records = await this.#runtime.run(
|
|
4534
|
-
async () => await
|
|
4647
|
+
async () => await listWikiGraphLibraries2()
|
|
4535
4648
|
);
|
|
4536
4649
|
return records.map((record) => new WikiGraphLibrary(this.#runtime, record));
|
|
4537
4650
|
}
|
|
@@ -4544,32 +4657,54 @@ var WikiGraphLibrary = class {
|
|
|
4544
4657
|
this.#record = record;
|
|
4545
4658
|
}
|
|
4546
4659
|
get snapshot() {
|
|
4547
|
-
return this.#record;
|
|
4660
|
+
return toLibrarySnapshot(this.#record);
|
|
4548
4661
|
}
|
|
4549
4662
|
get uri() {
|
|
4550
|
-
return
|
|
4663
|
+
return formatWikiGraphLibraryUri2(
|
|
4551
4664
|
this.#record.isDefault ? void 0 : this.#record.publicId
|
|
4552
4665
|
);
|
|
4553
4666
|
}
|
|
4554
4667
|
async archives() {
|
|
4555
4668
|
const target = requireLibraryTarget(this.uri);
|
|
4556
|
-
return await this.#runtime.run(
|
|
4557
|
-
async () => await
|
|
4558
|
-
);
|
|
4669
|
+
return (await this.#runtime.run(
|
|
4670
|
+
async () => await listWikiGraphLibraryArchives2(target)
|
|
4671
|
+
)).map(toArchiveSnapshot);
|
|
4559
4672
|
}
|
|
4560
4673
|
async scan() {
|
|
4561
4674
|
const target = requireLibraryTarget(this.uri);
|
|
4562
|
-
return
|
|
4563
|
-
async () => await scanWikiGraphLibrary(target)
|
|
4675
|
+
return toScanSnapshot(
|
|
4676
|
+
await this.#runtime.run(async () => await scanWikiGraphLibrary(target))
|
|
4564
4677
|
);
|
|
4565
4678
|
}
|
|
4566
4679
|
async remove() {
|
|
4567
4680
|
const target = requireLibraryTarget(this.uri);
|
|
4568
|
-
return
|
|
4569
|
-
async () => await removeWikiGraphLibrary(target)
|
|
4681
|
+
return toLibrarySnapshot(
|
|
4682
|
+
await this.#runtime.run(async () => await removeWikiGraphLibrary(target))
|
|
4570
4683
|
);
|
|
4571
4684
|
}
|
|
4572
4685
|
};
|
|
4686
|
+
function toLibrarySnapshot(record) {
|
|
4687
|
+
const compatible = record;
|
|
4688
|
+
return {
|
|
4689
|
+
id: record.id,
|
|
4690
|
+
publicId: record.publicId,
|
|
4691
|
+
uri: record.uri,
|
|
4692
|
+
folderPath: compatible.folderPath ?? (record.folder === void 0 ? "" : getNodeResourcePath(record.folder)),
|
|
4693
|
+
isDefault: record.isDefault,
|
|
4694
|
+
createdAt: record.createdAt,
|
|
4695
|
+
updatedAt: record.updatedAt
|
|
4696
|
+
};
|
|
4697
|
+
}
|
|
4698
|
+
function toArchiveSnapshot(record) {
|
|
4699
|
+
const { file: _file, ...snapshot } = record;
|
|
4700
|
+
return snapshot;
|
|
4701
|
+
}
|
|
4702
|
+
function toScanSnapshot(result) {
|
|
4703
|
+
return {
|
|
4704
|
+
...result.library === void 0 ? {} : { library: toLibrarySnapshot(result.library) },
|
|
4705
|
+
archives: result.archives.map(toArchiveSnapshot)
|
|
4706
|
+
};
|
|
4707
|
+
}
|
|
4573
4708
|
function requireLibraryTarget(value) {
|
|
4574
4709
|
if (typeof value !== "string") return value;
|
|
4575
4710
|
const target = parseWikiGraphLibraryUri2(value);
|
|
@@ -4585,7 +4720,7 @@ import {
|
|
|
4585
4720
|
parseWikiGraphLibraryUri as parseWikiGraphLibraryUri3,
|
|
4586
4721
|
upgradeWikiGraphMaintenanceTarget
|
|
4587
4722
|
} from "wiki-graph-core";
|
|
4588
|
-
import { basename as
|
|
4723
|
+
import { basename as basename5 } from "path";
|
|
4589
4724
|
var WikiGraphMaintenanceManager = class {
|
|
4590
4725
|
#runtime;
|
|
4591
4726
|
constructor(runtime) {
|
|
@@ -4609,7 +4744,7 @@ var WikiGraphMaintenanceManager = class {
|
|
|
4609
4744
|
target: parsed
|
|
4610
4745
|
});
|
|
4611
4746
|
}
|
|
4612
|
-
const path =
|
|
4747
|
+
const path = await assertStandaloneWikiGraphArchivePath(target.path);
|
|
4613
4748
|
return await upgradeWikiGraphMaintenanceTarget({
|
|
4614
4749
|
file: new NodeFile(path),
|
|
4615
4750
|
kind: "archive"
|
|
@@ -4625,7 +4760,7 @@ var WikiGraphMaintenanceManager = class {
|
|
|
4625
4760
|
const resolvedOutputPath = resolveWikiGraphRuntimePath(outputPath);
|
|
4626
4761
|
await migrateLegacySdpubToWikg(
|
|
4627
4762
|
new NodeFile(resolvedInputPath),
|
|
4628
|
-
new NodeFile(resolvedOutputPath,
|
|
4763
|
+
new NodeFile(resolvedOutputPath, basename5(resolvedOutputPath))
|
|
4629
4764
|
);
|
|
4630
4765
|
return {
|
|
4631
4766
|
inputPath: resolvedInputPath,
|
|
@@ -4707,16 +4842,16 @@ import {
|
|
|
4707
4842
|
formatError as formatError3,
|
|
4708
4843
|
formatLocatedChapterResourceUri,
|
|
4709
4844
|
formatLocatedChapterSourceCollectionUri,
|
|
4710
|
-
formatLocatedChapterUri as
|
|
4711
|
-
formatLocatedWikiGraphUri as
|
|
4845
|
+
formatLocatedChapterUri as formatLocatedChapterUri3,
|
|
4846
|
+
formatLocatedWikiGraphUri as formatLocatedWikiGraphUri3,
|
|
4712
4847
|
formatWikiGraphCommandUri,
|
|
4713
|
-
formatWikiGraphLibraryUri as
|
|
4848
|
+
formatWikiGraphLibraryUri as formatWikiGraphLibraryUri3,
|
|
4714
4849
|
isSourceLocatorScopeUri as isSourceLocatorScopeUri2,
|
|
4715
4850
|
isWikiGraphLibraryUri,
|
|
4716
4851
|
parseChapterPath,
|
|
4717
4852
|
parseChapterTreeInput,
|
|
4718
4853
|
parseChapterUriPath,
|
|
4719
|
-
parseLocatedWikiGraphUri as
|
|
4854
|
+
parseLocatedWikiGraphUri as parseLocatedWikiGraphUri3,
|
|
4720
4855
|
parseSourceTextJsonl,
|
|
4721
4856
|
parseWikiGraphLibraryUri as parseWikiGraphLibraryUri4,
|
|
4722
4857
|
parseWikiGraphUriSyntax
|
|
@@ -4735,6 +4870,7 @@ export {
|
|
|
4735
4870
|
WikiGraphArchiveExistsError,
|
|
4736
4871
|
WikiGraphArchiveHandle,
|
|
4737
4872
|
WikiGraphArchiveManager,
|
|
4873
|
+
WikiGraphArchiveOwnershipError,
|
|
4738
4874
|
WikiGraphConfigManager,
|
|
4739
4875
|
WikiGraphContinuationManager,
|
|
4740
4876
|
WikiGraphConversionManager,
|
|
@@ -4753,7 +4889,6 @@ export {
|
|
|
4753
4889
|
createNodeTemplateEnvironment,
|
|
4754
4890
|
createNodeWikiGraphStorage,
|
|
4755
4891
|
createStageLLM,
|
|
4756
|
-
createWikiGraphArchiveInspectReport,
|
|
4757
4892
|
createWikiGraphSDK,
|
|
4758
4893
|
deleteLocalConfigValue,
|
|
4759
4894
|
embedQueryText,
|
|
@@ -4764,13 +4899,14 @@ export {
|
|
|
4764
4899
|
formatGenerationPlanningModel,
|
|
4765
4900
|
formatLocatedChapterResourceUri,
|
|
4766
4901
|
formatLocatedChapterSourceCollectionUri,
|
|
4767
|
-
|
|
4768
|
-
|
|
4902
|
+
formatLocatedChapterUri3 as formatLocatedChapterUri,
|
|
4903
|
+
formatLocatedWikiGraphUri3 as formatLocatedWikiGraphUri,
|
|
4769
4904
|
formatWikiGraphCommandUri,
|
|
4770
|
-
|
|
4905
|
+
formatWikiGraphLibraryUri3 as formatWikiGraphLibraryUri,
|
|
4771
4906
|
getNodeResourcePath,
|
|
4772
4907
|
getWikiGraphSDKRuntimeContext,
|
|
4773
4908
|
getWikiGraphSDKStateDir,
|
|
4909
|
+
inspectWikiGraphArchive,
|
|
4774
4910
|
installNodeWikiGraphPlatform,
|
|
4775
4911
|
isSourceLocatorScopeUri2 as isSourceLocatorScopeUri,
|
|
4776
4912
|
isWikiGraphLibraryUri,
|
|
@@ -4784,7 +4920,7 @@ export {
|
|
|
4784
4920
|
parseChapterTreeInput,
|
|
4785
4921
|
parseChapterUriPath,
|
|
4786
4922
|
parseLocalConfigSection,
|
|
4787
|
-
|
|
4923
|
+
parseLocatedWikiGraphUri3 as parseLocatedWikiGraphUri,
|
|
4788
4924
|
parseSourceTextJsonl,
|
|
4789
4925
|
parseWikiGraphLibraryUri4 as parseWikiGraphLibraryUri,
|
|
4790
4926
|
parseWikiGraphUriSyntax,
|
|
@@ -4797,7 +4933,6 @@ export {
|
|
|
4797
4933
|
resolveExtractionPrompt,
|
|
4798
4934
|
resolveHostedProviderEndpoint,
|
|
4799
4935
|
resolveKnowledgeGraphRecallPrompt,
|
|
4800
|
-
resolveWikiGraphArchiveLocation,
|
|
4801
4936
|
resolveWikispineConfig,
|
|
4802
4937
|
runWikiGraphQueueWorker,
|
|
4803
4938
|
validateLocalConfigSection,
|