wiki-graph-sdk 0.7.0 → 0.8.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +16 -0
- package/dist/archives.cjs +470 -642
- package/dist/archives.cjs.map +1 -1
- package/dist/archives.d.ts +42 -80
- package/dist/archives.js +405 -570
- package/dist/archives.js.map +1 -1
- package/dist/conversions.cjs +223 -27
- package/dist/conversions.cjs.map +1 -1
- package/dist/conversions.d.ts +16 -4
- package/dist/conversions.js +228 -23
- package/dist/conversions.js.map +1 -1
- package/dist/default-worker.cjs +293 -55
- package/dist/default-worker.cjs.map +1 -1
- package/dist/default-worker.js +309 -54
- package/dist/default-worker.js.map +1 -1
- package/dist/index.cjs +1044 -923
- package/dist/index.cjs.map +1 -1
- package/dist/index.d.ts +36 -19
- package/dist/index.js +915 -780
- package/dist/index.js.map +1 -1
- package/dist/{jobs-Dd5PMSId.d.ts → jobs-xIm_kshM.d.ts} +22 -14
- package/dist/maintenance.cjs +138 -29
- package/dist/maintenance.cjs.map +1 -1
- package/dist/maintenance.d.ts +2 -1
- package/dist/maintenance.js +143 -25
- package/dist/maintenance.js.map +1 -1
- package/dist/target-CZ7elwmj.d.ts +14 -0
- package/dist/wikispine.cjs +14 -10
- package/dist/wikispine.cjs.map +1 -1
- package/dist/wikispine.js +14 -10
- package/dist/wikispine.js.map +1 -1
- package/dist/worker.cjs +328 -22
- package/dist/worker.cjs.map +1 -1
- package/dist/worker.d.ts +43 -4
- package/dist/worker.js +348 -27
- package/dist/worker.js.map +1 -1
- package/package.json +2 -2
package/dist/archives.js
CHANGED
|
@@ -4,9 +4,9 @@ globalThis.__WIKIGRAPH_DATA_DIR__ ??= __WIKIGRAPH_RESOLVE__(__WIKIGRAPH_FILE_URL
|
|
|
4
4
|
|
|
5
5
|
// src/archive/archive.ts
|
|
6
6
|
import { randomUUID as randomUUID2 } from "crypto";
|
|
7
|
-
import { link, mkdtemp, rename as rename2, rm as rm3, stat as
|
|
7
|
+
import { link, mkdtemp, rename as rename2, rm as rm3, stat as stat3 } from "fs/promises";
|
|
8
8
|
import { tmpdir } from "os";
|
|
9
|
-
import { basename as
|
|
9
|
+
import { basename as basename4, dirname as dirname3, join as join4 } from "path";
|
|
10
10
|
import { Readable as Readable2 } from "stream";
|
|
11
11
|
import {
|
|
12
12
|
addChapter,
|
|
@@ -15,9 +15,8 @@ import {
|
|
|
15
15
|
assertNoActiveBuildJobs,
|
|
16
16
|
deleteArchiveSearchSessions,
|
|
17
17
|
DirectoryDocument,
|
|
18
|
-
finalizeWikiGraphLibraryArchiveWrite,
|
|
19
18
|
findArchiveObjects,
|
|
20
|
-
formatLocatedWikiGraphUri,
|
|
19
|
+
formatLocatedWikiGraphUri as formatLocatedWikiGraphUri2,
|
|
21
20
|
isArchiveSearchIndexCurrent as isArchiveSearchIndexCurrent2,
|
|
22
21
|
isSourceLocatorScopeUri,
|
|
23
22
|
listArchiveCollection,
|
|
@@ -28,30 +27,29 @@ import {
|
|
|
28
27
|
listRelatedArchiveObjects,
|
|
29
28
|
moveChapter,
|
|
30
29
|
packArchiveContext,
|
|
31
|
-
parseLocatedWikiGraphUri,
|
|
32
|
-
parseWikiGraphLibraryUri,
|
|
30
|
+
parseLocatedWikiGraphUri as parseLocatedWikiGraphUri2,
|
|
33
31
|
readArchivePage,
|
|
34
32
|
readSearchIndexCapabilityStatus,
|
|
35
|
-
readWikiGraphLibraryIndexState,
|
|
36
33
|
rebuildArchiveSearchIndex,
|
|
37
|
-
rebuildWikiGraphLibraryIndex,
|
|
38
34
|
removeChapter,
|
|
39
35
|
resetChapter,
|
|
40
36
|
resolveChapterPathReadonly,
|
|
41
|
-
resolveWikiGraphLibraryArchiveFile,
|
|
42
37
|
setChapterSource,
|
|
43
38
|
setChapterSummary,
|
|
44
39
|
setChapterTitle,
|
|
45
40
|
getChapterTree,
|
|
46
41
|
WikiGraph as WikiGraph2,
|
|
47
|
-
WikiGraphArchiveFile,
|
|
42
|
+
WikiGraphArchiveFile as WikiGraphArchiveFile2,
|
|
48
43
|
TOC_FILE_VERSION,
|
|
49
44
|
writeWikgArchive,
|
|
50
45
|
ObjectMetadataKind
|
|
51
46
|
} from "wiki-graph-core";
|
|
52
47
|
|
|
53
48
|
// src/conversions.ts
|
|
49
|
+
import { createHash } from "crypto";
|
|
50
|
+
import { createReadStream } from "fs";
|
|
54
51
|
import { rm as rm2 } from "fs/promises";
|
|
52
|
+
import { basename as basename3 } from "path";
|
|
55
53
|
import {
|
|
56
54
|
WikiGraph
|
|
57
55
|
} from "wiki-graph-core";
|
|
@@ -380,42 +378,42 @@ var NodeDatabaseConnection = class {
|
|
|
380
378
|
}
|
|
381
379
|
database;
|
|
382
380
|
async close() {
|
|
383
|
-
await new Promise((
|
|
381
|
+
await new Promise((resolve4, reject) => {
|
|
384
382
|
this.database.close((error) => {
|
|
385
383
|
if (error) reject(error);
|
|
386
|
-
else
|
|
384
|
+
else resolve4();
|
|
387
385
|
});
|
|
388
386
|
});
|
|
389
387
|
}
|
|
390
388
|
async execute(sql) {
|
|
391
|
-
await new Promise((
|
|
389
|
+
await new Promise((resolve4, reject) => {
|
|
392
390
|
this.database.exec(sql, (error) => {
|
|
393
391
|
if (error) reject(error);
|
|
394
|
-
else
|
|
392
|
+
else resolve4();
|
|
395
393
|
});
|
|
396
394
|
});
|
|
397
395
|
}
|
|
398
396
|
async queryAll(sql, params = []) {
|
|
399
|
-
return await new Promise((
|
|
397
|
+
return await new Promise((resolve4, reject) => {
|
|
400
398
|
this.database.all(sql, [...params], (error, rows) => {
|
|
401
399
|
if (error) reject(error);
|
|
402
|
-
else
|
|
400
|
+
else resolve4(rows);
|
|
403
401
|
});
|
|
404
402
|
});
|
|
405
403
|
}
|
|
406
404
|
async queryOne(sql, params = []) {
|
|
407
|
-
return await new Promise((
|
|
405
|
+
return await new Promise((resolve4, reject) => {
|
|
408
406
|
this.database.get(sql, [...params], (error, row) => {
|
|
409
407
|
if (error) reject(error);
|
|
410
|
-
else
|
|
408
|
+
else resolve4(row);
|
|
411
409
|
});
|
|
412
410
|
});
|
|
413
411
|
}
|
|
414
412
|
async run(sql, params = []) {
|
|
415
|
-
await new Promise((
|
|
413
|
+
await new Promise((resolve4, reject) => {
|
|
416
414
|
this.database.run(sql, [...params], (error) => {
|
|
417
415
|
if (error) reject(error);
|
|
418
|
-
else
|
|
416
|
+
else resolve4();
|
|
419
417
|
});
|
|
420
418
|
});
|
|
421
419
|
}
|
|
@@ -429,10 +427,10 @@ async function openNativeNodeDatabase(file, flags) {
|
|
|
429
427
|
if (path === void 0) {
|
|
430
428
|
throw new TypeError("The Node database adapter requires a NodeFile");
|
|
431
429
|
}
|
|
432
|
-
return await new Promise((
|
|
430
|
+
return await new Promise((resolve4, reject) => {
|
|
433
431
|
const database = new nodeSqlite3.Database(path, flags, (error) => {
|
|
434
432
|
if (error) reject(error);
|
|
435
|
-
else
|
|
433
|
+
else resolve4(database);
|
|
436
434
|
});
|
|
437
435
|
});
|
|
438
436
|
}
|
|
@@ -441,19 +439,19 @@ async function openNodeZip(file) {
|
|
|
441
439
|
if (path === void 0) {
|
|
442
440
|
throw new TypeError("The Node ZIP adapter requires a NodeFile");
|
|
443
441
|
}
|
|
444
|
-
const zipFile = await new Promise((
|
|
442
|
+
const zipFile = await new Promise((resolve4, reject) => {
|
|
445
443
|
yauzl.open(
|
|
446
444
|
path,
|
|
447
445
|
{ autoClose: false, lazyEntries: true },
|
|
448
446
|
(error, opened) => {
|
|
449
447
|
if (error || opened === void 0)
|
|
450
448
|
reject(error ?? new Error("Cannot open ZIP"));
|
|
451
|
-
else
|
|
449
|
+
else resolve4(opened);
|
|
452
450
|
}
|
|
453
451
|
);
|
|
454
452
|
});
|
|
455
453
|
const entries = await new Promise(
|
|
456
|
-
(
|
|
454
|
+
(resolve4, reject) => {
|
|
457
455
|
const discovered = /* @__PURE__ */ new Map();
|
|
458
456
|
const rejectAndClose = (error) => {
|
|
459
457
|
zipFile.close();
|
|
@@ -463,7 +461,7 @@ async function openNodeZip(file) {
|
|
|
463
461
|
};
|
|
464
462
|
const resolveEntries = () => {
|
|
465
463
|
zipFile.off("error", rejectAndClose);
|
|
466
|
-
|
|
464
|
+
resolve4(discovered);
|
|
467
465
|
};
|
|
468
466
|
zipFile.on("entry", (entry) => {
|
|
469
467
|
if (!entry.fileName.endsWith("/")) {
|
|
@@ -573,12 +571,12 @@ async function collectDecodedNodeStreamRange(input, offset, length) {
|
|
|
573
571
|
return Buffer.concat(chunks, collected);
|
|
574
572
|
}
|
|
575
573
|
async function openNodeZipEntryStream(zipFile, entry, options) {
|
|
576
|
-
return await new Promise((
|
|
574
|
+
return await new Promise((resolve4, reject) => {
|
|
577
575
|
const callback = (error, opened) => {
|
|
578
576
|
if (error || opened === void 0) {
|
|
579
577
|
reject(error ?? new Error(`Cannot read ZIP entry: ${entry.fileName}`));
|
|
580
578
|
} else {
|
|
581
|
-
|
|
579
|
+
resolve4(opened);
|
|
582
580
|
}
|
|
583
581
|
};
|
|
584
582
|
if (options === void 0) zipFile.openReadStream(entry, callback);
|
|
@@ -709,11 +707,11 @@ async function writeNodeStream(input, writer) {
|
|
|
709
707
|
);
|
|
710
708
|
}
|
|
711
709
|
async function collectNodeStream(input) {
|
|
712
|
-
return await new Promise((
|
|
710
|
+
return await new Promise((resolve4, reject) => {
|
|
713
711
|
const chunks = [];
|
|
714
712
|
input.on("data", (chunk) => chunks.push(chunk));
|
|
715
713
|
input.once("error", reject);
|
|
716
|
-
input.once("end", () =>
|
|
714
|
+
input.once("end", () => resolve4(Buffer.concat(chunks)));
|
|
717
715
|
});
|
|
718
716
|
}
|
|
719
717
|
var nodeWikiGraphPlatform = {
|
|
@@ -1266,6 +1264,173 @@ function resolveWikiGraphRuntimePath(path) {
|
|
|
1266
1264
|
return isAbsolute2(path) ? path : resolve2(context.cwd, path);
|
|
1267
1265
|
}
|
|
1268
1266
|
|
|
1267
|
+
// src/archive/target.ts
|
|
1268
|
+
import { realpath, stat as stat2 } from "fs/promises";
|
|
1269
|
+
import { basename as basename2, dirname as dirname2, isAbsolute as isAbsolute3, join as join3, relative, resolve as resolve3 } from "path";
|
|
1270
|
+
import {
|
|
1271
|
+
formatLocatedWikiGraphUri,
|
|
1272
|
+
formatWikiGraphLibraryUri,
|
|
1273
|
+
listWikiGraphLibraries,
|
|
1274
|
+
listWikiGraphLibraryArchives,
|
|
1275
|
+
parseLocatedWikiGraphUri,
|
|
1276
|
+
parseWikiGraphLibraryUri,
|
|
1277
|
+
resolveWikiGraphLibrary,
|
|
1278
|
+
resolveWikiGraphLibraryArchiveFile
|
|
1279
|
+
} from "wiki-graph-core";
|
|
1280
|
+
var WikiGraphArchiveOwnershipError = class extends Error {
|
|
1281
|
+
code = "WIKI_GRAPH_ARCHIVE_OWNERSHIP_MISMATCH";
|
|
1282
|
+
constructor(message) {
|
|
1283
|
+
super(message);
|
|
1284
|
+
this.name = "WikiGraphArchiveOwnershipError";
|
|
1285
|
+
}
|
|
1286
|
+
};
|
|
1287
|
+
async function resolveWikiGraphArchiveLocation(target) {
|
|
1288
|
+
if (target.kind === "standalone") {
|
|
1289
|
+
return await resolveStandaloneLocation(target);
|
|
1290
|
+
}
|
|
1291
|
+
return await resolveLibraryLocation(target);
|
|
1292
|
+
}
|
|
1293
|
+
async function assertStandaloneWikiGraphArchivePath(inputPath) {
|
|
1294
|
+
const path = resolveWikiGraphRuntimePath(inputPath);
|
|
1295
|
+
const candidate = await canonicalPath(path);
|
|
1296
|
+
const candidateIdentity = await readFileIdentity(path);
|
|
1297
|
+
const libraries = await listWikiGraphLibraries();
|
|
1298
|
+
const matches = await findMatchingMembers(
|
|
1299
|
+
libraries,
|
|
1300
|
+
candidate,
|
|
1301
|
+
candidateIdentity
|
|
1302
|
+
);
|
|
1303
|
+
if (matches.length > 0) {
|
|
1304
|
+
const match = requireUnambiguousManagedMatch(path, matches);
|
|
1305
|
+
throw new WikiGraphArchiveOwnershipError(
|
|
1306
|
+
`Standalone archive path belongs to registered library ${formatWikiGraphLibraryUri(
|
|
1307
|
+
match.library.isDefault ? void 0 : match.library.publicId
|
|
1308
|
+
)}: ${path}. Use the library UUID URI ${match.archive.uri} instead.`
|
|
1309
|
+
);
|
|
1310
|
+
}
|
|
1311
|
+
for (const library of libraries) {
|
|
1312
|
+
const rootPath = getNodeResourcePath(library.folder);
|
|
1313
|
+
const root = await canonicalPath(rootPath);
|
|
1314
|
+
if (!isWithin(root, candidate)) continue;
|
|
1315
|
+
throw new WikiGraphArchiveOwnershipError(
|
|
1316
|
+
`Standalone archive path belongs to registered library ${formatWikiGraphLibraryUri(
|
|
1317
|
+
library.isDefault ? void 0 : library.publicId
|
|
1318
|
+
)}: ${path}. Scan the library, then address the archive by its library UUID URI.`
|
|
1319
|
+
);
|
|
1320
|
+
}
|
|
1321
|
+
return path;
|
|
1322
|
+
}
|
|
1323
|
+
async function resolveStandaloneLocation(target) {
|
|
1324
|
+
if (target.path.startsWith("wikg://lib/")) {
|
|
1325
|
+
throw new WikiGraphArchiveOwnershipError(
|
|
1326
|
+
`A library archive URI cannot be used as a standalone path: ${target.path}`
|
|
1327
|
+
);
|
|
1328
|
+
}
|
|
1329
|
+
const path = await assertStandaloneWikiGraphArchivePath(target.path);
|
|
1330
|
+
const normalized = {
|
|
1331
|
+
kind: "standalone",
|
|
1332
|
+
path,
|
|
1333
|
+
...target.objectUri === void 0 ? {} : { objectUri: target.objectUri }
|
|
1334
|
+
};
|
|
1335
|
+
return {
|
|
1336
|
+
archiveFile: new NodeFile(path),
|
|
1337
|
+
archiveKey: path,
|
|
1338
|
+
archivePath: path,
|
|
1339
|
+
indexScope: { archiveKey: path, archivePath: path, kind: "archive-index" },
|
|
1340
|
+
locatedUri: formatLocatedWikiGraphUri(path, target.objectUri),
|
|
1341
|
+
target: normalized
|
|
1342
|
+
};
|
|
1343
|
+
}
|
|
1344
|
+
async function resolveLibraryLocation(target) {
|
|
1345
|
+
const parsedLocated = parseLocatedWikiGraphUri(target.uri);
|
|
1346
|
+
const archiveUri = parsedLocated.archivePath ?? target.uri;
|
|
1347
|
+
const parsed = parseWikiGraphLibraryUri(archiveUri);
|
|
1348
|
+
if (parsed?.kind !== "archive") {
|
|
1349
|
+
throw new WikiGraphArchiveOwnershipError(
|
|
1350
|
+
`Library archive target must be a UUID archive URI: ${target.uri}`
|
|
1351
|
+
);
|
|
1352
|
+
}
|
|
1353
|
+
await resolveWikiGraphLibrary(parsed);
|
|
1354
|
+
const archiveFile = await resolveWikiGraphLibraryArchiveFile(archiveUri);
|
|
1355
|
+
const normalized = {
|
|
1356
|
+
kind: "library",
|
|
1357
|
+
uri: target.uri
|
|
1358
|
+
};
|
|
1359
|
+
return {
|
|
1360
|
+
archiveFile,
|
|
1361
|
+
archiveKey: archiveUri,
|
|
1362
|
+
archivePath: archiveUri,
|
|
1363
|
+
indexScope: {
|
|
1364
|
+
archiveKey: archiveUri,
|
|
1365
|
+
archivePath: archiveUri,
|
|
1366
|
+
kind: "archive-index"
|
|
1367
|
+
},
|
|
1368
|
+
libraryArchiveTarget: parsed,
|
|
1369
|
+
locatedUri: target.uri,
|
|
1370
|
+
publicArchiveUri: archiveUri,
|
|
1371
|
+
target: normalized
|
|
1372
|
+
};
|
|
1373
|
+
}
|
|
1374
|
+
async function canonicalPath(path) {
|
|
1375
|
+
try {
|
|
1376
|
+
return await realpath(path);
|
|
1377
|
+
} catch {
|
|
1378
|
+
const absolute = resolve3(path);
|
|
1379
|
+
const parent = dirname2(absolute);
|
|
1380
|
+
return parent === absolute ? absolute : join3(await canonicalPath(parent), basename2(absolute));
|
|
1381
|
+
}
|
|
1382
|
+
}
|
|
1383
|
+
function isWithin(root, candidate) {
|
|
1384
|
+
const child = relative(root, candidate);
|
|
1385
|
+
return child === "" || !child.startsWith("..") && !isAbsolute3(child);
|
|
1386
|
+
}
|
|
1387
|
+
async function findMatchingMembers(libraries, candidate, candidateIdentity) {
|
|
1388
|
+
const members = [];
|
|
1389
|
+
for (const library of libraries) {
|
|
1390
|
+
const archives = await listWikiGraphLibraryArchives({
|
|
1391
|
+
isDefault: library.isDefault,
|
|
1392
|
+
kind: "scope",
|
|
1393
|
+
...library.isDefault ? {} : { publicId: library.publicId }
|
|
1394
|
+
});
|
|
1395
|
+
for (const archive of archives) {
|
|
1396
|
+
if (archive.file === void 0) continue;
|
|
1397
|
+
members.push({
|
|
1398
|
+
archive,
|
|
1399
|
+
library,
|
|
1400
|
+
memberPath: getNodeResourcePath(archive.file)
|
|
1401
|
+
});
|
|
1402
|
+
}
|
|
1403
|
+
}
|
|
1404
|
+
const exact = [];
|
|
1405
|
+
for (const member of members) {
|
|
1406
|
+
if (await canonicalPath(member.memberPath) === candidate) {
|
|
1407
|
+
exact.push(member);
|
|
1408
|
+
}
|
|
1409
|
+
}
|
|
1410
|
+
if (exact.length > 0 || candidateIdentity === void 0) return exact;
|
|
1411
|
+
const identityMatches = [];
|
|
1412
|
+
for (const member of members) {
|
|
1413
|
+
if (await readFileIdentity(member.memberPath) === candidateIdentity) {
|
|
1414
|
+
identityMatches.push(member);
|
|
1415
|
+
}
|
|
1416
|
+
}
|
|
1417
|
+
return identityMatches;
|
|
1418
|
+
}
|
|
1419
|
+
function requireUnambiguousManagedMatch(path, matches) {
|
|
1420
|
+
if (matches.length === 1) return matches[0];
|
|
1421
|
+
throw new WikiGraphArchiveOwnershipError(
|
|
1422
|
+
`Archive file identity is ambiguous across managed archives: ${path}. Use an explicit library UUID URI (${matches.map((match) => match.archive.uri).join(", ")}).`
|
|
1423
|
+
);
|
|
1424
|
+
}
|
|
1425
|
+
async function readFileIdentity(path) {
|
|
1426
|
+
try {
|
|
1427
|
+
const value = await stat2(path);
|
|
1428
|
+
return `${value.dev}:${value.ino}`;
|
|
1429
|
+
} catch {
|
|
1430
|
+
return void 0;
|
|
1431
|
+
}
|
|
1432
|
+
}
|
|
1433
|
+
|
|
1269
1434
|
// src/conversions.ts
|
|
1270
1435
|
var WikiGraphConversionManager = class {
|
|
1271
1436
|
#runtime;
|
|
@@ -1280,6 +1445,9 @@ var WikiGraphConversionManager = class {
|
|
|
1280
1445
|
if (options.targetStage !== void 0 && options.output.format !== "wikg") {
|
|
1281
1446
|
throw new Error("targetStage is only supported for wikg output.");
|
|
1282
1447
|
}
|
|
1448
|
+
if (options.output.format === "pcex") {
|
|
1449
|
+
throw new Error("pcex is supported only as a conversion input.");
|
|
1450
|
+
}
|
|
1283
1451
|
if (options.targetStage !== void 0 && options.input.format === "wikg") {
|
|
1284
1452
|
throw new Error(
|
|
1285
1453
|
"targetStage is only supported when creating a wikg archive."
|
|
@@ -1298,7 +1466,17 @@ var WikiGraphConversionManager = class {
|
|
|
1298
1466
|
options.digestDirectory,
|
|
1299
1467
|
requiresDigest
|
|
1300
1468
|
);
|
|
1469
|
+
let chapterCount;
|
|
1470
|
+
let chapterPaths;
|
|
1301
1471
|
const write = async (archive) => {
|
|
1472
|
+
if (requiresDigest) await options.onSourceImported?.();
|
|
1473
|
+
if (typeof archive.readToc === "function") {
|
|
1474
|
+
const toc = await archive.readToc();
|
|
1475
|
+
chapterCount = toc === void 0 ? void 0 : countTocChapters(toc.items);
|
|
1476
|
+
if (options.output.format === "wikg") {
|
|
1477
|
+
chapterPaths = toc === void 0 ? [] : collectTocChapterPaths(toc.items);
|
|
1478
|
+
}
|
|
1479
|
+
}
|
|
1302
1480
|
await writeArchive(archive, options.output.path, options.output.format);
|
|
1303
1481
|
};
|
|
1304
1482
|
if (options.input.format === "wikg") {
|
|
@@ -1306,7 +1484,9 @@ var WikiGraphConversionManager = class {
|
|
|
1306
1484
|
throw new Error("wikg input requires a file path.");
|
|
1307
1485
|
}
|
|
1308
1486
|
await app.openSession(
|
|
1309
|
-
new NodeFile(
|
|
1487
|
+
new NodeFile(
|
|
1488
|
+
await assertStandaloneWikiGraphArchivePath(options.input.path)
|
|
1489
|
+
),
|
|
1310
1490
|
write
|
|
1311
1491
|
);
|
|
1312
1492
|
} else if ("stream" in options.input) {
|
|
@@ -1331,16 +1511,29 @@ var WikiGraphConversionManager = class {
|
|
|
1331
1511
|
};
|
|
1332
1512
|
if (options.input.format === "epub") {
|
|
1333
1513
|
await app.digestEpubSession(digestOptions, write);
|
|
1514
|
+
} else if (options.input.format === "pcex") {
|
|
1515
|
+
await app.digestPcexSession(
|
|
1516
|
+
{
|
|
1517
|
+
...digestOptions,
|
|
1518
|
+
pdfDigest: options.input.sourcePdf.digest,
|
|
1519
|
+
...options.input.sourcePdf.name === void 0 ? {} : { pdfName: options.input.sourcePdf.name }
|
|
1520
|
+
},
|
|
1521
|
+
write
|
|
1522
|
+
);
|
|
1334
1523
|
} else if (options.input.format === "markdown") {
|
|
1335
1524
|
await app.digestMarkdownSession(digestOptions, write);
|
|
1336
1525
|
} else {
|
|
1337
1526
|
await app.digestTxtSession(digestOptions, write);
|
|
1338
1527
|
}
|
|
1339
1528
|
}
|
|
1529
|
+
const sourceArtifact = await resolveSourceArtifact(options.input);
|
|
1340
1530
|
return {
|
|
1531
|
+
...chapterCount === void 0 ? {} : { chapterCount },
|
|
1532
|
+
...chapterPaths === void 0 ? {} : { chapterPaths },
|
|
1341
1533
|
inputFormat: options.input.format,
|
|
1342
1534
|
outputFormat: options.output.format,
|
|
1343
|
-
outputPath: options.output.path
|
|
1535
|
+
outputPath: options.output.path,
|
|
1536
|
+
...sourceArtifact === void 0 ? {} : { sourceArtifact }
|
|
1344
1537
|
};
|
|
1345
1538
|
}
|
|
1346
1539
|
};
|
|
@@ -1353,11 +1546,67 @@ async function prepareDigestDirectory(path, required) {
|
|
|
1353
1546
|
return new NodeDirectory(resolved);
|
|
1354
1547
|
}
|
|
1355
1548
|
async function writeArchive(archive, path, format) {
|
|
1356
|
-
const
|
|
1549
|
+
const outputPath = format === "wikg" ? await assertStandaloneWikiGraphArchivePath(path) : resolveWikiGraphRuntimePath(path);
|
|
1550
|
+
const file = new NodeFile(outputPath);
|
|
1357
1551
|
if (format === "epub") await archive.exportEpub(file);
|
|
1358
1552
|
else if (format === "wikg") await archive.saveAs(file);
|
|
1553
|
+
else if (format === "pcex") throw new Error("pcex output is not supported.");
|
|
1359
1554
|
else await archive.exportText(file);
|
|
1360
1555
|
}
|
|
1556
|
+
async function resolveSourceArtifact(input) {
|
|
1557
|
+
if (!("path" in input) || input.format === "wikg") return void 0;
|
|
1558
|
+
if (input.format === "pcex") {
|
|
1559
|
+
return {
|
|
1560
|
+
digest: input.sourcePdf.digest.trim().toLowerCase(),
|
|
1561
|
+
mediaType: "application/pdf",
|
|
1562
|
+
...input.sourcePdf.name === void 0 ? {} : { name: input.sourcePdf.name }
|
|
1563
|
+
};
|
|
1564
|
+
}
|
|
1565
|
+
if (input.format !== "epub") return void 0;
|
|
1566
|
+
const path = resolveWikiGraphRuntimePath(input.path);
|
|
1567
|
+
const hash = createHash("sha256");
|
|
1568
|
+
try {
|
|
1569
|
+
for await (const chunk of createReadStream(path)) {
|
|
1570
|
+
hash.update(chunk);
|
|
1571
|
+
}
|
|
1572
|
+
} catch (error) {
|
|
1573
|
+
if (isNodeErrorCode(error, "ENOENT")) return void 0;
|
|
1574
|
+
throw error;
|
|
1575
|
+
}
|
|
1576
|
+
return {
|
|
1577
|
+
digest: hash.digest("hex"),
|
|
1578
|
+
mediaType: "application/epub+zip",
|
|
1579
|
+
name: basename3(path)
|
|
1580
|
+
};
|
|
1581
|
+
}
|
|
1582
|
+
function isNodeErrorCode(error, code) {
|
|
1583
|
+
return typeof error === "object" && error !== null && "code" in error && error.code === code;
|
|
1584
|
+
}
|
|
1585
|
+
function countTocChapters(items) {
|
|
1586
|
+
let count = 0;
|
|
1587
|
+
for (const item of items) {
|
|
1588
|
+
if (item.serialId !== void 0) count += 1;
|
|
1589
|
+
count += countTocChapters(
|
|
1590
|
+
item.children
|
|
1591
|
+
);
|
|
1592
|
+
}
|
|
1593
|
+
return count;
|
|
1594
|
+
}
|
|
1595
|
+
function collectTocChapterPaths(items, ancestorKeys = []) {
|
|
1596
|
+
const paths = [];
|
|
1597
|
+
for (const item of items) {
|
|
1598
|
+
const key = item.key ?? `chapter-${item.serialId ?? "group"}`;
|
|
1599
|
+
const chapterPath = [...ancestorKeys, key];
|
|
1600
|
+
if (item.serialId !== void 0) paths.push(chapterPath.join("/"));
|
|
1601
|
+
paths.push(
|
|
1602
|
+
...collectTocChapterPaths(
|
|
1603
|
+
item.children,
|
|
1604
|
+
chapterPath
|
|
1605
|
+
)
|
|
1606
|
+
);
|
|
1607
|
+
}
|
|
1608
|
+
return paths;
|
|
1609
|
+
}
|
|
1361
1610
|
|
|
1362
1611
|
// src/query-runtime.ts
|
|
1363
1612
|
import {
|
|
@@ -1529,489 +1778,130 @@ import {
|
|
|
1529
1778
|
isArchiveSearchIndexCurrent,
|
|
1530
1779
|
listChapters
|
|
1531
1780
|
} from "wiki-graph-core";
|
|
1532
|
-
|
|
1533
|
-
|
|
1534
|
-
|
|
1535
|
-
var DEFAULT_GENERATION_REQUEST_CONCURRENCY = 6;
|
|
1536
|
-
function planGenerationTask(task, words, chapters, concurrent, model) {
|
|
1537
|
-
const profile = getGenerationPlanningProfile(task);
|
|
1538
|
-
const calls = Math.max(chapters, Math.ceil(words / profile.wordsPerCall));
|
|
1539
|
-
const effectiveConcurrency = Math.max(
|
|
1540
|
-
1,
|
|
1541
|
-
task === "reading-graph" ? 1 : concurrent.request
|
|
1542
|
-
);
|
|
1543
|
-
const callBatches = Math.ceil(calls / effectiveConcurrency);
|
|
1544
|
-
const wordSeconds = words * profile.secondsPerWord;
|
|
1545
|
-
return {
|
|
1546
|
-
model,
|
|
1547
|
-
timeSeconds: {
|
|
1548
|
-
max: Math.ceil(callBatches * profile.maxSecondsPerCall + wordSeconds),
|
|
1549
|
-
min: Math.ceil(callBatches * profile.minSecondsPerCall + wordSeconds)
|
|
1550
|
-
},
|
|
1551
|
-
tokens: {
|
|
1552
|
-
cacheableInput: Math.ceil(words * profile.cacheableInputTokenPerWord),
|
|
1553
|
-
input: Math.ceil(words * profile.inputTokenPerWord),
|
|
1554
|
-
output: Math.ceil(words * profile.outputTokenPerWord)
|
|
1555
|
-
}
|
|
1556
|
-
};
|
|
1557
|
-
}
|
|
1558
|
-
function createGenerationPerformanceHints(input) {
|
|
1559
|
-
if (!input.hasJobWork) {
|
|
1560
|
-
return [];
|
|
1561
|
-
}
|
|
1562
|
-
const hints = [];
|
|
1563
|
-
const recommendedRequest = input.concurrent.request < 4 ? DEFAULT_GENERATION_REQUEST_CONCURRENCY : input.concurrent.request < 8 ? 8 : void 0;
|
|
1564
|
-
if (input.hasRequestWork && recommendedRequest !== void 0) {
|
|
1565
|
-
hints.push({
|
|
1566
|
-
configKey: "request",
|
|
1567
|
-
configValue: recommendedRequest,
|
|
1568
|
-
current: input.concurrent.request,
|
|
1569
|
-
kind: "request",
|
|
1570
|
-
message: "LLM request concurrency can often be higher. Use at least 4; 6-8 is usually faster when the provider allows it.",
|
|
1571
|
-
recommended: recommendedRequest
|
|
1572
|
-
});
|
|
1573
|
-
}
|
|
1574
|
-
if (input.concurrent.job < 4 && input.chapters > 1) {
|
|
1575
|
-
hints.push({
|
|
1576
|
-
configKey: "job",
|
|
1577
|
-
configValue: 4,
|
|
1578
|
-
current: input.concurrent.job,
|
|
1579
|
-
kind: "job",
|
|
1580
|
-
message: "Multiple chapters need work and job concurrency is below 4. Raising it lets chapter jobs run in parallel.",
|
|
1581
|
-
recommended: 4
|
|
1582
|
-
});
|
|
1583
|
-
}
|
|
1584
|
-
return hints;
|
|
1585
|
-
}
|
|
1586
|
-
function formatGenerationPlanningModel(llm) {
|
|
1587
|
-
if (llm?.model === void 0) {
|
|
1588
|
-
return "not configured";
|
|
1589
|
-
}
|
|
1590
|
-
return llm.provider === void 0 ? llm.model : `${llm.provider}/${llm.model}`;
|
|
1591
|
-
}
|
|
1592
|
-
function getGenerationPlanningProfile(task) {
|
|
1593
|
-
switch (task) {
|
|
1594
|
-
case "index-fts":
|
|
1595
|
-
return {
|
|
1596
|
-
cacheableInputTokenPerWord: 0,
|
|
1597
|
-
inputTokenPerWord: 0,
|
|
1598
|
-
maxSecondsPerCall: 1,
|
|
1599
|
-
minSecondsPerCall: 0,
|
|
1600
|
-
outputTokenPerWord: 0,
|
|
1601
|
-
secondsPerWord: 5e-4,
|
|
1602
|
-
wordsPerCall: 1e5
|
|
1603
|
-
};
|
|
1604
|
-
case "index-embedding-source":
|
|
1605
|
-
case "index-embedding-summary":
|
|
1606
|
-
return {
|
|
1607
|
-
cacheableInputTokenPerWord: 0,
|
|
1608
|
-
inputTokenPerWord: 0,
|
|
1609
|
-
maxSecondsPerCall: 10,
|
|
1610
|
-
minSecondsPerCall: 1,
|
|
1611
|
-
outputTokenPerWord: 0,
|
|
1612
|
-
secondsPerWord: 2e-3,
|
|
1613
|
-
wordsPerCall: 1e4
|
|
1614
|
-
};
|
|
1615
|
-
case "knowledge-graph":
|
|
1616
|
-
return {
|
|
1617
|
-
cacheableInputTokenPerWord: 60,
|
|
1618
|
-
inputTokenPerWord: 80,
|
|
1619
|
-
maxSecondsPerCall: 35,
|
|
1620
|
-
minSecondsPerCall: 15,
|
|
1621
|
-
outputTokenPerWord: 25,
|
|
1622
|
-
secondsPerWord: 0.02,
|
|
1623
|
-
wordsPerCall: 35
|
|
1624
|
-
};
|
|
1625
|
-
case "reading-graph":
|
|
1626
|
-
return {
|
|
1627
|
-
cacheableInputTokenPerWord: 20,
|
|
1628
|
-
inputTokenPerWord: 25,
|
|
1629
|
-
maxSecondsPerCall: 45,
|
|
1630
|
-
minSecondsPerCall: 15,
|
|
1631
|
-
outputTokenPerWord: 4,
|
|
1632
|
-
secondsPerWord: 0.01,
|
|
1633
|
-
wordsPerCall: 160
|
|
1634
|
-
};
|
|
1635
|
-
case "reading-summary":
|
|
1636
|
-
return {
|
|
1637
|
-
cacheableInputTokenPerWord: 45,
|
|
1638
|
-
inputTokenPerWord: 55,
|
|
1639
|
-
maxSecondsPerCall: 25,
|
|
1640
|
-
minSecondsPerCall: 10,
|
|
1641
|
-
outputTokenPerWord: 4,
|
|
1642
|
-
secondsPerWord: 0.01,
|
|
1643
|
-
wordsPerCall: 110
|
|
1644
|
-
};
|
|
1645
|
-
}
|
|
1646
|
-
}
|
|
1647
|
-
|
|
1648
|
-
// src/archive/inspect.ts
|
|
1649
|
-
async function createWikiGraphArchiveInspectReport(document, options) {
|
|
1650
|
-
const { archiveUri } = options;
|
|
1651
|
-
const [
|
|
1652
|
-
chapters,
|
|
1653
|
-
summaryWords,
|
|
1654
|
-
indexCacheCurrent,
|
|
1655
|
-
config,
|
|
1656
|
-
ftsArtifactCoverage,
|
|
1657
|
-
sourceEmbeddingArtifactCoverage,
|
|
1658
|
-
summaryEmbeddingArtifactCoverage
|
|
1659
|
-
] = await Promise.all([
|
|
1660
|
-
readInspectChapters(document, options.chapterId),
|
|
1661
|
-
readSummaryWords(document, options.chapterId),
|
|
1781
|
+
async function inspectWikiGraphArchive(document, options) {
|
|
1782
|
+
const [allChapters, indexCacheCurrent, summaryWords, ...coverage] = await Promise.all([
|
|
1783
|
+
listChapters(document),
|
|
1662
1784
|
isArchiveSearchIndexCurrent(document),
|
|
1663
|
-
|
|
1664
|
-
|
|
1665
|
-
|
|
1666
|
-
|
|
1785
|
+
readSummaryWords(document),
|
|
1786
|
+
document.indexArtifacts.listCoverage("fts"),
|
|
1787
|
+
document.indexArtifacts.listCoverage("embedding-source"),
|
|
1788
|
+
document.indexArtifacts.listCoverage("embedding-summary")
|
|
1667
1789
|
]);
|
|
1668
|
-
const
|
|
1669
|
-
const
|
|
1670
|
-
|
|
1671
|
-
|
|
1672
|
-
|
|
1790
|
+
const chapters = selectChapters(allChapters, options.chapterId);
|
|
1791
|
+
const coverageByKind = {
|
|
1792
|
+
fts: byChapterId(coverage[0]),
|
|
1793
|
+
sourceEmbedding: byChapterId(coverage[1]),
|
|
1794
|
+
summaryEmbedding: byChapterId(coverage[2])
|
|
1673
1795
|
};
|
|
1674
|
-
const
|
|
1675
|
-
|
|
1676
|
-
|
|
1677
|
-
|
|
1678
|
-
|
|
1679
|
-
|
|
1680
|
-
|
|
1681
|
-
|
|
1682
|
-
|
|
1683
|
-
|
|
1684
|
-
|
|
1685
|
-
|
|
1686
|
-
|
|
1687
|
-
|
|
1688
|
-
|
|
1689
|
-
|
|
1690
|
-
|
|
1691
|
-
|
|
1692
|
-
|
|
1693
|
-
|
|
1694
|
-
|
|
1695
|
-
|
|
1696
|
-
|
|
1697
|
-
|
|
1698
|
-
|
|
1699
|
-
|
|
1700
|
-
|
|
1701
|
-
|
|
1702
|
-
|
|
1703
|
-
|
|
1704
|
-
(chapter) => createInspectChapterReference(archiveUri, chapter)
|
|
1796
|
+
const inspections = await Promise.all(
|
|
1797
|
+
chapters.map(async (chapter) => {
|
|
1798
|
+
const serial = await document.serials.getById(chapter.chapterId);
|
|
1799
|
+
return {
|
|
1800
|
+
capabilities: {
|
|
1801
|
+
knowledgeGraph: {
|
|
1802
|
+
completed: serial?.knowledgeGraphReady === true
|
|
1803
|
+
},
|
|
1804
|
+
readingGraph: { completed: serial?.topologyReady === true },
|
|
1805
|
+
readingSummary: { completed: chapter.stage === "summarized" }
|
|
1806
|
+
},
|
|
1807
|
+
chapter,
|
|
1808
|
+
indexes: {
|
|
1809
|
+
fts: inspectArtifact(
|
|
1810
|
+
"fts",
|
|
1811
|
+
coverageByKind.fts.get(chapter.chapterId)
|
|
1812
|
+
),
|
|
1813
|
+
sourceEmbedding: inspectArtifact(
|
|
1814
|
+
"embedding-source",
|
|
1815
|
+
coverageByKind.sourceEmbedding.get(chapter.chapterId)
|
|
1816
|
+
),
|
|
1817
|
+
summaryEmbedding: inspectArtifact(
|
|
1818
|
+
"embedding-summary",
|
|
1819
|
+
coverageByKind.summaryEmbedding.get(chapter.chapterId)
|
|
1820
|
+
)
|
|
1821
|
+
},
|
|
1822
|
+
revision: serial?.revision ?? 0,
|
|
1823
|
+
summaryWords: summaryWords.get(chapter.chapterId) ?? 0
|
|
1824
|
+
};
|
|
1825
|
+
})
|
|
1705
1826
|
);
|
|
1706
|
-
const
|
|
1707
|
-
const improvements = createInspectImprovements({
|
|
1708
|
-
concurrent,
|
|
1709
|
-
contentChapters,
|
|
1710
|
-
indexCacheCurrent,
|
|
1711
|
-
planningModel,
|
|
1712
|
-
queryBlockedChapters: queryBlockedContentChapters,
|
|
1713
|
-
summaryCovered,
|
|
1714
|
-
knowledgeGraphCovered,
|
|
1715
|
-
readingGraphCovered
|
|
1716
|
-
});
|
|
1717
|
-
const performanceHints = createGenerationPerformanceHints({
|
|
1718
|
-
chapters: Math.max(
|
|
1719
|
-
0,
|
|
1720
|
-
...improvements.map((improvement) => improvement.missingChapters ?? 0)
|
|
1721
|
-
),
|
|
1722
|
-
concurrent,
|
|
1723
|
-
hasJobWork: improvements.some(
|
|
1724
|
-
(improvement) => improvement.planning !== void 0
|
|
1725
|
-
),
|
|
1726
|
-
hasRequestWork: improvements.some(
|
|
1727
|
-
(improvement) => improvement.planning !== void 0
|
|
1728
|
-
)
|
|
1729
|
-
});
|
|
1827
|
+
const selected = inspections[0];
|
|
1730
1828
|
return {
|
|
1731
|
-
archiveUri,
|
|
1732
|
-
|
|
1829
|
+
archiveUri: options.archiveUri,
|
|
1830
|
+
chapters: inspections,
|
|
1831
|
+
localIndexCache: { current: indexCacheCurrent },
|
|
1733
1832
|
scope: options.chapterId === void 0 ? { type: "archive" } : { chapterId: options.chapterId, type: "chapter" },
|
|
1734
|
-
|
|
1735
|
-
chapters: {
|
|
1736
|
-
content: contentChapters.length,
|
|
1737
|
-
planned: chapters.length - contentChapters.length,
|
|
1738
|
-
total: chapters.length
|
|
1739
|
-
},
|
|
1740
|
-
sourceWords,
|
|
1741
|
-
summaryWords
|
|
1742
|
-
},
|
|
1743
|
-
localIndexCache: {
|
|
1744
|
-
current: indexCacheCurrent,
|
|
1745
|
-
...!queryReady ? {
|
|
1746
|
-
blockedBy: "missing-search-artifacts",
|
|
1747
|
-
impact: "The cache cannot be built until every content chapter has a current FTS or source embedding artifact."
|
|
1748
|
-
} : indexCacheCurrent ? {} : {
|
|
1749
|
-
impact: "The next archive query may need to sync index cache first.",
|
|
1750
|
-
resource: "local CPU/disk time only; no provider calls."
|
|
1751
|
-
},
|
|
1752
|
-
status: !queryReady ? "blocked" : indexCacheCurrent ? "current" : "missing-or-outdated",
|
|
1753
|
-
syncAvailable: queryReady && !indexCacheCurrent
|
|
1754
|
-
},
|
|
1755
|
-
coverage: {
|
|
1756
|
-
ftsIndexArtifact: createInspectCoverage(
|
|
1757
|
-
ftsArtifactCovered,
|
|
1758
|
-
contentChapters
|
|
1759
|
-
),
|
|
1760
|
-
knowledgeGraph: createInspectCoverage(
|
|
1761
|
-
knowledgeGraphCovered,
|
|
1762
|
-
contentChapters
|
|
1763
|
-
),
|
|
1764
|
-
readingGraph: createInspectCoverage(readingGraphCovered, contentChapters),
|
|
1765
|
-
summary: createInspectCoverage(summaryCovered, contentChapters),
|
|
1766
|
-
sourceEmbeddingIndexArtifact: createInspectCoverage(
|
|
1767
|
-
sourceEmbeddingArtifactCovered,
|
|
1768
|
-
contentChapters
|
|
1769
|
-
),
|
|
1770
|
-
summaryEmbeddingIndexArtifact: createInspectCoverage(
|
|
1771
|
-
summaryEmbeddingArtifactCovered,
|
|
1772
|
-
contentChapters
|
|
1773
|
-
)
|
|
1774
|
-
},
|
|
1775
|
-
query: {
|
|
1776
|
-
ready: queryReady
|
|
1777
|
-
},
|
|
1778
|
-
searchArtifacts: {
|
|
1779
|
-
blockedChapters: queryBlockedChapters,
|
|
1780
|
-
...!queryReady ? {
|
|
1781
|
-
resource: "local CPU/disk time only; no provider calls."
|
|
1782
|
-
} : {},
|
|
1783
|
-
ready: queryReady,
|
|
1784
|
-
status: queryReady ? "ready" : "incomplete"
|
|
1785
|
-
},
|
|
1786
|
-
retrievalGuidance: formatRetrievalGuidance({
|
|
1787
|
-
indexCacheCurrent,
|
|
1788
|
-
knowledgeGraphCovered,
|
|
1789
|
-
readingGraphCovered,
|
|
1790
|
-
contentChapters,
|
|
1791
|
-
queryReady,
|
|
1792
|
-
sourceWords
|
|
1793
|
-
}),
|
|
1794
|
-
improvements,
|
|
1795
|
-
performanceHints,
|
|
1796
|
-
summaryCoverageNote: "Summary coverage means a current summary artifact exists for the chapter; it does not guarantee that every source topic is represented in the summary text."
|
|
1797
|
-
};
|
|
1798
|
-
}
|
|
1799
|
-
function createInspectChapterReference(archiveUri, chapter) {
|
|
1800
|
-
return {
|
|
1801
|
-
chapterId: chapter.chapterId,
|
|
1802
|
-
locatedUri: formatInspectLocatedChapterUri(archiveUri, chapter),
|
|
1803
|
-
title: chapter.title,
|
|
1804
|
-
uri: chapter.uri
|
|
1833
|
+
uri: selected === void 0 || options.chapterId === void 0 ? options.archiveUri : formatLocatedChapterUri(options.archiveUri, selected.chapter)
|
|
1805
1834
|
};
|
|
1806
1835
|
}
|
|
1807
|
-
function
|
|
1808
|
-
|
|
1809
|
-
|
|
1810
|
-
async function readInspectChapters(document, chapterId) {
|
|
1811
|
-
const chapters = chapterId === void 0 ? await listChapters(document) : (await listChapters(document)).filter(
|
|
1836
|
+
function selectChapters(chapters, chapterId) {
|
|
1837
|
+
if (chapterId === void 0) return chapters;
|
|
1838
|
+
const selected = chapters.filter(
|
|
1812
1839
|
(chapter) => chapter.chapterId === chapterId
|
|
1813
1840
|
);
|
|
1814
|
-
if (
|
|
1841
|
+
if (selected.length === 0) {
|
|
1815
1842
|
throw new Error(`Chapter ${chapterId} does not exist.`);
|
|
1816
1843
|
}
|
|
1817
|
-
return
|
|
1818
|
-
chapters.map(async (chapter) => {
|
|
1819
|
-
const serial = await document.serials.getById(chapter.chapterId);
|
|
1820
|
-
return {
|
|
1821
|
-
...chapter,
|
|
1822
|
-
knowledgeGraphReady: serial?.knowledgeGraphReady === true,
|
|
1823
|
-
readingGraphReady: serial?.topologyReady === true,
|
|
1824
|
-
summaryReady: chapter.stage === "summarized"
|
|
1825
|
-
};
|
|
1826
|
-
})
|
|
1827
|
-
);
|
|
1828
|
-
}
|
|
1829
|
-
async function readSummaryWords(document, chapterId) {
|
|
1830
|
-
return await document.readDatabase(
|
|
1831
|
-
async (database) => await database.queryOne(
|
|
1832
|
-
`
|
|
1833
|
-
SELECT COALESCE(SUM(words_count), 0) AS words
|
|
1834
|
-
FROM text_sentence_records
|
|
1835
|
-
WHERE kind = 2
|
|
1836
|
-
${chapterId === void 0 ? "" : "AND chapter_id = ?"}
|
|
1837
|
-
`,
|
|
1838
|
-
chapterId === void 0 ? void 0 : [chapterId],
|
|
1839
|
-
(row) => Number(row.words)
|
|
1840
|
-
) ?? 0
|
|
1841
|
-
);
|
|
1842
|
-
}
|
|
1843
|
-
async function readIndexArtifactCoverage(document, kind, chapterId) {
|
|
1844
|
-
const coverage = await document.indexArtifacts.listCoverage(kind);
|
|
1845
|
-
return chapterId === void 0 ? coverage : coverage.filter((item) => item.serialId === chapterId);
|
|
1844
|
+
return selected;
|
|
1846
1845
|
}
|
|
1847
|
-
function
|
|
1848
|
-
|
|
1849
|
-
coverage.filter((item) => item.current).map((item) => item.serialId)
|
|
1850
|
-
);
|
|
1851
|
-
return chapters.filter((chapter) => currentChapterIds.has(chapter.chapterId));
|
|
1846
|
+
function byChapterId(coverage) {
|
|
1847
|
+
return new Map(coverage.map((item) => [item.serialId, item]));
|
|
1852
1848
|
}
|
|
1853
|
-
function
|
|
1854
|
-
const coveredWords = sumWords(covered);
|
|
1855
|
-
const totalWords = sumWords(total);
|
|
1849
|
+
function inspectArtifact(kind, coverage) {
|
|
1856
1850
|
return {
|
|
1857
|
-
|
|
1858
|
-
|
|
1859
|
-
|
|
1860
|
-
|
|
1861
|
-
totalWords
|
|
1851
|
+
current: coverage?.current === true,
|
|
1852
|
+
exists: coverage?.sourceRevision !== void 0,
|
|
1853
|
+
kind,
|
|
1854
|
+
...coverage?.sourceRevision === void 0 ? {} : { sourceRevision: coverage.sourceRevision }
|
|
1862
1855
|
};
|
|
1863
1856
|
}
|
|
1864
|
-
function
|
|
1865
|
-
|
|
1866
|
-
|
|
1867
|
-
|
|
1868
|
-
|
|
1869
|
-
|
|
1870
|
-
|
|
1871
|
-
|
|
1872
|
-
|
|
1873
|
-
|
|
1874
|
-
|
|
1875
|
-
|
|
1876
|
-
|
|
1877
|
-
|
|
1878
|
-
input.contentChapters
|
|
1879
|
-
)
|
|
1880
|
-
);
|
|
1881
|
-
lines.push(
|
|
1882
|
-
formatObjectSearchGuidance(
|
|
1883
|
-
"Entity/triple retrieval",
|
|
1884
|
-
input.knowledgeGraphCovered,
|
|
1885
|
-
input.contentChapters
|
|
1857
|
+
async function readSummaryWords(document) {
|
|
1858
|
+
const rows = await document.readDatabase(
|
|
1859
|
+
async (database) => await database.queryAll(
|
|
1860
|
+
`
|
|
1861
|
+
SELECT chapter_id, COALESCE(SUM(words_count), 0) AS words
|
|
1862
|
+
FROM text_sentence_records
|
|
1863
|
+
WHERE kind = 2
|
|
1864
|
+
GROUP BY chapter_id
|
|
1865
|
+
`,
|
|
1866
|
+
void 0,
|
|
1867
|
+
(row) => ({
|
|
1868
|
+
chapterId: Number(row.chapter_id),
|
|
1869
|
+
words: Number(row.words)
|
|
1870
|
+
})
|
|
1886
1871
|
)
|
|
1887
1872
|
);
|
|
1888
|
-
return
|
|
1889
|
-
}
|
|
1890
|
-
function formatObjectSearchGuidance(label, covered, total) {
|
|
1891
|
-
const coveredWords = sumWords(covered);
|
|
1892
|
-
const totalWords = sumWords(total);
|
|
1893
|
-
const ratio = totalWords === 0 ? 0 : coveredWords / totalWords;
|
|
1894
|
-
const coverage = formatPercent(coveredWords, totalWords);
|
|
1895
|
-
if (ratio >= 1) {
|
|
1896
|
-
return `${label}: covers all source content; it can represent the full scope for object retrieval.`;
|
|
1897
|
-
}
|
|
1898
|
-
if (ratio >= 0.9) {
|
|
1899
|
-
return `${label}: covers ${coverage}; use it as the main path, but source --query is needed for uncovered content.`;
|
|
1900
|
-
}
|
|
1901
|
-
if (ratio >= 0.5) {
|
|
1902
|
-
return `${label}: covers ${coverage}; use it as leads, not as a full-scope substitute for source --query.`;
|
|
1903
|
-
}
|
|
1904
|
-
return `${label}: covers ${coverage}; build missing graph coverage before relying on object retrieval.`;
|
|
1905
|
-
}
|
|
1906
|
-
function createInspectImprovements(input) {
|
|
1907
|
-
const improvements = [];
|
|
1908
|
-
if (input.contentChapters.length === 0) {
|
|
1909
|
-
improvements.push({
|
|
1910
|
-
kind: "add-source",
|
|
1911
|
-
recommendation: "No source content is available yet; add or import source text before graph or summary generation.",
|
|
1912
|
-
title: "Add source content"
|
|
1913
|
-
});
|
|
1914
|
-
return improvements;
|
|
1915
|
-
}
|
|
1916
|
-
if (input.queryBlockedChapters.length > 0) {
|
|
1917
|
-
improvements.push({
|
|
1918
|
-
kind: "build",
|
|
1919
|
-
missingChapters: input.queryBlockedChapters.length,
|
|
1920
|
-
missingWords: sumWords(input.queryBlockedChapters),
|
|
1921
|
-
recommendation: "Build local FTS artifacts to make this scope queryable. This does not call a provider.",
|
|
1922
|
-
task: "index-fts",
|
|
1923
|
-
title: "Enable query with local FTS"
|
|
1924
|
-
});
|
|
1925
|
-
} else if (!input.indexCacheCurrent) {
|
|
1926
|
-
improvements.push({
|
|
1927
|
-
kind: "sync-index",
|
|
1928
|
-
recommendation: "Sync the index cache from chapter index artifacts so query starts without a lazy cache rebuild.",
|
|
1929
|
-
title: "Sync index cache"
|
|
1930
|
-
});
|
|
1931
|
-
}
|
|
1932
|
-
improvements.push(
|
|
1933
|
-
...createGraphImprovement({
|
|
1934
|
-
concurrent: input.concurrent,
|
|
1935
|
-
covered: input.readingGraphCovered,
|
|
1936
|
-
planningModel: input.planningModel,
|
|
1937
|
-
task: "reading-graph",
|
|
1938
|
-
title: "Complete Reading Graph coverage",
|
|
1939
|
-
total: input.contentChapters
|
|
1940
|
-
})
|
|
1941
|
-
);
|
|
1942
|
-
improvements.push(
|
|
1943
|
-
...createGraphImprovement({
|
|
1944
|
-
concurrent: input.concurrent,
|
|
1945
|
-
covered: input.knowledgeGraphCovered,
|
|
1946
|
-
planningModel: input.planningModel,
|
|
1947
|
-
task: "knowledge-graph",
|
|
1948
|
-
title: "Complete Knowledge Graph coverage",
|
|
1949
|
-
total: input.contentChapters
|
|
1950
|
-
})
|
|
1951
|
-
);
|
|
1952
|
-
improvements.push(
|
|
1953
|
-
...createGraphImprovement({
|
|
1954
|
-
concurrent: input.concurrent,
|
|
1955
|
-
covered: input.summaryCovered,
|
|
1956
|
-
planningModel: input.planningModel,
|
|
1957
|
-
task: "reading-summary",
|
|
1958
|
-
title: "Complete Summary coverage",
|
|
1959
|
-
total: input.contentChapters
|
|
1960
|
-
})
|
|
1961
|
-
);
|
|
1962
|
-
return improvements;
|
|
1873
|
+
return new Map(rows.map((row) => [row.chapterId, row.words]));
|
|
1963
1874
|
}
|
|
1964
|
-
function
|
|
1965
|
-
|
|
1966
|
-
const missing = input.total.filter(
|
|
1967
|
-
(chapter) => !coveredIds.has(chapter.chapterId)
|
|
1968
|
-
);
|
|
1969
|
-
if (missing.length === 0) {
|
|
1970
|
-
return [];
|
|
1971
|
-
}
|
|
1972
|
-
const missingWords = sumWords(missing);
|
|
1973
|
-
return [
|
|
1974
|
-
{
|
|
1975
|
-
kind: "build",
|
|
1976
|
-
missingChapters: missing.length,
|
|
1977
|
-
missingWords,
|
|
1978
|
-
planning: planGenerationTask(
|
|
1979
|
-
input.task,
|
|
1980
|
-
missingWords,
|
|
1981
|
-
missing.length,
|
|
1982
|
-
input.concurrent,
|
|
1983
|
-
input.planningModel
|
|
1984
|
-
),
|
|
1985
|
-
recommendation: formatImprovementRecommendation(
|
|
1986
|
-
input.covered,
|
|
1987
|
-
input.total,
|
|
1988
|
-
missingWords
|
|
1989
|
-
),
|
|
1990
|
-
task: input.task,
|
|
1991
|
-
title: input.title
|
|
1992
|
-
}
|
|
1993
|
-
];
|
|
1994
|
-
}
|
|
1995
|
-
function formatImprovementRecommendation(covered, total, missingWords) {
|
|
1996
|
-
const totalWords = sumWords(total);
|
|
1997
|
-
const coveredWords = sumWords(covered);
|
|
1998
|
-
if (totalWords === 0) {
|
|
1999
|
-
return "No source content is available.";
|
|
2000
|
-
}
|
|
2001
|
-
if (total.length === 1 || coveredWords / totalWords >= 0.9 || missingWords <= 1e3) {
|
|
2002
|
-
return "Queue the full scope to finish the remaining gap.";
|
|
2003
|
-
}
|
|
2004
|
-
return "Queue selected chapters first if only part of the scope matters.";
|
|
2005
|
-
}
|
|
2006
|
-
function sumWords(chapters) {
|
|
2007
|
-
return chapters.reduce((total, chapter) => total + chapter.words, 0);
|
|
1875
|
+
function formatLocatedChapterUri(archiveUri, chapter) {
|
|
1876
|
+
return `${archiveUri.replace(/\/+$/u, "")}/${chapter.uri.replace(/^wikg:\/\//u, "")}`;
|
|
2008
1877
|
}
|
|
2009
|
-
|
|
2010
|
-
|
|
2011
|
-
|
|
1878
|
+
|
|
1879
|
+
// src/archive/write.ts
|
|
1880
|
+
import {
|
|
1881
|
+
WikiGraphArchiveFile,
|
|
1882
|
+
writeWikiGraphLibraryArchive
|
|
1883
|
+
} from "wiki-graph-core";
|
|
1884
|
+
async function writeWikiGraphArchiveLocation(location, operation, options = {}) {
|
|
1885
|
+
const cacheOnly = options.searchIndexWritebackPolicy === "cache";
|
|
1886
|
+
const write = async (archiveFile = location.archiveFile) => await new WikiGraphArchiveFile(archiveFile).write(operation, {
|
|
1887
|
+
...cacheOnly ? {} : { derivedStateKeys: [location.archiveKey, location.archivePath] },
|
|
1888
|
+
...options.invalidateContinuationCursors === void 0 ? {} : {
|
|
1889
|
+
invalidateContinuationCursors: options.invalidateContinuationCursors
|
|
1890
|
+
},
|
|
1891
|
+
...options.searchIndexWritebackPolicy === void 0 ? {} : { searchIndexWritebackPolicy: options.searchIndexWritebackPolicy }
|
|
1892
|
+
});
|
|
1893
|
+
if (location.libraryArchiveTarget === void 0 || cacheOnly || options.refreshLibraryIndex === false) {
|
|
1894
|
+
return await write();
|
|
2012
1895
|
}
|
|
2013
|
-
|
|
2014
|
-
|
|
1896
|
+
return await writeWikiGraphLibraryArchive({
|
|
1897
|
+
additionalDerivedStateKeys: [location.archiveKey, location.archivePath],
|
|
1898
|
+
...options.invalidateContinuationCursors === void 0 ? {} : {
|
|
1899
|
+
invalidateContinuationCursors: options.invalidateContinuationCursors
|
|
1900
|
+
},
|
|
1901
|
+
...options.onIndexSyncError === void 0 ? {} : { onIndexSyncError: options.onIndexSyncError },
|
|
1902
|
+
operation: async (archiveFile) => await write(archiveFile),
|
|
1903
|
+
target: location.libraryArchiveTarget
|
|
1904
|
+
});
|
|
2015
1905
|
}
|
|
2016
1906
|
|
|
2017
1907
|
// src/archive/archive.ts
|
|
@@ -2031,13 +1921,13 @@ var WikiGraphArchiveManager = class {
|
|
|
2031
1921
|
}
|
|
2032
1922
|
async create(options) {
|
|
2033
1923
|
return await this.#runtime.run(async () => {
|
|
2034
|
-
const path =
|
|
1924
|
+
const path = await assertStandaloneWikiGraphArchivePath(options.path);
|
|
2035
1925
|
if (options.replace !== true && await nodePathExists(path)) {
|
|
2036
1926
|
throw new WikiGraphArchiveExistsError(path);
|
|
2037
1927
|
}
|
|
2038
|
-
const outputPath =
|
|
2039
|
-
|
|
2040
|
-
`.${
|
|
1928
|
+
const outputPath = join4(
|
|
1929
|
+
dirname3(path),
|
|
1930
|
+
`.${basename4(path)}.${randomUUID2()}.tmp.wikg`
|
|
2041
1931
|
);
|
|
2042
1932
|
try {
|
|
2043
1933
|
if (options.importPath === void 0) {
|
|
@@ -2062,15 +1952,15 @@ var WikiGraphArchiveManager = class {
|
|
|
2062
1952
|
throw error;
|
|
2063
1953
|
}
|
|
2064
1954
|
}
|
|
2065
|
-
return { locatedUri:
|
|
1955
|
+
return { locatedUri: formatLocatedWikiGraphUri2(path), path };
|
|
2066
1956
|
} finally {
|
|
2067
1957
|
await rm3(outputPath, { force: true, recursive: true });
|
|
2068
1958
|
}
|
|
2069
1959
|
}, options.signal);
|
|
2070
1960
|
}
|
|
2071
|
-
async open(
|
|
1961
|
+
async open(target, options = {}) {
|
|
2072
1962
|
const location = await this.#runtime.run(
|
|
2073
|
-
async () => await resolveWikiGraphArchiveLocation(
|
|
1963
|
+
async () => await resolveWikiGraphArchiveLocation(target),
|
|
2074
1964
|
options.signal
|
|
2075
1965
|
);
|
|
2076
1966
|
return new WikiGraphArchiveHandle(this.#runtime, location);
|
|
@@ -2093,14 +1983,17 @@ var WikiGraphArchiveHandle = class {
|
|
|
2093
1983
|
return this.#location.locatedUri;
|
|
2094
1984
|
}
|
|
2095
1985
|
get objectUri() {
|
|
2096
|
-
return
|
|
1986
|
+
return parseLocatedWikiGraphUri2(this.#location.locatedUri).objectUri ?? "wikg://";
|
|
2097
1987
|
}
|
|
2098
1988
|
get path() {
|
|
2099
1989
|
return this.#location.archivePath;
|
|
2100
1990
|
}
|
|
1991
|
+
get target() {
|
|
1992
|
+
return this.#location.target;
|
|
1993
|
+
}
|
|
2101
1994
|
async inspect(options = {}) {
|
|
2102
1995
|
return await this.#readDocument(
|
|
2103
|
-
async (document) => await
|
|
1996
|
+
async (document) => await inspectWikiGraphArchive(document, {
|
|
2104
1997
|
archiveUri: this.#location.publicArchiveUri ?? this.locatedUri,
|
|
2105
1998
|
...options.chapterId === void 0 ? {} : { chapterId: options.chapterId }
|
|
2106
1999
|
}),
|
|
@@ -2401,7 +2294,7 @@ var WikiGraphArchiveHandle = class {
|
|
|
2401
2294
|
objectUri,
|
|
2402
2295
|
async (wikimediaOptions) => await this.#readDocument(
|
|
2403
2296
|
async (document) => await readArchivePage(document, objectUri, {
|
|
2404
|
-
...
|
|
2297
|
+
...options,
|
|
2405
2298
|
...wikimediaOptions
|
|
2406
2299
|
}),
|
|
2407
2300
|
options
|
|
@@ -2547,7 +2440,7 @@ var WikiGraphArchiveHandle = class {
|
|
|
2547
2440
|
}
|
|
2548
2441
|
async #readDocument(operation, options = {}) {
|
|
2549
2442
|
return await this.#runtime.run(
|
|
2550
|
-
async () => await new
|
|
2443
|
+
async () => await new WikiGraphArchiveFile2(this.#location.archiveFile).readDocument(
|
|
2551
2444
|
operation
|
|
2552
2445
|
),
|
|
2553
2446
|
options.signal
|
|
@@ -2555,17 +2448,13 @@ var WikiGraphArchiveHandle = class {
|
|
|
2555
2448
|
}
|
|
2556
2449
|
async #writeDocument(operation, options = {}) {
|
|
2557
2450
|
return await this.#runtime.run(async () => {
|
|
2558
|
-
|
|
2559
|
-
|
|
2560
|
-
|
|
2451
|
+
return await writeWikiGraphArchiveLocation(this.#location, operation, {
|
|
2452
|
+
...options.onIndexSyncError === void 0 ? {} : { onIndexSyncError: options.onIndexSyncError },
|
|
2453
|
+
...options.refreshLibraryIndex === void 0 ? {} : { refreshLibraryIndex: options.refreshLibraryIndex },
|
|
2561
2454
|
...options.searchIndexWritebackPolicy === void 0 ? {} : {
|
|
2562
2455
|
searchIndexWritebackPolicy: options.searchIndexWritebackPolicy
|
|
2563
2456
|
}
|
|
2564
2457
|
});
|
|
2565
|
-
if (options.refreshLibraryIndex !== false) {
|
|
2566
|
-
await this.#refreshLibraryIndex(options.onIndexSyncError);
|
|
2567
|
-
}
|
|
2568
|
-
return result;
|
|
2569
2458
|
}, options.signal);
|
|
2570
2459
|
}
|
|
2571
2460
|
async #resolveScope(document, options) {
|
|
@@ -2585,52 +2474,7 @@ var WikiGraphArchiveHandle = class {
|
|
|
2585
2474
|
}
|
|
2586
2475
|
return queryable;
|
|
2587
2476
|
}
|
|
2588
|
-
async #refreshLibraryIndex(onError) {
|
|
2589
|
-
if (this.#location.libraryDirtyTarget === void 0) return;
|
|
2590
|
-
if (this.#location.libraryArchiveTarget !== void 0) {
|
|
2591
|
-
await finalizeWikiGraphLibraryArchiveWrite({
|
|
2592
|
-
target: this.#location.libraryArchiveTarget
|
|
2593
|
-
});
|
|
2594
|
-
}
|
|
2595
|
-
try {
|
|
2596
|
-
const state = await readWikiGraphLibraryIndexState(
|
|
2597
|
-
this.#location.libraryDirtyTarget
|
|
2598
|
-
);
|
|
2599
|
-
if (state.status !== "missing") {
|
|
2600
|
-
await rebuildWikiGraphLibraryIndex(this.#location.libraryDirtyTarget);
|
|
2601
|
-
}
|
|
2602
|
-
} catch (error) {
|
|
2603
|
-
onError?.(error);
|
|
2604
|
-
}
|
|
2605
|
-
}
|
|
2606
2477
|
};
|
|
2607
|
-
async function resolveWikiGraphArchiveLocation(uriOrPath) {
|
|
2608
|
-
if (!uriOrPath.startsWith("wikg://")) {
|
|
2609
|
-
return createPathLocation(resolveWikiGraphRuntimePath(uriOrPath));
|
|
2610
|
-
}
|
|
2611
|
-
const parsed = parseLocatedWikiGraphUri(uriOrPath);
|
|
2612
|
-
const portableArchiveLocator = parsed.archivePath ?? uriOrPath;
|
|
2613
|
-
const archiveLocator = portableArchiveLocator.startsWith("wikg://lib/") ? portableArchiveLocator : resolveWikiGraphRuntimePath(portableArchiveLocator);
|
|
2614
|
-
const libraryArchiveTarget = archiveLocator.startsWith("wikg://lib/") ? parseWikiGraphLibraryUri(archiveLocator) : void 0;
|
|
2615
|
-
const archiveFile = libraryArchiveTarget?.kind === "archive" ? await resolveWikiGraphLibraryArchiveFile(archiveLocator) : new NodeFile(archiveLocator);
|
|
2616
|
-
const archivePath = getNodeResourcePath(archiveFile);
|
|
2617
|
-
return {
|
|
2618
|
-
archiveFile,
|
|
2619
|
-
archiveKey: archivePath,
|
|
2620
|
-
archivePath,
|
|
2621
|
-
indexScope: { archiveKey: archivePath, archivePath, kind: "archive-index" },
|
|
2622
|
-
...libraryArchiveTarget?.kind === "archive" ? {
|
|
2623
|
-
libraryArchiveTarget,
|
|
2624
|
-
libraryDirtyTarget: {
|
|
2625
|
-
isDefault: libraryArchiveTarget.isDefault,
|
|
2626
|
-
kind: "scope",
|
|
2627
|
-
...libraryArchiveTarget.publicId === void 0 ? {} : { publicId: libraryArchiveTarget.publicId }
|
|
2628
|
-
},
|
|
2629
|
-
publicArchiveUri: portableArchiveLocator
|
|
2630
|
-
} : {},
|
|
2631
|
-
locatedUri: formatLocatedWikiGraphUri(archivePath, parsed.objectUri)
|
|
2632
|
-
};
|
|
2633
|
-
}
|
|
2634
2478
|
async function ensureArchiveSearchIndex(document, options = {}) {
|
|
2635
2479
|
const scope = options.chapters === void 0 ? {} : { chapters: options.chapters };
|
|
2636
2480
|
if (await isArchiveSearchIndexCurrent2(document, scope)) return;
|
|
@@ -2669,18 +2513,9 @@ function selectChapterSubtree(chapters, rootChapterId, depth) {
|
|
|
2669
2513
|
(chapter) => chapter.chapterId === root.chapterId || chapter.path.startsWith(prefix) && (depth === void 0 || chapter.depth - root.depth <= depth)
|
|
2670
2514
|
);
|
|
2671
2515
|
}
|
|
2672
|
-
function createPathLocation(path) {
|
|
2673
|
-
return {
|
|
2674
|
-
archiveFile: new NodeFile(path),
|
|
2675
|
-
archiveKey: path,
|
|
2676
|
-
archivePath: path,
|
|
2677
|
-
indexScope: { archiveKey: path, archivePath: path, kind: "archive-index" },
|
|
2678
|
-
locatedUri: formatLocatedWikiGraphUri(path)
|
|
2679
|
-
};
|
|
2680
|
-
}
|
|
2681
2516
|
async function createEmptyArchiveFile(path) {
|
|
2682
2517
|
const directoryPath = await mkdtemp(
|
|
2683
|
-
|
|
2518
|
+
join4(tmpdir(), "wikigraph-archive-write-")
|
|
2684
2519
|
);
|
|
2685
2520
|
try {
|
|
2686
2521
|
const directory = new NodeDirectory(directoryPath);
|
|
@@ -2702,7 +2537,7 @@ function isNodeEEXISTError(error) {
|
|
|
2702
2537
|
}
|
|
2703
2538
|
async function nodePathExists(path) {
|
|
2704
2539
|
try {
|
|
2705
|
-
await
|
|
2540
|
+
await stat3(path);
|
|
2706
2541
|
return true;
|
|
2707
2542
|
} catch (error) {
|
|
2708
2543
|
if (isNodeENOENTError(error)) return false;
|
|
@@ -2808,7 +2643,7 @@ export {
|
|
|
2808
2643
|
WikiGraphArchiveExistsError,
|
|
2809
2644
|
WikiGraphArchiveHandle,
|
|
2810
2645
|
WikiGraphArchiveManager,
|
|
2811
|
-
|
|
2812
|
-
|
|
2646
|
+
WikiGraphArchiveOwnershipError,
|
|
2647
|
+
inspectWikiGraphArchive
|
|
2813
2648
|
};
|
|
2814
2649
|
//# sourceMappingURL=archives.js.map
|