wiki-graph-sdk 0.7.1 → 0.8.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/dist/archives.cjs +406 -642
- package/dist/archives.cjs.map +1 -1
- package/dist/archives.d.ts +42 -80
- package/dist/archives.js +342 -571
- package/dist/archives.js.map +1 -1
- package/dist/conversions.cjs +160 -28
- package/dist/conversions.cjs.map +1 -1
- package/dist/conversions.d.ts +3 -1
- package/dist/conversions.js +165 -24
- package/dist/conversions.js.map +1 -1
- package/dist/default-worker.cjs +289 -52
- package/dist/default-worker.cjs.map +1 -1
- package/dist/default-worker.js +303 -50
- package/dist/default-worker.js.map +1 -1
- package/dist/index.cjs +967 -911
- package/dist/index.cjs.map +1 -1
- package/dist/index.d.ts +36 -19
- package/dist/index.js +836 -767
- package/dist/index.js.map +1 -1
- package/dist/{jobs-Dd5PMSId.d.ts → jobs-xIm_kshM.d.ts} +22 -14
- package/dist/maintenance.cjs +138 -29
- package/dist/maintenance.cjs.map +1 -1
- package/dist/maintenance.d.ts +2 -1
- package/dist/maintenance.js +143 -25
- package/dist/maintenance.js.map +1 -1
- package/dist/target-CZ7elwmj.d.ts +14 -0
- package/dist/wikispine.cjs +14 -10
- package/dist/wikispine.cjs.map +1 -1
- package/dist/wikispine.js +14 -10
- package/dist/wikispine.js.map +1 -1
- package/dist/worker.cjs +294 -25
- package/dist/worker.cjs.map +1 -1
- package/dist/worker.d.ts +28 -5
- package/dist/worker.js +312 -28
- package/dist/worker.js.map +1 -1
- package/package.json +2 -2
package/dist/archives.js
CHANGED
|
@@ -4,9 +4,9 @@ globalThis.__WIKIGRAPH_DATA_DIR__ ??= __WIKIGRAPH_RESOLVE__(__WIKIGRAPH_FILE_URL
|
|
|
4
4
|
|
|
5
5
|
// src/archive/archive.ts
|
|
6
6
|
import { randomUUID as randomUUID2 } from "crypto";
|
|
7
|
-
import { link, mkdtemp, rename as rename2, rm as rm3, stat as
|
|
7
|
+
import { link, mkdtemp, rename as rename2, rm as rm3, stat as stat3 } from "fs/promises";
|
|
8
8
|
import { tmpdir } from "os";
|
|
9
|
-
import { basename as
|
|
9
|
+
import { basename as basename4, dirname as dirname3, join as join4 } from "path";
|
|
10
10
|
import { Readable as Readable2 } from "stream";
|
|
11
11
|
import {
|
|
12
12
|
addChapter,
|
|
@@ -15,9 +15,8 @@ import {
|
|
|
15
15
|
assertNoActiveBuildJobs,
|
|
16
16
|
deleteArchiveSearchSessions,
|
|
17
17
|
DirectoryDocument,
|
|
18
|
-
finalizeWikiGraphLibraryArchiveWrite,
|
|
19
18
|
findArchiveObjects,
|
|
20
|
-
formatLocatedWikiGraphUri,
|
|
19
|
+
formatLocatedWikiGraphUri as formatLocatedWikiGraphUri2,
|
|
21
20
|
isArchiveSearchIndexCurrent as isArchiveSearchIndexCurrent2,
|
|
22
21
|
isSourceLocatorScopeUri,
|
|
23
22
|
listArchiveCollection,
|
|
@@ -28,23 +27,19 @@ import {
|
|
|
28
27
|
listRelatedArchiveObjects,
|
|
29
28
|
moveChapter,
|
|
30
29
|
packArchiveContext,
|
|
31
|
-
parseLocatedWikiGraphUri,
|
|
32
|
-
parseWikiGraphLibraryUri,
|
|
30
|
+
parseLocatedWikiGraphUri as parseLocatedWikiGraphUri2,
|
|
33
31
|
readArchivePage,
|
|
34
32
|
readSearchIndexCapabilityStatus,
|
|
35
|
-
readWikiGraphLibraryIndexState,
|
|
36
33
|
rebuildArchiveSearchIndex,
|
|
37
|
-
rebuildWikiGraphLibraryIndex,
|
|
38
34
|
removeChapter,
|
|
39
35
|
resetChapter,
|
|
40
36
|
resolveChapterPathReadonly,
|
|
41
|
-
resolveWikiGraphLibraryArchiveFile,
|
|
42
37
|
setChapterSource,
|
|
43
38
|
setChapterSummary,
|
|
44
39
|
setChapterTitle,
|
|
45
40
|
getChapterTree,
|
|
46
41
|
WikiGraph as WikiGraph2,
|
|
47
|
-
WikiGraphArchiveFile,
|
|
42
|
+
WikiGraphArchiveFile as WikiGraphArchiveFile2,
|
|
48
43
|
TOC_FILE_VERSION,
|
|
49
44
|
writeWikgArchive,
|
|
50
45
|
ObjectMetadataKind
|
|
@@ -54,7 +49,7 @@ import {
|
|
|
54
49
|
import { createHash } from "crypto";
|
|
55
50
|
import { createReadStream } from "fs";
|
|
56
51
|
import { rm as rm2 } from "fs/promises";
|
|
57
|
-
import { basename as
|
|
52
|
+
import { basename as basename3 } from "path";
|
|
58
53
|
import {
|
|
59
54
|
WikiGraph
|
|
60
55
|
} from "wiki-graph-core";
|
|
@@ -383,42 +378,42 @@ var NodeDatabaseConnection = class {
|
|
|
383
378
|
}
|
|
384
379
|
database;
|
|
385
380
|
async close() {
|
|
386
|
-
await new Promise((
|
|
381
|
+
await new Promise((resolve4, reject) => {
|
|
387
382
|
this.database.close((error) => {
|
|
388
383
|
if (error) reject(error);
|
|
389
|
-
else
|
|
384
|
+
else resolve4();
|
|
390
385
|
});
|
|
391
386
|
});
|
|
392
387
|
}
|
|
393
388
|
async execute(sql) {
|
|
394
|
-
await new Promise((
|
|
389
|
+
await new Promise((resolve4, reject) => {
|
|
395
390
|
this.database.exec(sql, (error) => {
|
|
396
391
|
if (error) reject(error);
|
|
397
|
-
else
|
|
392
|
+
else resolve4();
|
|
398
393
|
});
|
|
399
394
|
});
|
|
400
395
|
}
|
|
401
396
|
async queryAll(sql, params = []) {
|
|
402
|
-
return await new Promise((
|
|
397
|
+
return await new Promise((resolve4, reject) => {
|
|
403
398
|
this.database.all(sql, [...params], (error, rows) => {
|
|
404
399
|
if (error) reject(error);
|
|
405
|
-
else
|
|
400
|
+
else resolve4(rows);
|
|
406
401
|
});
|
|
407
402
|
});
|
|
408
403
|
}
|
|
409
404
|
async queryOne(sql, params = []) {
|
|
410
|
-
return await new Promise((
|
|
405
|
+
return await new Promise((resolve4, reject) => {
|
|
411
406
|
this.database.get(sql, [...params], (error, row) => {
|
|
412
407
|
if (error) reject(error);
|
|
413
|
-
else
|
|
408
|
+
else resolve4(row);
|
|
414
409
|
});
|
|
415
410
|
});
|
|
416
411
|
}
|
|
417
412
|
async run(sql, params = []) {
|
|
418
|
-
await new Promise((
|
|
413
|
+
await new Promise((resolve4, reject) => {
|
|
419
414
|
this.database.run(sql, [...params], (error) => {
|
|
420
415
|
if (error) reject(error);
|
|
421
|
-
else
|
|
416
|
+
else resolve4();
|
|
422
417
|
});
|
|
423
418
|
});
|
|
424
419
|
}
|
|
@@ -432,10 +427,10 @@ async function openNativeNodeDatabase(file, flags) {
|
|
|
432
427
|
if (path === void 0) {
|
|
433
428
|
throw new TypeError("The Node database adapter requires a NodeFile");
|
|
434
429
|
}
|
|
435
|
-
return await new Promise((
|
|
430
|
+
return await new Promise((resolve4, reject) => {
|
|
436
431
|
const database = new nodeSqlite3.Database(path, flags, (error) => {
|
|
437
432
|
if (error) reject(error);
|
|
438
|
-
else
|
|
433
|
+
else resolve4(database);
|
|
439
434
|
});
|
|
440
435
|
});
|
|
441
436
|
}
|
|
@@ -444,19 +439,19 @@ async function openNodeZip(file) {
|
|
|
444
439
|
if (path === void 0) {
|
|
445
440
|
throw new TypeError("The Node ZIP adapter requires a NodeFile");
|
|
446
441
|
}
|
|
447
|
-
const zipFile = await new Promise((
|
|
442
|
+
const zipFile = await new Promise((resolve4, reject) => {
|
|
448
443
|
yauzl.open(
|
|
449
444
|
path,
|
|
450
445
|
{ autoClose: false, lazyEntries: true },
|
|
451
446
|
(error, opened) => {
|
|
452
447
|
if (error || opened === void 0)
|
|
453
448
|
reject(error ?? new Error("Cannot open ZIP"));
|
|
454
|
-
else
|
|
449
|
+
else resolve4(opened);
|
|
455
450
|
}
|
|
456
451
|
);
|
|
457
452
|
});
|
|
458
453
|
const entries = await new Promise(
|
|
459
|
-
(
|
|
454
|
+
(resolve4, reject) => {
|
|
460
455
|
const discovered = /* @__PURE__ */ new Map();
|
|
461
456
|
const rejectAndClose = (error) => {
|
|
462
457
|
zipFile.close();
|
|
@@ -466,7 +461,7 @@ async function openNodeZip(file) {
|
|
|
466
461
|
};
|
|
467
462
|
const resolveEntries = () => {
|
|
468
463
|
zipFile.off("error", rejectAndClose);
|
|
469
|
-
|
|
464
|
+
resolve4(discovered);
|
|
470
465
|
};
|
|
471
466
|
zipFile.on("entry", (entry) => {
|
|
472
467
|
if (!entry.fileName.endsWith("/")) {
|
|
@@ -576,12 +571,12 @@ async function collectDecodedNodeStreamRange(input, offset, length) {
|
|
|
576
571
|
return Buffer.concat(chunks, collected);
|
|
577
572
|
}
|
|
578
573
|
async function openNodeZipEntryStream(zipFile, entry, options) {
|
|
579
|
-
return await new Promise((
|
|
574
|
+
return await new Promise((resolve4, reject) => {
|
|
580
575
|
const callback = (error, opened) => {
|
|
581
576
|
if (error || opened === void 0) {
|
|
582
577
|
reject(error ?? new Error(`Cannot read ZIP entry: ${entry.fileName}`));
|
|
583
578
|
} else {
|
|
584
|
-
|
|
579
|
+
resolve4(opened);
|
|
585
580
|
}
|
|
586
581
|
};
|
|
587
582
|
if (options === void 0) zipFile.openReadStream(entry, callback);
|
|
@@ -712,11 +707,11 @@ async function writeNodeStream(input, writer) {
|
|
|
712
707
|
);
|
|
713
708
|
}
|
|
714
709
|
async function collectNodeStream(input) {
|
|
715
|
-
return await new Promise((
|
|
710
|
+
return await new Promise((resolve4, reject) => {
|
|
716
711
|
const chunks = [];
|
|
717
712
|
input.on("data", (chunk) => chunks.push(chunk));
|
|
718
713
|
input.once("error", reject);
|
|
719
|
-
input.once("end", () =>
|
|
714
|
+
input.once("end", () => resolve4(Buffer.concat(chunks)));
|
|
720
715
|
});
|
|
721
716
|
}
|
|
722
717
|
var nodeWikiGraphPlatform = {
|
|
@@ -1269,6 +1264,173 @@ function resolveWikiGraphRuntimePath(path) {
|
|
|
1269
1264
|
return isAbsolute2(path) ? path : resolve2(context.cwd, path);
|
|
1270
1265
|
}
|
|
1271
1266
|
|
|
1267
|
+
// src/archive/target.ts
|
|
1268
|
+
import { realpath, stat as stat2 } from "fs/promises";
|
|
1269
|
+
import { basename as basename2, dirname as dirname2, isAbsolute as isAbsolute3, join as join3, relative, resolve as resolve3 } from "path";
|
|
1270
|
+
import {
|
|
1271
|
+
formatLocatedWikiGraphUri,
|
|
1272
|
+
formatWikiGraphLibraryUri,
|
|
1273
|
+
listWikiGraphLibraries,
|
|
1274
|
+
listWikiGraphLibraryArchives,
|
|
1275
|
+
parseLocatedWikiGraphUri,
|
|
1276
|
+
parseWikiGraphLibraryUri,
|
|
1277
|
+
resolveWikiGraphLibrary,
|
|
1278
|
+
resolveWikiGraphLibraryArchiveFile
|
|
1279
|
+
} from "wiki-graph-core";
|
|
1280
|
+
var WikiGraphArchiveOwnershipError = class extends Error {
|
|
1281
|
+
code = "WIKI_GRAPH_ARCHIVE_OWNERSHIP_MISMATCH";
|
|
1282
|
+
constructor(message) {
|
|
1283
|
+
super(message);
|
|
1284
|
+
this.name = "WikiGraphArchiveOwnershipError";
|
|
1285
|
+
}
|
|
1286
|
+
};
|
|
1287
|
+
async function resolveWikiGraphArchiveLocation(target) {
|
|
1288
|
+
if (target.kind === "standalone") {
|
|
1289
|
+
return await resolveStandaloneLocation(target);
|
|
1290
|
+
}
|
|
1291
|
+
return await resolveLibraryLocation(target);
|
|
1292
|
+
}
|
|
1293
|
+
async function assertStandaloneWikiGraphArchivePath(inputPath) {
|
|
1294
|
+
const path = resolveWikiGraphRuntimePath(inputPath);
|
|
1295
|
+
const candidate = await canonicalPath(path);
|
|
1296
|
+
const candidateIdentity = await readFileIdentity(path);
|
|
1297
|
+
const libraries = await listWikiGraphLibraries();
|
|
1298
|
+
const matches = await findMatchingMembers(
|
|
1299
|
+
libraries,
|
|
1300
|
+
candidate,
|
|
1301
|
+
candidateIdentity
|
|
1302
|
+
);
|
|
1303
|
+
if (matches.length > 0) {
|
|
1304
|
+
const match = requireUnambiguousManagedMatch(path, matches);
|
|
1305
|
+
throw new WikiGraphArchiveOwnershipError(
|
|
1306
|
+
`Standalone archive path belongs to registered library ${formatWikiGraphLibraryUri(
|
|
1307
|
+
match.library.isDefault ? void 0 : match.library.publicId
|
|
1308
|
+
)}: ${path}. Use the library UUID URI ${match.archive.uri} instead.`
|
|
1309
|
+
);
|
|
1310
|
+
}
|
|
1311
|
+
for (const library of libraries) {
|
|
1312
|
+
const rootPath = getNodeResourcePath(library.folder);
|
|
1313
|
+
const root = await canonicalPath(rootPath);
|
|
1314
|
+
if (!isWithin(root, candidate)) continue;
|
|
1315
|
+
throw new WikiGraphArchiveOwnershipError(
|
|
1316
|
+
`Standalone archive path belongs to registered library ${formatWikiGraphLibraryUri(
|
|
1317
|
+
library.isDefault ? void 0 : library.publicId
|
|
1318
|
+
)}: ${path}. Scan the library, then address the archive by its library UUID URI.`
|
|
1319
|
+
);
|
|
1320
|
+
}
|
|
1321
|
+
return path;
|
|
1322
|
+
}
|
|
1323
|
+
async function resolveStandaloneLocation(target) {
|
|
1324
|
+
if (target.path.startsWith("wikg://lib/")) {
|
|
1325
|
+
throw new WikiGraphArchiveOwnershipError(
|
|
1326
|
+
`A library archive URI cannot be used as a standalone path: ${target.path}`
|
|
1327
|
+
);
|
|
1328
|
+
}
|
|
1329
|
+
const path = await assertStandaloneWikiGraphArchivePath(target.path);
|
|
1330
|
+
const normalized = {
|
|
1331
|
+
kind: "standalone",
|
|
1332
|
+
path,
|
|
1333
|
+
...target.objectUri === void 0 ? {} : { objectUri: target.objectUri }
|
|
1334
|
+
};
|
|
1335
|
+
return {
|
|
1336
|
+
archiveFile: new NodeFile(path),
|
|
1337
|
+
archiveKey: path,
|
|
1338
|
+
archivePath: path,
|
|
1339
|
+
indexScope: { archiveKey: path, archivePath: path, kind: "archive-index" },
|
|
1340
|
+
locatedUri: formatLocatedWikiGraphUri(path, target.objectUri),
|
|
1341
|
+
target: normalized
|
|
1342
|
+
};
|
|
1343
|
+
}
|
|
1344
|
+
async function resolveLibraryLocation(target) {
|
|
1345
|
+
const parsedLocated = parseLocatedWikiGraphUri(target.uri);
|
|
1346
|
+
const archiveUri = parsedLocated.archivePath ?? target.uri;
|
|
1347
|
+
const parsed = parseWikiGraphLibraryUri(archiveUri);
|
|
1348
|
+
if (parsed?.kind !== "archive") {
|
|
1349
|
+
throw new WikiGraphArchiveOwnershipError(
|
|
1350
|
+
`Library archive target must be a UUID archive URI: ${target.uri}`
|
|
1351
|
+
);
|
|
1352
|
+
}
|
|
1353
|
+
await resolveWikiGraphLibrary(parsed);
|
|
1354
|
+
const archiveFile = await resolveWikiGraphLibraryArchiveFile(archiveUri);
|
|
1355
|
+
const normalized = {
|
|
1356
|
+
kind: "library",
|
|
1357
|
+
uri: target.uri
|
|
1358
|
+
};
|
|
1359
|
+
return {
|
|
1360
|
+
archiveFile,
|
|
1361
|
+
archiveKey: archiveUri,
|
|
1362
|
+
archivePath: archiveUri,
|
|
1363
|
+
indexScope: {
|
|
1364
|
+
archiveKey: archiveUri,
|
|
1365
|
+
archivePath: archiveUri,
|
|
1366
|
+
kind: "archive-index"
|
|
1367
|
+
},
|
|
1368
|
+
libraryArchiveTarget: parsed,
|
|
1369
|
+
locatedUri: target.uri,
|
|
1370
|
+
publicArchiveUri: archiveUri,
|
|
1371
|
+
target: normalized
|
|
1372
|
+
};
|
|
1373
|
+
}
|
|
1374
|
+
async function canonicalPath(path) {
|
|
1375
|
+
try {
|
|
1376
|
+
return await realpath(path);
|
|
1377
|
+
} catch {
|
|
1378
|
+
const absolute = resolve3(path);
|
|
1379
|
+
const parent = dirname2(absolute);
|
|
1380
|
+
return parent === absolute ? absolute : join3(await canonicalPath(parent), basename2(absolute));
|
|
1381
|
+
}
|
|
1382
|
+
}
|
|
1383
|
+
function isWithin(root, candidate) {
|
|
1384
|
+
const child = relative(root, candidate);
|
|
1385
|
+
return child === "" || !child.startsWith("..") && !isAbsolute3(child);
|
|
1386
|
+
}
|
|
1387
|
+
async function findMatchingMembers(libraries, candidate, candidateIdentity) {
|
|
1388
|
+
const members = [];
|
|
1389
|
+
for (const library of libraries) {
|
|
1390
|
+
const archives = await listWikiGraphLibraryArchives({
|
|
1391
|
+
isDefault: library.isDefault,
|
|
1392
|
+
kind: "scope",
|
|
1393
|
+
...library.isDefault ? {} : { publicId: library.publicId }
|
|
1394
|
+
});
|
|
1395
|
+
for (const archive of archives) {
|
|
1396
|
+
if (archive.file === void 0) continue;
|
|
1397
|
+
members.push({
|
|
1398
|
+
archive,
|
|
1399
|
+
library,
|
|
1400
|
+
memberPath: getNodeResourcePath(archive.file)
|
|
1401
|
+
});
|
|
1402
|
+
}
|
|
1403
|
+
}
|
|
1404
|
+
const exact = [];
|
|
1405
|
+
for (const member of members) {
|
|
1406
|
+
if (await canonicalPath(member.memberPath) === candidate) {
|
|
1407
|
+
exact.push(member);
|
|
1408
|
+
}
|
|
1409
|
+
}
|
|
1410
|
+
if (exact.length > 0 || candidateIdentity === void 0) return exact;
|
|
1411
|
+
const identityMatches = [];
|
|
1412
|
+
for (const member of members) {
|
|
1413
|
+
if (await readFileIdentity(member.memberPath) === candidateIdentity) {
|
|
1414
|
+
identityMatches.push(member);
|
|
1415
|
+
}
|
|
1416
|
+
}
|
|
1417
|
+
return identityMatches;
|
|
1418
|
+
}
|
|
1419
|
+
function requireUnambiguousManagedMatch(path, matches) {
|
|
1420
|
+
if (matches.length === 1) return matches[0];
|
|
1421
|
+
throw new WikiGraphArchiveOwnershipError(
|
|
1422
|
+
`Archive file identity is ambiguous across managed archives: ${path}. Use an explicit library UUID URI (${matches.map((match) => match.archive.uri).join(", ")}).`
|
|
1423
|
+
);
|
|
1424
|
+
}
|
|
1425
|
+
async function readFileIdentity(path) {
|
|
1426
|
+
try {
|
|
1427
|
+
const value = await stat2(path);
|
|
1428
|
+
return `${value.dev}:${value.ino}`;
|
|
1429
|
+
} catch {
|
|
1430
|
+
return void 0;
|
|
1431
|
+
}
|
|
1432
|
+
}
|
|
1433
|
+
|
|
1272
1434
|
// src/conversions.ts
|
|
1273
1435
|
var WikiGraphConversionManager = class {
|
|
1274
1436
|
#runtime;
|
|
@@ -1305,11 +1467,15 @@ var WikiGraphConversionManager = class {
|
|
|
1305
1467
|
requiresDigest
|
|
1306
1468
|
);
|
|
1307
1469
|
let chapterCount;
|
|
1470
|
+
let chapterPaths;
|
|
1308
1471
|
const write = async (archive) => {
|
|
1309
1472
|
if (requiresDigest) await options.onSourceImported?.();
|
|
1310
1473
|
if (typeof archive.readToc === "function") {
|
|
1311
1474
|
const toc = await archive.readToc();
|
|
1312
1475
|
chapterCount = toc === void 0 ? void 0 : countTocChapters(toc.items);
|
|
1476
|
+
if (options.output.format === "wikg") {
|
|
1477
|
+
chapterPaths = toc === void 0 ? [] : collectTocChapterPaths(toc.items);
|
|
1478
|
+
}
|
|
1313
1479
|
}
|
|
1314
1480
|
await writeArchive(archive, options.output.path, options.output.format);
|
|
1315
1481
|
};
|
|
@@ -1318,7 +1484,9 @@ var WikiGraphConversionManager = class {
|
|
|
1318
1484
|
throw new Error("wikg input requires a file path.");
|
|
1319
1485
|
}
|
|
1320
1486
|
await app.openSession(
|
|
1321
|
-
new NodeFile(
|
|
1487
|
+
new NodeFile(
|
|
1488
|
+
await assertStandaloneWikiGraphArchivePath(options.input.path)
|
|
1489
|
+
),
|
|
1322
1490
|
write
|
|
1323
1491
|
);
|
|
1324
1492
|
} else if ("stream" in options.input) {
|
|
@@ -1361,6 +1529,7 @@ var WikiGraphConversionManager = class {
|
|
|
1361
1529
|
const sourceArtifact = await resolveSourceArtifact(options.input);
|
|
1362
1530
|
return {
|
|
1363
1531
|
...chapterCount === void 0 ? {} : { chapterCount },
|
|
1532
|
+
...chapterPaths === void 0 ? {} : { chapterPaths },
|
|
1364
1533
|
inputFormat: options.input.format,
|
|
1365
1534
|
outputFormat: options.output.format,
|
|
1366
1535
|
outputPath: options.output.path,
|
|
@@ -1377,7 +1546,8 @@ async function prepareDigestDirectory(path, required) {
|
|
|
1377
1546
|
return new NodeDirectory(resolved);
|
|
1378
1547
|
}
|
|
1379
1548
|
async function writeArchive(archive, path, format) {
|
|
1380
|
-
const
|
|
1549
|
+
const outputPath = format === "wikg" ? await assertStandaloneWikiGraphArchivePath(path) : resolveWikiGraphRuntimePath(path);
|
|
1550
|
+
const file = new NodeFile(outputPath);
|
|
1381
1551
|
if (format === "epub") await archive.exportEpub(file);
|
|
1382
1552
|
else if (format === "wikg") await archive.saveAs(file);
|
|
1383
1553
|
else if (format === "pcex") throw new Error("pcex output is not supported.");
|
|
@@ -1406,7 +1576,7 @@ async function resolveSourceArtifact(input) {
|
|
|
1406
1576
|
return {
|
|
1407
1577
|
digest: hash.digest("hex"),
|
|
1408
1578
|
mediaType: "application/epub+zip",
|
|
1409
|
-
name:
|
|
1579
|
+
name: basename3(path)
|
|
1410
1580
|
};
|
|
1411
1581
|
}
|
|
1412
1582
|
function isNodeErrorCode(error, code) {
|
|
@@ -1422,6 +1592,21 @@ function countTocChapters(items) {
|
|
|
1422
1592
|
}
|
|
1423
1593
|
return count;
|
|
1424
1594
|
}
|
|
1595
|
+
function collectTocChapterPaths(items, ancestorKeys = []) {
|
|
1596
|
+
const paths = [];
|
|
1597
|
+
for (const item of items) {
|
|
1598
|
+
const key = item.key ?? `chapter-${item.serialId ?? "group"}`;
|
|
1599
|
+
const chapterPath = [...ancestorKeys, key];
|
|
1600
|
+
if (item.serialId !== void 0) paths.push(chapterPath.join("/"));
|
|
1601
|
+
paths.push(
|
|
1602
|
+
...collectTocChapterPaths(
|
|
1603
|
+
item.children,
|
|
1604
|
+
chapterPath
|
|
1605
|
+
)
|
|
1606
|
+
);
|
|
1607
|
+
}
|
|
1608
|
+
return paths;
|
|
1609
|
+
}
|
|
1425
1610
|
|
|
1426
1611
|
// src/query-runtime.ts
|
|
1427
1612
|
import {
|
|
@@ -1593,489 +1778,130 @@ import {
|
|
|
1593
1778
|
isArchiveSearchIndexCurrent,
|
|
1594
1779
|
listChapters
|
|
1595
1780
|
} from "wiki-graph-core";
|
|
1596
|
-
|
|
1597
|
-
|
|
1598
|
-
|
|
1599
|
-
var DEFAULT_GENERATION_REQUEST_CONCURRENCY = 6;
|
|
1600
|
-
function planGenerationTask(task, words, chapters, concurrent, model) {
|
|
1601
|
-
const profile = getGenerationPlanningProfile(task);
|
|
1602
|
-
const calls = Math.max(chapters, Math.ceil(words / profile.wordsPerCall));
|
|
1603
|
-
const effectiveConcurrency = Math.max(
|
|
1604
|
-
1,
|
|
1605
|
-
task === "reading-graph" ? 1 : concurrent.request
|
|
1606
|
-
);
|
|
1607
|
-
const callBatches = Math.ceil(calls / effectiveConcurrency);
|
|
1608
|
-
const wordSeconds = words * profile.secondsPerWord;
|
|
1609
|
-
return {
|
|
1610
|
-
model,
|
|
1611
|
-
timeSeconds: {
|
|
1612
|
-
max: Math.ceil(callBatches * profile.maxSecondsPerCall + wordSeconds),
|
|
1613
|
-
min: Math.ceil(callBatches * profile.minSecondsPerCall + wordSeconds)
|
|
1614
|
-
},
|
|
1615
|
-
tokens: {
|
|
1616
|
-
cacheableInput: Math.ceil(words * profile.cacheableInputTokenPerWord),
|
|
1617
|
-
input: Math.ceil(words * profile.inputTokenPerWord),
|
|
1618
|
-
output: Math.ceil(words * profile.outputTokenPerWord)
|
|
1619
|
-
}
|
|
1620
|
-
};
|
|
1621
|
-
}
|
|
1622
|
-
function createGenerationPerformanceHints(input) {
|
|
1623
|
-
if (!input.hasJobWork) {
|
|
1624
|
-
return [];
|
|
1625
|
-
}
|
|
1626
|
-
const hints = [];
|
|
1627
|
-
const recommendedRequest = input.concurrent.request < 4 ? DEFAULT_GENERATION_REQUEST_CONCURRENCY : input.concurrent.request < 8 ? 8 : void 0;
|
|
1628
|
-
if (input.hasRequestWork && recommendedRequest !== void 0) {
|
|
1629
|
-
hints.push({
|
|
1630
|
-
configKey: "request",
|
|
1631
|
-
configValue: recommendedRequest,
|
|
1632
|
-
current: input.concurrent.request,
|
|
1633
|
-
kind: "request",
|
|
1634
|
-
message: "LLM request concurrency can often be higher. Use at least 4; 6-8 is usually faster when the provider allows it.",
|
|
1635
|
-
recommended: recommendedRequest
|
|
1636
|
-
});
|
|
1637
|
-
}
|
|
1638
|
-
if (input.concurrent.job < 4 && input.chapters > 1) {
|
|
1639
|
-
hints.push({
|
|
1640
|
-
configKey: "job",
|
|
1641
|
-
configValue: 4,
|
|
1642
|
-
current: input.concurrent.job,
|
|
1643
|
-
kind: "job",
|
|
1644
|
-
message: "Multiple chapters need work and job concurrency is below 4. Raising it lets chapter jobs run in parallel.",
|
|
1645
|
-
recommended: 4
|
|
1646
|
-
});
|
|
1647
|
-
}
|
|
1648
|
-
return hints;
|
|
1649
|
-
}
|
|
1650
|
-
function formatGenerationPlanningModel(llm) {
|
|
1651
|
-
if (llm?.model === void 0) {
|
|
1652
|
-
return "not configured";
|
|
1653
|
-
}
|
|
1654
|
-
return llm.provider === void 0 ? llm.model : `${llm.provider}/${llm.model}`;
|
|
1655
|
-
}
|
|
1656
|
-
function getGenerationPlanningProfile(task) {
|
|
1657
|
-
switch (task) {
|
|
1658
|
-
case "index-fts":
|
|
1659
|
-
return {
|
|
1660
|
-
cacheableInputTokenPerWord: 0,
|
|
1661
|
-
inputTokenPerWord: 0,
|
|
1662
|
-
maxSecondsPerCall: 1,
|
|
1663
|
-
minSecondsPerCall: 0,
|
|
1664
|
-
outputTokenPerWord: 0,
|
|
1665
|
-
secondsPerWord: 5e-4,
|
|
1666
|
-
wordsPerCall: 1e5
|
|
1667
|
-
};
|
|
1668
|
-
case "index-embedding-source":
|
|
1669
|
-
case "index-embedding-summary":
|
|
1670
|
-
return {
|
|
1671
|
-
cacheableInputTokenPerWord: 0,
|
|
1672
|
-
inputTokenPerWord: 0,
|
|
1673
|
-
maxSecondsPerCall: 10,
|
|
1674
|
-
minSecondsPerCall: 1,
|
|
1675
|
-
outputTokenPerWord: 0,
|
|
1676
|
-
secondsPerWord: 2e-3,
|
|
1677
|
-
wordsPerCall: 1e4
|
|
1678
|
-
};
|
|
1679
|
-
case "knowledge-graph":
|
|
1680
|
-
return {
|
|
1681
|
-
cacheableInputTokenPerWord: 60,
|
|
1682
|
-
inputTokenPerWord: 80,
|
|
1683
|
-
maxSecondsPerCall: 35,
|
|
1684
|
-
minSecondsPerCall: 15,
|
|
1685
|
-
outputTokenPerWord: 25,
|
|
1686
|
-
secondsPerWord: 0.02,
|
|
1687
|
-
wordsPerCall: 35
|
|
1688
|
-
};
|
|
1689
|
-
case "reading-graph":
|
|
1690
|
-
return {
|
|
1691
|
-
cacheableInputTokenPerWord: 20,
|
|
1692
|
-
inputTokenPerWord: 25,
|
|
1693
|
-
maxSecondsPerCall: 45,
|
|
1694
|
-
minSecondsPerCall: 15,
|
|
1695
|
-
outputTokenPerWord: 4,
|
|
1696
|
-
secondsPerWord: 0.01,
|
|
1697
|
-
wordsPerCall: 160
|
|
1698
|
-
};
|
|
1699
|
-
case "reading-summary":
|
|
1700
|
-
return {
|
|
1701
|
-
cacheableInputTokenPerWord: 45,
|
|
1702
|
-
inputTokenPerWord: 55,
|
|
1703
|
-
maxSecondsPerCall: 25,
|
|
1704
|
-
minSecondsPerCall: 10,
|
|
1705
|
-
outputTokenPerWord: 4,
|
|
1706
|
-
secondsPerWord: 0.01,
|
|
1707
|
-
wordsPerCall: 110
|
|
1708
|
-
};
|
|
1709
|
-
}
|
|
1710
|
-
}
|
|
1711
|
-
|
|
1712
|
-
// src/archive/inspect.ts
|
|
1713
|
-
async function createWikiGraphArchiveInspectReport(document, options) {
|
|
1714
|
-
const { archiveUri } = options;
|
|
1715
|
-
const [
|
|
1716
|
-
chapters,
|
|
1717
|
-
summaryWords,
|
|
1718
|
-
indexCacheCurrent,
|
|
1719
|
-
config,
|
|
1720
|
-
ftsArtifactCoverage,
|
|
1721
|
-
sourceEmbeddingArtifactCoverage,
|
|
1722
|
-
summaryEmbeddingArtifactCoverage
|
|
1723
|
-
] = await Promise.all([
|
|
1724
|
-
readInspectChapters(document, options.chapterId),
|
|
1725
|
-
readSummaryWords(document, options.chapterId),
|
|
1781
|
+
async function inspectWikiGraphArchive(document, options) {
|
|
1782
|
+
const [allChapters, indexCacheCurrent, summaryWords, ...coverage] = await Promise.all([
|
|
1783
|
+
listChapters(document),
|
|
1726
1784
|
isArchiveSearchIndexCurrent(document),
|
|
1727
|
-
|
|
1728
|
-
|
|
1729
|
-
|
|
1730
|
-
|
|
1785
|
+
readSummaryWords(document),
|
|
1786
|
+
document.indexArtifacts.listCoverage("fts"),
|
|
1787
|
+
document.indexArtifacts.listCoverage("embedding-source"),
|
|
1788
|
+
document.indexArtifacts.listCoverage("embedding-summary")
|
|
1731
1789
|
]);
|
|
1732
|
-
const
|
|
1733
|
-
const
|
|
1734
|
-
|
|
1735
|
-
|
|
1736
|
-
|
|
1790
|
+
const chapters = selectChapters(allChapters, options.chapterId);
|
|
1791
|
+
const coverageByKind = {
|
|
1792
|
+
fts: byChapterId(coverage[0]),
|
|
1793
|
+
sourceEmbedding: byChapterId(coverage[1]),
|
|
1794
|
+
summaryEmbedding: byChapterId(coverage[2])
|
|
1737
1795
|
};
|
|
1738
|
-
const
|
|
1739
|
-
|
|
1740
|
-
|
|
1741
|
-
|
|
1742
|
-
|
|
1743
|
-
|
|
1744
|
-
|
|
1745
|
-
|
|
1746
|
-
|
|
1747
|
-
|
|
1748
|
-
|
|
1749
|
-
|
|
1750
|
-
|
|
1751
|
-
|
|
1752
|
-
|
|
1753
|
-
|
|
1754
|
-
|
|
1755
|
-
|
|
1756
|
-
|
|
1757
|
-
|
|
1758
|
-
|
|
1759
|
-
|
|
1760
|
-
|
|
1761
|
-
|
|
1762
|
-
|
|
1763
|
-
|
|
1764
|
-
|
|
1765
|
-
|
|
1766
|
-
|
|
1767
|
-
|
|
1768
|
-
(chapter) => createInspectChapterReference(archiveUri, chapter)
|
|
1796
|
+
const inspections = await Promise.all(
|
|
1797
|
+
chapters.map(async (chapter) => {
|
|
1798
|
+
const serial = await document.serials.getById(chapter.chapterId);
|
|
1799
|
+
return {
|
|
1800
|
+
capabilities: {
|
|
1801
|
+
knowledgeGraph: {
|
|
1802
|
+
completed: serial?.knowledgeGraphReady === true
|
|
1803
|
+
},
|
|
1804
|
+
readingGraph: { completed: serial?.topologyReady === true },
|
|
1805
|
+
readingSummary: { completed: chapter.stage === "summarized" }
|
|
1806
|
+
},
|
|
1807
|
+
chapter,
|
|
1808
|
+
indexes: {
|
|
1809
|
+
fts: inspectArtifact(
|
|
1810
|
+
"fts",
|
|
1811
|
+
coverageByKind.fts.get(chapter.chapterId)
|
|
1812
|
+
),
|
|
1813
|
+
sourceEmbedding: inspectArtifact(
|
|
1814
|
+
"embedding-source",
|
|
1815
|
+
coverageByKind.sourceEmbedding.get(chapter.chapterId)
|
|
1816
|
+
),
|
|
1817
|
+
summaryEmbedding: inspectArtifact(
|
|
1818
|
+
"embedding-summary",
|
|
1819
|
+
coverageByKind.summaryEmbedding.get(chapter.chapterId)
|
|
1820
|
+
)
|
|
1821
|
+
},
|
|
1822
|
+
revision: serial?.revision ?? 0,
|
|
1823
|
+
summaryWords: summaryWords.get(chapter.chapterId) ?? 0
|
|
1824
|
+
};
|
|
1825
|
+
})
|
|
1769
1826
|
);
|
|
1770
|
-
const
|
|
1771
|
-
const improvements = createInspectImprovements({
|
|
1772
|
-
concurrent,
|
|
1773
|
-
contentChapters,
|
|
1774
|
-
indexCacheCurrent,
|
|
1775
|
-
planningModel,
|
|
1776
|
-
queryBlockedChapters: queryBlockedContentChapters,
|
|
1777
|
-
summaryCovered,
|
|
1778
|
-
knowledgeGraphCovered,
|
|
1779
|
-
readingGraphCovered
|
|
1780
|
-
});
|
|
1781
|
-
const performanceHints = createGenerationPerformanceHints({
|
|
1782
|
-
chapters: Math.max(
|
|
1783
|
-
0,
|
|
1784
|
-
...improvements.map((improvement) => improvement.missingChapters ?? 0)
|
|
1785
|
-
),
|
|
1786
|
-
concurrent,
|
|
1787
|
-
hasJobWork: improvements.some(
|
|
1788
|
-
(improvement) => improvement.planning !== void 0
|
|
1789
|
-
),
|
|
1790
|
-
hasRequestWork: improvements.some(
|
|
1791
|
-
(improvement) => improvement.planning !== void 0
|
|
1792
|
-
)
|
|
1793
|
-
});
|
|
1827
|
+
const selected = inspections[0];
|
|
1794
1828
|
return {
|
|
1795
|
-
archiveUri,
|
|
1796
|
-
|
|
1829
|
+
archiveUri: options.archiveUri,
|
|
1830
|
+
chapters: inspections,
|
|
1831
|
+
localIndexCache: { current: indexCacheCurrent },
|
|
1797
1832
|
scope: options.chapterId === void 0 ? { type: "archive" } : { chapterId: options.chapterId, type: "chapter" },
|
|
1798
|
-
|
|
1799
|
-
chapters: {
|
|
1800
|
-
content: contentChapters.length,
|
|
1801
|
-
planned: chapters.length - contentChapters.length,
|
|
1802
|
-
total: chapters.length
|
|
1803
|
-
},
|
|
1804
|
-
sourceWords,
|
|
1805
|
-
summaryWords
|
|
1806
|
-
},
|
|
1807
|
-
localIndexCache: {
|
|
1808
|
-
current: indexCacheCurrent,
|
|
1809
|
-
...!queryReady ? {
|
|
1810
|
-
blockedBy: "missing-search-artifacts",
|
|
1811
|
-
impact: "The cache cannot be built until every content chapter has a current FTS or source embedding artifact."
|
|
1812
|
-
} : indexCacheCurrent ? {} : {
|
|
1813
|
-
impact: "The next archive query may need to sync index cache first.",
|
|
1814
|
-
resource: "local CPU/disk time only; no provider calls."
|
|
1815
|
-
},
|
|
1816
|
-
status: !queryReady ? "blocked" : indexCacheCurrent ? "current" : "missing-or-outdated",
|
|
1817
|
-
syncAvailable: queryReady && !indexCacheCurrent
|
|
1818
|
-
},
|
|
1819
|
-
coverage: {
|
|
1820
|
-
ftsIndexArtifact: createInspectCoverage(
|
|
1821
|
-
ftsArtifactCovered,
|
|
1822
|
-
contentChapters
|
|
1823
|
-
),
|
|
1824
|
-
knowledgeGraph: createInspectCoverage(
|
|
1825
|
-
knowledgeGraphCovered,
|
|
1826
|
-
contentChapters
|
|
1827
|
-
),
|
|
1828
|
-
readingGraph: createInspectCoverage(readingGraphCovered, contentChapters),
|
|
1829
|
-
summary: createInspectCoverage(summaryCovered, contentChapters),
|
|
1830
|
-
sourceEmbeddingIndexArtifact: createInspectCoverage(
|
|
1831
|
-
sourceEmbeddingArtifactCovered,
|
|
1832
|
-
contentChapters
|
|
1833
|
-
),
|
|
1834
|
-
summaryEmbeddingIndexArtifact: createInspectCoverage(
|
|
1835
|
-
summaryEmbeddingArtifactCovered,
|
|
1836
|
-
contentChapters
|
|
1837
|
-
)
|
|
1838
|
-
},
|
|
1839
|
-
query: {
|
|
1840
|
-
ready: queryReady
|
|
1841
|
-
},
|
|
1842
|
-
searchArtifacts: {
|
|
1843
|
-
blockedChapters: queryBlockedChapters,
|
|
1844
|
-
...!queryReady ? {
|
|
1845
|
-
resource: "local CPU/disk time only; no provider calls."
|
|
1846
|
-
} : {},
|
|
1847
|
-
ready: queryReady,
|
|
1848
|
-
status: queryReady ? "ready" : "incomplete"
|
|
1849
|
-
},
|
|
1850
|
-
retrievalGuidance: formatRetrievalGuidance({
|
|
1851
|
-
indexCacheCurrent,
|
|
1852
|
-
knowledgeGraphCovered,
|
|
1853
|
-
readingGraphCovered,
|
|
1854
|
-
contentChapters,
|
|
1855
|
-
queryReady,
|
|
1856
|
-
sourceWords
|
|
1857
|
-
}),
|
|
1858
|
-
improvements,
|
|
1859
|
-
performanceHints,
|
|
1860
|
-
summaryCoverageNote: "Summary coverage means a current summary artifact exists for the chapter; it does not guarantee that every source topic is represented in the summary text."
|
|
1861
|
-
};
|
|
1862
|
-
}
|
|
1863
|
-
function createInspectChapterReference(archiveUri, chapter) {
|
|
1864
|
-
return {
|
|
1865
|
-
chapterId: chapter.chapterId,
|
|
1866
|
-
locatedUri: formatInspectLocatedChapterUri(archiveUri, chapter),
|
|
1867
|
-
title: chapter.title,
|
|
1868
|
-
uri: chapter.uri
|
|
1833
|
+
uri: selected === void 0 || options.chapterId === void 0 ? options.archiveUri : formatLocatedChapterUri(options.archiveUri, selected.chapter)
|
|
1869
1834
|
};
|
|
1870
1835
|
}
|
|
1871
|
-
function
|
|
1872
|
-
|
|
1873
|
-
|
|
1874
|
-
async function readInspectChapters(document, chapterId) {
|
|
1875
|
-
const chapters = chapterId === void 0 ? await listChapters(document) : (await listChapters(document)).filter(
|
|
1836
|
+
function selectChapters(chapters, chapterId) {
|
|
1837
|
+
if (chapterId === void 0) return chapters;
|
|
1838
|
+
const selected = chapters.filter(
|
|
1876
1839
|
(chapter) => chapter.chapterId === chapterId
|
|
1877
1840
|
);
|
|
1878
|
-
if (
|
|
1841
|
+
if (selected.length === 0) {
|
|
1879
1842
|
throw new Error(`Chapter ${chapterId} does not exist.`);
|
|
1880
1843
|
}
|
|
1881
|
-
return
|
|
1882
|
-
chapters.map(async (chapter) => {
|
|
1883
|
-
const serial = await document.serials.getById(chapter.chapterId);
|
|
1884
|
-
return {
|
|
1885
|
-
...chapter,
|
|
1886
|
-
knowledgeGraphReady: serial?.knowledgeGraphReady === true,
|
|
1887
|
-
readingGraphReady: serial?.topologyReady === true,
|
|
1888
|
-
summaryReady: chapter.stage === "summarized"
|
|
1889
|
-
};
|
|
1890
|
-
})
|
|
1891
|
-
);
|
|
1892
|
-
}
|
|
1893
|
-
async function readSummaryWords(document, chapterId) {
|
|
1894
|
-
return await document.readDatabase(
|
|
1895
|
-
async (database) => await database.queryOne(
|
|
1896
|
-
`
|
|
1897
|
-
SELECT COALESCE(SUM(words_count), 0) AS words
|
|
1898
|
-
FROM text_sentence_records
|
|
1899
|
-
WHERE kind = 2
|
|
1900
|
-
${chapterId === void 0 ? "" : "AND chapter_id = ?"}
|
|
1901
|
-
`,
|
|
1902
|
-
chapterId === void 0 ? void 0 : [chapterId],
|
|
1903
|
-
(row) => Number(row.words)
|
|
1904
|
-
) ?? 0
|
|
1905
|
-
);
|
|
1844
|
+
return selected;
|
|
1906
1845
|
}
|
|
1907
|
-
|
|
1908
|
-
|
|
1909
|
-
return chapterId === void 0 ? coverage : coverage.filter((item) => item.serialId === chapterId);
|
|
1846
|
+
function byChapterId(coverage) {
|
|
1847
|
+
return new Map(coverage.map((item) => [item.serialId, item]));
|
|
1910
1848
|
}
|
|
1911
|
-
function
|
|
1912
|
-
const currentChapterIds = new Set(
|
|
1913
|
-
coverage.filter((item) => item.current).map((item) => item.serialId)
|
|
1914
|
-
);
|
|
1915
|
-
return chapters.filter((chapter) => currentChapterIds.has(chapter.chapterId));
|
|
1916
|
-
}
|
|
1917
|
-
function createInspectCoverage(covered, total) {
|
|
1918
|
-
const coveredWords = sumWords(covered);
|
|
1919
|
-
const totalWords = sumWords(total);
|
|
1849
|
+
function inspectArtifact(kind, coverage) {
|
|
1920
1850
|
return {
|
|
1921
|
-
|
|
1922
|
-
|
|
1923
|
-
|
|
1924
|
-
|
|
1925
|
-
totalWords
|
|
1851
|
+
current: coverage?.current === true,
|
|
1852
|
+
exists: coverage?.sourceRevision !== void 0,
|
|
1853
|
+
kind,
|
|
1854
|
+
...coverage?.sourceRevision === void 0 ? {} : { sourceRevision: coverage.sourceRevision }
|
|
1926
1855
|
};
|
|
1927
1856
|
}
|
|
1928
|
-
function
|
|
1929
|
-
|
|
1930
|
-
|
|
1931
|
-
|
|
1932
|
-
|
|
1933
|
-
|
|
1934
|
-
|
|
1935
|
-
|
|
1936
|
-
|
|
1937
|
-
|
|
1938
|
-
|
|
1939
|
-
|
|
1940
|
-
|
|
1941
|
-
|
|
1942
|
-
input.contentChapters
|
|
1943
|
-
)
|
|
1944
|
-
);
|
|
1945
|
-
lines.push(
|
|
1946
|
-
formatObjectSearchGuidance(
|
|
1947
|
-
"Entity/triple retrieval",
|
|
1948
|
-
input.knowledgeGraphCovered,
|
|
1949
|
-
input.contentChapters
|
|
1857
|
+
async function readSummaryWords(document) {
|
|
1858
|
+
const rows = await document.readDatabase(
|
|
1859
|
+
async (database) => await database.queryAll(
|
|
1860
|
+
`
|
|
1861
|
+
SELECT chapter_id, COALESCE(SUM(words_count), 0) AS words
|
|
1862
|
+
FROM text_sentence_records
|
|
1863
|
+
WHERE kind = 2
|
|
1864
|
+
GROUP BY chapter_id
|
|
1865
|
+
`,
|
|
1866
|
+
void 0,
|
|
1867
|
+
(row) => ({
|
|
1868
|
+
chapterId: Number(row.chapter_id),
|
|
1869
|
+
words: Number(row.words)
|
|
1870
|
+
})
|
|
1950
1871
|
)
|
|
1951
1872
|
);
|
|
1952
|
-
return
|
|
1953
|
-
}
|
|
1954
|
-
function formatObjectSearchGuidance(label, covered, total) {
|
|
1955
|
-
const coveredWords = sumWords(covered);
|
|
1956
|
-
const totalWords = sumWords(total);
|
|
1957
|
-
const ratio = totalWords === 0 ? 0 : coveredWords / totalWords;
|
|
1958
|
-
const coverage = formatPercent(coveredWords, totalWords);
|
|
1959
|
-
if (ratio >= 1) {
|
|
1960
|
-
return `${label}: covers all source content; it can represent the full scope for object retrieval.`;
|
|
1961
|
-
}
|
|
1962
|
-
if (ratio >= 0.9) {
|
|
1963
|
-
return `${label}: covers ${coverage}; use it as the main path, but source --query is needed for uncovered content.`;
|
|
1964
|
-
}
|
|
1965
|
-
if (ratio >= 0.5) {
|
|
1966
|
-
return `${label}: covers ${coverage}; use it as leads, not as a full-scope substitute for source --query.`;
|
|
1967
|
-
}
|
|
1968
|
-
return `${label}: covers ${coverage}; build missing graph coverage before relying on object retrieval.`;
|
|
1969
|
-
}
|
|
1970
|
-
function createInspectImprovements(input) {
|
|
1971
|
-
const improvements = [];
|
|
1972
|
-
if (input.contentChapters.length === 0) {
|
|
1973
|
-
improvements.push({
|
|
1974
|
-
kind: "add-source",
|
|
1975
|
-
recommendation: "No source content is available yet; add or import source text before graph or summary generation.",
|
|
1976
|
-
title: "Add source content"
|
|
1977
|
-
});
|
|
1978
|
-
return improvements;
|
|
1979
|
-
}
|
|
1980
|
-
if (input.queryBlockedChapters.length > 0) {
|
|
1981
|
-
improvements.push({
|
|
1982
|
-
kind: "build",
|
|
1983
|
-
missingChapters: input.queryBlockedChapters.length,
|
|
1984
|
-
missingWords: sumWords(input.queryBlockedChapters),
|
|
1985
|
-
recommendation: "Build local FTS artifacts to make this scope queryable. This does not call a provider.",
|
|
1986
|
-
task: "index-fts",
|
|
1987
|
-
title: "Enable query with local FTS"
|
|
1988
|
-
});
|
|
1989
|
-
} else if (!input.indexCacheCurrent) {
|
|
1990
|
-
improvements.push({
|
|
1991
|
-
kind: "sync-index",
|
|
1992
|
-
recommendation: "Sync the index cache from chapter index artifacts so query starts without a lazy cache rebuild.",
|
|
1993
|
-
title: "Sync index cache"
|
|
1994
|
-
});
|
|
1995
|
-
}
|
|
1996
|
-
improvements.push(
|
|
1997
|
-
...createGraphImprovement({
|
|
1998
|
-
concurrent: input.concurrent,
|
|
1999
|
-
covered: input.readingGraphCovered,
|
|
2000
|
-
planningModel: input.planningModel,
|
|
2001
|
-
task: "reading-graph",
|
|
2002
|
-
title: "Complete Reading Graph coverage",
|
|
2003
|
-
total: input.contentChapters
|
|
2004
|
-
})
|
|
2005
|
-
);
|
|
2006
|
-
improvements.push(
|
|
2007
|
-
...createGraphImprovement({
|
|
2008
|
-
concurrent: input.concurrent,
|
|
2009
|
-
covered: input.knowledgeGraphCovered,
|
|
2010
|
-
planningModel: input.planningModel,
|
|
2011
|
-
task: "knowledge-graph",
|
|
2012
|
-
title: "Complete Knowledge Graph coverage",
|
|
2013
|
-
total: input.contentChapters
|
|
2014
|
-
})
|
|
2015
|
-
);
|
|
2016
|
-
improvements.push(
|
|
2017
|
-
...createGraphImprovement({
|
|
2018
|
-
concurrent: input.concurrent,
|
|
2019
|
-
covered: input.summaryCovered,
|
|
2020
|
-
planningModel: input.planningModel,
|
|
2021
|
-
task: "reading-summary",
|
|
2022
|
-
title: "Complete Summary coverage",
|
|
2023
|
-
total: input.contentChapters
|
|
2024
|
-
})
|
|
2025
|
-
);
|
|
2026
|
-
return improvements;
|
|
2027
|
-
}
|
|
2028
|
-
function createGraphImprovement(input) {
|
|
2029
|
-
const coveredIds = new Set(input.covered.map((chapter) => chapter.chapterId));
|
|
2030
|
-
const missing = input.total.filter(
|
|
2031
|
-
(chapter) => !coveredIds.has(chapter.chapterId)
|
|
2032
|
-
);
|
|
2033
|
-
if (missing.length === 0) {
|
|
2034
|
-
return [];
|
|
2035
|
-
}
|
|
2036
|
-
const missingWords = sumWords(missing);
|
|
2037
|
-
return [
|
|
2038
|
-
{
|
|
2039
|
-
kind: "build",
|
|
2040
|
-
missingChapters: missing.length,
|
|
2041
|
-
missingWords,
|
|
2042
|
-
planning: planGenerationTask(
|
|
2043
|
-
input.task,
|
|
2044
|
-
missingWords,
|
|
2045
|
-
missing.length,
|
|
2046
|
-
input.concurrent,
|
|
2047
|
-
input.planningModel
|
|
2048
|
-
),
|
|
2049
|
-
recommendation: formatImprovementRecommendation(
|
|
2050
|
-
input.covered,
|
|
2051
|
-
input.total,
|
|
2052
|
-
missingWords
|
|
2053
|
-
),
|
|
2054
|
-
task: input.task,
|
|
2055
|
-
title: input.title
|
|
2056
|
-
}
|
|
2057
|
-
];
|
|
2058
|
-
}
|
|
2059
|
-
function formatImprovementRecommendation(covered, total, missingWords) {
|
|
2060
|
-
const totalWords = sumWords(total);
|
|
2061
|
-
const coveredWords = sumWords(covered);
|
|
2062
|
-
if (totalWords === 0) {
|
|
2063
|
-
return "No source content is available.";
|
|
2064
|
-
}
|
|
2065
|
-
if (total.length === 1 || coveredWords / totalWords >= 0.9 || missingWords <= 1e3) {
|
|
2066
|
-
return "Queue the full scope to finish the remaining gap.";
|
|
2067
|
-
}
|
|
2068
|
-
return "Queue selected chapters first if only part of the scope matters.";
|
|
1873
|
+
return new Map(rows.map((row) => [row.chapterId, row.words]));
|
|
2069
1874
|
}
|
|
2070
|
-
function
|
|
2071
|
-
return
|
|
1875
|
+
function formatLocatedChapterUri(archiveUri, chapter) {
|
|
1876
|
+
return `${archiveUri.replace(/\/+$/u, "")}/${chapter.uri.replace(/^wikg:\/\//u, "")}`;
|
|
2072
1877
|
}
|
|
2073
|
-
|
|
2074
|
-
|
|
2075
|
-
|
|
1878
|
+
|
|
1879
|
+
// src/archive/write.ts
|
|
1880
|
+
import {
|
|
1881
|
+
WikiGraphArchiveFile,
|
|
1882
|
+
writeWikiGraphLibraryArchive
|
|
1883
|
+
} from "wiki-graph-core";
|
|
1884
|
+
async function writeWikiGraphArchiveLocation(location, operation, options = {}) {
|
|
1885
|
+
const cacheOnly = options.searchIndexWritebackPolicy === "cache";
|
|
1886
|
+
const write = async (archiveFile = location.archiveFile) => await new WikiGraphArchiveFile(archiveFile).write(operation, {
|
|
1887
|
+
...cacheOnly ? {} : { derivedStateKeys: [location.archiveKey, location.archivePath] },
|
|
1888
|
+
...options.invalidateContinuationCursors === void 0 ? {} : {
|
|
1889
|
+
invalidateContinuationCursors: options.invalidateContinuationCursors
|
|
1890
|
+
},
|
|
1891
|
+
...options.searchIndexWritebackPolicy === void 0 ? {} : { searchIndexWritebackPolicy: options.searchIndexWritebackPolicy }
|
|
1892
|
+
});
|
|
1893
|
+
if (location.libraryArchiveTarget === void 0 || cacheOnly || options.refreshLibraryIndex === false) {
|
|
1894
|
+
return await write();
|
|
2076
1895
|
}
|
|
2077
|
-
|
|
2078
|
-
|
|
1896
|
+
return await writeWikiGraphLibraryArchive({
|
|
1897
|
+
additionalDerivedStateKeys: [location.archiveKey, location.archivePath],
|
|
1898
|
+
...options.invalidateContinuationCursors === void 0 ? {} : {
|
|
1899
|
+
invalidateContinuationCursors: options.invalidateContinuationCursors
|
|
1900
|
+
},
|
|
1901
|
+
...options.onIndexSyncError === void 0 ? {} : { onIndexSyncError: options.onIndexSyncError },
|
|
1902
|
+
operation: async (archiveFile) => await write(archiveFile),
|
|
1903
|
+
target: location.libraryArchiveTarget
|
|
1904
|
+
});
|
|
2079
1905
|
}
|
|
2080
1906
|
|
|
2081
1907
|
// src/archive/archive.ts
|
|
@@ -2095,13 +1921,13 @@ var WikiGraphArchiveManager = class {
|
|
|
2095
1921
|
}
|
|
2096
1922
|
async create(options) {
|
|
2097
1923
|
return await this.#runtime.run(async () => {
|
|
2098
|
-
const path =
|
|
1924
|
+
const path = await assertStandaloneWikiGraphArchivePath(options.path);
|
|
2099
1925
|
if (options.replace !== true && await nodePathExists(path)) {
|
|
2100
1926
|
throw new WikiGraphArchiveExistsError(path);
|
|
2101
1927
|
}
|
|
2102
|
-
const outputPath =
|
|
2103
|
-
|
|
2104
|
-
`.${
|
|
1928
|
+
const outputPath = join4(
|
|
1929
|
+
dirname3(path),
|
|
1930
|
+
`.${basename4(path)}.${randomUUID2()}.tmp.wikg`
|
|
2105
1931
|
);
|
|
2106
1932
|
try {
|
|
2107
1933
|
if (options.importPath === void 0) {
|
|
@@ -2126,15 +1952,15 @@ var WikiGraphArchiveManager = class {
|
|
|
2126
1952
|
throw error;
|
|
2127
1953
|
}
|
|
2128
1954
|
}
|
|
2129
|
-
return { locatedUri:
|
|
1955
|
+
return { locatedUri: formatLocatedWikiGraphUri2(path), path };
|
|
2130
1956
|
} finally {
|
|
2131
1957
|
await rm3(outputPath, { force: true, recursive: true });
|
|
2132
1958
|
}
|
|
2133
1959
|
}, options.signal);
|
|
2134
1960
|
}
|
|
2135
|
-
async open(
|
|
1961
|
+
async open(target, options = {}) {
|
|
2136
1962
|
const location = await this.#runtime.run(
|
|
2137
|
-
async () => await resolveWikiGraphArchiveLocation(
|
|
1963
|
+
async () => await resolveWikiGraphArchiveLocation(target),
|
|
2138
1964
|
options.signal
|
|
2139
1965
|
);
|
|
2140
1966
|
return new WikiGraphArchiveHandle(this.#runtime, location);
|
|
@@ -2157,14 +1983,17 @@ var WikiGraphArchiveHandle = class {
|
|
|
2157
1983
|
return this.#location.locatedUri;
|
|
2158
1984
|
}
|
|
2159
1985
|
get objectUri() {
|
|
2160
|
-
return
|
|
1986
|
+
return parseLocatedWikiGraphUri2(this.#location.locatedUri).objectUri ?? "wikg://";
|
|
2161
1987
|
}
|
|
2162
1988
|
get path() {
|
|
2163
1989
|
return this.#location.archivePath;
|
|
2164
1990
|
}
|
|
1991
|
+
get target() {
|
|
1992
|
+
return this.#location.target;
|
|
1993
|
+
}
|
|
2165
1994
|
async inspect(options = {}) {
|
|
2166
1995
|
return await this.#readDocument(
|
|
2167
|
-
async (document) => await
|
|
1996
|
+
async (document) => await inspectWikiGraphArchive(document, {
|
|
2168
1997
|
archiveUri: this.#location.publicArchiveUri ?? this.locatedUri,
|
|
2169
1998
|
...options.chapterId === void 0 ? {} : { chapterId: options.chapterId }
|
|
2170
1999
|
}),
|
|
@@ -2465,7 +2294,7 @@ var WikiGraphArchiveHandle = class {
|
|
|
2465
2294
|
objectUri,
|
|
2466
2295
|
async (wikimediaOptions) => await this.#readDocument(
|
|
2467
2296
|
async (document) => await readArchivePage(document, objectUri, {
|
|
2468
|
-
...
|
|
2297
|
+
...options,
|
|
2469
2298
|
...wikimediaOptions
|
|
2470
2299
|
}),
|
|
2471
2300
|
options
|
|
@@ -2611,7 +2440,7 @@ var WikiGraphArchiveHandle = class {
|
|
|
2611
2440
|
}
|
|
2612
2441
|
async #readDocument(operation, options = {}) {
|
|
2613
2442
|
return await this.#runtime.run(
|
|
2614
|
-
async () => await new
|
|
2443
|
+
async () => await new WikiGraphArchiveFile2(this.#location.archiveFile).readDocument(
|
|
2615
2444
|
operation
|
|
2616
2445
|
),
|
|
2617
2446
|
options.signal
|
|
@@ -2619,17 +2448,13 @@ var WikiGraphArchiveHandle = class {
|
|
|
2619
2448
|
}
|
|
2620
2449
|
async #writeDocument(operation, options = {}) {
|
|
2621
2450
|
return await this.#runtime.run(async () => {
|
|
2622
|
-
|
|
2623
|
-
|
|
2624
|
-
|
|
2451
|
+
return await writeWikiGraphArchiveLocation(this.#location, operation, {
|
|
2452
|
+
...options.onIndexSyncError === void 0 ? {} : { onIndexSyncError: options.onIndexSyncError },
|
|
2453
|
+
...options.refreshLibraryIndex === void 0 ? {} : { refreshLibraryIndex: options.refreshLibraryIndex },
|
|
2625
2454
|
...options.searchIndexWritebackPolicy === void 0 ? {} : {
|
|
2626
2455
|
searchIndexWritebackPolicy: options.searchIndexWritebackPolicy
|
|
2627
2456
|
}
|
|
2628
2457
|
});
|
|
2629
|
-
if (options.refreshLibraryIndex !== false) {
|
|
2630
|
-
await this.#refreshLibraryIndex(options.onIndexSyncError);
|
|
2631
|
-
}
|
|
2632
|
-
return result;
|
|
2633
2458
|
}, options.signal);
|
|
2634
2459
|
}
|
|
2635
2460
|
async #resolveScope(document, options) {
|
|
@@ -2649,52 +2474,7 @@ var WikiGraphArchiveHandle = class {
|
|
|
2649
2474
|
}
|
|
2650
2475
|
return queryable;
|
|
2651
2476
|
}
|
|
2652
|
-
async #refreshLibraryIndex(onError) {
|
|
2653
|
-
if (this.#location.libraryDirtyTarget === void 0) return;
|
|
2654
|
-
if (this.#location.libraryArchiveTarget !== void 0) {
|
|
2655
|
-
await finalizeWikiGraphLibraryArchiveWrite({
|
|
2656
|
-
target: this.#location.libraryArchiveTarget
|
|
2657
|
-
});
|
|
2658
|
-
}
|
|
2659
|
-
try {
|
|
2660
|
-
const state = await readWikiGraphLibraryIndexState(
|
|
2661
|
-
this.#location.libraryDirtyTarget
|
|
2662
|
-
);
|
|
2663
|
-
if (state.status !== "missing") {
|
|
2664
|
-
await rebuildWikiGraphLibraryIndex(this.#location.libraryDirtyTarget);
|
|
2665
|
-
}
|
|
2666
|
-
} catch (error) {
|
|
2667
|
-
onError?.(error);
|
|
2668
|
-
}
|
|
2669
|
-
}
|
|
2670
2477
|
};
|
|
2671
|
-
async function resolveWikiGraphArchiveLocation(uriOrPath) {
|
|
2672
|
-
if (!uriOrPath.startsWith("wikg://")) {
|
|
2673
|
-
return createPathLocation(resolveWikiGraphRuntimePath(uriOrPath));
|
|
2674
|
-
}
|
|
2675
|
-
const parsed = parseLocatedWikiGraphUri(uriOrPath);
|
|
2676
|
-
const portableArchiveLocator = parsed.archivePath ?? uriOrPath;
|
|
2677
|
-
const archiveLocator = portableArchiveLocator.startsWith("wikg://lib/") ? portableArchiveLocator : resolveWikiGraphRuntimePath(portableArchiveLocator);
|
|
2678
|
-
const libraryArchiveTarget = archiveLocator.startsWith("wikg://lib/") ? parseWikiGraphLibraryUri(archiveLocator) : void 0;
|
|
2679
|
-
const archiveFile = libraryArchiveTarget?.kind === "archive" ? await resolveWikiGraphLibraryArchiveFile(archiveLocator) : new NodeFile(archiveLocator);
|
|
2680
|
-
const archivePath = getNodeResourcePath(archiveFile);
|
|
2681
|
-
return {
|
|
2682
|
-
archiveFile,
|
|
2683
|
-
archiveKey: archivePath,
|
|
2684
|
-
archivePath,
|
|
2685
|
-
indexScope: { archiveKey: archivePath, archivePath, kind: "archive-index" },
|
|
2686
|
-
...libraryArchiveTarget?.kind === "archive" ? {
|
|
2687
|
-
libraryArchiveTarget,
|
|
2688
|
-
libraryDirtyTarget: {
|
|
2689
|
-
isDefault: libraryArchiveTarget.isDefault,
|
|
2690
|
-
kind: "scope",
|
|
2691
|
-
...libraryArchiveTarget.publicId === void 0 ? {} : { publicId: libraryArchiveTarget.publicId }
|
|
2692
|
-
},
|
|
2693
|
-
publicArchiveUri: portableArchiveLocator
|
|
2694
|
-
} : {},
|
|
2695
|
-
locatedUri: formatLocatedWikiGraphUri(archivePath, parsed.objectUri)
|
|
2696
|
-
};
|
|
2697
|
-
}
|
|
2698
2478
|
async function ensureArchiveSearchIndex(document, options = {}) {
|
|
2699
2479
|
const scope = options.chapters === void 0 ? {} : { chapters: options.chapters };
|
|
2700
2480
|
if (await isArchiveSearchIndexCurrent2(document, scope)) return;
|
|
@@ -2733,18 +2513,9 @@ function selectChapterSubtree(chapters, rootChapterId, depth) {
|
|
|
2733
2513
|
(chapter) => chapter.chapterId === root.chapterId || chapter.path.startsWith(prefix) && (depth === void 0 || chapter.depth - root.depth <= depth)
|
|
2734
2514
|
);
|
|
2735
2515
|
}
|
|
2736
|
-
function createPathLocation(path) {
|
|
2737
|
-
return {
|
|
2738
|
-
archiveFile: new NodeFile(path),
|
|
2739
|
-
archiveKey: path,
|
|
2740
|
-
archivePath: path,
|
|
2741
|
-
indexScope: { archiveKey: path, archivePath: path, kind: "archive-index" },
|
|
2742
|
-
locatedUri: formatLocatedWikiGraphUri(path)
|
|
2743
|
-
};
|
|
2744
|
-
}
|
|
2745
2516
|
async function createEmptyArchiveFile(path) {
|
|
2746
2517
|
const directoryPath = await mkdtemp(
|
|
2747
|
-
|
|
2518
|
+
join4(tmpdir(), "wikigraph-archive-write-")
|
|
2748
2519
|
);
|
|
2749
2520
|
try {
|
|
2750
2521
|
const directory = new NodeDirectory(directoryPath);
|
|
@@ -2766,7 +2537,7 @@ function isNodeEEXISTError(error) {
|
|
|
2766
2537
|
}
|
|
2767
2538
|
async function nodePathExists(path) {
|
|
2768
2539
|
try {
|
|
2769
|
-
await
|
|
2540
|
+
await stat3(path);
|
|
2770
2541
|
return true;
|
|
2771
2542
|
} catch (error) {
|
|
2772
2543
|
if (isNodeENOENTError(error)) return false;
|
|
@@ -2872,7 +2643,7 @@ export {
|
|
|
2872
2643
|
WikiGraphArchiveExistsError,
|
|
2873
2644
|
WikiGraphArchiveHandle,
|
|
2874
2645
|
WikiGraphArchiveManager,
|
|
2875
|
-
|
|
2876
|
-
|
|
2646
|
+
WikiGraphArchiveOwnershipError,
|
|
2647
|
+
inspectWikiGraphArchive
|
|
2877
2648
|
};
|
|
2878
2649
|
//# sourceMappingURL=archives.js.map
|