pikelet-wasm 0.10.1 → 0.11.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +146 -0
- package/README.md +58 -200
- package/complete/builder.mjs +1 -0
- package/complete/format.mjs +4 -2
- package/complete/index.d.ts +38 -0
- package/complete/index.mjs +235 -116
- package/complete/index.node.mjs +15 -0
- package/complete/layer-reader.mjs +16 -1
- package/complete/node-fetch.mjs +113 -0
- package/complete/rowmap.mjs +62 -0
- package/complete/sources.mjs +49 -6
- package/dist/engine.scalar.wasm +0 -0
- package/dist/engine.wasm +0 -0
- package/package.json +7 -1
package/complete/index.mjs
CHANGED
|
@@ -25,6 +25,7 @@ import { loadStudentModel, embedTextWithStudent } from './student-embedder.mjs';
|
|
|
25
25
|
import { computeMatchQuality, computePreSearchAbstention } from './student-abstention.mjs';
|
|
26
26
|
import { createAbstentionScorer } from './retrieval-abstention.mjs';
|
|
27
27
|
import { openLexicalIndex, openLexicalIndexLazy } from './lexical.mjs';
|
|
28
|
+
import { parseRowMapSegment } from './rowmap.mjs';
|
|
28
29
|
import { PikeletSketchArtifact } from '../src/artifact/pikelet-artifact.js';
|
|
29
30
|
import { MAGIC, HEADER_BYTES, TABLE_ENTRY_BYTES, KINDS, KIND_NAMES } from './format.mjs';
|
|
30
31
|
|
|
@@ -769,7 +770,7 @@ export async function openPikeletFile(input, options = {}) {
|
|
|
769
770
|
// influences a returned result. Small segments (kinds 1/2) keep the
|
|
770
771
|
// one-read eager path unchanged.
|
|
771
772
|
const qiDeferrable = qi.length > LAZY_QI_BYTES;
|
|
772
|
-
const [qiFirst, sketch, tables, lexicalBytes] = await Promise.all([
|
|
773
|
+
const [qiFirst, sketch, tables, lexicalBytes, rowMapBytes] = await Promise.all([
|
|
773
774
|
(async () => {
|
|
774
775
|
if (qiDeferrable) {
|
|
775
776
|
return readChecked(source, qi.offset, Math.min(16, qi.length), 'query-interp header', maxReadBytes, fileBytes);
|
|
@@ -813,6 +814,17 @@ export async function openPikeletFile(input, options = {}) {
|
|
|
813
814
|
}
|
|
814
815
|
return { bytes };
|
|
815
816
|
})() : null,
|
|
817
|
+
// Row map (kind 8, OPTIONAL): a few bytes per record, read
|
|
818
|
+
// eagerly and digest-verified. Present when the index holds
|
|
819
|
+
// several vectors per record (sentence windows).
|
|
820
|
+
segments.has('rowmap') ? (async () => {
|
|
821
|
+
const seg = segments.get('rowmap');
|
|
822
|
+
const bytes = await readChecked(source, seg.offset, seg.length, 'row map segment', maxReadBytes, fileBytes);
|
|
823
|
+
if (await sha256hex(bytes) !== seg.sha256) {
|
|
824
|
+
throw new Error('.pikelet row map segment failed hash verification');
|
|
825
|
+
}
|
|
826
|
+
return bytes;
|
|
827
|
+
})() : null,
|
|
816
828
|
]);
|
|
817
829
|
|
|
818
830
|
const lexicalIndex = lexicalBytes
|
|
@@ -1057,7 +1069,18 @@ export async function openPikeletFile(input, options = {}) {
|
|
|
1057
1069
|
|
|
1058
1070
|
// Index: the embedded sketch artifact, opened in the wave above
|
|
1059
1071
|
// against a segment-windowed source (resident hash verified there).
|
|
1060
|
-
|
|
1072
|
+
// The index binds to the corpus positionally (3.4) unless a row map
|
|
1073
|
+
// (3.9) says how its rows group into records; the manifest names the
|
|
1074
|
+
// policy that produced the rows, so a row map can neither be dropped
|
|
1075
|
+
// nor added without changing the identity.
|
|
1076
|
+
const rowMapPolicy = manifest.index?.rowMap ?? null;
|
|
1077
|
+
if (!!rowMapPolicy !== !!rowMapBytes) {
|
|
1078
|
+
throw new Error(rowMapPolicy
|
|
1079
|
+
? '.pikelet manifest declares index.rowMap but carries no row map segment'
|
|
1080
|
+
: '.pikelet carries a row map segment its manifest does not declare (index.rowMap)');
|
|
1081
|
+
}
|
|
1082
|
+
const rowMap = rowMapBytes ? parseRowMapSegment(rowMapBytes, declaredRecords, sketch.count) : null;
|
|
1083
|
+
if (!rowMap && sketch.count !== declaredRecords) {
|
|
1061
1084
|
throw new Error(`.pikelet index count ${sketch.count} != corpus records ${declaredRecords}`);
|
|
1062
1085
|
}
|
|
1063
1086
|
if (sketch.dim !== undefined && sketch.dim !== dim) {
|
|
@@ -1256,6 +1279,167 @@ export async function openPikeletFile(input, options = {}) {
|
|
|
1256
1279
|
? Math.max(manifest.recommendedGap, Number(source.preferredGapBytes) || 0)
|
|
1257
1280
|
: undefined;
|
|
1258
1281
|
|
|
1282
|
+
// The one retrieval path behind query(): resident sketch scan plus
|
|
1283
|
+
// BM25 candidates, exact rerank, rank fusion and phrase pinning.
|
|
1284
|
+
// Compile-time calibration fits on exactly this (through
|
|
1285
|
+
// __calibrationEvidence below), so the evidence a verdict was fit on
|
|
1286
|
+
// is the evidence query() scores. excludeIds removes records from
|
|
1287
|
+
// both candidate sources before either is cut to size — calibration
|
|
1288
|
+
// ablations use it; query() never does.
|
|
1289
|
+
const retrieveCandidates = async (trimmed, context, k, queryOptions, excludeIds = null) => {
|
|
1290
|
+
let fusedFull = null;
|
|
1291
|
+
// Hybrid retrieval when the artifact carries a lexical
|
|
1292
|
+
// segment: the BM25 top matches join the sketch's exact
|
|
1293
|
+
// rerank as extra candidates (scored by true vector
|
|
1294
|
+
// distance like any candidate — this is what recovers
|
|
1295
|
+
// known-item lookups the sketch scan's top-C missed),
|
|
1296
|
+
// and the final ordering fuses the distance ranking
|
|
1297
|
+
// with the BM25 ranking by reciprocal rank (RRF, k=60,
|
|
1298
|
+
// untuned). Abstention still scores the distance-sorted
|
|
1299
|
+
// top-k exactly as calibrated; fusion affects result
|
|
1300
|
+
// order only. fullRerankOutput returns every reranked
|
|
1301
|
+
// candidate (all already fetched and exactly scored) so
|
|
1302
|
+
// fusion sees the true distance of every lexical match.
|
|
1303
|
+
// Common query terms give every matching document a
|
|
1304
|
+
// small, near-tied BM25 score (idf collapses them);
|
|
1305
|
+
// that flat mass carries no ranking information and
|
|
1306
|
+
// dilutes rank fusion. Keep only hits within a factor
|
|
1307
|
+
// of the best lexical score — for a known-item lookup
|
|
1308
|
+
// that is typically the one document holding the rare
|
|
1309
|
+
// terms.
|
|
1310
|
+
// retrieval: 'hybrid' (default when the artifact
|
|
1311
|
+
// carries a lexical segment) | 'vector' | 'lexical' —
|
|
1312
|
+
// the last two exist for measurement and debugging
|
|
1313
|
+
// (retrieval-quality bakeoffs run all three on one
|
|
1314
|
+
// artifact). Abstention scores the distance-sorted
|
|
1315
|
+
// top-k identically in every mode.
|
|
1316
|
+
// 'augmented' unions the lexical candidates into the
|
|
1317
|
+
// exact rerank but keeps pure distance order — the mode
|
|
1318
|
+
// for workloads scored against exact nearest neighbors,
|
|
1319
|
+
// where RRF's reordering reads as loss by definition.
|
|
1320
|
+
const retrieval = queryOptions.retrieval ?? 'hybrid';
|
|
1321
|
+
if (!['hybrid', 'vector', 'lexical', 'augmented'].includes(retrieval)) {
|
|
1322
|
+
throw new Error(`query() retrieval must be hybrid, vector, lexical, or augmented, got ${retrieval}`);
|
|
1323
|
+
}
|
|
1324
|
+
// An excluded record must not take a candidate slot on either
|
|
1325
|
+
// side, so both sources skip it before cutting to size — BM25
|
|
1326
|
+
// before its top-n sort (scored with the full corpus's term
|
|
1327
|
+
// statistics, the one way this differs from a corpus without
|
|
1328
|
+
// those records) and the sketch before its top-C cut.
|
|
1329
|
+
const excluded = excludeIds && excludeIds.size ? excludeIds : null;
|
|
1330
|
+
let excludeBits = null;
|
|
1331
|
+
if (excluded) {
|
|
1332
|
+
excludeBits = new Uint8Array(Math.ceil(sketch.count / 8));
|
|
1333
|
+
for (const id of excluded) {
|
|
1334
|
+
// With a row map, excluding a record masks every row it owns.
|
|
1335
|
+
const [from, to] = rowMap ? [rowMap.offsets[id], rowMap.offsets[id + 1]] : [id, id + 1];
|
|
1336
|
+
for (let row = from; row < to; row++) excludeBits[row >> 3] |= 1 << (row & 7);
|
|
1337
|
+
}
|
|
1338
|
+
}
|
|
1339
|
+
const lexicalRaw = lexicalIndex && retrieval !== 'vector'
|
|
1340
|
+
? await lexicalIndex.search(trimmed, LEXICAL_CANDIDATES, excluded ? { exclude: (id) => excluded.has(id) } : {})
|
|
1341
|
+
: [];
|
|
1342
|
+
const lexicalHits = lexicalRaw.filter((h) => h.score >= lexicalRaw[0].score / LEXICAL_CUTOFF);
|
|
1343
|
+
// With a row map (3.9) the sketch searches rows: a record's lexical
|
|
1344
|
+
// match joins the rerank with every row it owns, k is scaled to the
|
|
1345
|
+
// mean rows per record, and the reranked rows collapse to records
|
|
1346
|
+
// below, each scored by its best row.
|
|
1347
|
+
const extraRows = rowMap
|
|
1348
|
+
? lexicalHits.flatMap((h) => Array.from({ length: rowMap.offsets[h.id + 1] - rowMap.offsets[h.id] }, (_, i) => rowMap.offsets[h.id] + i))
|
|
1349
|
+
: lexicalHits.map((h) => h.id);
|
|
1350
|
+
const searchK = rowMap ? Math.min(sketch.count, Math.ceil(k * sketch.count / recordCount)) : k;
|
|
1351
|
+
const rerankedRows = (await sketch.search(context.vector, searchK, {
|
|
1352
|
+
rerank: queryOptions.rerank,
|
|
1353
|
+
parallelism: queryOptions.parallelism ?? queryOptions.rerankParallelism ?? options.rerankParallelism,
|
|
1354
|
+
gap: queryOptions.gap ?? queryOptions.rerankGap ?? options.rerankGap ?? rerankGapHint,
|
|
1355
|
+
maxRangeBytes: queryOptions.maxRangeBytes ?? options.rerankMaxRangeBytes,
|
|
1356
|
+
// Present once background staging finished; the
|
|
1357
|
+
// sketch reader ignores it for any query whose C
|
|
1358
|
+
// exceeds the scanner's buffers.
|
|
1359
|
+
...(scanner ? { scanner } : {}),
|
|
1360
|
+
...(extraRows.length ? { extraCandidates: extraRows } : {}),
|
|
1361
|
+
...(excludeBits ? { exclude: excludeBits } : {}),
|
|
1362
|
+
// Always return the full reranked candidate list
|
|
1363
|
+
// (already fetched and exactly scored) rather than
|
|
1364
|
+
// truncating to k: abstention is fit at a fixed
|
|
1365
|
+
// top-10 window (calibrate.mjs) independent of the
|
|
1366
|
+
// caller's k, and needs that full window to score
|
|
1367
|
+
// consistently — see retrieval-abstention.mjs.
|
|
1368
|
+
fullRerankOutput: true,
|
|
1369
|
+
})).results;
|
|
1370
|
+
// The rerank output is distance-ordered, so a record's first row
|
|
1371
|
+
// is its best: keeping first occurrences gives the records in
|
|
1372
|
+
// order of their best row's distance.
|
|
1373
|
+
let searched = rerankedRows;
|
|
1374
|
+
if (rowMap) {
|
|
1375
|
+
const seen = new Set();
|
|
1376
|
+
searched = [];
|
|
1377
|
+
for (const row of rerankedRows) {
|
|
1378
|
+
const id = rowMap.owner[row.id];
|
|
1379
|
+
if (!seen.has(id)) { seen.add(id); searched.push({ id, distance: row.distance }); }
|
|
1380
|
+
}
|
|
1381
|
+
}
|
|
1382
|
+
const hits = searched.slice(0, k);
|
|
1383
|
+
// Full-window fused order (not k-truncated) for scoring:
|
|
1384
|
+
// coverage grounds its verdict in the passage fusion
|
|
1385
|
+
// actually ranks first, and calibrate.mjs fits it the
|
|
1386
|
+
// same way — a k-truncated fused list would silently
|
|
1387
|
+
// drop the fused top hit for callers requesting a small k.
|
|
1388
|
+
if (retrieval === 'lexical') {
|
|
1389
|
+
const byId = new Map(searched.map((hit) => [hit.id, hit]));
|
|
1390
|
+
fusedFull = lexicalHits.map((h) => byId.get(h.id)).filter(Boolean);
|
|
1391
|
+
} else if (retrieval !== 'augmented' && lexicalHits.length) {
|
|
1392
|
+
const lexRank = new Map(lexicalHits.map((h, i) => [h.id, i]));
|
|
1393
|
+
fusedFull = fuseCandidates(searched, lexicalHits.map((h) => h.id));
|
|
1394
|
+
// One batched hydration for everything this block and
|
|
1395
|
+
// scoreQuality's coverage term (below) will need, in
|
|
1396
|
+
// place of two separate staged fetches (phrase-pin's
|
|
1397
|
+
// own, then a later cold start for coverage): over a
|
|
1398
|
+
// remote range-read source each hydrate() a caller
|
|
1399
|
+
// hasn't already paid for is its own network round
|
|
1400
|
+
// trip, and sequential/staged awaits pay that cost
|
|
1401
|
+
// once per stage instead of once per query. The
|
|
1402
|
+
// union of phrase-pin's candidates (top lexical
|
|
1403
|
+
// hits) and coverage's (fusedFull's top few — phrase
|
|
1404
|
+
// pinning only ever reorders that window, never
|
|
1405
|
+
// changes its membership) is known right here, so
|
|
1406
|
+
// both stages can share one round trip. A record
|
|
1407
|
+
// that fails to hydrate here does not fail the
|
|
1408
|
+
// query: the id is just missing from the map, and
|
|
1409
|
+
// each site below falls through exactly as it would
|
|
1410
|
+
// on a cold cache miss (recs[i] undefined; coverage
|
|
1411
|
+
// sees fewer usable passages) — hydrate() raising
|
|
1412
|
+
// for a bad id is still a real failure, but only
|
|
1413
|
+
// once a site actually needs that specific record,
|
|
1414
|
+
// not speculatively for every candidate touched here.
|
|
1415
|
+
const phraseCandidates = lexicalHits.slice(0, 3);
|
|
1416
|
+
const coverageCandidates = fusedFull.slice(0, COVERAGE_BATCH_COUNT);
|
|
1417
|
+
const batchIds = [...new Set([...phraseCandidates, ...coverageCandidates].map((h) => h.id))];
|
|
1418
|
+
const batch = new Map();
|
|
1419
|
+
await Promise.all(batchIds.map(async (id) => {
|
|
1420
|
+
try { batch.set(id, await hydrate(id)); } catch { /* see comment above */ }
|
|
1421
|
+
}));
|
|
1422
|
+
// Phrase pinning: a verbatim occurrence of the query (>= 3
|
|
1423
|
+
// content words) in one of the top lexical hits is the
|
|
1424
|
+
// strongest evidence of support there is; rank it first.
|
|
1425
|
+
const norm = (t) => String(t || '').toLowerCase().replace(/[^a-z0-9' ]+/g, ' ').replace(/\s+/g, ' ').trim();
|
|
1426
|
+
const phrase = norm(trimmed);
|
|
1427
|
+
if (phrase.split(' ').length >= 3) {
|
|
1428
|
+
for (const h of phraseCandidates) {
|
|
1429
|
+
if (norm(batch.get(h.id)?.text).includes(phrase)) {
|
|
1430
|
+
fusedFull = [searched.find((x) => x.id === h.id), ...fusedFull.filter((x) => x.id !== h.id)].filter(Boolean);
|
|
1431
|
+
break;
|
|
1432
|
+
}
|
|
1433
|
+
}
|
|
1434
|
+
}
|
|
1435
|
+
// Rank transparency: a consumer can see a verbatim hit sitting
|
|
1436
|
+
// at fused rank 3 behind two semantic near-misses.
|
|
1437
|
+
for (const [i, hit] of fusedFull.entries()) { hit.fusedRank = i + 1; hit.lexicalRank = lexRank.has(hit.id) ? lexRank.get(hit.id) + 1 : null; hit.vectorRank = searched.findIndex((x) => x.id === hit.id) + 1; }
|
|
1438
|
+
}
|
|
1439
|
+
const fused = retrieval === 'augmented' ? null : fusedFull?.slice(0, k) ?? null;
|
|
1440
|
+
return { hits, fused, searched, fusedFull };
|
|
1441
|
+
};
|
|
1442
|
+
|
|
1259
1443
|
return {
|
|
1260
1444
|
info() {
|
|
1261
1445
|
return {
|
|
@@ -1283,6 +1467,11 @@ export async function openPikeletFile(input, options = {}) {
|
|
|
1283
1467
|
license: typeof manifest.corpus?.provenance?.license === 'string'
|
|
1284
1468
|
? manifest.corpus.provenance.license : null,
|
|
1285
1469
|
records: recordCount,
|
|
1470
|
+
// Index rows and the policy that produced them when the
|
|
1471
|
+
// index holds several vectors per record (row map, 3.9);
|
|
1472
|
+
// otherwise one row per record and rowMap is null.
|
|
1473
|
+
indexRows: sketch.count,
|
|
1474
|
+
rowMap: rowMapPolicy ? { ...rowMapPolicy } : null,
|
|
1286
1475
|
dim,
|
|
1287
1476
|
metric: manifest.metric,
|
|
1288
1477
|
encoder: encoderInfo,
|
|
@@ -1316,6 +1505,16 @@ export async function openPikeletFile(input, options = {}) {
|
|
|
1316
1505
|
}
|
|
1317
1506
|
: null,
|
|
1318
1507
|
sampleQueries: Array.isArray(manifest.sampleQueries) ? manifest.sampleQueries : [],
|
|
1508
|
+
// Where each part of the file lives, in file order: the
|
|
1509
|
+
// header (fixed header, manifest and segment table, up to
|
|
1510
|
+
// the first segment's aligned offset) and every segment of
|
|
1511
|
+
// a known kind. Lets a viewer attribute range reads to
|
|
1512
|
+
// what they fetched.
|
|
1513
|
+
layout: [
|
|
1514
|
+
{ kind: 'header', offset: 0, length: align16(HEADER_BYTES + manifestBytes + tableBytes) },
|
|
1515
|
+
...[...segments].map(([kind, seg]) => ({ kind, offset: seg.offset, length: seg.length }))
|
|
1516
|
+
.sort((a, b) => a.offset - b.offset),
|
|
1517
|
+
],
|
|
1319
1518
|
};
|
|
1320
1519
|
},
|
|
1321
1520
|
|
|
@@ -1341,120 +1540,7 @@ export async function openPikeletFile(input, options = {}) {
|
|
|
1341
1540
|
let fused = null;
|
|
1342
1541
|
let searched = null;
|
|
1343
1542
|
let fusedFull = null;
|
|
1344
|
-
if (!pre) {
|
|
1345
|
-
// Hybrid retrieval when the artifact carries a lexical
|
|
1346
|
-
// segment: the BM25 top matches join the sketch's exact
|
|
1347
|
-
// rerank as extra candidates (scored by true vector
|
|
1348
|
-
// distance like any candidate — this is what recovers
|
|
1349
|
-
// known-item lookups the sketch scan's top-C missed),
|
|
1350
|
-
// and the final ordering fuses the distance ranking
|
|
1351
|
-
// with the BM25 ranking by reciprocal rank (RRF, k=60,
|
|
1352
|
-
// untuned). Abstention still scores the distance-sorted
|
|
1353
|
-
// top-k exactly as calibrated; fusion affects result
|
|
1354
|
-
// order only. fullRerankOutput returns every reranked
|
|
1355
|
-
// candidate (all already fetched and exactly scored) so
|
|
1356
|
-
// fusion sees the true distance of every lexical match.
|
|
1357
|
-
// Common query terms give every matching document a
|
|
1358
|
-
// small, near-tied BM25 score (idf collapses them);
|
|
1359
|
-
// that flat mass carries no ranking information and
|
|
1360
|
-
// dilutes rank fusion. Keep only hits within a factor
|
|
1361
|
-
// of the best lexical score — for a known-item lookup
|
|
1362
|
-
// that is typically the one document holding the rare
|
|
1363
|
-
// terms.
|
|
1364
|
-
// retrieval: 'hybrid' (default when the artifact
|
|
1365
|
-
// carries a lexical segment) | 'vector' | 'lexical' —
|
|
1366
|
-
// the last two exist for measurement and debugging
|
|
1367
|
-
// (retrieval-quality bakeoffs run all three on one
|
|
1368
|
-
// artifact). Abstention scores the distance-sorted
|
|
1369
|
-
// top-k identically in every mode.
|
|
1370
|
-
// 'augmented' unions the lexical candidates into the
|
|
1371
|
-
// exact rerank but keeps pure distance order — the mode
|
|
1372
|
-
// for workloads scored against exact nearest neighbors,
|
|
1373
|
-
// where RRF's reordering reads as loss by definition.
|
|
1374
|
-
const retrieval = queryOptions.retrieval ?? 'hybrid';
|
|
1375
|
-
if (!['hybrid', 'vector', 'lexical', 'augmented'].includes(retrieval)) {
|
|
1376
|
-
throw new Error(`query() retrieval must be hybrid, vector, lexical, or augmented, got ${retrieval}`);
|
|
1377
|
-
}
|
|
1378
|
-
const lexicalRaw = lexicalIndex && retrieval !== 'vector'
|
|
1379
|
-
? await lexicalIndex.search(trimmed, LEXICAL_CANDIDATES) : [];
|
|
1380
|
-
const lexicalHits = lexicalRaw.filter((h) => h.score >= lexicalRaw[0].score / LEXICAL_CUTOFF);
|
|
1381
|
-
searched = (await sketch.search(context.vector, k, {
|
|
1382
|
-
rerank: queryOptions.rerank,
|
|
1383
|
-
parallelism: queryOptions.parallelism ?? queryOptions.rerankParallelism ?? options.rerankParallelism,
|
|
1384
|
-
gap: queryOptions.gap ?? queryOptions.rerankGap ?? options.rerankGap ?? rerankGapHint,
|
|
1385
|
-
maxRangeBytes: queryOptions.maxRangeBytes ?? options.rerankMaxRangeBytes,
|
|
1386
|
-
// Present once background staging finished; the
|
|
1387
|
-
// sketch reader ignores it for any query whose C
|
|
1388
|
-
// exceeds the scanner's buffers.
|
|
1389
|
-
...(scanner ? { scanner } : {}),
|
|
1390
|
-
...(lexicalHits.length ? { extraCandidates: lexicalHits.map((h) => h.id) } : {}),
|
|
1391
|
-
// Always return the full reranked candidate list
|
|
1392
|
-
// (already fetched and exactly scored) rather than
|
|
1393
|
-
// truncating to k: abstention is fit at a fixed
|
|
1394
|
-
// top-10 window (calibrate.mjs) independent of the
|
|
1395
|
-
// caller's k, and needs that full window to score
|
|
1396
|
-
// consistently — see retrieval-abstention.mjs.
|
|
1397
|
-
fullRerankOutput: true,
|
|
1398
|
-
})).results;
|
|
1399
|
-
hits = searched.slice(0, k);
|
|
1400
|
-
// Full-window fused order (not k-truncated) for scoring:
|
|
1401
|
-
// coverage grounds its verdict in the passage fusion
|
|
1402
|
-
// actually ranks first, and calibrate.mjs fits it the
|
|
1403
|
-
// same way — a k-truncated fused list would silently
|
|
1404
|
-
// drop the fused top hit for callers requesting a small k.
|
|
1405
|
-
if (retrieval === 'lexical') {
|
|
1406
|
-
const byId = new Map(searched.map((hit) => [hit.id, hit]));
|
|
1407
|
-
fusedFull = lexicalHits.map((h) => byId.get(h.id)).filter(Boolean);
|
|
1408
|
-
} else if (retrieval !== 'augmented' && lexicalHits.length) {
|
|
1409
|
-
const lexRank = new Map(lexicalHits.map((h, i) => [h.id, i]));
|
|
1410
|
-
fusedFull = fuseCandidates(searched, lexicalHits.map((h) => h.id));
|
|
1411
|
-
// One batched hydration for everything this block and
|
|
1412
|
-
// scoreQuality's coverage term (below) will need, in
|
|
1413
|
-
// place of two separate staged fetches (phrase-pin's
|
|
1414
|
-
// own, then a later cold start for coverage): over a
|
|
1415
|
-
// remote range-read source each hydrate() a caller
|
|
1416
|
-
// hasn't already paid for is its own network round
|
|
1417
|
-
// trip, and sequential/staged awaits pay that cost
|
|
1418
|
-
// once per stage instead of once per query. The
|
|
1419
|
-
// union of phrase-pin's candidates (top lexical
|
|
1420
|
-
// hits) and coverage's (fusedFull's top few — phrase
|
|
1421
|
-
// pinning only ever reorders that window, never
|
|
1422
|
-
// changes its membership) is known right here, so
|
|
1423
|
-
// both stages can share one round trip. A record
|
|
1424
|
-
// that fails to hydrate here does not fail the
|
|
1425
|
-
// query: the id is just missing from the map, and
|
|
1426
|
-
// each site below falls through exactly as it would
|
|
1427
|
-
// on a cold cache miss (recs[i] undefined; coverage
|
|
1428
|
-
// sees fewer usable passages) — hydrate() raising
|
|
1429
|
-
// for a bad id is still a real failure, but only
|
|
1430
|
-
// once a site actually needs that specific record,
|
|
1431
|
-
// not speculatively for every candidate touched here.
|
|
1432
|
-
const phraseCandidates = lexicalHits.slice(0, 3);
|
|
1433
|
-
const coverageCandidates = fusedFull.slice(0, COVERAGE_BATCH_COUNT);
|
|
1434
|
-
const batchIds = [...new Set([...phraseCandidates, ...coverageCandidates].map((h) => h.id))];
|
|
1435
|
-
const batch = new Map();
|
|
1436
|
-
await Promise.all(batchIds.map(async (id) => {
|
|
1437
|
-
try { batch.set(id, await hydrate(id)); } catch { /* see comment above */ }
|
|
1438
|
-
}));
|
|
1439
|
-
// Phrase pinning: a verbatim occurrence of the query (>= 3
|
|
1440
|
-
// content words) in one of the top lexical hits is the
|
|
1441
|
-
// strongest evidence of support there is; rank it first.
|
|
1442
|
-
const norm = (t) => String(t || '').toLowerCase().replace(/[^a-z0-9' ]+/g, ' ').replace(/\s+/g, ' ').trim();
|
|
1443
|
-
const phrase = norm(trimmed);
|
|
1444
|
-
if (phrase.split(' ').length >= 3) {
|
|
1445
|
-
for (const h of phraseCandidates) {
|
|
1446
|
-
if (norm(batch.get(h.id)?.text).includes(phrase)) {
|
|
1447
|
-
fusedFull = [searched.find((x) => x.id === h.id), ...fusedFull.filter((x) => x.id !== h.id)].filter(Boolean);
|
|
1448
|
-
break;
|
|
1449
|
-
}
|
|
1450
|
-
}
|
|
1451
|
-
}
|
|
1452
|
-
// Rank transparency: a consumer can see a verbatim hit sitting
|
|
1453
|
-
// at fused rank 3 behind two semantic near-misses.
|
|
1454
|
-
for (const [i, hit] of fusedFull.entries()) { hit.fusedRank = i + 1; hit.lexicalRank = lexRank.has(hit.id) ? lexRank.get(hit.id) + 1 : null; hit.vectorRank = searched.findIndex((x) => x.id === hit.id) + 1; }
|
|
1455
|
-
}
|
|
1456
|
-
fused = retrieval === 'augmented' ? null : fusedFull?.slice(0, k) ?? null;
|
|
1457
|
-
}
|
|
1543
|
+
if (!pre) ({ hits, fused, searched, fusedFull } = await retrieveCandidates(trimmed, context, k, queryOptions));
|
|
1458
1544
|
// Abstention scores the full reranked candidate window
|
|
1459
1545
|
// (searched), not the k-truncated hits: calibrate.mjs fits
|
|
1460
1546
|
// its signals at a fixed top-10 (top-5 for coverage)
|
|
@@ -1490,12 +1576,42 @@ export async function openPikeletFile(input, options = {}) {
|
|
|
1490
1576
|
return { matchQuality: quality.match_quality, confidence: quality.confidence, results, ...(grounding ? { grounding } : {}) };
|
|
1491
1577
|
},
|
|
1492
1578
|
|
|
1579
|
+
/**
|
|
1580
|
+
* The retrieval evidence query() scores, for compile-time
|
|
1581
|
+
* calibration: `vector` is the distance-ordered rerank window
|
|
1582
|
+
* d0/margin/mean10 read, `fused` the order coverage reads its
|
|
1583
|
+
* passages from. Calibration consumes this instead of
|
|
1584
|
+
* reimplementing retrieval, so a verdict is fit on the evidence
|
|
1585
|
+
* it is later scored on. Not part of the documented reader API.
|
|
1586
|
+
*/
|
|
1587
|
+
async __calibrationEvidence(text, { excludeIds = null } = {}) {
|
|
1588
|
+
assertOpen();
|
|
1589
|
+
const trimmed = String(text || '').trim();
|
|
1590
|
+
if (!trimmed) throw new Error('query text is required');
|
|
1591
|
+
// The 10-wide window calibration fits at; any k up to the
|
|
1592
|
+
// pack's rerank depth gives query() the same candidates.
|
|
1593
|
+
const k = Math.min(10, Math.max(recordCount, 1));
|
|
1594
|
+
const { searched, fusedFull } = await retrieveCandidates(trimmed, await embed(trimmed), k, {}, excludeIds);
|
|
1595
|
+
const plain = (list) => list.map((hit) => ({ id: hit.id, distance: hit.distance }));
|
|
1596
|
+
return { vector: plain(searched), fused: plain(fusedFull?.length ? fusedFull : searched) };
|
|
1597
|
+
},
|
|
1598
|
+
|
|
1493
1599
|
/** Hydrate one corpus record by id (verified per record on format 2). */
|
|
1494
1600
|
async record(id) {
|
|
1495
1601
|
assertOpen();
|
|
1496
1602
|
return hydrate(id);
|
|
1497
1603
|
},
|
|
1498
1604
|
|
|
1605
|
+
/** The absolute byte range one corpus record's stored bytes occupy in the file. */
|
|
1606
|
+
recordRange(id) {
|
|
1607
|
+
assertOpen();
|
|
1608
|
+
if (!Number.isSafeInteger(id) || id < 0 || id >= recordCount) {
|
|
1609
|
+
throw new Error(`.pikelet record id ${id} is outside the corpus`);
|
|
1610
|
+
}
|
|
1611
|
+
const start = recordOffsets[id];
|
|
1612
|
+
return { offset: corpus.offset + start, length: recordOffsets[id + 1] - start };
|
|
1613
|
+
},
|
|
1614
|
+
|
|
1499
1615
|
/**
|
|
1500
1616
|
* Full pass over the index's lazy vector rows against the
|
|
1501
1617
|
* identity-anchored vectorsSha256 (streamed where possible).
|
|
@@ -1572,6 +1688,9 @@ export async function openPikeletFile(input, options = {}) {
|
|
|
1572
1688
|
recordCount,
|
|
1573
1689
|
dim,
|
|
1574
1690
|
sketch,
|
|
1691
|
+
// A chain maps sketch rows to records positionally, so
|
|
1692
|
+
// it needs to know when a member's rows are not records.
|
|
1693
|
+
rowMap,
|
|
1575
1694
|
lexicalIndex,
|
|
1576
1695
|
hydrate,
|
|
1577
1696
|
readRecordBytes,
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
// The Node entry of 'pikelet-wasm/complete': the portable reader, except
|
|
2
|
+
// that httpRangeSource defaults to nodeFetch (node:http/node:https, no
|
|
3
|
+
// automatic decompression, Content-Encoding other than identity refused)
|
|
4
|
+
// instead of the runtime's fetch, which decodes whatever a server labels as
|
|
5
|
+
// encoded. Pass options.fetch to choose otherwise.
|
|
6
|
+
|
|
7
|
+
import { httpRangeSource as portableHttpRangeSource } from './sources.mjs';
|
|
8
|
+
import { nodeFetch } from './node-fetch.mjs';
|
|
9
|
+
|
|
10
|
+
export * from './index.mjs';
|
|
11
|
+
export { nodeFetch };
|
|
12
|
+
|
|
13
|
+
export function httpRangeSource(url, options = {}) {
|
|
14
|
+
return portableHttpRangeSource(url, { fetch: nodeFetch, ...options });
|
|
15
|
+
}
|
|
@@ -56,7 +56,9 @@ export const CHAIN_DEFAULTS = Object.freeze({
|
|
|
56
56
|
* [base, layer1, layer2, ...]. Every member is opened through
|
|
57
57
|
* openPikeletFile(), so a string is a path and an object is a range source.
|
|
58
58
|
* @param {object} [options] forwarded to each member's open, plus:
|
|
59
|
-
* readerDriftLimit, maxDepth
|
|
59
|
+
* readerDriftLimit, maxDepth, and fetch (the transport for any parent the
|
|
60
|
+
* walk reaches over HTTP; default: the child source's own, else
|
|
61
|
+
* globalThis.fetch).
|
|
60
62
|
*/
|
|
61
63
|
/**
|
|
62
64
|
* 5.1.1: walk from a head to its base, resolving each parent by lineage
|
|
@@ -76,6 +78,7 @@ async function walkToBase(head, options) {
|
|
|
76
78
|
const chain = [head];
|
|
77
79
|
let cursor = head;
|
|
78
80
|
let expectIdentity = null;
|
|
81
|
+
let childFetch = null;
|
|
79
82
|
for (let hop = 0; hop <= maxDepth; hop++) {
|
|
80
83
|
const peek = await readMemberManifest(cursor, options);
|
|
81
84
|
if (expectIdentity !== null && peek.identity !== expectIdentity) {
|
|
@@ -89,6 +92,7 @@ async function walkToBase(head, options) {
|
|
|
89
92
|
// string as a file path sent that second hop to fileSource.
|
|
90
93
|
const childLocation = typeof cursor === 'string' ? cursor : (cursor.url ?? cursor.location ?? null);
|
|
91
94
|
const childIsFile = typeof cursor === 'string' && !isHttpUrl(cursor);
|
|
95
|
+
if (typeof cursor === 'object' && typeof cursor?.fetch === 'function') childFetch = cursor.fetch;
|
|
92
96
|
const located = await resolveParentLocation(peek.parent, {
|
|
93
97
|
lineage,
|
|
94
98
|
childLocation,
|
|
@@ -112,6 +116,9 @@ async function walkToBase(head, options) {
|
|
|
112
116
|
// may name any origin, so its redirects are not confined.
|
|
113
117
|
const confineTo = located.via === 'locator' ? childLocation : null;
|
|
114
118
|
cursor = httpRangeSource(cursor, {
|
|
119
|
+
// The walk's transport: the caller's, else the one the
|
|
120
|
+
// child source was opened with.
|
|
121
|
+
...((options.fetch || childFetch) ? { fetch: options.fetch || childFetch } : {}),
|
|
115
122
|
...(confineTo ? { redirectGuard: (target) => assertConfined(target, confineTo) } : {}),
|
|
116
123
|
});
|
|
117
124
|
await cursor.init();
|
|
@@ -200,6 +207,14 @@ export async function openPikeletChain(members, options = {}) {
|
|
|
200
207
|
if (base.manifest.profile === LAYER_PROFILE) {
|
|
201
208
|
throw new Error('the first chain member is a layer, not a base: a layer opened alone would serve a fraction of a corpus (4.1)');
|
|
202
209
|
}
|
|
210
|
+
// A chain maps each member's sketch rows to records positionally
|
|
211
|
+
// (rowBase + local id). A member whose index holds several rows per
|
|
212
|
+
// record (row map, COMPLETE_PROFILE.md 3.9) would be misread, so it is
|
|
213
|
+
// refused until the layered profile defines rows.
|
|
214
|
+
if (tiersRaw.some((t) => t && t.rowMap)) {
|
|
215
|
+
throw new Error('a pack whose index holds several rows per record (sentence windows) cannot be part of a layered chain yet: '
|
|
216
|
+
+ 'mount it on its own, or rebuild it without --sentence-windows to append to it');
|
|
217
|
+
}
|
|
203
218
|
|
|
204
219
|
const chainMembers = [{
|
|
205
220
|
depth: 0, identity: base.identity, rowBase: 0, records: base.recordCount,
|
|
@@ -0,0 +1,113 @@
|
|
|
1
|
+
// A fetch-shaped wrapper over node:http/node:https, for Node callers of the
|
|
2
|
+
// range source (and every other network read the pikelet CLI makes). The
|
|
3
|
+
// Node entry of 'pikelet-wasm/complete' makes it httpRangeSource's default.
|
|
4
|
+
//
|
|
5
|
+
// It exists to keep pack bytes away from the runtime's response decoder.
|
|
6
|
+
// Node's built-in fetch decompresses any body a server labels with a
|
|
7
|
+
// Content-Encoding; https.request does no automatic decompression, so the
|
|
8
|
+
// decoder never runs. On top of that it sends Accept-Encoding: identity and
|
|
9
|
+
// fails closed on any other Content-Encoding: a pack is content-addressed
|
|
10
|
+
// bytes, and a compressed range response cannot be sliced anyway. The one
|
|
11
|
+
// opt-in exception is options.inflateGzip, for HTML a crawler reads from a
|
|
12
|
+
// server that compresses whatever it was asked: gzip and deflate only,
|
|
13
|
+
// inflated by node:zlib, never Brotli or zstd.
|
|
14
|
+
//
|
|
15
|
+
// Only what the callers use is implemented: GET/HEAD without a request
|
|
16
|
+
// body, a headers object (or Headers), an AbortSignal, and the three
|
|
17
|
+
// redirect modes -- 'follow' for plain reads, 'manual' for the range
|
|
18
|
+
// source's HEAD walk (which reads Location itself), and 'error' for reads
|
|
19
|
+
// under a redirectGuard. The result is a real Response, so callers keep
|
|
20
|
+
// using status/ok/headers/body/arrayBuffer/text unchanged; an error on the
|
|
21
|
+
// response stream rejects those rather than hanging.
|
|
22
|
+
|
|
23
|
+
import http from 'node:http';
|
|
24
|
+
import https from 'node:https';
|
|
25
|
+
import zlib from 'node:zlib';
|
|
26
|
+
import { Readable } from 'node:stream';
|
|
27
|
+
|
|
28
|
+
const REDIRECT_STATUSES = new Set([301, 302, 303, 307, 308]);
|
|
29
|
+
const MAX_REDIRECTS = 20;
|
|
30
|
+
// Shared keep-alive agents bounded at the range source's default
|
|
31
|
+
// parallelism: connections are reused across a query's reads, and a burst
|
|
32
|
+
// opens at most 32 sockets per host instead of one per read (GitHub's
|
|
33
|
+
// release CDN answers unbounded bursts with 429s).
|
|
34
|
+
const MAX_SOCKETS = 32;
|
|
35
|
+
const agents = {
|
|
36
|
+
'http:': new http.Agent({ keepAlive: true, maxSockets: MAX_SOCKETS }),
|
|
37
|
+
'https:': new https.Agent({ keepAlive: true, maxSockets: MAX_SOCKETS }),
|
|
38
|
+
};
|
|
39
|
+
|
|
40
|
+
function requestOnce(url, { method, headers, signal }) {
|
|
41
|
+
const transport = url.protocol === 'https:' ? https : url.protocol === 'http:' ? http : null;
|
|
42
|
+
if (!transport) return Promise.reject(new TypeError(`unsupported protocol ${url.protocol} in ${url.href}`));
|
|
43
|
+
return new Promise((resolve, reject) => {
|
|
44
|
+
const req = transport.request(url, { method, headers, signal, agent: agents[url.protocol] }, resolve);
|
|
45
|
+
req.on('error', reject);
|
|
46
|
+
req.end();
|
|
47
|
+
});
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
// Content-Encoding is a list, applied in order; identity entries are no-ops.
|
|
51
|
+
export function contentCodings(value) {
|
|
52
|
+
return String(value || '').split(',').map((c) => c.trim().toLowerCase()).filter((c) => c && c !== 'identity');
|
|
53
|
+
}
|
|
54
|
+
|
|
55
|
+
function toResponse(res, method, body = res) {
|
|
56
|
+
const headers = new Headers();
|
|
57
|
+
for (const [name, value] of Object.entries(res.headers)) {
|
|
58
|
+
if (name === 'set-cookie') continue;
|
|
59
|
+
headers.set(name, Array.isArray(value) ? value.join(', ') : String(value));
|
|
60
|
+
}
|
|
61
|
+
if (body !== res) {
|
|
62
|
+
// Inflated: the encoded length and coding no longer describe the body.
|
|
63
|
+
headers.delete('content-encoding');
|
|
64
|
+
headers.delete('content-length');
|
|
65
|
+
}
|
|
66
|
+
// Null-body statuses: new Response() throws if handed a stream for them.
|
|
67
|
+
const bodyless = method === 'HEAD' || [204, 205, 304].includes(res.statusCode);
|
|
68
|
+
if (bodyless) res.resume();
|
|
69
|
+
return new Response(bodyless ? null : Readable.toWeb(body), {
|
|
70
|
+
status: res.statusCode,
|
|
71
|
+
statusText: res.statusMessage || '',
|
|
72
|
+
headers,
|
|
73
|
+
});
|
|
74
|
+
}
|
|
75
|
+
|
|
76
|
+
export async function nodeFetch(input, init = {}) {
|
|
77
|
+
const method = String(init.method || 'GET').toUpperCase();
|
|
78
|
+
if (method !== 'GET' && method !== 'HEAD') throw new TypeError(`nodeFetch supports GET and HEAD, not ${method}`);
|
|
79
|
+
const redirect = init.redirect || 'follow';
|
|
80
|
+
const headers = Object.fromEntries(new Headers(init.headers || {}));
|
|
81
|
+
headers['accept-encoding'] = 'identity';
|
|
82
|
+
headers['user-agent'] ??= 'pikelet';
|
|
83
|
+
let url = new URL(String(input));
|
|
84
|
+
for (let hop = 0; ; hop++) {
|
|
85
|
+
const res = await requestOnce(url, { method, headers, signal: init.signal });
|
|
86
|
+
const location = res.headers.location;
|
|
87
|
+
if (REDIRECT_STATUSES.has(res.statusCode) && location) {
|
|
88
|
+
res.resume();
|
|
89
|
+
if (redirect === 'manual') return toResponse(res, 'HEAD');
|
|
90
|
+
if (redirect === 'error') throw new TypeError(`redirect refused (redirect: 'error'): ${url.href} -> ${location}`);
|
|
91
|
+
if (hop >= MAX_REDIRECTS) throw new TypeError(`too many redirects from ${String(input)}`);
|
|
92
|
+
const next = new URL(location, url);
|
|
93
|
+
if (url.protocol === 'https:' && next.protocol !== 'https:') {
|
|
94
|
+
throw new TypeError(`refused redirect from https to ${next.protocol.slice(0, -1)}: ${url.href} -> ${next.href}`);
|
|
95
|
+
}
|
|
96
|
+
// Request headers (Range, If-Range) ride along unchanged: a signed-URL
|
|
97
|
+
// hop serves the same range the front door was asked for.
|
|
98
|
+
url = next;
|
|
99
|
+
continue;
|
|
100
|
+
}
|
|
101
|
+
const codings = contentCodings(res.headers['content-encoding']);
|
|
102
|
+
if (codings.length === 0) return toResponse(res, method);
|
|
103
|
+
if (init.inflateGzip && codings.length === 1 && ['gzip', 'x-gzip', 'deflate'].includes(codings[0])) {
|
|
104
|
+
if (method === 'HEAD') return toResponse(res, method);
|
|
105
|
+
const inflate = codings[0] === 'deflate' ? zlib.createInflate() : zlib.createGunzip();
|
|
106
|
+
res.on('error', (err) => inflate.destroy(err));
|
|
107
|
+
return toResponse(res, method, res.pipe(inflate));
|
|
108
|
+
}
|
|
109
|
+
res.destroy();
|
|
110
|
+
throw new Error(`${url.href} answered with Content-Encoding: ${res.headers['content-encoding']} after Accept-Encoding: identity; `
|
|
111
|
+
+ 'refusing encoded bytes (the server must serve this file unencoded)');
|
|
112
|
+
}
|
|
113
|
+
}
|