pikelet-wasm 0.7.0 → 0.8.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +19 -3
- package/NOTICE +5 -0
- package/README.md +397 -454
- package/complete/builder.d.ts +4 -4
- package/complete/builder.mjs +8 -8
- package/complete/encoder-kernels/encoder.node.wasm +0 -0
- package/complete/encoder-kernels/encoder.wasm +0 -0
- package/complete/index.d.ts +11 -7
- package/complete/index.mjs +161 -112
- package/complete/inline-transformer.mjs +22 -9
- package/complete/lexical.mjs +11 -11
- package/complete/retrieval-abstention.mjs +18 -6
- package/complete/sources.mjs +22 -1
- package/dist/engine.js +1 -1
- package/dist/engine.scalar.js +1 -1
- package/dist/engine.scalar.wasm +0 -0
- package/dist/engine.wasm +0 -0
- package/package.json +58 -51
- package/{pancake-artifact-common.js → pikelet-artifact-common.js} +42 -42
- package/{pancake-artifact-range.js → pikelet-artifact-range.js} +36 -36
- package/{pancake-artifact-sketch.js → pikelet-artifact-sketch.js} +76 -66
- package/{pancake-artifact.d.ts → pikelet-artifact.d.ts} +11 -11
- package/{pancake-artifact.js → pikelet-artifact.js} +10 -10
- package/{pancake-core.js → pikelet-core.js} +136 -136
- package/{pancake-errors.js → pikelet-errors.js} +9 -8
- package/{pancake.d.ts → pikelet.d.ts} +55 -55
- package/{pancake.js → pikelet.js} +44 -44
- package/{pancake.node.mjs → pikelet.node.mjs} +46 -46
- package/{pancake.web.d.ts → pikelet.web.d.ts} +13 -13
- package/{pancake.web.mjs → pikelet.web.mjs} +20 -20
- package/{pancake.workerd.mjs → pikelet.workerd.mjs} +18 -18
- /package/{pancake-loader.js → pikelet-loader.js} +0 -0
package/complete/builder.d.ts
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
/**
|
|
2
|
-
* pikelet-wasm/complete/builder — Node-only assembly of complete .
|
|
2
|
+
* pikelet-wasm/complete/builder — Node-only assembly of complete .pikelet
|
|
3
3
|
* artifacts (spec/COMPLETE_PROFILE.md). The wire-format constants here are
|
|
4
4
|
* the same objects the reader uses (complete/format.mjs).
|
|
5
5
|
*/
|
|
@@ -15,8 +15,8 @@ export declare function canonicalJson(value: unknown): string;
|
|
|
15
15
|
export declare const align16: (n: number) => number;
|
|
16
16
|
|
|
17
17
|
/** Manifest profile strings and the header formatVersion each one implies. */
|
|
18
|
-
export declare const PROFILE_V1: '
|
|
19
|
-
export declare const PROFILE_V2: '
|
|
18
|
+
export declare const PROFILE_V1: 'pikelet-complete-v1';
|
|
19
|
+
export declare const PROFILE_V2: 'pikelet-complete-v2';
|
|
20
20
|
export declare const FORMAT_VERSIONS: Record<string, number>;
|
|
21
21
|
/** Corpus layout name for format 2 (per-record digests behind a page table). */
|
|
22
22
|
export declare const CORPUS_LAYOUT_V2: 'records-v2';
|
|
@@ -52,7 +52,7 @@ export declare function buildInlineTransformerEncoderSegment(input: {
|
|
|
52
52
|
* buildCorpusSegment() corpus); mismatches throw. Segments with a kind this
|
|
53
53
|
* spec does not name must carry an explicit kindNumber (readers skip them).
|
|
54
54
|
*/
|
|
55
|
-
export declare function
|
|
55
|
+
export declare function assemblePikeletFile(
|
|
56
56
|
manifestFields: Record<string, unknown> & { profile: string; corpus: { records: number } & Record<string, unknown> },
|
|
57
57
|
segments: Array<{ kind: string; bytes: Buffer | Uint8Array; kindNumber?: number }>,
|
|
58
58
|
outPath: string
|
package/complete/builder.mjs
CHANGED
|
@@ -228,7 +228,7 @@ export function buildLexicalSegment(texts) {
|
|
|
228
228
|
};
|
|
229
229
|
}
|
|
230
230
|
|
|
231
|
-
// Corpus layout v1 (format 1, profile
|
|
231
|
+
// Corpus layout v1 (format 1, profile pikelet-complete-v1): count + offsets +
|
|
232
232
|
// records, integrity by whole-segment digest only. Kept for producing
|
|
233
233
|
// format-1 files (compatibility fixtures, readers that predate format 2);
|
|
234
234
|
// new artifacts use buildCorpusSegment() below.
|
|
@@ -248,7 +248,7 @@ export function buildCorpusSegmentFromBuffers(records) {
|
|
|
248
248
|
return out;
|
|
249
249
|
}
|
|
250
250
|
|
|
251
|
-
// Corpus layout v2 (format 2, profile
|
|
251
|
+
// Corpus layout v2 (format 2, profile pikelet-complete-v2): per-record
|
|
252
252
|
// integrity that a reader can check on the single range read that hydrates
|
|
253
253
|
// a record, without fetching the rest of the segment (contract 4.1 / 7):
|
|
254
254
|
//
|
|
@@ -352,7 +352,7 @@ export async function measureRecommendedRerank({
|
|
|
352
352
|
async read(offset, length) { return bytes.subarray(offset, offset + length); },
|
|
353
353
|
async close() {},
|
|
354
354
|
};
|
|
355
|
-
const sketch = await artifactModule.
|
|
355
|
+
const sketch = await artifactModule.PikeletSketchArtifact.open(source);
|
|
356
356
|
try {
|
|
357
357
|
const count = sketch.count;
|
|
358
358
|
if (count === 0) throw new Error('cannot measure rerank on an empty sketch');
|
|
@@ -432,16 +432,16 @@ function selfQueriesFromSnapshot(graph, maxQueries) {
|
|
|
432
432
|
return queries;
|
|
433
433
|
}
|
|
434
434
|
|
|
435
|
-
// Container format version follows the profile string:
|
|
436
|
-
// files carry corpus layout v1 and header version 1;
|
|
435
|
+
// Container format version follows the profile string: pikelet-complete-v1
|
|
436
|
+
// files carry corpus layout v1 and header version 1; pikelet-complete-v2
|
|
437
437
|
// files carry corpus layout records-v2 (buildCorpusSegment) and header
|
|
438
438
|
// version 2, so readers that predate per-record integrity reject them
|
|
439
439
|
// explicitly instead of misreading the corpus tables.
|
|
440
|
-
export const PROFILE_V1 = '
|
|
441
|
-
export const PROFILE_V2 = '
|
|
440
|
+
export const PROFILE_V1 = 'pikelet-complete-v1';
|
|
441
|
+
export const PROFILE_V2 = 'pikelet-complete-v2';
|
|
442
442
|
export const FORMAT_VERSIONS = { [PROFILE_V1]: 1, [PROFILE_V2]: 2 };
|
|
443
443
|
|
|
444
|
-
export function
|
|
444
|
+
export function assemblePikeletFile(manifestFields, segments, outPath) {
|
|
445
445
|
const formatVersion = FORMAT_VERSIONS[manifestFields.profile];
|
|
446
446
|
if (!formatVersion) {
|
|
447
447
|
throw new Error(`manifest.profile must be ${PROFILE_V1} or ${PROFILE_V2}, got ${manifestFields.profile}`);
|
|
Binary file
|
|
Binary file
|
package/complete/index.d.ts
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
/**
|
|
2
|
-
* pikelet-wasm/complete — reader for complete .
|
|
3
|
-
* (spec/COMPLETE_PROFILE.md, profile
|
|
2
|
+
* pikelet-wasm/complete — reader for complete .pikelet artifacts
|
|
3
|
+
* (spec/COMPLETE_PROFILE.md, profile pikelet-complete-v1). Opens any kind
|
|
4
4
|
* (1 student-inline, 2 declared-external, 3 inline-transformer) from a file
|
|
5
5
|
* path in Node or a range source anywhere.
|
|
6
6
|
*/
|
|
@@ -74,7 +74,7 @@ export interface CompleteSearch {
|
|
|
74
74
|
identity: string;
|
|
75
75
|
/** Container header formatVersion (1 or 2). */
|
|
76
76
|
formatVersion: number;
|
|
77
|
-
/** Manifest profile: '
|
|
77
|
+
/** Manifest profile: 'pikelet-complete-v1' | 'pikelet-complete-v2'. */
|
|
78
78
|
profile: string;
|
|
79
79
|
/**
|
|
80
80
|
* Human-readable pack name from the identity-verified manifest
|
|
@@ -158,6 +158,13 @@ export interface CompleteSearch {
|
|
|
158
158
|
parallelism?: number;
|
|
159
159
|
gap?: number;
|
|
160
160
|
maxRangeBytes?: number;
|
|
161
|
+
/**
|
|
162
|
+
* A matchQuality: 'none' verdict withholds results by default. Set
|
|
163
|
+
* true to see the raw retrieval anyway (e.g. to inspect why the
|
|
164
|
+
* classifier abstained) — matchQuality and confidence are unaffected,
|
|
165
|
+
* this only controls whether results is populated. Default false.
|
|
166
|
+
*/
|
|
167
|
+
showAbstained?: boolean;
|
|
161
168
|
}): Promise<CompleteQueryResult>;
|
|
162
169
|
/** Hydrate one corpus record by id (verified per record on format 2). */
|
|
163
170
|
record(id: number): Promise<Record<string, unknown>>;
|
|
@@ -166,7 +173,7 @@ export interface CompleteSearch {
|
|
|
166
173
|
close(): Promise<void>;
|
|
167
174
|
}
|
|
168
175
|
|
|
169
|
-
export declare function
|
|
176
|
+
export declare function openPikeletFile(
|
|
170
177
|
input: string | CompleteRangeSource,
|
|
171
178
|
options?: {
|
|
172
179
|
/**
|
|
@@ -284,6 +291,3 @@ export declare function buildInlineTestVectors(embedder: unknown, texts?: string
|
|
|
284
291
|
text: string; windows: number; embedding: number[]; tolerance: number;
|
|
285
292
|
}>>;
|
|
286
293
|
export declare function verifyInlineTestVectors(embedder: unknown): Promise<{ checked: number }>;
|
|
287
|
-
|
|
288
|
-
/** Documented alias for openPancakeFile — the project renamed to Pikelet; the reader's original export stays for compatibility. */
|
|
289
|
-
export declare const openPikeletFile: typeof openPancakeFile;
|