pikelet-wasm 0.7.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +868 -0
- package/LICENSE +191 -0
- package/README.md +590 -0
- package/complete/builder.d.ts +82 -0
- package/complete/builder.mjs +527 -0
- package/complete/encoder-kernels/encoder.mjs +2 -0
- package/complete/encoder-kernels/encoder.node.mjs +2 -0
- package/complete/encoder-kernels/encoder.node.wasm +0 -0
- package/complete/encoder-kernels/encoder.wasm +0 -0
- package/complete/format.mjs +8 -0
- package/complete/index.d.ts +289 -0
- package/complete/index.mjs +1124 -0
- package/complete/inline-transformer.mjs +205 -0
- package/complete/lexical.mjs +264 -0
- package/complete/retrieval-abstention.mjs +119 -0
- package/complete/sources.mjs +186 -0
- package/complete/student-abstention.mjs +102 -0
- package/complete/student-embedder.mjs +231 -0
- package/complete/wordpiece.mjs +68 -0
- package/dist/engine.js +2 -0
- package/dist/engine.scalar.js +2 -0
- package/dist/engine.scalar.wasm +0 -0
- package/dist/engine.wasm +0 -0
- package/package.json +168 -0
- package/pancake-artifact-common.js +561 -0
- package/pancake-artifact-range.js +799 -0
- package/pancake-artifact-sketch.js +1040 -0
- package/pancake-artifact.d.ts +109 -0
- package/pancake-artifact.js +31 -0
- package/pancake-core.js +1268 -0
- package/pancake-errors.js +40 -0
- package/pancake-loader.js +66 -0
- package/pancake.d.ts +691 -0
- package/pancake.js +254 -0
- package/pancake.node.mjs +259 -0
- package/pancake.web.d.ts +50 -0
- package/pancake.web.mjs +88 -0
- package/pancake.workerd.mjs +72 -0
|
@@ -0,0 +1,109 @@
|
|
|
1
|
+
// Types for the `pikelet-wasm/artifact` subpath: the engine-free CJS module
|
|
2
|
+
// holding the Search Artifact readers, builders, and snapshot parser. Unlike
|
|
3
|
+
// the entrypoints, this module DOES provide these as named exports.
|
|
4
|
+
|
|
5
|
+
import type {
|
|
6
|
+
NodeFileRangeSourceConstructor,
|
|
7
|
+
PancakeRangeArtifactConstructor,
|
|
8
|
+
PancakeSketchArtifact,
|
|
9
|
+
PancakeSketchArtifactConstructor,
|
|
10
|
+
RangeArtifactBuildManifest,
|
|
11
|
+
RangeArtifactBuildOptions,
|
|
12
|
+
SketchArtifactBuildManifest,
|
|
13
|
+
SketchArtifactBuildOptions,
|
|
14
|
+
SketchScanner,
|
|
15
|
+
SketchScannerOptions,
|
|
16
|
+
} from './pancake.js';
|
|
17
|
+
|
|
18
|
+
/** A parsed uint8 engine snapshot, as returned by {@link parseUint8Snapshot}. */
|
|
19
|
+
export interface Uint8SnapshotGraph {
|
|
20
|
+
readonly kind: 'u8';
|
|
21
|
+
readonly version: number;
|
|
22
|
+
readonly dim: number;
|
|
23
|
+
readonly count: number;
|
|
24
|
+
readonly entryPoint: number;
|
|
25
|
+
readonly maxLevel: number;
|
|
26
|
+
readonly M: number;
|
|
27
|
+
readonly M0: number;
|
|
28
|
+
/** 0 = l2, 1 = cosine. */
|
|
29
|
+
readonly metric: number;
|
|
30
|
+
readonly efConstruction: number;
|
|
31
|
+
readonly scales: Float32Array;
|
|
32
|
+
readonly offsets: Float32Array;
|
|
33
|
+
readonly qdata: Uint8Array;
|
|
34
|
+
readonly levels: Uint16Array;
|
|
35
|
+
readonly base: Uint32Array[];
|
|
36
|
+
readonly upper: Uint32Array[][];
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
/**
|
|
40
|
+
* Any index-like source of quantized rows: a {@link Uint8SnapshotGraph} or a
|
|
41
|
+
* live quantized index exposing the same row fields.
|
|
42
|
+
*/
|
|
43
|
+
export interface SketchRowSource {
|
|
44
|
+
readonly dim: number;
|
|
45
|
+
readonly count: number;
|
|
46
|
+
/** 0 = l2, 1 = cosine. */
|
|
47
|
+
readonly metric: number;
|
|
48
|
+
readonly qdata: Uint8Array;
|
|
49
|
+
readonly scales: Float32Array;
|
|
50
|
+
readonly offsets: Float32Array;
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
export declare const PancakeRangeArtifact: PancakeRangeArtifactConstructor;
|
|
54
|
+
export declare const PancakeSketchArtifact: PancakeSketchArtifactConstructor;
|
|
55
|
+
export declare const NodeFileRangeSource: NodeFileRangeSourceConstructor;
|
|
56
|
+
|
|
57
|
+
/**
|
|
58
|
+
* Raw scanner factory. The entrypoints curry the engine loader; here it must
|
|
59
|
+
* be passed explicitly as the runtime's engine-module loader.
|
|
60
|
+
*/
|
|
61
|
+
export declare function createSketchScanner(
|
|
62
|
+
loadEngine: () => Promise<unknown>,
|
|
63
|
+
artifact: PancakeSketchArtifact,
|
|
64
|
+
options?: SketchScannerOptions
|
|
65
|
+
): Promise<SketchScanner>;
|
|
66
|
+
|
|
67
|
+
/**
|
|
68
|
+
* @deprecated The `.pancake-range` profile is deprecated
|
|
69
|
+
* (spec/SEARCH_ARTIFACT_CONTRACT.md 9.2); build a `.pancake-sketch`
|
|
70
|
+
* artifact instead. Readers stay supported for existing artifacts.
|
|
71
|
+
*/
|
|
72
|
+
export declare function buildRangeArtifact(
|
|
73
|
+
snapshot: Uint8Array | ArrayBufferLike,
|
|
74
|
+
outPath: string,
|
|
75
|
+
opts?: RangeArtifactBuildOptions
|
|
76
|
+
): RangeArtifactBuildManifest;
|
|
77
|
+
|
|
78
|
+
/**
|
|
79
|
+
* @deprecated The `.pancake-range` profile is deprecated
|
|
80
|
+
* (spec/SEARCH_ARTIFACT_CONTRACT.md 9.2); build a `.pancake-sketch`
|
|
81
|
+
* artifact instead. Readers stay supported for existing artifacts.
|
|
82
|
+
*/
|
|
83
|
+
export declare function buildRangeArtifactFile(
|
|
84
|
+
snapshotPath: string,
|
|
85
|
+
outPath: string,
|
|
86
|
+
opts?: RangeArtifactBuildOptions
|
|
87
|
+
): RangeArtifactBuildManifest;
|
|
88
|
+
|
|
89
|
+
export declare function buildSketchArtifact(
|
|
90
|
+
snapshot: Uint8Array | ArrayBufferLike,
|
|
91
|
+
outPath: string,
|
|
92
|
+
opts?: SketchArtifactBuildOptions
|
|
93
|
+
): SketchArtifactBuildManifest;
|
|
94
|
+
|
|
95
|
+
export declare function buildSketchArtifactFile(
|
|
96
|
+
snapshotPath: string,
|
|
97
|
+
outPath: string,
|
|
98
|
+
opts?: SketchArtifactBuildOptions
|
|
99
|
+
): SketchArtifactBuildManifest;
|
|
100
|
+
|
|
101
|
+
/** Write a sketch artifact directly from quantized rows. */
|
|
102
|
+
export declare function exportSketchArtifact(
|
|
103
|
+
index: SketchRowSource,
|
|
104
|
+
outPath: string,
|
|
105
|
+
options?: SketchArtifactBuildOptions
|
|
106
|
+
): SketchArtifactBuildManifest;
|
|
107
|
+
|
|
108
|
+
/** Parse and validate an untrusted uint8 engine snapshot (fails closed). */
|
|
109
|
+
export declare function parseUint8Snapshot(bytes: Uint8Array | ArrayBufferLike): Uint8SnapshotGraph;
|
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
'use strict';
|
|
2
|
+
// pikelet-wasm/artifact — the Search Artifact layer (spec/SEARCH_ARTIFACT_CONTRACT.md).
|
|
3
|
+
// Public entry point; the implementation is split by profile:
|
|
4
|
+
// pancake-artifact-common.js read budgets + range validation, range sources
|
|
5
|
+
// (NodeFileRangeSource), result heap, snapshot
|
|
6
|
+
// parsing (parseUint8Snapshot), SHA-256 helpers
|
|
7
|
+
// pancake-artifact-range.js .pancake-range reader (PancakeRangeArtifact) + builder
|
|
8
|
+
// pancake-artifact-sketch.js .pancake-sketch reader (PancakeSketchArtifact),
|
|
9
|
+
// builders, and createSketchScanner
|
|
10
|
+
// The export list below is the package's contract (pancake-artifact.d.ts).
|
|
11
|
+
|
|
12
|
+
const { NodeFileRangeSource, parseUint8Snapshot } = require('./pancake-artifact-common.js');
|
|
13
|
+
const { PancakeRangeArtifact, buildRangeArtifact, buildRangeArtifactFile } = require('./pancake-artifact-range.js');
|
|
14
|
+
const {
|
|
15
|
+
PancakeSketchArtifact, createSketchScanner,
|
|
16
|
+
buildSketchArtifact, buildSketchArtifactBytes, buildSketchArtifactFile, exportSketchArtifact,
|
|
17
|
+
} = require('./pancake-artifact-sketch.js');
|
|
18
|
+
|
|
19
|
+
module.exports = {
|
|
20
|
+
PancakeRangeArtifact,
|
|
21
|
+
PancakeSketchArtifact,
|
|
22
|
+
createSketchScanner,
|
|
23
|
+
NodeFileRangeSource,
|
|
24
|
+
buildRangeArtifact,
|
|
25
|
+
buildRangeArtifactFile,
|
|
26
|
+
buildSketchArtifact,
|
|
27
|
+
buildSketchArtifactBytes,
|
|
28
|
+
buildSketchArtifactFile,
|
|
29
|
+
exportSketchArtifact,
|
|
30
|
+
parseUint8Snapshot,
|
|
31
|
+
};
|