pikelet-wasm 0.7.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +868 -0
- package/LICENSE +191 -0
- package/README.md +590 -0
- package/complete/builder.d.ts +82 -0
- package/complete/builder.mjs +527 -0
- package/complete/encoder-kernels/encoder.mjs +2 -0
- package/complete/encoder-kernels/encoder.node.mjs +2 -0
- package/complete/encoder-kernels/encoder.node.wasm +0 -0
- package/complete/encoder-kernels/encoder.wasm +0 -0
- package/complete/format.mjs +8 -0
- package/complete/index.d.ts +289 -0
- package/complete/index.mjs +1124 -0
- package/complete/inline-transformer.mjs +205 -0
- package/complete/lexical.mjs +264 -0
- package/complete/retrieval-abstention.mjs +119 -0
- package/complete/sources.mjs +186 -0
- package/complete/student-abstention.mjs +102 -0
- package/complete/student-embedder.mjs +231 -0
- package/complete/wordpiece.mjs +68 -0
- package/dist/engine.js +2 -0
- package/dist/engine.scalar.js +2 -0
- package/dist/engine.scalar.wasm +0 -0
- package/dist/engine.wasm +0 -0
- package/package.json +168 -0
- package/pancake-artifact-common.js +561 -0
- package/pancake-artifact-range.js +799 -0
- package/pancake-artifact-sketch.js +1040 -0
- package/pancake-artifact.d.ts +109 -0
- package/pancake-artifact.js +31 -0
- package/pancake-core.js +1268 -0
- package/pancake-errors.js +40 -0
- package/pancake-loader.js +66 -0
- package/pancake.d.ts +691 -0
- package/pancake.js +254 -0
- package/pancake.node.mjs +259 -0
- package/pancake.web.d.ts +50 -0
- package/pancake.web.mjs +88 -0
- package/pancake.workerd.mjs +72 -0
|
@@ -0,0 +1,186 @@
|
|
|
1
|
+
export function httpRangeSource(url, options = {}) {
|
|
2
|
+
const stats = { requests: 0, bytes: 0, acceptRanges: null, etag: null, fullFallback: false, retries: 0, redirects: 0 };
|
|
3
|
+
const maxFullFallbackBytes = options.maxFullFallbackBytes ?? 64 * 1024 * 1024;
|
|
4
|
+
// Transient statuses are retried with capped exponential backoff
|
|
5
|
+
// (Retry-After honored when sane): CDNs rate-limit the parallel range
|
|
6
|
+
// bursts this reader legitimately issues — GitHub release assets answer
|
|
7
|
+
// 429 under an open's prefetch + first-query fan-out — and failing a
|
|
8
|
+
// whole query over a pressure signal wastes everything already fetched.
|
|
9
|
+
const maxRetries = options.maxRetries ?? 6;
|
|
10
|
+
const RETRYABLE = new Set([429, 502, 503, 504]);
|
|
11
|
+
const retryDelayMs = (attempt, response) => {
|
|
12
|
+
// GitHub's release CDN answers rate-limited range bursts with 429
|
|
13
|
+
// and a Retry-After of up to a minute; honoring it (capped) beats
|
|
14
|
+
// guessing. Without one, exponential backoff capped at 15 s.
|
|
15
|
+
const after = Number(response.headers.get('retry-after'));
|
|
16
|
+
if (Number.isFinite(after) && after >= 0 && after <= 90) return after * 1000;
|
|
17
|
+
return Math.min(15000, 250 * 2 ** attempt);
|
|
18
|
+
};
|
|
19
|
+
const cacheKeyParam = options.cacheKeyParam === undefined ? 'r'
|
|
20
|
+
: (options.cacheKeyParam === null || options.cacheKeyParam === false) ? null
|
|
21
|
+
: String(options.cacheKeyParam);
|
|
22
|
+
let full = null;
|
|
23
|
+
|
|
24
|
+
// Redirect memoization. A host that answers with a redirect (GitHub
|
|
25
|
+
// release assets: github.com 302s every hit to a signed CDN URL) would
|
|
26
|
+
// otherwise charge its rate-limited front door once per range read —
|
|
27
|
+
// ~130 hits for one wiki query, and github.com's frontend throttling,
|
|
28
|
+
// not the CDN, is what 429s the burst; the blob host behind the
|
|
29
|
+
// redirect serves parallel 206s happily. Resolve the chain once, pin
|
|
30
|
+
// the final URL, and re-resolve only when it stops being accepted
|
|
31
|
+
// (401/403 — signed URLs expire). Browsers cannot read Location from a
|
|
32
|
+
// cross-origin redirect (opaqueredirect), so resolution degrades to
|
|
33
|
+
// the un-pinned auto-follow behavior there.
|
|
34
|
+
let resolvedTarget = null; // { target, viaRedirect } | null (unresolved or unresolvable)
|
|
35
|
+
let resolving = null;
|
|
36
|
+
let sizeFromHead = null;
|
|
37
|
+
const resolveTarget = async () => {
|
|
38
|
+
if (resolvedTarget) return resolvedTarget;
|
|
39
|
+
resolving ??= (async () => {
|
|
40
|
+
let target = url;
|
|
41
|
+
let viaRedirect = false;
|
|
42
|
+
try {
|
|
43
|
+
for (let hop = 0; hop < 5; hop++) {
|
|
44
|
+
const probe = await fetch(target, { method: 'HEAD', redirect: 'manual' });
|
|
45
|
+
const location = probe.headers.get('location');
|
|
46
|
+
if (probe.status >= 300 && probe.status < 400 && location) {
|
|
47
|
+
target = new URL(location, target).href;
|
|
48
|
+
viaRedirect = true;
|
|
49
|
+
stats.redirects += 1;
|
|
50
|
+
continue;
|
|
51
|
+
}
|
|
52
|
+
if (probe.type === 'opaqueredirect' || probe.status === 0) {
|
|
53
|
+
// Browser CORS: the chain is invisible; let fetch
|
|
54
|
+
// auto-follow per read as before.
|
|
55
|
+
target = url;
|
|
56
|
+
viaRedirect = false;
|
|
57
|
+
} else if (probe.ok) {
|
|
58
|
+
const len = Number(probe.headers.get('content-length'));
|
|
59
|
+
if (Number.isFinite(len) && len > 0) sizeFromHead = len;
|
|
60
|
+
stats.acceptRanges = probe.headers.get('accept-ranges');
|
|
61
|
+
stats.etag = probe.headers.get('etag');
|
|
62
|
+
}
|
|
63
|
+
break;
|
|
64
|
+
}
|
|
65
|
+
} catch {
|
|
66
|
+
// HEAD unsupported or blocked: fall back to auto-follow.
|
|
67
|
+
target = url;
|
|
68
|
+
viaRedirect = false;
|
|
69
|
+
}
|
|
70
|
+
resolvedTarget = { target, viaRedirect };
|
|
71
|
+
resolving = null;
|
|
72
|
+
return resolvedTarget;
|
|
73
|
+
})();
|
|
74
|
+
return resolving;
|
|
75
|
+
};
|
|
76
|
+
|
|
77
|
+
return {
|
|
78
|
+
stats,
|
|
79
|
+
size: undefined,
|
|
80
|
+
preferredParallelism: options.preferredParallelism ?? 32,
|
|
81
|
+
preferredGapBytes: options.preferredGapBytes ?? 16 * 1024,
|
|
82
|
+
|
|
83
|
+
async init() {
|
|
84
|
+
await resolveTarget();
|
|
85
|
+
if (sizeFromHead !== null) this.size = sizeFromHead;
|
|
86
|
+
},
|
|
87
|
+
|
|
88
|
+
async read(offset, length) {
|
|
89
|
+
// After a one-time full download (host ignored Range) every read
|
|
90
|
+
// is served from memory; no further requests go out.
|
|
91
|
+
if (full) return full.subarray(offset, offset + length);
|
|
92
|
+
stats.requests += 1;
|
|
93
|
+
stats.bytes += length;
|
|
94
|
+
const end = offset + length - 1;
|
|
95
|
+
const headers = { Range: `bytes=${offset}-${end}` };
|
|
96
|
+
if (stats.etag) headers['If-Range'] = stats.etag;
|
|
97
|
+
const pinned = await resolveTarget();
|
|
98
|
+
// The per-range cache-key query defeats Chromium's same-URL
|
|
99
|
+
// cache-entry lock (concurrent fetches of one cacheable URL
|
|
100
|
+
// serialize on it). Hosts that sign the full query string
|
|
101
|
+
// (S3 presigned URLs and similar) can turn it off with
|
|
102
|
+
// options.cacheKeyParam: null — range reads then share the
|
|
103
|
+
// unmodified URL. A redirect-pinned target never gets the
|
|
104
|
+
// param: signed URLs sign their query string, and appending
|
|
105
|
+
// anything invalidates them.
|
|
106
|
+
const targetFor = ({ target, viaRedirect }) => (cacheKeyParam && !viaRedirect
|
|
107
|
+
? `${target}${target.includes('?') ? '&' : '?'}${cacheKeyParam}=${offset}-${end}`
|
|
108
|
+
: target);
|
|
109
|
+
let response = await fetch(targetFor(pinned), { headers });
|
|
110
|
+
if ((response.status === 401 || response.status === 403) && pinned.viaRedirect) {
|
|
111
|
+
// The pinned signed URL expired; resolve a fresh one once
|
|
112
|
+
// and retry. Concurrent reads share the re-resolution.
|
|
113
|
+
if (resolvedTarget === pinned) resolvedTarget = null;
|
|
114
|
+
stats.retries += 1;
|
|
115
|
+
response = await fetch(targetFor(await resolveTarget()), { headers });
|
|
116
|
+
}
|
|
117
|
+
for (let attempt = 0; RETRYABLE.has(response.status) && attempt < maxRetries; attempt++) {
|
|
118
|
+
stats.retries += 1;
|
|
119
|
+
await new Promise((resolve) => setTimeout(resolve, retryDelayMs(attempt, response)));
|
|
120
|
+
response = await fetch(targetFor(resolvedTarget || pinned), { headers });
|
|
121
|
+
}
|
|
122
|
+
|
|
123
|
+
if (response.status === 206) {
|
|
124
|
+
// A 206 must be the range we asked for, and must say so: a
|
|
125
|
+
// host (or cache) answering with a different slice would
|
|
126
|
+
// otherwise be caught only downstream, by the digest on
|
|
127
|
+
// whatever that slice happens to be part of — and the lazy
|
|
128
|
+
// sketch rows have no per-row digest yet. RFC 9110 requires
|
|
129
|
+
// Content-Range on a 206, so its absence fails closed too.
|
|
130
|
+
const contentRange = response.headers.get('content-range');
|
|
131
|
+
const m = contentRange && /^bytes (\d+)-(\d+)\//.exec(contentRange);
|
|
132
|
+
if (!m || Number(m[1]) !== offset || Number(m[2]) > end) {
|
|
133
|
+
throw new Error(`range read returned Content-Range ${contentRange || '(missing)'}, requested bytes ${offset}-${end}`);
|
|
134
|
+
}
|
|
135
|
+
const body = new Uint8Array(await response.arrayBuffer());
|
|
136
|
+
if (body.length !== Number(m[2]) - Number(m[1]) + 1) {
|
|
137
|
+
throw new Error(`range read body is ${body.length} bytes for Content-Range ${contentRange}`);
|
|
138
|
+
}
|
|
139
|
+
return body;
|
|
140
|
+
}
|
|
141
|
+
if (response.status === 200) {
|
|
142
|
+
// An unknown size must refuse, not pass the gate: with no
|
|
143
|
+
// Content-Length (chunked body) and init() never called,
|
|
144
|
+
// size would be 0 and the cap check silently vacuous.
|
|
145
|
+
const size = Number(response.headers.get('content-length')) || this.size || 0;
|
|
146
|
+
if (size <= 0) {
|
|
147
|
+
throw new Error('host ignores Range and reports no size; refusing unbounded full download (call init() first or fix the host)');
|
|
148
|
+
}
|
|
149
|
+
if (size > maxFullFallbackBytes) {
|
|
150
|
+
throw new Error(`host ignores Range and the file is ${size} bytes; refusing full download`);
|
|
151
|
+
}
|
|
152
|
+
// Declared sizes are advisory (a chunked body can stream more
|
|
153
|
+
// than any header claimed), so the cap is enforced on the
|
|
154
|
+
// bytes actually received, aborting as soon as it is crossed.
|
|
155
|
+
const reader = response.body?.getReader?.();
|
|
156
|
+
if (!reader) {
|
|
157
|
+
const bytes = new Uint8Array(await response.arrayBuffer());
|
|
158
|
+
if (bytes.length > maxFullFallbackBytes) {
|
|
159
|
+
throw new Error(`host ignores Range and streamed ${bytes.length} bytes; refusing full download over ${maxFullFallbackBytes}`);
|
|
160
|
+
}
|
|
161
|
+
full = bytes;
|
|
162
|
+
} else {
|
|
163
|
+
const chunks = [];
|
|
164
|
+
let received = 0;
|
|
165
|
+
for (;;) {
|
|
166
|
+
const { done, value } = await reader.read();
|
|
167
|
+
if (done) break;
|
|
168
|
+
received += value.byteLength;
|
|
169
|
+
if (received > maxFullFallbackBytes) {
|
|
170
|
+
await reader.cancel().catch(() => {});
|
|
171
|
+
throw new Error(`host ignores Range and streamed over ${maxFullFallbackBytes} bytes; aborting full download`);
|
|
172
|
+
}
|
|
173
|
+
chunks.push(value);
|
|
174
|
+
}
|
|
175
|
+
full = new Uint8Array(received);
|
|
176
|
+
let at = 0;
|
|
177
|
+
for (const chunk of chunks) { full.set(chunk, at); at += chunk.byteLength; }
|
|
178
|
+
}
|
|
179
|
+
stats.fullFallback = true;
|
|
180
|
+
console.warn('.pancake host does not honor Range; fell back to a one-time full download');
|
|
181
|
+
return full.subarray(offset, offset + length);
|
|
182
|
+
}
|
|
183
|
+
throw new Error(`range read failed: HTTP ${response.status}`);
|
|
184
|
+
},
|
|
185
|
+
};
|
|
186
|
+
}
|
|
@@ -0,0 +1,102 @@
|
|
|
1
|
+
// Abstention scoring, extracted from 03's worker.js and parameterized on
|
|
2
|
+
// the calibration model. Pure JS, no runtime dependencies — shared by the
|
|
3
|
+
// Node spike facade and the browser one-file reader. The scoreQuery import
|
|
4
|
+
// is the only coupling to the encoder module (they form one
|
|
5
|
+
// query-interpretation unit; see spec/COMPLETE_PROFILE.md section 3.6).
|
|
6
|
+
|
|
7
|
+
import { scoreQuery } from './student-embedder.mjs';
|
|
8
|
+
|
|
9
|
+
export function buildKnownBucketTables(model) {
|
|
10
|
+
const word = new Set();
|
|
11
|
+
let maxIdf = 0;
|
|
12
|
+
for (const entry of model?.wordBuckets || []) {
|
|
13
|
+
if (!Array.isArray(entry) || entry.length < 2) continue;
|
|
14
|
+
const bucket = Number(entry[0]);
|
|
15
|
+
const idf = Number(entry[1]);
|
|
16
|
+
if (Number.isInteger(bucket) && bucket >= 0) {
|
|
17
|
+
word.add(bucket);
|
|
18
|
+
if (idf > maxIdf) maxIdf = idf;
|
|
19
|
+
}
|
|
20
|
+
}
|
|
21
|
+
const char = new Set();
|
|
22
|
+
for (const value of model?.charBuckets || []) {
|
|
23
|
+
const bucket = Number(value);
|
|
24
|
+
if (Number.isInteger(bucket) && bucket >= 0) char.add(bucket);
|
|
25
|
+
}
|
|
26
|
+
return { word, char, maxIdf: maxIdf || 255 };
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
export function computeKnownFractions(features, model) {
|
|
30
|
+
if (!model) return { known_word: 0, known_char: 0, n_feats: features.length };
|
|
31
|
+
const tables = model._knownBucketTables || (model._knownBucketTables = buildKnownBucketTables(model));
|
|
32
|
+
let wordKnown = 0;
|
|
33
|
+
let wordTotal = 0;
|
|
34
|
+
let charKnown = 0;
|
|
35
|
+
let charTotal = 0;
|
|
36
|
+
for (const feature of features) {
|
|
37
|
+
if (feature.family === 'word') {
|
|
38
|
+
wordTotal += 1;
|
|
39
|
+
if (tables.word.has(feature.bucket)) wordKnown += 1;
|
|
40
|
+
} else if (feature.family === 'char') {
|
|
41
|
+
charTotal += 1;
|
|
42
|
+
if (tables.char.has(feature.bucket)) charKnown += 1;
|
|
43
|
+
}
|
|
44
|
+
}
|
|
45
|
+
return {
|
|
46
|
+
known_word: wordTotal > 0 ? wordKnown / wordTotal : 0,
|
|
47
|
+
known_char: charTotal > 0 ? charKnown / charTotal : 0,
|
|
48
|
+
n_feats: features.length,
|
|
49
|
+
};
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
export function computeHiddenProbe(embedded, model) {
|
|
53
|
+
const probe = model?.hiddenProbe;
|
|
54
|
+
if (!probe || !Array.isArray(probe.weights) || !embedded.hidden) return 0;
|
|
55
|
+
let logit = Number(probe.bias) || 0;
|
|
56
|
+
const limit = Math.min(probe.weights.length, embedded.hidden.length);
|
|
57
|
+
for (let index = 0; index < limit; index++) {
|
|
58
|
+
logit += (Number(probe.weights[index]) || 0) * embedded.hidden[index];
|
|
59
|
+
}
|
|
60
|
+
if (logit >= 0) {
|
|
61
|
+
const z = Math.exp(-logit);
|
|
62
|
+
return 1 / (1 + z);
|
|
63
|
+
}
|
|
64
|
+
const z = Math.exp(logit);
|
|
65
|
+
return z / (1 + z);
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
export function computeMatchQuality(hits, embedded, model) {
|
|
69
|
+
if (!model) return { match_quality: 'unscored' };
|
|
70
|
+
const d0 = hits.length > 0 ? hits[0].distance : 1;
|
|
71
|
+
const marginIndex = Math.min(4, hits.length - 1);
|
|
72
|
+
const margin = marginIndex > 0 ? hits[marginIndex].distance - d0 : 0;
|
|
73
|
+
const known = computeKnownFractions(embedded.features, model);
|
|
74
|
+
const signals = {
|
|
75
|
+
d0,
|
|
76
|
+
margin,
|
|
77
|
+
pre_norm: embedded.preNorm,
|
|
78
|
+
known_word: known.known_word,
|
|
79
|
+
known_char: known.known_char,
|
|
80
|
+
hidden_probe: computeHiddenProbe(embedded, model),
|
|
81
|
+
n_feats: known.n_feats,
|
|
82
|
+
};
|
|
83
|
+
return scoreQuery(signals, model);
|
|
84
|
+
}
|
|
85
|
+
|
|
86
|
+
export function computePreSearchAbstention(embedded, model) {
|
|
87
|
+
if (!model) return null;
|
|
88
|
+
const thresholds = model.thresholds || {};
|
|
89
|
+
const minFeatures = Number.isInteger(thresholds.minFeatures) ? thresholds.minFeatures : 3;
|
|
90
|
+
const preNormFloor = Number.isFinite(thresholds.preNormFloor) ? thresholds.preNormFloor : 0.4;
|
|
91
|
+
if (embedded.features.length >= minFeatures && embedded.preNorm >= preNormFloor) return null;
|
|
92
|
+
const known = computeKnownFractions(embedded.features, model);
|
|
93
|
+
return scoreQuery({
|
|
94
|
+
d0: 1,
|
|
95
|
+
margin: 0,
|
|
96
|
+
pre_norm: embedded.preNorm,
|
|
97
|
+
known_word: known.known_word,
|
|
98
|
+
known_char: known.known_char,
|
|
99
|
+
hidden_probe: computeHiddenProbe(embedded, model),
|
|
100
|
+
n_feats: known.n_feats,
|
|
101
|
+
}, model);
|
|
102
|
+
}
|
|
@@ -0,0 +1,231 @@
|
|
|
1
|
+
const MAGIC = 'PSTU';
|
|
2
|
+
const VERSION = 1;
|
|
3
|
+
const HEADER_BYTES = 32;
|
|
4
|
+
const TOKEN_RE = /[a-z0-9]+/g;
|
|
5
|
+
|
|
6
|
+
function align4(value) {
|
|
7
|
+
return (value + 3) & ~3;
|
|
8
|
+
}
|
|
9
|
+
|
|
10
|
+
function hash32(text, seed) {
|
|
11
|
+
let hash = seed >>> 0;
|
|
12
|
+
for (let i = 0; i < text.length; i++) {
|
|
13
|
+
hash ^= text.charCodeAt(i);
|
|
14
|
+
hash = Math.imul(hash, 16777619);
|
|
15
|
+
}
|
|
16
|
+
return hash >>> 0;
|
|
17
|
+
}
|
|
18
|
+
|
|
19
|
+
export function extractStudentFeatures(text, model) {
|
|
20
|
+
return extractStudentFeatureRecords(text, model).map((feature) => feature.bucket);
|
|
21
|
+
}
|
|
22
|
+
|
|
23
|
+
export function extractStudentFeatureRecords(text, model) {
|
|
24
|
+
const tokens = String(text || '').toLowerCase().match(TOKEN_RE) || [];
|
|
25
|
+
const features = [];
|
|
26
|
+
const limit = model.maxFeatures;
|
|
27
|
+
|
|
28
|
+
const push = (feature, family) => {
|
|
29
|
+
if (features.length < limit) {
|
|
30
|
+
features.push({
|
|
31
|
+
bucket: hash32(feature, model.hashSeed) % model.bucketCount,
|
|
32
|
+
family
|
|
33
|
+
});
|
|
34
|
+
}
|
|
35
|
+
};
|
|
36
|
+
|
|
37
|
+
for (let tokenIndex = 0; tokenIndex < tokens.length && features.length < limit; tokenIndex++) {
|
|
38
|
+
const token = tokens[tokenIndex].slice(0, 48);
|
|
39
|
+
push(`w:${token}`, 'word');
|
|
40
|
+
|
|
41
|
+
const padded = `^${token}$`;
|
|
42
|
+
for (let width = 3; width <= 5 && features.length < limit; width++) {
|
|
43
|
+
for (let start = 0; start + width <= padded.length && features.length < limit; start++) {
|
|
44
|
+
push(`c${width}:${padded.slice(start, start + width)}`, 'char');
|
|
45
|
+
}
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
if (tokenIndex > 0) {
|
|
49
|
+
push(`b:${tokens[tokenIndex - 1].slice(0, 48)}:${token}`, 'word');
|
|
50
|
+
}
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
return features;
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
export function loadStudentModel(input) {
|
|
57
|
+
const bytes = input instanceof Uint8Array
|
|
58
|
+
? input
|
|
59
|
+
: input instanceof ArrayBuffer
|
|
60
|
+
? new Uint8Array(input)
|
|
61
|
+
: new Uint8Array(input.buffer, input.byteOffset, input.byteLength);
|
|
62
|
+
if (bytes.byteLength < HEADER_BYTES) throw new Error('Student model is truncated');
|
|
63
|
+
|
|
64
|
+
const magic = String.fromCharCode(...bytes.subarray(0, 4));
|
|
65
|
+
if (magic !== MAGIC) throw new Error(`Invalid student model magic: ${magic}`);
|
|
66
|
+
|
|
67
|
+
const view = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength);
|
|
68
|
+
const version = view.getUint32(4, true);
|
|
69
|
+
if (version !== VERSION) throw new Error(`Unsupported student model version: ${version}`);
|
|
70
|
+
|
|
71
|
+
const bucketCount = view.getUint32(8, true);
|
|
72
|
+
const hiddenDim = view.getUint32(12, true);
|
|
73
|
+
const outputDim = view.getUint32(16, true);
|
|
74
|
+
const hashSeed = view.getUint32(20, true);
|
|
75
|
+
const maxFeatures = view.getUint32(24, true);
|
|
76
|
+
if (!bucketCount || !hiddenDim || !outputDim || !maxFeatures) {
|
|
77
|
+
throw new Error('Student model has invalid dimensions');
|
|
78
|
+
}
|
|
79
|
+
|
|
80
|
+
let offset = HEADER_BYTES;
|
|
81
|
+
const embeddingScalesBytes = bucketCount * 4;
|
|
82
|
+
const embeddingBytes = bucketCount * hiddenDim;
|
|
83
|
+
const projectionScalesBytes = outputDim * 4;
|
|
84
|
+
const projectionBytes = outputDim * hiddenDim;
|
|
85
|
+
const biasBytes = outputDim * 4;
|
|
86
|
+
const required = align4(offset + embeddingScalesBytes + embeddingBytes)
|
|
87
|
+
+ projectionScalesBytes + projectionBytes + biasBytes;
|
|
88
|
+
if (required !== bytes.byteLength) {
|
|
89
|
+
throw new Error(`Student model size mismatch: expected ${required}, received ${bytes.byteLength}`);
|
|
90
|
+
}
|
|
91
|
+
|
|
92
|
+
const embeddingScales = new Float32Array(
|
|
93
|
+
bytes.buffer,
|
|
94
|
+
bytes.byteOffset + offset,
|
|
95
|
+
bucketCount
|
|
96
|
+
);
|
|
97
|
+
offset += embeddingScalesBytes;
|
|
98
|
+
const embeddingWeights = new Int8Array(
|
|
99
|
+
bytes.buffer,
|
|
100
|
+
bytes.byteOffset + offset,
|
|
101
|
+
embeddingBytes
|
|
102
|
+
);
|
|
103
|
+
offset = align4(offset + embeddingBytes);
|
|
104
|
+
const projectionScales = new Float32Array(
|
|
105
|
+
bytes.buffer,
|
|
106
|
+
bytes.byteOffset + offset,
|
|
107
|
+
outputDim
|
|
108
|
+
);
|
|
109
|
+
offset += projectionScalesBytes;
|
|
110
|
+
const projectionWeights = new Int8Array(
|
|
111
|
+
bytes.buffer,
|
|
112
|
+
bytes.byteOffset + offset,
|
|
113
|
+
projectionBytes
|
|
114
|
+
);
|
|
115
|
+
offset += projectionBytes;
|
|
116
|
+
const projectionBias = new Float32Array(
|
|
117
|
+
bytes.buffer,
|
|
118
|
+
bytes.byteOffset + offset,
|
|
119
|
+
outputDim
|
|
120
|
+
);
|
|
121
|
+
|
|
122
|
+
return {
|
|
123
|
+
version,
|
|
124
|
+
bucketCount,
|
|
125
|
+
hiddenDim,
|
|
126
|
+
outputDim,
|
|
127
|
+
hashSeed,
|
|
128
|
+
maxFeatures,
|
|
129
|
+
embeddingScales,
|
|
130
|
+
embeddingWeights,
|
|
131
|
+
projectionScales,
|
|
132
|
+
projectionWeights,
|
|
133
|
+
projectionBias,
|
|
134
|
+
byteLength: bytes.byteLength,
|
|
135
|
+
_bytes: bytes,
|
|
136
|
+
};
|
|
137
|
+
}
|
|
138
|
+
|
|
139
|
+
export function embedTextWithStudent(text, model) {
|
|
140
|
+
const features = extractStudentFeatureRecords(text, model);
|
|
141
|
+
const hidden = new Float32Array(model.hiddenDim);
|
|
142
|
+
const output = new Float32Array(model.outputDim);
|
|
143
|
+
if (features.length === 0) {
|
|
144
|
+
return { vector: output, preNorm: 0, features, hidden };
|
|
145
|
+
}
|
|
146
|
+
|
|
147
|
+
for (const { bucket } of features) {
|
|
148
|
+
const scale = model.embeddingScales[bucket];
|
|
149
|
+
const rowOffset = bucket * model.hiddenDim;
|
|
150
|
+
for (let column = 0; column < model.hiddenDim; column++) {
|
|
151
|
+
hidden[column] += model.embeddingWeights[rowOffset + column] * scale;
|
|
152
|
+
}
|
|
153
|
+
}
|
|
154
|
+
|
|
155
|
+
const inverseCount = 1 / features.length;
|
|
156
|
+
for (let column = 0; column < model.hiddenDim; column++) {
|
|
157
|
+
hidden[column] = Math.tanh(hidden[column] * inverseCount);
|
|
158
|
+
}
|
|
159
|
+
const hiddenOutput = new Float32Array(hidden);
|
|
160
|
+
|
|
161
|
+
let normSquared = 0;
|
|
162
|
+
for (let row = 0; row < model.outputDim; row++) {
|
|
163
|
+
const scale = model.projectionScales[row];
|
|
164
|
+
const rowOffset = row * model.hiddenDim;
|
|
165
|
+
let value = model.projectionBias[row];
|
|
166
|
+
for (let column = 0; column < model.hiddenDim; column++) {
|
|
167
|
+
value += model.projectionWeights[rowOffset + column] * scale * hidden[column];
|
|
168
|
+
}
|
|
169
|
+
output[row] = value;
|
|
170
|
+
normSquared += value * value;
|
|
171
|
+
}
|
|
172
|
+
|
|
173
|
+
const norm = Math.sqrt(normSquared);
|
|
174
|
+
if (!(norm > 0) || !Number.isFinite(norm)) {
|
|
175
|
+
return { vector: new Float32Array(model.outputDim), preNorm: 0, features, hidden: hiddenOutput };
|
|
176
|
+
}
|
|
177
|
+
const inverseNorm = 1 / norm;
|
|
178
|
+
for (let row = 0; row < model.outputDim; row++) output[row] *= inverseNorm;
|
|
179
|
+
return { vector: output, preNorm: norm, features, hidden: hiddenOutput };
|
|
180
|
+
}
|
|
181
|
+
|
|
182
|
+
function sigmoid(value) {
|
|
183
|
+
if (value >= 0) {
|
|
184
|
+
const z = Math.exp(-value);
|
|
185
|
+
return 1 / (1 + z);
|
|
186
|
+
}
|
|
187
|
+
const z = Math.exp(value);
|
|
188
|
+
return z / (1 + z);
|
|
189
|
+
}
|
|
190
|
+
|
|
191
|
+
export function scoreQuery(signals, abstention) {
|
|
192
|
+
if (!abstention) {
|
|
193
|
+
return { match_quality: 'unscored' };
|
|
194
|
+
}
|
|
195
|
+
const featureNames = abstention.features || ['d0', 'margin', 'pre_norm', 'known_word', 'known_char'];
|
|
196
|
+
const weights = abstention.weights || [];
|
|
197
|
+
let logit = Number(abstention.bias) || 0;
|
|
198
|
+
for (let i = 0; i < featureNames.length; i++) {
|
|
199
|
+
logit += (Number(weights[i]) || 0) * (Number(signals[featureNames[i]]) || 0);
|
|
200
|
+
}
|
|
201
|
+
|
|
202
|
+
const confidence = sigmoid(logit);
|
|
203
|
+
const thresholds = abstention.thresholds || {};
|
|
204
|
+
const hard = Number.isFinite(thresholds.hard) ? thresholds.hard : 0.05;
|
|
205
|
+
const weak = Number.isFinite(thresholds.weak) ? thresholds.weak : 0;
|
|
206
|
+
const preNormFloor = Number.isFinite(thresholds.preNormFloor) ? thresholds.preNormFloor : 0.4;
|
|
207
|
+
const minFeatures = Number.isInteger(thresholds.minFeatures) ? thresholds.minFeatures : 3;
|
|
208
|
+
const epsilon = 1e-6;
|
|
209
|
+
const floorTriggered = confidence < hard
|
|
210
|
+
|| (Number(signals.n_feats) || 0) < minFeatures
|
|
211
|
+
|| (Number(signals.pre_norm) || 0) < preNormFloor;
|
|
212
|
+
const match_quality = floorTriggered
|
|
213
|
+
? 'none'
|
|
214
|
+
: confidence + epsilon >= weak
|
|
215
|
+
? 'strong'
|
|
216
|
+
: 'weak';
|
|
217
|
+
return {
|
|
218
|
+
match_quality,
|
|
219
|
+
score: confidence,
|
|
220
|
+
confidence: Math.round(confidence * 1000) / 1000,
|
|
221
|
+
signals: {
|
|
222
|
+
d0: signals.d0,
|
|
223
|
+
margin: signals.margin,
|
|
224
|
+
pre_norm: signals.pre_norm,
|
|
225
|
+
known_word: signals.known_word,
|
|
226
|
+
known_char: signals.known_char,
|
|
227
|
+
hidden_probe: signals.hidden_probe,
|
|
228
|
+
n_feats: signals.n_feats
|
|
229
|
+
}
|
|
230
|
+
};
|
|
231
|
+
}
|
|
@@ -0,0 +1,68 @@
|
|
|
1
|
+
// Minimal BERT WordPiece tokenizer (uncased): lowercase, accent-strip via
|
|
2
|
+
// NFD, punctuation/CJK splitting, greedy longest-match with ## pieces,
|
|
3
|
+
// [CLS]/[SEP] framing. Parity-tested against the HF tokenizer on the wiki
|
|
4
|
+
// eval queries; pure JS so it runs wherever the reader runs.
|
|
5
|
+
|
|
6
|
+
export function createWordPiece(vocabText) {
|
|
7
|
+
const vocab = new Map();
|
|
8
|
+
vocabText.split('\n').forEach((token, id) => vocab.set(token, id));
|
|
9
|
+
const CLS = vocab.get('[CLS]');
|
|
10
|
+
const SEP = vocab.get('[SEP]');
|
|
11
|
+
const UNK = vocab.get('[UNK]');
|
|
12
|
+
const MAX_WORD = 100;
|
|
13
|
+
|
|
14
|
+
const isPunct = (c) => {
|
|
15
|
+
const cp = c.codePointAt(0);
|
|
16
|
+
if ((cp >= 33 && cp <= 47) || (cp >= 58 && cp <= 64)
|
|
17
|
+
|| (cp >= 91 && cp <= 96) || (cp >= 123 && cp <= 126)) return true;
|
|
18
|
+
return /\p{P}/u.test(c);
|
|
19
|
+
};
|
|
20
|
+
const isCjk = (c) => {
|
|
21
|
+
const cp = c.codePointAt(0);
|
|
22
|
+
return (cp >= 0x4e00 && cp <= 0x9fff) || (cp >= 0x3400 && cp <= 0x4dbf)
|
|
23
|
+
|| (cp >= 0xf900 && cp <= 0xfaff) || (cp >= 0x20000 && cp <= 0x2a6df);
|
|
24
|
+
};
|
|
25
|
+
|
|
26
|
+
function basicTokenize(text) {
|
|
27
|
+
const cleaned = text.normalize('NFD').replace(/\p{Mn}/gu, '').toLowerCase()
|
|
28
|
+
.replace(/[\u0000\uFFFD\u0001-\u0008\u000B\u000C\u000E-\u001F\u007F]/g, '');
|
|
29
|
+
const out = [];
|
|
30
|
+
let current = '';
|
|
31
|
+
const flush = () => { if (current) { out.push(current); current = ''; } };
|
|
32
|
+
for (const c of cleaned) {
|
|
33
|
+
if (/\s/.test(c)) { flush(); continue; }
|
|
34
|
+
if (isPunct(c) || isCjk(c)) { flush(); out.push(c); continue; }
|
|
35
|
+
current += c;
|
|
36
|
+
}
|
|
37
|
+
flush();
|
|
38
|
+
return out;
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
function wordpiece(word) {
|
|
42
|
+
if (word.length > MAX_WORD) return [UNK];
|
|
43
|
+
const pieces = [];
|
|
44
|
+
let start = 0;
|
|
45
|
+
while (start < word.length) {
|
|
46
|
+
let end = word.length;
|
|
47
|
+
let piece = null;
|
|
48
|
+
while (start < end) {
|
|
49
|
+
const candidate = (start > 0 ? '##' : '') + word.slice(start, end);
|
|
50
|
+
if (vocab.has(candidate)) { piece = vocab.get(candidate); break; }
|
|
51
|
+
end--;
|
|
52
|
+
}
|
|
53
|
+
if (piece === null) return [UNK];
|
|
54
|
+
pieces.push(piece);
|
|
55
|
+
start = end;
|
|
56
|
+
}
|
|
57
|
+
return pieces;
|
|
58
|
+
}
|
|
59
|
+
|
|
60
|
+
return {
|
|
61
|
+
encode(text) {
|
|
62
|
+
const ids = [CLS];
|
|
63
|
+
for (const word of basicTokenize(text)) ids.push(...wordpiece(word));
|
|
64
|
+
ids.push(SEP);
|
|
65
|
+
return ids;
|
|
66
|
+
},
|
|
67
|
+
};
|
|
68
|
+
}
|
package/dist/engine.js
ADDED
|
@@ -0,0 +1,2 @@
|
|
|
1
|
+
var P=(()=>{var _scriptName=globalThis.document?.currentScript?.src;return async function(moduleArg={}){var moduleRtn;var Module=moduleArg;var ENVIRONMENT_IS_WEB=!!globalThis.window;var ENVIRONMENT_IS_WORKER=!!globalThis.WorkerGlobalScope;var ENVIRONMENT_IS_NODE=false;var arguments_=[];var thisProgram="./this.program";var quit_=(status,toThrow)=>{throw toThrow};if(typeof __filename!="undefined"){_scriptName=__filename}else{}var scriptDirectory="";function locateFile(path){if(Module["locateFile"]){return Module["locateFile"](path,scriptDirectory)}return scriptDirectory+path}var readAsync,readBinary;if(ENVIRONMENT_IS_NODE){var fs=require("node:fs");scriptDirectory=__dirname+"/";readBinary=filename=>{filename=isFileURI(filename)?new URL(filename):filename;var ret=fs.readFileSync(filename);return ret};readAsync=async(filename,binary=true)=>{filename=isFileURI(filename)?new URL(filename):filename;var ret=fs.readFileSync(filename,binary?undefined:"utf8");return ret};if(process.argv.length>1){thisProgram=process.argv[1].replace(/\\/g,"/")}arguments_=process.argv.slice(2);quit_=(status,toThrow)=>{process.exitCode=status;throw toThrow}}else if(ENVIRONMENT_IS_WEB||ENVIRONMENT_IS_WORKER){try{scriptDirectory=new URL(".",_scriptName).href}catch{}{readAsync=async url=>{var response=await fetch(url,{credentials:"same-origin"});if(response.ok){return response.arrayBuffer()}throw new Error(response.status+" : "+response.url)}}}else{}var out=console.log.bind(console);var err=console.error.bind(console);var wasmBinary;var ABORT=false;var isFileURI=filename=>filename.startsWith("file://");var readyPromiseResolve,readyPromiseReject;var HEAP8,HEAPU8,HEAP16,HEAPU16,HEAP32,HEAPU32,HEAPF32,HEAPF64;var HEAP64,HEAPU64;var runtimeInitialized=false;function updateMemoryViews(){var b=wasmMemory.buffer;HEAP8=new Int8Array(b);HEAP16=new Int16Array(b);Module["HEAPU8"]=HEAPU8=new Uint8Array(b);HEAPU16=new Uint16Array(b);Module["HEAP32"]=HEAP32=new Int32Array(b);Module["HEAPU32"]=HEAPU32=new Uint32Array(b);Module["HEAPF32"]=HEAPF32=new Float32Array(b);HEAPF64=new Float64Array(b);HEAP64=new BigInt64Array(b);HEAPU64=new BigUint64Array(b)}function preRun(){if(Module["preRun"]){if(typeof Module["preRun"]=="function")Module["preRun"]=[Module["preRun"]];while(Module["preRun"].length){addOnPreRun(Module["preRun"].shift())}}callRuntimeCallbacks(onPreRuns)}function initRuntime(){runtimeInitialized=true;wasmExports["t"]()}function postRun(){if(Module["postRun"]){if(typeof Module["postRun"]=="function")Module["postRun"]=[Module["postRun"]];while(Module["postRun"].length){addOnPostRun(Module["postRun"].shift())}}callRuntimeCallbacks(onPostRuns)}function abort(what){Module["onAbort"]?.(what);what="Aborted("+what+")";err(what);ABORT=true;what+=". Build with -sASSERTIONS for more info.";var e=new WebAssembly.RuntimeError(what);readyPromiseReject?.(e);throw e}var wasmBinaryFile;function findWasmBinary(){return locateFile("engine.wasm")}function getBinarySync(file){if(file==wasmBinaryFile&&wasmBinary){return new Uint8Array(wasmBinary)}if(readBinary){return readBinary(file)}throw"both async and sync fetching of the wasm failed"}async function getWasmBinary(binaryFile){if(!wasmBinary){try{var response=await readAsync(binaryFile);return new Uint8Array(response)}catch{}}return getBinarySync(binaryFile)}async function instantiateArrayBuffer(binaryFile,imports){try{var binary=await getWasmBinary(binaryFile);var instance=await WebAssembly.instantiate(binary,imports);return instance}catch(reason){err(`failed to asynchronously prepare wasm: ${reason}`);abort(reason)}}async function instantiateAsync(binary,binaryFile,imports){if(!binary&&!ENVIRONMENT_IS_NODE){try{var response=fetch(binaryFile,{credentials:"same-origin"});var instantiationResult=await WebAssembly.instantiateStreaming(response,imports);return instantiationResult}catch(reason){err(`wasm streaming compile failed: ${reason}`);err("falling back to ArrayBuffer instantiation")}}return instantiateArrayBuffer(binaryFile,imports)}function getWasmImports(){var imports={a:wasmImports};return imports}async function createWasm(){function receiveInstance(instance,module){wasmExports=instance.exports;assignWasmExports(wasmExports);updateMemoryViews();return wasmExports}function receiveInstantiationResult(result){return receiveInstance(result["instance"])}var info=getWasmImports();if(Module["instantiateWasm"]){return new Promise((resolve,reject)=>{Module["instantiateWasm"](info,(inst,mod)=>{resolve(receiveInstance(inst,mod))})})}wasmBinaryFile??=findWasmBinary();var result=await instantiateAsync(wasmBinary,wasmBinaryFile,info);var exports=receiveInstantiationResult(result);return exports}class ExitStatus{name="ExitStatus";constructor(status){this.message=`Program terminated with exit(${status})`;this.status=status}}var callRuntimeCallbacks=callbacks=>{while(callbacks.length>0){callbacks.shift()(Module)}};var onPostRuns=[];var addOnPostRun=cb=>onPostRuns.push(cb);var onPreRuns=[];var addOnPreRun=cb=>onPreRuns.push(cb);var noExitRuntime=true;var stackRestore=val=>__emscripten_stack_restore(val);var stackSave=()=>_emscripten_stack_get_current();var exceptionCaught=[];var uncaughtExceptionCount=0;var ___cxa_begin_catch=ptr=>{var info=new ExceptionInfo(ptr);if(!info.get_caught()){info.set_caught(true);uncaughtExceptionCount--}info.set_rethrown(false);exceptionCaught.push(info);return ___cxa_get_exception_ptr(ptr)};var exceptionLast=0;var ___cxa_end_catch=()=>{_setThrew(0,0);var info=exceptionCaught.pop();___cxa_decrement_exception_refcount(info.excPtr);exceptionLast=0};class ExceptionInfo{constructor(excPtr){this.excPtr=excPtr;this.ptr=excPtr-24}set_type(type){HEAPU32[this.ptr+4>>2]=type}get_type(){return HEAPU32[this.ptr+4>>2]}set_destructor(destructor){HEAPU32[this.ptr+8>>2]=destructor}get_destructor(){return HEAPU32[this.ptr+8>>2]}set_caught(caught){caught=caught?1:0;HEAP8[this.ptr+12]=caught}get_caught(){return HEAP8[this.ptr+12]!=0}set_rethrown(rethrown){rethrown=rethrown?1:0;HEAP8[this.ptr+13]=rethrown}get_rethrown(){return HEAP8[this.ptr+13]!=0}init(type,destructor){this.set_adjusted_ptr(0);this.set_type(type);this.set_destructor(destructor)}set_adjusted_ptr(adjustedPtr){HEAPU32[this.ptr+16>>2]=adjustedPtr}get_adjusted_ptr(){return HEAPU32[this.ptr+16>>2]}}var setTempRet0=val=>__emscripten_tempret_set(val);var findMatchingCatch=args=>{var thrown=exceptionLast;if(!thrown){setTempRet0(0);return 0}var info=new ExceptionInfo(thrown);info.set_adjusted_ptr(thrown);var thrownType=info.get_type();if(!thrownType){setTempRet0(0);return thrown}for(var caughtType of args){if(caughtType===0||caughtType===thrownType){break}var adjusted_ptr_addr=info.ptr+16;if(___cxa_can_catch(caughtType,thrownType,adjusted_ptr_addr)){setTempRet0(caughtType);return thrown}}setTempRet0(thrownType);return thrown};var ___cxa_find_matching_catch_2=()=>findMatchingCatch([]);var ___cxa_find_matching_catch_3=arg0=>findMatchingCatch([arg0]);var ___cxa_throw=(ptr,type,destructor)=>{var info=new ExceptionInfo(ptr);info.init(type,destructor);___cxa_increment_exception_refcount(ptr);exceptionLast=ptr;uncaughtExceptionCount++;throw exceptionLast};var ___resumeException=ptr=>{if(!exceptionLast){exceptionLast=ptr}throw exceptionLast};var __abort_js=()=>abort("");var getHeapMax=()=>2147483648;var alignMemory=(size,alignment)=>Math.ceil(size/alignment)*alignment;var growMemory=size=>{var oldHeapSize=wasmMemory.buffer.byteLength;var pages=(size-oldHeapSize+65535)/65536|0;try{wasmMemory.grow(pages);updateMemoryViews();return 1}catch(e){}};var _emscripten_resize_heap=requestedSize=>{var oldSize=HEAPU8.length;requestedSize>>>=0;var maxHeapSize=getHeapMax();if(requestedSize>maxHeapSize){return false}for(var cutDown=1;cutDown<=4;cutDown*=2){var overGrownHeapSize=oldSize*(1+.2/cutDown);overGrownHeapSize=Math.min(overGrownHeapSize,requestedSize+100663296);var newSize=Math.min(maxHeapSize,alignMemory(Math.max(requestedSize,overGrownHeapSize),65536));var replacement=growMemory(newSize);if(replacement){return true}}return false};var wasmTableMirror=[];var getWasmTableEntry=funcPtr=>{var func=wasmTableMirror[funcPtr];if(!func){wasmTableMirror[funcPtr]=func=wasmTable.get(funcPtr)}return func};var getCFunc=ident=>{var func=Module["_"+ident];return func};var writeArrayToMemory=(array,buffer)=>{HEAP8.set(array,buffer)};var lengthBytesUTF8=str=>{var len=0;for(var i=0;i<str.length;++i){var c=str.charCodeAt(i);if(c<=127){len++}else if(c<=2047){len+=2}else if(c>=55296&&c<=57343){len+=4;++i}else{len+=3}}return len};var stringToUTF8Array=(str,heap,outIdx,maxBytesToWrite)=>{if(!(maxBytesToWrite>0))return 0;var startIdx=outIdx;var endIdx=outIdx+maxBytesToWrite-1;for(var i=0;i<str.length;++i){var u=str.codePointAt(i);if(u<=127){if(outIdx>=endIdx)break;heap[outIdx++]=u}else if(u<=2047){if(outIdx+1>=endIdx)break;heap[outIdx++]=192|u>>6;heap[outIdx++]=128|u&63}else if(u<=65535){if(outIdx+2>=endIdx)break;heap[outIdx++]=224|u>>12;heap[outIdx++]=128|u>>6&63;heap[outIdx++]=128|u&63}else{if(outIdx+3>=endIdx)break;heap[outIdx++]=240|u>>18;heap[outIdx++]=128|u>>12&63;heap[outIdx++]=128|u>>6&63;heap[outIdx++]=128|u&63;i++}}heap[outIdx]=0;return outIdx-startIdx};var stringToUTF8=(str,outPtr,maxBytesToWrite)=>stringToUTF8Array(str,HEAPU8,outPtr,maxBytesToWrite);var stackAlloc=sz=>__emscripten_stack_alloc(sz);var stringToUTF8OnStack=str=>{var size=lengthBytesUTF8(str)+1;var ret=stackAlloc(size);stringToUTF8(str,ret,size);return ret};var UTF8Decoder=globalThis.TextDecoder&&new TextDecoder;var findStringEnd=(heapOrArray,idx,maxBytesToRead,ignoreNul)=>{var maxIdx=idx+maxBytesToRead;if(ignoreNul)return maxIdx;while(heapOrArray[idx]&&!(idx>=maxIdx))++idx;return idx};var UTF8ArrayToString=(heapOrArray,idx=0,maxBytesToRead,ignoreNul)=>{var endPtr=findStringEnd(heapOrArray,idx,maxBytesToRead,ignoreNul);if(endPtr-idx>16&&heapOrArray.buffer&&UTF8Decoder){return UTF8Decoder.decode(heapOrArray.subarray(idx,endPtr))}var str="";while(idx<endPtr){var u0=heapOrArray[idx++];if(!(u0&128)){str+=String.fromCharCode(u0);continue}var u1=heapOrArray[idx++]&63;if((u0&224)==192){str+=String.fromCharCode((u0&31)<<6|u1);continue}var u2=heapOrArray[idx++]&63;if((u0&240)==224){u0=(u0&15)<<12|u1<<6|u2}else{u0=(u0&7)<<18|u1<<12|u2<<6|heapOrArray[idx++]&63}if(u0<65536){str+=String.fromCharCode(u0)}else{var ch=u0-65536;str+=String.fromCharCode(55296|ch>>10,56320|ch&1023)}}return str};var UTF8ToString=(ptr,maxBytesToRead,ignoreNul)=>ptr?UTF8ArrayToString(HEAPU8,ptr,maxBytesToRead,ignoreNul):"";var ccall=(ident,returnType,argTypes,args,opts)=>{var toC={string:str=>{var ret=0;if(str!==null&&str!==undefined&&str!==0){ret=stringToUTF8OnStack(str)}return ret},array:arr=>{var ret=stackAlloc(arr.length);writeArrayToMemory(arr,ret);return ret}};function convertReturnValue(ret){if(returnType==="string"){return UTF8ToString(ret)}if(returnType==="boolean")return Boolean(ret);return ret}var func=getCFunc(ident);var cArgs=[];var stack=0;if(args){for(var i=0;i<args.length;i++){var converter=toC[argTypes[i]];if(converter){if(stack===0)stack=stackSave();cArgs[i]=converter(args[i])}else{cArgs[i]=args[i]}}}var ret=func(...cArgs);function onDone(ret){if(stack!==0)stackRestore(stack);return convertReturnValue(ret)}ret=onDone(ret);return ret};var cwrap=(ident,returnType,argTypes,opts)=>{var numericArgs=!argTypes||argTypes.every(type=>type==="number"||type==="boolean");var numericRet=returnType!=="string";if(numericRet&&numericArgs&&!opts){return getCFunc(ident)}return(...args)=>ccall(ident,returnType,argTypes,args,opts)};{if(Module["noExitRuntime"])noExitRuntime=Module["noExitRuntime"];if(Module["print"])out=Module["print"];if(Module["printErr"])err=Module["printErr"];if(Module["wasmBinary"])wasmBinary=Module["wasmBinary"];if(Module["arguments"])arguments_=Module["arguments"];if(Module["thisProgram"])thisProgram=Module["thisProgram"];if(Module["preInit"]){if(typeof Module["preInit"]=="function")Module["preInit"]=[Module["preInit"]];while(Module["preInit"].length>0){Module["preInit"].shift()()}}}Module["ccall"]=ccall;Module["cwrap"]=cwrap;var _pancake_init,_pancake_add,_pancake_bulk_insert,_pancake_query,_pancake_query_filtered,_pancake_delete,_pancake_compact,_pancake_compact_remap,_pancake_count,_pancake_memory,_pancake_ghost_count,_pancake_ghost_ratio,_pancake_export,_pancake_import,_pancake_dispose,_pancake_dimension,_emsc_malloc,_emsc_free,_pancake_profile_print,_pancake_profile_reset,_pancake_sketch_scan,_pancake_shutdown_all,_shutdown_all,_setThrew,__emscripten_tempret_set,__emscripten_stack_restore,__emscripten_stack_alloc,_emscripten_stack_get_current,___cxa_increment_exception_refcount,___cxa_decrement_exception_refcount,___cxa_can_catch,___cxa_get_exception_ptr,memory,__indirect_function_table,wasmMemory,wasmTable;function assignWasmExports(wasmExports){_pancake_init=Module["_pancake_init"]=wasmExports["u"];_pancake_add=Module["_pancake_add"]=wasmExports["v"];_pancake_bulk_insert=Module["_pancake_bulk_insert"]=wasmExports["x"];_pancake_query=Module["_pancake_query"]=wasmExports["y"];_pancake_query_filtered=Module["_pancake_query_filtered"]=wasmExports["z"];_pancake_delete=Module["_pancake_delete"]=wasmExports["A"];_pancake_compact=Module["_pancake_compact"]=wasmExports["B"];_pancake_compact_remap=Module["_pancake_compact_remap"]=wasmExports["C"];_pancake_count=Module["_pancake_count"]=wasmExports["D"];_pancake_memory=Module["_pancake_memory"]=wasmExports["E"];_pancake_ghost_count=Module["_pancake_ghost_count"]=wasmExports["F"];_pancake_ghost_ratio=Module["_pancake_ghost_ratio"]=wasmExports["G"];_pancake_export=Module["_pancake_export"]=wasmExports["H"];_pancake_import=Module["_pancake_import"]=wasmExports["I"];_pancake_dispose=Module["_pancake_dispose"]=wasmExports["J"];_pancake_dimension=Module["_pancake_dimension"]=wasmExports["K"];_emsc_malloc=Module["_emsc_malloc"]=wasmExports["L"];_emsc_free=Module["_emsc_free"]=wasmExports["M"];_pancake_profile_print=Module["_pancake_profile_print"]=wasmExports["N"];_pancake_profile_reset=Module["_pancake_profile_reset"]=wasmExports["O"];_pancake_sketch_scan=Module["_pancake_sketch_scan"]=wasmExports["P"];_pancake_shutdown_all=Module["_pancake_shutdown_all"]=wasmExports["Q"];_shutdown_all=Module["_shutdown_all"]=wasmExports["R"];_setThrew=wasmExports["S"];__emscripten_tempret_set=wasmExports["T"];__emscripten_stack_restore=wasmExports["U"];__emscripten_stack_alloc=wasmExports["V"];_emscripten_stack_get_current=wasmExports["W"];___cxa_increment_exception_refcount=wasmExports["X"];___cxa_decrement_exception_refcount=wasmExports["Y"];___cxa_can_catch=wasmExports["Z"];___cxa_get_exception_ptr=wasmExports["_"];memory=wasmMemory=wasmExports["s"];__indirect_function_table=wasmTable=wasmExports["w"]}var wasmImports={l:___cxa_begin_catch,o:___cxa_end_catch,a:___cxa_find_matching_catch_2,i:___cxa_find_matching_catch_3,n:___cxa_throw,d:___resumeException,p:__abort_js,q:_emscripten_resize_heap,c:invoke_ii,m:invoke_iii,k:invoke_iiii,f:invoke_iiiiii,b:invoke_v,g:invoke_vii,e:invoke_viii,h:invoke_viiii,j:invoke_viiiii,r:invoke_viiiiiii};function invoke_ii(index,a1){var sp=stackSave();try{return getWasmTableEntry(index)(a1)}catch(e){stackRestore(sp);if(e!==e+0)throw e;_setThrew(1,0)}}function invoke_iiii(index,a1,a2,a3){var sp=stackSave();try{return getWasmTableEntry(index)(a1,a2,a3)}catch(e){stackRestore(sp);if(e!==e+0)throw e;_setThrew(1,0)}}function invoke_vii(index,a1,a2){var sp=stackSave();try{getWasmTableEntry(index)(a1,a2)}catch(e){stackRestore(sp);if(e!==e+0)throw e;_setThrew(1,0)}}function invoke_v(index){var sp=stackSave();try{getWasmTableEntry(index)()}catch(e){stackRestore(sp);if(e!==e+0)throw e;_setThrew(1,0)}}function invoke_viii(index,a1,a2,a3){var sp=stackSave();try{getWasmTableEntry(index)(a1,a2,a3)}catch(e){stackRestore(sp);if(e!==e+0)throw e;_setThrew(1,0)}}function invoke_iii(index,a1,a2){var sp=stackSave();try{return getWasmTableEntry(index)(a1,a2)}catch(e){stackRestore(sp);if(e!==e+0)throw e;_setThrew(1,0)}}function invoke_viiiiiii(index,a1,a2,a3,a4,a5,a6,a7){var sp=stackSave();try{getWasmTableEntry(index)(a1,a2,a3,a4,a5,a6,a7)}catch(e){stackRestore(sp);if(e!==e+0)throw e;_setThrew(1,0)}}function invoke_viiiii(index,a1,a2,a3,a4,a5){var sp=stackSave();try{getWasmTableEntry(index)(a1,a2,a3,a4,a5)}catch(e){stackRestore(sp);if(e!==e+0)throw e;_setThrew(1,0)}}function invoke_viiii(index,a1,a2,a3,a4){var sp=stackSave();try{getWasmTableEntry(index)(a1,a2,a3,a4)}catch(e){stackRestore(sp);if(e!==e+0)throw e;_setThrew(1,0)}}function invoke_iiiiii(index,a1,a2,a3,a4,a5){var sp=stackSave();try{return getWasmTableEntry(index)(a1,a2,a3,a4,a5)}catch(e){stackRestore(sp);if(e!==e+0)throw e;_setThrew(1,0)}}function run(){preRun();function doRun(){Module["calledRun"]=true;if(ABORT)return;initRuntime();readyPromiseResolve?.(Module);Module["onRuntimeInitialized"]?.();postRun()}if(Module["setStatus"]){Module["setStatus"]("Running...");setTimeout(()=>{setTimeout(()=>Module["setStatus"](""),1);doRun()},1)}else{doRun()}}var wasmExports;wasmExports=await (createWasm());run();if(runtimeInitialized){moduleRtn=Module}else{moduleRtn=new Promise((resolve,reject)=>{readyPromiseResolve=resolve;readyPromiseReject=reject})}
|
|
2
|
+
;return moduleRtn}})();if(typeof exports==="object"&&typeof module==="object"){module.exports=P;module.exports.default=P}else if(typeof define==="function"&&define["amd"])define([],()=>P);
|