sift-light 1.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +0 -0
- package/CONTRIBUTING.md +19 -0
- package/LICENSE +661 -0
- package/README.md +177 -0
- package/README.zh-CN.md +177 -0
- package/SECURITY.md +25 -0
- package/package.json +142 -0
- package/plugins/sift-light/.claude-plugin/plugin.json +10 -0
- package/plugins/sift-light/.codex-plugin/plugin.json +20 -0
- package/plugins/sift-light/.mcp.json +24 -0
- package/plugins/sift-light/LICENSE +661 -0
- package/plugins/sift-light/concept-worker.mjs +567 -0
- package/plugins/sift-light/hooks/THIRD_PARTY_NOTICES.txt +70 -0
- package/plugins/sift-light/hooks/hooks.json +16 -0
- package/plugins/sift-light/hooks/search-policy.mjs +4113 -0
- package/plugins/sift-light/hooks/tree-sitter-bash.wasm +0 -0
- package/plugins/sift-light/hooks/tree-sitter-powershell.wasm +0 -0
- package/plugins/sift-light/hooks/tree-sitter.wasm +0 -0
- package/plugins/sift-light/kimi.plugin.json +36 -0
- package/plugins/sift-light/mcp-server.mjs +1 -0
- package/plugins/sift-light/omp-extension.mjs +20914 -0
- package/plugins/sift-light/package.json +11 -0
- package/plugins/sift-light/syntax-worker.mjs +280 -0
- package/plugins/sift-light/syntax-worker.toml +2 -0
- package/src/analysis-evidence.ts +43 -0
- package/src/analysis-limits.ts +27 -0
- package/src/analysis-store.ts +559 -0
- package/src/analysis-term-pages.ts +73 -0
- package/src/analysis-types.ts +162 -0
- package/src/audit-search.ts +391 -0
- package/src/capped-lines.ts +112 -0
- package/src/concept-embedding-cache.ts +186 -0
- package/src/concept-inference.ts +218 -0
- package/src/concept-model.ts +96 -0
- package/src/concept-search.ts +577 -0
- package/src/concept-setup.ts +75 -0
- package/src/concept-source-generation.ts +268 -0
- package/src/concept-worker.mjs +567 -0
- package/src/concept-worker.ts +210 -0
- package/src/config-reader.ts +283 -0
- package/src/config.ts +37 -0
- package/src/context-budget.ts +26 -0
- package/src/discovery-errors.ts +2 -0
- package/src/errors.ts +95 -0
- package/src/evidence-candidates.ts +344 -0
- package/src/evidence-ranking.ts +48 -0
- package/src/evidence-service.ts +1047 -0
- package/src/evidence-validation.ts +466 -0
- package/src/evidence-validity.ts +33 -0
- package/src/excerpt.ts +53 -0
- package/src/file-discovery.ts +150 -0
- package/src/file-metadata-filter.ts +46 -0
- package/src/format.ts +407 -0
- package/src/git-diff.ts +195 -0
- package/src/git-process.ts +189 -0
- package/src/git-repository.ts +367 -0
- package/src/git-source.ts +385 -0
- package/src/historical-paths.ts +205 -0
- package/src/hybrid-search.ts +340 -0
- package/src/import-model.ts +611 -0
- package/src/import-navigation.ts +100 -0
- package/src/import-resolution.ts +240 -0
- package/src/index.ts +97 -0
- package/src/inspect-batch.ts +265 -0
- package/src/inspect.ts +271 -0
- package/src/language-capabilities.ts +157 -0
- package/src/language-capability-definitions.ts +204 -0
- package/src/literal-search.ts +25 -0
- package/src/mcp-cli.ts +21 -0
- package/src/mcp-model-output.ts +120 -0
- package/src/mcp-output.ts +9 -0
- package/src/mcp-semantic-judge.ts +39 -0
- package/src/mcp-server.mjs +14064 -0
- package/src/mcp-server.ts +157 -0
- package/src/mcp-stdio.ts +114 -0
- package/src/mcp.ts +631 -0
- package/src/model-error.ts +120 -0
- package/src/multi-term-search.ts +122 -0
- package/src/omp-index.ts +277 -0
- package/src/omp-marketplace-entry.ts +7 -0
- package/src/operation-lifecycle.ts +413 -0
- package/src/operation-output.ts +100 -0
- package/src/owned-parallel.ts +22 -0
- package/src/owned-process.ts +102 -0
- package/src/owned-task-queue.ts +38 -0
- package/src/package-version.ts +3 -0
- package/src/path-policy.ts +203 -0
- package/src/pi-search-policy.ts +33 -0
- package/src/prompt-guidelines.ts +54 -0
- package/src/python-outline.ts +231 -0
- package/src/record-value.ts +3 -0
- package/src/redaction.ts +95 -0
- package/src/request-contract-catalog.ts +215 -0
- package/src/request-contract-error.ts +96 -0
- package/src/request-contract.ts +481 -0
- package/src/request.ts +132 -0
- package/src/result-statistics.ts +178 -0
- package/src/rg.ts +574 -0
- package/src/ripgrep-diagnostics.ts +122 -0
- package/src/ripgrep-executable.ts +43 -0
- package/src/runtime.ts +53 -0
- package/src/scan-revisions.ts +148 -0
- package/src/script-runtime.ts +11 -0
- package/src/search-policy-commands.ts +170 -0
- package/src/search-policy-hook.ts +61 -0
- package/src/search-policy-recovery.ts +388 -0
- package/src/search-policy-shell.ts +280 -0
- package/src/search-policy.ts +102 -0
- package/src/search-retention.ts +77 -0
- package/src/semantic-judge-batches.ts +135 -0
- package/src/semantic-judge.ts +519 -0
- package/src/service.ts +945 -0
- package/src/session-summary.ts +100 -0
- package/src/snapshot-store.ts +203 -0
- package/src/source-access.ts +288 -0
- package/src/source-continuations.ts +168 -0
- package/src/source-document.ts +277 -0
- package/src/source-inspection.ts +731 -0
- package/src/source-pages.ts +112 -0
- package/src/source.ts +286 -0
- package/src/structural-search.ts +125 -0
- package/src/structure.ts +259 -0
- package/src/summary-previews.ts +96 -0
- package/src/summary.ts +79 -0
- package/src/syntax-facts.ts +464 -0
- package/src/syntax-fields.ts +161 -0
- package/src/syntax-search.ts +361 -0
- package/src/syntax-tree.ts +30 -0
- package/src/syntax-types.ts +72 -0
- package/src/syntax-worker.mjs +280 -0
- package/src/syntax-worker.toml +2 -0
- package/src/syntax-worker.ts +143 -0
- package/src/syntax.ts +259 -0
- package/src/test-navigation-facts.ts +261 -0
- package/src/test-navigation.ts +543 -0
- package/src/tool-schema.ts +321 -0
- package/src/tui/dashboard.ts +107 -0
- package/src/tui/layout.ts +291 -0
- package/src/tui/palette.ts +73 -0
- package/src/tui/presentation.ts +240 -0
- package/src/tui/renderers.ts +106 -0
- package/src/types.ts +368 -0
- package/src/validation-output.ts +148 -0
- package/src/validation-types.ts +48 -0
- package/src/workspace-files.ts +142 -0
|
@@ -0,0 +1,567 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
|
|
3
|
+
// src/concept-worker.ts
|
|
4
|
+
import { env, pipeline } from "@huggingface/transformers";
|
|
5
|
+
import { isUtf8 } from "node:buffer";
|
|
6
|
+
|
|
7
|
+
// src/concept-embedding-cache.ts
|
|
8
|
+
import { createHash as createHash2, randomUUID } from "node:crypto";
|
|
9
|
+
import { mkdir, readFile, readdir, rename, rm, stat, writeFile } from "node:fs/promises";
|
|
10
|
+
import { join as join2 } from "node:path";
|
|
11
|
+
|
|
12
|
+
// src/concept-model.ts
|
|
13
|
+
import { createHash } from "node:crypto";
|
|
14
|
+
import { createReadStream } from "node:fs";
|
|
15
|
+
import { homedir } from "node:os";
|
|
16
|
+
import { join, resolve } from "node:path";
|
|
17
|
+
|
|
18
|
+
// src/errors.ts
|
|
19
|
+
class SiftLightError extends Error {
|
|
20
|
+
constructor(message, options) {
|
|
21
|
+
super(message, options);
|
|
22
|
+
this.name = "SiftLightError";
|
|
23
|
+
}
|
|
24
|
+
}
|
|
25
|
+
|
|
26
|
+
// src/concept-model.ts
|
|
27
|
+
var CONCEPT_MODEL = "Xenova/multilingual-e5-small";
|
|
28
|
+
var CONCEPT_REVISION = "761b726dd34fb83930e26aab4e9ac3899aa1fa78";
|
|
29
|
+
var MAX_CONCEPT_CHARS = 1000;
|
|
30
|
+
var CONCEPT_MODEL_TOKENS = 512;
|
|
31
|
+
var CONCEPT_WINDOW_OVERLAP_TOKENS = 64;
|
|
32
|
+
var CONCEPT_EMBEDDING_DIMENSIONS = 384;
|
|
33
|
+
var CONCEPT_CACHE_VERSION = 1;
|
|
34
|
+
var CONCEPT_CACHE_MAX_BYTES = 512 * 1024 * 1024;
|
|
35
|
+
var CONCEPT_TIMEOUT_MS = 10 * 60000;
|
|
36
|
+
var MAX_CONCEPT_TIMEOUT_MS = 60 * 60000;
|
|
37
|
+
var MAX_CONCEPT_WORKER_INPUT_BYTES = 64 * 1024 * 1024;
|
|
38
|
+
var MAX_CONCEPT_WORKER_OUTPUT_BYTES = 4 * 1024 * 1024;
|
|
39
|
+
var CONCEPT_ASSETS = [
|
|
40
|
+
{
|
|
41
|
+
path: "config.json",
|
|
42
|
+
bytes: 658,
|
|
43
|
+
sha256: "cb99455288675345e1a4f411438d5d0adbba5fbd3a67ea4fb03c015433b996c1"
|
|
44
|
+
},
|
|
45
|
+
{
|
|
46
|
+
path: "tokenizer_config.json",
|
|
47
|
+
bytes: 443,
|
|
48
|
+
sha256: "a1d6bc8734a6f635dc158508bef000f8e2e5a759c7d92f984b2c86e5ff53425b"
|
|
49
|
+
},
|
|
50
|
+
{
|
|
51
|
+
path: "tokenizer.json",
|
|
52
|
+
bytes: 17082730,
|
|
53
|
+
sha256: "0b44a9d7b51c3c62626640cda0e2c2f70fdacdc25bbbd68038369d14ebdf4c39"
|
|
54
|
+
},
|
|
55
|
+
{
|
|
56
|
+
path: "onnx/model_quantized.onnx",
|
|
57
|
+
bytes: 118308185,
|
|
58
|
+
sha256: "f80102d3f2a1229f387d3c81909990d8945513e347b0eab049f7de3c6f98c193"
|
|
59
|
+
}
|
|
60
|
+
];
|
|
61
|
+
function conceptModelDirectory() {
|
|
62
|
+
return resolve(process.env.SIFT_LIGHT_MODEL_DIR ?? join(homedir(), ".cache", "sift-light", "models"), CONCEPT_REVISION);
|
|
63
|
+
}
|
|
64
|
+
function conceptCacheDirectory() {
|
|
65
|
+
return resolve(process.env.SIFT_LIGHT_MODEL_DIR ?? join(homedir(), ".cache", "sift-light", "models"), "concept-cache", `${CONCEPT_REVISION}-v${String(CONCEPT_CACHE_VERSION)}`);
|
|
66
|
+
}
|
|
67
|
+
async function verifyConceptModel(directory = conceptModelDirectory()) {
|
|
68
|
+
for (const asset of CONCEPT_ASSETS) {
|
|
69
|
+
const hash = createHash("sha256");
|
|
70
|
+
let size = 0;
|
|
71
|
+
try {
|
|
72
|
+
for await (const chunk of createReadStream(join(directory, asset.path))) {
|
|
73
|
+
const bytes = Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk);
|
|
74
|
+
size += bytes.length;
|
|
75
|
+
if (size > asset.bytes)
|
|
76
|
+
throw new Error("Asset exceeds pinned size");
|
|
77
|
+
hash.update(bytes);
|
|
78
|
+
}
|
|
79
|
+
if (size !== asset.bytes || hash.digest("hex") !== asset.sha256)
|
|
80
|
+
throw new Error("Pinned hash mismatch");
|
|
81
|
+
} catch (error) {
|
|
82
|
+
throw new SiftLightError(`Local concept model is missing or invalid (${asset.path}); run sift-light-model --install-model explicitly`, { cause: error });
|
|
83
|
+
}
|
|
84
|
+
}
|
|
85
|
+
}
|
|
86
|
+
|
|
87
|
+
// src/record-value.ts
|
|
88
|
+
function isRecordValue(value) {
|
|
89
|
+
return value !== null && typeof value === "object" && !Array.isArray(value);
|
|
90
|
+
}
|
|
91
|
+
|
|
92
|
+
// src/concept-embedding-cache.ts
|
|
93
|
+
function errorCode(error) {
|
|
94
|
+
return typeof error === "object" && error !== null && "code" in error ? error.code : undefined;
|
|
95
|
+
}
|
|
96
|
+
function conceptEmbeddingKey(role, text) {
|
|
97
|
+
return createHash2("sha256").update(role).update("\x00").update(text).digest("hex");
|
|
98
|
+
}
|
|
99
|
+
function cachePath(root, key) {
|
|
100
|
+
return join2(root, key.slice(0, 2), `${key}.json`);
|
|
101
|
+
}
|
|
102
|
+
function encodeVector(vector) {
|
|
103
|
+
const buffer = Buffer.allocUnsafe(vector.length * Float32Array.BYTES_PER_ELEMENT);
|
|
104
|
+
for (const [index, value] of vector.entries())
|
|
105
|
+
buffer.writeFloatLE(value, index * Float32Array.BYTES_PER_ELEMENT);
|
|
106
|
+
return buffer.toString("base64");
|
|
107
|
+
}
|
|
108
|
+
function decodeVector(encoded) {
|
|
109
|
+
const buffer = Buffer.from(encoded, "base64");
|
|
110
|
+
if (buffer.length !== CONCEPT_EMBEDDING_DIMENSIONS * Float32Array.BYTES_PER_ELEMENT)
|
|
111
|
+
return;
|
|
112
|
+
const vector = Array.from({ length: CONCEPT_EMBEDDING_DIMENSIONS }, (_, index) => buffer.readFloatLE(index * Float32Array.BYTES_PER_ELEMENT));
|
|
113
|
+
return vector.every(Number.isFinite) ? vector : undefined;
|
|
114
|
+
}
|
|
115
|
+
function storedEmbedding(value, key) {
|
|
116
|
+
if (!isRecordValue(value) || value.version !== CONCEPT_CACHE_VERSION || value.key !== key || !Array.isArray(value.windows) || value.windows.length === 0)
|
|
117
|
+
return;
|
|
118
|
+
const windows = [];
|
|
119
|
+
for (const item of value.windows) {
|
|
120
|
+
if (!isRecordValue(item) || typeof item.start !== "number" || !Number.isSafeInteger(item.start) || item.start < 0 || typeof item.end !== "number" || !Number.isSafeInteger(item.end) || item.end <= item.start || typeof item.vector !== "string")
|
|
121
|
+
return;
|
|
122
|
+
const vector = decodeVector(item.vector);
|
|
123
|
+
if (!vector)
|
|
124
|
+
return;
|
|
125
|
+
windows.push({ start: item.start, end: item.end, vector });
|
|
126
|
+
}
|
|
127
|
+
return { key, windows };
|
|
128
|
+
}
|
|
129
|
+
async function readConceptEmbedding(root, key) {
|
|
130
|
+
try {
|
|
131
|
+
return storedEmbedding(JSON.parse(await readFile(cachePath(root, key), "utf8")), key);
|
|
132
|
+
} catch (error) {
|
|
133
|
+
const code = errorCode(error);
|
|
134
|
+
if (code === "ENOENT" || error instanceof SyntaxError)
|
|
135
|
+
return;
|
|
136
|
+
throw error;
|
|
137
|
+
}
|
|
138
|
+
}
|
|
139
|
+
async function writeConceptEmbedding(root, embedding, stagingRoot = root) {
|
|
140
|
+
const directory = join2(root, embedding.key.slice(0, 2));
|
|
141
|
+
await mkdir(stagingRoot, { recursive: true });
|
|
142
|
+
await mkdir(directory, { recursive: true });
|
|
143
|
+
const destination = cachePath(root, embedding.key);
|
|
144
|
+
const temporary = join2(stagingRoot, `.${embedding.key}.${randomUUID()}.tmp`);
|
|
145
|
+
const stored = {
|
|
146
|
+
version: CONCEPT_CACHE_VERSION,
|
|
147
|
+
key: embedding.key,
|
|
148
|
+
windows: embedding.windows.map((window) => ({
|
|
149
|
+
start: window.start,
|
|
150
|
+
end: window.end,
|
|
151
|
+
vector: encodeVector(window.vector)
|
|
152
|
+
}))
|
|
153
|
+
};
|
|
154
|
+
try {
|
|
155
|
+
await writeFile(temporary, JSON.stringify(stored), { flag: "wx", mode: 384 });
|
|
156
|
+
try {
|
|
157
|
+
await rename(temporary, destination);
|
|
158
|
+
} catch (error) {
|
|
159
|
+
if (errorCode(error) !== "EEXIST")
|
|
160
|
+
throw error;
|
|
161
|
+
if (await readConceptEmbedding(root, embedding.key))
|
|
162
|
+
return;
|
|
163
|
+
await rm(destination, { force: true });
|
|
164
|
+
try {
|
|
165
|
+
await rename(temporary, destination);
|
|
166
|
+
} catch (replacementError) {
|
|
167
|
+
if (errorCode(replacementError) !== "EEXIST")
|
|
168
|
+
throw replacementError;
|
|
169
|
+
}
|
|
170
|
+
}
|
|
171
|
+
} finally {
|
|
172
|
+
await rm(temporary, { force: true });
|
|
173
|
+
}
|
|
174
|
+
}
|
|
175
|
+
async function enforceConceptCacheLimit(root, maximumBytes = CONCEPT_CACHE_MAX_BYTES) {
|
|
176
|
+
const entries = [];
|
|
177
|
+
let shards;
|
|
178
|
+
try {
|
|
179
|
+
shards = await readdir(root, { withFileTypes: true });
|
|
180
|
+
} catch (error) {
|
|
181
|
+
if (errorCode(error) === "ENOENT")
|
|
182
|
+
return 0;
|
|
183
|
+
throw error;
|
|
184
|
+
}
|
|
185
|
+
for (const shard of shards) {
|
|
186
|
+
if (!shard.isDirectory() || !/^[0-9a-f]{2}$/u.test(shard.name))
|
|
187
|
+
continue;
|
|
188
|
+
const directory = join2(root, shard.name);
|
|
189
|
+
let files;
|
|
190
|
+
try {
|
|
191
|
+
files = await readdir(directory, { withFileTypes: true });
|
|
192
|
+
} catch (error) {
|
|
193
|
+
if (errorCode(error) === "ENOENT")
|
|
194
|
+
continue;
|
|
195
|
+
throw error;
|
|
196
|
+
}
|
|
197
|
+
for (const file of files) {
|
|
198
|
+
if (!file.isFile() || !file.name.endsWith(".json"))
|
|
199
|
+
continue;
|
|
200
|
+
const path = join2(directory, file.name);
|
|
201
|
+
let metadata;
|
|
202
|
+
try {
|
|
203
|
+
metadata = await stat(path);
|
|
204
|
+
} catch (error) {
|
|
205
|
+
if (errorCode(error) === "ENOENT")
|
|
206
|
+
continue;
|
|
207
|
+
throw error;
|
|
208
|
+
}
|
|
209
|
+
entries.push({ path, bytes: metadata.size, modified: metadata.mtimeMs });
|
|
210
|
+
}
|
|
211
|
+
}
|
|
212
|
+
let bytes = entries.reduce((total, item) => total + item.bytes, 0);
|
|
213
|
+
if (bytes <= maximumBytes)
|
|
214
|
+
return bytes;
|
|
215
|
+
for (const item of entries.toSorted((a, b) => a.modified - b.modified || a.path.localeCompare(b.path))) {
|
|
216
|
+
await rm(item.path, { force: true });
|
|
217
|
+
bytes -= item.bytes;
|
|
218
|
+
if (bytes <= maximumBytes)
|
|
219
|
+
break;
|
|
220
|
+
}
|
|
221
|
+
return bytes;
|
|
222
|
+
}
|
|
223
|
+
|
|
224
|
+
// src/concept-inference.ts
|
|
225
|
+
var INFERENCE_BATCH_SIZE = 16;
|
|
226
|
+
function safeUtf16End(text, end) {
|
|
227
|
+
if (end <= 0 || end >= text.length)
|
|
228
|
+
return end;
|
|
229
|
+
const previous = text.charCodeAt(end - 1);
|
|
230
|
+
return previous >= 55296 && previous <= 56319 ? end - 1 : end;
|
|
231
|
+
}
|
|
232
|
+
function tokenCount(extractor, text) {
|
|
233
|
+
return extractor.tokenizer.encode(text).length;
|
|
234
|
+
}
|
|
235
|
+
function advanceLow(low, middle) {
|
|
236
|
+
return Math.max(low + 1, middle + 1);
|
|
237
|
+
}
|
|
238
|
+
function nextUtf16Boundary(text, start) {
|
|
239
|
+
const next = start + 1;
|
|
240
|
+
if (next >= text.length)
|
|
241
|
+
return next;
|
|
242
|
+
const first = text.charCodeAt(start);
|
|
243
|
+
const second = text.charCodeAt(next);
|
|
244
|
+
return first >= 55296 && first <= 56319 && second >= 56320 && second <= 57343 ? next + 1 : next;
|
|
245
|
+
}
|
|
246
|
+
function binarySearchBudget(span) {
|
|
247
|
+
return 2 * Math.max(span, 1) + 32;
|
|
248
|
+
}
|
|
249
|
+
function maximumTokenSafeEnd(extractor, prefix, text, start) {
|
|
250
|
+
let low = start + 1;
|
|
251
|
+
let high = text.length;
|
|
252
|
+
let accepted = start;
|
|
253
|
+
let iterations = 0;
|
|
254
|
+
const maxIterations = binarySearchBudget(high - low + 1);
|
|
255
|
+
while (low <= high) {
|
|
256
|
+
iterations += 1;
|
|
257
|
+
if (iterations > maxIterations)
|
|
258
|
+
throw new Error("Concept token window search failed to make progress");
|
|
259
|
+
const middle = safeUtf16End(text, Math.floor((low + high) / 2));
|
|
260
|
+
if (middle <= start) {
|
|
261
|
+
low += 1;
|
|
262
|
+
continue;
|
|
263
|
+
}
|
|
264
|
+
if (tokenCount(extractor, `${prefix}${text.slice(start, middle)}`) <= CONCEPT_MODEL_TOKENS) {
|
|
265
|
+
accepted = middle;
|
|
266
|
+
low = advanceLow(low, middle);
|
|
267
|
+
} else {
|
|
268
|
+
high = middle - 1;
|
|
269
|
+
}
|
|
270
|
+
}
|
|
271
|
+
return accepted;
|
|
272
|
+
}
|
|
273
|
+
function overlapStart(extractor, prefix, text, start, end) {
|
|
274
|
+
let low = start + 1;
|
|
275
|
+
let high = end;
|
|
276
|
+
let accepted = end;
|
|
277
|
+
let iterations = 0;
|
|
278
|
+
const maxIterations = binarySearchBudget(high - low + 1);
|
|
279
|
+
while (low <= high) {
|
|
280
|
+
iterations += 1;
|
|
281
|
+
if (iterations > maxIterations)
|
|
282
|
+
throw new Error("Concept overlap search failed to make progress");
|
|
283
|
+
const middle = safeUtf16End(text, Math.floor((low + high) / 2));
|
|
284
|
+
const tokens = tokenCount(extractor, `${prefix}${text.slice(middle, end)}`);
|
|
285
|
+
if (tokens <= CONCEPT_WINDOW_OVERLAP_TOKENS) {
|
|
286
|
+
accepted = middle;
|
|
287
|
+
high = middle - 1;
|
|
288
|
+
} else {
|
|
289
|
+
low = advanceLow(low, middle);
|
|
290
|
+
}
|
|
291
|
+
}
|
|
292
|
+
return Math.max(nextUtf16Boundary(text, start), accepted);
|
|
293
|
+
}
|
|
294
|
+
function tokenSafeWindows(extractor, pending) {
|
|
295
|
+
const prefix = `${pending.role}: `;
|
|
296
|
+
const windows = [];
|
|
297
|
+
let start = 0;
|
|
298
|
+
while (start < pending.text.length) {
|
|
299
|
+
const end = maximumTokenSafeEnd(extractor, prefix, pending.text, start);
|
|
300
|
+
if (end <= start)
|
|
301
|
+
throw new Error("Concept tokenizer could not admit one complete source character");
|
|
302
|
+
windows.push({ start, end, input: `${prefix}${pending.text.slice(start, end)}` });
|
|
303
|
+
if (end >= pending.text.length)
|
|
304
|
+
break;
|
|
305
|
+
const next = overlapStart(extractor, prefix, pending.text, start, end);
|
|
306
|
+
if (next <= start)
|
|
307
|
+
throw new Error("Concept window overlap failed to advance past a complete window");
|
|
308
|
+
start = next;
|
|
309
|
+
}
|
|
310
|
+
return windows;
|
|
311
|
+
}
|
|
312
|
+
async function embedConceptInputs(extractor, pending, callbacks = {}) {
|
|
313
|
+
const layouts = pending.map((item) => tokenSafeWindows(extractor, item));
|
|
314
|
+
const inputs = layouts.flatMap((windows) => windows.map((window) => window.input));
|
|
315
|
+
const vectors = [];
|
|
316
|
+
const layoutEnds = [];
|
|
317
|
+
let layoutEnd = 0;
|
|
318
|
+
for (const windows of layouts) {
|
|
319
|
+
layoutEnd += windows.length;
|
|
320
|
+
layoutEnds.push(layoutEnd);
|
|
321
|
+
}
|
|
322
|
+
let completedEmbedding = 0;
|
|
323
|
+
for (let offset = 0;offset < inputs.length; offset += INFERENCE_BATCH_SIZE) {
|
|
324
|
+
const batch = inputs.slice(offset, offset + INFERENCE_BATCH_SIZE);
|
|
325
|
+
const tensor = await extractor(batch, { pooling: "mean", normalize: true });
|
|
326
|
+
if (tensor.data.length !== batch.length * CONCEPT_EMBEDDING_DIMENSIONS)
|
|
327
|
+
throw new Error("Unexpected concept embedding dimensions");
|
|
328
|
+
for (let index = 0;index < batch.length; index += 1) {
|
|
329
|
+
const start = index * CONCEPT_EMBEDDING_DIMENSIONS;
|
|
330
|
+
vectors.push(Array.from(tensor.data.slice(start, start + CONCEPT_EMBEDDING_DIMENSIONS), Number));
|
|
331
|
+
}
|
|
332
|
+
const completedWindows = Math.min(offset + batch.length, inputs.length);
|
|
333
|
+
while (layoutEnds[completedEmbedding] !== undefined && layoutEnds[completedEmbedding] <= completedWindows) {
|
|
334
|
+
const start = completedEmbedding === 0 ? 0 : layoutEnds[completedEmbedding - 1];
|
|
335
|
+
const item = pending[completedEmbedding];
|
|
336
|
+
if (!item)
|
|
337
|
+
throw new Error("Missing completed concept embedding input");
|
|
338
|
+
await callbacks.onCompletedEmbedding?.({
|
|
339
|
+
key: item.key,
|
|
340
|
+
windows: (layouts[completedEmbedding] ?? []).map((window, index) => ({
|
|
341
|
+
start: window.start,
|
|
342
|
+
end: window.end,
|
|
343
|
+
vector: vectors[start + index] ?? []
|
|
344
|
+
}))
|
|
345
|
+
}, completedEmbedding + 1, pending.length);
|
|
346
|
+
completedEmbedding += 1;
|
|
347
|
+
}
|
|
348
|
+
callbacks.onBatch?.(completedWindows, inputs.length);
|
|
349
|
+
}
|
|
350
|
+
let vectorIndex = 0;
|
|
351
|
+
return pending.map((item, index) => ({
|
|
352
|
+
key: item.key,
|
|
353
|
+
windows: (layouts[index] ?? []).map((window) => ({
|
|
354
|
+
start: window.start,
|
|
355
|
+
end: window.end,
|
|
356
|
+
vector: vectors[vectorIndex++] ?? []
|
|
357
|
+
}))
|
|
358
|
+
}));
|
|
359
|
+
}
|
|
360
|
+
function cosineSimilarity(left, right) {
|
|
361
|
+
if (left.length !== CONCEPT_EMBEDDING_DIMENSIONS || right.length !== left.length)
|
|
362
|
+
throw new Error("Inconsistent concept embedding dimensions");
|
|
363
|
+
const score = left.reduce((sum, value, index) => sum + value * (right[index] ?? 0), 0);
|
|
364
|
+
if (!Number.isFinite(score))
|
|
365
|
+
throw new Error("Non-finite concept similarity");
|
|
366
|
+
return score;
|
|
367
|
+
}
|
|
368
|
+
function meanNormalized(vectors) {
|
|
369
|
+
if (!vectors.length)
|
|
370
|
+
throw new Error("Cannot pool an empty concept embedding");
|
|
371
|
+
const pooled = Array.from({ length: CONCEPT_EMBEDDING_DIMENSIONS }, (_, index) => vectors.reduce((sum, vector) => sum + (vector[index] ?? 0), 0));
|
|
372
|
+
const magnitude = Math.sqrt(pooled.reduce((sum, value) => sum + value * value, 0));
|
|
373
|
+
if (!Number.isFinite(magnitude) || magnitude === 0)
|
|
374
|
+
throw new Error("Invalid pooled concept query embedding");
|
|
375
|
+
return pooled.map((value) => value / magnitude);
|
|
376
|
+
}
|
|
377
|
+
|
|
378
|
+
// src/concept-setup.ts
|
|
379
|
+
import { mkdir as mkdir2, mkdtemp, open, rename as rename2, rm as rm2, stat as stat2 } from "node:fs/promises";
|
|
380
|
+
import { dirname, join as join3 } from "node:path";
|
|
381
|
+
async function installConceptModel() {
|
|
382
|
+
const destination = conceptModelDirectory();
|
|
383
|
+
try {
|
|
384
|
+
const existing = await stat2(destination);
|
|
385
|
+
if (!existing.isDirectory())
|
|
386
|
+
throw new Error("Concept model destination is not a directory");
|
|
387
|
+
await verifyConceptModel(destination);
|
|
388
|
+
process.stderr.write(`Concept model is already verified: ${destination}
|
|
389
|
+
`);
|
|
390
|
+
return;
|
|
391
|
+
} catch (error) {
|
|
392
|
+
if (!(error instanceof Error) || !("code" in error) || error.code !== "ENOENT")
|
|
393
|
+
throw error;
|
|
394
|
+
}
|
|
395
|
+
await mkdir2(dirname(destination), { recursive: true });
|
|
396
|
+
const stage = await mkdtemp(join3(dirname(destination), ".install-"));
|
|
397
|
+
const signal = AbortSignal.timeout(10 * 60000);
|
|
398
|
+
try {
|
|
399
|
+
for (const asset of CONCEPT_ASSETS) {
|
|
400
|
+
process.stderr.write(`Downloading pinned concept asset: ${asset.path} (${String(asset.bytes)} bytes)
|
|
401
|
+
`);
|
|
402
|
+
const response = await fetch(`https://huggingface.co/${CONCEPT_MODEL}/resolve/${CONCEPT_REVISION}/${asset.path}`, { signal });
|
|
403
|
+
if (!response.ok || !response.body)
|
|
404
|
+
throw new Error(`Model download failed: ${String(response.status)} ${asset.path}`);
|
|
405
|
+
const path = join3(stage, asset.path);
|
|
406
|
+
await mkdir2(dirname(path), { recursive: true });
|
|
407
|
+
const file = await open(path, "wx");
|
|
408
|
+
let size = 0;
|
|
409
|
+
try {
|
|
410
|
+
for await (const chunk of response.body) {
|
|
411
|
+
size += chunk.byteLength;
|
|
412
|
+
if (size > asset.bytes)
|
|
413
|
+
throw new Error(`Model asset exceeds pinned size: ${asset.path}`);
|
|
414
|
+
await file.writeFile(chunk);
|
|
415
|
+
}
|
|
416
|
+
await file.sync();
|
|
417
|
+
} finally {
|
|
418
|
+
await file.close();
|
|
419
|
+
}
|
|
420
|
+
}
|
|
421
|
+
await verifyConceptModel(stage);
|
|
422
|
+
try {
|
|
423
|
+
await rename2(stage, destination);
|
|
424
|
+
} catch (error) {
|
|
425
|
+
if (!(error instanceof Error) || !("code" in error) || !["EEXIST", "ENOTEMPTY"].includes(String(error.code)))
|
|
426
|
+
throw error;
|
|
427
|
+
await verifyConceptModel(destination);
|
|
428
|
+
}
|
|
429
|
+
process.stderr.write(`Concept model verified and installed: ${destination}
|
|
430
|
+
`);
|
|
431
|
+
} finally {
|
|
432
|
+
await rm2(stage, { recursive: true, force: true });
|
|
433
|
+
}
|
|
434
|
+
}
|
|
435
|
+
|
|
436
|
+
// src/concept-worker.ts
|
|
437
|
+
var CACHE_IO_CONCURRENCY = 64;
|
|
438
|
+
function emitProgress(progress) {
|
|
439
|
+
process.stdout.write(`${JSON.stringify({ type: "progress", ...progress })}
|
|
440
|
+
`);
|
|
441
|
+
}
|
|
442
|
+
async function readCachedEmbeddings(root, requested, onProgress) {
|
|
443
|
+
const cached = [];
|
|
444
|
+
for (let offset = 0;offset < requested.length; offset += CACHE_IO_CONCURRENCY) {
|
|
445
|
+
const batch = await Promise.all(requested.slice(offset, offset + CACHE_IO_CONCURRENCY).map((item) => readConceptEmbedding(root, item.key)));
|
|
446
|
+
cached.push(...batch);
|
|
447
|
+
onProgress?.(Math.min(offset + batch.length, requested.length), requested.length);
|
|
448
|
+
}
|
|
449
|
+
return cached;
|
|
450
|
+
}
|
|
451
|
+
async function requestFromStdin() {
|
|
452
|
+
const chunks = [];
|
|
453
|
+
let bytes = 0;
|
|
454
|
+
for await (const chunk of process.stdin) {
|
|
455
|
+
const buffer = Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk);
|
|
456
|
+
chunks.push(buffer);
|
|
457
|
+
bytes += buffer.length;
|
|
458
|
+
if (bytes > MAX_CONCEPT_WORKER_INPUT_BYTES)
|
|
459
|
+
throw new Error("Concept worker input exceeds its 64 MiB source protocol budget");
|
|
460
|
+
}
|
|
461
|
+
const request = JSON.parse(Buffer.concat(chunks).toString("utf8"));
|
|
462
|
+
if (!isRecordValue(request) || typeof request.query !== "string" || request.query.length === 0 || request.query.length > 256 || !request.query.isWellFormed() || /[\r\n\0]/u.test(request.query) || !Array.isArray(request.encodedPassages) || request.encodedPassages.some((item) => typeof item !== "string" || item.length === 0 || item.length % 4 !== 0 || item.length > Math.ceil((MAX_CONCEPT_CHARS + 256) * 4 / 3 * 4) + 4 || !/^[A-Za-z0-9+/]+={0,2}$/u.test(item)))
|
|
463
|
+
throw new Error("Invalid concept worker input");
|
|
464
|
+
const encodedPassages = request.encodedPassages.filter((item) => typeof item === "string");
|
|
465
|
+
const buffers = encodedPassages.map((item) => Buffer.from(item, "base64"));
|
|
466
|
+
if (buffers.some((item) => !isUtf8(item)))
|
|
467
|
+
throw new Error("Invalid UTF-8 concept worker passage");
|
|
468
|
+
const passages = buffers.map((item) => item.toString("utf8"));
|
|
469
|
+
if (passages.some((item) => !item.length || item.length > MAX_CONCEPT_CHARS + 256 || !item.isWellFormed()))
|
|
470
|
+
throw new Error("Invalid decoded concept worker passage");
|
|
471
|
+
return { query: request.query, passages };
|
|
472
|
+
}
|
|
473
|
+
async function search() {
|
|
474
|
+
const request = await requestFromStdin();
|
|
475
|
+
const directory = conceptModelDirectory();
|
|
476
|
+
const cacheRoot = conceptCacheDirectory();
|
|
477
|
+
const stagingRoot = process.env.SIFT_LIGHT_CONCEPT_CACHE_STAGING_DIR ?? cacheRoot;
|
|
478
|
+
await verifyConceptModel(directory);
|
|
479
|
+
const requested = [
|
|
480
|
+
{ key: conceptEmbeddingKey("query", request.query), role: "query", text: request.query },
|
|
481
|
+
...request.passages.map((text) => ({
|
|
482
|
+
key: conceptEmbeddingKey("passage", text),
|
|
483
|
+
role: "passage",
|
|
484
|
+
text
|
|
485
|
+
}))
|
|
486
|
+
];
|
|
487
|
+
const uniqueRequested = [...new Map(requested.map((item) => [item.key, item])).values()];
|
|
488
|
+
const progressContext = {
|
|
489
|
+
uniqueEmbeddings: uniqueRequested.length,
|
|
490
|
+
passages: request.passages.length
|
|
491
|
+
};
|
|
492
|
+
const cached = await readCachedEmbeddings(cacheRoot, uniqueRequested, (completed, total) => emitProgress({ phase: "cache-read", completed, total, ...progressContext }));
|
|
493
|
+
const missing = uniqueRequested.filter((_item, index) => !cached[index]);
|
|
494
|
+
const warnings = [];
|
|
495
|
+
let created = [];
|
|
496
|
+
let cacheWriteFailed = false;
|
|
497
|
+
if (missing.length) {
|
|
498
|
+
env.allowRemoteModels = false;
|
|
499
|
+
env.useFSCache = false;
|
|
500
|
+
env.useBrowserCache = false;
|
|
501
|
+
env.localModelPath = "/";
|
|
502
|
+
emitProgress({ phase: "model-loading", ...progressContext });
|
|
503
|
+
const extractor = await pipeline("feature-extraction", directory, {
|
|
504
|
+
local_files_only: true,
|
|
505
|
+
dtype: "q8",
|
|
506
|
+
device: "cpu",
|
|
507
|
+
session_options: { intraOpNumThreads: 2, interOpNumThreads: 1 }
|
|
508
|
+
});
|
|
509
|
+
try {
|
|
510
|
+
created = await embedConceptInputs(extractor, missing, {
|
|
511
|
+
onCompletedEmbedding: async (embedding, completed, total) => {
|
|
512
|
+
try {
|
|
513
|
+
await writeConceptEmbedding(cacheRoot, embedding, stagingRoot);
|
|
514
|
+
} catch {
|
|
515
|
+
cacheWriteFailed = true;
|
|
516
|
+
}
|
|
517
|
+
emitProgress({ phase: "cache-write", completed, total, ...progressContext });
|
|
518
|
+
},
|
|
519
|
+
onBatch: (completed, total) => emitProgress({ phase: "embedding", completed, total, ...progressContext })
|
|
520
|
+
});
|
|
521
|
+
} finally {
|
|
522
|
+
await extractor.dispose();
|
|
523
|
+
}
|
|
524
|
+
if (cacheWriteFailed)
|
|
525
|
+
warnings.push("Concept embedding cache write failed; ranking completed but some work may repeat");
|
|
526
|
+
}
|
|
527
|
+
const createdByKey = new Map(created.map((item) => [item.key, item]));
|
|
528
|
+
const embeddingsByKey = new Map(uniqueRequested.map((item, index) => [item.key, cached[index] ?? createdByKey.get(item.key)]));
|
|
529
|
+
const embeddings = requested.map((item) => embeddingsByKey.get(item.key));
|
|
530
|
+
if (embeddings.some((item) => !item))
|
|
531
|
+
throw new Error("Missing concept embedding result");
|
|
532
|
+
const queryWindows = embeddings[0]?.windows;
|
|
533
|
+
if (!queryWindows?.length)
|
|
534
|
+
throw new Error("Missing concept query embedding");
|
|
535
|
+
const query = meanNormalized(queryWindows.map((window) => window.vector));
|
|
536
|
+
const passageEmbeddings = embeddings.slice(1);
|
|
537
|
+
const scores = passageEmbeddings.map((embedding) => {
|
|
538
|
+
if (!embedding?.windows.length)
|
|
539
|
+
throw new Error("Missing concept passage embedding");
|
|
540
|
+
return Math.max(...embedding.windows.map((window) => cosineSimilarity(query, window.vector)));
|
|
541
|
+
});
|
|
542
|
+
let cacheBytes;
|
|
543
|
+
try {
|
|
544
|
+
emitProgress({ phase: "cache-cleanup", ...progressContext });
|
|
545
|
+
cacheBytes = await enforceConceptCacheLimit(cacheRoot);
|
|
546
|
+
} catch {
|
|
547
|
+
warnings.push("Concept embedding cache cleanup failed; the configured 512 MiB bound was not verified");
|
|
548
|
+
}
|
|
549
|
+
process.stdout.write(JSON.stringify({
|
|
550
|
+
type: "result",
|
|
551
|
+
scores,
|
|
552
|
+
cacheHits: cached.filter(Boolean).length,
|
|
553
|
+
cacheMisses: missing.length,
|
|
554
|
+
cacheMaxBytes: CONCEPT_CACHE_MAX_BYTES,
|
|
555
|
+
...cacheBytes === undefined ? {} : { cacheBytes },
|
|
556
|
+
windowsRanked: passageEmbeddings.reduce((total, embedding) => total + (embedding?.windows.length ?? 0), 0),
|
|
557
|
+
warnings: [...new Set(warnings)],
|
|
558
|
+
peakRssBytes: process.resourceUsage().maxRSS * 1024,
|
|
559
|
+
modelBytes: CONCEPT_ASSETS.reduce((sum, asset) => sum + asset.bytes, 0)
|
|
560
|
+
}));
|
|
561
|
+
}
|
|
562
|
+
if (process.argv.includes("--install-model"))
|
|
563
|
+
await installConceptModel();
|
|
564
|
+
else if (process.argv.includes("--infer"))
|
|
565
|
+
await search();
|
|
566
|
+
else
|
|
567
|
+
throw new Error("Usage: sift-light-model --install-model");
|