scream-code 0.18.7 → 0.18.8
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +6 -3
- package/dist/{app-IU7G9gs0.mjs → app-Db7f8Kja.mjs} +249 -193
- package/dist/{dispatch-7Mas_Tvb.mjs → dispatch-BPfUTccZ.mjs} +378 -20
- package/dist/{dispatch-BqBnrIMj.mjs → dispatch-zsFzFFoY.mjs} +1 -1
- package/dist/main.mjs +1 -1
- package/dist/{src-CQwXASmd.mjs → src-ClB_Hzp9.mjs} +280 -107
- package/dist/{src-ChTWl9Cn.mjs → src-DA34cXSv.mjs} +1 -1
- package/dist/{text-input-dialog-D7dm6kak.mjs → text-input-dialog-CFRUSCi6.mjs} +1 -1
- package/dist/{text-input-dialog-gDr3i7xz.mjs → text-input-dialog-DHDsHCqR.mjs} +58 -4
- package/package.json +3 -4
|
@@ -4,14 +4,14 @@ import { dirname as __cjsShimDirname } from 'node:path';
|
|
|
4
4
|
const __filename = __cjsShimFileURLToPath(import.meta.url);
|
|
5
5
|
const __dirname = __cjsShimDirname(__filename);
|
|
6
6
|
import { createHash, randomBytes } from "node:crypto";
|
|
7
|
-
import ro, { mkdir, readFile, readdir, rename, rm, rmdir, stat, unlink, writeFile } from "node:fs/promises";
|
|
7
|
+
import ro, { mkdir, open, readFile, readdir, rename, rm, rmdir, stat, unlink, writeFile } from "node:fs/promises";
|
|
8
8
|
import I from "fs";
|
|
9
9
|
import Qr, { EventEmitter } from "events";
|
|
10
10
|
import js, { dirname, parse } from "path";
|
|
11
11
|
import { Buffer as Buffer$1 } from "buffer";
|
|
12
12
|
import Cs from "node:stream";
|
|
13
|
-
import Kt, { createReadStream, existsSync, mkdirSync, rmSync, statSync } from "node:fs";
|
|
14
|
-
import path, { basename, join, posix, win32 } from "node:path";
|
|
13
|
+
import Kt, { createReadStream, existsSync, mkdirSync, readFileSync, rmSync, statSync } from "node:fs";
|
|
14
|
+
import path, { basename, join, posix, resolve, win32 } from "node:path";
|
|
15
15
|
import { DatabaseSync } from "node:sqlite";
|
|
16
16
|
import { EventEmitter as EventEmitter$1 } from "node:events";
|
|
17
17
|
import { StringDecoder } from "node:string_decoder";
|
|
@@ -3531,7 +3531,7 @@ function buildEmbeddingText(memo) {
|
|
|
3531
3531
|
*/
|
|
3532
3532
|
const engineCache = /* @__PURE__ */ new Map();
|
|
3533
3533
|
/**
|
|
3534
|
-
* Create an embedding engine backed by
|
|
3534
|
+
* Create an embedding engine backed by the in-process WASM pipeline.
|
|
3535
3535
|
* Lazily loads the model on first use so startup is not blocked.
|
|
3536
3536
|
* Engines are cached by cacheDir so repeated calls with the same cacheDir
|
|
3537
3537
|
* return the same instance, sharing model state and download progress.
|
|
@@ -3539,21 +3539,22 @@ const engineCache = /* @__PURE__ */ new Map();
|
|
|
3539
3539
|
* Defaults to "local_cache" (CWD-relative) if not provided — prefer
|
|
3540
3540
|
* passing an explicit path so the cache doesn't duplicate across CWDs.
|
|
3541
3541
|
*/
|
|
3542
|
-
function
|
|
3543
|
-
const key = cacheDir
|
|
3542
|
+
function createEmbeddingEngine(cacheDir) {
|
|
3543
|
+
const key = cacheDir === void 0 ? "" : resolve(cacheDir);
|
|
3544
3544
|
const cached = engineCache.get(key);
|
|
3545
3545
|
if (cached !== void 0) return cached;
|
|
3546
|
-
const engine =
|
|
3546
|
+
const engine = createEmbeddingEngineImpl(cacheDir);
|
|
3547
3547
|
engineCache.set(key, engine);
|
|
3548
3548
|
return engine;
|
|
3549
3549
|
}
|
|
3550
3550
|
/** Stable identity of the model used by every engine created here. */
|
|
3551
3551
|
const EMBEDDING_MODEL_NAME = "bge-small-zh-v1.5";
|
|
3552
|
-
function
|
|
3552
|
+
function createEmbeddingEngineImpl(cacheDir) {
|
|
3553
3553
|
let embedder = null;
|
|
3554
3554
|
let initPromise = null;
|
|
3555
3555
|
let loadFailed = false;
|
|
3556
3556
|
let lastError;
|
|
3557
|
+
let progressListener;
|
|
3557
3558
|
return {
|
|
3558
3559
|
modelName: EMBEDDING_MODEL_NAME,
|
|
3559
3560
|
get available() {
|
|
@@ -3562,6 +3563,9 @@ function createFastEmbedEngineImpl(cacheDir) {
|
|
|
3562
3563
|
get lastError() {
|
|
3563
3564
|
return lastError;
|
|
3564
3565
|
},
|
|
3566
|
+
setProgressListener(listener) {
|
|
3567
|
+
progressListener = listener;
|
|
3568
|
+
},
|
|
3565
3569
|
async embedBatch(texts) {
|
|
3566
3570
|
if (!this.available) return null;
|
|
3567
3571
|
if (texts.length === 0) return [];
|
|
@@ -3598,12 +3602,12 @@ function createFastEmbedEngineImpl(cacheDir) {
|
|
|
3598
3602
|
if (loadFailed || initPromise === null) {
|
|
3599
3603
|
loadFailed = false;
|
|
3600
3604
|
lastError = void 0;
|
|
3601
|
-
initPromise =
|
|
3605
|
+
initPromise = loadEmbeddingRuntime(cacheDir, (progress) => progressListener?.(progress));
|
|
3602
3606
|
}
|
|
3603
3607
|
embedder = await initPromise;
|
|
3604
3608
|
if (embedder === null) {
|
|
3605
3609
|
initPromise = null;
|
|
3606
|
-
lastError = "
|
|
3610
|
+
lastError = "embedding runtime returned an empty session";
|
|
3607
3611
|
return false;
|
|
3608
3612
|
}
|
|
3609
3613
|
loadFailed = false;
|
|
@@ -3618,42 +3622,157 @@ function createFastEmbedEngineImpl(cacheDir) {
|
|
|
3618
3622
|
}
|
|
3619
3623
|
};
|
|
3620
3624
|
}
|
|
3621
|
-
|
|
3622
|
-
|
|
3623
|
-
|
|
3624
|
-
|
|
3625
|
-
|
|
3626
|
-
|
|
3627
|
-
|
|
3628
|
-
|
|
3629
|
-
|
|
3630
|
-
|
|
3631
|
-
|
|
3632
|
-
|
|
3633
|
-
|
|
3634
|
-
|
|
3635
|
-
|
|
3636
|
-
|
|
3637
|
-
|
|
3638
|
-
|
|
3625
|
+
/** Maximum tokens per text; matches the model's position-embedding limit. */
|
|
3626
|
+
const MAX_SEQ_LEN = 512;
|
|
3627
|
+
/** Texts per inference batch: 16 × 512 × 512 floats ≈ 16MB of activations. */
|
|
3628
|
+
const EMBED_BATCH_SIZE = 16;
|
|
3629
|
+
/** Threads for the WASM backend. One keeps resolution deterministic in packaged
|
|
3630
|
+
* builds; `SCREAM_CODE_EMBEDDING_THREADS=4` is the supported opt-in. */
|
|
3631
|
+
const EMBEDDING_THREADS_ENV = "SCREAM_CODE_EMBEDDING_THREADS";
|
|
3632
|
+
/**
|
|
3633
|
+
* Token-level truncation that keeps the sentence boundary tokens.
|
|
3634
|
+
*
|
|
3635
|
+
* The tokenizer package does not apply the truncation policy itself and the
|
|
3636
|
+
* model package ships no truncation config, so this is the only place it can
|
|
3637
|
+
* happen. Keeping the head (which starts with `[CLS]`) and re-appending the
|
|
3638
|
+
* trailing special token (`[SEP]`) mirrors what a HuggingFace pipeline would
|
|
3639
|
+
* produce; slicing the tail off blindly would feed the encoder a sentence that
|
|
3640
|
+
* never closes and drift from the reference vectors on long inputs.
|
|
3641
|
+
*/
|
|
3642
|
+
function truncateEncoded(ids, types) {
|
|
3643
|
+
if (ids.length <= MAX_SEQ_LEN) return {
|
|
3644
|
+
ids: [...ids],
|
|
3645
|
+
types: [...types]
|
|
3646
|
+
};
|
|
3647
|
+
const headSize = MAX_SEQ_LEN - 1;
|
|
3648
|
+
const head = ids.slice(0, headSize);
|
|
3649
|
+
const headTypes = types.slice(0, headSize);
|
|
3650
|
+
return {
|
|
3651
|
+
ids: [...head, ids.at(-1)],
|
|
3652
|
+
types: [...headTypes, types.at(-1)]
|
|
3653
|
+
};
|
|
3654
|
+
}
|
|
3655
|
+
/**
|
|
3656
|
+
* Builds the embedding runtime in-process on `onnxruntime-web` (WASM) and the
|
|
3657
|
+
* pure-JS tokenizer. No native binaries, no install scripts: the whole stack is
|
|
3658
|
+
* JavaScript plus one WebAssembly artifact resolved from the package.
|
|
3659
|
+
*
|
|
3660
|
+
* The numeric pipeline is load-bearing and must stay compatible with the
|
|
3661
|
+
* reference embeddings of this model: token ids come from `tokenizer.json`, and
|
|
3662
|
+
* the sentence vector is the **CLS token** hidden state (`last_hidden_state[0]`)
|
|
3663
|
+
* followed by L2 normalization. Mean pooling produces visibly different vectors
|
|
3664
|
+
* (cosine ≈ 0.9 against the reference), so pooling is not a stylistic choice.
|
|
3665
|
+
*/
|
|
3666
|
+
async function createWasmRuntime(modelDir) {
|
|
3667
|
+
const ort = await import("onnxruntime-web");
|
|
3668
|
+
const { Tokenizer } = await import("@huggingface/tokenizers");
|
|
3669
|
+
const requestedThreads = Number(process.env[EMBEDDING_THREADS_ENV]);
|
|
3670
|
+
ort.env.wasm.numThreads = Number.isFinite(requestedThreads) && requestedThreads > 0 ? requestedThreads : 1;
|
|
3671
|
+
ort.env.logLevel = "error";
|
|
3672
|
+
const tokenizer = new Tokenizer(JSON.parse(readFileSync(join(modelDir, "tokenizer.json"), "utf8")), JSON.parse(readFileSync(join(modelDir, "tokenizer_config.json"), "utf8")));
|
|
3673
|
+
const session = await ort.InferenceSession.create(readFileSync(join(modelDir, "model_optimized.onnx")), { executionProviders: ["wasm"] });
|
|
3674
|
+
if (session === null || session === void 0) return null;
|
|
3675
|
+
return { async *embed(textStrings) {
|
|
3676
|
+
if (textStrings.length === 0) return;
|
|
3677
|
+
const encoded = textStrings.map((text, index) => {
|
|
3678
|
+
const encoding = tokenizer.encode(text, { return_token_type_ids: true });
|
|
3679
|
+
const types = encoding.token_type_ids ?? encoding.ids.map(() => 0);
|
|
3680
|
+
const truncated = truncateEncoded(encoding.ids, types);
|
|
3681
|
+
return {
|
|
3682
|
+
index,
|
|
3683
|
+
ids: truncated.ids,
|
|
3684
|
+
types: truncated.types
|
|
3685
|
+
};
|
|
3686
|
+
});
|
|
3687
|
+
const results = Array.from({ length: textStrings.length });
|
|
3688
|
+
const buckets = encoded.toSorted((a, b) => a.ids.length - b.ids.length);
|
|
3689
|
+
for (let start = 0; start < buckets.length; start += EMBED_BATCH_SIZE) {
|
|
3690
|
+
const group = buckets.slice(start, start + EMBED_BATCH_SIZE);
|
|
3691
|
+
const rows = group.length;
|
|
3692
|
+
const width = Math.max(...group.map((entry) => entry.ids.length));
|
|
3693
|
+
const ids = new BigInt64Array(rows * width);
|
|
3694
|
+
const mask = new BigInt64Array(rows * width);
|
|
3695
|
+
const types = new BigInt64Array(rows * width);
|
|
3696
|
+
group.forEach((entry, row) => {
|
|
3697
|
+
for (let column = 0; column < width; column++) {
|
|
3698
|
+
const offset = row * width + column;
|
|
3699
|
+
ids[offset] = BigInt(column < entry.ids.length ? entry.ids[column] : 0);
|
|
3700
|
+
mask[offset] = column < entry.ids.length ? 1n : 0n;
|
|
3701
|
+
types[offset] = BigInt(column < entry.types.length ? entry.types[column] : 0);
|
|
3702
|
+
}
|
|
3703
|
+
});
|
|
3704
|
+
const output = await session.run({
|
|
3705
|
+
input_ids: new ort.Tensor("int64", ids, [rows, width]),
|
|
3706
|
+
attention_mask: new ort.Tensor("int64", mask, [rows, width]),
|
|
3707
|
+
token_type_ids: new ort.Tensor("int64", types, [rows, width])
|
|
3708
|
+
});
|
|
3709
|
+
const hidden = output["last_hidden_state"] ?? output[Object.keys(output)[0]];
|
|
3710
|
+
const hiddenSize = hidden.dims.at(-1);
|
|
3711
|
+
const data = hidden.data;
|
|
3712
|
+
group.forEach((entry, row) => {
|
|
3713
|
+
const rowStart = row * width * hiddenSize;
|
|
3714
|
+
const vector = Array.from(data.subarray(rowStart, rowStart + hiddenSize));
|
|
3715
|
+
let norm = 0;
|
|
3716
|
+
for (const value of vector) norm += value * value;
|
|
3717
|
+
norm = Math.sqrt(norm) || 1;
|
|
3718
|
+
for (let i = 0; i < vector.length; i += 1) vector[i] = vector[i] / norm;
|
|
3719
|
+
results[entry.index] = vector;
|
|
3720
|
+
});
|
|
3721
|
+
}
|
|
3722
|
+
for (let start = 0; start < results.length; start += EMBED_BATCH_SIZE) yield results.slice(start, start + EMBED_BATCH_SIZE);
|
|
3723
|
+
} };
|
|
3724
|
+
}
|
|
3725
|
+
async function loadEmbeddingRuntime(cacheDir, emit) {
|
|
3726
|
+
if (cacheDir === void 0) {
|
|
3727
|
+
emit?.({ phase: "load" });
|
|
3728
|
+
return createWasmRuntime(join("local_cache", BGESMALLZH_CACHE_NAME));
|
|
3639
3729
|
}
|
|
3730
|
+
mkdirSync(cacheDir, { recursive: true });
|
|
3731
|
+
await ensureEmbeddingModelFiles(cacheDir, {
|
|
3732
|
+
onProgress: (received, total) => {
|
|
3733
|
+
emit?.(total === void 0 ? {
|
|
3734
|
+
phase: "download",
|
|
3735
|
+
received
|
|
3736
|
+
} : {
|
|
3737
|
+
phase: "download",
|
|
3738
|
+
received,
|
|
3739
|
+
total
|
|
3740
|
+
});
|
|
3741
|
+
},
|
|
3742
|
+
onPhase: (phase) => {
|
|
3743
|
+
if (phase === "verify") emit?.({ phase: "verify" });
|
|
3744
|
+
}
|
|
3745
|
+
});
|
|
3746
|
+
emit?.({ phase: "load" });
|
|
3747
|
+
return createWasmRuntime(join(cacheDir, BGESMALLZH_CACHE_NAME));
|
|
3640
3748
|
}
|
|
3641
3749
|
/**
|
|
3642
|
-
* Cache entry
|
|
3643
|
-
*
|
|
3750
|
+
* Cache entry name for the fixed BGESmallZH model: the staged archive extracts
|
|
3751
|
+
* to `<cacheDir>/<name>/` and the runtime loads from that directory.
|
|
3644
3752
|
*/
|
|
3645
3753
|
const BGESMALLZH_CACHE_NAME = "fast-bge-small-zh-v1.5";
|
|
3754
|
+
/** Files the runtime needs before the model can be considered installed. */
|
|
3755
|
+
const EMBEDDING_REQUIRED_FILES = [
|
|
3756
|
+
"model_optimized.onnx",
|
|
3757
|
+
"tokenizer.json",
|
|
3758
|
+
"tokenizer_config.json"
|
|
3759
|
+
];
|
|
3646
3760
|
/** Escape hatch for air-gapped installs and tests: points at an archive file. */
|
|
3647
3761
|
const EMBEDDING_MODEL_URL_ENV = "SCREAM_CODE_EMBEDDING_MODEL_URL";
|
|
3648
|
-
|
|
3649
|
-
|
|
3762
|
+
/** Default stall guards: a trickling connection must not spin a spinner forever. */
|
|
3763
|
+
const DEFAULT_IDLE_TIMEOUT_MS = 6e4;
|
|
3764
|
+
const DEFAULT_TOTAL_TIMEOUT_MS = 15 * 6e4;
|
|
3765
|
+
/** Progress callback throttle: frequent enough to move, rare enough to be free. */
|
|
3766
|
+
const PROGRESS_THROTTLE_MS = 100;
|
|
3767
|
+
function modelDirPath(cacheDir) {
|
|
3768
|
+
return join(cacheDir, BGESMALLZH_CACHE_NAME);
|
|
3650
3769
|
}
|
|
3651
3770
|
function sha256File(filePath) {
|
|
3652
3771
|
return readFile(filePath).then((data) => createHash("sha256").update(data).digest("hex"));
|
|
3653
3772
|
}
|
|
3654
3773
|
/**
|
|
3655
3774
|
* Stage the embedding model files when they are missing: download the pinned
|
|
3656
|
-
* archive (HTTP status verified
|
|
3775
|
+
* archive (HTTP status verified, unlike a naive downloader), extract,
|
|
3657
3776
|
* and verify the weights payload against the pinned sha256.
|
|
3658
3777
|
*
|
|
3659
3778
|
* A partially extracted tree or a hash mismatch is removed so the caller keeps
|
|
@@ -3663,62 +3782,149 @@ function sha256File(filePath) {
|
|
|
3663
3782
|
async function ensureEmbeddingModelFiles(cacheDir, options) {
|
|
3664
3783
|
if (hasEmbeddingModelCache(cacheDir)) return;
|
|
3665
3784
|
mkdirSync(cacheDir, { recursive: true });
|
|
3666
|
-
const modelDir =
|
|
3785
|
+
const modelDir = modelDirPath(cacheDir);
|
|
3667
3786
|
const stagingPath = join(cacheDir, `${BGESMALLZH_CACHE_NAME}.tar.gz.part`);
|
|
3668
3787
|
const url = options?.sourceUrl ?? process.env[EMBEDDING_MODEL_URL_ENV] ?? "https://github.com/LIUTod/scream-code/releases/download/embedding-models-v1/fast-bge-small-zh-v1.5.tar.gz";
|
|
3788
|
+
const idleTimeoutMs = options?.idleTimeoutMs ?? DEFAULT_IDLE_TIMEOUT_MS;
|
|
3789
|
+
const totalTimeoutMs = options?.totalTimeoutMs ?? DEFAULT_TOTAL_TIMEOUT_MS;
|
|
3790
|
+
const controller = new AbortController();
|
|
3791
|
+
const onExternalAbort = () => {
|
|
3792
|
+
controller.abort();
|
|
3793
|
+
};
|
|
3794
|
+
if (options?.signal !== void 0) if (options.signal.aborted) controller.abort();
|
|
3795
|
+
else options.signal.addEventListener("abort", onExternalAbort, { once: true });
|
|
3796
|
+
let idleTimer;
|
|
3797
|
+
const armIdleTimer = () => {
|
|
3798
|
+
if (idleTimer !== void 0) clearTimeout(idleTimer);
|
|
3799
|
+
idleTimer = setTimeout(() => {
|
|
3800
|
+
controller.abort();
|
|
3801
|
+
}, idleTimeoutMs);
|
|
3802
|
+
};
|
|
3803
|
+
const totalTimer = setTimeout(() => {
|
|
3804
|
+
controller.abort();
|
|
3805
|
+
}, totalTimeoutMs);
|
|
3806
|
+
let received = 0;
|
|
3807
|
+
let announcedTotal;
|
|
3808
|
+
let downloadCompleted = false;
|
|
3809
|
+
const announceProgress = () => {
|
|
3810
|
+
options?.onProgress?.(received, announcedTotal);
|
|
3811
|
+
};
|
|
3669
3812
|
try {
|
|
3670
3813
|
let response;
|
|
3671
3814
|
try {
|
|
3672
|
-
|
|
3815
|
+
armIdleTimer();
|
|
3816
|
+
response = await fetch(url, { signal: controller.signal });
|
|
3673
3817
|
} catch (error) {
|
|
3674
3818
|
const message = error instanceof Error ? error.message : String(error);
|
|
3675
3819
|
throw new Error(`fetch failed for ${url}: ${message}`, { cause: error });
|
|
3676
3820
|
}
|
|
3677
3821
|
if (!response.ok) throw new Error(`model source returned HTTP ${response.status} for ${url}`);
|
|
3678
|
-
|
|
3822
|
+
const contentLength = Number(response.headers.get("content-length"));
|
|
3823
|
+
announcedTotal = Number.isFinite(contentLength) && contentLength > 0 ? contentLength : void 0;
|
|
3824
|
+
announceProgress();
|
|
3825
|
+
const body = response.body;
|
|
3826
|
+
if (body === null) throw new Error(`model source returned an empty body for ${url}`);
|
|
3827
|
+
const reader = body.getReader();
|
|
3828
|
+
const abortRejection = new Promise((_, reject) => {
|
|
3829
|
+
if (controller.signal.aborted) {
|
|
3830
|
+
reject(/* @__PURE__ */ new Error("aborted"));
|
|
3831
|
+
return;
|
|
3832
|
+
}
|
|
3833
|
+
controller.signal.addEventListener("abort", () => {
|
|
3834
|
+
reject(/* @__PURE__ */ new Error("aborted"));
|
|
3835
|
+
}, { once: true });
|
|
3836
|
+
});
|
|
3837
|
+
const abortAwareRead = () => Promise.race([reader.read(), abortRejection]);
|
|
3838
|
+
let handle;
|
|
3839
|
+
try {
|
|
3840
|
+
handle = await open(stagingPath, "w");
|
|
3841
|
+
} catch (error) {
|
|
3842
|
+
await reader.cancel().catch(() => void 0);
|
|
3843
|
+
throw error;
|
|
3844
|
+
}
|
|
3845
|
+
let lastProgressAt = 0;
|
|
3846
|
+
try {
|
|
3847
|
+
await (async () => {
|
|
3848
|
+
for (;;) {
|
|
3849
|
+
let chunk;
|
|
3850
|
+
try {
|
|
3851
|
+
chunk = await abortAwareRead();
|
|
3852
|
+
} catch (error) {
|
|
3853
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
3854
|
+
throw new Error(`${message} (received ${received} of ${announcedTotal ?? "?"} bytes): ${url}`, { cause: error });
|
|
3855
|
+
}
|
|
3856
|
+
if (chunk.done) break;
|
|
3857
|
+
const bytes = chunk.value;
|
|
3858
|
+
armIdleTimer();
|
|
3859
|
+
await handle.write(bytes);
|
|
3860
|
+
received += bytes.byteLength;
|
|
3861
|
+
const now = Date.now();
|
|
3862
|
+
if (now - lastProgressAt >= PROGRESS_THROTTLE_MS) {
|
|
3863
|
+
lastProgressAt = now;
|
|
3864
|
+
announceProgress();
|
|
3865
|
+
}
|
|
3866
|
+
}
|
|
3867
|
+
announceProgress();
|
|
3868
|
+
downloadCompleted = true;
|
|
3869
|
+
})();
|
|
3870
|
+
} finally {
|
|
3871
|
+
if (idleTimer !== void 0) clearTimeout(idleTimer);
|
|
3872
|
+
await handle.close();
|
|
3873
|
+
await reader.cancel().catch(() => void 0);
|
|
3874
|
+
}
|
|
3875
|
+
options?.onPhase?.("verify");
|
|
3679
3876
|
await So({
|
|
3680
3877
|
file: stagingPath,
|
|
3681
3878
|
cwd: cacheDir
|
|
3682
3879
|
});
|
|
3683
|
-
const
|
|
3684
|
-
|
|
3685
|
-
|
|
3880
|
+
for (const fileName of EMBEDDING_REQUIRED_FILES) {
|
|
3881
|
+
const target = join(modelDir, fileName);
|
|
3882
|
+
if (!existsSync(target) || statSync(target).size === 0) throw new Error(`model package is missing ${fileName} after extraction: ${url}`);
|
|
3883
|
+
}
|
|
3884
|
+
const sha256 = await sha256File(join(modelDir, "model_optimized.onnx"));
|
|
3686
3885
|
const expected = options?.expectedOnnxSha256 ?? "1294ea4b6331115a353d81f96b85e8c8d7fdcc284453d5b2fab5b016230aad38";
|
|
3687
3886
|
if (sha256 !== expected) throw new Error(`model weights sha256 mismatch (${sha256.slice(0, 12)}… != ${expected.slice(0, 12)}…): ${url}`);
|
|
3688
3887
|
} catch (error) {
|
|
3689
|
-
|
|
3690
|
-
|
|
3691
|
-
|
|
3692
|
-
|
|
3693
|
-
|
|
3888
|
+
try {
|
|
3889
|
+
rmSync(modelDir, {
|
|
3890
|
+
recursive: true,
|
|
3891
|
+
force: true
|
|
3892
|
+
});
|
|
3893
|
+
} catch {}
|
|
3894
|
+
if (!downloadCompleted && controller.signal.aborted && !(options?.signal?.aborted ?? false)) throw new Error(`model download timed out (received ${received} of ${announcedTotal ?? "?"} bytes): ${url}`, { cause: error });
|
|
3895
|
+
throw error instanceof Error ? error : new Error(String(error), { cause: error });
|
|
3694
3896
|
} finally {
|
|
3897
|
+
clearTimeout(totalTimer);
|
|
3898
|
+
if (idleTimer !== void 0) clearTimeout(idleTimer);
|
|
3899
|
+
if (options?.signal !== void 0) options.signal.removeEventListener("abort", onExternalAbort);
|
|
3695
3900
|
rmSync(stagingPath, { force: true });
|
|
3696
3901
|
}
|
|
3697
3902
|
}
|
|
3698
|
-
const
|
|
3699
|
-
const ARCHIVE_FAILURE = /\bTAR_[A-Z_]+|incorrect header check|unexpected end of file|invalid tar|zlib/iu;
|
|
3700
|
-
const NETWORK_FAILURE = /\b(?:ENOTFOUND|EAI_AGAIN|ETIMEDOUT|ECONNRESET|ECONNREFUSED|EHOSTUNREACH|socket hang up|fetch failed)\b|network error|HTTP \d{3}/iu;
|
|
3903
|
+
const INSTALL_FAILURE = /Cannot find (?:package|module) '(?:onnxruntime-web|@huggingface\/tokenizers)'|ERR_MODULE_NOT_FOUND|Failed to resolve module specifier/iu;
|
|
3904
|
+
const ARCHIVE_FAILURE = /\bTAR_[A-Z_]+|incorrect header check|unexpected end of file|invalid tar|zlib|after extraction|sha256 mismatch/iu;
|
|
3905
|
+
const NETWORK_FAILURE = /\b(?:ENOTFOUND|EAI_AGAIN|ETIMEDOUT|ECONNRESET|ECONNREFUSED|EHOSTUNREACH|socket hang up|fetch failed)\b|network error|HTTP \d{3}|timed out|empty body/iu;
|
|
3701
3906
|
/**
|
|
3702
|
-
* Classify a load failure so callers can say something true about it:
|
|
3703
|
-
*
|
|
3704
|
-
*
|
|
3907
|
+
* Classify a load failure so callers can say something true about it: an
|
|
3908
|
+
* incomplete install, a corrupt cache, a network problem and anything else need
|
|
3909
|
+
* four different answers, and they used to be reported as one network problem.
|
|
3705
3910
|
*/
|
|
3706
3911
|
function classifyEmbeddingFailure(message) {
|
|
3707
3912
|
if (message === void 0) return "other";
|
|
3708
|
-
if (
|
|
3913
|
+
if (INSTALL_FAILURE.test(message)) return "install";
|
|
3709
3914
|
if (ARCHIVE_FAILURE.test(message)) return "archive";
|
|
3710
3915
|
if (NETWORK_FAILURE.test(message)) return "network";
|
|
3711
3916
|
return "other";
|
|
3712
3917
|
}
|
|
3713
3918
|
/**
|
|
3714
|
-
* Loads
|
|
3715
|
-
*
|
|
3716
|
-
* failed".
|
|
3717
|
-
*
|
|
3919
|
+
* Loads both runtime packages — no model download — so callers can tell "this
|
|
3920
|
+
* installation cannot run the local model at all" apart from "the download
|
|
3921
|
+
* failed". The runtime is JavaScript plus WebAssembly, so a failure here is an
|
|
3922
|
+
* incomplete installation rather than an unsupported platform.
|
|
3718
3923
|
*/
|
|
3719
3924
|
async function probeLocalEmbeddingSupport() {
|
|
3720
3925
|
try {
|
|
3721
|
-
await import("
|
|
3926
|
+
await import("onnxruntime-web");
|
|
3927
|
+
await import("@huggingface/tokenizers");
|
|
3722
3928
|
return { supported: true };
|
|
3723
3929
|
} catch (error) {
|
|
3724
3930
|
return {
|
|
@@ -3729,15 +3935,11 @@ async function probeLocalEmbeddingSupport() {
|
|
|
3729
3935
|
}
|
|
3730
3936
|
/**
|
|
3731
3937
|
* Remove every trace of a previous BGESmallZH download: the extracted model
|
|
3732
|
-
* directory
|
|
3938
|
+
* directory and the legacy staged archive name.
|
|
3733
3939
|
*
|
|
3734
|
-
* The
|
|
3735
|
-
*
|
|
3736
|
-
*
|
|
3737
|
-
* error) reject and leave whatever was written behind. So an archive sitting
|
|
3738
|
-
* next to a failed attempt is usually a partial leftover, and keeping it makes
|
|
3739
|
-
* every retry extract the same junk locally: no network traffic, a few
|
|
3740
|
-
* milliseconds, the same failure forever.
|
|
3940
|
+
* The legacy `<name>.tar.gz` file is still swept because older versions staged
|
|
3941
|
+
* the archive under that name; leaving it behind would waste 55MB per doomed
|
|
3942
|
+
* attempt. Our own downloader stages `.tar.gz.part` and always removes it.
|
|
3741
3943
|
*
|
|
3742
3944
|
* Never throws: a file locked by another process (on Windows a mapped ONNX file
|
|
3743
3945
|
* stays locked) must not replace the download error the caller is reporting.
|
|
@@ -3755,48 +3957,19 @@ function clearEmbeddingModelCache(cacheDir) {
|
|
|
3755
3957
|
}
|
|
3756
3958
|
}
|
|
3757
3959
|
/**
|
|
3758
|
-
* True when the
|
|
3759
|
-
*
|
|
3760
|
-
*
|
|
3761
|
-
*
|
|
3960
|
+
* True when every file the runtime needs is already on disk. Loading then never
|
|
3961
|
+
* downloads, so callers can safely trigger a local load without risking a
|
|
3962
|
+
* surprise mid-query download. All required files are checked (not just the
|
|
3963
|
+
* weights): a partially restored cache must count as missing so the setup flow
|
|
3964
|
+
* repairs it instead of failing at load time.
|
|
3762
3965
|
*/
|
|
3763
3966
|
function hasEmbeddingModelCache(cacheDir) {
|
|
3764
|
-
const modelDir =
|
|
3765
|
-
|
|
3766
|
-
|
|
3767
|
-
/**
|
|
3768
|
-
* Small config/tokenizer files that fastembed expects alongside model.onnx.
|
|
3769
|
-
* If these are missing (e.g. GCS download partially failed), fastembed throws.
|
|
3770
|
-
* We download them from HuggingFace so the model can load — this covers the
|
|
3771
|
-
* case where the GCS tarball was incomplete but the HF repo is reachable.
|
|
3772
|
-
*/
|
|
3773
|
-
const FASTEMBED_SIDECARS = [
|
|
3774
|
-
"config.json",
|
|
3775
|
-
"tokenizer.json",
|
|
3776
|
-
"tokenizer_config.json",
|
|
3777
|
-
"special_tokens_map.json"
|
|
3778
|
-
];
|
|
3779
|
-
const FASTEMBED_HF_REPOS = { "fast-bge-small-zh-v1.5": "BAAI/bge-small-zh-v1.5" };
|
|
3780
|
-
async function ensureFastembedModelSidecars(model, cacheDir) {
|
|
3781
|
-
const repo = FASTEMBED_HF_REPOS[model];
|
|
3782
|
-
if (repo === void 0) return;
|
|
3783
|
-
const modelDir = join(cacheDir ?? "local_cache", model);
|
|
3784
|
-
mkdirSync(modelDir, { recursive: true });
|
|
3785
|
-
for (const fileName of FASTEMBED_SIDECARS) {
|
|
3967
|
+
const modelDir = modelDirPath(cacheDir);
|
|
3968
|
+
if (!existsSync(modelDir)) return false;
|
|
3969
|
+
return EMBEDDING_REQUIRED_FILES.every((fileName) => {
|
|
3786
3970
|
const target = join(modelDir, fileName);
|
|
3787
|
-
|
|
3788
|
-
|
|
3789
|
-
await access(target);
|
|
3790
|
-
continue;
|
|
3791
|
-
} catch {}
|
|
3792
|
-
const hfUrl = `https://huggingface.co/${repo}/resolve/main/${fileName}`;
|
|
3793
|
-
try {
|
|
3794
|
-
const response = await fetch(hfUrl);
|
|
3795
|
-
if (!response.ok) continue;
|
|
3796
|
-
const { writeFile } = await import("node:fs/promises");
|
|
3797
|
-
await writeFile(target, Buffer.from(await response.arrayBuffer()));
|
|
3798
|
-
} catch {}
|
|
3799
|
-
}
|
|
3971
|
+
return existsSync(target) && statSync(target).size > 0;
|
|
3972
|
+
});
|
|
3800
3973
|
}
|
|
3801
3974
|
//#endregion
|
|
3802
3975
|
//#region ../../packages/memory/src/store.ts
|
|
@@ -5977,7 +6150,7 @@ function entryFor(homeDir) {
|
|
|
5977
6150
|
await store.init();
|
|
5978
6151
|
const cacheDir = join$1(homeDir, "cache", "fastembed");
|
|
5979
6152
|
mkdirSync(cacheDir, { recursive: true });
|
|
5980
|
-
store.setEmbeddingEngine(
|
|
6153
|
+
store.setEmbeddingEngine(createEmbeddingEngine(cacheDir));
|
|
5981
6154
|
} catch (error) {
|
|
5982
6155
|
entries.delete(homeDir);
|
|
5983
6156
|
throw error;
|
|
@@ -5994,7 +6167,7 @@ function entryFor(homeDir) {
|
|
|
5994
6167
|
*
|
|
5995
6168
|
* The map key is the raw homeDir string so the engine cache key
|
|
5996
6169
|
* join(homeDir, 'cache', 'fastembed') stays byte-identical with every other
|
|
5997
|
-
*
|
|
6170
|
+
* createEmbeddingEngine call site: same string → same engine instance.
|
|
5998
6171
|
*/
|
|
5999
6172
|
function sharedKnowledgeStore(homeDir) {
|
|
6000
6173
|
return entryFor(homeDir).store;
|
|
@@ -6008,7 +6181,7 @@ function sharedKnowledgeStoreReady(homeDir) {
|
|
|
6008
6181
|
const HEADING_RE = /^(#{1,6})\s+(.+?)\s*$/;
|
|
6009
6182
|
const CODE_FENCE_RE = /^(\s*)(`{3,}|~{3,})/;
|
|
6010
6183
|
/**
|
|
6011
|
-
* Soft cap on chunk token count. BGE-small-zh-v1.5 (the
|
|
6184
|
+
* Soft cap on chunk token count. BGE-small-zh-v1.5 (the embedding model used
|
|
6012
6185
|
* here) has a 512-token sequence limit; chunks longer than that get truncated
|
|
6013
6186
|
* at the embedding layer, hurting retrieval quality. We split at ~480 to leave
|
|
6014
6187
|
* headroom for the heading line that gets prepended at embed time.
|
|
@@ -7254,4 +7427,4 @@ async function chunkVectorFallback(store, queryVec, topK) {
|
|
|
7254
7427
|
return results;
|
|
7255
7428
|
}
|
|
7256
7429
|
//#endregion
|
|
7257
|
-
export { sharedMemoStore as A, buildProjectTagCloud as B, applyConsolidation as C, parseMemoryMemos as D, buildExitExtractionPrompt as E, probeLocalEmbeddingSupport as F, join$1 as G, basename$1 as H, createMemoryMemo as I, relative$1 as J, normalize as K, toSummary as L, clearEmbeddingModelCache as M,
|
|
7430
|
+
export { sharedMemoStore as A, buildProjectTagCloud as B, applyConsolidation as C, parseMemoryMemos as D, buildExitExtractionPrompt as E, probeLocalEmbeddingSupport as F, join$1 as G, basename$1 as H, createMemoryMemo as I, relative$1 as J, normalize as K, toSummary as L, clearEmbeddingModelCache as M, createEmbeddingEngine as N, stripMemoryTags as O, hasEmbeddingModelCache as P, generateTags as R, DreamTracker as S, EXIT_EXTRACTION_SYSTEM_PROMPT as T, dirname$2 as U, rankMemos as V, isAbsolute$1 as W, resolve$1 as Y, splitLargeSection as _, isSupportedFile as a, sharedKnowledgeStoreReady as b, buildExtractionUserPrompt as c, extractQueryEntities as d, parseExtractionResponse as f, estimateTokens as g, chunkText as h, ingestFile as i, classifyEmbeddingFailure as j, MemoryMemoStore as k, extractEventFromChunk as l, chunkMarkdown as m, multiSearchWithTrace as n, ENTITY_TYPES as o, rerankEventsWithLlm as p, parse$1 as q, ingestDirectory as r, EXTRACTION_SYSTEM_PROMPT as s, multiSearch as t, extractJsonFromText as u, stripMarkdown as v, buildConsolidationPlan as w, KnowledgeStore as x, sharedKnowledgeStore as y, normalizeTags as z };
|
|
@@ -3,5 +3,5 @@ import { fileURLToPath as __cjsShimFileURLToPath } from 'node:url';
|
|
|
3
3
|
import { dirname as __cjsShimDirname } from 'node:path';
|
|
4
4
|
const __filename = __cjsShimFileURLToPath(import.meta.url);
|
|
5
5
|
const __dirname = __cjsShimDirname(__filename);
|
|
6
|
-
import { n as multiSearchWithTrace } from "./src-
|
|
6
|
+
import { n as multiSearchWithTrace } from "./src-ClB_Hzp9.mjs";
|
|
7
7
|
export { multiSearchWithTrace };
|
|
@@ -3,5 +3,5 @@ import { fileURLToPath as __cjsShimFileURLToPath } from 'node:url';
|
|
|
3
3
|
import { dirname as __cjsShimDirname } from 'node:path';
|
|
4
4
|
const __filename = __cjsShimFileURLToPath(import.meta.url);
|
|
5
5
|
const __dirname = __cjsShimDirname(__filename);
|
|
6
|
-
import { t as TextInputDialogComponent } from "./text-input-dialog-
|
|
6
|
+
import { t as TextInputDialogComponent } from "./text-input-dialog-DHDsHCqR.mjs";
|
|
7
7
|
export { TextInputDialogComponent };
|