scream-code 0.18.7 → 0.18.8

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -4,14 +4,14 @@ import { dirname as __cjsShimDirname } from 'node:path';
4
4
  const __filename = __cjsShimFileURLToPath(import.meta.url);
5
5
  const __dirname = __cjsShimDirname(__filename);
6
6
  import { createHash, randomBytes } from "node:crypto";
7
- import ro, { mkdir, readFile, readdir, rename, rm, rmdir, stat, unlink, writeFile } from "node:fs/promises";
7
+ import ro, { mkdir, open, readFile, readdir, rename, rm, rmdir, stat, unlink, writeFile } from "node:fs/promises";
8
8
  import I from "fs";
9
9
  import Qr, { EventEmitter } from "events";
10
10
  import js, { dirname, parse } from "path";
11
11
  import { Buffer as Buffer$1 } from "buffer";
12
12
  import Cs from "node:stream";
13
- import Kt, { createReadStream, existsSync, mkdirSync, rmSync, statSync } from "node:fs";
14
- import path, { basename, join, posix, win32 } from "node:path";
13
+ import Kt, { createReadStream, existsSync, mkdirSync, readFileSync, rmSync, statSync } from "node:fs";
14
+ import path, { basename, join, posix, resolve, win32 } from "node:path";
15
15
  import { DatabaseSync } from "node:sqlite";
16
16
  import { EventEmitter as EventEmitter$1 } from "node:events";
17
17
  import { StringDecoder } from "node:string_decoder";
@@ -3531,7 +3531,7 @@ function buildEmbeddingText(memo) {
3531
3531
  */
3532
3532
  const engineCache = /* @__PURE__ */ new Map();
3533
3533
  /**
3534
- * Create an embedding engine backed by fastembed.
3534
+ * Create an embedding engine backed by the in-process WASM pipeline.
3535
3535
  * Lazily loads the model on first use so startup is not blocked.
3536
3536
  * Engines are cached by cacheDir so repeated calls with the same cacheDir
3537
3537
  * return the same instance, sharing model state and download progress.
@@ -3539,21 +3539,22 @@ const engineCache = /* @__PURE__ */ new Map();
3539
3539
  * Defaults to "local_cache" (CWD-relative) if not provided — prefer
3540
3540
  * passing an explicit path so the cache doesn't duplicate across CWDs.
3541
3541
  */
3542
- function createFastEmbedEngine(cacheDir) {
3543
- const key = cacheDir ?? "";
3542
+ function createEmbeddingEngine(cacheDir) {
3543
+ const key = cacheDir === void 0 ? "" : resolve(cacheDir);
3544
3544
  const cached = engineCache.get(key);
3545
3545
  if (cached !== void 0) return cached;
3546
- const engine = createFastEmbedEngineImpl(cacheDir);
3546
+ const engine = createEmbeddingEngineImpl(cacheDir);
3547
3547
  engineCache.set(key, engine);
3548
3548
  return engine;
3549
3549
  }
3550
3550
  /** Stable identity of the model used by every engine created here. */
3551
3551
  const EMBEDDING_MODEL_NAME = "bge-small-zh-v1.5";
3552
- function createFastEmbedEngineImpl(cacheDir) {
3552
+ function createEmbeddingEngineImpl(cacheDir) {
3553
3553
  let embedder = null;
3554
3554
  let initPromise = null;
3555
3555
  let loadFailed = false;
3556
3556
  let lastError;
3557
+ let progressListener;
3557
3558
  return {
3558
3559
  modelName: EMBEDDING_MODEL_NAME,
3559
3560
  get available() {
@@ -3562,6 +3563,9 @@ function createFastEmbedEngineImpl(cacheDir) {
3562
3563
  get lastError() {
3563
3564
  return lastError;
3564
3565
  },
3566
+ setProgressListener(listener) {
3567
+ progressListener = listener;
3568
+ },
3565
3569
  async embedBatch(texts) {
3566
3570
  if (!this.available) return null;
3567
3571
  if (texts.length === 0) return [];
@@ -3598,12 +3602,12 @@ function createFastEmbedEngineImpl(cacheDir) {
3598
3602
  if (loadFailed || initPromise === null) {
3599
3603
  loadFailed = false;
3600
3604
  lastError = void 0;
3601
- initPromise = loadEmbedder(cacheDir);
3605
+ initPromise = loadEmbeddingRuntime(cacheDir, (progress) => progressListener?.(progress));
3602
3606
  }
3603
3607
  embedder = await initPromise;
3604
3608
  if (embedder === null) {
3605
3609
  initPromise = null;
3606
- lastError = "fastembed returned an empty model";
3610
+ lastError = "embedding runtime returned an empty session";
3607
3611
  return false;
3608
3612
  }
3609
3613
  loadFailed = false;
@@ -3618,42 +3622,157 @@ function createFastEmbedEngineImpl(cacheDir) {
3618
3622
  }
3619
3623
  };
3620
3624
  }
3621
- async function loadEmbedder(cacheDir) {
3622
- const { FlagEmbedding, EmbeddingModel } = await import("fastembed");
3623
- if (cacheDir !== void 0) {
3624
- mkdirSync(cacheDir, { recursive: true });
3625
- await ensureEmbeddingModelFiles(cacheDir);
3626
- }
3627
- const model = EmbeddingModel.BGESmallZH;
3628
- const initOpts = cacheDir !== void 0 ? {
3629
- model,
3630
- cacheDir
3631
- } : { model };
3632
- try {
3633
- return await FlagEmbedding.init(initOpts);
3634
- } catch (initError) {
3635
- const msg = initError instanceof Error ? initError.message : "";
3636
- if (!/Config file not found|Tokenizer file not found|Tokens map file not found/iu.test(msg)) throw initError;
3637
- await ensureFastembedModelSidecars(model, cacheDir);
3638
- return FlagEmbedding.init(initOpts);
3625
+ /** Maximum tokens per text; matches the model's position-embedding limit. */
3626
+ const MAX_SEQ_LEN = 512;
3627
+ /** Texts per inference batch: 16 × 512 × 512 floats ≈ 16MB of activations. */
3628
+ const EMBED_BATCH_SIZE = 16;
3629
+ /** Threads for the WASM backend. One keeps resolution deterministic in packaged
3630
+ * builds; `SCREAM_CODE_EMBEDDING_THREADS=4` is the supported opt-in. */
3631
+ const EMBEDDING_THREADS_ENV = "SCREAM_CODE_EMBEDDING_THREADS";
3632
+ /**
3633
+ * Token-level truncation that keeps the sentence boundary tokens.
3634
+ *
3635
+ * The tokenizer package does not apply the truncation policy itself and the
3636
+ * model package ships no truncation config, so this is the only place it can
3637
+ * happen. Keeping the head (which starts with `[CLS]`) and re-appending the
3638
+ * trailing special token (`[SEP]`) mirrors what a HuggingFace pipeline would
3639
+ * produce; slicing the tail off blindly would feed the encoder a sentence that
3640
+ * never closes and drift from the reference vectors on long inputs.
3641
+ */
3642
+ function truncateEncoded(ids, types) {
3643
+ if (ids.length <= MAX_SEQ_LEN) return {
3644
+ ids: [...ids],
3645
+ types: [...types]
3646
+ };
3647
+ const headSize = MAX_SEQ_LEN - 1;
3648
+ const head = ids.slice(0, headSize);
3649
+ const headTypes = types.slice(0, headSize);
3650
+ return {
3651
+ ids: [...head, ids.at(-1)],
3652
+ types: [...headTypes, types.at(-1)]
3653
+ };
3654
+ }
3655
+ /**
3656
+ * Builds the embedding runtime in-process on `onnxruntime-web` (WASM) and the
3657
+ * pure-JS tokenizer. No native binaries, no install scripts: the whole stack is
3658
+ * JavaScript plus one WebAssembly artifact resolved from the package.
3659
+ *
3660
+ * The numeric pipeline is load-bearing and must stay compatible with the
3661
+ * reference embeddings of this model: token ids come from `tokenizer.json`, and
3662
+ * the sentence vector is the **CLS token** hidden state (`last_hidden_state[0]`)
3663
+ * followed by L2 normalization. Mean pooling produces visibly different vectors
3664
+ * (cosine ≈ 0.9 against the reference), so pooling is not a stylistic choice.
3665
+ */
3666
+ async function createWasmRuntime(modelDir) {
3667
+ const ort = await import("onnxruntime-web");
3668
+ const { Tokenizer } = await import("@huggingface/tokenizers");
3669
+ const requestedThreads = Number(process.env[EMBEDDING_THREADS_ENV]);
3670
+ ort.env.wasm.numThreads = Number.isFinite(requestedThreads) && requestedThreads > 0 ? requestedThreads : 1;
3671
+ ort.env.logLevel = "error";
3672
+ const tokenizer = new Tokenizer(JSON.parse(readFileSync(join(modelDir, "tokenizer.json"), "utf8")), JSON.parse(readFileSync(join(modelDir, "tokenizer_config.json"), "utf8")));
3673
+ const session = await ort.InferenceSession.create(readFileSync(join(modelDir, "model_optimized.onnx")), { executionProviders: ["wasm"] });
3674
+ if (session === null || session === void 0) return null;
3675
+ return { async *embed(textStrings) {
3676
+ if (textStrings.length === 0) return;
3677
+ const encoded = textStrings.map((text, index) => {
3678
+ const encoding = tokenizer.encode(text, { return_token_type_ids: true });
3679
+ const types = encoding.token_type_ids ?? encoding.ids.map(() => 0);
3680
+ const truncated = truncateEncoded(encoding.ids, types);
3681
+ return {
3682
+ index,
3683
+ ids: truncated.ids,
3684
+ types: truncated.types
3685
+ };
3686
+ });
3687
+ const results = Array.from({ length: textStrings.length });
3688
+ const buckets = encoded.toSorted((a, b) => a.ids.length - b.ids.length);
3689
+ for (let start = 0; start < buckets.length; start += EMBED_BATCH_SIZE) {
3690
+ const group = buckets.slice(start, start + EMBED_BATCH_SIZE);
3691
+ const rows = group.length;
3692
+ const width = Math.max(...group.map((entry) => entry.ids.length));
3693
+ const ids = new BigInt64Array(rows * width);
3694
+ const mask = new BigInt64Array(rows * width);
3695
+ const types = new BigInt64Array(rows * width);
3696
+ group.forEach((entry, row) => {
3697
+ for (let column = 0; column < width; column++) {
3698
+ const offset = row * width + column;
3699
+ ids[offset] = BigInt(column < entry.ids.length ? entry.ids[column] : 0);
3700
+ mask[offset] = column < entry.ids.length ? 1n : 0n;
3701
+ types[offset] = BigInt(column < entry.types.length ? entry.types[column] : 0);
3702
+ }
3703
+ });
3704
+ const output = await session.run({
3705
+ input_ids: new ort.Tensor("int64", ids, [rows, width]),
3706
+ attention_mask: new ort.Tensor("int64", mask, [rows, width]),
3707
+ token_type_ids: new ort.Tensor("int64", types, [rows, width])
3708
+ });
3709
+ const hidden = output["last_hidden_state"] ?? output[Object.keys(output)[0]];
3710
+ const hiddenSize = hidden.dims.at(-1);
3711
+ const data = hidden.data;
3712
+ group.forEach((entry, row) => {
3713
+ const rowStart = row * width * hiddenSize;
3714
+ const vector = Array.from(data.subarray(rowStart, rowStart + hiddenSize));
3715
+ let norm = 0;
3716
+ for (const value of vector) norm += value * value;
3717
+ norm = Math.sqrt(norm) || 1;
3718
+ for (let i = 0; i < vector.length; i += 1) vector[i] = vector[i] / norm;
3719
+ results[entry.index] = vector;
3720
+ });
3721
+ }
3722
+ for (let start = 0; start < results.length; start += EMBED_BATCH_SIZE) yield results.slice(start, start + EMBED_BATCH_SIZE);
3723
+ } };
3724
+ }
3725
+ async function loadEmbeddingRuntime(cacheDir, emit) {
3726
+ if (cacheDir === void 0) {
3727
+ emit?.({ phase: "load" });
3728
+ return createWasmRuntime(join("local_cache", BGESMALLZH_CACHE_NAME));
3639
3729
  }
3730
+ mkdirSync(cacheDir, { recursive: true });
3731
+ await ensureEmbeddingModelFiles(cacheDir, {
3732
+ onProgress: (received, total) => {
3733
+ emit?.(total === void 0 ? {
3734
+ phase: "download",
3735
+ received
3736
+ } : {
3737
+ phase: "download",
3738
+ received,
3739
+ total
3740
+ });
3741
+ },
3742
+ onPhase: (phase) => {
3743
+ if (phase === "verify") emit?.({ phase: "verify" });
3744
+ }
3745
+ });
3746
+ emit?.({ phase: "load" });
3747
+ return createWasmRuntime(join(cacheDir, BGESMALLZH_CACHE_NAME));
3640
3748
  }
3641
3749
  /**
3642
- * Cache entry names fastembed uses for the fixed BGESmallZH model: it extracts
3643
- * into `<cacheDir>/<name>/` from the staged archive `<cacheDir>/<name>.tar.gz`.
3750
+ * Cache entry name for the fixed BGESmallZH model: the staged archive extracts
3751
+ * to `<cacheDir>/<name>/` and the runtime loads from that directory.
3644
3752
  */
3645
3753
  const BGESMALLZH_CACHE_NAME = "fast-bge-small-zh-v1.5";
3754
+ /** Files the runtime needs before the model can be considered installed. */
3755
+ const EMBEDDING_REQUIRED_FILES = [
3756
+ "model_optimized.onnx",
3757
+ "tokenizer.json",
3758
+ "tokenizer_config.json"
3759
+ ];
3646
3760
  /** Escape hatch for air-gapped installs and tests: points at an archive file. */
3647
3761
  const EMBEDDING_MODEL_URL_ENV = "SCREAM_CODE_EMBEDDING_MODEL_URL";
3648
- function modelWeightsPath(cacheDir) {
3649
- return join(cacheDir, BGESMALLZH_CACHE_NAME, "model_optimized.onnx");
3762
+ /** Default stall guards: a trickling connection must not spin a spinner forever. */
3763
+ const DEFAULT_IDLE_TIMEOUT_MS = 6e4;
3764
+ const DEFAULT_TOTAL_TIMEOUT_MS = 15 * 6e4;
3765
+ /** Progress callback throttle: frequent enough to move, rare enough to be free. */
3766
+ const PROGRESS_THROTTLE_MS = 100;
3767
+ function modelDirPath(cacheDir) {
3768
+ return join(cacheDir, BGESMALLZH_CACHE_NAME);
3650
3769
  }
3651
3770
  function sha256File(filePath) {
3652
3771
  return readFile(filePath).then((data) => createHash("sha256").update(data).digest("hex"));
3653
3772
  }
3654
3773
  /**
3655
3774
  * Stage the embedding model files when they are missing: download the pinned
3656
- * archive (HTTP status verified — unlike fastembed's own downloader), extract,
3775
+ * archive (HTTP status verified, unlike a naive downloader), extract,
3657
3776
  * and verify the weights payload against the pinned sha256.
3658
3777
  *
3659
3778
  * A partially extracted tree or a hash mismatch is removed so the caller keeps
@@ -3663,62 +3782,149 @@ function sha256File(filePath) {
3663
3782
  async function ensureEmbeddingModelFiles(cacheDir, options) {
3664
3783
  if (hasEmbeddingModelCache(cacheDir)) return;
3665
3784
  mkdirSync(cacheDir, { recursive: true });
3666
- const modelDir = join(cacheDir, BGESMALLZH_CACHE_NAME);
3785
+ const modelDir = modelDirPath(cacheDir);
3667
3786
  const stagingPath = join(cacheDir, `${BGESMALLZH_CACHE_NAME}.tar.gz.part`);
3668
3787
  const url = options?.sourceUrl ?? process.env[EMBEDDING_MODEL_URL_ENV] ?? "https://github.com/LIUTod/scream-code/releases/download/embedding-models-v1/fast-bge-small-zh-v1.5.tar.gz";
3788
+ const idleTimeoutMs = options?.idleTimeoutMs ?? DEFAULT_IDLE_TIMEOUT_MS;
3789
+ const totalTimeoutMs = options?.totalTimeoutMs ?? DEFAULT_TOTAL_TIMEOUT_MS;
3790
+ const controller = new AbortController();
3791
+ const onExternalAbort = () => {
3792
+ controller.abort();
3793
+ };
3794
+ if (options?.signal !== void 0) if (options.signal.aborted) controller.abort();
3795
+ else options.signal.addEventListener("abort", onExternalAbort, { once: true });
3796
+ let idleTimer;
3797
+ const armIdleTimer = () => {
3798
+ if (idleTimer !== void 0) clearTimeout(idleTimer);
3799
+ idleTimer = setTimeout(() => {
3800
+ controller.abort();
3801
+ }, idleTimeoutMs);
3802
+ };
3803
+ const totalTimer = setTimeout(() => {
3804
+ controller.abort();
3805
+ }, totalTimeoutMs);
3806
+ let received = 0;
3807
+ let announcedTotal;
3808
+ let downloadCompleted = false;
3809
+ const announceProgress = () => {
3810
+ options?.onProgress?.(received, announcedTotal);
3811
+ };
3669
3812
  try {
3670
3813
  let response;
3671
3814
  try {
3672
- response = await fetch(url);
3815
+ armIdleTimer();
3816
+ response = await fetch(url, { signal: controller.signal });
3673
3817
  } catch (error) {
3674
3818
  const message = error instanceof Error ? error.message : String(error);
3675
3819
  throw new Error(`fetch failed for ${url}: ${message}`, { cause: error });
3676
3820
  }
3677
3821
  if (!response.ok) throw new Error(`model source returned HTTP ${response.status} for ${url}`);
3678
- await writeFile(stagingPath, Buffer.from(await response.arrayBuffer()));
3822
+ const contentLength = Number(response.headers.get("content-length"));
3823
+ announcedTotal = Number.isFinite(contentLength) && contentLength > 0 ? contentLength : void 0;
3824
+ announceProgress();
3825
+ const body = response.body;
3826
+ if (body === null) throw new Error(`model source returned an empty body for ${url}`);
3827
+ const reader = body.getReader();
3828
+ const abortRejection = new Promise((_, reject) => {
3829
+ if (controller.signal.aborted) {
3830
+ reject(/* @__PURE__ */ new Error("aborted"));
3831
+ return;
3832
+ }
3833
+ controller.signal.addEventListener("abort", () => {
3834
+ reject(/* @__PURE__ */ new Error("aborted"));
3835
+ }, { once: true });
3836
+ });
3837
+ const abortAwareRead = () => Promise.race([reader.read(), abortRejection]);
3838
+ let handle;
3839
+ try {
3840
+ handle = await open(stagingPath, "w");
3841
+ } catch (error) {
3842
+ await reader.cancel().catch(() => void 0);
3843
+ throw error;
3844
+ }
3845
+ let lastProgressAt = 0;
3846
+ try {
3847
+ await (async () => {
3848
+ for (;;) {
3849
+ let chunk;
3850
+ try {
3851
+ chunk = await abortAwareRead();
3852
+ } catch (error) {
3853
+ const message = error instanceof Error ? error.message : String(error);
3854
+ throw new Error(`${message} (received ${received} of ${announcedTotal ?? "?"} bytes): ${url}`, { cause: error });
3855
+ }
3856
+ if (chunk.done) break;
3857
+ const bytes = chunk.value;
3858
+ armIdleTimer();
3859
+ await handle.write(bytes);
3860
+ received += bytes.byteLength;
3861
+ const now = Date.now();
3862
+ if (now - lastProgressAt >= PROGRESS_THROTTLE_MS) {
3863
+ lastProgressAt = now;
3864
+ announceProgress();
3865
+ }
3866
+ }
3867
+ announceProgress();
3868
+ downloadCompleted = true;
3869
+ })();
3870
+ } finally {
3871
+ if (idleTimer !== void 0) clearTimeout(idleTimer);
3872
+ await handle.close();
3873
+ await reader.cancel().catch(() => void 0);
3874
+ }
3875
+ options?.onPhase?.("verify");
3679
3876
  await So({
3680
3877
  file: stagingPath,
3681
3878
  cwd: cacheDir
3682
3879
  });
3683
- const weightsPath = modelWeightsPath(cacheDir);
3684
- if (!existsSync(weightsPath) || statSync(weightsPath).size === 0) throw new Error(`model package did not contain weights after extraction: ${url}`);
3685
- const sha256 = await sha256File(weightsPath);
3880
+ for (const fileName of EMBEDDING_REQUIRED_FILES) {
3881
+ const target = join(modelDir, fileName);
3882
+ if (!existsSync(target) || statSync(target).size === 0) throw new Error(`model package is missing ${fileName} after extraction: ${url}`);
3883
+ }
3884
+ const sha256 = await sha256File(join(modelDir, "model_optimized.onnx"));
3686
3885
  const expected = options?.expectedOnnxSha256 ?? "1294ea4b6331115a353d81f96b85e8c8d7fdcc284453d5b2fab5b016230aad38";
3687
3886
  if (sha256 !== expected) throw new Error(`model weights sha256 mismatch (${sha256.slice(0, 12)}… != ${expected.slice(0, 12)}…): ${url}`);
3688
3887
  } catch (error) {
3689
- rmSync(modelDir, {
3690
- recursive: true,
3691
- force: true
3692
- });
3693
- throw error instanceof Error ? error : new Error(String(error));
3888
+ try {
3889
+ rmSync(modelDir, {
3890
+ recursive: true,
3891
+ force: true
3892
+ });
3893
+ } catch {}
3894
+ if (!downloadCompleted && controller.signal.aborted && !(options?.signal?.aborted ?? false)) throw new Error(`model download timed out (received ${received} of ${announcedTotal ?? "?"} bytes): ${url}`, { cause: error });
3895
+ throw error instanceof Error ? error : new Error(String(error), { cause: error });
3694
3896
  } finally {
3897
+ clearTimeout(totalTimer);
3898
+ if (idleTimer !== void 0) clearTimeout(idleTimer);
3899
+ if (options?.signal !== void 0) options.signal.removeEventListener("abort", onExternalAbort);
3695
3900
  rmSync(stagingPath, { force: true });
3696
3901
  }
3697
3902
  }
3698
- const PLATFORM_FAILURE = /@anush008\/tokenizers|Failed to load native binding|Unsupported (?:OS|platform|architecture)|ERR_DLOPEN_FAILED|invalid ELF header|Exec format error|is not a valid Win32 application|GLIBC_\d+[^"\n]*not found|Error loading shared library|cannot open shared object file/iu;
3699
- const ARCHIVE_FAILURE = /\bTAR_[A-Z_]+|incorrect header check|unexpected end of file|invalid tar|zlib/iu;
3700
- const NETWORK_FAILURE = /\b(?:ENOTFOUND|EAI_AGAIN|ETIMEDOUT|ECONNRESET|ECONNREFUSED|EHOSTUNREACH|socket hang up|fetch failed)\b|network error|HTTP \d{3}/iu;
3903
+ const INSTALL_FAILURE = /Cannot find (?:package|module) '(?:onnxruntime-web|@huggingface\/tokenizers)'|ERR_MODULE_NOT_FOUND|Failed to resolve module specifier/iu;
3904
+ const ARCHIVE_FAILURE = /\bTAR_[A-Z_]+|incorrect header check|unexpected end of file|invalid tar|zlib|after extraction|sha256 mismatch/iu;
3905
+ const NETWORK_FAILURE = /\b(?:ENOTFOUND|EAI_AGAIN|ETIMEDOUT|ECONNRESET|ECONNREFUSED|EHOSTUNREACH|socket hang up|fetch failed)\b|network error|HTTP \d{3}|timed out|empty body/iu;
3701
3906
  /**
3702
- * Classify a load failure so callers can say something true about it: the four
3703
- * causes need four different answers (unsupported platform, corrupt cache,
3704
- * network, unknown), and they used to be reported as one network problem.
3907
+ * Classify a load failure so callers can say something true about it: an
3908
+ * incomplete install, a corrupt cache, a network problem and anything else need
3909
+ * four different answers, and they used to be reported as one network problem.
3705
3910
  */
3706
3911
  function classifyEmbeddingFailure(message) {
3707
3912
  if (message === void 0) return "other";
3708
- if (PLATFORM_FAILURE.test(message)) return "platform";
3913
+ if (INSTALL_FAILURE.test(message)) return "install";
3709
3914
  if (ARCHIVE_FAILURE.test(message)) return "archive";
3710
3915
  if (NETWORK_FAILURE.test(message)) return "network";
3711
3916
  return "other";
3712
3917
  }
3713
3918
  /**
3714
- * Loads only the module entry point — no model download — so callers can tell
3715
- * "this machine cannot run the local model at all" apart from "the download
3716
- * failed". fastembed imports its native tokenizer binding at module scope, which
3717
- * is exactly the step a platform without a published binary cannot pass.
3919
+ * Loads both runtime packages — no model download — so callers can tell "this
3920
+ * installation cannot run the local model at all" apart from "the download
3921
+ * failed". The runtime is JavaScript plus WebAssembly, so a failure here is an
3922
+ * incomplete installation rather than an unsupported platform.
3718
3923
  */
3719
3924
  async function probeLocalEmbeddingSupport() {
3720
3925
  try {
3721
- await import("fastembed");
3926
+ await import("onnxruntime-web");
3927
+ await import("@huggingface/tokenizers");
3722
3928
  return { supported: true };
3723
3929
  } catch (error) {
3724
3930
  return {
@@ -3729,15 +3935,11 @@ async function probeLocalEmbeddingSupport() {
3729
3935
  }
3730
3936
  /**
3731
3937
  * Remove every trace of a previous BGESmallZH download: the extracted model
3732
- * directory *and* the staged archive it came from.
3938
+ * directory and the legacy staged archive name.
3733
3939
  *
3734
- * The archive has to go with it. fastembed skips the download whenever
3735
- * `<cacheDir>/<name>.tar.gz` exists, and it only deletes that file after a
3736
- * *successful* extract — its download error paths (response error, file stream
3737
- * error) reject and leave whatever was written behind. So an archive sitting
3738
- * next to a failed attempt is usually a partial leftover, and keeping it makes
3739
- * every retry extract the same junk locally: no network traffic, a few
3740
- * milliseconds, the same failure forever.
3940
+ * The legacy `<name>.tar.gz` file is still swept because older versions staged
3941
+ * the archive under that name; leaving it behind would waste 55MB per doomed
3942
+ * attempt. Our own downloader stages `.tar.gz.part` and always removes it.
3741
3943
  *
3742
3944
  * Never throws: a file locked by another process (on Windows a mapped ONNX file
3743
3945
  * stays locked) must not replace the download error the caller is reporting.
@@ -3755,48 +3957,19 @@ function clearEmbeddingModelCache(cacheDir) {
3755
3957
  }
3756
3958
  }
3757
3959
  /**
3758
- * True when the extracted model weights are already on disk. Loading then
3759
- * never downloads (fastembed skips the fetch whenever the model dir exists),
3760
- * so callers can safely trigger a local load without risking a surprise
3761
- * mid-query download.
3960
+ * True when every file the runtime needs is already on disk. Loading then never
3961
+ * downloads, so callers can safely trigger a local load without risking a
3962
+ * surprise mid-query download. All required files are checked (not just the
3963
+ * weights): a partially restored cache must count as missing so the setup flow
3964
+ * repairs it instead of failing at load time.
3762
3965
  */
3763
3966
  function hasEmbeddingModelCache(cacheDir) {
3764
- const modelDir = join(cacheDir, BGESMALLZH_CACHE_NAME);
3765
- return existsSync(modelDir) && existsSync(join(modelDir, "model_optimized.onnx"));
3766
- }
3767
- /**
3768
- * Small config/tokenizer files that fastembed expects alongside model.onnx.
3769
- * If these are missing (e.g. GCS download partially failed), fastembed throws.
3770
- * We download them from HuggingFace so the model can load — this covers the
3771
- * case where the GCS tarball was incomplete but the HF repo is reachable.
3772
- */
3773
- const FASTEMBED_SIDECARS = [
3774
- "config.json",
3775
- "tokenizer.json",
3776
- "tokenizer_config.json",
3777
- "special_tokens_map.json"
3778
- ];
3779
- const FASTEMBED_HF_REPOS = { "fast-bge-small-zh-v1.5": "BAAI/bge-small-zh-v1.5" };
3780
- async function ensureFastembedModelSidecars(model, cacheDir) {
3781
- const repo = FASTEMBED_HF_REPOS[model];
3782
- if (repo === void 0) return;
3783
- const modelDir = join(cacheDir ?? "local_cache", model);
3784
- mkdirSync(modelDir, { recursive: true });
3785
- for (const fileName of FASTEMBED_SIDECARS) {
3967
+ const modelDir = modelDirPath(cacheDir);
3968
+ if (!existsSync(modelDir)) return false;
3969
+ return EMBEDDING_REQUIRED_FILES.every((fileName) => {
3786
3970
  const target = join(modelDir, fileName);
3787
- try {
3788
- const { access } = await import("node:fs/promises");
3789
- await access(target);
3790
- continue;
3791
- } catch {}
3792
- const hfUrl = `https://huggingface.co/${repo}/resolve/main/${fileName}`;
3793
- try {
3794
- const response = await fetch(hfUrl);
3795
- if (!response.ok) continue;
3796
- const { writeFile } = await import("node:fs/promises");
3797
- await writeFile(target, Buffer.from(await response.arrayBuffer()));
3798
- } catch {}
3799
- }
3971
+ return existsSync(target) && statSync(target).size > 0;
3972
+ });
3800
3973
  }
3801
3974
  //#endregion
3802
3975
  //#region ../../packages/memory/src/store.ts
@@ -5977,7 +6150,7 @@ function entryFor(homeDir) {
5977
6150
  await store.init();
5978
6151
  const cacheDir = join$1(homeDir, "cache", "fastembed");
5979
6152
  mkdirSync(cacheDir, { recursive: true });
5980
- store.setEmbeddingEngine(createFastEmbedEngine(cacheDir));
6153
+ store.setEmbeddingEngine(createEmbeddingEngine(cacheDir));
5981
6154
  } catch (error) {
5982
6155
  entries.delete(homeDir);
5983
6156
  throw error;
@@ -5994,7 +6167,7 @@ function entryFor(homeDir) {
5994
6167
  *
5995
6168
  * The map key is the raw homeDir string so the engine cache key
5996
6169
  * join(homeDir, 'cache', 'fastembed') stays byte-identical with every other
5997
- * createFastEmbedEngine call site: same string → same engine instance.
6170
+ * createEmbeddingEngine call site: same string → same engine instance.
5998
6171
  */
5999
6172
  function sharedKnowledgeStore(homeDir) {
6000
6173
  return entryFor(homeDir).store;
@@ -6008,7 +6181,7 @@ function sharedKnowledgeStoreReady(homeDir) {
6008
6181
  const HEADING_RE = /^(#{1,6})\s+(.+?)\s*$/;
6009
6182
  const CODE_FENCE_RE = /^(\s*)(`{3,}|~{3,})/;
6010
6183
  /**
6011
- * Soft cap on chunk token count. BGE-small-zh-v1.5 (the fastembed model used
6184
+ * Soft cap on chunk token count. BGE-small-zh-v1.5 (the embedding model used
6012
6185
  * here) has a 512-token sequence limit; chunks longer than that get truncated
6013
6186
  * at the embedding layer, hurting retrieval quality. We split at ~480 to leave
6014
6187
  * headroom for the heading line that gets prepended at embed time.
@@ -7254,4 +7427,4 @@ async function chunkVectorFallback(store, queryVec, topK) {
7254
7427
  return results;
7255
7428
  }
7256
7429
  //#endregion
7257
- export { sharedMemoStore as A, buildProjectTagCloud as B, applyConsolidation as C, parseMemoryMemos as D, buildExitExtractionPrompt as E, probeLocalEmbeddingSupport as F, join$1 as G, basename$1 as H, createMemoryMemo as I, relative$1 as J, normalize as K, toSummary as L, clearEmbeddingModelCache as M, createFastEmbedEngine as N, stripMemoryTags as O, hasEmbeddingModelCache as P, generateTags as R, DreamTracker as S, EXIT_EXTRACTION_SYSTEM_PROMPT as T, dirname$2 as U, rankMemos as V, isAbsolute$1 as W, resolve$1 as Y, splitLargeSection as _, isSupportedFile as a, sharedKnowledgeStoreReady as b, buildExtractionUserPrompt as c, extractQueryEntities as d, parseExtractionResponse as f, estimateTokens as g, chunkText as h, ingestFile as i, classifyEmbeddingFailure as j, MemoryMemoStore as k, extractEventFromChunk as l, chunkMarkdown as m, multiSearchWithTrace as n, ENTITY_TYPES as o, rerankEventsWithLlm as p, parse$1 as q, ingestDirectory as r, EXTRACTION_SYSTEM_PROMPT as s, multiSearch as t, extractJsonFromText as u, stripMarkdown as v, buildConsolidationPlan as w, KnowledgeStore as x, sharedKnowledgeStore as y, normalizeTags as z };
7430
+ export { sharedMemoStore as A, buildProjectTagCloud as B, applyConsolidation as C, parseMemoryMemos as D, buildExitExtractionPrompt as E, probeLocalEmbeddingSupport as F, join$1 as G, basename$1 as H, createMemoryMemo as I, relative$1 as J, normalize as K, toSummary as L, clearEmbeddingModelCache as M, createEmbeddingEngine as N, stripMemoryTags as O, hasEmbeddingModelCache as P, generateTags as R, DreamTracker as S, EXIT_EXTRACTION_SYSTEM_PROMPT as T, dirname$2 as U, rankMemos as V, isAbsolute$1 as W, resolve$1 as Y, splitLargeSection as _, isSupportedFile as a, sharedKnowledgeStoreReady as b, buildExtractionUserPrompt as c, extractQueryEntities as d, parseExtractionResponse as f, estimateTokens as g, chunkText as h, ingestFile as i, classifyEmbeddingFailure as j, MemoryMemoStore as k, extractEventFromChunk as l, chunkMarkdown as m, multiSearchWithTrace as n, ENTITY_TYPES as o, rerankEventsWithLlm as p, parse$1 as q, ingestDirectory as r, EXTRACTION_SYSTEM_PROMPT as s, multiSearch as t, extractJsonFromText as u, stripMarkdown as v, buildConsolidationPlan as w, KnowledgeStore as x, sharedKnowledgeStore as y, normalizeTags as z };
@@ -3,5 +3,5 @@ import { fileURLToPath as __cjsShimFileURLToPath } from 'node:url';
3
3
  import { dirname as __cjsShimDirname } from 'node:path';
4
4
  const __filename = __cjsShimFileURLToPath(import.meta.url);
5
5
  const __dirname = __cjsShimDirname(__filename);
6
- import { n as multiSearchWithTrace } from "./src-CQwXASmd.mjs";
6
+ import { n as multiSearchWithTrace } from "./src-ClB_Hzp9.mjs";
7
7
  export { multiSearchWithTrace };
@@ -3,5 +3,5 @@ import { fileURLToPath as __cjsShimFileURLToPath } from 'node:url';
3
3
  import { dirname as __cjsShimDirname } from 'node:path';
4
4
  const __filename = __cjsShimFileURLToPath(import.meta.url);
5
5
  const __dirname = __cjsShimDirname(__filename);
6
- import { t as TextInputDialogComponent } from "./text-input-dialog-gDr3i7xz.mjs";
6
+ import { t as TextInputDialogComponent } from "./text-input-dialog-DHDsHCqR.mjs";
7
7
  export { TextInputDialogComponent };