sweet-search 2.4.2 → 2.5.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/core/cli.js +43 -5
- package/core/embedding/embedding-cache.js +266 -18
- package/core/embedding/embedding-service.js +45 -9
- package/core/graph/graph-expansion.js +52 -12
- package/core/graph/graph-extractor.js +30 -1
- package/core/indexing/ast-chunker.js +331 -16
- package/core/indexing/chunking/chunk-builder.js +34 -1
- package/core/indexing/index-codebase-v21.js +31 -2
- package/core/indexing/index.js +6 -3
- package/core/indexing/indexer-ann.js +45 -6
- package/core/indexing/indexer-build.js +9 -1
- package/core/indexing/indexer-phases.js +6 -4
- package/core/indexing/indexing-file-policy.js +140 -0
- package/core/indexing/li-skip-policy.js +11 -220
- package/core/infrastructure/codebase-repository.js +21 -0
- package/core/infrastructure/config/embedding.js +20 -1
- package/core/infrastructure/config/graph.js +2 -2
- package/core/infrastructure/config/ranking.js +10 -0
- package/core/infrastructure/config/vector-store.js +1 -1
- package/core/infrastructure/coreml-cascade.js +236 -30
- package/core/infrastructure/coreml-cascade.json +25 -0
- package/core/infrastructure/index.js +17 -0
- package/core/infrastructure/init-config.js +216 -0
- package/core/infrastructure/language-patterns/registry-core.js +18 -0
- package/core/infrastructure/model-registry.js +12 -0
- package/core/infrastructure/native-inference.js +143 -51
- package/core/infrastructure/tree-sitter-provider.js +92 -2
- package/core/ranking/cascaded-scorer.js +6 -2
- package/core/ranking/file-kind-ranking.js +264 -0
- package/core/ranking/late-interaction-index.js +10 -4
- package/core/ranking/late-interaction-policy.js +304 -0
- package/core/search/context-expander.js +267 -28
- package/core/search/index.js +4 -0
- package/core/search/search-cli.js +3 -1
- package/core/search/search-pattern.js +4 -3
- package/core/search/search-postprocess.js +189 -8
- package/core/search/search-read-semantic.js +734 -0
- package/core/search/search-read.js +481 -0
- package/core/search/search-server.js +153 -5
- package/core/search/sweet-search.js +133 -16
- package/core/start-server.js +13 -2
- package/mcp/server.js +41 -0
- package/mcp/tool-handlers.js +117 -6
- package/package.json +9 -7
- package/scripts/init.js +386 -5
- package/scripts/uninstall.js +152 -6
|
@@ -63,7 +63,9 @@ let _spec = null;
|
|
|
63
63
|
* `crates/sweet-search-native/src/inference/embedding_model.rs::parse_embed_variant_filename`
|
|
64
64
|
* (line ~140, `strip_prefix("nomic_bert_b")`) and
|
|
65
65
|
* `crates/sweet-search-native/src/inference/li_model.rs::parse_li_variant_filename`
|
|
66
|
-
* (
|
|
66
|
+
* (recognises BOTH `li_modernbert_b` for the standard `lateon-code`
|
|
67
|
+
* variant and `li_modernbert_edge_b` for the `lateon-code-edge`
|
|
68
|
+
* variant — the parser tries the longer "edge" prefix first).
|
|
67
69
|
*
|
|
68
70
|
* A future JSON edit that renames variants (e.g. `nomic_bert_v2`,
|
|
69
71
|
* `li_modernbert_short`) would silently disarm the cascade on end-user
|
|
@@ -74,8 +76,24 @@ let _spec = null;
|
|
|
74
76
|
*/
|
|
75
77
|
export const RUST_EMBED_PREFIX = 'nomic_bert_b';
|
|
76
78
|
export const RUST_LI_PREFIX = 'li_modernbert_b';
|
|
79
|
+
export const RUST_LI_EDGE_PREFIX = 'li_modernbert_edge_b';
|
|
77
80
|
export const RUST_VARIANT_SUFFIX = '_fp16.mlpackage';
|
|
78
81
|
|
|
82
|
+
/**
|
|
83
|
+
* Map an LI variant key from `LATE_INTERACTION_CONFIG.model` to its
|
|
84
|
+
* cascade section name in coreml-cascade.json. Pure helper.
|
|
85
|
+
*
|
|
86
|
+
* `lateon-code` (default) → `li`
|
|
87
|
+
* `lateon-code-edge` → `liEdge`
|
|
88
|
+
*
|
|
89
|
+
* Unknown / null variants fall back to the standard section so callers
|
|
90
|
+
* who don't pass a variant get the historical behaviour.
|
|
91
|
+
*/
|
|
92
|
+
export function liVariantToSectionKey(liVariantKey) {
|
|
93
|
+
if (liVariantKey === 'lateon-code-edge') return 'liEdge';
|
|
94
|
+
return 'li';
|
|
95
|
+
}
|
|
96
|
+
|
|
79
97
|
/**
|
|
80
98
|
* Validate a loaded cascade spec against every invariant the JS side,
|
|
81
99
|
* the Python trace script, and the Rust filename parser all depend on.
|
|
@@ -98,12 +116,21 @@ export function validateCascadeSpec(spec) {
|
|
|
98
116
|
throw new Error(`[CoremlCascade] invalid spec: hfRepo must be a non-empty string, got ${JSON.stringify(spec.hfRepo)}`);
|
|
99
117
|
}
|
|
100
118
|
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
119
|
+
// `embed` and `li` are required (the standard cascade has shipped
|
|
120
|
+
// since 2026-04-14). `liEdge` is OPTIONAL — older specs without it
|
|
121
|
+
// remain valid, runtime code uses `spec.liEdge?.variants ?? []`.
|
|
122
|
+
// When `liEdge` IS present, every invariant below applies in full.
|
|
123
|
+
for (const [side, requiredPrefix, optional] of [
|
|
124
|
+
['embed', RUST_EMBED_PREFIX, false],
|
|
125
|
+
['li', RUST_LI_PREFIX, false],
|
|
126
|
+
['liEdge', RUST_LI_EDGE_PREFIX, true],
|
|
104
127
|
]) {
|
|
105
128
|
const section = spec[side];
|
|
106
|
-
if (!section
|
|
129
|
+
if (!section) {
|
|
130
|
+
if (optional) continue;
|
|
131
|
+
throw new Error(`[CoremlCascade] invalid spec: missing or non-object \`${side}\` section`);
|
|
132
|
+
}
|
|
133
|
+
if (typeof section !== 'object') {
|
|
107
134
|
throw new Error(`[CoremlCascade] invalid spec: missing or non-object \`${side}\` section`);
|
|
108
135
|
}
|
|
109
136
|
if (typeof section.filePattern !== 'string') {
|
|
@@ -225,11 +252,26 @@ export function getCoremlEmbedDir() {
|
|
|
225
252
|
return join(getCoremlCascadeRoot(), 'embed');
|
|
226
253
|
}
|
|
227
254
|
|
|
228
|
-
/** Subdirectory holding LI variants. */
|
|
255
|
+
/** Subdirectory holding standard `lateon-code` LI variants. */
|
|
229
256
|
export function getCoremlLiDir() {
|
|
230
257
|
return join(getCoremlCascadeRoot(), 'li');
|
|
231
258
|
}
|
|
232
259
|
|
|
260
|
+
/** Subdirectory holding `lateon-code-edge` LI variants. */
|
|
261
|
+
export function getCoremlLiEdgeDir() {
|
|
262
|
+
return join(getCoremlCascadeRoot(), 'li-edge');
|
|
263
|
+
}
|
|
264
|
+
|
|
265
|
+
/**
|
|
266
|
+
* Resolve the on-disk LI cascade dir for a given LI variant. Used by
|
|
267
|
+
* native-inference.js to point the Rust addon at the right family.
|
|
268
|
+
*
|
|
269
|
+
* @param {string|undefined} liVariantKey - 'lateon-code' (default) or 'lateon-code-edge'
|
|
270
|
+
*/
|
|
271
|
+
export function getCoremlLiDirForVariant(liVariantKey) {
|
|
272
|
+
return liVariantKey === 'lateon-code-edge' ? getCoremlLiEdgeDir() : getCoremlLiDir();
|
|
273
|
+
}
|
|
274
|
+
|
|
233
275
|
/**
|
|
234
276
|
* Format a variant filename from the pattern in coreml-cascade.json.
|
|
235
277
|
* `{batch}` → batch, `{seq}` → seq. Kept in lockstep with the Rust parser.
|
|
@@ -283,14 +325,16 @@ export function isValidMlpackage(path) {
|
|
|
283
325
|
/**
|
|
284
326
|
* Enumerate every expected variant path for the current spec.
|
|
285
327
|
*
|
|
286
|
-
* Returns
|
|
287
|
-
* embedPaths:
|
|
288
|
-
* liPaths:
|
|
328
|
+
* Returns three lists:
|
|
329
|
+
* embedPaths: [{ batch, seq, filename, fullPath }, ...]
|
|
330
|
+
* liPaths: [{ batch, seq, filename, fullPath }, ...] — standard 128d
|
|
331
|
+
* liEdgePaths: [{ batch, seq, filename, fullPath }, ...] — edge 48d
|
|
289
332
|
*/
|
|
290
333
|
export function getExpectedVariantPaths() {
|
|
291
334
|
const spec = getCascadeSpec();
|
|
292
335
|
const embedDir = getCoremlEmbedDir();
|
|
293
336
|
const liDir = getCoremlLiDir();
|
|
337
|
+
const liEdgeDir = getCoremlLiEdgeDir();
|
|
294
338
|
|
|
295
339
|
const embedPaths = spec.embed.variants.map(v => {
|
|
296
340
|
const filename = formatVariantFilename(spec.embed.filePattern, v.batch, v.seq);
|
|
@@ -300,8 +344,12 @@ export function getExpectedVariantPaths() {
|
|
|
300
344
|
const filename = formatVariantFilename(spec.li.filePattern, v.batch, v.seq);
|
|
301
345
|
return { batch: v.batch, seq: v.seq, filename, fullPath: join(liDir, filename) };
|
|
302
346
|
});
|
|
347
|
+
const liEdgePaths = (spec.liEdge?.variants || []).map(v => {
|
|
348
|
+
const filename = formatVariantFilename(spec.liEdge.filePattern, v.batch, v.seq);
|
|
349
|
+
return { batch: v.batch, seq: v.seq, filename, fullPath: join(liEdgeDir, filename) };
|
|
350
|
+
});
|
|
303
351
|
|
|
304
|
-
return { embedPaths, liPaths };
|
|
352
|
+
return { embedPaths, liPaths, liEdgePaths };
|
|
305
353
|
}
|
|
306
354
|
|
|
307
355
|
/**
|
|
@@ -311,16 +359,24 @@ export function getExpectedVariantPaths() {
|
|
|
311
359
|
* Use `getCoremlCascadeResolvedDirs()` when you only need the
|
|
312
360
|
* (embedDir, liDir) pair for native-inference.js to pass to the addon.
|
|
313
361
|
*
|
|
362
|
+
* `complete` is true only when the embed cascade is complete AND at
|
|
363
|
+
* least one of the LI cascades (standard or edge) is complete — so a
|
|
364
|
+
* single-variant deployment that ships only one LI family still
|
|
365
|
+
* counts as a healthy cascade for the matching LI variant.
|
|
366
|
+
*
|
|
314
367
|
* @returns {{
|
|
315
368
|
* applicable: boolean,
|
|
316
369
|
* reason: string,
|
|
317
370
|
* root: string,
|
|
318
371
|
* embedDir: string,
|
|
319
372
|
* liDir: string,
|
|
373
|
+
* liEdgeDir: string,
|
|
320
374
|
* embedPresent: number,
|
|
321
375
|
* embedTotal: number,
|
|
322
376
|
* liPresent: number,
|
|
323
377
|
* liTotal: number,
|
|
378
|
+
* liEdgePresent: number,
|
|
379
|
+
* liEdgeTotal: number,
|
|
324
380
|
* complete: boolean,
|
|
325
381
|
* missing: string[],
|
|
326
382
|
* }}
|
|
@@ -330,7 +386,8 @@ export function getCoremlCascadeState() {
|
|
|
330
386
|
const root = getCoremlCascadeRoot();
|
|
331
387
|
const embedDir = getCoremlEmbedDir();
|
|
332
388
|
const liDir = getCoremlLiDir();
|
|
333
|
-
const
|
|
389
|
+
const liEdgeDir = getCoremlLiEdgeDir();
|
|
390
|
+
const { embedPaths, liPaths, liEdgePaths } = getExpectedVariantPaths();
|
|
334
391
|
|
|
335
392
|
const missing = [];
|
|
336
393
|
let embedPresent = 0;
|
|
@@ -343,10 +400,23 @@ export function getCoremlCascadeState() {
|
|
|
343
400
|
if (isValidMlpackage(v.fullPath)) liPresent++;
|
|
344
401
|
else missing.push(`li/${v.filename}`);
|
|
345
402
|
}
|
|
403
|
+
let liEdgePresent = 0;
|
|
404
|
+
for (const v of liEdgePaths) {
|
|
405
|
+
if (isValidMlpackage(v.fullPath)) liEdgePresent++;
|
|
406
|
+
else missing.push(`li-edge/${v.filename}`);
|
|
407
|
+
}
|
|
346
408
|
|
|
347
409
|
const embedTotal = embedPaths.length;
|
|
348
410
|
const liTotal = liPaths.length;
|
|
349
|
-
const
|
|
411
|
+
const liEdgeTotal = liEdgePaths.length;
|
|
412
|
+
// A cascade counts as "complete" when every advertised variant is on
|
|
413
|
+
// disk. Edge variants only contribute to completeness when the spec
|
|
414
|
+
// declares them — an `liEdgeTotal === 0` spec is treated as "edge
|
|
415
|
+
// not advertised" and complete reflects only embed + li.
|
|
416
|
+
const embedOk = embedPresent === embedTotal;
|
|
417
|
+
const liOk = liPresent === liTotal;
|
|
418
|
+
const liEdgeOk = liEdgeTotal === 0 || liEdgePresent === liEdgeTotal;
|
|
419
|
+
const complete = embedOk && liOk && liEdgeOk;
|
|
350
420
|
|
|
351
421
|
return {
|
|
352
422
|
applicable: hw.coremlCascadeEligible,
|
|
@@ -354,10 +424,13 @@ export function getCoremlCascadeState() {
|
|
|
354
424
|
root,
|
|
355
425
|
embedDir,
|
|
356
426
|
liDir,
|
|
427
|
+
liEdgeDir,
|
|
357
428
|
embedPresent,
|
|
358
429
|
embedTotal,
|
|
359
430
|
liPresent,
|
|
360
431
|
liTotal,
|
|
432
|
+
liEdgePresent,
|
|
433
|
+
liEdgeTotal,
|
|
361
434
|
complete,
|
|
362
435
|
missing,
|
|
363
436
|
};
|
|
@@ -373,9 +446,18 @@ export function getCoremlCascadeState() {
|
|
|
373
446
|
* (tests, benchmarks, diagnostic runs) should pass
|
|
374
447
|
* `SWEET_SEARCH_COREML_CASCADE=0` — see the env-var check below.
|
|
375
448
|
*
|
|
449
|
+
* `liVariantKey` selects which LI cascade family to point at:
|
|
450
|
+
* `lateon-code` (default) → `coreml-cascade/li/`,
|
|
451
|
+
* `lateon-code-edge` → `coreml-cascade/li-edge/`.
|
|
452
|
+
* Passed by `native-inference.js::resolveCoremlCascadeForAddon` after
|
|
453
|
+
* resolving the active variant from `LATE_INTERACTION_CONFIG`. Init,
|
|
454
|
+
* uninstall, and tests typically don't need to pass it — the default
|
|
455
|
+
* standard cascade is reported.
|
|
456
|
+
*
|
|
457
|
+
* @param {string} [liVariantKey='lateon-code'] - active LI variant
|
|
376
458
|
* @returns {{ embedDir: string | null, liDir: string | null, status: string }}
|
|
377
459
|
*/
|
|
378
|
-
export function getCoremlCascadeResolvedDirs() {
|
|
460
|
+
export function getCoremlCascadeResolvedDirs(liVariantKey = 'lateon-code') {
|
|
379
461
|
const rawFlag = (process.env.SWEET_SEARCH_COREML_CASCADE ?? '').trim().toLowerCase();
|
|
380
462
|
if (rawFlag === '0' || rawFlag === 'false' || rawFlag === 'off') {
|
|
381
463
|
return { embedDir: null, liDir: null, status: 'disabled-by-env' };
|
|
@@ -385,7 +467,13 @@ export function getCoremlCascadeResolvedDirs() {
|
|
|
385
467
|
if (!state.applicable) {
|
|
386
468
|
return { embedDir: null, liDir: null, status: 'hardware-ineligible' };
|
|
387
469
|
}
|
|
388
|
-
|
|
470
|
+
|
|
471
|
+
// Pick the LI cascade dir + present-count for the active variant.
|
|
472
|
+
const useEdge = liVariantKey === 'lateon-code-edge';
|
|
473
|
+
const liVariantPresent = useEdge ? state.liEdgePresent : state.liPresent;
|
|
474
|
+
const liVariantDir = useEdge ? state.liEdgeDir : state.liDir;
|
|
475
|
+
|
|
476
|
+
if (state.embedPresent === 0 && liVariantPresent === 0) {
|
|
389
477
|
return { embedDir: null, liDir: null, status: 'not-installed' };
|
|
390
478
|
}
|
|
391
479
|
|
|
@@ -395,10 +483,18 @@ export function getCoremlCascadeResolvedDirs() {
|
|
|
395
483
|
// dispatching through its own variants. Any call that picks a missing
|
|
396
484
|
// variant falls through to candle via `pick() → None`.
|
|
397
485
|
const embedDir = state.embedPresent > 0 ? state.embedDir : null;
|
|
398
|
-
const liDir =
|
|
486
|
+
const liDir = liVariantPresent > 0 ? liVariantDir : null;
|
|
487
|
+
|
|
488
|
+
// Per-variant completeness: status is `present` only when both embed
|
|
489
|
+
// and the active LI family are fully populated. Other LI families'
|
|
490
|
+
// state doesn't matter for *this* dispatch.
|
|
491
|
+
const liVariantComplete = useEdge
|
|
492
|
+
? state.liEdgeTotal > 0 && state.liEdgePresent === state.liEdgeTotal
|
|
493
|
+
: state.liPresent === state.liTotal;
|
|
494
|
+
const embedComplete = state.embedPresent === state.embedTotal;
|
|
399
495
|
|
|
400
496
|
let status;
|
|
401
|
-
if (
|
|
497
|
+
if (embedComplete && liVariantComplete) {
|
|
402
498
|
status = 'present';
|
|
403
499
|
} else if (embedDir && liDir) {
|
|
404
500
|
status = 'partial';
|
|
@@ -414,6 +510,12 @@ export function getCoremlCascadeResolvedDirs() {
|
|
|
414
510
|
/**
|
|
415
511
|
* Build a compact report used by `sweet-search init` to print a one-line
|
|
416
512
|
* status on completion and to record diagnostics in config.json.
|
|
513
|
+
*
|
|
514
|
+
* Counts are reported across all three families (embed, li, li-edge)
|
|
515
|
+
* so a single status line communicates whether the cascade is fully
|
|
516
|
+
* populated. Edge variants are folded into the totals only when the
|
|
517
|
+
* spec advertises them — older specs without `liEdge` remain
|
|
518
|
+
* backward-compatible.
|
|
417
519
|
*/
|
|
418
520
|
export function getCoremlCascadeReport() {
|
|
419
521
|
const state = getCoremlCascadeState();
|
|
@@ -435,17 +537,25 @@ export function getCoremlCascadeReport() {
|
|
|
435
537
|
reason: state.reason,
|
|
436
538
|
};
|
|
437
539
|
}
|
|
540
|
+
const totalAdvertised = state.embedTotal + state.liTotal + state.liEdgeTotal;
|
|
541
|
+
const totalPresent = state.embedPresent + state.liPresent + state.liEdgePresent;
|
|
542
|
+
const liEdgeReportFragment = state.liEdgeTotal > 0
|
|
543
|
+
? ` + ${state.liEdgePresent}/${state.liEdgeTotal} LI-edge`
|
|
544
|
+
: '';
|
|
438
545
|
if (state.complete) {
|
|
439
546
|
return {
|
|
440
547
|
status: 'present',
|
|
441
|
-
detail: `${
|
|
548
|
+
detail: `${totalPresent} variants ready (${state.embedTotal} embed + ${state.liTotal} LI`
|
|
549
|
+
+ (state.liEdgeTotal > 0 ? ` + ${state.liEdgeTotal} LI-edge` : '')
|
|
550
|
+
+ ')',
|
|
442
551
|
applicable: true,
|
|
443
552
|
reason: state.reason,
|
|
444
553
|
embedDir: state.embedDir,
|
|
445
554
|
liDir: state.liDir,
|
|
555
|
+
liEdgeDir: state.liEdgeTotal > 0 ? state.liEdgeDir : null,
|
|
446
556
|
};
|
|
447
557
|
}
|
|
448
|
-
if (
|
|
558
|
+
if (totalPresent === 0) {
|
|
449
559
|
return {
|
|
450
560
|
status: 'not-built',
|
|
451
561
|
detail: 'Run `node scripts/build-coreml-cascade.js` to build locally (~12 min, requires Python + coremltools)',
|
|
@@ -455,11 +565,14 @@ export function getCoremlCascadeReport() {
|
|
|
455
565
|
}
|
|
456
566
|
return {
|
|
457
567
|
status: 'partial',
|
|
458
|
-
detail: `${state.embedPresent}/${state.embedTotal} embed + ${state.liPresent}/${state.liTotal} LI
|
|
568
|
+
detail: `${state.embedPresent}/${state.embedTotal} embed + ${state.liPresent}/${state.liTotal} LI`
|
|
569
|
+
+ liEdgeReportFragment
|
|
570
|
+
+ ' present — rebuild with `node scripts/build-coreml-cascade.js`',
|
|
459
571
|
applicable: true,
|
|
460
572
|
reason: state.reason,
|
|
461
573
|
embedDir: state.embedPresent > 0 ? state.embedDir : null,
|
|
462
574
|
liDir: state.liPresent > 0 ? state.liDir : null,
|
|
575
|
+
liEdgeDir: state.liEdgePresent > 0 ? state.liEdgeDir : null,
|
|
463
576
|
missing: state.missing,
|
|
464
577
|
};
|
|
465
578
|
}
|
|
@@ -758,6 +871,38 @@ async function fetchAndExtractCascadeVariant({
|
|
|
758
871
|
}
|
|
759
872
|
}
|
|
760
873
|
|
|
874
|
+
/**
|
|
875
|
+
* Resolve which cascade families to fetch for a given LI variant. The
|
|
876
|
+
* embed cascade is shared, but only the LI family matching the active
|
|
877
|
+
* variant (or all families when `families: 'all'`) is fetched. Default
|
|
878
|
+
* behaviour: fetch embed + the LI family for the active LI variant.
|
|
879
|
+
*
|
|
880
|
+
* @param {string|string[]} families - 'auto' (default), 'all', or an
|
|
881
|
+
* explicit list of `'embed' | 'li' | 'li-edge'`
|
|
882
|
+
* @param {string} liVariantKey - Active LI variant key, used when
|
|
883
|
+
* `families==='auto'`. 'lateon-code-edge' selects the edge LI cascade,
|
|
884
|
+
* any other value selects the standard LI cascade.
|
|
885
|
+
* @returns {Set<string>} Set of family names to fetch
|
|
886
|
+
*/
|
|
887
|
+
export function resolveFamiliesToFetch(families, liVariantKey) {
|
|
888
|
+
const all = new Set(['embed', 'li', 'li-edge']);
|
|
889
|
+
if (families === 'all') return all;
|
|
890
|
+
if (Array.isArray(families)) {
|
|
891
|
+
const out = new Set();
|
|
892
|
+
for (const f of families) {
|
|
893
|
+
if (all.has(f)) out.add(f);
|
|
894
|
+
}
|
|
895
|
+
if (out.size > 0) return out;
|
|
896
|
+
}
|
|
897
|
+
// 'auto' (default): always embed, plus the LI family matching the
|
|
898
|
+
// active variant. Edge variant → edge LI cascade ONLY (no standard LI
|
|
899
|
+
// download). Anything else → standard LI cascade ONLY (no edge
|
|
900
|
+
// download).
|
|
901
|
+
return liVariantKey === 'lateon-code-edge'
|
|
902
|
+
? new Set(['embed', 'li-edge'])
|
|
903
|
+
: new Set(['embed', 'li']);
|
|
904
|
+
}
|
|
905
|
+
|
|
761
906
|
/**
|
|
762
907
|
* Fetch the CoreML cascade from the HF repo named in
|
|
763
908
|
* `coreml-cascade.json::hfRepo` and install the variants into the
|
|
@@ -768,29 +913,72 @@ async function fetchAndExtractCascadeVariant({
|
|
|
768
913
|
* Local builds via `scripts/build-coreml-cascade.js` remain supported
|
|
769
914
|
* as a developer path and for machines where HF fetch is unavailable.
|
|
770
915
|
*
|
|
916
|
+
* **Family gating (2026-05 release policy):** by default, only the LI
|
|
917
|
+
* family matching `LATE_INTERACTION_CONFIG.model` (or the explicit
|
|
918
|
+
* `liVariantKey` option) is fetched. Edge tarballs are NOT downloaded
|
|
919
|
+
* on a standard install, and standard tarballs are NOT downloaded on
|
|
920
|
+
* an edge install. Pass `families: 'all'` (or an explicit list) to
|
|
921
|
+
* fetch every family — used by the `--build-coreml-cascade` workflow
|
|
922
|
+
* and by re-publish tooling.
|
|
923
|
+
*
|
|
771
924
|
* Never throws. Any failure is captured in the returned `failures`
|
|
772
925
|
* array so the caller can log without aborting init.
|
|
773
926
|
*
|
|
774
927
|
* @param {object} [options]
|
|
775
|
-
* @param {boolean} [options.force]
|
|
928
|
+
* @param {boolean} [options.force] - Re-download even if the target already exists
|
|
929
|
+
* @param {string|string[]} [options.families] - 'auto' (default), 'all', or
|
|
930
|
+
* explicit array of `'embed'|'li'|'li-edge'`. See `resolveFamiliesToFetch`.
|
|
931
|
+
* @param {string} [options.liVariantKey] - Active LI variant key for
|
|
932
|
+
* `families==='auto'` resolution. Defaults to `LATE_INTERACTION_CONFIG.model`.
|
|
776
933
|
* @param {(variant: string, downloaded: number, total: number) => void} [options.onProgress]
|
|
777
934
|
* @returns {Promise<{
|
|
778
935
|
* status: string,
|
|
779
936
|
* fetched: number,
|
|
780
937
|
* cached: number,
|
|
781
938
|
* skipped: number,
|
|
939
|
+
* families: string[],
|
|
782
940
|
* failures: Array<{ variant: string, error: string }>,
|
|
783
941
|
* reason?: string,
|
|
784
942
|
* }>}
|
|
785
943
|
*/
|
|
786
944
|
export async function fetchCoremlCascade(options = {}) {
|
|
787
945
|
const hw = detectHardwareCapability();
|
|
946
|
+
const spec = (() => {
|
|
947
|
+
try { return getCascadeSpec(); } catch { return null; }
|
|
948
|
+
})();
|
|
949
|
+
|
|
950
|
+
// Resolve which families to fetch BEFORE counting "skipped" — only
|
|
951
|
+
// the selected families count toward the per-call total. Without
|
|
952
|
+
// this, an edge-only init would always report 6 standard variants
|
|
953
|
+
// as "skipped" even though the user never asked for them.
|
|
954
|
+
// Lazy import of LATE_INTERACTION_CONFIG to avoid a circular import
|
|
955
|
+
// at module load (config/index.js -> infrastructure/index.js ->
|
|
956
|
+
// coreml-cascade.js).
|
|
957
|
+
let activeLiVariant = options.liVariantKey;
|
|
958
|
+
if (activeLiVariant === undefined) {
|
|
959
|
+
try {
|
|
960
|
+
const { LATE_INTERACTION_CONFIG } = await import('./config/ranking.js');
|
|
961
|
+
activeLiVariant = LATE_INTERACTION_CONFIG.model;
|
|
962
|
+
} catch {
|
|
963
|
+
activeLiVariant = 'lateon-code';
|
|
964
|
+
}
|
|
965
|
+
}
|
|
966
|
+
const familiesToFetch = resolveFamiliesToFetch(options.families ?? 'auto', activeLiVariant);
|
|
967
|
+
const familiesArr = Array.from(familiesToFetch);
|
|
968
|
+
|
|
969
|
+
const advertisedTotal = spec
|
|
970
|
+
? (familiesToFetch.has('embed') ? (spec.embed?.variants?.length ?? 0) : 0)
|
|
971
|
+
+ (familiesToFetch.has('li') ? (spec.li?.variants?.length ?? 0) : 0)
|
|
972
|
+
+ (familiesToFetch.has('li-edge') ? (spec.liEdge?.variants?.length ?? 0) : 0)
|
|
973
|
+
: 0;
|
|
974
|
+
|
|
788
975
|
if (!hw.coremlCascadeEligible) {
|
|
789
976
|
return {
|
|
790
977
|
status: 'skipped',
|
|
791
978
|
fetched: 0,
|
|
792
979
|
cached: 0,
|
|
793
|
-
skipped:
|
|
980
|
+
skipped: advertisedTotal,
|
|
981
|
+
families: familiesArr,
|
|
794
982
|
failures: [],
|
|
795
983
|
reason: hw.coremlCascadeReason,
|
|
796
984
|
};
|
|
@@ -805,29 +993,47 @@ export async function fetchCoremlCascade(options = {}) {
|
|
|
805
993
|
status: 'skipped',
|
|
806
994
|
fetched: 0,
|
|
807
995
|
cached: 0,
|
|
808
|
-
skipped:
|
|
996
|
+
skipped: advertisedTotal,
|
|
997
|
+
families: familiesArr,
|
|
809
998
|
failures: [],
|
|
810
999
|
reason: 'Disabled via SWEET_SEARCH_COREML_CASCADE=0',
|
|
811
1000
|
};
|
|
812
1001
|
}
|
|
813
1002
|
|
|
814
|
-
|
|
1003
|
+
if (!spec) {
|
|
1004
|
+
return {
|
|
1005
|
+
status: 'not-configured',
|
|
1006
|
+
fetched: 0,
|
|
1007
|
+
cached: 0,
|
|
1008
|
+
skipped: 0,
|
|
1009
|
+
families: familiesArr,
|
|
1010
|
+
failures: [],
|
|
1011
|
+
reason: 'coreml-cascade.json failed to load',
|
|
1012
|
+
};
|
|
1013
|
+
}
|
|
815
1014
|
if (!spec.hfRepo) {
|
|
816
1015
|
return {
|
|
817
1016
|
status: 'not-configured',
|
|
818
1017
|
fetched: 0,
|
|
819
1018
|
cached: 0,
|
|
820
|
-
skipped:
|
|
1019
|
+
skipped: advertisedTotal,
|
|
1020
|
+
families: familiesArr,
|
|
821
1021
|
failures: [],
|
|
822
1022
|
reason: 'coreml-cascade.json has no hfRepo field',
|
|
823
1023
|
};
|
|
824
1024
|
}
|
|
825
1025
|
|
|
826
|
-
const { embedPaths, liPaths } = getExpectedVariantPaths();
|
|
827
|
-
const all = [
|
|
828
|
-
|
|
829
|
-
...
|
|
830
|
-
|
|
1026
|
+
const { embedPaths, liPaths, liEdgePaths } = getExpectedVariantPaths();
|
|
1027
|
+
const all = [];
|
|
1028
|
+
if (familiesToFetch.has('embed')) {
|
|
1029
|
+
all.push(...embedPaths.map(p => ({ ...p, category: 'embed', categorySpec: spec.embed })));
|
|
1030
|
+
}
|
|
1031
|
+
if (familiesToFetch.has('li')) {
|
|
1032
|
+
all.push(...liPaths.map(p => ({ ...p, category: 'li', categorySpec: spec.li })));
|
|
1033
|
+
}
|
|
1034
|
+
if (familiesToFetch.has('li-edge')) {
|
|
1035
|
+
all.push(...liEdgePaths.map(p => ({ ...p, category: 'li-edge', categorySpec: spec.liEdge })));
|
|
1036
|
+
}
|
|
831
1037
|
|
|
832
1038
|
let fetched = 0;
|
|
833
1039
|
let cached = 0;
|
|
@@ -896,7 +1102,7 @@ export async function fetchCoremlCascade(options = {}) {
|
|
|
896
1102
|
status = 'fetched';
|
|
897
1103
|
}
|
|
898
1104
|
|
|
899
|
-
return { status, fetched, cached, skipped, failures };
|
|
1105
|
+
return { status, fetched, cached, skipped, families: familiesArr, failures };
|
|
900
1106
|
}
|
|
901
1107
|
|
|
902
1108
|
/**
|
|
@@ -26,6 +26,7 @@
|
|
|
26
26
|
"modelKey": "modernbert-li",
|
|
27
27
|
"filePattern": "li_modernbert_b{batch}_s{seq}_fp16.mlpackage",
|
|
28
28
|
"tarballPattern": "li/li_modernbert_b{batch}_s{seq}_fp16.mlpackage.tar.gz",
|
|
29
|
+
"subdir": "li",
|
|
29
30
|
"tokenDim": 128,
|
|
30
31
|
"backboneDim": 768,
|
|
31
32
|
"traceSource": {
|
|
@@ -42,5 +43,29 @@
|
|
|
42
43
|
{ "batch": 4, "seq": 1024, "rationale": "long tail", "tarballSha256": "e618f8bd981d24e4984c61ee1788c02578aaf601efc16cd926885a2930833d40", "tarballSizeBytes": 275545042 },
|
|
43
44
|
{ "batch": 1, "seq": 2048, "rationale": "LI max length", "tarballSha256": "d5262bf49f145cadbdbddb82b7934499fc310f5c654c5aae34cfb9c2d90487d2", "tarballSizeBytes": 275743803 }
|
|
44
45
|
]
|
|
46
|
+
},
|
|
47
|
+
"liEdge": {
|
|
48
|
+
"$comment": "LateOn-Code-edge variant cascade. Mirrors `li` shapes but uses the smaller edge backbone (256d × 7 layers) with a 2-stage projection (256→512→48). Lives in a separate subdir so it never clashes with the standard 128d cascade. Tarball SHA256s and sizes are populated by `node scripts/build-coreml-cascade.js --li-edge-only` followed by the publish step (see INIT_STRATEGY.md republishing workflow).",
|
|
49
|
+
"modelKey": "modernbert-li-edge",
|
|
50
|
+
"filePattern": "li_modernbert_edge_b{batch}_s{seq}_fp16.mlpackage",
|
|
51
|
+
"tarballPattern": "li-edge/li_modernbert_edge_b{batch}_s{seq}_fp16.mlpackage.tar.gz",
|
|
52
|
+
"subdir": "li-edge",
|
|
53
|
+
"tokenDim": 48,
|
|
54
|
+
"backboneDim": 256,
|
|
55
|
+
"traceSource": {
|
|
56
|
+
"hfId": "lightonai/LateOn-Code-edge",
|
|
57
|
+
"backbone": "model.safetensors",
|
|
58
|
+
"projections": ["1_Dense/model.safetensors", "2_Dense/model.safetensors"],
|
|
59
|
+
"projectionDims": [512, 48],
|
|
60
|
+
"config": "config.json"
|
|
61
|
+
},
|
|
62
|
+
"variants": [
|
|
63
|
+
{ "batch": 128, "seq": 48, "rationale": "upperCap × very short — covers observed (128, 18..33)", "tarballSha256": "ca7e110e00576e76ed33be951cf3f32589adf5442646174a2ae656cfd082491c", "tarballSizeBytes": 31334197 },
|
|
64
|
+
{ "batch": 128, "seq": 128, "rationale": "upperCap × short", "tarballSha256": "16f189107a2733338a841d5bc73b972dcff0266c4c23e1f70d88c3a51d5f01a7", "tarballSizeBytes": 31341775 },
|
|
65
|
+
{ "batch": 64, "seq": 256, "rationale": "medium", "tarballSha256": "b41e5a7a75508a937bb8a0fca1c5a96cf5f9242a66eb2d087a6c40e6e55ff47d", "tarballSizeBytes": 31354136 },
|
|
66
|
+
{ "batch": 16, "seq": 512, "rationale": "long, cache-bound start", "tarballSha256": "4adb69f8dd64df1d369b129a3e418a827b0bee74439dca39d7f94a637571c24d", "tarballSizeBytes": 31377124 },
|
|
67
|
+
{ "batch": 4, "seq": 1024, "rationale": "long tail", "tarballSha256": "e66fa4ba8e29fd51d5d0d2e17d403d7b45ad7b81f5b636232192a4c0977d1bd7", "tarballSizeBytes": 31424956 },
|
|
68
|
+
{ "batch": 1, "seq": 2048, "rationale": "LI max length", "tarballSha256": "dce8507d32f14ea0b64daff963c7988b30afb1c19714de36e590f843ccce75be", "tarballSizeBytes": 31526197 }
|
|
69
|
+
]
|
|
45
70
|
}
|
|
46
71
|
}
|
|
@@ -57,6 +57,9 @@ export {
|
|
|
57
57
|
getCoremlCascadeRoot,
|
|
58
58
|
getCoremlEmbedDir,
|
|
59
59
|
getCoremlLiDir,
|
|
60
|
+
getCoremlLiEdgeDir,
|
|
61
|
+
getCoremlLiDirForVariant,
|
|
62
|
+
liVariantToSectionKey,
|
|
60
63
|
formatVariantFilename,
|
|
61
64
|
formatVariantTarballPath,
|
|
62
65
|
isCoremlCascadeApplicable,
|
|
@@ -67,8 +70,22 @@ export {
|
|
|
67
70
|
getCoremlCascadeReport,
|
|
68
71
|
getAllCoremlCachePaths,
|
|
69
72
|
fetchCoremlCascade,
|
|
73
|
+
resolveFamiliesToFetch,
|
|
70
74
|
} from './coreml-cascade.js';
|
|
71
75
|
|
|
76
|
+
// Persisted init config (.sweet-search/config.json) — written by scripts/init.js,
|
|
77
|
+
// read by SweetSearch runtime to honour the user's persisted LI policy choices.
|
|
78
|
+
export {
|
|
79
|
+
INIT_DATA_DIR_NAME,
|
|
80
|
+
INIT_CONFIG_FILE_NAME,
|
|
81
|
+
getInitConfigPath,
|
|
82
|
+
loadInitConfig,
|
|
83
|
+
writeInitConfig,
|
|
84
|
+
readPersistedLiPolicy,
|
|
85
|
+
resolveRuntimeLiModel,
|
|
86
|
+
applyPersistedLiModel,
|
|
87
|
+
} from './init-config.js';
|
|
88
|
+
|
|
72
89
|
// Language analysis
|
|
73
90
|
export {
|
|
74
91
|
getLanguageByPath, getLanguageByExtension, LANGUAGES, EXTENSION_MAP, FILENAME_MAP,
|