sweet-search 2.4.2 → 2.5.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/core/cli.js +43 -5
- package/core/embedding/embedding-cache.js +266 -18
- package/core/embedding/embedding-service.js +45 -9
- package/core/graph/graph-expansion.js +52 -12
- package/core/graph/graph-extractor.js +30 -1
- package/core/indexing/ast-chunker.js +331 -16
- package/core/indexing/chunking/chunk-builder.js +34 -1
- package/core/indexing/index-codebase-v21.js +31 -2
- package/core/indexing/index.js +6 -3
- package/core/indexing/indexer-ann.js +45 -6
- package/core/indexing/indexer-build.js +9 -1
- package/core/indexing/indexer-phases.js +6 -4
- package/core/indexing/indexing-file-policy.js +140 -0
- package/core/indexing/li-skip-policy.js +11 -220
- package/core/infrastructure/codebase-repository.js +21 -0
- package/core/infrastructure/config/embedding.js +20 -1
- package/core/infrastructure/config/graph.js +2 -2
- package/core/infrastructure/config/ranking.js +10 -0
- package/core/infrastructure/config/vector-store.js +1 -1
- package/core/infrastructure/coreml-cascade.js +236 -30
- package/core/infrastructure/coreml-cascade.json +25 -0
- package/core/infrastructure/index.js +17 -0
- package/core/infrastructure/init-config.js +216 -0
- package/core/infrastructure/language-patterns/registry-core.js +18 -0
- package/core/infrastructure/model-registry.js +12 -0
- package/core/infrastructure/native-inference.js +143 -51
- package/core/infrastructure/tree-sitter-provider.js +92 -2
- package/core/ranking/cascaded-scorer.js +6 -2
- package/core/ranking/file-kind-ranking.js +264 -0
- package/core/ranking/late-interaction-index.js +10 -4
- package/core/ranking/late-interaction-policy.js +304 -0
- package/core/search/context-expander.js +267 -28
- package/core/search/index.js +4 -0
- package/core/search/search-cli.js +3 -1
- package/core/search/search-pattern.js +4 -3
- package/core/search/search-postprocess.js +189 -8
- package/core/search/search-read-semantic.js +734 -0
- package/core/search/search-read.js +481 -0
- package/core/search/search-server.js +153 -5
- package/core/search/sweet-search.js +133 -16
- package/core/start-server.js +13 -2
- package/mcp/server.js +41 -0
- package/mcp/tool-handlers.js +117 -6
- package/package.json +9 -7
- package/scripts/init.js +386 -5
- package/scripts/uninstall.js +152 -6
|
@@ -21,6 +21,8 @@ import { HNSWIndex } from '../vector-store/hnsw-index.js';
|
|
|
21
21
|
import { BinaryHNSWIndex } from '../vector-store/binary-hnsw-index.js';
|
|
22
22
|
import { Reranker } from '../ranking/flashrank.js';
|
|
23
23
|
import { LateInteractionIndex } from '../ranking/late-interaction-index.js';
|
|
24
|
+
import { resolveSearchRerankPolicy } from '../ranking/late-interaction-policy.js';
|
|
25
|
+
import { applyPersistedLiModel, readPersistedLiPolicy } from '../infrastructure/index.js';
|
|
24
26
|
import { getEmbedding, getBinaryEmbedding, truncateForHNSW, int8CosineSimilarity, warmup as warmupEmbedding, isWarm, registerAutoPersistOnExit } from '../embedding/embedding-service.js';
|
|
25
27
|
import { FloatVectorStore, getFloatStorePath } from '../vector-store/float-vector-store.js';
|
|
26
28
|
import { recordQueryTelemetry } from '../embedding/embedding-cache.js';
|
|
@@ -41,6 +43,7 @@ import * as semantic from './search-semantic.js';
|
|
|
41
43
|
import * as hybrid from './search-hybrid.js';
|
|
42
44
|
import * as postprocess from './search-postprocess.js';
|
|
43
45
|
import * as pattern from './search-pattern.js';
|
|
46
|
+
import { packageForAgent } from './context-expander.js';
|
|
44
47
|
|
|
45
48
|
export { ROUTE_ALPHAS } from './search-fusion.js';
|
|
46
49
|
|
|
@@ -88,15 +91,26 @@ export class SweetSearch {
|
|
|
88
91
|
constructor(options = {}) {
|
|
89
92
|
const projectRoot = options.projectRoot || process.env.SWEET_SEARCH_PROJECT_ROOT || process.cwd();
|
|
90
93
|
this.projectRoot = projectRoot;
|
|
94
|
+
// Honor the user's persisted `runtime.li.model` choice from
|
|
95
|
+
// `.sweet-search/config.json` BEFORE we read `LATE_INTERACTION_CONFIG.model`
|
|
96
|
+
// for activeConfigModel below or any downstream consumer (encodeQuery,
|
|
97
|
+
// LateInteractionIndex header check, native LI loader, CoreML cascade
|
|
98
|
+
// dispatcher). Without this an edge-only init silently activates the
|
|
99
|
+
// standard model path on every search. Env var still wins; see
|
|
100
|
+
// applyPersistedLiModel for the full precedence ladder.
|
|
101
|
+
this._liModelApply = applyPersistedLiModel(projectRoot);
|
|
91
102
|
const projectConfig = loadProjectConfig(projectRoot);
|
|
92
103
|
const projectCascade = projectConfig.cascade || {};
|
|
93
104
|
const envOrProject = (envKey, cascadeKey, configKey) =>
|
|
94
105
|
process.env[envKey] != null ? CASCADE_CONFIG[configKey] : projectCascade[cascadeKey];
|
|
95
106
|
|
|
96
|
-
this.
|
|
97
|
-
this.
|
|
98
|
-
this.
|
|
99
|
-
this.
|
|
107
|
+
this.graphDbPath = options.graphDbPath || DB_PATHS.codeGraph;
|
|
108
|
+
this.graphSearch = new GraphSearch(this.graphDbPath);
|
|
109
|
+
this.codeGraphRepo = new CodeGraphRepository(this.graphDbPath);
|
|
110
|
+
this.hnswPath = options.hnswPath || DB_PATHS.hnswIndex;
|
|
111
|
+
this.binaryHnswPath = options.binaryHnswPath || DB_PATHS.binaryHnswIndex;
|
|
112
|
+
this.hnswIndex = new HNSWIndex({ indexPath: this.hnswPath });
|
|
113
|
+
this.binaryHnswIndex = new BinaryHNSWIndex({ indexPath: this.binaryHnswPath });
|
|
100
114
|
this.reranker = new Reranker(options);
|
|
101
115
|
this.lateInteractionIndex = new LateInteractionIndex(options.lateInteractionOptions || {});
|
|
102
116
|
this.router = new QueryRouter();
|
|
@@ -108,7 +122,25 @@ export class SweetSearch {
|
|
|
108
122
|
this.stage1Candidates = options.stage1Candidates ?? BINARY_HNSW_CONFIG.retrieval.stage1Candidates;
|
|
109
123
|
this.stage2Candidates = options.stage2Candidates ?? BINARY_HNSW_CONFIG.retrieval.stage2Candidates;
|
|
110
124
|
this.stage3Candidates = options.stage3Candidates ?? BINARY_HNSW_CONFIG.retrieval.stage3Candidates;
|
|
111
|
-
|
|
125
|
+
// Late-interaction search-rerank gating — see core/ranking/late-interaction-policy.js
|
|
126
|
+
// for the full precedence ladder. The constructor records the inputs and
|
|
127
|
+
// computes a tentative value (so callers reading `useLateInteraction`
|
|
128
|
+
// before init() get sensible defaults); init() recomputes once the LI
|
|
129
|
+
// index header has been loaded so the manifest's modelId can drive the
|
|
130
|
+
// auto policy.
|
|
131
|
+
this._liPolicyOptionOverride = typeof options.useLateInteraction === 'boolean'
|
|
132
|
+
? options.useLateInteraction
|
|
133
|
+
: undefined;
|
|
134
|
+
this._liPolicyPersisted = readPersistedLiPolicy(projectRoot);
|
|
135
|
+
const liInitial = resolveSearchRerankPolicy({
|
|
136
|
+
optionOverride: this._liPolicyOptionOverride,
|
|
137
|
+
env: process.env,
|
|
138
|
+
persisted: this._liPolicyPersisted,
|
|
139
|
+
indexManifest: null,
|
|
140
|
+
activeConfigModel: LATE_INTERACTION_CONFIG.model,
|
|
141
|
+
});
|
|
142
|
+
this.useLateInteraction = LATE_INTERACTION_CONFIG.enabled ? liInitial.effective : false;
|
|
143
|
+
this._liPolicyResolved = liInitial;
|
|
112
144
|
this.lateInteractionBlendWeight = options.lateInteractionBlendWeight ?? LATE_INTERACTION_CONFIG.blendWeight ?? 0.3;
|
|
113
145
|
this.returnSummaryFirst = options.returnSummaryFirst ?? HCGS_CONFIG.returnSummaryFirst;
|
|
114
146
|
this.summaryTokenBudget = options.summaryTokenBudget ?? HCGS_CONFIG.summaryTokenBudget;
|
|
@@ -152,9 +184,9 @@ export class SweetSearch {
|
|
|
152
184
|
if (this.initialized) return;
|
|
153
185
|
const start = Date.now();
|
|
154
186
|
|
|
155
|
-
this.hasGraphIndex = existsSync(
|
|
156
|
-
this.hasHnswIndex = existsSync(
|
|
157
|
-
this.hasBinaryHnswIndex = existsSync(
|
|
187
|
+
this.hasGraphIndex = existsSync(this.graphDbPath);
|
|
188
|
+
this.hasHnswIndex = existsSync(this.hnswPath.replace('.idx', '.meta.json'));
|
|
189
|
+
this.hasBinaryHnswIndex = existsSync(this.binaryHnswPath.replace('.idx', '.meta.json'));
|
|
158
190
|
this.hasCodebaseIndex = existsSync(this.codebaseDbPath);
|
|
159
191
|
this.hasLateInteractionIndex = existsSync(this.lateInteractionIndex.indexPath);
|
|
160
192
|
this.hasSparseGramIndex = existsSync(this.sparseGramIndexPath);
|
|
@@ -182,7 +214,7 @@ export class SweetSearch {
|
|
|
182
214
|
// disable the entire 3-stage pipeline. Stage 2.5 falls back to SQLite.
|
|
183
215
|
if (this.hasBinaryHnswIndex) {
|
|
184
216
|
try {
|
|
185
|
-
const floatStorePath = getFloatStorePath(
|
|
217
|
+
const floatStorePath = getFloatStorePath(this.binaryHnswPath);
|
|
186
218
|
const floatLoaded = await this.floatVectorStore.load(floatStorePath);
|
|
187
219
|
if (floatLoaded) {
|
|
188
220
|
const fStats = this.floatVectorStore.getStats();
|
|
@@ -212,15 +244,65 @@ export class SweetSearch {
|
|
|
212
244
|
const stats = this.lateInteractionIndex.getStats();
|
|
213
245
|
this.log(`LateInteraction: Loaded ${stats.documents} documents (${stats.estimatedSizeMB} MB, ${stats.avgTokensPerDoc} avg tokens)`);
|
|
214
246
|
|
|
215
|
-
//
|
|
216
|
-
//
|
|
217
|
-
|
|
218
|
-
|
|
219
|
-
this
|
|
247
|
+
// Re-resolve the rerank policy now that the index header has been
|
|
248
|
+
// loaded — the manifest's modelId is the source of truth for the
|
|
249
|
+
// auto rule (edge index → off, standard index → on, mismatch → off).
|
|
250
|
+
// The constructor only had the active LATE_INTERACTION_CONFIG.model
|
|
251
|
+
// to work with; this call corrects the decision when index and
|
|
252
|
+
// config disagree (env var changed, model bumped, etc.).
|
|
253
|
+
const manifest = {
|
|
254
|
+
modelId: this.lateInteractionIndex.modelId ?? null,
|
|
255
|
+
tokenDim: this.lateInteractionIndex.tokenDim ?? null,
|
|
256
|
+
modelMismatch: this.lateInteractionIndex.modelMismatch === true,
|
|
257
|
+
exists: true,
|
|
258
|
+
};
|
|
259
|
+
const resolved = resolveSearchRerankPolicy({
|
|
260
|
+
optionOverride: this._liPolicyOptionOverride,
|
|
261
|
+
env: process.env,
|
|
262
|
+
persisted: this._liPolicyPersisted,
|
|
263
|
+
indexManifest: manifest,
|
|
264
|
+
activeConfigModel: LATE_INTERACTION_CONFIG.model,
|
|
265
|
+
});
|
|
266
|
+
this._liPolicyResolved = resolved;
|
|
267
|
+
const previouslyOn = this.useLateInteraction;
|
|
268
|
+
this.useLateInteraction = LATE_INTERACTION_CONFIG.enabled && resolved.effective;
|
|
269
|
+
this.log(
|
|
270
|
+
`LateInteraction: rerank policy → ${this.useLateInteraction ? 'on' : 'off'} `
|
|
271
|
+
+ `(${resolved.reason}${manifest.modelId ? `, index=${manifest.modelId}` : ''})`,
|
|
272
|
+
);
|
|
273
|
+
if (resolved.warning) {
|
|
274
|
+
// One-line warning — keeps the log digestible, full guidance is
|
|
275
|
+
// in docs/BENCH_TODO.md.
|
|
276
|
+
console.warn(`[SweetSearch] ${resolved.warning}`);
|
|
277
|
+
}
|
|
278
|
+
|
|
279
|
+
if (this.useLateInteraction) {
|
|
280
|
+
// Preheat LI ONNX inference model (~900ms cold start otherwise).
|
|
281
|
+
// Only when we will actually rerank — saves cold-start cost when
|
|
282
|
+
// policy resolves to off post-manifest-inspection.
|
|
283
|
+
const { encodeQuery } = await import('../ranking/late-interaction-model.js');
|
|
284
|
+
await encodeQuery('warmup');
|
|
285
|
+
this.log('LateInteraction: ONNX model preheated');
|
|
286
|
+
} else if (previouslyOn) {
|
|
287
|
+
// We loaded the index because the constructor's tentative
|
|
288
|
+
// resolution said on, but the manifest just told us otherwise —
|
|
289
|
+
// log so the user understands the gap between "index present"
|
|
290
|
+
// and "rerank active".
|
|
291
|
+
this.log('LateInteraction: index loaded but search rerank disabled by policy (read-semantic + ColGrep still use the index)');
|
|
292
|
+
}
|
|
220
293
|
} catch (err) {
|
|
221
294
|
this.log(`LateInteraction: Failed to load: ${err.message}`);
|
|
222
295
|
this.hasLateInteractionIndex = false;
|
|
296
|
+
this.useLateInteraction = false;
|
|
223
297
|
}
|
|
298
|
+
} else if (this.hasLateInteractionIndex && !this.useLateInteraction) {
|
|
299
|
+
// Index present but constructor-time policy resolved to off.
|
|
300
|
+
// Skip the (expensive) load + encoder warmup — read-semantic and
|
|
301
|
+
// ColGrep both lazy-load their own LI handle when actually invoked.
|
|
302
|
+
this.log(
|
|
303
|
+
`LateInteraction: index present, search rerank disabled by policy `
|
|
304
|
+
+ `(${this._liPolicyResolved?.reason ?? 'unknown'})`,
|
|
305
|
+
);
|
|
224
306
|
}
|
|
225
307
|
|
|
226
308
|
if (this.hasSparseGramIndex) {
|
|
@@ -331,7 +413,12 @@ export class SweetSearch {
|
|
|
331
413
|
let searchMode;
|
|
332
414
|
if (mode === 'auto') {
|
|
333
415
|
searchMode = routing.mode;
|
|
334
|
-
stats.routing = {
|
|
416
|
+
stats.routing = {
|
|
417
|
+
mode: routing.mode,
|
|
418
|
+
confidence: routing.confidence,
|
|
419
|
+
latency_us: routing.routingLatency_us,
|
|
420
|
+
method: routing.method,
|
|
421
|
+
};
|
|
335
422
|
} else {
|
|
336
423
|
searchMode = mode;
|
|
337
424
|
stats.routing = {
|
|
@@ -419,9 +506,39 @@ export class SweetSearch {
|
|
|
419
506
|
}
|
|
420
507
|
|
|
421
508
|
// Step 3: Post-retrieval processing (delegated to extracted module)
|
|
422
|
-
|
|
509
|
+
const postRetrievalResult = await this._applyPostRetrieval(results, query, options, {
|
|
423
510
|
stats, semanticStats, searchMode, effectiveGraphExpand, intentPolicy, start,
|
|
424
511
|
});
|
|
512
|
+
|
|
513
|
+
// Step 4: Agent packaging (lexical/semantic/hybrid/structural).
|
|
514
|
+
// The pattern (colgrep) branch already returns its own pre-packaged response
|
|
515
|
+
// and short-circuits earlier in this method. For non-pattern modes, apply the
|
|
516
|
+
// shared packager when the caller explicitly asked for an agent format.
|
|
517
|
+
// Default behavior (no agent format) is unchanged.
|
|
518
|
+
const agentFormats = new Set(['agent', 'agent_preview', 'agent_full', 'agent_full_xl']);
|
|
519
|
+
if (agentFormats.has(options.format)) {
|
|
520
|
+
const finalResults = postRetrievalResult.results || [];
|
|
521
|
+
const finalStats = postRetrievalResult.stats || {};
|
|
522
|
+
const agentResponse = packageForAgent(finalResults, {
|
|
523
|
+
...finalStats,
|
|
524
|
+
candidatePoolSize: finalStats.results_count ?? finalResults.length,
|
|
525
|
+
}, {
|
|
526
|
+
query,
|
|
527
|
+
regex: regex || '',
|
|
528
|
+
mode: finalStats.path || searchMode,
|
|
529
|
+
format: options.format,
|
|
530
|
+
tokenBudget: options.tokenBudget,
|
|
531
|
+
codeGraphRepo: this.codeGraphRepo || null,
|
|
532
|
+
locationMap: null,
|
|
533
|
+
projectRoot: this.projectRoot,
|
|
534
|
+
ablations: options.ablations,
|
|
535
|
+
});
|
|
536
|
+
// Preserve the underlying retrieval stats so callers can inspect both layers
|
|
537
|
+
agentResponse.stats = finalStats;
|
|
538
|
+
return agentResponse;
|
|
539
|
+
}
|
|
540
|
+
|
|
541
|
+
return postRetrievalResult;
|
|
425
542
|
}
|
|
426
543
|
|
|
427
544
|
/** Structural search path (GraphRAG structural queries — opt-in via explicit flag) */
|
package/core/start-server.js
CHANGED
|
@@ -1,6 +1,17 @@
|
|
|
1
1
|
#!/usr/bin/env node
|
|
2
2
|
// Minimal server-start entry point — avoids the circular import in sweet-search.js.
|
|
3
|
-
// Used by the Rust CLI's auto_start_server() to spawn the background server
|
|
3
|
+
// Used by the Rust CLI's auto_start_server() to spawn the background server,
|
|
4
|
+
// and by the SessionStart daemon-prewarm hook (core/search/session-daemon-prewarm.mjs)
|
|
5
|
+
// when Claude Code opens a new session.
|
|
4
6
|
|
|
5
|
-
|
|
7
|
+
// Apply the user's persisted `runtime.li.model` from .sweet-search/config.json
|
|
8
|
+
// BEFORE importing search-server (which transitively imports session-warmup,
|
|
9
|
+
// which gates warmup steps on `LATE_INTERACTION_CONFIG.enabled` and triggers a
|
|
10
|
+
// warmup search using `LATE_INTERACTION_CONFIG.model`). Without this, an
|
|
11
|
+
// edge-only init still spawns a daemon that prewarms the standard model.
|
|
12
|
+
const projectRoot = process.env.SWEET_SEARCH_PROJECT_ROOT || process.cwd();
|
|
13
|
+
const { applyPersistedLiModel } = await import('./infrastructure/init-config.js');
|
|
14
|
+
applyPersistedLiModel(projectRoot);
|
|
15
|
+
|
|
16
|
+
const { startServer } = await import('./search/search-server.js');
|
|
6
17
|
await startServer();
|
package/mcp/server.js
CHANGED
|
@@ -18,11 +18,15 @@ import {
|
|
|
18
18
|
HealthOutputSchema,
|
|
19
19
|
RepoMapOutputSchema,
|
|
20
20
|
VocabPrewarmOutputSchema,
|
|
21
|
+
ReadOutputSchema,
|
|
22
|
+
ReadSemanticOutputSchema,
|
|
21
23
|
handleSearch,
|
|
22
24
|
handleIndex,
|
|
23
25
|
checkHealth,
|
|
24
26
|
handleRepoMap,
|
|
25
27
|
handleVocabPrewarm,
|
|
28
|
+
handleRead,
|
|
29
|
+
handleReadSemantic,
|
|
26
30
|
} from './tool-handlers.js';
|
|
27
31
|
|
|
28
32
|
const __filename = fileURLToPath(import.meta.url);
|
|
@@ -224,6 +228,43 @@ server.registerTool('vocab-prewarm', {
|
|
|
224
228
|
},
|
|
225
229
|
}, async (args) => handleVocabPrewarm(args, vocabDeps));
|
|
226
230
|
|
|
231
|
+
server.registerTool('read', {
|
|
232
|
+
description: 'Read one or more files for exact code understanding. Replaces the default Read tool for most code-reading workflows. Uses the filesystem as ground truth, supports line ranges and batching, and attaches symbol-aware chunk metadata when the file is indexed.',
|
|
233
|
+
inputSchema: {
|
|
234
|
+
files: z.array(z.object({
|
|
235
|
+
path: z.string().describe('File path relative to project root (or absolute)'),
|
|
236
|
+
startLine: z.number().int().min(1).optional().describe('Start line (1-based, inclusive)'),
|
|
237
|
+
endLine: z.number().int().min(1).optional().describe('End line (1-based, inclusive)'),
|
|
238
|
+
})).min(1).max(20).describe('Files to read (1-20)'),
|
|
239
|
+
includeMetadata: z.boolean().default(true).optional()
|
|
240
|
+
.describe('Attach symbol-aware chunk metadata when the file is indexed'),
|
|
241
|
+
},
|
|
242
|
+
outputSchema: ReadOutputSchema,
|
|
243
|
+
annotations: { readOnlyHint: true, destructiveHint: false, idempotentHint: true, openWorldHint: false },
|
|
244
|
+
}, async (args) => handleRead(args, { PROJECT_ROOT }));
|
|
245
|
+
|
|
246
|
+
server.registerTool('read-semantic', {
|
|
247
|
+
description: 'Read only the spans of a file relevant to a query. Selects spans via hybrid retrieval (lexical + symbol + ColBERT-style late-interaction MaxSim) with RRF fusion and LI re-rank, then re-reads exact lines from disk. Returns 1-N small spans instead of the full file. Falls back to a plain read if the file is not indexed.',
|
|
248
|
+
inputSchema: {
|
|
249
|
+
file: z.string().describe('File path (project-relative or absolute)'),
|
|
250
|
+
query: z.string().min(1).max(500).describe('What you want to understand about this file'),
|
|
251
|
+
topK: z.number().int().min(1).max(20).default(5).optional()
|
|
252
|
+
.describe('Maximum spans before merging (default: 5)'),
|
|
253
|
+
threshold: z.number().min(0).max(1).default(0.4).optional()
|
|
254
|
+
.describe('MaxSim score floor (default: 0.4)'),
|
|
255
|
+
contextLines: z.number().int().min(0).max(20).default(2).optional()
|
|
256
|
+
.describe('Pre/post context lines per span (default: 2)'),
|
|
257
|
+
maxChars: z.number().int().min(200).max(64000).default(8000).optional()
|
|
258
|
+
.describe('Hard cap on returned text (default: 8000 chars)'),
|
|
259
|
+
maxTokens: z.number().int().min(50).max(16000).optional()
|
|
260
|
+
.describe('Convenience cap (~chars/4)'),
|
|
261
|
+
verbose: z.boolean().default(false).optional()
|
|
262
|
+
.describe('Include timings + per-signal scores'),
|
|
263
|
+
},
|
|
264
|
+
outputSchema: ReadSemanticOutputSchema,
|
|
265
|
+
annotations: { readOnlyHint: true, destructiveHint: false, idempotentHint: true, openWorldHint: false },
|
|
266
|
+
}, async (args) => handleReadSemantic(args, { PROJECT_ROOT }));
|
|
267
|
+
|
|
227
268
|
// ---------------------------------------------------------------------------
|
|
228
269
|
// Resources
|
|
229
270
|
// ---------------------------------------------------------------------------
|
package/mcp/tool-handlers.js
CHANGED
|
@@ -88,6 +88,65 @@ export const VocabPrewarmOutputSchema = z.object({
|
|
|
88
88
|
dryRun: z.boolean().optional(),
|
|
89
89
|
});
|
|
90
90
|
|
|
91
|
+
const ReadFileResultSchema = z.object({
|
|
92
|
+
file: z.string(),
|
|
93
|
+
absolutePath: z.string().optional(),
|
|
94
|
+
ok: z.boolean(),
|
|
95
|
+
exact: z.boolean().optional(),
|
|
96
|
+
indexed: z.boolean().optional(),
|
|
97
|
+
language: z.string().nullable().optional(),
|
|
98
|
+
totalLines: z.number().int().optional(),
|
|
99
|
+
bytes: z.number().int().optional(),
|
|
100
|
+
mtimeMs: z.number().optional(),
|
|
101
|
+
range: z.object({
|
|
102
|
+
startLine: z.number().int(),
|
|
103
|
+
endLine: z.number().int(),
|
|
104
|
+
}).nullable().optional(),
|
|
105
|
+
text: z.string().optional(),
|
|
106
|
+
chunks: z.array(z.object({
|
|
107
|
+
id: z.string(),
|
|
108
|
+
symbol: z.string().nullable().optional(),
|
|
109
|
+
type: z.string().nullable().optional(),
|
|
110
|
+
startLine: z.number().int().nullable().optional(),
|
|
111
|
+
endLine: z.number().int().nullable().optional(),
|
|
112
|
+
signature: z.string().nullable().optional(),
|
|
113
|
+
})).optional(),
|
|
114
|
+
error: z.string().optional(),
|
|
115
|
+
timings: z.object({ totalMs: z.number() }).optional(),
|
|
116
|
+
});
|
|
117
|
+
|
|
118
|
+
export const ReadOutputSchema = z.object({
|
|
119
|
+
files: z.array(ReadFileResultSchema),
|
|
120
|
+
totalMs: z.number(),
|
|
121
|
+
});
|
|
122
|
+
|
|
123
|
+
const ReadSemanticSpanSchema = z.object({
|
|
124
|
+
startLine: z.number().int(),
|
|
125
|
+
endLine: z.number().int(),
|
|
126
|
+
score: z.number(),
|
|
127
|
+
symbols: z.array(z.string()).optional(),
|
|
128
|
+
types: z.array(z.string()).optional(),
|
|
129
|
+
chunkIds: z.array(z.string()).optional(),
|
|
130
|
+
text: z.string(),
|
|
131
|
+
truncated: z.boolean().optional(),
|
|
132
|
+
});
|
|
133
|
+
|
|
134
|
+
export const ReadSemanticOutputSchema = z.object({
|
|
135
|
+
file: z.string(),
|
|
136
|
+
query: z.string(),
|
|
137
|
+
ok: z.boolean(),
|
|
138
|
+
indexed: z.boolean(),
|
|
139
|
+
fellBack: z.boolean(),
|
|
140
|
+
reason: z.string().optional(),
|
|
141
|
+
language: z.string().nullable().optional(),
|
|
142
|
+
totalLines: z.number().int().optional(),
|
|
143
|
+
spans: z.array(ReadSemanticSpanSchema),
|
|
144
|
+
charsReturned: z.number().int().optional(),
|
|
145
|
+
approxTokensReturned: z.number().int().optional(),
|
|
146
|
+
signals: z.record(z.string(), z.any()).optional(),
|
|
147
|
+
timings: z.record(z.string(), z.number()).optional(),
|
|
148
|
+
});
|
|
149
|
+
|
|
91
150
|
// ---------------------------------------------------------------------------
|
|
92
151
|
// Internal state for health DB cache (module-scoped, not exported)
|
|
93
152
|
// ---------------------------------------------------------------------------
|
|
@@ -104,11 +163,6 @@ let _healthDb = null;
|
|
|
104
163
|
*/
|
|
105
164
|
export async function handleSearch({ query, k, mode, structural, regex, format, tokenBudget }, { getSearcher }) {
|
|
106
165
|
try {
|
|
107
|
-
// Agent format requires a regex (pattern search). If no regex, ignore the format
|
|
108
|
-
// to avoid silent fallback to non-pattern search without agent packaging.
|
|
109
|
-
const isAgentFormat = format && format.startsWith('agent');
|
|
110
|
-
const effectiveFormat = (isAgentFormat && !regex) ? undefined : format;
|
|
111
|
-
|
|
112
166
|
const searcher = await getSearcher();
|
|
113
167
|
const searchMode = structural ? 'structural' : mode;
|
|
114
168
|
const searchResult = await searcher.search(query, {
|
|
@@ -117,7 +171,7 @@ export async function handleSearch({ query, k, mode, structural, regex, format,
|
|
|
117
171
|
expand: true,
|
|
118
172
|
rerank: true,
|
|
119
173
|
...(regex && { regex }),
|
|
120
|
-
...(
|
|
174
|
+
...(format && { format }),
|
|
121
175
|
...(tokenBudget && { tokenBudget }),
|
|
122
176
|
});
|
|
123
177
|
|
|
@@ -474,3 +528,60 @@ export async function handleVocabPrewarm({ depth, modes, top, incremental, dryRu
|
|
|
474
528
|
};
|
|
475
529
|
}
|
|
476
530
|
}
|
|
531
|
+
|
|
532
|
+
// ---------------------------------------------------------------------------
|
|
533
|
+
// read — filesystem-grounded reader
|
|
534
|
+
// ---------------------------------------------------------------------------
|
|
535
|
+
|
|
536
|
+
/**
|
|
537
|
+
* @param {{ files: Array<{path: string, startLine?: number, endLine?: number}>, includeMetadata?: boolean }} args
|
|
538
|
+
* @param {{ PROJECT_ROOT: string }} deps
|
|
539
|
+
*/
|
|
540
|
+
export async function handleRead(args, deps) {
|
|
541
|
+
try {
|
|
542
|
+
const { readFiles, formatReadResults } = await import('../core/search/index.js');
|
|
543
|
+
const result = await readFiles(args.files || [], {
|
|
544
|
+
projectRoot: deps.PROJECT_ROOT,
|
|
545
|
+
includeMetadata: args.includeMetadata !== false,
|
|
546
|
+
});
|
|
547
|
+
return {
|
|
548
|
+
content: [{ type: 'text', text: formatReadResults(result, 'agent') }],
|
|
549
|
+
structuredContent: result,
|
|
550
|
+
};
|
|
551
|
+
} catch (err) {
|
|
552
|
+
const msg = (err.message || 'read failed').split('\n')[0];
|
|
553
|
+
return { content: [{ type: 'text', text: `read error: ${msg}` }], isError: true };
|
|
554
|
+
}
|
|
555
|
+
}
|
|
556
|
+
|
|
557
|
+
// ---------------------------------------------------------------------------
|
|
558
|
+
// read-semantic — hybrid span selection + filesystem-grounded re-read
|
|
559
|
+
// ---------------------------------------------------------------------------
|
|
560
|
+
|
|
561
|
+
/**
|
|
562
|
+
* @param {{ file: string, query: string, topK?: number, threshold?: number, contextLines?: number, maxChars?: number, maxTokens?: number, verbose?: boolean }} args
|
|
563
|
+
* @param {{ PROJECT_ROOT: string }} deps
|
|
564
|
+
*/
|
|
565
|
+
export async function handleReadSemantic(args, deps) {
|
|
566
|
+
try {
|
|
567
|
+
const { readSemantic, formatReadSemanticResult } = await import('../core/search/index.js');
|
|
568
|
+
const result = await readSemantic({
|
|
569
|
+
path: args.file,
|
|
570
|
+
query: args.query,
|
|
571
|
+
topK: args.topK,
|
|
572
|
+
threshold: args.threshold,
|
|
573
|
+
contextLines: args.contextLines,
|
|
574
|
+
maxChars: args.maxChars,
|
|
575
|
+
maxTokens: args.maxTokens,
|
|
576
|
+
projectRoot: deps.PROJECT_ROOT,
|
|
577
|
+
verbose: args.verbose,
|
|
578
|
+
});
|
|
579
|
+
return {
|
|
580
|
+
content: [{ type: 'text', text: formatReadSemanticResult(result, 'agent') }],
|
|
581
|
+
structuredContent: result,
|
|
582
|
+
};
|
|
583
|
+
} catch (err) {
|
|
584
|
+
const msg = (err.message || 'read-semantic failed').split('\n')[0];
|
|
585
|
+
return { content: [{ type: 'text', text: `read-semantic error: ${msg}` }], isError: true };
|
|
586
|
+
}
|
|
587
|
+
}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "sweet-search",
|
|
3
|
-
"version": "2.
|
|
3
|
+
"version": "2.5.2",
|
|
4
4
|
"description": "Sweet Search - SOTA Hybrid Code Search Engine with WASM CatBoost Query Router, Semantic/Lexical/Structural Search, and Multilingual Support",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"main": "core/search/sweet-search.js",
|
|
@@ -99,6 +99,8 @@
|
|
|
99
99
|
"eval:latency": "node eval/scripts/latency-stress.js",
|
|
100
100
|
"eval:multirepo": "node eval/scripts/multirepo-bench.js",
|
|
101
101
|
"eval:multirepo:test": "node eval/scripts/multirepo-bench.js --split=test",
|
|
102
|
+
"bench:read-workflows": "node eval/read-workflows/run-bench.js",
|
|
103
|
+
"bench:agent-read-workflows": "node eval/agent-read-workflows/run-bench.js",
|
|
102
104
|
"eval:fetch-repos": "node eval/scripts/fetch-benchmark-repos.js",
|
|
103
105
|
"features": "node core/training/query-router/features/extractor.js",
|
|
104
106
|
"features:benchmark": "node core/training/query-router/features/extractor.js --benchmark",
|
|
@@ -140,12 +142,12 @@
|
|
|
140
142
|
"vitest": "^4.0.16"
|
|
141
143
|
},
|
|
142
144
|
"optionalDependencies": {
|
|
143
|
-
"@sweet-search/native-darwin-arm64": "2.
|
|
144
|
-
"@sweet-search/native-darwin-x64": "2.
|
|
145
|
-
"@sweet-search/native-linux-arm64-gnu": "2.
|
|
146
|
-
"@sweet-search/native-linux-arm64-gnu-cuda": "2.
|
|
147
|
-
"@sweet-search/native-linux-x64-gnu": "2.
|
|
148
|
-
"@sweet-search/native-linux-x64-gnu-cuda": "2.
|
|
145
|
+
"@sweet-search/native-darwin-arm64": "2.5.2",
|
|
146
|
+
"@sweet-search/native-darwin-x64": "2.5.2",
|
|
147
|
+
"@sweet-search/native-linux-arm64-gnu": "2.5.2",
|
|
148
|
+
"@sweet-search/native-linux-arm64-gnu-cuda": "2.5.2",
|
|
149
|
+
"@sweet-search/native-linux-x64-gnu": "2.5.2",
|
|
150
|
+
"@sweet-search/native-linux-x64-gnu-cuda": "2.5.2"
|
|
149
151
|
},
|
|
150
152
|
"engines": {
|
|
151
153
|
"node": ">=18.0.0"
|