sweet-search 0.0.1 → 2.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (161) hide show
  1. package/LICENSE +190 -0
  2. package/NOTICE +23 -0
  3. package/core/cli.js +51 -0
  4. package/core/config.js +27 -0
  5. package/core/embedding/embedding-cache.js +467 -0
  6. package/core/embedding/embedding-local-model.js +845 -0
  7. package/core/embedding/embedding-remote.js +492 -0
  8. package/core/embedding/embedding-service.js +712 -0
  9. package/core/embedding/embedding-telemetry.js +219 -0
  10. package/core/embedding/index.js +40 -0
  11. package/core/graph/community-detector.js +294 -0
  12. package/core/graph/graph-expansion.js +839 -0
  13. package/core/graph/graph-extractor.js +2304 -0
  14. package/core/graph/graph-search.js +2148 -0
  15. package/core/graph/hcgs-generator.js +666 -0
  16. package/core/graph/index.js +16 -0
  17. package/core/graph/leiden-algorithm.js +547 -0
  18. package/core/graph/relationship-resolver.js +366 -0
  19. package/core/graph/repo-map.js +408 -0
  20. package/core/graph/summary-manager.js +549 -0
  21. package/core/indexing/artifact-builder.js +1054 -0
  22. package/core/indexing/ast-chunker.js +709 -0
  23. package/core/indexing/chunking/chunk-builder.js +170 -0
  24. package/core/indexing/chunking/markdown-chunker.js +503 -0
  25. package/core/indexing/chunking/plaintext-chunker.js +104 -0
  26. package/core/indexing/dedup/dedup-phase.js +159 -0
  27. package/core/indexing/dedup/exemplar-selector.js +65 -0
  28. package/core/indexing/document-chunker.js +56 -0
  29. package/core/indexing/incremental-parser.js +390 -0
  30. package/core/indexing/incremental-tracker.js +761 -0
  31. package/core/indexing/index-codebase-v21.js +472 -0
  32. package/core/indexing/index-maintainer.mjs +1674 -0
  33. package/core/indexing/index.js +90 -0
  34. package/core/indexing/indexer-ann.js +1077 -0
  35. package/core/indexing/indexer-build.js +742 -0
  36. package/core/indexing/indexer-phases.js +800 -0
  37. package/core/indexing/indexer-pool.js +764 -0
  38. package/core/indexing/indexer-sparse-gram.js +98 -0
  39. package/core/indexing/indexer-utils.js +536 -0
  40. package/core/indexing/indexer-worker.js +148 -0
  41. package/core/indexing/li-skip-policy.js +225 -0
  42. package/core/indexing/merkle-tracker.js +244 -0
  43. package/core/indexing/model-pool.js +166 -0
  44. package/core/infrastructure/code-graph-repository.js +120 -0
  45. package/core/infrastructure/codebase-repository.js +131 -0
  46. package/core/infrastructure/config/dedup.js +54 -0
  47. package/core/infrastructure/config/embedding.js +298 -0
  48. package/core/infrastructure/config/graph.js +80 -0
  49. package/core/infrastructure/config/index.js +82 -0
  50. package/core/infrastructure/config/indexing.js +8 -0
  51. package/core/infrastructure/config/platform.js +254 -0
  52. package/core/infrastructure/config/ranking.js +221 -0
  53. package/core/infrastructure/config/search.js +396 -0
  54. package/core/infrastructure/config/translation.js +89 -0
  55. package/core/infrastructure/config/vector-store.js +114 -0
  56. package/core/infrastructure/constants.js +86 -0
  57. package/core/infrastructure/coreml-cascade.js +909 -0
  58. package/core/infrastructure/coreml-cascade.json +46 -0
  59. package/core/infrastructure/coreml-provider.js +81 -0
  60. package/core/infrastructure/db-utils.js +69 -0
  61. package/core/infrastructure/dedup-hashing.js +83 -0
  62. package/core/infrastructure/hardware-capability.js +332 -0
  63. package/core/infrastructure/index.js +104 -0
  64. package/core/infrastructure/language-patterns/maps.js +121 -0
  65. package/core/infrastructure/language-patterns/registry-core.js +323 -0
  66. package/core/infrastructure/language-patterns/registry-data-query.js +155 -0
  67. package/core/infrastructure/language-patterns/registry-object-oriented.js +285 -0
  68. package/core/infrastructure/language-patterns/registry-tooling.js +240 -0
  69. package/core/infrastructure/language-patterns/registry-web-style.js +143 -0
  70. package/core/infrastructure/language-patterns/registry.js +19 -0
  71. package/core/infrastructure/language-patterns.js +141 -0
  72. package/core/infrastructure/llm-provider.js +733 -0
  73. package/core/infrastructure/manifest.json +46 -0
  74. package/core/infrastructure/maxsim.wasm +0 -0
  75. package/core/infrastructure/model-fetcher.js +423 -0
  76. package/core/infrastructure/model-registry.js +214 -0
  77. package/core/infrastructure/native-inference.js +587 -0
  78. package/core/infrastructure/native-resolver.js +187 -0
  79. package/core/infrastructure/native-sparse-gram.js +257 -0
  80. package/core/infrastructure/native-tokenizer.js +160 -0
  81. package/core/infrastructure/onnx-mutex.js +45 -0
  82. package/core/infrastructure/onnx-session-utils.js +261 -0
  83. package/core/infrastructure/ort-pipeline.js +111 -0
  84. package/core/infrastructure/project-detector.js +102 -0
  85. package/core/infrastructure/quantization.js +410 -0
  86. package/core/infrastructure/simd-distance.js +502 -0
  87. package/core/infrastructure/simd-distance.wasm +0 -0
  88. package/core/infrastructure/tree-sitter-provider.js +665 -0
  89. package/core/infrastructure/webgpu-maxsim.js +222 -0
  90. package/core/query/index.js +35 -0
  91. package/core/query/intent-detector.js +201 -0
  92. package/core/query/intent-router.js +156 -0
  93. package/core/query/query-router-catboost.js +222 -0
  94. package/core/query/query-router-ml.js +266 -0
  95. package/core/query/query-router.js +213 -0
  96. package/core/ranking/cascaded-scorer.js +379 -0
  97. package/core/ranking/flashrank.js +810 -0
  98. package/core/ranking/index.js +49 -0
  99. package/core/ranking/late-interaction-index.js +2383 -0
  100. package/core/ranking/late-interaction-model.js +812 -0
  101. package/core/ranking/local-reranker.js +374 -0
  102. package/core/ranking/mmr.js +379 -0
  103. package/core/ranking/quality-scorer.js +363 -0
  104. package/core/search/context-expander.js +1167 -0
  105. package/core/search/dedup/sibling-expander.js +327 -0
  106. package/core/search/index.js +16 -0
  107. package/core/search/search-boost.js +259 -0
  108. package/core/search/search-cli.js +544 -0
  109. package/core/search/search-format.js +282 -0
  110. package/core/search/search-fusion.js +327 -0
  111. package/core/search/search-hybrid.js +204 -0
  112. package/core/search/search-pattern-chunks.js +337 -0
  113. package/core/search/search-pattern-planner.js +439 -0
  114. package/core/search/search-pattern-prefilter.js +412 -0
  115. package/core/search/search-pattern-ripgrep.js +663 -0
  116. package/core/search/search-pattern.js +463 -0
  117. package/core/search/search-postprocess.js +452 -0
  118. package/core/search/search-semantic.js +706 -0
  119. package/core/search/search-server.js +554 -0
  120. package/core/search/session-daemon-prewarm.mjs +164 -0
  121. package/core/search/session-warmup.js +595 -0
  122. package/core/search/sweet-search.js +632 -0
  123. package/core/search/warmup-metrics.js +532 -0
  124. package/core/start-server.js +6 -0
  125. package/core/training/query-router/features/extractor.js +762 -0
  126. package/core/training/query-router/features/multilingual-patterns.js +431 -0
  127. package/core/training/query-router/features/text-segmenter.js +303 -0
  128. package/core/training/query-router/features/unicode-utils.js +383 -0
  129. package/core/training/query-router/output/v45_router_d4.js +11521 -0
  130. package/core/training/query-router/output/v46_router_d4.js +11498 -0
  131. package/core/vector-store/binary-heap.js +227 -0
  132. package/core/vector-store/binary-hnsw-index.js +1004 -0
  133. package/core/vector-store/float-vector-store.js +234 -0
  134. package/core/vector-store/hnsw-index.js +580 -0
  135. package/core/vector-store/index.js +39 -0
  136. package/core/vector-store/seismic-index.js +498 -0
  137. package/core/vocabulary/index.js +84 -0
  138. package/core/vocabulary/vocab-constants.js +20 -0
  139. package/core/vocabulary/vocab-miner-extractors.js +375 -0
  140. package/core/vocabulary/vocab-miner-nl.js +404 -0
  141. package/core/vocabulary/vocab-miner-utils.js +146 -0
  142. package/core/vocabulary/vocab-miner.js +574 -0
  143. package/core/vocabulary/vocab-prewarm-cli.js +110 -0
  144. package/core/vocabulary/vocab-ranker.js +492 -0
  145. package/core/vocabulary/vocab-warmer.js +523 -0
  146. package/core/vocabulary/vocab-warmup-orchestrator.js +425 -0
  147. package/core/vocabulary/vocabulary-utils.js +704 -0
  148. package/crates/wasm-router/pkg/package.json +13 -0
  149. package/crates/wasm-router/pkg/query_router_wasm.d.ts +36 -0
  150. package/crates/wasm-router/pkg/query_router_wasm.js +271 -0
  151. package/crates/wasm-router/pkg/query_router_wasm_bg.wasm +0 -0
  152. package/crates/wasm-router/pkg/query_router_wasm_bg.wasm.d.ts +19 -0
  153. package/mcp/config-gen.js +121 -0
  154. package/mcp/server.js +335 -0
  155. package/mcp/tool-handlers.js +476 -0
  156. package/package.json +131 -9
  157. package/scripts/benchmark-harness.js +794 -0
  158. package/scripts/init.js +1058 -0
  159. package/scripts/smoke-test.js +435 -0
  160. package/scripts/uninstall.js +478 -0
  161. package/scripts/verify-runtime.js +176 -0
package/mcp/server.js ADDED
@@ -0,0 +1,335 @@
1
+ #!/usr/bin/env node
2
+
3
+ // ESM-safe stdout protection: override console.log BEFORE any search module loads.
4
+ // Search modules call console.log at import time, which would corrupt stdio transport.
5
+ const _origLog = console.log;
6
+ console.log = (...args) => console.error(...args);
7
+
8
+ import { McpServer } from '@modelcontextprotocol/sdk/server/mcp.js';
9
+ import { StdioServerTransport } from '@modelcontextprotocol/sdk/server/stdio.js';
10
+ import { z } from 'zod';
11
+ import { existsSync, statSync, readFileSync } from 'node:fs';
12
+ import path from 'node:path';
13
+ import { fileURLToPath } from 'node:url';
14
+
15
+ import {
16
+ SearchOutputSchema,
17
+ IndexOutputSchema,
18
+ HealthOutputSchema,
19
+ RepoMapOutputSchema,
20
+ VocabPrewarmOutputSchema,
21
+ handleSearch,
22
+ handleIndex,
23
+ checkHealth,
24
+ handleRepoMap,
25
+ handleVocabPrewarm,
26
+ } from './tool-handlers.js';
27
+
28
+ const __filename = fileURLToPath(import.meta.url);
29
+ const __dirname = path.dirname(__filename);
30
+
31
+ const PKG_VERSION = (() => {
32
+ try {
33
+ const pkg = JSON.parse(readFileSync(path.join(__dirname, '..', 'package.json'), 'utf8'));
34
+ return pkg.version || '0.0.0';
35
+ } catch (err) {
36
+ if (process.env.DEBUG_CATCHES) process.stderr.write(`[non-fatal] ${err?.message || err}\n`);
37
+ return '0.0.0';
38
+ }
39
+ })();
40
+
41
+ // ---------------------------------------------------------------------------
42
+ // Project root resolution (once at startup)
43
+ // ---------------------------------------------------------------------------
44
+
45
+ function resolveProjectRoot() {
46
+ const args = process.argv.slice(2);
47
+ const rootFlagIdx = args.indexOf('--project-root');
48
+
49
+ let root;
50
+ if (rootFlagIdx !== -1 && args[rootFlagIdx + 1]) {
51
+ root = args[rootFlagIdx + 1];
52
+ } else if (process.env.SWEET_SEARCH_PROJECT_ROOT) {
53
+ root = process.env.SWEET_SEARCH_PROJECT_ROOT;
54
+ } else {
55
+ root = process.cwd();
56
+ }
57
+
58
+ root = path.resolve(root);
59
+
60
+ if (!existsSync(root) || !statSync(root).isDirectory()) {
61
+ console.error(`[sweet-search-mcp] Error: project root does not exist or is not a directory: ${root}`);
62
+ process.exit(1);
63
+ }
64
+
65
+ return root;
66
+ }
67
+
68
+ const PROJECT_ROOT = resolveProjectRoot();
69
+
70
+ // Fix #3: Propagate resolved root to environment so core/config.js picks it up
71
+ // even when --project-root CLI flag was used instead of env var.
72
+ process.env.SWEET_SEARCH_PROJECT_ROOT = PROJECT_ROOT;
73
+
74
+ // ---------------------------------------------------------------------------
75
+ // Lazy-loaded search engine (dynamic import to keep stdout safe)
76
+ // ---------------------------------------------------------------------------
77
+
78
+ let _searcher = null;
79
+
80
+ async function getSearcher() {
81
+ if (_searcher) return _searcher;
82
+
83
+ const { getWarmSearcher } = await import('../core/search/index.js');
84
+
85
+ _searcher = await getWarmSearcher({ verbose: false });
86
+ return _searcher;
87
+ }
88
+
89
+ async function getConfig() {
90
+ return import('../core/infrastructure/config/index.js');
91
+ }
92
+
93
+ // ---------------------------------------------------------------------------
94
+ // Shared dependency objects passed to handlers
95
+ // ---------------------------------------------------------------------------
96
+
97
+ const coreDir = path.join(__dirname, '..', 'core');
98
+
99
+ const searchDeps = { getSearcher };
100
+ const indexDeps = { PROJECT_ROOT, coreDir };
101
+ const healthDeps = { getConfig, PROJECT_ROOT };
102
+ const repoMapDeps = { coreDir };
103
+ const vocabDeps = { coreDir };
104
+
105
+ // ---------------------------------------------------------------------------
106
+ // MCP Server
107
+ // ---------------------------------------------------------------------------
108
+
109
+ const server = new McpServer({
110
+ name: 'sweet-search',
111
+ version: PKG_VERSION,
112
+ }, {
113
+ capabilities: {
114
+ tools: { listChanged: false },
115
+ resources: { subscribe: false, listChanged: false },
116
+ prompts: { listChanged: false },
117
+ },
118
+ });
119
+
120
+ // ---------------------------------------------------------------------------
121
+ // Tools
122
+ // ---------------------------------------------------------------------------
123
+
124
+ server.registerTool('search', {
125
+ description: 'Search the codebase using hybrid semantic/lexical/structural search. Use format="agent" with a regex for ColGrep pattern search that returns self-contained code blocks — eliminates follow-up file reads.',
126
+ inputSchema: {
127
+ query: z.string().min(1).max(1000).describe('Search query (1-1000 chars)'),
128
+ k: z.number().int().min(1).max(200).default(10).describe('Number of results (1-200)'),
129
+ mode: z.enum(['auto', 'lexical', 'semantic', 'hybrid']).default('auto')
130
+ .describe('Search mode'),
131
+ structural: z.boolean().default(false)
132
+ .describe('Force structural graph search mode (callers, callees, implementations)'),
133
+ regex: z.string().max(4096).optional()
134
+ .describe('Regex pattern for ColGrep pattern search (implies mode=pattern)'),
135
+ format: z.enum(['benchmark', 'agent', 'agent_preview', 'agent_full']).default('benchmark').optional()
136
+ .describe('Output format. "agent"/"agent_preview" returns bounded code blocks (4K budget). "agent_full" returns expanded code for top-3 (8K budget).'),
137
+ tokenBudget: z.number().int().min(500).max(16000).default(4000).optional()
138
+ .describe('Agent mode: total token budget for all results (default: 4000)'),
139
+ },
140
+ outputSchema: SearchOutputSchema,
141
+ annotations: {
142
+ readOnlyHint: true,
143
+ destructiveHint: false,
144
+ idempotentHint: true,
145
+ openWorldHint: false,
146
+ },
147
+ }, async (args) => handleSearch(args, searchDeps));
148
+
149
+ server.registerTool('index', {
150
+ description: 'Index or re-index the codebase',
151
+ inputSchema: {
152
+ mode: z.enum(['incremental', 'full']).default('incremental')
153
+ .describe('Indexing mode'),
154
+ },
155
+ outputSchema: IndexOutputSchema,
156
+ annotations: {
157
+ readOnlyHint: false,
158
+ destructiveHint: false,
159
+ idempotentHint: false,
160
+ openWorldHint: false,
161
+ },
162
+ }, async (args) => handleIndex(args, indexDeps));
163
+
164
+ server.registerTool('health', {
165
+ description: 'Check health status of all search subsystems',
166
+ outputSchema: HealthOutputSchema,
167
+ annotations: {
168
+ readOnlyHint: true,
169
+ destructiveHint: false,
170
+ idempotentHint: true,
171
+ openWorldHint: false,
172
+ },
173
+ }, async () => {
174
+ const structured = await checkHealth(healthDeps);
175
+
176
+ const statusLines = Object.entries(structured.subsystems).map(
177
+ ([name, s]) => ` ${s.status === 'ok' ? '+' : s.status === 'not_initialized' ? '-' : 'x'} ${name}: ${s.status}${s.details ? ' (' + s.details + ')' : ''}`
178
+ );
179
+ const text = `Health: ${structured.healthy ? 'OK' : 'DEGRADED'}\nProject: ${path.basename(PROJECT_ROOT)}\n\n${statusLines.join('\n')}`;
180
+
181
+ return {
182
+ content: [{ type: 'text', text }],
183
+ structuredContent: structured,
184
+ };
185
+ });
186
+
187
+ server.registerTool('repo-map', {
188
+ description: 'Generate a PageRank-scored repository map showing the most important symbols in the codebase, fitted to a token budget. Useful for giving LLMs a compressed structural overview.',
189
+ inputSchema: {
190
+ tokenBudget: z.number().int().min(100).max(100000).default(1024)
191
+ .describe('Maximum token budget for the output (default: 1024)'),
192
+ focusFiles: z.array(z.string()).optional()
193
+ .describe('Boost importance of entities in these files'),
194
+ focusEntities: z.array(z.string()).optional()
195
+ .describe('Boost importance of entities with these names'),
196
+ },
197
+ outputSchema: RepoMapOutputSchema,
198
+ annotations: {
199
+ readOnlyHint: true,
200
+ destructiveHint: false,
201
+ idempotentHint: true,
202
+ openWorldHint: false,
203
+ },
204
+ }, async (args) => handleRepoMap(args, repoMapDeps));
205
+
206
+ server.registerTool('vocab-prewarm', {
207
+ description: 'Mine the codebase for search vocabulary and warm all search modes (lexical, semantic, hybrid) with project-specific terms',
208
+ inputSchema: {
209
+ depth: z.enum(['light', 'medium', 'deep']).default('medium').describe('Mining depth'),
210
+ modes: z.array(z.enum(['lexical', 'semantic', 'hybrid'])).default(['lexical', 'semantic', 'hybrid']).describe('Search modes to warm'),
211
+ top: z.number().int().min(50).max(5000).default(1000).describe('Number of top terms to warm'),
212
+ incremental: z.boolean().default(true).describe('Only process changes since last warmup'),
213
+ dryRun: z.boolean().default(false).describe('Show what would be mined without actually warming'),
214
+ stats: z.boolean().default(false).describe('Return current warmup statistics'),
215
+ localWarmup: z.boolean().default(false).describe('Force local model for warmup even when remote provider is active').optional(),
216
+ provider: z.string().describe('Override embedding provider (voyage/mistral/jina/local)').optional(),
217
+ },
218
+ outputSchema: VocabPrewarmOutputSchema,
219
+ annotations: {
220
+ readOnlyHint: false,
221
+ destructiveHint: false,
222
+ idempotentHint: true,
223
+ openWorldHint: true,
224
+ },
225
+ }, async (args) => handleVocabPrewarm(args, vocabDeps));
226
+
227
+ // ---------------------------------------------------------------------------
228
+ // Resources
229
+ // ---------------------------------------------------------------------------
230
+
231
+ server.resource(
232
+ 'status',
233
+ 'sweet-search://status',
234
+ { mimeType: 'application/json', description: 'Index health and statistics' },
235
+ async () => {
236
+ const data = await checkHealth(healthDeps);
237
+ return { contents: [{ uri: 'sweet-search://status', mimeType: 'application/json', text: JSON.stringify(data, null, 2) }] };
238
+ },
239
+ );
240
+
241
+ server.resource(
242
+ 'config',
243
+ 'sweet-search://config',
244
+ { mimeType: 'application/json', description: 'Current search configuration' },
245
+ async () => {
246
+ let data = {};
247
+ try {
248
+ const config = await getConfig();
249
+ data = {
250
+ searchMode: 'auto',
251
+ reranker: {
252
+ flashrank: true,
253
+ localReranker: config.shouldUseLocalReranker?.() || false,
254
+ },
255
+ embeddingModel: config.EMBEDDING_CONFIG?.model || 'unknown',
256
+ supportedLanguages: ['en', 'de', 'fr', 'es', 'pl', 'ja', 'ko', 'zh', 'ru'],
257
+ projectRoot: path.basename(PROJECT_ROOT),
258
+ };
259
+ } catch (e) {
260
+ data = { error: e.message };
261
+ }
262
+ return { contents: [{ uri: 'sweet-search://config', mimeType: 'application/json', text: JSON.stringify(data, null, 2) }] };
263
+ },
264
+ );
265
+
266
+ // ---------------------------------------------------------------------------
267
+ // Prompts
268
+ // ---------------------------------------------------------------------------
269
+
270
+ server.prompt(
271
+ 'search-codebase',
272
+ 'Guided codebase search with focused results',
273
+ { query: z.string().describe('What to search for'), focus: z.string().optional().describe('Focus area: functions, types, tests, etc.') },
274
+ ({ query, focus }) => ({
275
+ messages: [
276
+ {
277
+ role: 'user',
278
+ content: {
279
+ type: 'text',
280
+ text: `Search this codebase for: "${query}"${focus ? ` (focus on ${focus})` : ''}\n\nUse the search tool to find relevant code, then summarize the findings with file paths and key code snippets.`,
281
+ },
282
+ },
283
+ ],
284
+ }),
285
+ );
286
+
287
+ server.prompt(
288
+ 'explain-code',
289
+ 'Find and explain code related to a topic',
290
+ { topic: z.string().describe('Topic to find and explain') },
291
+ ({ topic }) => ({
292
+ messages: [
293
+ {
294
+ role: 'user',
295
+ content: {
296
+ type: 'text',
297
+ text: `Find code related to "${topic}" in this codebase using the search tool, then explain how it works with context about the architecture and key patterns.`,
298
+ },
299
+ },
300
+ ],
301
+ }),
302
+ );
303
+
304
+ // ---------------------------------------------------------------------------
305
+ // Start server
306
+ // ---------------------------------------------------------------------------
307
+
308
+ // TODO: Streamable HTTP transport (--transport http --port <port>) — future work.
309
+ // Currently only stdio is implemented (universal baseline for Claude Code + Codex).
310
+
311
+ // ---------------------------------------------------------------------------
312
+ // Graceful shutdown (F-18)
313
+ // ---------------------------------------------------------------------------
314
+
315
+ function shutdown(signal) {
316
+ console.error(`[sweet-search-mcp] ${signal} received, shutting down`);
317
+ try { server.close(); } catch (err) {
318
+ if (process.env.DEBUG_CATCHES) process.stderr.write(`[non-fatal] ${err?.message || err}\n`);
319
+ }
320
+ process.exit(0);
321
+ }
322
+
323
+ process.on('SIGINT', () => shutdown('SIGINT'));
324
+ process.on('SIGTERM', () => shutdown('SIGTERM'));
325
+
326
+ async function main() {
327
+ const transport = new StdioServerTransport();
328
+ await server.connect(transport);
329
+ console.error(`[sweet-search-mcp] Server started (project: ${PROJECT_ROOT})`);
330
+ }
331
+
332
+ main().catch((err) => {
333
+ console.error('[sweet-search-mcp] Fatal:', err);
334
+ process.exit(1);
335
+ });