sweet-search 0.0.1 → 2.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (161) hide show
  1. package/LICENSE +190 -0
  2. package/NOTICE +23 -0
  3. package/core/cli.js +51 -0
  4. package/core/config.js +27 -0
  5. package/core/embedding/embedding-cache.js +467 -0
  6. package/core/embedding/embedding-local-model.js +845 -0
  7. package/core/embedding/embedding-remote.js +492 -0
  8. package/core/embedding/embedding-service.js +712 -0
  9. package/core/embedding/embedding-telemetry.js +219 -0
  10. package/core/embedding/index.js +40 -0
  11. package/core/graph/community-detector.js +294 -0
  12. package/core/graph/graph-expansion.js +839 -0
  13. package/core/graph/graph-extractor.js +2304 -0
  14. package/core/graph/graph-search.js +2148 -0
  15. package/core/graph/hcgs-generator.js +666 -0
  16. package/core/graph/index.js +16 -0
  17. package/core/graph/leiden-algorithm.js +547 -0
  18. package/core/graph/relationship-resolver.js +366 -0
  19. package/core/graph/repo-map.js +408 -0
  20. package/core/graph/summary-manager.js +549 -0
  21. package/core/indexing/artifact-builder.js +1054 -0
  22. package/core/indexing/ast-chunker.js +709 -0
  23. package/core/indexing/chunking/chunk-builder.js +170 -0
  24. package/core/indexing/chunking/markdown-chunker.js +503 -0
  25. package/core/indexing/chunking/plaintext-chunker.js +104 -0
  26. package/core/indexing/dedup/dedup-phase.js +159 -0
  27. package/core/indexing/dedup/exemplar-selector.js +65 -0
  28. package/core/indexing/document-chunker.js +56 -0
  29. package/core/indexing/incremental-parser.js +390 -0
  30. package/core/indexing/incremental-tracker.js +761 -0
  31. package/core/indexing/index-codebase-v21.js +472 -0
  32. package/core/indexing/index-maintainer.mjs +1674 -0
  33. package/core/indexing/index.js +90 -0
  34. package/core/indexing/indexer-ann.js +1077 -0
  35. package/core/indexing/indexer-build.js +742 -0
  36. package/core/indexing/indexer-phases.js +800 -0
  37. package/core/indexing/indexer-pool.js +764 -0
  38. package/core/indexing/indexer-sparse-gram.js +98 -0
  39. package/core/indexing/indexer-utils.js +536 -0
  40. package/core/indexing/indexer-worker.js +148 -0
  41. package/core/indexing/li-skip-policy.js +225 -0
  42. package/core/indexing/merkle-tracker.js +244 -0
  43. package/core/indexing/model-pool.js +166 -0
  44. package/core/infrastructure/code-graph-repository.js +120 -0
  45. package/core/infrastructure/codebase-repository.js +131 -0
  46. package/core/infrastructure/config/dedup.js +54 -0
  47. package/core/infrastructure/config/embedding.js +298 -0
  48. package/core/infrastructure/config/graph.js +80 -0
  49. package/core/infrastructure/config/index.js +82 -0
  50. package/core/infrastructure/config/indexing.js +8 -0
  51. package/core/infrastructure/config/platform.js +254 -0
  52. package/core/infrastructure/config/ranking.js +221 -0
  53. package/core/infrastructure/config/search.js +396 -0
  54. package/core/infrastructure/config/translation.js +89 -0
  55. package/core/infrastructure/config/vector-store.js +114 -0
  56. package/core/infrastructure/constants.js +86 -0
  57. package/core/infrastructure/coreml-cascade.js +909 -0
  58. package/core/infrastructure/coreml-cascade.json +46 -0
  59. package/core/infrastructure/coreml-provider.js +81 -0
  60. package/core/infrastructure/db-utils.js +69 -0
  61. package/core/infrastructure/dedup-hashing.js +83 -0
  62. package/core/infrastructure/hardware-capability.js +332 -0
  63. package/core/infrastructure/index.js +104 -0
  64. package/core/infrastructure/language-patterns/maps.js +121 -0
  65. package/core/infrastructure/language-patterns/registry-core.js +323 -0
  66. package/core/infrastructure/language-patterns/registry-data-query.js +155 -0
  67. package/core/infrastructure/language-patterns/registry-object-oriented.js +285 -0
  68. package/core/infrastructure/language-patterns/registry-tooling.js +240 -0
  69. package/core/infrastructure/language-patterns/registry-web-style.js +143 -0
  70. package/core/infrastructure/language-patterns/registry.js +19 -0
  71. package/core/infrastructure/language-patterns.js +141 -0
  72. package/core/infrastructure/llm-provider.js +733 -0
  73. package/core/infrastructure/manifest.json +46 -0
  74. package/core/infrastructure/maxsim.wasm +0 -0
  75. package/core/infrastructure/model-fetcher.js +423 -0
  76. package/core/infrastructure/model-registry.js +214 -0
  77. package/core/infrastructure/native-inference.js +587 -0
  78. package/core/infrastructure/native-resolver.js +187 -0
  79. package/core/infrastructure/native-sparse-gram.js +257 -0
  80. package/core/infrastructure/native-tokenizer.js +160 -0
  81. package/core/infrastructure/onnx-mutex.js +45 -0
  82. package/core/infrastructure/onnx-session-utils.js +261 -0
  83. package/core/infrastructure/ort-pipeline.js +111 -0
  84. package/core/infrastructure/project-detector.js +102 -0
  85. package/core/infrastructure/quantization.js +410 -0
  86. package/core/infrastructure/simd-distance.js +502 -0
  87. package/core/infrastructure/simd-distance.wasm +0 -0
  88. package/core/infrastructure/tree-sitter-provider.js +665 -0
  89. package/core/infrastructure/webgpu-maxsim.js +222 -0
  90. package/core/query/index.js +35 -0
  91. package/core/query/intent-detector.js +201 -0
  92. package/core/query/intent-router.js +156 -0
  93. package/core/query/query-router-catboost.js +222 -0
  94. package/core/query/query-router-ml.js +266 -0
  95. package/core/query/query-router.js +213 -0
  96. package/core/ranking/cascaded-scorer.js +379 -0
  97. package/core/ranking/flashrank.js +810 -0
  98. package/core/ranking/index.js +49 -0
  99. package/core/ranking/late-interaction-index.js +2383 -0
  100. package/core/ranking/late-interaction-model.js +812 -0
  101. package/core/ranking/local-reranker.js +374 -0
  102. package/core/ranking/mmr.js +379 -0
  103. package/core/ranking/quality-scorer.js +363 -0
  104. package/core/search/context-expander.js +1167 -0
  105. package/core/search/dedup/sibling-expander.js +327 -0
  106. package/core/search/index.js +16 -0
  107. package/core/search/search-boost.js +259 -0
  108. package/core/search/search-cli.js +544 -0
  109. package/core/search/search-format.js +282 -0
  110. package/core/search/search-fusion.js +327 -0
  111. package/core/search/search-hybrid.js +204 -0
  112. package/core/search/search-pattern-chunks.js +337 -0
  113. package/core/search/search-pattern-planner.js +439 -0
  114. package/core/search/search-pattern-prefilter.js +412 -0
  115. package/core/search/search-pattern-ripgrep.js +663 -0
  116. package/core/search/search-pattern.js +463 -0
  117. package/core/search/search-postprocess.js +452 -0
  118. package/core/search/search-semantic.js +706 -0
  119. package/core/search/search-server.js +554 -0
  120. package/core/search/session-daemon-prewarm.mjs +164 -0
  121. package/core/search/session-warmup.js +595 -0
  122. package/core/search/sweet-search.js +632 -0
  123. package/core/search/warmup-metrics.js +532 -0
  124. package/core/start-server.js +6 -0
  125. package/core/training/query-router/features/extractor.js +762 -0
  126. package/core/training/query-router/features/multilingual-patterns.js +431 -0
  127. package/core/training/query-router/features/text-segmenter.js +303 -0
  128. package/core/training/query-router/features/unicode-utils.js +383 -0
  129. package/core/training/query-router/output/v45_router_d4.js +11521 -0
  130. package/core/training/query-router/output/v46_router_d4.js +11498 -0
  131. package/core/vector-store/binary-heap.js +227 -0
  132. package/core/vector-store/binary-hnsw-index.js +1004 -0
  133. package/core/vector-store/float-vector-store.js +234 -0
  134. package/core/vector-store/hnsw-index.js +580 -0
  135. package/core/vector-store/index.js +39 -0
  136. package/core/vector-store/seismic-index.js +498 -0
  137. package/core/vocabulary/index.js +84 -0
  138. package/core/vocabulary/vocab-constants.js +20 -0
  139. package/core/vocabulary/vocab-miner-extractors.js +375 -0
  140. package/core/vocabulary/vocab-miner-nl.js +404 -0
  141. package/core/vocabulary/vocab-miner-utils.js +146 -0
  142. package/core/vocabulary/vocab-miner.js +574 -0
  143. package/core/vocabulary/vocab-prewarm-cli.js +110 -0
  144. package/core/vocabulary/vocab-ranker.js +492 -0
  145. package/core/vocabulary/vocab-warmer.js +523 -0
  146. package/core/vocabulary/vocab-warmup-orchestrator.js +425 -0
  147. package/core/vocabulary/vocabulary-utils.js +704 -0
  148. package/crates/wasm-router/pkg/package.json +13 -0
  149. package/crates/wasm-router/pkg/query_router_wasm.d.ts +36 -0
  150. package/crates/wasm-router/pkg/query_router_wasm.js +271 -0
  151. package/crates/wasm-router/pkg/query_router_wasm_bg.wasm +0 -0
  152. package/crates/wasm-router/pkg/query_router_wasm_bg.wasm.d.ts +19 -0
  153. package/mcp/config-gen.js +121 -0
  154. package/mcp/server.js +335 -0
  155. package/mcp/tool-handlers.js +476 -0
  156. package/package.json +131 -9
  157. package/scripts/benchmark-harness.js +794 -0
  158. package/scripts/init.js +1058 -0
  159. package/scripts/smoke-test.js +435 -0
  160. package/scripts/uninstall.js +478 -0
  161. package/scripts/verify-runtime.js +176 -0
@@ -0,0 +1,574 @@
1
+ /**
2
+ * Vocabulary Miner Module
3
+ *
4
+ * Extracts search vocabulary terms from multiple sources:
5
+ * 1. Structural – file paths, directories, package manifests
6
+ * 2. Symbols – identifiers from source code (imports, classes, functions)
7
+ * 3. Code Graph – entity names and hub detection from code-graph.db
8
+ * 4. NL Content – community-aware TF-IDF over comments/docstrings
9
+ * 5. Git – commit messages, branch names, frequently changed files
10
+ *
11
+ * Each miner returns { terms: [{ term, score, source }] } or equivalent.
12
+ * `mineAll()` merges results from all miners into a unified term list.
13
+ */
14
+
15
+ import { readFileSync, readdirSync, statSync, existsSync } from 'fs';
16
+ import { join, basename, extname, relative, sep } from 'path';
17
+ import { execFileSync } from 'child_process';
18
+ import Database from 'better-sqlite3';
19
+ import { DB_PATHS, PROJECT_ROOT } from '../infrastructure/config/index.js';
20
+ import { applyReadPragmas } from '../infrastructure/db-utils.js';
21
+ import { pageRank, loadGraph, buildAdjacency } from '../graph/repo-map.js';
22
+
23
+ // Import from sub-modules
24
+ import {
25
+ STOP_WORDS, SOURCE_EXTENSIONS,
26
+ splitIdentifier, addTerm, mergeTerms, termsToArray, walkShallow,
27
+ } from './vocab-miner-utils.js';
28
+
29
+ import {
30
+ extractImports, extractExports, extractDefinitions, extractConstants,
31
+ extractNpmDeps, extractCargoDeps, extractGoDeps, extractPipDeps, extractPyprojectDeps,
32
+ } from './vocab-miner-extractors.js';
33
+
34
+ import {
35
+ mineNLContent, computeNLContentHash, extractNLText, isSecretLike,
36
+ detectScript, tokenizeNL, extractBigrams, extractTrigrams,
37
+ SECRET_PATTERNS, NL_MINING_TIMEOUT_MS,
38
+ } from './vocab-miner-nl.js';
39
+
40
+ // ---------------------------------------------------------------------------
41
+ // Barrel Re-exports (backward compatibility layer)
42
+ // ---------------------------------------------------------------------------
43
+ // Prefer direct imports from leaf modules for new code:
44
+ // - ./vocab-miner-utils.js
45
+ // - ./vocab-miner-extractors.js
46
+ // - ./vocab-miner-nl.js
47
+ // This barrel remains to avoid breaking existing callers.
48
+
49
+ export {
50
+ splitIdentifier, STOP_WORDS, SOURCE_EXTENSIONS,
51
+ addTerm, mergeTerms, termsToArray, walkShallow,
52
+ } from './vocab-miner-utils.js';
53
+
54
+ export {
55
+ extractImports, extractExports, extractDefinitions, extractConstants,
56
+ extractNpmDeps, extractCargoDeps, extractGoDeps, extractPipDeps, extractPyprojectDeps,
57
+ } from './vocab-miner-extractors.js';
58
+
59
+ export {
60
+ mineNLContent, computeNLContentHash, extractNLText, isSecretLike,
61
+ detectScript, tokenizeNL, extractBigrams, extractTrigrams,
62
+ SECRET_PATTERNS, NL_MINING_TIMEOUT_MS,
63
+ } from './vocab-miner-nl.js';
64
+
65
+ // ---------------------------------------------------------------------------
66
+ // Constants
67
+ // ---------------------------------------------------------------------------
68
+
69
+ // Manifest files to mine
70
+ const MANIFEST_FILES = [
71
+ { file: 'package.json', extract: extractNpmDeps },
72
+ { file: 'Cargo.toml', extract: extractCargoDeps },
73
+ { file: 'go.mod', extract: extractGoDeps },
74
+ { file: 'requirements.txt', extract: extractPipDeps },
75
+ { file: 'pyproject.toml', extract: extractPyprojectDeps },
76
+ ];
77
+
78
+ const COUNT_SKIP_DIRS = new Set([
79
+ 'node_modules', '.git', '.sweet-search', '.agentic-qe',
80
+ 'dist', 'build', 'out', '.next', '.nuxt', 'coverage',
81
+ '__pycache__', '.venv', 'venv', 'target', '.swarm',
82
+ ]);
83
+
84
+ /**
85
+ * Count source files for ranking IDF.
86
+ *
87
+ * Fast path uses `git ls-files` (cheap and accurate for tracked files).
88
+ * Fallback performs iterative directory traversal (avoids recursion depth limits).
89
+ */
90
+ function estimateTotalSourceFiles(root) {
91
+ // Fast path: ask git for tracked files, then filter by source extensions.
92
+ try {
93
+ const listed = execFileSync(
94
+ 'git',
95
+ ['-C', root, 'ls-files'],
96
+ { encoding: 'utf-8', timeout: 3000, stdio: ['pipe', 'pipe', 'pipe'] }
97
+ );
98
+ if (listed) {
99
+ let count = 0;
100
+ for (const relPath of listed.split('\n')) {
101
+ if (!relPath) continue;
102
+ if (SOURCE_EXTENSIONS.has(extname(relPath))) count++;
103
+ }
104
+ if (count > 0) return count;
105
+ }
106
+ } catch (err) {
107
+ if (process.env.DEBUG_CATCHES) process.stderr.write(`[non-fatal] ${err?.message || err}\n`);
108
+ }
109
+
110
+ // Fallback: full traversal (still bounded by skip-dir rules).
111
+ const stack = [root];
112
+ let count = 0;
113
+
114
+ while (stack.length > 0) {
115
+ const dir = stack.pop();
116
+ let entries;
117
+ try {
118
+ entries = readdirSync(dir, { withFileTypes: true });
119
+ } catch (err) {
120
+ if (process.env.DEBUG_CATCHES) process.stderr.write(`[non-fatal] ${err?.message || err}\n`);
121
+ continue;
122
+ }
123
+
124
+ for (const entry of entries) {
125
+ if (entry.isDirectory()) {
126
+ if (COUNT_SKIP_DIRS.has(entry.name)) continue;
127
+ if (entry.name.startsWith('.') && dir !== root) continue;
128
+ stack.push(join(dir, entry.name));
129
+ } else if (entry.isFile()) {
130
+ if (SOURCE_EXTENSIONS.has(extname(entry.name))) count++;
131
+ }
132
+ }
133
+ }
134
+
135
+ return count;
136
+ }
137
+
138
+ // ---------------------------------------------------------------------------
139
+ // 1. Structural Mining
140
+ // ---------------------------------------------------------------------------
141
+
142
+ /**
143
+ * Mine vocabulary terms from file paths, directories, and package manifests.
144
+ *
145
+ * @param {string} [projectRoot] - Project root directory
146
+ * @returns {{ terms: Array<{term: string, score: number, source: string}> }}
147
+ */
148
+ export function mineStructural(projectRoot) {
149
+ const root = projectRoot || PROJECT_ROOT;
150
+ const terms = new Map(); // term -> { score, source }
151
+
152
+ // Mine top-level directory names
153
+ try {
154
+ const entries = readdirSync(root, { withFileTypes: true });
155
+ for (const entry of entries) {
156
+ if (entry.isDirectory() && !entry.name.startsWith('.') && entry.name !== 'node_modules') {
157
+ addTerm(terms, entry.name, 0.5, 'directory');
158
+ for (const part of splitIdentifier(entry.name)) {
159
+ if (part.length > 2 && !STOP_WORDS.has(part)) {
160
+ addTerm(terms, part, 0.3, 'directory-part');
161
+ }
162
+ }
163
+ }
164
+ }
165
+ } catch (err) {
166
+ if (process.env.DEBUG_CATCHES) process.stderr.write(`[non-fatal] ${err?.message || err}\n`);
167
+ }
168
+
169
+ // Mine source file paths (walk top 3 levels)
170
+ try {
171
+ const files = walkShallow(root, 3);
172
+ for (const filePath of files) {
173
+ const ext = extname(filePath);
174
+ if (!SOURCE_EXTENSIONS.has(ext)) continue;
175
+
176
+ const rel = relative(root, filePath);
177
+ const parts = rel.split(sep).filter(Boolean);
178
+
179
+ // File name tokens (without extension)
180
+ const fileName = basename(filePath, ext);
181
+ for (const part of splitIdentifier(fileName)) {
182
+ if (part.length > 2 && !STOP_WORDS.has(part)) {
183
+ addTerm(terms, part, 0.4, 'file-path');
184
+ }
185
+ }
186
+ // Full compound name (e.g. LoginController)
187
+ if (fileName.length > 3 && /[A-Z]/.test(fileName)) {
188
+ addTerm(terms, fileName, 0.6, 'file-name');
189
+ }
190
+
191
+ // Directory path tokens (skip root-level)
192
+ for (let i = 0; i < Math.min(parts.length - 1, 2); i++) {
193
+ for (const part of splitIdentifier(parts[i])) {
194
+ if (part.length > 2 && !STOP_WORDS.has(part)) {
195
+ addTerm(terms, part, 0.3, 'path-segment');
196
+ }
197
+ }
198
+ }
199
+ }
200
+ } catch (err) {
201
+ if (process.env.DEBUG_CATCHES) process.stderr.write(`[non-fatal] ${err?.message || err}\n`);
202
+ }
203
+
204
+ // Mine package manifests
205
+ for (const { file, extract } of MANIFEST_FILES) {
206
+ const manifestPath = join(root, file);
207
+ if (!existsSync(manifestPath)) continue;
208
+ try {
209
+ const content = readFileSync(manifestPath, 'utf-8');
210
+ const deps = extract(content);
211
+ for (const dep of deps) {
212
+ addTerm(terms, dep, 0.5, 'dependency');
213
+ // Split scoped/compound dep names
214
+ const depParts = dep.replace(/^@[^/]+\//, '').split(/[-_./]/);
215
+ for (const part of depParts) {
216
+ if (part.length > 2 && !STOP_WORDS.has(part.toLowerCase())) {
217
+ addTerm(terms, part.toLowerCase(), 0.3, 'dependency-part');
218
+ }
219
+ }
220
+ }
221
+ } catch (err) {
222
+ if (process.env.DEBUG_CATCHES) process.stderr.write(`[non-fatal] ${err?.message || err}\n`);
223
+ }
224
+ }
225
+
226
+ // Mine .env.example keys
227
+ try {
228
+ const envExPath = join(root, '.env.example');
229
+ if (existsSync(envExPath)) {
230
+ const content = readFileSync(envExPath, 'utf-8');
231
+ for (const line of content.split('\n')) {
232
+ const match = line.match(/^([A-Z][A-Z0-9_]+)\s*=/);
233
+ if (match) {
234
+ addTerm(terms, match[1], 0.3, 'env-key');
235
+ for (const part of splitIdentifier(match[1])) {
236
+ if (part.length > 2) addTerm(terms, part, 0.2, 'env-key-part');
237
+ }
238
+ }
239
+ }
240
+ }
241
+ } catch (err) {
242
+ if (process.env.DEBUG_CATCHES) process.stderr.write(`[non-fatal] ${err?.message || err}\n`);
243
+ }
244
+
245
+ return { terms: termsToArray(terms) };
246
+ }
247
+
248
+ // ---------------------------------------------------------------------------
249
+ // 2. Symbol Mining
250
+ // ---------------------------------------------------------------------------
251
+
252
+ /**
253
+ * Mine vocabulary from source code symbols: imports, exports, class/function names.
254
+ *
255
+ * @param {string} [projectRoot] - Project root directory
256
+ * @param {object} [options]
257
+ * @param {number} [options.maxFiles=500] - Max files to scan
258
+ * @param {number} [options.maxFileSize=100000] - Max bytes per file
259
+ * @returns {{ terms: Array<{term: string, score: number, source: string}> }}
260
+ */
261
+ export function mineSymbols(projectRoot, options = {}) {
262
+ const root = projectRoot || PROJECT_ROOT;
263
+ const maxFiles = options.maxFiles ?? 500;
264
+ const maxFileSize = options.maxFileSize ?? 100_000;
265
+ const terms = new Map();
266
+
267
+ let files;
268
+ try {
269
+ files = walkShallow(root, 4).filter(f => SOURCE_EXTENSIONS.has(extname(f)));
270
+ } catch (err) {
271
+ if (process.env.DEBUG_CATCHES) process.stderr.write(`[non-fatal] ${err?.message || err}\n`);
272
+ return { terms: [] };
273
+ }
274
+
275
+ // Limit file count
276
+ if (files.length > maxFiles) files = files.slice(0, maxFiles);
277
+
278
+ for (const filePath of files) {
279
+ try {
280
+ const stat = statSync(filePath);
281
+ if (stat.size > maxFileSize) continue;
282
+ } catch (err) {
283
+ if (process.env.DEBUG_CATCHES) process.stderr.write(`[non-fatal] ${err?.message || err}\n`);
284
+ continue;
285
+ }
286
+
287
+ let content;
288
+ try {
289
+ content = readFileSync(filePath, 'utf-8');
290
+ } catch (err) {
291
+ if (process.env.DEBUG_CATCHES) process.stderr.write(`[non-fatal] ${err?.message || err}\n`);
292
+ continue;
293
+ }
294
+
295
+ const ext = extname(filePath);
296
+
297
+ // Extract imports
298
+ extractImports(content, ext, terms);
299
+
300
+ // Extract exports
301
+ extractExports(content, ext, terms);
302
+
303
+ // Extract class/function/method names
304
+ extractDefinitions(content, ext, terms);
305
+
306
+ // Extract constants (SCREAMING_SNAKE_CASE)
307
+ extractConstants(content, terms);
308
+ }
309
+
310
+ return { terms: termsToArray(terms) };
311
+ }
312
+
313
+ // ---------------------------------------------------------------------------
314
+ // 3. Code Graph Mining
315
+ // ---------------------------------------------------------------------------
316
+
317
+ /**
318
+ * Mine vocabulary from the code-graph.db entity/relationship graph.
319
+ * Leverages PageRank scores to weight terms by graph importance.
320
+ *
321
+ * @param {string} [dbPath] - Path to code-graph.db
322
+ * @returns {{ terms: Array<{term: string, score: number, source: string}>, pageRankScores: Map<string, number> }}
323
+ */
324
+ export function mineCodeGraph(dbPath) {
325
+ const resolvedPath = dbPath || DB_PATHS.codeGraph;
326
+ const terms = new Map();
327
+ let pageRankScores = new Map();
328
+
329
+ let graph;
330
+ try {
331
+ graph = loadGraph(resolvedPath);
332
+ } catch (err) {
333
+ if (process.env.DEBUG_CATCHES) process.stderr.write(`[non-fatal] ${err?.message || err}\n`);
334
+ return { terms: [], pageRankScores };
335
+ }
336
+
337
+ if (graph.entities.length === 0) {
338
+ return { terms: [], pageRankScores };
339
+ }
340
+
341
+ // Run PageRank for importance scoring
342
+ const { outEdges, allNodes } = buildAdjacency(graph);
343
+ pageRankScores = pageRank(outEdges, allNodes);
344
+
345
+ // Compute in-degree for hub detection
346
+ const inDegree = new Map();
347
+ for (const rel of graph.relationships) {
348
+ if (rel.target_id) {
349
+ inDegree.set(rel.target_id, (inDegree.get(rel.target_id) || 0) + 1);
350
+ }
351
+ }
352
+
353
+ // Mine entity names weighted by PageRank
354
+ const maxPR = Math.max(...pageRankScores.values(), 0.001);
355
+
356
+ for (const ent of graph.entities) {
357
+ const pr = pageRankScores.get(ent.id) || 0;
358
+ const normalizedPR = pr / maxPR;
359
+ const inDeg = inDegree.get(ent.id) || 0;
360
+
361
+ // Base score from PageRank
362
+ let score = 0.3 + normalizedPR * 0.7; // 0.3 to 1.0 range
363
+
364
+ // Hub bonus: high in-degree entities are more important
365
+ if (inDeg > 5) score = Math.min(score * 1.3, 1.0);
366
+
367
+ // Public API boost (exported, capitalized Go, etc.)
368
+ const isPublic = ent.type === 'class' || ent.type === 'interface' ||
369
+ ent.type === 'enum' || ent.type === 'module' || ent.type === 'service';
370
+ if (isPublic) score = Math.min(score * 1.2, 1.0);
371
+
372
+ // Leaf penalty
373
+ const outDeg = (outEdges.get(ent.id)?.size) || 0;
374
+ if (inDeg === 0 && outDeg <= 1) score *= 0.5;
375
+
376
+ // Add entity name
377
+ addTerm(terms, ent.name, score, 'graph-entity');
378
+
379
+ // Add split parts
380
+ for (const part of splitIdentifier(ent.name)) {
381
+ if (part.length > 2 && !STOP_WORDS.has(part)) {
382
+ addTerm(terms, part, score * 0.6, 'graph-entity-part');
383
+ }
384
+ }
385
+ }
386
+
387
+ return { terms: termsToArray(terms), pageRankScores };
388
+ }
389
+
390
+ // ---------------------------------------------------------------------------
391
+ // 5. Git Mining
392
+ // ---------------------------------------------------------------------------
393
+
394
+ /**
395
+ * Mine vocabulary from git history: commit messages, branch names, hot files.
396
+ *
397
+ * @param {string} [projectRoot]
398
+ * @param {object} [options]
399
+ * @param {number} [options.maxCommits=200]
400
+ * @param {number} [options.maxDays=30]
401
+ * @returns {{ terms: Array<{term: string, score: number, source: string}> }}
402
+ */
403
+ export function mineGit(projectRoot, options = {}) {
404
+ const root = projectRoot || PROJECT_ROOT;
405
+ const maxCommits = options.maxCommits ?? 200;
406
+ const maxDays = options.maxDays ?? 30;
407
+ const terms = new Map();
408
+
409
+ // Commit messages
410
+ try {
411
+ const log = execFileSync(
412
+ 'git', ['log', `--format=%s`, '-n', String(maxCommits), `--since=${maxDays} days ago`],
413
+ { cwd: root, encoding: 'utf-8', timeout: 5000, stdio: ['pipe', 'pipe', 'pipe'] }
414
+ );
415
+
416
+ for (const line of log.split('\n').filter(Boolean)) {
417
+ const tokens = tokenizeNL(line);
418
+ for (const token of tokens) {
419
+ addTerm(terms, token, 0.3, 'git-commit');
420
+ }
421
+ }
422
+ } catch (err) {
423
+ if (process.env.DEBUG_CATCHES) process.stderr.write(`[non-fatal] ${err?.message || err}\n`);
424
+ }
425
+
426
+ // Branch names
427
+ try {
428
+ const branches = execFileSync(
429
+ 'git', ['branch', '--all', '--format=%(refname:short)'],
430
+ { cwd: root, encoding: 'utf-8', timeout: 3000, stdio: ['pipe', 'pipe', 'pipe'] }
431
+ );
432
+
433
+ for (const branch of branches.split('\n').filter(Boolean)) {
434
+ const name = branch.replace(/^origin\//, '');
435
+ if (name === 'main' || name === 'master' || name === 'HEAD') continue;
436
+ for (const part of splitIdentifier(name)) {
437
+ if (part.length > 2 && !STOP_WORDS.has(part)) {
438
+ addTerm(terms, part, 0.4, 'git-branch');
439
+ }
440
+ }
441
+ }
442
+ } catch (err) {
443
+ if (process.env.DEBUG_CATCHES) process.stderr.write(`[non-fatal] ${err?.message || err}\n`);
444
+ }
445
+
446
+ // Frequently changed files (hot files)
447
+ try {
448
+ const hotFiles = execFileSync(
449
+ 'git', ['log', '--format=', '--name-only', '-n', String(maxCommits), `--since=${maxDays} days ago`],
450
+ { cwd: root, encoding: 'utf-8', timeout: 5000, stdio: ['pipe', 'pipe', 'pipe'] }
451
+ );
452
+
453
+ const fileCounts = new Map();
454
+ for (const file of hotFiles.split('\n').filter(Boolean)) {
455
+ fileCounts.set(file, (fileCounts.get(file) || 0) + 1);
456
+ }
457
+
458
+ // Top 20 hottest files
459
+ const sorted = [...fileCounts.entries()].sort((a, b) => b[1] - a[1]).slice(0, 20);
460
+ const maxCount = sorted.length > 0 ? sorted[0][1] : 1;
461
+
462
+ for (const [filePath, count] of sorted) {
463
+ const score = 0.3 + 0.5 * (count / maxCount);
464
+ const name = basename(filePath, extname(filePath));
465
+ for (const part of splitIdentifier(name)) {
466
+ if (part.length > 2 && !STOP_WORDS.has(part)) {
467
+ addTerm(terms, part, score, 'git-hot-file');
468
+ }
469
+ }
470
+ }
471
+ } catch (err) {
472
+ if (process.env.DEBUG_CATCHES) process.stderr.write(`[non-fatal] ${err?.message || err}\n`);
473
+ }
474
+
475
+ return { terms: termsToArray(terms) };
476
+ }
477
+
478
+ // ---------------------------------------------------------------------------
479
+ // Unified API
480
+ // ---------------------------------------------------------------------------
481
+
482
+ /**
483
+ * Run all miners and merge results into a unified term list.
484
+ *
485
+ * @param {string} [projectRoot]
486
+ * @param {string} [dbPath]
487
+ * @param {Array} [communities] - From detectCommunities()
488
+ * @param {object} [options]
489
+ * @param {boolean} [options.deep=false] - Include git mining
490
+ * @param {boolean} [options.skipNL=false] - Skip NL mining
491
+ * @returns {{ terms: Array<{term: string, score: number, source: string}>, pageRankScores: Map, communityPhrases: Array, totalFiles: number }}
492
+ */
493
+ export function mineAll(projectRoot, dbPath, communities, options = {}) {
494
+ const root = projectRoot || PROJECT_ROOT;
495
+ const mergedTerms = new Map();
496
+ let pageRankScores = new Map();
497
+ let communityPhrases = [];
498
+ let totalFiles = 0;
499
+
500
+ // 1. Structural mining
501
+ const structural = mineStructural(root);
502
+ mergeTerms(mergedTerms, structural.terms);
503
+
504
+ // 2. Symbol mining
505
+ const symbols = mineSymbols(root, options);
506
+ mergeTerms(mergedTerms, symbols.terms);
507
+
508
+ // 3. Code graph mining
509
+ const graph = mineCodeGraph(dbPath);
510
+ mergeTerms(mergedTerms, graph.terms);
511
+ pageRankScores = graph.pageRankScores;
512
+
513
+ // Load entity names from code-graph.db so secret detection can exempt
514
+ // identifiers that are known to belong to the current code graph.
515
+ let codeGraphNames = null;
516
+ const resolvedDbPath = dbPath || DB_PATHS.codeGraph;
517
+ if (existsSync(resolvedDbPath)) {
518
+ try {
519
+ const db = new Database(resolvedDbPath, { readonly: true, timeout: 3000 });
520
+ applyReadPragmas(db);
521
+ try {
522
+ const rows = db.prepare('SELECT DISTINCT name FROM entities').all();
523
+ codeGraphNames = new Set(rows.map(r => r.name));
524
+ } finally { db.close(); }
525
+ } catch (err) {
526
+ if (process.env.DEBUG_CATCHES) process.stderr.write(`[non-fatal] ${err?.message || err}\n`);
527
+ }
528
+ }
529
+
530
+ // 4. Community NL mining
531
+ if (!options.skipNL && communities && communities.length > 0) {
532
+ const nl = mineNLContent(communities, root, { ...options, codeGraphNames });
533
+ communityPhrases = nl.communityPhrases;
534
+
535
+ // Flatten community phrases into terms
536
+ for (const cp of communityPhrases) {
537
+ for (const phrase of cp.phrases) {
538
+ addTerm(mergedTerms, phrase.text, phrase.score, phrase.type);
539
+ }
540
+ }
541
+ }
542
+
543
+ // 5. Git mining (deep mode only)
544
+ if (options.deep) {
545
+ const git = mineGit(root, options);
546
+ mergeTerms(mergedTerms, git.terms);
547
+ }
548
+
549
+ // Estimate real file count for BM25 IDF when ranking.
550
+ try {
551
+ totalFiles = estimateTotalSourceFiles(root);
552
+ } catch (err) {
553
+ if (process.env.DEBUG_CATCHES) process.stderr.write(`[non-fatal] ${err?.message || err}\n`);
554
+ totalFiles = 0;
555
+ }
556
+
557
+ return {
558
+ terms: termsToArray(mergedTerms),
559
+ pageRankScores,
560
+ communityPhrases,
561
+ totalFiles,
562
+ };
563
+ }
564
+
565
+ // Backward compatibility for existing default-import callers.
566
+ export default {
567
+ mineStructural,
568
+ mineSymbols,
569
+ mineCodeGraph,
570
+ mineNLContent,
571
+ mineGit,
572
+ mineAll,
573
+ splitIdentifier,
574
+ };
@@ -0,0 +1,110 @@
1
+ /**
2
+ * CLI handler for `sweet-search prewarm-vocab`.
3
+ *
4
+ * Pre-warms the vocabulary cache with terms for semantic search. Terms are
5
+ * sourced from, in priority order:
6
+ * 1. An explicit terms file path (first positional argument)
7
+ * 2. <PROJECT_ROOT>/.sweet-search/vocab-terms.json
8
+ * 3. A built-in generic programming term list (fallback)
9
+ *
10
+ * Usage: sweet-search prewarm-vocab [terms-file]
11
+ */
12
+
13
+ import { existsSync, readFileSync } from 'node:fs';
14
+ import { resolve } from 'node:path';
15
+ import { expandVocabulary } from '../embedding/index.js';
16
+ import { PROJECT_ROOT } from '../infrastructure/config/index.js';
17
+
18
+ const GENERIC_TERMS = [
19
+ 'authentication', 'authorization', 'login', 'logout',
20
+ 'user', 'config', 'database', 'connection',
21
+ 'request', 'response', 'handler', 'controller',
22
+ 'service', 'repository', 'model', 'entity',
23
+ 'error', 'exception', 'validation', 'middleware',
24
+ 'route', 'endpoint', 'API', 'REST',
25
+ 'test', 'mock', 'fixture', 'setup',
26
+ 'import', 'export', 'module', 'package',
27
+ 'class', 'interface', 'function', 'method',
28
+ 'constructor', 'initialize', 'factory', 'builder',
29
+ 'cache', 'queue', 'event', 'listener',
30
+ 'how does authentication work',
31
+ 'where is the database connection configured',
32
+ 'find all API endpoints',
33
+ 'error handling patterns',
34
+ ];
35
+
36
+ function loadTermsFromFile(filePath) {
37
+ const raw = readFileSync(filePath, 'utf-8');
38
+ const parsed = JSON.parse(raw);
39
+
40
+ if (!Array.isArray(parsed)) {
41
+ throw new Error(`Terms file must contain a JSON array, got ${typeof parsed}`);
42
+ }
43
+
44
+ const nonStrings = parsed.filter((t) => typeof t !== 'string');
45
+ if (nonStrings.length > 0) {
46
+ throw new Error(`Terms file contains ${nonStrings.length} non-string entries`);
47
+ }
48
+
49
+ return parsed;
50
+ }
51
+
52
+ function resolveTerms(cliArg) {
53
+ if (cliArg) {
54
+ const filePath = resolve(process.cwd(), cliArg);
55
+ if (!existsSync(filePath)) {
56
+ throw new Error(`terms file not found: ${filePath}`);
57
+ }
58
+ return { terms: loadTermsFromFile(filePath), source: `CLI argument (${filePath})` };
59
+ }
60
+
61
+ const defaultPath = resolve(PROJECT_ROOT, '.sweet-search', 'vocab-terms.json');
62
+ if (existsSync(defaultPath)) {
63
+ return { terms: loadTermsFromFile(defaultPath), source: `default file (${defaultPath})` };
64
+ }
65
+
66
+ return { terms: GENERIC_TERMS, source: 'generic fallback (built-in programming terms)' };
67
+ }
68
+
69
+ /**
70
+ * CLI entry point dispatched from `core/cli.js`.
71
+ *
72
+ * @param {string[]} args - Arguments after `prewarm-vocab`.
73
+ */
74
+ export async function handlePrewarmVocabCli(args) {
75
+ if (args.includes('--help') || args.includes('-h')) {
76
+ console.log(`sweet-search prewarm-vocab — pre-warm vocabulary for semantic search
77
+
78
+ Usage:
79
+ sweet-search prewarm-vocab [terms-file]
80
+
81
+ Arguments:
82
+ terms-file Optional path to a JSON array of terms to warm.
83
+ Defaults to .sweet-search/vocab-terms.json, then to a
84
+ built-in generic programming term list.`);
85
+ return;
86
+ }
87
+
88
+ const cliArg = args.find((a) => !a.startsWith('-'));
89
+
90
+ let terms, source;
91
+ try {
92
+ ({ terms, source } = resolveTerms(cliArg));
93
+ } catch (err) {
94
+ console.error(`Error: ${err.message}`);
95
+ process.exit(1);
96
+ }
97
+
98
+ console.log(`\nPre-warming vocabulary with ${terms.length} terms...`);
99
+ console.log(`Source: ${source}`);
100
+
101
+ const start = Date.now();
102
+
103
+ try {
104
+ await expandVocabulary(terms);
105
+ console.log(`Done in ${Date.now() - start}ms`);
106
+ } catch (err) {
107
+ console.error('Error:', err.message);
108
+ process.exit(1);
109
+ }
110
+ }