sweet-search 0.0.1 → 2.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +190 -0
- package/NOTICE +23 -0
- package/core/cli.js +51 -0
- package/core/config.js +27 -0
- package/core/embedding/embedding-cache.js +467 -0
- package/core/embedding/embedding-local-model.js +845 -0
- package/core/embedding/embedding-remote.js +492 -0
- package/core/embedding/embedding-service.js +712 -0
- package/core/embedding/embedding-telemetry.js +219 -0
- package/core/embedding/index.js +40 -0
- package/core/graph/community-detector.js +294 -0
- package/core/graph/graph-expansion.js +839 -0
- package/core/graph/graph-extractor.js +2304 -0
- package/core/graph/graph-search.js +2148 -0
- package/core/graph/hcgs-generator.js +666 -0
- package/core/graph/index.js +16 -0
- package/core/graph/leiden-algorithm.js +547 -0
- package/core/graph/relationship-resolver.js +366 -0
- package/core/graph/repo-map.js +408 -0
- package/core/graph/summary-manager.js +549 -0
- package/core/indexing/artifact-builder.js +1054 -0
- package/core/indexing/ast-chunker.js +709 -0
- package/core/indexing/chunking/chunk-builder.js +170 -0
- package/core/indexing/chunking/markdown-chunker.js +503 -0
- package/core/indexing/chunking/plaintext-chunker.js +104 -0
- package/core/indexing/dedup/dedup-phase.js +159 -0
- package/core/indexing/dedup/exemplar-selector.js +65 -0
- package/core/indexing/document-chunker.js +56 -0
- package/core/indexing/incremental-parser.js +390 -0
- package/core/indexing/incremental-tracker.js +761 -0
- package/core/indexing/index-codebase-v21.js +472 -0
- package/core/indexing/index-maintainer.mjs +1674 -0
- package/core/indexing/index.js +90 -0
- package/core/indexing/indexer-ann.js +1077 -0
- package/core/indexing/indexer-build.js +742 -0
- package/core/indexing/indexer-phases.js +800 -0
- package/core/indexing/indexer-pool.js +764 -0
- package/core/indexing/indexer-sparse-gram.js +98 -0
- package/core/indexing/indexer-utils.js +536 -0
- package/core/indexing/indexer-worker.js +148 -0
- package/core/indexing/li-skip-policy.js +225 -0
- package/core/indexing/merkle-tracker.js +244 -0
- package/core/indexing/model-pool.js +166 -0
- package/core/infrastructure/code-graph-repository.js +120 -0
- package/core/infrastructure/codebase-repository.js +131 -0
- package/core/infrastructure/config/dedup.js +54 -0
- package/core/infrastructure/config/embedding.js +298 -0
- package/core/infrastructure/config/graph.js +80 -0
- package/core/infrastructure/config/index.js +82 -0
- package/core/infrastructure/config/indexing.js +8 -0
- package/core/infrastructure/config/platform.js +254 -0
- package/core/infrastructure/config/ranking.js +221 -0
- package/core/infrastructure/config/search.js +396 -0
- package/core/infrastructure/config/translation.js +89 -0
- package/core/infrastructure/config/vector-store.js +114 -0
- package/core/infrastructure/constants.js +86 -0
- package/core/infrastructure/coreml-cascade.js +909 -0
- package/core/infrastructure/coreml-cascade.json +46 -0
- package/core/infrastructure/coreml-provider.js +81 -0
- package/core/infrastructure/db-utils.js +69 -0
- package/core/infrastructure/dedup-hashing.js +83 -0
- package/core/infrastructure/hardware-capability.js +332 -0
- package/core/infrastructure/index.js +104 -0
- package/core/infrastructure/language-patterns/maps.js +121 -0
- package/core/infrastructure/language-patterns/registry-core.js +323 -0
- package/core/infrastructure/language-patterns/registry-data-query.js +155 -0
- package/core/infrastructure/language-patterns/registry-object-oriented.js +285 -0
- package/core/infrastructure/language-patterns/registry-tooling.js +240 -0
- package/core/infrastructure/language-patterns/registry-web-style.js +143 -0
- package/core/infrastructure/language-patterns/registry.js +19 -0
- package/core/infrastructure/language-patterns.js +141 -0
- package/core/infrastructure/llm-provider.js +733 -0
- package/core/infrastructure/manifest.json +46 -0
- package/core/infrastructure/maxsim.wasm +0 -0
- package/core/infrastructure/model-fetcher.js +423 -0
- package/core/infrastructure/model-registry.js +214 -0
- package/core/infrastructure/native-inference.js +587 -0
- package/core/infrastructure/native-resolver.js +187 -0
- package/core/infrastructure/native-sparse-gram.js +257 -0
- package/core/infrastructure/native-tokenizer.js +160 -0
- package/core/infrastructure/onnx-mutex.js +45 -0
- package/core/infrastructure/onnx-session-utils.js +261 -0
- package/core/infrastructure/ort-pipeline.js +111 -0
- package/core/infrastructure/project-detector.js +102 -0
- package/core/infrastructure/quantization.js +410 -0
- package/core/infrastructure/simd-distance.js +502 -0
- package/core/infrastructure/simd-distance.wasm +0 -0
- package/core/infrastructure/tree-sitter-provider.js +665 -0
- package/core/infrastructure/webgpu-maxsim.js +222 -0
- package/core/query/index.js +35 -0
- package/core/query/intent-detector.js +201 -0
- package/core/query/intent-router.js +156 -0
- package/core/query/query-router-catboost.js +222 -0
- package/core/query/query-router-ml.js +266 -0
- package/core/query/query-router.js +213 -0
- package/core/ranking/cascaded-scorer.js +379 -0
- package/core/ranking/flashrank.js +810 -0
- package/core/ranking/index.js +49 -0
- package/core/ranking/late-interaction-index.js +2383 -0
- package/core/ranking/late-interaction-model.js +812 -0
- package/core/ranking/local-reranker.js +374 -0
- package/core/ranking/mmr.js +379 -0
- package/core/ranking/quality-scorer.js +363 -0
- package/core/search/context-expander.js +1167 -0
- package/core/search/dedup/sibling-expander.js +327 -0
- package/core/search/index.js +16 -0
- package/core/search/search-boost.js +259 -0
- package/core/search/search-cli.js +544 -0
- package/core/search/search-format.js +282 -0
- package/core/search/search-fusion.js +327 -0
- package/core/search/search-hybrid.js +204 -0
- package/core/search/search-pattern-chunks.js +337 -0
- package/core/search/search-pattern-planner.js +439 -0
- package/core/search/search-pattern-prefilter.js +412 -0
- package/core/search/search-pattern-ripgrep.js +663 -0
- package/core/search/search-pattern.js +463 -0
- package/core/search/search-postprocess.js +452 -0
- package/core/search/search-semantic.js +706 -0
- package/core/search/search-server.js +554 -0
- package/core/search/session-daemon-prewarm.mjs +164 -0
- package/core/search/session-warmup.js +595 -0
- package/core/search/sweet-search.js +632 -0
- package/core/search/warmup-metrics.js +532 -0
- package/core/start-server.js +6 -0
- package/core/training/query-router/features/extractor.js +762 -0
- package/core/training/query-router/features/multilingual-patterns.js +431 -0
- package/core/training/query-router/features/text-segmenter.js +303 -0
- package/core/training/query-router/features/unicode-utils.js +383 -0
- package/core/training/query-router/output/v45_router_d4.js +11521 -0
- package/core/training/query-router/output/v46_router_d4.js +11498 -0
- package/core/vector-store/binary-heap.js +227 -0
- package/core/vector-store/binary-hnsw-index.js +1004 -0
- package/core/vector-store/float-vector-store.js +234 -0
- package/core/vector-store/hnsw-index.js +580 -0
- package/core/vector-store/index.js +39 -0
- package/core/vector-store/seismic-index.js +498 -0
- package/core/vocabulary/index.js +84 -0
- package/core/vocabulary/vocab-constants.js +20 -0
- package/core/vocabulary/vocab-miner-extractors.js +375 -0
- package/core/vocabulary/vocab-miner-nl.js +404 -0
- package/core/vocabulary/vocab-miner-utils.js +146 -0
- package/core/vocabulary/vocab-miner.js +574 -0
- package/core/vocabulary/vocab-prewarm-cli.js +110 -0
- package/core/vocabulary/vocab-ranker.js +492 -0
- package/core/vocabulary/vocab-warmer.js +523 -0
- package/core/vocabulary/vocab-warmup-orchestrator.js +425 -0
- package/core/vocabulary/vocabulary-utils.js +704 -0
- package/crates/wasm-router/pkg/package.json +13 -0
- package/crates/wasm-router/pkg/query_router_wasm.d.ts +36 -0
- package/crates/wasm-router/pkg/query_router_wasm.js +271 -0
- package/crates/wasm-router/pkg/query_router_wasm_bg.wasm +0 -0
- package/crates/wasm-router/pkg/query_router_wasm_bg.wasm.d.ts +19 -0
- package/mcp/config-gen.js +121 -0
- package/mcp/server.js +335 -0
- package/mcp/tool-handlers.js +476 -0
- package/package.json +131 -9
- package/scripts/benchmark-harness.js +794 -0
- package/scripts/init.js +1058 -0
- package/scripts/smoke-test.js +435 -0
- package/scripts/uninstall.js +478 -0
- package/scripts/verify-runtime.js +176 -0
|
@@ -0,0 +1,574 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Vocabulary Miner Module
|
|
3
|
+
*
|
|
4
|
+
* Extracts search vocabulary terms from multiple sources:
|
|
5
|
+
* 1. Structural – file paths, directories, package manifests
|
|
6
|
+
* 2. Symbols – identifiers from source code (imports, classes, functions)
|
|
7
|
+
* 3. Code Graph – entity names and hub detection from code-graph.db
|
|
8
|
+
* 4. NL Content – community-aware TF-IDF over comments/docstrings
|
|
9
|
+
* 5. Git – commit messages, branch names, frequently changed files
|
|
10
|
+
*
|
|
11
|
+
* Each miner returns { terms: [{ term, score, source }] } or equivalent.
|
|
12
|
+
* `mineAll()` merges results from all miners into a unified term list.
|
|
13
|
+
*/
|
|
14
|
+
|
|
15
|
+
import { readFileSync, readdirSync, statSync, existsSync } from 'fs';
|
|
16
|
+
import { join, basename, extname, relative, sep } from 'path';
|
|
17
|
+
import { execFileSync } from 'child_process';
|
|
18
|
+
import Database from 'better-sqlite3';
|
|
19
|
+
import { DB_PATHS, PROJECT_ROOT } from '../infrastructure/config/index.js';
|
|
20
|
+
import { applyReadPragmas } from '../infrastructure/db-utils.js';
|
|
21
|
+
import { pageRank, loadGraph, buildAdjacency } from '../graph/repo-map.js';
|
|
22
|
+
|
|
23
|
+
// Import from sub-modules
|
|
24
|
+
import {
|
|
25
|
+
STOP_WORDS, SOURCE_EXTENSIONS,
|
|
26
|
+
splitIdentifier, addTerm, mergeTerms, termsToArray, walkShallow,
|
|
27
|
+
} from './vocab-miner-utils.js';
|
|
28
|
+
|
|
29
|
+
import {
|
|
30
|
+
extractImports, extractExports, extractDefinitions, extractConstants,
|
|
31
|
+
extractNpmDeps, extractCargoDeps, extractGoDeps, extractPipDeps, extractPyprojectDeps,
|
|
32
|
+
} from './vocab-miner-extractors.js';
|
|
33
|
+
|
|
34
|
+
import {
|
|
35
|
+
mineNLContent, computeNLContentHash, extractNLText, isSecretLike,
|
|
36
|
+
detectScript, tokenizeNL, extractBigrams, extractTrigrams,
|
|
37
|
+
SECRET_PATTERNS, NL_MINING_TIMEOUT_MS,
|
|
38
|
+
} from './vocab-miner-nl.js';
|
|
39
|
+
|
|
40
|
+
// ---------------------------------------------------------------------------
|
|
41
|
+
// Barrel Re-exports (backward compatibility layer)
|
|
42
|
+
// ---------------------------------------------------------------------------
|
|
43
|
+
// Prefer direct imports from leaf modules for new code:
|
|
44
|
+
// - ./vocab-miner-utils.js
|
|
45
|
+
// - ./vocab-miner-extractors.js
|
|
46
|
+
// - ./vocab-miner-nl.js
|
|
47
|
+
// This barrel remains to avoid breaking existing callers.
|
|
48
|
+
|
|
49
|
+
export {
|
|
50
|
+
splitIdentifier, STOP_WORDS, SOURCE_EXTENSIONS,
|
|
51
|
+
addTerm, mergeTerms, termsToArray, walkShallow,
|
|
52
|
+
} from './vocab-miner-utils.js';
|
|
53
|
+
|
|
54
|
+
export {
|
|
55
|
+
extractImports, extractExports, extractDefinitions, extractConstants,
|
|
56
|
+
extractNpmDeps, extractCargoDeps, extractGoDeps, extractPipDeps, extractPyprojectDeps,
|
|
57
|
+
} from './vocab-miner-extractors.js';
|
|
58
|
+
|
|
59
|
+
export {
|
|
60
|
+
mineNLContent, computeNLContentHash, extractNLText, isSecretLike,
|
|
61
|
+
detectScript, tokenizeNL, extractBigrams, extractTrigrams,
|
|
62
|
+
SECRET_PATTERNS, NL_MINING_TIMEOUT_MS,
|
|
63
|
+
} from './vocab-miner-nl.js';
|
|
64
|
+
|
|
65
|
+
// ---------------------------------------------------------------------------
|
|
66
|
+
// Constants
|
|
67
|
+
// ---------------------------------------------------------------------------
|
|
68
|
+
|
|
69
|
+
// Manifest files to mine
|
|
70
|
+
const MANIFEST_FILES = [
|
|
71
|
+
{ file: 'package.json', extract: extractNpmDeps },
|
|
72
|
+
{ file: 'Cargo.toml', extract: extractCargoDeps },
|
|
73
|
+
{ file: 'go.mod', extract: extractGoDeps },
|
|
74
|
+
{ file: 'requirements.txt', extract: extractPipDeps },
|
|
75
|
+
{ file: 'pyproject.toml', extract: extractPyprojectDeps },
|
|
76
|
+
];
|
|
77
|
+
|
|
78
|
+
const COUNT_SKIP_DIRS = new Set([
|
|
79
|
+
'node_modules', '.git', '.sweet-search', '.agentic-qe',
|
|
80
|
+
'dist', 'build', 'out', '.next', '.nuxt', 'coverage',
|
|
81
|
+
'__pycache__', '.venv', 'venv', 'target', '.swarm',
|
|
82
|
+
]);
|
|
83
|
+
|
|
84
|
+
/**
|
|
85
|
+
* Count source files for ranking IDF.
|
|
86
|
+
*
|
|
87
|
+
* Fast path uses `git ls-files` (cheap and accurate for tracked files).
|
|
88
|
+
* Fallback performs iterative directory traversal (avoids recursion depth limits).
|
|
89
|
+
*/
|
|
90
|
+
function estimateTotalSourceFiles(root) {
|
|
91
|
+
// Fast path: ask git for tracked files, then filter by source extensions.
|
|
92
|
+
try {
|
|
93
|
+
const listed = execFileSync(
|
|
94
|
+
'git',
|
|
95
|
+
['-C', root, 'ls-files'],
|
|
96
|
+
{ encoding: 'utf-8', timeout: 3000, stdio: ['pipe', 'pipe', 'pipe'] }
|
|
97
|
+
);
|
|
98
|
+
if (listed) {
|
|
99
|
+
let count = 0;
|
|
100
|
+
for (const relPath of listed.split('\n')) {
|
|
101
|
+
if (!relPath) continue;
|
|
102
|
+
if (SOURCE_EXTENSIONS.has(extname(relPath))) count++;
|
|
103
|
+
}
|
|
104
|
+
if (count > 0) return count;
|
|
105
|
+
}
|
|
106
|
+
} catch (err) {
|
|
107
|
+
if (process.env.DEBUG_CATCHES) process.stderr.write(`[non-fatal] ${err?.message || err}\n`);
|
|
108
|
+
}
|
|
109
|
+
|
|
110
|
+
// Fallback: full traversal (still bounded by skip-dir rules).
|
|
111
|
+
const stack = [root];
|
|
112
|
+
let count = 0;
|
|
113
|
+
|
|
114
|
+
while (stack.length > 0) {
|
|
115
|
+
const dir = stack.pop();
|
|
116
|
+
let entries;
|
|
117
|
+
try {
|
|
118
|
+
entries = readdirSync(dir, { withFileTypes: true });
|
|
119
|
+
} catch (err) {
|
|
120
|
+
if (process.env.DEBUG_CATCHES) process.stderr.write(`[non-fatal] ${err?.message || err}\n`);
|
|
121
|
+
continue;
|
|
122
|
+
}
|
|
123
|
+
|
|
124
|
+
for (const entry of entries) {
|
|
125
|
+
if (entry.isDirectory()) {
|
|
126
|
+
if (COUNT_SKIP_DIRS.has(entry.name)) continue;
|
|
127
|
+
if (entry.name.startsWith('.') && dir !== root) continue;
|
|
128
|
+
stack.push(join(dir, entry.name));
|
|
129
|
+
} else if (entry.isFile()) {
|
|
130
|
+
if (SOURCE_EXTENSIONS.has(extname(entry.name))) count++;
|
|
131
|
+
}
|
|
132
|
+
}
|
|
133
|
+
}
|
|
134
|
+
|
|
135
|
+
return count;
|
|
136
|
+
}
|
|
137
|
+
|
|
138
|
+
// ---------------------------------------------------------------------------
|
|
139
|
+
// 1. Structural Mining
|
|
140
|
+
// ---------------------------------------------------------------------------
|
|
141
|
+
|
|
142
|
+
/**
|
|
143
|
+
* Mine vocabulary terms from file paths, directories, and package manifests.
|
|
144
|
+
*
|
|
145
|
+
* @param {string} [projectRoot] - Project root directory
|
|
146
|
+
* @returns {{ terms: Array<{term: string, score: number, source: string}> }}
|
|
147
|
+
*/
|
|
148
|
+
export function mineStructural(projectRoot) {
|
|
149
|
+
const root = projectRoot || PROJECT_ROOT;
|
|
150
|
+
const terms = new Map(); // term -> { score, source }
|
|
151
|
+
|
|
152
|
+
// Mine top-level directory names
|
|
153
|
+
try {
|
|
154
|
+
const entries = readdirSync(root, { withFileTypes: true });
|
|
155
|
+
for (const entry of entries) {
|
|
156
|
+
if (entry.isDirectory() && !entry.name.startsWith('.') && entry.name !== 'node_modules') {
|
|
157
|
+
addTerm(terms, entry.name, 0.5, 'directory');
|
|
158
|
+
for (const part of splitIdentifier(entry.name)) {
|
|
159
|
+
if (part.length > 2 && !STOP_WORDS.has(part)) {
|
|
160
|
+
addTerm(terms, part, 0.3, 'directory-part');
|
|
161
|
+
}
|
|
162
|
+
}
|
|
163
|
+
}
|
|
164
|
+
}
|
|
165
|
+
} catch (err) {
|
|
166
|
+
if (process.env.DEBUG_CATCHES) process.stderr.write(`[non-fatal] ${err?.message || err}\n`);
|
|
167
|
+
}
|
|
168
|
+
|
|
169
|
+
// Mine source file paths (walk top 3 levels)
|
|
170
|
+
try {
|
|
171
|
+
const files = walkShallow(root, 3);
|
|
172
|
+
for (const filePath of files) {
|
|
173
|
+
const ext = extname(filePath);
|
|
174
|
+
if (!SOURCE_EXTENSIONS.has(ext)) continue;
|
|
175
|
+
|
|
176
|
+
const rel = relative(root, filePath);
|
|
177
|
+
const parts = rel.split(sep).filter(Boolean);
|
|
178
|
+
|
|
179
|
+
// File name tokens (without extension)
|
|
180
|
+
const fileName = basename(filePath, ext);
|
|
181
|
+
for (const part of splitIdentifier(fileName)) {
|
|
182
|
+
if (part.length > 2 && !STOP_WORDS.has(part)) {
|
|
183
|
+
addTerm(terms, part, 0.4, 'file-path');
|
|
184
|
+
}
|
|
185
|
+
}
|
|
186
|
+
// Full compound name (e.g. LoginController)
|
|
187
|
+
if (fileName.length > 3 && /[A-Z]/.test(fileName)) {
|
|
188
|
+
addTerm(terms, fileName, 0.6, 'file-name');
|
|
189
|
+
}
|
|
190
|
+
|
|
191
|
+
// Directory path tokens (skip root-level)
|
|
192
|
+
for (let i = 0; i < Math.min(parts.length - 1, 2); i++) {
|
|
193
|
+
for (const part of splitIdentifier(parts[i])) {
|
|
194
|
+
if (part.length > 2 && !STOP_WORDS.has(part)) {
|
|
195
|
+
addTerm(terms, part, 0.3, 'path-segment');
|
|
196
|
+
}
|
|
197
|
+
}
|
|
198
|
+
}
|
|
199
|
+
}
|
|
200
|
+
} catch (err) {
|
|
201
|
+
if (process.env.DEBUG_CATCHES) process.stderr.write(`[non-fatal] ${err?.message || err}\n`);
|
|
202
|
+
}
|
|
203
|
+
|
|
204
|
+
// Mine package manifests
|
|
205
|
+
for (const { file, extract } of MANIFEST_FILES) {
|
|
206
|
+
const manifestPath = join(root, file);
|
|
207
|
+
if (!existsSync(manifestPath)) continue;
|
|
208
|
+
try {
|
|
209
|
+
const content = readFileSync(manifestPath, 'utf-8');
|
|
210
|
+
const deps = extract(content);
|
|
211
|
+
for (const dep of deps) {
|
|
212
|
+
addTerm(terms, dep, 0.5, 'dependency');
|
|
213
|
+
// Split scoped/compound dep names
|
|
214
|
+
const depParts = dep.replace(/^@[^/]+\//, '').split(/[-_./]/);
|
|
215
|
+
for (const part of depParts) {
|
|
216
|
+
if (part.length > 2 && !STOP_WORDS.has(part.toLowerCase())) {
|
|
217
|
+
addTerm(terms, part.toLowerCase(), 0.3, 'dependency-part');
|
|
218
|
+
}
|
|
219
|
+
}
|
|
220
|
+
}
|
|
221
|
+
} catch (err) {
|
|
222
|
+
if (process.env.DEBUG_CATCHES) process.stderr.write(`[non-fatal] ${err?.message || err}\n`);
|
|
223
|
+
}
|
|
224
|
+
}
|
|
225
|
+
|
|
226
|
+
// Mine .env.example keys
|
|
227
|
+
try {
|
|
228
|
+
const envExPath = join(root, '.env.example');
|
|
229
|
+
if (existsSync(envExPath)) {
|
|
230
|
+
const content = readFileSync(envExPath, 'utf-8');
|
|
231
|
+
for (const line of content.split('\n')) {
|
|
232
|
+
const match = line.match(/^([A-Z][A-Z0-9_]+)\s*=/);
|
|
233
|
+
if (match) {
|
|
234
|
+
addTerm(terms, match[1], 0.3, 'env-key');
|
|
235
|
+
for (const part of splitIdentifier(match[1])) {
|
|
236
|
+
if (part.length > 2) addTerm(terms, part, 0.2, 'env-key-part');
|
|
237
|
+
}
|
|
238
|
+
}
|
|
239
|
+
}
|
|
240
|
+
}
|
|
241
|
+
} catch (err) {
|
|
242
|
+
if (process.env.DEBUG_CATCHES) process.stderr.write(`[non-fatal] ${err?.message || err}\n`);
|
|
243
|
+
}
|
|
244
|
+
|
|
245
|
+
return { terms: termsToArray(terms) };
|
|
246
|
+
}
|
|
247
|
+
|
|
248
|
+
// ---------------------------------------------------------------------------
|
|
249
|
+
// 2. Symbol Mining
|
|
250
|
+
// ---------------------------------------------------------------------------
|
|
251
|
+
|
|
252
|
+
/**
|
|
253
|
+
* Mine vocabulary from source code symbols: imports, exports, class/function names.
|
|
254
|
+
*
|
|
255
|
+
* @param {string} [projectRoot] - Project root directory
|
|
256
|
+
* @param {object} [options]
|
|
257
|
+
* @param {number} [options.maxFiles=500] - Max files to scan
|
|
258
|
+
* @param {number} [options.maxFileSize=100000] - Max bytes per file
|
|
259
|
+
* @returns {{ terms: Array<{term: string, score: number, source: string}> }}
|
|
260
|
+
*/
|
|
261
|
+
export function mineSymbols(projectRoot, options = {}) {
|
|
262
|
+
const root = projectRoot || PROJECT_ROOT;
|
|
263
|
+
const maxFiles = options.maxFiles ?? 500;
|
|
264
|
+
const maxFileSize = options.maxFileSize ?? 100_000;
|
|
265
|
+
const terms = new Map();
|
|
266
|
+
|
|
267
|
+
let files;
|
|
268
|
+
try {
|
|
269
|
+
files = walkShallow(root, 4).filter(f => SOURCE_EXTENSIONS.has(extname(f)));
|
|
270
|
+
} catch (err) {
|
|
271
|
+
if (process.env.DEBUG_CATCHES) process.stderr.write(`[non-fatal] ${err?.message || err}\n`);
|
|
272
|
+
return { terms: [] };
|
|
273
|
+
}
|
|
274
|
+
|
|
275
|
+
// Limit file count
|
|
276
|
+
if (files.length > maxFiles) files = files.slice(0, maxFiles);
|
|
277
|
+
|
|
278
|
+
for (const filePath of files) {
|
|
279
|
+
try {
|
|
280
|
+
const stat = statSync(filePath);
|
|
281
|
+
if (stat.size > maxFileSize) continue;
|
|
282
|
+
} catch (err) {
|
|
283
|
+
if (process.env.DEBUG_CATCHES) process.stderr.write(`[non-fatal] ${err?.message || err}\n`);
|
|
284
|
+
continue;
|
|
285
|
+
}
|
|
286
|
+
|
|
287
|
+
let content;
|
|
288
|
+
try {
|
|
289
|
+
content = readFileSync(filePath, 'utf-8');
|
|
290
|
+
} catch (err) {
|
|
291
|
+
if (process.env.DEBUG_CATCHES) process.stderr.write(`[non-fatal] ${err?.message || err}\n`);
|
|
292
|
+
continue;
|
|
293
|
+
}
|
|
294
|
+
|
|
295
|
+
const ext = extname(filePath);
|
|
296
|
+
|
|
297
|
+
// Extract imports
|
|
298
|
+
extractImports(content, ext, terms);
|
|
299
|
+
|
|
300
|
+
// Extract exports
|
|
301
|
+
extractExports(content, ext, terms);
|
|
302
|
+
|
|
303
|
+
// Extract class/function/method names
|
|
304
|
+
extractDefinitions(content, ext, terms);
|
|
305
|
+
|
|
306
|
+
// Extract constants (SCREAMING_SNAKE_CASE)
|
|
307
|
+
extractConstants(content, terms);
|
|
308
|
+
}
|
|
309
|
+
|
|
310
|
+
return { terms: termsToArray(terms) };
|
|
311
|
+
}
|
|
312
|
+
|
|
313
|
+
// ---------------------------------------------------------------------------
|
|
314
|
+
// 3. Code Graph Mining
|
|
315
|
+
// ---------------------------------------------------------------------------
|
|
316
|
+
|
|
317
|
+
/**
|
|
318
|
+
* Mine vocabulary from the code-graph.db entity/relationship graph.
|
|
319
|
+
* Leverages PageRank scores to weight terms by graph importance.
|
|
320
|
+
*
|
|
321
|
+
* @param {string} [dbPath] - Path to code-graph.db
|
|
322
|
+
* @returns {{ terms: Array<{term: string, score: number, source: string}>, pageRankScores: Map<string, number> }}
|
|
323
|
+
*/
|
|
324
|
+
export function mineCodeGraph(dbPath) {
|
|
325
|
+
const resolvedPath = dbPath || DB_PATHS.codeGraph;
|
|
326
|
+
const terms = new Map();
|
|
327
|
+
let pageRankScores = new Map();
|
|
328
|
+
|
|
329
|
+
let graph;
|
|
330
|
+
try {
|
|
331
|
+
graph = loadGraph(resolvedPath);
|
|
332
|
+
} catch (err) {
|
|
333
|
+
if (process.env.DEBUG_CATCHES) process.stderr.write(`[non-fatal] ${err?.message || err}\n`);
|
|
334
|
+
return { terms: [], pageRankScores };
|
|
335
|
+
}
|
|
336
|
+
|
|
337
|
+
if (graph.entities.length === 0) {
|
|
338
|
+
return { terms: [], pageRankScores };
|
|
339
|
+
}
|
|
340
|
+
|
|
341
|
+
// Run PageRank for importance scoring
|
|
342
|
+
const { outEdges, allNodes } = buildAdjacency(graph);
|
|
343
|
+
pageRankScores = pageRank(outEdges, allNodes);
|
|
344
|
+
|
|
345
|
+
// Compute in-degree for hub detection
|
|
346
|
+
const inDegree = new Map();
|
|
347
|
+
for (const rel of graph.relationships) {
|
|
348
|
+
if (rel.target_id) {
|
|
349
|
+
inDegree.set(rel.target_id, (inDegree.get(rel.target_id) || 0) + 1);
|
|
350
|
+
}
|
|
351
|
+
}
|
|
352
|
+
|
|
353
|
+
// Mine entity names weighted by PageRank
|
|
354
|
+
const maxPR = Math.max(...pageRankScores.values(), 0.001);
|
|
355
|
+
|
|
356
|
+
for (const ent of graph.entities) {
|
|
357
|
+
const pr = pageRankScores.get(ent.id) || 0;
|
|
358
|
+
const normalizedPR = pr / maxPR;
|
|
359
|
+
const inDeg = inDegree.get(ent.id) || 0;
|
|
360
|
+
|
|
361
|
+
// Base score from PageRank
|
|
362
|
+
let score = 0.3 + normalizedPR * 0.7; // 0.3 to 1.0 range
|
|
363
|
+
|
|
364
|
+
// Hub bonus: high in-degree entities are more important
|
|
365
|
+
if (inDeg > 5) score = Math.min(score * 1.3, 1.0);
|
|
366
|
+
|
|
367
|
+
// Public API boost (exported, capitalized Go, etc.)
|
|
368
|
+
const isPublic = ent.type === 'class' || ent.type === 'interface' ||
|
|
369
|
+
ent.type === 'enum' || ent.type === 'module' || ent.type === 'service';
|
|
370
|
+
if (isPublic) score = Math.min(score * 1.2, 1.0);
|
|
371
|
+
|
|
372
|
+
// Leaf penalty
|
|
373
|
+
const outDeg = (outEdges.get(ent.id)?.size) || 0;
|
|
374
|
+
if (inDeg === 0 && outDeg <= 1) score *= 0.5;
|
|
375
|
+
|
|
376
|
+
// Add entity name
|
|
377
|
+
addTerm(terms, ent.name, score, 'graph-entity');
|
|
378
|
+
|
|
379
|
+
// Add split parts
|
|
380
|
+
for (const part of splitIdentifier(ent.name)) {
|
|
381
|
+
if (part.length > 2 && !STOP_WORDS.has(part)) {
|
|
382
|
+
addTerm(terms, part, score * 0.6, 'graph-entity-part');
|
|
383
|
+
}
|
|
384
|
+
}
|
|
385
|
+
}
|
|
386
|
+
|
|
387
|
+
return { terms: termsToArray(terms), pageRankScores };
|
|
388
|
+
}
|
|
389
|
+
|
|
390
|
+
// ---------------------------------------------------------------------------
|
|
391
|
+
// 5. Git Mining
|
|
392
|
+
// ---------------------------------------------------------------------------
|
|
393
|
+
|
|
394
|
+
/**
|
|
395
|
+
* Mine vocabulary from git history: commit messages, branch names, hot files.
|
|
396
|
+
*
|
|
397
|
+
* @param {string} [projectRoot]
|
|
398
|
+
* @param {object} [options]
|
|
399
|
+
* @param {number} [options.maxCommits=200]
|
|
400
|
+
* @param {number} [options.maxDays=30]
|
|
401
|
+
* @returns {{ terms: Array<{term: string, score: number, source: string}> }}
|
|
402
|
+
*/
|
|
403
|
+
export function mineGit(projectRoot, options = {}) {
|
|
404
|
+
const root = projectRoot || PROJECT_ROOT;
|
|
405
|
+
const maxCommits = options.maxCommits ?? 200;
|
|
406
|
+
const maxDays = options.maxDays ?? 30;
|
|
407
|
+
const terms = new Map();
|
|
408
|
+
|
|
409
|
+
// Commit messages
|
|
410
|
+
try {
|
|
411
|
+
const log = execFileSync(
|
|
412
|
+
'git', ['log', `--format=%s`, '-n', String(maxCommits), `--since=${maxDays} days ago`],
|
|
413
|
+
{ cwd: root, encoding: 'utf-8', timeout: 5000, stdio: ['pipe', 'pipe', 'pipe'] }
|
|
414
|
+
);
|
|
415
|
+
|
|
416
|
+
for (const line of log.split('\n').filter(Boolean)) {
|
|
417
|
+
const tokens = tokenizeNL(line);
|
|
418
|
+
for (const token of tokens) {
|
|
419
|
+
addTerm(terms, token, 0.3, 'git-commit');
|
|
420
|
+
}
|
|
421
|
+
}
|
|
422
|
+
} catch (err) {
|
|
423
|
+
if (process.env.DEBUG_CATCHES) process.stderr.write(`[non-fatal] ${err?.message || err}\n`);
|
|
424
|
+
}
|
|
425
|
+
|
|
426
|
+
// Branch names
|
|
427
|
+
try {
|
|
428
|
+
const branches = execFileSync(
|
|
429
|
+
'git', ['branch', '--all', '--format=%(refname:short)'],
|
|
430
|
+
{ cwd: root, encoding: 'utf-8', timeout: 3000, stdio: ['pipe', 'pipe', 'pipe'] }
|
|
431
|
+
);
|
|
432
|
+
|
|
433
|
+
for (const branch of branches.split('\n').filter(Boolean)) {
|
|
434
|
+
const name = branch.replace(/^origin\//, '');
|
|
435
|
+
if (name === 'main' || name === 'master' || name === 'HEAD') continue;
|
|
436
|
+
for (const part of splitIdentifier(name)) {
|
|
437
|
+
if (part.length > 2 && !STOP_WORDS.has(part)) {
|
|
438
|
+
addTerm(terms, part, 0.4, 'git-branch');
|
|
439
|
+
}
|
|
440
|
+
}
|
|
441
|
+
}
|
|
442
|
+
} catch (err) {
|
|
443
|
+
if (process.env.DEBUG_CATCHES) process.stderr.write(`[non-fatal] ${err?.message || err}\n`);
|
|
444
|
+
}
|
|
445
|
+
|
|
446
|
+
// Frequently changed files (hot files)
|
|
447
|
+
try {
|
|
448
|
+
const hotFiles = execFileSync(
|
|
449
|
+
'git', ['log', '--format=', '--name-only', '-n', String(maxCommits), `--since=${maxDays} days ago`],
|
|
450
|
+
{ cwd: root, encoding: 'utf-8', timeout: 5000, stdio: ['pipe', 'pipe', 'pipe'] }
|
|
451
|
+
);
|
|
452
|
+
|
|
453
|
+
const fileCounts = new Map();
|
|
454
|
+
for (const file of hotFiles.split('\n').filter(Boolean)) {
|
|
455
|
+
fileCounts.set(file, (fileCounts.get(file) || 0) + 1);
|
|
456
|
+
}
|
|
457
|
+
|
|
458
|
+
// Top 20 hottest files
|
|
459
|
+
const sorted = [...fileCounts.entries()].sort((a, b) => b[1] - a[1]).slice(0, 20);
|
|
460
|
+
const maxCount = sorted.length > 0 ? sorted[0][1] : 1;
|
|
461
|
+
|
|
462
|
+
for (const [filePath, count] of sorted) {
|
|
463
|
+
const score = 0.3 + 0.5 * (count / maxCount);
|
|
464
|
+
const name = basename(filePath, extname(filePath));
|
|
465
|
+
for (const part of splitIdentifier(name)) {
|
|
466
|
+
if (part.length > 2 && !STOP_WORDS.has(part)) {
|
|
467
|
+
addTerm(terms, part, score, 'git-hot-file');
|
|
468
|
+
}
|
|
469
|
+
}
|
|
470
|
+
}
|
|
471
|
+
} catch (err) {
|
|
472
|
+
if (process.env.DEBUG_CATCHES) process.stderr.write(`[non-fatal] ${err?.message || err}\n`);
|
|
473
|
+
}
|
|
474
|
+
|
|
475
|
+
return { terms: termsToArray(terms) };
|
|
476
|
+
}
|
|
477
|
+
|
|
478
|
+
// ---------------------------------------------------------------------------
|
|
479
|
+
// Unified API
|
|
480
|
+
// ---------------------------------------------------------------------------
|
|
481
|
+
|
|
482
|
+
/**
|
|
483
|
+
* Run all miners and merge results into a unified term list.
|
|
484
|
+
*
|
|
485
|
+
* @param {string} [projectRoot]
|
|
486
|
+
* @param {string} [dbPath]
|
|
487
|
+
* @param {Array} [communities] - From detectCommunities()
|
|
488
|
+
* @param {object} [options]
|
|
489
|
+
* @param {boolean} [options.deep=false] - Include git mining
|
|
490
|
+
* @param {boolean} [options.skipNL=false] - Skip NL mining
|
|
491
|
+
* @returns {{ terms: Array<{term: string, score: number, source: string}>, pageRankScores: Map, communityPhrases: Array, totalFiles: number }}
|
|
492
|
+
*/
|
|
493
|
+
export function mineAll(projectRoot, dbPath, communities, options = {}) {
|
|
494
|
+
const root = projectRoot || PROJECT_ROOT;
|
|
495
|
+
const mergedTerms = new Map();
|
|
496
|
+
let pageRankScores = new Map();
|
|
497
|
+
let communityPhrases = [];
|
|
498
|
+
let totalFiles = 0;
|
|
499
|
+
|
|
500
|
+
// 1. Structural mining
|
|
501
|
+
const structural = mineStructural(root);
|
|
502
|
+
mergeTerms(mergedTerms, structural.terms);
|
|
503
|
+
|
|
504
|
+
// 2. Symbol mining
|
|
505
|
+
const symbols = mineSymbols(root, options);
|
|
506
|
+
mergeTerms(mergedTerms, symbols.terms);
|
|
507
|
+
|
|
508
|
+
// 3. Code graph mining
|
|
509
|
+
const graph = mineCodeGraph(dbPath);
|
|
510
|
+
mergeTerms(mergedTerms, graph.terms);
|
|
511
|
+
pageRankScores = graph.pageRankScores;
|
|
512
|
+
|
|
513
|
+
// Load entity names from code-graph.db so secret detection can exempt
|
|
514
|
+
// identifiers that are known to belong to the current code graph.
|
|
515
|
+
let codeGraphNames = null;
|
|
516
|
+
const resolvedDbPath = dbPath || DB_PATHS.codeGraph;
|
|
517
|
+
if (existsSync(resolvedDbPath)) {
|
|
518
|
+
try {
|
|
519
|
+
const db = new Database(resolvedDbPath, { readonly: true, timeout: 3000 });
|
|
520
|
+
applyReadPragmas(db);
|
|
521
|
+
try {
|
|
522
|
+
const rows = db.prepare('SELECT DISTINCT name FROM entities').all();
|
|
523
|
+
codeGraphNames = new Set(rows.map(r => r.name));
|
|
524
|
+
} finally { db.close(); }
|
|
525
|
+
} catch (err) {
|
|
526
|
+
if (process.env.DEBUG_CATCHES) process.stderr.write(`[non-fatal] ${err?.message || err}\n`);
|
|
527
|
+
}
|
|
528
|
+
}
|
|
529
|
+
|
|
530
|
+
// 4. Community NL mining
|
|
531
|
+
if (!options.skipNL && communities && communities.length > 0) {
|
|
532
|
+
const nl = mineNLContent(communities, root, { ...options, codeGraphNames });
|
|
533
|
+
communityPhrases = nl.communityPhrases;
|
|
534
|
+
|
|
535
|
+
// Flatten community phrases into terms
|
|
536
|
+
for (const cp of communityPhrases) {
|
|
537
|
+
for (const phrase of cp.phrases) {
|
|
538
|
+
addTerm(mergedTerms, phrase.text, phrase.score, phrase.type);
|
|
539
|
+
}
|
|
540
|
+
}
|
|
541
|
+
}
|
|
542
|
+
|
|
543
|
+
// 5. Git mining (deep mode only)
|
|
544
|
+
if (options.deep) {
|
|
545
|
+
const git = mineGit(root, options);
|
|
546
|
+
mergeTerms(mergedTerms, git.terms);
|
|
547
|
+
}
|
|
548
|
+
|
|
549
|
+
// Estimate real file count for BM25 IDF when ranking.
|
|
550
|
+
try {
|
|
551
|
+
totalFiles = estimateTotalSourceFiles(root);
|
|
552
|
+
} catch (err) {
|
|
553
|
+
if (process.env.DEBUG_CATCHES) process.stderr.write(`[non-fatal] ${err?.message || err}\n`);
|
|
554
|
+
totalFiles = 0;
|
|
555
|
+
}
|
|
556
|
+
|
|
557
|
+
return {
|
|
558
|
+
terms: termsToArray(mergedTerms),
|
|
559
|
+
pageRankScores,
|
|
560
|
+
communityPhrases,
|
|
561
|
+
totalFiles,
|
|
562
|
+
};
|
|
563
|
+
}
|
|
564
|
+
|
|
565
|
+
// Backward compatibility for existing default-import callers.
|
|
566
|
+
export default {
|
|
567
|
+
mineStructural,
|
|
568
|
+
mineSymbols,
|
|
569
|
+
mineCodeGraph,
|
|
570
|
+
mineNLContent,
|
|
571
|
+
mineGit,
|
|
572
|
+
mineAll,
|
|
573
|
+
splitIdentifier,
|
|
574
|
+
};
|
|
@@ -0,0 +1,110 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* CLI handler for `sweet-search prewarm-vocab`.
|
|
3
|
+
*
|
|
4
|
+
* Pre-warms the vocabulary cache with terms for semantic search. Terms are
|
|
5
|
+
* sourced from, in priority order:
|
|
6
|
+
* 1. An explicit terms file path (first positional argument)
|
|
7
|
+
* 2. <PROJECT_ROOT>/.sweet-search/vocab-terms.json
|
|
8
|
+
* 3. A built-in generic programming term list (fallback)
|
|
9
|
+
*
|
|
10
|
+
* Usage: sweet-search prewarm-vocab [terms-file]
|
|
11
|
+
*/
|
|
12
|
+
|
|
13
|
+
import { existsSync, readFileSync } from 'node:fs';
|
|
14
|
+
import { resolve } from 'node:path';
|
|
15
|
+
import { expandVocabulary } from '../embedding/index.js';
|
|
16
|
+
import { PROJECT_ROOT } from '../infrastructure/config/index.js';
|
|
17
|
+
|
|
18
|
+
const GENERIC_TERMS = [
|
|
19
|
+
'authentication', 'authorization', 'login', 'logout',
|
|
20
|
+
'user', 'config', 'database', 'connection',
|
|
21
|
+
'request', 'response', 'handler', 'controller',
|
|
22
|
+
'service', 'repository', 'model', 'entity',
|
|
23
|
+
'error', 'exception', 'validation', 'middleware',
|
|
24
|
+
'route', 'endpoint', 'API', 'REST',
|
|
25
|
+
'test', 'mock', 'fixture', 'setup',
|
|
26
|
+
'import', 'export', 'module', 'package',
|
|
27
|
+
'class', 'interface', 'function', 'method',
|
|
28
|
+
'constructor', 'initialize', 'factory', 'builder',
|
|
29
|
+
'cache', 'queue', 'event', 'listener',
|
|
30
|
+
'how does authentication work',
|
|
31
|
+
'where is the database connection configured',
|
|
32
|
+
'find all API endpoints',
|
|
33
|
+
'error handling patterns',
|
|
34
|
+
];
|
|
35
|
+
|
|
36
|
+
function loadTermsFromFile(filePath) {
|
|
37
|
+
const raw = readFileSync(filePath, 'utf-8');
|
|
38
|
+
const parsed = JSON.parse(raw);
|
|
39
|
+
|
|
40
|
+
if (!Array.isArray(parsed)) {
|
|
41
|
+
throw new Error(`Terms file must contain a JSON array, got ${typeof parsed}`);
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
const nonStrings = parsed.filter((t) => typeof t !== 'string');
|
|
45
|
+
if (nonStrings.length > 0) {
|
|
46
|
+
throw new Error(`Terms file contains ${nonStrings.length} non-string entries`);
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
return parsed;
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
function resolveTerms(cliArg) {
|
|
53
|
+
if (cliArg) {
|
|
54
|
+
const filePath = resolve(process.cwd(), cliArg);
|
|
55
|
+
if (!existsSync(filePath)) {
|
|
56
|
+
throw new Error(`terms file not found: ${filePath}`);
|
|
57
|
+
}
|
|
58
|
+
return { terms: loadTermsFromFile(filePath), source: `CLI argument (${filePath})` };
|
|
59
|
+
}
|
|
60
|
+
|
|
61
|
+
const defaultPath = resolve(PROJECT_ROOT, '.sweet-search', 'vocab-terms.json');
|
|
62
|
+
if (existsSync(defaultPath)) {
|
|
63
|
+
return { terms: loadTermsFromFile(defaultPath), source: `default file (${defaultPath})` };
|
|
64
|
+
}
|
|
65
|
+
|
|
66
|
+
return { terms: GENERIC_TERMS, source: 'generic fallback (built-in programming terms)' };
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
/**
|
|
70
|
+
* CLI entry point dispatched from `core/cli.js`.
|
|
71
|
+
*
|
|
72
|
+
* @param {string[]} args - Arguments after `prewarm-vocab`.
|
|
73
|
+
*/
|
|
74
|
+
export async function handlePrewarmVocabCli(args) {
|
|
75
|
+
if (args.includes('--help') || args.includes('-h')) {
|
|
76
|
+
console.log(`sweet-search prewarm-vocab — pre-warm vocabulary for semantic search
|
|
77
|
+
|
|
78
|
+
Usage:
|
|
79
|
+
sweet-search prewarm-vocab [terms-file]
|
|
80
|
+
|
|
81
|
+
Arguments:
|
|
82
|
+
terms-file Optional path to a JSON array of terms to warm.
|
|
83
|
+
Defaults to .sweet-search/vocab-terms.json, then to a
|
|
84
|
+
built-in generic programming term list.`);
|
|
85
|
+
return;
|
|
86
|
+
}
|
|
87
|
+
|
|
88
|
+
const cliArg = args.find((a) => !a.startsWith('-'));
|
|
89
|
+
|
|
90
|
+
let terms, source;
|
|
91
|
+
try {
|
|
92
|
+
({ terms, source } = resolveTerms(cliArg));
|
|
93
|
+
} catch (err) {
|
|
94
|
+
console.error(`Error: ${err.message}`);
|
|
95
|
+
process.exit(1);
|
|
96
|
+
}
|
|
97
|
+
|
|
98
|
+
console.log(`\nPre-warming vocabulary with ${terms.length} terms...`);
|
|
99
|
+
console.log(`Source: ${source}`);
|
|
100
|
+
|
|
101
|
+
const start = Date.now();
|
|
102
|
+
|
|
103
|
+
try {
|
|
104
|
+
await expandVocabulary(terms);
|
|
105
|
+
console.log(`Done in ${Date.now() - start}ms`);
|
|
106
|
+
} catch (err) {
|
|
107
|
+
console.error('Error:', err.message);
|
|
108
|
+
process.exit(1);
|
|
109
|
+
}
|
|
110
|
+
}
|