raggrep 0.3.0 → 0.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/app/indexer/index.d.ts +30 -4
- package/dist/cli/main.js +286 -108
- package/dist/cli/main.js.map +8 -7
- package/dist/domain/ports/embedding.d.ts +4 -1
- package/dist/domain/ports/index.d.ts +2 -1
- package/dist/domain/ports/logger.d.ts +66 -0
- package/dist/index.d.ts +45 -8
- package/dist/index.js +267 -82
- package/dist/index.js.map +8 -7
- package/dist/infrastructure/index.d.ts +1 -0
- package/dist/infrastructure/logger/index.d.ts +6 -0
- package/dist/infrastructure/logger/loggers.d.ts +75 -0
- package/dist/modules/language/typescript/index.d.ts +1 -0
- package/package.json +1 -1
package/dist/cli/main.js
CHANGED
|
@@ -1,5 +1,21 @@
|
|
|
1
1
|
#!/usr/bin/env node
|
|
2
|
+
import { createRequire } from "node:module";
|
|
3
|
+
var __create = Object.create;
|
|
4
|
+
var __getProtoOf = Object.getPrototypeOf;
|
|
2
5
|
var __defProp = Object.defineProperty;
|
|
6
|
+
var __getOwnPropNames = Object.getOwnPropertyNames;
|
|
7
|
+
var __hasOwnProp = Object.prototype.hasOwnProperty;
|
|
8
|
+
var __toESM = (mod, isNodeMode, target) => {
|
|
9
|
+
target = mod != null ? __create(__getProtoOf(mod)) : {};
|
|
10
|
+
const to = isNodeMode || !mod || !mod.__esModule ? __defProp(target, "default", { value: mod, enumerable: true }) : target;
|
|
11
|
+
for (let key of __getOwnPropNames(mod))
|
|
12
|
+
if (!__hasOwnProp.call(to, key))
|
|
13
|
+
__defProp(to, key, {
|
|
14
|
+
get: () => mod[key],
|
|
15
|
+
enumerable: true
|
|
16
|
+
});
|
|
17
|
+
return to;
|
|
18
|
+
};
|
|
3
19
|
var __export = (target, all) => {
|
|
4
20
|
for (var name in all)
|
|
5
21
|
__defProp(target, name, {
|
|
@@ -10,6 +26,7 @@ var __export = (target, all) => {
|
|
|
10
26
|
});
|
|
11
27
|
};
|
|
12
28
|
var __esm = (fn, res) => () => (fn && (res = fn(fn = 0)), res);
|
|
29
|
+
var __require = /* @__PURE__ */ createRequire(import.meta.url);
|
|
13
30
|
|
|
14
31
|
// src/infrastructure/embeddings/transformersEmbedding.ts
|
|
15
32
|
import {
|
|
@@ -27,7 +44,8 @@ class TransformersEmbeddingProvider {
|
|
|
27
44
|
constructor(config) {
|
|
28
45
|
this.config = {
|
|
29
46
|
model: config?.model ?? "all-MiniLM-L6-v2",
|
|
30
|
-
showProgress: config?.showProgress ?? false
|
|
47
|
+
showProgress: config?.showProgress ?? false,
|
|
48
|
+
logger: config?.logger
|
|
31
49
|
};
|
|
32
50
|
}
|
|
33
51
|
async initialize(config) {
|
|
@@ -49,29 +67,55 @@ class TransformersEmbeddingProvider {
|
|
|
49
67
|
this.isInitializing = true;
|
|
50
68
|
this.initPromise = (async () => {
|
|
51
69
|
const modelId = EMBEDDING_MODELS[this.config.model];
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
}
|
|
70
|
+
const logger = this.config.logger;
|
|
71
|
+
const showProgress = this.config.showProgress || !!logger;
|
|
72
|
+
const isCached = await isModelCached(this.config.model);
|
|
73
|
+
let hasDownloads = false;
|
|
57
74
|
try {
|
|
58
75
|
this.pipeline = await pipeline("feature-extraction", modelId, {
|
|
59
|
-
progress_callback:
|
|
76
|
+
progress_callback: showProgress && !isCached ? (progress) => {
|
|
60
77
|
if (progress.status === "progress" && progress.file) {
|
|
78
|
+
if (!hasDownloads) {
|
|
79
|
+
hasDownloads = true;
|
|
80
|
+
if (logger) {
|
|
81
|
+
logger.info(`Downloading embedding model: ${this.config.model}`);
|
|
82
|
+
} else {
|
|
83
|
+
console.log(`
|
|
84
|
+
Loading embedding model: ${this.config.model}`);
|
|
85
|
+
console.log(` Cache: ${CACHE_DIR}`);
|
|
86
|
+
}
|
|
87
|
+
}
|
|
61
88
|
const pct = progress.progress ? Math.round(progress.progress) : 0;
|
|
62
|
-
|
|
89
|
+
if (logger) {
|
|
90
|
+
logger.progress(` Downloading ${progress.file}: ${pct}%`);
|
|
91
|
+
} else {
|
|
92
|
+
process.stdout.write(`\r Downloading ${progress.file}: ${pct}% `);
|
|
93
|
+
}
|
|
63
94
|
} else if (progress.status === "done" && progress.file) {
|
|
64
|
-
|
|
95
|
+
if (logger) {
|
|
96
|
+
logger.clearProgress();
|
|
97
|
+
logger.info(` Downloaded ${progress.file}`);
|
|
98
|
+
} else if (hasDownloads) {
|
|
99
|
+
process.stdout.write(`\r Downloaded ${progress.file}
|
|
65
100
|
`);
|
|
101
|
+
}
|
|
66
102
|
}
|
|
67
103
|
} : undefined
|
|
68
104
|
});
|
|
69
|
-
if (
|
|
70
|
-
|
|
105
|
+
if (hasDownloads) {
|
|
106
|
+
if (logger) {
|
|
107
|
+
logger.clearProgress();
|
|
108
|
+
logger.info(`Model ready: ${this.config.model}`);
|
|
109
|
+
} else {
|
|
110
|
+
console.log(` Model ready.
|
|
71
111
|
`);
|
|
112
|
+
}
|
|
72
113
|
}
|
|
73
114
|
} catch (error) {
|
|
74
115
|
this.pipeline = null;
|
|
116
|
+
if (logger) {
|
|
117
|
+
logger.clearProgress();
|
|
118
|
+
}
|
|
75
119
|
throw new Error(`Failed to load embedding model: ${error}`);
|
|
76
120
|
} finally {
|
|
77
121
|
this.isInitializing = false;
|
|
@@ -125,9 +169,21 @@ class TransformersEmbeddingProvider {
|
|
|
125
169
|
function getCacheDir() {
|
|
126
170
|
return CACHE_DIR;
|
|
127
171
|
}
|
|
172
|
+
async function isModelCached(model) {
|
|
173
|
+
const modelId = EMBEDDING_MODELS[model];
|
|
174
|
+
const modelPath = path.join(CACHE_DIR, modelId);
|
|
175
|
+
try {
|
|
176
|
+
const fs = await import("fs/promises");
|
|
177
|
+
const onnxPath = path.join(modelPath, "onnx", "model_quantized.onnx");
|
|
178
|
+
await fs.access(onnxPath);
|
|
179
|
+
return true;
|
|
180
|
+
} catch {
|
|
181
|
+
return false;
|
|
182
|
+
}
|
|
183
|
+
}
|
|
128
184
|
function configureEmbeddings(config) {
|
|
129
185
|
const newConfig = { ...globalConfig, ...config };
|
|
130
|
-
if (newConfig.model !== globalConfig.model) {
|
|
186
|
+
if (newConfig.model !== globalConfig.model || newConfig.logger !== globalConfig.logger) {
|
|
131
187
|
globalProvider = null;
|
|
132
188
|
}
|
|
133
189
|
globalConfig = newConfig;
|
|
@@ -163,7 +219,8 @@ var init_transformersEmbedding = __esm(() => {
|
|
|
163
219
|
};
|
|
164
220
|
globalConfig = {
|
|
165
221
|
model: "all-MiniLM-L6-v2",
|
|
166
|
-
showProgress: false
|
|
222
|
+
showProgress: false,
|
|
223
|
+
logger: undefined
|
|
167
224
|
};
|
|
168
225
|
});
|
|
169
226
|
|
|
@@ -171,6 +228,96 @@ var init_transformersEmbedding = __esm(() => {
|
|
|
171
228
|
var init_embeddings = __esm(() => {
|
|
172
229
|
init_transformersEmbedding();
|
|
173
230
|
});
|
|
231
|
+
|
|
232
|
+
// src/infrastructure/logger/loggers.ts
|
|
233
|
+
class ConsoleLogger {
|
|
234
|
+
verbose;
|
|
235
|
+
constructor(options) {
|
|
236
|
+
this.verbose = options?.verbose ?? false;
|
|
237
|
+
}
|
|
238
|
+
info(message) {
|
|
239
|
+
console.log(message);
|
|
240
|
+
}
|
|
241
|
+
warn(message) {
|
|
242
|
+
console.warn(message);
|
|
243
|
+
}
|
|
244
|
+
error(message) {
|
|
245
|
+
console.error(message);
|
|
246
|
+
}
|
|
247
|
+
debug(message) {
|
|
248
|
+
if (this.verbose) {
|
|
249
|
+
console.log(message);
|
|
250
|
+
}
|
|
251
|
+
}
|
|
252
|
+
progress(message) {
|
|
253
|
+
console.log(message);
|
|
254
|
+
}
|
|
255
|
+
clearProgress() {}
|
|
256
|
+
}
|
|
257
|
+
|
|
258
|
+
class InlineProgressLogger {
|
|
259
|
+
verbose;
|
|
260
|
+
lastProgressLength = 0;
|
|
261
|
+
hasProgress = false;
|
|
262
|
+
constructor(options) {
|
|
263
|
+
this.verbose = options?.verbose ?? false;
|
|
264
|
+
}
|
|
265
|
+
info(message) {
|
|
266
|
+
this.clearProgress();
|
|
267
|
+
console.log(message);
|
|
268
|
+
}
|
|
269
|
+
warn(message) {
|
|
270
|
+
this.clearProgress();
|
|
271
|
+
console.warn(message);
|
|
272
|
+
}
|
|
273
|
+
error(message) {
|
|
274
|
+
this.clearProgress();
|
|
275
|
+
console.error(message);
|
|
276
|
+
}
|
|
277
|
+
debug(message) {
|
|
278
|
+
if (this.verbose) {
|
|
279
|
+
this.clearProgress();
|
|
280
|
+
console.log(message);
|
|
281
|
+
}
|
|
282
|
+
}
|
|
283
|
+
progress(message) {
|
|
284
|
+
process.stdout.write(`\r${message}`);
|
|
285
|
+
const padding = Math.max(0, this.lastProgressLength - message.length);
|
|
286
|
+
if (padding > 0) {
|
|
287
|
+
process.stdout.write(" ".repeat(padding));
|
|
288
|
+
}
|
|
289
|
+
this.lastProgressLength = message.length;
|
|
290
|
+
this.hasProgress = true;
|
|
291
|
+
}
|
|
292
|
+
clearProgress() {
|
|
293
|
+
if (this.hasProgress && this.lastProgressLength > 0) {
|
|
294
|
+
process.stdout.write("\r" + " ".repeat(this.lastProgressLength) + "\r");
|
|
295
|
+
this.lastProgressLength = 0;
|
|
296
|
+
this.hasProgress = false;
|
|
297
|
+
}
|
|
298
|
+
}
|
|
299
|
+
}
|
|
300
|
+
|
|
301
|
+
class SilentLogger {
|
|
302
|
+
info() {}
|
|
303
|
+
warn() {}
|
|
304
|
+
error() {}
|
|
305
|
+
debug() {}
|
|
306
|
+
progress() {}
|
|
307
|
+
clearProgress() {}
|
|
308
|
+
}
|
|
309
|
+
function createLogger(options) {
|
|
310
|
+
return new ConsoleLogger(options);
|
|
311
|
+
}
|
|
312
|
+
function createInlineLogger(options) {
|
|
313
|
+
return new InlineProgressLogger(options);
|
|
314
|
+
}
|
|
315
|
+
function createSilentLogger() {
|
|
316
|
+
return new SilentLogger;
|
|
317
|
+
}
|
|
318
|
+
|
|
319
|
+
// src/infrastructure/logger/index.ts
|
|
320
|
+
var init_logger = () => {};
|
|
174
321
|
// src/domain/entities/searchResult.ts
|
|
175
322
|
var DEFAULT_SEARCH_OPTIONS;
|
|
176
323
|
var init_searchResult = __esm(() => {
|
|
@@ -2769,8 +2916,16 @@ class TypeScriptModule {
|
|
|
2769
2916
|
symbolicIndex = null;
|
|
2770
2917
|
pendingSummaries = new Map;
|
|
2771
2918
|
rootDir = "";
|
|
2919
|
+
logger = undefined;
|
|
2772
2920
|
async initialize(config) {
|
|
2773
2921
|
this.embeddingConfig = getEmbeddingConfigFromModule(config);
|
|
2922
|
+
this.logger = config.options?.logger;
|
|
2923
|
+
if (this.logger) {
|
|
2924
|
+
this.embeddingConfig = {
|
|
2925
|
+
...this.embeddingConfig,
|
|
2926
|
+
logger: this.logger
|
|
2927
|
+
};
|
|
2928
|
+
}
|
|
2774
2929
|
configureEmbeddings(this.embeddingConfig);
|
|
2775
2930
|
this.pendingSummaries.clear();
|
|
2776
2931
|
}
|
|
@@ -3502,6 +3657,7 @@ var init_watcher = __esm(() => {
|
|
|
3502
3657
|
var exports_indexer = {};
|
|
3503
3658
|
__export(exports_indexer, {
|
|
3504
3659
|
watchDirectory: () => watchDirectory,
|
|
3660
|
+
resetIndex: () => resetIndex,
|
|
3505
3661
|
indexDirectory: () => indexDirectory,
|
|
3506
3662
|
getIndexStatus: () => getIndexStatus,
|
|
3507
3663
|
ensureIndexFresh: () => ensureIndexFresh,
|
|
@@ -3513,42 +3669,31 @@ import * as path12 from "path";
|
|
|
3513
3669
|
async function indexDirectory(rootDir, options = {}) {
|
|
3514
3670
|
const verbose = options.verbose ?? false;
|
|
3515
3671
|
const quiet = options.quiet ?? false;
|
|
3672
|
+
const logger = options.logger ? options.logger : quiet ? createSilentLogger() : createLogger({ verbose });
|
|
3516
3673
|
rootDir = path12.resolve(rootDir);
|
|
3517
3674
|
const location = getIndexLocation(rootDir);
|
|
3518
|
-
|
|
3519
|
-
|
|
3520
|
-
console.log(`Index location: ${location.indexDir}`);
|
|
3521
|
-
}
|
|
3675
|
+
logger.info(`Indexing directory: ${rootDir}`);
|
|
3676
|
+
logger.info(`Index location: ${location.indexDir}`);
|
|
3522
3677
|
const config = await loadConfig(rootDir);
|
|
3523
3678
|
const introspection = new IntrospectionIndex(rootDir);
|
|
3524
3679
|
await introspection.initialize();
|
|
3525
|
-
|
|
3526
|
-
|
|
3527
|
-
|
|
3528
|
-
console.log(`Detected monorepo with ${structure.projects.length} projects`);
|
|
3529
|
-
}
|
|
3680
|
+
const structure = introspection.getStructure();
|
|
3681
|
+
if (structure?.isMonorepo) {
|
|
3682
|
+
logger.debug(`Detected monorepo with ${structure.projects.length} projects`);
|
|
3530
3683
|
}
|
|
3531
3684
|
await registerBuiltInModules();
|
|
3532
3685
|
const enabledModules = registry.getEnabled(config);
|
|
3533
3686
|
if (enabledModules.length === 0) {
|
|
3534
|
-
|
|
3535
|
-
console.log("No modules enabled. Check your configuration.");
|
|
3536
|
-
}
|
|
3687
|
+
logger.info("No modules enabled. Check your configuration.");
|
|
3537
3688
|
return [];
|
|
3538
3689
|
}
|
|
3539
|
-
|
|
3540
|
-
console.log(`Enabled modules: ${enabledModules.map((m) => m.id).join(", ")}`);
|
|
3541
|
-
}
|
|
3690
|
+
logger.info(`Enabled modules: ${enabledModules.map((m) => m.id).join(", ")}`);
|
|
3542
3691
|
const files = await findFiles(rootDir, config);
|
|
3543
|
-
|
|
3544
|
-
console.log(`Found ${files.length} files to index`);
|
|
3545
|
-
}
|
|
3692
|
+
logger.info(`Found ${files.length} files to index`);
|
|
3546
3693
|
const results = [];
|
|
3547
3694
|
for (const module of enabledModules) {
|
|
3548
|
-
|
|
3549
|
-
console.log(`
|
|
3695
|
+
logger.info(`
|
|
3550
3696
|
[${module.name}] Starting indexing...`);
|
|
3551
|
-
}
|
|
3552
3697
|
const moduleConfig = getModuleConfig(config, module.id);
|
|
3553
3698
|
if (module.initialize && moduleConfig) {
|
|
3554
3699
|
const configWithOverrides = { ...moduleConfig };
|
|
@@ -3558,14 +3703,16 @@ async function indexDirectory(rootDir, options = {}) {
|
|
|
3558
3703
|
embeddingModel: options.model
|
|
3559
3704
|
};
|
|
3560
3705
|
}
|
|
3706
|
+
configWithOverrides.options = {
|
|
3707
|
+
...configWithOverrides.options,
|
|
3708
|
+
logger
|
|
3709
|
+
};
|
|
3561
3710
|
await module.initialize(configWithOverrides);
|
|
3562
3711
|
}
|
|
3563
|
-
const result = await indexWithModule(rootDir, files, module, config, verbose, introspection);
|
|
3712
|
+
const result = await indexWithModule(rootDir, files, module, config, verbose, introspection, logger);
|
|
3564
3713
|
results.push(result);
|
|
3565
3714
|
if (module.finalize) {
|
|
3566
|
-
|
|
3567
|
-
console.log(`[${module.name}] Building secondary indexes...`);
|
|
3568
|
-
}
|
|
3715
|
+
logger.info(`[${module.name}] Building secondary indexes...`);
|
|
3569
3716
|
const ctx = {
|
|
3570
3717
|
rootDir,
|
|
3571
3718
|
config,
|
|
@@ -3581,9 +3728,7 @@ async function indexDirectory(rootDir, options = {}) {
|
|
|
3581
3728
|
};
|
|
3582
3729
|
await module.finalize(ctx);
|
|
3583
3730
|
}
|
|
3584
|
-
|
|
3585
|
-
console.log(`[${module.name}] Complete: ${result.indexed} indexed, ${result.skipped} skipped, ${result.errors} errors`);
|
|
3586
|
-
}
|
|
3731
|
+
logger.info(`[${module.name}] Complete: ${result.indexed} indexed, ${result.skipped} skipped, ${result.errors} errors`);
|
|
3587
3732
|
}
|
|
3588
3733
|
await introspection.save(config);
|
|
3589
3734
|
await updateGlobalManifest(rootDir, enabledModules, config);
|
|
@@ -3606,28 +3751,37 @@ async function deleteIndex(rootDir) {
|
|
|
3606
3751
|
await fs6.rm(indexDir, { recursive: true, force: true });
|
|
3607
3752
|
} catch {}
|
|
3608
3753
|
}
|
|
3754
|
+
async function resetIndex(rootDir) {
|
|
3755
|
+
rootDir = path12.resolve(rootDir);
|
|
3756
|
+
const status = await getIndexStatus(rootDir);
|
|
3757
|
+
if (!status.exists) {
|
|
3758
|
+
throw new Error(`No index found for ${rootDir}`);
|
|
3759
|
+
}
|
|
3760
|
+
await deleteIndex(rootDir);
|
|
3761
|
+
return {
|
|
3762
|
+
success: true,
|
|
3763
|
+
indexDir: status.indexDir
|
|
3764
|
+
};
|
|
3765
|
+
}
|
|
3609
3766
|
async function ensureIndexFresh(rootDir, options = {}) {
|
|
3610
3767
|
const verbose = options.verbose ?? false;
|
|
3611
3768
|
const quiet = options.quiet ?? false;
|
|
3769
|
+
const logger = options.logger ? options.logger : quiet ? createSilentLogger() : createLogger({ verbose });
|
|
3612
3770
|
rootDir = path12.resolve(rootDir);
|
|
3613
3771
|
const status = await getIndexStatus(rootDir);
|
|
3614
3772
|
if (!status.exists) {
|
|
3615
|
-
|
|
3616
|
-
console.log(`No index found. Creating index...
|
|
3773
|
+
logger.info(`No index found. Creating index...
|
|
3617
3774
|
`);
|
|
3618
|
-
}
|
|
3619
|
-
const results = await indexDirectory(rootDir, { ...options, quiet });
|
|
3775
|
+
const results = await indexDirectory(rootDir, { ...options, logger });
|
|
3620
3776
|
const totalIndexed2 = results.reduce((sum, r) => sum + r.indexed, 0);
|
|
3621
3777
|
return { indexed: totalIndexed2, removed: 0, unchanged: 0 };
|
|
3622
3778
|
}
|
|
3623
3779
|
const versionCompatible = await isIndexVersionCompatible(rootDir);
|
|
3624
3780
|
if (!versionCompatible) {
|
|
3625
|
-
|
|
3626
|
-
console.log(`Index version incompatible. Rebuilding...
|
|
3781
|
+
logger.info(`Index version incompatible. Rebuilding...
|
|
3627
3782
|
`);
|
|
3628
|
-
}
|
|
3629
3783
|
await deleteIndex(rootDir);
|
|
3630
|
-
const results = await indexDirectory(rootDir, { ...options,
|
|
3784
|
+
const results = await indexDirectory(rootDir, { ...options, logger });
|
|
3631
3785
|
const totalIndexed2 = results.reduce((sum, r) => sum + r.indexed, 0);
|
|
3632
3786
|
return { indexed: totalIndexed2, removed: 0, unchanged: 0 };
|
|
3633
3787
|
}
|
|
@@ -3654,6 +3808,10 @@ async function ensureIndexFresh(rootDir, options = {}) {
|
|
|
3654
3808
|
embeddingModel: options.model
|
|
3655
3809
|
};
|
|
3656
3810
|
}
|
|
3811
|
+
configWithOverrides.options = {
|
|
3812
|
+
...configWithOverrides.options,
|
|
3813
|
+
logger
|
|
3814
|
+
};
|
|
3657
3815
|
await module.initialize(configWithOverrides);
|
|
3658
3816
|
}
|
|
3659
3817
|
const manifest = await loadModuleManifest(rootDir, module.id, config);
|
|
@@ -3665,9 +3823,7 @@ async function ensureIndexFresh(rootDir, options = {}) {
|
|
|
3665
3823
|
}
|
|
3666
3824
|
}
|
|
3667
3825
|
for (const filepath of filesToRemove) {
|
|
3668
|
-
|
|
3669
|
-
console.log(` Removing stale: ${filepath}`);
|
|
3670
|
-
}
|
|
3826
|
+
logger.debug(` Removing stale: ${filepath}`);
|
|
3671
3827
|
const indexFilePath = path12.join(indexPath, filepath.replace(/\.[^.]+$/, ".json"));
|
|
3672
3828
|
try {
|
|
3673
3829
|
await fs6.unlink(indexFilePath);
|
|
@@ -3693,8 +3849,11 @@ async function ensureIndexFresh(rootDir, options = {}) {
|
|
|
3693
3849
|
},
|
|
3694
3850
|
getIntrospection: (filepath) => introspection.getFile(filepath)
|
|
3695
3851
|
};
|
|
3696
|
-
|
|
3852
|
+
const totalFiles = currentFiles.length;
|
|
3853
|
+
for (let i = 0;i < currentFiles.length; i++) {
|
|
3854
|
+
const filepath = currentFiles[i];
|
|
3697
3855
|
const relativePath = path12.relative(rootDir, filepath);
|
|
3856
|
+
const progress = `[${i + 1}/${totalFiles}]`;
|
|
3698
3857
|
try {
|
|
3699
3858
|
const stats = await fs6.stat(filepath);
|
|
3700
3859
|
const lastModified = stats.mtime.toISOString();
|
|
@@ -3703,9 +3862,7 @@ async function ensureIndexFresh(rootDir, options = {}) {
|
|
|
3703
3862
|
totalUnchanged++;
|
|
3704
3863
|
continue;
|
|
3705
3864
|
}
|
|
3706
|
-
|
|
3707
|
-
console.log(` Indexing: ${relativePath}`);
|
|
3708
|
-
}
|
|
3865
|
+
logger.progress(` ${progress} Indexing: ${relativePath}`);
|
|
3709
3866
|
const content = await fs6.readFile(filepath, "utf-8");
|
|
3710
3867
|
introspection.addFile(relativePath, content);
|
|
3711
3868
|
const fileIndex = await module.indexFile(relativePath, content, ctx);
|
|
@@ -3718,11 +3875,11 @@ async function ensureIndexFresh(rootDir, options = {}) {
|
|
|
3718
3875
|
totalIndexed++;
|
|
3719
3876
|
}
|
|
3720
3877
|
} catch (error) {
|
|
3721
|
-
|
|
3722
|
-
|
|
3723
|
-
}
|
|
3878
|
+
logger.clearProgress();
|
|
3879
|
+
logger.error(` ${progress} Error indexing ${relativePath}: ${error}`);
|
|
3724
3880
|
}
|
|
3725
3881
|
}
|
|
3882
|
+
logger.clearProgress();
|
|
3726
3883
|
if (totalIndexed > 0 || totalRemoved > 0) {
|
|
3727
3884
|
manifest.lastUpdated = new Date().toISOString();
|
|
3728
3885
|
await writeModuleManifest(rootDir, module.id, manifest, config);
|
|
@@ -3746,7 +3903,7 @@ async function ensureIndexFresh(rootDir, options = {}) {
|
|
|
3746
3903
|
unchanged: totalUnchanged
|
|
3747
3904
|
};
|
|
3748
3905
|
}
|
|
3749
|
-
async function indexWithModule(rootDir, files, module, config, verbose, introspection) {
|
|
3906
|
+
async function indexWithModule(rootDir, files, module, config, verbose, introspection, logger) {
|
|
3750
3907
|
const result = {
|
|
3751
3908
|
moduleId: module.id,
|
|
3752
3909
|
indexed: 0,
|
|
@@ -3754,6 +3911,30 @@ async function indexWithModule(rootDir, files, module, config, verbose, introspe
|
|
|
3754
3911
|
errors: 0
|
|
3755
3912
|
};
|
|
3756
3913
|
const manifest = await loadModuleManifest(rootDir, module.id, config);
|
|
3914
|
+
const indexPath = getModuleIndexPath(rootDir, module.id, config);
|
|
3915
|
+
const currentFileSet = new Set(files.map((f) => path12.relative(rootDir, f)));
|
|
3916
|
+
const filesToRemove = [];
|
|
3917
|
+
for (const filepath of Object.keys(manifest.files)) {
|
|
3918
|
+
if (!currentFileSet.has(filepath)) {
|
|
3919
|
+
filesToRemove.push(filepath);
|
|
3920
|
+
}
|
|
3921
|
+
}
|
|
3922
|
+
if (filesToRemove.length > 0) {
|
|
3923
|
+
logger.info(` Removing ${filesToRemove.length} stale entries...`);
|
|
3924
|
+
for (const filepath of filesToRemove) {
|
|
3925
|
+
logger.debug(` Removing: ${filepath}`);
|
|
3926
|
+
const indexFilePath = path12.join(indexPath, filepath.replace(/\.[^.]+$/, ".json"));
|
|
3927
|
+
try {
|
|
3928
|
+
await fs6.unlink(indexFilePath);
|
|
3929
|
+
} catch {}
|
|
3930
|
+
const symbolicFilePath = path12.join(indexPath, "symbolic", filepath.replace(/\.[^.]+$/, ".json"));
|
|
3931
|
+
try {
|
|
3932
|
+
await fs6.unlink(symbolicFilePath);
|
|
3933
|
+
} catch {}
|
|
3934
|
+
delete manifest.files[filepath];
|
|
3935
|
+
}
|
|
3936
|
+
await cleanupEmptyDirectories(indexPath);
|
|
3937
|
+
}
|
|
3757
3938
|
const ctx = {
|
|
3758
3939
|
rootDir,
|
|
3759
3940
|
config,
|
|
@@ -3768,29 +3949,26 @@ async function indexWithModule(rootDir, files, module, config, verbose, introspe
|
|
|
3768
3949
|
},
|
|
3769
3950
|
getIntrospection: (filepath) => introspection.getFile(filepath)
|
|
3770
3951
|
};
|
|
3771
|
-
|
|
3952
|
+
const totalFiles = files.length;
|
|
3953
|
+
for (let i = 0;i < files.length; i++) {
|
|
3954
|
+
const filepath = files[i];
|
|
3772
3955
|
const relativePath = path12.relative(rootDir, filepath);
|
|
3956
|
+
const progress = `[${i + 1}/${totalFiles}]`;
|
|
3773
3957
|
try {
|
|
3774
3958
|
const stats = await fs6.stat(filepath);
|
|
3775
3959
|
const lastModified = stats.mtime.toISOString();
|
|
3776
3960
|
const existingEntry = manifest.files[relativePath];
|
|
3777
3961
|
if (existingEntry && existingEntry.lastModified === lastModified) {
|
|
3778
|
-
|
|
3779
|
-
console.log(` Skipped ${relativePath} (unchanged)`);
|
|
3780
|
-
}
|
|
3962
|
+
logger.debug(` ${progress} Skipped ${relativePath} (unchanged)`);
|
|
3781
3963
|
result.skipped++;
|
|
3782
3964
|
continue;
|
|
3783
3965
|
}
|
|
3784
3966
|
const content = await fs6.readFile(filepath, "utf-8");
|
|
3785
3967
|
introspection.addFile(relativePath, content);
|
|
3786
|
-
|
|
3787
|
-
console.log(` Processing ${relativePath}...`);
|
|
3788
|
-
}
|
|
3968
|
+
logger.progress(` ${progress} Processing: ${relativePath}`);
|
|
3789
3969
|
const fileIndex = await module.indexFile(relativePath, content, ctx);
|
|
3790
3970
|
if (!fileIndex) {
|
|
3791
|
-
|
|
3792
|
-
console.log(` Skipped ${relativePath} (no chunks)`);
|
|
3793
|
-
}
|
|
3971
|
+
logger.debug(` ${progress} Skipped ${relativePath} (no chunks)`);
|
|
3794
3972
|
result.skipped++;
|
|
3795
3973
|
continue;
|
|
3796
3974
|
}
|
|
@@ -3801,10 +3979,12 @@ async function indexWithModule(rootDir, files, module, config, verbose, introspe
|
|
|
3801
3979
|
};
|
|
3802
3980
|
result.indexed++;
|
|
3803
3981
|
} catch (error) {
|
|
3804
|
-
|
|
3982
|
+
logger.clearProgress();
|
|
3983
|
+
logger.error(` ${progress} Error indexing ${relativePath}: ${error}`);
|
|
3805
3984
|
result.errors++;
|
|
3806
3985
|
}
|
|
3807
3986
|
}
|
|
3987
|
+
logger.clearProgress();
|
|
3808
3988
|
manifest.lastUpdated = new Date().toISOString();
|
|
3809
3989
|
await writeModuleManifest(rootDir, module.id, manifest, config);
|
|
3810
3990
|
return result;
|
|
@@ -3860,26 +4040,27 @@ async function updateGlobalManifest(rootDir, modules, config) {
|
|
|
3860
4040
|
}
|
|
3861
4041
|
async function cleanupIndex(rootDir, options = {}) {
|
|
3862
4042
|
const verbose = options.verbose ?? false;
|
|
4043
|
+
const logger = options.logger ?? createLogger({ verbose });
|
|
3863
4044
|
rootDir = path12.resolve(rootDir);
|
|
3864
|
-
|
|
4045
|
+
logger.info(`Cleaning up index in: ${rootDir}`);
|
|
3865
4046
|
const config = await loadConfig(rootDir);
|
|
3866
4047
|
await registerBuiltInModules();
|
|
3867
4048
|
const enabledModules = registry.getEnabled(config);
|
|
3868
4049
|
if (enabledModules.length === 0) {
|
|
3869
|
-
|
|
4050
|
+
logger.info("No modules enabled.");
|
|
3870
4051
|
return [];
|
|
3871
4052
|
}
|
|
3872
4053
|
const results = [];
|
|
3873
4054
|
for (const module of enabledModules) {
|
|
3874
|
-
|
|
4055
|
+
logger.info(`
|
|
3875
4056
|
[${module.name}] Checking for stale entries...`);
|
|
3876
|
-
const result = await cleanupModuleIndex(rootDir, module.id, config,
|
|
4057
|
+
const result = await cleanupModuleIndex(rootDir, module.id, config, logger);
|
|
3877
4058
|
results.push(result);
|
|
3878
|
-
|
|
4059
|
+
logger.info(`[${module.name}] Removed ${result.removed} stale entries, kept ${result.kept} valid entries`);
|
|
3879
4060
|
}
|
|
3880
4061
|
return results;
|
|
3881
4062
|
}
|
|
3882
|
-
async function cleanupModuleIndex(rootDir, moduleId, config,
|
|
4063
|
+
async function cleanupModuleIndex(rootDir, moduleId, config, logger) {
|
|
3883
4064
|
const result = {
|
|
3884
4065
|
moduleId,
|
|
3885
4066
|
removed: 0,
|
|
@@ -3898,9 +4079,7 @@ async function cleanupModuleIndex(rootDir, moduleId, config, verbose) {
|
|
|
3898
4079
|
} catch {
|
|
3899
4080
|
filesToRemove.push(filepath);
|
|
3900
4081
|
result.removed++;
|
|
3901
|
-
|
|
3902
|
-
console.log(` Removing stale entry: ${filepath}`);
|
|
3903
|
-
}
|
|
4082
|
+
logger.debug(` Removing stale entry: ${filepath}`);
|
|
3904
4083
|
}
|
|
3905
4084
|
}
|
|
3906
4085
|
for (const filepath of filesToRemove) {
|
|
@@ -3996,6 +4175,7 @@ var init_indexer = __esm(() => {
|
|
|
3996
4175
|
init_config2();
|
|
3997
4176
|
init_registry();
|
|
3998
4177
|
init_introspection2();
|
|
4178
|
+
init_logger();
|
|
3999
4179
|
init_watcher();
|
|
4000
4180
|
});
|
|
4001
4181
|
|
|
@@ -4154,10 +4334,11 @@ var init_search = __esm(() => {
|
|
|
4154
4334
|
|
|
4155
4335
|
// src/app/cli/main.ts
|
|
4156
4336
|
init_embeddings();
|
|
4337
|
+
init_logger();
|
|
4157
4338
|
// package.json
|
|
4158
4339
|
var package_default = {
|
|
4159
4340
|
name: "raggrep",
|
|
4160
|
-
version: "0.
|
|
4341
|
+
version: "0.4.0",
|
|
4161
4342
|
description: "Local filesystem-based RAG system for codebases - semantic search using local embeddings",
|
|
4162
4343
|
type: "module",
|
|
4163
4344
|
main: "./dist/index.js",
|
|
@@ -4331,13 +4512,15 @@ Examples:
|
|
|
4331
4512
|
process.exit(0);
|
|
4332
4513
|
}
|
|
4333
4514
|
const { indexDirectory: indexDirectory2, watchDirectory: watchDirectory2 } = await Promise.resolve().then(() => (init_indexer(), exports_indexer));
|
|
4515
|
+
const logger = createInlineLogger({ verbose: flags.verbose });
|
|
4334
4516
|
console.log("RAGgrep Indexer");
|
|
4335
4517
|
console.log(`================
|
|
4336
4518
|
`);
|
|
4337
4519
|
try {
|
|
4338
4520
|
const results = await indexDirectory2(process.cwd(), {
|
|
4339
4521
|
model: flags.model,
|
|
4340
|
-
verbose: flags.verbose
|
|
4522
|
+
verbose: flags.verbose,
|
|
4523
|
+
logger
|
|
4341
4524
|
});
|
|
4342
4525
|
console.log(`
|
|
4343
4526
|
================`);
|
|
@@ -4422,9 +4605,11 @@ Examples:
|
|
|
4422
4605
|
process.exit(1);
|
|
4423
4606
|
}
|
|
4424
4607
|
try {
|
|
4608
|
+
const silentLogger = createSilentLogger();
|
|
4425
4609
|
const freshStats = await ensureIndexFresh2(process.cwd(), {
|
|
4426
4610
|
model: flags.model,
|
|
4427
|
-
quiet: true
|
|
4611
|
+
quiet: true,
|
|
4612
|
+
logger: silentLogger
|
|
4428
4613
|
});
|
|
4429
4614
|
console.log("RAGgrep Search");
|
|
4430
4615
|
console.log(`==============
|
|
@@ -4457,44 +4642,37 @@ Examples:
|
|
|
4457
4642
|
}
|
|
4458
4643
|
break;
|
|
4459
4644
|
}
|
|
4460
|
-
case "
|
|
4645
|
+
case "reset": {
|
|
4461
4646
|
if (flags.help) {
|
|
4462
4647
|
console.log(`
|
|
4463
|
-
raggrep
|
|
4648
|
+
raggrep reset - Clear the index for the current directory
|
|
4464
4649
|
|
|
4465
4650
|
Usage:
|
|
4466
|
-
raggrep
|
|
4651
|
+
raggrep reset [options]
|
|
4467
4652
|
|
|
4468
4653
|
Options:
|
|
4469
|
-
-v, --verbose Show detailed progress
|
|
4470
4654
|
-h, --help Show this help message
|
|
4471
4655
|
|
|
4472
4656
|
Description:
|
|
4473
|
-
|
|
4474
|
-
|
|
4657
|
+
Completely removes the index for the current directory.
|
|
4658
|
+
The next 'raggrep index' or 'raggrep query' will rebuild from scratch.
|
|
4475
4659
|
|
|
4476
4660
|
Examples:
|
|
4477
|
-
raggrep
|
|
4478
|
-
raggrep cleanup --verbose
|
|
4661
|
+
raggrep reset
|
|
4479
4662
|
`);
|
|
4480
4663
|
process.exit(0);
|
|
4481
4664
|
}
|
|
4482
|
-
const {
|
|
4483
|
-
console.log("RAGgrep Cleanup");
|
|
4484
|
-
console.log(`===============
|
|
4485
|
-
`);
|
|
4665
|
+
const { resetIndex: resetIndex2 } = await Promise.resolve().then(() => (init_indexer(), exports_indexer));
|
|
4486
4666
|
try {
|
|
4487
|
-
const
|
|
4488
|
-
|
|
4489
|
-
});
|
|
4490
|
-
console.log(`
|
|
4491
|
-
===============`);
|
|
4492
|
-
console.log("Summary:");
|
|
4493
|
-
for (const result of results) {
|
|
4494
|
-
console.log(` ${result.moduleId}: ${result.removed} removed, ${result.kept} kept`);
|
|
4495
|
-
}
|
|
4667
|
+
const result = await resetIndex2(process.cwd());
|
|
4668
|
+
console.log("Index cleared successfully.");
|
|
4669
|
+
console.log(` Removed: ${result.indexDir}`);
|
|
4496
4670
|
} catch (error) {
|
|
4497
|
-
|
|
4671
|
+
if (error instanceof Error && error.message.includes("No index found")) {
|
|
4672
|
+
console.error("Error: No index found for this directory.");
|
|
4673
|
+
process.exit(1);
|
|
4674
|
+
}
|
|
4675
|
+
console.error("Error during reset:", error);
|
|
4498
4676
|
process.exit(1);
|
|
4499
4677
|
}
|
|
4500
4678
|
break;
|
|
@@ -4573,7 +4751,7 @@ Commands:
|
|
|
4573
4751
|
index Index the current directory
|
|
4574
4752
|
query Search the indexed codebase
|
|
4575
4753
|
status Show the current state of the index
|
|
4576
|
-
|
|
4754
|
+
reset Clear the index for the current directory
|
|
4577
4755
|
|
|
4578
4756
|
Options:
|
|
4579
4757
|
-h, --help Show help for a command
|
|
@@ -4583,7 +4761,7 @@ Examples:
|
|
|
4583
4761
|
raggrep index
|
|
4584
4762
|
raggrep query "user login"
|
|
4585
4763
|
raggrep status
|
|
4586
|
-
raggrep
|
|
4764
|
+
raggrep reset
|
|
4587
4765
|
|
|
4588
4766
|
Run 'raggrep <command> --help' for more information.
|
|
4589
4767
|
`);
|
|
@@ -4595,4 +4773,4 @@ Run 'raggrep <command> --help' for more information.
|
|
|
4595
4773
|
}
|
|
4596
4774
|
main();
|
|
4597
4775
|
|
|
4598
|
-
//# debugId=
|
|
4776
|
+
//# debugId=B729BEE1B814E8D564756E2164756E21
|