raggrep 0.16.0 → 0.17.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +29 -5
- package/dist/cli/main.js +713 -458
- package/dist/cli/main.js.map +19 -13
- package/dist/domain/ports/embedding.d.ts +10 -0
- package/dist/domain/ports/index.d.ts +1 -1
- package/dist/index.js +524 -327
- package/dist/index.js.map +18 -12
- package/dist/infrastructure/config/configLoader.d.ts +9 -11
- package/dist/infrastructure/config/index.d.ts +1 -1
- package/dist/infrastructure/embeddings/embeddingPaths.d.ts +6 -0
- package/dist/infrastructure/embeddings/embeddingProviderFactory.d.ts +9 -0
- package/dist/infrastructure/embeddings/globalEmbeddings.d.ts +28 -0
- package/dist/infrastructure/embeddings/huggingfaceEmbeddingProvider.d.ts +21 -0
- package/dist/infrastructure/embeddings/index.d.ts +9 -2
- package/dist/infrastructure/embeddings/modelCache.d.ts +10 -0
- package/dist/infrastructure/embeddings/modelCatalog.d.ts +23 -0
- package/dist/infrastructure/embeddings/xenovaEmbeddingProvider.d.ts +23 -0
- package/dist/infrastructure/index.d.ts +1 -1
- package/package.json +7 -3
- package/dist/infrastructure/embeddings/transformersEmbedding.d.ts +0 -52
package/dist/cli/main.js
CHANGED
|
@@ -1,49 +1,89 @@
|
|
|
1
1
|
#!/usr/bin/env node
|
|
2
2
|
import { createRequire } from "node:module";
|
|
3
|
-
var __create = Object.create;
|
|
4
|
-
var __getProtoOf = Object.getPrototypeOf;
|
|
5
3
|
var __defProp = Object.defineProperty;
|
|
6
|
-
var
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
const to = isNodeMode || !mod || !mod.__esModule ? __defProp(target, "default", { value: mod, enumerable: true }) : target;
|
|
11
|
-
for (let key of __getOwnPropNames(mod))
|
|
12
|
-
if (!__hasOwnProp.call(to, key))
|
|
13
|
-
__defProp(to, key, {
|
|
14
|
-
get: () => mod[key],
|
|
15
|
-
enumerable: true
|
|
16
|
-
});
|
|
17
|
-
return to;
|
|
18
|
-
};
|
|
4
|
+
var __returnValue = (v) => v;
|
|
5
|
+
function __exportSetter(name, newValue) {
|
|
6
|
+
this[name] = __returnValue.bind(null, newValue);
|
|
7
|
+
}
|
|
19
8
|
var __export = (target, all) => {
|
|
20
9
|
for (var name in all)
|
|
21
10
|
__defProp(target, name, {
|
|
22
11
|
get: all[name],
|
|
23
12
|
enumerable: true,
|
|
24
13
|
configurable: true,
|
|
25
|
-
set: (
|
|
14
|
+
set: __exportSetter.bind(all, name)
|
|
26
15
|
});
|
|
27
16
|
};
|
|
28
17
|
var __esm = (fn, res) => () => (fn && (res = fn(fn = 0)), res);
|
|
29
18
|
var __require = /* @__PURE__ */ createRequire(import.meta.url);
|
|
30
19
|
|
|
31
|
-
// src/infrastructure/embeddings/
|
|
20
|
+
// src/infrastructure/embeddings/modelCatalog.ts
|
|
21
|
+
function getEmbeddingModelId(model) {
|
|
22
|
+
return EMBEDDING_MODEL_IDS[model];
|
|
23
|
+
}
|
|
24
|
+
function getEmbeddingDimension(model) {
|
|
25
|
+
return EMBEDDING_DIMENSIONS[model];
|
|
26
|
+
}
|
|
27
|
+
var EMBEDDING_MODEL_IDS, EMBEDDING_MODELS, EMBEDDING_DIMENSIONS;
|
|
28
|
+
var init_modelCatalog = __esm(() => {
|
|
29
|
+
EMBEDDING_MODEL_IDS = {
|
|
30
|
+
"all-MiniLM-L6-v2": "Xenova/all-MiniLM-L6-v2",
|
|
31
|
+
"all-MiniLM-L12-v2": "Xenova/all-MiniLM-L12-v2",
|
|
32
|
+
"bge-small-en-v1.5": "Xenova/bge-small-en-v1.5",
|
|
33
|
+
"paraphrase-MiniLM-L3-v2": "Xenova/paraphrase-MiniLM-L3-v2",
|
|
34
|
+
"nomic-embed-text-v1.5": "nomic-ai/nomic-embed-text-v1.5"
|
|
35
|
+
};
|
|
36
|
+
EMBEDDING_MODELS = EMBEDDING_MODEL_IDS;
|
|
37
|
+
EMBEDDING_DIMENSIONS = {
|
|
38
|
+
"all-MiniLM-L6-v2": 384,
|
|
39
|
+
"all-MiniLM-L12-v2": 384,
|
|
40
|
+
"bge-small-en-v1.5": 384,
|
|
41
|
+
"paraphrase-MiniLM-L3-v2": 384,
|
|
42
|
+
"nomic-embed-text-v1.5": 768
|
|
43
|
+
};
|
|
44
|
+
});
|
|
45
|
+
|
|
46
|
+
// src/infrastructure/embeddings/embeddingPaths.ts
|
|
47
|
+
import * as os from "os";
|
|
48
|
+
import * as path from "path";
|
|
49
|
+
var RAGGREP_MODEL_CACHE_DIR;
|
|
50
|
+
var init_embeddingPaths = __esm(() => {
|
|
51
|
+
RAGGREP_MODEL_CACHE_DIR = path.join(os.homedir(), ".cache", "raggrep", "models");
|
|
52
|
+
});
|
|
53
|
+
|
|
54
|
+
// src/infrastructure/embeddings/modelCache.ts
|
|
55
|
+
import * as fs from "fs/promises";
|
|
56
|
+
import * as path2 from "path";
|
|
57
|
+
async function isEmbeddingModelCached(model) {
|
|
58
|
+
const modelId = getEmbeddingModelId(model);
|
|
59
|
+
const onnxPath = path2.join(RAGGREP_MODEL_CACHE_DIR, modelId, "onnx", "model_quantized.onnx");
|
|
60
|
+
try {
|
|
61
|
+
await fs.access(onnxPath);
|
|
62
|
+
return true;
|
|
63
|
+
} catch {
|
|
64
|
+
return false;
|
|
65
|
+
}
|
|
66
|
+
}
|
|
67
|
+
var init_modelCache = __esm(() => {
|
|
68
|
+
init_embeddingPaths();
|
|
69
|
+
init_modelCatalog();
|
|
70
|
+
});
|
|
71
|
+
|
|
72
|
+
// src/infrastructure/embeddings/xenovaEmbeddingProvider.ts
|
|
32
73
|
import {
|
|
33
74
|
pipeline,
|
|
34
75
|
env
|
|
35
76
|
} from "@xenova/transformers";
|
|
36
|
-
import * as path from "path";
|
|
37
|
-
import * as os from "os";
|
|
38
77
|
|
|
39
|
-
class
|
|
40
|
-
|
|
78
|
+
class XenovaTransformersEmbeddingProvider {
|
|
79
|
+
extractor = null;
|
|
41
80
|
config;
|
|
42
81
|
isInitializing = false;
|
|
43
82
|
initPromise = null;
|
|
44
83
|
constructor(config) {
|
|
45
84
|
this.config = {
|
|
46
85
|
model: config?.model ?? "bge-small-en-v1.5",
|
|
86
|
+
runtime: config?.runtime ?? "xenova",
|
|
47
87
|
showProgress: config?.showProgress ?? false,
|
|
48
88
|
logger: config?.logger
|
|
49
89
|
};
|
|
@@ -51,14 +91,14 @@ class TransformersEmbeddingProvider {
|
|
|
51
91
|
async initialize(config) {
|
|
52
92
|
if (config) {
|
|
53
93
|
if (config.model !== this.config.model) {
|
|
54
|
-
this.
|
|
94
|
+
this.extractor = null;
|
|
55
95
|
}
|
|
56
96
|
this.config = { ...this.config, ...config };
|
|
57
97
|
}
|
|
58
|
-
await this.
|
|
98
|
+
await this.ensureExtractor();
|
|
59
99
|
}
|
|
60
|
-
async
|
|
61
|
-
if (this.
|
|
100
|
+
async ensureExtractor() {
|
|
101
|
+
if (this.extractor) {
|
|
62
102
|
return;
|
|
63
103
|
}
|
|
64
104
|
if (this.isInitializing && this.initPromise) {
|
|
@@ -66,14 +106,14 @@ class TransformersEmbeddingProvider {
|
|
|
66
106
|
}
|
|
67
107
|
this.isInitializing = true;
|
|
68
108
|
this.initPromise = (async () => {
|
|
69
|
-
const modelId =
|
|
109
|
+
const modelId = getEmbeddingModelId(this.config.model);
|
|
70
110
|
const logger = this.config.logger;
|
|
71
111
|
const showProgress = this.config.showProgress || !!logger;
|
|
72
|
-
const
|
|
112
|
+
const cached = await isEmbeddingModelCached(this.config.model);
|
|
73
113
|
let hasDownloads = false;
|
|
74
114
|
try {
|
|
75
|
-
this.
|
|
76
|
-
progress_callback: showProgress && !
|
|
115
|
+
this.extractor = await pipeline("feature-extraction", modelId, {
|
|
116
|
+
progress_callback: showProgress && !cached ? (progress) => {
|
|
77
117
|
if (progress.status === "progress" && progress.file) {
|
|
78
118
|
if (!hasDownloads) {
|
|
79
119
|
hasDownloads = true;
|
|
@@ -82,7 +122,7 @@ class TransformersEmbeddingProvider {
|
|
|
82
122
|
} else {
|
|
83
123
|
console.log(`
|
|
84
124
|
Loading embedding model: ${this.config.model}`);
|
|
85
|
-
console.log(` Cache: ${
|
|
125
|
+
console.log(` Cache: ${RAGGREP_MODEL_CACHE_DIR}`);
|
|
86
126
|
}
|
|
87
127
|
}
|
|
88
128
|
const pct = progress.progress ? Math.round(progress.progress) : 0;
|
|
@@ -112,9 +152,9 @@ class TransformersEmbeddingProvider {
|
|
|
112
152
|
}
|
|
113
153
|
}
|
|
114
154
|
} catch (error) {
|
|
115
|
-
this.
|
|
116
|
-
if (logger) {
|
|
117
|
-
logger.clearProgress();
|
|
155
|
+
this.extractor = null;
|
|
156
|
+
if (this.config.logger) {
|
|
157
|
+
this.config.logger.clearProgress();
|
|
118
158
|
}
|
|
119
159
|
throw new Error(`Failed to load embedding model: ${error}`);
|
|
120
160
|
} finally {
|
|
@@ -125,11 +165,11 @@ class TransformersEmbeddingProvider {
|
|
|
125
165
|
return this.initPromise;
|
|
126
166
|
}
|
|
127
167
|
async getEmbedding(text) {
|
|
128
|
-
await this.
|
|
129
|
-
if (!this.
|
|
168
|
+
await this.ensureExtractor();
|
|
169
|
+
if (!this.extractor) {
|
|
130
170
|
throw new Error("Embedding pipeline not initialized");
|
|
131
171
|
}
|
|
132
|
-
const output = await this.
|
|
172
|
+
const output = await this.extractor(text, {
|
|
133
173
|
pooling: "mean",
|
|
134
174
|
normalize: true
|
|
135
175
|
});
|
|
@@ -138,15 +178,15 @@ class TransformersEmbeddingProvider {
|
|
|
138
178
|
async getEmbeddings(texts) {
|
|
139
179
|
if (texts.length === 0)
|
|
140
180
|
return [];
|
|
141
|
-
await this.
|
|
142
|
-
if (!this.
|
|
181
|
+
await this.ensureExtractor();
|
|
182
|
+
if (!this.extractor) {
|
|
143
183
|
throw new Error("Embedding pipeline not initialized");
|
|
144
184
|
}
|
|
145
185
|
const results = [];
|
|
146
186
|
for (let i = 0;i < texts.length; i += BATCH_SIZE) {
|
|
147
187
|
const batch = texts.slice(i, i + BATCH_SIZE);
|
|
148
188
|
const outputs = await Promise.all(batch.map(async (text) => {
|
|
149
|
-
const output = await this.
|
|
189
|
+
const output = await this.extractor(text, {
|
|
150
190
|
pooling: "mean",
|
|
151
191
|
normalize: true
|
|
152
192
|
});
|
|
@@ -157,44 +197,210 @@ class TransformersEmbeddingProvider {
|
|
|
157
197
|
return results;
|
|
158
198
|
}
|
|
159
199
|
getDimension() {
|
|
160
|
-
return
|
|
200
|
+
return getEmbeddingDimension(this.config.model);
|
|
161
201
|
}
|
|
162
202
|
getModelName() {
|
|
163
203
|
return this.config.model;
|
|
164
204
|
}
|
|
165
205
|
async dispose() {
|
|
166
|
-
this.
|
|
206
|
+
this.extractor = null;
|
|
167
207
|
}
|
|
168
208
|
}
|
|
169
|
-
|
|
170
|
-
|
|
209
|
+
var BATCH_SIZE = 32;
|
|
210
|
+
var init_xenovaEmbeddingProvider = __esm(() => {
|
|
211
|
+
init_embeddingPaths();
|
|
212
|
+
init_modelCatalog();
|
|
213
|
+
init_modelCache();
|
|
214
|
+
env.cacheDir = RAGGREP_MODEL_CACHE_DIR;
|
|
215
|
+
env.allowLocalModels = true;
|
|
216
|
+
});
|
|
217
|
+
|
|
218
|
+
// src/infrastructure/embeddings/huggingfaceEmbeddingProvider.ts
|
|
219
|
+
import {
|
|
220
|
+
pipeline as pipeline2,
|
|
221
|
+
env as env2
|
|
222
|
+
} from "@huggingface/transformers";
|
|
223
|
+
|
|
224
|
+
class HuggingFaceTransformersEmbeddingProvider {
|
|
225
|
+
extractor = null;
|
|
226
|
+
config;
|
|
227
|
+
isInitializing = false;
|
|
228
|
+
initPromise = null;
|
|
229
|
+
constructor(config) {
|
|
230
|
+
this.config = {
|
|
231
|
+
model: config?.model ?? "bge-small-en-v1.5",
|
|
232
|
+
runtime: config?.runtime ?? "huggingface",
|
|
233
|
+
showProgress: config?.showProgress ?? false,
|
|
234
|
+
logger: config?.logger
|
|
235
|
+
};
|
|
236
|
+
}
|
|
237
|
+
async initialize(config) {
|
|
238
|
+
if (config) {
|
|
239
|
+
if (config.model !== this.config.model) {
|
|
240
|
+
this.extractor = null;
|
|
241
|
+
}
|
|
242
|
+
this.config = { ...this.config, ...config };
|
|
243
|
+
}
|
|
244
|
+
await this.ensureExtractor();
|
|
245
|
+
}
|
|
246
|
+
async ensureExtractor() {
|
|
247
|
+
if (this.extractor) {
|
|
248
|
+
return;
|
|
249
|
+
}
|
|
250
|
+
if (this.isInitializing && this.initPromise) {
|
|
251
|
+
return this.initPromise;
|
|
252
|
+
}
|
|
253
|
+
this.isInitializing = true;
|
|
254
|
+
this.initPromise = (async () => {
|
|
255
|
+
const modelId = getEmbeddingModelId(this.config.model);
|
|
256
|
+
const logger = this.config.logger;
|
|
257
|
+
const showProgress = this.config.showProgress || !!logger;
|
|
258
|
+
const cached = await isEmbeddingModelCached(this.config.model);
|
|
259
|
+
let hasDownloads = false;
|
|
260
|
+
try {
|
|
261
|
+
this.extractor = await pipeline2("feature-extraction", modelId, {
|
|
262
|
+
progress_callback: showProgress && !cached ? (progress) => {
|
|
263
|
+
if (progress.status === "progress" && progress.file) {
|
|
264
|
+
if (!hasDownloads) {
|
|
265
|
+
hasDownloads = true;
|
|
266
|
+
if (logger) {
|
|
267
|
+
logger.info(`Downloading embedding model: ${this.config.model}`);
|
|
268
|
+
} else {
|
|
269
|
+
console.log(`
|
|
270
|
+
Loading embedding model: ${this.config.model}`);
|
|
271
|
+
console.log(` Cache: ${RAGGREP_MODEL_CACHE_DIR}`);
|
|
272
|
+
}
|
|
273
|
+
}
|
|
274
|
+
const pct = progress.progress ? Math.round(progress.progress) : 0;
|
|
275
|
+
if (logger) {
|
|
276
|
+
logger.progress(` Downloading ${progress.file}: ${pct}%`);
|
|
277
|
+
} else {
|
|
278
|
+
process.stdout.write(`\r Downloading ${progress.file}: ${pct}% `);
|
|
279
|
+
}
|
|
280
|
+
} else if (progress.status === "done" && progress.file) {
|
|
281
|
+
if (logger) {
|
|
282
|
+
logger.clearProgress();
|
|
283
|
+
logger.info(` Downloaded ${progress.file}`);
|
|
284
|
+
} else if (hasDownloads) {
|
|
285
|
+
process.stdout.write(`\r Downloaded ${progress.file}
|
|
286
|
+
`);
|
|
287
|
+
}
|
|
288
|
+
}
|
|
289
|
+
} : undefined
|
|
290
|
+
});
|
|
291
|
+
if (hasDownloads) {
|
|
292
|
+
if (logger) {
|
|
293
|
+
logger.clearProgress();
|
|
294
|
+
logger.info(`Model ready: ${this.config.model}`);
|
|
295
|
+
} else {
|
|
296
|
+
console.log(` Model ready.
|
|
297
|
+
`);
|
|
298
|
+
}
|
|
299
|
+
}
|
|
300
|
+
} catch (error) {
|
|
301
|
+
this.extractor = null;
|
|
302
|
+
if (this.config.logger) {
|
|
303
|
+
this.config.logger.clearProgress();
|
|
304
|
+
}
|
|
305
|
+
throw new Error(`Failed to load embedding model: ${error}`);
|
|
306
|
+
} finally {
|
|
307
|
+
this.isInitializing = false;
|
|
308
|
+
this.initPromise = null;
|
|
309
|
+
}
|
|
310
|
+
})();
|
|
311
|
+
return this.initPromise;
|
|
312
|
+
}
|
|
313
|
+
async getEmbedding(text) {
|
|
314
|
+
await this.ensureExtractor();
|
|
315
|
+
if (!this.extractor) {
|
|
316
|
+
throw new Error("Embedding pipeline not initialized");
|
|
317
|
+
}
|
|
318
|
+
const output = await this.extractor(text, {
|
|
319
|
+
pooling: "mean",
|
|
320
|
+
normalize: true
|
|
321
|
+
});
|
|
322
|
+
return Array.from(output.data);
|
|
323
|
+
}
|
|
324
|
+
async getEmbeddings(texts) {
|
|
325
|
+
if (texts.length === 0)
|
|
326
|
+
return [];
|
|
327
|
+
await this.ensureExtractor();
|
|
328
|
+
if (!this.extractor) {
|
|
329
|
+
throw new Error("Embedding pipeline not initialized");
|
|
330
|
+
}
|
|
331
|
+
const results = [];
|
|
332
|
+
for (let i = 0;i < texts.length; i += BATCH_SIZE2) {
|
|
333
|
+
const batch = texts.slice(i, i + BATCH_SIZE2);
|
|
334
|
+
const outputs = await Promise.all(batch.map(async (text) => {
|
|
335
|
+
const output = await this.extractor(text, {
|
|
336
|
+
pooling: "mean",
|
|
337
|
+
normalize: true
|
|
338
|
+
});
|
|
339
|
+
return Array.from(output.data);
|
|
340
|
+
}));
|
|
341
|
+
results.push(...outputs);
|
|
342
|
+
}
|
|
343
|
+
return results;
|
|
344
|
+
}
|
|
345
|
+
getDimension() {
|
|
346
|
+
return getEmbeddingDimension(this.config.model);
|
|
347
|
+
}
|
|
348
|
+
getModelName() {
|
|
349
|
+
return this.config.model;
|
|
350
|
+
}
|
|
351
|
+
async dispose() {
|
|
352
|
+
this.extractor = null;
|
|
353
|
+
}
|
|
171
354
|
}
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
|
|
355
|
+
var BATCH_SIZE2 = 32;
|
|
356
|
+
var init_huggingfaceEmbeddingProvider = __esm(() => {
|
|
357
|
+
init_embeddingPaths();
|
|
358
|
+
init_modelCatalog();
|
|
359
|
+
init_modelCache();
|
|
360
|
+
env2.cacheDir = RAGGREP_MODEL_CACHE_DIR;
|
|
361
|
+
env2.allowLocalModels = true;
|
|
362
|
+
});
|
|
363
|
+
|
|
364
|
+
// src/infrastructure/embeddings/embeddingProviderFactory.ts
|
|
365
|
+
function resolveRuntime(config) {
|
|
366
|
+
return config.runtime ?? "huggingface";
|
|
367
|
+
}
|
|
368
|
+
function createEmbeddingProvider(config) {
|
|
369
|
+
const runtime = resolveRuntime(config);
|
|
370
|
+
if (runtime === "huggingface") {
|
|
371
|
+
return new HuggingFaceTransformersEmbeddingProvider(config);
|
|
182
372
|
}
|
|
373
|
+
return new XenovaTransformersEmbeddingProvider(config);
|
|
183
374
|
}
|
|
375
|
+
var init_embeddingProviderFactory = __esm(() => {
|
|
376
|
+
init_huggingfaceEmbeddingProvider();
|
|
377
|
+
init_xenovaEmbeddingProvider();
|
|
378
|
+
});
|
|
379
|
+
|
|
380
|
+
// src/infrastructure/embeddings/globalEmbeddings.ts
|
|
184
381
|
function configureEmbeddings(config) {
|
|
185
|
-
const
|
|
186
|
-
|
|
382
|
+
const merged = {
|
|
383
|
+
...globalConfig,
|
|
384
|
+
...config
|
|
385
|
+
};
|
|
386
|
+
if (merged.runtime === undefined) {
|
|
387
|
+
merged.runtime = "huggingface";
|
|
388
|
+
}
|
|
389
|
+
const needsReset = merged.model !== globalConfig.model || merged.runtime !== globalConfig.runtime || merged.logger !== globalConfig.logger;
|
|
390
|
+
if (needsReset) {
|
|
391
|
+
const prev = globalProvider;
|
|
187
392
|
globalProvider = null;
|
|
393
|
+
prev?.dispose?.();
|
|
188
394
|
}
|
|
189
|
-
globalConfig =
|
|
395
|
+
globalConfig = merged;
|
|
190
396
|
}
|
|
191
397
|
function getEmbeddingConfig() {
|
|
192
398
|
return { ...globalConfig };
|
|
193
399
|
}
|
|
194
400
|
async function ensureGlobalProvider() {
|
|
195
401
|
if (!globalProvider) {
|
|
196
|
-
globalProvider =
|
|
197
|
-
await globalProvider.initialize();
|
|
402
|
+
globalProvider = createEmbeddingProvider(globalConfig);
|
|
403
|
+
await globalProvider.initialize?.(globalConfig);
|
|
198
404
|
}
|
|
199
405
|
return globalProvider;
|
|
200
406
|
}
|
|
@@ -206,27 +412,17 @@ async function getEmbeddings(texts) {
|
|
|
206
412
|
const provider = await ensureGlobalProvider();
|
|
207
413
|
return provider.getEmbeddings(texts);
|
|
208
414
|
}
|
|
209
|
-
|
|
210
|
-
|
|
211
|
-
|
|
212
|
-
|
|
213
|
-
|
|
214
|
-
|
|
215
|
-
|
|
216
|
-
|
|
217
|
-
"bge-small-en-v1.5": "Xenova/bge-small-en-v1.5",
|
|
218
|
-
"paraphrase-MiniLM-L3-v2": "Xenova/paraphrase-MiniLM-L3-v2",
|
|
219
|
-
"nomic-embed-text-v1.5": "nomic-ai/nomic-embed-text-v1.5"
|
|
220
|
-
};
|
|
221
|
-
EMBEDDING_DIMENSIONS = {
|
|
222
|
-
"all-MiniLM-L6-v2": 384,
|
|
223
|
-
"all-MiniLM-L12-v2": 384,
|
|
224
|
-
"bge-small-en-v1.5": 384,
|
|
225
|
-
"paraphrase-MiniLM-L3-v2": 384,
|
|
226
|
-
"nomic-embed-text-v1.5": 768
|
|
227
|
-
};
|
|
415
|
+
function getCacheDir() {
|
|
416
|
+
return RAGGREP_MODEL_CACHE_DIR;
|
|
417
|
+
}
|
|
418
|
+
var globalProvider = null, globalConfig;
|
|
419
|
+
var init_globalEmbeddings = __esm(() => {
|
|
420
|
+
init_embeddingPaths();
|
|
421
|
+
init_embeddingProviderFactory();
|
|
422
|
+
init_modelCache();
|
|
228
423
|
globalConfig = {
|
|
229
424
|
model: "bge-small-en-v1.5",
|
|
425
|
+
runtime: "huggingface",
|
|
230
426
|
showProgress: false,
|
|
231
427
|
logger: undefined
|
|
232
428
|
};
|
|
@@ -234,7 +430,13 @@ var init_transformersEmbedding = __esm(() => {
|
|
|
234
430
|
|
|
235
431
|
// src/infrastructure/embeddings/index.ts
|
|
236
432
|
var init_embeddings = __esm(() => {
|
|
237
|
-
|
|
433
|
+
init_modelCatalog();
|
|
434
|
+
init_embeddingPaths();
|
|
435
|
+
init_xenovaEmbeddingProvider();
|
|
436
|
+
init_xenovaEmbeddingProvider();
|
|
437
|
+
init_huggingfaceEmbeddingProvider();
|
|
438
|
+
init_embeddingProviderFactory();
|
|
439
|
+
init_globalEmbeddings();
|
|
238
440
|
});
|
|
239
441
|
|
|
240
442
|
// src/infrastructure/logger/loggers.ts
|
|
@@ -457,29 +659,29 @@ var init_logger = () => {};
|
|
|
457
659
|
import { createRequire as createRequire2 } from "module";
|
|
458
660
|
import { basename, dirname, normalize, relative, resolve, sep } from "path";
|
|
459
661
|
import * as nativeFs from "fs";
|
|
460
|
-
function cleanPath(
|
|
461
|
-
let normalized = normalize(
|
|
662
|
+
function cleanPath(path3) {
|
|
663
|
+
let normalized = normalize(path3);
|
|
462
664
|
if (normalized.length > 1 && normalized[normalized.length - 1] === sep)
|
|
463
665
|
normalized = normalized.substring(0, normalized.length - 1);
|
|
464
666
|
return normalized;
|
|
465
667
|
}
|
|
466
|
-
function convertSlashes(
|
|
467
|
-
return
|
|
668
|
+
function convertSlashes(path3, separator) {
|
|
669
|
+
return path3.replace(SLASHES_REGEX, separator);
|
|
468
670
|
}
|
|
469
|
-
function isRootDirectory(
|
|
470
|
-
return
|
|
671
|
+
function isRootDirectory(path3) {
|
|
672
|
+
return path3 === "/" || WINDOWS_ROOT_DIR_REGEX.test(path3);
|
|
471
673
|
}
|
|
472
|
-
function normalizePath(
|
|
674
|
+
function normalizePath(path3, options) {
|
|
473
675
|
const { resolvePaths, normalizePath: normalizePath$1, pathSeparator } = options;
|
|
474
|
-
const pathNeedsCleaning = process.platform === "win32" &&
|
|
676
|
+
const pathNeedsCleaning = process.platform === "win32" && path3.includes("/") || path3.startsWith(".");
|
|
475
677
|
if (resolvePaths)
|
|
476
|
-
|
|
678
|
+
path3 = resolve(path3);
|
|
477
679
|
if (normalizePath$1 || pathNeedsCleaning)
|
|
478
|
-
|
|
479
|
-
if (
|
|
680
|
+
path3 = cleanPath(path3);
|
|
681
|
+
if (path3 === ".")
|
|
480
682
|
return "";
|
|
481
|
-
const needsSeperator =
|
|
482
|
-
return convertSlashes(needsSeperator ?
|
|
683
|
+
const needsSeperator = path3[path3.length - 1] !== pathSeparator;
|
|
684
|
+
return convertSlashes(needsSeperator ? path3 + pathSeparator : path3, pathSeparator);
|
|
483
685
|
}
|
|
484
686
|
function joinPathWithBasePath(filename, directoryPath) {
|
|
485
687
|
return directoryPath + filename;
|
|
@@ -545,10 +747,10 @@ function build$2(options, isSynchronous) {
|
|
|
545
747
|
return null;
|
|
546
748
|
return isSynchronous ? resolveSymlinks : resolveSymlinksAsync;
|
|
547
749
|
}
|
|
548
|
-
function isRecursive(
|
|
750
|
+
function isRecursive(path3, resolved, state) {
|
|
549
751
|
if (state.options.useRealPaths)
|
|
550
752
|
return isRecursiveUsingRealPaths(resolved, state);
|
|
551
|
-
let parent = dirname(
|
|
753
|
+
let parent = dirname(path3);
|
|
552
754
|
let depth = 1;
|
|
553
755
|
while (parent !== state.root && depth < 2) {
|
|
554
756
|
const resolvedPath = state.symlinks.get(parent);
|
|
@@ -558,7 +760,7 @@ function isRecursive(path2, resolved, state) {
|
|
|
558
760
|
else
|
|
559
761
|
parent = dirname(parent);
|
|
560
762
|
}
|
|
561
|
-
state.symlinks.set(
|
|
763
|
+
state.symlinks.set(path3, resolved);
|
|
562
764
|
return depth > 1;
|
|
563
765
|
}
|
|
564
766
|
function isRecursiveUsingRealPaths(resolved, state) {
|
|
@@ -604,9 +806,9 @@ function sync(root, options) {
|
|
|
604
806
|
var __require2, SLASHES_REGEX, WINDOWS_ROOT_DIR_REGEX, pushDirectory = (directoryPath, paths) => {
|
|
605
807
|
paths.push(directoryPath || ".");
|
|
606
808
|
}, pushDirectoryFilter = (directoryPath, paths, filters) => {
|
|
607
|
-
const
|
|
608
|
-
if (filters.every((filter) => filter(
|
|
609
|
-
paths.push(
|
|
809
|
+
const path3 = directoryPath || ".";
|
|
810
|
+
if (filters.every((filter) => filter(path3, true)))
|
|
811
|
+
paths.push(path3);
|
|
610
812
|
}, empty$2 = () => {}, pushFileFilterAndCount = (filename, _paths, counts, filters) => {
|
|
611
813
|
if (filters.every((filter) => filter(filename, false)))
|
|
612
814
|
counts.files++;
|
|
@@ -627,28 +829,28 @@ var __require2, SLASHES_REGEX, WINDOWS_ROOT_DIR_REGEX, pushDirectory = (director
|
|
|
627
829
|
files,
|
|
628
830
|
dir: directory
|
|
629
831
|
});
|
|
630
|
-
}, empty = () => {}, resolveSymlinksAsync = function(
|
|
631
|
-
const { queue, fs, options: { suppressErrors } } = state;
|
|
832
|
+
}, empty = () => {}, resolveSymlinksAsync = function(path3, state, callback$1) {
|
|
833
|
+
const { queue, fs: fs2, options: { suppressErrors } } = state;
|
|
632
834
|
queue.enqueue();
|
|
633
|
-
|
|
835
|
+
fs2.realpath(path3, (error, resolvedPath) => {
|
|
634
836
|
if (error)
|
|
635
837
|
return queue.dequeue(suppressErrors ? null : error, state);
|
|
636
|
-
|
|
838
|
+
fs2.stat(resolvedPath, (error$1, stat) => {
|
|
637
839
|
if (error$1)
|
|
638
840
|
return queue.dequeue(suppressErrors ? null : error$1, state);
|
|
639
|
-
if (stat.isDirectory() && isRecursive(
|
|
841
|
+
if (stat.isDirectory() && isRecursive(path3, resolvedPath, state))
|
|
640
842
|
return queue.dequeue(null, state);
|
|
641
843
|
callback$1(stat, resolvedPath);
|
|
642
844
|
queue.dequeue(null, state);
|
|
643
845
|
});
|
|
644
846
|
});
|
|
645
|
-
}, resolveSymlinks = function(
|
|
646
|
-
const { queue, fs, options: { suppressErrors } } = state;
|
|
847
|
+
}, resolveSymlinks = function(path3, state, callback$1) {
|
|
848
|
+
const { queue, fs: fs2, options: { suppressErrors } } = state;
|
|
647
849
|
queue.enqueue();
|
|
648
850
|
try {
|
|
649
|
-
const resolvedPath =
|
|
650
|
-
const stat =
|
|
651
|
-
if (stat.isDirectory() && isRecursive(
|
|
851
|
+
const resolvedPath = fs2.realpathSync(path3);
|
|
852
|
+
const stat = fs2.statSync(resolvedPath);
|
|
853
|
+
if (stat.isDirectory() && isRecursive(path3, resolvedPath, state))
|
|
652
854
|
return;
|
|
653
855
|
callback$1(stat, resolvedPath);
|
|
654
856
|
} catch (e) {
|
|
@@ -679,22 +881,22 @@ var __require2, SLASHES_REGEX, WINDOWS_ROOT_DIR_REGEX, pushDirectory = (director
|
|
|
679
881
|
state.queue.enqueue();
|
|
680
882
|
if (currentDepth < 0)
|
|
681
883
|
return state.queue.dequeue(null, state);
|
|
682
|
-
const { fs } = state;
|
|
884
|
+
const { fs: fs2 } = state;
|
|
683
885
|
state.visited.push(crawlPath);
|
|
684
886
|
state.counts.directories++;
|
|
685
|
-
|
|
887
|
+
fs2.readdir(crawlPath || ".", readdirOpts, (error, entries = []) => {
|
|
686
888
|
callback$1(entries, directoryPath, currentDepth);
|
|
687
889
|
state.queue.dequeue(state.options.suppressErrors ? null : error, state);
|
|
688
890
|
});
|
|
689
891
|
}, walkSync = (state, crawlPath, directoryPath, currentDepth, callback$1) => {
|
|
690
|
-
const { fs } = state;
|
|
892
|
+
const { fs: fs2 } = state;
|
|
691
893
|
if (currentDepth < 0)
|
|
692
894
|
return;
|
|
693
895
|
state.visited.push(crawlPath);
|
|
694
896
|
state.counts.directories++;
|
|
695
897
|
let entries = [];
|
|
696
898
|
try {
|
|
697
|
-
entries =
|
|
899
|
+
entries = fs2.readdirSync(crawlPath || ".", readdirOpts);
|
|
698
900
|
} catch (e) {
|
|
699
901
|
if (!state.options.suppressErrors)
|
|
700
902
|
throw e;
|
|
@@ -793,21 +995,21 @@ var __require2, SLASHES_REGEX, WINDOWS_ROOT_DIR_REGEX, pushDirectory = (director
|
|
|
793
995
|
const filename = this.joinPath(entry.name, directoryPath);
|
|
794
996
|
this.pushFile(filename, files, this.state.counts, filters);
|
|
795
997
|
} else if (entry.isDirectory()) {
|
|
796
|
-
let
|
|
797
|
-
if (exclude && exclude(entry.name,
|
|
998
|
+
let path3 = joinDirectoryPath(entry.name, directoryPath, this.state.options.pathSeparator);
|
|
999
|
+
if (exclude && exclude(entry.name, path3))
|
|
798
1000
|
continue;
|
|
799
|
-
this.pushDirectory(
|
|
800
|
-
this.walkDirectory(this.state,
|
|
1001
|
+
this.pushDirectory(path3, paths, filters);
|
|
1002
|
+
this.walkDirectory(this.state, path3, path3, depth - 1, this.walk);
|
|
801
1003
|
} else if (this.resolveSymlink && entry.isSymbolicLink()) {
|
|
802
|
-
let
|
|
803
|
-
this.resolveSymlink(
|
|
1004
|
+
let path3 = joinPathWithBasePath(entry.name, directoryPath);
|
|
1005
|
+
this.resolveSymlink(path3, this.state, (stat, resolvedPath) => {
|
|
804
1006
|
if (stat.isDirectory()) {
|
|
805
1007
|
resolvedPath = normalizePath(resolvedPath, this.state.options);
|
|
806
|
-
if (exclude && exclude(entry.name, useRealPaths ? resolvedPath :
|
|
1008
|
+
if (exclude && exclude(entry.name, useRealPaths ? resolvedPath : path3 + pathSeparator))
|
|
807
1009
|
return;
|
|
808
|
-
this.walkDirectory(this.state, resolvedPath, useRealPaths ? resolvedPath :
|
|
1010
|
+
this.walkDirectory(this.state, resolvedPath, useRealPaths ? resolvedPath : path3 + pathSeparator, depth - 1, this.walk);
|
|
809
1011
|
} else {
|
|
810
|
-
resolvedPath = useRealPaths ? resolvedPath :
|
|
1012
|
+
resolvedPath = useRealPaths ? resolvedPath : path3;
|
|
811
1013
|
const filename = basename(resolvedPath);
|
|
812
1014
|
const directoryPath$1 = normalizePath(dirname(resolvedPath), this.state.options);
|
|
813
1015
|
resolvedPath = this.joinPath(filename, directoryPath$1);
|
|
@@ -943,7 +1145,7 @@ var __require2, SLASHES_REGEX, WINDOWS_ROOT_DIR_REGEX, pushDirectory = (director
|
|
|
943
1145
|
isMatch = globFn(patterns, ...options);
|
|
944
1146
|
this.globCache[patterns.join("\x00")] = isMatch;
|
|
945
1147
|
}
|
|
946
|
-
this.options.filters.push((
|
|
1148
|
+
this.options.filters.push((path3) => isMatch(path3));
|
|
947
1149
|
return this;
|
|
948
1150
|
}
|
|
949
1151
|
};
|
|
@@ -985,49 +1187,37 @@ function createDefaultConfig() {
|
|
|
985
1187
|
{
|
|
986
1188
|
id: "language/typescript",
|
|
987
1189
|
enabled: true,
|
|
988
|
-
options: {
|
|
989
|
-
embeddingModel: "all-MiniLM-L6-v2"
|
|
990
|
-
}
|
|
1190
|
+
options: { ...DEFAULT_EMBEDDING_MODULE_OPTIONS }
|
|
991
1191
|
},
|
|
992
1192
|
{
|
|
993
1193
|
id: "language/python",
|
|
994
1194
|
enabled: true,
|
|
995
|
-
options: {
|
|
996
|
-
embeddingModel: "all-MiniLM-L6-v2"
|
|
997
|
-
}
|
|
1195
|
+
options: { ...DEFAULT_EMBEDDING_MODULE_OPTIONS }
|
|
998
1196
|
},
|
|
999
1197
|
{
|
|
1000
1198
|
id: "language/go",
|
|
1001
1199
|
enabled: true,
|
|
1002
|
-
options: {
|
|
1003
|
-
embeddingModel: "all-MiniLM-L6-v2"
|
|
1004
|
-
}
|
|
1200
|
+
options: { ...DEFAULT_EMBEDDING_MODULE_OPTIONS }
|
|
1005
1201
|
},
|
|
1006
1202
|
{
|
|
1007
1203
|
id: "language/rust",
|
|
1008
1204
|
enabled: true,
|
|
1009
|
-
options: {
|
|
1010
|
-
embeddingModel: "all-MiniLM-L6-v2"
|
|
1011
|
-
}
|
|
1205
|
+
options: { ...DEFAULT_EMBEDDING_MODULE_OPTIONS }
|
|
1012
1206
|
},
|
|
1013
1207
|
{
|
|
1014
1208
|
id: "data/json",
|
|
1015
1209
|
enabled: true,
|
|
1016
|
-
options: {
|
|
1017
|
-
embeddingModel: "all-MiniLM-L6-v2"
|
|
1018
|
-
}
|
|
1210
|
+
options: { ...DEFAULT_EMBEDDING_MODULE_OPTIONS }
|
|
1019
1211
|
},
|
|
1020
1212
|
{
|
|
1021
1213
|
id: "docs/markdown",
|
|
1022
1214
|
enabled: true,
|
|
1023
|
-
options: {
|
|
1024
|
-
embeddingModel: "all-MiniLM-L6-v2"
|
|
1025
|
-
}
|
|
1215
|
+
options: { ...DEFAULT_EMBEDDING_MODULE_OPTIONS }
|
|
1026
1216
|
}
|
|
1027
1217
|
]
|
|
1028
1218
|
};
|
|
1029
1219
|
}
|
|
1030
|
-
var DEFAULT_IGNORE_PATHS, DEFAULT_EXTENSIONS;
|
|
1220
|
+
var DEFAULT_IGNORE_PATHS, DEFAULT_EXTENSIONS, DEFAULT_EMBEDDING_MODULE_OPTIONS;
|
|
1031
1221
|
var init_config = __esm(() => {
|
|
1032
1222
|
DEFAULT_IGNORE_PATHS = [
|
|
1033
1223
|
"node_modules",
|
|
@@ -1097,6 +1287,10 @@ var init_config = __esm(() => {
|
|
|
1097
1287
|
".sql",
|
|
1098
1288
|
".txt"
|
|
1099
1289
|
];
|
|
1290
|
+
DEFAULT_EMBEDDING_MODULE_OPTIONS = {
|
|
1291
|
+
embeddingModel: "bge-small-en-v1.5",
|
|
1292
|
+
embeddingRuntime: "huggingface"
|
|
1293
|
+
};
|
|
1100
1294
|
});
|
|
1101
1295
|
|
|
1102
1296
|
// src/domain/entities/literal.ts
|
|
@@ -1127,47 +1321,45 @@ var init_entities = __esm(() => {
|
|
|
1127
1321
|
});
|
|
1128
1322
|
|
|
1129
1323
|
// src/infrastructure/config/configLoader.ts
|
|
1130
|
-
import * as
|
|
1131
|
-
import * as
|
|
1132
|
-
import * as os2 from "os";
|
|
1324
|
+
import * as path3 from "path";
|
|
1325
|
+
import * as fs2 from "fs/promises";
|
|
1133
1326
|
import * as crypto from "crypto";
|
|
1134
1327
|
function hashPath(inputPath) {
|
|
1135
1328
|
return crypto.createHash("sha256").update(inputPath).digest("hex").slice(0, 12);
|
|
1136
1329
|
}
|
|
1137
1330
|
function getRaggrepDir(rootDir, _config = DEFAULT_CONFIG) {
|
|
1138
|
-
const absoluteRoot =
|
|
1139
|
-
|
|
1140
|
-
return path2.join(RAGGREP_TEMP_BASE, projectHash);
|
|
1331
|
+
const absoluteRoot = path3.resolve(rootDir);
|
|
1332
|
+
return path3.join(absoluteRoot, RAGGREP_INDEX_DIR);
|
|
1141
1333
|
}
|
|
1142
1334
|
function getIndexLocation(rootDir) {
|
|
1143
|
-
const absoluteRoot =
|
|
1335
|
+
const absoluteRoot = path3.resolve(rootDir);
|
|
1144
1336
|
const projectHash = hashPath(absoluteRoot);
|
|
1145
1337
|
return {
|
|
1146
|
-
indexDir:
|
|
1338
|
+
indexDir: path3.join(absoluteRoot, RAGGREP_INDEX_DIR),
|
|
1147
1339
|
projectRoot: absoluteRoot,
|
|
1148
1340
|
projectHash
|
|
1149
1341
|
};
|
|
1150
1342
|
}
|
|
1151
1343
|
function getModuleIndexPath(rootDir, moduleId, config = DEFAULT_CONFIG) {
|
|
1152
1344
|
const indexDir = getRaggrepDir(rootDir, config);
|
|
1153
|
-
return
|
|
1345
|
+
return path3.join(indexDir, "index", moduleId);
|
|
1154
1346
|
}
|
|
1155
1347
|
function getModuleManifestPath(rootDir, moduleId, config = DEFAULT_CONFIG) {
|
|
1156
1348
|
const indexDir = getRaggrepDir(rootDir, config);
|
|
1157
|
-
return
|
|
1349
|
+
return path3.join(indexDir, "index", moduleId, "manifest.json");
|
|
1158
1350
|
}
|
|
1159
1351
|
function getGlobalManifestPath(rootDir, config = DEFAULT_CONFIG) {
|
|
1160
1352
|
const indexDir = getRaggrepDir(rootDir, config);
|
|
1161
|
-
return
|
|
1353
|
+
return path3.join(indexDir, "manifest.json");
|
|
1162
1354
|
}
|
|
1163
1355
|
function getConfigPath(rootDir, config = DEFAULT_CONFIG) {
|
|
1164
1356
|
const indexDir = getRaggrepDir(rootDir, config);
|
|
1165
|
-
return
|
|
1357
|
+
return path3.join(indexDir, "config.json");
|
|
1166
1358
|
}
|
|
1167
1359
|
async function loadConfig(rootDir) {
|
|
1168
1360
|
const configPath = getConfigPath(rootDir, DEFAULT_CONFIG);
|
|
1169
1361
|
try {
|
|
1170
|
-
const content = await
|
|
1362
|
+
const content = await fs2.readFile(configPath, "utf-8");
|
|
1171
1363
|
const savedConfig = JSON.parse(content);
|
|
1172
1364
|
return { ...DEFAULT_CONFIG, ...savedConfig };
|
|
1173
1365
|
} catch {
|
|
@@ -1180,27 +1372,28 @@ function getModuleConfig(config, moduleId) {
|
|
|
1180
1372
|
function getEmbeddingConfigFromModule(moduleConfig) {
|
|
1181
1373
|
const options = moduleConfig.options || {};
|
|
1182
1374
|
const modelName = options.embeddingModel || "bge-small-en-v1.5";
|
|
1183
|
-
if (!(modelName in
|
|
1375
|
+
if (!(modelName in EMBEDDING_MODELS)) {
|
|
1184
1376
|
console.warn(`Unknown embedding model: ${modelName}, falling back to bge-small-en-v1.5`);
|
|
1185
1377
|
return { model: "bge-small-en-v1.5" };
|
|
1186
1378
|
}
|
|
1379
|
+
const rt = options.embeddingRuntime;
|
|
1380
|
+
let runtime;
|
|
1381
|
+
if (rt === "xenova" || rt === "huggingface") {
|
|
1382
|
+
runtime = rt;
|
|
1383
|
+
} else if (rt !== undefined) {
|
|
1384
|
+
console.warn(`Unknown embeddingRuntime: ${rt}, falling back to default (huggingface)`);
|
|
1385
|
+
}
|
|
1187
1386
|
return {
|
|
1188
1387
|
model: modelName,
|
|
1388
|
+
...runtime ? { runtime } : {},
|
|
1189
1389
|
showProgress: options.showProgress === true
|
|
1190
1390
|
};
|
|
1191
1391
|
}
|
|
1192
|
-
var DEFAULT_CONFIG,
|
|
1392
|
+
var DEFAULT_CONFIG, RAGGREP_INDEX_DIR = ".raggrep";
|
|
1193
1393
|
var init_configLoader = __esm(() => {
|
|
1194
1394
|
init_entities();
|
|
1395
|
+
init_modelCatalog();
|
|
1195
1396
|
DEFAULT_CONFIG = createDefaultConfig();
|
|
1196
|
-
RAGGREP_TEMP_BASE = path2.join(os2.tmpdir(), "raggrep-indexes");
|
|
1197
|
-
EMBEDDING_MODELS2 = {
|
|
1198
|
-
"all-MiniLM-L6-v2": "Xenova/all-MiniLM-L6-v2",
|
|
1199
|
-
"all-MiniLM-L12-v2": "Xenova/all-MiniLM-L12-v2",
|
|
1200
|
-
"bge-small-en-v1.5": "Xenova/bge-small-en-v1.5",
|
|
1201
|
-
"paraphrase-MiniLM-L3-v2": "Xenova/paraphrase-MiniLM-L3-v2",
|
|
1202
|
-
"nomic-embed-text-v1.5": "nomic-ai/nomic-embed-text-v1.5"
|
|
1203
|
-
};
|
|
1204
1397
|
});
|
|
1205
1398
|
|
|
1206
1399
|
// src/infrastructure/config/index.ts
|
|
@@ -1338,10 +1531,10 @@ function normalizeScore(score, midpoint = 5) {
|
|
|
1338
1531
|
var BM25_K1 = 1.5, BM25_B = 0.75;
|
|
1339
1532
|
|
|
1340
1533
|
// src/domain/services/conventions/entryPoints.ts
|
|
1341
|
-
import * as
|
|
1534
|
+
import * as path4 from "path";
|
|
1342
1535
|
function getParentFolder(filepath) {
|
|
1343
|
-
const dir =
|
|
1344
|
-
return
|
|
1536
|
+
const dir = path4.dirname(filepath);
|
|
1537
|
+
return path4.basename(dir);
|
|
1345
1538
|
}
|
|
1346
1539
|
var entryPointConventions;
|
|
1347
1540
|
var init_entryPoints = __esm(() => {
|
|
@@ -2208,7 +2401,7 @@ var init_frameworks = __esm(() => {
|
|
|
2208
2401
|
});
|
|
2209
2402
|
|
|
2210
2403
|
// src/domain/services/conventions/index.ts
|
|
2211
|
-
import * as
|
|
2404
|
+
import * as path5 from "path";
|
|
2212
2405
|
function getConventions() {
|
|
2213
2406
|
return [
|
|
2214
2407
|
...entryPointConventions,
|
|
@@ -2220,8 +2413,8 @@ function getConventions() {
|
|
|
2220
2413
|
}
|
|
2221
2414
|
function getConventionKeywords(filepath) {
|
|
2222
2415
|
const conventions = getConventions();
|
|
2223
|
-
const filename =
|
|
2224
|
-
const extension =
|
|
2416
|
+
const filename = path5.basename(filepath);
|
|
2417
|
+
const extension = path5.extname(filepath);
|
|
2225
2418
|
const keywords = new Set;
|
|
2226
2419
|
for (const convention of conventions) {
|
|
2227
2420
|
try {
|
|
@@ -2309,13 +2502,13 @@ var init_conventions = __esm(() => {
|
|
|
2309
2502
|
});
|
|
2310
2503
|
|
|
2311
2504
|
// src/domain/services/introspection.ts
|
|
2312
|
-
import * as
|
|
2505
|
+
import * as path6 from "path";
|
|
2313
2506
|
function introspectFile(filepath, structure, options) {
|
|
2314
2507
|
const opts = typeof options === "string" ? { fileContent: options } : options || {};
|
|
2315
2508
|
const normalizedPath = filepath.replace(/\\/g, "/");
|
|
2316
2509
|
const segments = normalizedPath.split("/").filter((s) => s.length > 0);
|
|
2317
2510
|
const filename = segments[segments.length - 1] || "";
|
|
2318
|
-
const ext =
|
|
2511
|
+
const ext = path6.extname(filename);
|
|
2319
2512
|
const project = findProjectForFile(normalizedPath, structure);
|
|
2320
2513
|
const language = EXTENSION_TO_LANGUAGE[ext] || "unknown";
|
|
2321
2514
|
const layer = detectLayer(segments, filename);
|
|
@@ -2357,7 +2550,7 @@ function findNearestReadme(filepath, fileExists) {
|
|
|
2357
2550
|
}
|
|
2358
2551
|
function introspectionToKeywords(intro) {
|
|
2359
2552
|
const keywords = [];
|
|
2360
|
-
const filename =
|
|
2553
|
+
const filename = path6.basename(intro.filepath);
|
|
2361
2554
|
const filenameWithoutExt = filename.replace(/\.[^.]+$/, "");
|
|
2362
2555
|
const filenameParts = filenameWithoutExt.split(/[-_.]/).flatMap((part) => part.split(/(?=[A-Z])/)).map((part) => part.toLowerCase()).filter((part) => part.length > 1);
|
|
2363
2556
|
keywords.push(...filenameParts);
|
|
@@ -2836,8 +3029,8 @@ var exports_core = {};
|
|
|
2836
3029
|
__export(exports_core, {
|
|
2837
3030
|
CoreModule: () => CoreModule
|
|
2838
3031
|
});
|
|
2839
|
-
import * as
|
|
2840
|
-
import * as
|
|
3032
|
+
import * as path7 from "path";
|
|
3033
|
+
import * as fs3 from "fs/promises";
|
|
2841
3034
|
|
|
2842
3035
|
class CoreModule {
|
|
2843
3036
|
id = "core";
|
|
@@ -2934,8 +3127,8 @@ class CoreModule {
|
|
|
2934
3127
|
}
|
|
2935
3128
|
async finalize(ctx) {
|
|
2936
3129
|
const config = ctx.config;
|
|
2937
|
-
const coreDir =
|
|
2938
|
-
await
|
|
3130
|
+
const coreDir = path7.join(getRaggrepDir(ctx.rootDir, config), "index", "core");
|
|
3131
|
+
await fs3.mkdir(coreDir, { recursive: true });
|
|
2939
3132
|
this.bm25Index = new BM25Index;
|
|
2940
3133
|
for (const [filepath, entry] of this.symbolIndex) {
|
|
2941
3134
|
this.bm25Index.addDocument(filepath, entry.tokens);
|
|
@@ -2946,7 +3139,7 @@ class CoreModule {
|
|
|
2946
3139
|
files: Object.fromEntries(this.symbolIndex),
|
|
2947
3140
|
bm25Data: this.bm25Index.serialize()
|
|
2948
3141
|
};
|
|
2949
|
-
await
|
|
3142
|
+
await fs3.writeFile(path7.join(coreDir, "symbols.json"), JSON.stringify(symbolIndexData, null, 2));
|
|
2950
3143
|
}
|
|
2951
3144
|
async search(query, ctx, options) {
|
|
2952
3145
|
const config = ctx.config;
|
|
@@ -3042,10 +3235,10 @@ class CoreModule {
|
|
|
3042
3235
|
return bestChunk;
|
|
3043
3236
|
}
|
|
3044
3237
|
async loadSymbolIndex(rootDir, config) {
|
|
3045
|
-
const coreDir =
|
|
3046
|
-
const symbolsPath =
|
|
3238
|
+
const coreDir = path7.join(getRaggrepDir(rootDir, config), "index", "core");
|
|
3239
|
+
const symbolsPath = path7.join(coreDir, "symbols.json");
|
|
3047
3240
|
try {
|
|
3048
|
-
const content = await
|
|
3241
|
+
const content = await fs3.readFile(symbolsPath, "utf-8");
|
|
3049
3242
|
const data = JSON.parse(content);
|
|
3050
3243
|
this.symbolIndex = new Map(Object.entries(data.files));
|
|
3051
3244
|
if (data.bm25Data) {
|
|
@@ -3276,7 +3469,7 @@ function cosineSimilarity(a, b) {
|
|
|
3276
3469
|
}
|
|
3277
3470
|
|
|
3278
3471
|
// src/domain/services/queryIntent.ts
|
|
3279
|
-
import * as
|
|
3472
|
+
import * as path8 from "path";
|
|
3280
3473
|
function detectQueryIntent(queryTerms) {
|
|
3281
3474
|
const hasImplementationTerm = queryTerms.some((term) => IMPLEMENTATION_TERMS.includes(term));
|
|
3282
3475
|
const hasDocumentationTerm = queryTerms.some((term) => DOCUMENTATION_TERMS.includes(term));
|
|
@@ -3292,11 +3485,11 @@ function extractQueryTerms(query) {
|
|
|
3292
3485
|
return query.toLowerCase().split(/\s+/).filter((t) => t.length > 2);
|
|
3293
3486
|
}
|
|
3294
3487
|
function isSourceCodeFile(filepath) {
|
|
3295
|
-
const ext =
|
|
3488
|
+
const ext = path8.extname(filepath).toLowerCase();
|
|
3296
3489
|
return SOURCE_CODE_EXTENSIONS.includes(ext);
|
|
3297
3490
|
}
|
|
3298
3491
|
function isDocFile(filepath) {
|
|
3299
|
-
const ext =
|
|
3492
|
+
const ext = path8.extname(filepath).toLowerCase();
|
|
3300
3493
|
return DOC_EXTENSIONS.includes(ext);
|
|
3301
3494
|
}
|
|
3302
3495
|
function calculateFileTypeBoost(filepath, queryTerms) {
|
|
@@ -4371,8 +4564,8 @@ var init_lexicon2 = __esm(() => {
|
|
|
4371
4564
|
// src/domain/services/jsonPathExtractor.ts
|
|
4372
4565
|
function extractJsonPaths(obj, fileBasename) {
|
|
4373
4566
|
const paths = extractPathsRecursive(obj, fileBasename);
|
|
4374
|
-
return paths.map((
|
|
4375
|
-
value:
|
|
4567
|
+
return paths.map((path9) => ({
|
|
4568
|
+
value: path9,
|
|
4376
4569
|
type: "identifier",
|
|
4377
4570
|
matchType: "definition"
|
|
4378
4571
|
}));
|
|
@@ -4683,7 +4876,7 @@ var init_simpleSearch = __esm(() => {
|
|
|
4683
4876
|
});
|
|
4684
4877
|
|
|
4685
4878
|
// src/domain/services/chunkContext.ts
|
|
4686
|
-
import * as
|
|
4879
|
+
import * as path9 from "path";
|
|
4687
4880
|
function prepareChunkForEmbedding(options) {
|
|
4688
4881
|
const { filepath, content, name, docComment } = options;
|
|
4689
4882
|
const pathContext = parsePathContext(filepath);
|
|
@@ -4692,7 +4885,7 @@ function prepareChunkForEmbedding(options) {
|
|
|
4692
4885
|
if (pathPrefix) {
|
|
4693
4886
|
parts.push(pathPrefix);
|
|
4694
4887
|
}
|
|
4695
|
-
const filename =
|
|
4888
|
+
const filename = path9.basename(filepath);
|
|
4696
4889
|
const filenameWithoutExt = filename.replace(/\.[^.]+$/, "");
|
|
4697
4890
|
if (filenameWithoutExt && filenameWithoutExt.length > MIN_SEGMENT_LENGTH) {
|
|
4698
4891
|
const pathPrefixLower = pathPrefix.toLowerCase();
|
|
@@ -4922,11 +5115,12 @@ var init_parseCode = () => {};
|
|
|
4922
5115
|
// src/infrastructure/storage/fileIndexStorage.ts
|
|
4923
5116
|
var init_fileIndexStorage = __esm(() => {
|
|
4924
5117
|
init_entities();
|
|
5118
|
+
init_config2();
|
|
4925
5119
|
});
|
|
4926
5120
|
|
|
4927
5121
|
// src/infrastructure/storage/symbolicIndex.ts
|
|
4928
|
-
import * as
|
|
4929
|
-
import * as
|
|
5122
|
+
import * as fs4 from "fs/promises";
|
|
5123
|
+
import * as path10 from "path";
|
|
4930
5124
|
|
|
4931
5125
|
class SymbolicIndex {
|
|
4932
5126
|
meta = null;
|
|
@@ -4935,7 +5129,7 @@ class SymbolicIndex {
|
|
|
4935
5129
|
symbolicPath;
|
|
4936
5130
|
moduleId;
|
|
4937
5131
|
constructor(indexDir, moduleId) {
|
|
4938
|
-
this.symbolicPath =
|
|
5132
|
+
this.symbolicPath = path10.join(indexDir, "index", moduleId, "symbolic");
|
|
4939
5133
|
this.moduleId = moduleId;
|
|
4940
5134
|
}
|
|
4941
5135
|
async initialize() {
|
|
@@ -5021,13 +5215,13 @@ class SymbolicIndex {
|
|
|
5021
5215
|
if (this.bm25Index) {
|
|
5022
5216
|
this.meta.bm25Serialized = this.bm25Index.serialize();
|
|
5023
5217
|
}
|
|
5024
|
-
await
|
|
5025
|
-
const metaPath =
|
|
5026
|
-
await
|
|
5218
|
+
await fs4.mkdir(this.symbolicPath, { recursive: true });
|
|
5219
|
+
const metaPath = path10.join(this.symbolicPath, "_meta.json");
|
|
5220
|
+
await fs4.writeFile(metaPath, JSON.stringify(this.meta, null, 2));
|
|
5027
5221
|
for (const [filepath, summary] of this.fileSummaries) {
|
|
5028
5222
|
const summaryPath = this.getFileSummaryPath(filepath);
|
|
5029
|
-
await
|
|
5030
|
-
await
|
|
5223
|
+
await fs4.mkdir(path10.dirname(summaryPath), { recursive: true });
|
|
5224
|
+
await fs4.writeFile(summaryPath, JSON.stringify(summary, null, 2));
|
|
5031
5225
|
}
|
|
5032
5226
|
}
|
|
5033
5227
|
async saveIncremental(filepaths) {
|
|
@@ -5038,21 +5232,21 @@ class SymbolicIndex {
|
|
|
5038
5232
|
if (this.bm25Index) {
|
|
5039
5233
|
this.meta.bm25Serialized = this.bm25Index.serialize();
|
|
5040
5234
|
}
|
|
5041
|
-
await
|
|
5042
|
-
const metaPath =
|
|
5043
|
-
await
|
|
5235
|
+
await fs4.mkdir(this.symbolicPath, { recursive: true });
|
|
5236
|
+
const metaPath = path10.join(this.symbolicPath, "_meta.json");
|
|
5237
|
+
await fs4.writeFile(metaPath, JSON.stringify(this.meta, null, 2));
|
|
5044
5238
|
for (const filepath of filepaths) {
|
|
5045
5239
|
const summary = this.fileSummaries.get(filepath);
|
|
5046
5240
|
if (summary) {
|
|
5047
5241
|
const summaryPath = this.getFileSummaryPath(filepath);
|
|
5048
|
-
await
|
|
5049
|
-
await
|
|
5242
|
+
await fs4.mkdir(path10.dirname(summaryPath), { recursive: true });
|
|
5243
|
+
await fs4.writeFile(summaryPath, JSON.stringify(summary, null, 2));
|
|
5050
5244
|
}
|
|
5051
5245
|
}
|
|
5052
5246
|
}
|
|
5053
5247
|
async load() {
|
|
5054
|
-
const metaPath =
|
|
5055
|
-
const metaContent = await
|
|
5248
|
+
const metaPath = path10.join(this.symbolicPath, "_meta.json");
|
|
5249
|
+
const metaContent = await fs4.readFile(metaPath, "utf-8");
|
|
5056
5250
|
this.meta = JSON.parse(metaContent);
|
|
5057
5251
|
this.fileSummaries.clear();
|
|
5058
5252
|
await this.loadFileSummariesRecursive(this.symbolicPath);
|
|
@@ -5064,14 +5258,14 @@ class SymbolicIndex {
|
|
|
5064
5258
|
}
|
|
5065
5259
|
async loadFileSummariesRecursive(dir) {
|
|
5066
5260
|
try {
|
|
5067
|
-
const entries = await
|
|
5261
|
+
const entries = await fs4.readdir(dir, { withFileTypes: true });
|
|
5068
5262
|
for (const entry of entries) {
|
|
5069
|
-
const fullPath =
|
|
5263
|
+
const fullPath = path10.join(dir, entry.name);
|
|
5070
5264
|
if (entry.isDirectory()) {
|
|
5071
5265
|
await this.loadFileSummariesRecursive(fullPath);
|
|
5072
5266
|
} else if (entry.name.endsWith(".json") && entry.name !== "_meta.json") {
|
|
5073
5267
|
try {
|
|
5074
|
-
const content = await
|
|
5268
|
+
const content = await fs4.readFile(fullPath, "utf-8");
|
|
5075
5269
|
const summary = JSON.parse(content);
|
|
5076
5270
|
if (summary.filepath) {
|
|
5077
5271
|
this.fileSummaries.set(summary.filepath, summary);
|
|
@@ -5083,18 +5277,18 @@ class SymbolicIndex {
|
|
|
5083
5277
|
}
|
|
5084
5278
|
getFileSummaryPath(filepath) {
|
|
5085
5279
|
const jsonPath = filepath.replace(/\.[^.]+$/, ".json");
|
|
5086
|
-
return
|
|
5280
|
+
return path10.join(this.symbolicPath, jsonPath);
|
|
5087
5281
|
}
|
|
5088
5282
|
async deleteFileSummary(filepath) {
|
|
5089
5283
|
try {
|
|
5090
|
-
await
|
|
5284
|
+
await fs4.unlink(this.getFileSummaryPath(filepath));
|
|
5091
5285
|
} catch {}
|
|
5092
5286
|
this.fileSummaries.delete(filepath);
|
|
5093
5287
|
}
|
|
5094
5288
|
async exists() {
|
|
5095
5289
|
try {
|
|
5096
|
-
const metaPath =
|
|
5097
|
-
await
|
|
5290
|
+
const metaPath = path10.join(this.symbolicPath, "_meta.json");
|
|
5291
|
+
await fs4.access(metaPath);
|
|
5098
5292
|
return true;
|
|
5099
5293
|
} catch {
|
|
5100
5294
|
return false;
|
|
@@ -5126,8 +5320,8 @@ __export(exports_literalIndex, {
|
|
|
5126
5320
|
getLiteralIndexPath: () => getLiteralIndexPath,
|
|
5127
5321
|
LiteralIndex: () => LiteralIndex
|
|
5128
5322
|
});
|
|
5129
|
-
import * as
|
|
5130
|
-
import * as
|
|
5323
|
+
import * as fs5 from "fs/promises";
|
|
5324
|
+
import * as path11 from "path";
|
|
5131
5325
|
|
|
5132
5326
|
class LiteralIndex {
|
|
5133
5327
|
indexPath;
|
|
@@ -5136,7 +5330,7 @@ class LiteralIndex {
|
|
|
5136
5330
|
vocabularyIndex = new Map;
|
|
5137
5331
|
static VERSION = "1.1.0";
|
|
5138
5332
|
constructor(indexDir, moduleId) {
|
|
5139
|
-
this.indexPath =
|
|
5333
|
+
this.indexPath = path11.join(indexDir, "index", moduleId, "literals");
|
|
5140
5334
|
this.moduleId = moduleId;
|
|
5141
5335
|
}
|
|
5142
5336
|
async initialize() {
|
|
@@ -5297,17 +5491,17 @@ class LiteralIndex {
|
|
|
5297
5491
|
}));
|
|
5298
5492
|
}
|
|
5299
5493
|
async save() {
|
|
5300
|
-
await
|
|
5494
|
+
await fs5.mkdir(this.indexPath, { recursive: true });
|
|
5301
5495
|
const data = {
|
|
5302
5496
|
version: LiteralIndex.VERSION,
|
|
5303
5497
|
entries: Object.fromEntries(this.entries)
|
|
5304
5498
|
};
|
|
5305
|
-
const indexFile =
|
|
5306
|
-
await
|
|
5499
|
+
const indexFile = path11.join(this.indexPath, "_index.json");
|
|
5500
|
+
await fs5.writeFile(indexFile, JSON.stringify(data, null, 2));
|
|
5307
5501
|
}
|
|
5308
5502
|
async load() {
|
|
5309
|
-
const indexFile =
|
|
5310
|
-
const content = await
|
|
5503
|
+
const indexFile = path11.join(this.indexPath, "_index.json");
|
|
5504
|
+
const content = await fs5.readFile(indexFile, "utf-8");
|
|
5311
5505
|
const data = JSON.parse(content);
|
|
5312
5506
|
if (data.version !== LiteralIndex.VERSION) {
|
|
5313
5507
|
console.warn(`Literal index version mismatch: expected ${LiteralIndex.VERSION}, got ${data.version}`);
|
|
@@ -5316,8 +5510,8 @@ class LiteralIndex {
|
|
|
5316
5510
|
}
|
|
5317
5511
|
async exists() {
|
|
5318
5512
|
try {
|
|
5319
|
-
const indexFile =
|
|
5320
|
-
await
|
|
5513
|
+
const indexFile = path11.join(this.indexPath, "_index.json");
|
|
5514
|
+
await fs5.access(indexFile);
|
|
5321
5515
|
return true;
|
|
5322
5516
|
} catch {
|
|
5323
5517
|
return false;
|
|
@@ -5360,7 +5554,7 @@ function shouldReplaceMatchType(existing, incoming) {
|
|
|
5360
5554
|
return priority[incoming] > priority[existing];
|
|
5361
5555
|
}
|
|
5362
5556
|
function getLiteralIndexPath(rootDir, moduleId, indexDir = ".raggrep") {
|
|
5363
|
-
return
|
|
5557
|
+
return path11.join(rootDir, indexDir, "index", moduleId, "literals");
|
|
5364
5558
|
}
|
|
5365
5559
|
var init_literalIndex = () => {};
|
|
5366
5560
|
|
|
@@ -5381,9 +5575,9 @@ __export(exports_typescript, {
|
|
|
5381
5575
|
DEFAULT_TOP_K: () => DEFAULT_TOP_K2,
|
|
5382
5576
|
DEFAULT_MIN_SCORE: () => DEFAULT_MIN_SCORE2
|
|
5383
5577
|
});
|
|
5384
|
-
import * as
|
|
5578
|
+
import * as path12 from "path";
|
|
5385
5579
|
function isTypeScriptFile(filepath) {
|
|
5386
|
-
const ext =
|
|
5580
|
+
const ext = path12.extname(filepath).toLowerCase();
|
|
5387
5581
|
return TYPESCRIPT_EXTENSIONS.includes(ext);
|
|
5388
5582
|
}
|
|
5389
5583
|
function calculateChunkTypeBoost(chunk) {
|
|
@@ -5455,7 +5649,7 @@ class TypeScriptModule {
|
|
|
5455
5649
|
startLine: 1,
|
|
5456
5650
|
endLine: lines.length,
|
|
5457
5651
|
type: "file",
|
|
5458
|
-
name:
|
|
5652
|
+
name: path12.basename(filepath),
|
|
5459
5653
|
isExported: false
|
|
5460
5654
|
});
|
|
5461
5655
|
}
|
|
@@ -5782,16 +5976,16 @@ class TypeScriptModule {
|
|
|
5782
5976
|
while ((match = importRegex.exec(content)) !== null) {
|
|
5783
5977
|
const importPath = match[1];
|
|
5784
5978
|
if (importPath.startsWith(".")) {
|
|
5785
|
-
const dir =
|
|
5786
|
-
const resolved =
|
|
5979
|
+
const dir = path12.dirname(filepath);
|
|
5980
|
+
const resolved = path12.normalize(path12.join(dir, importPath));
|
|
5787
5981
|
references.push(resolved);
|
|
5788
5982
|
}
|
|
5789
5983
|
}
|
|
5790
5984
|
while ((match = requireRegex.exec(content)) !== null) {
|
|
5791
5985
|
const importPath = match[1];
|
|
5792
5986
|
if (importPath.startsWith(".")) {
|
|
5793
|
-
const dir =
|
|
5794
|
-
const resolved =
|
|
5987
|
+
const dir = path12.dirname(filepath);
|
|
5988
|
+
const resolved = path12.normalize(path12.join(dir, importPath));
|
|
5795
5989
|
references.push(resolved);
|
|
5796
5990
|
}
|
|
5797
5991
|
}
|
|
@@ -5819,7 +6013,7 @@ var init_typescript = __esm(() => {
|
|
|
5819
6013
|
});
|
|
5820
6014
|
|
|
5821
6015
|
// src/infrastructure/parsing/typescriptParser.ts
|
|
5822
|
-
import * as
|
|
6016
|
+
import * as path13 from "path";
|
|
5823
6017
|
|
|
5824
6018
|
class TypeScriptParser {
|
|
5825
6019
|
supportedLanguages = ["typescript", "javascript"];
|
|
@@ -5835,12 +6029,12 @@ class TypeScriptParser {
|
|
|
5835
6029
|
startLine: 1,
|
|
5836
6030
|
endLine: lines.length,
|
|
5837
6031
|
type: "file",
|
|
5838
|
-
name:
|
|
6032
|
+
name: path13.basename(filepath),
|
|
5839
6033
|
isExported: false
|
|
5840
6034
|
};
|
|
5841
6035
|
chunks.unshift(fullFileChunk);
|
|
5842
6036
|
}
|
|
5843
|
-
const ext =
|
|
6037
|
+
const ext = path13.extname(filepath).toLowerCase();
|
|
5844
6038
|
const language = ext === ".js" || ext === ".jsx" || ext === ".mjs" || ext === ".cjs" ? "javascript" : "typescript";
|
|
5845
6039
|
return {
|
|
5846
6040
|
chunks,
|
|
@@ -5857,7 +6051,7 @@ class TypeScriptParser {
|
|
|
5857
6051
|
}
|
|
5858
6052
|
}
|
|
5859
6053
|
canParse(filepath) {
|
|
5860
|
-
const ext =
|
|
6054
|
+
const ext = path13.extname(filepath).toLowerCase();
|
|
5861
6055
|
return TYPESCRIPT_EXTENSIONS2.includes(ext);
|
|
5862
6056
|
}
|
|
5863
6057
|
convertChunk(tc) {
|
|
@@ -5872,7 +6066,7 @@ class TypeScriptParser {
|
|
|
5872
6066
|
};
|
|
5873
6067
|
}
|
|
5874
6068
|
detectLanguage(filepath) {
|
|
5875
|
-
const ext =
|
|
6069
|
+
const ext = path13.extname(filepath).toLowerCase();
|
|
5876
6070
|
if ([".js", ".jsx", ".mjs", ".cjs"].includes(ext)) {
|
|
5877
6071
|
return "javascript";
|
|
5878
6072
|
}
|
|
@@ -6077,8 +6271,8 @@ var init_grammarManager = __esm(() => {
|
|
|
6077
6271
|
});
|
|
6078
6272
|
|
|
6079
6273
|
// src/infrastructure/parsing/treeSitterParser.ts
|
|
6080
|
-
import * as
|
|
6081
|
-
import * as
|
|
6274
|
+
import * as path14 from "path";
|
|
6275
|
+
import * as fs6 from "fs";
|
|
6082
6276
|
|
|
6083
6277
|
class TreeSitterParser {
|
|
6084
6278
|
supportedLanguages = [
|
|
@@ -6100,7 +6294,7 @@ class TreeSitterParser {
|
|
|
6100
6294
|
chunks: [],
|
|
6101
6295
|
language: "typescript",
|
|
6102
6296
|
success: false,
|
|
6103
|
-
error: `Unsupported file type: ${
|
|
6297
|
+
error: `Unsupported file type: ${path14.extname(filepath)}`
|
|
6104
6298
|
};
|
|
6105
6299
|
}
|
|
6106
6300
|
try {
|
|
@@ -6120,11 +6314,11 @@ class TreeSitterParser {
|
|
|
6120
6314
|
}
|
|
6121
6315
|
}
|
|
6122
6316
|
canParse(filepath) {
|
|
6123
|
-
const ext =
|
|
6317
|
+
const ext = path14.extname(filepath).toLowerCase();
|
|
6124
6318
|
return ext in EXTENSION_TO_LANGUAGE2;
|
|
6125
6319
|
}
|
|
6126
6320
|
detectLanguage(filepath) {
|
|
6127
|
-
const ext =
|
|
6321
|
+
const ext = path14.extname(filepath).toLowerCase();
|
|
6128
6322
|
return EXTENSION_TO_LANGUAGE2[ext] || null;
|
|
6129
6323
|
}
|
|
6130
6324
|
async ensureInitialized() {
|
|
@@ -6158,20 +6352,20 @@ class TreeSitterParser {
|
|
|
6158
6352
|
async resolveWasmPath() {
|
|
6159
6353
|
try {
|
|
6160
6354
|
const webTreeSitterPath = __require.resolve("web-tree-sitter");
|
|
6161
|
-
const wasmPath =
|
|
6162
|
-
if (
|
|
6355
|
+
const wasmPath = path14.join(path14.dirname(webTreeSitterPath), "web-tree-sitter.wasm");
|
|
6356
|
+
if (fs6.existsSync(wasmPath)) {
|
|
6163
6357
|
return wasmPath;
|
|
6164
6358
|
}
|
|
6165
6359
|
} catch {}
|
|
6166
6360
|
try {
|
|
6167
6361
|
const possiblePaths = [
|
|
6168
|
-
|
|
6169
|
-
|
|
6170
|
-
|
|
6171
|
-
|
|
6362
|
+
path14.join(__dirname, "../../../node_modules/web-tree-sitter/web-tree-sitter.wasm"),
|
|
6363
|
+
path14.join(__dirname, "../../node_modules/web-tree-sitter/web-tree-sitter.wasm"),
|
|
6364
|
+
path14.join(__dirname, "../../../../node_modules/web-tree-sitter/web-tree-sitter.wasm"),
|
|
6365
|
+
path14.join(__dirname, "web-tree-sitter.wasm")
|
|
6172
6366
|
];
|
|
6173
6367
|
for (const wasmPath of possiblePaths) {
|
|
6174
|
-
if (
|
|
6368
|
+
if (fs6.existsSync(wasmPath)) {
|
|
6175
6369
|
return wasmPath;
|
|
6176
6370
|
}
|
|
6177
6371
|
}
|
|
@@ -6212,7 +6406,7 @@ class TreeSitterParser {
|
|
|
6212
6406
|
startLine: 1,
|
|
6213
6407
|
endLine: lines.length,
|
|
6214
6408
|
type: "file",
|
|
6215
|
-
name:
|
|
6409
|
+
name: path14.basename(filepath),
|
|
6216
6410
|
isExported: false
|
|
6217
6411
|
});
|
|
6218
6412
|
}
|
|
@@ -6559,7 +6753,7 @@ class TreeSitterParser {
|
|
|
6559
6753
|
startLine: 1,
|
|
6560
6754
|
endLine: lines.length,
|
|
6561
6755
|
type: "file",
|
|
6562
|
-
name:
|
|
6756
|
+
name: path14.basename(filepath)
|
|
6563
6757
|
});
|
|
6564
6758
|
return {
|
|
6565
6759
|
chunks,
|
|
@@ -6568,7 +6762,7 @@ class TreeSitterParser {
|
|
|
6568
6762
|
};
|
|
6569
6763
|
}
|
|
6570
6764
|
}
|
|
6571
|
-
var __dirname = "/
|
|
6765
|
+
var __dirname = "/home/runner/work/raggrep/raggrep/src/infrastructure/parsing", EXTENSION_TO_LANGUAGE2;
|
|
6572
6766
|
var init_treeSitterParser = __esm(() => {
|
|
6573
6767
|
init_grammarManager();
|
|
6574
6768
|
EXTENSION_TO_LANGUAGE2 = {
|
|
@@ -6589,7 +6783,7 @@ var init_treeSitterParser = __esm(() => {
|
|
|
6589
6783
|
});
|
|
6590
6784
|
|
|
6591
6785
|
// src/infrastructure/parsing/parserFactory.ts
|
|
6592
|
-
import * as
|
|
6786
|
+
import * as path15 from "path";
|
|
6593
6787
|
function getTypeScriptParser() {
|
|
6594
6788
|
if (!typescriptParserInstance) {
|
|
6595
6789
|
typescriptParserInstance = new TypeScriptParser;
|
|
@@ -6603,7 +6797,7 @@ function getTreeSitterParser() {
|
|
|
6603
6797
|
return treeSitterParserInstance;
|
|
6604
6798
|
}
|
|
6605
6799
|
function createParserForFile(filepath) {
|
|
6606
|
-
const ext =
|
|
6800
|
+
const ext = path15.extname(filepath).toLowerCase();
|
|
6607
6801
|
const parserType = EXTENSION_PARSER_MAP[ext];
|
|
6608
6802
|
if (!parserType) {
|
|
6609
6803
|
return null;
|
|
@@ -6652,9 +6846,9 @@ __export(exports_python, {
|
|
|
6652
6846
|
DEFAULT_TOP_K: () => DEFAULT_TOP_K3,
|
|
6653
6847
|
DEFAULT_MIN_SCORE: () => DEFAULT_MIN_SCORE3
|
|
6654
6848
|
});
|
|
6655
|
-
import * as
|
|
6849
|
+
import * as path16 from "path";
|
|
6656
6850
|
function isPythonFile(filepath) {
|
|
6657
|
-
const ext =
|
|
6851
|
+
const ext = path16.extname(filepath).toLowerCase();
|
|
6658
6852
|
return PYTHON_EXTENSIONS.includes(ext);
|
|
6659
6853
|
}
|
|
6660
6854
|
function generateChunkId3(filepath, startLine, endLine) {
|
|
@@ -6738,7 +6932,7 @@ class PythonModule {
|
|
|
6738
6932
|
startLine: 1,
|
|
6739
6933
|
endLine: lines.length,
|
|
6740
6934
|
type: "file",
|
|
6741
|
-
name:
|
|
6935
|
+
name: path16.basename(filepath)
|
|
6742
6936
|
});
|
|
6743
6937
|
const funcRegex = /^(\s*)(async\s+)?def\s+(\w+)\s*\([^)]*\)\s*:/gm;
|
|
6744
6938
|
let match;
|
|
@@ -7112,9 +7306,9 @@ __export(exports_go, {
|
|
|
7112
7306
|
DEFAULT_TOP_K: () => DEFAULT_TOP_K4,
|
|
7113
7307
|
DEFAULT_MIN_SCORE: () => DEFAULT_MIN_SCORE4
|
|
7114
7308
|
});
|
|
7115
|
-
import * as
|
|
7309
|
+
import * as path17 from "path";
|
|
7116
7310
|
function isGoFile(filepath) {
|
|
7117
|
-
const ext =
|
|
7311
|
+
const ext = path17.extname(filepath).toLowerCase();
|
|
7118
7312
|
return GO_EXTENSIONS.includes(ext);
|
|
7119
7313
|
}
|
|
7120
7314
|
function generateChunkId4(filepath, startLine, endLine) {
|
|
@@ -7199,7 +7393,7 @@ class GoModule {
|
|
|
7199
7393
|
startLine: 1,
|
|
7200
7394
|
endLine: lines.length,
|
|
7201
7395
|
type: "file",
|
|
7202
|
-
name:
|
|
7396
|
+
name: path17.basename(filepath)
|
|
7203
7397
|
});
|
|
7204
7398
|
const funcRegex = /^func\s+(?:\(\s*\w+\s+\*?\w+\s*\)\s+)?(\w+)\s*\(/gm;
|
|
7205
7399
|
let match;
|
|
@@ -7593,9 +7787,9 @@ __export(exports_rust, {
|
|
|
7593
7787
|
DEFAULT_TOP_K: () => DEFAULT_TOP_K5,
|
|
7594
7788
|
DEFAULT_MIN_SCORE: () => DEFAULT_MIN_SCORE5
|
|
7595
7789
|
});
|
|
7596
|
-
import * as
|
|
7790
|
+
import * as path18 from "path";
|
|
7597
7791
|
function isRustFile(filepath) {
|
|
7598
|
-
const ext =
|
|
7792
|
+
const ext = path18.extname(filepath).toLowerCase();
|
|
7599
7793
|
return RUST_EXTENSIONS.includes(ext);
|
|
7600
7794
|
}
|
|
7601
7795
|
function generateChunkId5(filepath, startLine, endLine) {
|
|
@@ -7682,7 +7876,7 @@ class RustModule {
|
|
|
7682
7876
|
startLine: 1,
|
|
7683
7877
|
endLine: lines.length,
|
|
7684
7878
|
type: "file",
|
|
7685
|
-
name:
|
|
7879
|
+
name: path18.basename(filepath)
|
|
7686
7880
|
});
|
|
7687
7881
|
const funcRegex = /^(pub(?:\s*\([^)]*\))?\s+)?(?:async\s+)?fn\s+(\w+)/gm;
|
|
7688
7882
|
let match;
|
|
@@ -8153,9 +8347,9 @@ __export(exports_json, {
|
|
|
8153
8347
|
DEFAULT_TOP_K: () => DEFAULT_TOP_K6,
|
|
8154
8348
|
DEFAULT_MIN_SCORE: () => DEFAULT_MIN_SCORE6
|
|
8155
8349
|
});
|
|
8156
|
-
import * as
|
|
8350
|
+
import * as path19 from "path";
|
|
8157
8351
|
function isJsonFile(filepath) {
|
|
8158
|
-
const ext =
|
|
8352
|
+
const ext = path19.extname(filepath).toLowerCase();
|
|
8159
8353
|
return JSON_EXTENSIONS.includes(ext);
|
|
8160
8354
|
}
|
|
8161
8355
|
|
|
@@ -8189,7 +8383,7 @@ class JsonModule {
|
|
|
8189
8383
|
} catch {
|
|
8190
8384
|
return null;
|
|
8191
8385
|
}
|
|
8192
|
-
const fileBasename =
|
|
8386
|
+
const fileBasename = path19.basename(filepath, path19.extname(filepath));
|
|
8193
8387
|
const jsonPathLiterals = extractJsonPaths(parsed, fileBasename);
|
|
8194
8388
|
const lines = content.split(`
|
|
8195
8389
|
`);
|
|
@@ -8396,7 +8590,7 @@ __export(exports_markdown, {
|
|
|
8396
8590
|
DEFAULT_TOP_K: () => DEFAULT_TOP_K7,
|
|
8397
8591
|
DEFAULT_MIN_SCORE: () => DEFAULT_MIN_SCORE7
|
|
8398
8592
|
});
|
|
8399
|
-
import * as
|
|
8593
|
+
import * as path20 from "path";
|
|
8400
8594
|
function calculateHeadingLevelBoost(chunk) {
|
|
8401
8595
|
const metadata = chunk.metadata;
|
|
8402
8596
|
const level = metadata?.headingLevel ?? 0;
|
|
@@ -8416,7 +8610,7 @@ function calculateHeadingLevelBoost(chunk) {
|
|
|
8416
8610
|
}
|
|
8417
8611
|
}
|
|
8418
8612
|
function isMarkdownFile(filepath) {
|
|
8419
|
-
const ext =
|
|
8613
|
+
const ext = path20.extname(filepath).toLowerCase();
|
|
8420
8614
|
return MARKDOWN_EXTENSIONS.includes(ext);
|
|
8421
8615
|
}
|
|
8422
8616
|
function parseMarkdownHierarchical(content, maxDepth = 4) {
|
|
@@ -8758,15 +8952,15 @@ var init_registry = __esm(() => {
|
|
|
8758
8952
|
});
|
|
8759
8953
|
|
|
8760
8954
|
// src/infrastructure/introspection/projectDetector.ts
|
|
8761
|
-
import * as
|
|
8762
|
-
import * as
|
|
8955
|
+
import * as path21 from "path";
|
|
8956
|
+
import * as fs7 from "fs/promises";
|
|
8763
8957
|
async function scanForPackageJsons(rootDir, currentDir = "", depth = 0) {
|
|
8764
8958
|
if (depth > MAX_SCAN_DEPTH)
|
|
8765
8959
|
return [];
|
|
8766
8960
|
const results = [];
|
|
8767
|
-
const fullDir = currentDir ?
|
|
8961
|
+
const fullDir = currentDir ? path21.join(rootDir, currentDir) : rootDir;
|
|
8768
8962
|
try {
|
|
8769
|
-
const entries = await
|
|
8963
|
+
const entries = await fs7.readdir(fullDir, { withFileTypes: true });
|
|
8770
8964
|
const hasPackageJson = entries.some((e) => e.isFile() && e.name === "package.json");
|
|
8771
8965
|
if (hasPackageJson && currentDir) {
|
|
8772
8966
|
const info = await parsePackageJson(rootDir, currentDir);
|
|
@@ -8787,10 +8981,10 @@ async function scanForPackageJsons(rootDir, currentDir = "", depth = 0) {
|
|
|
8787
8981
|
}
|
|
8788
8982
|
async function parsePackageJson(rootDir, relativePath) {
|
|
8789
8983
|
try {
|
|
8790
|
-
const packageJsonPath =
|
|
8791
|
-
const content = await
|
|
8984
|
+
const packageJsonPath = path21.join(rootDir, relativePath, "package.json");
|
|
8985
|
+
const content = await fs7.readFile(packageJsonPath, "utf-8");
|
|
8792
8986
|
const pkg = JSON.parse(content);
|
|
8793
|
-
const name = pkg.name ||
|
|
8987
|
+
const name = pkg.name || path21.basename(relativePath);
|
|
8794
8988
|
const deps = { ...pkg.dependencies, ...pkg.devDependencies };
|
|
8795
8989
|
let type = "unknown";
|
|
8796
8990
|
if (deps["next"] || deps["react"] || deps["vue"] || deps["svelte"]) {
|
|
@@ -8826,7 +9020,7 @@ async function detectProjectStructure(rootDir) {
|
|
|
8826
9020
|
const projectMap = new Map;
|
|
8827
9021
|
let isMonorepo = false;
|
|
8828
9022
|
try {
|
|
8829
|
-
const entries = await
|
|
9023
|
+
const entries = await fs7.readdir(rootDir, { withFileTypes: true });
|
|
8830
9024
|
const dirNames = entries.filter((e) => e.isDirectory()).map((e) => e.name);
|
|
8831
9025
|
const monorepoPatterns = ["apps", "packages", "libs", "services"];
|
|
8832
9026
|
const hasMonorepoStructure = monorepoPatterns.some((p) => dirNames.includes(p));
|
|
@@ -8835,9 +9029,9 @@ async function detectProjectStructure(rootDir) {
|
|
|
8835
9029
|
for (const pattern of monorepoPatterns) {
|
|
8836
9030
|
if (!dirNames.includes(pattern))
|
|
8837
9031
|
continue;
|
|
8838
|
-
const patternDir =
|
|
9032
|
+
const patternDir = path21.join(rootDir, pattern);
|
|
8839
9033
|
try {
|
|
8840
|
-
const subDirs = await
|
|
9034
|
+
const subDirs = await fs7.readdir(patternDir, { withFileTypes: true });
|
|
8841
9035
|
for (const subDir of subDirs) {
|
|
8842
9036
|
if (!subDir.isDirectory())
|
|
8843
9037
|
continue;
|
|
@@ -8866,8 +9060,8 @@ async function detectProjectStructure(rootDir) {
|
|
|
8866
9060
|
}
|
|
8867
9061
|
let rootType = "unknown";
|
|
8868
9062
|
try {
|
|
8869
|
-
const rootPkgPath =
|
|
8870
|
-
const rootPkg = JSON.parse(await
|
|
9063
|
+
const rootPkgPath = path21.join(rootDir, "package.json");
|
|
9064
|
+
const rootPkg = JSON.parse(await fs7.readFile(rootPkgPath, "utf-8"));
|
|
8871
9065
|
if (rootPkg.workspaces)
|
|
8872
9066
|
isMonorepo = true;
|
|
8873
9067
|
const deps = { ...rootPkg.dependencies, ...rootPkg.devDependencies };
|
|
@@ -8906,8 +9100,8 @@ var init_projectDetector = __esm(() => {
|
|
|
8906
9100
|
});
|
|
8907
9101
|
|
|
8908
9102
|
// src/infrastructure/introspection/IntrospectionIndex.ts
|
|
8909
|
-
import * as
|
|
8910
|
-
import * as
|
|
9103
|
+
import * as path22 from "path";
|
|
9104
|
+
import * as fs8 from "fs/promises";
|
|
8911
9105
|
import * as fsSync from "fs";
|
|
8912
9106
|
|
|
8913
9107
|
class IntrospectionIndex {
|
|
@@ -8921,8 +9115,8 @@ class IntrospectionIndex {
|
|
|
8921
9115
|
async initialize() {
|
|
8922
9116
|
this.structure = await detectProjectStructure(this.rootDir);
|
|
8923
9117
|
try {
|
|
8924
|
-
const configPath =
|
|
8925
|
-
const configContent = await
|
|
9118
|
+
const configPath = path22.join(this.rootDir, ".raggrep", "config.json");
|
|
9119
|
+
const configContent = await fs8.readFile(configPath, "utf-8");
|
|
8926
9120
|
const config = JSON.parse(configContent);
|
|
8927
9121
|
this.config = config.introspection || {};
|
|
8928
9122
|
} catch {}
|
|
@@ -8936,7 +9130,7 @@ class IntrospectionIndex {
|
|
|
8936
9130
|
}
|
|
8937
9131
|
const fileExists = enableReadmeContext ? (relativePath) => {
|
|
8938
9132
|
try {
|
|
8939
|
-
const absolutePath =
|
|
9133
|
+
const absolutePath = path22.join(this.rootDir, relativePath);
|
|
8940
9134
|
return fsSync.existsSync(absolutePath);
|
|
8941
9135
|
} catch {
|
|
8942
9136
|
return false;
|
|
@@ -8972,28 +9166,28 @@ class IntrospectionIndex {
|
|
|
8972
9166
|
}
|
|
8973
9167
|
}
|
|
8974
9168
|
async save(config) {
|
|
8975
|
-
const introDir =
|
|
8976
|
-
await
|
|
8977
|
-
const projectPath =
|
|
8978
|
-
await
|
|
9169
|
+
const introDir = path22.join(getRaggrepDir(this.rootDir, config), "introspection");
|
|
9170
|
+
await fs8.mkdir(introDir, { recursive: true });
|
|
9171
|
+
const projectPath = path22.join(introDir, "_project.json");
|
|
9172
|
+
await fs8.writeFile(projectPath, JSON.stringify({
|
|
8979
9173
|
version: "1.0.0",
|
|
8980
9174
|
lastUpdated: new Date().toISOString(),
|
|
8981
9175
|
structure: this.structure
|
|
8982
9176
|
}, null, 2));
|
|
8983
9177
|
for (const [filepath, intro] of this.files) {
|
|
8984
|
-
const introFilePath =
|
|
8985
|
-
await
|
|
8986
|
-
await
|
|
9178
|
+
const introFilePath = path22.join(introDir, "files", filepath.replace(/\.[^.]+$/, ".json"));
|
|
9179
|
+
await fs8.mkdir(path22.dirname(introFilePath), { recursive: true });
|
|
9180
|
+
await fs8.writeFile(introFilePath, JSON.stringify(intro, null, 2));
|
|
8987
9181
|
}
|
|
8988
9182
|
}
|
|
8989
9183
|
async load(config) {
|
|
8990
|
-
const introDir =
|
|
9184
|
+
const introDir = path22.join(getRaggrepDir(this.rootDir, config), "introspection");
|
|
8991
9185
|
try {
|
|
8992
|
-
const projectPath =
|
|
8993
|
-
const projectContent = await
|
|
9186
|
+
const projectPath = path22.join(introDir, "_project.json");
|
|
9187
|
+
const projectContent = await fs8.readFile(projectPath, "utf-8");
|
|
8994
9188
|
const projectData = JSON.parse(projectContent);
|
|
8995
9189
|
this.structure = projectData.structure;
|
|
8996
|
-
await this.loadFilesRecursive(
|
|
9190
|
+
await this.loadFilesRecursive(path22.join(introDir, "files"), "");
|
|
8997
9191
|
} catch {
|
|
8998
9192
|
this.structure = null;
|
|
8999
9193
|
this.files.clear();
|
|
@@ -9001,14 +9195,14 @@ class IntrospectionIndex {
|
|
|
9001
9195
|
}
|
|
9002
9196
|
async loadFilesRecursive(basePath, prefix) {
|
|
9003
9197
|
try {
|
|
9004
|
-
const entries = await
|
|
9198
|
+
const entries = await fs8.readdir(basePath, { withFileTypes: true });
|
|
9005
9199
|
for (const entry of entries) {
|
|
9006
|
-
const entryPath =
|
|
9200
|
+
const entryPath = path22.join(basePath, entry.name);
|
|
9007
9201
|
const relativePath = prefix ? `${prefix}/${entry.name}` : entry.name;
|
|
9008
9202
|
if (entry.isDirectory()) {
|
|
9009
9203
|
await this.loadFilesRecursive(entryPath, relativePath);
|
|
9010
9204
|
} else if (entry.name.endsWith(".json")) {
|
|
9011
|
-
const content = await
|
|
9205
|
+
const content = await fs8.readFile(entryPath, "utf-8");
|
|
9012
9206
|
const intro = JSON.parse(content);
|
|
9013
9207
|
this.files.set(intro.filepath, intro);
|
|
9014
9208
|
}
|
|
@@ -9034,7 +9228,7 @@ var init_introspection2 = __esm(() => {
|
|
|
9034
9228
|
|
|
9035
9229
|
// src/app/indexer/watcher.ts
|
|
9036
9230
|
import { watch } from "chokidar";
|
|
9037
|
-
import * as
|
|
9231
|
+
import * as path23 from "path";
|
|
9038
9232
|
async function watchDirectory(rootDir, options = {}) {
|
|
9039
9233
|
const {
|
|
9040
9234
|
debounceMs = DEFAULT_DEBOUNCE_MS,
|
|
@@ -9045,17 +9239,17 @@ async function watchDirectory(rootDir, options = {}) {
|
|
|
9045
9239
|
onFileChange,
|
|
9046
9240
|
onError
|
|
9047
9241
|
} = options;
|
|
9048
|
-
rootDir =
|
|
9242
|
+
rootDir = path23.resolve(rootDir);
|
|
9049
9243
|
const config = await loadConfig(rootDir);
|
|
9050
|
-
const indexLocation = getIndexLocation(rootDir);
|
|
9051
9244
|
const validExtensions = new Set(config.extensions);
|
|
9052
9245
|
const ignorePatterns = [
|
|
9053
9246
|
...config.ignorePaths.map((p) => `**/${p}/**`),
|
|
9054
9247
|
"**/node_modules/**",
|
|
9055
|
-
"**/.git/**"
|
|
9248
|
+
"**/.git/**",
|
|
9249
|
+
"**/.raggrep/**"
|
|
9056
9250
|
];
|
|
9057
9251
|
function shouldWatchFile(filepath) {
|
|
9058
|
-
const ext =
|
|
9252
|
+
const ext = path23.extname(filepath);
|
|
9059
9253
|
return validExtensions.has(ext);
|
|
9060
9254
|
}
|
|
9061
9255
|
let isRunning = true;
|
|
@@ -9138,7 +9332,7 @@ async function watchDirectory(rootDir, options = {}) {
|
|
|
9138
9332
|
function handleFileEvent(event, filepath) {
|
|
9139
9333
|
if (!isRunning)
|
|
9140
9334
|
return;
|
|
9141
|
-
const relativePath =
|
|
9335
|
+
const relativePath = path23.relative(rootDir, filepath);
|
|
9142
9336
|
if (!shouldWatchFile(filepath)) {
|
|
9143
9337
|
return;
|
|
9144
9338
|
}
|
|
@@ -9216,9 +9410,9 @@ __export(exports_indexer, {
|
|
|
9216
9410
|
clearFreshnessCache: () => clearFreshnessCache,
|
|
9217
9411
|
cleanupIndex: () => cleanupIndex
|
|
9218
9412
|
});
|
|
9219
|
-
import * as
|
|
9220
|
-
import * as
|
|
9221
|
-
import * as
|
|
9413
|
+
import * as fs9 from "fs/promises";
|
|
9414
|
+
import * as path24 from "path";
|
|
9415
|
+
import * as os2 from "os";
|
|
9222
9416
|
import * as crypto2 from "crypto";
|
|
9223
9417
|
function clearFreshnessCache() {
|
|
9224
9418
|
freshnessCache = null;
|
|
@@ -9258,7 +9452,7 @@ function formatDuration(ms) {
|
|
|
9258
9452
|
return `${minutes}m ${remainingSeconds.toFixed(1)}s`;
|
|
9259
9453
|
}
|
|
9260
9454
|
function getOptimalConcurrency() {
|
|
9261
|
-
const cpuCount =
|
|
9455
|
+
const cpuCount = os2.cpus().length;
|
|
9262
9456
|
const optimal = Math.max(2, Math.min(16, Math.floor(cpuCount * 0.75)));
|
|
9263
9457
|
return optimal;
|
|
9264
9458
|
}
|
|
@@ -9269,7 +9463,7 @@ async function indexDirectory(rootDir, options = {}) {
|
|
|
9269
9463
|
const concurrency = options.concurrency ?? DEFAULT_CONCURRENCY;
|
|
9270
9464
|
clearFreshnessCache();
|
|
9271
9465
|
const logger = options.logger ? options.logger : quiet ? createSilentLogger() : createLogger({ verbose });
|
|
9272
|
-
rootDir =
|
|
9466
|
+
rootDir = path24.resolve(rootDir);
|
|
9273
9467
|
const location = getIndexLocation(rootDir);
|
|
9274
9468
|
logger.info(`Indexing directory: ${rootDir}`);
|
|
9275
9469
|
logger.info(`Index location: ${location.indexDir}`);
|
|
@@ -9321,12 +9515,12 @@ async function indexDirectory(rootDir, options = {}) {
|
|
|
9321
9515
|
rootDir,
|
|
9322
9516
|
config,
|
|
9323
9517
|
readFile: async (filepath) => {
|
|
9324
|
-
const fullPath =
|
|
9325
|
-
return
|
|
9518
|
+
const fullPath = path24.isAbsolute(filepath) ? filepath : path24.join(rootDir, filepath);
|
|
9519
|
+
return fs9.readFile(fullPath, "utf-8");
|
|
9326
9520
|
},
|
|
9327
9521
|
getFileStats: async (filepath) => {
|
|
9328
|
-
const fullPath =
|
|
9329
|
-
const stats = await
|
|
9522
|
+
const fullPath = path24.isAbsolute(filepath) ? filepath : path24.join(rootDir, filepath);
|
|
9523
|
+
const stats = await fs9.stat(fullPath);
|
|
9330
9524
|
return { lastModified: stats.mtime.toISOString() };
|
|
9331
9525
|
}
|
|
9332
9526
|
};
|
|
@@ -9351,7 +9545,7 @@ async function isIndexVersionCompatible(rootDir) {
|
|
|
9351
9545
|
const config = await loadConfig(rootDir);
|
|
9352
9546
|
const globalManifestPath = getGlobalManifestPath(rootDir, config);
|
|
9353
9547
|
try {
|
|
9354
|
-
const content = await
|
|
9548
|
+
const content = await fs9.readFile(globalManifestPath, "utf-8");
|
|
9355
9549
|
const manifest = JSON.parse(content);
|
|
9356
9550
|
return manifest.version === INDEX_SCHEMA_VERSION;
|
|
9357
9551
|
} catch {
|
|
@@ -9361,11 +9555,11 @@ async function isIndexVersionCompatible(rootDir) {
|
|
|
9361
9555
|
async function deleteIndex(rootDir) {
|
|
9362
9556
|
const indexDir = getRaggrepDir(rootDir);
|
|
9363
9557
|
try {
|
|
9364
|
-
await
|
|
9558
|
+
await fs9.rm(indexDir, { recursive: true, force: true });
|
|
9365
9559
|
} catch {}
|
|
9366
9560
|
}
|
|
9367
9561
|
async function resetIndex(rootDir) {
|
|
9368
|
-
rootDir =
|
|
9562
|
+
rootDir = path24.resolve(rootDir);
|
|
9369
9563
|
clearFreshnessCache();
|
|
9370
9564
|
const status = await getIndexStatus(rootDir);
|
|
9371
9565
|
if (!status.exists) {
|
|
@@ -9390,7 +9584,7 @@ async function ensureIndexFresh(rootDir, options = {}) {
|
|
|
9390
9584
|
let filesChanged = 0;
|
|
9391
9585
|
let filesReindexed = 0;
|
|
9392
9586
|
const logger = options.logger ? options.logger : quiet ? createSilentLogger() : createLogger({ verbose });
|
|
9393
|
-
rootDir =
|
|
9587
|
+
rootDir = path24.resolve(rootDir);
|
|
9394
9588
|
const status = await getIndexStatus(rootDir);
|
|
9395
9589
|
if (!status.exists) {
|
|
9396
9590
|
clearFreshnessCache();
|
|
@@ -9414,7 +9608,7 @@ async function ensureIndexFresh(rootDir, options = {}) {
|
|
|
9414
9608
|
const globalManifestPath = getGlobalManifestPath(rootDir, config);
|
|
9415
9609
|
let currentManifestMtime = 0;
|
|
9416
9610
|
try {
|
|
9417
|
-
const manifestStats = await
|
|
9611
|
+
const manifestStats = await fs9.stat(globalManifestPath);
|
|
9418
9612
|
currentManifestMtime = manifestStats.mtimeMs;
|
|
9419
9613
|
} catch {}
|
|
9420
9614
|
const now = Date.now();
|
|
@@ -9452,7 +9646,7 @@ async function ensureIndexFresh(rootDir, options = {}) {
|
|
|
9452
9646
|
const { allFiles: currentFiles, changedFiles, changedFileMtimes } = discoveryResult;
|
|
9453
9647
|
filesDiscovered = currentFiles.length;
|
|
9454
9648
|
filesChanged = changedFiles.length;
|
|
9455
|
-
const currentFileSet = new Set(currentFiles.map((f) =>
|
|
9649
|
+
const currentFileSet = new Set(currentFiles.map((f) => path24.relative(rootDir, f)));
|
|
9456
9650
|
const changedFileSet = new Set(changedFiles);
|
|
9457
9651
|
let totalIndexed = 0;
|
|
9458
9652
|
let totalRemoved = 0;
|
|
@@ -9486,11 +9680,11 @@ async function ensureIndexFresh(rootDir, options = {}) {
|
|
|
9486
9680
|
if (filesToRemove.length > 0) {
|
|
9487
9681
|
await Promise.all(filesToRemove.map(async (filepath) => {
|
|
9488
9682
|
logger.debug(` Removing stale: ${filepath}`);
|
|
9489
|
-
const indexFilePath =
|
|
9490
|
-
const symbolicFilePath =
|
|
9683
|
+
const indexFilePath = path24.join(indexPath, filepath.replace(/\.[^.]+$/, ".json"));
|
|
9684
|
+
const symbolicFilePath = path24.join(indexPath, "symbolic", filepath.replace(/\.[^.]+$/, ".json"));
|
|
9491
9685
|
await Promise.all([
|
|
9492
|
-
|
|
9493
|
-
|
|
9686
|
+
fs9.unlink(indexFilePath).catch(() => {}),
|
|
9687
|
+
fs9.unlink(symbolicFilePath).catch(() => {})
|
|
9494
9688
|
]);
|
|
9495
9689
|
delete manifest.files[filepath];
|
|
9496
9690
|
removedFilepaths.push(filepath);
|
|
@@ -9514,19 +9708,19 @@ async function ensureIndexFresh(rootDir, options = {}) {
|
|
|
9514
9708
|
rootDir,
|
|
9515
9709
|
config,
|
|
9516
9710
|
readFile: async (filepath) => {
|
|
9517
|
-
const fullPath =
|
|
9518
|
-
return
|
|
9711
|
+
const fullPath = path24.isAbsolute(filepath) ? filepath : path24.join(rootDir, filepath);
|
|
9712
|
+
return fs9.readFile(fullPath, "utf-8");
|
|
9519
9713
|
},
|
|
9520
9714
|
getFileStats: async (filepath) => {
|
|
9521
|
-
const fullPath =
|
|
9522
|
-
const stats = await
|
|
9715
|
+
const fullPath = path24.isAbsolute(filepath) ? filepath : path24.join(rootDir, filepath);
|
|
9716
|
+
const stats = await fs9.stat(fullPath);
|
|
9523
9717
|
return { lastModified: stats.mtime.toISOString() };
|
|
9524
9718
|
},
|
|
9525
9719
|
getIntrospection: (filepath) => introspection.getFile(filepath)
|
|
9526
9720
|
};
|
|
9527
9721
|
const moduleChangedFiles = module.supportsFile ? changedFiles.filter((f) => module.supportsFile(f)) : changedFiles;
|
|
9528
9722
|
const filesToProcess = moduleChangedFiles.map((filepath) => {
|
|
9529
|
-
const relativePath =
|
|
9723
|
+
const relativePath = path24.relative(rootDir, filepath);
|
|
9530
9724
|
const existingEntry = manifest.files[relativePath];
|
|
9531
9725
|
const lastModified = changedFileMtimes.get(filepath) || new Date().toISOString();
|
|
9532
9726
|
return {
|
|
@@ -9556,7 +9750,7 @@ async function ensureIndexFresh(rootDir, options = {}) {
|
|
|
9556
9750
|
return { relativePath, status: "unchanged" };
|
|
9557
9751
|
}
|
|
9558
9752
|
try {
|
|
9559
|
-
const content = await
|
|
9753
|
+
const content = await fs9.readFile(filepath, "utf-8");
|
|
9560
9754
|
const contentHash = computeContentHash(content);
|
|
9561
9755
|
if (!isNew && existingContentHash && existingContentHash === contentHash) {
|
|
9562
9756
|
completedCount++;
|
|
@@ -9683,7 +9877,7 @@ async function ensureIndexFresh(rootDir, options = {}) {
|
|
|
9683
9877
|
}
|
|
9684
9878
|
let finalManifestMtime = currentManifestMtime;
|
|
9685
9879
|
try {
|
|
9686
|
-
const manifestStats = await
|
|
9880
|
+
const manifestStats = await fs9.stat(globalManifestPath);
|
|
9687
9881
|
finalManifestMtime = manifestStats.mtimeMs;
|
|
9688
9882
|
} catch {}
|
|
9689
9883
|
freshnessCache = {
|
|
@@ -9703,7 +9897,7 @@ async function indexWithModule(rootDir, files, module, config, verbose, introspe
|
|
|
9703
9897
|
};
|
|
9704
9898
|
const manifest = await loadModuleManifest(rootDir, module.id, config);
|
|
9705
9899
|
const indexPath = getModuleIndexPath(rootDir, module.id, config);
|
|
9706
|
-
const currentFileSet = new Set(files.map((f) =>
|
|
9900
|
+
const currentFileSet = new Set(files.map((f) => path24.relative(rootDir, f)));
|
|
9707
9901
|
const filesToRemove = [];
|
|
9708
9902
|
for (const filepath of Object.keys(manifest.files)) {
|
|
9709
9903
|
if (!currentFileSet.has(filepath)) {
|
|
@@ -9714,13 +9908,13 @@ async function indexWithModule(rootDir, files, module, config, verbose, introspe
|
|
|
9714
9908
|
logger.info(` Removing ${filesToRemove.length} stale entries...`);
|
|
9715
9909
|
for (const filepath of filesToRemove) {
|
|
9716
9910
|
logger.debug(` Removing: ${filepath}`);
|
|
9717
|
-
const indexFilePath =
|
|
9911
|
+
const indexFilePath = path24.join(indexPath, filepath.replace(/\.[^.]+$/, ".json"));
|
|
9718
9912
|
try {
|
|
9719
|
-
await
|
|
9913
|
+
await fs9.unlink(indexFilePath);
|
|
9720
9914
|
} catch {}
|
|
9721
|
-
const symbolicFilePath =
|
|
9915
|
+
const symbolicFilePath = path24.join(indexPath, "symbolic", filepath.replace(/\.[^.]+$/, ".json"));
|
|
9722
9916
|
try {
|
|
9723
|
-
await
|
|
9917
|
+
await fs9.unlink(symbolicFilePath);
|
|
9724
9918
|
} catch {}
|
|
9725
9919
|
delete manifest.files[filepath];
|
|
9726
9920
|
}
|
|
@@ -9730,12 +9924,12 @@ async function indexWithModule(rootDir, files, module, config, verbose, introspe
|
|
|
9730
9924
|
rootDir,
|
|
9731
9925
|
config,
|
|
9732
9926
|
readFile: async (filepath) => {
|
|
9733
|
-
const fullPath =
|
|
9734
|
-
return
|
|
9927
|
+
const fullPath = path24.isAbsolute(filepath) ? filepath : path24.join(rootDir, filepath);
|
|
9928
|
+
return fs9.readFile(fullPath, "utf-8");
|
|
9735
9929
|
},
|
|
9736
9930
|
getFileStats: async (filepath) => {
|
|
9737
|
-
const fullPath =
|
|
9738
|
-
const stats = await
|
|
9931
|
+
const fullPath = path24.isAbsolute(filepath) ? filepath : path24.join(rootDir, filepath);
|
|
9932
|
+
const stats = await fs9.stat(fullPath);
|
|
9739
9933
|
return { lastModified: stats.mtime.toISOString() };
|
|
9740
9934
|
},
|
|
9741
9935
|
getIntrospection: (filepath) => introspection.getFile(filepath)
|
|
@@ -9747,9 +9941,9 @@ async function indexWithModule(rootDir, files, module, config, verbose, introspe
|
|
|
9747
9941
|
let indexedCount = 0;
|
|
9748
9942
|
let skippedCount = 0;
|
|
9749
9943
|
const processFile = async (filepath, _index) => {
|
|
9750
|
-
const relativePath =
|
|
9944
|
+
const relativePath = path24.relative(rootDir, filepath);
|
|
9751
9945
|
try {
|
|
9752
|
-
const stats = await
|
|
9946
|
+
const stats = await fs9.stat(filepath);
|
|
9753
9947
|
const lastModified = stats.mtime.toISOString();
|
|
9754
9948
|
const existingEntry = manifest.files[relativePath];
|
|
9755
9949
|
if (existingEntry && existingEntry.lastModified === lastModified) {
|
|
@@ -9759,7 +9953,7 @@ async function indexWithModule(rootDir, files, module, config, verbose, introspe
|
|
|
9759
9953
|
logger.debug(` [${completedCount}/${totalFiles}] Skipped ${relativePath} (unchanged)`);
|
|
9760
9954
|
return { relativePath, status: "skipped" };
|
|
9761
9955
|
}
|
|
9762
|
-
const content = await
|
|
9956
|
+
const content = await fs9.readFile(filepath, "utf-8");
|
|
9763
9957
|
const contentHash = computeContentHash(content);
|
|
9764
9958
|
if (existingEntry?.contentHash && existingEntry.contentHash === contentHash) {
|
|
9765
9959
|
completedCount++;
|
|
@@ -9840,8 +10034,8 @@ async function indexWithModule(rootDir, files, module, config, verbose, introspe
|
|
|
9840
10034
|
return result;
|
|
9841
10035
|
}
|
|
9842
10036
|
function isLikelyBinary(filepath) {
|
|
9843
|
-
const ext =
|
|
9844
|
-
const basename15 =
|
|
10037
|
+
const ext = path24.extname(filepath).toLowerCase();
|
|
10038
|
+
const basename15 = path24.basename(filepath).toLowerCase();
|
|
9845
10039
|
const binaryExtensions = new Set([
|
|
9846
10040
|
".png",
|
|
9847
10041
|
".jpg",
|
|
@@ -9917,7 +10111,7 @@ async function findFilesWithStats(rootDir, config, lastIndexStarted) {
|
|
|
9917
10111
|
const ignoreDirs = new Set(config.ignorePaths);
|
|
9918
10112
|
const lastIndexMs = lastIndexStarted?.getTime() ?? 0;
|
|
9919
10113
|
const crawler = new Builder().withFullPaths().exclude((dirName) => ignoreDirs.has(dirName)).filter((filePath) => {
|
|
9920
|
-
const ext =
|
|
10114
|
+
const ext = path24.extname(filePath);
|
|
9921
10115
|
return validExtensions.has(ext);
|
|
9922
10116
|
}).crawl(rootDir);
|
|
9923
10117
|
const allFiles = await crawler.withPromise();
|
|
@@ -9925,7 +10119,7 @@ async function findFilesWithStats(rootDir, config, lastIndexStarted) {
|
|
|
9925
10119
|
const changedFileMtimes2 = new Map;
|
|
9926
10120
|
await parallelMap(allFiles, async (filePath) => {
|
|
9927
10121
|
try {
|
|
9928
|
-
const stats = await
|
|
10122
|
+
const stats = await fs9.stat(filePath);
|
|
9929
10123
|
changedFileMtimes2.set(filePath, stats.mtime.toISOString());
|
|
9930
10124
|
} catch {}
|
|
9931
10125
|
}, STAT_CONCURRENCY);
|
|
@@ -9939,7 +10133,7 @@ async function findFilesWithStats(rootDir, config, lastIndexStarted) {
|
|
|
9939
10133
|
const changedFileMtimes = new Map;
|
|
9940
10134
|
await parallelMap(allFiles, async (filePath) => {
|
|
9941
10135
|
try {
|
|
9942
|
-
const stats = await
|
|
10136
|
+
const stats = await fs9.stat(filePath);
|
|
9943
10137
|
if (stats.mtimeMs > lastIndexMs) {
|
|
9944
10138
|
changedFiles.push(filePath);
|
|
9945
10139
|
changedFileMtimes.set(filePath, stats.mtime.toISOString());
|
|
@@ -9959,7 +10153,7 @@ async function findFiles(rootDir, config) {
|
|
|
9959
10153
|
async function loadModuleManifest(rootDir, moduleId, config) {
|
|
9960
10154
|
const manifestPath = getModuleManifestPath(rootDir, moduleId, config);
|
|
9961
10155
|
try {
|
|
9962
|
-
const content = await
|
|
10156
|
+
const content = await fs9.readFile(manifestPath, "utf-8");
|
|
9963
10157
|
return JSON.parse(content);
|
|
9964
10158
|
} catch {
|
|
9965
10159
|
return {
|
|
@@ -9972,19 +10166,19 @@ async function loadModuleManifest(rootDir, moduleId, config) {
|
|
|
9972
10166
|
}
|
|
9973
10167
|
async function writeModuleManifest(rootDir, moduleId, manifest, config) {
|
|
9974
10168
|
const manifestPath = getModuleManifestPath(rootDir, moduleId, config);
|
|
9975
|
-
await
|
|
9976
|
-
await
|
|
10169
|
+
await fs9.mkdir(path24.dirname(manifestPath), { recursive: true });
|
|
10170
|
+
await fs9.writeFile(manifestPath, JSON.stringify(manifest, null, 2));
|
|
9977
10171
|
}
|
|
9978
10172
|
async function writeFileIndex(rootDir, moduleId, filepath, fileIndex, config) {
|
|
9979
10173
|
const indexPath = getModuleIndexPath(rootDir, moduleId, config);
|
|
9980
|
-
const indexFilePath =
|
|
9981
|
-
await
|
|
9982
|
-
await
|
|
10174
|
+
const indexFilePath = path24.join(indexPath, filepath.replace(/\.[^.]+$/, ".json"));
|
|
10175
|
+
await fs9.mkdir(path24.dirname(indexFilePath), { recursive: true });
|
|
10176
|
+
await fs9.writeFile(indexFilePath, JSON.stringify(fileIndex, null, 2));
|
|
9983
10177
|
}
|
|
9984
10178
|
async function loadGlobalManifest(rootDir, config) {
|
|
9985
10179
|
const manifestPath = getGlobalManifestPath(rootDir, config);
|
|
9986
10180
|
try {
|
|
9987
|
-
const content = await
|
|
10181
|
+
const content = await fs9.readFile(manifestPath, "utf-8");
|
|
9988
10182
|
return JSON.parse(content);
|
|
9989
10183
|
} catch {
|
|
9990
10184
|
return null;
|
|
@@ -9998,13 +10192,13 @@ async function updateGlobalManifest(rootDir, modules, config, indexStartTime) {
|
|
|
9998
10192
|
lastIndexStarted: indexStartTime,
|
|
9999
10193
|
modules: modules.map((m) => m.id)
|
|
10000
10194
|
};
|
|
10001
|
-
await
|
|
10002
|
-
await
|
|
10195
|
+
await fs9.mkdir(path24.dirname(manifestPath), { recursive: true });
|
|
10196
|
+
await fs9.writeFile(manifestPath, JSON.stringify(manifest, null, 2));
|
|
10003
10197
|
}
|
|
10004
10198
|
async function cleanupIndex(rootDir, options = {}) {
|
|
10005
10199
|
const verbose = options.verbose ?? false;
|
|
10006
10200
|
const logger = options.logger ?? createLogger({ verbose });
|
|
10007
|
-
rootDir =
|
|
10201
|
+
rootDir = path24.resolve(rootDir);
|
|
10008
10202
|
logger.info(`Cleaning up index in: ${rootDir}`);
|
|
10009
10203
|
const config = await loadConfig(rootDir);
|
|
10010
10204
|
await registerBuiltInModules();
|
|
@@ -10034,9 +10228,9 @@ async function cleanupModuleIndex(rootDir, moduleId, config, logger) {
|
|
|
10034
10228
|
const filesToRemove = [];
|
|
10035
10229
|
const updatedFiles = {};
|
|
10036
10230
|
for (const [filepath, entry] of Object.entries(manifest.files)) {
|
|
10037
|
-
const fullPath =
|
|
10231
|
+
const fullPath = path24.join(rootDir, filepath);
|
|
10038
10232
|
try {
|
|
10039
|
-
await
|
|
10233
|
+
await fs9.access(fullPath);
|
|
10040
10234
|
updatedFiles[filepath] = entry;
|
|
10041
10235
|
result.kept++;
|
|
10042
10236
|
} catch {
|
|
@@ -10046,9 +10240,9 @@ async function cleanupModuleIndex(rootDir, moduleId, config, logger) {
|
|
|
10046
10240
|
}
|
|
10047
10241
|
}
|
|
10048
10242
|
for (const filepath of filesToRemove) {
|
|
10049
|
-
const indexFilePath =
|
|
10243
|
+
const indexFilePath = path24.join(indexPath, filepath.replace(/\.[^.]+$/, ".json"));
|
|
10050
10244
|
try {
|
|
10051
|
-
await
|
|
10245
|
+
await fs9.unlink(indexFilePath);
|
|
10052
10246
|
} catch {}
|
|
10053
10247
|
}
|
|
10054
10248
|
manifest.files = updatedFiles;
|
|
@@ -10059,16 +10253,16 @@ async function cleanupModuleIndex(rootDir, moduleId, config, logger) {
|
|
|
10059
10253
|
}
|
|
10060
10254
|
async function cleanupEmptyDirectories(dir) {
|
|
10061
10255
|
try {
|
|
10062
|
-
const entries = await
|
|
10256
|
+
const entries = await fs9.readdir(dir, { withFileTypes: true });
|
|
10063
10257
|
for (const entry of entries) {
|
|
10064
10258
|
if (entry.isDirectory()) {
|
|
10065
|
-
const subDir =
|
|
10259
|
+
const subDir = path24.join(dir, entry.name);
|
|
10066
10260
|
await cleanupEmptyDirectories(subDir);
|
|
10067
10261
|
}
|
|
10068
10262
|
}
|
|
10069
|
-
const remainingEntries = await
|
|
10263
|
+
const remainingEntries = await fs9.readdir(dir);
|
|
10070
10264
|
if (remainingEntries.length === 0) {
|
|
10071
|
-
await
|
|
10265
|
+
await fs9.rmdir(dir);
|
|
10072
10266
|
return true;
|
|
10073
10267
|
}
|
|
10074
10268
|
return false;
|
|
@@ -10077,7 +10271,7 @@ async function cleanupEmptyDirectories(dir) {
|
|
|
10077
10271
|
}
|
|
10078
10272
|
}
|
|
10079
10273
|
async function getIndexStatus(rootDir) {
|
|
10080
|
-
rootDir =
|
|
10274
|
+
rootDir = path24.resolve(rootDir);
|
|
10081
10275
|
const config = await loadConfig(rootDir);
|
|
10082
10276
|
const location = getIndexLocation(rootDir);
|
|
10083
10277
|
const indexDir = location.indexDir;
|
|
@@ -10089,13 +10283,13 @@ async function getIndexStatus(rootDir) {
|
|
|
10089
10283
|
totalFiles: 0
|
|
10090
10284
|
};
|
|
10091
10285
|
try {
|
|
10092
|
-
await
|
|
10286
|
+
await fs9.access(indexDir);
|
|
10093
10287
|
} catch {
|
|
10094
10288
|
return status;
|
|
10095
10289
|
}
|
|
10096
10290
|
try {
|
|
10097
10291
|
const globalManifestPath = getGlobalManifestPath(rootDir, config);
|
|
10098
|
-
const content = await
|
|
10292
|
+
const content = await fs9.readFile(globalManifestPath, "utf-8");
|
|
10099
10293
|
const globalManifest = JSON.parse(content);
|
|
10100
10294
|
status.exists = true;
|
|
10101
10295
|
status.lastUpdated = globalManifest.lastUpdated;
|
|
@@ -10113,7 +10307,7 @@ async function getIndexStatus(rootDir) {
|
|
|
10113
10307
|
}
|
|
10114
10308
|
} catch {
|
|
10115
10309
|
try {
|
|
10116
|
-
const entries = await
|
|
10310
|
+
const entries = await fs9.readdir(path24.join(indexDir, "index"));
|
|
10117
10311
|
if (entries.length > 0) {
|
|
10118
10312
|
status.exists = true;
|
|
10119
10313
|
for (const entry of entries) {
|
|
@@ -10133,7 +10327,7 @@ async function getIndexStatus(rootDir) {
|
|
|
10133
10327
|
}
|
|
10134
10328
|
return status;
|
|
10135
10329
|
}
|
|
10136
|
-
var FRESHNESS_CACHE_TTL_MS = 5000, freshnessCache = null, INDEX_SCHEMA_VERSION = "2.
|
|
10330
|
+
var FRESHNESS_CACHE_TTL_MS = 5000, freshnessCache = null, INDEX_SCHEMA_VERSION = "2.1.0", DEFAULT_CONCURRENCY, STAT_CONCURRENCY = 64;
|
|
10137
10331
|
var init_indexer = __esm(() => {
|
|
10138
10332
|
init_dist();
|
|
10139
10333
|
init_config2();
|
|
@@ -11479,7 +11673,7 @@ var minimatch = (p, pattern, options = {}) => {
|
|
|
11479
11673
|
}, qmarksTestNoExtDot = ([$0]) => {
|
|
11480
11674
|
const len = $0.length;
|
|
11481
11675
|
return (f) => f.length === len && f !== "." && f !== "..";
|
|
11482
|
-
}, defaultPlatform,
|
|
11676
|
+
}, defaultPlatform, path25, sep2, GLOBSTAR, qmark2 = "[^/]", star2, twoStarDot = "(?:(?!(?:\\/|^)(?:\\.{1,2})($|\\/)).)*?", twoStarNoDot = "(?:(?!(?:\\/|^)\\.).)*?", filter = (pattern, options = {}) => (p) => minimatch(p, pattern, options), ext = (a, b = {}) => Object.assign({}, a, b), defaults = (def) => {
|
|
11483
11677
|
if (!def || typeof def !== "object" || !Object.keys(def).length) {
|
|
11484
11678
|
return minimatch;
|
|
11485
11679
|
}
|
|
@@ -11537,11 +11731,11 @@ var init_esm2 = __esm(() => {
|
|
|
11537
11731
|
starRE = /^\*+$/;
|
|
11538
11732
|
qmarksRE = /^\?+([^+@!?\*\[\(]*)?$/;
|
|
11539
11733
|
defaultPlatform = typeof process === "object" && process ? typeof process.env === "object" && process.env && process.env.__MINIMATCH_TESTING_PLATFORM__ || process.platform : "posix";
|
|
11540
|
-
|
|
11734
|
+
path25 = {
|
|
11541
11735
|
win32: { sep: "\\" },
|
|
11542
11736
|
posix: { sep: "/" }
|
|
11543
11737
|
};
|
|
11544
|
-
sep2 = defaultPlatform === "win32" ?
|
|
11738
|
+
sep2 = defaultPlatform === "win32" ? path25.win32.sep : path25.posix.sep;
|
|
11545
11739
|
minimatch.sep = sep2;
|
|
11546
11740
|
GLOBSTAR = Symbol("globstar **");
|
|
11547
11741
|
minimatch.GLOBSTAR = GLOBSTAR;
|
|
@@ -11581,7 +11775,7 @@ function matchesPathFilter(relativePath, filters, matchFn) {
|
|
|
11581
11775
|
}
|
|
11582
11776
|
return false;
|
|
11583
11777
|
}
|
|
11584
|
-
async function executeExactSearch(
|
|
11778
|
+
async function executeExactSearch(fs10, options, matchFn) {
|
|
11585
11779
|
const {
|
|
11586
11780
|
rootDir,
|
|
11587
11781
|
literal,
|
|
@@ -11593,13 +11787,13 @@ async function executeExactSearch(fs9, options, matchFn) {
|
|
|
11593
11787
|
const files = new Map;
|
|
11594
11788
|
async function walkDir(dir, baseDir) {
|
|
11595
11789
|
try {
|
|
11596
|
-
const entries = await
|
|
11790
|
+
const entries = await fs10.readDir(dir);
|
|
11597
11791
|
for (const entry of entries) {
|
|
11598
|
-
const fullPath =
|
|
11599
|
-
const relativePath =
|
|
11792
|
+
const fullPath = fs10.join(dir, entry);
|
|
11793
|
+
const relativePath = fs10.relative(baseDir, fullPath);
|
|
11600
11794
|
let isDirectory = false;
|
|
11601
11795
|
try {
|
|
11602
|
-
const stats = await
|
|
11796
|
+
const stats = await fs10.getStats(fullPath);
|
|
11603
11797
|
isDirectory = stats.isDirectory ?? false;
|
|
11604
11798
|
} catch {
|
|
11605
11799
|
continue;
|
|
@@ -11616,7 +11810,7 @@ async function executeExactSearch(fs9, options, matchFn) {
|
|
|
11616
11810
|
}
|
|
11617
11811
|
}
|
|
11618
11812
|
try {
|
|
11619
|
-
const content = await
|
|
11813
|
+
const content = await fs10.readFile(fullPath);
|
|
11620
11814
|
if (isSearchableContent(content, fullPath)) {
|
|
11621
11815
|
files.set(relativePath, content);
|
|
11622
11816
|
}
|
|
@@ -11654,21 +11848,21 @@ var init_usecases = __esm(() => {
|
|
|
11654
11848
|
});
|
|
11655
11849
|
|
|
11656
11850
|
// src/infrastructure/filesystem/nodeFileSystem.ts
|
|
11657
|
-
import * as
|
|
11658
|
-
import * as
|
|
11851
|
+
import * as fs10 from "fs/promises";
|
|
11852
|
+
import * as path26 from "path";
|
|
11659
11853
|
import { glob } from "glob";
|
|
11660
11854
|
|
|
11661
11855
|
class NodeFileSystem {
|
|
11662
11856
|
async readFile(filepath) {
|
|
11663
|
-
return
|
|
11857
|
+
return fs10.readFile(filepath, "utf-8");
|
|
11664
11858
|
}
|
|
11665
11859
|
async writeFile(filepath, content) {
|
|
11666
|
-
await
|
|
11667
|
-
await
|
|
11860
|
+
await fs10.mkdir(path26.dirname(filepath), { recursive: true });
|
|
11861
|
+
await fs10.writeFile(filepath, content, "utf-8");
|
|
11668
11862
|
}
|
|
11669
11863
|
async deleteFile(filepath) {
|
|
11670
11864
|
try {
|
|
11671
|
-
await
|
|
11865
|
+
await fs10.unlink(filepath);
|
|
11672
11866
|
} catch (error) {
|
|
11673
11867
|
if (error.code !== "ENOENT") {
|
|
11674
11868
|
throw error;
|
|
@@ -11676,7 +11870,7 @@ class NodeFileSystem {
|
|
|
11676
11870
|
}
|
|
11677
11871
|
}
|
|
11678
11872
|
async getStats(filepath) {
|
|
11679
|
-
const stats = await
|
|
11873
|
+
const stats = await fs10.stat(filepath);
|
|
11680
11874
|
return {
|
|
11681
11875
|
lastModified: stats.mtime.toISOString(),
|
|
11682
11876
|
size: stats.isDirectory() ? undefined : stats.size,
|
|
@@ -11685,17 +11879,17 @@ class NodeFileSystem {
|
|
|
11685
11879
|
}
|
|
11686
11880
|
async exists(filepath) {
|
|
11687
11881
|
try {
|
|
11688
|
-
await
|
|
11882
|
+
await fs10.access(filepath);
|
|
11689
11883
|
return true;
|
|
11690
11884
|
} catch {
|
|
11691
11885
|
return false;
|
|
11692
11886
|
}
|
|
11693
11887
|
}
|
|
11694
11888
|
async mkdir(dirpath) {
|
|
11695
|
-
await
|
|
11889
|
+
await fs10.mkdir(dirpath, { recursive: true });
|
|
11696
11890
|
}
|
|
11697
11891
|
async readDir(dirpath) {
|
|
11698
|
-
return
|
|
11892
|
+
return fs10.readdir(dirpath);
|
|
11699
11893
|
}
|
|
11700
11894
|
async findFiles(rootDir, patterns, ignore) {
|
|
11701
11895
|
const ignorePatterns = ignore.map((p) => `**/${p}/**`);
|
|
@@ -11711,19 +11905,19 @@ class NodeFileSystem {
|
|
|
11711
11905
|
return [...new Set(files)];
|
|
11712
11906
|
}
|
|
11713
11907
|
join(...segments) {
|
|
11714
|
-
return
|
|
11908
|
+
return path26.join(...segments);
|
|
11715
11909
|
}
|
|
11716
11910
|
relative(from, to) {
|
|
11717
|
-
return
|
|
11911
|
+
return path26.relative(from, to);
|
|
11718
11912
|
}
|
|
11719
11913
|
resolve(...segments) {
|
|
11720
|
-
return
|
|
11914
|
+
return path26.resolve(...segments);
|
|
11721
11915
|
}
|
|
11722
11916
|
dirname(filepath) {
|
|
11723
|
-
return
|
|
11917
|
+
return path26.dirname(filepath);
|
|
11724
11918
|
}
|
|
11725
11919
|
extname(filepath) {
|
|
11726
|
-
return
|
|
11920
|
+
return path26.extname(filepath);
|
|
11727
11921
|
}
|
|
11728
11922
|
}
|
|
11729
11923
|
var nodeFileSystem;
|
|
@@ -11744,14 +11938,14 @@ __export(exports_search, {
|
|
|
11744
11938
|
formatSearchResults: () => formatSearchResults2,
|
|
11745
11939
|
formatHybridSearchResults: () => formatHybridSearchResults
|
|
11746
11940
|
});
|
|
11747
|
-
import * as
|
|
11748
|
-
import * as
|
|
11941
|
+
import * as fs11 from "fs/promises";
|
|
11942
|
+
import * as path27 from "path";
|
|
11749
11943
|
async function search(rootDir, query, options = {}) {
|
|
11750
11944
|
const hybridResults = await hybridSearch(rootDir, query, options);
|
|
11751
11945
|
return hybridResults.results;
|
|
11752
11946
|
}
|
|
11753
11947
|
async function hybridSearch(rootDir, query, options = {}) {
|
|
11754
|
-
rootDir =
|
|
11948
|
+
rootDir = path27.resolve(rootDir);
|
|
11755
11949
|
const ensureFresh = options.ensureFresh ?? DEFAULT_SEARCH_OPTIONS.ensureFresh;
|
|
11756
11950
|
if (ensureFresh) {
|
|
11757
11951
|
await ensureIndexFresh(rootDir, { quiet: true });
|
|
@@ -11828,15 +12022,15 @@ async function hybridSearch(rootDir, query, options = {}) {
|
|
|
11828
12022
|
};
|
|
11829
12023
|
}
|
|
11830
12024
|
async function performExactSearch(rootDir, literal, config, options) {
|
|
11831
|
-
const
|
|
11832
|
-
return executeExactSearch(
|
|
12025
|
+
const fs12 = new NodeFileSystem;
|
|
12026
|
+
return executeExactSearch(fs12, {
|
|
11833
12027
|
rootDir,
|
|
11834
12028
|
literal,
|
|
11835
12029
|
pathFilter: options.pathFilter,
|
|
11836
12030
|
maxFiles: 20,
|
|
11837
12031
|
maxOccurrencesPerFile: 5,
|
|
11838
12032
|
caseInsensitive: false
|
|
11839
|
-
}, (
|
|
12033
|
+
}, (path28, pattern) => minimatch(path28, pattern, { matchBase: true }));
|
|
11840
12034
|
}
|
|
11841
12035
|
function createSearchContext(rootDir, moduleId, config) {
|
|
11842
12036
|
const indexPath = getModuleIndexPath(rootDir, moduleId, config);
|
|
@@ -11845,9 +12039,9 @@ function createSearchContext(rootDir, moduleId, config) {
|
|
|
11845
12039
|
config,
|
|
11846
12040
|
loadFileIndex: async (filepath) => {
|
|
11847
12041
|
const hasExtension = /\.[^./]+$/.test(filepath);
|
|
11848
|
-
const indexFilePath = hasExtension ?
|
|
12042
|
+
const indexFilePath = hasExtension ? path27.join(indexPath, filepath.replace(/\.[^.]+$/, ".json")) : path27.join(indexPath, filepath + ".json");
|
|
11849
12043
|
try {
|
|
11850
|
-
const content = await
|
|
12044
|
+
const content = await fs11.readFile(indexFilePath, "utf-8");
|
|
11851
12045
|
return JSON.parse(content);
|
|
11852
12046
|
} catch {
|
|
11853
12047
|
return null;
|
|
@@ -11857,7 +12051,7 @@ function createSearchContext(rootDir, moduleId, config) {
|
|
|
11857
12051
|
const files = [];
|
|
11858
12052
|
await traverseDirectory(indexPath, files, indexPath);
|
|
11859
12053
|
return files.filter((f) => f.endsWith(".json") && !f.endsWith("manifest.json")).map((f) => {
|
|
11860
|
-
const relative6 =
|
|
12054
|
+
const relative6 = path27.relative(indexPath, f);
|
|
11861
12055
|
return relative6.replace(/\.json$/, "");
|
|
11862
12056
|
});
|
|
11863
12057
|
}
|
|
@@ -11865,9 +12059,9 @@ function createSearchContext(rootDir, moduleId, config) {
|
|
|
11865
12059
|
}
|
|
11866
12060
|
async function traverseDirectory(dir, files, basePath) {
|
|
11867
12061
|
try {
|
|
11868
|
-
const entries = await
|
|
12062
|
+
const entries = await fs11.readdir(dir, { withFileTypes: true });
|
|
11869
12063
|
for (const entry of entries) {
|
|
11870
|
-
const fullPath =
|
|
12064
|
+
const fullPath = path27.join(dir, entry.name);
|
|
11871
12065
|
if (entry.isDirectory()) {
|
|
11872
12066
|
await traverseDirectory(fullPath, files, basePath);
|
|
11873
12067
|
} else if (entry.isFile()) {
|
|
@@ -11879,7 +12073,7 @@ async function traverseDirectory(dir, files, basePath) {
|
|
|
11879
12073
|
async function loadGlobalManifest2(rootDir, config) {
|
|
11880
12074
|
const manifestPath = getGlobalManifestPath(rootDir, config);
|
|
11881
12075
|
try {
|
|
11882
|
-
const content = await
|
|
12076
|
+
const content = await fs11.readFile(manifestPath, "utf-8");
|
|
11883
12077
|
return JSON.parse(content);
|
|
11884
12078
|
} catch {
|
|
11885
12079
|
return null;
|
|
@@ -12048,18 +12242,18 @@ function getInstallationMethod(openCodeVersion) {
|
|
|
12048
12242
|
}
|
|
12049
12243
|
async function detectOpenCodeVersion() {
|
|
12050
12244
|
try {
|
|
12051
|
-
const
|
|
12052
|
-
const
|
|
12053
|
-
const
|
|
12054
|
-
const homeDir =
|
|
12245
|
+
const os3 = await import("os");
|
|
12246
|
+
const fs12 = await import("fs/promises");
|
|
12247
|
+
const path28 = await import("path");
|
|
12248
|
+
const homeDir = os3.homedir();
|
|
12055
12249
|
const possiblePaths = [
|
|
12056
|
-
|
|
12057
|
-
|
|
12058
|
-
|
|
12250
|
+
path28.join(homeDir, ".local", "share", "opencode", "package.json"),
|
|
12251
|
+
path28.join(homeDir, ".config", "opencode", "package.json"),
|
|
12252
|
+
path28.join(homeDir, ".npm", "global", "node_modules", "opencode", "package.json")
|
|
12059
12253
|
];
|
|
12060
12254
|
for (const packagePath of possiblePaths) {
|
|
12061
12255
|
try {
|
|
12062
|
-
const content = await
|
|
12256
|
+
const content = await fs12.readFile(packagePath, "utf-8");
|
|
12063
12257
|
const pkg = JSON.parse(content);
|
|
12064
12258
|
if (pkg.version) {
|
|
12065
12259
|
return pkg.version;
|
|
@@ -12097,12 +12291,12 @@ async function detectOpenCodeVersion() {
|
|
|
12097
12291
|
// src/app/cli/opencode/install-tool.ts
|
|
12098
12292
|
async function installTool(options = {}) {
|
|
12099
12293
|
const { logger, checkForOldSkill = true } = options;
|
|
12100
|
-
const
|
|
12101
|
-
const
|
|
12102
|
-
const
|
|
12103
|
-
const homeDir =
|
|
12104
|
-
const toolDir =
|
|
12105
|
-
const toolPath =
|
|
12294
|
+
const os3 = await import("os");
|
|
12295
|
+
const fs12 = await import("fs/promises");
|
|
12296
|
+
const path28 = await import("path");
|
|
12297
|
+
const homeDir = os3.homedir();
|
|
12298
|
+
const toolDir = path28.join(homeDir, ".config", "opencode", "tool");
|
|
12299
|
+
const toolPath = path28.join(toolDir, "raggrep.ts");
|
|
12106
12300
|
let removedOldSkill = false;
|
|
12107
12301
|
const toolContent = `import { tool } from "@opencode-ai/plugin";
|
|
12108
12302
|
|
|
@@ -12185,11 +12379,11 @@ Please install raggrep globally:
|
|
|
12185
12379
|
`;
|
|
12186
12380
|
try {
|
|
12187
12381
|
if (checkForOldSkill) {
|
|
12188
|
-
const oldSkillDir =
|
|
12189
|
-
const oldSkillPath =
|
|
12382
|
+
const oldSkillDir = path28.join(homeDir, ".config", "opencode", "skill", "raggrep");
|
|
12383
|
+
const oldSkillPath = path28.join(oldSkillDir, "SKILL.md");
|
|
12190
12384
|
let oldSkillExists = false;
|
|
12191
12385
|
try {
|
|
12192
|
-
await
|
|
12386
|
+
await fs12.access(oldSkillPath);
|
|
12193
12387
|
oldSkillExists = true;
|
|
12194
12388
|
} catch {}
|
|
12195
12389
|
if (oldSkillExists) {
|
|
@@ -12214,11 +12408,11 @@ Please install raggrep globally:
|
|
|
12214
12408
|
const shouldDelete = answer.toLowerCase() !== "n";
|
|
12215
12409
|
if (shouldDelete) {
|
|
12216
12410
|
try {
|
|
12217
|
-
await
|
|
12218
|
-
const skillDirContents = await
|
|
12411
|
+
await fs12.unlink(oldSkillPath);
|
|
12412
|
+
const skillDirContents = await fs12.readdir(oldSkillDir);
|
|
12219
12413
|
if (skillDirContents.length === 0) {
|
|
12220
12414
|
try {
|
|
12221
|
-
await
|
|
12415
|
+
await fs12.rmdir(oldSkillDir);
|
|
12222
12416
|
console.log("✓ Removed old skill directory.");
|
|
12223
12417
|
} catch (rmdirError) {
|
|
12224
12418
|
console.log("✓ Removed old skill file. (Directory not empty or other error)");
|
|
@@ -12255,8 +12449,8 @@ Please install raggrep globally:
|
|
|
12255
12449
|
}
|
|
12256
12450
|
}
|
|
12257
12451
|
}
|
|
12258
|
-
await
|
|
12259
|
-
await
|
|
12452
|
+
await fs12.mkdir(toolDir, { recursive: true });
|
|
12453
|
+
await fs12.writeFile(toolPath, toolContent, "utf-8");
|
|
12260
12454
|
const message = `Installed raggrep tool for OpenCode.
|
|
12261
12455
|
Location: ${toolPath}
|
|
12262
12456
|
|
|
@@ -12289,12 +12483,12 @@ The raggrep tool is now available in OpenCode.`;
|
|
|
12289
12483
|
// src/app/cli/opencode/install-skill.ts
|
|
12290
12484
|
async function installSkill(options = {}) {
|
|
12291
12485
|
const { logger, checkForOldTool = true } = options;
|
|
12292
|
-
const
|
|
12293
|
-
const
|
|
12294
|
-
const
|
|
12295
|
-
const homeDir =
|
|
12296
|
-
const skillDir =
|
|
12297
|
-
const skillPath =
|
|
12486
|
+
const os3 = await import("os");
|
|
12487
|
+
const fs12 = await import("fs/promises");
|
|
12488
|
+
const path28 = await import("path");
|
|
12489
|
+
const homeDir = os3.homedir();
|
|
12490
|
+
const skillDir = path28.join(homeDir, ".config", "opencode", "skill", "raggrep");
|
|
12491
|
+
const skillPath = path28.join(skillDir, "SKILL.md");
|
|
12298
12492
|
const skillContent = `---
|
|
12299
12493
|
name: raggrep
|
|
12300
12494
|
description: AST-powered semantic code search that understands intent, not just text. Superior to grep/rg - finds functions, classes, and logic even when keywords differ. Saves 10x tool calls by searching the actual code structure.
|
|
@@ -12334,19 +12528,32 @@ pnpm add -g raggrep
|
|
|
12334
12528
|
\`\`\`
|
|
12335
12529
|
|
|
12336
12530
|
### Step 1: Index your codebase
|
|
12531
|
+
|
|
12532
|
+
Index data is written to **\`<project>/.raggrep/\`** inside the directory you target. Add \`.raggrep/\` to \`.gitignore\` if you do not want it in version control.
|
|
12533
|
+
|
|
12534
|
+
From the repository root:
|
|
12337
12535
|
\`\`\`bash
|
|
12338
|
-
# Navigate to your project directory and index it
|
|
12339
12536
|
cd /path/to/your/project
|
|
12340
12537
|
raggrep index
|
|
12341
12538
|
\`\`\`
|
|
12342
12539
|
|
|
12540
|
+
Or pass the project path explicitly (no \`cd\` needed; same flag as \`git -C\`):
|
|
12541
|
+
\`\`\`bash
|
|
12542
|
+
raggrep index --dir /path/to/your/project
|
|
12543
|
+
# short form:
|
|
12544
|
+
raggrep index -C ~/projects/my-app
|
|
12545
|
+
\`\`\`
|
|
12546
|
+
|
|
12343
12547
|
### Step 2: Use semantic search
|
|
12548
|
+
|
|
12344
12549
|
\`\`\`bash
|
|
12345
|
-
# Search for specific functionality
|
|
12550
|
+
# Search for specific functionality (from project root, or use --dir / -C)
|
|
12346
12551
|
raggrep query "user authentication"
|
|
12552
|
+
raggrep query -C /path/to/your/project "user authentication"
|
|
12347
12553
|
|
|
12348
12554
|
# Search with filters
|
|
12349
12555
|
raggrep query "React hooks for data fetching" --filter "src/components"
|
|
12556
|
+
raggrep query -C /path/to/your/project "React hooks" --filter "src/components"
|
|
12350
12557
|
|
|
12351
12558
|
# Search with specific file types
|
|
12352
12559
|
raggrep query "database connection" --type ts
|
|
@@ -12355,6 +12562,8 @@ raggrep query "database connection" --type ts
|
|
|
12355
12562
|
raggrep query "error handling" --top 15
|
|
12356
12563
|
\`\`\`
|
|
12357
12564
|
|
|
12565
|
+
Use **\`-C\` / \`--dir\`** on \`raggrep query\`, \`raggrep index\`, \`raggrep status\`, and \`raggrep reset\` when the shell is not already in the repository root.
|
|
12566
|
+
|
|
12358
12567
|
### Step 3: Use in OpenCode agents
|
|
12359
12568
|
|
|
12360
12569
|
Load this skill in your agent conversation:
|
|
@@ -12390,6 +12599,7 @@ Instead of using grep/rg or manually reading files:
|
|
|
12390
12599
|
|
|
12391
12600
|
✅ **DO this instead:**
|
|
12392
12601
|
- \`raggrep query "Find the auth middleware"\` (finds ALL auth-related code)
|
|
12602
|
+
- \`raggrep query -C /path/to/repo "Find the auth middleware"\` (when not \`cd\`'d into the repo)
|
|
12393
12603
|
- \`raggrep query "Where are React components?"\`
|
|
12394
12604
|
- \`raggrep query "Database connection logic?"\`
|
|
12395
12605
|
- \`raggrep query "Error handling patterns"\`
|
|
@@ -12400,17 +12610,18 @@ Instead of using grep/rg or manually reading files:
|
|
|
12400
12610
|
2. **Use filters strategically**: \`--filter "src/auth"\`, \`--filter "*.test.ts"\`
|
|
12401
12611
|
3. **Adjust result count**: Use \`--top 5\` for focused results, \`--top 20\` for comprehensive search
|
|
12402
12612
|
4. **Replace grep/rg habits**: Instead of \`rg "pattern"\`, try \`raggrep query "what the code does"\`
|
|
12613
|
+
5. **Target the right tree**: If your shell is not in the repo root, use \`-C /path/to/repo\` or \`--dir\` on every command so the index under that repo's \`.raggrep/\` stays aligned with searches
|
|
12403
12614
|
|
|
12404
12615
|
**Result**: 10x fewer tool calls, BETTER results, deeper code understanding.
|
|
12405
12616
|
`;
|
|
12406
12617
|
let removedOldTool = false;
|
|
12407
12618
|
try {
|
|
12408
12619
|
if (checkForOldTool) {
|
|
12409
|
-
const oldToolDir =
|
|
12410
|
-
const oldToolPath =
|
|
12620
|
+
const oldToolDir = path28.join(homeDir, ".config", "opencode", "tool");
|
|
12621
|
+
const oldToolPath = path28.join(oldToolDir, "raggrep.ts");
|
|
12411
12622
|
let oldToolExists = false;
|
|
12412
12623
|
try {
|
|
12413
|
-
await
|
|
12624
|
+
await fs12.access(oldToolPath);
|
|
12414
12625
|
oldToolExists = true;
|
|
12415
12626
|
} catch {}
|
|
12416
12627
|
if (oldToolExists) {
|
|
@@ -12435,11 +12646,11 @@ Instead of using grep/rg or manually reading files:
|
|
|
12435
12646
|
const shouldDelete = answer.toLowerCase() !== "n";
|
|
12436
12647
|
if (shouldDelete) {
|
|
12437
12648
|
try {
|
|
12438
|
-
await
|
|
12439
|
-
const toolDirContents = await
|
|
12649
|
+
await fs12.unlink(oldToolPath);
|
|
12650
|
+
const toolDirContents = await fs12.readdir(oldToolDir);
|
|
12440
12651
|
if (toolDirContents.length === 0) {
|
|
12441
12652
|
try {
|
|
12442
|
-
await
|
|
12653
|
+
await fs12.rmdir(oldToolDir);
|
|
12443
12654
|
console.log("✓ Removed old tool directory.");
|
|
12444
12655
|
} catch (rmdirError) {
|
|
12445
12656
|
console.log("✓ Removed old tool file. (Directory not empty or other error)");
|
|
@@ -12472,8 +12683,8 @@ Instead of using grep/rg or manually reading files:
|
|
|
12472
12683
|
}
|
|
12473
12684
|
}
|
|
12474
12685
|
}
|
|
12475
|
-
await
|
|
12476
|
-
await
|
|
12686
|
+
await fs12.mkdir(skillDir, { recursive: true });
|
|
12687
|
+
await fs12.writeFile(skillPath, skillContent, "utf-8");
|
|
12477
12688
|
const message = `Installed raggrep skill for OpenCode.
|
|
12478
12689
|
Location: ${skillPath}
|
|
12479
12690
|
|
|
@@ -12481,7 +12692,7 @@ The raggrep skill is now available to OpenCode agents.
|
|
|
12481
12692
|
|
|
12482
12693
|
To use this skill:
|
|
12483
12694
|
1. Install raggrep: npm install -g raggrep
|
|
12484
|
-
2. Index your codebase: raggrep index
|
|
12695
|
+
2. Index your codebase: \`cd\` to the repo and \`raggrep index\`, or \`raggrep index --dir /path/to/repo\`
|
|
12485
12696
|
3. In OpenCode, load the skill: skill({ name: "raggrep" })`;
|
|
12486
12697
|
if (logger) {
|
|
12487
12698
|
logger.info(message);
|
|
@@ -12523,10 +12734,12 @@ var init_opencode = () => {};
|
|
|
12523
12734
|
// src/app/cli/main.ts
|
|
12524
12735
|
init_embeddings();
|
|
12525
12736
|
init_logger();
|
|
12737
|
+
import * as path28 from "path";
|
|
12738
|
+
import { stat as stat3 } from "fs/promises";
|
|
12526
12739
|
// package.json
|
|
12527
12740
|
var package_default = {
|
|
12528
12741
|
name: "raggrep",
|
|
12529
|
-
version: "0.
|
|
12742
|
+
version: "0.17.0",
|
|
12530
12743
|
description: "Local filesystem-based RAG system for codebases - semantic search using local embeddings",
|
|
12531
12744
|
type: "module",
|
|
12532
12745
|
main: "./dist/index.js",
|
|
@@ -12548,12 +12761,15 @@ var package_default = {
|
|
|
12548
12761
|
scripts: {
|
|
12549
12762
|
build: "bun run build:clean && bun run build:bundle && bun run build:types && bun run build:shebang",
|
|
12550
12763
|
"build:clean": "rm -rf dist",
|
|
12551
|
-
"build:bundle": "bun build src/index.ts --outdir dist --target node --sourcemap=external --external '@xenova/transformers' --external 'glob' --external 'typescript' --external 'chokidar' --external 'web-tree-sitter' && bun build src/app/cli/main.ts --outdir dist/cli --target node --sourcemap=external --external '@xenova/transformers' --external 'glob' --external 'typescript' --external 'chokidar' --external 'web-tree-sitter'",
|
|
12552
|
-
"build:types": "tsc --emitDeclarationOnly --outDir dist",
|
|
12764
|
+
"build:bundle": "bun build src/index.ts --outdir dist --target node --sourcemap=external --external '@xenova/transformers' --external '@huggingface/transformers' --external 'glob' --external 'typescript' --external 'chokidar' --external 'web-tree-sitter' && bun build src/app/cli/main.ts --outdir dist/cli --target node --sourcemap=external --external '@xenova/transformers' --external '@huggingface/transformers' --external 'glob' --external 'typescript' --external 'chokidar' --external 'web-tree-sitter'",
|
|
12765
|
+
"build:types": "tsc --project tsconfig.json --noEmit false --declaration true --emitDeclarationOnly true --rootDir ./src --outDir ./dist",
|
|
12553
12766
|
"build:shebang": "echo '#!/usr/bin/env node' | cat - dist/cli/main.js > temp && mv temp dist/cli/main.js && chmod +x dist/cli/main.js",
|
|
12554
12767
|
prepublishOnly: "bun run build",
|
|
12555
12768
|
raggrep: "bun run src/app/cli/main.ts",
|
|
12556
12769
|
test: "bun test",
|
|
12770
|
+
typecheck: "tsc --noEmit -p tsconfig.json && tsc --noEmit -p scripts/tsconfig.json",
|
|
12771
|
+
"bench:embeddings": "bun run scripts/benchmark-embedding-runtimes.ts",
|
|
12772
|
+
"bench:retrieval": "bun run scripts/benchmark-retrieval-quality.ts",
|
|
12557
12773
|
dev: "bun run src/app/cli/main.ts"
|
|
12558
12774
|
},
|
|
12559
12775
|
keywords: [
|
|
@@ -12581,6 +12797,7 @@ var package_default = {
|
|
|
12581
12797
|
node: ">=18.0.0"
|
|
12582
12798
|
},
|
|
12583
12799
|
dependencies: {
|
|
12800
|
+
"@huggingface/transformers": "^4.0.0",
|
|
12584
12801
|
"@xenova/transformers": "^2.17.0",
|
|
12585
12802
|
chokidar: "^5.0.0",
|
|
12586
12803
|
fdir: "^6.5.0",
|
|
@@ -12689,6 +12906,14 @@ function parseFlags(args2) {
|
|
|
12689
12906
|
console.error('--filter requires a path or glob pattern (e.g., src/auth, "*.ts")');
|
|
12690
12907
|
process.exit(1);
|
|
12691
12908
|
}
|
|
12909
|
+
} else if (arg === "--dir" || arg === "-C") {
|
|
12910
|
+
const dir = args2[++i];
|
|
12911
|
+
if (dir) {
|
|
12912
|
+
flags.projectDir = dir;
|
|
12913
|
+
} else {
|
|
12914
|
+
console.error("--dir / -C requires a path to the project directory to index or search.");
|
|
12915
|
+
process.exit(1);
|
|
12916
|
+
}
|
|
12692
12917
|
} else if (arg === "--tool") {
|
|
12693
12918
|
flags.forceTool = true;
|
|
12694
12919
|
} else if (arg === "--skill") {
|
|
@@ -12699,6 +12924,20 @@ function parseFlags(args2) {
|
|
|
12699
12924
|
}
|
|
12700
12925
|
return flags;
|
|
12701
12926
|
}
|
|
12927
|
+
async function resolveProjectRoot(flags) {
|
|
12928
|
+
const root = path28.resolve(flags.projectDir ?? process.cwd());
|
|
12929
|
+
try {
|
|
12930
|
+
const st = await stat3(root);
|
|
12931
|
+
if (!st.isDirectory()) {
|
|
12932
|
+
console.error(`Error: not a directory: ${root}`);
|
|
12933
|
+
process.exit(1);
|
|
12934
|
+
}
|
|
12935
|
+
} catch {
|
|
12936
|
+
console.error(`Error: directory does not exist or is not accessible: ${root}`);
|
|
12937
|
+
process.exit(1);
|
|
12938
|
+
}
|
|
12939
|
+
return root;
|
|
12940
|
+
}
|
|
12702
12941
|
async function main() {
|
|
12703
12942
|
const flags = parseFlags(args.slice(1));
|
|
12704
12943
|
switch (command) {
|
|
@@ -12706,14 +12945,15 @@ async function main() {
|
|
|
12706
12945
|
if (flags.help) {
|
|
12707
12946
|
const models = Object.keys(EMBEDDING_MODELS).join(", ");
|
|
12708
12947
|
console.log(`
|
|
12709
|
-
raggrep index - Index
|
|
12948
|
+
raggrep index - Index a directory for semantic search
|
|
12710
12949
|
|
|
12711
12950
|
Usage:
|
|
12712
12951
|
raggrep index [options]
|
|
12713
12952
|
|
|
12714
12953
|
Options:
|
|
12954
|
+
-C, --dir <path> Project directory to index (default: current directory)
|
|
12715
12955
|
-w, --watch Watch for file changes and re-index automatically
|
|
12716
|
-
-m, --model <name>
|
|
12956
|
+
-m, --model <name> Override embedding model for the TypeScript module (default: saved config, else bge-small-en-v1.5)
|
|
12717
12957
|
-c, --concurrency <n> Number of files to process in parallel (default: auto)
|
|
12718
12958
|
-v, --verbose Show detailed progress
|
|
12719
12959
|
-h, --help Show this help message
|
|
@@ -12725,6 +12965,7 @@ Model Cache: ${getCacheDir()}
|
|
|
12725
12965
|
|
|
12726
12966
|
Examples:
|
|
12727
12967
|
raggrep index
|
|
12968
|
+
raggrep index --dir ../other-repo
|
|
12728
12969
|
raggrep index --watch
|
|
12729
12970
|
raggrep index --model bge-small-en-v1.5
|
|
12730
12971
|
raggrep index --concurrency 8
|
|
@@ -12734,11 +12975,12 @@ Examples:
|
|
|
12734
12975
|
}
|
|
12735
12976
|
const { indexDirectory: indexDirectory3, watchDirectory: watchDirectory2 } = await Promise.resolve().then(() => (init_indexer(), exports_indexer));
|
|
12736
12977
|
const logger = createInlineLogger({ verbose: flags.verbose });
|
|
12978
|
+
const projectRoot = await resolveProjectRoot(flags);
|
|
12737
12979
|
console.log("RAGgrep Indexer");
|
|
12738
12980
|
console.log(`================
|
|
12739
12981
|
`);
|
|
12740
12982
|
try {
|
|
12741
|
-
const results = await indexDirectory3(
|
|
12983
|
+
const results = await indexDirectory3(projectRoot, {
|
|
12742
12984
|
model: flags.model,
|
|
12743
12985
|
verbose: flags.verbose,
|
|
12744
12986
|
concurrency: flags.concurrency,
|
|
@@ -12761,7 +13003,7 @@ Examples:
|
|
|
12761
13003
|
console.log(`└─────────────────────────────────────────┘
|
|
12762
13004
|
`);
|
|
12763
13005
|
try {
|
|
12764
|
-
const watcher = await watchDirectory2(
|
|
13006
|
+
const watcher = await watchDirectory2(projectRoot, {
|
|
12765
13007
|
model: flags.model,
|
|
12766
13008
|
verbose: flags.verbose,
|
|
12767
13009
|
onFileChange: (event, filepath) => {
|
|
@@ -12798,6 +13040,7 @@ Usage:
|
|
|
12798
13040
|
raggrep query <search query> [options]
|
|
12799
13041
|
|
|
12800
13042
|
Options:
|
|
13043
|
+
-C, --dir <path> Project directory to search (default: current directory)
|
|
12801
13044
|
-k, --top <n> Number of results to return (default: 10)
|
|
12802
13045
|
-s, --min-score <n> Minimum similarity score 0-1 (default: 0.15)
|
|
12803
13046
|
-t, --type <ext> Filter by file extension (e.g., ts, tsx, js)
|
|
@@ -12825,6 +13068,7 @@ Multiple Filters (OR logic):
|
|
|
12825
13068
|
|
|
12826
13069
|
Examples:
|
|
12827
13070
|
raggrep query "user authentication"
|
|
13071
|
+
raggrep query --dir ~/projects/my-app "user authentication"
|
|
12828
13072
|
raggrep query "handle errors" --top 5
|
|
12829
13073
|
raggrep query "database" --min-score 0.1
|
|
12830
13074
|
raggrep query "interface" --type ts
|
|
@@ -12850,9 +13094,10 @@ Examples:
|
|
|
12850
13094
|
console.error('Run "raggrep query --help" for more information.');
|
|
12851
13095
|
process.exit(1);
|
|
12852
13096
|
}
|
|
13097
|
+
const projectRoot = await resolveProjectRoot(flags);
|
|
12853
13098
|
try {
|
|
12854
13099
|
const silentLogger = createSilentLogger();
|
|
12855
|
-
const freshStats = await ensureIndexFresh2(
|
|
13100
|
+
const freshStats = await ensureIndexFresh2(projectRoot, {
|
|
12856
13101
|
model: flags.model,
|
|
12857
13102
|
quiet: true,
|
|
12858
13103
|
logger: silentLogger,
|
|
@@ -12893,7 +13138,7 @@ Examples:
|
|
|
12893
13138
|
}
|
|
12894
13139
|
const filePatterns = flags.fileType ? [`*.${flags.fileType}`] : undefined;
|
|
12895
13140
|
const { hybridSearch: hybridSearch2, formatHybridSearchResults: formatHybridSearchResults2 } = await Promise.resolve().then(() => (init_search(), exports_search));
|
|
12896
|
-
const hybridResults = await hybridSearch2(
|
|
13141
|
+
const hybridResults = await hybridSearch2(projectRoot, query, {
|
|
12897
13142
|
topK: flags.topK ?? 10,
|
|
12898
13143
|
minScore: flags.minScore,
|
|
12899
13144
|
filePatterns,
|
|
@@ -12910,26 +13155,29 @@ Examples:
|
|
|
12910
13155
|
case "reset": {
|
|
12911
13156
|
if (flags.help) {
|
|
12912
13157
|
console.log(`
|
|
12913
|
-
raggrep reset - Clear the index for
|
|
13158
|
+
raggrep reset - Clear the index for a project directory
|
|
12914
13159
|
|
|
12915
13160
|
Usage:
|
|
12916
13161
|
raggrep reset [options]
|
|
12917
13162
|
|
|
12918
13163
|
Options:
|
|
13164
|
+
-C, --dir <path> Project directory whose index to remove (default: current directory)
|
|
12919
13165
|
-h, --help Show this help message
|
|
12920
13166
|
|
|
12921
13167
|
Description:
|
|
12922
|
-
Completely removes the index for the
|
|
13168
|
+
Completely removes the .raggrep index for the project directory.
|
|
12923
13169
|
The next 'raggrep index' or 'raggrep query' will rebuild from scratch.
|
|
12924
13170
|
|
|
12925
13171
|
Examples:
|
|
12926
13172
|
raggrep reset
|
|
13173
|
+
raggrep reset --dir ../other-repo
|
|
12927
13174
|
`);
|
|
12928
13175
|
process.exit(0);
|
|
12929
13176
|
}
|
|
12930
13177
|
const { resetIndex: resetIndex2 } = await Promise.resolve().then(() => (init_indexer(), exports_indexer));
|
|
13178
|
+
const projectRoot = await resolveProjectRoot(flags);
|
|
12931
13179
|
try {
|
|
12932
|
-
const result = await resetIndex2(
|
|
13180
|
+
const result = await resetIndex2(projectRoot);
|
|
12933
13181
|
console.log("Index cleared successfully.");
|
|
12934
13182
|
console.log(` Removed: ${result.indexDir}`);
|
|
12935
13183
|
} catch (error) {
|
|
@@ -12951,21 +13199,24 @@ Usage:
|
|
|
12951
13199
|
raggrep status [options]
|
|
12952
13200
|
|
|
12953
13201
|
Options:
|
|
13202
|
+
-C, --dir <path> Project directory to report on (default: current directory)
|
|
12954
13203
|
-h, --help Show this help message
|
|
12955
13204
|
|
|
12956
13205
|
Description:
|
|
12957
|
-
Displays information about the index
|
|
13206
|
+
Displays information about the index for the project directory,
|
|
12958
13207
|
including whether it exists, how many files are indexed, and
|
|
12959
13208
|
when it was last updated.
|
|
12960
13209
|
|
|
12961
13210
|
Examples:
|
|
12962
13211
|
raggrep status
|
|
13212
|
+
raggrep status --dir ../other-repo
|
|
12963
13213
|
`);
|
|
12964
13214
|
process.exit(0);
|
|
12965
13215
|
}
|
|
12966
13216
|
const { getIndexStatus: getIndexStatus2 } = await Promise.resolve().then(() => (init_indexer(), exports_indexer));
|
|
13217
|
+
const projectRoot = await resolveProjectRoot(flags);
|
|
12967
13218
|
try {
|
|
12968
|
-
const status = await getIndexStatus2(
|
|
13219
|
+
const status = await getIndexStatus2(projectRoot);
|
|
12969
13220
|
if (!status.exists) {
|
|
12970
13221
|
console.log(`
|
|
12971
13222
|
┌─────────────────────────────────────────┐
|
|
@@ -13096,19 +13347,23 @@ Usage:
|
|
|
13096
13347
|
raggrep <command> [options]
|
|
13097
13348
|
|
|
13098
13349
|
Commands:
|
|
13099
|
-
index Index
|
|
13350
|
+
index Index a project directory
|
|
13100
13351
|
query Search the indexed codebase
|
|
13101
13352
|
status Show the current state of the index
|
|
13102
|
-
reset Clear the index for
|
|
13353
|
+
reset Clear the index for a project directory
|
|
13103
13354
|
opencode Manage opencode integration
|
|
13104
13355
|
|
|
13105
13356
|
Options:
|
|
13106
13357
|
-h, --help Show help for a command
|
|
13107
13358
|
-v, --version Show version number
|
|
13108
13359
|
|
|
13360
|
+
Project directory (index, query, status, reset):
|
|
13361
|
+
Use -C or --dir <path> to target a directory other than the current one.
|
|
13362
|
+
|
|
13109
13363
|
Examples:
|
|
13110
13364
|
raggrep index
|
|
13111
13365
|
raggrep query "user login"
|
|
13366
|
+
raggrep query -C ~/projects/app "user login"
|
|
13112
13367
|
raggrep status
|
|
13113
13368
|
raggrep reset
|
|
13114
13369
|
raggrep opencode install
|
|
@@ -13123,4 +13378,4 @@ Run 'raggrep <command> --help' for more information.
|
|
|
13123
13378
|
}
|
|
13124
13379
|
main();
|
|
13125
13380
|
|
|
13126
|
-
//# debugId=
|
|
13381
|
+
//# debugId=6B05E3A822FD1AE664756E2164756E21
|