raggrep 0.16.0 → 0.17.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +29 -5
- package/dist/cli/main.js +723 -460
- package/dist/cli/main.js.map +19 -13
- package/dist/domain/ports/embedding.d.ts +10 -0
- package/dist/domain/ports/index.d.ts +1 -1
- package/dist/index.js +529 -328
- package/dist/index.js.map +18 -12
- package/dist/infrastructure/config/configLoader.d.ts +9 -11
- package/dist/infrastructure/config/index.d.ts +1 -1
- package/dist/infrastructure/embeddings/embeddingPaths.d.ts +6 -0
- package/dist/infrastructure/embeddings/embeddingProviderFactory.d.ts +14 -0
- package/dist/infrastructure/embeddings/globalEmbeddings.d.ts +28 -0
- package/dist/infrastructure/embeddings/huggingfaceEmbeddingProvider.d.ts +21 -0
- package/dist/infrastructure/embeddings/index.d.ts +12 -2
- package/dist/infrastructure/embeddings/modelCache.d.ts +10 -0
- package/dist/infrastructure/embeddings/modelCatalog.d.ts +23 -0
- package/dist/infrastructure/embeddings/xenovaEmbeddingProvider.d.ts +23 -0
- package/dist/infrastructure/index.d.ts +1 -1
- package/package.json +12 -4
- package/dist/infrastructure/embeddings/transformersEmbedding.d.ts +0 -52
package/dist/cli/main.js
CHANGED
|
@@ -1,49 +1,89 @@
|
|
|
1
1
|
#!/usr/bin/env node
|
|
2
2
|
import { createRequire } from "node:module";
|
|
3
|
-
var __create = Object.create;
|
|
4
|
-
var __getProtoOf = Object.getPrototypeOf;
|
|
5
3
|
var __defProp = Object.defineProperty;
|
|
6
|
-
var
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
const to = isNodeMode || !mod || !mod.__esModule ? __defProp(target, "default", { value: mod, enumerable: true }) : target;
|
|
11
|
-
for (let key of __getOwnPropNames(mod))
|
|
12
|
-
if (!__hasOwnProp.call(to, key))
|
|
13
|
-
__defProp(to, key, {
|
|
14
|
-
get: () => mod[key],
|
|
15
|
-
enumerable: true
|
|
16
|
-
});
|
|
17
|
-
return to;
|
|
18
|
-
};
|
|
4
|
+
var __returnValue = (v) => v;
|
|
5
|
+
function __exportSetter(name, newValue) {
|
|
6
|
+
this[name] = __returnValue.bind(null, newValue);
|
|
7
|
+
}
|
|
19
8
|
var __export = (target, all) => {
|
|
20
9
|
for (var name in all)
|
|
21
10
|
__defProp(target, name, {
|
|
22
11
|
get: all[name],
|
|
23
12
|
enumerable: true,
|
|
24
13
|
configurable: true,
|
|
25
|
-
set: (
|
|
14
|
+
set: __exportSetter.bind(all, name)
|
|
26
15
|
});
|
|
27
16
|
};
|
|
28
17
|
var __esm = (fn, res) => () => (fn && (res = fn(fn = 0)), res);
|
|
29
18
|
var __require = /* @__PURE__ */ createRequire(import.meta.url);
|
|
30
19
|
|
|
31
|
-
// src/infrastructure/embeddings/
|
|
20
|
+
// src/infrastructure/embeddings/modelCatalog.ts
|
|
21
|
+
function getEmbeddingModelId(model) {
|
|
22
|
+
return EMBEDDING_MODEL_IDS[model];
|
|
23
|
+
}
|
|
24
|
+
function getEmbeddingDimension(model) {
|
|
25
|
+
return EMBEDDING_DIMENSIONS[model];
|
|
26
|
+
}
|
|
27
|
+
var EMBEDDING_MODEL_IDS, EMBEDDING_MODELS, EMBEDDING_DIMENSIONS;
|
|
28
|
+
var init_modelCatalog = __esm(() => {
|
|
29
|
+
EMBEDDING_MODEL_IDS = {
|
|
30
|
+
"all-MiniLM-L6-v2": "Xenova/all-MiniLM-L6-v2",
|
|
31
|
+
"all-MiniLM-L12-v2": "Xenova/all-MiniLM-L12-v2",
|
|
32
|
+
"bge-small-en-v1.5": "Xenova/bge-small-en-v1.5",
|
|
33
|
+
"paraphrase-MiniLM-L3-v2": "Xenova/paraphrase-MiniLM-L3-v2",
|
|
34
|
+
"nomic-embed-text-v1.5": "nomic-ai/nomic-embed-text-v1.5"
|
|
35
|
+
};
|
|
36
|
+
EMBEDDING_MODELS = EMBEDDING_MODEL_IDS;
|
|
37
|
+
EMBEDDING_DIMENSIONS = {
|
|
38
|
+
"all-MiniLM-L6-v2": 384,
|
|
39
|
+
"all-MiniLM-L12-v2": 384,
|
|
40
|
+
"bge-small-en-v1.5": 384,
|
|
41
|
+
"paraphrase-MiniLM-L3-v2": 384,
|
|
42
|
+
"nomic-embed-text-v1.5": 768
|
|
43
|
+
};
|
|
44
|
+
});
|
|
45
|
+
|
|
46
|
+
// src/infrastructure/embeddings/embeddingPaths.ts
|
|
47
|
+
import * as os from "os";
|
|
48
|
+
import * as path from "path";
|
|
49
|
+
var RAGGREP_MODEL_CACHE_DIR;
|
|
50
|
+
var init_embeddingPaths = __esm(() => {
|
|
51
|
+
RAGGREP_MODEL_CACHE_DIR = path.join(os.homedir(), ".cache", "raggrep", "models");
|
|
52
|
+
});
|
|
53
|
+
|
|
54
|
+
// src/infrastructure/embeddings/modelCache.ts
|
|
55
|
+
import * as fs from "fs/promises";
|
|
56
|
+
import * as path2 from "path";
|
|
57
|
+
async function isEmbeddingModelCached(model) {
|
|
58
|
+
const modelId = getEmbeddingModelId(model);
|
|
59
|
+
const onnxPath = path2.join(RAGGREP_MODEL_CACHE_DIR, modelId, "onnx", "model_quantized.onnx");
|
|
60
|
+
try {
|
|
61
|
+
await fs.access(onnxPath);
|
|
62
|
+
return true;
|
|
63
|
+
} catch {
|
|
64
|
+
return false;
|
|
65
|
+
}
|
|
66
|
+
}
|
|
67
|
+
var init_modelCache = __esm(() => {
|
|
68
|
+
init_embeddingPaths();
|
|
69
|
+
init_modelCatalog();
|
|
70
|
+
});
|
|
71
|
+
|
|
72
|
+
// src/infrastructure/embeddings/huggingfaceEmbeddingProvider.ts
|
|
32
73
|
import {
|
|
33
74
|
pipeline,
|
|
34
75
|
env
|
|
35
|
-
} from "@
|
|
36
|
-
import * as path from "path";
|
|
37
|
-
import * as os from "os";
|
|
76
|
+
} from "@huggingface/transformers";
|
|
38
77
|
|
|
39
|
-
class
|
|
40
|
-
|
|
78
|
+
class HuggingFaceTransformersEmbeddingProvider {
|
|
79
|
+
extractor = null;
|
|
41
80
|
config;
|
|
42
81
|
isInitializing = false;
|
|
43
82
|
initPromise = null;
|
|
44
83
|
constructor(config) {
|
|
45
84
|
this.config = {
|
|
46
85
|
model: config?.model ?? "bge-small-en-v1.5",
|
|
86
|
+
runtime: config?.runtime ?? "huggingface",
|
|
47
87
|
showProgress: config?.showProgress ?? false,
|
|
48
88
|
logger: config?.logger
|
|
49
89
|
};
|
|
@@ -51,14 +91,14 @@ class TransformersEmbeddingProvider {
|
|
|
51
91
|
async initialize(config) {
|
|
52
92
|
if (config) {
|
|
53
93
|
if (config.model !== this.config.model) {
|
|
54
|
-
this.
|
|
94
|
+
this.extractor = null;
|
|
55
95
|
}
|
|
56
96
|
this.config = { ...this.config, ...config };
|
|
57
97
|
}
|
|
58
|
-
await this.
|
|
98
|
+
await this.ensureExtractor();
|
|
59
99
|
}
|
|
60
|
-
async
|
|
61
|
-
if (this.
|
|
100
|
+
async ensureExtractor() {
|
|
101
|
+
if (this.extractor) {
|
|
62
102
|
return;
|
|
63
103
|
}
|
|
64
104
|
if (this.isInitializing && this.initPromise) {
|
|
@@ -66,14 +106,14 @@ class TransformersEmbeddingProvider {
|
|
|
66
106
|
}
|
|
67
107
|
this.isInitializing = true;
|
|
68
108
|
this.initPromise = (async () => {
|
|
69
|
-
const modelId =
|
|
109
|
+
const modelId = getEmbeddingModelId(this.config.model);
|
|
70
110
|
const logger = this.config.logger;
|
|
71
111
|
const showProgress = this.config.showProgress || !!logger;
|
|
72
|
-
const
|
|
112
|
+
const cached = await isEmbeddingModelCached(this.config.model);
|
|
73
113
|
let hasDownloads = false;
|
|
74
114
|
try {
|
|
75
|
-
this.
|
|
76
|
-
progress_callback: showProgress && !
|
|
115
|
+
this.extractor = await pipeline("feature-extraction", modelId, {
|
|
116
|
+
progress_callback: showProgress && !cached ? (progress) => {
|
|
77
117
|
if (progress.status === "progress" && progress.file) {
|
|
78
118
|
if (!hasDownloads) {
|
|
79
119
|
hasDownloads = true;
|
|
@@ -82,7 +122,7 @@ class TransformersEmbeddingProvider {
|
|
|
82
122
|
} else {
|
|
83
123
|
console.log(`
|
|
84
124
|
Loading embedding model: ${this.config.model}`);
|
|
85
|
-
console.log(` Cache: ${
|
|
125
|
+
console.log(` Cache: ${RAGGREP_MODEL_CACHE_DIR}`);
|
|
86
126
|
}
|
|
87
127
|
}
|
|
88
128
|
const pct = progress.progress ? Math.round(progress.progress) : 0;
|
|
@@ -112,9 +152,9 @@ class TransformersEmbeddingProvider {
|
|
|
112
152
|
}
|
|
113
153
|
}
|
|
114
154
|
} catch (error) {
|
|
115
|
-
this.
|
|
116
|
-
if (logger) {
|
|
117
|
-
logger.clearProgress();
|
|
155
|
+
this.extractor = null;
|
|
156
|
+
if (this.config.logger) {
|
|
157
|
+
this.config.logger.clearProgress();
|
|
118
158
|
}
|
|
119
159
|
throw new Error(`Failed to load embedding model: ${error}`);
|
|
120
160
|
} finally {
|
|
@@ -125,11 +165,11 @@ class TransformersEmbeddingProvider {
|
|
|
125
165
|
return this.initPromise;
|
|
126
166
|
}
|
|
127
167
|
async getEmbedding(text) {
|
|
128
|
-
await this.
|
|
129
|
-
if (!this.
|
|
168
|
+
await this.ensureExtractor();
|
|
169
|
+
if (!this.extractor) {
|
|
130
170
|
throw new Error("Embedding pipeline not initialized");
|
|
131
171
|
}
|
|
132
|
-
const output = await this.
|
|
172
|
+
const output = await this.extractor(text, {
|
|
133
173
|
pooling: "mean",
|
|
134
174
|
normalize: true
|
|
135
175
|
});
|
|
@@ -138,15 +178,15 @@ class TransformersEmbeddingProvider {
|
|
|
138
178
|
async getEmbeddings(texts) {
|
|
139
179
|
if (texts.length === 0)
|
|
140
180
|
return [];
|
|
141
|
-
await this.
|
|
142
|
-
if (!this.
|
|
181
|
+
await this.ensureExtractor();
|
|
182
|
+
if (!this.extractor) {
|
|
143
183
|
throw new Error("Embedding pipeline not initialized");
|
|
144
184
|
}
|
|
145
185
|
const results = [];
|
|
146
186
|
for (let i = 0;i < texts.length; i += BATCH_SIZE) {
|
|
147
187
|
const batch = texts.slice(i, i + BATCH_SIZE);
|
|
148
188
|
const outputs = await Promise.all(batch.map(async (text) => {
|
|
149
|
-
const output = await this.
|
|
189
|
+
const output = await this.extractor(text, {
|
|
150
190
|
pooling: "mean",
|
|
151
191
|
normalize: true
|
|
152
192
|
});
|
|
@@ -157,44 +197,216 @@ class TransformersEmbeddingProvider {
|
|
|
157
197
|
return results;
|
|
158
198
|
}
|
|
159
199
|
getDimension() {
|
|
160
|
-
return
|
|
200
|
+
return getEmbeddingDimension(this.config.model);
|
|
161
201
|
}
|
|
162
202
|
getModelName() {
|
|
163
203
|
return this.config.model;
|
|
164
204
|
}
|
|
165
205
|
async dispose() {
|
|
166
|
-
this.
|
|
206
|
+
this.extractor = null;
|
|
167
207
|
}
|
|
168
208
|
}
|
|
169
|
-
|
|
170
|
-
|
|
209
|
+
var BATCH_SIZE = 32;
|
|
210
|
+
var init_huggingfaceEmbeddingProvider = __esm(() => {
|
|
211
|
+
init_embeddingPaths();
|
|
212
|
+
init_modelCatalog();
|
|
213
|
+
init_modelCache();
|
|
214
|
+
env.cacheDir = RAGGREP_MODEL_CACHE_DIR;
|
|
215
|
+
env.allowLocalModels = true;
|
|
216
|
+
});
|
|
217
|
+
|
|
218
|
+
// src/infrastructure/embeddings/xenovaEmbeddingProvider.ts
|
|
219
|
+
var exports_xenovaEmbeddingProvider = {};
|
|
220
|
+
__export(exports_xenovaEmbeddingProvider, {
|
|
221
|
+
XenovaTransformersEmbeddingProvider: () => XenovaTransformersEmbeddingProvider,
|
|
222
|
+
TransformersEmbeddingProvider: () => TransformersEmbeddingProvider
|
|
223
|
+
});
|
|
224
|
+
import {
|
|
225
|
+
pipeline as pipeline2,
|
|
226
|
+
env as env2
|
|
227
|
+
} from "@xenova/transformers";
|
|
228
|
+
|
|
229
|
+
class XenovaTransformersEmbeddingProvider {
|
|
230
|
+
extractor = null;
|
|
231
|
+
config;
|
|
232
|
+
isInitializing = false;
|
|
233
|
+
initPromise = null;
|
|
234
|
+
constructor(config) {
|
|
235
|
+
this.config = {
|
|
236
|
+
model: config?.model ?? "bge-small-en-v1.5",
|
|
237
|
+
runtime: config?.runtime ?? "xenova",
|
|
238
|
+
showProgress: config?.showProgress ?? false,
|
|
239
|
+
logger: config?.logger
|
|
240
|
+
};
|
|
241
|
+
}
|
|
242
|
+
async initialize(config) {
|
|
243
|
+
if (config) {
|
|
244
|
+
if (config.model !== this.config.model) {
|
|
245
|
+
this.extractor = null;
|
|
246
|
+
}
|
|
247
|
+
this.config = { ...this.config, ...config };
|
|
248
|
+
}
|
|
249
|
+
await this.ensureExtractor();
|
|
250
|
+
}
|
|
251
|
+
async ensureExtractor() {
|
|
252
|
+
if (this.extractor) {
|
|
253
|
+
return;
|
|
254
|
+
}
|
|
255
|
+
if (this.isInitializing && this.initPromise) {
|
|
256
|
+
return this.initPromise;
|
|
257
|
+
}
|
|
258
|
+
this.isInitializing = true;
|
|
259
|
+
this.initPromise = (async () => {
|
|
260
|
+
const modelId = getEmbeddingModelId(this.config.model);
|
|
261
|
+
const logger = this.config.logger;
|
|
262
|
+
const showProgress = this.config.showProgress || !!logger;
|
|
263
|
+
const cached = await isEmbeddingModelCached(this.config.model);
|
|
264
|
+
let hasDownloads = false;
|
|
265
|
+
try {
|
|
266
|
+
this.extractor = await pipeline2("feature-extraction", modelId, {
|
|
267
|
+
progress_callback: showProgress && !cached ? (progress) => {
|
|
268
|
+
if (progress.status === "progress" && progress.file) {
|
|
269
|
+
if (!hasDownloads) {
|
|
270
|
+
hasDownloads = true;
|
|
271
|
+
if (logger) {
|
|
272
|
+
logger.info(`Downloading embedding model: ${this.config.model}`);
|
|
273
|
+
} else {
|
|
274
|
+
console.log(`
|
|
275
|
+
Loading embedding model: ${this.config.model}`);
|
|
276
|
+
console.log(` Cache: ${RAGGREP_MODEL_CACHE_DIR}`);
|
|
277
|
+
}
|
|
278
|
+
}
|
|
279
|
+
const pct = progress.progress ? Math.round(progress.progress) : 0;
|
|
280
|
+
if (logger) {
|
|
281
|
+
logger.progress(` Downloading ${progress.file}: ${pct}%`);
|
|
282
|
+
} else {
|
|
283
|
+
process.stdout.write(`\r Downloading ${progress.file}: ${pct}% `);
|
|
284
|
+
}
|
|
285
|
+
} else if (progress.status === "done" && progress.file) {
|
|
286
|
+
if (logger) {
|
|
287
|
+
logger.clearProgress();
|
|
288
|
+
logger.info(` Downloaded ${progress.file}`);
|
|
289
|
+
} else if (hasDownloads) {
|
|
290
|
+
process.stdout.write(`\r Downloaded ${progress.file}
|
|
291
|
+
`);
|
|
292
|
+
}
|
|
293
|
+
}
|
|
294
|
+
} : undefined
|
|
295
|
+
});
|
|
296
|
+
if (hasDownloads) {
|
|
297
|
+
if (logger) {
|
|
298
|
+
logger.clearProgress();
|
|
299
|
+
logger.info(`Model ready: ${this.config.model}`);
|
|
300
|
+
} else {
|
|
301
|
+
console.log(` Model ready.
|
|
302
|
+
`);
|
|
303
|
+
}
|
|
304
|
+
}
|
|
305
|
+
} catch (error) {
|
|
306
|
+
this.extractor = null;
|
|
307
|
+
if (this.config.logger) {
|
|
308
|
+
this.config.logger.clearProgress();
|
|
309
|
+
}
|
|
310
|
+
throw new Error(`Failed to load embedding model: ${error}`);
|
|
311
|
+
} finally {
|
|
312
|
+
this.isInitializing = false;
|
|
313
|
+
this.initPromise = null;
|
|
314
|
+
}
|
|
315
|
+
})();
|
|
316
|
+
return this.initPromise;
|
|
317
|
+
}
|
|
318
|
+
async getEmbedding(text) {
|
|
319
|
+
await this.ensureExtractor();
|
|
320
|
+
if (!this.extractor) {
|
|
321
|
+
throw new Error("Embedding pipeline not initialized");
|
|
322
|
+
}
|
|
323
|
+
const output = await this.extractor(text, {
|
|
324
|
+
pooling: "mean",
|
|
325
|
+
normalize: true
|
|
326
|
+
});
|
|
327
|
+
return Array.from(output.data);
|
|
328
|
+
}
|
|
329
|
+
async getEmbeddings(texts) {
|
|
330
|
+
if (texts.length === 0)
|
|
331
|
+
return [];
|
|
332
|
+
await this.ensureExtractor();
|
|
333
|
+
if (!this.extractor) {
|
|
334
|
+
throw new Error("Embedding pipeline not initialized");
|
|
335
|
+
}
|
|
336
|
+
const results = [];
|
|
337
|
+
for (let i = 0;i < texts.length; i += BATCH_SIZE2) {
|
|
338
|
+
const batch = texts.slice(i, i + BATCH_SIZE2);
|
|
339
|
+
const outputs = await Promise.all(batch.map(async (text) => {
|
|
340
|
+
const output = await this.extractor(text, {
|
|
341
|
+
pooling: "mean",
|
|
342
|
+
normalize: true
|
|
343
|
+
});
|
|
344
|
+
return Array.from(output.data);
|
|
345
|
+
}));
|
|
346
|
+
results.push(...outputs);
|
|
347
|
+
}
|
|
348
|
+
return results;
|
|
349
|
+
}
|
|
350
|
+
getDimension() {
|
|
351
|
+
return getEmbeddingDimension(this.config.model);
|
|
352
|
+
}
|
|
353
|
+
getModelName() {
|
|
354
|
+
return this.config.model;
|
|
355
|
+
}
|
|
356
|
+
async dispose() {
|
|
357
|
+
this.extractor = null;
|
|
358
|
+
}
|
|
171
359
|
}
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
|
|
360
|
+
var BATCH_SIZE2 = 32, TransformersEmbeddingProvider;
|
|
361
|
+
var init_xenovaEmbeddingProvider = __esm(() => {
|
|
362
|
+
init_embeddingPaths();
|
|
363
|
+
init_modelCatalog();
|
|
364
|
+
init_modelCache();
|
|
365
|
+
env2.cacheDir = RAGGREP_MODEL_CACHE_DIR;
|
|
366
|
+
env2.allowLocalModels = true;
|
|
367
|
+
TransformersEmbeddingProvider = XenovaTransformersEmbeddingProvider;
|
|
368
|
+
});
|
|
369
|
+
|
|
370
|
+
// src/infrastructure/embeddings/embeddingProviderFactory.ts
|
|
371
|
+
function resolveRuntime(config) {
|
|
372
|
+
return config.runtime ?? "huggingface";
|
|
373
|
+
}
|
|
374
|
+
async function createEmbeddingProvider(config) {
|
|
375
|
+
const runtime = resolveRuntime(config);
|
|
376
|
+
if (runtime === "huggingface") {
|
|
377
|
+
return new HuggingFaceTransformersEmbeddingProvider(config);
|
|
182
378
|
}
|
|
379
|
+
const { XenovaTransformersEmbeddingProvider: XenovaTransformersEmbeddingProvider2 } = await Promise.resolve().then(() => (init_xenovaEmbeddingProvider(), exports_xenovaEmbeddingProvider));
|
|
380
|
+
return new XenovaTransformersEmbeddingProvider2(config);
|
|
183
381
|
}
|
|
382
|
+
var init_embeddingProviderFactory = __esm(() => {
|
|
383
|
+
init_huggingfaceEmbeddingProvider();
|
|
384
|
+
});
|
|
385
|
+
|
|
386
|
+
// src/infrastructure/embeddings/globalEmbeddings.ts
|
|
184
387
|
function configureEmbeddings(config) {
|
|
185
|
-
const
|
|
186
|
-
|
|
388
|
+
const merged = {
|
|
389
|
+
...globalConfig,
|
|
390
|
+
...config
|
|
391
|
+
};
|
|
392
|
+
if (merged.runtime === undefined) {
|
|
393
|
+
merged.runtime = "huggingface";
|
|
394
|
+
}
|
|
395
|
+
const needsReset = merged.model !== globalConfig.model || merged.runtime !== globalConfig.runtime || merged.logger !== globalConfig.logger;
|
|
396
|
+
if (needsReset) {
|
|
397
|
+
const prev = globalProvider;
|
|
187
398
|
globalProvider = null;
|
|
399
|
+
prev?.dispose?.();
|
|
188
400
|
}
|
|
189
|
-
globalConfig =
|
|
401
|
+
globalConfig = merged;
|
|
190
402
|
}
|
|
191
403
|
function getEmbeddingConfig() {
|
|
192
404
|
return { ...globalConfig };
|
|
193
405
|
}
|
|
194
406
|
async function ensureGlobalProvider() {
|
|
195
407
|
if (!globalProvider) {
|
|
196
|
-
globalProvider =
|
|
197
|
-
await globalProvider.initialize();
|
|
408
|
+
globalProvider = await createEmbeddingProvider(globalConfig);
|
|
409
|
+
await globalProvider.initialize?.(globalConfig);
|
|
198
410
|
}
|
|
199
411
|
return globalProvider;
|
|
200
412
|
}
|
|
@@ -206,27 +418,17 @@ async function getEmbeddings(texts) {
|
|
|
206
418
|
const provider = await ensureGlobalProvider();
|
|
207
419
|
return provider.getEmbeddings(texts);
|
|
208
420
|
}
|
|
209
|
-
|
|
210
|
-
|
|
211
|
-
|
|
212
|
-
|
|
213
|
-
|
|
214
|
-
|
|
215
|
-
|
|
216
|
-
|
|
217
|
-
"bge-small-en-v1.5": "Xenova/bge-small-en-v1.5",
|
|
218
|
-
"paraphrase-MiniLM-L3-v2": "Xenova/paraphrase-MiniLM-L3-v2",
|
|
219
|
-
"nomic-embed-text-v1.5": "nomic-ai/nomic-embed-text-v1.5"
|
|
220
|
-
};
|
|
221
|
-
EMBEDDING_DIMENSIONS = {
|
|
222
|
-
"all-MiniLM-L6-v2": 384,
|
|
223
|
-
"all-MiniLM-L12-v2": 384,
|
|
224
|
-
"bge-small-en-v1.5": 384,
|
|
225
|
-
"paraphrase-MiniLM-L3-v2": 384,
|
|
226
|
-
"nomic-embed-text-v1.5": 768
|
|
227
|
-
};
|
|
421
|
+
function getCacheDir() {
|
|
422
|
+
return RAGGREP_MODEL_CACHE_DIR;
|
|
423
|
+
}
|
|
424
|
+
var globalProvider = null, globalConfig;
|
|
425
|
+
var init_globalEmbeddings = __esm(() => {
|
|
426
|
+
init_embeddingPaths();
|
|
427
|
+
init_embeddingProviderFactory();
|
|
428
|
+
init_modelCache();
|
|
228
429
|
globalConfig = {
|
|
229
430
|
model: "bge-small-en-v1.5",
|
|
431
|
+
runtime: "huggingface",
|
|
230
432
|
showProgress: false,
|
|
231
433
|
logger: undefined
|
|
232
434
|
};
|
|
@@ -234,7 +436,11 @@ var init_transformersEmbedding = __esm(() => {
|
|
|
234
436
|
|
|
235
437
|
// src/infrastructure/embeddings/index.ts
|
|
236
438
|
var init_embeddings = __esm(() => {
|
|
237
|
-
|
|
439
|
+
init_modelCatalog();
|
|
440
|
+
init_embeddingPaths();
|
|
441
|
+
init_huggingfaceEmbeddingProvider();
|
|
442
|
+
init_embeddingProviderFactory();
|
|
443
|
+
init_globalEmbeddings();
|
|
238
444
|
});
|
|
239
445
|
|
|
240
446
|
// src/infrastructure/logger/loggers.ts
|
|
@@ -457,29 +663,29 @@ var init_logger = () => {};
|
|
|
457
663
|
import { createRequire as createRequire2 } from "module";
|
|
458
664
|
import { basename, dirname, normalize, relative, resolve, sep } from "path";
|
|
459
665
|
import * as nativeFs from "fs";
|
|
460
|
-
function cleanPath(
|
|
461
|
-
let normalized = normalize(
|
|
666
|
+
function cleanPath(path3) {
|
|
667
|
+
let normalized = normalize(path3);
|
|
462
668
|
if (normalized.length > 1 && normalized[normalized.length - 1] === sep)
|
|
463
669
|
normalized = normalized.substring(0, normalized.length - 1);
|
|
464
670
|
return normalized;
|
|
465
671
|
}
|
|
466
|
-
function convertSlashes(
|
|
467
|
-
return
|
|
672
|
+
function convertSlashes(path3, separator) {
|
|
673
|
+
return path3.replace(SLASHES_REGEX, separator);
|
|
468
674
|
}
|
|
469
|
-
function isRootDirectory(
|
|
470
|
-
return
|
|
675
|
+
function isRootDirectory(path3) {
|
|
676
|
+
return path3 === "/" || WINDOWS_ROOT_DIR_REGEX.test(path3);
|
|
471
677
|
}
|
|
472
|
-
function normalizePath(
|
|
678
|
+
function normalizePath(path3, options) {
|
|
473
679
|
const { resolvePaths, normalizePath: normalizePath$1, pathSeparator } = options;
|
|
474
|
-
const pathNeedsCleaning = process.platform === "win32" &&
|
|
680
|
+
const pathNeedsCleaning = process.platform === "win32" && path3.includes("/") || path3.startsWith(".");
|
|
475
681
|
if (resolvePaths)
|
|
476
|
-
|
|
682
|
+
path3 = resolve(path3);
|
|
477
683
|
if (normalizePath$1 || pathNeedsCleaning)
|
|
478
|
-
|
|
479
|
-
if (
|
|
684
|
+
path3 = cleanPath(path3);
|
|
685
|
+
if (path3 === ".")
|
|
480
686
|
return "";
|
|
481
|
-
const needsSeperator =
|
|
482
|
-
return convertSlashes(needsSeperator ?
|
|
687
|
+
const needsSeperator = path3[path3.length - 1] !== pathSeparator;
|
|
688
|
+
return convertSlashes(needsSeperator ? path3 + pathSeparator : path3, pathSeparator);
|
|
483
689
|
}
|
|
484
690
|
function joinPathWithBasePath(filename, directoryPath) {
|
|
485
691
|
return directoryPath + filename;
|
|
@@ -545,10 +751,10 @@ function build$2(options, isSynchronous) {
|
|
|
545
751
|
return null;
|
|
546
752
|
return isSynchronous ? resolveSymlinks : resolveSymlinksAsync;
|
|
547
753
|
}
|
|
548
|
-
function isRecursive(
|
|
754
|
+
function isRecursive(path3, resolved, state) {
|
|
549
755
|
if (state.options.useRealPaths)
|
|
550
756
|
return isRecursiveUsingRealPaths(resolved, state);
|
|
551
|
-
let parent = dirname(
|
|
757
|
+
let parent = dirname(path3);
|
|
552
758
|
let depth = 1;
|
|
553
759
|
while (parent !== state.root && depth < 2) {
|
|
554
760
|
const resolvedPath = state.symlinks.get(parent);
|
|
@@ -558,7 +764,7 @@ function isRecursive(path2, resolved, state) {
|
|
|
558
764
|
else
|
|
559
765
|
parent = dirname(parent);
|
|
560
766
|
}
|
|
561
|
-
state.symlinks.set(
|
|
767
|
+
state.symlinks.set(path3, resolved);
|
|
562
768
|
return depth > 1;
|
|
563
769
|
}
|
|
564
770
|
function isRecursiveUsingRealPaths(resolved, state) {
|
|
@@ -604,9 +810,9 @@ function sync(root, options) {
|
|
|
604
810
|
var __require2, SLASHES_REGEX, WINDOWS_ROOT_DIR_REGEX, pushDirectory = (directoryPath, paths) => {
|
|
605
811
|
paths.push(directoryPath || ".");
|
|
606
812
|
}, pushDirectoryFilter = (directoryPath, paths, filters) => {
|
|
607
|
-
const
|
|
608
|
-
if (filters.every((filter) => filter(
|
|
609
|
-
paths.push(
|
|
813
|
+
const path3 = directoryPath || ".";
|
|
814
|
+
if (filters.every((filter) => filter(path3, true)))
|
|
815
|
+
paths.push(path3);
|
|
610
816
|
}, empty$2 = () => {}, pushFileFilterAndCount = (filename, _paths, counts, filters) => {
|
|
611
817
|
if (filters.every((filter) => filter(filename, false)))
|
|
612
818
|
counts.files++;
|
|
@@ -627,28 +833,28 @@ var __require2, SLASHES_REGEX, WINDOWS_ROOT_DIR_REGEX, pushDirectory = (director
|
|
|
627
833
|
files,
|
|
628
834
|
dir: directory
|
|
629
835
|
});
|
|
630
|
-
}, empty = () => {}, resolveSymlinksAsync = function(
|
|
631
|
-
const { queue, fs, options: { suppressErrors } } = state;
|
|
836
|
+
}, empty = () => {}, resolveSymlinksAsync = function(path3, state, callback$1) {
|
|
837
|
+
const { queue, fs: fs2, options: { suppressErrors } } = state;
|
|
632
838
|
queue.enqueue();
|
|
633
|
-
|
|
839
|
+
fs2.realpath(path3, (error, resolvedPath) => {
|
|
634
840
|
if (error)
|
|
635
841
|
return queue.dequeue(suppressErrors ? null : error, state);
|
|
636
|
-
|
|
842
|
+
fs2.stat(resolvedPath, (error$1, stat) => {
|
|
637
843
|
if (error$1)
|
|
638
844
|
return queue.dequeue(suppressErrors ? null : error$1, state);
|
|
639
|
-
if (stat.isDirectory() && isRecursive(
|
|
845
|
+
if (stat.isDirectory() && isRecursive(path3, resolvedPath, state))
|
|
640
846
|
return queue.dequeue(null, state);
|
|
641
847
|
callback$1(stat, resolvedPath);
|
|
642
848
|
queue.dequeue(null, state);
|
|
643
849
|
});
|
|
644
850
|
});
|
|
645
|
-
}, resolveSymlinks = function(
|
|
646
|
-
const { queue, fs, options: { suppressErrors } } = state;
|
|
851
|
+
}, resolveSymlinks = function(path3, state, callback$1) {
|
|
852
|
+
const { queue, fs: fs2, options: { suppressErrors } } = state;
|
|
647
853
|
queue.enqueue();
|
|
648
854
|
try {
|
|
649
|
-
const resolvedPath =
|
|
650
|
-
const stat =
|
|
651
|
-
if (stat.isDirectory() && isRecursive(
|
|
855
|
+
const resolvedPath = fs2.realpathSync(path3);
|
|
856
|
+
const stat = fs2.statSync(resolvedPath);
|
|
857
|
+
if (stat.isDirectory() && isRecursive(path3, resolvedPath, state))
|
|
652
858
|
return;
|
|
653
859
|
callback$1(stat, resolvedPath);
|
|
654
860
|
} catch (e) {
|
|
@@ -679,22 +885,22 @@ var __require2, SLASHES_REGEX, WINDOWS_ROOT_DIR_REGEX, pushDirectory = (director
|
|
|
679
885
|
state.queue.enqueue();
|
|
680
886
|
if (currentDepth < 0)
|
|
681
887
|
return state.queue.dequeue(null, state);
|
|
682
|
-
const { fs } = state;
|
|
888
|
+
const { fs: fs2 } = state;
|
|
683
889
|
state.visited.push(crawlPath);
|
|
684
890
|
state.counts.directories++;
|
|
685
|
-
|
|
891
|
+
fs2.readdir(crawlPath || ".", readdirOpts, (error, entries = []) => {
|
|
686
892
|
callback$1(entries, directoryPath, currentDepth);
|
|
687
893
|
state.queue.dequeue(state.options.suppressErrors ? null : error, state);
|
|
688
894
|
});
|
|
689
895
|
}, walkSync = (state, crawlPath, directoryPath, currentDepth, callback$1) => {
|
|
690
|
-
const { fs } = state;
|
|
896
|
+
const { fs: fs2 } = state;
|
|
691
897
|
if (currentDepth < 0)
|
|
692
898
|
return;
|
|
693
899
|
state.visited.push(crawlPath);
|
|
694
900
|
state.counts.directories++;
|
|
695
901
|
let entries = [];
|
|
696
902
|
try {
|
|
697
|
-
entries =
|
|
903
|
+
entries = fs2.readdirSync(crawlPath || ".", readdirOpts);
|
|
698
904
|
} catch (e) {
|
|
699
905
|
if (!state.options.suppressErrors)
|
|
700
906
|
throw e;
|
|
@@ -793,21 +999,21 @@ var __require2, SLASHES_REGEX, WINDOWS_ROOT_DIR_REGEX, pushDirectory = (director
|
|
|
793
999
|
const filename = this.joinPath(entry.name, directoryPath);
|
|
794
1000
|
this.pushFile(filename, files, this.state.counts, filters);
|
|
795
1001
|
} else if (entry.isDirectory()) {
|
|
796
|
-
let
|
|
797
|
-
if (exclude && exclude(entry.name,
|
|
1002
|
+
let path3 = joinDirectoryPath(entry.name, directoryPath, this.state.options.pathSeparator);
|
|
1003
|
+
if (exclude && exclude(entry.name, path3))
|
|
798
1004
|
continue;
|
|
799
|
-
this.pushDirectory(
|
|
800
|
-
this.walkDirectory(this.state,
|
|
1005
|
+
this.pushDirectory(path3, paths, filters);
|
|
1006
|
+
this.walkDirectory(this.state, path3, path3, depth - 1, this.walk);
|
|
801
1007
|
} else if (this.resolveSymlink && entry.isSymbolicLink()) {
|
|
802
|
-
let
|
|
803
|
-
this.resolveSymlink(
|
|
1008
|
+
let path3 = joinPathWithBasePath(entry.name, directoryPath);
|
|
1009
|
+
this.resolveSymlink(path3, this.state, (stat, resolvedPath) => {
|
|
804
1010
|
if (stat.isDirectory()) {
|
|
805
1011
|
resolvedPath = normalizePath(resolvedPath, this.state.options);
|
|
806
|
-
if (exclude && exclude(entry.name, useRealPaths ? resolvedPath :
|
|
1012
|
+
if (exclude && exclude(entry.name, useRealPaths ? resolvedPath : path3 + pathSeparator))
|
|
807
1013
|
return;
|
|
808
|
-
this.walkDirectory(this.state, resolvedPath, useRealPaths ? resolvedPath :
|
|
1014
|
+
this.walkDirectory(this.state, resolvedPath, useRealPaths ? resolvedPath : path3 + pathSeparator, depth - 1, this.walk);
|
|
809
1015
|
} else {
|
|
810
|
-
resolvedPath = useRealPaths ? resolvedPath :
|
|
1016
|
+
resolvedPath = useRealPaths ? resolvedPath : path3;
|
|
811
1017
|
const filename = basename(resolvedPath);
|
|
812
1018
|
const directoryPath$1 = normalizePath(dirname(resolvedPath), this.state.options);
|
|
813
1019
|
resolvedPath = this.joinPath(filename, directoryPath$1);
|
|
@@ -943,7 +1149,7 @@ var __require2, SLASHES_REGEX, WINDOWS_ROOT_DIR_REGEX, pushDirectory = (director
|
|
|
943
1149
|
isMatch = globFn(patterns, ...options);
|
|
944
1150
|
this.globCache[patterns.join("\x00")] = isMatch;
|
|
945
1151
|
}
|
|
946
|
-
this.options.filters.push((
|
|
1152
|
+
this.options.filters.push((path3) => isMatch(path3));
|
|
947
1153
|
return this;
|
|
948
1154
|
}
|
|
949
1155
|
};
|
|
@@ -985,49 +1191,37 @@ function createDefaultConfig() {
|
|
|
985
1191
|
{
|
|
986
1192
|
id: "language/typescript",
|
|
987
1193
|
enabled: true,
|
|
988
|
-
options: {
|
|
989
|
-
embeddingModel: "all-MiniLM-L6-v2"
|
|
990
|
-
}
|
|
1194
|
+
options: { ...DEFAULT_EMBEDDING_MODULE_OPTIONS }
|
|
991
1195
|
},
|
|
992
1196
|
{
|
|
993
1197
|
id: "language/python",
|
|
994
1198
|
enabled: true,
|
|
995
|
-
options: {
|
|
996
|
-
embeddingModel: "all-MiniLM-L6-v2"
|
|
997
|
-
}
|
|
1199
|
+
options: { ...DEFAULT_EMBEDDING_MODULE_OPTIONS }
|
|
998
1200
|
},
|
|
999
1201
|
{
|
|
1000
1202
|
id: "language/go",
|
|
1001
1203
|
enabled: true,
|
|
1002
|
-
options: {
|
|
1003
|
-
embeddingModel: "all-MiniLM-L6-v2"
|
|
1004
|
-
}
|
|
1204
|
+
options: { ...DEFAULT_EMBEDDING_MODULE_OPTIONS }
|
|
1005
1205
|
},
|
|
1006
1206
|
{
|
|
1007
1207
|
id: "language/rust",
|
|
1008
1208
|
enabled: true,
|
|
1009
|
-
options: {
|
|
1010
|
-
embeddingModel: "all-MiniLM-L6-v2"
|
|
1011
|
-
}
|
|
1209
|
+
options: { ...DEFAULT_EMBEDDING_MODULE_OPTIONS }
|
|
1012
1210
|
},
|
|
1013
1211
|
{
|
|
1014
1212
|
id: "data/json",
|
|
1015
1213
|
enabled: true,
|
|
1016
|
-
options: {
|
|
1017
|
-
embeddingModel: "all-MiniLM-L6-v2"
|
|
1018
|
-
}
|
|
1214
|
+
options: { ...DEFAULT_EMBEDDING_MODULE_OPTIONS }
|
|
1019
1215
|
},
|
|
1020
1216
|
{
|
|
1021
1217
|
id: "docs/markdown",
|
|
1022
1218
|
enabled: true,
|
|
1023
|
-
options: {
|
|
1024
|
-
embeddingModel: "all-MiniLM-L6-v2"
|
|
1025
|
-
}
|
|
1219
|
+
options: { ...DEFAULT_EMBEDDING_MODULE_OPTIONS }
|
|
1026
1220
|
}
|
|
1027
1221
|
]
|
|
1028
1222
|
};
|
|
1029
1223
|
}
|
|
1030
|
-
var DEFAULT_IGNORE_PATHS, DEFAULT_EXTENSIONS;
|
|
1224
|
+
var DEFAULT_IGNORE_PATHS, DEFAULT_EXTENSIONS, DEFAULT_EMBEDDING_MODULE_OPTIONS;
|
|
1031
1225
|
var init_config = __esm(() => {
|
|
1032
1226
|
DEFAULT_IGNORE_PATHS = [
|
|
1033
1227
|
"node_modules",
|
|
@@ -1097,6 +1291,10 @@ var init_config = __esm(() => {
|
|
|
1097
1291
|
".sql",
|
|
1098
1292
|
".txt"
|
|
1099
1293
|
];
|
|
1294
|
+
DEFAULT_EMBEDDING_MODULE_OPTIONS = {
|
|
1295
|
+
embeddingModel: "bge-small-en-v1.5",
|
|
1296
|
+
embeddingRuntime: "huggingface"
|
|
1297
|
+
};
|
|
1100
1298
|
});
|
|
1101
1299
|
|
|
1102
1300
|
// src/domain/entities/literal.ts
|
|
@@ -1127,47 +1325,45 @@ var init_entities = __esm(() => {
|
|
|
1127
1325
|
});
|
|
1128
1326
|
|
|
1129
1327
|
// src/infrastructure/config/configLoader.ts
|
|
1130
|
-
import * as
|
|
1131
|
-
import * as
|
|
1132
|
-
import * as os2 from "os";
|
|
1328
|
+
import * as path3 from "path";
|
|
1329
|
+
import * as fs2 from "fs/promises";
|
|
1133
1330
|
import * as crypto from "crypto";
|
|
1134
1331
|
function hashPath(inputPath) {
|
|
1135
1332
|
return crypto.createHash("sha256").update(inputPath).digest("hex").slice(0, 12);
|
|
1136
1333
|
}
|
|
1137
1334
|
function getRaggrepDir(rootDir, _config = DEFAULT_CONFIG) {
|
|
1138
|
-
const absoluteRoot =
|
|
1139
|
-
|
|
1140
|
-
return path2.join(RAGGREP_TEMP_BASE, projectHash);
|
|
1335
|
+
const absoluteRoot = path3.resolve(rootDir);
|
|
1336
|
+
return path3.join(absoluteRoot, RAGGREP_INDEX_DIR);
|
|
1141
1337
|
}
|
|
1142
1338
|
function getIndexLocation(rootDir) {
|
|
1143
|
-
const absoluteRoot =
|
|
1339
|
+
const absoluteRoot = path3.resolve(rootDir);
|
|
1144
1340
|
const projectHash = hashPath(absoluteRoot);
|
|
1145
1341
|
return {
|
|
1146
|
-
indexDir:
|
|
1342
|
+
indexDir: path3.join(absoluteRoot, RAGGREP_INDEX_DIR),
|
|
1147
1343
|
projectRoot: absoluteRoot,
|
|
1148
1344
|
projectHash
|
|
1149
1345
|
};
|
|
1150
1346
|
}
|
|
1151
1347
|
function getModuleIndexPath(rootDir, moduleId, config = DEFAULT_CONFIG) {
|
|
1152
1348
|
const indexDir = getRaggrepDir(rootDir, config);
|
|
1153
|
-
return
|
|
1349
|
+
return path3.join(indexDir, "index", moduleId);
|
|
1154
1350
|
}
|
|
1155
1351
|
function getModuleManifestPath(rootDir, moduleId, config = DEFAULT_CONFIG) {
|
|
1156
1352
|
const indexDir = getRaggrepDir(rootDir, config);
|
|
1157
|
-
return
|
|
1353
|
+
return path3.join(indexDir, "index", moduleId, "manifest.json");
|
|
1158
1354
|
}
|
|
1159
1355
|
function getGlobalManifestPath(rootDir, config = DEFAULT_CONFIG) {
|
|
1160
1356
|
const indexDir = getRaggrepDir(rootDir, config);
|
|
1161
|
-
return
|
|
1357
|
+
return path3.join(indexDir, "manifest.json");
|
|
1162
1358
|
}
|
|
1163
1359
|
function getConfigPath(rootDir, config = DEFAULT_CONFIG) {
|
|
1164
1360
|
const indexDir = getRaggrepDir(rootDir, config);
|
|
1165
|
-
return
|
|
1361
|
+
return path3.join(indexDir, "config.json");
|
|
1166
1362
|
}
|
|
1167
1363
|
async function loadConfig(rootDir) {
|
|
1168
1364
|
const configPath = getConfigPath(rootDir, DEFAULT_CONFIG);
|
|
1169
1365
|
try {
|
|
1170
|
-
const content = await
|
|
1366
|
+
const content = await fs2.readFile(configPath, "utf-8");
|
|
1171
1367
|
const savedConfig = JSON.parse(content);
|
|
1172
1368
|
return { ...DEFAULT_CONFIG, ...savedConfig };
|
|
1173
1369
|
} catch {
|
|
@@ -1180,27 +1376,28 @@ function getModuleConfig(config, moduleId) {
|
|
|
1180
1376
|
function getEmbeddingConfigFromModule(moduleConfig) {
|
|
1181
1377
|
const options = moduleConfig.options || {};
|
|
1182
1378
|
const modelName = options.embeddingModel || "bge-small-en-v1.5";
|
|
1183
|
-
if (!(modelName in
|
|
1379
|
+
if (!(modelName in EMBEDDING_MODELS)) {
|
|
1184
1380
|
console.warn(`Unknown embedding model: ${modelName}, falling back to bge-small-en-v1.5`);
|
|
1185
1381
|
return { model: "bge-small-en-v1.5" };
|
|
1186
1382
|
}
|
|
1383
|
+
const rt = options.embeddingRuntime;
|
|
1384
|
+
let runtime;
|
|
1385
|
+
if (rt === "xenova" || rt === "huggingface") {
|
|
1386
|
+
runtime = rt;
|
|
1387
|
+
} else if (rt !== undefined) {
|
|
1388
|
+
console.warn(`Unknown embeddingRuntime: ${rt}, falling back to default (huggingface)`);
|
|
1389
|
+
}
|
|
1187
1390
|
return {
|
|
1188
1391
|
model: modelName,
|
|
1392
|
+
...runtime ? { runtime } : {},
|
|
1189
1393
|
showProgress: options.showProgress === true
|
|
1190
1394
|
};
|
|
1191
1395
|
}
|
|
1192
|
-
var DEFAULT_CONFIG,
|
|
1396
|
+
var DEFAULT_CONFIG, RAGGREP_INDEX_DIR = ".raggrep";
|
|
1193
1397
|
var init_configLoader = __esm(() => {
|
|
1194
1398
|
init_entities();
|
|
1399
|
+
init_modelCatalog();
|
|
1195
1400
|
DEFAULT_CONFIG = createDefaultConfig();
|
|
1196
|
-
RAGGREP_TEMP_BASE = path2.join(os2.tmpdir(), "raggrep-indexes");
|
|
1197
|
-
EMBEDDING_MODELS2 = {
|
|
1198
|
-
"all-MiniLM-L6-v2": "Xenova/all-MiniLM-L6-v2",
|
|
1199
|
-
"all-MiniLM-L12-v2": "Xenova/all-MiniLM-L12-v2",
|
|
1200
|
-
"bge-small-en-v1.5": "Xenova/bge-small-en-v1.5",
|
|
1201
|
-
"paraphrase-MiniLM-L3-v2": "Xenova/paraphrase-MiniLM-L3-v2",
|
|
1202
|
-
"nomic-embed-text-v1.5": "nomic-ai/nomic-embed-text-v1.5"
|
|
1203
|
-
};
|
|
1204
1401
|
});
|
|
1205
1402
|
|
|
1206
1403
|
// src/infrastructure/config/index.ts
|
|
@@ -1338,10 +1535,10 @@ function normalizeScore(score, midpoint = 5) {
|
|
|
1338
1535
|
var BM25_K1 = 1.5, BM25_B = 0.75;
|
|
1339
1536
|
|
|
1340
1537
|
// src/domain/services/conventions/entryPoints.ts
|
|
1341
|
-
import * as
|
|
1538
|
+
import * as path4 from "path";
|
|
1342
1539
|
function getParentFolder(filepath) {
|
|
1343
|
-
const dir =
|
|
1344
|
-
return
|
|
1540
|
+
const dir = path4.dirname(filepath);
|
|
1541
|
+
return path4.basename(dir);
|
|
1345
1542
|
}
|
|
1346
1543
|
var entryPointConventions;
|
|
1347
1544
|
var init_entryPoints = __esm(() => {
|
|
@@ -2208,7 +2405,7 @@ var init_frameworks = __esm(() => {
|
|
|
2208
2405
|
});
|
|
2209
2406
|
|
|
2210
2407
|
// src/domain/services/conventions/index.ts
|
|
2211
|
-
import * as
|
|
2408
|
+
import * as path5 from "path";
|
|
2212
2409
|
function getConventions() {
|
|
2213
2410
|
return [
|
|
2214
2411
|
...entryPointConventions,
|
|
@@ -2220,8 +2417,8 @@ function getConventions() {
|
|
|
2220
2417
|
}
|
|
2221
2418
|
function getConventionKeywords(filepath) {
|
|
2222
2419
|
const conventions = getConventions();
|
|
2223
|
-
const filename =
|
|
2224
|
-
const extension =
|
|
2420
|
+
const filename = path5.basename(filepath);
|
|
2421
|
+
const extension = path5.extname(filepath);
|
|
2225
2422
|
const keywords = new Set;
|
|
2226
2423
|
for (const convention of conventions) {
|
|
2227
2424
|
try {
|
|
@@ -2309,13 +2506,13 @@ var init_conventions = __esm(() => {
|
|
|
2309
2506
|
});
|
|
2310
2507
|
|
|
2311
2508
|
// src/domain/services/introspection.ts
|
|
2312
|
-
import * as
|
|
2509
|
+
import * as path6 from "path";
|
|
2313
2510
|
function introspectFile(filepath, structure, options) {
|
|
2314
2511
|
const opts = typeof options === "string" ? { fileContent: options } : options || {};
|
|
2315
2512
|
const normalizedPath = filepath.replace(/\\/g, "/");
|
|
2316
2513
|
const segments = normalizedPath.split("/").filter((s) => s.length > 0);
|
|
2317
2514
|
const filename = segments[segments.length - 1] || "";
|
|
2318
|
-
const ext =
|
|
2515
|
+
const ext = path6.extname(filename);
|
|
2319
2516
|
const project = findProjectForFile(normalizedPath, structure);
|
|
2320
2517
|
const language = EXTENSION_TO_LANGUAGE[ext] || "unknown";
|
|
2321
2518
|
const layer = detectLayer(segments, filename);
|
|
@@ -2357,7 +2554,7 @@ function findNearestReadme(filepath, fileExists) {
|
|
|
2357
2554
|
}
|
|
2358
2555
|
function introspectionToKeywords(intro) {
|
|
2359
2556
|
const keywords = [];
|
|
2360
|
-
const filename =
|
|
2557
|
+
const filename = path6.basename(intro.filepath);
|
|
2361
2558
|
const filenameWithoutExt = filename.replace(/\.[^.]+$/, "");
|
|
2362
2559
|
const filenameParts = filenameWithoutExt.split(/[-_.]/).flatMap((part) => part.split(/(?=[A-Z])/)).map((part) => part.toLowerCase()).filter((part) => part.length > 1);
|
|
2363
2560
|
keywords.push(...filenameParts);
|
|
@@ -2836,8 +3033,8 @@ var exports_core = {};
|
|
|
2836
3033
|
__export(exports_core, {
|
|
2837
3034
|
CoreModule: () => CoreModule
|
|
2838
3035
|
});
|
|
2839
|
-
import * as
|
|
2840
|
-
import * as
|
|
3036
|
+
import * as path7 from "path";
|
|
3037
|
+
import * as fs3 from "fs/promises";
|
|
2841
3038
|
|
|
2842
3039
|
class CoreModule {
|
|
2843
3040
|
id = "core";
|
|
@@ -2934,8 +3131,8 @@ class CoreModule {
|
|
|
2934
3131
|
}
|
|
2935
3132
|
async finalize(ctx) {
|
|
2936
3133
|
const config = ctx.config;
|
|
2937
|
-
const coreDir =
|
|
2938
|
-
await
|
|
3134
|
+
const coreDir = path7.join(getRaggrepDir(ctx.rootDir, config), "index", "core");
|
|
3135
|
+
await fs3.mkdir(coreDir, { recursive: true });
|
|
2939
3136
|
this.bm25Index = new BM25Index;
|
|
2940
3137
|
for (const [filepath, entry] of this.symbolIndex) {
|
|
2941
3138
|
this.bm25Index.addDocument(filepath, entry.tokens);
|
|
@@ -2946,7 +3143,7 @@ class CoreModule {
|
|
|
2946
3143
|
files: Object.fromEntries(this.symbolIndex),
|
|
2947
3144
|
bm25Data: this.bm25Index.serialize()
|
|
2948
3145
|
};
|
|
2949
|
-
await
|
|
3146
|
+
await fs3.writeFile(path7.join(coreDir, "symbols.json"), JSON.stringify(symbolIndexData, null, 2));
|
|
2950
3147
|
}
|
|
2951
3148
|
async search(query, ctx, options) {
|
|
2952
3149
|
const config = ctx.config;
|
|
@@ -3042,10 +3239,10 @@ class CoreModule {
|
|
|
3042
3239
|
return bestChunk;
|
|
3043
3240
|
}
|
|
3044
3241
|
async loadSymbolIndex(rootDir, config) {
|
|
3045
|
-
const coreDir =
|
|
3046
|
-
const symbolsPath =
|
|
3242
|
+
const coreDir = path7.join(getRaggrepDir(rootDir, config), "index", "core");
|
|
3243
|
+
const symbolsPath = path7.join(coreDir, "symbols.json");
|
|
3047
3244
|
try {
|
|
3048
|
-
const content = await
|
|
3245
|
+
const content = await fs3.readFile(symbolsPath, "utf-8");
|
|
3049
3246
|
const data = JSON.parse(content);
|
|
3050
3247
|
this.symbolIndex = new Map(Object.entries(data.files));
|
|
3051
3248
|
if (data.bm25Data) {
|
|
@@ -3276,7 +3473,7 @@ function cosineSimilarity(a, b) {
|
|
|
3276
3473
|
}
|
|
3277
3474
|
|
|
3278
3475
|
// src/domain/services/queryIntent.ts
|
|
3279
|
-
import * as
|
|
3476
|
+
import * as path8 from "path";
|
|
3280
3477
|
function detectQueryIntent(queryTerms) {
|
|
3281
3478
|
const hasImplementationTerm = queryTerms.some((term) => IMPLEMENTATION_TERMS.includes(term));
|
|
3282
3479
|
const hasDocumentationTerm = queryTerms.some((term) => DOCUMENTATION_TERMS.includes(term));
|
|
@@ -3292,11 +3489,11 @@ function extractQueryTerms(query) {
|
|
|
3292
3489
|
return query.toLowerCase().split(/\s+/).filter((t) => t.length > 2);
|
|
3293
3490
|
}
|
|
3294
3491
|
function isSourceCodeFile(filepath) {
|
|
3295
|
-
const ext =
|
|
3492
|
+
const ext = path8.extname(filepath).toLowerCase();
|
|
3296
3493
|
return SOURCE_CODE_EXTENSIONS.includes(ext);
|
|
3297
3494
|
}
|
|
3298
3495
|
function isDocFile(filepath) {
|
|
3299
|
-
const ext =
|
|
3496
|
+
const ext = path8.extname(filepath).toLowerCase();
|
|
3300
3497
|
return DOC_EXTENSIONS.includes(ext);
|
|
3301
3498
|
}
|
|
3302
3499
|
function calculateFileTypeBoost(filepath, queryTerms) {
|
|
@@ -4371,8 +4568,8 @@ var init_lexicon2 = __esm(() => {
|
|
|
4371
4568
|
// src/domain/services/jsonPathExtractor.ts
|
|
4372
4569
|
function extractJsonPaths(obj, fileBasename) {
|
|
4373
4570
|
const paths = extractPathsRecursive(obj, fileBasename);
|
|
4374
|
-
return paths.map((
|
|
4375
|
-
value:
|
|
4571
|
+
return paths.map((path9) => ({
|
|
4572
|
+
value: path9,
|
|
4376
4573
|
type: "identifier",
|
|
4377
4574
|
matchType: "definition"
|
|
4378
4575
|
}));
|
|
@@ -4683,7 +4880,7 @@ var init_simpleSearch = __esm(() => {
|
|
|
4683
4880
|
});
|
|
4684
4881
|
|
|
4685
4882
|
// src/domain/services/chunkContext.ts
|
|
4686
|
-
import * as
|
|
4883
|
+
import * as path9 from "path";
|
|
4687
4884
|
function prepareChunkForEmbedding(options) {
|
|
4688
4885
|
const { filepath, content, name, docComment } = options;
|
|
4689
4886
|
const pathContext = parsePathContext(filepath);
|
|
@@ -4692,7 +4889,7 @@ function prepareChunkForEmbedding(options) {
|
|
|
4692
4889
|
if (pathPrefix) {
|
|
4693
4890
|
parts.push(pathPrefix);
|
|
4694
4891
|
}
|
|
4695
|
-
const filename =
|
|
4892
|
+
const filename = path9.basename(filepath);
|
|
4696
4893
|
const filenameWithoutExt = filename.replace(/\.[^.]+$/, "");
|
|
4697
4894
|
if (filenameWithoutExt && filenameWithoutExt.length > MIN_SEGMENT_LENGTH) {
|
|
4698
4895
|
const pathPrefixLower = pathPrefix.toLowerCase();
|
|
@@ -4922,11 +5119,12 @@ var init_parseCode = () => {};
|
|
|
4922
5119
|
// src/infrastructure/storage/fileIndexStorage.ts
|
|
4923
5120
|
var init_fileIndexStorage = __esm(() => {
|
|
4924
5121
|
init_entities();
|
|
5122
|
+
init_config2();
|
|
4925
5123
|
});
|
|
4926
5124
|
|
|
4927
5125
|
// src/infrastructure/storage/symbolicIndex.ts
|
|
4928
|
-
import * as
|
|
4929
|
-
import * as
|
|
5126
|
+
import * as fs4 from "fs/promises";
|
|
5127
|
+
import * as path10 from "path";
|
|
4930
5128
|
|
|
4931
5129
|
class SymbolicIndex {
|
|
4932
5130
|
meta = null;
|
|
@@ -4935,7 +5133,7 @@ class SymbolicIndex {
|
|
|
4935
5133
|
symbolicPath;
|
|
4936
5134
|
moduleId;
|
|
4937
5135
|
constructor(indexDir, moduleId) {
|
|
4938
|
-
this.symbolicPath =
|
|
5136
|
+
this.symbolicPath = path10.join(indexDir, "index", moduleId, "symbolic");
|
|
4939
5137
|
this.moduleId = moduleId;
|
|
4940
5138
|
}
|
|
4941
5139
|
async initialize() {
|
|
@@ -5021,13 +5219,13 @@ class SymbolicIndex {
|
|
|
5021
5219
|
if (this.bm25Index) {
|
|
5022
5220
|
this.meta.bm25Serialized = this.bm25Index.serialize();
|
|
5023
5221
|
}
|
|
5024
|
-
await
|
|
5025
|
-
const metaPath =
|
|
5026
|
-
await
|
|
5222
|
+
await fs4.mkdir(this.symbolicPath, { recursive: true });
|
|
5223
|
+
const metaPath = path10.join(this.symbolicPath, "_meta.json");
|
|
5224
|
+
await fs4.writeFile(metaPath, JSON.stringify(this.meta, null, 2));
|
|
5027
5225
|
for (const [filepath, summary] of this.fileSummaries) {
|
|
5028
5226
|
const summaryPath = this.getFileSummaryPath(filepath);
|
|
5029
|
-
await
|
|
5030
|
-
await
|
|
5227
|
+
await fs4.mkdir(path10.dirname(summaryPath), { recursive: true });
|
|
5228
|
+
await fs4.writeFile(summaryPath, JSON.stringify(summary, null, 2));
|
|
5031
5229
|
}
|
|
5032
5230
|
}
|
|
5033
5231
|
async saveIncremental(filepaths) {
|
|
@@ -5038,21 +5236,21 @@ class SymbolicIndex {
|
|
|
5038
5236
|
if (this.bm25Index) {
|
|
5039
5237
|
this.meta.bm25Serialized = this.bm25Index.serialize();
|
|
5040
5238
|
}
|
|
5041
|
-
await
|
|
5042
|
-
const metaPath =
|
|
5043
|
-
await
|
|
5239
|
+
await fs4.mkdir(this.symbolicPath, { recursive: true });
|
|
5240
|
+
const metaPath = path10.join(this.symbolicPath, "_meta.json");
|
|
5241
|
+
await fs4.writeFile(metaPath, JSON.stringify(this.meta, null, 2));
|
|
5044
5242
|
for (const filepath of filepaths) {
|
|
5045
5243
|
const summary = this.fileSummaries.get(filepath);
|
|
5046
5244
|
if (summary) {
|
|
5047
5245
|
const summaryPath = this.getFileSummaryPath(filepath);
|
|
5048
|
-
await
|
|
5049
|
-
await
|
|
5246
|
+
await fs4.mkdir(path10.dirname(summaryPath), { recursive: true });
|
|
5247
|
+
await fs4.writeFile(summaryPath, JSON.stringify(summary, null, 2));
|
|
5050
5248
|
}
|
|
5051
5249
|
}
|
|
5052
5250
|
}
|
|
5053
5251
|
async load() {
|
|
5054
|
-
const metaPath =
|
|
5055
|
-
const metaContent = await
|
|
5252
|
+
const metaPath = path10.join(this.symbolicPath, "_meta.json");
|
|
5253
|
+
const metaContent = await fs4.readFile(metaPath, "utf-8");
|
|
5056
5254
|
this.meta = JSON.parse(metaContent);
|
|
5057
5255
|
this.fileSummaries.clear();
|
|
5058
5256
|
await this.loadFileSummariesRecursive(this.symbolicPath);
|
|
@@ -5064,14 +5262,14 @@ class SymbolicIndex {
|
|
|
5064
5262
|
}
|
|
5065
5263
|
async loadFileSummariesRecursive(dir) {
|
|
5066
5264
|
try {
|
|
5067
|
-
const entries = await
|
|
5265
|
+
const entries = await fs4.readdir(dir, { withFileTypes: true });
|
|
5068
5266
|
for (const entry of entries) {
|
|
5069
|
-
const fullPath =
|
|
5267
|
+
const fullPath = path10.join(dir, entry.name);
|
|
5070
5268
|
if (entry.isDirectory()) {
|
|
5071
5269
|
await this.loadFileSummariesRecursive(fullPath);
|
|
5072
5270
|
} else if (entry.name.endsWith(".json") && entry.name !== "_meta.json") {
|
|
5073
5271
|
try {
|
|
5074
|
-
const content = await
|
|
5272
|
+
const content = await fs4.readFile(fullPath, "utf-8");
|
|
5075
5273
|
const summary = JSON.parse(content);
|
|
5076
5274
|
if (summary.filepath) {
|
|
5077
5275
|
this.fileSummaries.set(summary.filepath, summary);
|
|
@@ -5083,18 +5281,18 @@ class SymbolicIndex {
|
|
|
5083
5281
|
}
|
|
5084
5282
|
getFileSummaryPath(filepath) {
|
|
5085
5283
|
const jsonPath = filepath.replace(/\.[^.]+$/, ".json");
|
|
5086
|
-
return
|
|
5284
|
+
return path10.join(this.symbolicPath, jsonPath);
|
|
5087
5285
|
}
|
|
5088
5286
|
async deleteFileSummary(filepath) {
|
|
5089
5287
|
try {
|
|
5090
|
-
await
|
|
5288
|
+
await fs4.unlink(this.getFileSummaryPath(filepath));
|
|
5091
5289
|
} catch {}
|
|
5092
5290
|
this.fileSummaries.delete(filepath);
|
|
5093
5291
|
}
|
|
5094
5292
|
async exists() {
|
|
5095
5293
|
try {
|
|
5096
|
-
const metaPath =
|
|
5097
|
-
await
|
|
5294
|
+
const metaPath = path10.join(this.symbolicPath, "_meta.json");
|
|
5295
|
+
await fs4.access(metaPath);
|
|
5098
5296
|
return true;
|
|
5099
5297
|
} catch {
|
|
5100
5298
|
return false;
|
|
@@ -5126,8 +5324,8 @@ __export(exports_literalIndex, {
|
|
|
5126
5324
|
getLiteralIndexPath: () => getLiteralIndexPath,
|
|
5127
5325
|
LiteralIndex: () => LiteralIndex
|
|
5128
5326
|
});
|
|
5129
|
-
import * as
|
|
5130
|
-
import * as
|
|
5327
|
+
import * as fs5 from "fs/promises";
|
|
5328
|
+
import * as path11 from "path";
|
|
5131
5329
|
|
|
5132
5330
|
class LiteralIndex {
|
|
5133
5331
|
indexPath;
|
|
@@ -5136,7 +5334,7 @@ class LiteralIndex {
|
|
|
5136
5334
|
vocabularyIndex = new Map;
|
|
5137
5335
|
static VERSION = "1.1.0";
|
|
5138
5336
|
constructor(indexDir, moduleId) {
|
|
5139
|
-
this.indexPath =
|
|
5337
|
+
this.indexPath = path11.join(indexDir, "index", moduleId, "literals");
|
|
5140
5338
|
this.moduleId = moduleId;
|
|
5141
5339
|
}
|
|
5142
5340
|
async initialize() {
|
|
@@ -5297,17 +5495,17 @@ class LiteralIndex {
|
|
|
5297
5495
|
}));
|
|
5298
5496
|
}
|
|
5299
5497
|
async save() {
|
|
5300
|
-
await
|
|
5498
|
+
await fs5.mkdir(this.indexPath, { recursive: true });
|
|
5301
5499
|
const data = {
|
|
5302
5500
|
version: LiteralIndex.VERSION,
|
|
5303
5501
|
entries: Object.fromEntries(this.entries)
|
|
5304
5502
|
};
|
|
5305
|
-
const indexFile =
|
|
5306
|
-
await
|
|
5503
|
+
const indexFile = path11.join(this.indexPath, "_index.json");
|
|
5504
|
+
await fs5.writeFile(indexFile, JSON.stringify(data, null, 2));
|
|
5307
5505
|
}
|
|
5308
5506
|
async load() {
|
|
5309
|
-
const indexFile =
|
|
5310
|
-
const content = await
|
|
5507
|
+
const indexFile = path11.join(this.indexPath, "_index.json");
|
|
5508
|
+
const content = await fs5.readFile(indexFile, "utf-8");
|
|
5311
5509
|
const data = JSON.parse(content);
|
|
5312
5510
|
if (data.version !== LiteralIndex.VERSION) {
|
|
5313
5511
|
console.warn(`Literal index version mismatch: expected ${LiteralIndex.VERSION}, got ${data.version}`);
|
|
@@ -5316,8 +5514,8 @@ class LiteralIndex {
|
|
|
5316
5514
|
}
|
|
5317
5515
|
async exists() {
|
|
5318
5516
|
try {
|
|
5319
|
-
const indexFile =
|
|
5320
|
-
await
|
|
5517
|
+
const indexFile = path11.join(this.indexPath, "_index.json");
|
|
5518
|
+
await fs5.access(indexFile);
|
|
5321
5519
|
return true;
|
|
5322
5520
|
} catch {
|
|
5323
5521
|
return false;
|
|
@@ -5360,7 +5558,7 @@ function shouldReplaceMatchType(existing, incoming) {
|
|
|
5360
5558
|
return priority[incoming] > priority[existing];
|
|
5361
5559
|
}
|
|
5362
5560
|
function getLiteralIndexPath(rootDir, moduleId, indexDir = ".raggrep") {
|
|
5363
|
-
return
|
|
5561
|
+
return path11.join(rootDir, indexDir, "index", moduleId, "literals");
|
|
5364
5562
|
}
|
|
5365
5563
|
var init_literalIndex = () => {};
|
|
5366
5564
|
|
|
@@ -5381,9 +5579,9 @@ __export(exports_typescript, {
|
|
|
5381
5579
|
DEFAULT_TOP_K: () => DEFAULT_TOP_K2,
|
|
5382
5580
|
DEFAULT_MIN_SCORE: () => DEFAULT_MIN_SCORE2
|
|
5383
5581
|
});
|
|
5384
|
-
import * as
|
|
5582
|
+
import * as path12 from "path";
|
|
5385
5583
|
function isTypeScriptFile(filepath) {
|
|
5386
|
-
const ext =
|
|
5584
|
+
const ext = path12.extname(filepath).toLowerCase();
|
|
5387
5585
|
return TYPESCRIPT_EXTENSIONS.includes(ext);
|
|
5388
5586
|
}
|
|
5389
5587
|
function calculateChunkTypeBoost(chunk) {
|
|
@@ -5455,7 +5653,7 @@ class TypeScriptModule {
|
|
|
5455
5653
|
startLine: 1,
|
|
5456
5654
|
endLine: lines.length,
|
|
5457
5655
|
type: "file",
|
|
5458
|
-
name:
|
|
5656
|
+
name: path12.basename(filepath),
|
|
5459
5657
|
isExported: false
|
|
5460
5658
|
});
|
|
5461
5659
|
}
|
|
@@ -5782,16 +5980,16 @@ class TypeScriptModule {
|
|
|
5782
5980
|
while ((match = importRegex.exec(content)) !== null) {
|
|
5783
5981
|
const importPath = match[1];
|
|
5784
5982
|
if (importPath.startsWith(".")) {
|
|
5785
|
-
const dir =
|
|
5786
|
-
const resolved =
|
|
5983
|
+
const dir = path12.dirname(filepath);
|
|
5984
|
+
const resolved = path12.normalize(path12.join(dir, importPath));
|
|
5787
5985
|
references.push(resolved);
|
|
5788
5986
|
}
|
|
5789
5987
|
}
|
|
5790
5988
|
while ((match = requireRegex.exec(content)) !== null) {
|
|
5791
5989
|
const importPath = match[1];
|
|
5792
5990
|
if (importPath.startsWith(".")) {
|
|
5793
|
-
const dir =
|
|
5794
|
-
const resolved =
|
|
5991
|
+
const dir = path12.dirname(filepath);
|
|
5992
|
+
const resolved = path12.normalize(path12.join(dir, importPath));
|
|
5795
5993
|
references.push(resolved);
|
|
5796
5994
|
}
|
|
5797
5995
|
}
|
|
@@ -5819,7 +6017,7 @@ var init_typescript = __esm(() => {
|
|
|
5819
6017
|
});
|
|
5820
6018
|
|
|
5821
6019
|
// src/infrastructure/parsing/typescriptParser.ts
|
|
5822
|
-
import * as
|
|
6020
|
+
import * as path13 from "path";
|
|
5823
6021
|
|
|
5824
6022
|
class TypeScriptParser {
|
|
5825
6023
|
supportedLanguages = ["typescript", "javascript"];
|
|
@@ -5835,12 +6033,12 @@ class TypeScriptParser {
|
|
|
5835
6033
|
startLine: 1,
|
|
5836
6034
|
endLine: lines.length,
|
|
5837
6035
|
type: "file",
|
|
5838
|
-
name:
|
|
6036
|
+
name: path13.basename(filepath),
|
|
5839
6037
|
isExported: false
|
|
5840
6038
|
};
|
|
5841
6039
|
chunks.unshift(fullFileChunk);
|
|
5842
6040
|
}
|
|
5843
|
-
const ext =
|
|
6041
|
+
const ext = path13.extname(filepath).toLowerCase();
|
|
5844
6042
|
const language = ext === ".js" || ext === ".jsx" || ext === ".mjs" || ext === ".cjs" ? "javascript" : "typescript";
|
|
5845
6043
|
return {
|
|
5846
6044
|
chunks,
|
|
@@ -5857,7 +6055,7 @@ class TypeScriptParser {
|
|
|
5857
6055
|
}
|
|
5858
6056
|
}
|
|
5859
6057
|
canParse(filepath) {
|
|
5860
|
-
const ext =
|
|
6058
|
+
const ext = path13.extname(filepath).toLowerCase();
|
|
5861
6059
|
return TYPESCRIPT_EXTENSIONS2.includes(ext);
|
|
5862
6060
|
}
|
|
5863
6061
|
convertChunk(tc) {
|
|
@@ -5872,7 +6070,7 @@ class TypeScriptParser {
|
|
|
5872
6070
|
};
|
|
5873
6071
|
}
|
|
5874
6072
|
detectLanguage(filepath) {
|
|
5875
|
-
const ext =
|
|
6073
|
+
const ext = path13.extname(filepath).toLowerCase();
|
|
5876
6074
|
if ([".js", ".jsx", ".mjs", ".cjs"].includes(ext)) {
|
|
5877
6075
|
return "javascript";
|
|
5878
6076
|
}
|
|
@@ -6077,8 +6275,8 @@ var init_grammarManager = __esm(() => {
|
|
|
6077
6275
|
});
|
|
6078
6276
|
|
|
6079
6277
|
// src/infrastructure/parsing/treeSitterParser.ts
|
|
6080
|
-
import * as
|
|
6081
|
-
import * as
|
|
6278
|
+
import * as path14 from "path";
|
|
6279
|
+
import * as fs6 from "fs";
|
|
6082
6280
|
|
|
6083
6281
|
class TreeSitterParser {
|
|
6084
6282
|
supportedLanguages = [
|
|
@@ -6100,7 +6298,7 @@ class TreeSitterParser {
|
|
|
6100
6298
|
chunks: [],
|
|
6101
6299
|
language: "typescript",
|
|
6102
6300
|
success: false,
|
|
6103
|
-
error: `Unsupported file type: ${
|
|
6301
|
+
error: `Unsupported file type: ${path14.extname(filepath)}`
|
|
6104
6302
|
};
|
|
6105
6303
|
}
|
|
6106
6304
|
try {
|
|
@@ -6120,11 +6318,11 @@ class TreeSitterParser {
|
|
|
6120
6318
|
}
|
|
6121
6319
|
}
|
|
6122
6320
|
canParse(filepath) {
|
|
6123
|
-
const ext =
|
|
6321
|
+
const ext = path14.extname(filepath).toLowerCase();
|
|
6124
6322
|
return ext in EXTENSION_TO_LANGUAGE2;
|
|
6125
6323
|
}
|
|
6126
6324
|
detectLanguage(filepath) {
|
|
6127
|
-
const ext =
|
|
6325
|
+
const ext = path14.extname(filepath).toLowerCase();
|
|
6128
6326
|
return EXTENSION_TO_LANGUAGE2[ext] || null;
|
|
6129
6327
|
}
|
|
6130
6328
|
async ensureInitialized() {
|
|
@@ -6158,20 +6356,20 @@ class TreeSitterParser {
|
|
|
6158
6356
|
async resolveWasmPath() {
|
|
6159
6357
|
try {
|
|
6160
6358
|
const webTreeSitterPath = __require.resolve("web-tree-sitter");
|
|
6161
|
-
const wasmPath =
|
|
6162
|
-
if (
|
|
6359
|
+
const wasmPath = path14.join(path14.dirname(webTreeSitterPath), "web-tree-sitter.wasm");
|
|
6360
|
+
if (fs6.existsSync(wasmPath)) {
|
|
6163
6361
|
return wasmPath;
|
|
6164
6362
|
}
|
|
6165
6363
|
} catch {}
|
|
6166
6364
|
try {
|
|
6167
6365
|
const possiblePaths = [
|
|
6168
|
-
|
|
6169
|
-
|
|
6170
|
-
|
|
6171
|
-
|
|
6366
|
+
path14.join(__dirname, "../../../node_modules/web-tree-sitter/web-tree-sitter.wasm"),
|
|
6367
|
+
path14.join(__dirname, "../../node_modules/web-tree-sitter/web-tree-sitter.wasm"),
|
|
6368
|
+
path14.join(__dirname, "../../../../node_modules/web-tree-sitter/web-tree-sitter.wasm"),
|
|
6369
|
+
path14.join(__dirname, "web-tree-sitter.wasm")
|
|
6172
6370
|
];
|
|
6173
6371
|
for (const wasmPath of possiblePaths) {
|
|
6174
|
-
if (
|
|
6372
|
+
if (fs6.existsSync(wasmPath)) {
|
|
6175
6373
|
return wasmPath;
|
|
6176
6374
|
}
|
|
6177
6375
|
}
|
|
@@ -6212,7 +6410,7 @@ class TreeSitterParser {
|
|
|
6212
6410
|
startLine: 1,
|
|
6213
6411
|
endLine: lines.length,
|
|
6214
6412
|
type: "file",
|
|
6215
|
-
name:
|
|
6413
|
+
name: path14.basename(filepath),
|
|
6216
6414
|
isExported: false
|
|
6217
6415
|
});
|
|
6218
6416
|
}
|
|
@@ -6559,7 +6757,7 @@ class TreeSitterParser {
|
|
|
6559
6757
|
startLine: 1,
|
|
6560
6758
|
endLine: lines.length,
|
|
6561
6759
|
type: "file",
|
|
6562
|
-
name:
|
|
6760
|
+
name: path14.basename(filepath)
|
|
6563
6761
|
});
|
|
6564
6762
|
return {
|
|
6565
6763
|
chunks,
|
|
@@ -6568,7 +6766,7 @@ class TreeSitterParser {
|
|
|
6568
6766
|
};
|
|
6569
6767
|
}
|
|
6570
6768
|
}
|
|
6571
|
-
var __dirname = "/
|
|
6769
|
+
var __dirname = "/home/runner/work/raggrep/raggrep/src/infrastructure/parsing", EXTENSION_TO_LANGUAGE2;
|
|
6572
6770
|
var init_treeSitterParser = __esm(() => {
|
|
6573
6771
|
init_grammarManager();
|
|
6574
6772
|
EXTENSION_TO_LANGUAGE2 = {
|
|
@@ -6589,7 +6787,7 @@ var init_treeSitterParser = __esm(() => {
|
|
|
6589
6787
|
});
|
|
6590
6788
|
|
|
6591
6789
|
// src/infrastructure/parsing/parserFactory.ts
|
|
6592
|
-
import * as
|
|
6790
|
+
import * as path15 from "path";
|
|
6593
6791
|
function getTypeScriptParser() {
|
|
6594
6792
|
if (!typescriptParserInstance) {
|
|
6595
6793
|
typescriptParserInstance = new TypeScriptParser;
|
|
@@ -6603,7 +6801,7 @@ function getTreeSitterParser() {
|
|
|
6603
6801
|
return treeSitterParserInstance;
|
|
6604
6802
|
}
|
|
6605
6803
|
function createParserForFile(filepath) {
|
|
6606
|
-
const ext =
|
|
6804
|
+
const ext = path15.extname(filepath).toLowerCase();
|
|
6607
6805
|
const parserType = EXTENSION_PARSER_MAP[ext];
|
|
6608
6806
|
if (!parserType) {
|
|
6609
6807
|
return null;
|
|
@@ -6652,9 +6850,9 @@ __export(exports_python, {
|
|
|
6652
6850
|
DEFAULT_TOP_K: () => DEFAULT_TOP_K3,
|
|
6653
6851
|
DEFAULT_MIN_SCORE: () => DEFAULT_MIN_SCORE3
|
|
6654
6852
|
});
|
|
6655
|
-
import * as
|
|
6853
|
+
import * as path16 from "path";
|
|
6656
6854
|
function isPythonFile(filepath) {
|
|
6657
|
-
const ext =
|
|
6855
|
+
const ext = path16.extname(filepath).toLowerCase();
|
|
6658
6856
|
return PYTHON_EXTENSIONS.includes(ext);
|
|
6659
6857
|
}
|
|
6660
6858
|
function generateChunkId3(filepath, startLine, endLine) {
|
|
@@ -6738,7 +6936,7 @@ class PythonModule {
|
|
|
6738
6936
|
startLine: 1,
|
|
6739
6937
|
endLine: lines.length,
|
|
6740
6938
|
type: "file",
|
|
6741
|
-
name:
|
|
6939
|
+
name: path16.basename(filepath)
|
|
6742
6940
|
});
|
|
6743
6941
|
const funcRegex = /^(\s*)(async\s+)?def\s+(\w+)\s*\([^)]*\)\s*:/gm;
|
|
6744
6942
|
let match;
|
|
@@ -7112,9 +7310,9 @@ __export(exports_go, {
|
|
|
7112
7310
|
DEFAULT_TOP_K: () => DEFAULT_TOP_K4,
|
|
7113
7311
|
DEFAULT_MIN_SCORE: () => DEFAULT_MIN_SCORE4
|
|
7114
7312
|
});
|
|
7115
|
-
import * as
|
|
7313
|
+
import * as path17 from "path";
|
|
7116
7314
|
function isGoFile(filepath) {
|
|
7117
|
-
const ext =
|
|
7315
|
+
const ext = path17.extname(filepath).toLowerCase();
|
|
7118
7316
|
return GO_EXTENSIONS.includes(ext);
|
|
7119
7317
|
}
|
|
7120
7318
|
function generateChunkId4(filepath, startLine, endLine) {
|
|
@@ -7199,7 +7397,7 @@ class GoModule {
|
|
|
7199
7397
|
startLine: 1,
|
|
7200
7398
|
endLine: lines.length,
|
|
7201
7399
|
type: "file",
|
|
7202
|
-
name:
|
|
7400
|
+
name: path17.basename(filepath)
|
|
7203
7401
|
});
|
|
7204
7402
|
const funcRegex = /^func\s+(?:\(\s*\w+\s+\*?\w+\s*\)\s+)?(\w+)\s*\(/gm;
|
|
7205
7403
|
let match;
|
|
@@ -7593,9 +7791,9 @@ __export(exports_rust, {
|
|
|
7593
7791
|
DEFAULT_TOP_K: () => DEFAULT_TOP_K5,
|
|
7594
7792
|
DEFAULT_MIN_SCORE: () => DEFAULT_MIN_SCORE5
|
|
7595
7793
|
});
|
|
7596
|
-
import * as
|
|
7794
|
+
import * as path18 from "path";
|
|
7597
7795
|
function isRustFile(filepath) {
|
|
7598
|
-
const ext =
|
|
7796
|
+
const ext = path18.extname(filepath).toLowerCase();
|
|
7599
7797
|
return RUST_EXTENSIONS.includes(ext);
|
|
7600
7798
|
}
|
|
7601
7799
|
function generateChunkId5(filepath, startLine, endLine) {
|
|
@@ -7682,7 +7880,7 @@ class RustModule {
|
|
|
7682
7880
|
startLine: 1,
|
|
7683
7881
|
endLine: lines.length,
|
|
7684
7882
|
type: "file",
|
|
7685
|
-
name:
|
|
7883
|
+
name: path18.basename(filepath)
|
|
7686
7884
|
});
|
|
7687
7885
|
const funcRegex = /^(pub(?:\s*\([^)]*\))?\s+)?(?:async\s+)?fn\s+(\w+)/gm;
|
|
7688
7886
|
let match;
|
|
@@ -8153,9 +8351,9 @@ __export(exports_json, {
|
|
|
8153
8351
|
DEFAULT_TOP_K: () => DEFAULT_TOP_K6,
|
|
8154
8352
|
DEFAULT_MIN_SCORE: () => DEFAULT_MIN_SCORE6
|
|
8155
8353
|
});
|
|
8156
|
-
import * as
|
|
8354
|
+
import * as path19 from "path";
|
|
8157
8355
|
function isJsonFile(filepath) {
|
|
8158
|
-
const ext =
|
|
8356
|
+
const ext = path19.extname(filepath).toLowerCase();
|
|
8159
8357
|
return JSON_EXTENSIONS.includes(ext);
|
|
8160
8358
|
}
|
|
8161
8359
|
|
|
@@ -8189,7 +8387,7 @@ class JsonModule {
|
|
|
8189
8387
|
} catch {
|
|
8190
8388
|
return null;
|
|
8191
8389
|
}
|
|
8192
|
-
const fileBasename =
|
|
8390
|
+
const fileBasename = path19.basename(filepath, path19.extname(filepath));
|
|
8193
8391
|
const jsonPathLiterals = extractJsonPaths(parsed, fileBasename);
|
|
8194
8392
|
const lines = content.split(`
|
|
8195
8393
|
`);
|
|
@@ -8396,7 +8594,7 @@ __export(exports_markdown, {
|
|
|
8396
8594
|
DEFAULT_TOP_K: () => DEFAULT_TOP_K7,
|
|
8397
8595
|
DEFAULT_MIN_SCORE: () => DEFAULT_MIN_SCORE7
|
|
8398
8596
|
});
|
|
8399
|
-
import * as
|
|
8597
|
+
import * as path20 from "path";
|
|
8400
8598
|
function calculateHeadingLevelBoost(chunk) {
|
|
8401
8599
|
const metadata = chunk.metadata;
|
|
8402
8600
|
const level = metadata?.headingLevel ?? 0;
|
|
@@ -8416,7 +8614,7 @@ function calculateHeadingLevelBoost(chunk) {
|
|
|
8416
8614
|
}
|
|
8417
8615
|
}
|
|
8418
8616
|
function isMarkdownFile(filepath) {
|
|
8419
|
-
const ext =
|
|
8617
|
+
const ext = path20.extname(filepath).toLowerCase();
|
|
8420
8618
|
return MARKDOWN_EXTENSIONS.includes(ext);
|
|
8421
8619
|
}
|
|
8422
8620
|
function parseMarkdownHierarchical(content, maxDepth = 4) {
|
|
@@ -8758,15 +8956,15 @@ var init_registry = __esm(() => {
|
|
|
8758
8956
|
});
|
|
8759
8957
|
|
|
8760
8958
|
// src/infrastructure/introspection/projectDetector.ts
|
|
8761
|
-
import * as
|
|
8762
|
-
import * as
|
|
8959
|
+
import * as path21 from "path";
|
|
8960
|
+
import * as fs7 from "fs/promises";
|
|
8763
8961
|
async function scanForPackageJsons(rootDir, currentDir = "", depth = 0) {
|
|
8764
8962
|
if (depth > MAX_SCAN_DEPTH)
|
|
8765
8963
|
return [];
|
|
8766
8964
|
const results = [];
|
|
8767
|
-
const fullDir = currentDir ?
|
|
8965
|
+
const fullDir = currentDir ? path21.join(rootDir, currentDir) : rootDir;
|
|
8768
8966
|
try {
|
|
8769
|
-
const entries = await
|
|
8967
|
+
const entries = await fs7.readdir(fullDir, { withFileTypes: true });
|
|
8770
8968
|
const hasPackageJson = entries.some((e) => e.isFile() && e.name === "package.json");
|
|
8771
8969
|
if (hasPackageJson && currentDir) {
|
|
8772
8970
|
const info = await parsePackageJson(rootDir, currentDir);
|
|
@@ -8787,10 +8985,10 @@ async function scanForPackageJsons(rootDir, currentDir = "", depth = 0) {
|
|
|
8787
8985
|
}
|
|
8788
8986
|
async function parsePackageJson(rootDir, relativePath) {
|
|
8789
8987
|
try {
|
|
8790
|
-
const packageJsonPath =
|
|
8791
|
-
const content = await
|
|
8988
|
+
const packageJsonPath = path21.join(rootDir, relativePath, "package.json");
|
|
8989
|
+
const content = await fs7.readFile(packageJsonPath, "utf-8");
|
|
8792
8990
|
const pkg = JSON.parse(content);
|
|
8793
|
-
const name = pkg.name ||
|
|
8991
|
+
const name = pkg.name || path21.basename(relativePath);
|
|
8794
8992
|
const deps = { ...pkg.dependencies, ...pkg.devDependencies };
|
|
8795
8993
|
let type = "unknown";
|
|
8796
8994
|
if (deps["next"] || deps["react"] || deps["vue"] || deps["svelte"]) {
|
|
@@ -8826,7 +9024,7 @@ async function detectProjectStructure(rootDir) {
|
|
|
8826
9024
|
const projectMap = new Map;
|
|
8827
9025
|
let isMonorepo = false;
|
|
8828
9026
|
try {
|
|
8829
|
-
const entries = await
|
|
9027
|
+
const entries = await fs7.readdir(rootDir, { withFileTypes: true });
|
|
8830
9028
|
const dirNames = entries.filter((e) => e.isDirectory()).map((e) => e.name);
|
|
8831
9029
|
const monorepoPatterns = ["apps", "packages", "libs", "services"];
|
|
8832
9030
|
const hasMonorepoStructure = monorepoPatterns.some((p) => dirNames.includes(p));
|
|
@@ -8835,9 +9033,9 @@ async function detectProjectStructure(rootDir) {
|
|
|
8835
9033
|
for (const pattern of monorepoPatterns) {
|
|
8836
9034
|
if (!dirNames.includes(pattern))
|
|
8837
9035
|
continue;
|
|
8838
|
-
const patternDir =
|
|
9036
|
+
const patternDir = path21.join(rootDir, pattern);
|
|
8839
9037
|
try {
|
|
8840
|
-
const subDirs = await
|
|
9038
|
+
const subDirs = await fs7.readdir(patternDir, { withFileTypes: true });
|
|
8841
9039
|
for (const subDir of subDirs) {
|
|
8842
9040
|
if (!subDir.isDirectory())
|
|
8843
9041
|
continue;
|
|
@@ -8866,8 +9064,8 @@ async function detectProjectStructure(rootDir) {
|
|
|
8866
9064
|
}
|
|
8867
9065
|
let rootType = "unknown";
|
|
8868
9066
|
try {
|
|
8869
|
-
const rootPkgPath =
|
|
8870
|
-
const rootPkg = JSON.parse(await
|
|
9067
|
+
const rootPkgPath = path21.join(rootDir, "package.json");
|
|
9068
|
+
const rootPkg = JSON.parse(await fs7.readFile(rootPkgPath, "utf-8"));
|
|
8871
9069
|
if (rootPkg.workspaces)
|
|
8872
9070
|
isMonorepo = true;
|
|
8873
9071
|
const deps = { ...rootPkg.dependencies, ...rootPkg.devDependencies };
|
|
@@ -8906,8 +9104,8 @@ var init_projectDetector = __esm(() => {
|
|
|
8906
9104
|
});
|
|
8907
9105
|
|
|
8908
9106
|
// src/infrastructure/introspection/IntrospectionIndex.ts
|
|
8909
|
-
import * as
|
|
8910
|
-
import * as
|
|
9107
|
+
import * as path22 from "path";
|
|
9108
|
+
import * as fs8 from "fs/promises";
|
|
8911
9109
|
import * as fsSync from "fs";
|
|
8912
9110
|
|
|
8913
9111
|
class IntrospectionIndex {
|
|
@@ -8921,8 +9119,8 @@ class IntrospectionIndex {
|
|
|
8921
9119
|
async initialize() {
|
|
8922
9120
|
this.structure = await detectProjectStructure(this.rootDir);
|
|
8923
9121
|
try {
|
|
8924
|
-
const configPath =
|
|
8925
|
-
const configContent = await
|
|
9122
|
+
const configPath = path22.join(this.rootDir, ".raggrep", "config.json");
|
|
9123
|
+
const configContent = await fs8.readFile(configPath, "utf-8");
|
|
8926
9124
|
const config = JSON.parse(configContent);
|
|
8927
9125
|
this.config = config.introspection || {};
|
|
8928
9126
|
} catch {}
|
|
@@ -8936,7 +9134,7 @@ class IntrospectionIndex {
|
|
|
8936
9134
|
}
|
|
8937
9135
|
const fileExists = enableReadmeContext ? (relativePath) => {
|
|
8938
9136
|
try {
|
|
8939
|
-
const absolutePath =
|
|
9137
|
+
const absolutePath = path22.join(this.rootDir, relativePath);
|
|
8940
9138
|
return fsSync.existsSync(absolutePath);
|
|
8941
9139
|
} catch {
|
|
8942
9140
|
return false;
|
|
@@ -8972,28 +9170,28 @@ class IntrospectionIndex {
|
|
|
8972
9170
|
}
|
|
8973
9171
|
}
|
|
8974
9172
|
async save(config) {
|
|
8975
|
-
const introDir =
|
|
8976
|
-
await
|
|
8977
|
-
const projectPath =
|
|
8978
|
-
await
|
|
9173
|
+
const introDir = path22.join(getRaggrepDir(this.rootDir, config), "introspection");
|
|
9174
|
+
await fs8.mkdir(introDir, { recursive: true });
|
|
9175
|
+
const projectPath = path22.join(introDir, "_project.json");
|
|
9176
|
+
await fs8.writeFile(projectPath, JSON.stringify({
|
|
8979
9177
|
version: "1.0.0",
|
|
8980
9178
|
lastUpdated: new Date().toISOString(),
|
|
8981
9179
|
structure: this.structure
|
|
8982
9180
|
}, null, 2));
|
|
8983
9181
|
for (const [filepath, intro] of this.files) {
|
|
8984
|
-
const introFilePath =
|
|
8985
|
-
await
|
|
8986
|
-
await
|
|
9182
|
+
const introFilePath = path22.join(introDir, "files", filepath.replace(/\.[^.]+$/, ".json"));
|
|
9183
|
+
await fs8.mkdir(path22.dirname(introFilePath), { recursive: true });
|
|
9184
|
+
await fs8.writeFile(introFilePath, JSON.stringify(intro, null, 2));
|
|
8987
9185
|
}
|
|
8988
9186
|
}
|
|
8989
9187
|
async load(config) {
|
|
8990
|
-
const introDir =
|
|
9188
|
+
const introDir = path22.join(getRaggrepDir(this.rootDir, config), "introspection");
|
|
8991
9189
|
try {
|
|
8992
|
-
const projectPath =
|
|
8993
|
-
const projectContent = await
|
|
9190
|
+
const projectPath = path22.join(introDir, "_project.json");
|
|
9191
|
+
const projectContent = await fs8.readFile(projectPath, "utf-8");
|
|
8994
9192
|
const projectData = JSON.parse(projectContent);
|
|
8995
9193
|
this.structure = projectData.structure;
|
|
8996
|
-
await this.loadFilesRecursive(
|
|
9194
|
+
await this.loadFilesRecursive(path22.join(introDir, "files"), "");
|
|
8997
9195
|
} catch {
|
|
8998
9196
|
this.structure = null;
|
|
8999
9197
|
this.files.clear();
|
|
@@ -9001,14 +9199,14 @@ class IntrospectionIndex {
|
|
|
9001
9199
|
}
|
|
9002
9200
|
async loadFilesRecursive(basePath, prefix) {
|
|
9003
9201
|
try {
|
|
9004
|
-
const entries = await
|
|
9202
|
+
const entries = await fs8.readdir(basePath, { withFileTypes: true });
|
|
9005
9203
|
for (const entry of entries) {
|
|
9006
|
-
const entryPath =
|
|
9204
|
+
const entryPath = path22.join(basePath, entry.name);
|
|
9007
9205
|
const relativePath = prefix ? `${prefix}/${entry.name}` : entry.name;
|
|
9008
9206
|
if (entry.isDirectory()) {
|
|
9009
9207
|
await this.loadFilesRecursive(entryPath, relativePath);
|
|
9010
9208
|
} else if (entry.name.endsWith(".json")) {
|
|
9011
|
-
const content = await
|
|
9209
|
+
const content = await fs8.readFile(entryPath, "utf-8");
|
|
9012
9210
|
const intro = JSON.parse(content);
|
|
9013
9211
|
this.files.set(intro.filepath, intro);
|
|
9014
9212
|
}
|
|
@@ -9034,7 +9232,7 @@ var init_introspection2 = __esm(() => {
|
|
|
9034
9232
|
|
|
9035
9233
|
// src/app/indexer/watcher.ts
|
|
9036
9234
|
import { watch } from "chokidar";
|
|
9037
|
-
import * as
|
|
9235
|
+
import * as path23 from "path";
|
|
9038
9236
|
async function watchDirectory(rootDir, options = {}) {
|
|
9039
9237
|
const {
|
|
9040
9238
|
debounceMs = DEFAULT_DEBOUNCE_MS,
|
|
@@ -9045,17 +9243,17 @@ async function watchDirectory(rootDir, options = {}) {
|
|
|
9045
9243
|
onFileChange,
|
|
9046
9244
|
onError
|
|
9047
9245
|
} = options;
|
|
9048
|
-
rootDir =
|
|
9246
|
+
rootDir = path23.resolve(rootDir);
|
|
9049
9247
|
const config = await loadConfig(rootDir);
|
|
9050
|
-
const indexLocation = getIndexLocation(rootDir);
|
|
9051
9248
|
const validExtensions = new Set(config.extensions);
|
|
9052
9249
|
const ignorePatterns = [
|
|
9053
9250
|
...config.ignorePaths.map((p) => `**/${p}/**`),
|
|
9054
9251
|
"**/node_modules/**",
|
|
9055
|
-
"**/.git/**"
|
|
9252
|
+
"**/.git/**",
|
|
9253
|
+
"**/.raggrep/**"
|
|
9056
9254
|
];
|
|
9057
9255
|
function shouldWatchFile(filepath) {
|
|
9058
|
-
const ext =
|
|
9256
|
+
const ext = path23.extname(filepath);
|
|
9059
9257
|
return validExtensions.has(ext);
|
|
9060
9258
|
}
|
|
9061
9259
|
let isRunning = true;
|
|
@@ -9138,7 +9336,7 @@ async function watchDirectory(rootDir, options = {}) {
|
|
|
9138
9336
|
function handleFileEvent(event, filepath) {
|
|
9139
9337
|
if (!isRunning)
|
|
9140
9338
|
return;
|
|
9141
|
-
const relativePath =
|
|
9339
|
+
const relativePath = path23.relative(rootDir, filepath);
|
|
9142
9340
|
if (!shouldWatchFile(filepath)) {
|
|
9143
9341
|
return;
|
|
9144
9342
|
}
|
|
@@ -9216,9 +9414,9 @@ __export(exports_indexer, {
|
|
|
9216
9414
|
clearFreshnessCache: () => clearFreshnessCache,
|
|
9217
9415
|
cleanupIndex: () => cleanupIndex
|
|
9218
9416
|
});
|
|
9219
|
-
import * as
|
|
9220
|
-
import * as
|
|
9221
|
-
import * as
|
|
9417
|
+
import * as fs9 from "fs/promises";
|
|
9418
|
+
import * as path24 from "path";
|
|
9419
|
+
import * as os2 from "os";
|
|
9222
9420
|
import * as crypto2 from "crypto";
|
|
9223
9421
|
function clearFreshnessCache() {
|
|
9224
9422
|
freshnessCache = null;
|
|
@@ -9258,7 +9456,7 @@ function formatDuration(ms) {
|
|
|
9258
9456
|
return `${minutes}m ${remainingSeconds.toFixed(1)}s`;
|
|
9259
9457
|
}
|
|
9260
9458
|
function getOptimalConcurrency() {
|
|
9261
|
-
const cpuCount =
|
|
9459
|
+
const cpuCount = os2.cpus().length;
|
|
9262
9460
|
const optimal = Math.max(2, Math.min(16, Math.floor(cpuCount * 0.75)));
|
|
9263
9461
|
return optimal;
|
|
9264
9462
|
}
|
|
@@ -9269,7 +9467,7 @@ async function indexDirectory(rootDir, options = {}) {
|
|
|
9269
9467
|
const concurrency = options.concurrency ?? DEFAULT_CONCURRENCY;
|
|
9270
9468
|
clearFreshnessCache();
|
|
9271
9469
|
const logger = options.logger ? options.logger : quiet ? createSilentLogger() : createLogger({ verbose });
|
|
9272
|
-
rootDir =
|
|
9470
|
+
rootDir = path24.resolve(rootDir);
|
|
9273
9471
|
const location = getIndexLocation(rootDir);
|
|
9274
9472
|
logger.info(`Indexing directory: ${rootDir}`);
|
|
9275
9473
|
logger.info(`Index location: ${location.indexDir}`);
|
|
@@ -9321,12 +9519,12 @@ async function indexDirectory(rootDir, options = {}) {
|
|
|
9321
9519
|
rootDir,
|
|
9322
9520
|
config,
|
|
9323
9521
|
readFile: async (filepath) => {
|
|
9324
|
-
const fullPath =
|
|
9325
|
-
return
|
|
9522
|
+
const fullPath = path24.isAbsolute(filepath) ? filepath : path24.join(rootDir, filepath);
|
|
9523
|
+
return fs9.readFile(fullPath, "utf-8");
|
|
9326
9524
|
},
|
|
9327
9525
|
getFileStats: async (filepath) => {
|
|
9328
|
-
const fullPath =
|
|
9329
|
-
const stats = await
|
|
9526
|
+
const fullPath = path24.isAbsolute(filepath) ? filepath : path24.join(rootDir, filepath);
|
|
9527
|
+
const stats = await fs9.stat(fullPath);
|
|
9330
9528
|
return { lastModified: stats.mtime.toISOString() };
|
|
9331
9529
|
}
|
|
9332
9530
|
};
|
|
@@ -9351,7 +9549,7 @@ async function isIndexVersionCompatible(rootDir) {
|
|
|
9351
9549
|
const config = await loadConfig(rootDir);
|
|
9352
9550
|
const globalManifestPath = getGlobalManifestPath(rootDir, config);
|
|
9353
9551
|
try {
|
|
9354
|
-
const content = await
|
|
9552
|
+
const content = await fs9.readFile(globalManifestPath, "utf-8");
|
|
9355
9553
|
const manifest = JSON.parse(content);
|
|
9356
9554
|
return manifest.version === INDEX_SCHEMA_VERSION;
|
|
9357
9555
|
} catch {
|
|
@@ -9361,11 +9559,11 @@ async function isIndexVersionCompatible(rootDir) {
|
|
|
9361
9559
|
async function deleteIndex(rootDir) {
|
|
9362
9560
|
const indexDir = getRaggrepDir(rootDir);
|
|
9363
9561
|
try {
|
|
9364
|
-
await
|
|
9562
|
+
await fs9.rm(indexDir, { recursive: true, force: true });
|
|
9365
9563
|
} catch {}
|
|
9366
9564
|
}
|
|
9367
9565
|
async function resetIndex(rootDir) {
|
|
9368
|
-
rootDir =
|
|
9566
|
+
rootDir = path24.resolve(rootDir);
|
|
9369
9567
|
clearFreshnessCache();
|
|
9370
9568
|
const status = await getIndexStatus(rootDir);
|
|
9371
9569
|
if (!status.exists) {
|
|
@@ -9390,7 +9588,7 @@ async function ensureIndexFresh(rootDir, options = {}) {
|
|
|
9390
9588
|
let filesChanged = 0;
|
|
9391
9589
|
let filesReindexed = 0;
|
|
9392
9590
|
const logger = options.logger ? options.logger : quiet ? createSilentLogger() : createLogger({ verbose });
|
|
9393
|
-
rootDir =
|
|
9591
|
+
rootDir = path24.resolve(rootDir);
|
|
9394
9592
|
const status = await getIndexStatus(rootDir);
|
|
9395
9593
|
if (!status.exists) {
|
|
9396
9594
|
clearFreshnessCache();
|
|
@@ -9414,7 +9612,7 @@ async function ensureIndexFresh(rootDir, options = {}) {
|
|
|
9414
9612
|
const globalManifestPath = getGlobalManifestPath(rootDir, config);
|
|
9415
9613
|
let currentManifestMtime = 0;
|
|
9416
9614
|
try {
|
|
9417
|
-
const manifestStats = await
|
|
9615
|
+
const manifestStats = await fs9.stat(globalManifestPath);
|
|
9418
9616
|
currentManifestMtime = manifestStats.mtimeMs;
|
|
9419
9617
|
} catch {}
|
|
9420
9618
|
const now = Date.now();
|
|
@@ -9452,7 +9650,7 @@ async function ensureIndexFresh(rootDir, options = {}) {
|
|
|
9452
9650
|
const { allFiles: currentFiles, changedFiles, changedFileMtimes } = discoveryResult;
|
|
9453
9651
|
filesDiscovered = currentFiles.length;
|
|
9454
9652
|
filesChanged = changedFiles.length;
|
|
9455
|
-
const currentFileSet = new Set(currentFiles.map((f) =>
|
|
9653
|
+
const currentFileSet = new Set(currentFiles.map((f) => path24.relative(rootDir, f)));
|
|
9456
9654
|
const changedFileSet = new Set(changedFiles);
|
|
9457
9655
|
let totalIndexed = 0;
|
|
9458
9656
|
let totalRemoved = 0;
|
|
@@ -9486,11 +9684,11 @@ async function ensureIndexFresh(rootDir, options = {}) {
|
|
|
9486
9684
|
if (filesToRemove.length > 0) {
|
|
9487
9685
|
await Promise.all(filesToRemove.map(async (filepath) => {
|
|
9488
9686
|
logger.debug(` Removing stale: ${filepath}`);
|
|
9489
|
-
const indexFilePath =
|
|
9490
|
-
const symbolicFilePath =
|
|
9687
|
+
const indexFilePath = path24.join(indexPath, filepath.replace(/\.[^.]+$/, ".json"));
|
|
9688
|
+
const symbolicFilePath = path24.join(indexPath, "symbolic", filepath.replace(/\.[^.]+$/, ".json"));
|
|
9491
9689
|
await Promise.all([
|
|
9492
|
-
|
|
9493
|
-
|
|
9690
|
+
fs9.unlink(indexFilePath).catch(() => {}),
|
|
9691
|
+
fs9.unlink(symbolicFilePath).catch(() => {})
|
|
9494
9692
|
]);
|
|
9495
9693
|
delete manifest.files[filepath];
|
|
9496
9694
|
removedFilepaths.push(filepath);
|
|
@@ -9514,19 +9712,19 @@ async function ensureIndexFresh(rootDir, options = {}) {
|
|
|
9514
9712
|
rootDir,
|
|
9515
9713
|
config,
|
|
9516
9714
|
readFile: async (filepath) => {
|
|
9517
|
-
const fullPath =
|
|
9518
|
-
return
|
|
9715
|
+
const fullPath = path24.isAbsolute(filepath) ? filepath : path24.join(rootDir, filepath);
|
|
9716
|
+
return fs9.readFile(fullPath, "utf-8");
|
|
9519
9717
|
},
|
|
9520
9718
|
getFileStats: async (filepath) => {
|
|
9521
|
-
const fullPath =
|
|
9522
|
-
const stats = await
|
|
9719
|
+
const fullPath = path24.isAbsolute(filepath) ? filepath : path24.join(rootDir, filepath);
|
|
9720
|
+
const stats = await fs9.stat(fullPath);
|
|
9523
9721
|
return { lastModified: stats.mtime.toISOString() };
|
|
9524
9722
|
},
|
|
9525
9723
|
getIntrospection: (filepath) => introspection.getFile(filepath)
|
|
9526
9724
|
};
|
|
9527
9725
|
const moduleChangedFiles = module.supportsFile ? changedFiles.filter((f) => module.supportsFile(f)) : changedFiles;
|
|
9528
9726
|
const filesToProcess = moduleChangedFiles.map((filepath) => {
|
|
9529
|
-
const relativePath =
|
|
9727
|
+
const relativePath = path24.relative(rootDir, filepath);
|
|
9530
9728
|
const existingEntry = manifest.files[relativePath];
|
|
9531
9729
|
const lastModified = changedFileMtimes.get(filepath) || new Date().toISOString();
|
|
9532
9730
|
return {
|
|
@@ -9556,7 +9754,7 @@ async function ensureIndexFresh(rootDir, options = {}) {
|
|
|
9556
9754
|
return { relativePath, status: "unchanged" };
|
|
9557
9755
|
}
|
|
9558
9756
|
try {
|
|
9559
|
-
const content = await
|
|
9757
|
+
const content = await fs9.readFile(filepath, "utf-8");
|
|
9560
9758
|
const contentHash = computeContentHash(content);
|
|
9561
9759
|
if (!isNew && existingContentHash && existingContentHash === contentHash) {
|
|
9562
9760
|
completedCount++;
|
|
@@ -9683,7 +9881,7 @@ async function ensureIndexFresh(rootDir, options = {}) {
|
|
|
9683
9881
|
}
|
|
9684
9882
|
let finalManifestMtime = currentManifestMtime;
|
|
9685
9883
|
try {
|
|
9686
|
-
const manifestStats = await
|
|
9884
|
+
const manifestStats = await fs9.stat(globalManifestPath);
|
|
9687
9885
|
finalManifestMtime = manifestStats.mtimeMs;
|
|
9688
9886
|
} catch {}
|
|
9689
9887
|
freshnessCache = {
|
|
@@ -9703,7 +9901,7 @@ async function indexWithModule(rootDir, files, module, config, verbose, introspe
|
|
|
9703
9901
|
};
|
|
9704
9902
|
const manifest = await loadModuleManifest(rootDir, module.id, config);
|
|
9705
9903
|
const indexPath = getModuleIndexPath(rootDir, module.id, config);
|
|
9706
|
-
const currentFileSet = new Set(files.map((f) =>
|
|
9904
|
+
const currentFileSet = new Set(files.map((f) => path24.relative(rootDir, f)));
|
|
9707
9905
|
const filesToRemove = [];
|
|
9708
9906
|
for (const filepath of Object.keys(manifest.files)) {
|
|
9709
9907
|
if (!currentFileSet.has(filepath)) {
|
|
@@ -9714,13 +9912,13 @@ async function indexWithModule(rootDir, files, module, config, verbose, introspe
|
|
|
9714
9912
|
logger.info(` Removing ${filesToRemove.length} stale entries...`);
|
|
9715
9913
|
for (const filepath of filesToRemove) {
|
|
9716
9914
|
logger.debug(` Removing: ${filepath}`);
|
|
9717
|
-
const indexFilePath =
|
|
9915
|
+
const indexFilePath = path24.join(indexPath, filepath.replace(/\.[^.]+$/, ".json"));
|
|
9718
9916
|
try {
|
|
9719
|
-
await
|
|
9917
|
+
await fs9.unlink(indexFilePath);
|
|
9720
9918
|
} catch {}
|
|
9721
|
-
const symbolicFilePath =
|
|
9919
|
+
const symbolicFilePath = path24.join(indexPath, "symbolic", filepath.replace(/\.[^.]+$/, ".json"));
|
|
9722
9920
|
try {
|
|
9723
|
-
await
|
|
9921
|
+
await fs9.unlink(symbolicFilePath);
|
|
9724
9922
|
} catch {}
|
|
9725
9923
|
delete manifest.files[filepath];
|
|
9726
9924
|
}
|
|
@@ -9730,12 +9928,12 @@ async function indexWithModule(rootDir, files, module, config, verbose, introspe
|
|
|
9730
9928
|
rootDir,
|
|
9731
9929
|
config,
|
|
9732
9930
|
readFile: async (filepath) => {
|
|
9733
|
-
const fullPath =
|
|
9734
|
-
return
|
|
9931
|
+
const fullPath = path24.isAbsolute(filepath) ? filepath : path24.join(rootDir, filepath);
|
|
9932
|
+
return fs9.readFile(fullPath, "utf-8");
|
|
9735
9933
|
},
|
|
9736
9934
|
getFileStats: async (filepath) => {
|
|
9737
|
-
const fullPath =
|
|
9738
|
-
const stats = await
|
|
9935
|
+
const fullPath = path24.isAbsolute(filepath) ? filepath : path24.join(rootDir, filepath);
|
|
9936
|
+
const stats = await fs9.stat(fullPath);
|
|
9739
9937
|
return { lastModified: stats.mtime.toISOString() };
|
|
9740
9938
|
},
|
|
9741
9939
|
getIntrospection: (filepath) => introspection.getFile(filepath)
|
|
@@ -9747,9 +9945,9 @@ async function indexWithModule(rootDir, files, module, config, verbose, introspe
|
|
|
9747
9945
|
let indexedCount = 0;
|
|
9748
9946
|
let skippedCount = 0;
|
|
9749
9947
|
const processFile = async (filepath, _index) => {
|
|
9750
|
-
const relativePath =
|
|
9948
|
+
const relativePath = path24.relative(rootDir, filepath);
|
|
9751
9949
|
try {
|
|
9752
|
-
const stats = await
|
|
9950
|
+
const stats = await fs9.stat(filepath);
|
|
9753
9951
|
const lastModified = stats.mtime.toISOString();
|
|
9754
9952
|
const existingEntry = manifest.files[relativePath];
|
|
9755
9953
|
if (existingEntry && existingEntry.lastModified === lastModified) {
|
|
@@ -9759,7 +9957,7 @@ async function indexWithModule(rootDir, files, module, config, verbose, introspe
|
|
|
9759
9957
|
logger.debug(` [${completedCount}/${totalFiles}] Skipped ${relativePath} (unchanged)`);
|
|
9760
9958
|
return { relativePath, status: "skipped" };
|
|
9761
9959
|
}
|
|
9762
|
-
const content = await
|
|
9960
|
+
const content = await fs9.readFile(filepath, "utf-8");
|
|
9763
9961
|
const contentHash = computeContentHash(content);
|
|
9764
9962
|
if (existingEntry?.contentHash && existingEntry.contentHash === contentHash) {
|
|
9765
9963
|
completedCount++;
|
|
@@ -9840,8 +10038,8 @@ async function indexWithModule(rootDir, files, module, config, verbose, introspe
|
|
|
9840
10038
|
return result;
|
|
9841
10039
|
}
|
|
9842
10040
|
function isLikelyBinary(filepath) {
|
|
9843
|
-
const ext =
|
|
9844
|
-
const basename15 =
|
|
10041
|
+
const ext = path24.extname(filepath).toLowerCase();
|
|
10042
|
+
const basename15 = path24.basename(filepath).toLowerCase();
|
|
9845
10043
|
const binaryExtensions = new Set([
|
|
9846
10044
|
".png",
|
|
9847
10045
|
".jpg",
|
|
@@ -9917,7 +10115,7 @@ async function findFilesWithStats(rootDir, config, lastIndexStarted) {
|
|
|
9917
10115
|
const ignoreDirs = new Set(config.ignorePaths);
|
|
9918
10116
|
const lastIndexMs = lastIndexStarted?.getTime() ?? 0;
|
|
9919
10117
|
const crawler = new Builder().withFullPaths().exclude((dirName) => ignoreDirs.has(dirName)).filter((filePath) => {
|
|
9920
|
-
const ext =
|
|
10118
|
+
const ext = path24.extname(filePath);
|
|
9921
10119
|
return validExtensions.has(ext);
|
|
9922
10120
|
}).crawl(rootDir);
|
|
9923
10121
|
const allFiles = await crawler.withPromise();
|
|
@@ -9925,7 +10123,7 @@ async function findFilesWithStats(rootDir, config, lastIndexStarted) {
|
|
|
9925
10123
|
const changedFileMtimes2 = new Map;
|
|
9926
10124
|
await parallelMap(allFiles, async (filePath) => {
|
|
9927
10125
|
try {
|
|
9928
|
-
const stats = await
|
|
10126
|
+
const stats = await fs9.stat(filePath);
|
|
9929
10127
|
changedFileMtimes2.set(filePath, stats.mtime.toISOString());
|
|
9930
10128
|
} catch {}
|
|
9931
10129
|
}, STAT_CONCURRENCY);
|
|
@@ -9939,7 +10137,7 @@ async function findFilesWithStats(rootDir, config, lastIndexStarted) {
|
|
|
9939
10137
|
const changedFileMtimes = new Map;
|
|
9940
10138
|
await parallelMap(allFiles, async (filePath) => {
|
|
9941
10139
|
try {
|
|
9942
|
-
const stats = await
|
|
10140
|
+
const stats = await fs9.stat(filePath);
|
|
9943
10141
|
if (stats.mtimeMs > lastIndexMs) {
|
|
9944
10142
|
changedFiles.push(filePath);
|
|
9945
10143
|
changedFileMtimes.set(filePath, stats.mtime.toISOString());
|
|
@@ -9959,7 +10157,7 @@ async function findFiles(rootDir, config) {
|
|
|
9959
10157
|
async function loadModuleManifest(rootDir, moduleId, config) {
|
|
9960
10158
|
const manifestPath = getModuleManifestPath(rootDir, moduleId, config);
|
|
9961
10159
|
try {
|
|
9962
|
-
const content = await
|
|
10160
|
+
const content = await fs9.readFile(manifestPath, "utf-8");
|
|
9963
10161
|
return JSON.parse(content);
|
|
9964
10162
|
} catch {
|
|
9965
10163
|
return {
|
|
@@ -9972,19 +10170,19 @@ async function loadModuleManifest(rootDir, moduleId, config) {
|
|
|
9972
10170
|
}
|
|
9973
10171
|
async function writeModuleManifest(rootDir, moduleId, manifest, config) {
|
|
9974
10172
|
const manifestPath = getModuleManifestPath(rootDir, moduleId, config);
|
|
9975
|
-
await
|
|
9976
|
-
await
|
|
10173
|
+
await fs9.mkdir(path24.dirname(manifestPath), { recursive: true });
|
|
10174
|
+
await fs9.writeFile(manifestPath, JSON.stringify(manifest, null, 2));
|
|
9977
10175
|
}
|
|
9978
10176
|
async function writeFileIndex(rootDir, moduleId, filepath, fileIndex, config) {
|
|
9979
10177
|
const indexPath = getModuleIndexPath(rootDir, moduleId, config);
|
|
9980
|
-
const indexFilePath =
|
|
9981
|
-
await
|
|
9982
|
-
await
|
|
10178
|
+
const indexFilePath = path24.join(indexPath, filepath.replace(/\.[^.]+$/, ".json"));
|
|
10179
|
+
await fs9.mkdir(path24.dirname(indexFilePath), { recursive: true });
|
|
10180
|
+
await fs9.writeFile(indexFilePath, JSON.stringify(fileIndex, null, 2));
|
|
9983
10181
|
}
|
|
9984
10182
|
async function loadGlobalManifest(rootDir, config) {
|
|
9985
10183
|
const manifestPath = getGlobalManifestPath(rootDir, config);
|
|
9986
10184
|
try {
|
|
9987
|
-
const content = await
|
|
10185
|
+
const content = await fs9.readFile(manifestPath, "utf-8");
|
|
9988
10186
|
return JSON.parse(content);
|
|
9989
10187
|
} catch {
|
|
9990
10188
|
return null;
|
|
@@ -9998,13 +10196,13 @@ async function updateGlobalManifest(rootDir, modules, config, indexStartTime) {
|
|
|
9998
10196
|
lastIndexStarted: indexStartTime,
|
|
9999
10197
|
modules: modules.map((m) => m.id)
|
|
10000
10198
|
};
|
|
10001
|
-
await
|
|
10002
|
-
await
|
|
10199
|
+
await fs9.mkdir(path24.dirname(manifestPath), { recursive: true });
|
|
10200
|
+
await fs9.writeFile(manifestPath, JSON.stringify(manifest, null, 2));
|
|
10003
10201
|
}
|
|
10004
10202
|
async function cleanupIndex(rootDir, options = {}) {
|
|
10005
10203
|
const verbose = options.verbose ?? false;
|
|
10006
10204
|
const logger = options.logger ?? createLogger({ verbose });
|
|
10007
|
-
rootDir =
|
|
10205
|
+
rootDir = path24.resolve(rootDir);
|
|
10008
10206
|
logger.info(`Cleaning up index in: ${rootDir}`);
|
|
10009
10207
|
const config = await loadConfig(rootDir);
|
|
10010
10208
|
await registerBuiltInModules();
|
|
@@ -10034,9 +10232,9 @@ async function cleanupModuleIndex(rootDir, moduleId, config, logger) {
|
|
|
10034
10232
|
const filesToRemove = [];
|
|
10035
10233
|
const updatedFiles = {};
|
|
10036
10234
|
for (const [filepath, entry] of Object.entries(manifest.files)) {
|
|
10037
|
-
const fullPath =
|
|
10235
|
+
const fullPath = path24.join(rootDir, filepath);
|
|
10038
10236
|
try {
|
|
10039
|
-
await
|
|
10237
|
+
await fs9.access(fullPath);
|
|
10040
10238
|
updatedFiles[filepath] = entry;
|
|
10041
10239
|
result.kept++;
|
|
10042
10240
|
} catch {
|
|
@@ -10046,9 +10244,9 @@ async function cleanupModuleIndex(rootDir, moduleId, config, logger) {
|
|
|
10046
10244
|
}
|
|
10047
10245
|
}
|
|
10048
10246
|
for (const filepath of filesToRemove) {
|
|
10049
|
-
const indexFilePath =
|
|
10247
|
+
const indexFilePath = path24.join(indexPath, filepath.replace(/\.[^.]+$/, ".json"));
|
|
10050
10248
|
try {
|
|
10051
|
-
await
|
|
10249
|
+
await fs9.unlink(indexFilePath);
|
|
10052
10250
|
} catch {}
|
|
10053
10251
|
}
|
|
10054
10252
|
manifest.files = updatedFiles;
|
|
@@ -10059,16 +10257,16 @@ async function cleanupModuleIndex(rootDir, moduleId, config, logger) {
|
|
|
10059
10257
|
}
|
|
10060
10258
|
async function cleanupEmptyDirectories(dir) {
|
|
10061
10259
|
try {
|
|
10062
|
-
const entries = await
|
|
10260
|
+
const entries = await fs9.readdir(dir, { withFileTypes: true });
|
|
10063
10261
|
for (const entry of entries) {
|
|
10064
10262
|
if (entry.isDirectory()) {
|
|
10065
|
-
const subDir =
|
|
10263
|
+
const subDir = path24.join(dir, entry.name);
|
|
10066
10264
|
await cleanupEmptyDirectories(subDir);
|
|
10067
10265
|
}
|
|
10068
10266
|
}
|
|
10069
|
-
const remainingEntries = await
|
|
10267
|
+
const remainingEntries = await fs9.readdir(dir);
|
|
10070
10268
|
if (remainingEntries.length === 0) {
|
|
10071
|
-
await
|
|
10269
|
+
await fs9.rmdir(dir);
|
|
10072
10270
|
return true;
|
|
10073
10271
|
}
|
|
10074
10272
|
return false;
|
|
@@ -10077,7 +10275,7 @@ async function cleanupEmptyDirectories(dir) {
|
|
|
10077
10275
|
}
|
|
10078
10276
|
}
|
|
10079
10277
|
async function getIndexStatus(rootDir) {
|
|
10080
|
-
rootDir =
|
|
10278
|
+
rootDir = path24.resolve(rootDir);
|
|
10081
10279
|
const config = await loadConfig(rootDir);
|
|
10082
10280
|
const location = getIndexLocation(rootDir);
|
|
10083
10281
|
const indexDir = location.indexDir;
|
|
@@ -10089,13 +10287,13 @@ async function getIndexStatus(rootDir) {
|
|
|
10089
10287
|
totalFiles: 0
|
|
10090
10288
|
};
|
|
10091
10289
|
try {
|
|
10092
|
-
await
|
|
10290
|
+
await fs9.access(indexDir);
|
|
10093
10291
|
} catch {
|
|
10094
10292
|
return status;
|
|
10095
10293
|
}
|
|
10096
10294
|
try {
|
|
10097
10295
|
const globalManifestPath = getGlobalManifestPath(rootDir, config);
|
|
10098
|
-
const content = await
|
|
10296
|
+
const content = await fs9.readFile(globalManifestPath, "utf-8");
|
|
10099
10297
|
const globalManifest = JSON.parse(content);
|
|
10100
10298
|
status.exists = true;
|
|
10101
10299
|
status.lastUpdated = globalManifest.lastUpdated;
|
|
@@ -10113,7 +10311,7 @@ async function getIndexStatus(rootDir) {
|
|
|
10113
10311
|
}
|
|
10114
10312
|
} catch {
|
|
10115
10313
|
try {
|
|
10116
|
-
const entries = await
|
|
10314
|
+
const entries = await fs9.readdir(path24.join(indexDir, "index"));
|
|
10117
10315
|
if (entries.length > 0) {
|
|
10118
10316
|
status.exists = true;
|
|
10119
10317
|
for (const entry of entries) {
|
|
@@ -10133,7 +10331,7 @@ async function getIndexStatus(rootDir) {
|
|
|
10133
10331
|
}
|
|
10134
10332
|
return status;
|
|
10135
10333
|
}
|
|
10136
|
-
var FRESHNESS_CACHE_TTL_MS = 5000, freshnessCache = null, INDEX_SCHEMA_VERSION = "2.
|
|
10334
|
+
var FRESHNESS_CACHE_TTL_MS = 5000, freshnessCache = null, INDEX_SCHEMA_VERSION = "2.1.0", DEFAULT_CONCURRENCY, STAT_CONCURRENCY = 64;
|
|
10137
10335
|
var init_indexer = __esm(() => {
|
|
10138
10336
|
init_dist();
|
|
10139
10337
|
init_config2();
|
|
@@ -11479,7 +11677,7 @@ var minimatch = (p, pattern, options = {}) => {
|
|
|
11479
11677
|
}, qmarksTestNoExtDot = ([$0]) => {
|
|
11480
11678
|
const len = $0.length;
|
|
11481
11679
|
return (f) => f.length === len && f !== "." && f !== "..";
|
|
11482
|
-
}, defaultPlatform,
|
|
11680
|
+
}, defaultPlatform, path25, sep2, GLOBSTAR, qmark2 = "[^/]", star2, twoStarDot = "(?:(?!(?:\\/|^)(?:\\.{1,2})($|\\/)).)*?", twoStarNoDot = "(?:(?!(?:\\/|^)\\.).)*?", filter = (pattern, options = {}) => (p) => minimatch(p, pattern, options), ext = (a, b = {}) => Object.assign({}, a, b), defaults = (def) => {
|
|
11483
11681
|
if (!def || typeof def !== "object" || !Object.keys(def).length) {
|
|
11484
11682
|
return minimatch;
|
|
11485
11683
|
}
|
|
@@ -11537,11 +11735,11 @@ var init_esm2 = __esm(() => {
|
|
|
11537
11735
|
starRE = /^\*+$/;
|
|
11538
11736
|
qmarksRE = /^\?+([^+@!?\*\[\(]*)?$/;
|
|
11539
11737
|
defaultPlatform = typeof process === "object" && process ? typeof process.env === "object" && process.env && process.env.__MINIMATCH_TESTING_PLATFORM__ || process.platform : "posix";
|
|
11540
|
-
|
|
11738
|
+
path25 = {
|
|
11541
11739
|
win32: { sep: "\\" },
|
|
11542
11740
|
posix: { sep: "/" }
|
|
11543
11741
|
};
|
|
11544
|
-
sep2 = defaultPlatform === "win32" ?
|
|
11742
|
+
sep2 = defaultPlatform === "win32" ? path25.win32.sep : path25.posix.sep;
|
|
11545
11743
|
minimatch.sep = sep2;
|
|
11546
11744
|
GLOBSTAR = Symbol("globstar **");
|
|
11547
11745
|
minimatch.GLOBSTAR = GLOBSTAR;
|
|
@@ -11581,7 +11779,7 @@ function matchesPathFilter(relativePath, filters, matchFn) {
|
|
|
11581
11779
|
}
|
|
11582
11780
|
return false;
|
|
11583
11781
|
}
|
|
11584
|
-
async function executeExactSearch(
|
|
11782
|
+
async function executeExactSearch(fs10, options, matchFn) {
|
|
11585
11783
|
const {
|
|
11586
11784
|
rootDir,
|
|
11587
11785
|
literal,
|
|
@@ -11593,13 +11791,13 @@ async function executeExactSearch(fs9, options, matchFn) {
|
|
|
11593
11791
|
const files = new Map;
|
|
11594
11792
|
async function walkDir(dir, baseDir) {
|
|
11595
11793
|
try {
|
|
11596
|
-
const entries = await
|
|
11794
|
+
const entries = await fs10.readDir(dir);
|
|
11597
11795
|
for (const entry of entries) {
|
|
11598
|
-
const fullPath =
|
|
11599
|
-
const relativePath =
|
|
11796
|
+
const fullPath = fs10.join(dir, entry);
|
|
11797
|
+
const relativePath = fs10.relative(baseDir, fullPath);
|
|
11600
11798
|
let isDirectory = false;
|
|
11601
11799
|
try {
|
|
11602
|
-
const stats = await
|
|
11800
|
+
const stats = await fs10.getStats(fullPath);
|
|
11603
11801
|
isDirectory = stats.isDirectory ?? false;
|
|
11604
11802
|
} catch {
|
|
11605
11803
|
continue;
|
|
@@ -11616,7 +11814,7 @@ async function executeExactSearch(fs9, options, matchFn) {
|
|
|
11616
11814
|
}
|
|
11617
11815
|
}
|
|
11618
11816
|
try {
|
|
11619
|
-
const content = await
|
|
11817
|
+
const content = await fs10.readFile(fullPath);
|
|
11620
11818
|
if (isSearchableContent(content, fullPath)) {
|
|
11621
11819
|
files.set(relativePath, content);
|
|
11622
11820
|
}
|
|
@@ -11654,21 +11852,21 @@ var init_usecases = __esm(() => {
|
|
|
11654
11852
|
});
|
|
11655
11853
|
|
|
11656
11854
|
// src/infrastructure/filesystem/nodeFileSystem.ts
|
|
11657
|
-
import * as
|
|
11658
|
-
import * as
|
|
11855
|
+
import * as fs10 from "fs/promises";
|
|
11856
|
+
import * as path26 from "path";
|
|
11659
11857
|
import { glob } from "glob";
|
|
11660
11858
|
|
|
11661
11859
|
class NodeFileSystem {
|
|
11662
11860
|
async readFile(filepath) {
|
|
11663
|
-
return
|
|
11861
|
+
return fs10.readFile(filepath, "utf-8");
|
|
11664
11862
|
}
|
|
11665
11863
|
async writeFile(filepath, content) {
|
|
11666
|
-
await
|
|
11667
|
-
await
|
|
11864
|
+
await fs10.mkdir(path26.dirname(filepath), { recursive: true });
|
|
11865
|
+
await fs10.writeFile(filepath, content, "utf-8");
|
|
11668
11866
|
}
|
|
11669
11867
|
async deleteFile(filepath) {
|
|
11670
11868
|
try {
|
|
11671
|
-
await
|
|
11869
|
+
await fs10.unlink(filepath);
|
|
11672
11870
|
} catch (error) {
|
|
11673
11871
|
if (error.code !== "ENOENT") {
|
|
11674
11872
|
throw error;
|
|
@@ -11676,7 +11874,7 @@ class NodeFileSystem {
|
|
|
11676
11874
|
}
|
|
11677
11875
|
}
|
|
11678
11876
|
async getStats(filepath) {
|
|
11679
|
-
const stats = await
|
|
11877
|
+
const stats = await fs10.stat(filepath);
|
|
11680
11878
|
return {
|
|
11681
11879
|
lastModified: stats.mtime.toISOString(),
|
|
11682
11880
|
size: stats.isDirectory() ? undefined : stats.size,
|
|
@@ -11685,17 +11883,17 @@ class NodeFileSystem {
|
|
|
11685
11883
|
}
|
|
11686
11884
|
async exists(filepath) {
|
|
11687
11885
|
try {
|
|
11688
|
-
await
|
|
11886
|
+
await fs10.access(filepath);
|
|
11689
11887
|
return true;
|
|
11690
11888
|
} catch {
|
|
11691
11889
|
return false;
|
|
11692
11890
|
}
|
|
11693
11891
|
}
|
|
11694
11892
|
async mkdir(dirpath) {
|
|
11695
|
-
await
|
|
11893
|
+
await fs10.mkdir(dirpath, { recursive: true });
|
|
11696
11894
|
}
|
|
11697
11895
|
async readDir(dirpath) {
|
|
11698
|
-
return
|
|
11896
|
+
return fs10.readdir(dirpath);
|
|
11699
11897
|
}
|
|
11700
11898
|
async findFiles(rootDir, patterns, ignore) {
|
|
11701
11899
|
const ignorePatterns = ignore.map((p) => `**/${p}/**`);
|
|
@@ -11711,19 +11909,19 @@ class NodeFileSystem {
|
|
|
11711
11909
|
return [...new Set(files)];
|
|
11712
11910
|
}
|
|
11713
11911
|
join(...segments) {
|
|
11714
|
-
return
|
|
11912
|
+
return path26.join(...segments);
|
|
11715
11913
|
}
|
|
11716
11914
|
relative(from, to) {
|
|
11717
|
-
return
|
|
11915
|
+
return path26.relative(from, to);
|
|
11718
11916
|
}
|
|
11719
11917
|
resolve(...segments) {
|
|
11720
|
-
return
|
|
11918
|
+
return path26.resolve(...segments);
|
|
11721
11919
|
}
|
|
11722
11920
|
dirname(filepath) {
|
|
11723
|
-
return
|
|
11921
|
+
return path26.dirname(filepath);
|
|
11724
11922
|
}
|
|
11725
11923
|
extname(filepath) {
|
|
11726
|
-
return
|
|
11924
|
+
return path26.extname(filepath);
|
|
11727
11925
|
}
|
|
11728
11926
|
}
|
|
11729
11927
|
var nodeFileSystem;
|
|
@@ -11744,14 +11942,14 @@ __export(exports_search, {
|
|
|
11744
11942
|
formatSearchResults: () => formatSearchResults2,
|
|
11745
11943
|
formatHybridSearchResults: () => formatHybridSearchResults
|
|
11746
11944
|
});
|
|
11747
|
-
import * as
|
|
11748
|
-
import * as
|
|
11945
|
+
import * as fs11 from "fs/promises";
|
|
11946
|
+
import * as path27 from "path";
|
|
11749
11947
|
async function search(rootDir, query, options = {}) {
|
|
11750
11948
|
const hybridResults = await hybridSearch(rootDir, query, options);
|
|
11751
11949
|
return hybridResults.results;
|
|
11752
11950
|
}
|
|
11753
11951
|
async function hybridSearch(rootDir, query, options = {}) {
|
|
11754
|
-
rootDir =
|
|
11952
|
+
rootDir = path27.resolve(rootDir);
|
|
11755
11953
|
const ensureFresh = options.ensureFresh ?? DEFAULT_SEARCH_OPTIONS.ensureFresh;
|
|
11756
11954
|
if (ensureFresh) {
|
|
11757
11955
|
await ensureIndexFresh(rootDir, { quiet: true });
|
|
@@ -11828,15 +12026,15 @@ async function hybridSearch(rootDir, query, options = {}) {
|
|
|
11828
12026
|
};
|
|
11829
12027
|
}
|
|
11830
12028
|
async function performExactSearch(rootDir, literal, config, options) {
|
|
11831
|
-
const
|
|
11832
|
-
return executeExactSearch(
|
|
12029
|
+
const fs12 = new NodeFileSystem;
|
|
12030
|
+
return executeExactSearch(fs12, {
|
|
11833
12031
|
rootDir,
|
|
11834
12032
|
literal,
|
|
11835
12033
|
pathFilter: options.pathFilter,
|
|
11836
12034
|
maxFiles: 20,
|
|
11837
12035
|
maxOccurrencesPerFile: 5,
|
|
11838
12036
|
caseInsensitive: false
|
|
11839
|
-
}, (
|
|
12037
|
+
}, (path28, pattern) => minimatch(path28, pattern, { matchBase: true }));
|
|
11840
12038
|
}
|
|
11841
12039
|
function createSearchContext(rootDir, moduleId, config) {
|
|
11842
12040
|
const indexPath = getModuleIndexPath(rootDir, moduleId, config);
|
|
@@ -11845,9 +12043,9 @@ function createSearchContext(rootDir, moduleId, config) {
|
|
|
11845
12043
|
config,
|
|
11846
12044
|
loadFileIndex: async (filepath) => {
|
|
11847
12045
|
const hasExtension = /\.[^./]+$/.test(filepath);
|
|
11848
|
-
const indexFilePath = hasExtension ?
|
|
12046
|
+
const indexFilePath = hasExtension ? path27.join(indexPath, filepath.replace(/\.[^.]+$/, ".json")) : path27.join(indexPath, filepath + ".json");
|
|
11849
12047
|
try {
|
|
11850
|
-
const content = await
|
|
12048
|
+
const content = await fs11.readFile(indexFilePath, "utf-8");
|
|
11851
12049
|
return JSON.parse(content);
|
|
11852
12050
|
} catch {
|
|
11853
12051
|
return null;
|
|
@@ -11857,7 +12055,7 @@ function createSearchContext(rootDir, moduleId, config) {
|
|
|
11857
12055
|
const files = [];
|
|
11858
12056
|
await traverseDirectory(indexPath, files, indexPath);
|
|
11859
12057
|
return files.filter((f) => f.endsWith(".json") && !f.endsWith("manifest.json")).map((f) => {
|
|
11860
|
-
const relative6 =
|
|
12058
|
+
const relative6 = path27.relative(indexPath, f);
|
|
11861
12059
|
return relative6.replace(/\.json$/, "");
|
|
11862
12060
|
});
|
|
11863
12061
|
}
|
|
@@ -11865,9 +12063,9 @@ function createSearchContext(rootDir, moduleId, config) {
|
|
|
11865
12063
|
}
|
|
11866
12064
|
async function traverseDirectory(dir, files, basePath) {
|
|
11867
12065
|
try {
|
|
11868
|
-
const entries = await
|
|
12066
|
+
const entries = await fs11.readdir(dir, { withFileTypes: true });
|
|
11869
12067
|
for (const entry of entries) {
|
|
11870
|
-
const fullPath =
|
|
12068
|
+
const fullPath = path27.join(dir, entry.name);
|
|
11871
12069
|
if (entry.isDirectory()) {
|
|
11872
12070
|
await traverseDirectory(fullPath, files, basePath);
|
|
11873
12071
|
} else if (entry.isFile()) {
|
|
@@ -11879,7 +12077,7 @@ async function traverseDirectory(dir, files, basePath) {
|
|
|
11879
12077
|
async function loadGlobalManifest2(rootDir, config) {
|
|
11880
12078
|
const manifestPath = getGlobalManifestPath(rootDir, config);
|
|
11881
12079
|
try {
|
|
11882
|
-
const content = await
|
|
12080
|
+
const content = await fs11.readFile(manifestPath, "utf-8");
|
|
11883
12081
|
return JSON.parse(content);
|
|
11884
12082
|
} catch {
|
|
11885
12083
|
return null;
|
|
@@ -12048,18 +12246,18 @@ function getInstallationMethod(openCodeVersion) {
|
|
|
12048
12246
|
}
|
|
12049
12247
|
async function detectOpenCodeVersion() {
|
|
12050
12248
|
try {
|
|
12051
|
-
const
|
|
12052
|
-
const
|
|
12053
|
-
const
|
|
12054
|
-
const homeDir =
|
|
12249
|
+
const os3 = await import("os");
|
|
12250
|
+
const fs12 = await import("fs/promises");
|
|
12251
|
+
const path28 = await import("path");
|
|
12252
|
+
const homeDir = os3.homedir();
|
|
12055
12253
|
const possiblePaths = [
|
|
12056
|
-
|
|
12057
|
-
|
|
12058
|
-
|
|
12254
|
+
path28.join(homeDir, ".local", "share", "opencode", "package.json"),
|
|
12255
|
+
path28.join(homeDir, ".config", "opencode", "package.json"),
|
|
12256
|
+
path28.join(homeDir, ".npm", "global", "node_modules", "opencode", "package.json")
|
|
12059
12257
|
];
|
|
12060
12258
|
for (const packagePath of possiblePaths) {
|
|
12061
12259
|
try {
|
|
12062
|
-
const content = await
|
|
12260
|
+
const content = await fs12.readFile(packagePath, "utf-8");
|
|
12063
12261
|
const pkg = JSON.parse(content);
|
|
12064
12262
|
if (pkg.version) {
|
|
12065
12263
|
return pkg.version;
|
|
@@ -12097,12 +12295,12 @@ async function detectOpenCodeVersion() {
|
|
|
12097
12295
|
// src/app/cli/opencode/install-tool.ts
|
|
12098
12296
|
async function installTool(options = {}) {
|
|
12099
12297
|
const { logger, checkForOldSkill = true } = options;
|
|
12100
|
-
const
|
|
12101
|
-
const
|
|
12102
|
-
const
|
|
12103
|
-
const homeDir =
|
|
12104
|
-
const toolDir =
|
|
12105
|
-
const toolPath =
|
|
12298
|
+
const os3 = await import("os");
|
|
12299
|
+
const fs12 = await import("fs/promises");
|
|
12300
|
+
const path28 = await import("path");
|
|
12301
|
+
const homeDir = os3.homedir();
|
|
12302
|
+
const toolDir = path28.join(homeDir, ".config", "opencode", "tool");
|
|
12303
|
+
const toolPath = path28.join(toolDir, "raggrep.ts");
|
|
12106
12304
|
let removedOldSkill = false;
|
|
12107
12305
|
const toolContent = `import { tool } from "@opencode-ai/plugin";
|
|
12108
12306
|
|
|
@@ -12185,11 +12383,11 @@ Please install raggrep globally:
|
|
|
12185
12383
|
`;
|
|
12186
12384
|
try {
|
|
12187
12385
|
if (checkForOldSkill) {
|
|
12188
|
-
const oldSkillDir =
|
|
12189
|
-
const oldSkillPath =
|
|
12386
|
+
const oldSkillDir = path28.join(homeDir, ".config", "opencode", "skill", "raggrep");
|
|
12387
|
+
const oldSkillPath = path28.join(oldSkillDir, "SKILL.md");
|
|
12190
12388
|
let oldSkillExists = false;
|
|
12191
12389
|
try {
|
|
12192
|
-
await
|
|
12390
|
+
await fs12.access(oldSkillPath);
|
|
12193
12391
|
oldSkillExists = true;
|
|
12194
12392
|
} catch {}
|
|
12195
12393
|
if (oldSkillExists) {
|
|
@@ -12214,11 +12412,11 @@ Please install raggrep globally:
|
|
|
12214
12412
|
const shouldDelete = answer.toLowerCase() !== "n";
|
|
12215
12413
|
if (shouldDelete) {
|
|
12216
12414
|
try {
|
|
12217
|
-
await
|
|
12218
|
-
const skillDirContents = await
|
|
12415
|
+
await fs12.unlink(oldSkillPath);
|
|
12416
|
+
const skillDirContents = await fs12.readdir(oldSkillDir);
|
|
12219
12417
|
if (skillDirContents.length === 0) {
|
|
12220
12418
|
try {
|
|
12221
|
-
await
|
|
12419
|
+
await fs12.rmdir(oldSkillDir);
|
|
12222
12420
|
console.log("✓ Removed old skill directory.");
|
|
12223
12421
|
} catch (rmdirError) {
|
|
12224
12422
|
console.log("✓ Removed old skill file. (Directory not empty or other error)");
|
|
@@ -12255,8 +12453,8 @@ Please install raggrep globally:
|
|
|
12255
12453
|
}
|
|
12256
12454
|
}
|
|
12257
12455
|
}
|
|
12258
|
-
await
|
|
12259
|
-
await
|
|
12456
|
+
await fs12.mkdir(toolDir, { recursive: true });
|
|
12457
|
+
await fs12.writeFile(toolPath, toolContent, "utf-8");
|
|
12260
12458
|
const message = `Installed raggrep tool for OpenCode.
|
|
12261
12459
|
Location: ${toolPath}
|
|
12262
12460
|
|
|
@@ -12289,12 +12487,12 @@ The raggrep tool is now available in OpenCode.`;
|
|
|
12289
12487
|
// src/app/cli/opencode/install-skill.ts
|
|
12290
12488
|
async function installSkill(options = {}) {
|
|
12291
12489
|
const { logger, checkForOldTool = true } = options;
|
|
12292
|
-
const
|
|
12293
|
-
const
|
|
12294
|
-
const
|
|
12295
|
-
const homeDir =
|
|
12296
|
-
const skillDir =
|
|
12297
|
-
const skillPath =
|
|
12490
|
+
const os3 = await import("os");
|
|
12491
|
+
const fs12 = await import("fs/promises");
|
|
12492
|
+
const path28 = await import("path");
|
|
12493
|
+
const homeDir = os3.homedir();
|
|
12494
|
+
const skillDir = path28.join(homeDir, ".config", "opencode", "skill", "raggrep");
|
|
12495
|
+
const skillPath = path28.join(skillDir, "SKILL.md");
|
|
12298
12496
|
const skillContent = `---
|
|
12299
12497
|
name: raggrep
|
|
12300
12498
|
description: AST-powered semantic code search that understands intent, not just text. Superior to grep/rg - finds functions, classes, and logic even when keywords differ. Saves 10x tool calls by searching the actual code structure.
|
|
@@ -12334,19 +12532,32 @@ pnpm add -g raggrep
|
|
|
12334
12532
|
\`\`\`
|
|
12335
12533
|
|
|
12336
12534
|
### Step 1: Index your codebase
|
|
12535
|
+
|
|
12536
|
+
Index data is written to **\`<project>/.raggrep/\`** inside the directory you target. Add \`.raggrep/\` to \`.gitignore\` if you do not want it in version control.
|
|
12537
|
+
|
|
12538
|
+
From the repository root:
|
|
12337
12539
|
\`\`\`bash
|
|
12338
|
-
# Navigate to your project directory and index it
|
|
12339
12540
|
cd /path/to/your/project
|
|
12340
12541
|
raggrep index
|
|
12341
12542
|
\`\`\`
|
|
12342
12543
|
|
|
12544
|
+
Or pass the project path explicitly (no \`cd\` needed; same flag as \`git -C\`):
|
|
12545
|
+
\`\`\`bash
|
|
12546
|
+
raggrep index --dir /path/to/your/project
|
|
12547
|
+
# short form:
|
|
12548
|
+
raggrep index -C ~/projects/my-app
|
|
12549
|
+
\`\`\`
|
|
12550
|
+
|
|
12343
12551
|
### Step 2: Use semantic search
|
|
12552
|
+
|
|
12344
12553
|
\`\`\`bash
|
|
12345
|
-
# Search for specific functionality
|
|
12554
|
+
# Search for specific functionality (from project root, or use --dir / -C)
|
|
12346
12555
|
raggrep query "user authentication"
|
|
12556
|
+
raggrep query -C /path/to/your/project "user authentication"
|
|
12347
12557
|
|
|
12348
12558
|
# Search with filters
|
|
12349
12559
|
raggrep query "React hooks for data fetching" --filter "src/components"
|
|
12560
|
+
raggrep query -C /path/to/your/project "React hooks" --filter "src/components"
|
|
12350
12561
|
|
|
12351
12562
|
# Search with specific file types
|
|
12352
12563
|
raggrep query "database connection" --type ts
|
|
@@ -12355,6 +12566,8 @@ raggrep query "database connection" --type ts
|
|
|
12355
12566
|
raggrep query "error handling" --top 15
|
|
12356
12567
|
\`\`\`
|
|
12357
12568
|
|
|
12569
|
+
Use **\`-C\` / \`--dir\`** on \`raggrep query\`, \`raggrep index\`, \`raggrep status\`, and \`raggrep reset\` when the shell is not already in the repository root.
|
|
12570
|
+
|
|
12358
12571
|
### Step 3: Use in OpenCode agents
|
|
12359
12572
|
|
|
12360
12573
|
Load this skill in your agent conversation:
|
|
@@ -12390,6 +12603,7 @@ Instead of using grep/rg or manually reading files:
|
|
|
12390
12603
|
|
|
12391
12604
|
✅ **DO this instead:**
|
|
12392
12605
|
- \`raggrep query "Find the auth middleware"\` (finds ALL auth-related code)
|
|
12606
|
+
- \`raggrep query -C /path/to/repo "Find the auth middleware"\` (when not \`cd\`'d into the repo)
|
|
12393
12607
|
- \`raggrep query "Where are React components?"\`
|
|
12394
12608
|
- \`raggrep query "Database connection logic?"\`
|
|
12395
12609
|
- \`raggrep query "Error handling patterns"\`
|
|
@@ -12400,17 +12614,18 @@ Instead of using grep/rg or manually reading files:
|
|
|
12400
12614
|
2. **Use filters strategically**: \`--filter "src/auth"\`, \`--filter "*.test.ts"\`
|
|
12401
12615
|
3. **Adjust result count**: Use \`--top 5\` for focused results, \`--top 20\` for comprehensive search
|
|
12402
12616
|
4. **Replace grep/rg habits**: Instead of \`rg "pattern"\`, try \`raggrep query "what the code does"\`
|
|
12617
|
+
5. **Target the right tree**: If your shell is not in the repo root, use \`-C /path/to/repo\` or \`--dir\` on every command so the index under that repo's \`.raggrep/\` stays aligned with searches
|
|
12403
12618
|
|
|
12404
12619
|
**Result**: 10x fewer tool calls, BETTER results, deeper code understanding.
|
|
12405
12620
|
`;
|
|
12406
12621
|
let removedOldTool = false;
|
|
12407
12622
|
try {
|
|
12408
12623
|
if (checkForOldTool) {
|
|
12409
|
-
const oldToolDir =
|
|
12410
|
-
const oldToolPath =
|
|
12624
|
+
const oldToolDir = path28.join(homeDir, ".config", "opencode", "tool");
|
|
12625
|
+
const oldToolPath = path28.join(oldToolDir, "raggrep.ts");
|
|
12411
12626
|
let oldToolExists = false;
|
|
12412
12627
|
try {
|
|
12413
|
-
await
|
|
12628
|
+
await fs12.access(oldToolPath);
|
|
12414
12629
|
oldToolExists = true;
|
|
12415
12630
|
} catch {}
|
|
12416
12631
|
if (oldToolExists) {
|
|
@@ -12435,11 +12650,11 @@ Instead of using grep/rg or manually reading files:
|
|
|
12435
12650
|
const shouldDelete = answer.toLowerCase() !== "n";
|
|
12436
12651
|
if (shouldDelete) {
|
|
12437
12652
|
try {
|
|
12438
|
-
await
|
|
12439
|
-
const toolDirContents = await
|
|
12653
|
+
await fs12.unlink(oldToolPath);
|
|
12654
|
+
const toolDirContents = await fs12.readdir(oldToolDir);
|
|
12440
12655
|
if (toolDirContents.length === 0) {
|
|
12441
12656
|
try {
|
|
12442
|
-
await
|
|
12657
|
+
await fs12.rmdir(oldToolDir);
|
|
12443
12658
|
console.log("✓ Removed old tool directory.");
|
|
12444
12659
|
} catch (rmdirError) {
|
|
12445
12660
|
console.log("✓ Removed old tool file. (Directory not empty or other error)");
|
|
@@ -12472,8 +12687,8 @@ Instead of using grep/rg or manually reading files:
|
|
|
12472
12687
|
}
|
|
12473
12688
|
}
|
|
12474
12689
|
}
|
|
12475
|
-
await
|
|
12476
|
-
await
|
|
12690
|
+
await fs12.mkdir(skillDir, { recursive: true });
|
|
12691
|
+
await fs12.writeFile(skillPath, skillContent, "utf-8");
|
|
12477
12692
|
const message = `Installed raggrep skill for OpenCode.
|
|
12478
12693
|
Location: ${skillPath}
|
|
12479
12694
|
|
|
@@ -12481,7 +12696,7 @@ The raggrep skill is now available to OpenCode agents.
|
|
|
12481
12696
|
|
|
12482
12697
|
To use this skill:
|
|
12483
12698
|
1. Install raggrep: npm install -g raggrep
|
|
12484
|
-
2. Index your codebase: raggrep index
|
|
12699
|
+
2. Index your codebase: \`cd\` to the repo and \`raggrep index\`, or \`raggrep index --dir /path/to/repo\`
|
|
12485
12700
|
3. In OpenCode, load the skill: skill({ name: "raggrep" })`;
|
|
12486
12701
|
if (logger) {
|
|
12487
12702
|
logger.info(message);
|
|
@@ -12523,10 +12738,12 @@ var init_opencode = () => {};
|
|
|
12523
12738
|
// src/app/cli/main.ts
|
|
12524
12739
|
init_embeddings();
|
|
12525
12740
|
init_logger();
|
|
12741
|
+
import * as path28 from "path";
|
|
12742
|
+
import { stat as stat3 } from "fs/promises";
|
|
12526
12743
|
// package.json
|
|
12527
12744
|
var package_default = {
|
|
12528
12745
|
name: "raggrep",
|
|
12529
|
-
version: "0.
|
|
12746
|
+
version: "0.17.1",
|
|
12530
12747
|
description: "Local filesystem-based RAG system for codebases - semantic search using local embeddings",
|
|
12531
12748
|
type: "module",
|
|
12532
12749
|
main: "./dist/index.js",
|
|
@@ -12548,12 +12765,15 @@ var package_default = {
|
|
|
12548
12765
|
scripts: {
|
|
12549
12766
|
build: "bun run build:clean && bun run build:bundle && bun run build:types && bun run build:shebang",
|
|
12550
12767
|
"build:clean": "rm -rf dist",
|
|
12551
|
-
"build:bundle": "bun build src/index.ts --outdir dist --target node --sourcemap=external --external '@xenova/transformers' --external 'glob' --external 'typescript' --external 'chokidar' --external 'web-tree-sitter' && bun build src/app/cli/main.ts --outdir dist/cli --target node --sourcemap=external --external '@xenova/transformers' --external 'glob' --external 'typescript' --external 'chokidar' --external 'web-tree-sitter'",
|
|
12552
|
-
"build:types": "tsc --emitDeclarationOnly --outDir dist",
|
|
12768
|
+
"build:bundle": "bun build src/index.ts --outdir dist --target node --sourcemap=external --external '@xenova/transformers' --external '@huggingface/transformers' --external 'glob' --external 'typescript' --external 'chokidar' --external 'web-tree-sitter' && bun build src/app/cli/main.ts --outdir dist/cli --target node --sourcemap=external --external '@xenova/transformers' --external '@huggingface/transformers' --external 'glob' --external 'typescript' --external 'chokidar' --external 'web-tree-sitter'",
|
|
12769
|
+
"build:types": "tsc --project tsconfig.json --noEmit false --declaration true --emitDeclarationOnly true --rootDir ./src --outDir ./dist",
|
|
12553
12770
|
"build:shebang": "echo '#!/usr/bin/env node' | cat - dist/cli/main.js > temp && mv temp dist/cli/main.js && chmod +x dist/cli/main.js",
|
|
12554
12771
|
prepublishOnly: "bun run build",
|
|
12555
12772
|
raggrep: "bun run src/app/cli/main.ts",
|
|
12556
12773
|
test: "bun test",
|
|
12774
|
+
typecheck: "tsc --noEmit -p tsconfig.json && tsc --noEmit -p scripts/tsconfig.json",
|
|
12775
|
+
"bench:embeddings": "bun run scripts/benchmark-embedding-runtimes.ts",
|
|
12776
|
+
"bench:retrieval": "bun run scripts/benchmark-retrieval-quality.ts",
|
|
12557
12777
|
dev: "bun run src/app/cli/main.ts"
|
|
12558
12778
|
},
|
|
12559
12779
|
keywords: [
|
|
@@ -12581,10 +12801,11 @@ var package_default = {
|
|
|
12581
12801
|
node: ">=18.0.0"
|
|
12582
12802
|
},
|
|
12583
12803
|
dependencies: {
|
|
12804
|
+
"@huggingface/transformers": "^4.0.0",
|
|
12584
12805
|
"@xenova/transformers": "^2.17.0",
|
|
12585
12806
|
chokidar: "^5.0.0",
|
|
12586
12807
|
fdir: "^6.5.0",
|
|
12587
|
-
glob: "^
|
|
12808
|
+
glob: "^11.0.0",
|
|
12588
12809
|
minimatch: "^10.1.1",
|
|
12589
12810
|
typescript: "^5.0.0",
|
|
12590
12811
|
"web-tree-sitter": "^0.26.3"
|
|
@@ -12592,6 +12813,10 @@ var package_default = {
|
|
|
12592
12813
|
devDependencies: {
|
|
12593
12814
|
"@types/bun": "latest",
|
|
12594
12815
|
"@types/node": "^20.0.0"
|
|
12816
|
+
},
|
|
12817
|
+
overrides: {
|
|
12818
|
+
sharp: "^0.34.5",
|
|
12819
|
+
"global-agent": "^4.1.3"
|
|
12595
12820
|
}
|
|
12596
12821
|
};
|
|
12597
12822
|
|
|
@@ -12689,6 +12914,14 @@ function parseFlags(args2) {
|
|
|
12689
12914
|
console.error('--filter requires a path or glob pattern (e.g., src/auth, "*.ts")');
|
|
12690
12915
|
process.exit(1);
|
|
12691
12916
|
}
|
|
12917
|
+
} else if (arg === "--dir" || arg === "-C") {
|
|
12918
|
+
const dir = args2[++i];
|
|
12919
|
+
if (dir) {
|
|
12920
|
+
flags.projectDir = dir;
|
|
12921
|
+
} else {
|
|
12922
|
+
console.error("--dir / -C requires a path to the project directory to index or search.");
|
|
12923
|
+
process.exit(1);
|
|
12924
|
+
}
|
|
12692
12925
|
} else if (arg === "--tool") {
|
|
12693
12926
|
flags.forceTool = true;
|
|
12694
12927
|
} else if (arg === "--skill") {
|
|
@@ -12699,6 +12932,20 @@ function parseFlags(args2) {
|
|
|
12699
12932
|
}
|
|
12700
12933
|
return flags;
|
|
12701
12934
|
}
|
|
12935
|
+
async function resolveProjectRoot(flags) {
|
|
12936
|
+
const root = path28.resolve(flags.projectDir ?? process.cwd());
|
|
12937
|
+
try {
|
|
12938
|
+
const st = await stat3(root);
|
|
12939
|
+
if (!st.isDirectory()) {
|
|
12940
|
+
console.error(`Error: not a directory: ${root}`);
|
|
12941
|
+
process.exit(1);
|
|
12942
|
+
}
|
|
12943
|
+
} catch {
|
|
12944
|
+
console.error(`Error: directory does not exist or is not accessible: ${root}`);
|
|
12945
|
+
process.exit(1);
|
|
12946
|
+
}
|
|
12947
|
+
return root;
|
|
12948
|
+
}
|
|
12702
12949
|
async function main() {
|
|
12703
12950
|
const flags = parseFlags(args.slice(1));
|
|
12704
12951
|
switch (command) {
|
|
@@ -12706,14 +12953,15 @@ async function main() {
|
|
|
12706
12953
|
if (flags.help) {
|
|
12707
12954
|
const models = Object.keys(EMBEDDING_MODELS).join(", ");
|
|
12708
12955
|
console.log(`
|
|
12709
|
-
raggrep index - Index
|
|
12956
|
+
raggrep index - Index a directory for semantic search
|
|
12710
12957
|
|
|
12711
12958
|
Usage:
|
|
12712
12959
|
raggrep index [options]
|
|
12713
12960
|
|
|
12714
12961
|
Options:
|
|
12962
|
+
-C, --dir <path> Project directory to index (default: current directory)
|
|
12715
12963
|
-w, --watch Watch for file changes and re-index automatically
|
|
12716
|
-
-m, --model <name>
|
|
12964
|
+
-m, --model <name> Override embedding model for the TypeScript module (default: saved config, else bge-small-en-v1.5)
|
|
12717
12965
|
-c, --concurrency <n> Number of files to process in parallel (default: auto)
|
|
12718
12966
|
-v, --verbose Show detailed progress
|
|
12719
12967
|
-h, --help Show this help message
|
|
@@ -12725,6 +12973,7 @@ Model Cache: ${getCacheDir()}
|
|
|
12725
12973
|
|
|
12726
12974
|
Examples:
|
|
12727
12975
|
raggrep index
|
|
12976
|
+
raggrep index --dir ../other-repo
|
|
12728
12977
|
raggrep index --watch
|
|
12729
12978
|
raggrep index --model bge-small-en-v1.5
|
|
12730
12979
|
raggrep index --concurrency 8
|
|
@@ -12734,11 +12983,12 @@ Examples:
|
|
|
12734
12983
|
}
|
|
12735
12984
|
const { indexDirectory: indexDirectory3, watchDirectory: watchDirectory2 } = await Promise.resolve().then(() => (init_indexer(), exports_indexer));
|
|
12736
12985
|
const logger = createInlineLogger({ verbose: flags.verbose });
|
|
12986
|
+
const projectRoot = await resolveProjectRoot(flags);
|
|
12737
12987
|
console.log("RAGgrep Indexer");
|
|
12738
12988
|
console.log(`================
|
|
12739
12989
|
`);
|
|
12740
12990
|
try {
|
|
12741
|
-
const results = await indexDirectory3(
|
|
12991
|
+
const results = await indexDirectory3(projectRoot, {
|
|
12742
12992
|
model: flags.model,
|
|
12743
12993
|
verbose: flags.verbose,
|
|
12744
12994
|
concurrency: flags.concurrency,
|
|
@@ -12761,7 +13011,7 @@ Examples:
|
|
|
12761
13011
|
console.log(`└─────────────────────────────────────────┘
|
|
12762
13012
|
`);
|
|
12763
13013
|
try {
|
|
12764
|
-
const watcher = await watchDirectory2(
|
|
13014
|
+
const watcher = await watchDirectory2(projectRoot, {
|
|
12765
13015
|
model: flags.model,
|
|
12766
13016
|
verbose: flags.verbose,
|
|
12767
13017
|
onFileChange: (event, filepath) => {
|
|
@@ -12798,6 +13048,7 @@ Usage:
|
|
|
12798
13048
|
raggrep query <search query> [options]
|
|
12799
13049
|
|
|
12800
13050
|
Options:
|
|
13051
|
+
-C, --dir <path> Project directory to search (default: current directory)
|
|
12801
13052
|
-k, --top <n> Number of results to return (default: 10)
|
|
12802
13053
|
-s, --min-score <n> Minimum similarity score 0-1 (default: 0.15)
|
|
12803
13054
|
-t, --type <ext> Filter by file extension (e.g., ts, tsx, js)
|
|
@@ -12825,6 +13076,7 @@ Multiple Filters (OR logic):
|
|
|
12825
13076
|
|
|
12826
13077
|
Examples:
|
|
12827
13078
|
raggrep query "user authentication"
|
|
13079
|
+
raggrep query --dir ~/projects/my-app "user authentication"
|
|
12828
13080
|
raggrep query "handle errors" --top 5
|
|
12829
13081
|
raggrep query "database" --min-score 0.1
|
|
12830
13082
|
raggrep query "interface" --type ts
|
|
@@ -12850,9 +13102,10 @@ Examples:
|
|
|
12850
13102
|
console.error('Run "raggrep query --help" for more information.');
|
|
12851
13103
|
process.exit(1);
|
|
12852
13104
|
}
|
|
13105
|
+
const projectRoot = await resolveProjectRoot(flags);
|
|
12853
13106
|
try {
|
|
12854
13107
|
const silentLogger = createSilentLogger();
|
|
12855
|
-
const freshStats = await ensureIndexFresh2(
|
|
13108
|
+
const freshStats = await ensureIndexFresh2(projectRoot, {
|
|
12856
13109
|
model: flags.model,
|
|
12857
13110
|
quiet: true,
|
|
12858
13111
|
logger: silentLogger,
|
|
@@ -12893,7 +13146,7 @@ Examples:
|
|
|
12893
13146
|
}
|
|
12894
13147
|
const filePatterns = flags.fileType ? [`*.${flags.fileType}`] : undefined;
|
|
12895
13148
|
const { hybridSearch: hybridSearch2, formatHybridSearchResults: formatHybridSearchResults2 } = await Promise.resolve().then(() => (init_search(), exports_search));
|
|
12896
|
-
const hybridResults = await hybridSearch2(
|
|
13149
|
+
const hybridResults = await hybridSearch2(projectRoot, query, {
|
|
12897
13150
|
topK: flags.topK ?? 10,
|
|
12898
13151
|
minScore: flags.minScore,
|
|
12899
13152
|
filePatterns,
|
|
@@ -12910,26 +13163,29 @@ Examples:
|
|
|
12910
13163
|
case "reset": {
|
|
12911
13164
|
if (flags.help) {
|
|
12912
13165
|
console.log(`
|
|
12913
|
-
raggrep reset - Clear the index for
|
|
13166
|
+
raggrep reset - Clear the index for a project directory
|
|
12914
13167
|
|
|
12915
13168
|
Usage:
|
|
12916
13169
|
raggrep reset [options]
|
|
12917
13170
|
|
|
12918
13171
|
Options:
|
|
13172
|
+
-C, --dir <path> Project directory whose index to remove (default: current directory)
|
|
12919
13173
|
-h, --help Show this help message
|
|
12920
13174
|
|
|
12921
13175
|
Description:
|
|
12922
|
-
Completely removes the index for the
|
|
13176
|
+
Completely removes the .raggrep index for the project directory.
|
|
12923
13177
|
The next 'raggrep index' or 'raggrep query' will rebuild from scratch.
|
|
12924
13178
|
|
|
12925
13179
|
Examples:
|
|
12926
13180
|
raggrep reset
|
|
13181
|
+
raggrep reset --dir ../other-repo
|
|
12927
13182
|
`);
|
|
12928
13183
|
process.exit(0);
|
|
12929
13184
|
}
|
|
12930
13185
|
const { resetIndex: resetIndex2 } = await Promise.resolve().then(() => (init_indexer(), exports_indexer));
|
|
13186
|
+
const projectRoot = await resolveProjectRoot(flags);
|
|
12931
13187
|
try {
|
|
12932
|
-
const result = await resetIndex2(
|
|
13188
|
+
const result = await resetIndex2(projectRoot);
|
|
12933
13189
|
console.log("Index cleared successfully.");
|
|
12934
13190
|
console.log(` Removed: ${result.indexDir}`);
|
|
12935
13191
|
} catch (error) {
|
|
@@ -12951,21 +13207,24 @@ Usage:
|
|
|
12951
13207
|
raggrep status [options]
|
|
12952
13208
|
|
|
12953
13209
|
Options:
|
|
13210
|
+
-C, --dir <path> Project directory to report on (default: current directory)
|
|
12954
13211
|
-h, --help Show this help message
|
|
12955
13212
|
|
|
12956
13213
|
Description:
|
|
12957
|
-
Displays information about the index
|
|
13214
|
+
Displays information about the index for the project directory,
|
|
12958
13215
|
including whether it exists, how many files are indexed, and
|
|
12959
13216
|
when it was last updated.
|
|
12960
13217
|
|
|
12961
13218
|
Examples:
|
|
12962
13219
|
raggrep status
|
|
13220
|
+
raggrep status --dir ../other-repo
|
|
12963
13221
|
`);
|
|
12964
13222
|
process.exit(0);
|
|
12965
13223
|
}
|
|
12966
13224
|
const { getIndexStatus: getIndexStatus2 } = await Promise.resolve().then(() => (init_indexer(), exports_indexer));
|
|
13225
|
+
const projectRoot = await resolveProjectRoot(flags);
|
|
12967
13226
|
try {
|
|
12968
|
-
const status = await getIndexStatus2(
|
|
13227
|
+
const status = await getIndexStatus2(projectRoot);
|
|
12969
13228
|
if (!status.exists) {
|
|
12970
13229
|
console.log(`
|
|
12971
13230
|
┌─────────────────────────────────────────┐
|
|
@@ -13096,19 +13355,23 @@ Usage:
|
|
|
13096
13355
|
raggrep <command> [options]
|
|
13097
13356
|
|
|
13098
13357
|
Commands:
|
|
13099
|
-
index Index
|
|
13358
|
+
index Index a project directory
|
|
13100
13359
|
query Search the indexed codebase
|
|
13101
13360
|
status Show the current state of the index
|
|
13102
|
-
reset Clear the index for
|
|
13361
|
+
reset Clear the index for a project directory
|
|
13103
13362
|
opencode Manage opencode integration
|
|
13104
13363
|
|
|
13105
13364
|
Options:
|
|
13106
13365
|
-h, --help Show help for a command
|
|
13107
13366
|
-v, --version Show version number
|
|
13108
13367
|
|
|
13368
|
+
Project directory (index, query, status, reset):
|
|
13369
|
+
Use -C or --dir <path> to target a directory other than the current one.
|
|
13370
|
+
|
|
13109
13371
|
Examples:
|
|
13110
13372
|
raggrep index
|
|
13111
13373
|
raggrep query "user login"
|
|
13374
|
+
raggrep query -C ~/projects/app "user login"
|
|
13112
13375
|
raggrep status
|
|
13113
13376
|
raggrep reset
|
|
13114
13377
|
raggrep opencode install
|
|
@@ -13123,4 +13386,4 @@ Run 'raggrep <command> --help' for more information.
|
|
|
13123
13386
|
}
|
|
13124
13387
|
main();
|
|
13125
13388
|
|
|
13126
|
-
//# debugId=
|
|
13389
|
+
//# debugId=1CBA39A0838887E764756E2164756E21
|