raggrep 0.15.0 → 0.17.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +112 -8
- package/dist/cli/main.js +827 -521
- package/dist/cli/main.js.map +26 -19
- package/dist/domain/ports/embedding.d.ts +10 -0
- package/dist/domain/ports/index.d.ts +1 -1
- package/dist/domain/services/chunkContext.d.ts +76 -0
- package/dist/domain/services/index.d.ts +1 -0
- package/dist/index.js +638 -390
- package/dist/index.js.map +25 -18
- package/dist/infrastructure/config/configLoader.d.ts +9 -11
- package/dist/infrastructure/config/index.d.ts +1 -1
- package/dist/infrastructure/embeddings/embeddingPaths.d.ts +6 -0
- package/dist/infrastructure/embeddings/embeddingProviderFactory.d.ts +9 -0
- package/dist/infrastructure/embeddings/globalEmbeddings.d.ts +28 -0
- package/dist/infrastructure/embeddings/huggingfaceEmbeddingProvider.d.ts +21 -0
- package/dist/infrastructure/embeddings/index.d.ts +9 -2
- package/dist/infrastructure/embeddings/modelCache.d.ts +10 -0
- package/dist/infrastructure/embeddings/modelCatalog.d.ts +23 -0
- package/dist/infrastructure/embeddings/xenovaEmbeddingProvider.d.ts +23 -0
- package/dist/infrastructure/index.d.ts +1 -1
- package/package.json +7 -3
- package/dist/infrastructure/embeddings/transformersEmbedding.d.ts +0 -52
package/dist/cli/main.js
CHANGED
|
@@ -1,49 +1,89 @@
|
|
|
1
1
|
#!/usr/bin/env node
|
|
2
2
|
import { createRequire } from "node:module";
|
|
3
|
-
var __create = Object.create;
|
|
4
|
-
var __getProtoOf = Object.getPrototypeOf;
|
|
5
3
|
var __defProp = Object.defineProperty;
|
|
6
|
-
var
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
const to = isNodeMode || !mod || !mod.__esModule ? __defProp(target, "default", { value: mod, enumerable: true }) : target;
|
|
11
|
-
for (let key of __getOwnPropNames(mod))
|
|
12
|
-
if (!__hasOwnProp.call(to, key))
|
|
13
|
-
__defProp(to, key, {
|
|
14
|
-
get: () => mod[key],
|
|
15
|
-
enumerable: true
|
|
16
|
-
});
|
|
17
|
-
return to;
|
|
18
|
-
};
|
|
4
|
+
var __returnValue = (v) => v;
|
|
5
|
+
function __exportSetter(name, newValue) {
|
|
6
|
+
this[name] = __returnValue.bind(null, newValue);
|
|
7
|
+
}
|
|
19
8
|
var __export = (target, all) => {
|
|
20
9
|
for (var name in all)
|
|
21
10
|
__defProp(target, name, {
|
|
22
11
|
get: all[name],
|
|
23
12
|
enumerable: true,
|
|
24
13
|
configurable: true,
|
|
25
|
-
set: (
|
|
14
|
+
set: __exportSetter.bind(all, name)
|
|
26
15
|
});
|
|
27
16
|
};
|
|
28
17
|
var __esm = (fn, res) => () => (fn && (res = fn(fn = 0)), res);
|
|
29
18
|
var __require = /* @__PURE__ */ createRequire(import.meta.url);
|
|
30
19
|
|
|
31
|
-
// src/infrastructure/embeddings/
|
|
20
|
+
// src/infrastructure/embeddings/modelCatalog.ts
|
|
21
|
+
function getEmbeddingModelId(model) {
|
|
22
|
+
return EMBEDDING_MODEL_IDS[model];
|
|
23
|
+
}
|
|
24
|
+
function getEmbeddingDimension(model) {
|
|
25
|
+
return EMBEDDING_DIMENSIONS[model];
|
|
26
|
+
}
|
|
27
|
+
var EMBEDDING_MODEL_IDS, EMBEDDING_MODELS, EMBEDDING_DIMENSIONS;
|
|
28
|
+
var init_modelCatalog = __esm(() => {
|
|
29
|
+
EMBEDDING_MODEL_IDS = {
|
|
30
|
+
"all-MiniLM-L6-v2": "Xenova/all-MiniLM-L6-v2",
|
|
31
|
+
"all-MiniLM-L12-v2": "Xenova/all-MiniLM-L12-v2",
|
|
32
|
+
"bge-small-en-v1.5": "Xenova/bge-small-en-v1.5",
|
|
33
|
+
"paraphrase-MiniLM-L3-v2": "Xenova/paraphrase-MiniLM-L3-v2",
|
|
34
|
+
"nomic-embed-text-v1.5": "nomic-ai/nomic-embed-text-v1.5"
|
|
35
|
+
};
|
|
36
|
+
EMBEDDING_MODELS = EMBEDDING_MODEL_IDS;
|
|
37
|
+
EMBEDDING_DIMENSIONS = {
|
|
38
|
+
"all-MiniLM-L6-v2": 384,
|
|
39
|
+
"all-MiniLM-L12-v2": 384,
|
|
40
|
+
"bge-small-en-v1.5": 384,
|
|
41
|
+
"paraphrase-MiniLM-L3-v2": 384,
|
|
42
|
+
"nomic-embed-text-v1.5": 768
|
|
43
|
+
};
|
|
44
|
+
});
|
|
45
|
+
|
|
46
|
+
// src/infrastructure/embeddings/embeddingPaths.ts
|
|
47
|
+
import * as os from "os";
|
|
48
|
+
import * as path from "path";
|
|
49
|
+
var RAGGREP_MODEL_CACHE_DIR;
|
|
50
|
+
var init_embeddingPaths = __esm(() => {
|
|
51
|
+
RAGGREP_MODEL_CACHE_DIR = path.join(os.homedir(), ".cache", "raggrep", "models");
|
|
52
|
+
});
|
|
53
|
+
|
|
54
|
+
// src/infrastructure/embeddings/modelCache.ts
|
|
55
|
+
import * as fs from "fs/promises";
|
|
56
|
+
import * as path2 from "path";
|
|
57
|
+
async function isEmbeddingModelCached(model) {
|
|
58
|
+
const modelId = getEmbeddingModelId(model);
|
|
59
|
+
const onnxPath = path2.join(RAGGREP_MODEL_CACHE_DIR, modelId, "onnx", "model_quantized.onnx");
|
|
60
|
+
try {
|
|
61
|
+
await fs.access(onnxPath);
|
|
62
|
+
return true;
|
|
63
|
+
} catch {
|
|
64
|
+
return false;
|
|
65
|
+
}
|
|
66
|
+
}
|
|
67
|
+
var init_modelCache = __esm(() => {
|
|
68
|
+
init_embeddingPaths();
|
|
69
|
+
init_modelCatalog();
|
|
70
|
+
});
|
|
71
|
+
|
|
72
|
+
// src/infrastructure/embeddings/xenovaEmbeddingProvider.ts
|
|
32
73
|
import {
|
|
33
74
|
pipeline,
|
|
34
75
|
env
|
|
35
76
|
} from "@xenova/transformers";
|
|
36
|
-
import * as path from "path";
|
|
37
|
-
import * as os from "os";
|
|
38
77
|
|
|
39
|
-
class
|
|
40
|
-
|
|
78
|
+
class XenovaTransformersEmbeddingProvider {
|
|
79
|
+
extractor = null;
|
|
41
80
|
config;
|
|
42
81
|
isInitializing = false;
|
|
43
82
|
initPromise = null;
|
|
44
83
|
constructor(config) {
|
|
45
84
|
this.config = {
|
|
46
85
|
model: config?.model ?? "bge-small-en-v1.5",
|
|
86
|
+
runtime: config?.runtime ?? "xenova",
|
|
47
87
|
showProgress: config?.showProgress ?? false,
|
|
48
88
|
logger: config?.logger
|
|
49
89
|
};
|
|
@@ -51,14 +91,14 @@ class TransformersEmbeddingProvider {
|
|
|
51
91
|
async initialize(config) {
|
|
52
92
|
if (config) {
|
|
53
93
|
if (config.model !== this.config.model) {
|
|
54
|
-
this.
|
|
94
|
+
this.extractor = null;
|
|
55
95
|
}
|
|
56
96
|
this.config = { ...this.config, ...config };
|
|
57
97
|
}
|
|
58
|
-
await this.
|
|
98
|
+
await this.ensureExtractor();
|
|
59
99
|
}
|
|
60
|
-
async
|
|
61
|
-
if (this.
|
|
100
|
+
async ensureExtractor() {
|
|
101
|
+
if (this.extractor) {
|
|
62
102
|
return;
|
|
63
103
|
}
|
|
64
104
|
if (this.isInitializing && this.initPromise) {
|
|
@@ -66,14 +106,14 @@ class TransformersEmbeddingProvider {
|
|
|
66
106
|
}
|
|
67
107
|
this.isInitializing = true;
|
|
68
108
|
this.initPromise = (async () => {
|
|
69
|
-
const modelId =
|
|
109
|
+
const modelId = getEmbeddingModelId(this.config.model);
|
|
70
110
|
const logger = this.config.logger;
|
|
71
111
|
const showProgress = this.config.showProgress || !!logger;
|
|
72
|
-
const
|
|
112
|
+
const cached = await isEmbeddingModelCached(this.config.model);
|
|
73
113
|
let hasDownloads = false;
|
|
74
114
|
try {
|
|
75
|
-
this.
|
|
76
|
-
progress_callback: showProgress && !
|
|
115
|
+
this.extractor = await pipeline("feature-extraction", modelId, {
|
|
116
|
+
progress_callback: showProgress && !cached ? (progress) => {
|
|
77
117
|
if (progress.status === "progress" && progress.file) {
|
|
78
118
|
if (!hasDownloads) {
|
|
79
119
|
hasDownloads = true;
|
|
@@ -82,7 +122,7 @@ class TransformersEmbeddingProvider {
|
|
|
82
122
|
} else {
|
|
83
123
|
console.log(`
|
|
84
124
|
Loading embedding model: ${this.config.model}`);
|
|
85
|
-
console.log(` Cache: ${
|
|
125
|
+
console.log(` Cache: ${RAGGREP_MODEL_CACHE_DIR}`);
|
|
86
126
|
}
|
|
87
127
|
}
|
|
88
128
|
const pct = progress.progress ? Math.round(progress.progress) : 0;
|
|
@@ -112,9 +152,9 @@ class TransformersEmbeddingProvider {
|
|
|
112
152
|
}
|
|
113
153
|
}
|
|
114
154
|
} catch (error) {
|
|
115
|
-
this.
|
|
116
|
-
if (logger) {
|
|
117
|
-
logger.clearProgress();
|
|
155
|
+
this.extractor = null;
|
|
156
|
+
if (this.config.logger) {
|
|
157
|
+
this.config.logger.clearProgress();
|
|
118
158
|
}
|
|
119
159
|
throw new Error(`Failed to load embedding model: ${error}`);
|
|
120
160
|
} finally {
|
|
@@ -125,11 +165,11 @@ class TransformersEmbeddingProvider {
|
|
|
125
165
|
return this.initPromise;
|
|
126
166
|
}
|
|
127
167
|
async getEmbedding(text) {
|
|
128
|
-
await this.
|
|
129
|
-
if (!this.
|
|
168
|
+
await this.ensureExtractor();
|
|
169
|
+
if (!this.extractor) {
|
|
130
170
|
throw new Error("Embedding pipeline not initialized");
|
|
131
171
|
}
|
|
132
|
-
const output = await this.
|
|
172
|
+
const output = await this.extractor(text, {
|
|
133
173
|
pooling: "mean",
|
|
134
174
|
normalize: true
|
|
135
175
|
});
|
|
@@ -138,15 +178,15 @@ class TransformersEmbeddingProvider {
|
|
|
138
178
|
async getEmbeddings(texts) {
|
|
139
179
|
if (texts.length === 0)
|
|
140
180
|
return [];
|
|
141
|
-
await this.
|
|
142
|
-
if (!this.
|
|
181
|
+
await this.ensureExtractor();
|
|
182
|
+
if (!this.extractor) {
|
|
143
183
|
throw new Error("Embedding pipeline not initialized");
|
|
144
184
|
}
|
|
145
185
|
const results = [];
|
|
146
186
|
for (let i = 0;i < texts.length; i += BATCH_SIZE) {
|
|
147
187
|
const batch = texts.slice(i, i + BATCH_SIZE);
|
|
148
188
|
const outputs = await Promise.all(batch.map(async (text) => {
|
|
149
|
-
const output = await this.
|
|
189
|
+
const output = await this.extractor(text, {
|
|
150
190
|
pooling: "mean",
|
|
151
191
|
normalize: true
|
|
152
192
|
});
|
|
@@ -157,44 +197,210 @@ class TransformersEmbeddingProvider {
|
|
|
157
197
|
return results;
|
|
158
198
|
}
|
|
159
199
|
getDimension() {
|
|
160
|
-
return
|
|
200
|
+
return getEmbeddingDimension(this.config.model);
|
|
161
201
|
}
|
|
162
202
|
getModelName() {
|
|
163
203
|
return this.config.model;
|
|
164
204
|
}
|
|
165
205
|
async dispose() {
|
|
166
|
-
this.
|
|
206
|
+
this.extractor = null;
|
|
167
207
|
}
|
|
168
208
|
}
|
|
169
|
-
|
|
170
|
-
|
|
209
|
+
var BATCH_SIZE = 32;
|
|
210
|
+
var init_xenovaEmbeddingProvider = __esm(() => {
|
|
211
|
+
init_embeddingPaths();
|
|
212
|
+
init_modelCatalog();
|
|
213
|
+
init_modelCache();
|
|
214
|
+
env.cacheDir = RAGGREP_MODEL_CACHE_DIR;
|
|
215
|
+
env.allowLocalModels = true;
|
|
216
|
+
});
|
|
217
|
+
|
|
218
|
+
// src/infrastructure/embeddings/huggingfaceEmbeddingProvider.ts
|
|
219
|
+
import {
|
|
220
|
+
pipeline as pipeline2,
|
|
221
|
+
env as env2
|
|
222
|
+
} from "@huggingface/transformers";
|
|
223
|
+
|
|
224
|
+
class HuggingFaceTransformersEmbeddingProvider {
|
|
225
|
+
extractor = null;
|
|
226
|
+
config;
|
|
227
|
+
isInitializing = false;
|
|
228
|
+
initPromise = null;
|
|
229
|
+
constructor(config) {
|
|
230
|
+
this.config = {
|
|
231
|
+
model: config?.model ?? "bge-small-en-v1.5",
|
|
232
|
+
runtime: config?.runtime ?? "huggingface",
|
|
233
|
+
showProgress: config?.showProgress ?? false,
|
|
234
|
+
logger: config?.logger
|
|
235
|
+
};
|
|
236
|
+
}
|
|
237
|
+
async initialize(config) {
|
|
238
|
+
if (config) {
|
|
239
|
+
if (config.model !== this.config.model) {
|
|
240
|
+
this.extractor = null;
|
|
241
|
+
}
|
|
242
|
+
this.config = { ...this.config, ...config };
|
|
243
|
+
}
|
|
244
|
+
await this.ensureExtractor();
|
|
245
|
+
}
|
|
246
|
+
async ensureExtractor() {
|
|
247
|
+
if (this.extractor) {
|
|
248
|
+
return;
|
|
249
|
+
}
|
|
250
|
+
if (this.isInitializing && this.initPromise) {
|
|
251
|
+
return this.initPromise;
|
|
252
|
+
}
|
|
253
|
+
this.isInitializing = true;
|
|
254
|
+
this.initPromise = (async () => {
|
|
255
|
+
const modelId = getEmbeddingModelId(this.config.model);
|
|
256
|
+
const logger = this.config.logger;
|
|
257
|
+
const showProgress = this.config.showProgress || !!logger;
|
|
258
|
+
const cached = await isEmbeddingModelCached(this.config.model);
|
|
259
|
+
let hasDownloads = false;
|
|
260
|
+
try {
|
|
261
|
+
this.extractor = await pipeline2("feature-extraction", modelId, {
|
|
262
|
+
progress_callback: showProgress && !cached ? (progress) => {
|
|
263
|
+
if (progress.status === "progress" && progress.file) {
|
|
264
|
+
if (!hasDownloads) {
|
|
265
|
+
hasDownloads = true;
|
|
266
|
+
if (logger) {
|
|
267
|
+
logger.info(`Downloading embedding model: ${this.config.model}`);
|
|
268
|
+
} else {
|
|
269
|
+
console.log(`
|
|
270
|
+
Loading embedding model: ${this.config.model}`);
|
|
271
|
+
console.log(` Cache: ${RAGGREP_MODEL_CACHE_DIR}`);
|
|
272
|
+
}
|
|
273
|
+
}
|
|
274
|
+
const pct = progress.progress ? Math.round(progress.progress) : 0;
|
|
275
|
+
if (logger) {
|
|
276
|
+
logger.progress(` Downloading ${progress.file}: ${pct}%`);
|
|
277
|
+
} else {
|
|
278
|
+
process.stdout.write(`\r Downloading ${progress.file}: ${pct}% `);
|
|
279
|
+
}
|
|
280
|
+
} else if (progress.status === "done" && progress.file) {
|
|
281
|
+
if (logger) {
|
|
282
|
+
logger.clearProgress();
|
|
283
|
+
logger.info(` Downloaded ${progress.file}`);
|
|
284
|
+
} else if (hasDownloads) {
|
|
285
|
+
process.stdout.write(`\r Downloaded ${progress.file}
|
|
286
|
+
`);
|
|
287
|
+
}
|
|
288
|
+
}
|
|
289
|
+
} : undefined
|
|
290
|
+
});
|
|
291
|
+
if (hasDownloads) {
|
|
292
|
+
if (logger) {
|
|
293
|
+
logger.clearProgress();
|
|
294
|
+
logger.info(`Model ready: ${this.config.model}`);
|
|
295
|
+
} else {
|
|
296
|
+
console.log(` Model ready.
|
|
297
|
+
`);
|
|
298
|
+
}
|
|
299
|
+
}
|
|
300
|
+
} catch (error) {
|
|
301
|
+
this.extractor = null;
|
|
302
|
+
if (this.config.logger) {
|
|
303
|
+
this.config.logger.clearProgress();
|
|
304
|
+
}
|
|
305
|
+
throw new Error(`Failed to load embedding model: ${error}`);
|
|
306
|
+
} finally {
|
|
307
|
+
this.isInitializing = false;
|
|
308
|
+
this.initPromise = null;
|
|
309
|
+
}
|
|
310
|
+
})();
|
|
311
|
+
return this.initPromise;
|
|
312
|
+
}
|
|
313
|
+
async getEmbedding(text) {
|
|
314
|
+
await this.ensureExtractor();
|
|
315
|
+
if (!this.extractor) {
|
|
316
|
+
throw new Error("Embedding pipeline not initialized");
|
|
317
|
+
}
|
|
318
|
+
const output = await this.extractor(text, {
|
|
319
|
+
pooling: "mean",
|
|
320
|
+
normalize: true
|
|
321
|
+
});
|
|
322
|
+
return Array.from(output.data);
|
|
323
|
+
}
|
|
324
|
+
async getEmbeddings(texts) {
|
|
325
|
+
if (texts.length === 0)
|
|
326
|
+
return [];
|
|
327
|
+
await this.ensureExtractor();
|
|
328
|
+
if (!this.extractor) {
|
|
329
|
+
throw new Error("Embedding pipeline not initialized");
|
|
330
|
+
}
|
|
331
|
+
const results = [];
|
|
332
|
+
for (let i = 0;i < texts.length; i += BATCH_SIZE2) {
|
|
333
|
+
const batch = texts.slice(i, i + BATCH_SIZE2);
|
|
334
|
+
const outputs = await Promise.all(batch.map(async (text) => {
|
|
335
|
+
const output = await this.extractor(text, {
|
|
336
|
+
pooling: "mean",
|
|
337
|
+
normalize: true
|
|
338
|
+
});
|
|
339
|
+
return Array.from(output.data);
|
|
340
|
+
}));
|
|
341
|
+
results.push(...outputs);
|
|
342
|
+
}
|
|
343
|
+
return results;
|
|
344
|
+
}
|
|
345
|
+
getDimension() {
|
|
346
|
+
return getEmbeddingDimension(this.config.model);
|
|
347
|
+
}
|
|
348
|
+
getModelName() {
|
|
349
|
+
return this.config.model;
|
|
350
|
+
}
|
|
351
|
+
async dispose() {
|
|
352
|
+
this.extractor = null;
|
|
353
|
+
}
|
|
171
354
|
}
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
|
|
355
|
+
var BATCH_SIZE2 = 32;
|
|
356
|
+
var init_huggingfaceEmbeddingProvider = __esm(() => {
|
|
357
|
+
init_embeddingPaths();
|
|
358
|
+
init_modelCatalog();
|
|
359
|
+
init_modelCache();
|
|
360
|
+
env2.cacheDir = RAGGREP_MODEL_CACHE_DIR;
|
|
361
|
+
env2.allowLocalModels = true;
|
|
362
|
+
});
|
|
363
|
+
|
|
364
|
+
// src/infrastructure/embeddings/embeddingProviderFactory.ts
|
|
365
|
+
function resolveRuntime(config) {
|
|
366
|
+
return config.runtime ?? "huggingface";
|
|
367
|
+
}
|
|
368
|
+
function createEmbeddingProvider(config) {
|
|
369
|
+
const runtime = resolveRuntime(config);
|
|
370
|
+
if (runtime === "huggingface") {
|
|
371
|
+
return new HuggingFaceTransformersEmbeddingProvider(config);
|
|
182
372
|
}
|
|
373
|
+
return new XenovaTransformersEmbeddingProvider(config);
|
|
183
374
|
}
|
|
375
|
+
var init_embeddingProviderFactory = __esm(() => {
|
|
376
|
+
init_huggingfaceEmbeddingProvider();
|
|
377
|
+
init_xenovaEmbeddingProvider();
|
|
378
|
+
});
|
|
379
|
+
|
|
380
|
+
// src/infrastructure/embeddings/globalEmbeddings.ts
|
|
184
381
|
function configureEmbeddings(config) {
|
|
185
|
-
const
|
|
186
|
-
|
|
382
|
+
const merged = {
|
|
383
|
+
...globalConfig,
|
|
384
|
+
...config
|
|
385
|
+
};
|
|
386
|
+
if (merged.runtime === undefined) {
|
|
387
|
+
merged.runtime = "huggingface";
|
|
388
|
+
}
|
|
389
|
+
const needsReset = merged.model !== globalConfig.model || merged.runtime !== globalConfig.runtime || merged.logger !== globalConfig.logger;
|
|
390
|
+
if (needsReset) {
|
|
391
|
+
const prev = globalProvider;
|
|
187
392
|
globalProvider = null;
|
|
393
|
+
prev?.dispose?.();
|
|
188
394
|
}
|
|
189
|
-
globalConfig =
|
|
395
|
+
globalConfig = merged;
|
|
190
396
|
}
|
|
191
397
|
function getEmbeddingConfig() {
|
|
192
398
|
return { ...globalConfig };
|
|
193
399
|
}
|
|
194
400
|
async function ensureGlobalProvider() {
|
|
195
401
|
if (!globalProvider) {
|
|
196
|
-
globalProvider =
|
|
197
|
-
await globalProvider.initialize();
|
|
402
|
+
globalProvider = createEmbeddingProvider(globalConfig);
|
|
403
|
+
await globalProvider.initialize?.(globalConfig);
|
|
198
404
|
}
|
|
199
405
|
return globalProvider;
|
|
200
406
|
}
|
|
@@ -206,27 +412,17 @@ async function getEmbeddings(texts) {
|
|
|
206
412
|
const provider = await ensureGlobalProvider();
|
|
207
413
|
return provider.getEmbeddings(texts);
|
|
208
414
|
}
|
|
209
|
-
|
|
210
|
-
|
|
211
|
-
|
|
212
|
-
|
|
213
|
-
|
|
214
|
-
|
|
215
|
-
|
|
216
|
-
|
|
217
|
-
"bge-small-en-v1.5": "Xenova/bge-small-en-v1.5",
|
|
218
|
-
"paraphrase-MiniLM-L3-v2": "Xenova/paraphrase-MiniLM-L3-v2",
|
|
219
|
-
"nomic-embed-text-v1.5": "nomic-ai/nomic-embed-text-v1.5"
|
|
220
|
-
};
|
|
221
|
-
EMBEDDING_DIMENSIONS = {
|
|
222
|
-
"all-MiniLM-L6-v2": 384,
|
|
223
|
-
"all-MiniLM-L12-v2": 384,
|
|
224
|
-
"bge-small-en-v1.5": 384,
|
|
225
|
-
"paraphrase-MiniLM-L3-v2": 384,
|
|
226
|
-
"nomic-embed-text-v1.5": 768
|
|
227
|
-
};
|
|
415
|
+
function getCacheDir() {
|
|
416
|
+
return RAGGREP_MODEL_CACHE_DIR;
|
|
417
|
+
}
|
|
418
|
+
var globalProvider = null, globalConfig;
|
|
419
|
+
var init_globalEmbeddings = __esm(() => {
|
|
420
|
+
init_embeddingPaths();
|
|
421
|
+
init_embeddingProviderFactory();
|
|
422
|
+
init_modelCache();
|
|
228
423
|
globalConfig = {
|
|
229
424
|
model: "bge-small-en-v1.5",
|
|
425
|
+
runtime: "huggingface",
|
|
230
426
|
showProgress: false,
|
|
231
427
|
logger: undefined
|
|
232
428
|
};
|
|
@@ -234,7 +430,13 @@ var init_transformersEmbedding = __esm(() => {
|
|
|
234
430
|
|
|
235
431
|
// src/infrastructure/embeddings/index.ts
|
|
236
432
|
var init_embeddings = __esm(() => {
|
|
237
|
-
|
|
433
|
+
init_modelCatalog();
|
|
434
|
+
init_embeddingPaths();
|
|
435
|
+
init_xenovaEmbeddingProvider();
|
|
436
|
+
init_xenovaEmbeddingProvider();
|
|
437
|
+
init_huggingfaceEmbeddingProvider();
|
|
438
|
+
init_embeddingProviderFactory();
|
|
439
|
+
init_globalEmbeddings();
|
|
238
440
|
});
|
|
239
441
|
|
|
240
442
|
// src/infrastructure/logger/loggers.ts
|
|
@@ -457,29 +659,29 @@ var init_logger = () => {};
|
|
|
457
659
|
import { createRequire as createRequire2 } from "module";
|
|
458
660
|
import { basename, dirname, normalize, relative, resolve, sep } from "path";
|
|
459
661
|
import * as nativeFs from "fs";
|
|
460
|
-
function cleanPath(
|
|
461
|
-
let normalized = normalize(
|
|
662
|
+
function cleanPath(path3) {
|
|
663
|
+
let normalized = normalize(path3);
|
|
462
664
|
if (normalized.length > 1 && normalized[normalized.length - 1] === sep)
|
|
463
665
|
normalized = normalized.substring(0, normalized.length - 1);
|
|
464
666
|
return normalized;
|
|
465
667
|
}
|
|
466
|
-
function convertSlashes(
|
|
467
|
-
return
|
|
668
|
+
function convertSlashes(path3, separator) {
|
|
669
|
+
return path3.replace(SLASHES_REGEX, separator);
|
|
468
670
|
}
|
|
469
|
-
function isRootDirectory(
|
|
470
|
-
return
|
|
671
|
+
function isRootDirectory(path3) {
|
|
672
|
+
return path3 === "/" || WINDOWS_ROOT_DIR_REGEX.test(path3);
|
|
471
673
|
}
|
|
472
|
-
function normalizePath(
|
|
674
|
+
function normalizePath(path3, options) {
|
|
473
675
|
const { resolvePaths, normalizePath: normalizePath$1, pathSeparator } = options;
|
|
474
|
-
const pathNeedsCleaning = process.platform === "win32" &&
|
|
676
|
+
const pathNeedsCleaning = process.platform === "win32" && path3.includes("/") || path3.startsWith(".");
|
|
475
677
|
if (resolvePaths)
|
|
476
|
-
|
|
678
|
+
path3 = resolve(path3);
|
|
477
679
|
if (normalizePath$1 || pathNeedsCleaning)
|
|
478
|
-
|
|
479
|
-
if (
|
|
680
|
+
path3 = cleanPath(path3);
|
|
681
|
+
if (path3 === ".")
|
|
480
682
|
return "";
|
|
481
|
-
const needsSeperator =
|
|
482
|
-
return convertSlashes(needsSeperator ?
|
|
683
|
+
const needsSeperator = path3[path3.length - 1] !== pathSeparator;
|
|
684
|
+
return convertSlashes(needsSeperator ? path3 + pathSeparator : path3, pathSeparator);
|
|
483
685
|
}
|
|
484
686
|
function joinPathWithBasePath(filename, directoryPath) {
|
|
485
687
|
return directoryPath + filename;
|
|
@@ -545,10 +747,10 @@ function build$2(options, isSynchronous) {
|
|
|
545
747
|
return null;
|
|
546
748
|
return isSynchronous ? resolveSymlinks : resolveSymlinksAsync;
|
|
547
749
|
}
|
|
548
|
-
function isRecursive(
|
|
750
|
+
function isRecursive(path3, resolved, state) {
|
|
549
751
|
if (state.options.useRealPaths)
|
|
550
752
|
return isRecursiveUsingRealPaths(resolved, state);
|
|
551
|
-
let parent = dirname(
|
|
753
|
+
let parent = dirname(path3);
|
|
552
754
|
let depth = 1;
|
|
553
755
|
while (parent !== state.root && depth < 2) {
|
|
554
756
|
const resolvedPath = state.symlinks.get(parent);
|
|
@@ -558,7 +760,7 @@ function isRecursive(path2, resolved, state) {
|
|
|
558
760
|
else
|
|
559
761
|
parent = dirname(parent);
|
|
560
762
|
}
|
|
561
|
-
state.symlinks.set(
|
|
763
|
+
state.symlinks.set(path3, resolved);
|
|
562
764
|
return depth > 1;
|
|
563
765
|
}
|
|
564
766
|
function isRecursiveUsingRealPaths(resolved, state) {
|
|
@@ -604,9 +806,9 @@ function sync(root, options) {
|
|
|
604
806
|
var __require2, SLASHES_REGEX, WINDOWS_ROOT_DIR_REGEX, pushDirectory = (directoryPath, paths) => {
|
|
605
807
|
paths.push(directoryPath || ".");
|
|
606
808
|
}, pushDirectoryFilter = (directoryPath, paths, filters) => {
|
|
607
|
-
const
|
|
608
|
-
if (filters.every((filter) => filter(
|
|
609
|
-
paths.push(
|
|
809
|
+
const path3 = directoryPath || ".";
|
|
810
|
+
if (filters.every((filter) => filter(path3, true)))
|
|
811
|
+
paths.push(path3);
|
|
610
812
|
}, empty$2 = () => {}, pushFileFilterAndCount = (filename, _paths, counts, filters) => {
|
|
611
813
|
if (filters.every((filter) => filter(filename, false)))
|
|
612
814
|
counts.files++;
|
|
@@ -627,28 +829,28 @@ var __require2, SLASHES_REGEX, WINDOWS_ROOT_DIR_REGEX, pushDirectory = (director
|
|
|
627
829
|
files,
|
|
628
830
|
dir: directory
|
|
629
831
|
});
|
|
630
|
-
}, empty = () => {}, resolveSymlinksAsync = function(
|
|
631
|
-
const { queue, fs, options: { suppressErrors } } = state;
|
|
832
|
+
}, empty = () => {}, resolveSymlinksAsync = function(path3, state, callback$1) {
|
|
833
|
+
const { queue, fs: fs2, options: { suppressErrors } } = state;
|
|
632
834
|
queue.enqueue();
|
|
633
|
-
|
|
835
|
+
fs2.realpath(path3, (error, resolvedPath) => {
|
|
634
836
|
if (error)
|
|
635
837
|
return queue.dequeue(suppressErrors ? null : error, state);
|
|
636
|
-
|
|
838
|
+
fs2.stat(resolvedPath, (error$1, stat) => {
|
|
637
839
|
if (error$1)
|
|
638
840
|
return queue.dequeue(suppressErrors ? null : error$1, state);
|
|
639
|
-
if (stat.isDirectory() && isRecursive(
|
|
841
|
+
if (stat.isDirectory() && isRecursive(path3, resolvedPath, state))
|
|
640
842
|
return queue.dequeue(null, state);
|
|
641
843
|
callback$1(stat, resolvedPath);
|
|
642
844
|
queue.dequeue(null, state);
|
|
643
845
|
});
|
|
644
846
|
});
|
|
645
|
-
}, resolveSymlinks = function(
|
|
646
|
-
const { queue, fs, options: { suppressErrors } } = state;
|
|
847
|
+
}, resolveSymlinks = function(path3, state, callback$1) {
|
|
848
|
+
const { queue, fs: fs2, options: { suppressErrors } } = state;
|
|
647
849
|
queue.enqueue();
|
|
648
850
|
try {
|
|
649
|
-
const resolvedPath =
|
|
650
|
-
const stat =
|
|
651
|
-
if (stat.isDirectory() && isRecursive(
|
|
851
|
+
const resolvedPath = fs2.realpathSync(path3);
|
|
852
|
+
const stat = fs2.statSync(resolvedPath);
|
|
853
|
+
if (stat.isDirectory() && isRecursive(path3, resolvedPath, state))
|
|
652
854
|
return;
|
|
653
855
|
callback$1(stat, resolvedPath);
|
|
654
856
|
} catch (e) {
|
|
@@ -679,22 +881,22 @@ var __require2, SLASHES_REGEX, WINDOWS_ROOT_DIR_REGEX, pushDirectory = (director
|
|
|
679
881
|
state.queue.enqueue();
|
|
680
882
|
if (currentDepth < 0)
|
|
681
883
|
return state.queue.dequeue(null, state);
|
|
682
|
-
const { fs } = state;
|
|
884
|
+
const { fs: fs2 } = state;
|
|
683
885
|
state.visited.push(crawlPath);
|
|
684
886
|
state.counts.directories++;
|
|
685
|
-
|
|
887
|
+
fs2.readdir(crawlPath || ".", readdirOpts, (error, entries = []) => {
|
|
686
888
|
callback$1(entries, directoryPath, currentDepth);
|
|
687
889
|
state.queue.dequeue(state.options.suppressErrors ? null : error, state);
|
|
688
890
|
});
|
|
689
891
|
}, walkSync = (state, crawlPath, directoryPath, currentDepth, callback$1) => {
|
|
690
|
-
const { fs } = state;
|
|
892
|
+
const { fs: fs2 } = state;
|
|
691
893
|
if (currentDepth < 0)
|
|
692
894
|
return;
|
|
693
895
|
state.visited.push(crawlPath);
|
|
694
896
|
state.counts.directories++;
|
|
695
897
|
let entries = [];
|
|
696
898
|
try {
|
|
697
|
-
entries =
|
|
899
|
+
entries = fs2.readdirSync(crawlPath || ".", readdirOpts);
|
|
698
900
|
} catch (e) {
|
|
699
901
|
if (!state.options.suppressErrors)
|
|
700
902
|
throw e;
|
|
@@ -793,21 +995,21 @@ var __require2, SLASHES_REGEX, WINDOWS_ROOT_DIR_REGEX, pushDirectory = (director
|
|
|
793
995
|
const filename = this.joinPath(entry.name, directoryPath);
|
|
794
996
|
this.pushFile(filename, files, this.state.counts, filters);
|
|
795
997
|
} else if (entry.isDirectory()) {
|
|
796
|
-
let
|
|
797
|
-
if (exclude && exclude(entry.name,
|
|
998
|
+
let path3 = joinDirectoryPath(entry.name, directoryPath, this.state.options.pathSeparator);
|
|
999
|
+
if (exclude && exclude(entry.name, path3))
|
|
798
1000
|
continue;
|
|
799
|
-
this.pushDirectory(
|
|
800
|
-
this.walkDirectory(this.state,
|
|
1001
|
+
this.pushDirectory(path3, paths, filters);
|
|
1002
|
+
this.walkDirectory(this.state, path3, path3, depth - 1, this.walk);
|
|
801
1003
|
} else if (this.resolveSymlink && entry.isSymbolicLink()) {
|
|
802
|
-
let
|
|
803
|
-
this.resolveSymlink(
|
|
1004
|
+
let path3 = joinPathWithBasePath(entry.name, directoryPath);
|
|
1005
|
+
this.resolveSymlink(path3, this.state, (stat, resolvedPath) => {
|
|
804
1006
|
if (stat.isDirectory()) {
|
|
805
1007
|
resolvedPath = normalizePath(resolvedPath, this.state.options);
|
|
806
|
-
if (exclude && exclude(entry.name, useRealPaths ? resolvedPath :
|
|
1008
|
+
if (exclude && exclude(entry.name, useRealPaths ? resolvedPath : path3 + pathSeparator))
|
|
807
1009
|
return;
|
|
808
|
-
this.walkDirectory(this.state, resolvedPath, useRealPaths ? resolvedPath :
|
|
1010
|
+
this.walkDirectory(this.state, resolvedPath, useRealPaths ? resolvedPath : path3 + pathSeparator, depth - 1, this.walk);
|
|
809
1011
|
} else {
|
|
810
|
-
resolvedPath = useRealPaths ? resolvedPath :
|
|
1012
|
+
resolvedPath = useRealPaths ? resolvedPath : path3;
|
|
811
1013
|
const filename = basename(resolvedPath);
|
|
812
1014
|
const directoryPath$1 = normalizePath(dirname(resolvedPath), this.state.options);
|
|
813
1015
|
resolvedPath = this.joinPath(filename, directoryPath$1);
|
|
@@ -943,7 +1145,7 @@ var __require2, SLASHES_REGEX, WINDOWS_ROOT_DIR_REGEX, pushDirectory = (director
|
|
|
943
1145
|
isMatch = globFn(patterns, ...options);
|
|
944
1146
|
this.globCache[patterns.join("\x00")] = isMatch;
|
|
945
1147
|
}
|
|
946
|
-
this.options.filters.push((
|
|
1148
|
+
this.options.filters.push((path3) => isMatch(path3));
|
|
947
1149
|
return this;
|
|
948
1150
|
}
|
|
949
1151
|
};
|
|
@@ -985,49 +1187,37 @@ function createDefaultConfig() {
|
|
|
985
1187
|
{
|
|
986
1188
|
id: "language/typescript",
|
|
987
1189
|
enabled: true,
|
|
988
|
-
options: {
|
|
989
|
-
embeddingModel: "all-MiniLM-L6-v2"
|
|
990
|
-
}
|
|
1190
|
+
options: { ...DEFAULT_EMBEDDING_MODULE_OPTIONS }
|
|
991
1191
|
},
|
|
992
1192
|
{
|
|
993
1193
|
id: "language/python",
|
|
994
1194
|
enabled: true,
|
|
995
|
-
options: {
|
|
996
|
-
embeddingModel: "all-MiniLM-L6-v2"
|
|
997
|
-
}
|
|
1195
|
+
options: { ...DEFAULT_EMBEDDING_MODULE_OPTIONS }
|
|
998
1196
|
},
|
|
999
1197
|
{
|
|
1000
1198
|
id: "language/go",
|
|
1001
1199
|
enabled: true,
|
|
1002
|
-
options: {
|
|
1003
|
-
embeddingModel: "all-MiniLM-L6-v2"
|
|
1004
|
-
}
|
|
1200
|
+
options: { ...DEFAULT_EMBEDDING_MODULE_OPTIONS }
|
|
1005
1201
|
},
|
|
1006
1202
|
{
|
|
1007
1203
|
id: "language/rust",
|
|
1008
1204
|
enabled: true,
|
|
1009
|
-
options: {
|
|
1010
|
-
embeddingModel: "all-MiniLM-L6-v2"
|
|
1011
|
-
}
|
|
1205
|
+
options: { ...DEFAULT_EMBEDDING_MODULE_OPTIONS }
|
|
1012
1206
|
},
|
|
1013
1207
|
{
|
|
1014
1208
|
id: "data/json",
|
|
1015
1209
|
enabled: true,
|
|
1016
|
-
options: {
|
|
1017
|
-
embeddingModel: "all-MiniLM-L6-v2"
|
|
1018
|
-
}
|
|
1210
|
+
options: { ...DEFAULT_EMBEDDING_MODULE_OPTIONS }
|
|
1019
1211
|
},
|
|
1020
1212
|
{
|
|
1021
1213
|
id: "docs/markdown",
|
|
1022
1214
|
enabled: true,
|
|
1023
|
-
options: {
|
|
1024
|
-
embeddingModel: "all-MiniLM-L6-v2"
|
|
1025
|
-
}
|
|
1215
|
+
options: { ...DEFAULT_EMBEDDING_MODULE_OPTIONS }
|
|
1026
1216
|
}
|
|
1027
1217
|
]
|
|
1028
1218
|
};
|
|
1029
1219
|
}
|
|
1030
|
-
var DEFAULT_IGNORE_PATHS, DEFAULT_EXTENSIONS;
|
|
1220
|
+
var DEFAULT_IGNORE_PATHS, DEFAULT_EXTENSIONS, DEFAULT_EMBEDDING_MODULE_OPTIONS;
|
|
1031
1221
|
var init_config = __esm(() => {
|
|
1032
1222
|
DEFAULT_IGNORE_PATHS = [
|
|
1033
1223
|
"node_modules",
|
|
@@ -1097,6 +1287,10 @@ var init_config = __esm(() => {
|
|
|
1097
1287
|
".sql",
|
|
1098
1288
|
".txt"
|
|
1099
1289
|
];
|
|
1290
|
+
DEFAULT_EMBEDDING_MODULE_OPTIONS = {
|
|
1291
|
+
embeddingModel: "bge-small-en-v1.5",
|
|
1292
|
+
embeddingRuntime: "huggingface"
|
|
1293
|
+
};
|
|
1100
1294
|
});
|
|
1101
1295
|
|
|
1102
1296
|
// src/domain/entities/literal.ts
|
|
@@ -1127,47 +1321,45 @@ var init_entities = __esm(() => {
|
|
|
1127
1321
|
});
|
|
1128
1322
|
|
|
1129
1323
|
// src/infrastructure/config/configLoader.ts
|
|
1130
|
-
import * as
|
|
1131
|
-
import * as
|
|
1132
|
-
import * as os2 from "os";
|
|
1324
|
+
import * as path3 from "path";
|
|
1325
|
+
import * as fs2 from "fs/promises";
|
|
1133
1326
|
import * as crypto from "crypto";
|
|
1134
1327
|
function hashPath(inputPath) {
|
|
1135
1328
|
return crypto.createHash("sha256").update(inputPath).digest("hex").slice(0, 12);
|
|
1136
1329
|
}
|
|
1137
1330
|
function getRaggrepDir(rootDir, _config = DEFAULT_CONFIG) {
|
|
1138
|
-
const absoluteRoot =
|
|
1139
|
-
|
|
1140
|
-
return path2.join(RAGGREP_TEMP_BASE, projectHash);
|
|
1331
|
+
const absoluteRoot = path3.resolve(rootDir);
|
|
1332
|
+
return path3.join(absoluteRoot, RAGGREP_INDEX_DIR);
|
|
1141
1333
|
}
|
|
1142
1334
|
function getIndexLocation(rootDir) {
|
|
1143
|
-
const absoluteRoot =
|
|
1335
|
+
const absoluteRoot = path3.resolve(rootDir);
|
|
1144
1336
|
const projectHash = hashPath(absoluteRoot);
|
|
1145
1337
|
return {
|
|
1146
|
-
indexDir:
|
|
1338
|
+
indexDir: path3.join(absoluteRoot, RAGGREP_INDEX_DIR),
|
|
1147
1339
|
projectRoot: absoluteRoot,
|
|
1148
1340
|
projectHash
|
|
1149
1341
|
};
|
|
1150
1342
|
}
|
|
1151
1343
|
function getModuleIndexPath(rootDir, moduleId, config = DEFAULT_CONFIG) {
|
|
1152
1344
|
const indexDir = getRaggrepDir(rootDir, config);
|
|
1153
|
-
return
|
|
1345
|
+
return path3.join(indexDir, "index", moduleId);
|
|
1154
1346
|
}
|
|
1155
1347
|
function getModuleManifestPath(rootDir, moduleId, config = DEFAULT_CONFIG) {
|
|
1156
1348
|
const indexDir = getRaggrepDir(rootDir, config);
|
|
1157
|
-
return
|
|
1349
|
+
return path3.join(indexDir, "index", moduleId, "manifest.json");
|
|
1158
1350
|
}
|
|
1159
1351
|
function getGlobalManifestPath(rootDir, config = DEFAULT_CONFIG) {
|
|
1160
1352
|
const indexDir = getRaggrepDir(rootDir, config);
|
|
1161
|
-
return
|
|
1353
|
+
return path3.join(indexDir, "manifest.json");
|
|
1162
1354
|
}
|
|
1163
1355
|
function getConfigPath(rootDir, config = DEFAULT_CONFIG) {
|
|
1164
1356
|
const indexDir = getRaggrepDir(rootDir, config);
|
|
1165
|
-
return
|
|
1357
|
+
return path3.join(indexDir, "config.json");
|
|
1166
1358
|
}
|
|
1167
1359
|
async function loadConfig(rootDir) {
|
|
1168
1360
|
const configPath = getConfigPath(rootDir, DEFAULT_CONFIG);
|
|
1169
1361
|
try {
|
|
1170
|
-
const content = await
|
|
1362
|
+
const content = await fs2.readFile(configPath, "utf-8");
|
|
1171
1363
|
const savedConfig = JSON.parse(content);
|
|
1172
1364
|
return { ...DEFAULT_CONFIG, ...savedConfig };
|
|
1173
1365
|
} catch {
|
|
@@ -1180,27 +1372,28 @@ function getModuleConfig(config, moduleId) {
|
|
|
1180
1372
|
function getEmbeddingConfigFromModule(moduleConfig) {
|
|
1181
1373
|
const options = moduleConfig.options || {};
|
|
1182
1374
|
const modelName = options.embeddingModel || "bge-small-en-v1.5";
|
|
1183
|
-
if (!(modelName in
|
|
1375
|
+
if (!(modelName in EMBEDDING_MODELS)) {
|
|
1184
1376
|
console.warn(`Unknown embedding model: ${modelName}, falling back to bge-small-en-v1.5`);
|
|
1185
1377
|
return { model: "bge-small-en-v1.5" };
|
|
1186
1378
|
}
|
|
1379
|
+
const rt = options.embeddingRuntime;
|
|
1380
|
+
let runtime;
|
|
1381
|
+
if (rt === "xenova" || rt === "huggingface") {
|
|
1382
|
+
runtime = rt;
|
|
1383
|
+
} else if (rt !== undefined) {
|
|
1384
|
+
console.warn(`Unknown embeddingRuntime: ${rt}, falling back to default (huggingface)`);
|
|
1385
|
+
}
|
|
1187
1386
|
return {
|
|
1188
1387
|
model: modelName,
|
|
1388
|
+
...runtime ? { runtime } : {},
|
|
1189
1389
|
showProgress: options.showProgress === true
|
|
1190
1390
|
};
|
|
1191
1391
|
}
|
|
1192
|
-
var DEFAULT_CONFIG,
|
|
1392
|
+
var DEFAULT_CONFIG, RAGGREP_INDEX_DIR = ".raggrep";
|
|
1193
1393
|
var init_configLoader = __esm(() => {
|
|
1194
1394
|
init_entities();
|
|
1395
|
+
init_modelCatalog();
|
|
1195
1396
|
DEFAULT_CONFIG = createDefaultConfig();
|
|
1196
|
-
RAGGREP_TEMP_BASE = path2.join(os2.tmpdir(), "raggrep-indexes");
|
|
1197
|
-
EMBEDDING_MODELS2 = {
|
|
1198
|
-
"all-MiniLM-L6-v2": "Xenova/all-MiniLM-L6-v2",
|
|
1199
|
-
"all-MiniLM-L12-v2": "Xenova/all-MiniLM-L12-v2",
|
|
1200
|
-
"bge-small-en-v1.5": "Xenova/bge-small-en-v1.5",
|
|
1201
|
-
"paraphrase-MiniLM-L3-v2": "Xenova/paraphrase-MiniLM-L3-v2",
|
|
1202
|
-
"nomic-embed-text-v1.5": "nomic-ai/nomic-embed-text-v1.5"
|
|
1203
|
-
};
|
|
1204
1397
|
});
|
|
1205
1398
|
|
|
1206
1399
|
// src/infrastructure/config/index.ts
|
|
@@ -1338,10 +1531,10 @@ function normalizeScore(score, midpoint = 5) {
|
|
|
1338
1531
|
var BM25_K1 = 1.5, BM25_B = 0.75;
|
|
1339
1532
|
|
|
1340
1533
|
// src/domain/services/conventions/entryPoints.ts
|
|
1341
|
-
import * as
|
|
1534
|
+
import * as path4 from "path";
|
|
1342
1535
|
function getParentFolder(filepath) {
|
|
1343
|
-
const dir =
|
|
1344
|
-
return
|
|
1536
|
+
const dir = path4.dirname(filepath);
|
|
1537
|
+
return path4.basename(dir);
|
|
1345
1538
|
}
|
|
1346
1539
|
var entryPointConventions;
|
|
1347
1540
|
var init_entryPoints = __esm(() => {
|
|
@@ -2208,7 +2401,7 @@ var init_frameworks = __esm(() => {
|
|
|
2208
2401
|
});
|
|
2209
2402
|
|
|
2210
2403
|
// src/domain/services/conventions/index.ts
|
|
2211
|
-
import * as
|
|
2404
|
+
import * as path5 from "path";
|
|
2212
2405
|
function getConventions() {
|
|
2213
2406
|
return [
|
|
2214
2407
|
...entryPointConventions,
|
|
@@ -2220,8 +2413,8 @@ function getConventions() {
|
|
|
2220
2413
|
}
|
|
2221
2414
|
function getConventionKeywords(filepath) {
|
|
2222
2415
|
const conventions = getConventions();
|
|
2223
|
-
const filename =
|
|
2224
|
-
const extension =
|
|
2416
|
+
const filename = path5.basename(filepath);
|
|
2417
|
+
const extension = path5.extname(filepath);
|
|
2225
2418
|
const keywords = new Set;
|
|
2226
2419
|
for (const convention of conventions) {
|
|
2227
2420
|
try {
|
|
@@ -2309,13 +2502,13 @@ var init_conventions = __esm(() => {
|
|
|
2309
2502
|
});
|
|
2310
2503
|
|
|
2311
2504
|
// src/domain/services/introspection.ts
|
|
2312
|
-
import * as
|
|
2505
|
+
import * as path6 from "path";
|
|
2313
2506
|
function introspectFile(filepath, structure, options) {
|
|
2314
2507
|
const opts = typeof options === "string" ? { fileContent: options } : options || {};
|
|
2315
2508
|
const normalizedPath = filepath.replace(/\\/g, "/");
|
|
2316
2509
|
const segments = normalizedPath.split("/").filter((s) => s.length > 0);
|
|
2317
2510
|
const filename = segments[segments.length - 1] || "";
|
|
2318
|
-
const ext =
|
|
2511
|
+
const ext = path6.extname(filename);
|
|
2319
2512
|
const project = findProjectForFile(normalizedPath, structure);
|
|
2320
2513
|
const language = EXTENSION_TO_LANGUAGE[ext] || "unknown";
|
|
2321
2514
|
const layer = detectLayer(segments, filename);
|
|
@@ -2357,7 +2550,7 @@ function findNearestReadme(filepath, fileExists) {
|
|
|
2357
2550
|
}
|
|
2358
2551
|
function introspectionToKeywords(intro) {
|
|
2359
2552
|
const keywords = [];
|
|
2360
|
-
const filename =
|
|
2553
|
+
const filename = path6.basename(intro.filepath);
|
|
2361
2554
|
const filenameWithoutExt = filename.replace(/\.[^.]+$/, "");
|
|
2362
2555
|
const filenameParts = filenameWithoutExt.split(/[-_.]/).flatMap((part) => part.split(/(?=[A-Z])/)).map((part) => part.toLowerCase()).filter((part) => part.length > 1);
|
|
2363
2556
|
keywords.push(...filenameParts);
|
|
@@ -2836,8 +3029,8 @@ var exports_core = {};
|
|
|
2836
3029
|
__export(exports_core, {
|
|
2837
3030
|
CoreModule: () => CoreModule
|
|
2838
3031
|
});
|
|
2839
|
-
import * as
|
|
2840
|
-
import * as
|
|
3032
|
+
import * as path7 from "path";
|
|
3033
|
+
import * as fs3 from "fs/promises";
|
|
2841
3034
|
|
|
2842
3035
|
class CoreModule {
|
|
2843
3036
|
id = "core";
|
|
@@ -2934,8 +3127,8 @@ class CoreModule {
|
|
|
2934
3127
|
}
|
|
2935
3128
|
async finalize(ctx) {
|
|
2936
3129
|
const config = ctx.config;
|
|
2937
|
-
const coreDir =
|
|
2938
|
-
await
|
|
3130
|
+
const coreDir = path7.join(getRaggrepDir(ctx.rootDir, config), "index", "core");
|
|
3131
|
+
await fs3.mkdir(coreDir, { recursive: true });
|
|
2939
3132
|
this.bm25Index = new BM25Index;
|
|
2940
3133
|
for (const [filepath, entry] of this.symbolIndex) {
|
|
2941
3134
|
this.bm25Index.addDocument(filepath, entry.tokens);
|
|
@@ -2946,7 +3139,7 @@ class CoreModule {
|
|
|
2946
3139
|
files: Object.fromEntries(this.symbolIndex),
|
|
2947
3140
|
bm25Data: this.bm25Index.serialize()
|
|
2948
3141
|
};
|
|
2949
|
-
await
|
|
3142
|
+
await fs3.writeFile(path7.join(coreDir, "symbols.json"), JSON.stringify(symbolIndexData, null, 2));
|
|
2950
3143
|
}
|
|
2951
3144
|
async search(query, ctx, options) {
|
|
2952
3145
|
const config = ctx.config;
|
|
@@ -3042,10 +3235,10 @@ class CoreModule {
|
|
|
3042
3235
|
return bestChunk;
|
|
3043
3236
|
}
|
|
3044
3237
|
async loadSymbolIndex(rootDir, config) {
|
|
3045
|
-
const coreDir =
|
|
3046
|
-
const symbolsPath =
|
|
3238
|
+
const coreDir = path7.join(getRaggrepDir(rootDir, config), "index", "core");
|
|
3239
|
+
const symbolsPath = path7.join(coreDir, "symbols.json");
|
|
3047
3240
|
try {
|
|
3048
|
-
const content = await
|
|
3241
|
+
const content = await fs3.readFile(symbolsPath, "utf-8");
|
|
3049
3242
|
const data = JSON.parse(content);
|
|
3050
3243
|
this.symbolIndex = new Map(Object.entries(data.files));
|
|
3051
3244
|
if (data.bm25Data) {
|
|
@@ -3276,7 +3469,7 @@ function cosineSimilarity(a, b) {
|
|
|
3276
3469
|
}
|
|
3277
3470
|
|
|
3278
3471
|
// src/domain/services/queryIntent.ts
|
|
3279
|
-
import * as
|
|
3472
|
+
import * as path8 from "path";
|
|
3280
3473
|
function detectQueryIntent(queryTerms) {
|
|
3281
3474
|
const hasImplementationTerm = queryTerms.some((term) => IMPLEMENTATION_TERMS.includes(term));
|
|
3282
3475
|
const hasDocumentationTerm = queryTerms.some((term) => DOCUMENTATION_TERMS.includes(term));
|
|
@@ -3292,11 +3485,11 @@ function extractQueryTerms(query) {
|
|
|
3292
3485
|
return query.toLowerCase().split(/\s+/).filter((t) => t.length > 2);
|
|
3293
3486
|
}
|
|
3294
3487
|
function isSourceCodeFile(filepath) {
|
|
3295
|
-
const ext =
|
|
3488
|
+
const ext = path8.extname(filepath).toLowerCase();
|
|
3296
3489
|
return SOURCE_CODE_EXTENSIONS.includes(ext);
|
|
3297
3490
|
}
|
|
3298
3491
|
function isDocFile(filepath) {
|
|
3299
|
-
const ext =
|
|
3492
|
+
const ext = path8.extname(filepath).toLowerCase();
|
|
3300
3493
|
return DOC_EXTENSIONS.includes(ext);
|
|
3301
3494
|
}
|
|
3302
3495
|
function calculateFileTypeBoost(filepath, queryTerms) {
|
|
@@ -4371,8 +4564,8 @@ var init_lexicon2 = __esm(() => {
|
|
|
4371
4564
|
// src/domain/services/jsonPathExtractor.ts
|
|
4372
4565
|
function extractJsonPaths(obj, fileBasename) {
|
|
4373
4566
|
const paths = extractPathsRecursive(obj, fileBasename);
|
|
4374
|
-
return paths.map((
|
|
4375
|
-
value:
|
|
4567
|
+
return paths.map((path9) => ({
|
|
4568
|
+
value: path9,
|
|
4376
4569
|
type: "identifier",
|
|
4377
4570
|
matchType: "definition"
|
|
4378
4571
|
}));
|
|
@@ -4682,6 +4875,62 @@ var init_simpleSearch = __esm(() => {
|
|
|
4682
4875
|
};
|
|
4683
4876
|
});
|
|
4684
4877
|
|
|
4878
|
+
// src/domain/services/chunkContext.ts
|
|
4879
|
+
import * as path9 from "path";
|
|
4880
|
+
function prepareChunkForEmbedding(options) {
|
|
4881
|
+
const { filepath, content, name, docComment } = options;
|
|
4882
|
+
const pathContext = parsePathContext(filepath);
|
|
4883
|
+
const pathPrefix = formatPathContextForEmbedding(pathContext);
|
|
4884
|
+
const parts = [];
|
|
4885
|
+
if (pathPrefix) {
|
|
4886
|
+
parts.push(pathPrefix);
|
|
4887
|
+
}
|
|
4888
|
+
const filename = path9.basename(filepath);
|
|
4889
|
+
const filenameWithoutExt = filename.replace(/\.[^.]+$/, "");
|
|
4890
|
+
if (filenameWithoutExt && filenameWithoutExt.length > MIN_SEGMENT_LENGTH) {
|
|
4891
|
+
const pathPrefixLower = pathPrefix.toLowerCase();
|
|
4892
|
+
if (!pathPrefixLower.includes(filenameWithoutExt.toLowerCase())) {
|
|
4893
|
+
parts.push(filenameWithoutExt);
|
|
4894
|
+
}
|
|
4895
|
+
}
|
|
4896
|
+
if (name) {
|
|
4897
|
+
parts.push(`${name}:`);
|
|
4898
|
+
}
|
|
4899
|
+
if (docComment) {
|
|
4900
|
+
parts.push(docComment);
|
|
4901
|
+
}
|
|
4902
|
+
parts.push(content);
|
|
4903
|
+
return parts.join(" ");
|
|
4904
|
+
}
|
|
4905
|
+
function extractPathKeywordsForFileSummary(filepath) {
|
|
4906
|
+
const keywords = extractPathKeywords(filepath);
|
|
4907
|
+
const filtered = keywords.filter((k) => k.length >= MIN_SEGMENT_LENGTH && !GENERIC_SEGMENTS.has(k));
|
|
4908
|
+
return [...new Set(filtered)];
|
|
4909
|
+
}
|
|
4910
|
+
function getPathContextForFileSummary(filepath) {
|
|
4911
|
+
const pathContext = parsePathContext(filepath);
|
|
4912
|
+
return {
|
|
4913
|
+
segments: pathContext.segments,
|
|
4914
|
+
layer: pathContext.layer,
|
|
4915
|
+
domain: pathContext.domain,
|
|
4916
|
+
depth: pathContext.depth
|
|
4917
|
+
};
|
|
4918
|
+
}
|
|
4919
|
+
var GENERIC_SEGMENTS, MIN_SEGMENT_LENGTH = 2;
|
|
4920
|
+
var init_chunkContext = __esm(() => {
|
|
4921
|
+
init_keywords();
|
|
4922
|
+
GENERIC_SEGMENTS = new Set([
|
|
4923
|
+
"src",
|
|
4924
|
+
"lib",
|
|
4925
|
+
"app",
|
|
4926
|
+
"index",
|
|
4927
|
+
"dist",
|
|
4928
|
+
"build",
|
|
4929
|
+
"out",
|
|
4930
|
+
"node_modules"
|
|
4931
|
+
]);
|
|
4932
|
+
});
|
|
4933
|
+
|
|
4685
4934
|
// src/domain/services/index.ts
|
|
4686
4935
|
var init_services = __esm(() => {
|
|
4687
4936
|
init_keywords();
|
|
@@ -4694,6 +4943,7 @@ var init_services = __esm(() => {
|
|
|
4694
4943
|
init_configValidator();
|
|
4695
4944
|
init_phraseMatch();
|
|
4696
4945
|
init_simpleSearch();
|
|
4946
|
+
init_chunkContext();
|
|
4697
4947
|
});
|
|
4698
4948
|
|
|
4699
4949
|
// src/modules/language/typescript/parseCode.ts
|
|
@@ -4865,11 +5115,12 @@ var init_parseCode = () => {};
|
|
|
4865
5115
|
// src/infrastructure/storage/fileIndexStorage.ts
|
|
4866
5116
|
var init_fileIndexStorage = __esm(() => {
|
|
4867
5117
|
init_entities();
|
|
5118
|
+
init_config2();
|
|
4868
5119
|
});
|
|
4869
5120
|
|
|
4870
5121
|
// src/infrastructure/storage/symbolicIndex.ts
|
|
4871
|
-
import * as
|
|
4872
|
-
import * as
|
|
5122
|
+
import * as fs4 from "fs/promises";
|
|
5123
|
+
import * as path10 from "path";
|
|
4873
5124
|
|
|
4874
5125
|
class SymbolicIndex {
|
|
4875
5126
|
meta = null;
|
|
@@ -4878,7 +5129,7 @@ class SymbolicIndex {
|
|
|
4878
5129
|
symbolicPath;
|
|
4879
5130
|
moduleId;
|
|
4880
5131
|
constructor(indexDir, moduleId) {
|
|
4881
|
-
this.symbolicPath =
|
|
5132
|
+
this.symbolicPath = path10.join(indexDir, "index", moduleId, "symbolic");
|
|
4882
5133
|
this.moduleId = moduleId;
|
|
4883
5134
|
}
|
|
4884
5135
|
async initialize() {
|
|
@@ -4964,13 +5215,13 @@ class SymbolicIndex {
|
|
|
4964
5215
|
if (this.bm25Index) {
|
|
4965
5216
|
this.meta.bm25Serialized = this.bm25Index.serialize();
|
|
4966
5217
|
}
|
|
4967
|
-
await
|
|
4968
|
-
const metaPath =
|
|
4969
|
-
await
|
|
5218
|
+
await fs4.mkdir(this.symbolicPath, { recursive: true });
|
|
5219
|
+
const metaPath = path10.join(this.symbolicPath, "_meta.json");
|
|
5220
|
+
await fs4.writeFile(metaPath, JSON.stringify(this.meta, null, 2));
|
|
4970
5221
|
for (const [filepath, summary] of this.fileSummaries) {
|
|
4971
5222
|
const summaryPath = this.getFileSummaryPath(filepath);
|
|
4972
|
-
await
|
|
4973
|
-
await
|
|
5223
|
+
await fs4.mkdir(path10.dirname(summaryPath), { recursive: true });
|
|
5224
|
+
await fs4.writeFile(summaryPath, JSON.stringify(summary, null, 2));
|
|
4974
5225
|
}
|
|
4975
5226
|
}
|
|
4976
5227
|
async saveIncremental(filepaths) {
|
|
@@ -4981,21 +5232,21 @@ class SymbolicIndex {
|
|
|
4981
5232
|
if (this.bm25Index) {
|
|
4982
5233
|
this.meta.bm25Serialized = this.bm25Index.serialize();
|
|
4983
5234
|
}
|
|
4984
|
-
await
|
|
4985
|
-
const metaPath =
|
|
4986
|
-
await
|
|
5235
|
+
await fs4.mkdir(this.symbolicPath, { recursive: true });
|
|
5236
|
+
const metaPath = path10.join(this.symbolicPath, "_meta.json");
|
|
5237
|
+
await fs4.writeFile(metaPath, JSON.stringify(this.meta, null, 2));
|
|
4987
5238
|
for (const filepath of filepaths) {
|
|
4988
5239
|
const summary = this.fileSummaries.get(filepath);
|
|
4989
5240
|
if (summary) {
|
|
4990
5241
|
const summaryPath = this.getFileSummaryPath(filepath);
|
|
4991
|
-
await
|
|
4992
|
-
await
|
|
5242
|
+
await fs4.mkdir(path10.dirname(summaryPath), { recursive: true });
|
|
5243
|
+
await fs4.writeFile(summaryPath, JSON.stringify(summary, null, 2));
|
|
4993
5244
|
}
|
|
4994
5245
|
}
|
|
4995
5246
|
}
|
|
4996
5247
|
async load() {
|
|
4997
|
-
const metaPath =
|
|
4998
|
-
const metaContent = await
|
|
5248
|
+
const metaPath = path10.join(this.symbolicPath, "_meta.json");
|
|
5249
|
+
const metaContent = await fs4.readFile(metaPath, "utf-8");
|
|
4999
5250
|
this.meta = JSON.parse(metaContent);
|
|
5000
5251
|
this.fileSummaries.clear();
|
|
5001
5252
|
await this.loadFileSummariesRecursive(this.symbolicPath);
|
|
@@ -5007,14 +5258,14 @@ class SymbolicIndex {
|
|
|
5007
5258
|
}
|
|
5008
5259
|
async loadFileSummariesRecursive(dir) {
|
|
5009
5260
|
try {
|
|
5010
|
-
const entries = await
|
|
5261
|
+
const entries = await fs4.readdir(dir, { withFileTypes: true });
|
|
5011
5262
|
for (const entry of entries) {
|
|
5012
|
-
const fullPath =
|
|
5263
|
+
const fullPath = path10.join(dir, entry.name);
|
|
5013
5264
|
if (entry.isDirectory()) {
|
|
5014
5265
|
await this.loadFileSummariesRecursive(fullPath);
|
|
5015
5266
|
} else if (entry.name.endsWith(".json") && entry.name !== "_meta.json") {
|
|
5016
5267
|
try {
|
|
5017
|
-
const content = await
|
|
5268
|
+
const content = await fs4.readFile(fullPath, "utf-8");
|
|
5018
5269
|
const summary = JSON.parse(content);
|
|
5019
5270
|
if (summary.filepath) {
|
|
5020
5271
|
this.fileSummaries.set(summary.filepath, summary);
|
|
@@ -5026,18 +5277,18 @@ class SymbolicIndex {
|
|
|
5026
5277
|
}
|
|
5027
5278
|
getFileSummaryPath(filepath) {
|
|
5028
5279
|
const jsonPath = filepath.replace(/\.[^.]+$/, ".json");
|
|
5029
|
-
return
|
|
5280
|
+
return path10.join(this.symbolicPath, jsonPath);
|
|
5030
5281
|
}
|
|
5031
5282
|
async deleteFileSummary(filepath) {
|
|
5032
5283
|
try {
|
|
5033
|
-
await
|
|
5284
|
+
await fs4.unlink(this.getFileSummaryPath(filepath));
|
|
5034
5285
|
} catch {}
|
|
5035
5286
|
this.fileSummaries.delete(filepath);
|
|
5036
5287
|
}
|
|
5037
5288
|
async exists() {
|
|
5038
5289
|
try {
|
|
5039
|
-
const metaPath =
|
|
5040
|
-
await
|
|
5290
|
+
const metaPath = path10.join(this.symbolicPath, "_meta.json");
|
|
5291
|
+
await fs4.access(metaPath);
|
|
5041
5292
|
return true;
|
|
5042
5293
|
} catch {
|
|
5043
5294
|
return false;
|
|
@@ -5069,8 +5320,8 @@ __export(exports_literalIndex, {
|
|
|
5069
5320
|
getLiteralIndexPath: () => getLiteralIndexPath,
|
|
5070
5321
|
LiteralIndex: () => LiteralIndex
|
|
5071
5322
|
});
|
|
5072
|
-
import * as
|
|
5073
|
-
import * as
|
|
5323
|
+
import * as fs5 from "fs/promises";
|
|
5324
|
+
import * as path11 from "path";
|
|
5074
5325
|
|
|
5075
5326
|
class LiteralIndex {
|
|
5076
5327
|
indexPath;
|
|
@@ -5079,7 +5330,7 @@ class LiteralIndex {
|
|
|
5079
5330
|
vocabularyIndex = new Map;
|
|
5080
5331
|
static VERSION = "1.1.0";
|
|
5081
5332
|
constructor(indexDir, moduleId) {
|
|
5082
|
-
this.indexPath =
|
|
5333
|
+
this.indexPath = path11.join(indexDir, "index", moduleId, "literals");
|
|
5083
5334
|
this.moduleId = moduleId;
|
|
5084
5335
|
}
|
|
5085
5336
|
async initialize() {
|
|
@@ -5240,17 +5491,17 @@ class LiteralIndex {
|
|
|
5240
5491
|
}));
|
|
5241
5492
|
}
|
|
5242
5493
|
async save() {
|
|
5243
|
-
await
|
|
5494
|
+
await fs5.mkdir(this.indexPath, { recursive: true });
|
|
5244
5495
|
const data = {
|
|
5245
5496
|
version: LiteralIndex.VERSION,
|
|
5246
5497
|
entries: Object.fromEntries(this.entries)
|
|
5247
5498
|
};
|
|
5248
|
-
const indexFile =
|
|
5249
|
-
await
|
|
5499
|
+
const indexFile = path11.join(this.indexPath, "_index.json");
|
|
5500
|
+
await fs5.writeFile(indexFile, JSON.stringify(data, null, 2));
|
|
5250
5501
|
}
|
|
5251
5502
|
async load() {
|
|
5252
|
-
const indexFile =
|
|
5253
|
-
const content = await
|
|
5503
|
+
const indexFile = path11.join(this.indexPath, "_index.json");
|
|
5504
|
+
const content = await fs5.readFile(indexFile, "utf-8");
|
|
5254
5505
|
const data = JSON.parse(content);
|
|
5255
5506
|
if (data.version !== LiteralIndex.VERSION) {
|
|
5256
5507
|
console.warn(`Literal index version mismatch: expected ${LiteralIndex.VERSION}, got ${data.version}`);
|
|
@@ -5259,8 +5510,8 @@ class LiteralIndex {
|
|
|
5259
5510
|
}
|
|
5260
5511
|
async exists() {
|
|
5261
5512
|
try {
|
|
5262
|
-
const indexFile =
|
|
5263
|
-
await
|
|
5513
|
+
const indexFile = path11.join(this.indexPath, "_index.json");
|
|
5514
|
+
await fs5.access(indexFile);
|
|
5264
5515
|
return true;
|
|
5265
5516
|
} catch {
|
|
5266
5517
|
return false;
|
|
@@ -5303,7 +5554,7 @@ function shouldReplaceMatchType(existing, incoming) {
|
|
|
5303
5554
|
return priority[incoming] > priority[existing];
|
|
5304
5555
|
}
|
|
5305
5556
|
function getLiteralIndexPath(rootDir, moduleId, indexDir = ".raggrep") {
|
|
5306
|
-
return
|
|
5557
|
+
return path11.join(rootDir, indexDir, "index", moduleId, "literals");
|
|
5307
5558
|
}
|
|
5308
5559
|
var init_literalIndex = () => {};
|
|
5309
5560
|
|
|
@@ -5324,9 +5575,9 @@ __export(exports_typescript, {
|
|
|
5324
5575
|
DEFAULT_TOP_K: () => DEFAULT_TOP_K2,
|
|
5325
5576
|
DEFAULT_MIN_SCORE: () => DEFAULT_MIN_SCORE2
|
|
5326
5577
|
});
|
|
5327
|
-
import * as
|
|
5578
|
+
import * as path12 from "path";
|
|
5328
5579
|
function isTypeScriptFile(filepath) {
|
|
5329
|
-
const ext =
|
|
5580
|
+
const ext = path12.extname(filepath).toLowerCase();
|
|
5330
5581
|
return TYPESCRIPT_EXTENSIONS.includes(ext);
|
|
5331
5582
|
}
|
|
5332
5583
|
function calculateChunkTypeBoost(chunk) {
|
|
@@ -5388,8 +5639,6 @@ class TypeScriptModule {
|
|
|
5388
5639
|
if (parsedChunks.length === 0) {
|
|
5389
5640
|
return null;
|
|
5390
5641
|
}
|
|
5391
|
-
const pathContext = parsePathContext(filepath);
|
|
5392
|
-
const pathPrefix = formatPathContextForEmbedding(pathContext);
|
|
5393
5642
|
const includeFullFileChunk = parsedChunks.length > 1;
|
|
5394
5643
|
const allParsedChunks = [...parsedChunks];
|
|
5395
5644
|
if (includeFullFileChunk) {
|
|
@@ -5400,13 +5649,17 @@ class TypeScriptModule {
|
|
|
5400
5649
|
startLine: 1,
|
|
5401
5650
|
endLine: lines.length,
|
|
5402
5651
|
type: "file",
|
|
5403
|
-
name:
|
|
5652
|
+
name: path12.basename(filepath),
|
|
5404
5653
|
isExported: false
|
|
5405
5654
|
});
|
|
5406
5655
|
}
|
|
5407
5656
|
const chunkContents = allParsedChunks.map((c) => {
|
|
5408
|
-
|
|
5409
|
-
|
|
5657
|
+
return prepareChunkForEmbedding({
|
|
5658
|
+
filepath,
|
|
5659
|
+
content: c.content,
|
|
5660
|
+
name: c.name,
|
|
5661
|
+
docComment: c.jsDoc
|
|
5662
|
+
});
|
|
5410
5663
|
});
|
|
5411
5664
|
const embeddings = await getEmbeddings(chunkContents);
|
|
5412
5665
|
const chunks = allParsedChunks.map((pc) => ({
|
|
@@ -5430,25 +5683,21 @@ class TypeScriptModule {
|
|
|
5430
5683
|
...new Set(parsedChunks.map((pc) => pc.type))
|
|
5431
5684
|
];
|
|
5432
5685
|
const exports = parsedChunks.filter((pc) => pc.isExported && pc.name).map((pc) => pc.name);
|
|
5433
|
-
const
|
|
5686
|
+
const contentKeywords = new Set;
|
|
5434
5687
|
for (const pc of parsedChunks) {
|
|
5435
5688
|
const keywords = extractKeywords(pc.content, pc.name);
|
|
5436
|
-
keywords.forEach((k) =>
|
|
5689
|
+
keywords.forEach((k) => contentKeywords.add(k));
|
|
5437
5690
|
}
|
|
5438
|
-
|
|
5691
|
+
const pathKeywords = extractPathKeywordsForFileSummary(filepath);
|
|
5692
|
+
const allKeywords = [...contentKeywords, ...pathKeywords];
|
|
5439
5693
|
const fileSummary = {
|
|
5440
5694
|
filepath,
|
|
5441
5695
|
chunkCount: chunks.length,
|
|
5442
5696
|
chunkTypes,
|
|
5443
|
-
keywords:
|
|
5697
|
+
keywords: [...new Set(allKeywords)],
|
|
5444
5698
|
exports,
|
|
5445
5699
|
lastModified: stats.lastModified,
|
|
5446
|
-
pathContext:
|
|
5447
|
-
segments: pathContext.segments,
|
|
5448
|
-
layer: pathContext.layer,
|
|
5449
|
-
domain: pathContext.domain,
|
|
5450
|
-
depth: pathContext.depth
|
|
5451
|
-
}
|
|
5700
|
+
pathContext: getPathContextForFileSummary(filepath)
|
|
5452
5701
|
};
|
|
5453
5702
|
this.pendingSummaries.set(filepath, fileSummary);
|
|
5454
5703
|
for (const chunk of chunks) {
|
|
@@ -5727,16 +5976,16 @@ class TypeScriptModule {
|
|
|
5727
5976
|
while ((match = importRegex.exec(content)) !== null) {
|
|
5728
5977
|
const importPath = match[1];
|
|
5729
5978
|
if (importPath.startsWith(".")) {
|
|
5730
|
-
const dir =
|
|
5731
|
-
const resolved =
|
|
5979
|
+
const dir = path12.dirname(filepath);
|
|
5980
|
+
const resolved = path12.normalize(path12.join(dir, importPath));
|
|
5732
5981
|
references.push(resolved);
|
|
5733
5982
|
}
|
|
5734
5983
|
}
|
|
5735
5984
|
while ((match = requireRegex.exec(content)) !== null) {
|
|
5736
5985
|
const importPath = match[1];
|
|
5737
5986
|
if (importPath.startsWith(".")) {
|
|
5738
|
-
const dir =
|
|
5739
|
-
const resolved =
|
|
5987
|
+
const dir = path12.dirname(filepath);
|
|
5988
|
+
const resolved = path12.normalize(path12.join(dir, importPath));
|
|
5740
5989
|
references.push(resolved);
|
|
5741
5990
|
}
|
|
5742
5991
|
}
|
|
@@ -5764,7 +6013,7 @@ var init_typescript = __esm(() => {
|
|
|
5764
6013
|
});
|
|
5765
6014
|
|
|
5766
6015
|
// src/infrastructure/parsing/typescriptParser.ts
|
|
5767
|
-
import * as
|
|
6016
|
+
import * as path13 from "path";
|
|
5768
6017
|
|
|
5769
6018
|
class TypeScriptParser {
|
|
5770
6019
|
supportedLanguages = ["typescript", "javascript"];
|
|
@@ -5780,12 +6029,12 @@ class TypeScriptParser {
|
|
|
5780
6029
|
startLine: 1,
|
|
5781
6030
|
endLine: lines.length,
|
|
5782
6031
|
type: "file",
|
|
5783
|
-
name:
|
|
6032
|
+
name: path13.basename(filepath),
|
|
5784
6033
|
isExported: false
|
|
5785
6034
|
};
|
|
5786
6035
|
chunks.unshift(fullFileChunk);
|
|
5787
6036
|
}
|
|
5788
|
-
const ext =
|
|
6037
|
+
const ext = path13.extname(filepath).toLowerCase();
|
|
5789
6038
|
const language = ext === ".js" || ext === ".jsx" || ext === ".mjs" || ext === ".cjs" ? "javascript" : "typescript";
|
|
5790
6039
|
return {
|
|
5791
6040
|
chunks,
|
|
@@ -5802,7 +6051,7 @@ class TypeScriptParser {
|
|
|
5802
6051
|
}
|
|
5803
6052
|
}
|
|
5804
6053
|
canParse(filepath) {
|
|
5805
|
-
const ext =
|
|
6054
|
+
const ext = path13.extname(filepath).toLowerCase();
|
|
5806
6055
|
return TYPESCRIPT_EXTENSIONS2.includes(ext);
|
|
5807
6056
|
}
|
|
5808
6057
|
convertChunk(tc) {
|
|
@@ -5817,7 +6066,7 @@ class TypeScriptParser {
|
|
|
5817
6066
|
};
|
|
5818
6067
|
}
|
|
5819
6068
|
detectLanguage(filepath) {
|
|
5820
|
-
const ext =
|
|
6069
|
+
const ext = path13.extname(filepath).toLowerCase();
|
|
5821
6070
|
if ([".js", ".jsx", ".mjs", ".cjs"].includes(ext)) {
|
|
5822
6071
|
return "javascript";
|
|
5823
6072
|
}
|
|
@@ -6022,8 +6271,8 @@ var init_grammarManager = __esm(() => {
|
|
|
6022
6271
|
});
|
|
6023
6272
|
|
|
6024
6273
|
// src/infrastructure/parsing/treeSitterParser.ts
|
|
6025
|
-
import * as
|
|
6026
|
-
import * as
|
|
6274
|
+
import * as path14 from "path";
|
|
6275
|
+
import * as fs6 from "fs";
|
|
6027
6276
|
|
|
6028
6277
|
class TreeSitterParser {
|
|
6029
6278
|
supportedLanguages = [
|
|
@@ -6045,7 +6294,7 @@ class TreeSitterParser {
|
|
|
6045
6294
|
chunks: [],
|
|
6046
6295
|
language: "typescript",
|
|
6047
6296
|
success: false,
|
|
6048
|
-
error: `Unsupported file type: ${
|
|
6297
|
+
error: `Unsupported file type: ${path14.extname(filepath)}`
|
|
6049
6298
|
};
|
|
6050
6299
|
}
|
|
6051
6300
|
try {
|
|
@@ -6065,11 +6314,11 @@ class TreeSitterParser {
|
|
|
6065
6314
|
}
|
|
6066
6315
|
}
|
|
6067
6316
|
canParse(filepath) {
|
|
6068
|
-
const ext =
|
|
6317
|
+
const ext = path14.extname(filepath).toLowerCase();
|
|
6069
6318
|
return ext in EXTENSION_TO_LANGUAGE2;
|
|
6070
6319
|
}
|
|
6071
6320
|
detectLanguage(filepath) {
|
|
6072
|
-
const ext =
|
|
6321
|
+
const ext = path14.extname(filepath).toLowerCase();
|
|
6073
6322
|
return EXTENSION_TO_LANGUAGE2[ext] || null;
|
|
6074
6323
|
}
|
|
6075
6324
|
async ensureInitialized() {
|
|
@@ -6103,20 +6352,20 @@ class TreeSitterParser {
|
|
|
6103
6352
|
async resolveWasmPath() {
|
|
6104
6353
|
try {
|
|
6105
6354
|
const webTreeSitterPath = __require.resolve("web-tree-sitter");
|
|
6106
|
-
const wasmPath =
|
|
6107
|
-
if (
|
|
6355
|
+
const wasmPath = path14.join(path14.dirname(webTreeSitterPath), "web-tree-sitter.wasm");
|
|
6356
|
+
if (fs6.existsSync(wasmPath)) {
|
|
6108
6357
|
return wasmPath;
|
|
6109
6358
|
}
|
|
6110
6359
|
} catch {}
|
|
6111
6360
|
try {
|
|
6112
6361
|
const possiblePaths = [
|
|
6113
|
-
|
|
6114
|
-
|
|
6115
|
-
|
|
6116
|
-
|
|
6362
|
+
path14.join(__dirname, "../../../node_modules/web-tree-sitter/web-tree-sitter.wasm"),
|
|
6363
|
+
path14.join(__dirname, "../../node_modules/web-tree-sitter/web-tree-sitter.wasm"),
|
|
6364
|
+
path14.join(__dirname, "../../../../node_modules/web-tree-sitter/web-tree-sitter.wasm"),
|
|
6365
|
+
path14.join(__dirname, "web-tree-sitter.wasm")
|
|
6117
6366
|
];
|
|
6118
6367
|
for (const wasmPath of possiblePaths) {
|
|
6119
|
-
if (
|
|
6368
|
+
if (fs6.existsSync(wasmPath)) {
|
|
6120
6369
|
return wasmPath;
|
|
6121
6370
|
}
|
|
6122
6371
|
}
|
|
@@ -6157,7 +6406,7 @@ class TreeSitterParser {
|
|
|
6157
6406
|
startLine: 1,
|
|
6158
6407
|
endLine: lines.length,
|
|
6159
6408
|
type: "file",
|
|
6160
|
-
name:
|
|
6409
|
+
name: path14.basename(filepath),
|
|
6161
6410
|
isExported: false
|
|
6162
6411
|
});
|
|
6163
6412
|
}
|
|
@@ -6504,7 +6753,7 @@ class TreeSitterParser {
|
|
|
6504
6753
|
startLine: 1,
|
|
6505
6754
|
endLine: lines.length,
|
|
6506
6755
|
type: "file",
|
|
6507
|
-
name:
|
|
6756
|
+
name: path14.basename(filepath)
|
|
6508
6757
|
});
|
|
6509
6758
|
return {
|
|
6510
6759
|
chunks,
|
|
@@ -6513,7 +6762,7 @@ class TreeSitterParser {
|
|
|
6513
6762
|
};
|
|
6514
6763
|
}
|
|
6515
6764
|
}
|
|
6516
|
-
var __dirname = "/
|
|
6765
|
+
var __dirname = "/home/runner/work/raggrep/raggrep/src/infrastructure/parsing", EXTENSION_TO_LANGUAGE2;
|
|
6517
6766
|
var init_treeSitterParser = __esm(() => {
|
|
6518
6767
|
init_grammarManager();
|
|
6519
6768
|
EXTENSION_TO_LANGUAGE2 = {
|
|
@@ -6534,7 +6783,7 @@ var init_treeSitterParser = __esm(() => {
|
|
|
6534
6783
|
});
|
|
6535
6784
|
|
|
6536
6785
|
// src/infrastructure/parsing/parserFactory.ts
|
|
6537
|
-
import * as
|
|
6786
|
+
import * as path15 from "path";
|
|
6538
6787
|
function getTypeScriptParser() {
|
|
6539
6788
|
if (!typescriptParserInstance) {
|
|
6540
6789
|
typescriptParserInstance = new TypeScriptParser;
|
|
@@ -6548,7 +6797,7 @@ function getTreeSitterParser() {
|
|
|
6548
6797
|
return treeSitterParserInstance;
|
|
6549
6798
|
}
|
|
6550
6799
|
function createParserForFile(filepath) {
|
|
6551
|
-
const ext =
|
|
6800
|
+
const ext = path15.extname(filepath).toLowerCase();
|
|
6552
6801
|
const parserType = EXTENSION_PARSER_MAP[ext];
|
|
6553
6802
|
if (!parserType) {
|
|
6554
6803
|
return null;
|
|
@@ -6597,9 +6846,9 @@ __export(exports_python, {
|
|
|
6597
6846
|
DEFAULT_TOP_K: () => DEFAULT_TOP_K3,
|
|
6598
6847
|
DEFAULT_MIN_SCORE: () => DEFAULT_MIN_SCORE3
|
|
6599
6848
|
});
|
|
6600
|
-
import * as
|
|
6849
|
+
import * as path16 from "path";
|
|
6601
6850
|
function isPythonFile(filepath) {
|
|
6602
|
-
const ext =
|
|
6851
|
+
const ext = path16.extname(filepath).toLowerCase();
|
|
6603
6852
|
return PYTHON_EXTENSIONS.includes(ext);
|
|
6604
6853
|
}
|
|
6605
6854
|
function generateChunkId3(filepath, startLine, endLine) {
|
|
@@ -6683,7 +6932,7 @@ class PythonModule {
|
|
|
6683
6932
|
startLine: 1,
|
|
6684
6933
|
endLine: lines.length,
|
|
6685
6934
|
type: "file",
|
|
6686
|
-
name:
|
|
6935
|
+
name: path16.basename(filepath)
|
|
6687
6936
|
});
|
|
6688
6937
|
const funcRegex = /^(\s*)(async\s+)?def\s+(\w+)\s*\([^)]*\)\s*:/gm;
|
|
6689
6938
|
let match;
|
|
@@ -6761,12 +7010,13 @@ class PythonModule {
|
|
|
6761
7010
|
return chunks;
|
|
6762
7011
|
}
|
|
6763
7012
|
async createFileIndex(filepath, content, parsedChunks, ctx) {
|
|
6764
|
-
const pathContext = parsePathContext(filepath);
|
|
6765
|
-
const pathPrefix = formatPathContextForEmbedding(pathContext);
|
|
6766
7013
|
const chunkContents = parsedChunks.map((c) => {
|
|
6767
|
-
|
|
6768
|
-
|
|
6769
|
-
|
|
7014
|
+
return prepareChunkForEmbedding({
|
|
7015
|
+
filepath,
|
|
7016
|
+
content: c.content,
|
|
7017
|
+
name: c.name,
|
|
7018
|
+
docComment: c.docComment
|
|
7019
|
+
});
|
|
6770
7020
|
});
|
|
6771
7021
|
const embeddings = await getEmbeddings(chunkContents);
|
|
6772
7022
|
const chunks = parsedChunks.map((pc) => ({
|
|
@@ -6789,25 +7039,21 @@ class PythonModule {
|
|
|
6789
7039
|
...new Set(parsedChunks.map((pc) => pc.type))
|
|
6790
7040
|
];
|
|
6791
7041
|
const exports = parsedChunks.filter((pc) => pc.isExported && pc.name).map((pc) => pc.name);
|
|
6792
|
-
const
|
|
7042
|
+
const contentKeywords = new Set;
|
|
6793
7043
|
for (const pc of parsedChunks) {
|
|
6794
7044
|
const keywords = extractKeywords(pc.content, pc.name);
|
|
6795
|
-
keywords.forEach((k) =>
|
|
7045
|
+
keywords.forEach((k) => contentKeywords.add(k));
|
|
6796
7046
|
}
|
|
6797
|
-
|
|
7047
|
+
const pathKeywords = extractPathKeywordsForFileSummary(filepath);
|
|
7048
|
+
const allKeywords = [...contentKeywords, ...pathKeywords];
|
|
6798
7049
|
const fileSummary = {
|
|
6799
7050
|
filepath,
|
|
6800
7051
|
chunkCount: chunks.length,
|
|
6801
7052
|
chunkTypes,
|
|
6802
|
-
keywords:
|
|
7053
|
+
keywords: [...new Set(allKeywords)],
|
|
6803
7054
|
exports,
|
|
6804
7055
|
lastModified: stats.lastModified,
|
|
6805
|
-
pathContext:
|
|
6806
|
-
segments: pathContext.segments,
|
|
6807
|
-
layer: pathContext.layer,
|
|
6808
|
-
domain: pathContext.domain,
|
|
6809
|
-
depth: pathContext.depth
|
|
6810
|
-
}
|
|
7056
|
+
pathContext: getPathContextForFileSummary(filepath)
|
|
6811
7057
|
};
|
|
6812
7058
|
this.pendingSummaries.set(filepath, fileSummary);
|
|
6813
7059
|
for (const chunk of chunks) {
|
|
@@ -7060,9 +7306,9 @@ __export(exports_go, {
|
|
|
7060
7306
|
DEFAULT_TOP_K: () => DEFAULT_TOP_K4,
|
|
7061
7307
|
DEFAULT_MIN_SCORE: () => DEFAULT_MIN_SCORE4
|
|
7062
7308
|
});
|
|
7063
|
-
import * as
|
|
7309
|
+
import * as path17 from "path";
|
|
7064
7310
|
function isGoFile(filepath) {
|
|
7065
|
-
const ext =
|
|
7311
|
+
const ext = path17.extname(filepath).toLowerCase();
|
|
7066
7312
|
return GO_EXTENSIONS.includes(ext);
|
|
7067
7313
|
}
|
|
7068
7314
|
function generateChunkId4(filepath, startLine, endLine) {
|
|
@@ -7147,7 +7393,7 @@ class GoModule {
|
|
|
7147
7393
|
startLine: 1,
|
|
7148
7394
|
endLine: lines.length,
|
|
7149
7395
|
type: "file",
|
|
7150
|
-
name:
|
|
7396
|
+
name: path17.basename(filepath)
|
|
7151
7397
|
});
|
|
7152
7398
|
const funcRegex = /^func\s+(?:\(\s*\w+\s+\*?\w+\s*\)\s+)?(\w+)\s*\(/gm;
|
|
7153
7399
|
let match;
|
|
@@ -7296,12 +7542,13 @@ class GoModule {
|
|
|
7296
7542
|
return chunks;
|
|
7297
7543
|
}
|
|
7298
7544
|
async createFileIndex(filepath, content, parsedChunks, ctx) {
|
|
7299
|
-
const pathContext = parsePathContext(filepath);
|
|
7300
|
-
const pathPrefix = formatPathContextForEmbedding(pathContext);
|
|
7301
7545
|
const chunkContents = parsedChunks.map((c) => {
|
|
7302
|
-
|
|
7303
|
-
|
|
7304
|
-
|
|
7546
|
+
return prepareChunkForEmbedding({
|
|
7547
|
+
filepath,
|
|
7548
|
+
content: c.content,
|
|
7549
|
+
name: c.name,
|
|
7550
|
+
docComment: c.docComment
|
|
7551
|
+
});
|
|
7305
7552
|
});
|
|
7306
7553
|
const embeddings = await getEmbeddings(chunkContents);
|
|
7307
7554
|
const chunks = parsedChunks.map((pc) => ({
|
|
@@ -7324,25 +7571,21 @@ class GoModule {
|
|
|
7324
7571
|
...new Set(parsedChunks.map((pc) => pc.type))
|
|
7325
7572
|
];
|
|
7326
7573
|
const exports = parsedChunks.filter((pc) => pc.isExported && pc.name).map((pc) => pc.name);
|
|
7327
|
-
const
|
|
7574
|
+
const contentKeywords = new Set;
|
|
7328
7575
|
for (const pc of parsedChunks) {
|
|
7329
7576
|
const keywords = extractKeywords(pc.content, pc.name);
|
|
7330
|
-
keywords.forEach((k) =>
|
|
7577
|
+
keywords.forEach((k) => contentKeywords.add(k));
|
|
7331
7578
|
}
|
|
7332
|
-
|
|
7579
|
+
const pathKeywords = extractPathKeywordsForFileSummary(filepath);
|
|
7580
|
+
const allKeywords = [...contentKeywords, ...pathKeywords];
|
|
7333
7581
|
const fileSummary = {
|
|
7334
7582
|
filepath,
|
|
7335
7583
|
chunkCount: chunks.length,
|
|
7336
7584
|
chunkTypes,
|
|
7337
|
-
keywords:
|
|
7585
|
+
keywords: [...new Set(allKeywords)],
|
|
7338
7586
|
exports,
|
|
7339
7587
|
lastModified: stats.lastModified,
|
|
7340
|
-
pathContext:
|
|
7341
|
-
segments: pathContext.segments,
|
|
7342
|
-
layer: pathContext.layer,
|
|
7343
|
-
domain: pathContext.domain,
|
|
7344
|
-
depth: pathContext.depth
|
|
7345
|
-
}
|
|
7588
|
+
pathContext: getPathContextForFileSummary(filepath)
|
|
7346
7589
|
};
|
|
7347
7590
|
this.pendingSummaries.set(filepath, fileSummary);
|
|
7348
7591
|
for (const chunk of chunks) {
|
|
@@ -7544,9 +7787,9 @@ __export(exports_rust, {
|
|
|
7544
7787
|
DEFAULT_TOP_K: () => DEFAULT_TOP_K5,
|
|
7545
7788
|
DEFAULT_MIN_SCORE: () => DEFAULT_MIN_SCORE5
|
|
7546
7789
|
});
|
|
7547
|
-
import * as
|
|
7790
|
+
import * as path18 from "path";
|
|
7548
7791
|
function isRustFile(filepath) {
|
|
7549
|
-
const ext =
|
|
7792
|
+
const ext = path18.extname(filepath).toLowerCase();
|
|
7550
7793
|
return RUST_EXTENSIONS.includes(ext);
|
|
7551
7794
|
}
|
|
7552
7795
|
function generateChunkId5(filepath, startLine, endLine) {
|
|
@@ -7633,7 +7876,7 @@ class RustModule {
|
|
|
7633
7876
|
startLine: 1,
|
|
7634
7877
|
endLine: lines.length,
|
|
7635
7878
|
type: "file",
|
|
7636
|
-
name:
|
|
7879
|
+
name: path18.basename(filepath)
|
|
7637
7880
|
});
|
|
7638
7881
|
const funcRegex = /^(pub(?:\s*\([^)]*\))?\s+)?(?:async\s+)?fn\s+(\w+)/gm;
|
|
7639
7882
|
let match;
|
|
@@ -7859,12 +8102,13 @@ class RustModule {
|
|
|
7859
8102
|
return chunks;
|
|
7860
8103
|
}
|
|
7861
8104
|
async createFileIndex(filepath, content, parsedChunks, ctx) {
|
|
7862
|
-
const pathContext = parsePathContext(filepath);
|
|
7863
|
-
const pathPrefix = formatPathContextForEmbedding(pathContext);
|
|
7864
8105
|
const chunkContents = parsedChunks.map((c) => {
|
|
7865
|
-
|
|
7866
|
-
|
|
7867
|
-
|
|
8106
|
+
return prepareChunkForEmbedding({
|
|
8107
|
+
filepath,
|
|
8108
|
+
content: c.content,
|
|
8109
|
+
name: c.name,
|
|
8110
|
+
docComment: c.docComment
|
|
8111
|
+
});
|
|
7868
8112
|
});
|
|
7869
8113
|
const embeddings = await getEmbeddings(chunkContents);
|
|
7870
8114
|
const chunks = parsedChunks.map((pc) => ({
|
|
@@ -7887,25 +8131,21 @@ class RustModule {
|
|
|
7887
8131
|
...new Set(parsedChunks.map((pc) => pc.type))
|
|
7888
8132
|
];
|
|
7889
8133
|
const exports = parsedChunks.filter((pc) => pc.isExported && pc.name).map((pc) => pc.name);
|
|
7890
|
-
const
|
|
8134
|
+
const contentKeywords = new Set;
|
|
7891
8135
|
for (const pc of parsedChunks) {
|
|
7892
8136
|
const keywords = extractKeywords(pc.content, pc.name);
|
|
7893
|
-
keywords.forEach((k) =>
|
|
8137
|
+
keywords.forEach((k) => contentKeywords.add(k));
|
|
7894
8138
|
}
|
|
7895
|
-
|
|
8139
|
+
const pathKeywords = extractPathKeywordsForFileSummary(filepath);
|
|
8140
|
+
const allKeywords = [...contentKeywords, ...pathKeywords];
|
|
7896
8141
|
const fileSummary = {
|
|
7897
8142
|
filepath,
|
|
7898
8143
|
chunkCount: chunks.length,
|
|
7899
8144
|
chunkTypes,
|
|
7900
|
-
keywords:
|
|
8145
|
+
keywords: [...new Set(allKeywords)],
|
|
7901
8146
|
exports,
|
|
7902
8147
|
lastModified: stats.lastModified,
|
|
7903
|
-
pathContext:
|
|
7904
|
-
segments: pathContext.segments,
|
|
7905
|
-
layer: pathContext.layer,
|
|
7906
|
-
domain: pathContext.domain,
|
|
7907
|
-
depth: pathContext.depth
|
|
7908
|
-
}
|
|
8148
|
+
pathContext: getPathContextForFileSummary(filepath)
|
|
7909
8149
|
};
|
|
7910
8150
|
this.pendingSummaries.set(filepath, fileSummary);
|
|
7911
8151
|
for (const chunk of chunks) {
|
|
@@ -8107,9 +8347,9 @@ __export(exports_json, {
|
|
|
8107
8347
|
DEFAULT_TOP_K: () => DEFAULT_TOP_K6,
|
|
8108
8348
|
DEFAULT_MIN_SCORE: () => DEFAULT_MIN_SCORE6
|
|
8109
8349
|
});
|
|
8110
|
-
import * as
|
|
8350
|
+
import * as path19 from "path";
|
|
8111
8351
|
function isJsonFile(filepath) {
|
|
8112
|
-
const ext =
|
|
8352
|
+
const ext = path19.extname(filepath).toLowerCase();
|
|
8113
8353
|
return JSON_EXTENSIONS.includes(ext);
|
|
8114
8354
|
}
|
|
8115
8355
|
|
|
@@ -8143,7 +8383,7 @@ class JsonModule {
|
|
|
8143
8383
|
} catch {
|
|
8144
8384
|
return null;
|
|
8145
8385
|
}
|
|
8146
|
-
const fileBasename =
|
|
8386
|
+
const fileBasename = path19.basename(filepath, path19.extname(filepath));
|
|
8147
8387
|
const jsonPathLiterals = extractJsonPaths(parsed, fileBasename);
|
|
8148
8388
|
const lines = content.split(`
|
|
8149
8389
|
`);
|
|
@@ -8350,7 +8590,7 @@ __export(exports_markdown, {
|
|
|
8350
8590
|
DEFAULT_TOP_K: () => DEFAULT_TOP_K7,
|
|
8351
8591
|
DEFAULT_MIN_SCORE: () => DEFAULT_MIN_SCORE7
|
|
8352
8592
|
});
|
|
8353
|
-
import * as
|
|
8593
|
+
import * as path20 from "path";
|
|
8354
8594
|
function calculateHeadingLevelBoost(chunk) {
|
|
8355
8595
|
const metadata = chunk.metadata;
|
|
8356
8596
|
const level = metadata?.headingLevel ?? 0;
|
|
@@ -8370,7 +8610,7 @@ function calculateHeadingLevelBoost(chunk) {
|
|
|
8370
8610
|
}
|
|
8371
8611
|
}
|
|
8372
8612
|
function isMarkdownFile(filepath) {
|
|
8373
|
-
const ext =
|
|
8613
|
+
const ext = path20.extname(filepath).toLowerCase();
|
|
8374
8614
|
return MARKDOWN_EXTENSIONS.includes(ext);
|
|
8375
8615
|
}
|
|
8376
8616
|
function parseMarkdownHierarchical(content, maxDepth = 4) {
|
|
@@ -8502,9 +8742,11 @@ class MarkdownModule {
|
|
|
8502
8742
|
return null;
|
|
8503
8743
|
}
|
|
8504
8744
|
const chunkContents = hierarchicalChunks.map((s) => {
|
|
8505
|
-
|
|
8506
|
-
|
|
8507
|
-
|
|
8745
|
+
return prepareChunkForEmbedding({
|
|
8746
|
+
filepath,
|
|
8747
|
+
content: s.content,
|
|
8748
|
+
name: s.heading || undefined
|
|
8749
|
+
});
|
|
8508
8750
|
});
|
|
8509
8751
|
const embeddings = await getEmbeddings(chunkContents);
|
|
8510
8752
|
const chunks = hierarchicalChunks.map((section) => ({
|
|
@@ -8528,14 +8770,17 @@ class MarkdownModule {
|
|
|
8528
8770
|
embeddingModel: currentConfig.model,
|
|
8529
8771
|
headings: uniqueHeadings
|
|
8530
8772
|
};
|
|
8531
|
-
const
|
|
8773
|
+
const contentKeywords = extractMarkdownKeywords(content);
|
|
8774
|
+
const pathKeywords = extractPathKeywordsForFileSummary(filepath);
|
|
8775
|
+
const allKeywords = [...new Set([...contentKeywords, ...pathKeywords])];
|
|
8532
8776
|
const fileSummary = {
|
|
8533
8777
|
filepath,
|
|
8534
8778
|
chunkCount: chunks.length,
|
|
8535
8779
|
chunkTypes: ["block"],
|
|
8536
|
-
keywords,
|
|
8780
|
+
keywords: allKeywords,
|
|
8537
8781
|
exports: uniqueHeadings,
|
|
8538
|
-
lastModified: stats.lastModified
|
|
8782
|
+
lastModified: stats.lastModified,
|
|
8783
|
+
pathContext: getPathContextForFileSummary(filepath)
|
|
8539
8784
|
};
|
|
8540
8785
|
this.pendingSummaries.set(filepath, fileSummary);
|
|
8541
8786
|
return {
|
|
@@ -8707,15 +8952,15 @@ var init_registry = __esm(() => {
|
|
|
8707
8952
|
});
|
|
8708
8953
|
|
|
8709
8954
|
// src/infrastructure/introspection/projectDetector.ts
|
|
8710
|
-
import * as
|
|
8711
|
-
import * as
|
|
8955
|
+
import * as path21 from "path";
|
|
8956
|
+
import * as fs7 from "fs/promises";
|
|
8712
8957
|
async function scanForPackageJsons(rootDir, currentDir = "", depth = 0) {
|
|
8713
8958
|
if (depth > MAX_SCAN_DEPTH)
|
|
8714
8959
|
return [];
|
|
8715
8960
|
const results = [];
|
|
8716
|
-
const fullDir = currentDir ?
|
|
8961
|
+
const fullDir = currentDir ? path21.join(rootDir, currentDir) : rootDir;
|
|
8717
8962
|
try {
|
|
8718
|
-
const entries = await
|
|
8963
|
+
const entries = await fs7.readdir(fullDir, { withFileTypes: true });
|
|
8719
8964
|
const hasPackageJson = entries.some((e) => e.isFile() && e.name === "package.json");
|
|
8720
8965
|
if (hasPackageJson && currentDir) {
|
|
8721
8966
|
const info = await parsePackageJson(rootDir, currentDir);
|
|
@@ -8736,10 +8981,10 @@ async function scanForPackageJsons(rootDir, currentDir = "", depth = 0) {
|
|
|
8736
8981
|
}
|
|
8737
8982
|
async function parsePackageJson(rootDir, relativePath) {
|
|
8738
8983
|
try {
|
|
8739
|
-
const packageJsonPath =
|
|
8740
|
-
const content = await
|
|
8984
|
+
const packageJsonPath = path21.join(rootDir, relativePath, "package.json");
|
|
8985
|
+
const content = await fs7.readFile(packageJsonPath, "utf-8");
|
|
8741
8986
|
const pkg = JSON.parse(content);
|
|
8742
|
-
const name = pkg.name ||
|
|
8987
|
+
const name = pkg.name || path21.basename(relativePath);
|
|
8743
8988
|
const deps = { ...pkg.dependencies, ...pkg.devDependencies };
|
|
8744
8989
|
let type = "unknown";
|
|
8745
8990
|
if (deps["next"] || deps["react"] || deps["vue"] || deps["svelte"]) {
|
|
@@ -8775,7 +9020,7 @@ async function detectProjectStructure(rootDir) {
|
|
|
8775
9020
|
const projectMap = new Map;
|
|
8776
9021
|
let isMonorepo = false;
|
|
8777
9022
|
try {
|
|
8778
|
-
const entries = await
|
|
9023
|
+
const entries = await fs7.readdir(rootDir, { withFileTypes: true });
|
|
8779
9024
|
const dirNames = entries.filter((e) => e.isDirectory()).map((e) => e.name);
|
|
8780
9025
|
const monorepoPatterns = ["apps", "packages", "libs", "services"];
|
|
8781
9026
|
const hasMonorepoStructure = monorepoPatterns.some((p) => dirNames.includes(p));
|
|
@@ -8784,9 +9029,9 @@ async function detectProjectStructure(rootDir) {
|
|
|
8784
9029
|
for (const pattern of monorepoPatterns) {
|
|
8785
9030
|
if (!dirNames.includes(pattern))
|
|
8786
9031
|
continue;
|
|
8787
|
-
const patternDir =
|
|
9032
|
+
const patternDir = path21.join(rootDir, pattern);
|
|
8788
9033
|
try {
|
|
8789
|
-
const subDirs = await
|
|
9034
|
+
const subDirs = await fs7.readdir(patternDir, { withFileTypes: true });
|
|
8790
9035
|
for (const subDir of subDirs) {
|
|
8791
9036
|
if (!subDir.isDirectory())
|
|
8792
9037
|
continue;
|
|
@@ -8815,8 +9060,8 @@ async function detectProjectStructure(rootDir) {
|
|
|
8815
9060
|
}
|
|
8816
9061
|
let rootType = "unknown";
|
|
8817
9062
|
try {
|
|
8818
|
-
const rootPkgPath =
|
|
8819
|
-
const rootPkg = JSON.parse(await
|
|
9063
|
+
const rootPkgPath = path21.join(rootDir, "package.json");
|
|
9064
|
+
const rootPkg = JSON.parse(await fs7.readFile(rootPkgPath, "utf-8"));
|
|
8820
9065
|
if (rootPkg.workspaces)
|
|
8821
9066
|
isMonorepo = true;
|
|
8822
9067
|
const deps = { ...rootPkg.dependencies, ...rootPkg.devDependencies };
|
|
@@ -8855,8 +9100,8 @@ var init_projectDetector = __esm(() => {
|
|
|
8855
9100
|
});
|
|
8856
9101
|
|
|
8857
9102
|
// src/infrastructure/introspection/IntrospectionIndex.ts
|
|
8858
|
-
import * as
|
|
8859
|
-
import * as
|
|
9103
|
+
import * as path22 from "path";
|
|
9104
|
+
import * as fs8 from "fs/promises";
|
|
8860
9105
|
import * as fsSync from "fs";
|
|
8861
9106
|
|
|
8862
9107
|
class IntrospectionIndex {
|
|
@@ -8870,8 +9115,8 @@ class IntrospectionIndex {
|
|
|
8870
9115
|
async initialize() {
|
|
8871
9116
|
this.structure = await detectProjectStructure(this.rootDir);
|
|
8872
9117
|
try {
|
|
8873
|
-
const configPath =
|
|
8874
|
-
const configContent = await
|
|
9118
|
+
const configPath = path22.join(this.rootDir, ".raggrep", "config.json");
|
|
9119
|
+
const configContent = await fs8.readFile(configPath, "utf-8");
|
|
8875
9120
|
const config = JSON.parse(configContent);
|
|
8876
9121
|
this.config = config.introspection || {};
|
|
8877
9122
|
} catch {}
|
|
@@ -8885,7 +9130,7 @@ class IntrospectionIndex {
|
|
|
8885
9130
|
}
|
|
8886
9131
|
const fileExists = enableReadmeContext ? (relativePath) => {
|
|
8887
9132
|
try {
|
|
8888
|
-
const absolutePath =
|
|
9133
|
+
const absolutePath = path22.join(this.rootDir, relativePath);
|
|
8889
9134
|
return fsSync.existsSync(absolutePath);
|
|
8890
9135
|
} catch {
|
|
8891
9136
|
return false;
|
|
@@ -8921,28 +9166,28 @@ class IntrospectionIndex {
|
|
|
8921
9166
|
}
|
|
8922
9167
|
}
|
|
8923
9168
|
async save(config) {
|
|
8924
|
-
const introDir =
|
|
8925
|
-
await
|
|
8926
|
-
const projectPath =
|
|
8927
|
-
await
|
|
9169
|
+
const introDir = path22.join(getRaggrepDir(this.rootDir, config), "introspection");
|
|
9170
|
+
await fs8.mkdir(introDir, { recursive: true });
|
|
9171
|
+
const projectPath = path22.join(introDir, "_project.json");
|
|
9172
|
+
await fs8.writeFile(projectPath, JSON.stringify({
|
|
8928
9173
|
version: "1.0.0",
|
|
8929
9174
|
lastUpdated: new Date().toISOString(),
|
|
8930
9175
|
structure: this.structure
|
|
8931
9176
|
}, null, 2));
|
|
8932
9177
|
for (const [filepath, intro] of this.files) {
|
|
8933
|
-
const introFilePath =
|
|
8934
|
-
await
|
|
8935
|
-
await
|
|
9178
|
+
const introFilePath = path22.join(introDir, "files", filepath.replace(/\.[^.]+$/, ".json"));
|
|
9179
|
+
await fs8.mkdir(path22.dirname(introFilePath), { recursive: true });
|
|
9180
|
+
await fs8.writeFile(introFilePath, JSON.stringify(intro, null, 2));
|
|
8936
9181
|
}
|
|
8937
9182
|
}
|
|
8938
9183
|
async load(config) {
|
|
8939
|
-
const introDir =
|
|
9184
|
+
const introDir = path22.join(getRaggrepDir(this.rootDir, config), "introspection");
|
|
8940
9185
|
try {
|
|
8941
|
-
const projectPath =
|
|
8942
|
-
const projectContent = await
|
|
9186
|
+
const projectPath = path22.join(introDir, "_project.json");
|
|
9187
|
+
const projectContent = await fs8.readFile(projectPath, "utf-8");
|
|
8943
9188
|
const projectData = JSON.parse(projectContent);
|
|
8944
9189
|
this.structure = projectData.structure;
|
|
8945
|
-
await this.loadFilesRecursive(
|
|
9190
|
+
await this.loadFilesRecursive(path22.join(introDir, "files"), "");
|
|
8946
9191
|
} catch {
|
|
8947
9192
|
this.structure = null;
|
|
8948
9193
|
this.files.clear();
|
|
@@ -8950,14 +9195,14 @@ class IntrospectionIndex {
|
|
|
8950
9195
|
}
|
|
8951
9196
|
async loadFilesRecursive(basePath, prefix) {
|
|
8952
9197
|
try {
|
|
8953
|
-
const entries = await
|
|
9198
|
+
const entries = await fs8.readdir(basePath, { withFileTypes: true });
|
|
8954
9199
|
for (const entry of entries) {
|
|
8955
|
-
const entryPath =
|
|
9200
|
+
const entryPath = path22.join(basePath, entry.name);
|
|
8956
9201
|
const relativePath = prefix ? `${prefix}/${entry.name}` : entry.name;
|
|
8957
9202
|
if (entry.isDirectory()) {
|
|
8958
9203
|
await this.loadFilesRecursive(entryPath, relativePath);
|
|
8959
9204
|
} else if (entry.name.endsWith(".json")) {
|
|
8960
|
-
const content = await
|
|
9205
|
+
const content = await fs8.readFile(entryPath, "utf-8");
|
|
8961
9206
|
const intro = JSON.parse(content);
|
|
8962
9207
|
this.files.set(intro.filepath, intro);
|
|
8963
9208
|
}
|
|
@@ -8983,7 +9228,7 @@ var init_introspection2 = __esm(() => {
|
|
|
8983
9228
|
|
|
8984
9229
|
// src/app/indexer/watcher.ts
|
|
8985
9230
|
import { watch } from "chokidar";
|
|
8986
|
-
import * as
|
|
9231
|
+
import * as path23 from "path";
|
|
8987
9232
|
async function watchDirectory(rootDir, options = {}) {
|
|
8988
9233
|
const {
|
|
8989
9234
|
debounceMs = DEFAULT_DEBOUNCE_MS,
|
|
@@ -8994,17 +9239,17 @@ async function watchDirectory(rootDir, options = {}) {
|
|
|
8994
9239
|
onFileChange,
|
|
8995
9240
|
onError
|
|
8996
9241
|
} = options;
|
|
8997
|
-
rootDir =
|
|
9242
|
+
rootDir = path23.resolve(rootDir);
|
|
8998
9243
|
const config = await loadConfig(rootDir);
|
|
8999
|
-
const indexLocation = getIndexLocation(rootDir);
|
|
9000
9244
|
const validExtensions = new Set(config.extensions);
|
|
9001
9245
|
const ignorePatterns = [
|
|
9002
9246
|
...config.ignorePaths.map((p) => `**/${p}/**`),
|
|
9003
9247
|
"**/node_modules/**",
|
|
9004
|
-
"**/.git/**"
|
|
9248
|
+
"**/.git/**",
|
|
9249
|
+
"**/.raggrep/**"
|
|
9005
9250
|
];
|
|
9006
9251
|
function shouldWatchFile(filepath) {
|
|
9007
|
-
const ext =
|
|
9252
|
+
const ext = path23.extname(filepath);
|
|
9008
9253
|
return validExtensions.has(ext);
|
|
9009
9254
|
}
|
|
9010
9255
|
let isRunning = true;
|
|
@@ -9087,7 +9332,7 @@ async function watchDirectory(rootDir, options = {}) {
|
|
|
9087
9332
|
function handleFileEvent(event, filepath) {
|
|
9088
9333
|
if (!isRunning)
|
|
9089
9334
|
return;
|
|
9090
|
-
const relativePath =
|
|
9335
|
+
const relativePath = path23.relative(rootDir, filepath);
|
|
9091
9336
|
if (!shouldWatchFile(filepath)) {
|
|
9092
9337
|
return;
|
|
9093
9338
|
}
|
|
@@ -9165,9 +9410,9 @@ __export(exports_indexer, {
|
|
|
9165
9410
|
clearFreshnessCache: () => clearFreshnessCache,
|
|
9166
9411
|
cleanupIndex: () => cleanupIndex
|
|
9167
9412
|
});
|
|
9168
|
-
import * as
|
|
9169
|
-
import * as
|
|
9170
|
-
import * as
|
|
9413
|
+
import * as fs9 from "fs/promises";
|
|
9414
|
+
import * as path24 from "path";
|
|
9415
|
+
import * as os2 from "os";
|
|
9171
9416
|
import * as crypto2 from "crypto";
|
|
9172
9417
|
function clearFreshnessCache() {
|
|
9173
9418
|
freshnessCache = null;
|
|
@@ -9207,7 +9452,7 @@ function formatDuration(ms) {
|
|
|
9207
9452
|
return `${minutes}m ${remainingSeconds.toFixed(1)}s`;
|
|
9208
9453
|
}
|
|
9209
9454
|
function getOptimalConcurrency() {
|
|
9210
|
-
const cpuCount =
|
|
9455
|
+
const cpuCount = os2.cpus().length;
|
|
9211
9456
|
const optimal = Math.max(2, Math.min(16, Math.floor(cpuCount * 0.75)));
|
|
9212
9457
|
return optimal;
|
|
9213
9458
|
}
|
|
@@ -9218,7 +9463,7 @@ async function indexDirectory(rootDir, options = {}) {
|
|
|
9218
9463
|
const concurrency = options.concurrency ?? DEFAULT_CONCURRENCY;
|
|
9219
9464
|
clearFreshnessCache();
|
|
9220
9465
|
const logger = options.logger ? options.logger : quiet ? createSilentLogger() : createLogger({ verbose });
|
|
9221
|
-
rootDir =
|
|
9466
|
+
rootDir = path24.resolve(rootDir);
|
|
9222
9467
|
const location = getIndexLocation(rootDir);
|
|
9223
9468
|
logger.info(`Indexing directory: ${rootDir}`);
|
|
9224
9469
|
logger.info(`Index location: ${location.indexDir}`);
|
|
@@ -9270,12 +9515,12 @@ async function indexDirectory(rootDir, options = {}) {
|
|
|
9270
9515
|
rootDir,
|
|
9271
9516
|
config,
|
|
9272
9517
|
readFile: async (filepath) => {
|
|
9273
|
-
const fullPath =
|
|
9274
|
-
return
|
|
9518
|
+
const fullPath = path24.isAbsolute(filepath) ? filepath : path24.join(rootDir, filepath);
|
|
9519
|
+
return fs9.readFile(fullPath, "utf-8");
|
|
9275
9520
|
},
|
|
9276
9521
|
getFileStats: async (filepath) => {
|
|
9277
|
-
const fullPath =
|
|
9278
|
-
const stats = await
|
|
9522
|
+
const fullPath = path24.isAbsolute(filepath) ? filepath : path24.join(rootDir, filepath);
|
|
9523
|
+
const stats = await fs9.stat(fullPath);
|
|
9279
9524
|
return { lastModified: stats.mtime.toISOString() };
|
|
9280
9525
|
}
|
|
9281
9526
|
};
|
|
@@ -9300,7 +9545,7 @@ async function isIndexVersionCompatible(rootDir) {
|
|
|
9300
9545
|
const config = await loadConfig(rootDir);
|
|
9301
9546
|
const globalManifestPath = getGlobalManifestPath(rootDir, config);
|
|
9302
9547
|
try {
|
|
9303
|
-
const content = await
|
|
9548
|
+
const content = await fs9.readFile(globalManifestPath, "utf-8");
|
|
9304
9549
|
const manifest = JSON.parse(content);
|
|
9305
9550
|
return manifest.version === INDEX_SCHEMA_VERSION;
|
|
9306
9551
|
} catch {
|
|
@@ -9310,11 +9555,11 @@ async function isIndexVersionCompatible(rootDir) {
|
|
|
9310
9555
|
async function deleteIndex(rootDir) {
|
|
9311
9556
|
const indexDir = getRaggrepDir(rootDir);
|
|
9312
9557
|
try {
|
|
9313
|
-
await
|
|
9558
|
+
await fs9.rm(indexDir, { recursive: true, force: true });
|
|
9314
9559
|
} catch {}
|
|
9315
9560
|
}
|
|
9316
9561
|
async function resetIndex(rootDir) {
|
|
9317
|
-
rootDir =
|
|
9562
|
+
rootDir = path24.resolve(rootDir);
|
|
9318
9563
|
clearFreshnessCache();
|
|
9319
9564
|
const status = await getIndexStatus(rootDir);
|
|
9320
9565
|
if (!status.exists) {
|
|
@@ -9339,7 +9584,7 @@ async function ensureIndexFresh(rootDir, options = {}) {
|
|
|
9339
9584
|
let filesChanged = 0;
|
|
9340
9585
|
let filesReindexed = 0;
|
|
9341
9586
|
const logger = options.logger ? options.logger : quiet ? createSilentLogger() : createLogger({ verbose });
|
|
9342
|
-
rootDir =
|
|
9587
|
+
rootDir = path24.resolve(rootDir);
|
|
9343
9588
|
const status = await getIndexStatus(rootDir);
|
|
9344
9589
|
if (!status.exists) {
|
|
9345
9590
|
clearFreshnessCache();
|
|
@@ -9363,7 +9608,7 @@ async function ensureIndexFresh(rootDir, options = {}) {
|
|
|
9363
9608
|
const globalManifestPath = getGlobalManifestPath(rootDir, config);
|
|
9364
9609
|
let currentManifestMtime = 0;
|
|
9365
9610
|
try {
|
|
9366
|
-
const manifestStats = await
|
|
9611
|
+
const manifestStats = await fs9.stat(globalManifestPath);
|
|
9367
9612
|
currentManifestMtime = manifestStats.mtimeMs;
|
|
9368
9613
|
} catch {}
|
|
9369
9614
|
const now = Date.now();
|
|
@@ -9401,7 +9646,7 @@ async function ensureIndexFresh(rootDir, options = {}) {
|
|
|
9401
9646
|
const { allFiles: currentFiles, changedFiles, changedFileMtimes } = discoveryResult;
|
|
9402
9647
|
filesDiscovered = currentFiles.length;
|
|
9403
9648
|
filesChanged = changedFiles.length;
|
|
9404
|
-
const currentFileSet = new Set(currentFiles.map((f) =>
|
|
9649
|
+
const currentFileSet = new Set(currentFiles.map((f) => path24.relative(rootDir, f)));
|
|
9405
9650
|
const changedFileSet = new Set(changedFiles);
|
|
9406
9651
|
let totalIndexed = 0;
|
|
9407
9652
|
let totalRemoved = 0;
|
|
@@ -9435,11 +9680,11 @@ async function ensureIndexFresh(rootDir, options = {}) {
|
|
|
9435
9680
|
if (filesToRemove.length > 0) {
|
|
9436
9681
|
await Promise.all(filesToRemove.map(async (filepath) => {
|
|
9437
9682
|
logger.debug(` Removing stale: ${filepath}`);
|
|
9438
|
-
const indexFilePath =
|
|
9439
|
-
const symbolicFilePath =
|
|
9683
|
+
const indexFilePath = path24.join(indexPath, filepath.replace(/\.[^.]+$/, ".json"));
|
|
9684
|
+
const symbolicFilePath = path24.join(indexPath, "symbolic", filepath.replace(/\.[^.]+$/, ".json"));
|
|
9440
9685
|
await Promise.all([
|
|
9441
|
-
|
|
9442
|
-
|
|
9686
|
+
fs9.unlink(indexFilePath).catch(() => {}),
|
|
9687
|
+
fs9.unlink(symbolicFilePath).catch(() => {})
|
|
9443
9688
|
]);
|
|
9444
9689
|
delete manifest.files[filepath];
|
|
9445
9690
|
removedFilepaths.push(filepath);
|
|
@@ -9463,19 +9708,19 @@ async function ensureIndexFresh(rootDir, options = {}) {
|
|
|
9463
9708
|
rootDir,
|
|
9464
9709
|
config,
|
|
9465
9710
|
readFile: async (filepath) => {
|
|
9466
|
-
const fullPath =
|
|
9467
|
-
return
|
|
9711
|
+
const fullPath = path24.isAbsolute(filepath) ? filepath : path24.join(rootDir, filepath);
|
|
9712
|
+
return fs9.readFile(fullPath, "utf-8");
|
|
9468
9713
|
},
|
|
9469
9714
|
getFileStats: async (filepath) => {
|
|
9470
|
-
const fullPath =
|
|
9471
|
-
const stats = await
|
|
9715
|
+
const fullPath = path24.isAbsolute(filepath) ? filepath : path24.join(rootDir, filepath);
|
|
9716
|
+
const stats = await fs9.stat(fullPath);
|
|
9472
9717
|
return { lastModified: stats.mtime.toISOString() };
|
|
9473
9718
|
},
|
|
9474
9719
|
getIntrospection: (filepath) => introspection.getFile(filepath)
|
|
9475
9720
|
};
|
|
9476
9721
|
const moduleChangedFiles = module.supportsFile ? changedFiles.filter((f) => module.supportsFile(f)) : changedFiles;
|
|
9477
9722
|
const filesToProcess = moduleChangedFiles.map((filepath) => {
|
|
9478
|
-
const relativePath =
|
|
9723
|
+
const relativePath = path24.relative(rootDir, filepath);
|
|
9479
9724
|
const existingEntry = manifest.files[relativePath];
|
|
9480
9725
|
const lastModified = changedFileMtimes.get(filepath) || new Date().toISOString();
|
|
9481
9726
|
return {
|
|
@@ -9505,7 +9750,7 @@ async function ensureIndexFresh(rootDir, options = {}) {
|
|
|
9505
9750
|
return { relativePath, status: "unchanged" };
|
|
9506
9751
|
}
|
|
9507
9752
|
try {
|
|
9508
|
-
const content = await
|
|
9753
|
+
const content = await fs9.readFile(filepath, "utf-8");
|
|
9509
9754
|
const contentHash = computeContentHash(content);
|
|
9510
9755
|
if (!isNew && existingContentHash && existingContentHash === contentHash) {
|
|
9511
9756
|
completedCount++;
|
|
@@ -9632,7 +9877,7 @@ async function ensureIndexFresh(rootDir, options = {}) {
|
|
|
9632
9877
|
}
|
|
9633
9878
|
let finalManifestMtime = currentManifestMtime;
|
|
9634
9879
|
try {
|
|
9635
|
-
const manifestStats = await
|
|
9880
|
+
const manifestStats = await fs9.stat(globalManifestPath);
|
|
9636
9881
|
finalManifestMtime = manifestStats.mtimeMs;
|
|
9637
9882
|
} catch {}
|
|
9638
9883
|
freshnessCache = {
|
|
@@ -9652,7 +9897,7 @@ async function indexWithModule(rootDir, files, module, config, verbose, introspe
|
|
|
9652
9897
|
};
|
|
9653
9898
|
const manifest = await loadModuleManifest(rootDir, module.id, config);
|
|
9654
9899
|
const indexPath = getModuleIndexPath(rootDir, module.id, config);
|
|
9655
|
-
const currentFileSet = new Set(files.map((f) =>
|
|
9900
|
+
const currentFileSet = new Set(files.map((f) => path24.relative(rootDir, f)));
|
|
9656
9901
|
const filesToRemove = [];
|
|
9657
9902
|
for (const filepath of Object.keys(manifest.files)) {
|
|
9658
9903
|
if (!currentFileSet.has(filepath)) {
|
|
@@ -9663,13 +9908,13 @@ async function indexWithModule(rootDir, files, module, config, verbose, introspe
|
|
|
9663
9908
|
logger.info(` Removing ${filesToRemove.length} stale entries...`);
|
|
9664
9909
|
for (const filepath of filesToRemove) {
|
|
9665
9910
|
logger.debug(` Removing: ${filepath}`);
|
|
9666
|
-
const indexFilePath =
|
|
9911
|
+
const indexFilePath = path24.join(indexPath, filepath.replace(/\.[^.]+$/, ".json"));
|
|
9667
9912
|
try {
|
|
9668
|
-
await
|
|
9913
|
+
await fs9.unlink(indexFilePath);
|
|
9669
9914
|
} catch {}
|
|
9670
|
-
const symbolicFilePath =
|
|
9915
|
+
const symbolicFilePath = path24.join(indexPath, "symbolic", filepath.replace(/\.[^.]+$/, ".json"));
|
|
9671
9916
|
try {
|
|
9672
|
-
await
|
|
9917
|
+
await fs9.unlink(symbolicFilePath);
|
|
9673
9918
|
} catch {}
|
|
9674
9919
|
delete manifest.files[filepath];
|
|
9675
9920
|
}
|
|
@@ -9679,12 +9924,12 @@ async function indexWithModule(rootDir, files, module, config, verbose, introspe
|
|
|
9679
9924
|
rootDir,
|
|
9680
9925
|
config,
|
|
9681
9926
|
readFile: async (filepath) => {
|
|
9682
|
-
const fullPath =
|
|
9683
|
-
return
|
|
9927
|
+
const fullPath = path24.isAbsolute(filepath) ? filepath : path24.join(rootDir, filepath);
|
|
9928
|
+
return fs9.readFile(fullPath, "utf-8");
|
|
9684
9929
|
},
|
|
9685
9930
|
getFileStats: async (filepath) => {
|
|
9686
|
-
const fullPath =
|
|
9687
|
-
const stats = await
|
|
9931
|
+
const fullPath = path24.isAbsolute(filepath) ? filepath : path24.join(rootDir, filepath);
|
|
9932
|
+
const stats = await fs9.stat(fullPath);
|
|
9688
9933
|
return { lastModified: stats.mtime.toISOString() };
|
|
9689
9934
|
},
|
|
9690
9935
|
getIntrospection: (filepath) => introspection.getFile(filepath)
|
|
@@ -9696,9 +9941,9 @@ async function indexWithModule(rootDir, files, module, config, verbose, introspe
|
|
|
9696
9941
|
let indexedCount = 0;
|
|
9697
9942
|
let skippedCount = 0;
|
|
9698
9943
|
const processFile = async (filepath, _index) => {
|
|
9699
|
-
const relativePath =
|
|
9944
|
+
const relativePath = path24.relative(rootDir, filepath);
|
|
9700
9945
|
try {
|
|
9701
|
-
const stats = await
|
|
9946
|
+
const stats = await fs9.stat(filepath);
|
|
9702
9947
|
const lastModified = stats.mtime.toISOString();
|
|
9703
9948
|
const existingEntry = manifest.files[relativePath];
|
|
9704
9949
|
if (existingEntry && existingEntry.lastModified === lastModified) {
|
|
@@ -9708,7 +9953,7 @@ async function indexWithModule(rootDir, files, module, config, verbose, introspe
|
|
|
9708
9953
|
logger.debug(` [${completedCount}/${totalFiles}] Skipped ${relativePath} (unchanged)`);
|
|
9709
9954
|
return { relativePath, status: "skipped" };
|
|
9710
9955
|
}
|
|
9711
|
-
const content = await
|
|
9956
|
+
const content = await fs9.readFile(filepath, "utf-8");
|
|
9712
9957
|
const contentHash = computeContentHash(content);
|
|
9713
9958
|
if (existingEntry?.contentHash && existingEntry.contentHash === contentHash) {
|
|
9714
9959
|
completedCount++;
|
|
@@ -9789,8 +10034,8 @@ async function indexWithModule(rootDir, files, module, config, verbose, introspe
|
|
|
9789
10034
|
return result;
|
|
9790
10035
|
}
|
|
9791
10036
|
function isLikelyBinary(filepath) {
|
|
9792
|
-
const ext =
|
|
9793
|
-
const basename15 =
|
|
10037
|
+
const ext = path24.extname(filepath).toLowerCase();
|
|
10038
|
+
const basename15 = path24.basename(filepath).toLowerCase();
|
|
9794
10039
|
const binaryExtensions = new Set([
|
|
9795
10040
|
".png",
|
|
9796
10041
|
".jpg",
|
|
@@ -9866,7 +10111,7 @@ async function findFilesWithStats(rootDir, config, lastIndexStarted) {
|
|
|
9866
10111
|
const ignoreDirs = new Set(config.ignorePaths);
|
|
9867
10112
|
const lastIndexMs = lastIndexStarted?.getTime() ?? 0;
|
|
9868
10113
|
const crawler = new Builder().withFullPaths().exclude((dirName) => ignoreDirs.has(dirName)).filter((filePath) => {
|
|
9869
|
-
const ext =
|
|
10114
|
+
const ext = path24.extname(filePath);
|
|
9870
10115
|
return validExtensions.has(ext);
|
|
9871
10116
|
}).crawl(rootDir);
|
|
9872
10117
|
const allFiles = await crawler.withPromise();
|
|
@@ -9874,7 +10119,7 @@ async function findFilesWithStats(rootDir, config, lastIndexStarted) {
|
|
|
9874
10119
|
const changedFileMtimes2 = new Map;
|
|
9875
10120
|
await parallelMap(allFiles, async (filePath) => {
|
|
9876
10121
|
try {
|
|
9877
|
-
const stats = await
|
|
10122
|
+
const stats = await fs9.stat(filePath);
|
|
9878
10123
|
changedFileMtimes2.set(filePath, stats.mtime.toISOString());
|
|
9879
10124
|
} catch {}
|
|
9880
10125
|
}, STAT_CONCURRENCY);
|
|
@@ -9888,7 +10133,7 @@ async function findFilesWithStats(rootDir, config, lastIndexStarted) {
|
|
|
9888
10133
|
const changedFileMtimes = new Map;
|
|
9889
10134
|
await parallelMap(allFiles, async (filePath) => {
|
|
9890
10135
|
try {
|
|
9891
|
-
const stats = await
|
|
10136
|
+
const stats = await fs9.stat(filePath);
|
|
9892
10137
|
if (stats.mtimeMs > lastIndexMs) {
|
|
9893
10138
|
changedFiles.push(filePath);
|
|
9894
10139
|
changedFileMtimes.set(filePath, stats.mtime.toISOString());
|
|
@@ -9908,7 +10153,7 @@ async function findFiles(rootDir, config) {
|
|
|
9908
10153
|
async function loadModuleManifest(rootDir, moduleId, config) {
|
|
9909
10154
|
const manifestPath = getModuleManifestPath(rootDir, moduleId, config);
|
|
9910
10155
|
try {
|
|
9911
|
-
const content = await
|
|
10156
|
+
const content = await fs9.readFile(manifestPath, "utf-8");
|
|
9912
10157
|
return JSON.parse(content);
|
|
9913
10158
|
} catch {
|
|
9914
10159
|
return {
|
|
@@ -9921,19 +10166,19 @@ async function loadModuleManifest(rootDir, moduleId, config) {
|
|
|
9921
10166
|
}
|
|
9922
10167
|
async function writeModuleManifest(rootDir, moduleId, manifest, config) {
|
|
9923
10168
|
const manifestPath = getModuleManifestPath(rootDir, moduleId, config);
|
|
9924
|
-
await
|
|
9925
|
-
await
|
|
10169
|
+
await fs9.mkdir(path24.dirname(manifestPath), { recursive: true });
|
|
10170
|
+
await fs9.writeFile(manifestPath, JSON.stringify(manifest, null, 2));
|
|
9926
10171
|
}
|
|
9927
10172
|
async function writeFileIndex(rootDir, moduleId, filepath, fileIndex, config) {
|
|
9928
10173
|
const indexPath = getModuleIndexPath(rootDir, moduleId, config);
|
|
9929
|
-
const indexFilePath =
|
|
9930
|
-
await
|
|
9931
|
-
await
|
|
10174
|
+
const indexFilePath = path24.join(indexPath, filepath.replace(/\.[^.]+$/, ".json"));
|
|
10175
|
+
await fs9.mkdir(path24.dirname(indexFilePath), { recursive: true });
|
|
10176
|
+
await fs9.writeFile(indexFilePath, JSON.stringify(fileIndex, null, 2));
|
|
9932
10177
|
}
|
|
9933
10178
|
async function loadGlobalManifest(rootDir, config) {
|
|
9934
10179
|
const manifestPath = getGlobalManifestPath(rootDir, config);
|
|
9935
10180
|
try {
|
|
9936
|
-
const content = await
|
|
10181
|
+
const content = await fs9.readFile(manifestPath, "utf-8");
|
|
9937
10182
|
return JSON.parse(content);
|
|
9938
10183
|
} catch {
|
|
9939
10184
|
return null;
|
|
@@ -9947,13 +10192,13 @@ async function updateGlobalManifest(rootDir, modules, config, indexStartTime) {
|
|
|
9947
10192
|
lastIndexStarted: indexStartTime,
|
|
9948
10193
|
modules: modules.map((m) => m.id)
|
|
9949
10194
|
};
|
|
9950
|
-
await
|
|
9951
|
-
await
|
|
10195
|
+
await fs9.mkdir(path24.dirname(manifestPath), { recursive: true });
|
|
10196
|
+
await fs9.writeFile(manifestPath, JSON.stringify(manifest, null, 2));
|
|
9952
10197
|
}
|
|
9953
10198
|
async function cleanupIndex(rootDir, options = {}) {
|
|
9954
10199
|
const verbose = options.verbose ?? false;
|
|
9955
10200
|
const logger = options.logger ?? createLogger({ verbose });
|
|
9956
|
-
rootDir =
|
|
10201
|
+
rootDir = path24.resolve(rootDir);
|
|
9957
10202
|
logger.info(`Cleaning up index in: ${rootDir}`);
|
|
9958
10203
|
const config = await loadConfig(rootDir);
|
|
9959
10204
|
await registerBuiltInModules();
|
|
@@ -9983,9 +10228,9 @@ async function cleanupModuleIndex(rootDir, moduleId, config, logger) {
|
|
|
9983
10228
|
const filesToRemove = [];
|
|
9984
10229
|
const updatedFiles = {};
|
|
9985
10230
|
for (const [filepath, entry] of Object.entries(manifest.files)) {
|
|
9986
|
-
const fullPath =
|
|
10231
|
+
const fullPath = path24.join(rootDir, filepath);
|
|
9987
10232
|
try {
|
|
9988
|
-
await
|
|
10233
|
+
await fs9.access(fullPath);
|
|
9989
10234
|
updatedFiles[filepath] = entry;
|
|
9990
10235
|
result.kept++;
|
|
9991
10236
|
} catch {
|
|
@@ -9995,9 +10240,9 @@ async function cleanupModuleIndex(rootDir, moduleId, config, logger) {
|
|
|
9995
10240
|
}
|
|
9996
10241
|
}
|
|
9997
10242
|
for (const filepath of filesToRemove) {
|
|
9998
|
-
const indexFilePath =
|
|
10243
|
+
const indexFilePath = path24.join(indexPath, filepath.replace(/\.[^.]+$/, ".json"));
|
|
9999
10244
|
try {
|
|
10000
|
-
await
|
|
10245
|
+
await fs9.unlink(indexFilePath);
|
|
10001
10246
|
} catch {}
|
|
10002
10247
|
}
|
|
10003
10248
|
manifest.files = updatedFiles;
|
|
@@ -10008,16 +10253,16 @@ async function cleanupModuleIndex(rootDir, moduleId, config, logger) {
|
|
|
10008
10253
|
}
|
|
10009
10254
|
async function cleanupEmptyDirectories(dir) {
|
|
10010
10255
|
try {
|
|
10011
|
-
const entries = await
|
|
10256
|
+
const entries = await fs9.readdir(dir, { withFileTypes: true });
|
|
10012
10257
|
for (const entry of entries) {
|
|
10013
10258
|
if (entry.isDirectory()) {
|
|
10014
|
-
const subDir =
|
|
10259
|
+
const subDir = path24.join(dir, entry.name);
|
|
10015
10260
|
await cleanupEmptyDirectories(subDir);
|
|
10016
10261
|
}
|
|
10017
10262
|
}
|
|
10018
|
-
const remainingEntries = await
|
|
10263
|
+
const remainingEntries = await fs9.readdir(dir);
|
|
10019
10264
|
if (remainingEntries.length === 0) {
|
|
10020
|
-
await
|
|
10265
|
+
await fs9.rmdir(dir);
|
|
10021
10266
|
return true;
|
|
10022
10267
|
}
|
|
10023
10268
|
return false;
|
|
@@ -10026,7 +10271,7 @@ async function cleanupEmptyDirectories(dir) {
|
|
|
10026
10271
|
}
|
|
10027
10272
|
}
|
|
10028
10273
|
async function getIndexStatus(rootDir) {
|
|
10029
|
-
rootDir =
|
|
10274
|
+
rootDir = path24.resolve(rootDir);
|
|
10030
10275
|
const config = await loadConfig(rootDir);
|
|
10031
10276
|
const location = getIndexLocation(rootDir);
|
|
10032
10277
|
const indexDir = location.indexDir;
|
|
@@ -10038,13 +10283,13 @@ async function getIndexStatus(rootDir) {
|
|
|
10038
10283
|
totalFiles: 0
|
|
10039
10284
|
};
|
|
10040
10285
|
try {
|
|
10041
|
-
await
|
|
10286
|
+
await fs9.access(indexDir);
|
|
10042
10287
|
} catch {
|
|
10043
10288
|
return status;
|
|
10044
10289
|
}
|
|
10045
10290
|
try {
|
|
10046
10291
|
const globalManifestPath = getGlobalManifestPath(rootDir, config);
|
|
10047
|
-
const content = await
|
|
10292
|
+
const content = await fs9.readFile(globalManifestPath, "utf-8");
|
|
10048
10293
|
const globalManifest = JSON.parse(content);
|
|
10049
10294
|
status.exists = true;
|
|
10050
10295
|
status.lastUpdated = globalManifest.lastUpdated;
|
|
@@ -10062,7 +10307,7 @@ async function getIndexStatus(rootDir) {
|
|
|
10062
10307
|
}
|
|
10063
10308
|
} catch {
|
|
10064
10309
|
try {
|
|
10065
|
-
const entries = await
|
|
10310
|
+
const entries = await fs9.readdir(path24.join(indexDir, "index"));
|
|
10066
10311
|
if (entries.length > 0) {
|
|
10067
10312
|
status.exists = true;
|
|
10068
10313
|
for (const entry of entries) {
|
|
@@ -10082,7 +10327,7 @@ async function getIndexStatus(rootDir) {
|
|
|
10082
10327
|
}
|
|
10083
10328
|
return status;
|
|
10084
10329
|
}
|
|
10085
|
-
var FRESHNESS_CACHE_TTL_MS = 5000, freshnessCache = null, INDEX_SCHEMA_VERSION = "2.
|
|
10330
|
+
var FRESHNESS_CACHE_TTL_MS = 5000, freshnessCache = null, INDEX_SCHEMA_VERSION = "2.1.0", DEFAULT_CONCURRENCY, STAT_CONCURRENCY = 64;
|
|
10086
10331
|
var init_indexer = __esm(() => {
|
|
10087
10332
|
init_dist();
|
|
10088
10333
|
init_config2();
|
|
@@ -11428,7 +11673,7 @@ var minimatch = (p, pattern, options = {}) => {
|
|
|
11428
11673
|
}, qmarksTestNoExtDot = ([$0]) => {
|
|
11429
11674
|
const len = $0.length;
|
|
11430
11675
|
return (f) => f.length === len && f !== "." && f !== "..";
|
|
11431
|
-
}, defaultPlatform,
|
|
11676
|
+
}, defaultPlatform, path25, sep2, GLOBSTAR, qmark2 = "[^/]", star2, twoStarDot = "(?:(?!(?:\\/|^)(?:\\.{1,2})($|\\/)).)*?", twoStarNoDot = "(?:(?!(?:\\/|^)\\.).)*?", filter = (pattern, options = {}) => (p) => minimatch(p, pattern, options), ext = (a, b = {}) => Object.assign({}, a, b), defaults = (def) => {
|
|
11432
11677
|
if (!def || typeof def !== "object" || !Object.keys(def).length) {
|
|
11433
11678
|
return minimatch;
|
|
11434
11679
|
}
|
|
@@ -11486,11 +11731,11 @@ var init_esm2 = __esm(() => {
|
|
|
11486
11731
|
starRE = /^\*+$/;
|
|
11487
11732
|
qmarksRE = /^\?+([^+@!?\*\[\(]*)?$/;
|
|
11488
11733
|
defaultPlatform = typeof process === "object" && process ? typeof process.env === "object" && process.env && process.env.__MINIMATCH_TESTING_PLATFORM__ || process.platform : "posix";
|
|
11489
|
-
|
|
11734
|
+
path25 = {
|
|
11490
11735
|
win32: { sep: "\\" },
|
|
11491
11736
|
posix: { sep: "/" }
|
|
11492
11737
|
};
|
|
11493
|
-
sep2 = defaultPlatform === "win32" ?
|
|
11738
|
+
sep2 = defaultPlatform === "win32" ? path25.win32.sep : path25.posix.sep;
|
|
11494
11739
|
minimatch.sep = sep2;
|
|
11495
11740
|
GLOBSTAR = Symbol("globstar **");
|
|
11496
11741
|
minimatch.GLOBSTAR = GLOBSTAR;
|
|
@@ -11530,7 +11775,7 @@ function matchesPathFilter(relativePath, filters, matchFn) {
|
|
|
11530
11775
|
}
|
|
11531
11776
|
return false;
|
|
11532
11777
|
}
|
|
11533
|
-
async function executeExactSearch(
|
|
11778
|
+
async function executeExactSearch(fs10, options, matchFn) {
|
|
11534
11779
|
const {
|
|
11535
11780
|
rootDir,
|
|
11536
11781
|
literal,
|
|
@@ -11542,13 +11787,13 @@ async function executeExactSearch(fs9, options, matchFn) {
|
|
|
11542
11787
|
const files = new Map;
|
|
11543
11788
|
async function walkDir(dir, baseDir) {
|
|
11544
11789
|
try {
|
|
11545
|
-
const entries = await
|
|
11790
|
+
const entries = await fs10.readDir(dir);
|
|
11546
11791
|
for (const entry of entries) {
|
|
11547
|
-
const fullPath =
|
|
11548
|
-
const relativePath =
|
|
11792
|
+
const fullPath = fs10.join(dir, entry);
|
|
11793
|
+
const relativePath = fs10.relative(baseDir, fullPath);
|
|
11549
11794
|
let isDirectory = false;
|
|
11550
11795
|
try {
|
|
11551
|
-
const stats = await
|
|
11796
|
+
const stats = await fs10.getStats(fullPath);
|
|
11552
11797
|
isDirectory = stats.isDirectory ?? false;
|
|
11553
11798
|
} catch {
|
|
11554
11799
|
continue;
|
|
@@ -11565,7 +11810,7 @@ async function executeExactSearch(fs9, options, matchFn) {
|
|
|
11565
11810
|
}
|
|
11566
11811
|
}
|
|
11567
11812
|
try {
|
|
11568
|
-
const content = await
|
|
11813
|
+
const content = await fs10.readFile(fullPath);
|
|
11569
11814
|
if (isSearchableContent(content, fullPath)) {
|
|
11570
11815
|
files.set(relativePath, content);
|
|
11571
11816
|
}
|
|
@@ -11603,21 +11848,21 @@ var init_usecases = __esm(() => {
|
|
|
11603
11848
|
});
|
|
11604
11849
|
|
|
11605
11850
|
// src/infrastructure/filesystem/nodeFileSystem.ts
|
|
11606
|
-
import * as
|
|
11607
|
-
import * as
|
|
11851
|
+
import * as fs10 from "fs/promises";
|
|
11852
|
+
import * as path26 from "path";
|
|
11608
11853
|
import { glob } from "glob";
|
|
11609
11854
|
|
|
11610
11855
|
class NodeFileSystem {
|
|
11611
11856
|
async readFile(filepath) {
|
|
11612
|
-
return
|
|
11857
|
+
return fs10.readFile(filepath, "utf-8");
|
|
11613
11858
|
}
|
|
11614
11859
|
async writeFile(filepath, content) {
|
|
11615
|
-
await
|
|
11616
|
-
await
|
|
11860
|
+
await fs10.mkdir(path26.dirname(filepath), { recursive: true });
|
|
11861
|
+
await fs10.writeFile(filepath, content, "utf-8");
|
|
11617
11862
|
}
|
|
11618
11863
|
async deleteFile(filepath) {
|
|
11619
11864
|
try {
|
|
11620
|
-
await
|
|
11865
|
+
await fs10.unlink(filepath);
|
|
11621
11866
|
} catch (error) {
|
|
11622
11867
|
if (error.code !== "ENOENT") {
|
|
11623
11868
|
throw error;
|
|
@@ -11625,7 +11870,7 @@ class NodeFileSystem {
|
|
|
11625
11870
|
}
|
|
11626
11871
|
}
|
|
11627
11872
|
async getStats(filepath) {
|
|
11628
|
-
const stats = await
|
|
11873
|
+
const stats = await fs10.stat(filepath);
|
|
11629
11874
|
return {
|
|
11630
11875
|
lastModified: stats.mtime.toISOString(),
|
|
11631
11876
|
size: stats.isDirectory() ? undefined : stats.size,
|
|
@@ -11634,17 +11879,17 @@ class NodeFileSystem {
|
|
|
11634
11879
|
}
|
|
11635
11880
|
async exists(filepath) {
|
|
11636
11881
|
try {
|
|
11637
|
-
await
|
|
11882
|
+
await fs10.access(filepath);
|
|
11638
11883
|
return true;
|
|
11639
11884
|
} catch {
|
|
11640
11885
|
return false;
|
|
11641
11886
|
}
|
|
11642
11887
|
}
|
|
11643
11888
|
async mkdir(dirpath) {
|
|
11644
|
-
await
|
|
11889
|
+
await fs10.mkdir(dirpath, { recursive: true });
|
|
11645
11890
|
}
|
|
11646
11891
|
async readDir(dirpath) {
|
|
11647
|
-
return
|
|
11892
|
+
return fs10.readdir(dirpath);
|
|
11648
11893
|
}
|
|
11649
11894
|
async findFiles(rootDir, patterns, ignore) {
|
|
11650
11895
|
const ignorePatterns = ignore.map((p) => `**/${p}/**`);
|
|
@@ -11660,19 +11905,19 @@ class NodeFileSystem {
|
|
|
11660
11905
|
return [...new Set(files)];
|
|
11661
11906
|
}
|
|
11662
11907
|
join(...segments) {
|
|
11663
|
-
return
|
|
11908
|
+
return path26.join(...segments);
|
|
11664
11909
|
}
|
|
11665
11910
|
relative(from, to) {
|
|
11666
|
-
return
|
|
11911
|
+
return path26.relative(from, to);
|
|
11667
11912
|
}
|
|
11668
11913
|
resolve(...segments) {
|
|
11669
|
-
return
|
|
11914
|
+
return path26.resolve(...segments);
|
|
11670
11915
|
}
|
|
11671
11916
|
dirname(filepath) {
|
|
11672
|
-
return
|
|
11917
|
+
return path26.dirname(filepath);
|
|
11673
11918
|
}
|
|
11674
11919
|
extname(filepath) {
|
|
11675
|
-
return
|
|
11920
|
+
return path26.extname(filepath);
|
|
11676
11921
|
}
|
|
11677
11922
|
}
|
|
11678
11923
|
var nodeFileSystem;
|
|
@@ -11693,14 +11938,14 @@ __export(exports_search, {
|
|
|
11693
11938
|
formatSearchResults: () => formatSearchResults2,
|
|
11694
11939
|
formatHybridSearchResults: () => formatHybridSearchResults
|
|
11695
11940
|
});
|
|
11696
|
-
import * as
|
|
11697
|
-
import * as
|
|
11941
|
+
import * as fs11 from "fs/promises";
|
|
11942
|
+
import * as path27 from "path";
|
|
11698
11943
|
async function search(rootDir, query, options = {}) {
|
|
11699
11944
|
const hybridResults = await hybridSearch(rootDir, query, options);
|
|
11700
11945
|
return hybridResults.results;
|
|
11701
11946
|
}
|
|
11702
11947
|
async function hybridSearch(rootDir, query, options = {}) {
|
|
11703
|
-
rootDir =
|
|
11948
|
+
rootDir = path27.resolve(rootDir);
|
|
11704
11949
|
const ensureFresh = options.ensureFresh ?? DEFAULT_SEARCH_OPTIONS.ensureFresh;
|
|
11705
11950
|
if (ensureFresh) {
|
|
11706
11951
|
await ensureIndexFresh(rootDir, { quiet: true });
|
|
@@ -11777,15 +12022,15 @@ async function hybridSearch(rootDir, query, options = {}) {
|
|
|
11777
12022
|
};
|
|
11778
12023
|
}
|
|
11779
12024
|
async function performExactSearch(rootDir, literal, config, options) {
|
|
11780
|
-
const
|
|
11781
|
-
return executeExactSearch(
|
|
12025
|
+
const fs12 = new NodeFileSystem;
|
|
12026
|
+
return executeExactSearch(fs12, {
|
|
11782
12027
|
rootDir,
|
|
11783
12028
|
literal,
|
|
11784
12029
|
pathFilter: options.pathFilter,
|
|
11785
12030
|
maxFiles: 20,
|
|
11786
12031
|
maxOccurrencesPerFile: 5,
|
|
11787
12032
|
caseInsensitive: false
|
|
11788
|
-
}, (
|
|
12033
|
+
}, (path28, pattern) => minimatch(path28, pattern, { matchBase: true }));
|
|
11789
12034
|
}
|
|
11790
12035
|
function createSearchContext(rootDir, moduleId, config) {
|
|
11791
12036
|
const indexPath = getModuleIndexPath(rootDir, moduleId, config);
|
|
@@ -11794,9 +12039,9 @@ function createSearchContext(rootDir, moduleId, config) {
|
|
|
11794
12039
|
config,
|
|
11795
12040
|
loadFileIndex: async (filepath) => {
|
|
11796
12041
|
const hasExtension = /\.[^./]+$/.test(filepath);
|
|
11797
|
-
const indexFilePath = hasExtension ?
|
|
12042
|
+
const indexFilePath = hasExtension ? path27.join(indexPath, filepath.replace(/\.[^.]+$/, ".json")) : path27.join(indexPath, filepath + ".json");
|
|
11798
12043
|
try {
|
|
11799
|
-
const content = await
|
|
12044
|
+
const content = await fs11.readFile(indexFilePath, "utf-8");
|
|
11800
12045
|
return JSON.parse(content);
|
|
11801
12046
|
} catch {
|
|
11802
12047
|
return null;
|
|
@@ -11806,7 +12051,7 @@ function createSearchContext(rootDir, moduleId, config) {
|
|
|
11806
12051
|
const files = [];
|
|
11807
12052
|
await traverseDirectory(indexPath, files, indexPath);
|
|
11808
12053
|
return files.filter((f) => f.endsWith(".json") && !f.endsWith("manifest.json")).map((f) => {
|
|
11809
|
-
const relative6 =
|
|
12054
|
+
const relative6 = path27.relative(indexPath, f);
|
|
11810
12055
|
return relative6.replace(/\.json$/, "");
|
|
11811
12056
|
});
|
|
11812
12057
|
}
|
|
@@ -11814,9 +12059,9 @@ function createSearchContext(rootDir, moduleId, config) {
|
|
|
11814
12059
|
}
|
|
11815
12060
|
async function traverseDirectory(dir, files, basePath) {
|
|
11816
12061
|
try {
|
|
11817
|
-
const entries = await
|
|
12062
|
+
const entries = await fs11.readdir(dir, { withFileTypes: true });
|
|
11818
12063
|
for (const entry of entries) {
|
|
11819
|
-
const fullPath =
|
|
12064
|
+
const fullPath = path27.join(dir, entry.name);
|
|
11820
12065
|
if (entry.isDirectory()) {
|
|
11821
12066
|
await traverseDirectory(fullPath, files, basePath);
|
|
11822
12067
|
} else if (entry.isFile()) {
|
|
@@ -11828,7 +12073,7 @@ async function traverseDirectory(dir, files, basePath) {
|
|
|
11828
12073
|
async function loadGlobalManifest2(rootDir, config) {
|
|
11829
12074
|
const manifestPath = getGlobalManifestPath(rootDir, config);
|
|
11830
12075
|
try {
|
|
11831
|
-
const content = await
|
|
12076
|
+
const content = await fs11.readFile(manifestPath, "utf-8");
|
|
11832
12077
|
return JSON.parse(content);
|
|
11833
12078
|
} catch {
|
|
11834
12079
|
return null;
|
|
@@ -11997,18 +12242,18 @@ function getInstallationMethod(openCodeVersion) {
|
|
|
11997
12242
|
}
|
|
11998
12243
|
async function detectOpenCodeVersion() {
|
|
11999
12244
|
try {
|
|
12000
|
-
const
|
|
12001
|
-
const
|
|
12002
|
-
const
|
|
12003
|
-
const homeDir =
|
|
12245
|
+
const os3 = await import("os");
|
|
12246
|
+
const fs12 = await import("fs/promises");
|
|
12247
|
+
const path28 = await import("path");
|
|
12248
|
+
const homeDir = os3.homedir();
|
|
12004
12249
|
const possiblePaths = [
|
|
12005
|
-
|
|
12006
|
-
|
|
12007
|
-
|
|
12250
|
+
path28.join(homeDir, ".local", "share", "opencode", "package.json"),
|
|
12251
|
+
path28.join(homeDir, ".config", "opencode", "package.json"),
|
|
12252
|
+
path28.join(homeDir, ".npm", "global", "node_modules", "opencode", "package.json")
|
|
12008
12253
|
];
|
|
12009
12254
|
for (const packagePath of possiblePaths) {
|
|
12010
12255
|
try {
|
|
12011
|
-
const content = await
|
|
12256
|
+
const content = await fs12.readFile(packagePath, "utf-8");
|
|
12012
12257
|
const pkg = JSON.parse(content);
|
|
12013
12258
|
if (pkg.version) {
|
|
12014
12259
|
return pkg.version;
|
|
@@ -12046,12 +12291,12 @@ async function detectOpenCodeVersion() {
|
|
|
12046
12291
|
// src/app/cli/opencode/install-tool.ts
|
|
12047
12292
|
async function installTool(options = {}) {
|
|
12048
12293
|
const { logger, checkForOldSkill = true } = options;
|
|
12049
|
-
const
|
|
12050
|
-
const
|
|
12051
|
-
const
|
|
12052
|
-
const homeDir =
|
|
12053
|
-
const toolDir =
|
|
12054
|
-
const toolPath =
|
|
12294
|
+
const os3 = await import("os");
|
|
12295
|
+
const fs12 = await import("fs/promises");
|
|
12296
|
+
const path28 = await import("path");
|
|
12297
|
+
const homeDir = os3.homedir();
|
|
12298
|
+
const toolDir = path28.join(homeDir, ".config", "opencode", "tool");
|
|
12299
|
+
const toolPath = path28.join(toolDir, "raggrep.ts");
|
|
12055
12300
|
let removedOldSkill = false;
|
|
12056
12301
|
const toolContent = `import { tool } from "@opencode-ai/plugin";
|
|
12057
12302
|
|
|
@@ -12134,11 +12379,11 @@ Please install raggrep globally:
|
|
|
12134
12379
|
`;
|
|
12135
12380
|
try {
|
|
12136
12381
|
if (checkForOldSkill) {
|
|
12137
|
-
const oldSkillDir =
|
|
12138
|
-
const oldSkillPath =
|
|
12382
|
+
const oldSkillDir = path28.join(homeDir, ".config", "opencode", "skill", "raggrep");
|
|
12383
|
+
const oldSkillPath = path28.join(oldSkillDir, "SKILL.md");
|
|
12139
12384
|
let oldSkillExists = false;
|
|
12140
12385
|
try {
|
|
12141
|
-
await
|
|
12386
|
+
await fs12.access(oldSkillPath);
|
|
12142
12387
|
oldSkillExists = true;
|
|
12143
12388
|
} catch {}
|
|
12144
12389
|
if (oldSkillExists) {
|
|
@@ -12163,11 +12408,11 @@ Please install raggrep globally:
|
|
|
12163
12408
|
const shouldDelete = answer.toLowerCase() !== "n";
|
|
12164
12409
|
if (shouldDelete) {
|
|
12165
12410
|
try {
|
|
12166
|
-
await
|
|
12167
|
-
const skillDirContents = await
|
|
12411
|
+
await fs12.unlink(oldSkillPath);
|
|
12412
|
+
const skillDirContents = await fs12.readdir(oldSkillDir);
|
|
12168
12413
|
if (skillDirContents.length === 0) {
|
|
12169
12414
|
try {
|
|
12170
|
-
await
|
|
12415
|
+
await fs12.rmdir(oldSkillDir);
|
|
12171
12416
|
console.log("✓ Removed old skill directory.");
|
|
12172
12417
|
} catch (rmdirError) {
|
|
12173
12418
|
console.log("✓ Removed old skill file. (Directory not empty or other error)");
|
|
@@ -12204,8 +12449,8 @@ Please install raggrep globally:
|
|
|
12204
12449
|
}
|
|
12205
12450
|
}
|
|
12206
12451
|
}
|
|
12207
|
-
await
|
|
12208
|
-
await
|
|
12452
|
+
await fs12.mkdir(toolDir, { recursive: true });
|
|
12453
|
+
await fs12.writeFile(toolPath, toolContent, "utf-8");
|
|
12209
12454
|
const message = `Installed raggrep tool for OpenCode.
|
|
12210
12455
|
Location: ${toolPath}
|
|
12211
12456
|
|
|
@@ -12238,12 +12483,12 @@ The raggrep tool is now available in OpenCode.`;
|
|
|
12238
12483
|
// src/app/cli/opencode/install-skill.ts
|
|
12239
12484
|
async function installSkill(options = {}) {
|
|
12240
12485
|
const { logger, checkForOldTool = true } = options;
|
|
12241
|
-
const
|
|
12242
|
-
const
|
|
12243
|
-
const
|
|
12244
|
-
const homeDir =
|
|
12245
|
-
const skillDir =
|
|
12246
|
-
const skillPath =
|
|
12486
|
+
const os3 = await import("os");
|
|
12487
|
+
const fs12 = await import("fs/promises");
|
|
12488
|
+
const path28 = await import("path");
|
|
12489
|
+
const homeDir = os3.homedir();
|
|
12490
|
+
const skillDir = path28.join(homeDir, ".config", "opencode", "skill", "raggrep");
|
|
12491
|
+
const skillPath = path28.join(skillDir, "SKILL.md");
|
|
12247
12492
|
const skillContent = `---
|
|
12248
12493
|
name: raggrep
|
|
12249
12494
|
description: AST-powered semantic code search that understands intent, not just text. Superior to grep/rg - finds functions, classes, and logic even when keywords differ. Saves 10x tool calls by searching the actual code structure.
|
|
@@ -12283,19 +12528,32 @@ pnpm add -g raggrep
|
|
|
12283
12528
|
\`\`\`
|
|
12284
12529
|
|
|
12285
12530
|
### Step 1: Index your codebase
|
|
12531
|
+
|
|
12532
|
+
Index data is written to **\`<project>/.raggrep/\`** inside the directory you target. Add \`.raggrep/\` to \`.gitignore\` if you do not want it in version control.
|
|
12533
|
+
|
|
12534
|
+
From the repository root:
|
|
12286
12535
|
\`\`\`bash
|
|
12287
|
-
# Navigate to your project directory and index it
|
|
12288
12536
|
cd /path/to/your/project
|
|
12289
12537
|
raggrep index
|
|
12290
12538
|
\`\`\`
|
|
12291
12539
|
|
|
12540
|
+
Or pass the project path explicitly (no \`cd\` needed; same flag as \`git -C\`):
|
|
12541
|
+
\`\`\`bash
|
|
12542
|
+
raggrep index --dir /path/to/your/project
|
|
12543
|
+
# short form:
|
|
12544
|
+
raggrep index -C ~/projects/my-app
|
|
12545
|
+
\`\`\`
|
|
12546
|
+
|
|
12292
12547
|
### Step 2: Use semantic search
|
|
12548
|
+
|
|
12293
12549
|
\`\`\`bash
|
|
12294
|
-
# Search for specific functionality
|
|
12550
|
+
# Search for specific functionality (from project root, or use --dir / -C)
|
|
12295
12551
|
raggrep query "user authentication"
|
|
12552
|
+
raggrep query -C /path/to/your/project "user authentication"
|
|
12296
12553
|
|
|
12297
12554
|
# Search with filters
|
|
12298
12555
|
raggrep query "React hooks for data fetching" --filter "src/components"
|
|
12556
|
+
raggrep query -C /path/to/your/project "React hooks" --filter "src/components"
|
|
12299
12557
|
|
|
12300
12558
|
# Search with specific file types
|
|
12301
12559
|
raggrep query "database connection" --type ts
|
|
@@ -12304,6 +12562,8 @@ raggrep query "database connection" --type ts
|
|
|
12304
12562
|
raggrep query "error handling" --top 15
|
|
12305
12563
|
\`\`\`
|
|
12306
12564
|
|
|
12565
|
+
Use **\`-C\` / \`--dir\`** on \`raggrep query\`, \`raggrep index\`, \`raggrep status\`, and \`raggrep reset\` when the shell is not already in the repository root.
|
|
12566
|
+
|
|
12307
12567
|
### Step 3: Use in OpenCode agents
|
|
12308
12568
|
|
|
12309
12569
|
Load this skill in your agent conversation:
|
|
@@ -12339,6 +12599,7 @@ Instead of using grep/rg or manually reading files:
|
|
|
12339
12599
|
|
|
12340
12600
|
✅ **DO this instead:**
|
|
12341
12601
|
- \`raggrep query "Find the auth middleware"\` (finds ALL auth-related code)
|
|
12602
|
+
- \`raggrep query -C /path/to/repo "Find the auth middleware"\` (when not \`cd\`'d into the repo)
|
|
12342
12603
|
- \`raggrep query "Where are React components?"\`
|
|
12343
12604
|
- \`raggrep query "Database connection logic?"\`
|
|
12344
12605
|
- \`raggrep query "Error handling patterns"\`
|
|
@@ -12349,17 +12610,18 @@ Instead of using grep/rg or manually reading files:
|
|
|
12349
12610
|
2. **Use filters strategically**: \`--filter "src/auth"\`, \`--filter "*.test.ts"\`
|
|
12350
12611
|
3. **Adjust result count**: Use \`--top 5\` for focused results, \`--top 20\` for comprehensive search
|
|
12351
12612
|
4. **Replace grep/rg habits**: Instead of \`rg "pattern"\`, try \`raggrep query "what the code does"\`
|
|
12613
|
+
5. **Target the right tree**: If your shell is not in the repo root, use \`-C /path/to/repo\` or \`--dir\` on every command so the index under that repo's \`.raggrep/\` stays aligned with searches
|
|
12352
12614
|
|
|
12353
12615
|
**Result**: 10x fewer tool calls, BETTER results, deeper code understanding.
|
|
12354
12616
|
`;
|
|
12355
12617
|
let removedOldTool = false;
|
|
12356
12618
|
try {
|
|
12357
12619
|
if (checkForOldTool) {
|
|
12358
|
-
const oldToolDir =
|
|
12359
|
-
const oldToolPath =
|
|
12620
|
+
const oldToolDir = path28.join(homeDir, ".config", "opencode", "tool");
|
|
12621
|
+
const oldToolPath = path28.join(oldToolDir, "raggrep.ts");
|
|
12360
12622
|
let oldToolExists = false;
|
|
12361
12623
|
try {
|
|
12362
|
-
await
|
|
12624
|
+
await fs12.access(oldToolPath);
|
|
12363
12625
|
oldToolExists = true;
|
|
12364
12626
|
} catch {}
|
|
12365
12627
|
if (oldToolExists) {
|
|
@@ -12384,11 +12646,11 @@ Instead of using grep/rg or manually reading files:
|
|
|
12384
12646
|
const shouldDelete = answer.toLowerCase() !== "n";
|
|
12385
12647
|
if (shouldDelete) {
|
|
12386
12648
|
try {
|
|
12387
|
-
await
|
|
12388
|
-
const toolDirContents = await
|
|
12649
|
+
await fs12.unlink(oldToolPath);
|
|
12650
|
+
const toolDirContents = await fs12.readdir(oldToolDir);
|
|
12389
12651
|
if (toolDirContents.length === 0) {
|
|
12390
12652
|
try {
|
|
12391
|
-
await
|
|
12653
|
+
await fs12.rmdir(oldToolDir);
|
|
12392
12654
|
console.log("✓ Removed old tool directory.");
|
|
12393
12655
|
} catch (rmdirError) {
|
|
12394
12656
|
console.log("✓ Removed old tool file. (Directory not empty or other error)");
|
|
@@ -12421,8 +12683,8 @@ Instead of using grep/rg or manually reading files:
|
|
|
12421
12683
|
}
|
|
12422
12684
|
}
|
|
12423
12685
|
}
|
|
12424
|
-
await
|
|
12425
|
-
await
|
|
12686
|
+
await fs12.mkdir(skillDir, { recursive: true });
|
|
12687
|
+
await fs12.writeFile(skillPath, skillContent, "utf-8");
|
|
12426
12688
|
const message = `Installed raggrep skill for OpenCode.
|
|
12427
12689
|
Location: ${skillPath}
|
|
12428
12690
|
|
|
@@ -12430,7 +12692,7 @@ The raggrep skill is now available to OpenCode agents.
|
|
|
12430
12692
|
|
|
12431
12693
|
To use this skill:
|
|
12432
12694
|
1. Install raggrep: npm install -g raggrep
|
|
12433
|
-
2. Index your codebase: raggrep index
|
|
12695
|
+
2. Index your codebase: \`cd\` to the repo and \`raggrep index\`, or \`raggrep index --dir /path/to/repo\`
|
|
12434
12696
|
3. In OpenCode, load the skill: skill({ name: "raggrep" })`;
|
|
12435
12697
|
if (logger) {
|
|
12436
12698
|
logger.info(message);
|
|
@@ -12472,10 +12734,12 @@ var init_opencode = () => {};
|
|
|
12472
12734
|
// src/app/cli/main.ts
|
|
12473
12735
|
init_embeddings();
|
|
12474
12736
|
init_logger();
|
|
12737
|
+
import * as path28 from "path";
|
|
12738
|
+
import { stat as stat3 } from "fs/promises";
|
|
12475
12739
|
// package.json
|
|
12476
12740
|
var package_default = {
|
|
12477
12741
|
name: "raggrep",
|
|
12478
|
-
version: "0.
|
|
12742
|
+
version: "0.17.0",
|
|
12479
12743
|
description: "Local filesystem-based RAG system for codebases - semantic search using local embeddings",
|
|
12480
12744
|
type: "module",
|
|
12481
12745
|
main: "./dist/index.js",
|
|
@@ -12497,12 +12761,15 @@ var package_default = {
|
|
|
12497
12761
|
scripts: {
|
|
12498
12762
|
build: "bun run build:clean && bun run build:bundle && bun run build:types && bun run build:shebang",
|
|
12499
12763
|
"build:clean": "rm -rf dist",
|
|
12500
|
-
"build:bundle": "bun build src/index.ts --outdir dist --target node --sourcemap=external --external '@xenova/transformers' --external 'glob' --external 'typescript' --external 'chokidar' --external 'web-tree-sitter' && bun build src/app/cli/main.ts --outdir dist/cli --target node --sourcemap=external --external '@xenova/transformers' --external 'glob' --external 'typescript' --external 'chokidar' --external 'web-tree-sitter'",
|
|
12501
|
-
"build:types": "tsc --emitDeclarationOnly --outDir dist",
|
|
12764
|
+
"build:bundle": "bun build src/index.ts --outdir dist --target node --sourcemap=external --external '@xenova/transformers' --external '@huggingface/transformers' --external 'glob' --external 'typescript' --external 'chokidar' --external 'web-tree-sitter' && bun build src/app/cli/main.ts --outdir dist/cli --target node --sourcemap=external --external '@xenova/transformers' --external '@huggingface/transformers' --external 'glob' --external 'typescript' --external 'chokidar' --external 'web-tree-sitter'",
|
|
12765
|
+
"build:types": "tsc --project tsconfig.json --noEmit false --declaration true --emitDeclarationOnly true --rootDir ./src --outDir ./dist",
|
|
12502
12766
|
"build:shebang": "echo '#!/usr/bin/env node' | cat - dist/cli/main.js > temp && mv temp dist/cli/main.js && chmod +x dist/cli/main.js",
|
|
12503
12767
|
prepublishOnly: "bun run build",
|
|
12504
12768
|
raggrep: "bun run src/app/cli/main.ts",
|
|
12505
12769
|
test: "bun test",
|
|
12770
|
+
typecheck: "tsc --noEmit -p tsconfig.json && tsc --noEmit -p scripts/tsconfig.json",
|
|
12771
|
+
"bench:embeddings": "bun run scripts/benchmark-embedding-runtimes.ts",
|
|
12772
|
+
"bench:retrieval": "bun run scripts/benchmark-retrieval-quality.ts",
|
|
12506
12773
|
dev: "bun run src/app/cli/main.ts"
|
|
12507
12774
|
},
|
|
12508
12775
|
keywords: [
|
|
@@ -12530,6 +12797,7 @@ var package_default = {
|
|
|
12530
12797
|
node: ">=18.0.0"
|
|
12531
12798
|
},
|
|
12532
12799
|
dependencies: {
|
|
12800
|
+
"@huggingface/transformers": "^4.0.0",
|
|
12533
12801
|
"@xenova/transformers": "^2.17.0",
|
|
12534
12802
|
chokidar: "^5.0.0",
|
|
12535
12803
|
fdir: "^6.5.0",
|
|
@@ -12638,6 +12906,14 @@ function parseFlags(args2) {
|
|
|
12638
12906
|
console.error('--filter requires a path or glob pattern (e.g., src/auth, "*.ts")');
|
|
12639
12907
|
process.exit(1);
|
|
12640
12908
|
}
|
|
12909
|
+
} else if (arg === "--dir" || arg === "-C") {
|
|
12910
|
+
const dir = args2[++i];
|
|
12911
|
+
if (dir) {
|
|
12912
|
+
flags.projectDir = dir;
|
|
12913
|
+
} else {
|
|
12914
|
+
console.error("--dir / -C requires a path to the project directory to index or search.");
|
|
12915
|
+
process.exit(1);
|
|
12916
|
+
}
|
|
12641
12917
|
} else if (arg === "--tool") {
|
|
12642
12918
|
flags.forceTool = true;
|
|
12643
12919
|
} else if (arg === "--skill") {
|
|
@@ -12648,6 +12924,20 @@ function parseFlags(args2) {
|
|
|
12648
12924
|
}
|
|
12649
12925
|
return flags;
|
|
12650
12926
|
}
|
|
12927
|
+
async function resolveProjectRoot(flags) {
|
|
12928
|
+
const root = path28.resolve(flags.projectDir ?? process.cwd());
|
|
12929
|
+
try {
|
|
12930
|
+
const st = await stat3(root);
|
|
12931
|
+
if (!st.isDirectory()) {
|
|
12932
|
+
console.error(`Error: not a directory: ${root}`);
|
|
12933
|
+
process.exit(1);
|
|
12934
|
+
}
|
|
12935
|
+
} catch {
|
|
12936
|
+
console.error(`Error: directory does not exist or is not accessible: ${root}`);
|
|
12937
|
+
process.exit(1);
|
|
12938
|
+
}
|
|
12939
|
+
return root;
|
|
12940
|
+
}
|
|
12651
12941
|
async function main() {
|
|
12652
12942
|
const flags = parseFlags(args.slice(1));
|
|
12653
12943
|
switch (command) {
|
|
@@ -12655,14 +12945,15 @@ async function main() {
|
|
|
12655
12945
|
if (flags.help) {
|
|
12656
12946
|
const models = Object.keys(EMBEDDING_MODELS).join(", ");
|
|
12657
12947
|
console.log(`
|
|
12658
|
-
raggrep index - Index
|
|
12948
|
+
raggrep index - Index a directory for semantic search
|
|
12659
12949
|
|
|
12660
12950
|
Usage:
|
|
12661
12951
|
raggrep index [options]
|
|
12662
12952
|
|
|
12663
12953
|
Options:
|
|
12954
|
+
-C, --dir <path> Project directory to index (default: current directory)
|
|
12664
12955
|
-w, --watch Watch for file changes and re-index automatically
|
|
12665
|
-
-m, --model <name>
|
|
12956
|
+
-m, --model <name> Override embedding model for the TypeScript module (default: saved config, else bge-small-en-v1.5)
|
|
12666
12957
|
-c, --concurrency <n> Number of files to process in parallel (default: auto)
|
|
12667
12958
|
-v, --verbose Show detailed progress
|
|
12668
12959
|
-h, --help Show this help message
|
|
@@ -12674,6 +12965,7 @@ Model Cache: ${getCacheDir()}
|
|
|
12674
12965
|
|
|
12675
12966
|
Examples:
|
|
12676
12967
|
raggrep index
|
|
12968
|
+
raggrep index --dir ../other-repo
|
|
12677
12969
|
raggrep index --watch
|
|
12678
12970
|
raggrep index --model bge-small-en-v1.5
|
|
12679
12971
|
raggrep index --concurrency 8
|
|
@@ -12683,11 +12975,12 @@ Examples:
|
|
|
12683
12975
|
}
|
|
12684
12976
|
const { indexDirectory: indexDirectory3, watchDirectory: watchDirectory2 } = await Promise.resolve().then(() => (init_indexer(), exports_indexer));
|
|
12685
12977
|
const logger = createInlineLogger({ verbose: flags.verbose });
|
|
12978
|
+
const projectRoot = await resolveProjectRoot(flags);
|
|
12686
12979
|
console.log("RAGgrep Indexer");
|
|
12687
12980
|
console.log(`================
|
|
12688
12981
|
`);
|
|
12689
12982
|
try {
|
|
12690
|
-
const results = await indexDirectory3(
|
|
12983
|
+
const results = await indexDirectory3(projectRoot, {
|
|
12691
12984
|
model: flags.model,
|
|
12692
12985
|
verbose: flags.verbose,
|
|
12693
12986
|
concurrency: flags.concurrency,
|
|
@@ -12710,7 +13003,7 @@ Examples:
|
|
|
12710
13003
|
console.log(`└─────────────────────────────────────────┘
|
|
12711
13004
|
`);
|
|
12712
13005
|
try {
|
|
12713
|
-
const watcher = await watchDirectory2(
|
|
13006
|
+
const watcher = await watchDirectory2(projectRoot, {
|
|
12714
13007
|
model: flags.model,
|
|
12715
13008
|
verbose: flags.verbose,
|
|
12716
13009
|
onFileChange: (event, filepath) => {
|
|
@@ -12747,6 +13040,7 @@ Usage:
|
|
|
12747
13040
|
raggrep query <search query> [options]
|
|
12748
13041
|
|
|
12749
13042
|
Options:
|
|
13043
|
+
-C, --dir <path> Project directory to search (default: current directory)
|
|
12750
13044
|
-k, --top <n> Number of results to return (default: 10)
|
|
12751
13045
|
-s, --min-score <n> Minimum similarity score 0-1 (default: 0.15)
|
|
12752
13046
|
-t, --type <ext> Filter by file extension (e.g., ts, tsx, js)
|
|
@@ -12774,6 +13068,7 @@ Multiple Filters (OR logic):
|
|
|
12774
13068
|
|
|
12775
13069
|
Examples:
|
|
12776
13070
|
raggrep query "user authentication"
|
|
13071
|
+
raggrep query --dir ~/projects/my-app "user authentication"
|
|
12777
13072
|
raggrep query "handle errors" --top 5
|
|
12778
13073
|
raggrep query "database" --min-score 0.1
|
|
12779
13074
|
raggrep query "interface" --type ts
|
|
@@ -12799,9 +13094,10 @@ Examples:
|
|
|
12799
13094
|
console.error('Run "raggrep query --help" for more information.');
|
|
12800
13095
|
process.exit(1);
|
|
12801
13096
|
}
|
|
13097
|
+
const projectRoot = await resolveProjectRoot(flags);
|
|
12802
13098
|
try {
|
|
12803
13099
|
const silentLogger = createSilentLogger();
|
|
12804
|
-
const freshStats = await ensureIndexFresh2(
|
|
13100
|
+
const freshStats = await ensureIndexFresh2(projectRoot, {
|
|
12805
13101
|
model: flags.model,
|
|
12806
13102
|
quiet: true,
|
|
12807
13103
|
logger: silentLogger,
|
|
@@ -12842,7 +13138,7 @@ Examples:
|
|
|
12842
13138
|
}
|
|
12843
13139
|
const filePatterns = flags.fileType ? [`*.${flags.fileType}`] : undefined;
|
|
12844
13140
|
const { hybridSearch: hybridSearch2, formatHybridSearchResults: formatHybridSearchResults2 } = await Promise.resolve().then(() => (init_search(), exports_search));
|
|
12845
|
-
const hybridResults = await hybridSearch2(
|
|
13141
|
+
const hybridResults = await hybridSearch2(projectRoot, query, {
|
|
12846
13142
|
topK: flags.topK ?? 10,
|
|
12847
13143
|
minScore: flags.minScore,
|
|
12848
13144
|
filePatterns,
|
|
@@ -12859,26 +13155,29 @@ Examples:
|
|
|
12859
13155
|
case "reset": {
|
|
12860
13156
|
if (flags.help) {
|
|
12861
13157
|
console.log(`
|
|
12862
|
-
raggrep reset - Clear the index for
|
|
13158
|
+
raggrep reset - Clear the index for a project directory
|
|
12863
13159
|
|
|
12864
13160
|
Usage:
|
|
12865
13161
|
raggrep reset [options]
|
|
12866
13162
|
|
|
12867
13163
|
Options:
|
|
13164
|
+
-C, --dir <path> Project directory whose index to remove (default: current directory)
|
|
12868
13165
|
-h, --help Show this help message
|
|
12869
13166
|
|
|
12870
13167
|
Description:
|
|
12871
|
-
Completely removes the index for the
|
|
13168
|
+
Completely removes the .raggrep index for the project directory.
|
|
12872
13169
|
The next 'raggrep index' or 'raggrep query' will rebuild from scratch.
|
|
12873
13170
|
|
|
12874
13171
|
Examples:
|
|
12875
13172
|
raggrep reset
|
|
13173
|
+
raggrep reset --dir ../other-repo
|
|
12876
13174
|
`);
|
|
12877
13175
|
process.exit(0);
|
|
12878
13176
|
}
|
|
12879
13177
|
const { resetIndex: resetIndex2 } = await Promise.resolve().then(() => (init_indexer(), exports_indexer));
|
|
13178
|
+
const projectRoot = await resolveProjectRoot(flags);
|
|
12880
13179
|
try {
|
|
12881
|
-
const result = await resetIndex2(
|
|
13180
|
+
const result = await resetIndex2(projectRoot);
|
|
12882
13181
|
console.log("Index cleared successfully.");
|
|
12883
13182
|
console.log(` Removed: ${result.indexDir}`);
|
|
12884
13183
|
} catch (error) {
|
|
@@ -12900,21 +13199,24 @@ Usage:
|
|
|
12900
13199
|
raggrep status [options]
|
|
12901
13200
|
|
|
12902
13201
|
Options:
|
|
13202
|
+
-C, --dir <path> Project directory to report on (default: current directory)
|
|
12903
13203
|
-h, --help Show this help message
|
|
12904
13204
|
|
|
12905
13205
|
Description:
|
|
12906
|
-
Displays information about the index
|
|
13206
|
+
Displays information about the index for the project directory,
|
|
12907
13207
|
including whether it exists, how many files are indexed, and
|
|
12908
13208
|
when it was last updated.
|
|
12909
13209
|
|
|
12910
13210
|
Examples:
|
|
12911
13211
|
raggrep status
|
|
13212
|
+
raggrep status --dir ../other-repo
|
|
12912
13213
|
`);
|
|
12913
13214
|
process.exit(0);
|
|
12914
13215
|
}
|
|
12915
13216
|
const { getIndexStatus: getIndexStatus2 } = await Promise.resolve().then(() => (init_indexer(), exports_indexer));
|
|
13217
|
+
const projectRoot = await resolveProjectRoot(flags);
|
|
12916
13218
|
try {
|
|
12917
|
-
const status = await getIndexStatus2(
|
|
13219
|
+
const status = await getIndexStatus2(projectRoot);
|
|
12918
13220
|
if (!status.exists) {
|
|
12919
13221
|
console.log(`
|
|
12920
13222
|
┌─────────────────────────────────────────┐
|
|
@@ -13045,19 +13347,23 @@ Usage:
|
|
|
13045
13347
|
raggrep <command> [options]
|
|
13046
13348
|
|
|
13047
13349
|
Commands:
|
|
13048
|
-
index Index
|
|
13350
|
+
index Index a project directory
|
|
13049
13351
|
query Search the indexed codebase
|
|
13050
13352
|
status Show the current state of the index
|
|
13051
|
-
reset Clear the index for
|
|
13353
|
+
reset Clear the index for a project directory
|
|
13052
13354
|
opencode Manage opencode integration
|
|
13053
13355
|
|
|
13054
13356
|
Options:
|
|
13055
13357
|
-h, --help Show help for a command
|
|
13056
13358
|
-v, --version Show version number
|
|
13057
13359
|
|
|
13360
|
+
Project directory (index, query, status, reset):
|
|
13361
|
+
Use -C or --dir <path> to target a directory other than the current one.
|
|
13362
|
+
|
|
13058
13363
|
Examples:
|
|
13059
13364
|
raggrep index
|
|
13060
13365
|
raggrep query "user login"
|
|
13366
|
+
raggrep query -C ~/projects/app "user login"
|
|
13061
13367
|
raggrep status
|
|
13062
13368
|
raggrep reset
|
|
13063
13369
|
raggrep opencode install
|
|
@@ -13072,4 +13378,4 @@ Run 'raggrep <command> --help' for more information.
|
|
|
13072
13378
|
}
|
|
13073
13379
|
main();
|
|
13074
13380
|
|
|
13075
|
-
//# debugId=
|
|
13381
|
+
//# debugId=6B05E3A822FD1AE664756E2164756E21
|