raggrep 0.16.0 → 0.17.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +29 -5
- package/dist/cli/main.js +713 -458
- package/dist/cli/main.js.map +19 -13
- package/dist/domain/ports/embedding.d.ts +10 -0
- package/dist/domain/ports/index.d.ts +1 -1
- package/dist/index.js +524 -327
- package/dist/index.js.map +18 -12
- package/dist/infrastructure/config/configLoader.d.ts +9 -11
- package/dist/infrastructure/config/index.d.ts +1 -1
- package/dist/infrastructure/embeddings/embeddingPaths.d.ts +6 -0
- package/dist/infrastructure/embeddings/embeddingProviderFactory.d.ts +9 -0
- package/dist/infrastructure/embeddings/globalEmbeddings.d.ts +28 -0
- package/dist/infrastructure/embeddings/huggingfaceEmbeddingProvider.d.ts +21 -0
- package/dist/infrastructure/embeddings/index.d.ts +9 -2
- package/dist/infrastructure/embeddings/modelCache.d.ts +10 -0
- package/dist/infrastructure/embeddings/modelCatalog.d.ts +23 -0
- package/dist/infrastructure/embeddings/xenovaEmbeddingProvider.d.ts +23 -0
- package/dist/infrastructure/index.d.ts +1 -1
- package/package.json +7 -3
- package/dist/infrastructure/embeddings/transformersEmbedding.d.ts +0 -52
package/dist/index.js
CHANGED
|
@@ -1,27 +1,16 @@
|
|
|
1
1
|
import { createRequire } from "node:module";
|
|
2
|
-
var __create = Object.create;
|
|
3
|
-
var __getProtoOf = Object.getPrototypeOf;
|
|
4
2
|
var __defProp = Object.defineProperty;
|
|
5
|
-
var
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
const to = isNodeMode || !mod || !mod.__esModule ? __defProp(target, "default", { value: mod, enumerable: true }) : target;
|
|
10
|
-
for (let key of __getOwnPropNames(mod))
|
|
11
|
-
if (!__hasOwnProp.call(to, key))
|
|
12
|
-
__defProp(to, key, {
|
|
13
|
-
get: () => mod[key],
|
|
14
|
-
enumerable: true
|
|
15
|
-
});
|
|
16
|
-
return to;
|
|
17
|
-
};
|
|
3
|
+
var __returnValue = (v) => v;
|
|
4
|
+
function __exportSetter(name, newValue) {
|
|
5
|
+
this[name] = __returnValue.bind(null, newValue);
|
|
6
|
+
}
|
|
18
7
|
var __export = (target, all) => {
|
|
19
8
|
for (var name in all)
|
|
20
9
|
__defProp(target, name, {
|
|
21
10
|
get: all[name],
|
|
22
11
|
enumerable: true,
|
|
23
12
|
configurable: true,
|
|
24
|
-
set: (
|
|
13
|
+
set: __exportSetter.bind(all, name)
|
|
25
14
|
});
|
|
26
15
|
};
|
|
27
16
|
var __esm = (fn, res) => () => (fn && (res = fn(fn = 0)), res);
|
|
@@ -54,49 +43,37 @@ function createDefaultConfig() {
|
|
|
54
43
|
{
|
|
55
44
|
id: "language/typescript",
|
|
56
45
|
enabled: true,
|
|
57
|
-
options: {
|
|
58
|
-
embeddingModel: "all-MiniLM-L6-v2"
|
|
59
|
-
}
|
|
46
|
+
options: { ...DEFAULT_EMBEDDING_MODULE_OPTIONS }
|
|
60
47
|
},
|
|
61
48
|
{
|
|
62
49
|
id: "language/python",
|
|
63
50
|
enabled: true,
|
|
64
|
-
options: {
|
|
65
|
-
embeddingModel: "all-MiniLM-L6-v2"
|
|
66
|
-
}
|
|
51
|
+
options: { ...DEFAULT_EMBEDDING_MODULE_OPTIONS }
|
|
67
52
|
},
|
|
68
53
|
{
|
|
69
54
|
id: "language/go",
|
|
70
55
|
enabled: true,
|
|
71
|
-
options: {
|
|
72
|
-
embeddingModel: "all-MiniLM-L6-v2"
|
|
73
|
-
}
|
|
56
|
+
options: { ...DEFAULT_EMBEDDING_MODULE_OPTIONS }
|
|
74
57
|
},
|
|
75
58
|
{
|
|
76
59
|
id: "language/rust",
|
|
77
60
|
enabled: true,
|
|
78
|
-
options: {
|
|
79
|
-
embeddingModel: "all-MiniLM-L6-v2"
|
|
80
|
-
}
|
|
61
|
+
options: { ...DEFAULT_EMBEDDING_MODULE_OPTIONS }
|
|
81
62
|
},
|
|
82
63
|
{
|
|
83
64
|
id: "data/json",
|
|
84
65
|
enabled: true,
|
|
85
|
-
options: {
|
|
86
|
-
embeddingModel: "all-MiniLM-L6-v2"
|
|
87
|
-
}
|
|
66
|
+
options: { ...DEFAULT_EMBEDDING_MODULE_OPTIONS }
|
|
88
67
|
},
|
|
89
68
|
{
|
|
90
69
|
id: "docs/markdown",
|
|
91
70
|
enabled: true,
|
|
92
|
-
options: {
|
|
93
|
-
embeddingModel: "all-MiniLM-L6-v2"
|
|
94
|
-
}
|
|
71
|
+
options: { ...DEFAULT_EMBEDDING_MODULE_OPTIONS }
|
|
95
72
|
}
|
|
96
73
|
]
|
|
97
74
|
};
|
|
98
75
|
}
|
|
99
|
-
var DEFAULT_IGNORE_PATHS, DEFAULT_EXTENSIONS;
|
|
76
|
+
var DEFAULT_IGNORE_PATHS, DEFAULT_EXTENSIONS, DEFAULT_EMBEDDING_MODULE_OPTIONS;
|
|
100
77
|
var init_config = __esm(() => {
|
|
101
78
|
DEFAULT_IGNORE_PATHS = [
|
|
102
79
|
"node_modules",
|
|
@@ -166,6 +143,10 @@ var init_config = __esm(() => {
|
|
|
166
143
|
".sql",
|
|
167
144
|
".txt"
|
|
168
145
|
];
|
|
146
|
+
DEFAULT_EMBEDDING_MODULE_OPTIONS = {
|
|
147
|
+
embeddingModel: "bge-small-en-v1.5",
|
|
148
|
+
embeddingRuntime: "huggingface"
|
|
149
|
+
};
|
|
169
150
|
});
|
|
170
151
|
|
|
171
152
|
// src/domain/entities/literal.ts
|
|
@@ -195,24 +176,48 @@ var init_entities = __esm(() => {
|
|
|
195
176
|
init_lexicon();
|
|
196
177
|
});
|
|
197
178
|
|
|
179
|
+
// src/infrastructure/embeddings/modelCatalog.ts
|
|
180
|
+
function getEmbeddingModelId(model) {
|
|
181
|
+
return EMBEDDING_MODEL_IDS[model];
|
|
182
|
+
}
|
|
183
|
+
function getEmbeddingDimension(model) {
|
|
184
|
+
return EMBEDDING_DIMENSIONS[model];
|
|
185
|
+
}
|
|
186
|
+
var EMBEDDING_MODEL_IDS, EMBEDDING_MODELS, EMBEDDING_DIMENSIONS;
|
|
187
|
+
var init_modelCatalog = __esm(() => {
|
|
188
|
+
EMBEDDING_MODEL_IDS = {
|
|
189
|
+
"all-MiniLM-L6-v2": "Xenova/all-MiniLM-L6-v2",
|
|
190
|
+
"all-MiniLM-L12-v2": "Xenova/all-MiniLM-L12-v2",
|
|
191
|
+
"bge-small-en-v1.5": "Xenova/bge-small-en-v1.5",
|
|
192
|
+
"paraphrase-MiniLM-L3-v2": "Xenova/paraphrase-MiniLM-L3-v2",
|
|
193
|
+
"nomic-embed-text-v1.5": "nomic-ai/nomic-embed-text-v1.5"
|
|
194
|
+
};
|
|
195
|
+
EMBEDDING_MODELS = EMBEDDING_MODEL_IDS;
|
|
196
|
+
EMBEDDING_DIMENSIONS = {
|
|
197
|
+
"all-MiniLM-L6-v2": 384,
|
|
198
|
+
"all-MiniLM-L12-v2": 384,
|
|
199
|
+
"bge-small-en-v1.5": 384,
|
|
200
|
+
"paraphrase-MiniLM-L3-v2": 384,
|
|
201
|
+
"nomic-embed-text-v1.5": 768
|
|
202
|
+
};
|
|
203
|
+
});
|
|
204
|
+
|
|
198
205
|
// src/infrastructure/config/configLoader.ts
|
|
199
206
|
import * as path from "path";
|
|
200
207
|
import * as fs from "fs/promises";
|
|
201
|
-
import * as os from "os";
|
|
202
208
|
import * as crypto from "crypto";
|
|
203
209
|
function hashPath(inputPath) {
|
|
204
210
|
return crypto.createHash("sha256").update(inputPath).digest("hex").slice(0, 12);
|
|
205
211
|
}
|
|
206
212
|
function getRaggrepDir(rootDir, _config = DEFAULT_CONFIG) {
|
|
207
213
|
const absoluteRoot = path.resolve(rootDir);
|
|
208
|
-
|
|
209
|
-
return path.join(RAGGREP_TEMP_BASE, projectHash);
|
|
214
|
+
return path.join(absoluteRoot, RAGGREP_INDEX_DIR);
|
|
210
215
|
}
|
|
211
216
|
function getIndexLocation(rootDir) {
|
|
212
217
|
const absoluteRoot = path.resolve(rootDir);
|
|
213
218
|
const projectHash = hashPath(absoluteRoot);
|
|
214
219
|
return {
|
|
215
|
-
indexDir: path.join(
|
|
220
|
+
indexDir: path.join(absoluteRoot, RAGGREP_INDEX_DIR),
|
|
216
221
|
projectRoot: absoluteRoot,
|
|
217
222
|
projectHash
|
|
218
223
|
};
|
|
@@ -253,23 +258,24 @@ function getEmbeddingConfigFromModule(moduleConfig) {
|
|
|
253
258
|
console.warn(`Unknown embedding model: ${modelName}, falling back to bge-small-en-v1.5`);
|
|
254
259
|
return { model: "bge-small-en-v1.5" };
|
|
255
260
|
}
|
|
261
|
+
const rt = options.embeddingRuntime;
|
|
262
|
+
let runtime;
|
|
263
|
+
if (rt === "xenova" || rt === "huggingface") {
|
|
264
|
+
runtime = rt;
|
|
265
|
+
} else if (rt !== undefined) {
|
|
266
|
+
console.warn(`Unknown embeddingRuntime: ${rt}, falling back to default (huggingface)`);
|
|
267
|
+
}
|
|
256
268
|
return {
|
|
257
269
|
model: modelName,
|
|
270
|
+
...runtime ? { runtime } : {},
|
|
258
271
|
showProgress: options.showProgress === true
|
|
259
272
|
};
|
|
260
273
|
}
|
|
261
|
-
var DEFAULT_CONFIG,
|
|
274
|
+
var DEFAULT_CONFIG, RAGGREP_INDEX_DIR = ".raggrep";
|
|
262
275
|
var init_configLoader = __esm(() => {
|
|
263
276
|
init_entities();
|
|
277
|
+
init_modelCatalog();
|
|
264
278
|
DEFAULT_CONFIG = createDefaultConfig();
|
|
265
|
-
RAGGREP_TEMP_BASE = path.join(os.tmpdir(), "raggrep-indexes");
|
|
266
|
-
EMBEDDING_MODELS = {
|
|
267
|
-
"all-MiniLM-L6-v2": "Xenova/all-MiniLM-L6-v2",
|
|
268
|
-
"all-MiniLM-L12-v2": "Xenova/all-MiniLM-L12-v2",
|
|
269
|
-
"bge-small-en-v1.5": "Xenova/bge-small-en-v1.5",
|
|
270
|
-
"paraphrase-MiniLM-L3-v2": "Xenova/paraphrase-MiniLM-L3-v2",
|
|
271
|
-
"nomic-embed-text-v1.5": "nomic-ai/nomic-embed-text-v1.5"
|
|
272
|
-
};
|
|
273
279
|
});
|
|
274
280
|
|
|
275
281
|
// src/infrastructure/config/index.ts
|
|
@@ -2137,22 +2143,47 @@ var init_core = __esm(() => {
|
|
|
2137
2143
|
init_symbols();
|
|
2138
2144
|
});
|
|
2139
2145
|
|
|
2140
|
-
// src/infrastructure/embeddings/
|
|
2146
|
+
// src/infrastructure/embeddings/embeddingPaths.ts
|
|
2147
|
+
import * as os from "os";
|
|
2148
|
+
import * as path6 from "path";
|
|
2149
|
+
var RAGGREP_MODEL_CACHE_DIR;
|
|
2150
|
+
var init_embeddingPaths = __esm(() => {
|
|
2151
|
+
RAGGREP_MODEL_CACHE_DIR = path6.join(os.homedir(), ".cache", "raggrep", "models");
|
|
2152
|
+
});
|
|
2153
|
+
|
|
2154
|
+
// src/infrastructure/embeddings/modelCache.ts
|
|
2155
|
+
import * as fs3 from "fs/promises";
|
|
2156
|
+
import * as path7 from "path";
|
|
2157
|
+
async function isEmbeddingModelCached(model) {
|
|
2158
|
+
const modelId = getEmbeddingModelId(model);
|
|
2159
|
+
const onnxPath = path7.join(RAGGREP_MODEL_CACHE_DIR, modelId, "onnx", "model_quantized.onnx");
|
|
2160
|
+
try {
|
|
2161
|
+
await fs3.access(onnxPath);
|
|
2162
|
+
return true;
|
|
2163
|
+
} catch {
|
|
2164
|
+
return false;
|
|
2165
|
+
}
|
|
2166
|
+
}
|
|
2167
|
+
var init_modelCache = __esm(() => {
|
|
2168
|
+
init_embeddingPaths();
|
|
2169
|
+
init_modelCatalog();
|
|
2170
|
+
});
|
|
2171
|
+
|
|
2172
|
+
// src/infrastructure/embeddings/xenovaEmbeddingProvider.ts
|
|
2141
2173
|
import {
|
|
2142
2174
|
pipeline,
|
|
2143
2175
|
env
|
|
2144
2176
|
} from "@xenova/transformers";
|
|
2145
|
-
import * as path6 from "path";
|
|
2146
|
-
import * as os2 from "os";
|
|
2147
2177
|
|
|
2148
|
-
class
|
|
2149
|
-
|
|
2178
|
+
class XenovaTransformersEmbeddingProvider {
|
|
2179
|
+
extractor = null;
|
|
2150
2180
|
config;
|
|
2151
2181
|
isInitializing = false;
|
|
2152
2182
|
initPromise = null;
|
|
2153
2183
|
constructor(config) {
|
|
2154
2184
|
this.config = {
|
|
2155
2185
|
model: config?.model ?? "bge-small-en-v1.5",
|
|
2186
|
+
runtime: config?.runtime ?? "xenova",
|
|
2156
2187
|
showProgress: config?.showProgress ?? false,
|
|
2157
2188
|
logger: config?.logger
|
|
2158
2189
|
};
|
|
@@ -2160,14 +2191,14 @@ class TransformersEmbeddingProvider {
|
|
|
2160
2191
|
async initialize(config) {
|
|
2161
2192
|
if (config) {
|
|
2162
2193
|
if (config.model !== this.config.model) {
|
|
2163
|
-
this.
|
|
2194
|
+
this.extractor = null;
|
|
2164
2195
|
}
|
|
2165
2196
|
this.config = { ...this.config, ...config };
|
|
2166
2197
|
}
|
|
2167
|
-
await this.
|
|
2198
|
+
await this.ensureExtractor();
|
|
2168
2199
|
}
|
|
2169
|
-
async
|
|
2170
|
-
if (this.
|
|
2200
|
+
async ensureExtractor() {
|
|
2201
|
+
if (this.extractor) {
|
|
2171
2202
|
return;
|
|
2172
2203
|
}
|
|
2173
2204
|
if (this.isInitializing && this.initPromise) {
|
|
@@ -2175,14 +2206,14 @@ class TransformersEmbeddingProvider {
|
|
|
2175
2206
|
}
|
|
2176
2207
|
this.isInitializing = true;
|
|
2177
2208
|
this.initPromise = (async () => {
|
|
2178
|
-
const modelId =
|
|
2209
|
+
const modelId = getEmbeddingModelId(this.config.model);
|
|
2179
2210
|
const logger = this.config.logger;
|
|
2180
2211
|
const showProgress = this.config.showProgress || !!logger;
|
|
2181
|
-
const
|
|
2212
|
+
const cached = await isEmbeddingModelCached(this.config.model);
|
|
2182
2213
|
let hasDownloads = false;
|
|
2183
2214
|
try {
|
|
2184
|
-
this.
|
|
2185
|
-
progress_callback: showProgress && !
|
|
2215
|
+
this.extractor = await pipeline("feature-extraction", modelId, {
|
|
2216
|
+
progress_callback: showProgress && !cached ? (progress) => {
|
|
2186
2217
|
if (progress.status === "progress" && progress.file) {
|
|
2187
2218
|
if (!hasDownloads) {
|
|
2188
2219
|
hasDownloads = true;
|
|
@@ -2191,7 +2222,7 @@ class TransformersEmbeddingProvider {
|
|
|
2191
2222
|
} else {
|
|
2192
2223
|
console.log(`
|
|
2193
2224
|
Loading embedding model: ${this.config.model}`);
|
|
2194
|
-
console.log(` Cache: ${
|
|
2225
|
+
console.log(` Cache: ${RAGGREP_MODEL_CACHE_DIR}`);
|
|
2195
2226
|
}
|
|
2196
2227
|
}
|
|
2197
2228
|
const pct = progress.progress ? Math.round(progress.progress) : 0;
|
|
@@ -2221,9 +2252,9 @@ class TransformersEmbeddingProvider {
|
|
|
2221
2252
|
}
|
|
2222
2253
|
}
|
|
2223
2254
|
} catch (error) {
|
|
2224
|
-
this.
|
|
2225
|
-
if (logger) {
|
|
2226
|
-
logger.clearProgress();
|
|
2255
|
+
this.extractor = null;
|
|
2256
|
+
if (this.config.logger) {
|
|
2257
|
+
this.config.logger.clearProgress();
|
|
2227
2258
|
}
|
|
2228
2259
|
throw new Error(`Failed to load embedding model: ${error}`);
|
|
2229
2260
|
} finally {
|
|
@@ -2234,11 +2265,11 @@ class TransformersEmbeddingProvider {
|
|
|
2234
2265
|
return this.initPromise;
|
|
2235
2266
|
}
|
|
2236
2267
|
async getEmbedding(text) {
|
|
2237
|
-
await this.
|
|
2238
|
-
if (!this.
|
|
2268
|
+
await this.ensureExtractor();
|
|
2269
|
+
if (!this.extractor) {
|
|
2239
2270
|
throw new Error("Embedding pipeline not initialized");
|
|
2240
2271
|
}
|
|
2241
|
-
const output = await this.
|
|
2272
|
+
const output = await this.extractor(text, {
|
|
2242
2273
|
pooling: "mean",
|
|
2243
2274
|
normalize: true
|
|
2244
2275
|
});
|
|
@@ -2247,15 +2278,15 @@ class TransformersEmbeddingProvider {
|
|
|
2247
2278
|
async getEmbeddings(texts) {
|
|
2248
2279
|
if (texts.length === 0)
|
|
2249
2280
|
return [];
|
|
2250
|
-
await this.
|
|
2251
|
-
if (!this.
|
|
2281
|
+
await this.ensureExtractor();
|
|
2282
|
+
if (!this.extractor) {
|
|
2252
2283
|
throw new Error("Embedding pipeline not initialized");
|
|
2253
2284
|
}
|
|
2254
2285
|
const results = [];
|
|
2255
2286
|
for (let i = 0;i < texts.length; i += BATCH_SIZE) {
|
|
2256
2287
|
const batch = texts.slice(i, i + BATCH_SIZE);
|
|
2257
2288
|
const outputs = await Promise.all(batch.map(async (text) => {
|
|
2258
|
-
const output = await this.
|
|
2289
|
+
const output = await this.extractor(text, {
|
|
2259
2290
|
pooling: "mean",
|
|
2260
2291
|
normalize: true
|
|
2261
2292
|
});
|
|
@@ -2266,41 +2297,210 @@ class TransformersEmbeddingProvider {
|
|
|
2266
2297
|
return results;
|
|
2267
2298
|
}
|
|
2268
2299
|
getDimension() {
|
|
2269
|
-
return
|
|
2300
|
+
return getEmbeddingDimension(this.config.model);
|
|
2270
2301
|
}
|
|
2271
2302
|
getModelName() {
|
|
2272
2303
|
return this.config.model;
|
|
2273
2304
|
}
|
|
2274
2305
|
async dispose() {
|
|
2275
|
-
this.
|
|
2306
|
+
this.extractor = null;
|
|
2276
2307
|
}
|
|
2277
2308
|
}
|
|
2278
|
-
|
|
2279
|
-
|
|
2280
|
-
|
|
2281
|
-
|
|
2282
|
-
|
|
2283
|
-
|
|
2284
|
-
|
|
2285
|
-
|
|
2286
|
-
|
|
2287
|
-
|
|
2309
|
+
var BATCH_SIZE = 32;
|
|
2310
|
+
var init_xenovaEmbeddingProvider = __esm(() => {
|
|
2311
|
+
init_embeddingPaths();
|
|
2312
|
+
init_modelCatalog();
|
|
2313
|
+
init_modelCache();
|
|
2314
|
+
env.cacheDir = RAGGREP_MODEL_CACHE_DIR;
|
|
2315
|
+
env.allowLocalModels = true;
|
|
2316
|
+
});
|
|
2317
|
+
|
|
2318
|
+
// src/infrastructure/embeddings/huggingfaceEmbeddingProvider.ts
|
|
2319
|
+
import {
|
|
2320
|
+
pipeline as pipeline2,
|
|
2321
|
+
env as env2
|
|
2322
|
+
} from "@huggingface/transformers";
|
|
2323
|
+
|
|
2324
|
+
class HuggingFaceTransformersEmbeddingProvider {
|
|
2325
|
+
extractor = null;
|
|
2326
|
+
config;
|
|
2327
|
+
isInitializing = false;
|
|
2328
|
+
initPromise = null;
|
|
2329
|
+
constructor(config) {
|
|
2330
|
+
this.config = {
|
|
2331
|
+
model: config?.model ?? "bge-small-en-v1.5",
|
|
2332
|
+
runtime: config?.runtime ?? "huggingface",
|
|
2333
|
+
showProgress: config?.showProgress ?? false,
|
|
2334
|
+
logger: config?.logger
|
|
2335
|
+
};
|
|
2336
|
+
}
|
|
2337
|
+
async initialize(config) {
|
|
2338
|
+
if (config) {
|
|
2339
|
+
if (config.model !== this.config.model) {
|
|
2340
|
+
this.extractor = null;
|
|
2341
|
+
}
|
|
2342
|
+
this.config = { ...this.config, ...config };
|
|
2343
|
+
}
|
|
2344
|
+
await this.ensureExtractor();
|
|
2345
|
+
}
|
|
2346
|
+
async ensureExtractor() {
|
|
2347
|
+
if (this.extractor) {
|
|
2348
|
+
return;
|
|
2349
|
+
}
|
|
2350
|
+
if (this.isInitializing && this.initPromise) {
|
|
2351
|
+
return this.initPromise;
|
|
2352
|
+
}
|
|
2353
|
+
this.isInitializing = true;
|
|
2354
|
+
this.initPromise = (async () => {
|
|
2355
|
+
const modelId = getEmbeddingModelId(this.config.model);
|
|
2356
|
+
const logger = this.config.logger;
|
|
2357
|
+
const showProgress = this.config.showProgress || !!logger;
|
|
2358
|
+
const cached = await isEmbeddingModelCached(this.config.model);
|
|
2359
|
+
let hasDownloads = false;
|
|
2360
|
+
try {
|
|
2361
|
+
this.extractor = await pipeline2("feature-extraction", modelId, {
|
|
2362
|
+
progress_callback: showProgress && !cached ? (progress) => {
|
|
2363
|
+
if (progress.status === "progress" && progress.file) {
|
|
2364
|
+
if (!hasDownloads) {
|
|
2365
|
+
hasDownloads = true;
|
|
2366
|
+
if (logger) {
|
|
2367
|
+
logger.info(`Downloading embedding model: ${this.config.model}`);
|
|
2368
|
+
} else {
|
|
2369
|
+
console.log(`
|
|
2370
|
+
Loading embedding model: ${this.config.model}`);
|
|
2371
|
+
console.log(` Cache: ${RAGGREP_MODEL_CACHE_DIR}`);
|
|
2372
|
+
}
|
|
2373
|
+
}
|
|
2374
|
+
const pct = progress.progress ? Math.round(progress.progress) : 0;
|
|
2375
|
+
if (logger) {
|
|
2376
|
+
logger.progress(` Downloading ${progress.file}: ${pct}%`);
|
|
2377
|
+
} else {
|
|
2378
|
+
process.stdout.write(`\r Downloading ${progress.file}: ${pct}% `);
|
|
2379
|
+
}
|
|
2380
|
+
} else if (progress.status === "done" && progress.file) {
|
|
2381
|
+
if (logger) {
|
|
2382
|
+
logger.clearProgress();
|
|
2383
|
+
logger.info(` Downloaded ${progress.file}`);
|
|
2384
|
+
} else if (hasDownloads) {
|
|
2385
|
+
process.stdout.write(`\r Downloaded ${progress.file}
|
|
2386
|
+
`);
|
|
2387
|
+
}
|
|
2388
|
+
}
|
|
2389
|
+
} : undefined
|
|
2390
|
+
});
|
|
2391
|
+
if (hasDownloads) {
|
|
2392
|
+
if (logger) {
|
|
2393
|
+
logger.clearProgress();
|
|
2394
|
+
logger.info(`Model ready: ${this.config.model}`);
|
|
2395
|
+
} else {
|
|
2396
|
+
console.log(` Model ready.
|
|
2397
|
+
`);
|
|
2398
|
+
}
|
|
2399
|
+
}
|
|
2400
|
+
} catch (error) {
|
|
2401
|
+
this.extractor = null;
|
|
2402
|
+
if (this.config.logger) {
|
|
2403
|
+
this.config.logger.clearProgress();
|
|
2404
|
+
}
|
|
2405
|
+
throw new Error(`Failed to load embedding model: ${error}`);
|
|
2406
|
+
} finally {
|
|
2407
|
+
this.isInitializing = false;
|
|
2408
|
+
this.initPromise = null;
|
|
2409
|
+
}
|
|
2410
|
+
})();
|
|
2411
|
+
return this.initPromise;
|
|
2288
2412
|
}
|
|
2413
|
+
async getEmbedding(text) {
|
|
2414
|
+
await this.ensureExtractor();
|
|
2415
|
+
if (!this.extractor) {
|
|
2416
|
+
throw new Error("Embedding pipeline not initialized");
|
|
2417
|
+
}
|
|
2418
|
+
const output = await this.extractor(text, {
|
|
2419
|
+
pooling: "mean",
|
|
2420
|
+
normalize: true
|
|
2421
|
+
});
|
|
2422
|
+
return Array.from(output.data);
|
|
2423
|
+
}
|
|
2424
|
+
async getEmbeddings(texts) {
|
|
2425
|
+
if (texts.length === 0)
|
|
2426
|
+
return [];
|
|
2427
|
+
await this.ensureExtractor();
|
|
2428
|
+
if (!this.extractor) {
|
|
2429
|
+
throw new Error("Embedding pipeline not initialized");
|
|
2430
|
+
}
|
|
2431
|
+
const results = [];
|
|
2432
|
+
for (let i = 0;i < texts.length; i += BATCH_SIZE2) {
|
|
2433
|
+
const batch = texts.slice(i, i + BATCH_SIZE2);
|
|
2434
|
+
const outputs = await Promise.all(batch.map(async (text) => {
|
|
2435
|
+
const output = await this.extractor(text, {
|
|
2436
|
+
pooling: "mean",
|
|
2437
|
+
normalize: true
|
|
2438
|
+
});
|
|
2439
|
+
return Array.from(output.data);
|
|
2440
|
+
}));
|
|
2441
|
+
results.push(...outputs);
|
|
2442
|
+
}
|
|
2443
|
+
return results;
|
|
2444
|
+
}
|
|
2445
|
+
getDimension() {
|
|
2446
|
+
return getEmbeddingDimension(this.config.model);
|
|
2447
|
+
}
|
|
2448
|
+
getModelName() {
|
|
2449
|
+
return this.config.model;
|
|
2450
|
+
}
|
|
2451
|
+
async dispose() {
|
|
2452
|
+
this.extractor = null;
|
|
2453
|
+
}
|
|
2454
|
+
}
|
|
2455
|
+
var BATCH_SIZE2 = 32;
|
|
2456
|
+
var init_huggingfaceEmbeddingProvider = __esm(() => {
|
|
2457
|
+
init_embeddingPaths();
|
|
2458
|
+
init_modelCatalog();
|
|
2459
|
+
init_modelCache();
|
|
2460
|
+
env2.cacheDir = RAGGREP_MODEL_CACHE_DIR;
|
|
2461
|
+
env2.allowLocalModels = true;
|
|
2462
|
+
});
|
|
2463
|
+
|
|
2464
|
+
// src/infrastructure/embeddings/embeddingProviderFactory.ts
|
|
2465
|
+
function resolveRuntime(config) {
|
|
2466
|
+
return config.runtime ?? "huggingface";
|
|
2467
|
+
}
|
|
2468
|
+
function createEmbeddingProvider(config) {
|
|
2469
|
+
const runtime = resolveRuntime(config);
|
|
2470
|
+
if (runtime === "huggingface") {
|
|
2471
|
+
return new HuggingFaceTransformersEmbeddingProvider(config);
|
|
2472
|
+
}
|
|
2473
|
+
return new XenovaTransformersEmbeddingProvider(config);
|
|
2289
2474
|
}
|
|
2475
|
+
var init_embeddingProviderFactory = __esm(() => {
|
|
2476
|
+
init_huggingfaceEmbeddingProvider();
|
|
2477
|
+
init_xenovaEmbeddingProvider();
|
|
2478
|
+
});
|
|
2479
|
+
|
|
2480
|
+
// src/infrastructure/embeddings/globalEmbeddings.ts
|
|
2290
2481
|
function configureEmbeddings(config) {
|
|
2291
|
-
const
|
|
2292
|
-
|
|
2482
|
+
const merged = {
|
|
2483
|
+
...globalConfig,
|
|
2484
|
+
...config
|
|
2485
|
+
};
|
|
2486
|
+
if (merged.runtime === undefined) {
|
|
2487
|
+
merged.runtime = "huggingface";
|
|
2488
|
+
}
|
|
2489
|
+
const needsReset = merged.model !== globalConfig.model || merged.runtime !== globalConfig.runtime || merged.logger !== globalConfig.logger;
|
|
2490
|
+
if (needsReset) {
|
|
2491
|
+
const prev = globalProvider;
|
|
2293
2492
|
globalProvider = null;
|
|
2493
|
+
prev?.dispose?.();
|
|
2294
2494
|
}
|
|
2295
|
-
globalConfig =
|
|
2495
|
+
globalConfig = merged;
|
|
2296
2496
|
}
|
|
2297
2497
|
function getEmbeddingConfig() {
|
|
2298
2498
|
return { ...globalConfig };
|
|
2299
2499
|
}
|
|
2300
2500
|
async function ensureGlobalProvider() {
|
|
2301
2501
|
if (!globalProvider) {
|
|
2302
|
-
globalProvider =
|
|
2303
|
-
await globalProvider.initialize();
|
|
2502
|
+
globalProvider = createEmbeddingProvider(globalConfig);
|
|
2503
|
+
await globalProvider.initialize?.(globalConfig);
|
|
2304
2504
|
}
|
|
2305
2505
|
return globalProvider;
|
|
2306
2506
|
}
|
|
@@ -2312,27 +2512,14 @@ async function getEmbeddings(texts) {
|
|
|
2312
2512
|
const provider = await ensureGlobalProvider();
|
|
2313
2513
|
return provider.getEmbeddings(texts);
|
|
2314
2514
|
}
|
|
2315
|
-
var
|
|
2316
|
-
var
|
|
2317
|
-
|
|
2318
|
-
|
|
2319
|
-
|
|
2320
|
-
EMBEDDING_MODELS2 = {
|
|
2321
|
-
"all-MiniLM-L6-v2": "Xenova/all-MiniLM-L6-v2",
|
|
2322
|
-
"all-MiniLM-L12-v2": "Xenova/all-MiniLM-L12-v2",
|
|
2323
|
-
"bge-small-en-v1.5": "Xenova/bge-small-en-v1.5",
|
|
2324
|
-
"paraphrase-MiniLM-L3-v2": "Xenova/paraphrase-MiniLM-L3-v2",
|
|
2325
|
-
"nomic-embed-text-v1.5": "nomic-ai/nomic-embed-text-v1.5"
|
|
2326
|
-
};
|
|
2327
|
-
EMBEDDING_DIMENSIONS = {
|
|
2328
|
-
"all-MiniLM-L6-v2": 384,
|
|
2329
|
-
"all-MiniLM-L12-v2": 384,
|
|
2330
|
-
"bge-small-en-v1.5": 384,
|
|
2331
|
-
"paraphrase-MiniLM-L3-v2": 384,
|
|
2332
|
-
"nomic-embed-text-v1.5": 768
|
|
2333
|
-
};
|
|
2515
|
+
var globalProvider = null, globalConfig;
|
|
2516
|
+
var init_globalEmbeddings = __esm(() => {
|
|
2517
|
+
init_embeddingPaths();
|
|
2518
|
+
init_embeddingProviderFactory();
|
|
2519
|
+
init_modelCache();
|
|
2334
2520
|
globalConfig = {
|
|
2335
2521
|
model: "bge-small-en-v1.5",
|
|
2522
|
+
runtime: "huggingface",
|
|
2336
2523
|
showProgress: false,
|
|
2337
2524
|
logger: undefined
|
|
2338
2525
|
};
|
|
@@ -2340,7 +2527,13 @@ var init_transformersEmbedding = __esm(() => {
|
|
|
2340
2527
|
|
|
2341
2528
|
// src/infrastructure/embeddings/index.ts
|
|
2342
2529
|
var init_embeddings = __esm(() => {
|
|
2343
|
-
|
|
2530
|
+
init_modelCatalog();
|
|
2531
|
+
init_embeddingPaths();
|
|
2532
|
+
init_xenovaEmbeddingProvider();
|
|
2533
|
+
init_xenovaEmbeddingProvider();
|
|
2534
|
+
init_huggingfaceEmbeddingProvider();
|
|
2535
|
+
init_embeddingProviderFactory();
|
|
2536
|
+
init_globalEmbeddings();
|
|
2344
2537
|
});
|
|
2345
2538
|
|
|
2346
2539
|
// src/domain/services/keywords.ts
|
|
@@ -2551,7 +2744,7 @@ function cosineSimilarity(a, b) {
|
|
|
2551
2744
|
}
|
|
2552
2745
|
|
|
2553
2746
|
// src/domain/services/queryIntent.ts
|
|
2554
|
-
import * as
|
|
2747
|
+
import * as path8 from "path";
|
|
2555
2748
|
function detectQueryIntent(queryTerms) {
|
|
2556
2749
|
const hasImplementationTerm = queryTerms.some((term) => IMPLEMENTATION_TERMS.includes(term));
|
|
2557
2750
|
const hasDocumentationTerm = queryTerms.some((term) => DOCUMENTATION_TERMS.includes(term));
|
|
@@ -2567,11 +2760,11 @@ function extractQueryTerms(query) {
|
|
|
2567
2760
|
return query.toLowerCase().split(/\s+/).filter((t) => t.length > 2);
|
|
2568
2761
|
}
|
|
2569
2762
|
function isSourceCodeFile(filepath) {
|
|
2570
|
-
const ext =
|
|
2763
|
+
const ext = path8.extname(filepath).toLowerCase();
|
|
2571
2764
|
return SOURCE_CODE_EXTENSIONS.includes(ext);
|
|
2572
2765
|
}
|
|
2573
2766
|
function isDocFile(filepath) {
|
|
2574
|
-
const ext =
|
|
2767
|
+
const ext = path8.extname(filepath).toLowerCase();
|
|
2575
2768
|
return DOC_EXTENSIONS.includes(ext);
|
|
2576
2769
|
}
|
|
2577
2770
|
function calculateFileTypeBoost(filepath, queryTerms) {
|
|
@@ -3646,8 +3839,8 @@ var init_lexicon2 = __esm(() => {
|
|
|
3646
3839
|
// src/domain/services/jsonPathExtractor.ts
|
|
3647
3840
|
function extractJsonPaths(obj, fileBasename) {
|
|
3648
3841
|
const paths = extractPathsRecursive(obj, fileBasename);
|
|
3649
|
-
return paths.map((
|
|
3650
|
-
value:
|
|
3842
|
+
return paths.map((path9) => ({
|
|
3843
|
+
value: path9,
|
|
3651
3844
|
type: "identifier",
|
|
3652
3845
|
matchType: "definition"
|
|
3653
3846
|
}));
|
|
@@ -3958,7 +4151,7 @@ var init_simpleSearch = __esm(() => {
|
|
|
3958
4151
|
});
|
|
3959
4152
|
|
|
3960
4153
|
// src/domain/services/chunkContext.ts
|
|
3961
|
-
import * as
|
|
4154
|
+
import * as path9 from "path";
|
|
3962
4155
|
function prepareChunkForEmbedding(options) {
|
|
3963
4156
|
const { filepath, content, name, docComment } = options;
|
|
3964
4157
|
const pathContext = parsePathContext(filepath);
|
|
@@ -3967,7 +4160,7 @@ function prepareChunkForEmbedding(options) {
|
|
|
3967
4160
|
if (pathPrefix) {
|
|
3968
4161
|
parts.push(pathPrefix);
|
|
3969
4162
|
}
|
|
3970
|
-
const filename =
|
|
4163
|
+
const filename = path9.basename(filepath);
|
|
3971
4164
|
const filenameWithoutExt = filename.replace(/\.[^.]+$/, "");
|
|
3972
4165
|
if (filenameWithoutExt && filenameWithoutExt.length > MIN_SEGMENT_LENGTH) {
|
|
3973
4166
|
const pathPrefixLower = pathPrefix.toLowerCase();
|
|
@@ -4197,11 +4390,12 @@ var init_parseCode = () => {};
|
|
|
4197
4390
|
// src/infrastructure/storage/fileIndexStorage.ts
|
|
4198
4391
|
var init_fileIndexStorage = __esm(() => {
|
|
4199
4392
|
init_entities();
|
|
4393
|
+
init_config2();
|
|
4200
4394
|
});
|
|
4201
4395
|
|
|
4202
4396
|
// src/infrastructure/storage/symbolicIndex.ts
|
|
4203
|
-
import * as
|
|
4204
|
-
import * as
|
|
4397
|
+
import * as fs4 from "fs/promises";
|
|
4398
|
+
import * as path10 from "path";
|
|
4205
4399
|
|
|
4206
4400
|
class SymbolicIndex {
|
|
4207
4401
|
meta = null;
|
|
@@ -4210,7 +4404,7 @@ class SymbolicIndex {
|
|
|
4210
4404
|
symbolicPath;
|
|
4211
4405
|
moduleId;
|
|
4212
4406
|
constructor(indexDir, moduleId) {
|
|
4213
|
-
this.symbolicPath =
|
|
4407
|
+
this.symbolicPath = path10.join(indexDir, "index", moduleId, "symbolic");
|
|
4214
4408
|
this.moduleId = moduleId;
|
|
4215
4409
|
}
|
|
4216
4410
|
async initialize() {
|
|
@@ -4296,13 +4490,13 @@ class SymbolicIndex {
|
|
|
4296
4490
|
if (this.bm25Index) {
|
|
4297
4491
|
this.meta.bm25Serialized = this.bm25Index.serialize();
|
|
4298
4492
|
}
|
|
4299
|
-
await
|
|
4300
|
-
const metaPath =
|
|
4301
|
-
await
|
|
4493
|
+
await fs4.mkdir(this.symbolicPath, { recursive: true });
|
|
4494
|
+
const metaPath = path10.join(this.symbolicPath, "_meta.json");
|
|
4495
|
+
await fs4.writeFile(metaPath, JSON.stringify(this.meta, null, 2));
|
|
4302
4496
|
for (const [filepath, summary] of this.fileSummaries) {
|
|
4303
4497
|
const summaryPath = this.getFileSummaryPath(filepath);
|
|
4304
|
-
await
|
|
4305
|
-
await
|
|
4498
|
+
await fs4.mkdir(path10.dirname(summaryPath), { recursive: true });
|
|
4499
|
+
await fs4.writeFile(summaryPath, JSON.stringify(summary, null, 2));
|
|
4306
4500
|
}
|
|
4307
4501
|
}
|
|
4308
4502
|
async saveIncremental(filepaths) {
|
|
@@ -4313,21 +4507,21 @@ class SymbolicIndex {
|
|
|
4313
4507
|
if (this.bm25Index) {
|
|
4314
4508
|
this.meta.bm25Serialized = this.bm25Index.serialize();
|
|
4315
4509
|
}
|
|
4316
|
-
await
|
|
4317
|
-
const metaPath =
|
|
4318
|
-
await
|
|
4510
|
+
await fs4.mkdir(this.symbolicPath, { recursive: true });
|
|
4511
|
+
const metaPath = path10.join(this.symbolicPath, "_meta.json");
|
|
4512
|
+
await fs4.writeFile(metaPath, JSON.stringify(this.meta, null, 2));
|
|
4319
4513
|
for (const filepath of filepaths) {
|
|
4320
4514
|
const summary = this.fileSummaries.get(filepath);
|
|
4321
4515
|
if (summary) {
|
|
4322
4516
|
const summaryPath = this.getFileSummaryPath(filepath);
|
|
4323
|
-
await
|
|
4324
|
-
await
|
|
4517
|
+
await fs4.mkdir(path10.dirname(summaryPath), { recursive: true });
|
|
4518
|
+
await fs4.writeFile(summaryPath, JSON.stringify(summary, null, 2));
|
|
4325
4519
|
}
|
|
4326
4520
|
}
|
|
4327
4521
|
}
|
|
4328
4522
|
async load() {
|
|
4329
|
-
const metaPath =
|
|
4330
|
-
const metaContent = await
|
|
4523
|
+
const metaPath = path10.join(this.symbolicPath, "_meta.json");
|
|
4524
|
+
const metaContent = await fs4.readFile(metaPath, "utf-8");
|
|
4331
4525
|
this.meta = JSON.parse(metaContent);
|
|
4332
4526
|
this.fileSummaries.clear();
|
|
4333
4527
|
await this.loadFileSummariesRecursive(this.symbolicPath);
|
|
@@ -4339,14 +4533,14 @@ class SymbolicIndex {
|
|
|
4339
4533
|
}
|
|
4340
4534
|
async loadFileSummariesRecursive(dir) {
|
|
4341
4535
|
try {
|
|
4342
|
-
const entries = await
|
|
4536
|
+
const entries = await fs4.readdir(dir, { withFileTypes: true });
|
|
4343
4537
|
for (const entry of entries) {
|
|
4344
|
-
const fullPath =
|
|
4538
|
+
const fullPath = path10.join(dir, entry.name);
|
|
4345
4539
|
if (entry.isDirectory()) {
|
|
4346
4540
|
await this.loadFileSummariesRecursive(fullPath);
|
|
4347
4541
|
} else if (entry.name.endsWith(".json") && entry.name !== "_meta.json") {
|
|
4348
4542
|
try {
|
|
4349
|
-
const content = await
|
|
4543
|
+
const content = await fs4.readFile(fullPath, "utf-8");
|
|
4350
4544
|
const summary = JSON.parse(content);
|
|
4351
4545
|
if (summary.filepath) {
|
|
4352
4546
|
this.fileSummaries.set(summary.filepath, summary);
|
|
@@ -4358,18 +4552,18 @@ class SymbolicIndex {
|
|
|
4358
4552
|
}
|
|
4359
4553
|
getFileSummaryPath(filepath) {
|
|
4360
4554
|
const jsonPath = filepath.replace(/\.[^.]+$/, ".json");
|
|
4361
|
-
return
|
|
4555
|
+
return path10.join(this.symbolicPath, jsonPath);
|
|
4362
4556
|
}
|
|
4363
4557
|
async deleteFileSummary(filepath) {
|
|
4364
4558
|
try {
|
|
4365
|
-
await
|
|
4559
|
+
await fs4.unlink(this.getFileSummaryPath(filepath));
|
|
4366
4560
|
} catch {}
|
|
4367
4561
|
this.fileSummaries.delete(filepath);
|
|
4368
4562
|
}
|
|
4369
4563
|
async exists() {
|
|
4370
4564
|
try {
|
|
4371
|
-
const metaPath =
|
|
4372
|
-
await
|
|
4565
|
+
const metaPath = path10.join(this.symbolicPath, "_meta.json");
|
|
4566
|
+
await fs4.access(metaPath);
|
|
4373
4567
|
return true;
|
|
4374
4568
|
} catch {
|
|
4375
4569
|
return false;
|
|
@@ -4401,8 +4595,8 @@ __export(exports_literalIndex, {
|
|
|
4401
4595
|
getLiteralIndexPath: () => getLiteralIndexPath,
|
|
4402
4596
|
LiteralIndex: () => LiteralIndex
|
|
4403
4597
|
});
|
|
4404
|
-
import * as
|
|
4405
|
-
import * as
|
|
4598
|
+
import * as fs5 from "fs/promises";
|
|
4599
|
+
import * as path11 from "path";
|
|
4406
4600
|
|
|
4407
4601
|
class LiteralIndex {
|
|
4408
4602
|
indexPath;
|
|
@@ -4411,7 +4605,7 @@ class LiteralIndex {
|
|
|
4411
4605
|
vocabularyIndex = new Map;
|
|
4412
4606
|
static VERSION = "1.1.0";
|
|
4413
4607
|
constructor(indexDir, moduleId) {
|
|
4414
|
-
this.indexPath =
|
|
4608
|
+
this.indexPath = path11.join(indexDir, "index", moduleId, "literals");
|
|
4415
4609
|
this.moduleId = moduleId;
|
|
4416
4610
|
}
|
|
4417
4611
|
async initialize() {
|
|
@@ -4572,17 +4766,17 @@ class LiteralIndex {
|
|
|
4572
4766
|
}));
|
|
4573
4767
|
}
|
|
4574
4768
|
async save() {
|
|
4575
|
-
await
|
|
4769
|
+
await fs5.mkdir(this.indexPath, { recursive: true });
|
|
4576
4770
|
const data = {
|
|
4577
4771
|
version: LiteralIndex.VERSION,
|
|
4578
4772
|
entries: Object.fromEntries(this.entries)
|
|
4579
4773
|
};
|
|
4580
|
-
const indexFile =
|
|
4581
|
-
await
|
|
4774
|
+
const indexFile = path11.join(this.indexPath, "_index.json");
|
|
4775
|
+
await fs5.writeFile(indexFile, JSON.stringify(data, null, 2));
|
|
4582
4776
|
}
|
|
4583
4777
|
async load() {
|
|
4584
|
-
const indexFile =
|
|
4585
|
-
const content = await
|
|
4778
|
+
const indexFile = path11.join(this.indexPath, "_index.json");
|
|
4779
|
+
const content = await fs5.readFile(indexFile, "utf-8");
|
|
4586
4780
|
const data = JSON.parse(content);
|
|
4587
4781
|
if (data.version !== LiteralIndex.VERSION) {
|
|
4588
4782
|
console.warn(`Literal index version mismatch: expected ${LiteralIndex.VERSION}, got ${data.version}`);
|
|
@@ -4591,8 +4785,8 @@ class LiteralIndex {
|
|
|
4591
4785
|
}
|
|
4592
4786
|
async exists() {
|
|
4593
4787
|
try {
|
|
4594
|
-
const indexFile =
|
|
4595
|
-
await
|
|
4788
|
+
const indexFile = path11.join(this.indexPath, "_index.json");
|
|
4789
|
+
await fs5.access(indexFile);
|
|
4596
4790
|
return true;
|
|
4597
4791
|
} catch {
|
|
4598
4792
|
return false;
|
|
@@ -4635,7 +4829,7 @@ function shouldReplaceMatchType(existing, incoming) {
|
|
|
4635
4829
|
return priority[incoming] > priority[existing];
|
|
4636
4830
|
}
|
|
4637
4831
|
function getLiteralIndexPath(rootDir, moduleId, indexDir = ".raggrep") {
|
|
4638
|
-
return
|
|
4832
|
+
return path11.join(rootDir, indexDir, "index", moduleId, "literals");
|
|
4639
4833
|
}
|
|
4640
4834
|
var init_literalIndex = () => {};
|
|
4641
4835
|
|
|
@@ -4656,9 +4850,9 @@ __export(exports_typescript, {
|
|
|
4656
4850
|
DEFAULT_TOP_K: () => DEFAULT_TOP_K2,
|
|
4657
4851
|
DEFAULT_MIN_SCORE: () => DEFAULT_MIN_SCORE2
|
|
4658
4852
|
});
|
|
4659
|
-
import * as
|
|
4853
|
+
import * as path12 from "path";
|
|
4660
4854
|
function isTypeScriptFile(filepath) {
|
|
4661
|
-
const ext =
|
|
4855
|
+
const ext = path12.extname(filepath).toLowerCase();
|
|
4662
4856
|
return TYPESCRIPT_EXTENSIONS.includes(ext);
|
|
4663
4857
|
}
|
|
4664
4858
|
function calculateChunkTypeBoost(chunk) {
|
|
@@ -4730,7 +4924,7 @@ class TypeScriptModule {
|
|
|
4730
4924
|
startLine: 1,
|
|
4731
4925
|
endLine: lines.length,
|
|
4732
4926
|
type: "file",
|
|
4733
|
-
name:
|
|
4927
|
+
name: path12.basename(filepath),
|
|
4734
4928
|
isExported: false
|
|
4735
4929
|
});
|
|
4736
4930
|
}
|
|
@@ -5057,16 +5251,16 @@ class TypeScriptModule {
|
|
|
5057
5251
|
while ((match = importRegex.exec(content)) !== null) {
|
|
5058
5252
|
const importPath = match[1];
|
|
5059
5253
|
if (importPath.startsWith(".")) {
|
|
5060
|
-
const dir =
|
|
5061
|
-
const resolved =
|
|
5254
|
+
const dir = path12.dirname(filepath);
|
|
5255
|
+
const resolved = path12.normalize(path12.join(dir, importPath));
|
|
5062
5256
|
references.push(resolved);
|
|
5063
5257
|
}
|
|
5064
5258
|
}
|
|
5065
5259
|
while ((match = requireRegex.exec(content)) !== null) {
|
|
5066
5260
|
const importPath = match[1];
|
|
5067
5261
|
if (importPath.startsWith(".")) {
|
|
5068
|
-
const dir =
|
|
5069
|
-
const resolved =
|
|
5262
|
+
const dir = path12.dirname(filepath);
|
|
5263
|
+
const resolved = path12.normalize(path12.join(dir, importPath));
|
|
5070
5264
|
references.push(resolved);
|
|
5071
5265
|
}
|
|
5072
5266
|
}
|
|
@@ -5094,7 +5288,7 @@ var init_typescript = __esm(() => {
|
|
|
5094
5288
|
});
|
|
5095
5289
|
|
|
5096
5290
|
// src/infrastructure/parsing/typescriptParser.ts
|
|
5097
|
-
import * as
|
|
5291
|
+
import * as path13 from "path";
|
|
5098
5292
|
|
|
5099
5293
|
class TypeScriptParser {
|
|
5100
5294
|
supportedLanguages = ["typescript", "javascript"];
|
|
@@ -5110,12 +5304,12 @@ class TypeScriptParser {
|
|
|
5110
5304
|
startLine: 1,
|
|
5111
5305
|
endLine: lines.length,
|
|
5112
5306
|
type: "file",
|
|
5113
|
-
name:
|
|
5307
|
+
name: path13.basename(filepath),
|
|
5114
5308
|
isExported: false
|
|
5115
5309
|
};
|
|
5116
5310
|
chunks.unshift(fullFileChunk);
|
|
5117
5311
|
}
|
|
5118
|
-
const ext =
|
|
5312
|
+
const ext = path13.extname(filepath).toLowerCase();
|
|
5119
5313
|
const language = ext === ".js" || ext === ".jsx" || ext === ".mjs" || ext === ".cjs" ? "javascript" : "typescript";
|
|
5120
5314
|
return {
|
|
5121
5315
|
chunks,
|
|
@@ -5132,7 +5326,7 @@ class TypeScriptParser {
|
|
|
5132
5326
|
}
|
|
5133
5327
|
}
|
|
5134
5328
|
canParse(filepath) {
|
|
5135
|
-
const ext =
|
|
5329
|
+
const ext = path13.extname(filepath).toLowerCase();
|
|
5136
5330
|
return TYPESCRIPT_EXTENSIONS2.includes(ext);
|
|
5137
5331
|
}
|
|
5138
5332
|
convertChunk(tc) {
|
|
@@ -5147,7 +5341,7 @@ class TypeScriptParser {
|
|
|
5147
5341
|
};
|
|
5148
5342
|
}
|
|
5149
5343
|
detectLanguage(filepath) {
|
|
5150
|
-
const ext =
|
|
5344
|
+
const ext = path13.extname(filepath).toLowerCase();
|
|
5151
5345
|
if ([".js", ".jsx", ".mjs", ".cjs"].includes(ext)) {
|
|
5152
5346
|
return "javascript";
|
|
5153
5347
|
}
|
|
@@ -5352,8 +5546,8 @@ var init_grammarManager = __esm(() => {
|
|
|
5352
5546
|
});
|
|
5353
5547
|
|
|
5354
5548
|
// src/infrastructure/parsing/treeSitterParser.ts
|
|
5355
|
-
import * as
|
|
5356
|
-
import * as
|
|
5549
|
+
import * as path14 from "path";
|
|
5550
|
+
import * as fs6 from "fs";
|
|
5357
5551
|
|
|
5358
5552
|
class TreeSitterParser {
|
|
5359
5553
|
supportedLanguages = [
|
|
@@ -5375,7 +5569,7 @@ class TreeSitterParser {
|
|
|
5375
5569
|
chunks: [],
|
|
5376
5570
|
language: "typescript",
|
|
5377
5571
|
success: false,
|
|
5378
|
-
error: `Unsupported file type: ${
|
|
5572
|
+
error: `Unsupported file type: ${path14.extname(filepath)}`
|
|
5379
5573
|
};
|
|
5380
5574
|
}
|
|
5381
5575
|
try {
|
|
@@ -5395,11 +5589,11 @@ class TreeSitterParser {
|
|
|
5395
5589
|
}
|
|
5396
5590
|
}
|
|
5397
5591
|
canParse(filepath) {
|
|
5398
|
-
const ext =
|
|
5592
|
+
const ext = path14.extname(filepath).toLowerCase();
|
|
5399
5593
|
return ext in EXTENSION_TO_LANGUAGE2;
|
|
5400
5594
|
}
|
|
5401
5595
|
detectLanguage(filepath) {
|
|
5402
|
-
const ext =
|
|
5596
|
+
const ext = path14.extname(filepath).toLowerCase();
|
|
5403
5597
|
return EXTENSION_TO_LANGUAGE2[ext] || null;
|
|
5404
5598
|
}
|
|
5405
5599
|
async ensureInitialized() {
|
|
@@ -5433,20 +5627,20 @@ class TreeSitterParser {
|
|
|
5433
5627
|
async resolveWasmPath() {
|
|
5434
5628
|
try {
|
|
5435
5629
|
const webTreeSitterPath = __require.resolve("web-tree-sitter");
|
|
5436
|
-
const wasmPath =
|
|
5437
|
-
if (
|
|
5630
|
+
const wasmPath = path14.join(path14.dirname(webTreeSitterPath), "web-tree-sitter.wasm");
|
|
5631
|
+
if (fs6.existsSync(wasmPath)) {
|
|
5438
5632
|
return wasmPath;
|
|
5439
5633
|
}
|
|
5440
5634
|
} catch {}
|
|
5441
5635
|
try {
|
|
5442
5636
|
const possiblePaths = [
|
|
5443
|
-
|
|
5444
|
-
|
|
5445
|
-
|
|
5446
|
-
|
|
5637
|
+
path14.join(__dirname, "../../../node_modules/web-tree-sitter/web-tree-sitter.wasm"),
|
|
5638
|
+
path14.join(__dirname, "../../node_modules/web-tree-sitter/web-tree-sitter.wasm"),
|
|
5639
|
+
path14.join(__dirname, "../../../../node_modules/web-tree-sitter/web-tree-sitter.wasm"),
|
|
5640
|
+
path14.join(__dirname, "web-tree-sitter.wasm")
|
|
5447
5641
|
];
|
|
5448
5642
|
for (const wasmPath of possiblePaths) {
|
|
5449
|
-
if (
|
|
5643
|
+
if (fs6.existsSync(wasmPath)) {
|
|
5450
5644
|
return wasmPath;
|
|
5451
5645
|
}
|
|
5452
5646
|
}
|
|
@@ -5487,7 +5681,7 @@ class TreeSitterParser {
|
|
|
5487
5681
|
startLine: 1,
|
|
5488
5682
|
endLine: lines.length,
|
|
5489
5683
|
type: "file",
|
|
5490
|
-
name:
|
|
5684
|
+
name: path14.basename(filepath),
|
|
5491
5685
|
isExported: false
|
|
5492
5686
|
});
|
|
5493
5687
|
}
|
|
@@ -5834,7 +6028,7 @@ class TreeSitterParser {
|
|
|
5834
6028
|
startLine: 1,
|
|
5835
6029
|
endLine: lines.length,
|
|
5836
6030
|
type: "file",
|
|
5837
|
-
name:
|
|
6031
|
+
name: path14.basename(filepath)
|
|
5838
6032
|
});
|
|
5839
6033
|
return {
|
|
5840
6034
|
chunks,
|
|
@@ -5843,7 +6037,7 @@ class TreeSitterParser {
|
|
|
5843
6037
|
};
|
|
5844
6038
|
}
|
|
5845
6039
|
}
|
|
5846
|
-
var __dirname = "/
|
|
6040
|
+
var __dirname = "/home/runner/work/raggrep/raggrep/src/infrastructure/parsing", EXTENSION_TO_LANGUAGE2;
|
|
5847
6041
|
var init_treeSitterParser = __esm(() => {
|
|
5848
6042
|
init_grammarManager();
|
|
5849
6043
|
EXTENSION_TO_LANGUAGE2 = {
|
|
@@ -5864,7 +6058,7 @@ var init_treeSitterParser = __esm(() => {
|
|
|
5864
6058
|
});
|
|
5865
6059
|
|
|
5866
6060
|
// src/infrastructure/parsing/parserFactory.ts
|
|
5867
|
-
import * as
|
|
6061
|
+
import * as path15 from "path";
|
|
5868
6062
|
function getTypeScriptParser() {
|
|
5869
6063
|
if (!typescriptParserInstance) {
|
|
5870
6064
|
typescriptParserInstance = new TypeScriptParser;
|
|
@@ -5878,7 +6072,7 @@ function getTreeSitterParser() {
|
|
|
5878
6072
|
return treeSitterParserInstance;
|
|
5879
6073
|
}
|
|
5880
6074
|
function createParserForFile(filepath) {
|
|
5881
|
-
const ext =
|
|
6075
|
+
const ext = path15.extname(filepath).toLowerCase();
|
|
5882
6076
|
const parserType = EXTENSION_PARSER_MAP[ext];
|
|
5883
6077
|
if (!parserType) {
|
|
5884
6078
|
return null;
|
|
@@ -5927,9 +6121,9 @@ __export(exports_python, {
|
|
|
5927
6121
|
DEFAULT_TOP_K: () => DEFAULT_TOP_K3,
|
|
5928
6122
|
DEFAULT_MIN_SCORE: () => DEFAULT_MIN_SCORE3
|
|
5929
6123
|
});
|
|
5930
|
-
import * as
|
|
6124
|
+
import * as path16 from "path";
|
|
5931
6125
|
function isPythonFile(filepath) {
|
|
5932
|
-
const ext =
|
|
6126
|
+
const ext = path16.extname(filepath).toLowerCase();
|
|
5933
6127
|
return PYTHON_EXTENSIONS.includes(ext);
|
|
5934
6128
|
}
|
|
5935
6129
|
function generateChunkId3(filepath, startLine, endLine) {
|
|
@@ -6013,7 +6207,7 @@ class PythonModule {
|
|
|
6013
6207
|
startLine: 1,
|
|
6014
6208
|
endLine: lines.length,
|
|
6015
6209
|
type: "file",
|
|
6016
|
-
name:
|
|
6210
|
+
name: path16.basename(filepath)
|
|
6017
6211
|
});
|
|
6018
6212
|
const funcRegex = /^(\s*)(async\s+)?def\s+(\w+)\s*\([^)]*\)\s*:/gm;
|
|
6019
6213
|
let match;
|
|
@@ -6387,9 +6581,9 @@ __export(exports_go, {
|
|
|
6387
6581
|
DEFAULT_TOP_K: () => DEFAULT_TOP_K4,
|
|
6388
6582
|
DEFAULT_MIN_SCORE: () => DEFAULT_MIN_SCORE4
|
|
6389
6583
|
});
|
|
6390
|
-
import * as
|
|
6584
|
+
import * as path17 from "path";
|
|
6391
6585
|
function isGoFile(filepath) {
|
|
6392
|
-
const ext =
|
|
6586
|
+
const ext = path17.extname(filepath).toLowerCase();
|
|
6393
6587
|
return GO_EXTENSIONS.includes(ext);
|
|
6394
6588
|
}
|
|
6395
6589
|
function generateChunkId4(filepath, startLine, endLine) {
|
|
@@ -6474,7 +6668,7 @@ class GoModule {
|
|
|
6474
6668
|
startLine: 1,
|
|
6475
6669
|
endLine: lines.length,
|
|
6476
6670
|
type: "file",
|
|
6477
|
-
name:
|
|
6671
|
+
name: path17.basename(filepath)
|
|
6478
6672
|
});
|
|
6479
6673
|
const funcRegex = /^func\s+(?:\(\s*\w+\s+\*?\w+\s*\)\s+)?(\w+)\s*\(/gm;
|
|
6480
6674
|
let match;
|
|
@@ -6868,9 +7062,9 @@ __export(exports_rust, {
|
|
|
6868
7062
|
DEFAULT_TOP_K: () => DEFAULT_TOP_K5,
|
|
6869
7063
|
DEFAULT_MIN_SCORE: () => DEFAULT_MIN_SCORE5
|
|
6870
7064
|
});
|
|
6871
|
-
import * as
|
|
7065
|
+
import * as path18 from "path";
|
|
6872
7066
|
function isRustFile(filepath) {
|
|
6873
|
-
const ext =
|
|
7067
|
+
const ext = path18.extname(filepath).toLowerCase();
|
|
6874
7068
|
return RUST_EXTENSIONS.includes(ext);
|
|
6875
7069
|
}
|
|
6876
7070
|
function generateChunkId5(filepath, startLine, endLine) {
|
|
@@ -6957,7 +7151,7 @@ class RustModule {
|
|
|
6957
7151
|
startLine: 1,
|
|
6958
7152
|
endLine: lines.length,
|
|
6959
7153
|
type: "file",
|
|
6960
|
-
name:
|
|
7154
|
+
name: path18.basename(filepath)
|
|
6961
7155
|
});
|
|
6962
7156
|
const funcRegex = /^(pub(?:\s*\([^)]*\))?\s+)?(?:async\s+)?fn\s+(\w+)/gm;
|
|
6963
7157
|
let match;
|
|
@@ -7428,9 +7622,9 @@ __export(exports_json, {
|
|
|
7428
7622
|
DEFAULT_TOP_K: () => DEFAULT_TOP_K6,
|
|
7429
7623
|
DEFAULT_MIN_SCORE: () => DEFAULT_MIN_SCORE6
|
|
7430
7624
|
});
|
|
7431
|
-
import * as
|
|
7625
|
+
import * as path19 from "path";
|
|
7432
7626
|
function isJsonFile(filepath) {
|
|
7433
|
-
const ext =
|
|
7627
|
+
const ext = path19.extname(filepath).toLowerCase();
|
|
7434
7628
|
return JSON_EXTENSIONS.includes(ext);
|
|
7435
7629
|
}
|
|
7436
7630
|
|
|
@@ -7464,7 +7658,7 @@ class JsonModule {
|
|
|
7464
7658
|
} catch {
|
|
7465
7659
|
return null;
|
|
7466
7660
|
}
|
|
7467
|
-
const fileBasename =
|
|
7661
|
+
const fileBasename = path19.basename(filepath, path19.extname(filepath));
|
|
7468
7662
|
const jsonPathLiterals = extractJsonPaths(parsed, fileBasename);
|
|
7469
7663
|
const lines = content.split(`
|
|
7470
7664
|
`);
|
|
@@ -7671,7 +7865,7 @@ __export(exports_markdown, {
|
|
|
7671
7865
|
DEFAULT_TOP_K: () => DEFAULT_TOP_K7,
|
|
7672
7866
|
DEFAULT_MIN_SCORE: () => DEFAULT_MIN_SCORE7
|
|
7673
7867
|
});
|
|
7674
|
-
import * as
|
|
7868
|
+
import * as path20 from "path";
|
|
7675
7869
|
function calculateHeadingLevelBoost(chunk) {
|
|
7676
7870
|
const metadata = chunk.metadata;
|
|
7677
7871
|
const level = metadata?.headingLevel ?? 0;
|
|
@@ -7691,7 +7885,7 @@ function calculateHeadingLevelBoost(chunk) {
|
|
|
7691
7885
|
}
|
|
7692
7886
|
}
|
|
7693
7887
|
function isMarkdownFile(filepath) {
|
|
7694
|
-
const ext =
|
|
7888
|
+
const ext = path20.extname(filepath).toLowerCase();
|
|
7695
7889
|
return MARKDOWN_EXTENSIONS.includes(ext);
|
|
7696
7890
|
}
|
|
7697
7891
|
function parseMarkdownHierarchical(content, maxDepth = 4) {
|
|
@@ -7992,6 +8186,11 @@ var init_markdown = __esm(() => {
|
|
|
7992
8186
|
supportsFile6 = isMarkdownFile;
|
|
7993
8187
|
});
|
|
7994
8188
|
|
|
8189
|
+
// src/types.ts
|
|
8190
|
+
var init_types = __esm(() => {
|
|
8191
|
+
init_entities();
|
|
8192
|
+
});
|
|
8193
|
+
|
|
7995
8194
|
// node_modules/fdir/dist/index.mjs
|
|
7996
8195
|
import { createRequire as createRequire2 } from "module";
|
|
7997
8196
|
import { basename, dirname, normalize, relative, resolve, sep } from "path";
|
|
@@ -8527,9 +8726,9 @@ var Builder = class {
|
|
|
8527
8726
|
|
|
8528
8727
|
// src/app/indexer/index.ts
|
|
8529
8728
|
init_config2();
|
|
8530
|
-
import * as
|
|
8531
|
-
import * as
|
|
8532
|
-
import * as
|
|
8729
|
+
import * as fs9 from "fs/promises";
|
|
8730
|
+
import * as path23 from "path";
|
|
8731
|
+
import * as os2 from "os";
|
|
8533
8732
|
import * as crypto2 from "crypto";
|
|
8534
8733
|
|
|
8535
8734
|
// src/modules/registry.ts
|
|
@@ -8570,13 +8769,13 @@ async function registerBuiltInModules() {
|
|
|
8570
8769
|
}
|
|
8571
8770
|
|
|
8572
8771
|
// src/infrastructure/introspection/IntrospectionIndex.ts
|
|
8573
|
-
import * as
|
|
8574
|
-
import * as
|
|
8772
|
+
import * as path22 from "path";
|
|
8773
|
+
import * as fs8 from "fs/promises";
|
|
8575
8774
|
import * as fsSync from "fs";
|
|
8576
8775
|
|
|
8577
8776
|
// src/infrastructure/introspection/projectDetector.ts
|
|
8578
|
-
import * as
|
|
8579
|
-
import * as
|
|
8777
|
+
import * as path21 from "path";
|
|
8778
|
+
import * as fs7 from "fs/promises";
|
|
8580
8779
|
var MAX_SCAN_DEPTH = 4;
|
|
8581
8780
|
var SKIP_DIRS = new Set([
|
|
8582
8781
|
"node_modules",
|
|
@@ -8592,9 +8791,9 @@ async function scanForPackageJsons(rootDir, currentDir = "", depth = 0) {
|
|
|
8592
8791
|
if (depth > MAX_SCAN_DEPTH)
|
|
8593
8792
|
return [];
|
|
8594
8793
|
const results = [];
|
|
8595
|
-
const fullDir = currentDir ?
|
|
8794
|
+
const fullDir = currentDir ? path21.join(rootDir, currentDir) : rootDir;
|
|
8596
8795
|
try {
|
|
8597
|
-
const entries = await
|
|
8796
|
+
const entries = await fs7.readdir(fullDir, { withFileTypes: true });
|
|
8598
8797
|
const hasPackageJson = entries.some((e) => e.isFile() && e.name === "package.json");
|
|
8599
8798
|
if (hasPackageJson && currentDir) {
|
|
8600
8799
|
const info = await parsePackageJson(rootDir, currentDir);
|
|
@@ -8615,10 +8814,10 @@ async function scanForPackageJsons(rootDir, currentDir = "", depth = 0) {
|
|
|
8615
8814
|
}
|
|
8616
8815
|
async function parsePackageJson(rootDir, relativePath) {
|
|
8617
8816
|
try {
|
|
8618
|
-
const packageJsonPath =
|
|
8619
|
-
const content = await
|
|
8817
|
+
const packageJsonPath = path21.join(rootDir, relativePath, "package.json");
|
|
8818
|
+
const content = await fs7.readFile(packageJsonPath, "utf-8");
|
|
8620
8819
|
const pkg = JSON.parse(content);
|
|
8621
|
-
const name = pkg.name ||
|
|
8820
|
+
const name = pkg.name || path21.basename(relativePath);
|
|
8622
8821
|
const deps = { ...pkg.dependencies, ...pkg.devDependencies };
|
|
8623
8822
|
let type = "unknown";
|
|
8624
8823
|
if (deps["next"] || deps["react"] || deps["vue"] || deps["svelte"]) {
|
|
@@ -8654,7 +8853,7 @@ async function detectProjectStructure(rootDir) {
|
|
|
8654
8853
|
const projectMap = new Map;
|
|
8655
8854
|
let isMonorepo = false;
|
|
8656
8855
|
try {
|
|
8657
|
-
const entries = await
|
|
8856
|
+
const entries = await fs7.readdir(rootDir, { withFileTypes: true });
|
|
8658
8857
|
const dirNames = entries.filter((e) => e.isDirectory()).map((e) => e.name);
|
|
8659
8858
|
const monorepoPatterns = ["apps", "packages", "libs", "services"];
|
|
8660
8859
|
const hasMonorepoStructure = monorepoPatterns.some((p) => dirNames.includes(p));
|
|
@@ -8663,9 +8862,9 @@ async function detectProjectStructure(rootDir) {
|
|
|
8663
8862
|
for (const pattern of monorepoPatterns) {
|
|
8664
8863
|
if (!dirNames.includes(pattern))
|
|
8665
8864
|
continue;
|
|
8666
|
-
const patternDir =
|
|
8865
|
+
const patternDir = path21.join(rootDir, pattern);
|
|
8667
8866
|
try {
|
|
8668
|
-
const subDirs = await
|
|
8867
|
+
const subDirs = await fs7.readdir(patternDir, { withFileTypes: true });
|
|
8669
8868
|
for (const subDir of subDirs) {
|
|
8670
8869
|
if (!subDir.isDirectory())
|
|
8671
8870
|
continue;
|
|
@@ -8694,8 +8893,8 @@ async function detectProjectStructure(rootDir) {
|
|
|
8694
8893
|
}
|
|
8695
8894
|
let rootType = "unknown";
|
|
8696
8895
|
try {
|
|
8697
|
-
const rootPkgPath =
|
|
8698
|
-
const rootPkg = JSON.parse(await
|
|
8896
|
+
const rootPkgPath = path21.join(rootDir, "package.json");
|
|
8897
|
+
const rootPkg = JSON.parse(await fs7.readFile(rootPkgPath, "utf-8"));
|
|
8699
8898
|
if (rootPkg.workspaces)
|
|
8700
8899
|
isMonorepo = true;
|
|
8701
8900
|
const deps = { ...rootPkg.dependencies, ...rootPkg.devDependencies };
|
|
@@ -8735,8 +8934,8 @@ class IntrospectionIndex {
|
|
|
8735
8934
|
async initialize() {
|
|
8736
8935
|
this.structure = await detectProjectStructure(this.rootDir);
|
|
8737
8936
|
try {
|
|
8738
|
-
const configPath =
|
|
8739
|
-
const configContent = await
|
|
8937
|
+
const configPath = path22.join(this.rootDir, ".raggrep", "config.json");
|
|
8938
|
+
const configContent = await fs8.readFile(configPath, "utf-8");
|
|
8740
8939
|
const config = JSON.parse(configContent);
|
|
8741
8940
|
this.config = config.introspection || {};
|
|
8742
8941
|
} catch {}
|
|
@@ -8750,7 +8949,7 @@ class IntrospectionIndex {
|
|
|
8750
8949
|
}
|
|
8751
8950
|
const fileExists = enableReadmeContext ? (relativePath) => {
|
|
8752
8951
|
try {
|
|
8753
|
-
const absolutePath =
|
|
8952
|
+
const absolutePath = path22.join(this.rootDir, relativePath);
|
|
8754
8953
|
return fsSync.existsSync(absolutePath);
|
|
8755
8954
|
} catch {
|
|
8756
8955
|
return false;
|
|
@@ -8786,28 +8985,28 @@ class IntrospectionIndex {
|
|
|
8786
8985
|
}
|
|
8787
8986
|
}
|
|
8788
8987
|
async save(config) {
|
|
8789
|
-
const introDir =
|
|
8790
|
-
await
|
|
8791
|
-
const projectPath =
|
|
8792
|
-
await
|
|
8988
|
+
const introDir = path22.join(getRaggrepDir(this.rootDir, config), "introspection");
|
|
8989
|
+
await fs8.mkdir(introDir, { recursive: true });
|
|
8990
|
+
const projectPath = path22.join(introDir, "_project.json");
|
|
8991
|
+
await fs8.writeFile(projectPath, JSON.stringify({
|
|
8793
8992
|
version: "1.0.0",
|
|
8794
8993
|
lastUpdated: new Date().toISOString(),
|
|
8795
8994
|
structure: this.structure
|
|
8796
8995
|
}, null, 2));
|
|
8797
8996
|
for (const [filepath, intro] of this.files) {
|
|
8798
|
-
const introFilePath =
|
|
8799
|
-
await
|
|
8800
|
-
await
|
|
8997
|
+
const introFilePath = path22.join(introDir, "files", filepath.replace(/\.[^.]+$/, ".json"));
|
|
8998
|
+
await fs8.mkdir(path22.dirname(introFilePath), { recursive: true });
|
|
8999
|
+
await fs8.writeFile(introFilePath, JSON.stringify(intro, null, 2));
|
|
8801
9000
|
}
|
|
8802
9001
|
}
|
|
8803
9002
|
async load(config) {
|
|
8804
|
-
const introDir =
|
|
9003
|
+
const introDir = path22.join(getRaggrepDir(this.rootDir, config), "introspection");
|
|
8805
9004
|
try {
|
|
8806
|
-
const projectPath =
|
|
8807
|
-
const projectContent = await
|
|
9005
|
+
const projectPath = path22.join(introDir, "_project.json");
|
|
9006
|
+
const projectContent = await fs8.readFile(projectPath, "utf-8");
|
|
8808
9007
|
const projectData = JSON.parse(projectContent);
|
|
8809
9008
|
this.structure = projectData.structure;
|
|
8810
|
-
await this.loadFilesRecursive(
|
|
9009
|
+
await this.loadFilesRecursive(path22.join(introDir, "files"), "");
|
|
8811
9010
|
} catch {
|
|
8812
9011
|
this.structure = null;
|
|
8813
9012
|
this.files.clear();
|
|
@@ -8815,14 +9014,14 @@ class IntrospectionIndex {
|
|
|
8815
9014
|
}
|
|
8816
9015
|
async loadFilesRecursive(basePath, prefix) {
|
|
8817
9016
|
try {
|
|
8818
|
-
const entries = await
|
|
9017
|
+
const entries = await fs8.readdir(basePath, { withFileTypes: true });
|
|
8819
9018
|
for (const entry of entries) {
|
|
8820
|
-
const entryPath =
|
|
9019
|
+
const entryPath = path22.join(basePath, entry.name);
|
|
8821
9020
|
const relativePath = prefix ? `${prefix}/${entry.name}` : entry.name;
|
|
8822
9021
|
if (entry.isDirectory()) {
|
|
8823
9022
|
await this.loadFilesRecursive(entryPath, relativePath);
|
|
8824
9023
|
} else if (entry.name.endsWith(".json")) {
|
|
8825
|
-
const content = await
|
|
9024
|
+
const content = await fs8.readFile(entryPath, "utf-8");
|
|
8826
9025
|
const intro = JSON.parse(content);
|
|
8827
9026
|
this.files.set(intro.filepath, intro);
|
|
8828
9027
|
}
|
|
@@ -9078,7 +9277,7 @@ async function parallelMap(items, processor, concurrency) {
|
|
|
9078
9277
|
await Promise.all(workers);
|
|
9079
9278
|
return results;
|
|
9080
9279
|
}
|
|
9081
|
-
var INDEX_SCHEMA_VERSION = "2.
|
|
9280
|
+
var INDEX_SCHEMA_VERSION = "2.1.0";
|
|
9082
9281
|
function formatDuration(ms) {
|
|
9083
9282
|
if (ms < 1000) {
|
|
9084
9283
|
return `${ms}ms`;
|
|
@@ -9092,7 +9291,7 @@ function formatDuration(ms) {
|
|
|
9092
9291
|
return `${minutes}m ${remainingSeconds.toFixed(1)}s`;
|
|
9093
9292
|
}
|
|
9094
9293
|
function getOptimalConcurrency() {
|
|
9095
|
-
const cpuCount =
|
|
9294
|
+
const cpuCount = os2.cpus().length;
|
|
9096
9295
|
const optimal = Math.max(2, Math.min(16, Math.floor(cpuCount * 0.75)));
|
|
9097
9296
|
return optimal;
|
|
9098
9297
|
}
|
|
@@ -9104,7 +9303,7 @@ async function indexDirectory(rootDir, options = {}) {
|
|
|
9104
9303
|
const concurrency = options.concurrency ?? DEFAULT_CONCURRENCY;
|
|
9105
9304
|
clearFreshnessCache();
|
|
9106
9305
|
const logger = options.logger ? options.logger : quiet ? createSilentLogger() : createLogger({ verbose });
|
|
9107
|
-
rootDir =
|
|
9306
|
+
rootDir = path23.resolve(rootDir);
|
|
9108
9307
|
const location = getIndexLocation(rootDir);
|
|
9109
9308
|
logger.info(`Indexing directory: ${rootDir}`);
|
|
9110
9309
|
logger.info(`Index location: ${location.indexDir}`);
|
|
@@ -9156,12 +9355,12 @@ async function indexDirectory(rootDir, options = {}) {
|
|
|
9156
9355
|
rootDir,
|
|
9157
9356
|
config,
|
|
9158
9357
|
readFile: async (filepath) => {
|
|
9159
|
-
const fullPath =
|
|
9160
|
-
return
|
|
9358
|
+
const fullPath = path23.isAbsolute(filepath) ? filepath : path23.join(rootDir, filepath);
|
|
9359
|
+
return fs9.readFile(fullPath, "utf-8");
|
|
9161
9360
|
},
|
|
9162
9361
|
getFileStats: async (filepath) => {
|
|
9163
|
-
const fullPath =
|
|
9164
|
-
const stats = await
|
|
9362
|
+
const fullPath = path23.isAbsolute(filepath) ? filepath : path23.join(rootDir, filepath);
|
|
9363
|
+
const stats = await fs9.stat(fullPath);
|
|
9165
9364
|
return { lastModified: stats.mtime.toISOString() };
|
|
9166
9365
|
}
|
|
9167
9366
|
};
|
|
@@ -9186,7 +9385,7 @@ async function isIndexVersionCompatible(rootDir) {
|
|
|
9186
9385
|
const config = await loadConfig(rootDir);
|
|
9187
9386
|
const globalManifestPath = getGlobalManifestPath(rootDir, config);
|
|
9188
9387
|
try {
|
|
9189
|
-
const content = await
|
|
9388
|
+
const content = await fs9.readFile(globalManifestPath, "utf-8");
|
|
9190
9389
|
const manifest = JSON.parse(content);
|
|
9191
9390
|
return manifest.version === INDEX_SCHEMA_VERSION;
|
|
9192
9391
|
} catch {
|
|
@@ -9196,11 +9395,11 @@ async function isIndexVersionCompatible(rootDir) {
|
|
|
9196
9395
|
async function deleteIndex(rootDir) {
|
|
9197
9396
|
const indexDir = getRaggrepDir(rootDir);
|
|
9198
9397
|
try {
|
|
9199
|
-
await
|
|
9398
|
+
await fs9.rm(indexDir, { recursive: true, force: true });
|
|
9200
9399
|
} catch {}
|
|
9201
9400
|
}
|
|
9202
9401
|
async function resetIndex(rootDir) {
|
|
9203
|
-
rootDir =
|
|
9402
|
+
rootDir = path23.resolve(rootDir);
|
|
9204
9403
|
clearFreshnessCache();
|
|
9205
9404
|
const status = await getIndexStatus(rootDir);
|
|
9206
9405
|
if (!status.exists) {
|
|
@@ -9225,7 +9424,7 @@ async function ensureIndexFresh(rootDir, options = {}) {
|
|
|
9225
9424
|
let filesChanged = 0;
|
|
9226
9425
|
let filesReindexed = 0;
|
|
9227
9426
|
const logger = options.logger ? options.logger : quiet ? createSilentLogger() : createLogger({ verbose });
|
|
9228
|
-
rootDir =
|
|
9427
|
+
rootDir = path23.resolve(rootDir);
|
|
9229
9428
|
const status = await getIndexStatus(rootDir);
|
|
9230
9429
|
if (!status.exists) {
|
|
9231
9430
|
clearFreshnessCache();
|
|
@@ -9249,7 +9448,7 @@ async function ensureIndexFresh(rootDir, options = {}) {
|
|
|
9249
9448
|
const globalManifestPath = getGlobalManifestPath(rootDir, config);
|
|
9250
9449
|
let currentManifestMtime = 0;
|
|
9251
9450
|
try {
|
|
9252
|
-
const manifestStats = await
|
|
9451
|
+
const manifestStats = await fs9.stat(globalManifestPath);
|
|
9253
9452
|
currentManifestMtime = manifestStats.mtimeMs;
|
|
9254
9453
|
} catch {}
|
|
9255
9454
|
const now = Date.now();
|
|
@@ -9287,7 +9486,7 @@ async function ensureIndexFresh(rootDir, options = {}) {
|
|
|
9287
9486
|
const { allFiles: currentFiles, changedFiles, changedFileMtimes } = discoveryResult;
|
|
9288
9487
|
filesDiscovered = currentFiles.length;
|
|
9289
9488
|
filesChanged = changedFiles.length;
|
|
9290
|
-
const currentFileSet = new Set(currentFiles.map((f) =>
|
|
9489
|
+
const currentFileSet = new Set(currentFiles.map((f) => path23.relative(rootDir, f)));
|
|
9291
9490
|
const changedFileSet = new Set(changedFiles);
|
|
9292
9491
|
let totalIndexed = 0;
|
|
9293
9492
|
let totalRemoved = 0;
|
|
@@ -9321,11 +9520,11 @@ async function ensureIndexFresh(rootDir, options = {}) {
|
|
|
9321
9520
|
if (filesToRemove.length > 0) {
|
|
9322
9521
|
await Promise.all(filesToRemove.map(async (filepath) => {
|
|
9323
9522
|
logger.debug(` Removing stale: ${filepath}`);
|
|
9324
|
-
const indexFilePath =
|
|
9325
|
-
const symbolicFilePath =
|
|
9523
|
+
const indexFilePath = path23.join(indexPath, filepath.replace(/\.[^.]+$/, ".json"));
|
|
9524
|
+
const symbolicFilePath = path23.join(indexPath, "symbolic", filepath.replace(/\.[^.]+$/, ".json"));
|
|
9326
9525
|
await Promise.all([
|
|
9327
|
-
|
|
9328
|
-
|
|
9526
|
+
fs9.unlink(indexFilePath).catch(() => {}),
|
|
9527
|
+
fs9.unlink(symbolicFilePath).catch(() => {})
|
|
9329
9528
|
]);
|
|
9330
9529
|
delete manifest.files[filepath];
|
|
9331
9530
|
removedFilepaths.push(filepath);
|
|
@@ -9349,19 +9548,19 @@ async function ensureIndexFresh(rootDir, options = {}) {
|
|
|
9349
9548
|
rootDir,
|
|
9350
9549
|
config,
|
|
9351
9550
|
readFile: async (filepath) => {
|
|
9352
|
-
const fullPath =
|
|
9353
|
-
return
|
|
9551
|
+
const fullPath = path23.isAbsolute(filepath) ? filepath : path23.join(rootDir, filepath);
|
|
9552
|
+
return fs9.readFile(fullPath, "utf-8");
|
|
9354
9553
|
},
|
|
9355
9554
|
getFileStats: async (filepath) => {
|
|
9356
|
-
const fullPath =
|
|
9357
|
-
const stats = await
|
|
9555
|
+
const fullPath = path23.isAbsolute(filepath) ? filepath : path23.join(rootDir, filepath);
|
|
9556
|
+
const stats = await fs9.stat(fullPath);
|
|
9358
9557
|
return { lastModified: stats.mtime.toISOString() };
|
|
9359
9558
|
},
|
|
9360
9559
|
getIntrospection: (filepath) => introspection.getFile(filepath)
|
|
9361
9560
|
};
|
|
9362
9561
|
const moduleChangedFiles = module.supportsFile ? changedFiles.filter((f) => module.supportsFile(f)) : changedFiles;
|
|
9363
9562
|
const filesToProcess = moduleChangedFiles.map((filepath) => {
|
|
9364
|
-
const relativePath =
|
|
9563
|
+
const relativePath = path23.relative(rootDir, filepath);
|
|
9365
9564
|
const existingEntry = manifest.files[relativePath];
|
|
9366
9565
|
const lastModified = changedFileMtimes.get(filepath) || new Date().toISOString();
|
|
9367
9566
|
return {
|
|
@@ -9391,7 +9590,7 @@ async function ensureIndexFresh(rootDir, options = {}) {
|
|
|
9391
9590
|
return { relativePath, status: "unchanged" };
|
|
9392
9591
|
}
|
|
9393
9592
|
try {
|
|
9394
|
-
const content = await
|
|
9593
|
+
const content = await fs9.readFile(filepath, "utf-8");
|
|
9395
9594
|
const contentHash = computeContentHash(content);
|
|
9396
9595
|
if (!isNew && existingContentHash && existingContentHash === contentHash) {
|
|
9397
9596
|
completedCount++;
|
|
@@ -9518,7 +9717,7 @@ async function ensureIndexFresh(rootDir, options = {}) {
|
|
|
9518
9717
|
}
|
|
9519
9718
|
let finalManifestMtime = currentManifestMtime;
|
|
9520
9719
|
try {
|
|
9521
|
-
const manifestStats = await
|
|
9720
|
+
const manifestStats = await fs9.stat(globalManifestPath);
|
|
9522
9721
|
finalManifestMtime = manifestStats.mtimeMs;
|
|
9523
9722
|
} catch {}
|
|
9524
9723
|
freshnessCache = {
|
|
@@ -9538,7 +9737,7 @@ async function indexWithModule(rootDir, files, module, config, verbose, introspe
|
|
|
9538
9737
|
};
|
|
9539
9738
|
const manifest = await loadModuleManifest(rootDir, module.id, config);
|
|
9540
9739
|
const indexPath = getModuleIndexPath(rootDir, module.id, config);
|
|
9541
|
-
const currentFileSet = new Set(files.map((f) =>
|
|
9740
|
+
const currentFileSet = new Set(files.map((f) => path23.relative(rootDir, f)));
|
|
9542
9741
|
const filesToRemove = [];
|
|
9543
9742
|
for (const filepath of Object.keys(manifest.files)) {
|
|
9544
9743
|
if (!currentFileSet.has(filepath)) {
|
|
@@ -9549,13 +9748,13 @@ async function indexWithModule(rootDir, files, module, config, verbose, introspe
|
|
|
9549
9748
|
logger.info(` Removing ${filesToRemove.length} stale entries...`);
|
|
9550
9749
|
for (const filepath of filesToRemove) {
|
|
9551
9750
|
logger.debug(` Removing: ${filepath}`);
|
|
9552
|
-
const indexFilePath =
|
|
9751
|
+
const indexFilePath = path23.join(indexPath, filepath.replace(/\.[^.]+$/, ".json"));
|
|
9553
9752
|
try {
|
|
9554
|
-
await
|
|
9753
|
+
await fs9.unlink(indexFilePath);
|
|
9555
9754
|
} catch {}
|
|
9556
|
-
const symbolicFilePath =
|
|
9755
|
+
const symbolicFilePath = path23.join(indexPath, "symbolic", filepath.replace(/\.[^.]+$/, ".json"));
|
|
9557
9756
|
try {
|
|
9558
|
-
await
|
|
9757
|
+
await fs9.unlink(symbolicFilePath);
|
|
9559
9758
|
} catch {}
|
|
9560
9759
|
delete manifest.files[filepath];
|
|
9561
9760
|
}
|
|
@@ -9565,12 +9764,12 @@ async function indexWithModule(rootDir, files, module, config, verbose, introspe
|
|
|
9565
9764
|
rootDir,
|
|
9566
9765
|
config,
|
|
9567
9766
|
readFile: async (filepath) => {
|
|
9568
|
-
const fullPath =
|
|
9569
|
-
return
|
|
9767
|
+
const fullPath = path23.isAbsolute(filepath) ? filepath : path23.join(rootDir, filepath);
|
|
9768
|
+
return fs9.readFile(fullPath, "utf-8");
|
|
9570
9769
|
},
|
|
9571
9770
|
getFileStats: async (filepath) => {
|
|
9572
|
-
const fullPath =
|
|
9573
|
-
const stats = await
|
|
9771
|
+
const fullPath = path23.isAbsolute(filepath) ? filepath : path23.join(rootDir, filepath);
|
|
9772
|
+
const stats = await fs9.stat(fullPath);
|
|
9574
9773
|
return { lastModified: stats.mtime.toISOString() };
|
|
9575
9774
|
},
|
|
9576
9775
|
getIntrospection: (filepath) => introspection.getFile(filepath)
|
|
@@ -9582,9 +9781,9 @@ async function indexWithModule(rootDir, files, module, config, verbose, introspe
|
|
|
9582
9781
|
let indexedCount = 0;
|
|
9583
9782
|
let skippedCount = 0;
|
|
9584
9783
|
const processFile = async (filepath, _index) => {
|
|
9585
|
-
const relativePath =
|
|
9784
|
+
const relativePath = path23.relative(rootDir, filepath);
|
|
9586
9785
|
try {
|
|
9587
|
-
const stats = await
|
|
9786
|
+
const stats = await fs9.stat(filepath);
|
|
9588
9787
|
const lastModified = stats.mtime.toISOString();
|
|
9589
9788
|
const existingEntry = manifest.files[relativePath];
|
|
9590
9789
|
if (existingEntry && existingEntry.lastModified === lastModified) {
|
|
@@ -9594,7 +9793,7 @@ async function indexWithModule(rootDir, files, module, config, verbose, introspe
|
|
|
9594
9793
|
logger.debug(` [${completedCount}/${totalFiles}] Skipped ${relativePath} (unchanged)`);
|
|
9595
9794
|
return { relativePath, status: "skipped" };
|
|
9596
9795
|
}
|
|
9597
|
-
const content = await
|
|
9796
|
+
const content = await fs9.readFile(filepath, "utf-8");
|
|
9598
9797
|
const contentHash = computeContentHash(content);
|
|
9599
9798
|
if (existingEntry?.contentHash && existingEntry.contentHash === contentHash) {
|
|
9600
9799
|
completedCount++;
|
|
@@ -9676,8 +9875,8 @@ async function indexWithModule(rootDir, files, module, config, verbose, introspe
|
|
|
9676
9875
|
}
|
|
9677
9876
|
var STAT_CONCURRENCY = 64;
|
|
9678
9877
|
function isLikelyBinary(filepath) {
|
|
9679
|
-
const ext =
|
|
9680
|
-
const basename15 =
|
|
9878
|
+
const ext = path23.extname(filepath).toLowerCase();
|
|
9879
|
+
const basename15 = path23.basename(filepath).toLowerCase();
|
|
9681
9880
|
const binaryExtensions = new Set([
|
|
9682
9881
|
".png",
|
|
9683
9882
|
".jpg",
|
|
@@ -9753,7 +9952,7 @@ async function findFilesWithStats(rootDir, config, lastIndexStarted) {
|
|
|
9753
9952
|
const ignoreDirs = new Set(config.ignorePaths);
|
|
9754
9953
|
const lastIndexMs = lastIndexStarted?.getTime() ?? 0;
|
|
9755
9954
|
const crawler = new Builder().withFullPaths().exclude((dirName) => ignoreDirs.has(dirName)).filter((filePath) => {
|
|
9756
|
-
const ext =
|
|
9955
|
+
const ext = path23.extname(filePath);
|
|
9757
9956
|
return validExtensions.has(ext);
|
|
9758
9957
|
}).crawl(rootDir);
|
|
9759
9958
|
const allFiles = await crawler.withPromise();
|
|
@@ -9761,7 +9960,7 @@ async function findFilesWithStats(rootDir, config, lastIndexStarted) {
|
|
|
9761
9960
|
const changedFileMtimes2 = new Map;
|
|
9762
9961
|
await parallelMap(allFiles, async (filePath) => {
|
|
9763
9962
|
try {
|
|
9764
|
-
const stats = await
|
|
9963
|
+
const stats = await fs9.stat(filePath);
|
|
9765
9964
|
changedFileMtimes2.set(filePath, stats.mtime.toISOString());
|
|
9766
9965
|
} catch {}
|
|
9767
9966
|
}, STAT_CONCURRENCY);
|
|
@@ -9775,7 +9974,7 @@ async function findFilesWithStats(rootDir, config, lastIndexStarted) {
|
|
|
9775
9974
|
const changedFileMtimes = new Map;
|
|
9776
9975
|
await parallelMap(allFiles, async (filePath) => {
|
|
9777
9976
|
try {
|
|
9778
|
-
const stats = await
|
|
9977
|
+
const stats = await fs9.stat(filePath);
|
|
9779
9978
|
if (stats.mtimeMs > lastIndexMs) {
|
|
9780
9979
|
changedFiles.push(filePath);
|
|
9781
9980
|
changedFileMtimes.set(filePath, stats.mtime.toISOString());
|
|
@@ -9795,7 +9994,7 @@ async function findFiles(rootDir, config) {
|
|
|
9795
9994
|
async function loadModuleManifest(rootDir, moduleId, config) {
|
|
9796
9995
|
const manifestPath = getModuleManifestPath(rootDir, moduleId, config);
|
|
9797
9996
|
try {
|
|
9798
|
-
const content = await
|
|
9997
|
+
const content = await fs9.readFile(manifestPath, "utf-8");
|
|
9799
9998
|
return JSON.parse(content);
|
|
9800
9999
|
} catch {
|
|
9801
10000
|
return {
|
|
@@ -9808,19 +10007,19 @@ async function loadModuleManifest(rootDir, moduleId, config) {
|
|
|
9808
10007
|
}
|
|
9809
10008
|
async function writeModuleManifest(rootDir, moduleId, manifest, config) {
|
|
9810
10009
|
const manifestPath = getModuleManifestPath(rootDir, moduleId, config);
|
|
9811
|
-
await
|
|
9812
|
-
await
|
|
10010
|
+
await fs9.mkdir(path23.dirname(manifestPath), { recursive: true });
|
|
10011
|
+
await fs9.writeFile(manifestPath, JSON.stringify(manifest, null, 2));
|
|
9813
10012
|
}
|
|
9814
10013
|
async function writeFileIndex(rootDir, moduleId, filepath, fileIndex, config) {
|
|
9815
10014
|
const indexPath = getModuleIndexPath(rootDir, moduleId, config);
|
|
9816
|
-
const indexFilePath =
|
|
9817
|
-
await
|
|
9818
|
-
await
|
|
10015
|
+
const indexFilePath = path23.join(indexPath, filepath.replace(/\.[^.]+$/, ".json"));
|
|
10016
|
+
await fs9.mkdir(path23.dirname(indexFilePath), { recursive: true });
|
|
10017
|
+
await fs9.writeFile(indexFilePath, JSON.stringify(fileIndex, null, 2));
|
|
9819
10018
|
}
|
|
9820
10019
|
async function loadGlobalManifest(rootDir, config) {
|
|
9821
10020
|
const manifestPath = getGlobalManifestPath(rootDir, config);
|
|
9822
10021
|
try {
|
|
9823
|
-
const content = await
|
|
10022
|
+
const content = await fs9.readFile(manifestPath, "utf-8");
|
|
9824
10023
|
return JSON.parse(content);
|
|
9825
10024
|
} catch {
|
|
9826
10025
|
return null;
|
|
@@ -9834,13 +10033,13 @@ async function updateGlobalManifest(rootDir, modules, config, indexStartTime) {
|
|
|
9834
10033
|
lastIndexStarted: indexStartTime,
|
|
9835
10034
|
modules: modules.map((m) => m.id)
|
|
9836
10035
|
};
|
|
9837
|
-
await
|
|
9838
|
-
await
|
|
10036
|
+
await fs9.mkdir(path23.dirname(manifestPath), { recursive: true });
|
|
10037
|
+
await fs9.writeFile(manifestPath, JSON.stringify(manifest, null, 2));
|
|
9839
10038
|
}
|
|
9840
10039
|
async function cleanupIndex(rootDir, options = {}) {
|
|
9841
10040
|
const verbose = options.verbose ?? false;
|
|
9842
10041
|
const logger = options.logger ?? createLogger({ verbose });
|
|
9843
|
-
rootDir =
|
|
10042
|
+
rootDir = path23.resolve(rootDir);
|
|
9844
10043
|
logger.info(`Cleaning up index in: ${rootDir}`);
|
|
9845
10044
|
const config = await loadConfig(rootDir);
|
|
9846
10045
|
await registerBuiltInModules();
|
|
@@ -9870,9 +10069,9 @@ async function cleanupModuleIndex(rootDir, moduleId, config, logger) {
|
|
|
9870
10069
|
const filesToRemove = [];
|
|
9871
10070
|
const updatedFiles = {};
|
|
9872
10071
|
for (const [filepath, entry] of Object.entries(manifest.files)) {
|
|
9873
|
-
const fullPath =
|
|
10072
|
+
const fullPath = path23.join(rootDir, filepath);
|
|
9874
10073
|
try {
|
|
9875
|
-
await
|
|
10074
|
+
await fs9.access(fullPath);
|
|
9876
10075
|
updatedFiles[filepath] = entry;
|
|
9877
10076
|
result.kept++;
|
|
9878
10077
|
} catch {
|
|
@@ -9882,9 +10081,9 @@ async function cleanupModuleIndex(rootDir, moduleId, config, logger) {
|
|
|
9882
10081
|
}
|
|
9883
10082
|
}
|
|
9884
10083
|
for (const filepath of filesToRemove) {
|
|
9885
|
-
const indexFilePath =
|
|
10084
|
+
const indexFilePath = path23.join(indexPath, filepath.replace(/\.[^.]+$/, ".json"));
|
|
9886
10085
|
try {
|
|
9887
|
-
await
|
|
10086
|
+
await fs9.unlink(indexFilePath);
|
|
9888
10087
|
} catch {}
|
|
9889
10088
|
}
|
|
9890
10089
|
manifest.files = updatedFiles;
|
|
@@ -9895,16 +10094,16 @@ async function cleanupModuleIndex(rootDir, moduleId, config, logger) {
|
|
|
9895
10094
|
}
|
|
9896
10095
|
async function cleanupEmptyDirectories(dir) {
|
|
9897
10096
|
try {
|
|
9898
|
-
const entries = await
|
|
10097
|
+
const entries = await fs9.readdir(dir, { withFileTypes: true });
|
|
9899
10098
|
for (const entry of entries) {
|
|
9900
10099
|
if (entry.isDirectory()) {
|
|
9901
|
-
const subDir =
|
|
10100
|
+
const subDir = path23.join(dir, entry.name);
|
|
9902
10101
|
await cleanupEmptyDirectories(subDir);
|
|
9903
10102
|
}
|
|
9904
10103
|
}
|
|
9905
|
-
const remainingEntries = await
|
|
10104
|
+
const remainingEntries = await fs9.readdir(dir);
|
|
9906
10105
|
if (remainingEntries.length === 0) {
|
|
9907
|
-
await
|
|
10106
|
+
await fs9.rmdir(dir);
|
|
9908
10107
|
return true;
|
|
9909
10108
|
}
|
|
9910
10109
|
return false;
|
|
@@ -9913,7 +10112,7 @@ async function cleanupEmptyDirectories(dir) {
|
|
|
9913
10112
|
}
|
|
9914
10113
|
}
|
|
9915
10114
|
async function getIndexStatus(rootDir) {
|
|
9916
|
-
rootDir =
|
|
10115
|
+
rootDir = path23.resolve(rootDir);
|
|
9917
10116
|
const config = await loadConfig(rootDir);
|
|
9918
10117
|
const location = getIndexLocation(rootDir);
|
|
9919
10118
|
const indexDir = location.indexDir;
|
|
@@ -9925,13 +10124,13 @@ async function getIndexStatus(rootDir) {
|
|
|
9925
10124
|
totalFiles: 0
|
|
9926
10125
|
};
|
|
9927
10126
|
try {
|
|
9928
|
-
await
|
|
10127
|
+
await fs9.access(indexDir);
|
|
9929
10128
|
} catch {
|
|
9930
10129
|
return status;
|
|
9931
10130
|
}
|
|
9932
10131
|
try {
|
|
9933
10132
|
const globalManifestPath = getGlobalManifestPath(rootDir, config);
|
|
9934
|
-
const content = await
|
|
10133
|
+
const content = await fs9.readFile(globalManifestPath, "utf-8");
|
|
9935
10134
|
const globalManifest = JSON.parse(content);
|
|
9936
10135
|
status.exists = true;
|
|
9937
10136
|
status.lastUpdated = globalManifest.lastUpdated;
|
|
@@ -9949,7 +10148,7 @@ async function getIndexStatus(rootDir) {
|
|
|
9949
10148
|
}
|
|
9950
10149
|
} catch {
|
|
9951
10150
|
try {
|
|
9952
|
-
const entries = await
|
|
10151
|
+
const entries = await fs9.readdir(path23.join(indexDir, "index"));
|
|
9953
10152
|
if (entries.length > 0) {
|
|
9954
10153
|
status.exists = true;
|
|
9955
10154
|
for (const entry of entries) {
|
|
@@ -9971,8 +10170,8 @@ async function getIndexStatus(rootDir) {
|
|
|
9971
10170
|
}
|
|
9972
10171
|
|
|
9973
10172
|
// src/app/search/index.ts
|
|
9974
|
-
import * as
|
|
9975
|
-
import * as
|
|
10173
|
+
import * as fs11 from "fs/promises";
|
|
10174
|
+
import * as path26 from "path";
|
|
9976
10175
|
|
|
9977
10176
|
// node_modules/@isaacs/balanced-match/dist/esm/index.js
|
|
9978
10177
|
var balanced = (a, b, str) => {
|
|
@@ -10775,11 +10974,11 @@ var qmarksTestNoExtDot = ([$0]) => {
|
|
|
10775
10974
|
return (f) => f.length === len && f !== "." && f !== "..";
|
|
10776
10975
|
};
|
|
10777
10976
|
var defaultPlatform = typeof process === "object" && process ? typeof process.env === "object" && process.env && process.env.__MINIMATCH_TESTING_PLATFORM__ || process.platform : "posix";
|
|
10778
|
-
var
|
|
10977
|
+
var path24 = {
|
|
10779
10978
|
win32: { sep: "\\" },
|
|
10780
10979
|
posix: { sep: "/" }
|
|
10781
10980
|
};
|
|
10782
|
-
var sep2 = defaultPlatform === "win32" ?
|
|
10981
|
+
var sep2 = defaultPlatform === "win32" ? path24.win32.sep : path24.posix.sep;
|
|
10783
10982
|
minimatch.sep = sep2;
|
|
10784
10983
|
var GLOBSTAR = Symbol("globstar **");
|
|
10785
10984
|
minimatch.GLOBSTAR = GLOBSTAR;
|
|
@@ -11408,10 +11607,8 @@ minimatch.Minimatch = Minimatch;
|
|
|
11408
11607
|
minimatch.escape = escape;
|
|
11409
11608
|
minimatch.unescape = unescape;
|
|
11410
11609
|
|
|
11411
|
-
// src/types.ts
|
|
11412
|
-
init_entities();
|
|
11413
|
-
|
|
11414
11610
|
// src/app/search/index.ts
|
|
11611
|
+
init_types();
|
|
11415
11612
|
init_config2();
|
|
11416
11613
|
init_services();
|
|
11417
11614
|
// src/domain/usecases/exactSearch.ts
|
|
@@ -11445,7 +11642,7 @@ function matchesPathFilter(relativePath, filters, matchFn) {
|
|
|
11445
11642
|
}
|
|
11446
11643
|
return false;
|
|
11447
11644
|
}
|
|
11448
|
-
async function executeExactSearch(
|
|
11645
|
+
async function executeExactSearch(fs10, options, matchFn) {
|
|
11449
11646
|
const {
|
|
11450
11647
|
rootDir,
|
|
11451
11648
|
literal,
|
|
@@ -11457,13 +11654,13 @@ async function executeExactSearch(fs9, options, matchFn) {
|
|
|
11457
11654
|
const files = new Map;
|
|
11458
11655
|
async function walkDir(dir, baseDir) {
|
|
11459
11656
|
try {
|
|
11460
|
-
const entries = await
|
|
11657
|
+
const entries = await fs10.readDir(dir);
|
|
11461
11658
|
for (const entry of entries) {
|
|
11462
|
-
const fullPath =
|
|
11463
|
-
const relativePath =
|
|
11659
|
+
const fullPath = fs10.join(dir, entry);
|
|
11660
|
+
const relativePath = fs10.relative(baseDir, fullPath);
|
|
11464
11661
|
let isDirectory = false;
|
|
11465
11662
|
try {
|
|
11466
|
-
const stats = await
|
|
11663
|
+
const stats = await fs10.getStats(fullPath);
|
|
11467
11664
|
isDirectory = stats.isDirectory ?? false;
|
|
11468
11665
|
} catch {
|
|
11469
11666
|
continue;
|
|
@@ -11480,7 +11677,7 @@ async function executeExactSearch(fs9, options, matchFn) {
|
|
|
11480
11677
|
}
|
|
11481
11678
|
}
|
|
11482
11679
|
try {
|
|
11483
|
-
const content = await
|
|
11680
|
+
const content = await fs10.readFile(fullPath);
|
|
11484
11681
|
if (isSearchableContent(content, fullPath)) {
|
|
11485
11682
|
files.set(relativePath, content);
|
|
11486
11683
|
}
|
|
@@ -11497,21 +11694,21 @@ async function executeExactSearch(fs9, options, matchFn) {
|
|
|
11497
11694
|
});
|
|
11498
11695
|
}
|
|
11499
11696
|
// src/infrastructure/filesystem/nodeFileSystem.ts
|
|
11500
|
-
import * as
|
|
11501
|
-
import * as
|
|
11697
|
+
import * as fs10 from "fs/promises";
|
|
11698
|
+
import * as path25 from "path";
|
|
11502
11699
|
import { glob } from "glob";
|
|
11503
11700
|
|
|
11504
11701
|
class NodeFileSystem {
|
|
11505
11702
|
async readFile(filepath) {
|
|
11506
|
-
return
|
|
11703
|
+
return fs10.readFile(filepath, "utf-8");
|
|
11507
11704
|
}
|
|
11508
11705
|
async writeFile(filepath, content) {
|
|
11509
|
-
await
|
|
11510
|
-
await
|
|
11706
|
+
await fs10.mkdir(path25.dirname(filepath), { recursive: true });
|
|
11707
|
+
await fs10.writeFile(filepath, content, "utf-8");
|
|
11511
11708
|
}
|
|
11512
11709
|
async deleteFile(filepath) {
|
|
11513
11710
|
try {
|
|
11514
|
-
await
|
|
11711
|
+
await fs10.unlink(filepath);
|
|
11515
11712
|
} catch (error) {
|
|
11516
11713
|
if (error.code !== "ENOENT") {
|
|
11517
11714
|
throw error;
|
|
@@ -11519,7 +11716,7 @@ class NodeFileSystem {
|
|
|
11519
11716
|
}
|
|
11520
11717
|
}
|
|
11521
11718
|
async getStats(filepath) {
|
|
11522
|
-
const stats = await
|
|
11719
|
+
const stats = await fs10.stat(filepath);
|
|
11523
11720
|
return {
|
|
11524
11721
|
lastModified: stats.mtime.toISOString(),
|
|
11525
11722
|
size: stats.isDirectory() ? undefined : stats.size,
|
|
@@ -11528,17 +11725,17 @@ class NodeFileSystem {
|
|
|
11528
11725
|
}
|
|
11529
11726
|
async exists(filepath) {
|
|
11530
11727
|
try {
|
|
11531
|
-
await
|
|
11728
|
+
await fs10.access(filepath);
|
|
11532
11729
|
return true;
|
|
11533
11730
|
} catch {
|
|
11534
11731
|
return false;
|
|
11535
11732
|
}
|
|
11536
11733
|
}
|
|
11537
11734
|
async mkdir(dirpath) {
|
|
11538
|
-
await
|
|
11735
|
+
await fs10.mkdir(dirpath, { recursive: true });
|
|
11539
11736
|
}
|
|
11540
11737
|
async readDir(dirpath) {
|
|
11541
|
-
return
|
|
11738
|
+
return fs10.readdir(dirpath);
|
|
11542
11739
|
}
|
|
11543
11740
|
async findFiles(rootDir, patterns, ignore) {
|
|
11544
11741
|
const ignorePatterns = ignore.map((p) => `**/${p}/**`);
|
|
@@ -11554,19 +11751,19 @@ class NodeFileSystem {
|
|
|
11554
11751
|
return [...new Set(files)];
|
|
11555
11752
|
}
|
|
11556
11753
|
join(...segments) {
|
|
11557
|
-
return
|
|
11754
|
+
return path25.join(...segments);
|
|
11558
11755
|
}
|
|
11559
11756
|
relative(from, to) {
|
|
11560
|
-
return
|
|
11757
|
+
return path25.relative(from, to);
|
|
11561
11758
|
}
|
|
11562
11759
|
resolve(...segments) {
|
|
11563
|
-
return
|
|
11760
|
+
return path25.resolve(...segments);
|
|
11564
11761
|
}
|
|
11565
11762
|
dirname(filepath) {
|
|
11566
|
-
return
|
|
11763
|
+
return path25.dirname(filepath);
|
|
11567
11764
|
}
|
|
11568
11765
|
extname(filepath) {
|
|
11569
|
-
return
|
|
11766
|
+
return path25.extname(filepath);
|
|
11570
11767
|
}
|
|
11571
11768
|
}
|
|
11572
11769
|
var nodeFileSystem = new NodeFileSystem;
|
|
@@ -11576,7 +11773,7 @@ async function search(rootDir, query, options = {}) {
|
|
|
11576
11773
|
return hybridResults.results;
|
|
11577
11774
|
}
|
|
11578
11775
|
async function hybridSearch(rootDir, query, options = {}) {
|
|
11579
|
-
rootDir =
|
|
11776
|
+
rootDir = path26.resolve(rootDir);
|
|
11580
11777
|
const ensureFresh = options.ensureFresh ?? DEFAULT_SEARCH_OPTIONS.ensureFresh;
|
|
11581
11778
|
if (ensureFresh) {
|
|
11582
11779
|
await ensureIndexFresh(rootDir, { quiet: true });
|
|
@@ -11653,15 +11850,15 @@ async function hybridSearch(rootDir, query, options = {}) {
|
|
|
11653
11850
|
};
|
|
11654
11851
|
}
|
|
11655
11852
|
async function performExactSearch(rootDir, literal, config, options) {
|
|
11656
|
-
const
|
|
11657
|
-
return executeExactSearch(
|
|
11853
|
+
const fs12 = new NodeFileSystem;
|
|
11854
|
+
return executeExactSearch(fs12, {
|
|
11658
11855
|
rootDir,
|
|
11659
11856
|
literal,
|
|
11660
11857
|
pathFilter: options.pathFilter,
|
|
11661
11858
|
maxFiles: 20,
|
|
11662
11859
|
maxOccurrencesPerFile: 5,
|
|
11663
11860
|
caseInsensitive: false
|
|
11664
|
-
}, (
|
|
11861
|
+
}, (path27, pattern) => minimatch(path27, pattern, { matchBase: true }));
|
|
11665
11862
|
}
|
|
11666
11863
|
function createSearchContext(rootDir, moduleId, config) {
|
|
11667
11864
|
const indexPath = getModuleIndexPath(rootDir, moduleId, config);
|
|
@@ -11670,9 +11867,9 @@ function createSearchContext(rootDir, moduleId, config) {
|
|
|
11670
11867
|
config,
|
|
11671
11868
|
loadFileIndex: async (filepath) => {
|
|
11672
11869
|
const hasExtension = /\.[^./]+$/.test(filepath);
|
|
11673
|
-
const indexFilePath = hasExtension ?
|
|
11870
|
+
const indexFilePath = hasExtension ? path26.join(indexPath, filepath.replace(/\.[^.]+$/, ".json")) : path26.join(indexPath, filepath + ".json");
|
|
11674
11871
|
try {
|
|
11675
|
-
const content = await
|
|
11872
|
+
const content = await fs11.readFile(indexFilePath, "utf-8");
|
|
11676
11873
|
return JSON.parse(content);
|
|
11677
11874
|
} catch {
|
|
11678
11875
|
return null;
|
|
@@ -11682,7 +11879,7 @@ function createSearchContext(rootDir, moduleId, config) {
|
|
|
11682
11879
|
const files = [];
|
|
11683
11880
|
await traverseDirectory(indexPath, files, indexPath);
|
|
11684
11881
|
return files.filter((f) => f.endsWith(".json") && !f.endsWith("manifest.json")).map((f) => {
|
|
11685
|
-
const relative5 =
|
|
11882
|
+
const relative5 = path26.relative(indexPath, f);
|
|
11686
11883
|
return relative5.replace(/\.json$/, "");
|
|
11687
11884
|
});
|
|
11688
11885
|
}
|
|
@@ -11690,9 +11887,9 @@ function createSearchContext(rootDir, moduleId, config) {
|
|
|
11690
11887
|
}
|
|
11691
11888
|
async function traverseDirectory(dir, files, basePath) {
|
|
11692
11889
|
try {
|
|
11693
|
-
const entries = await
|
|
11890
|
+
const entries = await fs11.readdir(dir, { withFileTypes: true });
|
|
11694
11891
|
for (const entry of entries) {
|
|
11695
|
-
const fullPath =
|
|
11892
|
+
const fullPath = path26.join(dir, entry.name);
|
|
11696
11893
|
if (entry.isDirectory()) {
|
|
11697
11894
|
await traverseDirectory(fullPath, files, basePath);
|
|
11698
11895
|
} else if (entry.isFile()) {
|
|
@@ -11704,7 +11901,7 @@ async function traverseDirectory(dir, files, basePath) {
|
|
|
11704
11901
|
async function loadGlobalManifest2(rootDir, config) {
|
|
11705
11902
|
const manifestPath = getGlobalManifestPath(rootDir, config);
|
|
11706
11903
|
try {
|
|
11707
|
-
const content = await
|
|
11904
|
+
const content = await fs11.readFile(manifestPath, "utf-8");
|
|
11708
11905
|
return JSON.parse(content);
|
|
11709
11906
|
} catch {
|
|
11710
11907
|
return null;
|
|
@@ -11873,4 +12070,4 @@ export {
|
|
|
11873
12070
|
ConsoleLogger
|
|
11874
12071
|
};
|
|
11875
12072
|
|
|
11876
|
-
//# debugId=
|
|
12073
|
+
//# debugId=5C9139B3A95BD0AB64756E2164756E21
|