raggrep 0.16.0 → 0.17.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +29 -5
- package/dist/cli/main.js +723 -460
- package/dist/cli/main.js.map +19 -13
- package/dist/domain/ports/embedding.d.ts +10 -0
- package/dist/domain/ports/index.d.ts +1 -1
- package/dist/index.js +529 -328
- package/dist/index.js.map +18 -12
- package/dist/infrastructure/config/configLoader.d.ts +9 -11
- package/dist/infrastructure/config/index.d.ts +1 -1
- package/dist/infrastructure/embeddings/embeddingPaths.d.ts +6 -0
- package/dist/infrastructure/embeddings/embeddingProviderFactory.d.ts +14 -0
- package/dist/infrastructure/embeddings/globalEmbeddings.d.ts +28 -0
- package/dist/infrastructure/embeddings/huggingfaceEmbeddingProvider.d.ts +21 -0
- package/dist/infrastructure/embeddings/index.d.ts +12 -2
- package/dist/infrastructure/embeddings/modelCache.d.ts +10 -0
- package/dist/infrastructure/embeddings/modelCatalog.d.ts +23 -0
- package/dist/infrastructure/embeddings/xenovaEmbeddingProvider.d.ts +23 -0
- package/dist/infrastructure/index.d.ts +1 -1
- package/package.json +12 -4
- package/dist/infrastructure/embeddings/transformersEmbedding.d.ts +0 -52
package/dist/index.js
CHANGED
|
@@ -1,27 +1,16 @@
|
|
|
1
1
|
import { createRequire } from "node:module";
|
|
2
|
-
var __create = Object.create;
|
|
3
|
-
var __getProtoOf = Object.getPrototypeOf;
|
|
4
2
|
var __defProp = Object.defineProperty;
|
|
5
|
-
var
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
const to = isNodeMode || !mod || !mod.__esModule ? __defProp(target, "default", { value: mod, enumerable: true }) : target;
|
|
10
|
-
for (let key of __getOwnPropNames(mod))
|
|
11
|
-
if (!__hasOwnProp.call(to, key))
|
|
12
|
-
__defProp(to, key, {
|
|
13
|
-
get: () => mod[key],
|
|
14
|
-
enumerable: true
|
|
15
|
-
});
|
|
16
|
-
return to;
|
|
17
|
-
};
|
|
3
|
+
var __returnValue = (v) => v;
|
|
4
|
+
function __exportSetter(name, newValue) {
|
|
5
|
+
this[name] = __returnValue.bind(null, newValue);
|
|
6
|
+
}
|
|
18
7
|
var __export = (target, all) => {
|
|
19
8
|
for (var name in all)
|
|
20
9
|
__defProp(target, name, {
|
|
21
10
|
get: all[name],
|
|
22
11
|
enumerable: true,
|
|
23
12
|
configurable: true,
|
|
24
|
-
set: (
|
|
13
|
+
set: __exportSetter.bind(all, name)
|
|
25
14
|
});
|
|
26
15
|
};
|
|
27
16
|
var __esm = (fn, res) => () => (fn && (res = fn(fn = 0)), res);
|
|
@@ -54,49 +43,37 @@ function createDefaultConfig() {
|
|
|
54
43
|
{
|
|
55
44
|
id: "language/typescript",
|
|
56
45
|
enabled: true,
|
|
57
|
-
options: {
|
|
58
|
-
embeddingModel: "all-MiniLM-L6-v2"
|
|
59
|
-
}
|
|
46
|
+
options: { ...DEFAULT_EMBEDDING_MODULE_OPTIONS }
|
|
60
47
|
},
|
|
61
48
|
{
|
|
62
49
|
id: "language/python",
|
|
63
50
|
enabled: true,
|
|
64
|
-
options: {
|
|
65
|
-
embeddingModel: "all-MiniLM-L6-v2"
|
|
66
|
-
}
|
|
51
|
+
options: { ...DEFAULT_EMBEDDING_MODULE_OPTIONS }
|
|
67
52
|
},
|
|
68
53
|
{
|
|
69
54
|
id: "language/go",
|
|
70
55
|
enabled: true,
|
|
71
|
-
options: {
|
|
72
|
-
embeddingModel: "all-MiniLM-L6-v2"
|
|
73
|
-
}
|
|
56
|
+
options: { ...DEFAULT_EMBEDDING_MODULE_OPTIONS }
|
|
74
57
|
},
|
|
75
58
|
{
|
|
76
59
|
id: "language/rust",
|
|
77
60
|
enabled: true,
|
|
78
|
-
options: {
|
|
79
|
-
embeddingModel: "all-MiniLM-L6-v2"
|
|
80
|
-
}
|
|
61
|
+
options: { ...DEFAULT_EMBEDDING_MODULE_OPTIONS }
|
|
81
62
|
},
|
|
82
63
|
{
|
|
83
64
|
id: "data/json",
|
|
84
65
|
enabled: true,
|
|
85
|
-
options: {
|
|
86
|
-
embeddingModel: "all-MiniLM-L6-v2"
|
|
87
|
-
}
|
|
66
|
+
options: { ...DEFAULT_EMBEDDING_MODULE_OPTIONS }
|
|
88
67
|
},
|
|
89
68
|
{
|
|
90
69
|
id: "docs/markdown",
|
|
91
70
|
enabled: true,
|
|
92
|
-
options: {
|
|
93
|
-
embeddingModel: "all-MiniLM-L6-v2"
|
|
94
|
-
}
|
|
71
|
+
options: { ...DEFAULT_EMBEDDING_MODULE_OPTIONS }
|
|
95
72
|
}
|
|
96
73
|
]
|
|
97
74
|
};
|
|
98
75
|
}
|
|
99
|
-
var DEFAULT_IGNORE_PATHS, DEFAULT_EXTENSIONS;
|
|
76
|
+
var DEFAULT_IGNORE_PATHS, DEFAULT_EXTENSIONS, DEFAULT_EMBEDDING_MODULE_OPTIONS;
|
|
100
77
|
var init_config = __esm(() => {
|
|
101
78
|
DEFAULT_IGNORE_PATHS = [
|
|
102
79
|
"node_modules",
|
|
@@ -166,6 +143,10 @@ var init_config = __esm(() => {
|
|
|
166
143
|
".sql",
|
|
167
144
|
".txt"
|
|
168
145
|
];
|
|
146
|
+
DEFAULT_EMBEDDING_MODULE_OPTIONS = {
|
|
147
|
+
embeddingModel: "bge-small-en-v1.5",
|
|
148
|
+
embeddingRuntime: "huggingface"
|
|
149
|
+
};
|
|
169
150
|
});
|
|
170
151
|
|
|
171
152
|
// src/domain/entities/literal.ts
|
|
@@ -195,24 +176,48 @@ var init_entities = __esm(() => {
|
|
|
195
176
|
init_lexicon();
|
|
196
177
|
});
|
|
197
178
|
|
|
179
|
+
// src/infrastructure/embeddings/modelCatalog.ts
|
|
180
|
+
function getEmbeddingModelId(model) {
|
|
181
|
+
return EMBEDDING_MODEL_IDS[model];
|
|
182
|
+
}
|
|
183
|
+
function getEmbeddingDimension(model) {
|
|
184
|
+
return EMBEDDING_DIMENSIONS[model];
|
|
185
|
+
}
|
|
186
|
+
var EMBEDDING_MODEL_IDS, EMBEDDING_MODELS, EMBEDDING_DIMENSIONS;
|
|
187
|
+
var init_modelCatalog = __esm(() => {
|
|
188
|
+
EMBEDDING_MODEL_IDS = {
|
|
189
|
+
"all-MiniLM-L6-v2": "Xenova/all-MiniLM-L6-v2",
|
|
190
|
+
"all-MiniLM-L12-v2": "Xenova/all-MiniLM-L12-v2",
|
|
191
|
+
"bge-small-en-v1.5": "Xenova/bge-small-en-v1.5",
|
|
192
|
+
"paraphrase-MiniLM-L3-v2": "Xenova/paraphrase-MiniLM-L3-v2",
|
|
193
|
+
"nomic-embed-text-v1.5": "nomic-ai/nomic-embed-text-v1.5"
|
|
194
|
+
};
|
|
195
|
+
EMBEDDING_MODELS = EMBEDDING_MODEL_IDS;
|
|
196
|
+
EMBEDDING_DIMENSIONS = {
|
|
197
|
+
"all-MiniLM-L6-v2": 384,
|
|
198
|
+
"all-MiniLM-L12-v2": 384,
|
|
199
|
+
"bge-small-en-v1.5": 384,
|
|
200
|
+
"paraphrase-MiniLM-L3-v2": 384,
|
|
201
|
+
"nomic-embed-text-v1.5": 768
|
|
202
|
+
};
|
|
203
|
+
});
|
|
204
|
+
|
|
198
205
|
// src/infrastructure/config/configLoader.ts
|
|
199
206
|
import * as path from "path";
|
|
200
207
|
import * as fs from "fs/promises";
|
|
201
|
-
import * as os from "os";
|
|
202
208
|
import * as crypto from "crypto";
|
|
203
209
|
function hashPath(inputPath) {
|
|
204
210
|
return crypto.createHash("sha256").update(inputPath).digest("hex").slice(0, 12);
|
|
205
211
|
}
|
|
206
212
|
function getRaggrepDir(rootDir, _config = DEFAULT_CONFIG) {
|
|
207
213
|
const absoluteRoot = path.resolve(rootDir);
|
|
208
|
-
|
|
209
|
-
return path.join(RAGGREP_TEMP_BASE, projectHash);
|
|
214
|
+
return path.join(absoluteRoot, RAGGREP_INDEX_DIR);
|
|
210
215
|
}
|
|
211
216
|
function getIndexLocation(rootDir) {
|
|
212
217
|
const absoluteRoot = path.resolve(rootDir);
|
|
213
218
|
const projectHash = hashPath(absoluteRoot);
|
|
214
219
|
return {
|
|
215
|
-
indexDir: path.join(
|
|
220
|
+
indexDir: path.join(absoluteRoot, RAGGREP_INDEX_DIR),
|
|
216
221
|
projectRoot: absoluteRoot,
|
|
217
222
|
projectHash
|
|
218
223
|
};
|
|
@@ -253,23 +258,24 @@ function getEmbeddingConfigFromModule(moduleConfig) {
|
|
|
253
258
|
console.warn(`Unknown embedding model: ${modelName}, falling back to bge-small-en-v1.5`);
|
|
254
259
|
return { model: "bge-small-en-v1.5" };
|
|
255
260
|
}
|
|
261
|
+
const rt = options.embeddingRuntime;
|
|
262
|
+
let runtime;
|
|
263
|
+
if (rt === "xenova" || rt === "huggingface") {
|
|
264
|
+
runtime = rt;
|
|
265
|
+
} else if (rt !== undefined) {
|
|
266
|
+
console.warn(`Unknown embeddingRuntime: ${rt}, falling back to default (huggingface)`);
|
|
267
|
+
}
|
|
256
268
|
return {
|
|
257
269
|
model: modelName,
|
|
270
|
+
...runtime ? { runtime } : {},
|
|
258
271
|
showProgress: options.showProgress === true
|
|
259
272
|
};
|
|
260
273
|
}
|
|
261
|
-
var DEFAULT_CONFIG,
|
|
274
|
+
var DEFAULT_CONFIG, RAGGREP_INDEX_DIR = ".raggrep";
|
|
262
275
|
var init_configLoader = __esm(() => {
|
|
263
276
|
init_entities();
|
|
277
|
+
init_modelCatalog();
|
|
264
278
|
DEFAULT_CONFIG = createDefaultConfig();
|
|
265
|
-
RAGGREP_TEMP_BASE = path.join(os.tmpdir(), "raggrep-indexes");
|
|
266
|
-
EMBEDDING_MODELS = {
|
|
267
|
-
"all-MiniLM-L6-v2": "Xenova/all-MiniLM-L6-v2",
|
|
268
|
-
"all-MiniLM-L12-v2": "Xenova/all-MiniLM-L12-v2",
|
|
269
|
-
"bge-small-en-v1.5": "Xenova/bge-small-en-v1.5",
|
|
270
|
-
"paraphrase-MiniLM-L3-v2": "Xenova/paraphrase-MiniLM-L3-v2",
|
|
271
|
-
"nomic-embed-text-v1.5": "nomic-ai/nomic-embed-text-v1.5"
|
|
272
|
-
};
|
|
273
279
|
});
|
|
274
280
|
|
|
275
281
|
// src/infrastructure/config/index.ts
|
|
@@ -2137,22 +2143,47 @@ var init_core = __esm(() => {
|
|
|
2137
2143
|
init_symbols();
|
|
2138
2144
|
});
|
|
2139
2145
|
|
|
2140
|
-
// src/infrastructure/embeddings/
|
|
2146
|
+
// src/infrastructure/embeddings/embeddingPaths.ts
|
|
2147
|
+
import * as os from "os";
|
|
2148
|
+
import * as path6 from "path";
|
|
2149
|
+
var RAGGREP_MODEL_CACHE_DIR;
|
|
2150
|
+
var init_embeddingPaths = __esm(() => {
|
|
2151
|
+
RAGGREP_MODEL_CACHE_DIR = path6.join(os.homedir(), ".cache", "raggrep", "models");
|
|
2152
|
+
});
|
|
2153
|
+
|
|
2154
|
+
// src/infrastructure/embeddings/modelCache.ts
|
|
2155
|
+
import * as fs3 from "fs/promises";
|
|
2156
|
+
import * as path7 from "path";
|
|
2157
|
+
async function isEmbeddingModelCached(model) {
|
|
2158
|
+
const modelId = getEmbeddingModelId(model);
|
|
2159
|
+
const onnxPath = path7.join(RAGGREP_MODEL_CACHE_DIR, modelId, "onnx", "model_quantized.onnx");
|
|
2160
|
+
try {
|
|
2161
|
+
await fs3.access(onnxPath);
|
|
2162
|
+
return true;
|
|
2163
|
+
} catch {
|
|
2164
|
+
return false;
|
|
2165
|
+
}
|
|
2166
|
+
}
|
|
2167
|
+
var init_modelCache = __esm(() => {
|
|
2168
|
+
init_embeddingPaths();
|
|
2169
|
+
init_modelCatalog();
|
|
2170
|
+
});
|
|
2171
|
+
|
|
2172
|
+
// src/infrastructure/embeddings/huggingfaceEmbeddingProvider.ts
|
|
2141
2173
|
import {
|
|
2142
2174
|
pipeline,
|
|
2143
2175
|
env
|
|
2144
|
-
} from "@
|
|
2145
|
-
import * as path6 from "path";
|
|
2146
|
-
import * as os2 from "os";
|
|
2176
|
+
} from "@huggingface/transformers";
|
|
2147
2177
|
|
|
2148
|
-
class
|
|
2149
|
-
|
|
2178
|
+
class HuggingFaceTransformersEmbeddingProvider {
|
|
2179
|
+
extractor = null;
|
|
2150
2180
|
config;
|
|
2151
2181
|
isInitializing = false;
|
|
2152
2182
|
initPromise = null;
|
|
2153
2183
|
constructor(config) {
|
|
2154
2184
|
this.config = {
|
|
2155
2185
|
model: config?.model ?? "bge-small-en-v1.5",
|
|
2186
|
+
runtime: config?.runtime ?? "huggingface",
|
|
2156
2187
|
showProgress: config?.showProgress ?? false,
|
|
2157
2188
|
logger: config?.logger
|
|
2158
2189
|
};
|
|
@@ -2160,14 +2191,14 @@ class TransformersEmbeddingProvider {
|
|
|
2160
2191
|
async initialize(config) {
|
|
2161
2192
|
if (config) {
|
|
2162
2193
|
if (config.model !== this.config.model) {
|
|
2163
|
-
this.
|
|
2194
|
+
this.extractor = null;
|
|
2164
2195
|
}
|
|
2165
2196
|
this.config = { ...this.config, ...config };
|
|
2166
2197
|
}
|
|
2167
|
-
await this.
|
|
2198
|
+
await this.ensureExtractor();
|
|
2168
2199
|
}
|
|
2169
|
-
async
|
|
2170
|
-
if (this.
|
|
2200
|
+
async ensureExtractor() {
|
|
2201
|
+
if (this.extractor) {
|
|
2171
2202
|
return;
|
|
2172
2203
|
}
|
|
2173
2204
|
if (this.isInitializing && this.initPromise) {
|
|
@@ -2175,14 +2206,14 @@ class TransformersEmbeddingProvider {
|
|
|
2175
2206
|
}
|
|
2176
2207
|
this.isInitializing = true;
|
|
2177
2208
|
this.initPromise = (async () => {
|
|
2178
|
-
const modelId =
|
|
2209
|
+
const modelId = getEmbeddingModelId(this.config.model);
|
|
2179
2210
|
const logger = this.config.logger;
|
|
2180
2211
|
const showProgress = this.config.showProgress || !!logger;
|
|
2181
|
-
const
|
|
2212
|
+
const cached = await isEmbeddingModelCached(this.config.model);
|
|
2182
2213
|
let hasDownloads = false;
|
|
2183
2214
|
try {
|
|
2184
|
-
this.
|
|
2185
|
-
progress_callback: showProgress && !
|
|
2215
|
+
this.extractor = await pipeline("feature-extraction", modelId, {
|
|
2216
|
+
progress_callback: showProgress && !cached ? (progress) => {
|
|
2186
2217
|
if (progress.status === "progress" && progress.file) {
|
|
2187
2218
|
if (!hasDownloads) {
|
|
2188
2219
|
hasDownloads = true;
|
|
@@ -2191,7 +2222,7 @@ class TransformersEmbeddingProvider {
|
|
|
2191
2222
|
} else {
|
|
2192
2223
|
console.log(`
|
|
2193
2224
|
Loading embedding model: ${this.config.model}`);
|
|
2194
|
-
console.log(` Cache: ${
|
|
2225
|
+
console.log(` Cache: ${RAGGREP_MODEL_CACHE_DIR}`);
|
|
2195
2226
|
}
|
|
2196
2227
|
}
|
|
2197
2228
|
const pct = progress.progress ? Math.round(progress.progress) : 0;
|
|
@@ -2221,9 +2252,9 @@ class TransformersEmbeddingProvider {
|
|
|
2221
2252
|
}
|
|
2222
2253
|
}
|
|
2223
2254
|
} catch (error) {
|
|
2224
|
-
this.
|
|
2225
|
-
if (logger) {
|
|
2226
|
-
logger.clearProgress();
|
|
2255
|
+
this.extractor = null;
|
|
2256
|
+
if (this.config.logger) {
|
|
2257
|
+
this.config.logger.clearProgress();
|
|
2227
2258
|
}
|
|
2228
2259
|
throw new Error(`Failed to load embedding model: ${error}`);
|
|
2229
2260
|
} finally {
|
|
@@ -2234,11 +2265,11 @@ class TransformersEmbeddingProvider {
|
|
|
2234
2265
|
return this.initPromise;
|
|
2235
2266
|
}
|
|
2236
2267
|
async getEmbedding(text) {
|
|
2237
|
-
await this.
|
|
2238
|
-
if (!this.
|
|
2268
|
+
await this.ensureExtractor();
|
|
2269
|
+
if (!this.extractor) {
|
|
2239
2270
|
throw new Error("Embedding pipeline not initialized");
|
|
2240
2271
|
}
|
|
2241
|
-
const output = await this.
|
|
2272
|
+
const output = await this.extractor(text, {
|
|
2242
2273
|
pooling: "mean",
|
|
2243
2274
|
normalize: true
|
|
2244
2275
|
});
|
|
@@ -2247,15 +2278,15 @@ class TransformersEmbeddingProvider {
|
|
|
2247
2278
|
async getEmbeddings(texts) {
|
|
2248
2279
|
if (texts.length === 0)
|
|
2249
2280
|
return [];
|
|
2250
|
-
await this.
|
|
2251
|
-
if (!this.
|
|
2281
|
+
await this.ensureExtractor();
|
|
2282
|
+
if (!this.extractor) {
|
|
2252
2283
|
throw new Error("Embedding pipeline not initialized");
|
|
2253
2284
|
}
|
|
2254
2285
|
const results = [];
|
|
2255
2286
|
for (let i = 0;i < texts.length; i += BATCH_SIZE) {
|
|
2256
2287
|
const batch = texts.slice(i, i + BATCH_SIZE);
|
|
2257
2288
|
const outputs = await Promise.all(batch.map(async (text) => {
|
|
2258
|
-
const output = await this.
|
|
2289
|
+
const output = await this.extractor(text, {
|
|
2259
2290
|
pooling: "mean",
|
|
2260
2291
|
normalize: true
|
|
2261
2292
|
});
|
|
@@ -2266,41 +2297,216 @@ class TransformersEmbeddingProvider {
|
|
|
2266
2297
|
return results;
|
|
2267
2298
|
}
|
|
2268
2299
|
getDimension() {
|
|
2269
|
-
return
|
|
2300
|
+
return getEmbeddingDimension(this.config.model);
|
|
2270
2301
|
}
|
|
2271
2302
|
getModelName() {
|
|
2272
2303
|
return this.config.model;
|
|
2273
2304
|
}
|
|
2274
2305
|
async dispose() {
|
|
2275
|
-
this.
|
|
2306
|
+
this.extractor = null;
|
|
2276
2307
|
}
|
|
2277
2308
|
}
|
|
2278
|
-
|
|
2279
|
-
|
|
2280
|
-
|
|
2281
|
-
|
|
2282
|
-
|
|
2283
|
-
|
|
2284
|
-
|
|
2285
|
-
|
|
2286
|
-
|
|
2287
|
-
|
|
2309
|
+
var BATCH_SIZE = 32;
|
|
2310
|
+
var init_huggingfaceEmbeddingProvider = __esm(() => {
|
|
2311
|
+
init_embeddingPaths();
|
|
2312
|
+
init_modelCatalog();
|
|
2313
|
+
init_modelCache();
|
|
2314
|
+
env.cacheDir = RAGGREP_MODEL_CACHE_DIR;
|
|
2315
|
+
env.allowLocalModels = true;
|
|
2316
|
+
});
|
|
2317
|
+
|
|
2318
|
+
// src/infrastructure/embeddings/xenovaEmbeddingProvider.ts
|
|
2319
|
+
var exports_xenovaEmbeddingProvider = {};
|
|
2320
|
+
__export(exports_xenovaEmbeddingProvider, {
|
|
2321
|
+
XenovaTransformersEmbeddingProvider: () => XenovaTransformersEmbeddingProvider,
|
|
2322
|
+
TransformersEmbeddingProvider: () => TransformersEmbeddingProvider
|
|
2323
|
+
});
|
|
2324
|
+
import {
|
|
2325
|
+
pipeline as pipeline2,
|
|
2326
|
+
env as env2
|
|
2327
|
+
} from "@xenova/transformers";
|
|
2328
|
+
|
|
2329
|
+
class XenovaTransformersEmbeddingProvider {
|
|
2330
|
+
extractor = null;
|
|
2331
|
+
config;
|
|
2332
|
+
isInitializing = false;
|
|
2333
|
+
initPromise = null;
|
|
2334
|
+
constructor(config) {
|
|
2335
|
+
this.config = {
|
|
2336
|
+
model: config?.model ?? "bge-small-en-v1.5",
|
|
2337
|
+
runtime: config?.runtime ?? "xenova",
|
|
2338
|
+
showProgress: config?.showProgress ?? false,
|
|
2339
|
+
logger: config?.logger
|
|
2340
|
+
};
|
|
2341
|
+
}
|
|
2342
|
+
async initialize(config) {
|
|
2343
|
+
if (config) {
|
|
2344
|
+
if (config.model !== this.config.model) {
|
|
2345
|
+
this.extractor = null;
|
|
2346
|
+
}
|
|
2347
|
+
this.config = { ...this.config, ...config };
|
|
2348
|
+
}
|
|
2349
|
+
await this.ensureExtractor();
|
|
2350
|
+
}
|
|
2351
|
+
async ensureExtractor() {
|
|
2352
|
+
if (this.extractor) {
|
|
2353
|
+
return;
|
|
2354
|
+
}
|
|
2355
|
+
if (this.isInitializing && this.initPromise) {
|
|
2356
|
+
return this.initPromise;
|
|
2357
|
+
}
|
|
2358
|
+
this.isInitializing = true;
|
|
2359
|
+
this.initPromise = (async () => {
|
|
2360
|
+
const modelId = getEmbeddingModelId(this.config.model);
|
|
2361
|
+
const logger = this.config.logger;
|
|
2362
|
+
const showProgress = this.config.showProgress || !!logger;
|
|
2363
|
+
const cached = await isEmbeddingModelCached(this.config.model);
|
|
2364
|
+
let hasDownloads = false;
|
|
2365
|
+
try {
|
|
2366
|
+
this.extractor = await pipeline2("feature-extraction", modelId, {
|
|
2367
|
+
progress_callback: showProgress && !cached ? (progress) => {
|
|
2368
|
+
if (progress.status === "progress" && progress.file) {
|
|
2369
|
+
if (!hasDownloads) {
|
|
2370
|
+
hasDownloads = true;
|
|
2371
|
+
if (logger) {
|
|
2372
|
+
logger.info(`Downloading embedding model: ${this.config.model}`);
|
|
2373
|
+
} else {
|
|
2374
|
+
console.log(`
|
|
2375
|
+
Loading embedding model: ${this.config.model}`);
|
|
2376
|
+
console.log(` Cache: ${RAGGREP_MODEL_CACHE_DIR}`);
|
|
2377
|
+
}
|
|
2378
|
+
}
|
|
2379
|
+
const pct = progress.progress ? Math.round(progress.progress) : 0;
|
|
2380
|
+
if (logger) {
|
|
2381
|
+
logger.progress(` Downloading ${progress.file}: ${pct}%`);
|
|
2382
|
+
} else {
|
|
2383
|
+
process.stdout.write(`\r Downloading ${progress.file}: ${pct}% `);
|
|
2384
|
+
}
|
|
2385
|
+
} else if (progress.status === "done" && progress.file) {
|
|
2386
|
+
if (logger) {
|
|
2387
|
+
logger.clearProgress();
|
|
2388
|
+
logger.info(` Downloaded ${progress.file}`);
|
|
2389
|
+
} else if (hasDownloads) {
|
|
2390
|
+
process.stdout.write(`\r Downloaded ${progress.file}
|
|
2391
|
+
`);
|
|
2392
|
+
}
|
|
2393
|
+
}
|
|
2394
|
+
} : undefined
|
|
2395
|
+
});
|
|
2396
|
+
if (hasDownloads) {
|
|
2397
|
+
if (logger) {
|
|
2398
|
+
logger.clearProgress();
|
|
2399
|
+
logger.info(`Model ready: ${this.config.model}`);
|
|
2400
|
+
} else {
|
|
2401
|
+
console.log(` Model ready.
|
|
2402
|
+
`);
|
|
2403
|
+
}
|
|
2404
|
+
}
|
|
2405
|
+
} catch (error) {
|
|
2406
|
+
this.extractor = null;
|
|
2407
|
+
if (this.config.logger) {
|
|
2408
|
+
this.config.logger.clearProgress();
|
|
2409
|
+
}
|
|
2410
|
+
throw new Error(`Failed to load embedding model: ${error}`);
|
|
2411
|
+
} finally {
|
|
2412
|
+
this.isInitializing = false;
|
|
2413
|
+
this.initPromise = null;
|
|
2414
|
+
}
|
|
2415
|
+
})();
|
|
2416
|
+
return this.initPromise;
|
|
2417
|
+
}
|
|
2418
|
+
async getEmbedding(text) {
|
|
2419
|
+
await this.ensureExtractor();
|
|
2420
|
+
if (!this.extractor) {
|
|
2421
|
+
throw new Error("Embedding pipeline not initialized");
|
|
2422
|
+
}
|
|
2423
|
+
const output = await this.extractor(text, {
|
|
2424
|
+
pooling: "mean",
|
|
2425
|
+
normalize: true
|
|
2426
|
+
});
|
|
2427
|
+
return Array.from(output.data);
|
|
2428
|
+
}
|
|
2429
|
+
async getEmbeddings(texts) {
|
|
2430
|
+
if (texts.length === 0)
|
|
2431
|
+
return [];
|
|
2432
|
+
await this.ensureExtractor();
|
|
2433
|
+
if (!this.extractor) {
|
|
2434
|
+
throw new Error("Embedding pipeline not initialized");
|
|
2435
|
+
}
|
|
2436
|
+
const results = [];
|
|
2437
|
+
for (let i = 0;i < texts.length; i += BATCH_SIZE2) {
|
|
2438
|
+
const batch = texts.slice(i, i + BATCH_SIZE2);
|
|
2439
|
+
const outputs = await Promise.all(batch.map(async (text) => {
|
|
2440
|
+
const output = await this.extractor(text, {
|
|
2441
|
+
pooling: "mean",
|
|
2442
|
+
normalize: true
|
|
2443
|
+
});
|
|
2444
|
+
return Array.from(output.data);
|
|
2445
|
+
}));
|
|
2446
|
+
results.push(...outputs);
|
|
2447
|
+
}
|
|
2448
|
+
return results;
|
|
2449
|
+
}
|
|
2450
|
+
getDimension() {
|
|
2451
|
+
return getEmbeddingDimension(this.config.model);
|
|
2452
|
+
}
|
|
2453
|
+
getModelName() {
|
|
2454
|
+
return this.config.model;
|
|
2455
|
+
}
|
|
2456
|
+
async dispose() {
|
|
2457
|
+
this.extractor = null;
|
|
2288
2458
|
}
|
|
2289
2459
|
}
|
|
2460
|
+
var BATCH_SIZE2 = 32, TransformersEmbeddingProvider;
|
|
2461
|
+
var init_xenovaEmbeddingProvider = __esm(() => {
|
|
2462
|
+
init_embeddingPaths();
|
|
2463
|
+
init_modelCatalog();
|
|
2464
|
+
init_modelCache();
|
|
2465
|
+
env2.cacheDir = RAGGREP_MODEL_CACHE_DIR;
|
|
2466
|
+
env2.allowLocalModels = true;
|
|
2467
|
+
TransformersEmbeddingProvider = XenovaTransformersEmbeddingProvider;
|
|
2468
|
+
});
|
|
2469
|
+
|
|
2470
|
+
// src/infrastructure/embeddings/embeddingProviderFactory.ts
|
|
2471
|
+
function resolveRuntime(config) {
|
|
2472
|
+
return config.runtime ?? "huggingface";
|
|
2473
|
+
}
|
|
2474
|
+
async function createEmbeddingProvider(config) {
|
|
2475
|
+
const runtime = resolveRuntime(config);
|
|
2476
|
+
if (runtime === "huggingface") {
|
|
2477
|
+
return new HuggingFaceTransformersEmbeddingProvider(config);
|
|
2478
|
+
}
|
|
2479
|
+
const { XenovaTransformersEmbeddingProvider: XenovaTransformersEmbeddingProvider2 } = await Promise.resolve().then(() => (init_xenovaEmbeddingProvider(), exports_xenovaEmbeddingProvider));
|
|
2480
|
+
return new XenovaTransformersEmbeddingProvider2(config);
|
|
2481
|
+
}
|
|
2482
|
+
var init_embeddingProviderFactory = __esm(() => {
|
|
2483
|
+
init_huggingfaceEmbeddingProvider();
|
|
2484
|
+
});
|
|
2485
|
+
|
|
2486
|
+
// src/infrastructure/embeddings/globalEmbeddings.ts
|
|
2290
2487
|
function configureEmbeddings(config) {
|
|
2291
|
-
const
|
|
2292
|
-
|
|
2488
|
+
const merged = {
|
|
2489
|
+
...globalConfig,
|
|
2490
|
+
...config
|
|
2491
|
+
};
|
|
2492
|
+
if (merged.runtime === undefined) {
|
|
2493
|
+
merged.runtime = "huggingface";
|
|
2494
|
+
}
|
|
2495
|
+
const needsReset = merged.model !== globalConfig.model || merged.runtime !== globalConfig.runtime || merged.logger !== globalConfig.logger;
|
|
2496
|
+
if (needsReset) {
|
|
2497
|
+
const prev = globalProvider;
|
|
2293
2498
|
globalProvider = null;
|
|
2499
|
+
prev?.dispose?.();
|
|
2294
2500
|
}
|
|
2295
|
-
globalConfig =
|
|
2501
|
+
globalConfig = merged;
|
|
2296
2502
|
}
|
|
2297
2503
|
function getEmbeddingConfig() {
|
|
2298
2504
|
return { ...globalConfig };
|
|
2299
2505
|
}
|
|
2300
2506
|
async function ensureGlobalProvider() {
|
|
2301
2507
|
if (!globalProvider) {
|
|
2302
|
-
globalProvider =
|
|
2303
|
-
await globalProvider.initialize();
|
|
2508
|
+
globalProvider = await createEmbeddingProvider(globalConfig);
|
|
2509
|
+
await globalProvider.initialize?.(globalConfig);
|
|
2304
2510
|
}
|
|
2305
2511
|
return globalProvider;
|
|
2306
2512
|
}
|
|
@@ -2312,27 +2518,14 @@ async function getEmbeddings(texts) {
|
|
|
2312
2518
|
const provider = await ensureGlobalProvider();
|
|
2313
2519
|
return provider.getEmbeddings(texts);
|
|
2314
2520
|
}
|
|
2315
|
-
var
|
|
2316
|
-
var
|
|
2317
|
-
|
|
2318
|
-
|
|
2319
|
-
|
|
2320
|
-
EMBEDDING_MODELS2 = {
|
|
2321
|
-
"all-MiniLM-L6-v2": "Xenova/all-MiniLM-L6-v2",
|
|
2322
|
-
"all-MiniLM-L12-v2": "Xenova/all-MiniLM-L12-v2",
|
|
2323
|
-
"bge-small-en-v1.5": "Xenova/bge-small-en-v1.5",
|
|
2324
|
-
"paraphrase-MiniLM-L3-v2": "Xenova/paraphrase-MiniLM-L3-v2",
|
|
2325
|
-
"nomic-embed-text-v1.5": "nomic-ai/nomic-embed-text-v1.5"
|
|
2326
|
-
};
|
|
2327
|
-
EMBEDDING_DIMENSIONS = {
|
|
2328
|
-
"all-MiniLM-L6-v2": 384,
|
|
2329
|
-
"all-MiniLM-L12-v2": 384,
|
|
2330
|
-
"bge-small-en-v1.5": 384,
|
|
2331
|
-
"paraphrase-MiniLM-L3-v2": 384,
|
|
2332
|
-
"nomic-embed-text-v1.5": 768
|
|
2333
|
-
};
|
|
2521
|
+
var globalProvider = null, globalConfig;
|
|
2522
|
+
var init_globalEmbeddings = __esm(() => {
|
|
2523
|
+
init_embeddingPaths();
|
|
2524
|
+
init_embeddingProviderFactory();
|
|
2525
|
+
init_modelCache();
|
|
2334
2526
|
globalConfig = {
|
|
2335
2527
|
model: "bge-small-en-v1.5",
|
|
2528
|
+
runtime: "huggingface",
|
|
2336
2529
|
showProgress: false,
|
|
2337
2530
|
logger: undefined
|
|
2338
2531
|
};
|
|
@@ -2340,7 +2533,11 @@ var init_transformersEmbedding = __esm(() => {
|
|
|
2340
2533
|
|
|
2341
2534
|
// src/infrastructure/embeddings/index.ts
|
|
2342
2535
|
var init_embeddings = __esm(() => {
|
|
2343
|
-
|
|
2536
|
+
init_modelCatalog();
|
|
2537
|
+
init_embeddingPaths();
|
|
2538
|
+
init_huggingfaceEmbeddingProvider();
|
|
2539
|
+
init_embeddingProviderFactory();
|
|
2540
|
+
init_globalEmbeddings();
|
|
2344
2541
|
});
|
|
2345
2542
|
|
|
2346
2543
|
// src/domain/services/keywords.ts
|
|
@@ -2551,7 +2748,7 @@ function cosineSimilarity(a, b) {
|
|
|
2551
2748
|
}
|
|
2552
2749
|
|
|
2553
2750
|
// src/domain/services/queryIntent.ts
|
|
2554
|
-
import * as
|
|
2751
|
+
import * as path8 from "path";
|
|
2555
2752
|
function detectQueryIntent(queryTerms) {
|
|
2556
2753
|
const hasImplementationTerm = queryTerms.some((term) => IMPLEMENTATION_TERMS.includes(term));
|
|
2557
2754
|
const hasDocumentationTerm = queryTerms.some((term) => DOCUMENTATION_TERMS.includes(term));
|
|
@@ -2567,11 +2764,11 @@ function extractQueryTerms(query) {
|
|
|
2567
2764
|
return query.toLowerCase().split(/\s+/).filter((t) => t.length > 2);
|
|
2568
2765
|
}
|
|
2569
2766
|
function isSourceCodeFile(filepath) {
|
|
2570
|
-
const ext =
|
|
2767
|
+
const ext = path8.extname(filepath).toLowerCase();
|
|
2571
2768
|
return SOURCE_CODE_EXTENSIONS.includes(ext);
|
|
2572
2769
|
}
|
|
2573
2770
|
function isDocFile(filepath) {
|
|
2574
|
-
const ext =
|
|
2771
|
+
const ext = path8.extname(filepath).toLowerCase();
|
|
2575
2772
|
return DOC_EXTENSIONS.includes(ext);
|
|
2576
2773
|
}
|
|
2577
2774
|
function calculateFileTypeBoost(filepath, queryTerms) {
|
|
@@ -3646,8 +3843,8 @@ var init_lexicon2 = __esm(() => {
|
|
|
3646
3843
|
// src/domain/services/jsonPathExtractor.ts
|
|
3647
3844
|
function extractJsonPaths(obj, fileBasename) {
|
|
3648
3845
|
const paths = extractPathsRecursive(obj, fileBasename);
|
|
3649
|
-
return paths.map((
|
|
3650
|
-
value:
|
|
3846
|
+
return paths.map((path9) => ({
|
|
3847
|
+
value: path9,
|
|
3651
3848
|
type: "identifier",
|
|
3652
3849
|
matchType: "definition"
|
|
3653
3850
|
}));
|
|
@@ -3958,7 +4155,7 @@ var init_simpleSearch = __esm(() => {
|
|
|
3958
4155
|
});
|
|
3959
4156
|
|
|
3960
4157
|
// src/domain/services/chunkContext.ts
|
|
3961
|
-
import * as
|
|
4158
|
+
import * as path9 from "path";
|
|
3962
4159
|
function prepareChunkForEmbedding(options) {
|
|
3963
4160
|
const { filepath, content, name, docComment } = options;
|
|
3964
4161
|
const pathContext = parsePathContext(filepath);
|
|
@@ -3967,7 +4164,7 @@ function prepareChunkForEmbedding(options) {
|
|
|
3967
4164
|
if (pathPrefix) {
|
|
3968
4165
|
parts.push(pathPrefix);
|
|
3969
4166
|
}
|
|
3970
|
-
const filename =
|
|
4167
|
+
const filename = path9.basename(filepath);
|
|
3971
4168
|
const filenameWithoutExt = filename.replace(/\.[^.]+$/, "");
|
|
3972
4169
|
if (filenameWithoutExt && filenameWithoutExt.length > MIN_SEGMENT_LENGTH) {
|
|
3973
4170
|
const pathPrefixLower = pathPrefix.toLowerCase();
|
|
@@ -4197,11 +4394,12 @@ var init_parseCode = () => {};
|
|
|
4197
4394
|
// src/infrastructure/storage/fileIndexStorage.ts
|
|
4198
4395
|
var init_fileIndexStorage = __esm(() => {
|
|
4199
4396
|
init_entities();
|
|
4397
|
+
init_config2();
|
|
4200
4398
|
});
|
|
4201
4399
|
|
|
4202
4400
|
// src/infrastructure/storage/symbolicIndex.ts
|
|
4203
|
-
import * as
|
|
4204
|
-
import * as
|
|
4401
|
+
import * as fs4 from "fs/promises";
|
|
4402
|
+
import * as path10 from "path";
|
|
4205
4403
|
|
|
4206
4404
|
class SymbolicIndex {
|
|
4207
4405
|
meta = null;
|
|
@@ -4210,7 +4408,7 @@ class SymbolicIndex {
|
|
|
4210
4408
|
symbolicPath;
|
|
4211
4409
|
moduleId;
|
|
4212
4410
|
constructor(indexDir, moduleId) {
|
|
4213
|
-
this.symbolicPath =
|
|
4411
|
+
this.symbolicPath = path10.join(indexDir, "index", moduleId, "symbolic");
|
|
4214
4412
|
this.moduleId = moduleId;
|
|
4215
4413
|
}
|
|
4216
4414
|
async initialize() {
|
|
@@ -4296,13 +4494,13 @@ class SymbolicIndex {
|
|
|
4296
4494
|
if (this.bm25Index) {
|
|
4297
4495
|
this.meta.bm25Serialized = this.bm25Index.serialize();
|
|
4298
4496
|
}
|
|
4299
|
-
await
|
|
4300
|
-
const metaPath =
|
|
4301
|
-
await
|
|
4497
|
+
await fs4.mkdir(this.symbolicPath, { recursive: true });
|
|
4498
|
+
const metaPath = path10.join(this.symbolicPath, "_meta.json");
|
|
4499
|
+
await fs4.writeFile(metaPath, JSON.stringify(this.meta, null, 2));
|
|
4302
4500
|
for (const [filepath, summary] of this.fileSummaries) {
|
|
4303
4501
|
const summaryPath = this.getFileSummaryPath(filepath);
|
|
4304
|
-
await
|
|
4305
|
-
await
|
|
4502
|
+
await fs4.mkdir(path10.dirname(summaryPath), { recursive: true });
|
|
4503
|
+
await fs4.writeFile(summaryPath, JSON.stringify(summary, null, 2));
|
|
4306
4504
|
}
|
|
4307
4505
|
}
|
|
4308
4506
|
async saveIncremental(filepaths) {
|
|
@@ -4313,21 +4511,21 @@ class SymbolicIndex {
|
|
|
4313
4511
|
if (this.bm25Index) {
|
|
4314
4512
|
this.meta.bm25Serialized = this.bm25Index.serialize();
|
|
4315
4513
|
}
|
|
4316
|
-
await
|
|
4317
|
-
const metaPath =
|
|
4318
|
-
await
|
|
4514
|
+
await fs4.mkdir(this.symbolicPath, { recursive: true });
|
|
4515
|
+
const metaPath = path10.join(this.symbolicPath, "_meta.json");
|
|
4516
|
+
await fs4.writeFile(metaPath, JSON.stringify(this.meta, null, 2));
|
|
4319
4517
|
for (const filepath of filepaths) {
|
|
4320
4518
|
const summary = this.fileSummaries.get(filepath);
|
|
4321
4519
|
if (summary) {
|
|
4322
4520
|
const summaryPath = this.getFileSummaryPath(filepath);
|
|
4323
|
-
await
|
|
4324
|
-
await
|
|
4521
|
+
await fs4.mkdir(path10.dirname(summaryPath), { recursive: true });
|
|
4522
|
+
await fs4.writeFile(summaryPath, JSON.stringify(summary, null, 2));
|
|
4325
4523
|
}
|
|
4326
4524
|
}
|
|
4327
4525
|
}
|
|
4328
4526
|
async load() {
|
|
4329
|
-
const metaPath =
|
|
4330
|
-
const metaContent = await
|
|
4527
|
+
const metaPath = path10.join(this.symbolicPath, "_meta.json");
|
|
4528
|
+
const metaContent = await fs4.readFile(metaPath, "utf-8");
|
|
4331
4529
|
this.meta = JSON.parse(metaContent);
|
|
4332
4530
|
this.fileSummaries.clear();
|
|
4333
4531
|
await this.loadFileSummariesRecursive(this.symbolicPath);
|
|
@@ -4339,14 +4537,14 @@ class SymbolicIndex {
|
|
|
4339
4537
|
}
|
|
4340
4538
|
async loadFileSummariesRecursive(dir) {
|
|
4341
4539
|
try {
|
|
4342
|
-
const entries = await
|
|
4540
|
+
const entries = await fs4.readdir(dir, { withFileTypes: true });
|
|
4343
4541
|
for (const entry of entries) {
|
|
4344
|
-
const fullPath =
|
|
4542
|
+
const fullPath = path10.join(dir, entry.name);
|
|
4345
4543
|
if (entry.isDirectory()) {
|
|
4346
4544
|
await this.loadFileSummariesRecursive(fullPath);
|
|
4347
4545
|
} else if (entry.name.endsWith(".json") && entry.name !== "_meta.json") {
|
|
4348
4546
|
try {
|
|
4349
|
-
const content = await
|
|
4547
|
+
const content = await fs4.readFile(fullPath, "utf-8");
|
|
4350
4548
|
const summary = JSON.parse(content);
|
|
4351
4549
|
if (summary.filepath) {
|
|
4352
4550
|
this.fileSummaries.set(summary.filepath, summary);
|
|
@@ -4358,18 +4556,18 @@ class SymbolicIndex {
|
|
|
4358
4556
|
}
|
|
4359
4557
|
getFileSummaryPath(filepath) {
|
|
4360
4558
|
const jsonPath = filepath.replace(/\.[^.]+$/, ".json");
|
|
4361
|
-
return
|
|
4559
|
+
return path10.join(this.symbolicPath, jsonPath);
|
|
4362
4560
|
}
|
|
4363
4561
|
async deleteFileSummary(filepath) {
|
|
4364
4562
|
try {
|
|
4365
|
-
await
|
|
4563
|
+
await fs4.unlink(this.getFileSummaryPath(filepath));
|
|
4366
4564
|
} catch {}
|
|
4367
4565
|
this.fileSummaries.delete(filepath);
|
|
4368
4566
|
}
|
|
4369
4567
|
async exists() {
|
|
4370
4568
|
try {
|
|
4371
|
-
const metaPath =
|
|
4372
|
-
await
|
|
4569
|
+
const metaPath = path10.join(this.symbolicPath, "_meta.json");
|
|
4570
|
+
await fs4.access(metaPath);
|
|
4373
4571
|
return true;
|
|
4374
4572
|
} catch {
|
|
4375
4573
|
return false;
|
|
@@ -4401,8 +4599,8 @@ __export(exports_literalIndex, {
|
|
|
4401
4599
|
getLiteralIndexPath: () => getLiteralIndexPath,
|
|
4402
4600
|
LiteralIndex: () => LiteralIndex
|
|
4403
4601
|
});
|
|
4404
|
-
import * as
|
|
4405
|
-
import * as
|
|
4602
|
+
import * as fs5 from "fs/promises";
|
|
4603
|
+
import * as path11 from "path";
|
|
4406
4604
|
|
|
4407
4605
|
class LiteralIndex {
|
|
4408
4606
|
indexPath;
|
|
@@ -4411,7 +4609,7 @@ class LiteralIndex {
|
|
|
4411
4609
|
vocabularyIndex = new Map;
|
|
4412
4610
|
static VERSION = "1.1.0";
|
|
4413
4611
|
constructor(indexDir, moduleId) {
|
|
4414
|
-
this.indexPath =
|
|
4612
|
+
this.indexPath = path11.join(indexDir, "index", moduleId, "literals");
|
|
4415
4613
|
this.moduleId = moduleId;
|
|
4416
4614
|
}
|
|
4417
4615
|
async initialize() {
|
|
@@ -4572,17 +4770,17 @@ class LiteralIndex {
|
|
|
4572
4770
|
}));
|
|
4573
4771
|
}
|
|
4574
4772
|
async save() {
|
|
4575
|
-
await
|
|
4773
|
+
await fs5.mkdir(this.indexPath, { recursive: true });
|
|
4576
4774
|
const data = {
|
|
4577
4775
|
version: LiteralIndex.VERSION,
|
|
4578
4776
|
entries: Object.fromEntries(this.entries)
|
|
4579
4777
|
};
|
|
4580
|
-
const indexFile =
|
|
4581
|
-
await
|
|
4778
|
+
const indexFile = path11.join(this.indexPath, "_index.json");
|
|
4779
|
+
await fs5.writeFile(indexFile, JSON.stringify(data, null, 2));
|
|
4582
4780
|
}
|
|
4583
4781
|
async load() {
|
|
4584
|
-
const indexFile =
|
|
4585
|
-
const content = await
|
|
4782
|
+
const indexFile = path11.join(this.indexPath, "_index.json");
|
|
4783
|
+
const content = await fs5.readFile(indexFile, "utf-8");
|
|
4586
4784
|
const data = JSON.parse(content);
|
|
4587
4785
|
if (data.version !== LiteralIndex.VERSION) {
|
|
4588
4786
|
console.warn(`Literal index version mismatch: expected ${LiteralIndex.VERSION}, got ${data.version}`);
|
|
@@ -4591,8 +4789,8 @@ class LiteralIndex {
|
|
|
4591
4789
|
}
|
|
4592
4790
|
async exists() {
|
|
4593
4791
|
try {
|
|
4594
|
-
const indexFile =
|
|
4595
|
-
await
|
|
4792
|
+
const indexFile = path11.join(this.indexPath, "_index.json");
|
|
4793
|
+
await fs5.access(indexFile);
|
|
4596
4794
|
return true;
|
|
4597
4795
|
} catch {
|
|
4598
4796
|
return false;
|
|
@@ -4635,7 +4833,7 @@ function shouldReplaceMatchType(existing, incoming) {
|
|
|
4635
4833
|
return priority[incoming] > priority[existing];
|
|
4636
4834
|
}
|
|
4637
4835
|
function getLiteralIndexPath(rootDir, moduleId, indexDir = ".raggrep") {
|
|
4638
|
-
return
|
|
4836
|
+
return path11.join(rootDir, indexDir, "index", moduleId, "literals");
|
|
4639
4837
|
}
|
|
4640
4838
|
var init_literalIndex = () => {};
|
|
4641
4839
|
|
|
@@ -4656,9 +4854,9 @@ __export(exports_typescript, {
|
|
|
4656
4854
|
DEFAULT_TOP_K: () => DEFAULT_TOP_K2,
|
|
4657
4855
|
DEFAULT_MIN_SCORE: () => DEFAULT_MIN_SCORE2
|
|
4658
4856
|
});
|
|
4659
|
-
import * as
|
|
4857
|
+
import * as path12 from "path";
|
|
4660
4858
|
function isTypeScriptFile(filepath) {
|
|
4661
|
-
const ext =
|
|
4859
|
+
const ext = path12.extname(filepath).toLowerCase();
|
|
4662
4860
|
return TYPESCRIPT_EXTENSIONS.includes(ext);
|
|
4663
4861
|
}
|
|
4664
4862
|
function calculateChunkTypeBoost(chunk) {
|
|
@@ -4730,7 +4928,7 @@ class TypeScriptModule {
|
|
|
4730
4928
|
startLine: 1,
|
|
4731
4929
|
endLine: lines.length,
|
|
4732
4930
|
type: "file",
|
|
4733
|
-
name:
|
|
4931
|
+
name: path12.basename(filepath),
|
|
4734
4932
|
isExported: false
|
|
4735
4933
|
});
|
|
4736
4934
|
}
|
|
@@ -5057,16 +5255,16 @@ class TypeScriptModule {
|
|
|
5057
5255
|
while ((match = importRegex.exec(content)) !== null) {
|
|
5058
5256
|
const importPath = match[1];
|
|
5059
5257
|
if (importPath.startsWith(".")) {
|
|
5060
|
-
const dir =
|
|
5061
|
-
const resolved =
|
|
5258
|
+
const dir = path12.dirname(filepath);
|
|
5259
|
+
const resolved = path12.normalize(path12.join(dir, importPath));
|
|
5062
5260
|
references.push(resolved);
|
|
5063
5261
|
}
|
|
5064
5262
|
}
|
|
5065
5263
|
while ((match = requireRegex.exec(content)) !== null) {
|
|
5066
5264
|
const importPath = match[1];
|
|
5067
5265
|
if (importPath.startsWith(".")) {
|
|
5068
|
-
const dir =
|
|
5069
|
-
const resolved =
|
|
5266
|
+
const dir = path12.dirname(filepath);
|
|
5267
|
+
const resolved = path12.normalize(path12.join(dir, importPath));
|
|
5070
5268
|
references.push(resolved);
|
|
5071
5269
|
}
|
|
5072
5270
|
}
|
|
@@ -5094,7 +5292,7 @@ var init_typescript = __esm(() => {
|
|
|
5094
5292
|
});
|
|
5095
5293
|
|
|
5096
5294
|
// src/infrastructure/parsing/typescriptParser.ts
|
|
5097
|
-
import * as
|
|
5295
|
+
import * as path13 from "path";
|
|
5098
5296
|
|
|
5099
5297
|
class TypeScriptParser {
|
|
5100
5298
|
supportedLanguages = ["typescript", "javascript"];
|
|
@@ -5110,12 +5308,12 @@ class TypeScriptParser {
|
|
|
5110
5308
|
startLine: 1,
|
|
5111
5309
|
endLine: lines.length,
|
|
5112
5310
|
type: "file",
|
|
5113
|
-
name:
|
|
5311
|
+
name: path13.basename(filepath),
|
|
5114
5312
|
isExported: false
|
|
5115
5313
|
};
|
|
5116
5314
|
chunks.unshift(fullFileChunk);
|
|
5117
5315
|
}
|
|
5118
|
-
const ext =
|
|
5316
|
+
const ext = path13.extname(filepath).toLowerCase();
|
|
5119
5317
|
const language = ext === ".js" || ext === ".jsx" || ext === ".mjs" || ext === ".cjs" ? "javascript" : "typescript";
|
|
5120
5318
|
return {
|
|
5121
5319
|
chunks,
|
|
@@ -5132,7 +5330,7 @@ class TypeScriptParser {
|
|
|
5132
5330
|
}
|
|
5133
5331
|
}
|
|
5134
5332
|
canParse(filepath) {
|
|
5135
|
-
const ext =
|
|
5333
|
+
const ext = path13.extname(filepath).toLowerCase();
|
|
5136
5334
|
return TYPESCRIPT_EXTENSIONS2.includes(ext);
|
|
5137
5335
|
}
|
|
5138
5336
|
convertChunk(tc) {
|
|
@@ -5147,7 +5345,7 @@ class TypeScriptParser {
|
|
|
5147
5345
|
};
|
|
5148
5346
|
}
|
|
5149
5347
|
detectLanguage(filepath) {
|
|
5150
|
-
const ext =
|
|
5348
|
+
const ext = path13.extname(filepath).toLowerCase();
|
|
5151
5349
|
if ([".js", ".jsx", ".mjs", ".cjs"].includes(ext)) {
|
|
5152
5350
|
return "javascript";
|
|
5153
5351
|
}
|
|
@@ -5352,8 +5550,8 @@ var init_grammarManager = __esm(() => {
|
|
|
5352
5550
|
});
|
|
5353
5551
|
|
|
5354
5552
|
// src/infrastructure/parsing/treeSitterParser.ts
|
|
5355
|
-
import * as
|
|
5356
|
-
import * as
|
|
5553
|
+
import * as path14 from "path";
|
|
5554
|
+
import * as fs6 from "fs";
|
|
5357
5555
|
|
|
5358
5556
|
class TreeSitterParser {
|
|
5359
5557
|
supportedLanguages = [
|
|
@@ -5375,7 +5573,7 @@ class TreeSitterParser {
|
|
|
5375
5573
|
chunks: [],
|
|
5376
5574
|
language: "typescript",
|
|
5377
5575
|
success: false,
|
|
5378
|
-
error: `Unsupported file type: ${
|
|
5576
|
+
error: `Unsupported file type: ${path14.extname(filepath)}`
|
|
5379
5577
|
};
|
|
5380
5578
|
}
|
|
5381
5579
|
try {
|
|
@@ -5395,11 +5593,11 @@ class TreeSitterParser {
|
|
|
5395
5593
|
}
|
|
5396
5594
|
}
|
|
5397
5595
|
canParse(filepath) {
|
|
5398
|
-
const ext =
|
|
5596
|
+
const ext = path14.extname(filepath).toLowerCase();
|
|
5399
5597
|
return ext in EXTENSION_TO_LANGUAGE2;
|
|
5400
5598
|
}
|
|
5401
5599
|
detectLanguage(filepath) {
|
|
5402
|
-
const ext =
|
|
5600
|
+
const ext = path14.extname(filepath).toLowerCase();
|
|
5403
5601
|
return EXTENSION_TO_LANGUAGE2[ext] || null;
|
|
5404
5602
|
}
|
|
5405
5603
|
async ensureInitialized() {
|
|
@@ -5433,20 +5631,20 @@ class TreeSitterParser {
|
|
|
5433
5631
|
async resolveWasmPath() {
|
|
5434
5632
|
try {
|
|
5435
5633
|
const webTreeSitterPath = __require.resolve("web-tree-sitter");
|
|
5436
|
-
const wasmPath =
|
|
5437
|
-
if (
|
|
5634
|
+
const wasmPath = path14.join(path14.dirname(webTreeSitterPath), "web-tree-sitter.wasm");
|
|
5635
|
+
if (fs6.existsSync(wasmPath)) {
|
|
5438
5636
|
return wasmPath;
|
|
5439
5637
|
}
|
|
5440
5638
|
} catch {}
|
|
5441
5639
|
try {
|
|
5442
5640
|
const possiblePaths = [
|
|
5443
|
-
|
|
5444
|
-
|
|
5445
|
-
|
|
5446
|
-
|
|
5641
|
+
path14.join(__dirname, "../../../node_modules/web-tree-sitter/web-tree-sitter.wasm"),
|
|
5642
|
+
path14.join(__dirname, "../../node_modules/web-tree-sitter/web-tree-sitter.wasm"),
|
|
5643
|
+
path14.join(__dirname, "../../../../node_modules/web-tree-sitter/web-tree-sitter.wasm"),
|
|
5644
|
+
path14.join(__dirname, "web-tree-sitter.wasm")
|
|
5447
5645
|
];
|
|
5448
5646
|
for (const wasmPath of possiblePaths) {
|
|
5449
|
-
if (
|
|
5647
|
+
if (fs6.existsSync(wasmPath)) {
|
|
5450
5648
|
return wasmPath;
|
|
5451
5649
|
}
|
|
5452
5650
|
}
|
|
@@ -5487,7 +5685,7 @@ class TreeSitterParser {
|
|
|
5487
5685
|
startLine: 1,
|
|
5488
5686
|
endLine: lines.length,
|
|
5489
5687
|
type: "file",
|
|
5490
|
-
name:
|
|
5688
|
+
name: path14.basename(filepath),
|
|
5491
5689
|
isExported: false
|
|
5492
5690
|
});
|
|
5493
5691
|
}
|
|
@@ -5834,7 +6032,7 @@ class TreeSitterParser {
|
|
|
5834
6032
|
startLine: 1,
|
|
5835
6033
|
endLine: lines.length,
|
|
5836
6034
|
type: "file",
|
|
5837
|
-
name:
|
|
6035
|
+
name: path14.basename(filepath)
|
|
5838
6036
|
});
|
|
5839
6037
|
return {
|
|
5840
6038
|
chunks,
|
|
@@ -5843,7 +6041,7 @@ class TreeSitterParser {
|
|
|
5843
6041
|
};
|
|
5844
6042
|
}
|
|
5845
6043
|
}
|
|
5846
|
-
var __dirname = "/
|
|
6044
|
+
var __dirname = "/home/runner/work/raggrep/raggrep/src/infrastructure/parsing", EXTENSION_TO_LANGUAGE2;
|
|
5847
6045
|
var init_treeSitterParser = __esm(() => {
|
|
5848
6046
|
init_grammarManager();
|
|
5849
6047
|
EXTENSION_TO_LANGUAGE2 = {
|
|
@@ -5864,7 +6062,7 @@ var init_treeSitterParser = __esm(() => {
|
|
|
5864
6062
|
});
|
|
5865
6063
|
|
|
5866
6064
|
// src/infrastructure/parsing/parserFactory.ts
|
|
5867
|
-
import * as
|
|
6065
|
+
import * as path15 from "path";
|
|
5868
6066
|
function getTypeScriptParser() {
|
|
5869
6067
|
if (!typescriptParserInstance) {
|
|
5870
6068
|
typescriptParserInstance = new TypeScriptParser;
|
|
@@ -5878,7 +6076,7 @@ function getTreeSitterParser() {
|
|
|
5878
6076
|
return treeSitterParserInstance;
|
|
5879
6077
|
}
|
|
5880
6078
|
function createParserForFile(filepath) {
|
|
5881
|
-
const ext =
|
|
6079
|
+
const ext = path15.extname(filepath).toLowerCase();
|
|
5882
6080
|
const parserType = EXTENSION_PARSER_MAP[ext];
|
|
5883
6081
|
if (!parserType) {
|
|
5884
6082
|
return null;
|
|
@@ -5927,9 +6125,9 @@ __export(exports_python, {
|
|
|
5927
6125
|
DEFAULT_TOP_K: () => DEFAULT_TOP_K3,
|
|
5928
6126
|
DEFAULT_MIN_SCORE: () => DEFAULT_MIN_SCORE3
|
|
5929
6127
|
});
|
|
5930
|
-
import * as
|
|
6128
|
+
import * as path16 from "path";
|
|
5931
6129
|
function isPythonFile(filepath) {
|
|
5932
|
-
const ext =
|
|
6130
|
+
const ext = path16.extname(filepath).toLowerCase();
|
|
5933
6131
|
return PYTHON_EXTENSIONS.includes(ext);
|
|
5934
6132
|
}
|
|
5935
6133
|
function generateChunkId3(filepath, startLine, endLine) {
|
|
@@ -6013,7 +6211,7 @@ class PythonModule {
|
|
|
6013
6211
|
startLine: 1,
|
|
6014
6212
|
endLine: lines.length,
|
|
6015
6213
|
type: "file",
|
|
6016
|
-
name:
|
|
6214
|
+
name: path16.basename(filepath)
|
|
6017
6215
|
});
|
|
6018
6216
|
const funcRegex = /^(\s*)(async\s+)?def\s+(\w+)\s*\([^)]*\)\s*:/gm;
|
|
6019
6217
|
let match;
|
|
@@ -6387,9 +6585,9 @@ __export(exports_go, {
|
|
|
6387
6585
|
DEFAULT_TOP_K: () => DEFAULT_TOP_K4,
|
|
6388
6586
|
DEFAULT_MIN_SCORE: () => DEFAULT_MIN_SCORE4
|
|
6389
6587
|
});
|
|
6390
|
-
import * as
|
|
6588
|
+
import * as path17 from "path";
|
|
6391
6589
|
function isGoFile(filepath) {
|
|
6392
|
-
const ext =
|
|
6590
|
+
const ext = path17.extname(filepath).toLowerCase();
|
|
6393
6591
|
return GO_EXTENSIONS.includes(ext);
|
|
6394
6592
|
}
|
|
6395
6593
|
function generateChunkId4(filepath, startLine, endLine) {
|
|
@@ -6474,7 +6672,7 @@ class GoModule {
|
|
|
6474
6672
|
startLine: 1,
|
|
6475
6673
|
endLine: lines.length,
|
|
6476
6674
|
type: "file",
|
|
6477
|
-
name:
|
|
6675
|
+
name: path17.basename(filepath)
|
|
6478
6676
|
});
|
|
6479
6677
|
const funcRegex = /^func\s+(?:\(\s*\w+\s+\*?\w+\s*\)\s+)?(\w+)\s*\(/gm;
|
|
6480
6678
|
let match;
|
|
@@ -6868,9 +7066,9 @@ __export(exports_rust, {
|
|
|
6868
7066
|
DEFAULT_TOP_K: () => DEFAULT_TOP_K5,
|
|
6869
7067
|
DEFAULT_MIN_SCORE: () => DEFAULT_MIN_SCORE5
|
|
6870
7068
|
});
|
|
6871
|
-
import * as
|
|
7069
|
+
import * as path18 from "path";
|
|
6872
7070
|
function isRustFile(filepath) {
|
|
6873
|
-
const ext =
|
|
7071
|
+
const ext = path18.extname(filepath).toLowerCase();
|
|
6874
7072
|
return RUST_EXTENSIONS.includes(ext);
|
|
6875
7073
|
}
|
|
6876
7074
|
function generateChunkId5(filepath, startLine, endLine) {
|
|
@@ -6957,7 +7155,7 @@ class RustModule {
|
|
|
6957
7155
|
startLine: 1,
|
|
6958
7156
|
endLine: lines.length,
|
|
6959
7157
|
type: "file",
|
|
6960
|
-
name:
|
|
7158
|
+
name: path18.basename(filepath)
|
|
6961
7159
|
});
|
|
6962
7160
|
const funcRegex = /^(pub(?:\s*\([^)]*\))?\s+)?(?:async\s+)?fn\s+(\w+)/gm;
|
|
6963
7161
|
let match;
|
|
@@ -7428,9 +7626,9 @@ __export(exports_json, {
|
|
|
7428
7626
|
DEFAULT_TOP_K: () => DEFAULT_TOP_K6,
|
|
7429
7627
|
DEFAULT_MIN_SCORE: () => DEFAULT_MIN_SCORE6
|
|
7430
7628
|
});
|
|
7431
|
-
import * as
|
|
7629
|
+
import * as path19 from "path";
|
|
7432
7630
|
function isJsonFile(filepath) {
|
|
7433
|
-
const ext =
|
|
7631
|
+
const ext = path19.extname(filepath).toLowerCase();
|
|
7434
7632
|
return JSON_EXTENSIONS.includes(ext);
|
|
7435
7633
|
}
|
|
7436
7634
|
|
|
@@ -7464,7 +7662,7 @@ class JsonModule {
|
|
|
7464
7662
|
} catch {
|
|
7465
7663
|
return null;
|
|
7466
7664
|
}
|
|
7467
|
-
const fileBasename =
|
|
7665
|
+
const fileBasename = path19.basename(filepath, path19.extname(filepath));
|
|
7468
7666
|
const jsonPathLiterals = extractJsonPaths(parsed, fileBasename);
|
|
7469
7667
|
const lines = content.split(`
|
|
7470
7668
|
`);
|
|
@@ -7671,7 +7869,7 @@ __export(exports_markdown, {
|
|
|
7671
7869
|
DEFAULT_TOP_K: () => DEFAULT_TOP_K7,
|
|
7672
7870
|
DEFAULT_MIN_SCORE: () => DEFAULT_MIN_SCORE7
|
|
7673
7871
|
});
|
|
7674
|
-
import * as
|
|
7872
|
+
import * as path20 from "path";
|
|
7675
7873
|
function calculateHeadingLevelBoost(chunk) {
|
|
7676
7874
|
const metadata = chunk.metadata;
|
|
7677
7875
|
const level = metadata?.headingLevel ?? 0;
|
|
@@ -7691,7 +7889,7 @@ function calculateHeadingLevelBoost(chunk) {
|
|
|
7691
7889
|
}
|
|
7692
7890
|
}
|
|
7693
7891
|
function isMarkdownFile(filepath) {
|
|
7694
|
-
const ext =
|
|
7892
|
+
const ext = path20.extname(filepath).toLowerCase();
|
|
7695
7893
|
return MARKDOWN_EXTENSIONS.includes(ext);
|
|
7696
7894
|
}
|
|
7697
7895
|
function parseMarkdownHierarchical(content, maxDepth = 4) {
|
|
@@ -7992,6 +8190,11 @@ var init_markdown = __esm(() => {
|
|
|
7992
8190
|
supportsFile6 = isMarkdownFile;
|
|
7993
8191
|
});
|
|
7994
8192
|
|
|
8193
|
+
// src/types.ts
|
|
8194
|
+
var init_types = __esm(() => {
|
|
8195
|
+
init_entities();
|
|
8196
|
+
});
|
|
8197
|
+
|
|
7995
8198
|
// node_modules/fdir/dist/index.mjs
|
|
7996
8199
|
import { createRequire as createRequire2 } from "module";
|
|
7997
8200
|
import { basename, dirname, normalize, relative, resolve, sep } from "path";
|
|
@@ -8527,9 +8730,9 @@ var Builder = class {
|
|
|
8527
8730
|
|
|
8528
8731
|
// src/app/indexer/index.ts
|
|
8529
8732
|
init_config2();
|
|
8530
|
-
import * as
|
|
8531
|
-
import * as
|
|
8532
|
-
import * as
|
|
8733
|
+
import * as fs9 from "fs/promises";
|
|
8734
|
+
import * as path23 from "path";
|
|
8735
|
+
import * as os2 from "os";
|
|
8533
8736
|
import * as crypto2 from "crypto";
|
|
8534
8737
|
|
|
8535
8738
|
// src/modules/registry.ts
|
|
@@ -8570,13 +8773,13 @@ async function registerBuiltInModules() {
|
|
|
8570
8773
|
}
|
|
8571
8774
|
|
|
8572
8775
|
// src/infrastructure/introspection/IntrospectionIndex.ts
|
|
8573
|
-
import * as
|
|
8574
|
-
import * as
|
|
8776
|
+
import * as path22 from "path";
|
|
8777
|
+
import * as fs8 from "fs/promises";
|
|
8575
8778
|
import * as fsSync from "fs";
|
|
8576
8779
|
|
|
8577
8780
|
// src/infrastructure/introspection/projectDetector.ts
|
|
8578
|
-
import * as
|
|
8579
|
-
import * as
|
|
8781
|
+
import * as path21 from "path";
|
|
8782
|
+
import * as fs7 from "fs/promises";
|
|
8580
8783
|
var MAX_SCAN_DEPTH = 4;
|
|
8581
8784
|
var SKIP_DIRS = new Set([
|
|
8582
8785
|
"node_modules",
|
|
@@ -8592,9 +8795,9 @@ async function scanForPackageJsons(rootDir, currentDir = "", depth = 0) {
|
|
|
8592
8795
|
if (depth > MAX_SCAN_DEPTH)
|
|
8593
8796
|
return [];
|
|
8594
8797
|
const results = [];
|
|
8595
|
-
const fullDir = currentDir ?
|
|
8798
|
+
const fullDir = currentDir ? path21.join(rootDir, currentDir) : rootDir;
|
|
8596
8799
|
try {
|
|
8597
|
-
const entries = await
|
|
8800
|
+
const entries = await fs7.readdir(fullDir, { withFileTypes: true });
|
|
8598
8801
|
const hasPackageJson = entries.some((e) => e.isFile() && e.name === "package.json");
|
|
8599
8802
|
if (hasPackageJson && currentDir) {
|
|
8600
8803
|
const info = await parsePackageJson(rootDir, currentDir);
|
|
@@ -8615,10 +8818,10 @@ async function scanForPackageJsons(rootDir, currentDir = "", depth = 0) {
|
|
|
8615
8818
|
}
|
|
8616
8819
|
async function parsePackageJson(rootDir, relativePath) {
|
|
8617
8820
|
try {
|
|
8618
|
-
const packageJsonPath =
|
|
8619
|
-
const content = await
|
|
8821
|
+
const packageJsonPath = path21.join(rootDir, relativePath, "package.json");
|
|
8822
|
+
const content = await fs7.readFile(packageJsonPath, "utf-8");
|
|
8620
8823
|
const pkg = JSON.parse(content);
|
|
8621
|
-
const name = pkg.name ||
|
|
8824
|
+
const name = pkg.name || path21.basename(relativePath);
|
|
8622
8825
|
const deps = { ...pkg.dependencies, ...pkg.devDependencies };
|
|
8623
8826
|
let type = "unknown";
|
|
8624
8827
|
if (deps["next"] || deps["react"] || deps["vue"] || deps["svelte"]) {
|
|
@@ -8654,7 +8857,7 @@ async function detectProjectStructure(rootDir) {
|
|
|
8654
8857
|
const projectMap = new Map;
|
|
8655
8858
|
let isMonorepo = false;
|
|
8656
8859
|
try {
|
|
8657
|
-
const entries = await
|
|
8860
|
+
const entries = await fs7.readdir(rootDir, { withFileTypes: true });
|
|
8658
8861
|
const dirNames = entries.filter((e) => e.isDirectory()).map((e) => e.name);
|
|
8659
8862
|
const monorepoPatterns = ["apps", "packages", "libs", "services"];
|
|
8660
8863
|
const hasMonorepoStructure = monorepoPatterns.some((p) => dirNames.includes(p));
|
|
@@ -8663,9 +8866,9 @@ async function detectProjectStructure(rootDir) {
|
|
|
8663
8866
|
for (const pattern of monorepoPatterns) {
|
|
8664
8867
|
if (!dirNames.includes(pattern))
|
|
8665
8868
|
continue;
|
|
8666
|
-
const patternDir =
|
|
8869
|
+
const patternDir = path21.join(rootDir, pattern);
|
|
8667
8870
|
try {
|
|
8668
|
-
const subDirs = await
|
|
8871
|
+
const subDirs = await fs7.readdir(patternDir, { withFileTypes: true });
|
|
8669
8872
|
for (const subDir of subDirs) {
|
|
8670
8873
|
if (!subDir.isDirectory())
|
|
8671
8874
|
continue;
|
|
@@ -8694,8 +8897,8 @@ async function detectProjectStructure(rootDir) {
|
|
|
8694
8897
|
}
|
|
8695
8898
|
let rootType = "unknown";
|
|
8696
8899
|
try {
|
|
8697
|
-
const rootPkgPath =
|
|
8698
|
-
const rootPkg = JSON.parse(await
|
|
8900
|
+
const rootPkgPath = path21.join(rootDir, "package.json");
|
|
8901
|
+
const rootPkg = JSON.parse(await fs7.readFile(rootPkgPath, "utf-8"));
|
|
8699
8902
|
if (rootPkg.workspaces)
|
|
8700
8903
|
isMonorepo = true;
|
|
8701
8904
|
const deps = { ...rootPkg.dependencies, ...rootPkg.devDependencies };
|
|
@@ -8735,8 +8938,8 @@ class IntrospectionIndex {
|
|
|
8735
8938
|
async initialize() {
|
|
8736
8939
|
this.structure = await detectProjectStructure(this.rootDir);
|
|
8737
8940
|
try {
|
|
8738
|
-
const configPath =
|
|
8739
|
-
const configContent = await
|
|
8941
|
+
const configPath = path22.join(this.rootDir, ".raggrep", "config.json");
|
|
8942
|
+
const configContent = await fs8.readFile(configPath, "utf-8");
|
|
8740
8943
|
const config = JSON.parse(configContent);
|
|
8741
8944
|
this.config = config.introspection || {};
|
|
8742
8945
|
} catch {}
|
|
@@ -8750,7 +8953,7 @@ class IntrospectionIndex {
|
|
|
8750
8953
|
}
|
|
8751
8954
|
const fileExists = enableReadmeContext ? (relativePath) => {
|
|
8752
8955
|
try {
|
|
8753
|
-
const absolutePath =
|
|
8956
|
+
const absolutePath = path22.join(this.rootDir, relativePath);
|
|
8754
8957
|
return fsSync.existsSync(absolutePath);
|
|
8755
8958
|
} catch {
|
|
8756
8959
|
return false;
|
|
@@ -8786,28 +8989,28 @@ class IntrospectionIndex {
|
|
|
8786
8989
|
}
|
|
8787
8990
|
}
|
|
8788
8991
|
async save(config) {
|
|
8789
|
-
const introDir =
|
|
8790
|
-
await
|
|
8791
|
-
const projectPath =
|
|
8792
|
-
await
|
|
8992
|
+
const introDir = path22.join(getRaggrepDir(this.rootDir, config), "introspection");
|
|
8993
|
+
await fs8.mkdir(introDir, { recursive: true });
|
|
8994
|
+
const projectPath = path22.join(introDir, "_project.json");
|
|
8995
|
+
await fs8.writeFile(projectPath, JSON.stringify({
|
|
8793
8996
|
version: "1.0.0",
|
|
8794
8997
|
lastUpdated: new Date().toISOString(),
|
|
8795
8998
|
structure: this.structure
|
|
8796
8999
|
}, null, 2));
|
|
8797
9000
|
for (const [filepath, intro] of this.files) {
|
|
8798
|
-
const introFilePath =
|
|
8799
|
-
await
|
|
8800
|
-
await
|
|
9001
|
+
const introFilePath = path22.join(introDir, "files", filepath.replace(/\.[^.]+$/, ".json"));
|
|
9002
|
+
await fs8.mkdir(path22.dirname(introFilePath), { recursive: true });
|
|
9003
|
+
await fs8.writeFile(introFilePath, JSON.stringify(intro, null, 2));
|
|
8801
9004
|
}
|
|
8802
9005
|
}
|
|
8803
9006
|
async load(config) {
|
|
8804
|
-
const introDir =
|
|
9007
|
+
const introDir = path22.join(getRaggrepDir(this.rootDir, config), "introspection");
|
|
8805
9008
|
try {
|
|
8806
|
-
const projectPath =
|
|
8807
|
-
const projectContent = await
|
|
9009
|
+
const projectPath = path22.join(introDir, "_project.json");
|
|
9010
|
+
const projectContent = await fs8.readFile(projectPath, "utf-8");
|
|
8808
9011
|
const projectData = JSON.parse(projectContent);
|
|
8809
9012
|
this.structure = projectData.structure;
|
|
8810
|
-
await this.loadFilesRecursive(
|
|
9013
|
+
await this.loadFilesRecursive(path22.join(introDir, "files"), "");
|
|
8811
9014
|
} catch {
|
|
8812
9015
|
this.structure = null;
|
|
8813
9016
|
this.files.clear();
|
|
@@ -8815,14 +9018,14 @@ class IntrospectionIndex {
|
|
|
8815
9018
|
}
|
|
8816
9019
|
async loadFilesRecursive(basePath, prefix) {
|
|
8817
9020
|
try {
|
|
8818
|
-
const entries = await
|
|
9021
|
+
const entries = await fs8.readdir(basePath, { withFileTypes: true });
|
|
8819
9022
|
for (const entry of entries) {
|
|
8820
|
-
const entryPath =
|
|
9023
|
+
const entryPath = path22.join(basePath, entry.name);
|
|
8821
9024
|
const relativePath = prefix ? `${prefix}/${entry.name}` : entry.name;
|
|
8822
9025
|
if (entry.isDirectory()) {
|
|
8823
9026
|
await this.loadFilesRecursive(entryPath, relativePath);
|
|
8824
9027
|
} else if (entry.name.endsWith(".json")) {
|
|
8825
|
-
const content = await
|
|
9028
|
+
const content = await fs8.readFile(entryPath, "utf-8");
|
|
8826
9029
|
const intro = JSON.parse(content);
|
|
8827
9030
|
this.files.set(intro.filepath, intro);
|
|
8828
9031
|
}
|
|
@@ -9078,7 +9281,7 @@ async function parallelMap(items, processor, concurrency) {
|
|
|
9078
9281
|
await Promise.all(workers);
|
|
9079
9282
|
return results;
|
|
9080
9283
|
}
|
|
9081
|
-
var INDEX_SCHEMA_VERSION = "2.
|
|
9284
|
+
var INDEX_SCHEMA_VERSION = "2.1.0";
|
|
9082
9285
|
function formatDuration(ms) {
|
|
9083
9286
|
if (ms < 1000) {
|
|
9084
9287
|
return `${ms}ms`;
|
|
@@ -9092,7 +9295,7 @@ function formatDuration(ms) {
|
|
|
9092
9295
|
return `${minutes}m ${remainingSeconds.toFixed(1)}s`;
|
|
9093
9296
|
}
|
|
9094
9297
|
function getOptimalConcurrency() {
|
|
9095
|
-
const cpuCount =
|
|
9298
|
+
const cpuCount = os2.cpus().length;
|
|
9096
9299
|
const optimal = Math.max(2, Math.min(16, Math.floor(cpuCount * 0.75)));
|
|
9097
9300
|
return optimal;
|
|
9098
9301
|
}
|
|
@@ -9104,7 +9307,7 @@ async function indexDirectory(rootDir, options = {}) {
|
|
|
9104
9307
|
const concurrency = options.concurrency ?? DEFAULT_CONCURRENCY;
|
|
9105
9308
|
clearFreshnessCache();
|
|
9106
9309
|
const logger = options.logger ? options.logger : quiet ? createSilentLogger() : createLogger({ verbose });
|
|
9107
|
-
rootDir =
|
|
9310
|
+
rootDir = path23.resolve(rootDir);
|
|
9108
9311
|
const location = getIndexLocation(rootDir);
|
|
9109
9312
|
logger.info(`Indexing directory: ${rootDir}`);
|
|
9110
9313
|
logger.info(`Index location: ${location.indexDir}`);
|
|
@@ -9156,12 +9359,12 @@ async function indexDirectory(rootDir, options = {}) {
|
|
|
9156
9359
|
rootDir,
|
|
9157
9360
|
config,
|
|
9158
9361
|
readFile: async (filepath) => {
|
|
9159
|
-
const fullPath =
|
|
9160
|
-
return
|
|
9362
|
+
const fullPath = path23.isAbsolute(filepath) ? filepath : path23.join(rootDir, filepath);
|
|
9363
|
+
return fs9.readFile(fullPath, "utf-8");
|
|
9161
9364
|
},
|
|
9162
9365
|
getFileStats: async (filepath) => {
|
|
9163
|
-
const fullPath =
|
|
9164
|
-
const stats = await
|
|
9366
|
+
const fullPath = path23.isAbsolute(filepath) ? filepath : path23.join(rootDir, filepath);
|
|
9367
|
+
const stats = await fs9.stat(fullPath);
|
|
9165
9368
|
return { lastModified: stats.mtime.toISOString() };
|
|
9166
9369
|
}
|
|
9167
9370
|
};
|
|
@@ -9186,7 +9389,7 @@ async function isIndexVersionCompatible(rootDir) {
|
|
|
9186
9389
|
const config = await loadConfig(rootDir);
|
|
9187
9390
|
const globalManifestPath = getGlobalManifestPath(rootDir, config);
|
|
9188
9391
|
try {
|
|
9189
|
-
const content = await
|
|
9392
|
+
const content = await fs9.readFile(globalManifestPath, "utf-8");
|
|
9190
9393
|
const manifest = JSON.parse(content);
|
|
9191
9394
|
return manifest.version === INDEX_SCHEMA_VERSION;
|
|
9192
9395
|
} catch {
|
|
@@ -9196,11 +9399,11 @@ async function isIndexVersionCompatible(rootDir) {
|
|
|
9196
9399
|
async function deleteIndex(rootDir) {
|
|
9197
9400
|
const indexDir = getRaggrepDir(rootDir);
|
|
9198
9401
|
try {
|
|
9199
|
-
await
|
|
9402
|
+
await fs9.rm(indexDir, { recursive: true, force: true });
|
|
9200
9403
|
} catch {}
|
|
9201
9404
|
}
|
|
9202
9405
|
async function resetIndex(rootDir) {
|
|
9203
|
-
rootDir =
|
|
9406
|
+
rootDir = path23.resolve(rootDir);
|
|
9204
9407
|
clearFreshnessCache();
|
|
9205
9408
|
const status = await getIndexStatus(rootDir);
|
|
9206
9409
|
if (!status.exists) {
|
|
@@ -9225,7 +9428,7 @@ async function ensureIndexFresh(rootDir, options = {}) {
|
|
|
9225
9428
|
let filesChanged = 0;
|
|
9226
9429
|
let filesReindexed = 0;
|
|
9227
9430
|
const logger = options.logger ? options.logger : quiet ? createSilentLogger() : createLogger({ verbose });
|
|
9228
|
-
rootDir =
|
|
9431
|
+
rootDir = path23.resolve(rootDir);
|
|
9229
9432
|
const status = await getIndexStatus(rootDir);
|
|
9230
9433
|
if (!status.exists) {
|
|
9231
9434
|
clearFreshnessCache();
|
|
@@ -9249,7 +9452,7 @@ async function ensureIndexFresh(rootDir, options = {}) {
|
|
|
9249
9452
|
const globalManifestPath = getGlobalManifestPath(rootDir, config);
|
|
9250
9453
|
let currentManifestMtime = 0;
|
|
9251
9454
|
try {
|
|
9252
|
-
const manifestStats = await
|
|
9455
|
+
const manifestStats = await fs9.stat(globalManifestPath);
|
|
9253
9456
|
currentManifestMtime = manifestStats.mtimeMs;
|
|
9254
9457
|
} catch {}
|
|
9255
9458
|
const now = Date.now();
|
|
@@ -9287,7 +9490,7 @@ async function ensureIndexFresh(rootDir, options = {}) {
|
|
|
9287
9490
|
const { allFiles: currentFiles, changedFiles, changedFileMtimes } = discoveryResult;
|
|
9288
9491
|
filesDiscovered = currentFiles.length;
|
|
9289
9492
|
filesChanged = changedFiles.length;
|
|
9290
|
-
const currentFileSet = new Set(currentFiles.map((f) =>
|
|
9493
|
+
const currentFileSet = new Set(currentFiles.map((f) => path23.relative(rootDir, f)));
|
|
9291
9494
|
const changedFileSet = new Set(changedFiles);
|
|
9292
9495
|
let totalIndexed = 0;
|
|
9293
9496
|
let totalRemoved = 0;
|
|
@@ -9321,11 +9524,11 @@ async function ensureIndexFresh(rootDir, options = {}) {
|
|
|
9321
9524
|
if (filesToRemove.length > 0) {
|
|
9322
9525
|
await Promise.all(filesToRemove.map(async (filepath) => {
|
|
9323
9526
|
logger.debug(` Removing stale: ${filepath}`);
|
|
9324
|
-
const indexFilePath =
|
|
9325
|
-
const symbolicFilePath =
|
|
9527
|
+
const indexFilePath = path23.join(indexPath, filepath.replace(/\.[^.]+$/, ".json"));
|
|
9528
|
+
const symbolicFilePath = path23.join(indexPath, "symbolic", filepath.replace(/\.[^.]+$/, ".json"));
|
|
9326
9529
|
await Promise.all([
|
|
9327
|
-
|
|
9328
|
-
|
|
9530
|
+
fs9.unlink(indexFilePath).catch(() => {}),
|
|
9531
|
+
fs9.unlink(symbolicFilePath).catch(() => {})
|
|
9329
9532
|
]);
|
|
9330
9533
|
delete manifest.files[filepath];
|
|
9331
9534
|
removedFilepaths.push(filepath);
|
|
@@ -9349,19 +9552,19 @@ async function ensureIndexFresh(rootDir, options = {}) {
|
|
|
9349
9552
|
rootDir,
|
|
9350
9553
|
config,
|
|
9351
9554
|
readFile: async (filepath) => {
|
|
9352
|
-
const fullPath =
|
|
9353
|
-
return
|
|
9555
|
+
const fullPath = path23.isAbsolute(filepath) ? filepath : path23.join(rootDir, filepath);
|
|
9556
|
+
return fs9.readFile(fullPath, "utf-8");
|
|
9354
9557
|
},
|
|
9355
9558
|
getFileStats: async (filepath) => {
|
|
9356
|
-
const fullPath =
|
|
9357
|
-
const stats = await
|
|
9559
|
+
const fullPath = path23.isAbsolute(filepath) ? filepath : path23.join(rootDir, filepath);
|
|
9560
|
+
const stats = await fs9.stat(fullPath);
|
|
9358
9561
|
return { lastModified: stats.mtime.toISOString() };
|
|
9359
9562
|
},
|
|
9360
9563
|
getIntrospection: (filepath) => introspection.getFile(filepath)
|
|
9361
9564
|
};
|
|
9362
9565
|
const moduleChangedFiles = module.supportsFile ? changedFiles.filter((f) => module.supportsFile(f)) : changedFiles;
|
|
9363
9566
|
const filesToProcess = moduleChangedFiles.map((filepath) => {
|
|
9364
|
-
const relativePath =
|
|
9567
|
+
const relativePath = path23.relative(rootDir, filepath);
|
|
9365
9568
|
const existingEntry = manifest.files[relativePath];
|
|
9366
9569
|
const lastModified = changedFileMtimes.get(filepath) || new Date().toISOString();
|
|
9367
9570
|
return {
|
|
@@ -9391,7 +9594,7 @@ async function ensureIndexFresh(rootDir, options = {}) {
|
|
|
9391
9594
|
return { relativePath, status: "unchanged" };
|
|
9392
9595
|
}
|
|
9393
9596
|
try {
|
|
9394
|
-
const content = await
|
|
9597
|
+
const content = await fs9.readFile(filepath, "utf-8");
|
|
9395
9598
|
const contentHash = computeContentHash(content);
|
|
9396
9599
|
if (!isNew && existingContentHash && existingContentHash === contentHash) {
|
|
9397
9600
|
completedCount++;
|
|
@@ -9518,7 +9721,7 @@ async function ensureIndexFresh(rootDir, options = {}) {
|
|
|
9518
9721
|
}
|
|
9519
9722
|
let finalManifestMtime = currentManifestMtime;
|
|
9520
9723
|
try {
|
|
9521
|
-
const manifestStats = await
|
|
9724
|
+
const manifestStats = await fs9.stat(globalManifestPath);
|
|
9522
9725
|
finalManifestMtime = manifestStats.mtimeMs;
|
|
9523
9726
|
} catch {}
|
|
9524
9727
|
freshnessCache = {
|
|
@@ -9538,7 +9741,7 @@ async function indexWithModule(rootDir, files, module, config, verbose, introspe
|
|
|
9538
9741
|
};
|
|
9539
9742
|
const manifest = await loadModuleManifest(rootDir, module.id, config);
|
|
9540
9743
|
const indexPath = getModuleIndexPath(rootDir, module.id, config);
|
|
9541
|
-
const currentFileSet = new Set(files.map((f) =>
|
|
9744
|
+
const currentFileSet = new Set(files.map((f) => path23.relative(rootDir, f)));
|
|
9542
9745
|
const filesToRemove = [];
|
|
9543
9746
|
for (const filepath of Object.keys(manifest.files)) {
|
|
9544
9747
|
if (!currentFileSet.has(filepath)) {
|
|
@@ -9549,13 +9752,13 @@ async function indexWithModule(rootDir, files, module, config, verbose, introspe
|
|
|
9549
9752
|
logger.info(` Removing ${filesToRemove.length} stale entries...`);
|
|
9550
9753
|
for (const filepath of filesToRemove) {
|
|
9551
9754
|
logger.debug(` Removing: ${filepath}`);
|
|
9552
|
-
const indexFilePath =
|
|
9755
|
+
const indexFilePath = path23.join(indexPath, filepath.replace(/\.[^.]+$/, ".json"));
|
|
9553
9756
|
try {
|
|
9554
|
-
await
|
|
9757
|
+
await fs9.unlink(indexFilePath);
|
|
9555
9758
|
} catch {}
|
|
9556
|
-
const symbolicFilePath =
|
|
9759
|
+
const symbolicFilePath = path23.join(indexPath, "symbolic", filepath.replace(/\.[^.]+$/, ".json"));
|
|
9557
9760
|
try {
|
|
9558
|
-
await
|
|
9761
|
+
await fs9.unlink(symbolicFilePath);
|
|
9559
9762
|
} catch {}
|
|
9560
9763
|
delete manifest.files[filepath];
|
|
9561
9764
|
}
|
|
@@ -9565,12 +9768,12 @@ async function indexWithModule(rootDir, files, module, config, verbose, introspe
|
|
|
9565
9768
|
rootDir,
|
|
9566
9769
|
config,
|
|
9567
9770
|
readFile: async (filepath) => {
|
|
9568
|
-
const fullPath =
|
|
9569
|
-
return
|
|
9771
|
+
const fullPath = path23.isAbsolute(filepath) ? filepath : path23.join(rootDir, filepath);
|
|
9772
|
+
return fs9.readFile(fullPath, "utf-8");
|
|
9570
9773
|
},
|
|
9571
9774
|
getFileStats: async (filepath) => {
|
|
9572
|
-
const fullPath =
|
|
9573
|
-
const stats = await
|
|
9775
|
+
const fullPath = path23.isAbsolute(filepath) ? filepath : path23.join(rootDir, filepath);
|
|
9776
|
+
const stats = await fs9.stat(fullPath);
|
|
9574
9777
|
return { lastModified: stats.mtime.toISOString() };
|
|
9575
9778
|
},
|
|
9576
9779
|
getIntrospection: (filepath) => introspection.getFile(filepath)
|
|
@@ -9582,9 +9785,9 @@ async function indexWithModule(rootDir, files, module, config, verbose, introspe
|
|
|
9582
9785
|
let indexedCount = 0;
|
|
9583
9786
|
let skippedCount = 0;
|
|
9584
9787
|
const processFile = async (filepath, _index) => {
|
|
9585
|
-
const relativePath =
|
|
9788
|
+
const relativePath = path23.relative(rootDir, filepath);
|
|
9586
9789
|
try {
|
|
9587
|
-
const stats = await
|
|
9790
|
+
const stats = await fs9.stat(filepath);
|
|
9588
9791
|
const lastModified = stats.mtime.toISOString();
|
|
9589
9792
|
const existingEntry = manifest.files[relativePath];
|
|
9590
9793
|
if (existingEntry && existingEntry.lastModified === lastModified) {
|
|
@@ -9594,7 +9797,7 @@ async function indexWithModule(rootDir, files, module, config, verbose, introspe
|
|
|
9594
9797
|
logger.debug(` [${completedCount}/${totalFiles}] Skipped ${relativePath} (unchanged)`);
|
|
9595
9798
|
return { relativePath, status: "skipped" };
|
|
9596
9799
|
}
|
|
9597
|
-
const content = await
|
|
9800
|
+
const content = await fs9.readFile(filepath, "utf-8");
|
|
9598
9801
|
const contentHash = computeContentHash(content);
|
|
9599
9802
|
if (existingEntry?.contentHash && existingEntry.contentHash === contentHash) {
|
|
9600
9803
|
completedCount++;
|
|
@@ -9676,8 +9879,8 @@ async function indexWithModule(rootDir, files, module, config, verbose, introspe
|
|
|
9676
9879
|
}
|
|
9677
9880
|
var STAT_CONCURRENCY = 64;
|
|
9678
9881
|
function isLikelyBinary(filepath) {
|
|
9679
|
-
const ext =
|
|
9680
|
-
const basename15 =
|
|
9882
|
+
const ext = path23.extname(filepath).toLowerCase();
|
|
9883
|
+
const basename15 = path23.basename(filepath).toLowerCase();
|
|
9681
9884
|
const binaryExtensions = new Set([
|
|
9682
9885
|
".png",
|
|
9683
9886
|
".jpg",
|
|
@@ -9753,7 +9956,7 @@ async function findFilesWithStats(rootDir, config, lastIndexStarted) {
|
|
|
9753
9956
|
const ignoreDirs = new Set(config.ignorePaths);
|
|
9754
9957
|
const lastIndexMs = lastIndexStarted?.getTime() ?? 0;
|
|
9755
9958
|
const crawler = new Builder().withFullPaths().exclude((dirName) => ignoreDirs.has(dirName)).filter((filePath) => {
|
|
9756
|
-
const ext =
|
|
9959
|
+
const ext = path23.extname(filePath);
|
|
9757
9960
|
return validExtensions.has(ext);
|
|
9758
9961
|
}).crawl(rootDir);
|
|
9759
9962
|
const allFiles = await crawler.withPromise();
|
|
@@ -9761,7 +9964,7 @@ async function findFilesWithStats(rootDir, config, lastIndexStarted) {
|
|
|
9761
9964
|
const changedFileMtimes2 = new Map;
|
|
9762
9965
|
await parallelMap(allFiles, async (filePath) => {
|
|
9763
9966
|
try {
|
|
9764
|
-
const stats = await
|
|
9967
|
+
const stats = await fs9.stat(filePath);
|
|
9765
9968
|
changedFileMtimes2.set(filePath, stats.mtime.toISOString());
|
|
9766
9969
|
} catch {}
|
|
9767
9970
|
}, STAT_CONCURRENCY);
|
|
@@ -9775,7 +9978,7 @@ async function findFilesWithStats(rootDir, config, lastIndexStarted) {
|
|
|
9775
9978
|
const changedFileMtimes = new Map;
|
|
9776
9979
|
await parallelMap(allFiles, async (filePath) => {
|
|
9777
9980
|
try {
|
|
9778
|
-
const stats = await
|
|
9981
|
+
const stats = await fs9.stat(filePath);
|
|
9779
9982
|
if (stats.mtimeMs > lastIndexMs) {
|
|
9780
9983
|
changedFiles.push(filePath);
|
|
9781
9984
|
changedFileMtimes.set(filePath, stats.mtime.toISOString());
|
|
@@ -9795,7 +9998,7 @@ async function findFiles(rootDir, config) {
|
|
|
9795
9998
|
async function loadModuleManifest(rootDir, moduleId, config) {
|
|
9796
9999
|
const manifestPath = getModuleManifestPath(rootDir, moduleId, config);
|
|
9797
10000
|
try {
|
|
9798
|
-
const content = await
|
|
10001
|
+
const content = await fs9.readFile(manifestPath, "utf-8");
|
|
9799
10002
|
return JSON.parse(content);
|
|
9800
10003
|
} catch {
|
|
9801
10004
|
return {
|
|
@@ -9808,19 +10011,19 @@ async function loadModuleManifest(rootDir, moduleId, config) {
|
|
|
9808
10011
|
}
|
|
9809
10012
|
async function writeModuleManifest(rootDir, moduleId, manifest, config) {
|
|
9810
10013
|
const manifestPath = getModuleManifestPath(rootDir, moduleId, config);
|
|
9811
|
-
await
|
|
9812
|
-
await
|
|
10014
|
+
await fs9.mkdir(path23.dirname(manifestPath), { recursive: true });
|
|
10015
|
+
await fs9.writeFile(manifestPath, JSON.stringify(manifest, null, 2));
|
|
9813
10016
|
}
|
|
9814
10017
|
async function writeFileIndex(rootDir, moduleId, filepath, fileIndex, config) {
|
|
9815
10018
|
const indexPath = getModuleIndexPath(rootDir, moduleId, config);
|
|
9816
|
-
const indexFilePath =
|
|
9817
|
-
await
|
|
9818
|
-
await
|
|
10019
|
+
const indexFilePath = path23.join(indexPath, filepath.replace(/\.[^.]+$/, ".json"));
|
|
10020
|
+
await fs9.mkdir(path23.dirname(indexFilePath), { recursive: true });
|
|
10021
|
+
await fs9.writeFile(indexFilePath, JSON.stringify(fileIndex, null, 2));
|
|
9819
10022
|
}
|
|
9820
10023
|
async function loadGlobalManifest(rootDir, config) {
|
|
9821
10024
|
const manifestPath = getGlobalManifestPath(rootDir, config);
|
|
9822
10025
|
try {
|
|
9823
|
-
const content = await
|
|
10026
|
+
const content = await fs9.readFile(manifestPath, "utf-8");
|
|
9824
10027
|
return JSON.parse(content);
|
|
9825
10028
|
} catch {
|
|
9826
10029
|
return null;
|
|
@@ -9834,13 +10037,13 @@ async function updateGlobalManifest(rootDir, modules, config, indexStartTime) {
|
|
|
9834
10037
|
lastIndexStarted: indexStartTime,
|
|
9835
10038
|
modules: modules.map((m) => m.id)
|
|
9836
10039
|
};
|
|
9837
|
-
await
|
|
9838
|
-
await
|
|
10040
|
+
await fs9.mkdir(path23.dirname(manifestPath), { recursive: true });
|
|
10041
|
+
await fs9.writeFile(manifestPath, JSON.stringify(manifest, null, 2));
|
|
9839
10042
|
}
|
|
9840
10043
|
async function cleanupIndex(rootDir, options = {}) {
|
|
9841
10044
|
const verbose = options.verbose ?? false;
|
|
9842
10045
|
const logger = options.logger ?? createLogger({ verbose });
|
|
9843
|
-
rootDir =
|
|
10046
|
+
rootDir = path23.resolve(rootDir);
|
|
9844
10047
|
logger.info(`Cleaning up index in: ${rootDir}`);
|
|
9845
10048
|
const config = await loadConfig(rootDir);
|
|
9846
10049
|
await registerBuiltInModules();
|
|
@@ -9870,9 +10073,9 @@ async function cleanupModuleIndex(rootDir, moduleId, config, logger) {
|
|
|
9870
10073
|
const filesToRemove = [];
|
|
9871
10074
|
const updatedFiles = {};
|
|
9872
10075
|
for (const [filepath, entry] of Object.entries(manifest.files)) {
|
|
9873
|
-
const fullPath =
|
|
10076
|
+
const fullPath = path23.join(rootDir, filepath);
|
|
9874
10077
|
try {
|
|
9875
|
-
await
|
|
10078
|
+
await fs9.access(fullPath);
|
|
9876
10079
|
updatedFiles[filepath] = entry;
|
|
9877
10080
|
result.kept++;
|
|
9878
10081
|
} catch {
|
|
@@ -9882,9 +10085,9 @@ async function cleanupModuleIndex(rootDir, moduleId, config, logger) {
|
|
|
9882
10085
|
}
|
|
9883
10086
|
}
|
|
9884
10087
|
for (const filepath of filesToRemove) {
|
|
9885
|
-
const indexFilePath =
|
|
10088
|
+
const indexFilePath = path23.join(indexPath, filepath.replace(/\.[^.]+$/, ".json"));
|
|
9886
10089
|
try {
|
|
9887
|
-
await
|
|
10090
|
+
await fs9.unlink(indexFilePath);
|
|
9888
10091
|
} catch {}
|
|
9889
10092
|
}
|
|
9890
10093
|
manifest.files = updatedFiles;
|
|
@@ -9895,16 +10098,16 @@ async function cleanupModuleIndex(rootDir, moduleId, config, logger) {
|
|
|
9895
10098
|
}
|
|
9896
10099
|
async function cleanupEmptyDirectories(dir) {
|
|
9897
10100
|
try {
|
|
9898
|
-
const entries = await
|
|
10101
|
+
const entries = await fs9.readdir(dir, { withFileTypes: true });
|
|
9899
10102
|
for (const entry of entries) {
|
|
9900
10103
|
if (entry.isDirectory()) {
|
|
9901
|
-
const subDir =
|
|
10104
|
+
const subDir = path23.join(dir, entry.name);
|
|
9902
10105
|
await cleanupEmptyDirectories(subDir);
|
|
9903
10106
|
}
|
|
9904
10107
|
}
|
|
9905
|
-
const remainingEntries = await
|
|
10108
|
+
const remainingEntries = await fs9.readdir(dir);
|
|
9906
10109
|
if (remainingEntries.length === 0) {
|
|
9907
|
-
await
|
|
10110
|
+
await fs9.rmdir(dir);
|
|
9908
10111
|
return true;
|
|
9909
10112
|
}
|
|
9910
10113
|
return false;
|
|
@@ -9913,7 +10116,7 @@ async function cleanupEmptyDirectories(dir) {
|
|
|
9913
10116
|
}
|
|
9914
10117
|
}
|
|
9915
10118
|
async function getIndexStatus(rootDir) {
|
|
9916
|
-
rootDir =
|
|
10119
|
+
rootDir = path23.resolve(rootDir);
|
|
9917
10120
|
const config = await loadConfig(rootDir);
|
|
9918
10121
|
const location = getIndexLocation(rootDir);
|
|
9919
10122
|
const indexDir = location.indexDir;
|
|
@@ -9925,13 +10128,13 @@ async function getIndexStatus(rootDir) {
|
|
|
9925
10128
|
totalFiles: 0
|
|
9926
10129
|
};
|
|
9927
10130
|
try {
|
|
9928
|
-
await
|
|
10131
|
+
await fs9.access(indexDir);
|
|
9929
10132
|
} catch {
|
|
9930
10133
|
return status;
|
|
9931
10134
|
}
|
|
9932
10135
|
try {
|
|
9933
10136
|
const globalManifestPath = getGlobalManifestPath(rootDir, config);
|
|
9934
|
-
const content = await
|
|
10137
|
+
const content = await fs9.readFile(globalManifestPath, "utf-8");
|
|
9935
10138
|
const globalManifest = JSON.parse(content);
|
|
9936
10139
|
status.exists = true;
|
|
9937
10140
|
status.lastUpdated = globalManifest.lastUpdated;
|
|
@@ -9949,7 +10152,7 @@ async function getIndexStatus(rootDir) {
|
|
|
9949
10152
|
}
|
|
9950
10153
|
} catch {
|
|
9951
10154
|
try {
|
|
9952
|
-
const entries = await
|
|
10155
|
+
const entries = await fs9.readdir(path23.join(indexDir, "index"));
|
|
9953
10156
|
if (entries.length > 0) {
|
|
9954
10157
|
status.exists = true;
|
|
9955
10158
|
for (const entry of entries) {
|
|
@@ -9971,8 +10174,8 @@ async function getIndexStatus(rootDir) {
|
|
|
9971
10174
|
}
|
|
9972
10175
|
|
|
9973
10176
|
// src/app/search/index.ts
|
|
9974
|
-
import * as
|
|
9975
|
-
import * as
|
|
10177
|
+
import * as fs11 from "fs/promises";
|
|
10178
|
+
import * as path26 from "path";
|
|
9976
10179
|
|
|
9977
10180
|
// node_modules/@isaacs/balanced-match/dist/esm/index.js
|
|
9978
10181
|
var balanced = (a, b, str) => {
|
|
@@ -10775,11 +10978,11 @@ var qmarksTestNoExtDot = ([$0]) => {
|
|
|
10775
10978
|
return (f) => f.length === len && f !== "." && f !== "..";
|
|
10776
10979
|
};
|
|
10777
10980
|
var defaultPlatform = typeof process === "object" && process ? typeof process.env === "object" && process.env && process.env.__MINIMATCH_TESTING_PLATFORM__ || process.platform : "posix";
|
|
10778
|
-
var
|
|
10981
|
+
var path24 = {
|
|
10779
10982
|
win32: { sep: "\\" },
|
|
10780
10983
|
posix: { sep: "/" }
|
|
10781
10984
|
};
|
|
10782
|
-
var sep2 = defaultPlatform === "win32" ?
|
|
10985
|
+
var sep2 = defaultPlatform === "win32" ? path24.win32.sep : path24.posix.sep;
|
|
10783
10986
|
minimatch.sep = sep2;
|
|
10784
10987
|
var GLOBSTAR = Symbol("globstar **");
|
|
10785
10988
|
minimatch.GLOBSTAR = GLOBSTAR;
|
|
@@ -11408,10 +11611,8 @@ minimatch.Minimatch = Minimatch;
|
|
|
11408
11611
|
minimatch.escape = escape;
|
|
11409
11612
|
minimatch.unescape = unescape;
|
|
11410
11613
|
|
|
11411
|
-
// src/types.ts
|
|
11412
|
-
init_entities();
|
|
11413
|
-
|
|
11414
11614
|
// src/app/search/index.ts
|
|
11615
|
+
init_types();
|
|
11415
11616
|
init_config2();
|
|
11416
11617
|
init_services();
|
|
11417
11618
|
// src/domain/usecases/exactSearch.ts
|
|
@@ -11445,7 +11646,7 @@ function matchesPathFilter(relativePath, filters, matchFn) {
|
|
|
11445
11646
|
}
|
|
11446
11647
|
return false;
|
|
11447
11648
|
}
|
|
11448
|
-
async function executeExactSearch(
|
|
11649
|
+
async function executeExactSearch(fs10, options, matchFn) {
|
|
11449
11650
|
const {
|
|
11450
11651
|
rootDir,
|
|
11451
11652
|
literal,
|
|
@@ -11457,13 +11658,13 @@ async function executeExactSearch(fs9, options, matchFn) {
|
|
|
11457
11658
|
const files = new Map;
|
|
11458
11659
|
async function walkDir(dir, baseDir) {
|
|
11459
11660
|
try {
|
|
11460
|
-
const entries = await
|
|
11661
|
+
const entries = await fs10.readDir(dir);
|
|
11461
11662
|
for (const entry of entries) {
|
|
11462
|
-
const fullPath =
|
|
11463
|
-
const relativePath =
|
|
11663
|
+
const fullPath = fs10.join(dir, entry);
|
|
11664
|
+
const relativePath = fs10.relative(baseDir, fullPath);
|
|
11464
11665
|
let isDirectory = false;
|
|
11465
11666
|
try {
|
|
11466
|
-
const stats = await
|
|
11667
|
+
const stats = await fs10.getStats(fullPath);
|
|
11467
11668
|
isDirectory = stats.isDirectory ?? false;
|
|
11468
11669
|
} catch {
|
|
11469
11670
|
continue;
|
|
@@ -11480,7 +11681,7 @@ async function executeExactSearch(fs9, options, matchFn) {
|
|
|
11480
11681
|
}
|
|
11481
11682
|
}
|
|
11482
11683
|
try {
|
|
11483
|
-
const content = await
|
|
11684
|
+
const content = await fs10.readFile(fullPath);
|
|
11484
11685
|
if (isSearchableContent(content, fullPath)) {
|
|
11485
11686
|
files.set(relativePath, content);
|
|
11486
11687
|
}
|
|
@@ -11497,21 +11698,21 @@ async function executeExactSearch(fs9, options, matchFn) {
|
|
|
11497
11698
|
});
|
|
11498
11699
|
}
|
|
11499
11700
|
// src/infrastructure/filesystem/nodeFileSystem.ts
|
|
11500
|
-
import * as
|
|
11501
|
-
import * as
|
|
11701
|
+
import * as fs10 from "fs/promises";
|
|
11702
|
+
import * as path25 from "path";
|
|
11502
11703
|
import { glob } from "glob";
|
|
11503
11704
|
|
|
11504
11705
|
class NodeFileSystem {
|
|
11505
11706
|
async readFile(filepath) {
|
|
11506
|
-
return
|
|
11707
|
+
return fs10.readFile(filepath, "utf-8");
|
|
11507
11708
|
}
|
|
11508
11709
|
async writeFile(filepath, content) {
|
|
11509
|
-
await
|
|
11510
|
-
await
|
|
11710
|
+
await fs10.mkdir(path25.dirname(filepath), { recursive: true });
|
|
11711
|
+
await fs10.writeFile(filepath, content, "utf-8");
|
|
11511
11712
|
}
|
|
11512
11713
|
async deleteFile(filepath) {
|
|
11513
11714
|
try {
|
|
11514
|
-
await
|
|
11715
|
+
await fs10.unlink(filepath);
|
|
11515
11716
|
} catch (error) {
|
|
11516
11717
|
if (error.code !== "ENOENT") {
|
|
11517
11718
|
throw error;
|
|
@@ -11519,7 +11720,7 @@ class NodeFileSystem {
|
|
|
11519
11720
|
}
|
|
11520
11721
|
}
|
|
11521
11722
|
async getStats(filepath) {
|
|
11522
|
-
const stats = await
|
|
11723
|
+
const stats = await fs10.stat(filepath);
|
|
11523
11724
|
return {
|
|
11524
11725
|
lastModified: stats.mtime.toISOString(),
|
|
11525
11726
|
size: stats.isDirectory() ? undefined : stats.size,
|
|
@@ -11528,17 +11729,17 @@ class NodeFileSystem {
|
|
|
11528
11729
|
}
|
|
11529
11730
|
async exists(filepath) {
|
|
11530
11731
|
try {
|
|
11531
|
-
await
|
|
11732
|
+
await fs10.access(filepath);
|
|
11532
11733
|
return true;
|
|
11533
11734
|
} catch {
|
|
11534
11735
|
return false;
|
|
11535
11736
|
}
|
|
11536
11737
|
}
|
|
11537
11738
|
async mkdir(dirpath) {
|
|
11538
|
-
await
|
|
11739
|
+
await fs10.mkdir(dirpath, { recursive: true });
|
|
11539
11740
|
}
|
|
11540
11741
|
async readDir(dirpath) {
|
|
11541
|
-
return
|
|
11742
|
+
return fs10.readdir(dirpath);
|
|
11542
11743
|
}
|
|
11543
11744
|
async findFiles(rootDir, patterns, ignore) {
|
|
11544
11745
|
const ignorePatterns = ignore.map((p) => `**/${p}/**`);
|
|
@@ -11554,19 +11755,19 @@ class NodeFileSystem {
|
|
|
11554
11755
|
return [...new Set(files)];
|
|
11555
11756
|
}
|
|
11556
11757
|
join(...segments) {
|
|
11557
|
-
return
|
|
11758
|
+
return path25.join(...segments);
|
|
11558
11759
|
}
|
|
11559
11760
|
relative(from, to) {
|
|
11560
|
-
return
|
|
11761
|
+
return path25.relative(from, to);
|
|
11561
11762
|
}
|
|
11562
11763
|
resolve(...segments) {
|
|
11563
|
-
return
|
|
11764
|
+
return path25.resolve(...segments);
|
|
11564
11765
|
}
|
|
11565
11766
|
dirname(filepath) {
|
|
11566
|
-
return
|
|
11767
|
+
return path25.dirname(filepath);
|
|
11567
11768
|
}
|
|
11568
11769
|
extname(filepath) {
|
|
11569
|
-
return
|
|
11770
|
+
return path25.extname(filepath);
|
|
11570
11771
|
}
|
|
11571
11772
|
}
|
|
11572
11773
|
var nodeFileSystem = new NodeFileSystem;
|
|
@@ -11576,7 +11777,7 @@ async function search(rootDir, query, options = {}) {
|
|
|
11576
11777
|
return hybridResults.results;
|
|
11577
11778
|
}
|
|
11578
11779
|
async function hybridSearch(rootDir, query, options = {}) {
|
|
11579
|
-
rootDir =
|
|
11780
|
+
rootDir = path26.resolve(rootDir);
|
|
11580
11781
|
const ensureFresh = options.ensureFresh ?? DEFAULT_SEARCH_OPTIONS.ensureFresh;
|
|
11581
11782
|
if (ensureFresh) {
|
|
11582
11783
|
await ensureIndexFresh(rootDir, { quiet: true });
|
|
@@ -11653,15 +11854,15 @@ async function hybridSearch(rootDir, query, options = {}) {
|
|
|
11653
11854
|
};
|
|
11654
11855
|
}
|
|
11655
11856
|
async function performExactSearch(rootDir, literal, config, options) {
|
|
11656
|
-
const
|
|
11657
|
-
return executeExactSearch(
|
|
11857
|
+
const fs12 = new NodeFileSystem;
|
|
11858
|
+
return executeExactSearch(fs12, {
|
|
11658
11859
|
rootDir,
|
|
11659
11860
|
literal,
|
|
11660
11861
|
pathFilter: options.pathFilter,
|
|
11661
11862
|
maxFiles: 20,
|
|
11662
11863
|
maxOccurrencesPerFile: 5,
|
|
11663
11864
|
caseInsensitive: false
|
|
11664
|
-
}, (
|
|
11865
|
+
}, (path27, pattern) => minimatch(path27, pattern, { matchBase: true }));
|
|
11665
11866
|
}
|
|
11666
11867
|
function createSearchContext(rootDir, moduleId, config) {
|
|
11667
11868
|
const indexPath = getModuleIndexPath(rootDir, moduleId, config);
|
|
@@ -11670,9 +11871,9 @@ function createSearchContext(rootDir, moduleId, config) {
|
|
|
11670
11871
|
config,
|
|
11671
11872
|
loadFileIndex: async (filepath) => {
|
|
11672
11873
|
const hasExtension = /\.[^./]+$/.test(filepath);
|
|
11673
|
-
const indexFilePath = hasExtension ?
|
|
11874
|
+
const indexFilePath = hasExtension ? path26.join(indexPath, filepath.replace(/\.[^.]+$/, ".json")) : path26.join(indexPath, filepath + ".json");
|
|
11674
11875
|
try {
|
|
11675
|
-
const content = await
|
|
11876
|
+
const content = await fs11.readFile(indexFilePath, "utf-8");
|
|
11676
11877
|
return JSON.parse(content);
|
|
11677
11878
|
} catch {
|
|
11678
11879
|
return null;
|
|
@@ -11682,7 +11883,7 @@ function createSearchContext(rootDir, moduleId, config) {
|
|
|
11682
11883
|
const files = [];
|
|
11683
11884
|
await traverseDirectory(indexPath, files, indexPath);
|
|
11684
11885
|
return files.filter((f) => f.endsWith(".json") && !f.endsWith("manifest.json")).map((f) => {
|
|
11685
|
-
const relative5 =
|
|
11886
|
+
const relative5 = path26.relative(indexPath, f);
|
|
11686
11887
|
return relative5.replace(/\.json$/, "");
|
|
11687
11888
|
});
|
|
11688
11889
|
}
|
|
@@ -11690,9 +11891,9 @@ function createSearchContext(rootDir, moduleId, config) {
|
|
|
11690
11891
|
}
|
|
11691
11892
|
async function traverseDirectory(dir, files, basePath) {
|
|
11692
11893
|
try {
|
|
11693
|
-
const entries = await
|
|
11894
|
+
const entries = await fs11.readdir(dir, { withFileTypes: true });
|
|
11694
11895
|
for (const entry of entries) {
|
|
11695
|
-
const fullPath =
|
|
11896
|
+
const fullPath = path26.join(dir, entry.name);
|
|
11696
11897
|
if (entry.isDirectory()) {
|
|
11697
11898
|
await traverseDirectory(fullPath, files, basePath);
|
|
11698
11899
|
} else if (entry.isFile()) {
|
|
@@ -11704,7 +11905,7 @@ async function traverseDirectory(dir, files, basePath) {
|
|
|
11704
11905
|
async function loadGlobalManifest2(rootDir, config) {
|
|
11705
11906
|
const manifestPath = getGlobalManifestPath(rootDir, config);
|
|
11706
11907
|
try {
|
|
11707
|
-
const content = await
|
|
11908
|
+
const content = await fs11.readFile(manifestPath, "utf-8");
|
|
11708
11909
|
return JSON.parse(content);
|
|
11709
11910
|
} catch {
|
|
11710
11911
|
return null;
|
|
@@ -11873,4 +12074,4 @@ export {
|
|
|
11873
12074
|
ConsoleLogger
|
|
11874
12075
|
};
|
|
11875
12076
|
|
|
11876
|
-
//# debugId=
|
|
12077
|
+
//# debugId=C5BB91833412BA2664756E2164756E21
|