raggrep 0.15.0 → 0.17.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +112 -8
- package/dist/cli/main.js +827 -521
- package/dist/cli/main.js.map +26 -19
- package/dist/domain/ports/embedding.d.ts +10 -0
- package/dist/domain/ports/index.d.ts +1 -1
- package/dist/domain/services/chunkContext.d.ts +76 -0
- package/dist/domain/services/index.d.ts +1 -0
- package/dist/index.js +638 -390
- package/dist/index.js.map +25 -18
- package/dist/infrastructure/config/configLoader.d.ts +9 -11
- package/dist/infrastructure/config/index.d.ts +1 -1
- package/dist/infrastructure/embeddings/embeddingPaths.d.ts +6 -0
- package/dist/infrastructure/embeddings/embeddingProviderFactory.d.ts +9 -0
- package/dist/infrastructure/embeddings/globalEmbeddings.d.ts +28 -0
- package/dist/infrastructure/embeddings/huggingfaceEmbeddingProvider.d.ts +21 -0
- package/dist/infrastructure/embeddings/index.d.ts +9 -2
- package/dist/infrastructure/embeddings/modelCache.d.ts +10 -0
- package/dist/infrastructure/embeddings/modelCatalog.d.ts +23 -0
- package/dist/infrastructure/embeddings/xenovaEmbeddingProvider.d.ts +23 -0
- package/dist/infrastructure/index.d.ts +1 -1
- package/package.json +7 -3
- package/dist/infrastructure/embeddings/transformersEmbedding.d.ts +0 -52
package/dist/index.js
CHANGED
|
@@ -1,27 +1,16 @@
|
|
|
1
1
|
import { createRequire } from "node:module";
|
|
2
|
-
var __create = Object.create;
|
|
3
|
-
var __getProtoOf = Object.getPrototypeOf;
|
|
4
2
|
var __defProp = Object.defineProperty;
|
|
5
|
-
var
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
const to = isNodeMode || !mod || !mod.__esModule ? __defProp(target, "default", { value: mod, enumerable: true }) : target;
|
|
10
|
-
for (let key of __getOwnPropNames(mod))
|
|
11
|
-
if (!__hasOwnProp.call(to, key))
|
|
12
|
-
__defProp(to, key, {
|
|
13
|
-
get: () => mod[key],
|
|
14
|
-
enumerable: true
|
|
15
|
-
});
|
|
16
|
-
return to;
|
|
17
|
-
};
|
|
3
|
+
var __returnValue = (v) => v;
|
|
4
|
+
function __exportSetter(name, newValue) {
|
|
5
|
+
this[name] = __returnValue.bind(null, newValue);
|
|
6
|
+
}
|
|
18
7
|
var __export = (target, all) => {
|
|
19
8
|
for (var name in all)
|
|
20
9
|
__defProp(target, name, {
|
|
21
10
|
get: all[name],
|
|
22
11
|
enumerable: true,
|
|
23
12
|
configurable: true,
|
|
24
|
-
set: (
|
|
13
|
+
set: __exportSetter.bind(all, name)
|
|
25
14
|
});
|
|
26
15
|
};
|
|
27
16
|
var __esm = (fn, res) => () => (fn && (res = fn(fn = 0)), res);
|
|
@@ -54,49 +43,37 @@ function createDefaultConfig() {
|
|
|
54
43
|
{
|
|
55
44
|
id: "language/typescript",
|
|
56
45
|
enabled: true,
|
|
57
|
-
options: {
|
|
58
|
-
embeddingModel: "all-MiniLM-L6-v2"
|
|
59
|
-
}
|
|
46
|
+
options: { ...DEFAULT_EMBEDDING_MODULE_OPTIONS }
|
|
60
47
|
},
|
|
61
48
|
{
|
|
62
49
|
id: "language/python",
|
|
63
50
|
enabled: true,
|
|
64
|
-
options: {
|
|
65
|
-
embeddingModel: "all-MiniLM-L6-v2"
|
|
66
|
-
}
|
|
51
|
+
options: { ...DEFAULT_EMBEDDING_MODULE_OPTIONS }
|
|
67
52
|
},
|
|
68
53
|
{
|
|
69
54
|
id: "language/go",
|
|
70
55
|
enabled: true,
|
|
71
|
-
options: {
|
|
72
|
-
embeddingModel: "all-MiniLM-L6-v2"
|
|
73
|
-
}
|
|
56
|
+
options: { ...DEFAULT_EMBEDDING_MODULE_OPTIONS }
|
|
74
57
|
},
|
|
75
58
|
{
|
|
76
59
|
id: "language/rust",
|
|
77
60
|
enabled: true,
|
|
78
|
-
options: {
|
|
79
|
-
embeddingModel: "all-MiniLM-L6-v2"
|
|
80
|
-
}
|
|
61
|
+
options: { ...DEFAULT_EMBEDDING_MODULE_OPTIONS }
|
|
81
62
|
},
|
|
82
63
|
{
|
|
83
64
|
id: "data/json",
|
|
84
65
|
enabled: true,
|
|
85
|
-
options: {
|
|
86
|
-
embeddingModel: "all-MiniLM-L6-v2"
|
|
87
|
-
}
|
|
66
|
+
options: { ...DEFAULT_EMBEDDING_MODULE_OPTIONS }
|
|
88
67
|
},
|
|
89
68
|
{
|
|
90
69
|
id: "docs/markdown",
|
|
91
70
|
enabled: true,
|
|
92
|
-
options: {
|
|
93
|
-
embeddingModel: "all-MiniLM-L6-v2"
|
|
94
|
-
}
|
|
71
|
+
options: { ...DEFAULT_EMBEDDING_MODULE_OPTIONS }
|
|
95
72
|
}
|
|
96
73
|
]
|
|
97
74
|
};
|
|
98
75
|
}
|
|
99
|
-
var DEFAULT_IGNORE_PATHS, DEFAULT_EXTENSIONS;
|
|
76
|
+
var DEFAULT_IGNORE_PATHS, DEFAULT_EXTENSIONS, DEFAULT_EMBEDDING_MODULE_OPTIONS;
|
|
100
77
|
var init_config = __esm(() => {
|
|
101
78
|
DEFAULT_IGNORE_PATHS = [
|
|
102
79
|
"node_modules",
|
|
@@ -166,6 +143,10 @@ var init_config = __esm(() => {
|
|
|
166
143
|
".sql",
|
|
167
144
|
".txt"
|
|
168
145
|
];
|
|
146
|
+
DEFAULT_EMBEDDING_MODULE_OPTIONS = {
|
|
147
|
+
embeddingModel: "bge-small-en-v1.5",
|
|
148
|
+
embeddingRuntime: "huggingface"
|
|
149
|
+
};
|
|
169
150
|
});
|
|
170
151
|
|
|
171
152
|
// src/domain/entities/literal.ts
|
|
@@ -195,24 +176,48 @@ var init_entities = __esm(() => {
|
|
|
195
176
|
init_lexicon();
|
|
196
177
|
});
|
|
197
178
|
|
|
179
|
+
// src/infrastructure/embeddings/modelCatalog.ts
|
|
180
|
+
function getEmbeddingModelId(model) {
|
|
181
|
+
return EMBEDDING_MODEL_IDS[model];
|
|
182
|
+
}
|
|
183
|
+
function getEmbeddingDimension(model) {
|
|
184
|
+
return EMBEDDING_DIMENSIONS[model];
|
|
185
|
+
}
|
|
186
|
+
var EMBEDDING_MODEL_IDS, EMBEDDING_MODELS, EMBEDDING_DIMENSIONS;
|
|
187
|
+
var init_modelCatalog = __esm(() => {
|
|
188
|
+
EMBEDDING_MODEL_IDS = {
|
|
189
|
+
"all-MiniLM-L6-v2": "Xenova/all-MiniLM-L6-v2",
|
|
190
|
+
"all-MiniLM-L12-v2": "Xenova/all-MiniLM-L12-v2",
|
|
191
|
+
"bge-small-en-v1.5": "Xenova/bge-small-en-v1.5",
|
|
192
|
+
"paraphrase-MiniLM-L3-v2": "Xenova/paraphrase-MiniLM-L3-v2",
|
|
193
|
+
"nomic-embed-text-v1.5": "nomic-ai/nomic-embed-text-v1.5"
|
|
194
|
+
};
|
|
195
|
+
EMBEDDING_MODELS = EMBEDDING_MODEL_IDS;
|
|
196
|
+
EMBEDDING_DIMENSIONS = {
|
|
197
|
+
"all-MiniLM-L6-v2": 384,
|
|
198
|
+
"all-MiniLM-L12-v2": 384,
|
|
199
|
+
"bge-small-en-v1.5": 384,
|
|
200
|
+
"paraphrase-MiniLM-L3-v2": 384,
|
|
201
|
+
"nomic-embed-text-v1.5": 768
|
|
202
|
+
};
|
|
203
|
+
});
|
|
204
|
+
|
|
198
205
|
// src/infrastructure/config/configLoader.ts
|
|
199
206
|
import * as path from "path";
|
|
200
207
|
import * as fs from "fs/promises";
|
|
201
|
-
import * as os from "os";
|
|
202
208
|
import * as crypto from "crypto";
|
|
203
209
|
function hashPath(inputPath) {
|
|
204
210
|
return crypto.createHash("sha256").update(inputPath).digest("hex").slice(0, 12);
|
|
205
211
|
}
|
|
206
212
|
function getRaggrepDir(rootDir, _config = DEFAULT_CONFIG) {
|
|
207
213
|
const absoluteRoot = path.resolve(rootDir);
|
|
208
|
-
|
|
209
|
-
return path.join(RAGGREP_TEMP_BASE, projectHash);
|
|
214
|
+
return path.join(absoluteRoot, RAGGREP_INDEX_DIR);
|
|
210
215
|
}
|
|
211
216
|
function getIndexLocation(rootDir) {
|
|
212
217
|
const absoluteRoot = path.resolve(rootDir);
|
|
213
218
|
const projectHash = hashPath(absoluteRoot);
|
|
214
219
|
return {
|
|
215
|
-
indexDir: path.join(
|
|
220
|
+
indexDir: path.join(absoluteRoot, RAGGREP_INDEX_DIR),
|
|
216
221
|
projectRoot: absoluteRoot,
|
|
217
222
|
projectHash
|
|
218
223
|
};
|
|
@@ -253,23 +258,24 @@ function getEmbeddingConfigFromModule(moduleConfig) {
|
|
|
253
258
|
console.warn(`Unknown embedding model: ${modelName}, falling back to bge-small-en-v1.5`);
|
|
254
259
|
return { model: "bge-small-en-v1.5" };
|
|
255
260
|
}
|
|
261
|
+
const rt = options.embeddingRuntime;
|
|
262
|
+
let runtime;
|
|
263
|
+
if (rt === "xenova" || rt === "huggingface") {
|
|
264
|
+
runtime = rt;
|
|
265
|
+
} else if (rt !== undefined) {
|
|
266
|
+
console.warn(`Unknown embeddingRuntime: ${rt}, falling back to default (huggingface)`);
|
|
267
|
+
}
|
|
256
268
|
return {
|
|
257
269
|
model: modelName,
|
|
270
|
+
...runtime ? { runtime } : {},
|
|
258
271
|
showProgress: options.showProgress === true
|
|
259
272
|
};
|
|
260
273
|
}
|
|
261
|
-
var DEFAULT_CONFIG,
|
|
274
|
+
var DEFAULT_CONFIG, RAGGREP_INDEX_DIR = ".raggrep";
|
|
262
275
|
var init_configLoader = __esm(() => {
|
|
263
276
|
init_entities();
|
|
277
|
+
init_modelCatalog();
|
|
264
278
|
DEFAULT_CONFIG = createDefaultConfig();
|
|
265
|
-
RAGGREP_TEMP_BASE = path.join(os.tmpdir(), "raggrep-indexes");
|
|
266
|
-
EMBEDDING_MODELS = {
|
|
267
|
-
"all-MiniLM-L6-v2": "Xenova/all-MiniLM-L6-v2",
|
|
268
|
-
"all-MiniLM-L12-v2": "Xenova/all-MiniLM-L12-v2",
|
|
269
|
-
"bge-small-en-v1.5": "Xenova/bge-small-en-v1.5",
|
|
270
|
-
"paraphrase-MiniLM-L3-v2": "Xenova/paraphrase-MiniLM-L3-v2",
|
|
271
|
-
"nomic-embed-text-v1.5": "nomic-ai/nomic-embed-text-v1.5"
|
|
272
|
-
};
|
|
273
279
|
});
|
|
274
280
|
|
|
275
281
|
// src/infrastructure/config/index.ts
|
|
@@ -2137,22 +2143,47 @@ var init_core = __esm(() => {
|
|
|
2137
2143
|
init_symbols();
|
|
2138
2144
|
});
|
|
2139
2145
|
|
|
2140
|
-
// src/infrastructure/embeddings/
|
|
2146
|
+
// src/infrastructure/embeddings/embeddingPaths.ts
|
|
2147
|
+
import * as os from "os";
|
|
2148
|
+
import * as path6 from "path";
|
|
2149
|
+
var RAGGREP_MODEL_CACHE_DIR;
|
|
2150
|
+
var init_embeddingPaths = __esm(() => {
|
|
2151
|
+
RAGGREP_MODEL_CACHE_DIR = path6.join(os.homedir(), ".cache", "raggrep", "models");
|
|
2152
|
+
});
|
|
2153
|
+
|
|
2154
|
+
// src/infrastructure/embeddings/modelCache.ts
|
|
2155
|
+
import * as fs3 from "fs/promises";
|
|
2156
|
+
import * as path7 from "path";
|
|
2157
|
+
async function isEmbeddingModelCached(model) {
|
|
2158
|
+
const modelId = getEmbeddingModelId(model);
|
|
2159
|
+
const onnxPath = path7.join(RAGGREP_MODEL_CACHE_DIR, modelId, "onnx", "model_quantized.onnx");
|
|
2160
|
+
try {
|
|
2161
|
+
await fs3.access(onnxPath);
|
|
2162
|
+
return true;
|
|
2163
|
+
} catch {
|
|
2164
|
+
return false;
|
|
2165
|
+
}
|
|
2166
|
+
}
|
|
2167
|
+
var init_modelCache = __esm(() => {
|
|
2168
|
+
init_embeddingPaths();
|
|
2169
|
+
init_modelCatalog();
|
|
2170
|
+
});
|
|
2171
|
+
|
|
2172
|
+
// src/infrastructure/embeddings/xenovaEmbeddingProvider.ts
|
|
2141
2173
|
import {
|
|
2142
2174
|
pipeline,
|
|
2143
2175
|
env
|
|
2144
2176
|
} from "@xenova/transformers";
|
|
2145
|
-
import * as path6 from "path";
|
|
2146
|
-
import * as os2 from "os";
|
|
2147
2177
|
|
|
2148
|
-
class
|
|
2149
|
-
|
|
2178
|
+
class XenovaTransformersEmbeddingProvider {
|
|
2179
|
+
extractor = null;
|
|
2150
2180
|
config;
|
|
2151
2181
|
isInitializing = false;
|
|
2152
2182
|
initPromise = null;
|
|
2153
2183
|
constructor(config) {
|
|
2154
2184
|
this.config = {
|
|
2155
2185
|
model: config?.model ?? "bge-small-en-v1.5",
|
|
2186
|
+
runtime: config?.runtime ?? "xenova",
|
|
2156
2187
|
showProgress: config?.showProgress ?? false,
|
|
2157
2188
|
logger: config?.logger
|
|
2158
2189
|
};
|
|
@@ -2160,14 +2191,14 @@ class TransformersEmbeddingProvider {
|
|
|
2160
2191
|
async initialize(config) {
|
|
2161
2192
|
if (config) {
|
|
2162
2193
|
if (config.model !== this.config.model) {
|
|
2163
|
-
this.
|
|
2194
|
+
this.extractor = null;
|
|
2164
2195
|
}
|
|
2165
2196
|
this.config = { ...this.config, ...config };
|
|
2166
2197
|
}
|
|
2167
|
-
await this.
|
|
2198
|
+
await this.ensureExtractor();
|
|
2168
2199
|
}
|
|
2169
|
-
async
|
|
2170
|
-
if (this.
|
|
2200
|
+
async ensureExtractor() {
|
|
2201
|
+
if (this.extractor) {
|
|
2171
2202
|
return;
|
|
2172
2203
|
}
|
|
2173
2204
|
if (this.isInitializing && this.initPromise) {
|
|
@@ -2175,14 +2206,14 @@ class TransformersEmbeddingProvider {
|
|
|
2175
2206
|
}
|
|
2176
2207
|
this.isInitializing = true;
|
|
2177
2208
|
this.initPromise = (async () => {
|
|
2178
|
-
const modelId =
|
|
2209
|
+
const modelId = getEmbeddingModelId(this.config.model);
|
|
2179
2210
|
const logger = this.config.logger;
|
|
2180
2211
|
const showProgress = this.config.showProgress || !!logger;
|
|
2181
|
-
const
|
|
2212
|
+
const cached = await isEmbeddingModelCached(this.config.model);
|
|
2182
2213
|
let hasDownloads = false;
|
|
2183
2214
|
try {
|
|
2184
|
-
this.
|
|
2185
|
-
progress_callback: showProgress && !
|
|
2215
|
+
this.extractor = await pipeline("feature-extraction", modelId, {
|
|
2216
|
+
progress_callback: showProgress && !cached ? (progress) => {
|
|
2186
2217
|
if (progress.status === "progress" && progress.file) {
|
|
2187
2218
|
if (!hasDownloads) {
|
|
2188
2219
|
hasDownloads = true;
|
|
@@ -2191,7 +2222,7 @@ class TransformersEmbeddingProvider {
|
|
|
2191
2222
|
} else {
|
|
2192
2223
|
console.log(`
|
|
2193
2224
|
Loading embedding model: ${this.config.model}`);
|
|
2194
|
-
console.log(` Cache: ${
|
|
2225
|
+
console.log(` Cache: ${RAGGREP_MODEL_CACHE_DIR}`);
|
|
2195
2226
|
}
|
|
2196
2227
|
}
|
|
2197
2228
|
const pct = progress.progress ? Math.round(progress.progress) : 0;
|
|
@@ -2221,9 +2252,9 @@ class TransformersEmbeddingProvider {
|
|
|
2221
2252
|
}
|
|
2222
2253
|
}
|
|
2223
2254
|
} catch (error) {
|
|
2224
|
-
this.
|
|
2225
|
-
if (logger) {
|
|
2226
|
-
logger.clearProgress();
|
|
2255
|
+
this.extractor = null;
|
|
2256
|
+
if (this.config.logger) {
|
|
2257
|
+
this.config.logger.clearProgress();
|
|
2227
2258
|
}
|
|
2228
2259
|
throw new Error(`Failed to load embedding model: ${error}`);
|
|
2229
2260
|
} finally {
|
|
@@ -2234,11 +2265,11 @@ class TransformersEmbeddingProvider {
|
|
|
2234
2265
|
return this.initPromise;
|
|
2235
2266
|
}
|
|
2236
2267
|
async getEmbedding(text) {
|
|
2237
|
-
await this.
|
|
2238
|
-
if (!this.
|
|
2268
|
+
await this.ensureExtractor();
|
|
2269
|
+
if (!this.extractor) {
|
|
2239
2270
|
throw new Error("Embedding pipeline not initialized");
|
|
2240
2271
|
}
|
|
2241
|
-
const output = await this.
|
|
2272
|
+
const output = await this.extractor(text, {
|
|
2242
2273
|
pooling: "mean",
|
|
2243
2274
|
normalize: true
|
|
2244
2275
|
});
|
|
@@ -2247,15 +2278,15 @@ class TransformersEmbeddingProvider {
|
|
|
2247
2278
|
async getEmbeddings(texts) {
|
|
2248
2279
|
if (texts.length === 0)
|
|
2249
2280
|
return [];
|
|
2250
|
-
await this.
|
|
2251
|
-
if (!this.
|
|
2281
|
+
await this.ensureExtractor();
|
|
2282
|
+
if (!this.extractor) {
|
|
2252
2283
|
throw new Error("Embedding pipeline not initialized");
|
|
2253
2284
|
}
|
|
2254
2285
|
const results = [];
|
|
2255
2286
|
for (let i = 0;i < texts.length; i += BATCH_SIZE) {
|
|
2256
2287
|
const batch = texts.slice(i, i + BATCH_SIZE);
|
|
2257
2288
|
const outputs = await Promise.all(batch.map(async (text) => {
|
|
2258
|
-
const output = await this.
|
|
2289
|
+
const output = await this.extractor(text, {
|
|
2259
2290
|
pooling: "mean",
|
|
2260
2291
|
normalize: true
|
|
2261
2292
|
});
|
|
@@ -2266,41 +2297,210 @@ class TransformersEmbeddingProvider {
|
|
|
2266
2297
|
return results;
|
|
2267
2298
|
}
|
|
2268
2299
|
getDimension() {
|
|
2269
|
-
return
|
|
2300
|
+
return getEmbeddingDimension(this.config.model);
|
|
2270
2301
|
}
|
|
2271
2302
|
getModelName() {
|
|
2272
2303
|
return this.config.model;
|
|
2273
2304
|
}
|
|
2274
2305
|
async dispose() {
|
|
2275
|
-
this.
|
|
2306
|
+
this.extractor = null;
|
|
2276
2307
|
}
|
|
2277
2308
|
}
|
|
2278
|
-
|
|
2279
|
-
|
|
2280
|
-
|
|
2281
|
-
|
|
2282
|
-
|
|
2283
|
-
|
|
2284
|
-
|
|
2285
|
-
|
|
2286
|
-
|
|
2287
|
-
|
|
2309
|
+
var BATCH_SIZE = 32;
|
|
2310
|
+
var init_xenovaEmbeddingProvider = __esm(() => {
|
|
2311
|
+
init_embeddingPaths();
|
|
2312
|
+
init_modelCatalog();
|
|
2313
|
+
init_modelCache();
|
|
2314
|
+
env.cacheDir = RAGGREP_MODEL_CACHE_DIR;
|
|
2315
|
+
env.allowLocalModels = true;
|
|
2316
|
+
});
|
|
2317
|
+
|
|
2318
|
+
// src/infrastructure/embeddings/huggingfaceEmbeddingProvider.ts
|
|
2319
|
+
import {
|
|
2320
|
+
pipeline as pipeline2,
|
|
2321
|
+
env as env2
|
|
2322
|
+
} from "@huggingface/transformers";
|
|
2323
|
+
|
|
2324
|
+
class HuggingFaceTransformersEmbeddingProvider {
|
|
2325
|
+
extractor = null;
|
|
2326
|
+
config;
|
|
2327
|
+
isInitializing = false;
|
|
2328
|
+
initPromise = null;
|
|
2329
|
+
constructor(config) {
|
|
2330
|
+
this.config = {
|
|
2331
|
+
model: config?.model ?? "bge-small-en-v1.5",
|
|
2332
|
+
runtime: config?.runtime ?? "huggingface",
|
|
2333
|
+
showProgress: config?.showProgress ?? false,
|
|
2334
|
+
logger: config?.logger
|
|
2335
|
+
};
|
|
2336
|
+
}
|
|
2337
|
+
async initialize(config) {
|
|
2338
|
+
if (config) {
|
|
2339
|
+
if (config.model !== this.config.model) {
|
|
2340
|
+
this.extractor = null;
|
|
2341
|
+
}
|
|
2342
|
+
this.config = { ...this.config, ...config };
|
|
2343
|
+
}
|
|
2344
|
+
await this.ensureExtractor();
|
|
2345
|
+
}
|
|
2346
|
+
async ensureExtractor() {
|
|
2347
|
+
if (this.extractor) {
|
|
2348
|
+
return;
|
|
2349
|
+
}
|
|
2350
|
+
if (this.isInitializing && this.initPromise) {
|
|
2351
|
+
return this.initPromise;
|
|
2352
|
+
}
|
|
2353
|
+
this.isInitializing = true;
|
|
2354
|
+
this.initPromise = (async () => {
|
|
2355
|
+
const modelId = getEmbeddingModelId(this.config.model);
|
|
2356
|
+
const logger = this.config.logger;
|
|
2357
|
+
const showProgress = this.config.showProgress || !!logger;
|
|
2358
|
+
const cached = await isEmbeddingModelCached(this.config.model);
|
|
2359
|
+
let hasDownloads = false;
|
|
2360
|
+
try {
|
|
2361
|
+
this.extractor = await pipeline2("feature-extraction", modelId, {
|
|
2362
|
+
progress_callback: showProgress && !cached ? (progress) => {
|
|
2363
|
+
if (progress.status === "progress" && progress.file) {
|
|
2364
|
+
if (!hasDownloads) {
|
|
2365
|
+
hasDownloads = true;
|
|
2366
|
+
if (logger) {
|
|
2367
|
+
logger.info(`Downloading embedding model: ${this.config.model}`);
|
|
2368
|
+
} else {
|
|
2369
|
+
console.log(`
|
|
2370
|
+
Loading embedding model: ${this.config.model}`);
|
|
2371
|
+
console.log(` Cache: ${RAGGREP_MODEL_CACHE_DIR}`);
|
|
2372
|
+
}
|
|
2373
|
+
}
|
|
2374
|
+
const pct = progress.progress ? Math.round(progress.progress) : 0;
|
|
2375
|
+
if (logger) {
|
|
2376
|
+
logger.progress(` Downloading ${progress.file}: ${pct}%`);
|
|
2377
|
+
} else {
|
|
2378
|
+
process.stdout.write(`\r Downloading ${progress.file}: ${pct}% `);
|
|
2379
|
+
}
|
|
2380
|
+
} else if (progress.status === "done" && progress.file) {
|
|
2381
|
+
if (logger) {
|
|
2382
|
+
logger.clearProgress();
|
|
2383
|
+
logger.info(` Downloaded ${progress.file}`);
|
|
2384
|
+
} else if (hasDownloads) {
|
|
2385
|
+
process.stdout.write(`\r Downloaded ${progress.file}
|
|
2386
|
+
`);
|
|
2387
|
+
}
|
|
2388
|
+
}
|
|
2389
|
+
} : undefined
|
|
2390
|
+
});
|
|
2391
|
+
if (hasDownloads) {
|
|
2392
|
+
if (logger) {
|
|
2393
|
+
logger.clearProgress();
|
|
2394
|
+
logger.info(`Model ready: ${this.config.model}`);
|
|
2395
|
+
} else {
|
|
2396
|
+
console.log(` Model ready.
|
|
2397
|
+
`);
|
|
2398
|
+
}
|
|
2399
|
+
}
|
|
2400
|
+
} catch (error) {
|
|
2401
|
+
this.extractor = null;
|
|
2402
|
+
if (this.config.logger) {
|
|
2403
|
+
this.config.logger.clearProgress();
|
|
2404
|
+
}
|
|
2405
|
+
throw new Error(`Failed to load embedding model: ${error}`);
|
|
2406
|
+
} finally {
|
|
2407
|
+
this.isInitializing = false;
|
|
2408
|
+
this.initPromise = null;
|
|
2409
|
+
}
|
|
2410
|
+
})();
|
|
2411
|
+
return this.initPromise;
|
|
2412
|
+
}
|
|
2413
|
+
async getEmbedding(text) {
|
|
2414
|
+
await this.ensureExtractor();
|
|
2415
|
+
if (!this.extractor) {
|
|
2416
|
+
throw new Error("Embedding pipeline not initialized");
|
|
2417
|
+
}
|
|
2418
|
+
const output = await this.extractor(text, {
|
|
2419
|
+
pooling: "mean",
|
|
2420
|
+
normalize: true
|
|
2421
|
+
});
|
|
2422
|
+
return Array.from(output.data);
|
|
2423
|
+
}
|
|
2424
|
+
async getEmbeddings(texts) {
|
|
2425
|
+
if (texts.length === 0)
|
|
2426
|
+
return [];
|
|
2427
|
+
await this.ensureExtractor();
|
|
2428
|
+
if (!this.extractor) {
|
|
2429
|
+
throw new Error("Embedding pipeline not initialized");
|
|
2430
|
+
}
|
|
2431
|
+
const results = [];
|
|
2432
|
+
for (let i = 0;i < texts.length; i += BATCH_SIZE2) {
|
|
2433
|
+
const batch = texts.slice(i, i + BATCH_SIZE2);
|
|
2434
|
+
const outputs = await Promise.all(batch.map(async (text) => {
|
|
2435
|
+
const output = await this.extractor(text, {
|
|
2436
|
+
pooling: "mean",
|
|
2437
|
+
normalize: true
|
|
2438
|
+
});
|
|
2439
|
+
return Array.from(output.data);
|
|
2440
|
+
}));
|
|
2441
|
+
results.push(...outputs);
|
|
2442
|
+
}
|
|
2443
|
+
return results;
|
|
2444
|
+
}
|
|
2445
|
+
getDimension() {
|
|
2446
|
+
return getEmbeddingDimension(this.config.model);
|
|
2447
|
+
}
|
|
2448
|
+
getModelName() {
|
|
2449
|
+
return this.config.model;
|
|
2450
|
+
}
|
|
2451
|
+
async dispose() {
|
|
2452
|
+
this.extractor = null;
|
|
2288
2453
|
}
|
|
2289
2454
|
}
|
|
2455
|
+
var BATCH_SIZE2 = 32;
|
|
2456
|
+
var init_huggingfaceEmbeddingProvider = __esm(() => {
|
|
2457
|
+
init_embeddingPaths();
|
|
2458
|
+
init_modelCatalog();
|
|
2459
|
+
init_modelCache();
|
|
2460
|
+
env2.cacheDir = RAGGREP_MODEL_CACHE_DIR;
|
|
2461
|
+
env2.allowLocalModels = true;
|
|
2462
|
+
});
|
|
2463
|
+
|
|
2464
|
+
// src/infrastructure/embeddings/embeddingProviderFactory.ts
|
|
2465
|
+
function resolveRuntime(config) {
|
|
2466
|
+
return config.runtime ?? "huggingface";
|
|
2467
|
+
}
|
|
2468
|
+
function createEmbeddingProvider(config) {
|
|
2469
|
+
const runtime = resolveRuntime(config);
|
|
2470
|
+
if (runtime === "huggingface") {
|
|
2471
|
+
return new HuggingFaceTransformersEmbeddingProvider(config);
|
|
2472
|
+
}
|
|
2473
|
+
return new XenovaTransformersEmbeddingProvider(config);
|
|
2474
|
+
}
|
|
2475
|
+
var init_embeddingProviderFactory = __esm(() => {
|
|
2476
|
+
init_huggingfaceEmbeddingProvider();
|
|
2477
|
+
init_xenovaEmbeddingProvider();
|
|
2478
|
+
});
|
|
2479
|
+
|
|
2480
|
+
// src/infrastructure/embeddings/globalEmbeddings.ts
|
|
2290
2481
|
function configureEmbeddings(config) {
|
|
2291
|
-
const
|
|
2292
|
-
|
|
2482
|
+
const merged = {
|
|
2483
|
+
...globalConfig,
|
|
2484
|
+
...config
|
|
2485
|
+
};
|
|
2486
|
+
if (merged.runtime === undefined) {
|
|
2487
|
+
merged.runtime = "huggingface";
|
|
2488
|
+
}
|
|
2489
|
+
const needsReset = merged.model !== globalConfig.model || merged.runtime !== globalConfig.runtime || merged.logger !== globalConfig.logger;
|
|
2490
|
+
if (needsReset) {
|
|
2491
|
+
const prev = globalProvider;
|
|
2293
2492
|
globalProvider = null;
|
|
2493
|
+
prev?.dispose?.();
|
|
2294
2494
|
}
|
|
2295
|
-
globalConfig =
|
|
2495
|
+
globalConfig = merged;
|
|
2296
2496
|
}
|
|
2297
2497
|
function getEmbeddingConfig() {
|
|
2298
2498
|
return { ...globalConfig };
|
|
2299
2499
|
}
|
|
2300
2500
|
async function ensureGlobalProvider() {
|
|
2301
2501
|
if (!globalProvider) {
|
|
2302
|
-
globalProvider =
|
|
2303
|
-
await globalProvider.initialize();
|
|
2502
|
+
globalProvider = createEmbeddingProvider(globalConfig);
|
|
2503
|
+
await globalProvider.initialize?.(globalConfig);
|
|
2304
2504
|
}
|
|
2305
2505
|
return globalProvider;
|
|
2306
2506
|
}
|
|
@@ -2312,27 +2512,14 @@ async function getEmbeddings(texts) {
|
|
|
2312
2512
|
const provider = await ensureGlobalProvider();
|
|
2313
2513
|
return provider.getEmbeddings(texts);
|
|
2314
2514
|
}
|
|
2315
|
-
var
|
|
2316
|
-
var
|
|
2317
|
-
|
|
2318
|
-
|
|
2319
|
-
|
|
2320
|
-
EMBEDDING_MODELS2 = {
|
|
2321
|
-
"all-MiniLM-L6-v2": "Xenova/all-MiniLM-L6-v2",
|
|
2322
|
-
"all-MiniLM-L12-v2": "Xenova/all-MiniLM-L12-v2",
|
|
2323
|
-
"bge-small-en-v1.5": "Xenova/bge-small-en-v1.5",
|
|
2324
|
-
"paraphrase-MiniLM-L3-v2": "Xenova/paraphrase-MiniLM-L3-v2",
|
|
2325
|
-
"nomic-embed-text-v1.5": "nomic-ai/nomic-embed-text-v1.5"
|
|
2326
|
-
};
|
|
2327
|
-
EMBEDDING_DIMENSIONS = {
|
|
2328
|
-
"all-MiniLM-L6-v2": 384,
|
|
2329
|
-
"all-MiniLM-L12-v2": 384,
|
|
2330
|
-
"bge-small-en-v1.5": 384,
|
|
2331
|
-
"paraphrase-MiniLM-L3-v2": 384,
|
|
2332
|
-
"nomic-embed-text-v1.5": 768
|
|
2333
|
-
};
|
|
2515
|
+
var globalProvider = null, globalConfig;
|
|
2516
|
+
var init_globalEmbeddings = __esm(() => {
|
|
2517
|
+
init_embeddingPaths();
|
|
2518
|
+
init_embeddingProviderFactory();
|
|
2519
|
+
init_modelCache();
|
|
2334
2520
|
globalConfig = {
|
|
2335
2521
|
model: "bge-small-en-v1.5",
|
|
2522
|
+
runtime: "huggingface",
|
|
2336
2523
|
showProgress: false,
|
|
2337
2524
|
logger: undefined
|
|
2338
2525
|
};
|
|
@@ -2340,7 +2527,13 @@ var init_transformersEmbedding = __esm(() => {
|
|
|
2340
2527
|
|
|
2341
2528
|
// src/infrastructure/embeddings/index.ts
|
|
2342
2529
|
var init_embeddings = __esm(() => {
|
|
2343
|
-
|
|
2530
|
+
init_modelCatalog();
|
|
2531
|
+
init_embeddingPaths();
|
|
2532
|
+
init_xenovaEmbeddingProvider();
|
|
2533
|
+
init_xenovaEmbeddingProvider();
|
|
2534
|
+
init_huggingfaceEmbeddingProvider();
|
|
2535
|
+
init_embeddingProviderFactory();
|
|
2536
|
+
init_globalEmbeddings();
|
|
2344
2537
|
});
|
|
2345
2538
|
|
|
2346
2539
|
// src/domain/services/keywords.ts
|
|
@@ -2551,7 +2744,7 @@ function cosineSimilarity(a, b) {
|
|
|
2551
2744
|
}
|
|
2552
2745
|
|
|
2553
2746
|
// src/domain/services/queryIntent.ts
|
|
2554
|
-
import * as
|
|
2747
|
+
import * as path8 from "path";
|
|
2555
2748
|
function detectQueryIntent(queryTerms) {
|
|
2556
2749
|
const hasImplementationTerm = queryTerms.some((term) => IMPLEMENTATION_TERMS.includes(term));
|
|
2557
2750
|
const hasDocumentationTerm = queryTerms.some((term) => DOCUMENTATION_TERMS.includes(term));
|
|
@@ -2567,11 +2760,11 @@ function extractQueryTerms(query) {
|
|
|
2567
2760
|
return query.toLowerCase().split(/\s+/).filter((t) => t.length > 2);
|
|
2568
2761
|
}
|
|
2569
2762
|
function isSourceCodeFile(filepath) {
|
|
2570
|
-
const ext =
|
|
2763
|
+
const ext = path8.extname(filepath).toLowerCase();
|
|
2571
2764
|
return SOURCE_CODE_EXTENSIONS.includes(ext);
|
|
2572
2765
|
}
|
|
2573
2766
|
function isDocFile(filepath) {
|
|
2574
|
-
const ext =
|
|
2767
|
+
const ext = path8.extname(filepath).toLowerCase();
|
|
2575
2768
|
return DOC_EXTENSIONS.includes(ext);
|
|
2576
2769
|
}
|
|
2577
2770
|
function calculateFileTypeBoost(filepath, queryTerms) {
|
|
@@ -3646,8 +3839,8 @@ var init_lexicon2 = __esm(() => {
|
|
|
3646
3839
|
// src/domain/services/jsonPathExtractor.ts
|
|
3647
3840
|
function extractJsonPaths(obj, fileBasename) {
|
|
3648
3841
|
const paths = extractPathsRecursive(obj, fileBasename);
|
|
3649
|
-
return paths.map((
|
|
3650
|
-
value:
|
|
3842
|
+
return paths.map((path9) => ({
|
|
3843
|
+
value: path9,
|
|
3651
3844
|
type: "identifier",
|
|
3652
3845
|
matchType: "definition"
|
|
3653
3846
|
}));
|
|
@@ -3957,6 +4150,62 @@ var init_simpleSearch = __esm(() => {
|
|
|
3957
4150
|
};
|
|
3958
4151
|
});
|
|
3959
4152
|
|
|
4153
|
+
// src/domain/services/chunkContext.ts
|
|
4154
|
+
import * as path9 from "path";
|
|
4155
|
+
function prepareChunkForEmbedding(options) {
|
|
4156
|
+
const { filepath, content, name, docComment } = options;
|
|
4157
|
+
const pathContext = parsePathContext(filepath);
|
|
4158
|
+
const pathPrefix = formatPathContextForEmbedding(pathContext);
|
|
4159
|
+
const parts = [];
|
|
4160
|
+
if (pathPrefix) {
|
|
4161
|
+
parts.push(pathPrefix);
|
|
4162
|
+
}
|
|
4163
|
+
const filename = path9.basename(filepath);
|
|
4164
|
+
const filenameWithoutExt = filename.replace(/\.[^.]+$/, "");
|
|
4165
|
+
if (filenameWithoutExt && filenameWithoutExt.length > MIN_SEGMENT_LENGTH) {
|
|
4166
|
+
const pathPrefixLower = pathPrefix.toLowerCase();
|
|
4167
|
+
if (!pathPrefixLower.includes(filenameWithoutExt.toLowerCase())) {
|
|
4168
|
+
parts.push(filenameWithoutExt);
|
|
4169
|
+
}
|
|
4170
|
+
}
|
|
4171
|
+
if (name) {
|
|
4172
|
+
parts.push(`${name}:`);
|
|
4173
|
+
}
|
|
4174
|
+
if (docComment) {
|
|
4175
|
+
parts.push(docComment);
|
|
4176
|
+
}
|
|
4177
|
+
parts.push(content);
|
|
4178
|
+
return parts.join(" ");
|
|
4179
|
+
}
|
|
4180
|
+
function extractPathKeywordsForFileSummary(filepath) {
|
|
4181
|
+
const keywords = extractPathKeywords(filepath);
|
|
4182
|
+
const filtered = keywords.filter((k) => k.length >= MIN_SEGMENT_LENGTH && !GENERIC_SEGMENTS.has(k));
|
|
4183
|
+
return [...new Set(filtered)];
|
|
4184
|
+
}
|
|
4185
|
+
function getPathContextForFileSummary(filepath) {
|
|
4186
|
+
const pathContext = parsePathContext(filepath);
|
|
4187
|
+
return {
|
|
4188
|
+
segments: pathContext.segments,
|
|
4189
|
+
layer: pathContext.layer,
|
|
4190
|
+
domain: pathContext.domain,
|
|
4191
|
+
depth: pathContext.depth
|
|
4192
|
+
};
|
|
4193
|
+
}
|
|
4194
|
+
var GENERIC_SEGMENTS, MIN_SEGMENT_LENGTH = 2;
|
|
4195
|
+
var init_chunkContext = __esm(() => {
|
|
4196
|
+
init_keywords();
|
|
4197
|
+
GENERIC_SEGMENTS = new Set([
|
|
4198
|
+
"src",
|
|
4199
|
+
"lib",
|
|
4200
|
+
"app",
|
|
4201
|
+
"index",
|
|
4202
|
+
"dist",
|
|
4203
|
+
"build",
|
|
4204
|
+
"out",
|
|
4205
|
+
"node_modules"
|
|
4206
|
+
]);
|
|
4207
|
+
});
|
|
4208
|
+
|
|
3960
4209
|
// src/domain/services/index.ts
|
|
3961
4210
|
var init_services = __esm(() => {
|
|
3962
4211
|
init_keywords();
|
|
@@ -3969,6 +4218,7 @@ var init_services = __esm(() => {
|
|
|
3969
4218
|
init_configValidator();
|
|
3970
4219
|
init_phraseMatch();
|
|
3971
4220
|
init_simpleSearch();
|
|
4221
|
+
init_chunkContext();
|
|
3972
4222
|
});
|
|
3973
4223
|
|
|
3974
4224
|
// src/modules/language/typescript/parseCode.ts
|
|
@@ -4140,11 +4390,12 @@ var init_parseCode = () => {};
|
|
|
4140
4390
|
// src/infrastructure/storage/fileIndexStorage.ts
|
|
4141
4391
|
var init_fileIndexStorage = __esm(() => {
|
|
4142
4392
|
init_entities();
|
|
4393
|
+
init_config2();
|
|
4143
4394
|
});
|
|
4144
4395
|
|
|
4145
4396
|
// src/infrastructure/storage/symbolicIndex.ts
|
|
4146
|
-
import * as
|
|
4147
|
-
import * as
|
|
4397
|
+
import * as fs4 from "fs/promises";
|
|
4398
|
+
import * as path10 from "path";
|
|
4148
4399
|
|
|
4149
4400
|
class SymbolicIndex {
|
|
4150
4401
|
meta = null;
|
|
@@ -4153,7 +4404,7 @@ class SymbolicIndex {
|
|
|
4153
4404
|
symbolicPath;
|
|
4154
4405
|
moduleId;
|
|
4155
4406
|
constructor(indexDir, moduleId) {
|
|
4156
|
-
this.symbolicPath =
|
|
4407
|
+
this.symbolicPath = path10.join(indexDir, "index", moduleId, "symbolic");
|
|
4157
4408
|
this.moduleId = moduleId;
|
|
4158
4409
|
}
|
|
4159
4410
|
async initialize() {
|
|
@@ -4239,13 +4490,13 @@ class SymbolicIndex {
|
|
|
4239
4490
|
if (this.bm25Index) {
|
|
4240
4491
|
this.meta.bm25Serialized = this.bm25Index.serialize();
|
|
4241
4492
|
}
|
|
4242
|
-
await
|
|
4243
|
-
const metaPath =
|
|
4244
|
-
await
|
|
4493
|
+
await fs4.mkdir(this.symbolicPath, { recursive: true });
|
|
4494
|
+
const metaPath = path10.join(this.symbolicPath, "_meta.json");
|
|
4495
|
+
await fs4.writeFile(metaPath, JSON.stringify(this.meta, null, 2));
|
|
4245
4496
|
for (const [filepath, summary] of this.fileSummaries) {
|
|
4246
4497
|
const summaryPath = this.getFileSummaryPath(filepath);
|
|
4247
|
-
await
|
|
4248
|
-
await
|
|
4498
|
+
await fs4.mkdir(path10.dirname(summaryPath), { recursive: true });
|
|
4499
|
+
await fs4.writeFile(summaryPath, JSON.stringify(summary, null, 2));
|
|
4249
4500
|
}
|
|
4250
4501
|
}
|
|
4251
4502
|
async saveIncremental(filepaths) {
|
|
@@ -4256,21 +4507,21 @@ class SymbolicIndex {
|
|
|
4256
4507
|
if (this.bm25Index) {
|
|
4257
4508
|
this.meta.bm25Serialized = this.bm25Index.serialize();
|
|
4258
4509
|
}
|
|
4259
|
-
await
|
|
4260
|
-
const metaPath =
|
|
4261
|
-
await
|
|
4510
|
+
await fs4.mkdir(this.symbolicPath, { recursive: true });
|
|
4511
|
+
const metaPath = path10.join(this.symbolicPath, "_meta.json");
|
|
4512
|
+
await fs4.writeFile(metaPath, JSON.stringify(this.meta, null, 2));
|
|
4262
4513
|
for (const filepath of filepaths) {
|
|
4263
4514
|
const summary = this.fileSummaries.get(filepath);
|
|
4264
4515
|
if (summary) {
|
|
4265
4516
|
const summaryPath = this.getFileSummaryPath(filepath);
|
|
4266
|
-
await
|
|
4267
|
-
await
|
|
4517
|
+
await fs4.mkdir(path10.dirname(summaryPath), { recursive: true });
|
|
4518
|
+
await fs4.writeFile(summaryPath, JSON.stringify(summary, null, 2));
|
|
4268
4519
|
}
|
|
4269
4520
|
}
|
|
4270
4521
|
}
|
|
4271
4522
|
async load() {
|
|
4272
|
-
const metaPath =
|
|
4273
|
-
const metaContent = await
|
|
4523
|
+
const metaPath = path10.join(this.symbolicPath, "_meta.json");
|
|
4524
|
+
const metaContent = await fs4.readFile(metaPath, "utf-8");
|
|
4274
4525
|
this.meta = JSON.parse(metaContent);
|
|
4275
4526
|
this.fileSummaries.clear();
|
|
4276
4527
|
await this.loadFileSummariesRecursive(this.symbolicPath);
|
|
@@ -4282,14 +4533,14 @@ class SymbolicIndex {
|
|
|
4282
4533
|
}
|
|
4283
4534
|
async loadFileSummariesRecursive(dir) {
|
|
4284
4535
|
try {
|
|
4285
|
-
const entries = await
|
|
4536
|
+
const entries = await fs4.readdir(dir, { withFileTypes: true });
|
|
4286
4537
|
for (const entry of entries) {
|
|
4287
|
-
const fullPath =
|
|
4538
|
+
const fullPath = path10.join(dir, entry.name);
|
|
4288
4539
|
if (entry.isDirectory()) {
|
|
4289
4540
|
await this.loadFileSummariesRecursive(fullPath);
|
|
4290
4541
|
} else if (entry.name.endsWith(".json") && entry.name !== "_meta.json") {
|
|
4291
4542
|
try {
|
|
4292
|
-
const content = await
|
|
4543
|
+
const content = await fs4.readFile(fullPath, "utf-8");
|
|
4293
4544
|
const summary = JSON.parse(content);
|
|
4294
4545
|
if (summary.filepath) {
|
|
4295
4546
|
this.fileSummaries.set(summary.filepath, summary);
|
|
@@ -4301,18 +4552,18 @@ class SymbolicIndex {
|
|
|
4301
4552
|
}
|
|
4302
4553
|
getFileSummaryPath(filepath) {
|
|
4303
4554
|
const jsonPath = filepath.replace(/\.[^.]+$/, ".json");
|
|
4304
|
-
return
|
|
4555
|
+
return path10.join(this.symbolicPath, jsonPath);
|
|
4305
4556
|
}
|
|
4306
4557
|
async deleteFileSummary(filepath) {
|
|
4307
4558
|
try {
|
|
4308
|
-
await
|
|
4559
|
+
await fs4.unlink(this.getFileSummaryPath(filepath));
|
|
4309
4560
|
} catch {}
|
|
4310
4561
|
this.fileSummaries.delete(filepath);
|
|
4311
4562
|
}
|
|
4312
4563
|
async exists() {
|
|
4313
4564
|
try {
|
|
4314
|
-
const metaPath =
|
|
4315
|
-
await
|
|
4565
|
+
const metaPath = path10.join(this.symbolicPath, "_meta.json");
|
|
4566
|
+
await fs4.access(metaPath);
|
|
4316
4567
|
return true;
|
|
4317
4568
|
} catch {
|
|
4318
4569
|
return false;
|
|
@@ -4344,8 +4595,8 @@ __export(exports_literalIndex, {
|
|
|
4344
4595
|
getLiteralIndexPath: () => getLiteralIndexPath,
|
|
4345
4596
|
LiteralIndex: () => LiteralIndex
|
|
4346
4597
|
});
|
|
4347
|
-
import * as
|
|
4348
|
-
import * as
|
|
4598
|
+
import * as fs5 from "fs/promises";
|
|
4599
|
+
import * as path11 from "path";
|
|
4349
4600
|
|
|
4350
4601
|
class LiteralIndex {
|
|
4351
4602
|
indexPath;
|
|
@@ -4354,7 +4605,7 @@ class LiteralIndex {
|
|
|
4354
4605
|
vocabularyIndex = new Map;
|
|
4355
4606
|
static VERSION = "1.1.0";
|
|
4356
4607
|
constructor(indexDir, moduleId) {
|
|
4357
|
-
this.indexPath =
|
|
4608
|
+
this.indexPath = path11.join(indexDir, "index", moduleId, "literals");
|
|
4358
4609
|
this.moduleId = moduleId;
|
|
4359
4610
|
}
|
|
4360
4611
|
async initialize() {
|
|
@@ -4515,17 +4766,17 @@ class LiteralIndex {
|
|
|
4515
4766
|
}));
|
|
4516
4767
|
}
|
|
4517
4768
|
async save() {
|
|
4518
|
-
await
|
|
4769
|
+
await fs5.mkdir(this.indexPath, { recursive: true });
|
|
4519
4770
|
const data = {
|
|
4520
4771
|
version: LiteralIndex.VERSION,
|
|
4521
4772
|
entries: Object.fromEntries(this.entries)
|
|
4522
4773
|
};
|
|
4523
|
-
const indexFile =
|
|
4524
|
-
await
|
|
4774
|
+
const indexFile = path11.join(this.indexPath, "_index.json");
|
|
4775
|
+
await fs5.writeFile(indexFile, JSON.stringify(data, null, 2));
|
|
4525
4776
|
}
|
|
4526
4777
|
async load() {
|
|
4527
|
-
const indexFile =
|
|
4528
|
-
const content = await
|
|
4778
|
+
const indexFile = path11.join(this.indexPath, "_index.json");
|
|
4779
|
+
const content = await fs5.readFile(indexFile, "utf-8");
|
|
4529
4780
|
const data = JSON.parse(content);
|
|
4530
4781
|
if (data.version !== LiteralIndex.VERSION) {
|
|
4531
4782
|
console.warn(`Literal index version mismatch: expected ${LiteralIndex.VERSION}, got ${data.version}`);
|
|
@@ -4534,8 +4785,8 @@ class LiteralIndex {
|
|
|
4534
4785
|
}
|
|
4535
4786
|
async exists() {
|
|
4536
4787
|
try {
|
|
4537
|
-
const indexFile =
|
|
4538
|
-
await
|
|
4788
|
+
const indexFile = path11.join(this.indexPath, "_index.json");
|
|
4789
|
+
await fs5.access(indexFile);
|
|
4539
4790
|
return true;
|
|
4540
4791
|
} catch {
|
|
4541
4792
|
return false;
|
|
@@ -4578,7 +4829,7 @@ function shouldReplaceMatchType(existing, incoming) {
|
|
|
4578
4829
|
return priority[incoming] > priority[existing];
|
|
4579
4830
|
}
|
|
4580
4831
|
function getLiteralIndexPath(rootDir, moduleId, indexDir = ".raggrep") {
|
|
4581
|
-
return
|
|
4832
|
+
return path11.join(rootDir, indexDir, "index", moduleId, "literals");
|
|
4582
4833
|
}
|
|
4583
4834
|
var init_literalIndex = () => {};
|
|
4584
4835
|
|
|
@@ -4599,9 +4850,9 @@ __export(exports_typescript, {
|
|
|
4599
4850
|
DEFAULT_TOP_K: () => DEFAULT_TOP_K2,
|
|
4600
4851
|
DEFAULT_MIN_SCORE: () => DEFAULT_MIN_SCORE2
|
|
4601
4852
|
});
|
|
4602
|
-
import * as
|
|
4853
|
+
import * as path12 from "path";
|
|
4603
4854
|
function isTypeScriptFile(filepath) {
|
|
4604
|
-
const ext =
|
|
4855
|
+
const ext = path12.extname(filepath).toLowerCase();
|
|
4605
4856
|
return TYPESCRIPT_EXTENSIONS.includes(ext);
|
|
4606
4857
|
}
|
|
4607
4858
|
function calculateChunkTypeBoost(chunk) {
|
|
@@ -4663,8 +4914,6 @@ class TypeScriptModule {
|
|
|
4663
4914
|
if (parsedChunks.length === 0) {
|
|
4664
4915
|
return null;
|
|
4665
4916
|
}
|
|
4666
|
-
const pathContext = parsePathContext(filepath);
|
|
4667
|
-
const pathPrefix = formatPathContextForEmbedding(pathContext);
|
|
4668
4917
|
const includeFullFileChunk = parsedChunks.length > 1;
|
|
4669
4918
|
const allParsedChunks = [...parsedChunks];
|
|
4670
4919
|
if (includeFullFileChunk) {
|
|
@@ -4675,13 +4924,17 @@ class TypeScriptModule {
|
|
|
4675
4924
|
startLine: 1,
|
|
4676
4925
|
endLine: lines.length,
|
|
4677
4926
|
type: "file",
|
|
4678
|
-
name:
|
|
4927
|
+
name: path12.basename(filepath),
|
|
4679
4928
|
isExported: false
|
|
4680
4929
|
});
|
|
4681
4930
|
}
|
|
4682
4931
|
const chunkContents = allParsedChunks.map((c) => {
|
|
4683
|
-
|
|
4684
|
-
|
|
4932
|
+
return prepareChunkForEmbedding({
|
|
4933
|
+
filepath,
|
|
4934
|
+
content: c.content,
|
|
4935
|
+
name: c.name,
|
|
4936
|
+
docComment: c.jsDoc
|
|
4937
|
+
});
|
|
4685
4938
|
});
|
|
4686
4939
|
const embeddings = await getEmbeddings(chunkContents);
|
|
4687
4940
|
const chunks = allParsedChunks.map((pc) => ({
|
|
@@ -4705,25 +4958,21 @@ class TypeScriptModule {
|
|
|
4705
4958
|
...new Set(parsedChunks.map((pc) => pc.type))
|
|
4706
4959
|
];
|
|
4707
4960
|
const exports = parsedChunks.filter((pc) => pc.isExported && pc.name).map((pc) => pc.name);
|
|
4708
|
-
const
|
|
4961
|
+
const contentKeywords = new Set;
|
|
4709
4962
|
for (const pc of parsedChunks) {
|
|
4710
4963
|
const keywords = extractKeywords(pc.content, pc.name);
|
|
4711
|
-
keywords.forEach((k) =>
|
|
4964
|
+
keywords.forEach((k) => contentKeywords.add(k));
|
|
4712
4965
|
}
|
|
4713
|
-
|
|
4966
|
+
const pathKeywords = extractPathKeywordsForFileSummary(filepath);
|
|
4967
|
+
const allKeywords = [...contentKeywords, ...pathKeywords];
|
|
4714
4968
|
const fileSummary = {
|
|
4715
4969
|
filepath,
|
|
4716
4970
|
chunkCount: chunks.length,
|
|
4717
4971
|
chunkTypes,
|
|
4718
|
-
keywords:
|
|
4972
|
+
keywords: [...new Set(allKeywords)],
|
|
4719
4973
|
exports,
|
|
4720
4974
|
lastModified: stats.lastModified,
|
|
4721
|
-
pathContext:
|
|
4722
|
-
segments: pathContext.segments,
|
|
4723
|
-
layer: pathContext.layer,
|
|
4724
|
-
domain: pathContext.domain,
|
|
4725
|
-
depth: pathContext.depth
|
|
4726
|
-
}
|
|
4975
|
+
pathContext: getPathContextForFileSummary(filepath)
|
|
4727
4976
|
};
|
|
4728
4977
|
this.pendingSummaries.set(filepath, fileSummary);
|
|
4729
4978
|
for (const chunk of chunks) {
|
|
@@ -5002,16 +5251,16 @@ class TypeScriptModule {
|
|
|
5002
5251
|
while ((match = importRegex.exec(content)) !== null) {
|
|
5003
5252
|
const importPath = match[1];
|
|
5004
5253
|
if (importPath.startsWith(".")) {
|
|
5005
|
-
const dir =
|
|
5006
|
-
const resolved =
|
|
5254
|
+
const dir = path12.dirname(filepath);
|
|
5255
|
+
const resolved = path12.normalize(path12.join(dir, importPath));
|
|
5007
5256
|
references.push(resolved);
|
|
5008
5257
|
}
|
|
5009
5258
|
}
|
|
5010
5259
|
while ((match = requireRegex.exec(content)) !== null) {
|
|
5011
5260
|
const importPath = match[1];
|
|
5012
5261
|
if (importPath.startsWith(".")) {
|
|
5013
|
-
const dir =
|
|
5014
|
-
const resolved =
|
|
5262
|
+
const dir = path12.dirname(filepath);
|
|
5263
|
+
const resolved = path12.normalize(path12.join(dir, importPath));
|
|
5015
5264
|
references.push(resolved);
|
|
5016
5265
|
}
|
|
5017
5266
|
}
|
|
@@ -5039,7 +5288,7 @@ var init_typescript = __esm(() => {
|
|
|
5039
5288
|
});
|
|
5040
5289
|
|
|
5041
5290
|
// src/infrastructure/parsing/typescriptParser.ts
|
|
5042
|
-
import * as
|
|
5291
|
+
import * as path13 from "path";
|
|
5043
5292
|
|
|
5044
5293
|
class TypeScriptParser {
|
|
5045
5294
|
supportedLanguages = ["typescript", "javascript"];
|
|
@@ -5055,12 +5304,12 @@ class TypeScriptParser {
|
|
|
5055
5304
|
startLine: 1,
|
|
5056
5305
|
endLine: lines.length,
|
|
5057
5306
|
type: "file",
|
|
5058
|
-
name:
|
|
5307
|
+
name: path13.basename(filepath),
|
|
5059
5308
|
isExported: false
|
|
5060
5309
|
};
|
|
5061
5310
|
chunks.unshift(fullFileChunk);
|
|
5062
5311
|
}
|
|
5063
|
-
const ext =
|
|
5312
|
+
const ext = path13.extname(filepath).toLowerCase();
|
|
5064
5313
|
const language = ext === ".js" || ext === ".jsx" || ext === ".mjs" || ext === ".cjs" ? "javascript" : "typescript";
|
|
5065
5314
|
return {
|
|
5066
5315
|
chunks,
|
|
@@ -5077,7 +5326,7 @@ class TypeScriptParser {
|
|
|
5077
5326
|
}
|
|
5078
5327
|
}
|
|
5079
5328
|
canParse(filepath) {
|
|
5080
|
-
const ext =
|
|
5329
|
+
const ext = path13.extname(filepath).toLowerCase();
|
|
5081
5330
|
return TYPESCRIPT_EXTENSIONS2.includes(ext);
|
|
5082
5331
|
}
|
|
5083
5332
|
convertChunk(tc) {
|
|
@@ -5092,7 +5341,7 @@ class TypeScriptParser {
|
|
|
5092
5341
|
};
|
|
5093
5342
|
}
|
|
5094
5343
|
detectLanguage(filepath) {
|
|
5095
|
-
const ext =
|
|
5344
|
+
const ext = path13.extname(filepath).toLowerCase();
|
|
5096
5345
|
if ([".js", ".jsx", ".mjs", ".cjs"].includes(ext)) {
|
|
5097
5346
|
return "javascript";
|
|
5098
5347
|
}
|
|
@@ -5297,8 +5546,8 @@ var init_grammarManager = __esm(() => {
|
|
|
5297
5546
|
});
|
|
5298
5547
|
|
|
5299
5548
|
// src/infrastructure/parsing/treeSitterParser.ts
|
|
5300
|
-
import * as
|
|
5301
|
-
import * as
|
|
5549
|
+
import * as path14 from "path";
|
|
5550
|
+
import * as fs6 from "fs";
|
|
5302
5551
|
|
|
5303
5552
|
class TreeSitterParser {
|
|
5304
5553
|
supportedLanguages = [
|
|
@@ -5320,7 +5569,7 @@ class TreeSitterParser {
|
|
|
5320
5569
|
chunks: [],
|
|
5321
5570
|
language: "typescript",
|
|
5322
5571
|
success: false,
|
|
5323
|
-
error: `Unsupported file type: ${
|
|
5572
|
+
error: `Unsupported file type: ${path14.extname(filepath)}`
|
|
5324
5573
|
};
|
|
5325
5574
|
}
|
|
5326
5575
|
try {
|
|
@@ -5340,11 +5589,11 @@ class TreeSitterParser {
|
|
|
5340
5589
|
}
|
|
5341
5590
|
}
|
|
5342
5591
|
canParse(filepath) {
|
|
5343
|
-
const ext =
|
|
5592
|
+
const ext = path14.extname(filepath).toLowerCase();
|
|
5344
5593
|
return ext in EXTENSION_TO_LANGUAGE2;
|
|
5345
5594
|
}
|
|
5346
5595
|
detectLanguage(filepath) {
|
|
5347
|
-
const ext =
|
|
5596
|
+
const ext = path14.extname(filepath).toLowerCase();
|
|
5348
5597
|
return EXTENSION_TO_LANGUAGE2[ext] || null;
|
|
5349
5598
|
}
|
|
5350
5599
|
async ensureInitialized() {
|
|
@@ -5378,20 +5627,20 @@ class TreeSitterParser {
|
|
|
5378
5627
|
async resolveWasmPath() {
|
|
5379
5628
|
try {
|
|
5380
5629
|
const webTreeSitterPath = __require.resolve("web-tree-sitter");
|
|
5381
|
-
const wasmPath =
|
|
5382
|
-
if (
|
|
5630
|
+
const wasmPath = path14.join(path14.dirname(webTreeSitterPath), "web-tree-sitter.wasm");
|
|
5631
|
+
if (fs6.existsSync(wasmPath)) {
|
|
5383
5632
|
return wasmPath;
|
|
5384
5633
|
}
|
|
5385
5634
|
} catch {}
|
|
5386
5635
|
try {
|
|
5387
5636
|
const possiblePaths = [
|
|
5388
|
-
|
|
5389
|
-
|
|
5390
|
-
|
|
5391
|
-
|
|
5637
|
+
path14.join(__dirname, "../../../node_modules/web-tree-sitter/web-tree-sitter.wasm"),
|
|
5638
|
+
path14.join(__dirname, "../../node_modules/web-tree-sitter/web-tree-sitter.wasm"),
|
|
5639
|
+
path14.join(__dirname, "../../../../node_modules/web-tree-sitter/web-tree-sitter.wasm"),
|
|
5640
|
+
path14.join(__dirname, "web-tree-sitter.wasm")
|
|
5392
5641
|
];
|
|
5393
5642
|
for (const wasmPath of possiblePaths) {
|
|
5394
|
-
if (
|
|
5643
|
+
if (fs6.existsSync(wasmPath)) {
|
|
5395
5644
|
return wasmPath;
|
|
5396
5645
|
}
|
|
5397
5646
|
}
|
|
@@ -5432,7 +5681,7 @@ class TreeSitterParser {
|
|
|
5432
5681
|
startLine: 1,
|
|
5433
5682
|
endLine: lines.length,
|
|
5434
5683
|
type: "file",
|
|
5435
|
-
name:
|
|
5684
|
+
name: path14.basename(filepath),
|
|
5436
5685
|
isExported: false
|
|
5437
5686
|
});
|
|
5438
5687
|
}
|
|
@@ -5779,7 +6028,7 @@ class TreeSitterParser {
|
|
|
5779
6028
|
startLine: 1,
|
|
5780
6029
|
endLine: lines.length,
|
|
5781
6030
|
type: "file",
|
|
5782
|
-
name:
|
|
6031
|
+
name: path14.basename(filepath)
|
|
5783
6032
|
});
|
|
5784
6033
|
return {
|
|
5785
6034
|
chunks,
|
|
@@ -5788,7 +6037,7 @@ class TreeSitterParser {
|
|
|
5788
6037
|
};
|
|
5789
6038
|
}
|
|
5790
6039
|
}
|
|
5791
|
-
var __dirname = "/
|
|
6040
|
+
var __dirname = "/home/runner/work/raggrep/raggrep/src/infrastructure/parsing", EXTENSION_TO_LANGUAGE2;
|
|
5792
6041
|
var init_treeSitterParser = __esm(() => {
|
|
5793
6042
|
init_grammarManager();
|
|
5794
6043
|
EXTENSION_TO_LANGUAGE2 = {
|
|
@@ -5809,7 +6058,7 @@ var init_treeSitterParser = __esm(() => {
|
|
|
5809
6058
|
});
|
|
5810
6059
|
|
|
5811
6060
|
// src/infrastructure/parsing/parserFactory.ts
|
|
5812
|
-
import * as
|
|
6061
|
+
import * as path15 from "path";
|
|
5813
6062
|
function getTypeScriptParser() {
|
|
5814
6063
|
if (!typescriptParserInstance) {
|
|
5815
6064
|
typescriptParserInstance = new TypeScriptParser;
|
|
@@ -5823,7 +6072,7 @@ function getTreeSitterParser() {
|
|
|
5823
6072
|
return treeSitterParserInstance;
|
|
5824
6073
|
}
|
|
5825
6074
|
function createParserForFile(filepath) {
|
|
5826
|
-
const ext =
|
|
6075
|
+
const ext = path15.extname(filepath).toLowerCase();
|
|
5827
6076
|
const parserType = EXTENSION_PARSER_MAP[ext];
|
|
5828
6077
|
if (!parserType) {
|
|
5829
6078
|
return null;
|
|
@@ -5872,9 +6121,9 @@ __export(exports_python, {
|
|
|
5872
6121
|
DEFAULT_TOP_K: () => DEFAULT_TOP_K3,
|
|
5873
6122
|
DEFAULT_MIN_SCORE: () => DEFAULT_MIN_SCORE3
|
|
5874
6123
|
});
|
|
5875
|
-
import * as
|
|
6124
|
+
import * as path16 from "path";
|
|
5876
6125
|
function isPythonFile(filepath) {
|
|
5877
|
-
const ext =
|
|
6126
|
+
const ext = path16.extname(filepath).toLowerCase();
|
|
5878
6127
|
return PYTHON_EXTENSIONS.includes(ext);
|
|
5879
6128
|
}
|
|
5880
6129
|
function generateChunkId3(filepath, startLine, endLine) {
|
|
@@ -5958,7 +6207,7 @@ class PythonModule {
|
|
|
5958
6207
|
startLine: 1,
|
|
5959
6208
|
endLine: lines.length,
|
|
5960
6209
|
type: "file",
|
|
5961
|
-
name:
|
|
6210
|
+
name: path16.basename(filepath)
|
|
5962
6211
|
});
|
|
5963
6212
|
const funcRegex = /^(\s*)(async\s+)?def\s+(\w+)\s*\([^)]*\)\s*:/gm;
|
|
5964
6213
|
let match;
|
|
@@ -6036,12 +6285,13 @@ class PythonModule {
|
|
|
6036
6285
|
return chunks;
|
|
6037
6286
|
}
|
|
6038
6287
|
async createFileIndex(filepath, content, parsedChunks, ctx) {
|
|
6039
|
-
const pathContext = parsePathContext(filepath);
|
|
6040
|
-
const pathPrefix = formatPathContextForEmbedding(pathContext);
|
|
6041
6288
|
const chunkContents = parsedChunks.map((c) => {
|
|
6042
|
-
|
|
6043
|
-
|
|
6044
|
-
|
|
6289
|
+
return prepareChunkForEmbedding({
|
|
6290
|
+
filepath,
|
|
6291
|
+
content: c.content,
|
|
6292
|
+
name: c.name,
|
|
6293
|
+
docComment: c.docComment
|
|
6294
|
+
});
|
|
6045
6295
|
});
|
|
6046
6296
|
const embeddings = await getEmbeddings(chunkContents);
|
|
6047
6297
|
const chunks = parsedChunks.map((pc) => ({
|
|
@@ -6064,25 +6314,21 @@ class PythonModule {
|
|
|
6064
6314
|
...new Set(parsedChunks.map((pc) => pc.type))
|
|
6065
6315
|
];
|
|
6066
6316
|
const exports = parsedChunks.filter((pc) => pc.isExported && pc.name).map((pc) => pc.name);
|
|
6067
|
-
const
|
|
6317
|
+
const contentKeywords = new Set;
|
|
6068
6318
|
for (const pc of parsedChunks) {
|
|
6069
6319
|
const keywords = extractKeywords(pc.content, pc.name);
|
|
6070
|
-
keywords.forEach((k) =>
|
|
6320
|
+
keywords.forEach((k) => contentKeywords.add(k));
|
|
6071
6321
|
}
|
|
6072
|
-
|
|
6322
|
+
const pathKeywords = extractPathKeywordsForFileSummary(filepath);
|
|
6323
|
+
const allKeywords = [...contentKeywords, ...pathKeywords];
|
|
6073
6324
|
const fileSummary = {
|
|
6074
6325
|
filepath,
|
|
6075
6326
|
chunkCount: chunks.length,
|
|
6076
6327
|
chunkTypes,
|
|
6077
|
-
keywords:
|
|
6328
|
+
keywords: [...new Set(allKeywords)],
|
|
6078
6329
|
exports,
|
|
6079
6330
|
lastModified: stats.lastModified,
|
|
6080
|
-
pathContext:
|
|
6081
|
-
segments: pathContext.segments,
|
|
6082
|
-
layer: pathContext.layer,
|
|
6083
|
-
domain: pathContext.domain,
|
|
6084
|
-
depth: pathContext.depth
|
|
6085
|
-
}
|
|
6331
|
+
pathContext: getPathContextForFileSummary(filepath)
|
|
6086
6332
|
};
|
|
6087
6333
|
this.pendingSummaries.set(filepath, fileSummary);
|
|
6088
6334
|
for (const chunk of chunks) {
|
|
@@ -6335,9 +6581,9 @@ __export(exports_go, {
|
|
|
6335
6581
|
DEFAULT_TOP_K: () => DEFAULT_TOP_K4,
|
|
6336
6582
|
DEFAULT_MIN_SCORE: () => DEFAULT_MIN_SCORE4
|
|
6337
6583
|
});
|
|
6338
|
-
import * as
|
|
6584
|
+
import * as path17 from "path";
|
|
6339
6585
|
function isGoFile(filepath) {
|
|
6340
|
-
const ext =
|
|
6586
|
+
const ext = path17.extname(filepath).toLowerCase();
|
|
6341
6587
|
return GO_EXTENSIONS.includes(ext);
|
|
6342
6588
|
}
|
|
6343
6589
|
function generateChunkId4(filepath, startLine, endLine) {
|
|
@@ -6422,7 +6668,7 @@ class GoModule {
|
|
|
6422
6668
|
startLine: 1,
|
|
6423
6669
|
endLine: lines.length,
|
|
6424
6670
|
type: "file",
|
|
6425
|
-
name:
|
|
6671
|
+
name: path17.basename(filepath)
|
|
6426
6672
|
});
|
|
6427
6673
|
const funcRegex = /^func\s+(?:\(\s*\w+\s+\*?\w+\s*\)\s+)?(\w+)\s*\(/gm;
|
|
6428
6674
|
let match;
|
|
@@ -6571,12 +6817,13 @@ class GoModule {
|
|
|
6571
6817
|
return chunks;
|
|
6572
6818
|
}
|
|
6573
6819
|
async createFileIndex(filepath, content, parsedChunks, ctx) {
|
|
6574
|
-
const pathContext = parsePathContext(filepath);
|
|
6575
|
-
const pathPrefix = formatPathContextForEmbedding(pathContext);
|
|
6576
6820
|
const chunkContents = parsedChunks.map((c) => {
|
|
6577
|
-
|
|
6578
|
-
|
|
6579
|
-
|
|
6821
|
+
return prepareChunkForEmbedding({
|
|
6822
|
+
filepath,
|
|
6823
|
+
content: c.content,
|
|
6824
|
+
name: c.name,
|
|
6825
|
+
docComment: c.docComment
|
|
6826
|
+
});
|
|
6580
6827
|
});
|
|
6581
6828
|
const embeddings = await getEmbeddings(chunkContents);
|
|
6582
6829
|
const chunks = parsedChunks.map((pc) => ({
|
|
@@ -6599,25 +6846,21 @@ class GoModule {
|
|
|
6599
6846
|
...new Set(parsedChunks.map((pc) => pc.type))
|
|
6600
6847
|
];
|
|
6601
6848
|
const exports = parsedChunks.filter((pc) => pc.isExported && pc.name).map((pc) => pc.name);
|
|
6602
|
-
const
|
|
6849
|
+
const contentKeywords = new Set;
|
|
6603
6850
|
for (const pc of parsedChunks) {
|
|
6604
6851
|
const keywords = extractKeywords(pc.content, pc.name);
|
|
6605
|
-
keywords.forEach((k) =>
|
|
6852
|
+
keywords.forEach((k) => contentKeywords.add(k));
|
|
6606
6853
|
}
|
|
6607
|
-
|
|
6854
|
+
const pathKeywords = extractPathKeywordsForFileSummary(filepath);
|
|
6855
|
+
const allKeywords = [...contentKeywords, ...pathKeywords];
|
|
6608
6856
|
const fileSummary = {
|
|
6609
6857
|
filepath,
|
|
6610
6858
|
chunkCount: chunks.length,
|
|
6611
6859
|
chunkTypes,
|
|
6612
|
-
keywords:
|
|
6860
|
+
keywords: [...new Set(allKeywords)],
|
|
6613
6861
|
exports,
|
|
6614
6862
|
lastModified: stats.lastModified,
|
|
6615
|
-
pathContext:
|
|
6616
|
-
segments: pathContext.segments,
|
|
6617
|
-
layer: pathContext.layer,
|
|
6618
|
-
domain: pathContext.domain,
|
|
6619
|
-
depth: pathContext.depth
|
|
6620
|
-
}
|
|
6863
|
+
pathContext: getPathContextForFileSummary(filepath)
|
|
6621
6864
|
};
|
|
6622
6865
|
this.pendingSummaries.set(filepath, fileSummary);
|
|
6623
6866
|
for (const chunk of chunks) {
|
|
@@ -6819,9 +7062,9 @@ __export(exports_rust, {
|
|
|
6819
7062
|
DEFAULT_TOP_K: () => DEFAULT_TOP_K5,
|
|
6820
7063
|
DEFAULT_MIN_SCORE: () => DEFAULT_MIN_SCORE5
|
|
6821
7064
|
});
|
|
6822
|
-
import * as
|
|
7065
|
+
import * as path18 from "path";
|
|
6823
7066
|
function isRustFile(filepath) {
|
|
6824
|
-
const ext =
|
|
7067
|
+
const ext = path18.extname(filepath).toLowerCase();
|
|
6825
7068
|
return RUST_EXTENSIONS.includes(ext);
|
|
6826
7069
|
}
|
|
6827
7070
|
function generateChunkId5(filepath, startLine, endLine) {
|
|
@@ -6908,7 +7151,7 @@ class RustModule {
|
|
|
6908
7151
|
startLine: 1,
|
|
6909
7152
|
endLine: lines.length,
|
|
6910
7153
|
type: "file",
|
|
6911
|
-
name:
|
|
7154
|
+
name: path18.basename(filepath)
|
|
6912
7155
|
});
|
|
6913
7156
|
const funcRegex = /^(pub(?:\s*\([^)]*\))?\s+)?(?:async\s+)?fn\s+(\w+)/gm;
|
|
6914
7157
|
let match;
|
|
@@ -7134,12 +7377,13 @@ class RustModule {
|
|
|
7134
7377
|
return chunks;
|
|
7135
7378
|
}
|
|
7136
7379
|
async createFileIndex(filepath, content, parsedChunks, ctx) {
|
|
7137
|
-
const pathContext = parsePathContext(filepath);
|
|
7138
|
-
const pathPrefix = formatPathContextForEmbedding(pathContext);
|
|
7139
7380
|
const chunkContents = parsedChunks.map((c) => {
|
|
7140
|
-
|
|
7141
|
-
|
|
7142
|
-
|
|
7381
|
+
return prepareChunkForEmbedding({
|
|
7382
|
+
filepath,
|
|
7383
|
+
content: c.content,
|
|
7384
|
+
name: c.name,
|
|
7385
|
+
docComment: c.docComment
|
|
7386
|
+
});
|
|
7143
7387
|
});
|
|
7144
7388
|
const embeddings = await getEmbeddings(chunkContents);
|
|
7145
7389
|
const chunks = parsedChunks.map((pc) => ({
|
|
@@ -7162,25 +7406,21 @@ class RustModule {
|
|
|
7162
7406
|
...new Set(parsedChunks.map((pc) => pc.type))
|
|
7163
7407
|
];
|
|
7164
7408
|
const exports = parsedChunks.filter((pc) => pc.isExported && pc.name).map((pc) => pc.name);
|
|
7165
|
-
const
|
|
7409
|
+
const contentKeywords = new Set;
|
|
7166
7410
|
for (const pc of parsedChunks) {
|
|
7167
7411
|
const keywords = extractKeywords(pc.content, pc.name);
|
|
7168
|
-
keywords.forEach((k) =>
|
|
7412
|
+
keywords.forEach((k) => contentKeywords.add(k));
|
|
7169
7413
|
}
|
|
7170
|
-
|
|
7414
|
+
const pathKeywords = extractPathKeywordsForFileSummary(filepath);
|
|
7415
|
+
const allKeywords = [...contentKeywords, ...pathKeywords];
|
|
7171
7416
|
const fileSummary = {
|
|
7172
7417
|
filepath,
|
|
7173
7418
|
chunkCount: chunks.length,
|
|
7174
7419
|
chunkTypes,
|
|
7175
|
-
keywords:
|
|
7420
|
+
keywords: [...new Set(allKeywords)],
|
|
7176
7421
|
exports,
|
|
7177
7422
|
lastModified: stats.lastModified,
|
|
7178
|
-
pathContext:
|
|
7179
|
-
segments: pathContext.segments,
|
|
7180
|
-
layer: pathContext.layer,
|
|
7181
|
-
domain: pathContext.domain,
|
|
7182
|
-
depth: pathContext.depth
|
|
7183
|
-
}
|
|
7423
|
+
pathContext: getPathContextForFileSummary(filepath)
|
|
7184
7424
|
};
|
|
7185
7425
|
this.pendingSummaries.set(filepath, fileSummary);
|
|
7186
7426
|
for (const chunk of chunks) {
|
|
@@ -7382,9 +7622,9 @@ __export(exports_json, {
|
|
|
7382
7622
|
DEFAULT_TOP_K: () => DEFAULT_TOP_K6,
|
|
7383
7623
|
DEFAULT_MIN_SCORE: () => DEFAULT_MIN_SCORE6
|
|
7384
7624
|
});
|
|
7385
|
-
import * as
|
|
7625
|
+
import * as path19 from "path";
|
|
7386
7626
|
function isJsonFile(filepath) {
|
|
7387
|
-
const ext =
|
|
7627
|
+
const ext = path19.extname(filepath).toLowerCase();
|
|
7388
7628
|
return JSON_EXTENSIONS.includes(ext);
|
|
7389
7629
|
}
|
|
7390
7630
|
|
|
@@ -7418,7 +7658,7 @@ class JsonModule {
|
|
|
7418
7658
|
} catch {
|
|
7419
7659
|
return null;
|
|
7420
7660
|
}
|
|
7421
|
-
const fileBasename =
|
|
7661
|
+
const fileBasename = path19.basename(filepath, path19.extname(filepath));
|
|
7422
7662
|
const jsonPathLiterals = extractJsonPaths(parsed, fileBasename);
|
|
7423
7663
|
const lines = content.split(`
|
|
7424
7664
|
`);
|
|
@@ -7625,7 +7865,7 @@ __export(exports_markdown, {
|
|
|
7625
7865
|
DEFAULT_TOP_K: () => DEFAULT_TOP_K7,
|
|
7626
7866
|
DEFAULT_MIN_SCORE: () => DEFAULT_MIN_SCORE7
|
|
7627
7867
|
});
|
|
7628
|
-
import * as
|
|
7868
|
+
import * as path20 from "path";
|
|
7629
7869
|
function calculateHeadingLevelBoost(chunk) {
|
|
7630
7870
|
const metadata = chunk.metadata;
|
|
7631
7871
|
const level = metadata?.headingLevel ?? 0;
|
|
@@ -7645,7 +7885,7 @@ function calculateHeadingLevelBoost(chunk) {
|
|
|
7645
7885
|
}
|
|
7646
7886
|
}
|
|
7647
7887
|
function isMarkdownFile(filepath) {
|
|
7648
|
-
const ext =
|
|
7888
|
+
const ext = path20.extname(filepath).toLowerCase();
|
|
7649
7889
|
return MARKDOWN_EXTENSIONS.includes(ext);
|
|
7650
7890
|
}
|
|
7651
7891
|
function parseMarkdownHierarchical(content, maxDepth = 4) {
|
|
@@ -7777,9 +8017,11 @@ class MarkdownModule {
|
|
|
7777
8017
|
return null;
|
|
7778
8018
|
}
|
|
7779
8019
|
const chunkContents = hierarchicalChunks.map((s) => {
|
|
7780
|
-
|
|
7781
|
-
|
|
7782
|
-
|
|
8020
|
+
return prepareChunkForEmbedding({
|
|
8021
|
+
filepath,
|
|
8022
|
+
content: s.content,
|
|
8023
|
+
name: s.heading || undefined
|
|
8024
|
+
});
|
|
7783
8025
|
});
|
|
7784
8026
|
const embeddings = await getEmbeddings(chunkContents);
|
|
7785
8027
|
const chunks = hierarchicalChunks.map((section) => ({
|
|
@@ -7803,14 +8045,17 @@ class MarkdownModule {
|
|
|
7803
8045
|
embeddingModel: currentConfig.model,
|
|
7804
8046
|
headings: uniqueHeadings
|
|
7805
8047
|
};
|
|
7806
|
-
const
|
|
8048
|
+
const contentKeywords = extractMarkdownKeywords(content);
|
|
8049
|
+
const pathKeywords = extractPathKeywordsForFileSummary(filepath);
|
|
8050
|
+
const allKeywords = [...new Set([...contentKeywords, ...pathKeywords])];
|
|
7807
8051
|
const fileSummary = {
|
|
7808
8052
|
filepath,
|
|
7809
8053
|
chunkCount: chunks.length,
|
|
7810
8054
|
chunkTypes: ["block"],
|
|
7811
|
-
keywords,
|
|
8055
|
+
keywords: allKeywords,
|
|
7812
8056
|
exports: uniqueHeadings,
|
|
7813
|
-
lastModified: stats.lastModified
|
|
8057
|
+
lastModified: stats.lastModified,
|
|
8058
|
+
pathContext: getPathContextForFileSummary(filepath)
|
|
7814
8059
|
};
|
|
7815
8060
|
this.pendingSummaries.set(filepath, fileSummary);
|
|
7816
8061
|
return {
|
|
@@ -7941,6 +8186,11 @@ var init_markdown = __esm(() => {
|
|
|
7941
8186
|
supportsFile6 = isMarkdownFile;
|
|
7942
8187
|
});
|
|
7943
8188
|
|
|
8189
|
+
// src/types.ts
|
|
8190
|
+
var init_types = __esm(() => {
|
|
8191
|
+
init_entities();
|
|
8192
|
+
});
|
|
8193
|
+
|
|
7944
8194
|
// node_modules/fdir/dist/index.mjs
|
|
7945
8195
|
import { createRequire as createRequire2 } from "module";
|
|
7946
8196
|
import { basename, dirname, normalize, relative, resolve, sep } from "path";
|
|
@@ -8476,9 +8726,9 @@ var Builder = class {
|
|
|
8476
8726
|
|
|
8477
8727
|
// src/app/indexer/index.ts
|
|
8478
8728
|
init_config2();
|
|
8479
|
-
import * as
|
|
8480
|
-
import * as
|
|
8481
|
-
import * as
|
|
8729
|
+
import * as fs9 from "fs/promises";
|
|
8730
|
+
import * as path23 from "path";
|
|
8731
|
+
import * as os2 from "os";
|
|
8482
8732
|
import * as crypto2 from "crypto";
|
|
8483
8733
|
|
|
8484
8734
|
// src/modules/registry.ts
|
|
@@ -8519,13 +8769,13 @@ async function registerBuiltInModules() {
|
|
|
8519
8769
|
}
|
|
8520
8770
|
|
|
8521
8771
|
// src/infrastructure/introspection/IntrospectionIndex.ts
|
|
8522
|
-
import * as
|
|
8523
|
-
import * as
|
|
8772
|
+
import * as path22 from "path";
|
|
8773
|
+
import * as fs8 from "fs/promises";
|
|
8524
8774
|
import * as fsSync from "fs";
|
|
8525
8775
|
|
|
8526
8776
|
// src/infrastructure/introspection/projectDetector.ts
|
|
8527
|
-
import * as
|
|
8528
|
-
import * as
|
|
8777
|
+
import * as path21 from "path";
|
|
8778
|
+
import * as fs7 from "fs/promises";
|
|
8529
8779
|
var MAX_SCAN_DEPTH = 4;
|
|
8530
8780
|
var SKIP_DIRS = new Set([
|
|
8531
8781
|
"node_modules",
|
|
@@ -8541,9 +8791,9 @@ async function scanForPackageJsons(rootDir, currentDir = "", depth = 0) {
|
|
|
8541
8791
|
if (depth > MAX_SCAN_DEPTH)
|
|
8542
8792
|
return [];
|
|
8543
8793
|
const results = [];
|
|
8544
|
-
const fullDir = currentDir ?
|
|
8794
|
+
const fullDir = currentDir ? path21.join(rootDir, currentDir) : rootDir;
|
|
8545
8795
|
try {
|
|
8546
|
-
const entries = await
|
|
8796
|
+
const entries = await fs7.readdir(fullDir, { withFileTypes: true });
|
|
8547
8797
|
const hasPackageJson = entries.some((e) => e.isFile() && e.name === "package.json");
|
|
8548
8798
|
if (hasPackageJson && currentDir) {
|
|
8549
8799
|
const info = await parsePackageJson(rootDir, currentDir);
|
|
@@ -8564,10 +8814,10 @@ async function scanForPackageJsons(rootDir, currentDir = "", depth = 0) {
|
|
|
8564
8814
|
}
|
|
8565
8815
|
async function parsePackageJson(rootDir, relativePath) {
|
|
8566
8816
|
try {
|
|
8567
|
-
const packageJsonPath =
|
|
8568
|
-
const content = await
|
|
8817
|
+
const packageJsonPath = path21.join(rootDir, relativePath, "package.json");
|
|
8818
|
+
const content = await fs7.readFile(packageJsonPath, "utf-8");
|
|
8569
8819
|
const pkg = JSON.parse(content);
|
|
8570
|
-
const name = pkg.name ||
|
|
8820
|
+
const name = pkg.name || path21.basename(relativePath);
|
|
8571
8821
|
const deps = { ...pkg.dependencies, ...pkg.devDependencies };
|
|
8572
8822
|
let type = "unknown";
|
|
8573
8823
|
if (deps["next"] || deps["react"] || deps["vue"] || deps["svelte"]) {
|
|
@@ -8603,7 +8853,7 @@ async function detectProjectStructure(rootDir) {
|
|
|
8603
8853
|
const projectMap = new Map;
|
|
8604
8854
|
let isMonorepo = false;
|
|
8605
8855
|
try {
|
|
8606
|
-
const entries = await
|
|
8856
|
+
const entries = await fs7.readdir(rootDir, { withFileTypes: true });
|
|
8607
8857
|
const dirNames = entries.filter((e) => e.isDirectory()).map((e) => e.name);
|
|
8608
8858
|
const monorepoPatterns = ["apps", "packages", "libs", "services"];
|
|
8609
8859
|
const hasMonorepoStructure = monorepoPatterns.some((p) => dirNames.includes(p));
|
|
@@ -8612,9 +8862,9 @@ async function detectProjectStructure(rootDir) {
|
|
|
8612
8862
|
for (const pattern of monorepoPatterns) {
|
|
8613
8863
|
if (!dirNames.includes(pattern))
|
|
8614
8864
|
continue;
|
|
8615
|
-
const patternDir =
|
|
8865
|
+
const patternDir = path21.join(rootDir, pattern);
|
|
8616
8866
|
try {
|
|
8617
|
-
const subDirs = await
|
|
8867
|
+
const subDirs = await fs7.readdir(patternDir, { withFileTypes: true });
|
|
8618
8868
|
for (const subDir of subDirs) {
|
|
8619
8869
|
if (!subDir.isDirectory())
|
|
8620
8870
|
continue;
|
|
@@ -8643,8 +8893,8 @@ async function detectProjectStructure(rootDir) {
|
|
|
8643
8893
|
}
|
|
8644
8894
|
let rootType = "unknown";
|
|
8645
8895
|
try {
|
|
8646
|
-
const rootPkgPath =
|
|
8647
|
-
const rootPkg = JSON.parse(await
|
|
8896
|
+
const rootPkgPath = path21.join(rootDir, "package.json");
|
|
8897
|
+
const rootPkg = JSON.parse(await fs7.readFile(rootPkgPath, "utf-8"));
|
|
8648
8898
|
if (rootPkg.workspaces)
|
|
8649
8899
|
isMonorepo = true;
|
|
8650
8900
|
const deps = { ...rootPkg.dependencies, ...rootPkg.devDependencies };
|
|
@@ -8684,8 +8934,8 @@ class IntrospectionIndex {
|
|
|
8684
8934
|
async initialize() {
|
|
8685
8935
|
this.structure = await detectProjectStructure(this.rootDir);
|
|
8686
8936
|
try {
|
|
8687
|
-
const configPath =
|
|
8688
|
-
const configContent = await
|
|
8937
|
+
const configPath = path22.join(this.rootDir, ".raggrep", "config.json");
|
|
8938
|
+
const configContent = await fs8.readFile(configPath, "utf-8");
|
|
8689
8939
|
const config = JSON.parse(configContent);
|
|
8690
8940
|
this.config = config.introspection || {};
|
|
8691
8941
|
} catch {}
|
|
@@ -8699,7 +8949,7 @@ class IntrospectionIndex {
|
|
|
8699
8949
|
}
|
|
8700
8950
|
const fileExists = enableReadmeContext ? (relativePath) => {
|
|
8701
8951
|
try {
|
|
8702
|
-
const absolutePath =
|
|
8952
|
+
const absolutePath = path22.join(this.rootDir, relativePath);
|
|
8703
8953
|
return fsSync.existsSync(absolutePath);
|
|
8704
8954
|
} catch {
|
|
8705
8955
|
return false;
|
|
@@ -8735,28 +8985,28 @@ class IntrospectionIndex {
|
|
|
8735
8985
|
}
|
|
8736
8986
|
}
|
|
8737
8987
|
async save(config) {
|
|
8738
|
-
const introDir =
|
|
8739
|
-
await
|
|
8740
|
-
const projectPath =
|
|
8741
|
-
await
|
|
8988
|
+
const introDir = path22.join(getRaggrepDir(this.rootDir, config), "introspection");
|
|
8989
|
+
await fs8.mkdir(introDir, { recursive: true });
|
|
8990
|
+
const projectPath = path22.join(introDir, "_project.json");
|
|
8991
|
+
await fs8.writeFile(projectPath, JSON.stringify({
|
|
8742
8992
|
version: "1.0.0",
|
|
8743
8993
|
lastUpdated: new Date().toISOString(),
|
|
8744
8994
|
structure: this.structure
|
|
8745
8995
|
}, null, 2));
|
|
8746
8996
|
for (const [filepath, intro] of this.files) {
|
|
8747
|
-
const introFilePath =
|
|
8748
|
-
await
|
|
8749
|
-
await
|
|
8997
|
+
const introFilePath = path22.join(introDir, "files", filepath.replace(/\.[^.]+$/, ".json"));
|
|
8998
|
+
await fs8.mkdir(path22.dirname(introFilePath), { recursive: true });
|
|
8999
|
+
await fs8.writeFile(introFilePath, JSON.stringify(intro, null, 2));
|
|
8750
9000
|
}
|
|
8751
9001
|
}
|
|
8752
9002
|
async load(config) {
|
|
8753
|
-
const introDir =
|
|
9003
|
+
const introDir = path22.join(getRaggrepDir(this.rootDir, config), "introspection");
|
|
8754
9004
|
try {
|
|
8755
|
-
const projectPath =
|
|
8756
|
-
const projectContent = await
|
|
9005
|
+
const projectPath = path22.join(introDir, "_project.json");
|
|
9006
|
+
const projectContent = await fs8.readFile(projectPath, "utf-8");
|
|
8757
9007
|
const projectData = JSON.parse(projectContent);
|
|
8758
9008
|
this.structure = projectData.structure;
|
|
8759
|
-
await this.loadFilesRecursive(
|
|
9009
|
+
await this.loadFilesRecursive(path22.join(introDir, "files"), "");
|
|
8760
9010
|
} catch {
|
|
8761
9011
|
this.structure = null;
|
|
8762
9012
|
this.files.clear();
|
|
@@ -8764,14 +9014,14 @@ class IntrospectionIndex {
|
|
|
8764
9014
|
}
|
|
8765
9015
|
async loadFilesRecursive(basePath, prefix) {
|
|
8766
9016
|
try {
|
|
8767
|
-
const entries = await
|
|
9017
|
+
const entries = await fs8.readdir(basePath, { withFileTypes: true });
|
|
8768
9018
|
for (const entry of entries) {
|
|
8769
|
-
const entryPath =
|
|
9019
|
+
const entryPath = path22.join(basePath, entry.name);
|
|
8770
9020
|
const relativePath = prefix ? `${prefix}/${entry.name}` : entry.name;
|
|
8771
9021
|
if (entry.isDirectory()) {
|
|
8772
9022
|
await this.loadFilesRecursive(entryPath, relativePath);
|
|
8773
9023
|
} else if (entry.name.endsWith(".json")) {
|
|
8774
|
-
const content = await
|
|
9024
|
+
const content = await fs8.readFile(entryPath, "utf-8");
|
|
8775
9025
|
const intro = JSON.parse(content);
|
|
8776
9026
|
this.files.set(intro.filepath, intro);
|
|
8777
9027
|
}
|
|
@@ -9027,7 +9277,7 @@ async function parallelMap(items, processor, concurrency) {
|
|
|
9027
9277
|
await Promise.all(workers);
|
|
9028
9278
|
return results;
|
|
9029
9279
|
}
|
|
9030
|
-
var INDEX_SCHEMA_VERSION = "2.
|
|
9280
|
+
var INDEX_SCHEMA_VERSION = "2.1.0";
|
|
9031
9281
|
function formatDuration(ms) {
|
|
9032
9282
|
if (ms < 1000) {
|
|
9033
9283
|
return `${ms}ms`;
|
|
@@ -9041,7 +9291,7 @@ function formatDuration(ms) {
|
|
|
9041
9291
|
return `${minutes}m ${remainingSeconds.toFixed(1)}s`;
|
|
9042
9292
|
}
|
|
9043
9293
|
function getOptimalConcurrency() {
|
|
9044
|
-
const cpuCount =
|
|
9294
|
+
const cpuCount = os2.cpus().length;
|
|
9045
9295
|
const optimal = Math.max(2, Math.min(16, Math.floor(cpuCount * 0.75)));
|
|
9046
9296
|
return optimal;
|
|
9047
9297
|
}
|
|
@@ -9053,7 +9303,7 @@ async function indexDirectory(rootDir, options = {}) {
|
|
|
9053
9303
|
const concurrency = options.concurrency ?? DEFAULT_CONCURRENCY;
|
|
9054
9304
|
clearFreshnessCache();
|
|
9055
9305
|
const logger = options.logger ? options.logger : quiet ? createSilentLogger() : createLogger({ verbose });
|
|
9056
|
-
rootDir =
|
|
9306
|
+
rootDir = path23.resolve(rootDir);
|
|
9057
9307
|
const location = getIndexLocation(rootDir);
|
|
9058
9308
|
logger.info(`Indexing directory: ${rootDir}`);
|
|
9059
9309
|
logger.info(`Index location: ${location.indexDir}`);
|
|
@@ -9105,12 +9355,12 @@ async function indexDirectory(rootDir, options = {}) {
|
|
|
9105
9355
|
rootDir,
|
|
9106
9356
|
config,
|
|
9107
9357
|
readFile: async (filepath) => {
|
|
9108
|
-
const fullPath =
|
|
9109
|
-
return
|
|
9358
|
+
const fullPath = path23.isAbsolute(filepath) ? filepath : path23.join(rootDir, filepath);
|
|
9359
|
+
return fs9.readFile(fullPath, "utf-8");
|
|
9110
9360
|
},
|
|
9111
9361
|
getFileStats: async (filepath) => {
|
|
9112
|
-
const fullPath =
|
|
9113
|
-
const stats = await
|
|
9362
|
+
const fullPath = path23.isAbsolute(filepath) ? filepath : path23.join(rootDir, filepath);
|
|
9363
|
+
const stats = await fs9.stat(fullPath);
|
|
9114
9364
|
return { lastModified: stats.mtime.toISOString() };
|
|
9115
9365
|
}
|
|
9116
9366
|
};
|
|
@@ -9135,7 +9385,7 @@ async function isIndexVersionCompatible(rootDir) {
|
|
|
9135
9385
|
const config = await loadConfig(rootDir);
|
|
9136
9386
|
const globalManifestPath = getGlobalManifestPath(rootDir, config);
|
|
9137
9387
|
try {
|
|
9138
|
-
const content = await
|
|
9388
|
+
const content = await fs9.readFile(globalManifestPath, "utf-8");
|
|
9139
9389
|
const manifest = JSON.parse(content);
|
|
9140
9390
|
return manifest.version === INDEX_SCHEMA_VERSION;
|
|
9141
9391
|
} catch {
|
|
@@ -9145,11 +9395,11 @@ async function isIndexVersionCompatible(rootDir) {
|
|
|
9145
9395
|
async function deleteIndex(rootDir) {
|
|
9146
9396
|
const indexDir = getRaggrepDir(rootDir);
|
|
9147
9397
|
try {
|
|
9148
|
-
await
|
|
9398
|
+
await fs9.rm(indexDir, { recursive: true, force: true });
|
|
9149
9399
|
} catch {}
|
|
9150
9400
|
}
|
|
9151
9401
|
async function resetIndex(rootDir) {
|
|
9152
|
-
rootDir =
|
|
9402
|
+
rootDir = path23.resolve(rootDir);
|
|
9153
9403
|
clearFreshnessCache();
|
|
9154
9404
|
const status = await getIndexStatus(rootDir);
|
|
9155
9405
|
if (!status.exists) {
|
|
@@ -9174,7 +9424,7 @@ async function ensureIndexFresh(rootDir, options = {}) {
|
|
|
9174
9424
|
let filesChanged = 0;
|
|
9175
9425
|
let filesReindexed = 0;
|
|
9176
9426
|
const logger = options.logger ? options.logger : quiet ? createSilentLogger() : createLogger({ verbose });
|
|
9177
|
-
rootDir =
|
|
9427
|
+
rootDir = path23.resolve(rootDir);
|
|
9178
9428
|
const status = await getIndexStatus(rootDir);
|
|
9179
9429
|
if (!status.exists) {
|
|
9180
9430
|
clearFreshnessCache();
|
|
@@ -9198,7 +9448,7 @@ async function ensureIndexFresh(rootDir, options = {}) {
|
|
|
9198
9448
|
const globalManifestPath = getGlobalManifestPath(rootDir, config);
|
|
9199
9449
|
let currentManifestMtime = 0;
|
|
9200
9450
|
try {
|
|
9201
|
-
const manifestStats = await
|
|
9451
|
+
const manifestStats = await fs9.stat(globalManifestPath);
|
|
9202
9452
|
currentManifestMtime = manifestStats.mtimeMs;
|
|
9203
9453
|
} catch {}
|
|
9204
9454
|
const now = Date.now();
|
|
@@ -9236,7 +9486,7 @@ async function ensureIndexFresh(rootDir, options = {}) {
|
|
|
9236
9486
|
const { allFiles: currentFiles, changedFiles, changedFileMtimes } = discoveryResult;
|
|
9237
9487
|
filesDiscovered = currentFiles.length;
|
|
9238
9488
|
filesChanged = changedFiles.length;
|
|
9239
|
-
const currentFileSet = new Set(currentFiles.map((f) =>
|
|
9489
|
+
const currentFileSet = new Set(currentFiles.map((f) => path23.relative(rootDir, f)));
|
|
9240
9490
|
const changedFileSet = new Set(changedFiles);
|
|
9241
9491
|
let totalIndexed = 0;
|
|
9242
9492
|
let totalRemoved = 0;
|
|
@@ -9270,11 +9520,11 @@ async function ensureIndexFresh(rootDir, options = {}) {
|
|
|
9270
9520
|
if (filesToRemove.length > 0) {
|
|
9271
9521
|
await Promise.all(filesToRemove.map(async (filepath) => {
|
|
9272
9522
|
logger.debug(` Removing stale: ${filepath}`);
|
|
9273
|
-
const indexFilePath =
|
|
9274
|
-
const symbolicFilePath =
|
|
9523
|
+
const indexFilePath = path23.join(indexPath, filepath.replace(/\.[^.]+$/, ".json"));
|
|
9524
|
+
const symbolicFilePath = path23.join(indexPath, "symbolic", filepath.replace(/\.[^.]+$/, ".json"));
|
|
9275
9525
|
await Promise.all([
|
|
9276
|
-
|
|
9277
|
-
|
|
9526
|
+
fs9.unlink(indexFilePath).catch(() => {}),
|
|
9527
|
+
fs9.unlink(symbolicFilePath).catch(() => {})
|
|
9278
9528
|
]);
|
|
9279
9529
|
delete manifest.files[filepath];
|
|
9280
9530
|
removedFilepaths.push(filepath);
|
|
@@ -9298,19 +9548,19 @@ async function ensureIndexFresh(rootDir, options = {}) {
|
|
|
9298
9548
|
rootDir,
|
|
9299
9549
|
config,
|
|
9300
9550
|
readFile: async (filepath) => {
|
|
9301
|
-
const fullPath =
|
|
9302
|
-
return
|
|
9551
|
+
const fullPath = path23.isAbsolute(filepath) ? filepath : path23.join(rootDir, filepath);
|
|
9552
|
+
return fs9.readFile(fullPath, "utf-8");
|
|
9303
9553
|
},
|
|
9304
9554
|
getFileStats: async (filepath) => {
|
|
9305
|
-
const fullPath =
|
|
9306
|
-
const stats = await
|
|
9555
|
+
const fullPath = path23.isAbsolute(filepath) ? filepath : path23.join(rootDir, filepath);
|
|
9556
|
+
const stats = await fs9.stat(fullPath);
|
|
9307
9557
|
return { lastModified: stats.mtime.toISOString() };
|
|
9308
9558
|
},
|
|
9309
9559
|
getIntrospection: (filepath) => introspection.getFile(filepath)
|
|
9310
9560
|
};
|
|
9311
9561
|
const moduleChangedFiles = module.supportsFile ? changedFiles.filter((f) => module.supportsFile(f)) : changedFiles;
|
|
9312
9562
|
const filesToProcess = moduleChangedFiles.map((filepath) => {
|
|
9313
|
-
const relativePath =
|
|
9563
|
+
const relativePath = path23.relative(rootDir, filepath);
|
|
9314
9564
|
const existingEntry = manifest.files[relativePath];
|
|
9315
9565
|
const lastModified = changedFileMtimes.get(filepath) || new Date().toISOString();
|
|
9316
9566
|
return {
|
|
@@ -9340,7 +9590,7 @@ async function ensureIndexFresh(rootDir, options = {}) {
|
|
|
9340
9590
|
return { relativePath, status: "unchanged" };
|
|
9341
9591
|
}
|
|
9342
9592
|
try {
|
|
9343
|
-
const content = await
|
|
9593
|
+
const content = await fs9.readFile(filepath, "utf-8");
|
|
9344
9594
|
const contentHash = computeContentHash(content);
|
|
9345
9595
|
if (!isNew && existingContentHash && existingContentHash === contentHash) {
|
|
9346
9596
|
completedCount++;
|
|
@@ -9467,7 +9717,7 @@ async function ensureIndexFresh(rootDir, options = {}) {
|
|
|
9467
9717
|
}
|
|
9468
9718
|
let finalManifestMtime = currentManifestMtime;
|
|
9469
9719
|
try {
|
|
9470
|
-
const manifestStats = await
|
|
9720
|
+
const manifestStats = await fs9.stat(globalManifestPath);
|
|
9471
9721
|
finalManifestMtime = manifestStats.mtimeMs;
|
|
9472
9722
|
} catch {}
|
|
9473
9723
|
freshnessCache = {
|
|
@@ -9487,7 +9737,7 @@ async function indexWithModule(rootDir, files, module, config, verbose, introspe
|
|
|
9487
9737
|
};
|
|
9488
9738
|
const manifest = await loadModuleManifest(rootDir, module.id, config);
|
|
9489
9739
|
const indexPath = getModuleIndexPath(rootDir, module.id, config);
|
|
9490
|
-
const currentFileSet = new Set(files.map((f) =>
|
|
9740
|
+
const currentFileSet = new Set(files.map((f) => path23.relative(rootDir, f)));
|
|
9491
9741
|
const filesToRemove = [];
|
|
9492
9742
|
for (const filepath of Object.keys(manifest.files)) {
|
|
9493
9743
|
if (!currentFileSet.has(filepath)) {
|
|
@@ -9498,13 +9748,13 @@ async function indexWithModule(rootDir, files, module, config, verbose, introspe
|
|
|
9498
9748
|
logger.info(` Removing ${filesToRemove.length} stale entries...`);
|
|
9499
9749
|
for (const filepath of filesToRemove) {
|
|
9500
9750
|
logger.debug(` Removing: ${filepath}`);
|
|
9501
|
-
const indexFilePath =
|
|
9751
|
+
const indexFilePath = path23.join(indexPath, filepath.replace(/\.[^.]+$/, ".json"));
|
|
9502
9752
|
try {
|
|
9503
|
-
await
|
|
9753
|
+
await fs9.unlink(indexFilePath);
|
|
9504
9754
|
} catch {}
|
|
9505
|
-
const symbolicFilePath =
|
|
9755
|
+
const symbolicFilePath = path23.join(indexPath, "symbolic", filepath.replace(/\.[^.]+$/, ".json"));
|
|
9506
9756
|
try {
|
|
9507
|
-
await
|
|
9757
|
+
await fs9.unlink(symbolicFilePath);
|
|
9508
9758
|
} catch {}
|
|
9509
9759
|
delete manifest.files[filepath];
|
|
9510
9760
|
}
|
|
@@ -9514,12 +9764,12 @@ async function indexWithModule(rootDir, files, module, config, verbose, introspe
|
|
|
9514
9764
|
rootDir,
|
|
9515
9765
|
config,
|
|
9516
9766
|
readFile: async (filepath) => {
|
|
9517
|
-
const fullPath =
|
|
9518
|
-
return
|
|
9767
|
+
const fullPath = path23.isAbsolute(filepath) ? filepath : path23.join(rootDir, filepath);
|
|
9768
|
+
return fs9.readFile(fullPath, "utf-8");
|
|
9519
9769
|
},
|
|
9520
9770
|
getFileStats: async (filepath) => {
|
|
9521
|
-
const fullPath =
|
|
9522
|
-
const stats = await
|
|
9771
|
+
const fullPath = path23.isAbsolute(filepath) ? filepath : path23.join(rootDir, filepath);
|
|
9772
|
+
const stats = await fs9.stat(fullPath);
|
|
9523
9773
|
return { lastModified: stats.mtime.toISOString() };
|
|
9524
9774
|
},
|
|
9525
9775
|
getIntrospection: (filepath) => introspection.getFile(filepath)
|
|
@@ -9531,9 +9781,9 @@ async function indexWithModule(rootDir, files, module, config, verbose, introspe
|
|
|
9531
9781
|
let indexedCount = 0;
|
|
9532
9782
|
let skippedCount = 0;
|
|
9533
9783
|
const processFile = async (filepath, _index) => {
|
|
9534
|
-
const relativePath =
|
|
9784
|
+
const relativePath = path23.relative(rootDir, filepath);
|
|
9535
9785
|
try {
|
|
9536
|
-
const stats = await
|
|
9786
|
+
const stats = await fs9.stat(filepath);
|
|
9537
9787
|
const lastModified = stats.mtime.toISOString();
|
|
9538
9788
|
const existingEntry = manifest.files[relativePath];
|
|
9539
9789
|
if (existingEntry && existingEntry.lastModified === lastModified) {
|
|
@@ -9543,7 +9793,7 @@ async function indexWithModule(rootDir, files, module, config, verbose, introspe
|
|
|
9543
9793
|
logger.debug(` [${completedCount}/${totalFiles}] Skipped ${relativePath} (unchanged)`);
|
|
9544
9794
|
return { relativePath, status: "skipped" };
|
|
9545
9795
|
}
|
|
9546
|
-
const content = await
|
|
9796
|
+
const content = await fs9.readFile(filepath, "utf-8");
|
|
9547
9797
|
const contentHash = computeContentHash(content);
|
|
9548
9798
|
if (existingEntry?.contentHash && existingEntry.contentHash === contentHash) {
|
|
9549
9799
|
completedCount++;
|
|
@@ -9625,8 +9875,8 @@ async function indexWithModule(rootDir, files, module, config, verbose, introspe
|
|
|
9625
9875
|
}
|
|
9626
9876
|
var STAT_CONCURRENCY = 64;
|
|
9627
9877
|
function isLikelyBinary(filepath) {
|
|
9628
|
-
const ext =
|
|
9629
|
-
const basename15 =
|
|
9878
|
+
const ext = path23.extname(filepath).toLowerCase();
|
|
9879
|
+
const basename15 = path23.basename(filepath).toLowerCase();
|
|
9630
9880
|
const binaryExtensions = new Set([
|
|
9631
9881
|
".png",
|
|
9632
9882
|
".jpg",
|
|
@@ -9702,7 +9952,7 @@ async function findFilesWithStats(rootDir, config, lastIndexStarted) {
|
|
|
9702
9952
|
const ignoreDirs = new Set(config.ignorePaths);
|
|
9703
9953
|
const lastIndexMs = lastIndexStarted?.getTime() ?? 0;
|
|
9704
9954
|
const crawler = new Builder().withFullPaths().exclude((dirName) => ignoreDirs.has(dirName)).filter((filePath) => {
|
|
9705
|
-
const ext =
|
|
9955
|
+
const ext = path23.extname(filePath);
|
|
9706
9956
|
return validExtensions.has(ext);
|
|
9707
9957
|
}).crawl(rootDir);
|
|
9708
9958
|
const allFiles = await crawler.withPromise();
|
|
@@ -9710,7 +9960,7 @@ async function findFilesWithStats(rootDir, config, lastIndexStarted) {
|
|
|
9710
9960
|
const changedFileMtimes2 = new Map;
|
|
9711
9961
|
await parallelMap(allFiles, async (filePath) => {
|
|
9712
9962
|
try {
|
|
9713
|
-
const stats = await
|
|
9963
|
+
const stats = await fs9.stat(filePath);
|
|
9714
9964
|
changedFileMtimes2.set(filePath, stats.mtime.toISOString());
|
|
9715
9965
|
} catch {}
|
|
9716
9966
|
}, STAT_CONCURRENCY);
|
|
@@ -9724,7 +9974,7 @@ async function findFilesWithStats(rootDir, config, lastIndexStarted) {
|
|
|
9724
9974
|
const changedFileMtimes = new Map;
|
|
9725
9975
|
await parallelMap(allFiles, async (filePath) => {
|
|
9726
9976
|
try {
|
|
9727
|
-
const stats = await
|
|
9977
|
+
const stats = await fs9.stat(filePath);
|
|
9728
9978
|
if (stats.mtimeMs > lastIndexMs) {
|
|
9729
9979
|
changedFiles.push(filePath);
|
|
9730
9980
|
changedFileMtimes.set(filePath, stats.mtime.toISOString());
|
|
@@ -9744,7 +9994,7 @@ async function findFiles(rootDir, config) {
|
|
|
9744
9994
|
async function loadModuleManifest(rootDir, moduleId, config) {
|
|
9745
9995
|
const manifestPath = getModuleManifestPath(rootDir, moduleId, config);
|
|
9746
9996
|
try {
|
|
9747
|
-
const content = await
|
|
9997
|
+
const content = await fs9.readFile(manifestPath, "utf-8");
|
|
9748
9998
|
return JSON.parse(content);
|
|
9749
9999
|
} catch {
|
|
9750
10000
|
return {
|
|
@@ -9757,19 +10007,19 @@ async function loadModuleManifest(rootDir, moduleId, config) {
|
|
|
9757
10007
|
}
|
|
9758
10008
|
async function writeModuleManifest(rootDir, moduleId, manifest, config) {
|
|
9759
10009
|
const manifestPath = getModuleManifestPath(rootDir, moduleId, config);
|
|
9760
|
-
await
|
|
9761
|
-
await
|
|
10010
|
+
await fs9.mkdir(path23.dirname(manifestPath), { recursive: true });
|
|
10011
|
+
await fs9.writeFile(manifestPath, JSON.stringify(manifest, null, 2));
|
|
9762
10012
|
}
|
|
9763
10013
|
async function writeFileIndex(rootDir, moduleId, filepath, fileIndex, config) {
|
|
9764
10014
|
const indexPath = getModuleIndexPath(rootDir, moduleId, config);
|
|
9765
|
-
const indexFilePath =
|
|
9766
|
-
await
|
|
9767
|
-
await
|
|
10015
|
+
const indexFilePath = path23.join(indexPath, filepath.replace(/\.[^.]+$/, ".json"));
|
|
10016
|
+
await fs9.mkdir(path23.dirname(indexFilePath), { recursive: true });
|
|
10017
|
+
await fs9.writeFile(indexFilePath, JSON.stringify(fileIndex, null, 2));
|
|
9768
10018
|
}
|
|
9769
10019
|
async function loadGlobalManifest(rootDir, config) {
|
|
9770
10020
|
const manifestPath = getGlobalManifestPath(rootDir, config);
|
|
9771
10021
|
try {
|
|
9772
|
-
const content = await
|
|
10022
|
+
const content = await fs9.readFile(manifestPath, "utf-8");
|
|
9773
10023
|
return JSON.parse(content);
|
|
9774
10024
|
} catch {
|
|
9775
10025
|
return null;
|
|
@@ -9783,13 +10033,13 @@ async function updateGlobalManifest(rootDir, modules, config, indexStartTime) {
|
|
|
9783
10033
|
lastIndexStarted: indexStartTime,
|
|
9784
10034
|
modules: modules.map((m) => m.id)
|
|
9785
10035
|
};
|
|
9786
|
-
await
|
|
9787
|
-
await
|
|
10036
|
+
await fs9.mkdir(path23.dirname(manifestPath), { recursive: true });
|
|
10037
|
+
await fs9.writeFile(manifestPath, JSON.stringify(manifest, null, 2));
|
|
9788
10038
|
}
|
|
9789
10039
|
async function cleanupIndex(rootDir, options = {}) {
|
|
9790
10040
|
const verbose = options.verbose ?? false;
|
|
9791
10041
|
const logger = options.logger ?? createLogger({ verbose });
|
|
9792
|
-
rootDir =
|
|
10042
|
+
rootDir = path23.resolve(rootDir);
|
|
9793
10043
|
logger.info(`Cleaning up index in: ${rootDir}`);
|
|
9794
10044
|
const config = await loadConfig(rootDir);
|
|
9795
10045
|
await registerBuiltInModules();
|
|
@@ -9819,9 +10069,9 @@ async function cleanupModuleIndex(rootDir, moduleId, config, logger) {
|
|
|
9819
10069
|
const filesToRemove = [];
|
|
9820
10070
|
const updatedFiles = {};
|
|
9821
10071
|
for (const [filepath, entry] of Object.entries(manifest.files)) {
|
|
9822
|
-
const fullPath =
|
|
10072
|
+
const fullPath = path23.join(rootDir, filepath);
|
|
9823
10073
|
try {
|
|
9824
|
-
await
|
|
10074
|
+
await fs9.access(fullPath);
|
|
9825
10075
|
updatedFiles[filepath] = entry;
|
|
9826
10076
|
result.kept++;
|
|
9827
10077
|
} catch {
|
|
@@ -9831,9 +10081,9 @@ async function cleanupModuleIndex(rootDir, moduleId, config, logger) {
|
|
|
9831
10081
|
}
|
|
9832
10082
|
}
|
|
9833
10083
|
for (const filepath of filesToRemove) {
|
|
9834
|
-
const indexFilePath =
|
|
10084
|
+
const indexFilePath = path23.join(indexPath, filepath.replace(/\.[^.]+$/, ".json"));
|
|
9835
10085
|
try {
|
|
9836
|
-
await
|
|
10086
|
+
await fs9.unlink(indexFilePath);
|
|
9837
10087
|
} catch {}
|
|
9838
10088
|
}
|
|
9839
10089
|
manifest.files = updatedFiles;
|
|
@@ -9844,16 +10094,16 @@ async function cleanupModuleIndex(rootDir, moduleId, config, logger) {
|
|
|
9844
10094
|
}
|
|
9845
10095
|
async function cleanupEmptyDirectories(dir) {
|
|
9846
10096
|
try {
|
|
9847
|
-
const entries = await
|
|
10097
|
+
const entries = await fs9.readdir(dir, { withFileTypes: true });
|
|
9848
10098
|
for (const entry of entries) {
|
|
9849
10099
|
if (entry.isDirectory()) {
|
|
9850
|
-
const subDir =
|
|
10100
|
+
const subDir = path23.join(dir, entry.name);
|
|
9851
10101
|
await cleanupEmptyDirectories(subDir);
|
|
9852
10102
|
}
|
|
9853
10103
|
}
|
|
9854
|
-
const remainingEntries = await
|
|
10104
|
+
const remainingEntries = await fs9.readdir(dir);
|
|
9855
10105
|
if (remainingEntries.length === 0) {
|
|
9856
|
-
await
|
|
10106
|
+
await fs9.rmdir(dir);
|
|
9857
10107
|
return true;
|
|
9858
10108
|
}
|
|
9859
10109
|
return false;
|
|
@@ -9862,7 +10112,7 @@ async function cleanupEmptyDirectories(dir) {
|
|
|
9862
10112
|
}
|
|
9863
10113
|
}
|
|
9864
10114
|
async function getIndexStatus(rootDir) {
|
|
9865
|
-
rootDir =
|
|
10115
|
+
rootDir = path23.resolve(rootDir);
|
|
9866
10116
|
const config = await loadConfig(rootDir);
|
|
9867
10117
|
const location = getIndexLocation(rootDir);
|
|
9868
10118
|
const indexDir = location.indexDir;
|
|
@@ -9874,13 +10124,13 @@ async function getIndexStatus(rootDir) {
|
|
|
9874
10124
|
totalFiles: 0
|
|
9875
10125
|
};
|
|
9876
10126
|
try {
|
|
9877
|
-
await
|
|
10127
|
+
await fs9.access(indexDir);
|
|
9878
10128
|
} catch {
|
|
9879
10129
|
return status;
|
|
9880
10130
|
}
|
|
9881
10131
|
try {
|
|
9882
10132
|
const globalManifestPath = getGlobalManifestPath(rootDir, config);
|
|
9883
|
-
const content = await
|
|
10133
|
+
const content = await fs9.readFile(globalManifestPath, "utf-8");
|
|
9884
10134
|
const globalManifest = JSON.parse(content);
|
|
9885
10135
|
status.exists = true;
|
|
9886
10136
|
status.lastUpdated = globalManifest.lastUpdated;
|
|
@@ -9898,7 +10148,7 @@ async function getIndexStatus(rootDir) {
|
|
|
9898
10148
|
}
|
|
9899
10149
|
} catch {
|
|
9900
10150
|
try {
|
|
9901
|
-
const entries = await
|
|
10151
|
+
const entries = await fs9.readdir(path23.join(indexDir, "index"));
|
|
9902
10152
|
if (entries.length > 0) {
|
|
9903
10153
|
status.exists = true;
|
|
9904
10154
|
for (const entry of entries) {
|
|
@@ -9920,8 +10170,8 @@ async function getIndexStatus(rootDir) {
|
|
|
9920
10170
|
}
|
|
9921
10171
|
|
|
9922
10172
|
// src/app/search/index.ts
|
|
9923
|
-
import * as
|
|
9924
|
-
import * as
|
|
10173
|
+
import * as fs11 from "fs/promises";
|
|
10174
|
+
import * as path26 from "path";
|
|
9925
10175
|
|
|
9926
10176
|
// node_modules/@isaacs/balanced-match/dist/esm/index.js
|
|
9927
10177
|
var balanced = (a, b, str) => {
|
|
@@ -10724,11 +10974,11 @@ var qmarksTestNoExtDot = ([$0]) => {
|
|
|
10724
10974
|
return (f) => f.length === len && f !== "." && f !== "..";
|
|
10725
10975
|
};
|
|
10726
10976
|
var defaultPlatform = typeof process === "object" && process ? typeof process.env === "object" && process.env && process.env.__MINIMATCH_TESTING_PLATFORM__ || process.platform : "posix";
|
|
10727
|
-
var
|
|
10977
|
+
var path24 = {
|
|
10728
10978
|
win32: { sep: "\\" },
|
|
10729
10979
|
posix: { sep: "/" }
|
|
10730
10980
|
};
|
|
10731
|
-
var sep2 = defaultPlatform === "win32" ?
|
|
10981
|
+
var sep2 = defaultPlatform === "win32" ? path24.win32.sep : path24.posix.sep;
|
|
10732
10982
|
minimatch.sep = sep2;
|
|
10733
10983
|
var GLOBSTAR = Symbol("globstar **");
|
|
10734
10984
|
minimatch.GLOBSTAR = GLOBSTAR;
|
|
@@ -11357,10 +11607,8 @@ minimatch.Minimatch = Minimatch;
|
|
|
11357
11607
|
minimatch.escape = escape;
|
|
11358
11608
|
minimatch.unescape = unescape;
|
|
11359
11609
|
|
|
11360
|
-
// src/types.ts
|
|
11361
|
-
init_entities();
|
|
11362
|
-
|
|
11363
11610
|
// src/app/search/index.ts
|
|
11611
|
+
init_types();
|
|
11364
11612
|
init_config2();
|
|
11365
11613
|
init_services();
|
|
11366
11614
|
// src/domain/usecases/exactSearch.ts
|
|
@@ -11394,7 +11642,7 @@ function matchesPathFilter(relativePath, filters, matchFn) {
|
|
|
11394
11642
|
}
|
|
11395
11643
|
return false;
|
|
11396
11644
|
}
|
|
11397
|
-
async function executeExactSearch(
|
|
11645
|
+
async function executeExactSearch(fs10, options, matchFn) {
|
|
11398
11646
|
const {
|
|
11399
11647
|
rootDir,
|
|
11400
11648
|
literal,
|
|
@@ -11406,13 +11654,13 @@ async function executeExactSearch(fs9, options, matchFn) {
|
|
|
11406
11654
|
const files = new Map;
|
|
11407
11655
|
async function walkDir(dir, baseDir) {
|
|
11408
11656
|
try {
|
|
11409
|
-
const entries = await
|
|
11657
|
+
const entries = await fs10.readDir(dir);
|
|
11410
11658
|
for (const entry of entries) {
|
|
11411
|
-
const fullPath =
|
|
11412
|
-
const relativePath =
|
|
11659
|
+
const fullPath = fs10.join(dir, entry);
|
|
11660
|
+
const relativePath = fs10.relative(baseDir, fullPath);
|
|
11413
11661
|
let isDirectory = false;
|
|
11414
11662
|
try {
|
|
11415
|
-
const stats = await
|
|
11663
|
+
const stats = await fs10.getStats(fullPath);
|
|
11416
11664
|
isDirectory = stats.isDirectory ?? false;
|
|
11417
11665
|
} catch {
|
|
11418
11666
|
continue;
|
|
@@ -11429,7 +11677,7 @@ async function executeExactSearch(fs9, options, matchFn) {
|
|
|
11429
11677
|
}
|
|
11430
11678
|
}
|
|
11431
11679
|
try {
|
|
11432
|
-
const content = await
|
|
11680
|
+
const content = await fs10.readFile(fullPath);
|
|
11433
11681
|
if (isSearchableContent(content, fullPath)) {
|
|
11434
11682
|
files.set(relativePath, content);
|
|
11435
11683
|
}
|
|
@@ -11446,21 +11694,21 @@ async function executeExactSearch(fs9, options, matchFn) {
|
|
|
11446
11694
|
});
|
|
11447
11695
|
}
|
|
11448
11696
|
// src/infrastructure/filesystem/nodeFileSystem.ts
|
|
11449
|
-
import * as
|
|
11450
|
-
import * as
|
|
11697
|
+
import * as fs10 from "fs/promises";
|
|
11698
|
+
import * as path25 from "path";
|
|
11451
11699
|
import { glob } from "glob";
|
|
11452
11700
|
|
|
11453
11701
|
class NodeFileSystem {
|
|
11454
11702
|
async readFile(filepath) {
|
|
11455
|
-
return
|
|
11703
|
+
return fs10.readFile(filepath, "utf-8");
|
|
11456
11704
|
}
|
|
11457
11705
|
async writeFile(filepath, content) {
|
|
11458
|
-
await
|
|
11459
|
-
await
|
|
11706
|
+
await fs10.mkdir(path25.dirname(filepath), { recursive: true });
|
|
11707
|
+
await fs10.writeFile(filepath, content, "utf-8");
|
|
11460
11708
|
}
|
|
11461
11709
|
async deleteFile(filepath) {
|
|
11462
11710
|
try {
|
|
11463
|
-
await
|
|
11711
|
+
await fs10.unlink(filepath);
|
|
11464
11712
|
} catch (error) {
|
|
11465
11713
|
if (error.code !== "ENOENT") {
|
|
11466
11714
|
throw error;
|
|
@@ -11468,7 +11716,7 @@ class NodeFileSystem {
|
|
|
11468
11716
|
}
|
|
11469
11717
|
}
|
|
11470
11718
|
async getStats(filepath) {
|
|
11471
|
-
const stats = await
|
|
11719
|
+
const stats = await fs10.stat(filepath);
|
|
11472
11720
|
return {
|
|
11473
11721
|
lastModified: stats.mtime.toISOString(),
|
|
11474
11722
|
size: stats.isDirectory() ? undefined : stats.size,
|
|
@@ -11477,17 +11725,17 @@ class NodeFileSystem {
|
|
|
11477
11725
|
}
|
|
11478
11726
|
async exists(filepath) {
|
|
11479
11727
|
try {
|
|
11480
|
-
await
|
|
11728
|
+
await fs10.access(filepath);
|
|
11481
11729
|
return true;
|
|
11482
11730
|
} catch {
|
|
11483
11731
|
return false;
|
|
11484
11732
|
}
|
|
11485
11733
|
}
|
|
11486
11734
|
async mkdir(dirpath) {
|
|
11487
|
-
await
|
|
11735
|
+
await fs10.mkdir(dirpath, { recursive: true });
|
|
11488
11736
|
}
|
|
11489
11737
|
async readDir(dirpath) {
|
|
11490
|
-
return
|
|
11738
|
+
return fs10.readdir(dirpath);
|
|
11491
11739
|
}
|
|
11492
11740
|
async findFiles(rootDir, patterns, ignore) {
|
|
11493
11741
|
const ignorePatterns = ignore.map((p) => `**/${p}/**`);
|
|
@@ -11503,19 +11751,19 @@ class NodeFileSystem {
|
|
|
11503
11751
|
return [...new Set(files)];
|
|
11504
11752
|
}
|
|
11505
11753
|
join(...segments) {
|
|
11506
|
-
return
|
|
11754
|
+
return path25.join(...segments);
|
|
11507
11755
|
}
|
|
11508
11756
|
relative(from, to) {
|
|
11509
|
-
return
|
|
11757
|
+
return path25.relative(from, to);
|
|
11510
11758
|
}
|
|
11511
11759
|
resolve(...segments) {
|
|
11512
|
-
return
|
|
11760
|
+
return path25.resolve(...segments);
|
|
11513
11761
|
}
|
|
11514
11762
|
dirname(filepath) {
|
|
11515
|
-
return
|
|
11763
|
+
return path25.dirname(filepath);
|
|
11516
11764
|
}
|
|
11517
11765
|
extname(filepath) {
|
|
11518
|
-
return
|
|
11766
|
+
return path25.extname(filepath);
|
|
11519
11767
|
}
|
|
11520
11768
|
}
|
|
11521
11769
|
var nodeFileSystem = new NodeFileSystem;
|
|
@@ -11525,7 +11773,7 @@ async function search(rootDir, query, options = {}) {
|
|
|
11525
11773
|
return hybridResults.results;
|
|
11526
11774
|
}
|
|
11527
11775
|
async function hybridSearch(rootDir, query, options = {}) {
|
|
11528
|
-
rootDir =
|
|
11776
|
+
rootDir = path26.resolve(rootDir);
|
|
11529
11777
|
const ensureFresh = options.ensureFresh ?? DEFAULT_SEARCH_OPTIONS.ensureFresh;
|
|
11530
11778
|
if (ensureFresh) {
|
|
11531
11779
|
await ensureIndexFresh(rootDir, { quiet: true });
|
|
@@ -11602,15 +11850,15 @@ async function hybridSearch(rootDir, query, options = {}) {
|
|
|
11602
11850
|
};
|
|
11603
11851
|
}
|
|
11604
11852
|
async function performExactSearch(rootDir, literal, config, options) {
|
|
11605
|
-
const
|
|
11606
|
-
return executeExactSearch(
|
|
11853
|
+
const fs12 = new NodeFileSystem;
|
|
11854
|
+
return executeExactSearch(fs12, {
|
|
11607
11855
|
rootDir,
|
|
11608
11856
|
literal,
|
|
11609
11857
|
pathFilter: options.pathFilter,
|
|
11610
11858
|
maxFiles: 20,
|
|
11611
11859
|
maxOccurrencesPerFile: 5,
|
|
11612
11860
|
caseInsensitive: false
|
|
11613
|
-
}, (
|
|
11861
|
+
}, (path27, pattern) => minimatch(path27, pattern, { matchBase: true }));
|
|
11614
11862
|
}
|
|
11615
11863
|
function createSearchContext(rootDir, moduleId, config) {
|
|
11616
11864
|
const indexPath = getModuleIndexPath(rootDir, moduleId, config);
|
|
@@ -11619,9 +11867,9 @@ function createSearchContext(rootDir, moduleId, config) {
|
|
|
11619
11867
|
config,
|
|
11620
11868
|
loadFileIndex: async (filepath) => {
|
|
11621
11869
|
const hasExtension = /\.[^./]+$/.test(filepath);
|
|
11622
|
-
const indexFilePath = hasExtension ?
|
|
11870
|
+
const indexFilePath = hasExtension ? path26.join(indexPath, filepath.replace(/\.[^.]+$/, ".json")) : path26.join(indexPath, filepath + ".json");
|
|
11623
11871
|
try {
|
|
11624
|
-
const content = await
|
|
11872
|
+
const content = await fs11.readFile(indexFilePath, "utf-8");
|
|
11625
11873
|
return JSON.parse(content);
|
|
11626
11874
|
} catch {
|
|
11627
11875
|
return null;
|
|
@@ -11631,7 +11879,7 @@ function createSearchContext(rootDir, moduleId, config) {
|
|
|
11631
11879
|
const files = [];
|
|
11632
11880
|
await traverseDirectory(indexPath, files, indexPath);
|
|
11633
11881
|
return files.filter((f) => f.endsWith(".json") && !f.endsWith("manifest.json")).map((f) => {
|
|
11634
|
-
const relative5 =
|
|
11882
|
+
const relative5 = path26.relative(indexPath, f);
|
|
11635
11883
|
return relative5.replace(/\.json$/, "");
|
|
11636
11884
|
});
|
|
11637
11885
|
}
|
|
@@ -11639,9 +11887,9 @@ function createSearchContext(rootDir, moduleId, config) {
|
|
|
11639
11887
|
}
|
|
11640
11888
|
async function traverseDirectory(dir, files, basePath) {
|
|
11641
11889
|
try {
|
|
11642
|
-
const entries = await
|
|
11890
|
+
const entries = await fs11.readdir(dir, { withFileTypes: true });
|
|
11643
11891
|
for (const entry of entries) {
|
|
11644
|
-
const fullPath =
|
|
11892
|
+
const fullPath = path26.join(dir, entry.name);
|
|
11645
11893
|
if (entry.isDirectory()) {
|
|
11646
11894
|
await traverseDirectory(fullPath, files, basePath);
|
|
11647
11895
|
} else if (entry.isFile()) {
|
|
@@ -11653,7 +11901,7 @@ async function traverseDirectory(dir, files, basePath) {
|
|
|
11653
11901
|
async function loadGlobalManifest2(rootDir, config) {
|
|
11654
11902
|
const manifestPath = getGlobalManifestPath(rootDir, config);
|
|
11655
11903
|
try {
|
|
11656
|
-
const content = await
|
|
11904
|
+
const content = await fs11.readFile(manifestPath, "utf-8");
|
|
11657
11905
|
return JSON.parse(content);
|
|
11658
11906
|
} catch {
|
|
11659
11907
|
return null;
|
|
@@ -11822,4 +12070,4 @@ export {
|
|
|
11822
12070
|
ConsoleLogger
|
|
11823
12071
|
};
|
|
11824
12072
|
|
|
11825
|
-
//# debugId=
|
|
12073
|
+
//# debugId=5C9139B3A95BD0AB64756E2164756E21
|