src-mcp 1.0.1 → 1.0.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/bin.d.mts +1 -0
- package/dist/bin.mjs +205 -0
- package/dist/bin.mjs.map +1 -0
- package/dist/index.d.mts +1 -0
- package/dist/index.mjs +8 -0
- package/dist/index.mjs.map +1 -0
- package/dist/server-B2Ms4jQx.mjs +3889 -0
- package/dist/server-B2Ms4jQx.mjs.map +1 -0
- package/package.json +10 -9
- package/dist/bin.d.ts +0 -3
- package/dist/bin.d.ts.map +0 -1
- package/dist/bin.js +0 -4
- package/dist/bin.js.map +0 -1
- package/dist/cli/adapter.d.ts +0 -7
- package/dist/cli/adapter.d.ts.map +0 -1
- package/dist/cli/adapter.js +0 -39
- package/dist/cli/adapter.js.map +0 -1
- package/dist/cli/commands/index.d.ts +0 -24
- package/dist/cli/commands/index.d.ts.map +0 -1
- package/dist/cli/commands/index.js +0 -13
- package/dist/cli/commands/index.js.map +0 -1
- package/dist/cli/commands/serve.command.d.ts +0 -21
- package/dist/cli/commands/serve.command.d.ts.map +0 -1
- package/dist/cli/commands/serve.command.js +0 -62
- package/dist/cli/commands/serve.command.js.map +0 -1
- package/dist/cli/commands/version.command.d.ts +0 -2
- package/dist/cli/commands/version.command.d.ts.map +0 -1
- package/dist/cli/commands/version.command.js +0 -12
- package/dist/cli/commands/version.command.js.map +0 -1
- package/dist/cli/index.d.ts +0 -2
- package/dist/cli/index.d.ts.map +0 -1
- package/dist/cli/index.js +0 -15
- package/dist/cli/index.js.map +0 -1
- package/dist/cli/parser.d.ts +0 -7
- package/dist/cli/parser.d.ts.map +0 -1
- package/dist/cli/parser.js +0 -99
- package/dist/cli/parser.js.map +0 -1
- package/dist/config/index.d.ts +0 -24
- package/dist/config/index.d.ts.map +0 -1
- package/dist/config/index.js +0 -38
- package/dist/config/index.js.map +0 -1
- package/dist/core/ast/index.d.ts +0 -82
- package/dist/core/ast/index.d.ts.map +0 -1
- package/dist/core/ast/index.js +0 -204
- package/dist/core/ast/index.js.map +0 -1
- package/dist/core/ast/types.d.ts +0 -152
- package/dist/core/ast/types.d.ts.map +0 -1
- package/dist/core/ast/types.js +0 -5
- package/dist/core/ast/types.js.map +0 -1
- package/dist/core/constants.d.ts +0 -17
- package/dist/core/constants.d.ts.map +0 -1
- package/dist/core/constants.js +0 -49
- package/dist/core/constants.js.map +0 -1
- package/dist/core/embeddings/callgraph.d.ts +0 -98
- package/dist/core/embeddings/callgraph.d.ts.map +0 -1
- package/dist/core/embeddings/callgraph.js +0 -415
- package/dist/core/embeddings/callgraph.js.map +0 -1
- package/dist/core/embeddings/chunker.d.ts +0 -37
- package/dist/core/embeddings/chunker.d.ts.map +0 -1
- package/dist/core/embeddings/chunker.js +0 -298
- package/dist/core/embeddings/chunker.js.map +0 -1
- package/dist/core/embeddings/client.d.ts +0 -30
- package/dist/core/embeddings/client.d.ts.map +0 -1
- package/dist/core/embeddings/client.js +0 -65
- package/dist/core/embeddings/client.js.map +0 -1
- package/dist/core/embeddings/crossfile.d.ts +0 -58
- package/dist/core/embeddings/crossfile.d.ts.map +0 -1
- package/dist/core/embeddings/crossfile.js +0 -202
- package/dist/core/embeddings/crossfile.js.map +0 -1
- package/dist/core/embeddings/enricher.d.ts +0 -53
- package/dist/core/embeddings/enricher.d.ts.map +0 -1
- package/dist/core/embeddings/enricher.js +0 -308
- package/dist/core/embeddings/enricher.js.map +0 -1
- package/dist/core/embeddings/index.d.ts +0 -13
- package/dist/core/embeddings/index.d.ts.map +0 -1
- package/dist/core/embeddings/index.js +0 -20
- package/dist/core/embeddings/index.js.map +0 -1
- package/dist/core/embeddings/reranker.d.ts +0 -41
- package/dist/core/embeddings/reranker.d.ts.map +0 -1
- package/dist/core/embeddings/reranker.js +0 -117
- package/dist/core/embeddings/reranker.js.map +0 -1
- package/dist/core/embeddings/store.d.ts +0 -93
- package/dist/core/embeddings/store.d.ts.map +0 -1
- package/dist/core/embeddings/store.js +0 -304
- package/dist/core/embeddings/store.js.map +0 -1
- package/dist/core/embeddings/types.d.ts +0 -77
- package/dist/core/embeddings/types.d.ts.map +0 -1
- package/dist/core/embeddings/types.js +0 -5
- package/dist/core/embeddings/types.js.map +0 -1
- package/dist/core/embeddings/watcher.d.ts +0 -130
- package/dist/core/embeddings/watcher.d.ts.map +0 -1
- package/dist/core/embeddings/watcher.js +0 -448
- package/dist/core/embeddings/watcher.js.map +0 -1
- package/dist/core/fallback/index.d.ts +0 -26
- package/dist/core/fallback/index.d.ts.map +0 -1
- package/dist/core/fallback/index.js +0 -76
- package/dist/core/fallback/index.js.map +0 -1
- package/dist/core/parser/index.d.ts +0 -64
- package/dist/core/parser/index.d.ts.map +0 -1
- package/dist/core/parser/index.js +0 -205
- package/dist/core/parser/index.js.map +0 -1
- package/dist/core/parser/languages.d.ts +0 -26
- package/dist/core/parser/languages.d.ts.map +0 -1
- package/dist/core/parser/languages.js +0 -101
- package/dist/core/parser/languages.js.map +0 -1
- package/dist/core/queries/helpers.d.ts +0 -72
- package/dist/core/queries/helpers.d.ts.map +0 -1
- package/dist/core/queries/helpers.js +0 -101
- package/dist/core/queries/helpers.js.map +0 -1
- package/dist/core/queries/index.d.ts +0 -144
- package/dist/core/queries/index.d.ts.map +0 -1
- package/dist/core/queries/index.js +0 -396
- package/dist/core/queries/index.js.map +0 -1
- package/dist/core/queries/loader.d.ts +0 -46
- package/dist/core/queries/loader.d.ts.map +0 -1
- package/dist/core/queries/loader.js +0 -216
- package/dist/core/queries/loader.js.map +0 -1
- package/dist/core/queries/patterns.d.ts +0 -10
- package/dist/core/queries/patterns.d.ts.map +0 -1
- package/dist/core/queries/patterns.js +0 -112
- package/dist/core/queries/patterns.js.map +0 -1
- package/dist/core/symbols/index.d.ts +0 -70
- package/dist/core/symbols/index.d.ts.map +0 -1
- package/dist/core/symbols/index.js +0 -359
- package/dist/core/symbols/index.js.map +0 -1
- package/dist/core/unified/index.d.ts +0 -118
- package/dist/core/unified/index.d.ts.map +0 -1
- package/dist/core/unified/index.js +0 -428
- package/dist/core/unified/index.js.map +0 -1
- package/dist/core/utils/assets.d.ts +0 -34
- package/dist/core/utils/assets.d.ts.map +0 -1
- package/dist/core/utils/assets.js +0 -85
- package/dist/core/utils/assets.js.map +0 -1
- package/dist/core/utils/cache.d.ts +0 -43
- package/dist/core/utils/cache.d.ts.map +0 -1
- package/dist/core/utils/cache.js +0 -60
- package/dist/core/utils/cache.js.map +0 -1
- package/dist/core/utils/index.d.ts +0 -7
- package/dist/core/utils/index.d.ts.map +0 -1
- package/dist/core/utils/index.js +0 -10
- package/dist/core/utils/index.js.map +0 -1
- package/dist/core/utils/tsconfig.d.ts +0 -34
- package/dist/core/utils/tsconfig.d.ts.map +0 -1
- package/dist/core/utils/tsconfig.js +0 -173
- package/dist/core/utils/tsconfig.js.map +0 -1
- package/dist/features/analyze-file/index.d.ts +0 -15
- package/dist/features/analyze-file/index.d.ts.map +0 -1
- package/dist/features/analyze-file/index.js +0 -164
- package/dist/features/analyze-file/index.js.map +0 -1
- package/dist/features/get-call-graph/index.d.ts +0 -24
- package/dist/features/get-call-graph/index.d.ts.map +0 -1
- package/dist/features/get-call-graph/index.js +0 -246
- package/dist/features/get-call-graph/index.js.map +0 -1
- package/dist/features/get-index-status/index.d.ts +0 -20
- package/dist/features/get-index-status/index.d.ts.map +0 -1
- package/dist/features/get-index-status/index.js +0 -90
- package/dist/features/get-index-status/index.js.map +0 -1
- package/dist/features/index-codebase/index.d.ts +0 -24
- package/dist/features/index-codebase/index.d.ts.map +0 -1
- package/dist/features/index-codebase/index.js +0 -283
- package/dist/features/index-codebase/index.js.map +0 -1
- package/dist/features/index.d.ts +0 -15
- package/dist/features/index.d.ts.map +0 -1
- package/dist/features/index.js +0 -28
- package/dist/features/index.js.map +0 -1
- package/dist/features/info/index.d.ts +0 -19
- package/dist/features/info/index.d.ts.map +0 -1
- package/dist/features/info/index.js +0 -41
- package/dist/features/info/index.js.map +0 -1
- package/dist/features/list-symbols/index.d.ts +0 -22
- package/dist/features/list-symbols/index.d.ts.map +0 -1
- package/dist/features/list-symbols/index.js +0 -74
- package/dist/features/list-symbols/index.js.map +0 -1
- package/dist/features/parse-ast/index.d.ts +0 -12
- package/dist/features/parse-ast/index.d.ts.map +0 -1
- package/dist/features/parse-ast/index.js +0 -71
- package/dist/features/parse-ast/index.js.map +0 -1
- package/dist/features/query-code/index.d.ts +0 -23
- package/dist/features/query-code/index.d.ts.map +0 -1
- package/dist/features/query-code/index.js +0 -96
- package/dist/features/query-code/index.js.map +0 -1
- package/dist/features/search-code/index.d.ts +0 -39
- package/dist/features/search-code/index.d.ts.map +0 -1
- package/dist/features/search-code/index.js +0 -258
- package/dist/features/search-code/index.js.map +0 -1
- package/dist/features/types.d.ts +0 -14
- package/dist/features/types.d.ts.map +0 -1
- package/dist/features/types.js +0 -2
- package/dist/features/types.js.map +0 -1
- package/dist/features/update-index/index.d.ts +0 -24
- package/dist/features/update-index/index.d.ts.map +0 -1
- package/dist/features/update-index/index.js +0 -358
- package/dist/features/update-index/index.js.map +0 -1
- package/dist/features/utils/content.d.ts +0 -30
- package/dist/features/utils/content.d.ts.map +0 -1
- package/dist/features/utils/content.js +0 -49
- package/dist/features/utils/content.js.map +0 -1
- package/dist/features/utils/index.d.ts +0 -6
- package/dist/features/utils/index.d.ts.map +0 -1
- package/dist/features/utils/index.js +0 -8
- package/dist/features/utils/index.js.map +0 -1
- package/dist/features/utils/result.d.ts +0 -37
- package/dist/features/utils/result.d.ts.map +0 -1
- package/dist/features/utils/result.js +0 -53
- package/dist/features/utils/result.js.map +0 -1
- package/dist/index.d.ts +0 -2
- package/dist/index.d.ts.map +0 -1
- package/dist/index.js +0 -4
- package/dist/index.js.map +0 -1
- package/dist/prompts/index.d.ts +0 -9
- package/dist/prompts/index.d.ts.map +0 -1
- package/dist/prompts/index.js +0 -188
- package/dist/prompts/index.js.map +0 -1
- package/dist/resources/index.d.ts +0 -3
- package/dist/resources/index.d.ts.map +0 -1
- package/dist/resources/index.js +0 -17
- package/dist/resources/index.js.map +0 -1
- package/dist/server.d.ts +0 -4
- package/dist/server.d.ts.map +0 -1
- package/dist/server.js +0 -24
- package/dist/server.js.map +0 -1
- package/dist/tools/adapter.d.ts +0 -4
- package/dist/tools/adapter.d.ts.map +0 -1
- package/dist/tools/adapter.js +0 -28
- package/dist/tools/adapter.js.map +0 -1
- package/dist/tools/index.d.ts +0 -5
- package/dist/tools/index.d.ts.map +0 -1
- package/dist/tools/index.js +0 -9
- package/dist/tools/index.js.map +0 -1
- package/dist/types/index.d.ts +0 -20
- package/dist/types/index.d.ts.map +0 -1
- package/dist/types/index.js +0 -2
- package/dist/types/index.js.map +0 -1
- package/dist/utils/colors.d.ts +0 -24
- package/dist/utils/colors.d.ts.map +0 -1
- package/dist/utils/colors.js +0 -30
- package/dist/utils/colors.js.map +0 -1
- package/dist/utils/index.d.ts +0 -4
- package/dist/utils/index.d.ts.map +0 -1
- package/dist/utils/index.js +0 -4
- package/dist/utils/index.js.map +0 -1
- package/dist/utils/logger.d.ts +0 -8
- package/dist/utils/logger.d.ts.map +0 -1
- package/dist/utils/logger.js +0 -57
- package/dist/utils/logger.js.map +0 -1
- package/dist/utils/spinner.d.ts +0 -11
- package/dist/utils/spinner.d.ts.map +0 -1
- package/dist/utils/spinner.js +0 -36
- package/dist/utils/spinner.js.map +0 -1
|
@@ -1,448 +0,0 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* File watcher for automatic index updates
|
|
3
|
-
*
|
|
4
|
-
* Features:
|
|
5
|
-
* - SHA-256 hash comparison to detect real content changes
|
|
6
|
-
* - Debounce (5s default) to handle rapid changes
|
|
7
|
-
* - Persistent hash cache to avoid unnecessary re-indexing
|
|
8
|
-
* - fast-glob for efficient file scanning
|
|
9
|
-
*/
|
|
10
|
-
import * as fs from "node:fs";
|
|
11
|
-
import * as path from "node:path";
|
|
12
|
-
import * as crypto from "node:crypto";
|
|
13
|
-
import { watch } from "chokidar";
|
|
14
|
-
import fg from "fast-glob";
|
|
15
|
-
import ignore from "ignore";
|
|
16
|
-
import { OllamaClient } from "@core/embeddings/client";
|
|
17
|
-
import { VectorStore } from "@core/embeddings/store";
|
|
18
|
-
import { chunkFile, shouldIndexFile, SUPPORTED_EXTENSIONS, } from "@core/embeddings/chunker";
|
|
19
|
-
import { enrichChunksFromFile } from "@core/embeddings/enricher";
|
|
20
|
-
import { logger } from "@utils";
|
|
21
|
-
/** Default debounce delay in milliseconds */
|
|
22
|
-
const DEFAULT_DEBOUNCE_MS = 5000;
|
|
23
|
-
/** Cache file name for storing hashes */
|
|
24
|
-
const HASH_CACHE_FILE = ".src-index-hashes.json";
|
|
25
|
-
export class IndexWatcher {
|
|
26
|
-
directory;
|
|
27
|
-
config;
|
|
28
|
-
debounceMs;
|
|
29
|
-
ollamaClient;
|
|
30
|
-
vectorStore;
|
|
31
|
-
watcher = null;
|
|
32
|
-
ig;
|
|
33
|
-
isProcessing = false;
|
|
34
|
-
hashCache = {};
|
|
35
|
-
pendingChanges = new Map();
|
|
36
|
-
operationQueue = [];
|
|
37
|
-
onReady;
|
|
38
|
-
onError;
|
|
39
|
-
onIndexed;
|
|
40
|
-
onRemoved;
|
|
41
|
-
constructor(options) {
|
|
42
|
-
this.directory = path.resolve(options.directory);
|
|
43
|
-
this.config = options.config;
|
|
44
|
-
this.debounceMs = options.debounceMs ?? DEFAULT_DEBOUNCE_MS;
|
|
45
|
-
this.ollamaClient = new OllamaClient(options.config);
|
|
46
|
-
this.vectorStore = new VectorStore(this.directory, options.config);
|
|
47
|
-
this.ig = this.createIgnoreFilter();
|
|
48
|
-
this.onReady = options.onReady;
|
|
49
|
-
this.onError = options.onError;
|
|
50
|
-
this.onIndexed = options.onIndexed;
|
|
51
|
-
this.onRemoved = options.onRemoved;
|
|
52
|
-
this.loadHashCache();
|
|
53
|
-
}
|
|
54
|
-
/**
|
|
55
|
-
* Compute SHA-256 hash of content
|
|
56
|
-
*/
|
|
57
|
-
computeHash(content) {
|
|
58
|
-
return crypto.createHash("sha256").update(content, "utf8").digest("hex");
|
|
59
|
-
}
|
|
60
|
-
/**
|
|
61
|
-
* Get hash cache file path
|
|
62
|
-
*/
|
|
63
|
-
getHashCachePath() {
|
|
64
|
-
return path.join(this.directory, ".src-index", HASH_CACHE_FILE);
|
|
65
|
-
}
|
|
66
|
-
/**
|
|
67
|
-
* Load hash cache from disk
|
|
68
|
-
*/
|
|
69
|
-
loadHashCache() {
|
|
70
|
-
const cachePath = this.getHashCachePath();
|
|
71
|
-
if (fs.existsSync(cachePath)) {
|
|
72
|
-
try {
|
|
73
|
-
const content = fs.readFileSync(cachePath, "utf-8");
|
|
74
|
-
this.hashCache = JSON.parse(content);
|
|
75
|
-
logger.debug(`Loaded ${String(Object.keys(this.hashCache).length)} cached hashes`);
|
|
76
|
-
}
|
|
77
|
-
catch {
|
|
78
|
-
this.hashCache = {};
|
|
79
|
-
}
|
|
80
|
-
}
|
|
81
|
-
}
|
|
82
|
-
/**
|
|
83
|
-
* Save hash cache to disk
|
|
84
|
-
*/
|
|
85
|
-
saveHashCache() {
|
|
86
|
-
const cachePath = this.getHashCachePath();
|
|
87
|
-
const cacheDir = path.dirname(cachePath);
|
|
88
|
-
try {
|
|
89
|
-
if (!fs.existsSync(cacheDir)) {
|
|
90
|
-
fs.mkdirSync(cacheDir, { recursive: true });
|
|
91
|
-
}
|
|
92
|
-
fs.writeFileSync(cachePath, JSON.stringify(this.hashCache, null, 2));
|
|
93
|
-
}
|
|
94
|
-
catch (err) {
|
|
95
|
-
const error = err instanceof Error ? err : new Error(String(err));
|
|
96
|
-
logger.debug(`Failed to save hash cache: ${error.message}`);
|
|
97
|
-
}
|
|
98
|
-
}
|
|
99
|
-
/**
|
|
100
|
-
* Check if file content has changed by comparing hashes
|
|
101
|
-
*/
|
|
102
|
-
hasContentChanged(filePath, content) {
|
|
103
|
-
const newHash = this.computeHash(content);
|
|
104
|
-
const oldHash = this.hashCache[filePath];
|
|
105
|
-
if (oldHash === newHash) {
|
|
106
|
-
return false;
|
|
107
|
-
}
|
|
108
|
-
this.hashCache[filePath] = newHash;
|
|
109
|
-
return true;
|
|
110
|
-
}
|
|
111
|
-
/**
|
|
112
|
-
* Remove file from hash cache
|
|
113
|
-
*/
|
|
114
|
-
removeFromHashCache(filePath) {
|
|
115
|
-
const { [filePath]: _, ...rest } = this.hashCache;
|
|
116
|
-
this.hashCache = rest;
|
|
117
|
-
}
|
|
118
|
-
/**
|
|
119
|
-
* Create ignore filter from .gitignore
|
|
120
|
-
*/
|
|
121
|
-
createIgnoreFilter() {
|
|
122
|
-
const ig = ignore();
|
|
123
|
-
const gitignorePath = path.join(this.directory, ".gitignore");
|
|
124
|
-
if (fs.existsSync(gitignorePath)) {
|
|
125
|
-
try {
|
|
126
|
-
const content = fs.readFileSync(gitignorePath, "utf-8");
|
|
127
|
-
ig.add(content);
|
|
128
|
-
}
|
|
129
|
-
catch {
|
|
130
|
-
// Ignore read errors
|
|
131
|
-
}
|
|
132
|
-
}
|
|
133
|
-
return ig;
|
|
134
|
-
}
|
|
135
|
-
/**
|
|
136
|
-
* Check if a file should be indexed
|
|
137
|
-
*/
|
|
138
|
-
shouldIndex(filePath) {
|
|
139
|
-
const relativePath = path
|
|
140
|
-
.relative(this.directory, filePath)
|
|
141
|
-
.replace(/\\/g, "/");
|
|
142
|
-
// Skip hidden files/folders
|
|
143
|
-
if (relativePath.split("/").some((part) => part.startsWith("."))) {
|
|
144
|
-
return false;
|
|
145
|
-
}
|
|
146
|
-
// Skip gitignore patterns
|
|
147
|
-
if (this.ig.ignores(relativePath)) {
|
|
148
|
-
return false;
|
|
149
|
-
}
|
|
150
|
-
return shouldIndexFile(filePath);
|
|
151
|
-
}
|
|
152
|
-
/**
|
|
153
|
-
* Schedule a file change with debouncing
|
|
154
|
-
*/
|
|
155
|
-
scheduleChange(type, filePath) {
|
|
156
|
-
const existing = this.pendingChanges.get(filePath);
|
|
157
|
-
if (existing) {
|
|
158
|
-
clearTimeout(existing.timer);
|
|
159
|
-
}
|
|
160
|
-
const timer = setTimeout(() => {
|
|
161
|
-
this.pendingChanges.delete(filePath);
|
|
162
|
-
this.queueOperation(async () => this.processChange(type, filePath));
|
|
163
|
-
}, this.debounceMs);
|
|
164
|
-
this.pendingChanges.set(filePath, { type, filePath, timer });
|
|
165
|
-
logger.debug(`Scheduled ${type}: ${path.basename(filePath)} (${String(this.debounceMs)}ms)`);
|
|
166
|
-
}
|
|
167
|
-
/**
|
|
168
|
-
* Process a file change after debounce
|
|
169
|
-
*/
|
|
170
|
-
async processChange(type, filePath) {
|
|
171
|
-
if (type === "unlink") {
|
|
172
|
-
await this.removeFile(filePath);
|
|
173
|
-
}
|
|
174
|
-
else {
|
|
175
|
-
await this.indexFile(filePath);
|
|
176
|
-
}
|
|
177
|
-
}
|
|
178
|
-
/**
|
|
179
|
-
* Index a single file
|
|
180
|
-
*/
|
|
181
|
-
async indexFile(filePath) {
|
|
182
|
-
if (!this.shouldIndex(filePath)) {
|
|
183
|
-
return;
|
|
184
|
-
}
|
|
185
|
-
try {
|
|
186
|
-
const content = fs.readFileSync(filePath, "utf-8");
|
|
187
|
-
// Skip if content unchanged
|
|
188
|
-
if (!this.hasContentChanged(filePath, content)) {
|
|
189
|
-
logger.debug(`Skipped (unchanged): ${path.basename(filePath)}`);
|
|
190
|
-
return;
|
|
191
|
-
}
|
|
192
|
-
const chunks = await chunkFile(filePath, content, this.config);
|
|
193
|
-
if (chunks.length === 0) {
|
|
194
|
-
return;
|
|
195
|
-
}
|
|
196
|
-
// Enrich chunks with semantic metadata
|
|
197
|
-
const enrichedChunks = await enrichChunksFromFile(chunks, content);
|
|
198
|
-
// Use enrichedContent for embedding
|
|
199
|
-
const texts = enrichedChunks.map((c) => c.enrichedContent);
|
|
200
|
-
const embeddings = await this.ollamaClient.embedBatch(texts);
|
|
201
|
-
// Store original chunk data (without enrichedContent)
|
|
202
|
-
const embeddedChunks = enrichedChunks.map((chunk, i) => ({
|
|
203
|
-
id: chunk.id,
|
|
204
|
-
content: chunk.content,
|
|
205
|
-
filePath: chunk.filePath,
|
|
206
|
-
language: chunk.language,
|
|
207
|
-
startLine: chunk.startLine,
|
|
208
|
-
endLine: chunk.endLine,
|
|
209
|
-
symbolName: chunk.symbolName,
|
|
210
|
-
symbolType: chunk.symbolType,
|
|
211
|
-
vector: embeddings[i] ?? [],
|
|
212
|
-
}));
|
|
213
|
-
await this.vectorStore.deleteByFilePath(filePath);
|
|
214
|
-
await this.vectorStore.addChunks(embeddedChunks);
|
|
215
|
-
this.saveHashCache();
|
|
216
|
-
logger.debug(`Indexed: ${path.relative(this.directory, filePath)}`);
|
|
217
|
-
this.onIndexed?.(filePath);
|
|
218
|
-
}
|
|
219
|
-
catch (err) {
|
|
220
|
-
const error = err instanceof Error ? err : new Error(String(err));
|
|
221
|
-
logger.error(`Failed to index ${filePath}: ${error.message}`);
|
|
222
|
-
this.onError?.(error);
|
|
223
|
-
}
|
|
224
|
-
}
|
|
225
|
-
/**
|
|
226
|
-
* Remove a file from the index
|
|
227
|
-
*/
|
|
228
|
-
async removeFile(filePath) {
|
|
229
|
-
try {
|
|
230
|
-
await this.vectorStore.deleteByFilePath(filePath);
|
|
231
|
-
this.removeFromHashCache(filePath);
|
|
232
|
-
this.saveHashCache();
|
|
233
|
-
logger.debug(`Removed: ${path.relative(this.directory, filePath)}`);
|
|
234
|
-
this.onRemoved?.(filePath);
|
|
235
|
-
}
|
|
236
|
-
catch (err) {
|
|
237
|
-
const error = err instanceof Error ? err : new Error(String(err));
|
|
238
|
-
logger.error(`Failed to remove ${filePath}: ${error.message}`);
|
|
239
|
-
this.onError?.(error);
|
|
240
|
-
}
|
|
241
|
-
}
|
|
242
|
-
/**
|
|
243
|
-
* Queue an operation to prevent concurrent modifications
|
|
244
|
-
*/
|
|
245
|
-
queueOperation(operation) {
|
|
246
|
-
this.operationQueue.push(operation);
|
|
247
|
-
void this.processQueue();
|
|
248
|
-
}
|
|
249
|
-
/**
|
|
250
|
-
* Process queued operations sequentially
|
|
251
|
-
*/
|
|
252
|
-
async processQueue() {
|
|
253
|
-
if (this.isProcessing) {
|
|
254
|
-
return;
|
|
255
|
-
}
|
|
256
|
-
this.isProcessing = true;
|
|
257
|
-
while (this.operationQueue.length > 0) {
|
|
258
|
-
const operation = this.operationQueue.shift();
|
|
259
|
-
if (operation) {
|
|
260
|
-
try {
|
|
261
|
-
await operation();
|
|
262
|
-
}
|
|
263
|
-
catch (err) {
|
|
264
|
-
const error = err instanceof Error ? err : new Error(String(err));
|
|
265
|
-
logger.error(`Operation failed: ${error.message}`);
|
|
266
|
-
}
|
|
267
|
-
}
|
|
268
|
-
}
|
|
269
|
-
this.isProcessing = false;
|
|
270
|
-
}
|
|
271
|
-
/**
|
|
272
|
-
* Collect files using fast-glob
|
|
273
|
-
*/
|
|
274
|
-
async collectFilesWithGlob() {
|
|
275
|
-
const extensions = SUPPORTED_EXTENSIONS.map((ext) => ext.slice(1));
|
|
276
|
-
const pattern = `**/*.{${extensions.join(",")}}`;
|
|
277
|
-
const files = await fg(pattern, {
|
|
278
|
-
cwd: this.directory,
|
|
279
|
-
absolute: true,
|
|
280
|
-
ignore: ["**/.*", "**/.*/**"],
|
|
281
|
-
dot: false,
|
|
282
|
-
onlyFiles: true,
|
|
283
|
-
followSymbolicLinks: false,
|
|
284
|
-
});
|
|
285
|
-
// Filter by gitignore
|
|
286
|
-
return files.filter((file) => {
|
|
287
|
-
const relativePath = path
|
|
288
|
-
.relative(this.directory, file)
|
|
289
|
-
.replace(/\\/g, "/");
|
|
290
|
-
return !this.ig.ignores(relativePath);
|
|
291
|
-
});
|
|
292
|
-
}
|
|
293
|
-
/**
|
|
294
|
-
* Perform full initial indexing
|
|
295
|
-
*/
|
|
296
|
-
async fullIndex() {
|
|
297
|
-
logger.info("Starting full index...");
|
|
298
|
-
const files = await this.collectFilesWithGlob();
|
|
299
|
-
let indexed = 0;
|
|
300
|
-
let skipped = 0;
|
|
301
|
-
for (const filePath of files) {
|
|
302
|
-
try {
|
|
303
|
-
const content = fs.readFileSync(filePath, "utf-8");
|
|
304
|
-
if (!this.hasContentChanged(filePath, content)) {
|
|
305
|
-
skipped++;
|
|
306
|
-
continue;
|
|
307
|
-
}
|
|
308
|
-
const chunks = await chunkFile(filePath, content, this.config);
|
|
309
|
-
if (chunks.length === 0) {
|
|
310
|
-
continue;
|
|
311
|
-
}
|
|
312
|
-
// Enrich chunks with semantic metadata
|
|
313
|
-
const enrichedChunks = await enrichChunksFromFile(chunks, content);
|
|
314
|
-
// Use enrichedContent for embedding
|
|
315
|
-
const texts = enrichedChunks.map((c) => c.enrichedContent);
|
|
316
|
-
const embeddings = await this.ollamaClient.embedBatch(texts);
|
|
317
|
-
// Store original chunk data (without enrichedContent)
|
|
318
|
-
const embeddedChunks = enrichedChunks.map((chunk, i) => ({
|
|
319
|
-
id: chunk.id,
|
|
320
|
-
content: chunk.content,
|
|
321
|
-
filePath: chunk.filePath,
|
|
322
|
-
language: chunk.language,
|
|
323
|
-
startLine: chunk.startLine,
|
|
324
|
-
endLine: chunk.endLine,
|
|
325
|
-
symbolName: chunk.symbolName,
|
|
326
|
-
symbolType: chunk.symbolType,
|
|
327
|
-
vector: embeddings[i] ?? [],
|
|
328
|
-
}));
|
|
329
|
-
await this.vectorStore.addChunks(embeddedChunks);
|
|
330
|
-
indexed++;
|
|
331
|
-
}
|
|
332
|
-
catch (err) {
|
|
333
|
-
const error = err instanceof Error ? err : new Error(String(err));
|
|
334
|
-
logger.debug(`Error indexing ${filePath}: ${error.message}`);
|
|
335
|
-
}
|
|
336
|
-
}
|
|
337
|
-
this.saveHashCache();
|
|
338
|
-
logger.info(`Full index: ${String(indexed)} indexed, ${String(skipped)} skipped`);
|
|
339
|
-
}
|
|
340
|
-
/**
|
|
341
|
-
* Start watching for file changes
|
|
342
|
-
*/
|
|
343
|
-
async start() {
|
|
344
|
-
const health = await this.ollamaClient.healthCheck();
|
|
345
|
-
if (!health.ok) {
|
|
346
|
-
throw new Error(health.error ?? "Ollama is not available");
|
|
347
|
-
}
|
|
348
|
-
await this.vectorStore.connect();
|
|
349
|
-
if (!this.vectorStore.exists()) {
|
|
350
|
-
await this.fullIndex();
|
|
351
|
-
}
|
|
352
|
-
this.watcher = watch(this.directory, {
|
|
353
|
-
ignored: (filePath) => {
|
|
354
|
-
const relativePath = path
|
|
355
|
-
.relative(this.directory, filePath)
|
|
356
|
-
.replace(/\\/g, "/");
|
|
357
|
-
// Skip empty paths or root directory
|
|
358
|
-
if (!relativePath) {
|
|
359
|
-
return false;
|
|
360
|
-
}
|
|
361
|
-
// Skip hidden files/folders
|
|
362
|
-
if (relativePath.split("/").some((part) => part.startsWith("."))) {
|
|
363
|
-
return true;
|
|
364
|
-
}
|
|
365
|
-
return this.ig.ignores(relativePath);
|
|
366
|
-
},
|
|
367
|
-
persistent: true,
|
|
368
|
-
ignoreInitial: true,
|
|
369
|
-
awaitWriteFinish: {
|
|
370
|
-
stabilityThreshold: 500,
|
|
371
|
-
pollInterval: 100,
|
|
372
|
-
},
|
|
373
|
-
});
|
|
374
|
-
this.watcher.on("add", (filePath) => {
|
|
375
|
-
if (shouldIndexFile(filePath)) {
|
|
376
|
-
this.scheduleChange("add", filePath);
|
|
377
|
-
}
|
|
378
|
-
});
|
|
379
|
-
this.watcher.on("change", (filePath) => {
|
|
380
|
-
if (shouldIndexFile(filePath)) {
|
|
381
|
-
this.scheduleChange("change", filePath);
|
|
382
|
-
}
|
|
383
|
-
});
|
|
384
|
-
this.watcher.on("unlink", (filePath) => {
|
|
385
|
-
if (shouldIndexFile(filePath)) {
|
|
386
|
-
this.scheduleChange("unlink", filePath);
|
|
387
|
-
}
|
|
388
|
-
});
|
|
389
|
-
this.watcher.on("ready", () => {
|
|
390
|
-
logger.info(`Watching: ${this.directory} (${String(this.debounceMs)}ms debounce)`);
|
|
391
|
-
this.onReady?.();
|
|
392
|
-
});
|
|
393
|
-
this.watcher.on("error", (err) => {
|
|
394
|
-
const error = err instanceof Error ? err : new Error(String(err));
|
|
395
|
-
logger.error(`Watcher error: ${error.message}`);
|
|
396
|
-
this.onError?.(error);
|
|
397
|
-
});
|
|
398
|
-
}
|
|
399
|
-
/**
|
|
400
|
-
* Stop watching and cleanup
|
|
401
|
-
*/
|
|
402
|
-
async stop() {
|
|
403
|
-
for (const pending of this.pendingChanges.values()) {
|
|
404
|
-
clearTimeout(pending.timer);
|
|
405
|
-
}
|
|
406
|
-
this.pendingChanges.clear();
|
|
407
|
-
this.saveHashCache();
|
|
408
|
-
if (this.watcher) {
|
|
409
|
-
await this.watcher.close();
|
|
410
|
-
this.watcher = null;
|
|
411
|
-
}
|
|
412
|
-
this.vectorStore.close();
|
|
413
|
-
logger.info("Watcher stopped");
|
|
414
|
-
}
|
|
415
|
-
/**
|
|
416
|
-
* Check if watcher is running
|
|
417
|
-
*/
|
|
418
|
-
isRunning() {
|
|
419
|
-
return this.watcher !== null;
|
|
420
|
-
}
|
|
421
|
-
/**
|
|
422
|
-
* Clear the hash cache
|
|
423
|
-
*/
|
|
424
|
-
clearCache() {
|
|
425
|
-
this.hashCache = {};
|
|
426
|
-
const cachePath = this.getHashCachePath();
|
|
427
|
-
if (fs.existsSync(cachePath)) {
|
|
428
|
-
fs.unlinkSync(cachePath);
|
|
429
|
-
}
|
|
430
|
-
logger.info("Hash cache cleared");
|
|
431
|
-
}
|
|
432
|
-
/**
|
|
433
|
-
* Get cache statistics
|
|
434
|
-
*/
|
|
435
|
-
getCacheStats() {
|
|
436
|
-
return {
|
|
437
|
-
cachedFiles: Object.keys(this.hashCache).length,
|
|
438
|
-
cacheSize: JSON.stringify(this.hashCache).length,
|
|
439
|
-
};
|
|
440
|
-
}
|
|
441
|
-
}
|
|
442
|
-
/**
|
|
443
|
-
* Create a new index watcher
|
|
444
|
-
*/
|
|
445
|
-
export function createIndexWatcher(options) {
|
|
446
|
-
return new IndexWatcher(options);
|
|
447
|
-
}
|
|
448
|
-
//# sourceMappingURL=watcher.js.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"watcher.js","sourceRoot":"","sources":["../../../src/core/embeddings/watcher.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AAEH,OAAO,KAAK,EAAE,MAAM,SAAS,CAAC;AAC9B,OAAO,KAAK,IAAI,MAAM,WAAW,CAAC;AAClC,OAAO,KAAK,MAAM,MAAM,aAAa,CAAC;AACtC,OAAO,EAAE,KAAK,EAAkB,MAAM,UAAU,CAAC;AACjD,OAAO,EAAE,MAAM,WAAW,CAAC;AAC3B,OAAO,MAAuB,MAAM,QAAQ,CAAC;AAE7C,OAAO,EAAE,YAAY,EAAE,MAAM,yBAAyB,CAAC;AACvD,OAAO,EAAE,WAAW,EAAE,MAAM,wBAAwB,CAAC;AACrD,OAAO,EACL,SAAS,EACT,eAAe,EACf,oBAAoB,GACrB,MAAM,0BAA0B,CAAC;AAClC,OAAO,EAAE,oBAAoB,EAAE,MAAM,2BAA2B,CAAC;AACjE,OAAO,EAAE,MAAM,EAAE,MAAM,QAAQ,CAAC;AAEhC,6CAA6C;AAC7C,MAAM,mBAAmB,GAAG,IAAI,CAAC;AAEjC,yCAAyC;AACzC,MAAM,eAAe,GAAG,wBAAwB,CAAC;AAqBjD,MAAM,OAAO,YAAY;IACN,SAAS,CAAS;IAClB,MAAM,CAAkB;IACxB,UAAU,CAAS;IACnB,YAAY,CAAe;IAC3B,WAAW,CAAc;IAClC,OAAO,GAAqB,IAAI,CAAC;IACjC,EAAE,CAAS;IACX,YAAY,GAAG,KAAK,CAAC;IACrB,SAAS,GAAc,EAAE,CAAC;IAC1B,cAAc,GAAG,IAAI,GAAG,EAAyB,CAAC;IAClD,cAAc,GAA4B,EAAE,CAAC;IAEpC,OAAO,CAAc;IACrB,OAAO,CAA0B;IACjC,SAAS,CAA8B;IACvC,SAAS,CAA8B;IAExD,YAAY,OAAuB;QACjC,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC;QACjD,IAAI,CAAC,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC;QAC7B,IAAI,CAAC,UAAU,GAAG,OAAO,CAAC,UAAU,IAAI,mBAAmB,CAAC;QAC5D,IAAI,CAAC,YAAY,GAAG,IAAI,YAAY,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC;QACrD,IAAI,CAAC,WAAW,GAAG,IAAI,WAAW,CAAC,IAAI,CAAC,SAAS,EAAE,OAAO,CAAC,MAAM,CAAC,CAAC;QACnE,IAAI,CAAC,EAAE,GAAG,IAAI,CAAC,kBAAkB,EAAE,CAAC;QAEpC,IAAI,CAAC,OAAO,GAAG,OAAO,CAAC,OAAO,CAAC;QAC/B,IAAI,CAAC,OAAO,GAAG,OAAO,CAAC,OAAO,CAAC;QAC/B,IAAI,CAAC,SAAS,GAAG,OAAO,CAAC,SAAS,CAAC;QACnC,IAAI,CAAC,SAAS,GAAG,OAAO,CAAC,SAAS,CAAC;QAEnC,IAAI,CAAC,aAAa,EAAE,CAAC;IACvB,CAAC;IAED;;OAEG;IACK,WAAW,CAAC,OAAe;QACjC,OAAO,MAAM,CAAC,UAAU,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;IAC3E,CAAC;IAED;;OAEG;IACK,gBAAgB;QACtB,OAAO,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,YAAY,EAAE,eAAe,CAAC,CAAC;IAClE,CAAC;IAED;;OAEG;IACK,aAAa;QACnB,MAAM,SAAS,GAAG,IAAI,CAAC,gBAAgB,EAAE,CAAC;QAE1C,IAAI,EAAE,CAAC,UAAU,CAAC,SAAS,CAAC,EAAE,CAAC;YAC7B,IAAI,CAAC;gBACH,MAAM,OAAO,GAAG,EAAE,CAAC,YAAY,CAAC,SAAS,EAAE,OAAO,CAAC,CAAC;gBACpD,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC,KAAK,CAAC,OAAO,CAAc,CAAC;gBAClD,MAAM,CAAC,KAAK,CACV,UAAU,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,MAAM,CAAC,gBAAgB,CACrE,CAAC;YACJ,CAAC;YAAC,MAAM,CAAC;gBACP,IAAI,CAAC,SAAS,GAAG,EAAE,CAAC;YACtB,CAAC;QACH,CAAC;IACH,CAAC;IAED;;OAEG;IACK,aAAa;QACnB,MAAM,SAAS,GAAG,IAAI,CAAC,gBAAgB,EAAE,CAAC;QAC1C,MAAM,QAAQ,GAAG,IAAI,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC;QAEzC,IAAI,CAAC;YACH,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,QAAQ,CAAC,EAAE,CAAC;gBAC7B,EAAE,CAAC,SAAS,CAAC,QAAQ,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;YAC9C,CAAC;YACD,EAAE,CAAC,aAAa,CAAC,SAAS,EAAE,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC;QACvE,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,MAAM,KAAK,GAAG,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,KAAK,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC;YAClE,MAAM,CAAC,KAAK,CAAC,8BAA8B,KAAK,CAAC,OAAO,EAAE,CAAC,CAAC;QAC9D,CAAC;IACH,CAAC;IAED;;OAEG;IACK,iBAAiB,CAAC,QAAgB,EAAE,OAAe;QACzD,MAAM,OAAO,GAAG,IAAI,CAAC,WAAW,CAAC,OAAO,CAAC,CAAC;QAC1C,MAAM,OAAO,GAAG,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC,CAAC;QAEzC,IAAI,OAAO,KAAK,OAAO,EAAE,CAAC;YACxB,OAAO,KAAK,CAAC;QACf,CAAC;QAED,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC,GAAG,OAAO,CAAC;QACnC,OAAO,IAAI,CAAC;IACd,CAAC;IAED;;OAEG;IACK,mBAAmB,CAAC,QAAgB;QAC1C,MAAM,EAAE,CAAC,QAAQ,CAAC,EAAE,CAAC,EAAE,GAAG,IAAI,EAAE,GAAG,IAAI,CAAC,SAAS,CAAC;QAClD,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC;IACxB,CAAC;IAED;;OAEG;IACK,kBAAkB;QACxB,MAAM,EAAE,GAAG,MAAM,EAAE,CAAC;QACpB,MAAM,aAAa,GAAG,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,YAAY,CAAC,CAAC;QAE9D,IAAI,EAAE,CAAC,UAAU,CAAC,aAAa,CAAC,EAAE,CAAC;YACjC,IAAI,CAAC;gBACH,MAAM,OAAO,GAAG,EAAE,CAAC,YAAY,CAAC,aAAa,EAAE,OAAO,CAAC,CAAC;gBACxD,EAAE,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;YAClB,CAAC;YAAC,MAAM,CAAC;gBACP,qBAAqB;YACvB,CAAC;QACH,CAAC;QAED,OAAO,EAAE,CAAC;IACZ,CAAC;IAED;;OAEG;IACK,WAAW,CAAC,QAAgB;QAClC,MAAM,YAAY,GAAG,IAAI;aACtB,QAAQ,CAAC,IAAI,CAAC,SAAS,EAAE,QAAQ,CAAC;aAClC,OAAO,CAAC,KAAK,EAAE,GAAG,CAAC,CAAC;QAEvB,4BAA4B;QAC5B,IAAI,YAAY,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC;YACjE,OAAO,KAAK,CAAC;QACf,CAAC;QAED,0BAA0B;QAC1B,IAAI,IAAI,CAAC,EAAE,CAAC,OAAO,CAAC,YAAY,CAAC,EAAE,CAAC;YAClC,OAAO,KAAK,CAAC;QACf,CAAC;QAED,OAAO,eAAe,CAAC,QAAQ,CAAC,CAAC;IACnC,CAAC;IAED;;OAEG;IACK,cAAc,CACpB,IAAiC,EACjC,QAAgB;QAEhB,MAAM,QAAQ,GAAG,IAAI,CAAC,cAAc,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;QACnD,IAAI,QAAQ,EAAE,CAAC;YACb,YAAY,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC;QAC/B,CAAC;QAED,MAAM,KAAK,GAAG,UAAU,CAAC,GAAG,EAAE;YAC5B,IAAI,CAAC,cAAc,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC;YACrC,IAAI,CAAC,cAAc,CAAC,KAAK,IAAI,EAAE,CAAC,IAAI,CAAC,aAAa,CAAC,IAAI,EAAE,QAAQ,CAAC,CAAC,CAAC;QACtE,CAAC,EAAE,IAAI,CAAC,UAAU,CAAC,CAAC;QAEpB,IAAI,CAAC,cAAc,CAAC,GAAG,CAAC,QAAQ,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,KAAK,EAAE,CAAC,CAAC;QAE7D,MAAM,CAAC,KAAK,CACV,aAAa,IAAI,KAAK,IAAI,CAAC,QAAQ,CAAC,QAAQ,CAAC,KAAK,MAAM,CAAC,IAAI,CAAC,UAAU,CAAC,KAAK,CAC/E,CAAC;IACJ,CAAC;IAED;;OAEG;IACK,KAAK,CAAC,aAAa,CACzB,IAAiC,EACjC,QAAgB;QAEhB,IAAI,IAAI,KAAK,QAAQ,EAAE,CAAC;YACtB,MAAM,IAAI,CAAC,UAAU,CAAC,QAAQ,CAAC,CAAC;QAClC,CAAC;aAAM,CAAC;YACN,MAAM,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC,CAAC;QACjC,CAAC;IACH,CAAC;IAED;;OAEG;IACK,KAAK,CAAC,SAAS,CAAC,QAAgB;QACtC,IAAI,CAAC,IAAI,CAAC,WAAW,CAAC,QAAQ,CAAC,EAAE,CAAC;YAChC,OAAO;QACT,CAAC;QAED,IAAI,CAAC;YACH,MAAM,OAAO,GAAG,EAAE,CAAC,YAAY,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;YAEnD,4BAA4B;YAC5B,IAAI,CAAC,IAAI,CAAC,iBAAiB,CAAC,QAAQ,EAAE,OAAO,CAAC,EAAE,CAAC;gBAC/C,MAAM,CAAC,KAAK,CAAC,wBAAwB,IAAI,CAAC,QAAQ,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC;gBAChE,OAAO;YACT,CAAC;YAED,MAAM,MAAM,GAAG,MAAM,SAAS,CAAC,QAAQ,EAAE,OAAO,EAAE,IAAI,CAAC,MAAM,CAAC,CAAC;YAE/D,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;gBACxB,OAAO;YACT,CAAC;YAED,uCAAuC;YACvC,MAAM,cAAc,GAAG,MAAM,oBAAoB,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;YAEnE,oCAAoC;YACpC,MAAM,KAAK,GAAG,cAAc,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,eAAe,CAAC,CAAC;YAC3D,MAAM,UAAU,GAAG,MAAM,IAAI,CAAC,YAAY,CAAC,UAAU,CAAC,KAAK,CAAC,CAAC;YAE7D,sDAAsD;YACtD,MAAM,cAAc,GAAG,cAAc,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC;gBACvD,EAAE,EAAE,KAAK,CAAC,EAAE;gBACZ,OAAO,EAAE,KAAK,CAAC,OAAO;gBACtB,QAAQ,EAAE,KAAK,CAAC,QAAQ;gBACxB,QAAQ,EAAE,KAAK,CAAC,QAAQ;gBACxB,SAAS,EAAE,KAAK,CAAC,SAAS;gBAC1B,OAAO,EAAE,KAAK,CAAC,OAAO;gBACtB,UAAU,EAAE,KAAK,CAAC,UAAU;gBAC5B,UAAU,EAAE,KAAK,CAAC,UAAU;gBAC5B,MAAM,EAAE,UAAU,CAAC,CAAC,CAAC,IAAI,EAAE;aAC5B,CAAC,CAAC,CAAC;YAEJ,MAAM,IAAI,CAAC,WAAW,CAAC,gBAAgB,CAAC,QAAQ,CAAC,CAAC;YAClD,MAAM,IAAI,CAAC,WAAW,CAAC,SAAS,CAAC,cAAc,CAAC,CAAC;YAEjD,IAAI,CAAC,aAAa,EAAE,CAAC;YAErB,MAAM,CAAC,KAAK,CAAC,YAAY,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,SAAS,EAAE,QAAQ,CAAC,EAAE,CAAC,CAAC;YACpE,IAAI,CAAC,SAAS,EAAE,CAAC,QAAQ,CAAC,CAAC;QAC7B,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,MAAM,KAAK,GAAG,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,KAAK,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC;YAClE,MAAM,CAAC,KAAK,CAAC,mBAAmB,QAAQ,KAAK,KAAK,CAAC,OAAO,EAAE,CAAC,CAAC;YAC9D,IAAI,CAAC,OAAO,EAAE,CAAC,KAAK,CAAC,CAAC;QACxB,CAAC;IACH,CAAC;IAED;;OAEG;IACK,KAAK,CAAC,UAAU,CAAC,QAAgB;QACvC,IAAI,CAAC;YACH,MAAM,IAAI,CAAC,WAAW,CAAC,gBAAgB,CAAC,QAAQ,CAAC,CAAC;YAClD,IAAI,CAAC,mBAAmB,CAAC,QAAQ,CAAC,CAAC;YACnC,IAAI,CAAC,aAAa,EAAE,CAAC;YAErB,MAAM,CAAC,KAAK,CAAC,YAAY,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,SAAS,EAAE,QAAQ,CAAC,EAAE,CAAC,CAAC;YACpE,IAAI,CAAC,SAAS,EAAE,CAAC,QAAQ,CAAC,CAAC;QAC7B,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,MAAM,KAAK,GAAG,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,KAAK,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC;YAClE,MAAM,CAAC,KAAK,CAAC,oBAAoB,QAAQ,KAAK,KAAK,CAAC,OAAO,EAAE,CAAC,CAAC;YAC/D,IAAI,CAAC,OAAO,EAAE,CAAC,KAAK,CAAC,CAAC;QACxB,CAAC;IACH,CAAC;IAED;;OAEG;IACK,cAAc,CAAC,SAA8B;QACnD,IAAI,CAAC,cAAc,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;QACpC,KAAK,IAAI,CAAC,YAAY,EAAE,CAAC;IAC3B,CAAC;IAED;;OAEG;IACK,KAAK,CAAC,YAAY;QACxB,IAAI,IAAI,CAAC,YAAY,EAAE,CAAC;YACtB,OAAO;QACT,CAAC;QAED,IAAI,CAAC,YAAY,GAAG,IAAI,CAAC;QAEzB,OAAO,IAAI,CAAC,cAAc,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACtC,MAAM,SAAS,GAAG,IAAI,CAAC,cAAc,CAAC,KAAK,EAAE,CAAC;YAC9C,IAAI,SAAS,EAAE,CAAC;gBACd,IAAI,CAAC;oBACH,MAAM,SAAS,EAAE,CAAC;gBACpB,CAAC;gBAAC,OAAO,GAAG,EAAE,CAAC;oBACb,MAAM,KAAK,GAAG,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,KAAK,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC;oBAClE,MAAM,CAAC,KAAK,CAAC,qBAAqB,KAAK,CAAC,OAAO,EAAE,CAAC,CAAC;gBACrD,CAAC;YACH,CAAC;QACH,CAAC;QAED,IAAI,CAAC,YAAY,GAAG,KAAK,CAAC;IAC5B,CAAC;IAED;;OAEG;IACK,KAAK,CAAC,oBAAoB;QAChC,MAAM,UAAU,GAAG,oBAAoB,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC;QACnE,MAAM,OAAO,GAAG,SAAS,UAAU,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC;QAEjD,MAAM,KAAK,GAAG,MAAM,EAAE,CAAC,OAAO,EAAE;YAC9B,GAAG,EAAE,IAAI,CAAC,SAAS;YACnB,QAAQ,EAAE,IAAI;YACd,MAAM,EAAE,CAAC,OAAO,EAAE,UAAU,CAAC;YAC7B,GAAG,EAAE,KAAK;YACV,SAAS,EAAE,IAAI;YACf,mBAAmB,EAAE,KAAK;SAC3B,CAAC,CAAC;QAEH,sBAAsB;QACtB,OAAO,KAAK,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE;YAC3B,MAAM,YAAY,GAAG,IAAI;iBACtB,QAAQ,CAAC,IAAI,CAAC,SAAS,EAAE,IAAI,CAAC;iBAC9B,OAAO,CAAC,KAAK,EAAE,GAAG,CAAC,CAAC;YACvB,OAAO,CAAC,IAAI,CAAC,EAAE,CAAC,OAAO,CAAC,YAAY,CAAC,CAAC;QACxC,CAAC,CAAC,CAAC;IACL,CAAC;IAED;;OAEG;IACK,KAAK,CAAC,SAAS;QACrB,MAAM,CAAC,IAAI,CAAC,wBAAwB,CAAC,CAAC;QAEtC,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,oBAAoB,EAAE,CAAC;QAChD,IAAI,OAAO,GAAG,CAAC,CAAC;QAChB,IAAI,OAAO,GAAG,CAAC,CAAC;QAEhB,KAAK,MAAM,QAAQ,IAAI,KAAK,EAAE,CAAC;YAC7B,IAAI,CAAC;gBACH,MAAM,OAAO,GAAG,EAAE,CAAC,YAAY,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;gBAEnD,IAAI,CAAC,IAAI,CAAC,iBAAiB,CAAC,QAAQ,EAAE,OAAO,CAAC,EAAE,CAAC;oBAC/C,OAAO,EAAE,CAAC;oBACV,SAAS;gBACX,CAAC;gBAED,MAAM,MAAM,GAAG,MAAM,SAAS,CAAC,QAAQ,EAAE,OAAO,EAAE,IAAI,CAAC,MAAM,CAAC,CAAC;gBAE/D,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;oBACxB,SAAS;gBACX,CAAC;gBAED,uCAAuC;gBACvC,MAAM,cAAc,GAAG,MAAM,oBAAoB,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;gBAEnE,oCAAoC;gBACpC,MAAM,KAAK,GAAG,cAAc,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,eAAe,CAAC,CAAC;gBAC3D,MAAM,UAAU,GAAG,MAAM,IAAI,CAAC,YAAY,CAAC,UAAU,CAAC,KAAK,CAAC,CAAC;gBAE7D,sDAAsD;gBACtD,MAAM,cAAc,GAAG,cAAc,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC;oBACvD,EAAE,EAAE,KAAK,CAAC,EAAE;oBACZ,OAAO,EAAE,KAAK,CAAC,OAAO;oBACtB,QAAQ,EAAE,KAAK,CAAC,QAAQ;oBACxB,QAAQ,EAAE,KAAK,CAAC,QAAQ;oBACxB,SAAS,EAAE,KAAK,CAAC,SAAS;oBAC1B,OAAO,EAAE,KAAK,CAAC,OAAO;oBACtB,UAAU,EAAE,KAAK,CAAC,UAAU;oBAC5B,UAAU,EAAE,KAAK,CAAC,UAAU;oBAC5B,MAAM,EAAE,UAAU,CAAC,CAAC,CAAC,IAAI,EAAE;iBAC5B,CAAC,CAAC,CAAC;gBAEJ,MAAM,IAAI,CAAC,WAAW,CAAC,SAAS,CAAC,cAAc,CAAC,CAAC;gBACjD,OAAO,EAAE,CAAC;YACZ,CAAC;YAAC,OAAO,GAAG,EAAE,CAAC;gBACb,MAAM,KAAK,GAAG,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,KAAK,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC;gBAClE,MAAM,CAAC,KAAK,CAAC,kBAAkB,QAAQ,KAAK,KAAK,CAAC,OAAO,EAAE,CAAC,CAAC;YAC/D,CAAC;QACH,CAAC;QAED,IAAI,CAAC,aAAa,EAAE,CAAC;QAErB,MAAM,CAAC,IAAI,CACT,eAAe,MAAM,CAAC,OAAO,CAAC,aAAa,MAAM,CAAC,OAAO,CAAC,UAAU,CACrE,CAAC;IACJ,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,KAAK;QACT,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,YAAY,CAAC,WAAW,EAAE,CAAC;QACrD,IAAI,CAAC,MAAM,CAAC,EAAE,EAAE,CAAC;YACf,MAAM,IAAI,KAAK,CAAC,MAAM,CAAC,KAAK,IAAI,yBAAyB,CAAC,CAAC;QAC7D,CAAC;QAED,MAAM,IAAI,CAAC,WAAW,CAAC,OAAO,EAAE,CAAC;QAEjC,IAAI,CAAC,IAAI,CAAC,WAAW,CAAC,MAAM,EAAE,EAAE,CAAC;YAC/B,MAAM,IAAI,CAAC,SAAS,EAAE,CAAC;QACzB,CAAC;QAED,IAAI,CAAC,OAAO,GAAG,KAAK,CAAC,IAAI,CAAC,SAAS,EAAE;YACnC,OAAO,EAAE,CAAC,QAAgB,EAAE,EAAE;gBAC5B,MAAM,YAAY,GAAG,IAAI;qBACtB,QAAQ,CAAC,IAAI,CAAC,SAAS,EAAE,QAAQ,CAAC;qBAClC,OAAO,CAAC,KAAK,EAAE,GAAG,CAAC,CAAC;gBACvB,qCAAqC;gBACrC,IAAI,CAAC,YAAY,EAAE,CAAC;oBAClB,OAAO,KAAK,CAAC;gBACf,CAAC;gBACD,4BAA4B;gBAC5B,IAAI,YAAY,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC;oBACjE,OAAO,IAAI,CAAC;gBACd,CAAC;gBACD,OAAO,IAAI,CAAC,EAAE,CAAC,OAAO,CAAC,YAAY,CAAC,CAAC;YACvC,CAAC;YACD,UAAU,EAAE,IAAI;YAChB,aAAa,EAAE,IAAI;YACnB,gBAAgB,EAAE;gBAChB,kBAAkB,EAAE,GAAG;gBACvB,YAAY,EAAE,GAAG;aAClB;SACF,CAAC,CAAC;QAEH,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC,KAAK,EAAE,CAAC,QAAgB,EAAE,EAAE;YAC1C,IAAI,eAAe,CAAC,QAAQ,CAAC,EAAE,CAAC;gBAC9B,IAAI,CAAC,cAAc,CAAC,KAAK,EAAE,QAAQ,CAAC,CAAC;YACvC,CAAC;QACH,CAAC,CAAC,CAAC;QAEH,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAgB,EAAE,EAAE;YAC7C,IAAI,eAAe,CAAC,QAAQ,CAAC,EAAE,CAAC;gBAC9B,IAAI,CAAC,cAAc,CAAC,QAAQ,EAAE,QAAQ,CAAC,CAAC;YAC1C,CAAC;QACH,CAAC,CAAC,CAAC;QAEH,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAgB,EAAE,EAAE;YAC7C,IAAI,eAAe,CAAC,QAAQ,CAAC,EAAE,CAAC;gBAC9B,IAAI,CAAC,cAAc,CAAC,QAAQ,EAAE,QAAQ,CAAC,CAAC;YAC1C,CAAC;QACH,CAAC,CAAC,CAAC;QAEH,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC,OAAO,EAAE,GAAG,EAAE;YAC5B,MAAM,CAAC,IAAI,CACT,aAAa,IAAI,CAAC,SAAS,KAAK,MAAM,CAAC,IAAI,CAAC,UAAU,CAAC,cAAc,CACtE,CAAC;YACF,IAAI,CAAC,OAAO,EAAE,EAAE,CAAC;QACnB,CAAC,CAAC,CAAC;QAEH,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,GAAY,EAAE,EAAE;YACxC,MAAM,KAAK,GAAG,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,KAAK,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC;YAClE,MAAM,CAAC,KAAK,CAAC,kBAAkB,KAAK,CAAC,OAAO,EAAE,CAAC,CAAC;YAChD,IAAI,CAAC,OAAO,EAAE,CAAC,KAAK,CAAC,CAAC;QACxB,CAAC,CAAC,CAAC;IACL,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,IAAI;QACR,KAAK,MAAM,OAAO,IAAI,IAAI,CAAC,cAAc,CAAC,MAAM,EAAE,EAAE,CAAC;YACnD,YAAY,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;QAC9B,CAAC;QACD,IAAI,CAAC,cAAc,CAAC,KAAK,EAAE,CAAC;QAE5B,IAAI,CAAC,aAAa,EAAE,CAAC;QAErB,IAAI,IAAI,CAAC,OAAO,EAAE,CAAC;YACjB,MAAM,IAAI,CAAC,OAAO,CAAC,KAAK,EAAE,CAAC;YAC3B,IAAI,CAAC,OAAO,GAAG,IAAI,CAAC;QACtB,CAAC;QACD,IAAI,CAAC,WAAW,CAAC,KAAK,EAAE,CAAC;QACzB,MAAM,CAAC,IAAI,CAAC,iBAAiB,CAAC,CAAC;IACjC,CAAC;IAED;;OAEG;IACH,SAAS;QACP,OAAO,IAAI,CAAC,OAAO,KAAK,IAAI,CAAC;IAC/B,CAAC;IAED;;OAEG;IACH,UAAU;QACR,IAAI,CAAC,SAAS,GAAG,EAAE,CAAC;QACpB,MAAM,SAAS,GAAG,IAAI,CAAC,gBAAgB,EAAE,CAAC;QAC1C,IAAI,EAAE,CAAC,UAAU,CAAC,SAAS,CAAC,EAAE,CAAC;YAC7B,EAAE,CAAC,UAAU,CAAC,SAAS,CAAC,CAAC;QAC3B,CAAC;QACD,MAAM,CAAC,IAAI,CAAC,oBAAoB,CAAC,CAAC;IACpC,CAAC;IAED;;OAEG;IACH,aAAa;QACX,OAAO;YACL,WAAW,EAAE,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,MAAM;YAC/C,SAAS,EAAE,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,MAAM;SACjD,CAAC;IACJ,CAAC;CACF;AAED;;GAEG;AACH,MAAM,UAAU,kBAAkB,CAAC,OAAuB;IACxD,OAAO,IAAI,YAAY,CAAC,OAAO,CAAC,CAAC;AACnC,CAAC"}
|
|
@@ -1,26 +0,0 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* LangChain text splitter fallback for languages without Tree-sitter support
|
|
3
|
-
*/
|
|
4
|
-
import { type SupportedTextSplitterLanguage } from "@langchain/textsplitters";
|
|
5
|
-
export interface TextChunk {
|
|
6
|
-
content: string;
|
|
7
|
-
startLine: number;
|
|
8
|
-
endLine: number;
|
|
9
|
-
index: number;
|
|
10
|
-
}
|
|
11
|
-
export interface TextSplitResult {
|
|
12
|
-
chunks: TextChunk[];
|
|
13
|
-
count: number;
|
|
14
|
-
language: string;
|
|
15
|
-
method: "text-splitter";
|
|
16
|
-
}
|
|
17
|
-
export interface TextSplitOptions {
|
|
18
|
-
chunkSize?: number;
|
|
19
|
-
chunkOverlap?: number;
|
|
20
|
-
}
|
|
21
|
-
export declare function isTextSplitterLanguage(language: string): boolean;
|
|
22
|
-
export declare function getTextSplitterLanguage(language: string): SupportedTextSplitterLanguage | undefined;
|
|
23
|
-
export declare function splitCode(content: string, language: string, options?: TextSplitOptions): Promise<TextSplitResult>;
|
|
24
|
-
export declare function getSeparators(language: string): string[];
|
|
25
|
-
export declare function clearConfigCache(): void;
|
|
26
|
-
//# sourceMappingURL=index.d.ts.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/core/fallback/index.ts"],"names":[],"mappings":"AAAA;;GAEG;AACH,OAAO,EAEL,KAAK,6BAA6B,EACnC,MAAM,0BAA0B,CAAC;AAgClC,MAAM,WAAW,SAAS;IACxB,OAAO,EAAE,MAAM,CAAC;IAChB,SAAS,EAAE,MAAM,CAAC;IAClB,OAAO,EAAE,MAAM,CAAC;IAChB,KAAK,EAAE,MAAM,CAAC;CACf;AAED,MAAM,WAAW,eAAe;IAC9B,MAAM,EAAE,SAAS,EAAE,CAAC;IACpB,KAAK,EAAE,MAAM,CAAC;IACd,QAAQ,EAAE,MAAM,CAAC;IACjB,MAAM,EAAE,eAAe,CAAC;CACzB;AAED,MAAM,WAAW,gBAAgB;IAC/B,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,YAAY,CAAC,EAAE,MAAM,CAAC;CACvB;AAED,wBAAgB,sBAAsB,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAOhE;AAED,wBAAgB,uBAAuB,CACrC,QAAQ,EAAE,MAAM,GACf,6BAA6B,GAAG,SAAS,CAU3C;AAmBD,wBAAsB,SAAS,CAC7B,OAAO,EAAE,MAAM,EACf,QAAQ,EAAE,MAAM,EAChB,OAAO,GAAE,gBAAqB,GAC7B,OAAO,CAAC,eAAe,CAAC,CA0B1B;AAED,wBAAgB,aAAa,CAAC,QAAQ,EAAE,MAAM,GAAG,MAAM,EAAE,CAKxD;AAED,wBAAgB,gBAAgB,IAAI,IAAI,CAEvC"}
|
|
@@ -1,76 +0,0 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* LangChain text splitter fallback for languages without Tree-sitter support
|
|
3
|
-
*/
|
|
4
|
-
import { RecursiveCharacterTextSplitter, } from "@langchain/textsplitters";
|
|
5
|
-
import { loadJsonConfig, registerCache } from "@core/utils";
|
|
6
|
-
let langchainConfig = null;
|
|
7
|
-
function loadConfig() {
|
|
8
|
-
if (langchainConfig) {
|
|
9
|
-
return langchainConfig;
|
|
10
|
-
}
|
|
11
|
-
const defaultConfig = {
|
|
12
|
-
langchain: { mapping: {}, supported: [], generic: [] },
|
|
13
|
-
};
|
|
14
|
-
const fullConfig = loadJsonConfig("languages.json", defaultConfig);
|
|
15
|
-
langchainConfig = fullConfig.langchain;
|
|
16
|
-
return langchainConfig;
|
|
17
|
-
}
|
|
18
|
-
export function isTextSplitterLanguage(language) {
|
|
19
|
-
const config = loadConfig();
|
|
20
|
-
return (language in config.mapping ||
|
|
21
|
-
config.supported.includes(language) ||
|
|
22
|
-
config.generic.includes(language));
|
|
23
|
-
}
|
|
24
|
-
export function getTextSplitterLanguage(language) {
|
|
25
|
-
const config = loadConfig();
|
|
26
|
-
if (language in config.mapping) {
|
|
27
|
-
return config.mapping[language];
|
|
28
|
-
}
|
|
29
|
-
if (config.supported.includes(language)) {
|
|
30
|
-
return language;
|
|
31
|
-
}
|
|
32
|
-
return undefined;
|
|
33
|
-
}
|
|
34
|
-
function calculateLineNumbers(fullContent, chunkContent, startSearchIndex) {
|
|
35
|
-
const chunkIndex = fullContent.indexOf(chunkContent, startSearchIndex);
|
|
36
|
-
const actualIndex = chunkIndex >= 0 ? chunkIndex : startSearchIndex;
|
|
37
|
-
const beforeChunk = fullContent.slice(0, actualIndex);
|
|
38
|
-
const startLine = (beforeChunk.match(/\n/g) ?? []).length + 1;
|
|
39
|
-
const chunkLines = (chunkContent.match(/\n/g) ?? []).length;
|
|
40
|
-
return {
|
|
41
|
-
startLine,
|
|
42
|
-
endLine: startLine + chunkLines,
|
|
43
|
-
foundIndex: actualIndex + chunkContent.length,
|
|
44
|
-
};
|
|
45
|
-
}
|
|
46
|
-
export async function splitCode(content, language, options = {}) {
|
|
47
|
-
const { chunkSize = 1000, chunkOverlap = 200 } = options;
|
|
48
|
-
const splitterLanguage = getTextSplitterLanguage(language);
|
|
49
|
-
const splitter = splitterLanguage
|
|
50
|
-
? RecursiveCharacterTextSplitter.fromLanguage(splitterLanguage, {
|
|
51
|
-
chunkSize,
|
|
52
|
-
chunkOverlap,
|
|
53
|
-
})
|
|
54
|
-
: new RecursiveCharacterTextSplitter({ chunkSize, chunkOverlap });
|
|
55
|
-
const docs = await splitter.createDocuments([content]);
|
|
56
|
-
const chunks = [];
|
|
57
|
-
let searchIndex = 0;
|
|
58
|
-
for (const [i, doc] of docs.entries()) {
|
|
59
|
-
const { startLine, endLine, foundIndex } = calculateLineNumbers(content, doc.pageContent, searchIndex);
|
|
60
|
-
searchIndex = foundIndex;
|
|
61
|
-
chunks.push({ content: doc.pageContent, startLine, endLine, index: i });
|
|
62
|
-
}
|
|
63
|
-
return { chunks, count: chunks.length, language, method: "text-splitter" };
|
|
64
|
-
}
|
|
65
|
-
export function getSeparators(language) {
|
|
66
|
-
const splitterLanguage = getTextSplitterLanguage(language);
|
|
67
|
-
return splitterLanguage
|
|
68
|
-
? RecursiveCharacterTextSplitter.getSeparatorsForLanguage(splitterLanguage)
|
|
69
|
-
: [];
|
|
70
|
-
}
|
|
71
|
-
export function clearConfigCache() {
|
|
72
|
-
langchainConfig = null;
|
|
73
|
-
}
|
|
74
|
-
// Register cache for centralized clearing
|
|
75
|
-
registerCache("fallback:config", clearConfigCache);
|
|
76
|
-
//# sourceMappingURL=index.js.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../src/core/fallback/index.ts"],"names":[],"mappings":"AAAA;;GAEG;AACH,OAAO,EACL,8BAA8B,GAE/B,MAAM,0BAA0B,CAAC;AAElC,OAAO,EAAE,cAAc,EAAE,aAAa,EAAE,MAAM,aAAa,CAAC;AAY5D,IAAI,eAAe,GAA2B,IAAI,CAAC;AAEnD,SAAS,UAAU;IACjB,IAAI,eAAe,EAAE,CAAC;QACpB,OAAO,eAAe,CAAC;IACzB,CAAC;IAED,MAAM,aAAa,GAAe;QAChC,SAAS,EAAE,EAAE,OAAO,EAAE,EAAE,EAAE,SAAS,EAAE,EAAE,EAAE,OAAO,EAAE,EAAE,EAAE;KACvD,CAAC;IACF,MAAM,UAAU,GAAG,cAAc,CAC/B,gBAAgB,EAChB,aAAa,CACd,CAAC;IACF,eAAe,GAAG,UAAU,CAAC,SAAS,CAAC;IACvC,OAAO,eAAe,CAAC;AACzB,CAAC;AAqBD,MAAM,UAAU,sBAAsB,CAAC,QAAgB;IACrD,MAAM,MAAM,GAAG,UAAU,EAAE,CAAC;IAC5B,OAAO,CACL,QAAQ,IAAI,MAAM,CAAC,OAAO;QAC1B,MAAM,CAAC,SAAS,CAAC,QAAQ,CAAC,QAAQ,CAAC;QACnC,MAAM,CAAC,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAClC,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,uBAAuB,CACrC,QAAgB;IAEhB,MAAM,MAAM,GAAG,UAAU,EAAE,CAAC;IAE5B,IAAI,QAAQ,IAAI,MAAM,CAAC,OAAO,EAAE,CAAC;QAC/B,OAAO,MAAM,CAAC,OAAO,CAAC,QAAQ,CAAkC,CAAC;IACnE,CAAC;IACD,IAAI,MAAM,CAAC,SAAS,CAAC,QAAQ,CAAC,QAAQ,CAAC,EAAE,CAAC;QACxC,OAAO,QAAyC,CAAC;IACnD,CAAC;IACD,OAAO,SAAS,CAAC;AACnB,CAAC;AAED,SAAS,oBAAoB,CAC3B,WAAmB,EACnB,YAAoB,EACpB,gBAAwB;IAExB,MAAM,UAAU,GAAG,WAAW,CAAC,OAAO,CAAC,YAAY,EAAE,gBAAgB,CAAC,CAAC;IACvE,MAAM,WAAW,GAAG,UAAU,IAAI,CAAC,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,gBAAgB,CAAC;IACpE,MAAM,WAAW,GAAG,WAAW,CAAC,KAAK,CAAC,CAAC,EAAE,WAAW,CAAC,CAAC;IACtD,MAAM,SAAS,GAAG,CAAC,WAAW,CAAC,KAAK,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC;IAC9D,MAAM,UAAU,GAAG,CAAC,YAAY,CAAC,KAAK,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC,CAAC,MAAM,CAAC;IAC5D,OAAO;QACL,SAAS;QACT,OAAO,EAAE,SAAS,GAAG,UAAU;QAC/B,UAAU,EAAE,WAAW,GAAG,YAAY,CAAC,MAAM;KAC9C,CAAC;AACJ,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,SAAS,CAC7B,OAAe,EACf,QAAgB,EAChB,UAA4B,EAAE;IAE9B,MAAM,EAAE,SAAS,GAAG,IAAI,EAAE,YAAY,GAAG,GAAG,EAAE,GAAG,OAAO,CAAC;IACzD,MAAM,gBAAgB,GAAG,uBAAuB,CAAC,QAAQ,CAAC,CAAC;IAE3D,MAAM,QAAQ,GAAG,gBAAgB;QAC/B,CAAC,CAAC,8BAA8B,CAAC,YAAY,CAAC,gBAAgB,EAAE;YAC5D,SAAS;YACT,YAAY;SACb,CAAC;QACJ,CAAC,CAAC,IAAI,8BAA8B,CAAC,EAAE,SAAS,EAAE,YAAY,EAAE,CAAC,CAAC;IAEpE,MAAM,IAAI,GAAG,MAAM,QAAQ,CAAC,eAAe,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC;IACvD,MAAM,MAAM,GAAgB,EAAE,CAAC;IAC/B,IAAI,WAAW,GAAG,CAAC,CAAC;IAEpB,KAAK,MAAM,CAAC,CAAC,EAAE,GAAG,CAAC,IAAI,IAAI,CAAC,OAAO,EAAE,EAAE,CAAC;QACtC,MAAM,EAAE,SAAS,EAAE,OAAO,EAAE,UAAU,EAAE,GAAG,oBAAoB,CAC7D,OAAO,EACP,GAAG,CAAC,WAAW,EACf,WAAW,CACZ,CAAC;QACF,WAAW,GAAG,UAAU,CAAC;QACzB,MAAM,CAAC,IAAI,CAAC,EAAE,OAAO,EAAE,GAAG,CAAC,WAAW,EAAE,SAAS,EAAE,OAAO,EAAE,KAAK,EAAE,CAAC,EAAE,CAAC,CAAC;IAC1E,CAAC;IAED,OAAO,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,CAAC,MAAM,EAAE,QAAQ,EAAE,MAAM,EAAE,eAAe,EAAE,CAAC;AAC7E,CAAC;AAED,MAAM,UAAU,aAAa,CAAC,QAAgB;IAC5C,MAAM,gBAAgB,GAAG,uBAAuB,CAAC,QAAQ,CAAC,CAAC;IAC3D,OAAO,gBAAgB;QACrB,CAAC,CAAC,8BAA8B,CAAC,wBAAwB,CAAC,gBAAgB,CAAC;QAC3E,CAAC,CAAC,EAAE,CAAC;AACT,CAAC;AAED,MAAM,UAAU,gBAAgB;IAC9B,eAAe,GAAG,IAAI,CAAC;AACzB,CAAC;AAED,0CAA0C;AAC1C,aAAa,CAAC,iBAAiB,EAAE,gBAAgB,CAAC,CAAC"}
|
|
@@ -1,64 +0,0 @@
|
|
|
1
|
-
import { Language, type Node, Parser, type Point, type Tree } from "web-tree-sitter";
|
|
2
|
-
import type { ASTNode, Position } from "@core/ast/types";
|
|
3
|
-
export * from "./languages";
|
|
4
|
-
export type { Language, Node, Point, Tree };
|
|
5
|
-
/**
|
|
6
|
-
* Initialize the Tree-sitter WASM module
|
|
7
|
-
* Must be called before any parsing operations
|
|
8
|
-
*/
|
|
9
|
-
export declare function initializeParser(): Promise<void>;
|
|
10
|
-
/**
|
|
11
|
-
* Check if the parser is initialized
|
|
12
|
-
*/
|
|
13
|
-
export declare function isParserInitialized(): boolean;
|
|
14
|
-
/**
|
|
15
|
-
* Parse result
|
|
16
|
-
*/
|
|
17
|
-
export interface ParseResult {
|
|
18
|
-
/** The Tree-sitter tree */
|
|
19
|
-
tree: Tree;
|
|
20
|
-
/** Language that was used */
|
|
21
|
-
language: string;
|
|
22
|
-
/** The parser instance (for queries) */
|
|
23
|
-
parser: Parser;
|
|
24
|
-
/** The language instance (for queries) */
|
|
25
|
-
languageInstance: Language;
|
|
26
|
-
}
|
|
27
|
-
/**
|
|
28
|
-
* Parse options
|
|
29
|
-
*/
|
|
30
|
-
export interface ParseOptions {
|
|
31
|
-
/** Language name (auto-detected from file path if not provided) */
|
|
32
|
-
language?: string;
|
|
33
|
-
/** File path (for language detection) */
|
|
34
|
-
filePath?: string;
|
|
35
|
-
}
|
|
36
|
-
/**
|
|
37
|
-
* Parse code content
|
|
38
|
-
*/
|
|
39
|
-
export declare function parseCode(content: string, options?: ParseOptions): Promise<ParseResult>;
|
|
40
|
-
/**
|
|
41
|
-
* Convert Tree-sitter position to our Position type
|
|
42
|
-
*/
|
|
43
|
-
export declare function toPosition(point: Point, offset: number): Position;
|
|
44
|
-
/**
|
|
45
|
-
* Convert Tree-sitter node to ASTNode
|
|
46
|
-
*/
|
|
47
|
-
export declare function toASTNode(node: Node, maxDepth?: number, currentDepth?: number): ASTNode;
|
|
48
|
-
/**
|
|
49
|
-
* Get the root ASTNode from a parse result
|
|
50
|
-
*/
|
|
51
|
-
export declare function getASTRoot(parseResult: ParseResult, maxDepth?: number): ASTNode;
|
|
52
|
-
/**
|
|
53
|
-
* Count nodes in the tree
|
|
54
|
-
*/
|
|
55
|
-
export declare function countNodes(node: Node): number;
|
|
56
|
-
/**
|
|
57
|
-
* Clear the language cache (useful for testing)
|
|
58
|
-
*/
|
|
59
|
-
export declare function clearLanguageCache(): void;
|
|
60
|
-
/**
|
|
61
|
-
* Reset the parser state (useful for testing)
|
|
62
|
-
*/
|
|
63
|
-
export declare function resetParser(): void;
|
|
64
|
-
//# sourceMappingURL=index.d.ts.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/core/parser/index.ts"],"names":[],"mappings":"AASA,OAAO,EACL,QAAQ,EACR,KAAK,IAAI,EACT,MAAM,EACN,KAAK,KAAK,EACV,KAAK,IAAI,EACV,MAAM,iBAAiB,CAAC;AAEzB,OAAO,KAAK,EAAE,OAAO,EAAE,QAAQ,EAAE,MAAM,iBAAiB,CAAC;AAUzD,cAAc,aAAa,CAAC;AAG5B,YAAY,EAAE,QAAQ,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC;AAkB5C;;;GAGG;AACH,wBAAsB,gBAAgB,IAAI,OAAO,CAAC,IAAI,CAAC,CAiBtD;AAED;;GAEG;AACH,wBAAgB,mBAAmB,IAAI,OAAO,CAE7C;AA2CD;;GAEG;AACH,MAAM,WAAW,WAAW;IAC1B,2BAA2B;IAC3B,IAAI,EAAE,IAAI,CAAC;IACX,6BAA6B;IAC7B,QAAQ,EAAE,MAAM,CAAC;IACjB,wCAAwC;IACxC,MAAM,EAAE,MAAM,CAAC;IACf,0CAA0C;IAC1C,gBAAgB,EAAE,QAAQ,CAAC;CAC5B;AAED;;GAEG;AACH,MAAM,WAAW,YAAY;IAC3B,mEAAmE;IACnE,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,yCAAyC;IACzC,QAAQ,CAAC,EAAE,MAAM,CAAC;CACnB;AAED;;GAEG;AACH,wBAAsB,SAAS,CAC7B,OAAO,EAAE,MAAM,EACf,OAAO,GAAE,YAAiB,GACzB,OAAO,CAAC,WAAW,CAAC,CAwCtB;AAED;;GAEG;AACH,wBAAgB,UAAU,CAAC,KAAK,EAAE,KAAK,EAAE,MAAM,EAAE,MAAM,GAAG,QAAQ,CAMjE;AAED;;GAEG;AACH,wBAAgB,SAAS,CACvB,IAAI,EAAE,IAAI,EACV,QAAQ,CAAC,EAAE,MAAM,EACjB,YAAY,SAAI,GACf,OAAO,CA2CT;AAED;;GAEG;AACH,wBAAgB,UAAU,CACxB,WAAW,EAAE,WAAW,EACxB,QAAQ,CAAC,EAAE,MAAM,GAChB,OAAO,CAET;AAED;;GAEG;AACH,wBAAgB,UAAU,CAAC,IAAI,EAAE,IAAI,GAAG,MAAM,CAM7C;AAED;;GAEG;AACH,wBAAgB,kBAAkB,IAAI,IAAI,CAEzC;AAED;;GAEG;AACH,wBAAgB,WAAW,IAAI,IAAI,CAKlC"}
|