src-mcp 1.0.1

package/dist/core/embeddings/watcher.js

@@ -0,0 +1,448 @@
+/**
+ * File watcher for automatic index updates
+ *
+ * Features:
+ * - SHA-256 hash comparison to detect real content changes
+ * - Debounce (5s default) to handle rapid changes
+ * - Persistent hash cache to avoid unnecessary re-indexing
+ * - fast-glob for efficient file scanning
+ */
+import * as fs from "node:fs";
+import * as path from "node:path";
+import * as crypto from "node:crypto";
+import { watch } from "chokidar";
+import fg from "fast-glob";
+import ignore from "ignore";
+import { OllamaClient } from "@core/embeddings/client";
+import { VectorStore } from "@core/embeddings/store";
+import { chunkFile, shouldIndexFile, SUPPORTED_EXTENSIONS, } from "@core/embeddings/chunker";
+import { enrichChunksFromFile } from "@core/embeddings/enricher";
+import { logger } from "@utils";
+/** Default debounce delay in milliseconds */
+const DEFAULT_DEBOUNCE_MS = 5000;
+/** Cache file name for storing hashes */
+const HASH_CACHE_FILE = ".src-index-hashes.json";
+export class IndexWatcher {
+    directory;
+    config;
+    debounceMs;
+    ollamaClient;
+    vectorStore;
+    watcher = null;
+    ig;
+    isProcessing = false;
+    hashCache = {};
+    pendingChanges = new Map();
+    operationQueue = [];
+    onReady;
+    onError;
+    onIndexed;
+    onRemoved;
+    constructor(options) {
+        this.directory = path.resolve(options.directory);
+        this.config = options.config;
+        this.debounceMs = options.debounceMs ?? DEFAULT_DEBOUNCE_MS;
+        this.ollamaClient = new OllamaClient(options.config);
+        this.vectorStore = new VectorStore(this.directory, options.config);
+        this.ig = this.createIgnoreFilter();
+        this.onReady = options.onReady;
+        this.onError = options.onError;
+        this.onIndexed = options.onIndexed;
+        this.onRemoved = options.onRemoved;
+        this.loadHashCache();
+    }
+    /**
+     * Compute SHA-256 hash of content
+     */
+    computeHash(content) {
+        return crypto.createHash("sha256").update(content, "utf8").digest("hex");
+    }
+    /**
+     * Get hash cache file path
+     */
+    getHashCachePath() {
+        return path.join(this.directory, ".src-index", HASH_CACHE_FILE);
+    }
+    /**
+     * Load hash cache from disk
+     */
+    loadHashCache() {
+        const cachePath = this.getHashCachePath();
+        if (fs.existsSync(cachePath)) {
+            try {
+                const content = fs.readFileSync(cachePath, "utf-8");
+                this.hashCache = JSON.parse(content);
+                logger.debug(`Loaded ${String(Object.keys(this.hashCache).length)} cached hashes`);
+            }
+            catch {
+                this.hashCache = {};
+            }
+        }
+    }
+    /**
+     * Save hash cache to disk
+     */
+    saveHashCache() {
+        const cachePath = this.getHashCachePath();
+        const cacheDir = path.dirname(cachePath);
+        try {
+            if (!fs.existsSync(cacheDir)) {
+                fs.mkdirSync(cacheDir, { recursive: true });
+            }
+            fs.writeFileSync(cachePath, JSON.stringify(this.hashCache, null, 2));
+        }
+        catch (err) {
+            const error = err instanceof Error ? err : new Error(String(err));
+            logger.debug(`Failed to save hash cache: ${error.message}`);
+        }
+    }
+    /**
+     * Check if file content has changed by comparing hashes
+     */
+    hasContentChanged(filePath, content) {
+        const newHash = this.computeHash(content);
+        const oldHash = this.hashCache[filePath];
+        if (oldHash === newHash) {
+            return false;
+        }
+        this.hashCache[filePath] = newHash;
+        return true;
+    }
+    /**
+     * Remove file from hash cache
+     */
+    removeFromHashCache(filePath) {
+        const { [filePath]: _, ...rest } = this.hashCache;
+        this.hashCache = rest;
+    }
+    /**
+     * Create ignore filter from .gitignore
+     */
+    createIgnoreFilter() {
+        const ig = ignore();
+        const gitignorePath = path.join(this.directory, ".gitignore");
+        if (fs.existsSync(gitignorePath)) {
+            try {
+                const content = fs.readFileSync(gitignorePath, "utf-8");
+                ig.add(content);
+            }
+            catch {
+                // Ignore read errors
+            }
+        }
+        return ig;
+    }
+    /**
+     * Check if a file should be indexed
+     */
+    shouldIndex(filePath) {
+        const relativePath = path
+            .relative(this.directory, filePath)
+            .replace(/\\/g, "/");
+        // Skip hidden files/folders
+        if (relativePath.split("/").some((part) => part.startsWith("."))) {
+            return false;
+        }
+        // Skip gitignore patterns
+        if (this.ig.ignores(relativePath)) {
+            return false;
+        }
+        return shouldIndexFile(filePath);
+    }
+    /**
+     * Schedule a file change with debouncing
+     */
+    scheduleChange(type, filePath) {
+        const existing = this.pendingChanges.get(filePath);
+        if (existing) {
+            clearTimeout(existing.timer);
+        }
+        const timer = setTimeout(() => {
+            this.pendingChanges.delete(filePath);
+            this.queueOperation(async () => this.processChange(type, filePath));
+        }, this.debounceMs);
+        this.pendingChanges.set(filePath, { type, filePath, timer });
+        logger.debug(`Scheduled ${type}: ${path.basename(filePath)} (${String(this.debounceMs)}ms)`);
+    }
+    /**
+     * Process a file change after debounce
+     */
+    async processChange(type, filePath) {
+        if (type === "unlink") {
+            await this.removeFile(filePath);
+        }
+        else {
+            await this.indexFile(filePath);
+        }
+    }
+    /**
+     * Index a single file
+     */
+    async indexFile(filePath) {
+        if (!this.shouldIndex(filePath)) {
+            return;
+        }
+        try {
+            const content = fs.readFileSync(filePath, "utf-8");
+            // Skip if content unchanged
+            if (!this.hasContentChanged(filePath, content)) {
+                logger.debug(`Skipped (unchanged): ${path.basename(filePath)}`);
+                return;
+            }
+            const chunks = await chunkFile(filePath, content, this.config);
+            if (chunks.length === 0) {
+                return;
+            }
+            // Enrich chunks with semantic metadata
+            const enrichedChunks = await enrichChunksFromFile(chunks, content);
+            // Use enrichedContent for embedding
+            const texts = enrichedChunks.map((c) => c.enrichedContent);
+            const embeddings = await this.ollamaClient.embedBatch(texts);
+            // Store original chunk data (without enrichedContent)
+            const embeddedChunks = enrichedChunks.map((chunk, i) => ({
+                id: chunk.id,
+                content: chunk.content,
+                filePath: chunk.filePath,
+                language: chunk.language,
+                startLine: chunk.startLine,
+                endLine: chunk.endLine,
+                symbolName: chunk.symbolName,
+                symbolType: chunk.symbolType,
+                vector: embeddings[i] ?? [],
+            }));
+            await this.vectorStore.deleteByFilePath(filePath);
+            await this.vectorStore.addChunks(embeddedChunks);
+            this.saveHashCache();
+            logger.debug(`Indexed: ${path.relative(this.directory, filePath)}`);
+            this.onIndexed?.(filePath);
+        }
+        catch (err) {
+            const error = err instanceof Error ? err : new Error(String(err));
+            logger.error(`Failed to index ${filePath}: ${error.message}`);
+            this.onError?.(error);
+        }
+    }
+    /**
+     * Remove a file from the index
+     */
+    async removeFile(filePath) {
+        try {
+            await this.vectorStore.deleteByFilePath(filePath);
+            this.removeFromHashCache(filePath);
+            this.saveHashCache();
+            logger.debug(`Removed: ${path.relative(this.directory, filePath)}`);
+            this.onRemoved?.(filePath);
+        }
+        catch (err) {
+            const error = err instanceof Error ? err : new Error(String(err));
+            logger.error(`Failed to remove ${filePath}: ${error.message}`);
+            this.onError?.(error);
+        }
+    }
+    /**
+     * Queue an operation to prevent concurrent modifications
+     */
+    queueOperation(operation) {
+        this.operationQueue.push(operation);
+        void this.processQueue();
+    }
+    /**
+     * Process queued operations sequentially
+     */
+    async processQueue() {
+        if (this.isProcessing) {
+            return;
+        }
+        this.isProcessing = true;
+        while (this.operationQueue.length > 0) {
+            const operation = this.operationQueue.shift();
+            if (operation) {
+                try {
+                    await operation();
+                }
+                catch (err) {
+                    const error = err instanceof Error ? err : new Error(String(err));
+                    logger.error(`Operation failed: ${error.message}`);
+                }
+            }
+        }
+        this.isProcessing = false;
+    }
+    /**
+     * Collect files using fast-glob
+     */
+    async collectFilesWithGlob() {
+        const extensions = SUPPORTED_EXTENSIONS.map((ext) => ext.slice(1));
+        const pattern = `**/*.{${extensions.join(",")}}`;
+        const files = await fg(pattern, {
+            cwd: this.directory,
+            absolute: true,
+            ignore: ["**/.*", "**/.*/**"],
+            dot: false,
+            onlyFiles: true,
+            followSymbolicLinks: false,
+        });
+        // Filter by gitignore
+        return files.filter((file) => {
+            const relativePath = path
+                .relative(this.directory, file)
+                .replace(/\\/g, "/");
+            return !this.ig.ignores(relativePath);
+        });
+    }
+    /**
+     * Perform full initial indexing
+     */
+    async fullIndex() {
+        logger.info("Starting full index...");
+        const files = await this.collectFilesWithGlob();
+        let indexed = 0;
+        let skipped = 0;
+        for (const filePath of files) {
+            try {
+                const content = fs.readFileSync(filePath, "utf-8");
+                if (!this.hasContentChanged(filePath, content)) {
+                    skipped++;
+                    continue;
+                }
+                const chunks = await chunkFile(filePath, content, this.config);
+                if (chunks.length === 0) {
+                    continue;
+                }
+                // Enrich chunks with semantic metadata
+                const enrichedChunks = await enrichChunksFromFile(chunks, content);
+                // Use enrichedContent for embedding
+                const texts = enrichedChunks.map((c) => c.enrichedContent);
+                const embeddings = await this.ollamaClient.embedBatch(texts);
+                // Store original chunk data (without enrichedContent)
+                const embeddedChunks = enrichedChunks.map((chunk, i) => ({
+                    id: chunk.id,
+                    content: chunk.content,
+                    filePath: chunk.filePath,
+                    language: chunk.language,
+                    startLine: chunk.startLine,
+                    endLine: chunk.endLine,
+                    symbolName: chunk.symbolName,
+                    symbolType: chunk.symbolType,
+                    vector: embeddings[i] ?? [],
+                }));
+                await this.vectorStore.addChunks(embeddedChunks);
+                indexed++;
+            }
+            catch (err) {
+                const error = err instanceof Error ? err : new Error(String(err));
+                logger.debug(`Error indexing ${filePath}: ${error.message}`);
+            }
+        }
+        this.saveHashCache();
+        logger.info(`Full index: ${String(indexed)} indexed, ${String(skipped)} skipped`);
+    }
+    /**
+     * Start watching for file changes
+     */
+    async start() {
+        const health = await this.ollamaClient.healthCheck();
+        if (!health.ok) {
+            throw new Error(health.error ?? "Ollama is not available");
+        }
+        await this.vectorStore.connect();
+        if (!this.vectorStore.exists()) {
+            await this.fullIndex();
+        }
+        this.watcher = watch(this.directory, {
+            ignored: (filePath) => {
+                const relativePath = path
+                    .relative(this.directory, filePath)
+                    .replace(/\\/g, "/");
+                // Skip empty paths or root directory
+                if (!relativePath) {
+                    return false;
+                }
+                // Skip hidden files/folders
+                if (relativePath.split("/").some((part) => part.startsWith("."))) {
+                    return true;
+                }
+                return this.ig.ignores(relativePath);
+            },
+            persistent: true,
+            ignoreInitial: true,
+            awaitWriteFinish: {
+                stabilityThreshold: 500,
+                pollInterval: 100,
+            },
+        });
+        this.watcher.on("add", (filePath) => {
+            if (shouldIndexFile(filePath)) {
+                this.scheduleChange("add", filePath);
+            }
+        });
+        this.watcher.on("change", (filePath) => {
+            if (shouldIndexFile(filePath)) {
+                this.scheduleChange("change", filePath);
+            }
+        });
+        this.watcher.on("unlink", (filePath) => {
+            if (shouldIndexFile(filePath)) {
+                this.scheduleChange("unlink", filePath);
+            }
+        });
+        this.watcher.on("ready", () => {
+            logger.info(`Watching: ${this.directory} (${String(this.debounceMs)}ms debounce)`);
+            this.onReady?.();
+        });
+        this.watcher.on("error", (err) => {
+            const error = err instanceof Error ? err : new Error(String(err));
+            logger.error(`Watcher error: ${error.message}`);
+            this.onError?.(error);
+        });
+    }
+    /**
+     * Stop watching and cleanup
+     */
+    async stop() {
+        for (const pending of this.pendingChanges.values()) {
+            clearTimeout(pending.timer);
+        }
+        this.pendingChanges.clear();
+        this.saveHashCache();
+        if (this.watcher) {
+            await this.watcher.close();
+            this.watcher = null;
+        }
+        this.vectorStore.close();
+        logger.info("Watcher stopped");
+    }
+    /**
+     * Check if watcher is running
+     */
+    isRunning() {
+        return this.watcher !== null;
+    }
+    /**
+     * Clear the hash cache
+     */
+    clearCache() {
+        this.hashCache = {};
+        const cachePath = this.getHashCachePath();
+        if (fs.existsSync(cachePath)) {
+            fs.unlinkSync(cachePath);
+        }
+        logger.info("Hash cache cleared");
+    }
+    /**
+     * Get cache statistics
+     */
+    getCacheStats() {
+        return {
+            cachedFiles: Object.keys(this.hashCache).length,
+            cacheSize: JSON.stringify(this.hashCache).length,
+        };
+    }
+}
+/**
+ * Create a new index watcher
+ */
+export function createIndexWatcher(options) {
+    return new IndexWatcher(options);
+}
+//# sourceMappingURL=watcher.js.map

package/dist/core/embeddings/watcher.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"watcher.js","sourceRoot":"","sources":["../../../src/core/embeddings/watcher.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AAEH,OAAO,KAAK,EAAE,MAAM,SAAS,CAAC;AAC9B,OAAO,KAAK,IAAI,MAAM,WAAW,CAAC;AAClC,OAAO,KAAK,MAAM,MAAM,aAAa,CAAC;AACtC,OAAO,EAAE,KAAK,EAAkB,MAAM,UAAU,CAAC;AACjD,OAAO,EAAE,MAAM,WAAW,CAAC;AAC3B,OAAO,MAAuB,MAAM,QAAQ,CAAC;AAE7C,OAAO,EAAE,YAAY,EAAE,MAAM,yBAAyB,CAAC;AACvD,OAAO,EAAE,WAAW,EAAE,MAAM,wBAAwB,CAAC;AACrD,OAAO,EACL,SAAS,EACT,eAAe,EACf,oBAAoB,GACrB,MAAM,0BAA0B,CAAC;AAClC,OAAO,EAAE,oBAAoB,EAAE,MAAM,2BAA2B,CAAC;AACjE,OAAO,EAAE,MAAM,EAAE,MAAM,QAAQ,CAAC;AAEhC,6CAA6C;AAC7C,MAAM,mBAAmB,GAAG,IAAI,CAAC;AAEjC,yCAAyC;AACzC,MAAM,eAAe,GAAG,wBAAwB,CAAC;AAqBjD,MAAM,OAAO,YAAY;IACN,SAAS,CAAS;IAClB,MAAM,CAAkB;IACxB,UAAU,CAAS;IACnB,YAAY,CAAe;IAC3B,WAAW,CAAc;IAClC,OAAO,GAAqB,IAAI,CAAC;IACjC,EAAE,CAAS;IACX,YAAY,GAAG,KAAK,CAAC;IACrB,SAAS,GAAc,EAAE,CAAC;IAC1B,cAAc,GAAG,IAAI,GAAG,EAAyB,CAAC;IAClD,cAAc,GAA4B,EAAE,CAAC;IAEpC,OAAO,CAAc;IACrB,OAAO,CAA0B;IACjC,SAAS,CAA8B;IACvC,SAAS,CAA8B;IAExD,YAAY,OAAuB;QACjC,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC;QACjD,IAAI,CAAC,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC;QAC7B,IAAI,CAAC,UAAU,GAAG,OAAO,CAAC,UAAU,IAAI,mBAAmB,CAAC;QAC5D,IAAI,CAAC,YAAY,GAAG,IAAI,YAAY,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC;QACrD,IAAI,CAAC,WAAW,GAAG,IAAI,WAAW,CAAC,IAAI,CAAC,SAAS,EAAE,OAAO,CAAC,MAAM,CAAC,CAAC;QACnE,IAAI,CAAC,EAAE,GAAG,IAAI,CAAC,kBAAkB,EAAE,CAAC;QAEpC,IAAI,CAAC,OAAO,GAAG,OAAO,CAAC,OAAO,CAAC;QAC/B,IAAI,CAAC,OAAO,GAAG,OAAO,CAAC,OAAO,CAAC;QAC/B,IAAI,CAAC,SAAS,GAAG,OAAO,CAAC,SAAS,CAAC;QACnC,IAAI,CAAC,SAAS,GAAG,OAAO,CAAC,SAAS,CAAC;QAEnC,IAAI,CAAC,aAAa,EAAE,CAAC;IACvB,CAAC;IAED;;OAEG;IACK,WAAW,CAAC,OAAe;QACjC,OAAO,MAAM,CAAC,UAAU,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;IAC3E,CAAC;IAED;;OAEG;IACK,gBAAgB;QACtB,OAAO,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,YAAY,EAAE,eAAe,CAAC,CAAC;IAClE,CAAC;IAED;;OAEG;IACK,aAAa;QACnB,MAAM,SAAS,GAAG,IAAI,CAAC,gBAAgB,EAAE,CAAC;QAE1C,IAAI,EAAE,CAAC,UAAU,CAAC,SAAS,CAAC,EAAE,CAAC;YAC7B,IAAI,CAAC;gBACH,MAAM,OAAO,GAAG,EAAE,CAAC,YAAY,CAAC,SAAS,EAAE,OAAO,CAAC,CAAC;gBACpD,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC,KAAK,CAAC,OAAO,CAAc,CAAC;gBAClD,MAAM,CAAC,KAAK,CACV,UAAU,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,MAAM,CAAC,gBAAgB,CACrE,CAAC;YACJ,CAAC;YAAC,MAAM,CAAC;gBACP,IAAI,CAAC,SAAS,GAAG,EAAE,CAAC;YACtB,CAAC;QACH,CAAC;IACH,CAAC;IAED;;OAEG;IACK,aAAa;QACnB,MAAM,SAAS,GAAG,IAAI,CAAC,gBAAgB,EAAE,CAAC;QAC1C,MAAM,QAAQ,GAAG,IAAI,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC;QAEzC,IAAI,CAAC;YACH,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,QAAQ,CAAC,EAAE,CAAC;gBAC7B,EAAE,CAAC,SAAS,CAAC,QAAQ,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;YAC9C,CAAC;YACD,EAAE,CAAC,aAAa,CAAC,SAAS,EAAE,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC;QACvE,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,MAAM,KAAK,GAAG,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,KAAK,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC;YAClE,MAAM,CAAC,KAAK,CAAC,8BAA8B,KAAK,CAAC,OAAO,EAAE,CAAC,CAAC;QAC9D,CAAC;IACH,CAAC;IAED;;OAEG;IACK,iBAAiB,CAAC,QAAgB,EAAE,OAAe;QACzD,MAAM,OAAO,GAAG,IAAI,CAAC,WAAW,CAAC,OAAO,CAAC,CAAC;QAC1C,MAAM,OAAO,GAAG,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC,CAAC;QAEzC,IAAI,OAAO,KAAK,OAAO,EAAE,CAAC;YACxB,OAAO,KAAK,CAAC;QACf,CAAC;QAED,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC,GAAG,OAAO,CAAC;QACnC,OAAO,IAAI,CAAC;IACd,CAAC;IAED;;OAEG;IACK,mBAAmB,CAAC,QAAgB;QAC1C,MAAM,EAAE,CAAC,QAAQ,CAAC,EAAE,CAAC,EAAE,GAAG,IAAI,EAAE,GAAG,IAAI,CAAC,SAAS,CAAC;QAClD,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC;IACxB,CAAC;IAED;;OAEG;IACK,kBAAkB;QACxB,MAAM,EAAE,GAAG,MAAM,EAAE,CAAC;QACpB,MAAM,aAAa,GAAG,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,YAAY,CAAC,CAAC;QAE9D,IAAI,EAAE,CAAC,UAAU,CAAC,aAAa,CAAC,EAAE,CAAC;YACjC,IAAI,CAAC;gBACH,MAAM,OAAO,GAAG,EAAE,CAAC,YAAY,CAAC,aAAa,EAAE,OAAO,CAAC,CAAC;gBACxD,EAAE,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;YAClB,CAAC;YAAC,MAAM,CAAC;gBACP,qBAAqB;YACvB,CAAC;QACH,CAAC;QAED,OAAO,EAAE,CAAC;IACZ,CAAC;IAED;;OAEG;IACK,WAAW,CAAC,QAAgB;QAClC,MAAM,YAAY,GAAG,IAAI;aACtB,QAAQ,CAAC,IAAI,CAAC,SAAS,EAAE,QAAQ,CAAC;aAClC,OAAO,CAAC,KAAK,EAAE,GAAG,CAAC,CAAC;QAEvB,4BAA4B;QAC5B,IAAI,YAAY,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC;YACjE,OAAO,KAAK,CAAC;QACf,CAAC;QAED,0BAA0B;QAC1B,IAAI,IAAI,CAAC,EAAE,CAAC,OAAO,CAAC,YAAY,CAAC,EAAE,CAAC;YAClC,OAAO,KAAK,CAAC;QACf,CAAC;QAED,OAAO,eAAe,CAAC,QAAQ,CAAC,CAAC;IACnC,CAAC;IAED;;OAEG;IACK,cAAc,CACpB,IAAiC,EACjC,QAAgB;QAEhB,MAAM,QAAQ,GAAG,IAAI,CAAC,cAAc,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;QACnD,IAAI,QAAQ,EAAE,CAAC;YACb,YAAY,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC;QAC/B,CAAC;QAED,MAAM,KAAK,GAAG,UAAU,CAAC,GAAG,EAAE;YAC5B,IAAI,CAAC,cAAc,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC;YACrC,IAAI,CAAC,cAAc,CAAC,KAAK,IAAI,EAAE,CAAC,IAAI,CAAC,aAAa,CAAC,IAAI,EAAE,QAAQ,CAAC,CAAC,CAAC;QACtE,CAAC,EAAE,IAAI,CAAC,UAAU,CAAC,CAAC;QAEpB,IAAI,CAAC,cAAc,CAAC,GAAG,CAAC,QAAQ,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,KAAK,EAAE,CAAC,CAAC;QAE7D,MAAM,CAAC,KAAK,CACV,aAAa,IAAI,KAAK,IAAI,CAAC,QAAQ,CAAC,QAAQ,CAAC,KAAK,MAAM,CAAC,IAAI,CAAC,UAAU,CAAC,KAAK,CAC/E,CAAC;IACJ,CAAC;IAED;;OAEG;IACK,KAAK,CAAC,aAAa,CACzB,IAAiC,EACjC,QAAgB;QAEhB,IAAI,IAAI,KAAK,QAAQ,EAAE,CAAC;YACtB,MAAM,IAAI,CAAC,UAAU,CAAC,QAAQ,CAAC,CAAC;QAClC,CAAC;aAAM,CAAC;YACN,MAAM,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC,CAAC;QACjC,CAAC;IACH,CAAC;IAED;;OAEG;IACK,KAAK,CAAC,SAAS,CAAC,QAAgB;QACtC,IAAI,CAAC,IAAI,CAAC,WAAW,CAAC,QAAQ,CAAC,EAAE,CAAC;YAChC,OAAO;QACT,CAAC;QAED,IAAI,CAAC;YACH,MAAM,OAAO,GAAG,EAAE,CAAC,YAAY,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;YAEnD,4BAA4B;YAC5B,IAAI,CAAC,IAAI,CAAC,iBAAiB,CAAC,QAAQ,EAAE,OAAO,CAAC,EAAE,CAAC;gBAC/C,MAAM,CAAC,KAAK,CAAC,wBAAwB,IAAI,CAAC,QAAQ,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC;gBAChE,OAAO;YACT,CAAC;YAED,MAAM,MAAM,GAAG,MAAM,SAAS,CAAC,QAAQ,EAAE,OAAO,EAAE,IAAI,CAAC,MAAM,CAAC,CAAC;YAE/D,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;gBACxB,OAAO;YACT,CAAC;YAED,uCAAuC;YACvC,MAAM,cAAc,GAAG,MAAM,oBAAoB,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;YAEnE,oCAAoC;YACpC,MAAM,KAAK,GAAG,cAAc,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,eAAe,CAAC,CAAC;YAC3D,MAAM,UAAU,GAAG,MAAM,IAAI,CAAC,YAAY,CAAC,UAAU,CAAC,KAAK,CAAC,CAAC;YAE7D,sDAAsD;YACtD,MAAM,cAAc,GAAG,cAAc,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC;gBACvD,EAAE,EAAE,KAAK,CAAC,EAAE;gBACZ,OAAO,EAAE,KAAK,CAAC,OAAO;gBACtB,QAAQ,EAAE,KAAK,CAAC,QAAQ;gBACxB,QAAQ,EAAE,KAAK,CAAC,QAAQ;gBACxB,SAAS,EAAE,KAAK,CAAC,SAAS;gBAC1B,OAAO,EAAE,KAAK,CAAC,OAAO;gBACtB,UAAU,EAAE,KAAK,CAAC,UAAU;gBAC5B,UAAU,EAAE,KAAK,CAAC,UAAU;gBAC5B,MAAM,EAAE,UAAU,CAAC,CAAC,CAAC,IAAI,EAAE;aAC5B,CAAC,CAAC,CAAC;YAEJ,MAAM,IAAI,CAAC,WAAW,CAAC,gBAAgB,CAAC,QAAQ,CAAC,CAAC;YAClD,MAAM,IAAI,CAAC,WAAW,CAAC,SAAS,CAAC,cAAc,CAAC,CAAC;YAEjD,IAAI,CAAC,aAAa,EAAE,CAAC;YAErB,MAAM,CAAC,KAAK,CAAC,YAAY,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,SAAS,EAAE,QAAQ,CAAC,EAAE,CAAC,CAAC;YACpE,IAAI,CAAC,SAAS,EAAE,CAAC,QAAQ,CAAC,CAAC;QAC7B,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,MAAM,KAAK,GAAG,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,KAAK,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC;YAClE,MAAM,CAAC,KAAK,CAAC,mBAAmB,QAAQ,KAAK,KAAK,CAAC,OAAO,EAAE,CAAC,CAAC;YAC9D,IAAI,CAAC,OAAO,EAAE,CAAC,KAAK,CAAC,CAAC;QACxB,CAAC;IACH,CAAC;IAED;;OAEG;IACK,KAAK,CAAC,UAAU,CAAC,QAAgB;QACvC,IAAI,CAAC;YACH,MAAM,IAAI,CAAC,WAAW,CAAC,gBAAgB,CAAC,QAAQ,CAAC,CAAC;YAClD,IAAI,CAAC,mBAAmB,CAAC,QAAQ,CAAC,CAAC;YACnC,IAAI,CAAC,aAAa,EAAE,CAAC;YAErB,MAAM,CAAC,KAAK,CAAC,YAAY,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,SAAS,EAAE,QAAQ,CAAC,EAAE,CAAC,CAAC;YACpE,IAAI,CAAC,SAAS,EAAE,CAAC,QAAQ,CAAC,CAAC;QAC7B,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,MAAM,KAAK,GAAG,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,KAAK,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC;YAClE,MAAM,CAAC,KAAK,CAAC,oBAAoB,QAAQ,KAAK,KAAK,CAAC,OAAO,EAAE,CAAC,CAAC;YAC/D,IAAI,CAAC,OAAO,EAAE,CAAC,KAAK,CAAC,CAAC;QACxB,CAAC;IACH,CAAC;IAED;;OAEG;IACK,cAAc,CAAC,SAA8B;QACnD,IAAI,CAAC,cAAc,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;QACpC,KAAK,IAAI,CAAC,YAAY,EAAE,CAAC;IAC3B,CAAC;IAED;;OAEG;IACK,KAAK,CAAC,YAAY;QACxB,IAAI,IAAI,CAAC,YAAY,EAAE,CAAC;YACtB,OAAO;QACT,CAAC;QAED,IAAI,CAAC,YAAY,GAAG,IAAI,CAAC;QAEzB,OAAO,IAAI,CAAC,cAAc,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACtC,MAAM,SAAS,GAAG,IAAI,CAAC,cAAc,CAAC,KAAK,EAAE,CAAC;YAC9C,IAAI,SAAS,EAAE,CAAC;gBACd,IAAI,CAAC;oBACH,MAAM,SAAS,EAAE,CAAC;gBACpB,CAAC;gBAAC,OAAO,GAAG,EAAE,CAAC;oBACb,MAAM,KAAK,GAAG,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,KAAK,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC;oBAClE,MAAM,CAAC,KAAK,CAAC,qBAAqB,KAAK,CAAC,OAAO,EAAE,CAAC,CAAC;gBACrD,CAAC;YACH,CAAC;QACH,CAAC;QAED,IAAI,CAAC,YAAY,GAAG,KAAK,CAAC;IAC5B,CAAC;IAED;;OAEG;IACK,KAAK,CAAC,oBAAoB;QAChC,MAAM,UAAU,GAAG,oBAAoB,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC;QACnE,MAAM,OAAO,GAAG,SAAS,UAAU,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC;QAEjD,MAAM,KAAK,GAAG,MAAM,EAAE,CAAC,OAAO,EAAE;YAC9B,GAAG,EAAE,IAAI,CAAC,SAAS;YACnB,QAAQ,EAAE,IAAI;YACd,MAAM,EAAE,CAAC,OAAO,EAAE,UAAU,CAAC;YAC7B,GAAG,EAAE,KAAK;YACV,SAAS,EAAE,IAAI;YACf,mBAAmB,EAAE,KAAK;SAC3B,CAAC,CAAC;QAEH,sBAAsB;QACtB,OAAO,KAAK,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE;YAC3B,MAAM,YAAY,GAAG,IAAI;iBACtB,QAAQ,CAAC,IAAI,CAAC,SAAS,EAAE,IAAI,CAAC;iBAC9B,OAAO,CAAC,KAAK,EAAE,GAAG,CAAC,CAAC;YACvB,OAAO,CAAC,IAAI,CAAC,EAAE,CAAC,OAAO,CAAC,YAAY,CAAC,CAAC;QACxC,CAAC,CAAC,CAAC;IACL,CAAC;IAED;;OAEG;IACK,KAAK,CAAC,SAAS;QACrB,MAAM,CAAC,IAAI,CAAC,wBAAwB,CAAC,CAAC;QAEtC,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,oBAAoB,EAAE,CAAC;QAChD,IAAI,OAAO,GAAG,CAAC,CAAC;QAChB,IAAI,OAAO,GAAG,CAAC,CAAC;QAEhB,KAAK,MAAM,QAAQ,IAAI,KAAK,EAAE,CAAC;YAC7B,IAAI,CAAC;gBACH,MAAM,OAAO,GAAG,EAAE,CAAC,YAAY,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;gBAEnD,IAAI,CAAC,IAAI,CAAC,iBAAiB,CAAC,QAAQ,EAAE,OAAO,CAAC,EAAE,CAAC;oBAC/C,OAAO,EAAE,CAAC;oBACV,SAAS;gBACX,CAAC;gBAED,MAAM,MAAM,GAAG,MAAM,SAAS,CAAC,QAAQ,EAAE,OAAO,EAAE,IAAI,CAAC,MAAM,CAAC,CAAC;gBAE/D,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;oBACxB,SAAS;gBACX,CAAC;gBAED,uCAAuC;gBACvC,MAAM,cAAc,GAAG,MAAM,oBAAoB,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;gBAEnE,oCAAoC;gBACpC,MAAM,KAAK,GAAG,cAAc,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,eAAe,CAAC,CAAC;gBAC3D,MAAM,UAAU,GAAG,MAAM,IAAI,CAAC,YAAY,CAAC,UAAU,CAAC,KAAK,CAAC,CAAC;gBAE7D,sDAAsD;gBACtD,MAAM,cAAc,GAAG,cAAc,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC;oBACvD,EAAE,EAAE,KAAK,CAAC,EAAE;oBACZ,OAAO,EAAE,KAAK,CAAC,OAAO;oBACtB,QAAQ,EAAE,KAAK,CAAC,QAAQ;oBACxB,QAAQ,EAAE,KAAK,CAAC,QAAQ;oBACxB,SAAS,EAAE,KAAK,CAAC,SAAS;oBAC1B,OAAO,EAAE,KAAK,CAAC,OAAO;oBACtB,UAAU,EAAE,KAAK,CAAC,UAAU;oBAC5B,UAAU,EAAE,KAAK,CAAC,UAAU;oBAC5B,MAAM,EAAE,UAAU,CAAC,CAAC,CAAC,IAAI,EAAE;iBAC5B,CAAC,CAAC,CAAC;gBAEJ,MAAM,IAAI,CAAC,WAAW,CAAC,SAAS,CAAC,cAAc,CAAC,CAAC;gBACjD,OAAO,EAAE,CAAC;YACZ,CAAC;YAAC,OAAO,GAAG,EAAE,CAAC;gBACb,MAAM,KAAK,GAAG,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,KAAK,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC;gBAClE,MAAM,CAAC,KAAK,CAAC,kBAAkB,QAAQ,KAAK,KAAK,CAAC,OAAO,EAAE,CAAC,CAAC;YAC/D,CAAC;QACH,CAAC;QAED,IAAI,CAAC,aAAa,EAAE,CAAC;QAErB,MAAM,CAAC,IAAI,CACT,eAAe,MAAM,CAAC,OAAO,CAAC,aAAa,MAAM,CAAC,OAAO,CAAC,UAAU,CACrE,CAAC;IACJ,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,KAAK;QACT,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,YAAY,CAAC,WAAW,EAAE,CAAC;QACrD,IAAI,CAAC,MAAM,CAAC,EAAE,EAAE,CAAC;YACf,MAAM,IAAI,KAAK,CAAC,MAAM,CAAC,KAAK,IAAI,yBAAyB,CAAC,CAAC;QAC7D,CAAC;QAED,MAAM,IAAI,CAAC,WAAW,CAAC,OAAO,EAAE,CAAC;QAEjC,IAAI,CAAC,IAAI,CAAC,WAAW,CAAC,MAAM,EAAE,EAAE,CAAC;YAC/B,MAAM,IAAI,CAAC,SAAS,EAAE,CAAC;QACzB,CAAC;QAED,IAAI,CAAC,OAAO,GAAG,KAAK,CAAC,IAAI,CAAC,SAAS,EAAE;YACnC,OAAO,EAAE,CAAC,QAAgB,EAAE,EAAE;gBAC5B,MAAM,YAAY,GAAG,IAAI;qBACtB,QAAQ,CAAC,IAAI,CAAC,SAAS,EAAE,QAAQ,CAAC;qBAClC,OAAO,CAAC,KAAK,EAAE,GAAG,CAAC,CAAC;gBACvB,qCAAqC;gBACrC,IAAI,CAAC,YAAY,EAAE,CAAC;oBAClB,OAAO,KAAK,CAAC;gBACf,CAAC;gBACD,4BAA4B;gBAC5B,IAAI,YAAY,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC;oBACjE,OAAO,IAAI,CAAC;gBACd,CAAC;gBACD,OAAO,IAAI,CAAC,EAAE,CAAC,OAAO,CAAC,YAAY,CAAC,CAAC;YACvC,CAAC;YACD,UAAU,EAAE,IAAI;YAChB,aAAa,EAAE,IAAI;YACnB,gBAAgB,EAAE;gBAChB,kBAAkB,EAAE,GAAG;gBACvB,YAAY,EAAE,GAAG;aAClB;SACF,CAAC,CAAC;QAEH,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC,KAAK,EAAE,CAAC,QAAgB,EAAE,EAAE;YAC1C,IAAI,eAAe,CAAC,QAAQ,CAAC,EAAE,CAAC;gBAC9B,IAAI,CAAC,cAAc,CAAC,KAAK,EAAE,QAAQ,CAAC,CAAC;YACvC,CAAC;QACH,CAAC,CAAC,CAAC;QAEH,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAgB,EAAE,EAAE;YAC7C,IAAI,eAAe,CAAC,QAAQ,CAAC,EAAE,CAAC;gBAC9B,IAAI,CAAC,cAAc,CAAC,QAAQ,EAAE,QAAQ,CAAC,CAAC;YAC1C,CAAC;QACH,CAAC,CAAC,CAAC;QAEH,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAgB,EAAE,EAAE;YAC7C,IAAI,eAAe,CAAC,QAAQ,CAAC,EAAE,CAAC;gBAC9B,IAAI,CAAC,cAAc,CAAC,QAAQ,EAAE,QAAQ,CAAC,CAAC;YAC1C,CAAC;QACH,CAAC,CAAC,CAAC;QAEH,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC,OAAO,EAAE,GAAG,EAAE;YAC5B,MAAM,CAAC,IAAI,CACT,aAAa,IAAI,CAAC,SAAS,KAAK,MAAM,CAAC,IAAI,CAAC,UAAU,CAAC,cAAc,CACtE,CAAC;YACF,IAAI,CAAC,OAAO,EAAE,EAAE,CAAC;QACnB,CAAC,CAAC,CAAC;QAEH,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,GAAY,EAAE,EAAE;YACxC,MAAM,KAAK,GAAG,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,KAAK,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC;YAClE,MAAM,CAAC,KAAK,CAAC,kBAAkB,KAAK,CAAC,OAAO,EAAE,CAAC,CAAC;YAChD,IAAI,CAAC,OAAO,EAAE,CAAC,KAAK,CAAC,CAAC;QACxB,CAAC,CAAC,CAAC;IACL,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,IAAI;QACR,KAAK,MAAM,OAAO,IAAI,IAAI,CAAC,cAAc,CAAC,MAAM,EAAE,EAAE,CAAC;YACnD,YAAY,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;QAC9B,CAAC;QACD,IAAI,CAAC,cAAc,CAAC,KAAK,EAAE,CAAC;QAE5B,IAAI,CAAC,aAAa,EAAE,CAAC;QAErB,IAAI,IAAI,CAAC,OAAO,EAAE,CAAC;YACjB,MAAM,IAAI,CAAC,OAAO,CAAC,KAAK,EAAE,CAAC;YAC3B,IAAI,CAAC,OAAO,GAAG,IAAI,CAAC;QACtB,CAAC;QACD,IAAI,CAAC,WAAW,CAAC,KAAK,EAAE,CAAC;QACzB,MAAM,CAAC,IAAI,CAAC,iBAAiB,CAAC,CAAC;IACjC,CAAC;IAED;;OAEG;IACH,SAAS;QACP,OAAO,IAAI,CAAC,OAAO,KAAK,IAAI,CAAC;IAC/B,CAAC;IAED;;OAEG;IACH,UAAU;QACR,IAAI,CAAC,SAAS,GAAG,EAAE,CAAC;QACpB,MAAM,SAAS,GAAG,IAAI,CAAC,gBAAgB,EAAE,CAAC;QAC1C,IAAI,EAAE,CAAC,UAAU,CAAC,SAAS,CAAC,EAAE,CAAC;YAC7B,EAAE,CAAC,UAAU,CAAC,SAAS,CAAC,CAAC;QAC3B,CAAC;QACD,MAAM,CAAC,IAAI,CAAC,oBAAoB,CAAC,CAAC;IACpC,CAAC;IAED;;OAEG;IACH,aAAa;QACX,OAAO;YACL,WAAW,EAAE,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,MAAM;YAC/C,SAAS,EAAE,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,MAAM;SACjD,CAAC;IACJ,CAAC;CACF;AAED;;GAEG;AACH,MAAM,UAAU,kBAAkB,CAAC,OAAuB;IACxD,OAAO,IAAI,YAAY,CAAC,OAAO,CAAC,CAAC;AACnC,CAAC"}

package/dist/core/fallback/index.d.ts

@@ -0,0 +1,26 @@
+/**
+ * LangChain text splitter fallback for languages without Tree-sitter support
+ */
+import { type SupportedTextSplitterLanguage } from "@langchain/textsplitters";
+export interface TextChunk {
+    content: string;
+    startLine: number;
+    endLine: number;
+    index: number;
+}
+export interface TextSplitResult {
+    chunks: TextChunk[];
+    count: number;
+    language: string;
+    method: "text-splitter";
+}
+export interface TextSplitOptions {
+    chunkSize?: number;
+    chunkOverlap?: number;
+}
+export declare function isTextSplitterLanguage(language: string): boolean;
+export declare function getTextSplitterLanguage(language: string): SupportedTextSplitterLanguage | undefined;
+export declare function splitCode(content: string, language: string, options?: TextSplitOptions): Promise<TextSplitResult>;
+export declare function getSeparators(language: string): string[];
+export declare function clearConfigCache(): void;
+//# sourceMappingURL=index.d.ts.map

package/dist/core/fallback/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/core/fallback/index.ts"],"names":[],"mappings":"AAAA;;GAEG;AACH,OAAO,EAEL,KAAK,6BAA6B,EACnC,MAAM,0BAA0B,CAAC;AAgClC,MAAM,WAAW,SAAS;IACxB,OAAO,EAAE,MAAM,CAAC;IAChB,SAAS,EAAE,MAAM,CAAC;IAClB,OAAO,EAAE,MAAM,CAAC;IAChB,KAAK,EAAE,MAAM,CAAC;CACf;AAED,MAAM,WAAW,eAAe;IAC9B,MAAM,EAAE,SAAS,EAAE,CAAC;IACpB,KAAK,EAAE,MAAM,CAAC;IACd,QAAQ,EAAE,MAAM,CAAC;IACjB,MAAM,EAAE,eAAe,CAAC;CACzB;AAED,MAAM,WAAW,gBAAgB;IAC/B,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,YAAY,CAAC,EAAE,MAAM,CAAC;CACvB;AAED,wBAAgB,sBAAsB,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAOhE;AAED,wBAAgB,uBAAuB,CACrC,QAAQ,EAAE,MAAM,GACf,6BAA6B,GAAG,SAAS,CAU3C;AAmBD,wBAAsB,SAAS,CAC7B,OAAO,EAAE,MAAM,EACf,QAAQ,EAAE,MAAM,EAChB,OAAO,GAAE,gBAAqB,GAC7B,OAAO,CAAC,eAAe,CAAC,CA0B1B;AAED,wBAAgB,aAAa,CAAC,QAAQ,EAAE,MAAM,GAAG,MAAM,EAAE,CAKxD;AAED,wBAAgB,gBAAgB,IAAI,IAAI,CAEvC"}

package/dist/core/fallback/index.js

@@ -0,0 +1,76 @@
+/**
+ * LangChain text splitter fallback for languages without Tree-sitter support
+ */
+import { RecursiveCharacterTextSplitter, } from "@langchain/textsplitters";
+import { loadJsonConfig, registerCache } from "@core/utils";
+let langchainConfig = null;
+function loadConfig() {
+    if (langchainConfig) {
+        return langchainConfig;
+    }
+    const defaultConfig = {
+        langchain: { mapping: {}, supported: [], generic: [] },
+    };
+    const fullConfig = loadJsonConfig("languages.json", defaultConfig);
+    langchainConfig = fullConfig.langchain;
+    return langchainConfig;
+}
+export function isTextSplitterLanguage(language) {
+    const config = loadConfig();
+    return (language in config.mapping ||
+        config.supported.includes(language) ||
+        config.generic.includes(language));
+}
+export function getTextSplitterLanguage(language) {
+    const config = loadConfig();
+    if (language in config.mapping) {
+        return config.mapping[language];
+    }
+    if (config.supported.includes(language)) {
+        return language;
+    }
+    return undefined;
+}
+function calculateLineNumbers(fullContent, chunkContent, startSearchIndex) {
+    const chunkIndex = fullContent.indexOf(chunkContent, startSearchIndex);
+    const actualIndex = chunkIndex >= 0 ? chunkIndex : startSearchIndex;
+    const beforeChunk = fullContent.slice(0, actualIndex);
+    const startLine = (beforeChunk.match(/\n/g) ?? []).length + 1;
+    const chunkLines = (chunkContent.match(/\n/g) ?? []).length;
+    return {
+        startLine,
+        endLine: startLine + chunkLines,
+        foundIndex: actualIndex + chunkContent.length,
+    };
+}
+export async function splitCode(content, language, options = {}) {
+    const { chunkSize = 1000, chunkOverlap = 200 } = options;
+    const splitterLanguage = getTextSplitterLanguage(language);
+    const splitter = splitterLanguage
+        ? RecursiveCharacterTextSplitter.fromLanguage(splitterLanguage, {
+            chunkSize,
+            chunkOverlap,
+        })
+        : new RecursiveCharacterTextSplitter({ chunkSize, chunkOverlap });
+    const docs = await splitter.createDocuments([content]);
+    const chunks = [];
+    let searchIndex = 0;
+    for (const [i, doc] of docs.entries()) {
+        const { startLine, endLine, foundIndex } = calculateLineNumbers(content, doc.pageContent, searchIndex);
+        searchIndex = foundIndex;
+        chunks.push({ content: doc.pageContent, startLine, endLine, index: i });
+    }
+    return { chunks, count: chunks.length, language, method: "text-splitter" };
+}
+export function getSeparators(language) {
+    const splitterLanguage = getTextSplitterLanguage(language);
+    return splitterLanguage
+        ? RecursiveCharacterTextSplitter.getSeparatorsForLanguage(splitterLanguage)
+        : [];
+}
+export function clearConfigCache() {
+    langchainConfig = null;
+}
+// Register cache for centralized clearing
+registerCache("fallback:config", clearConfigCache);
+//# sourceMappingURL=index.js.map

package/dist/core/fallback/index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../src/core/fallback/index.ts"],"names":[],"mappings":"AAAA;;GAEG;AACH,OAAO,EACL,8BAA8B,GAE/B,MAAM,0BAA0B,CAAC;AAElC,OAAO,EAAE,cAAc,EAAE,aAAa,EAAE,MAAM,aAAa,CAAC;AAY5D,IAAI,eAAe,GAA2B,IAAI,CAAC;AAEnD,SAAS,UAAU;IACjB,IAAI,eAAe,EAAE,CAAC;QACpB,OAAO,eAAe,CAAC;IACzB,CAAC;IAED,MAAM,aAAa,GAAe;QAChC,SAAS,EAAE,EAAE,OAAO,EAAE,EAAE,EAAE,SAAS,EAAE,EAAE,EAAE,OAAO,EAAE,EAAE,EAAE;KACvD,CAAC;IACF,MAAM,UAAU,GAAG,cAAc,CAC/B,gBAAgB,EAChB,aAAa,CACd,CAAC;IACF,eAAe,GAAG,UAAU,CAAC,SAAS,CAAC;IACvC,OAAO,eAAe,CAAC;AACzB,CAAC;AAqBD,MAAM,UAAU,sBAAsB,CAAC,QAAgB;IACrD,MAAM,MAAM,GAAG,UAAU,EAAE,CAAC;IAC5B,OAAO,CACL,QAAQ,IAAI,MAAM,CAAC,OAAO;QAC1B,MAAM,CAAC,SAAS,CAAC,QAAQ,CAAC,QAAQ,CAAC;QACnC,MAAM,CAAC,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAClC,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,uBAAuB,CACrC,QAAgB;IAEhB,MAAM,MAAM,GAAG,UAAU,EAAE,CAAC;IAE5B,IAAI,QAAQ,IAAI,MAAM,CAAC,OAAO,EAAE,CAAC;QAC/B,OAAO,MAAM,CAAC,OAAO,CAAC,QAAQ,CAAkC,CAAC;IACnE,CAAC;IACD,IAAI,MAAM,CAAC,SAAS,CAAC,QAAQ,CAAC,QAAQ,CAAC,EAAE,CAAC;QACxC,OAAO,QAAyC,CAAC;IACnD,CAAC;IACD,OAAO,SAAS,CAAC;AACnB,CAAC;AAED,SAAS,oBAAoB,CAC3B,WAAmB,EACnB,YAAoB,EACpB,gBAAwB;IAExB,MAAM,UAAU,GAAG,WAAW,CAAC,OAAO,CAAC,YAAY,EAAE,gBAAgB,CAAC,CAAC;IACvE,MAAM,WAAW,GAAG,UAAU,IAAI,CAAC,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,gBAAgB,CAAC;IACpE,MAAM,WAAW,GAAG,WAAW,CAAC,KAAK,CAAC,CAAC,EAAE,WAAW,CAAC,CAAC;IACtD,MAAM,SAAS,GAAG,CAAC,WAAW,CAAC,KAAK,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC;IAC9D,MAAM,UAAU,GAAG,CAAC,YAAY,CAAC,KAAK,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC,CAAC,MAAM,CAAC;IAC5D,OAAO;QACL,SAAS;QACT,OAAO,EAAE,SAAS,GAAG,UAAU;QAC/B,UAAU,EAAE,WAAW,GAAG,YAAY,CAAC,MAAM;KAC9C,CAAC;AACJ,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,SAAS,CAC7B,OAAe,EACf,QAAgB,EAChB,UAA4B,EAAE;IAE9B,MAAM,EAAE,SAAS,GAAG,IAAI,EAAE,YAAY,GAAG,GAAG,EAAE,GAAG,OAAO,CAAC;IACzD,MAAM,gBAAgB,GAAG,uBAAuB,CAAC,QAAQ,CAAC,CAAC;IAE3D,MAAM,QAAQ,GAAG,gBAAgB;QAC/B,CAAC,CAAC,8BAA8B,CAAC,YAAY,CAAC,gBAAgB,EAAE;YAC5D,SAAS;YACT,YAAY;SACb,CAAC;QACJ,CAAC,CAAC,IAAI,8BAA8B,CAAC,EAAE,SAAS,EAAE,YAAY,EAAE,CAAC,CAAC;IAEpE,MAAM,IAAI,GAAG,MAAM,QAAQ,CAAC,eAAe,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC;IACvD,MAAM,MAAM,GAAgB,EAAE,CAAC;IAC/B,IAAI,WAAW,GAAG,CAAC,CAAC;IAEpB,KAAK,MAAM,CAAC,CAAC,EAAE,GAAG,CAAC,IAAI,IAAI,CAAC,OAAO,EAAE,EAAE,CAAC;QACtC,MAAM,EAAE,SAAS,EAAE,OAAO,EAAE,UAAU,EAAE,GAAG,oBAAoB,CAC7D,OAAO,EACP,GAAG,CAAC,WAAW,EACf,WAAW,CACZ,CAAC;QACF,WAAW,GAAG,UAAU,CAAC;QACzB,MAAM,CAAC,IAAI,CAAC,EAAE,OAAO,EAAE,GAAG,CAAC,WAAW,EAAE,SAAS,EAAE,OAAO,EAAE,KAAK,EAAE,CAAC,EAAE,CAAC,CAAC;IAC1E,CAAC;IAED,OAAO,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,CAAC,MAAM,EAAE,QAAQ,EAAE,MAAM,EAAE,eAAe,EAAE,CAAC;AAC7E,CAAC;AAED,MAAM,UAAU,aAAa,CAAC,QAAgB;IAC5C,MAAM,gBAAgB,GAAG,uBAAuB,CAAC,QAAQ,CAAC,CAAC;IAC3D,OAAO,gBAAgB;QACrB,CAAC,CAAC,8BAA8B,CAAC,wBAAwB,CAAC,gBAAgB,CAAC;QAC3E,CAAC,CAAC,EAAE,CAAC;AACT,CAAC;AAED,MAAM,UAAU,gBAAgB;IAC9B,eAAe,GAAG,IAAI,CAAC;AACzB,CAAC;AAED,0CAA0C;AAC1C,aAAa,CAAC,iBAAiB,EAAE,gBAAgB,CAAC,CAAC"}

package/dist/core/parser/index.d.ts

@@ -0,0 +1,64 @@
+import { Language, type Node, Parser, type Point, type Tree } from "web-tree-sitter";
+import type { ASTNode, Position } from "@core/ast/types";
+export * from "./languages";
+export type { Language, Node, Point, Tree };
+/**
+ * Initialize the Tree-sitter WASM module
+ * Must be called before any parsing operations
+ */
+export declare function initializeParser(): Promise<void>;
+/**
+ * Check if the parser is initialized
+ */
+export declare function isParserInitialized(): boolean;
+/**
+ * Parse result
+ */
+export interface ParseResult {
+    /** The Tree-sitter tree */
+    tree: Tree;
+    /** Language that was used */
+    language: string;
+    /** The parser instance (for queries) */
+    parser: Parser;
+    /** The language instance (for queries) */
+    languageInstance: Language;
+}
+/**
+ * Parse options
+ */
+export interface ParseOptions {
+    /** Language name (auto-detected from file path if not provided) */
+    language?: string;
+    /** File path (for language detection) */
+    filePath?: string;
+}
+/**
+ * Parse code content
+ */
+export declare function parseCode(content: string, options?: ParseOptions): Promise<ParseResult>;
+/**
+ * Convert Tree-sitter position to our Position type
+ */
+export declare function toPosition(point: Point, offset: number): Position;
+/**
+ * Convert Tree-sitter node to ASTNode
+ */
+export declare function toASTNode(node: Node, maxDepth?: number, currentDepth?: number): ASTNode;
+/**
+ * Get the root ASTNode from a parse result
+ */
+export declare function getASTRoot(parseResult: ParseResult, maxDepth?: number): ASTNode;
+/**
+ * Count nodes in the tree
+ */
+export declare function countNodes(node: Node): number;
+/**
+ * Clear the language cache (useful for testing)
+ */
+export declare function clearLanguageCache(): void;
+/**
+ * Reset the parser state (useful for testing)
+ */
+export declare function resetParser(): void;
+//# sourceMappingURL=index.d.ts.map

package/dist/core/parser/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/core/parser/index.ts"],"names":[],"mappings":"AASA,OAAO,EACL,QAAQ,EACR,KAAK,IAAI,EACT,MAAM,EACN,KAAK,KAAK,EACV,KAAK,IAAI,EACV,MAAM,iBAAiB,CAAC;AAEzB,OAAO,KAAK,EAAE,OAAO,EAAE,QAAQ,EAAE,MAAM,iBAAiB,CAAC;AAUzD,cAAc,aAAa,CAAC;AAG5B,YAAY,EAAE,QAAQ,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC;AAkB5C;;;GAGG;AACH,wBAAsB,gBAAgB,IAAI,OAAO,CAAC,IAAI,CAAC,CAiBtD;AAED;;GAEG;AACH,wBAAgB,mBAAmB,IAAI,OAAO,CAE7C;AA2CD;;GAEG;AACH,MAAM,WAAW,WAAW;IAC1B,2BAA2B;IAC3B,IAAI,EAAE,IAAI,CAAC;IACX,6BAA6B;IAC7B,QAAQ,EAAE,MAAM,CAAC;IACjB,wCAAwC;IACxC,MAAM,EAAE,MAAM,CAAC;IACf,0CAA0C;IAC1C,gBAAgB,EAAE,QAAQ,CAAC;CAC5B;AAED;;GAEG;AACH,MAAM,WAAW,YAAY;IAC3B,mEAAmE;IACnE,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,yCAAyC;IACzC,QAAQ,CAAC,EAAE,MAAM,CAAC;CACnB;AAED;;GAEG;AACH,wBAAsB,SAAS,CAC7B,OAAO,EAAE,MAAM,EACf,OAAO,GAAE,YAAiB,GACzB,OAAO,CAAC,WAAW,CAAC,CAwCtB;AAED;;GAEG;AACH,wBAAgB,UAAU,CAAC,KAAK,EAAE,KAAK,EAAE,MAAM,EAAE,MAAM,GAAG,QAAQ,CAMjE;AAED;;GAEG;AACH,wBAAgB,SAAS,CACvB,IAAI,EAAE,IAAI,EACV,QAAQ,CAAC,EAAE,MAAM,EACjB,YAAY,SAAI,GACf,OAAO,CA2CT;AAED;;GAEG;AACH,wBAAgB,UAAU,CACxB,WAAW,EAAE,WAAW,EACxB,QAAQ,CAAC,EAAE,MAAM,GAChB,OAAO,CAET;AAED;;GAEG;AACH,wBAAgB,UAAU,CAAC,IAAI,EAAE,IAAI,GAAG,MAAM,CAM7C;AAED;;GAEG;AACH,wBAAgB,kBAAkB,IAAI,IAAI,CAEzC;AAED;;GAEG;AACH,wBAAgB,WAAW,IAAI,IAAI,CAKlC"}