toolnet-memory 0.2.1 → 0.2.3

package/dist/src/code-intelligence/rich/rich-graph-enricher.js

@@ -1,463 +0,0 @@
-import { createHash, } from "node:crypto";
-import { dirname, relative, resolve, } from "node:path";
-import ts from "typescript";
-function normalize(value) {
-    return value
-        .replaceAll("\\", "/")
-        .replace(/^\.\//, "");
-}
-function id(...parts) {
-    return createHash("sha256")
-        .update(parts.join(":"))
-        .digest("hex")
-        .slice(0, 24);
-}
-function lineOf(source, node) {
-    return source
-        .getLineAndCharacterOfPosition(node.getStart(source)).line + 1;
-}
-const ROUTE_METHODS = new Set([
-    "get",
-    "post",
-    "put",
-    "patch",
-    "delete",
-    "options",
-    "head",
-]);
-const ASSIGNMENTS = new Set([
-    ts.SyntaxKind.EqualsToken,
-    ts.SyntaxKind.PlusEqualsToken,
-    ts.SyntaxKind.MinusEqualsToken,
-    ts.SyntaxKind.AsteriskEqualsToken,
-    ts.SyntaxKind.SlashEqualsToken,
-    ts.SyntaxKind.PercentEqualsToken,
-    ts.SyntaxKind.QuestionQuestionEqualsToken,
-    ts.SyntaxKind.AmpersandAmpersandEqualsToken,
-    ts.SyntaxKind.BarBarEqualsToken,
-]);
-export class RichGraphEnricher {
-    graph;
-    constructor(graph) {
-        this.graph = graph;
-    }
-    enrich(projectId, rootPath, resolution) {
-        const stats = {
-            routes: 0,
-            tests: 0,
-            usesType: 0,
-            writes: 0,
-            callReferences: 0,
-            properties: 0,
-        };
-        const config = this.loadConfig(rootPath);
-        const program = ts.createProgram({
-            rootNames: config.fileNames,
-            options: config.options,
-        });
-        const checker = program.getTypeChecker();
-        for (const source of program.getSourceFiles()) {
-            if (source.isDeclarationFile) {
-                continue;
-            }
-            const filePath = normalize(relative(rootPath, resolve(source.fileName)));
-            if (filePath.startsWith("../") ||
-                filePath.includes("node_modules/")) {
-                continue;
-            }
-            const fileNode = this.fileNode(projectId, filePath);
-            if (!fileNode) {
-                continue;
-            }
-            const visit = (node) => {
-                if (ts.isCallExpression(node)) {
-                    if (this.addRoute(projectId, rootPath, source, fileNode, node, checker)) {
-                        stats.routes++;
-                    }
-                }
-                if (ts.isTypeReferenceNode(node)) {
-                    if (this.addTypeUse(projectId, rootPath, source, node, checker)) {
-                        stats.usesType++;
-                    }
-                }
-                if (ts.isBinaryExpression(node) &&
-                    ASSIGNMENTS.has(node.operatorToken.kind)) {
-                    const result = this.addWrite(projectId, rootPath, source, node.left, checker);
-                    if (result.added) {
-                        stats.writes++;
-                        if (result.propertyCreated) {
-                            stats.properties++;
-                        }
-                    }
-                }
-                if ((ts.isPrefixUnaryExpression(node) ||
-                    ts.isPostfixUnaryExpression(node)) &&
-                    (node.operator ===
-                        ts.SyntaxKind.PlusPlusToken ||
-                        node.operator ===
-                            ts.SyntaxKind.MinusMinusToken)) {
-                    const result = this.addWrite(projectId, rootPath, source, node.operand, checker);
-                    if (result.added) {
-                        stats.writes++;
-                        if (result.propertyCreated) {
-                            stats.properties++;
-                        }
-                    }
-                }
-                ts.forEachChild(node, visit);
-            };
-            visit(source);
-        }
-        stats.tests +=
-            this.addTestEdges(projectId);
-        if (resolution) {
-            stats.callReferences +=
-                this.addResolvedCalls(projectId, resolution);
-        }
-        return stats;
-    }
-    addRoute(projectId, rootPath, source, fileNode, node, checker) {
-        if (!ts.isPropertyAccessExpression(node.expression)) {
-            return false;
-        }
-        const method = node.expression
-            .name.text
-            .toLowerCase();
-        if (!ROUTE_METHODS.has(method)) {
-            return false;
-        }
-        const first = node.arguments[0];
-        if (!first ||
-            !(ts.isStringLiteral(first) ||
-                ts.isNoSubstitutionTemplateLiteral(first))) {
-            return false;
-        }
-        const path = first.text;
-        if (!path.startsWith("/")) {
-            return false;
-        }
-        const sourceLine = lineOf(source, node);
-        const routeId = id(projectId, source.fileName, sourceLine, method, path);
-        if (!this.graph.getSymbol(routeId)) {
-            this.graph.addSymbol({
-                id: routeId,
-                projectId,
-                name: `${method.toUpperCase()} ${path}`,
-                qualifiedName: `${method.toUpperCase()} ${path}`,
-                type: "route",
-                filePath: fileNode.filePath,
-                startLine: sourceLine,
-                endLine: sourceLine,
-            });
-            this.graph.addEdge(this.edge(projectId, fileNode.id, "DEFINES", routeId));
-        }
-        const handler = node.arguments.at(-1);
-        if (handler) {
-            const target = this.resolveNodeSymbol(projectId, rootPath, handler, checker);
-            if (target) {
-                this.graph.addEdge(this.edge(projectId, routeId, "ROUTE", target.id, {
-                    method: method.toUpperCase(),
-                    path,
-                }));
-            }
-        }
-        return true;
-    }
-    addTypeUse(projectId, rootPath, source, node, checker) {
-        const target = this.resolveNodeSymbol(projectId, rootPath, node.typeName, checker);
-        if (!target) {
-            return false;
-        }
-        const sourceSymbol = this.containingSymbol(projectId, normalize(relative(rootPath, resolve(source.fileName))), lineOf(source, node));
-        if (!sourceSymbol ||
-            sourceSymbol.id ===
-                target.id) {
-            return false;
-        }
-        this.graph.addEdge(this.edge(projectId, sourceSymbol.id, "USES_TYPE", target.id));
-        return true;
-    }
-    addWrite(projectId, rootPath, source, targetNode, checker) {
-        const symbol = this.resolveTsSymbol(targetNode, checker);
-        const declaration = symbol
-            ?.valueDeclaration ??
-            symbol
-                ?.declarations?.[0];
-        if (!declaration) {
-            return {
-                added: false,
-                propertyCreated: false,
-            };
-        }
-        const declarationSource = declaration
-            .getSourceFile();
-        if (declarationSource
-            .isDeclarationFile) {
-            return {
-                added: false,
-                propertyCreated: false,
-            };
-        }
-        const targetFile = normalize(relative(rootPath, resolve(declarationSource
-            .fileName)));
-        if (targetFile.startsWith("../") ||
-            targetFile.includes("node_modules/")) {
-            return {
-                added: false,
-                propertyCreated: false,
-            };
-        }
-        const targetLine = lineOf(declarationSource, declaration);
-        let target = this.findSymbolAt(projectId, targetFile, targetLine);
-        let propertyCreated = false;
-        if (!target ||
-            target.type === "file") {
-            const name = symbol?.getName() ??
-                targetNode.getText(source);
-            const propertyId = id(projectId, targetFile, "property", name, targetLine);
-            target =
-                this.graph.getSymbol(propertyId);
-            if (!target) {
-                target = {
-                    id: propertyId,
-                    projectId,
-                    name,
-                    qualifiedName: name,
-                    type: "property",
-                    filePath: targetFile,
-                    startLine: targetLine,
-                    endLine: targetLine,
-                };
-                this.graph.addSymbol(target);
-                const targetFileNode = this.fileNode(projectId, targetFile);
-                if (targetFileNode) {
-                    this.graph.addEdge(this.edge(projectId, targetFileNode.id, "DEFINES", target.id));
-                }
-                propertyCreated =
-                    true;
-            }
-        }
-        if (!target) {
-            return {
-                added: false,
-                propertyCreated,
-            };
-        }
-        const sourceFile = normalize(relative(rootPath, resolve(source.fileName)));
-        const writer = this.containingSymbol(projectId, sourceFile, lineOf(source, targetNode));
-        if (!writer ||
-            writer.id === target.id) {
-            return {
-                added: false,
-                propertyCreated,
-            };
-        }
-        this.graph.addEdge(this.edge(projectId, writer.id, "WRITES", target.id, {
-            expression: targetNode.getText(source),
-        }));
-        return {
-            added: true,
-            propertyCreated,
-        };
-    }
-    addTestEdges(projectId) {
-        const edges = this.graph.allEdges(projectId);
-        let added = 0;
-        for (const edge of edges) {
-            if (edge.type !==
-                "IMPORTS") {
-                continue;
-            }
-            const from = this.graph.getSymbol(edge.from);
-            const to = this.graph.getSymbol(edge.to);
-            if (!from ||
-                !to ||
-                !this.isTestFile(from.filePath) ||
-                this.isTestFile(to.filePath)) {
-                continue;
-            }
-            this.graph.addEdge(this.edge(projectId, from.id, "TESTS", to.id));
-            added++;
-        }
-        return added;
-    }
-    addResolvedCalls(projectId, resolution) {
-        let added = 0;
-        for (const item of resolution.resolutions) {
-            if (item.kind !== "CALL" ||
-                !item.targetSymbolId) {
-                continue;
-            }
-            const target = this.graph.getSymbol(item.targetSymbolId);
-            if (!target) {
-                continue;
-            }
-            const source = this.containingSymbol(projectId, item.sourceFile, item.sourceLine);
-            if (!source ||
-                source.id === target.id) {
-                continue;
-            }
-            /*
-             * Nếu CALLS hiện tại đã đúng target
-             * thì không tạo edge trùng.
-             */
-            const exactAlreadyExists = this.graph
-                .allEdges(projectId)
-                .some((edge) => edge.type ===
-                "CALLS" &&
-                edge.from ===
-                    source.id &&
-                edge.to ===
-                    target.id);
-            if (exactAlreadyExists) {
-                continue;
-            }
-            this.graph.addEdge(this.edge(projectId, source.id, "CALL_REFERENCE", target.id, {
-                expression: item.expression,
-                confidence: item.confidence,
-                resolver: item.resolver,
-            }));
-            added++;
-        }
-        return added;
-    }
-    resolveNodeSymbol(projectId, rootPath, node, checker) {
-        const symbol = this.resolveTsSymbol(node, checker);
-        const declaration = symbol
-            ?.valueDeclaration ??
-            symbol
-                ?.declarations?.[0];
-        if (!declaration) {
-            return undefined;
-        }
-        const targetSource = declaration
-            .getSourceFile();
-        if (targetSource
-            .isDeclarationFile) {
-            return undefined;
-        }
-        const filePath = normalize(relative(rootPath, resolve(targetSource
-            .fileName)));
-        if (filePath.startsWith("../") ||
-            filePath.includes("node_modules/")) {
-            return undefined;
-        }
-        return this.findSymbolAt(projectId, filePath, lineOf(targetSource, declaration), symbol?.getName());
-    }
-    resolveTsSymbol(node, checker) {
-        let symbol = checker.getSymbolAtLocation(node);
-        if (symbol &&
-            (symbol.flags &
-                ts.SymbolFlags.Alias)) {
-            try {
-                symbol =
-                    checker.getAliasedSymbol(symbol);
-            }
-            catch {
-                // keep original
-            }
-        }
-        return symbol;
-    }
-    containingSymbol(projectId, filePath, line) {
-        return this.graph
-            .allSymbols(projectId)
-            .filter((symbol) => normalize(symbol.filePath) ===
-            normalize(filePath) &&
-            typeof symbol.startLine ===
-                "number" &&
-            typeof symbol.endLine ===
-                "number" &&
-            symbol.startLine <= line &&
-            symbol.endLine >= line)
-            .sort((a, b) => ((a.endLine ?? 0) -
-            (a.startLine ?? 0)) -
-            ((b.endLine ?? 0) -
-                (b.startLine ?? 0)))[0];
-    }
-    findSymbolAt(projectId, filePath, line, name) {
-        const candidates = this.graph
-            .allSymbols(projectId)
-            .filter((symbol) => normalize(symbol.filePath) ===
-            normalize(filePath) &&
-            typeof symbol.startLine ===
-                "number" &&
-            typeof symbol.endLine ===
-                "number" &&
-            symbol.startLine <= line &&
-            symbol.endLine >= line)
-            .sort((a, b) => ((a.endLine ?? 0) -
-            (a.startLine ?? 0)) -
-            ((b.endLine ?? 0) -
-                (b.startLine ?? 0)));
-        if (name) {
-            const named = candidates.find((symbol) => symbol.name === name ||
-                (symbol.qualifiedName
-                    ?.endsWith(`.${name}`) ?? false));
-            if (named) {
-                return named;
-            }
-        }
-        return candidates[0];
-    }
-    fileNode(projectId, filePath) {
-        return this.graph
-            .allSymbols(projectId)
-            .find((symbol) => symbol.type ===
-            "file" &&
-            normalize(symbol.filePath) ===
-                normalize(filePath));
-    }
-    isTestFile(filePath) {
-        const path = normalize(filePath).toLowerCase();
-        return (path.startsWith("tests/") ||
-            path.includes("/tests/") ||
-            path.includes("__tests__") ||
-            /\.(test|spec)\.[^.]+$/
-                .test(path));
-    }
-    edge(projectId, from, type, to, metadata) {
-        return {
-            id: id(projectId, from, type, to, JSON.stringify(metadata ?? {})),
-            projectId,
-            from,
-            to,
-            type,
-            metadata,
-        };
-    }
-    loadConfig(rootPath) {
-        const configPath = ts.findConfigFile(rootPath, ts.sys.fileExists, "tsconfig.json");
-        if (configPath) {
-            const raw = ts.readConfigFile(configPath, ts.sys.readFile);
-            return ts.parseJsonConfigFileContent(raw.config, ts.sys, dirname(configPath));
-        }
-        return {
-            fileNames: ts.sys.readDirectory(rootPath, [
-                ".ts",
-                ".tsx",
-                ".js",
-                ".jsx",
-                ".mts",
-                ".cts",
-                ".mjs",
-                ".cjs",
-            ], [
-                "node_modules",
-                ".git",
-                "dist",
-                "build",
-                "coverage",
-            ]),
-            options: {
-                allowJs: true,
-                noEmit: true,
-                skipLibCheck: true,
-                target: ts.ScriptTarget.ES2022,
-                module: ts.ModuleKind.NodeNext,
-                moduleResolution: ts.ModuleResolutionKind
-                    .NodeNext,
-            },
-        };
-    }
-}

package/dist/src/code-intelligence/semantic/index.js

@@ -1,2 +0,0 @@
-export * from "./semantic-code-engine.js";
-export * from "./source-priority.js";

package/dist/src/code-intelligence/semantic/semantic-code-engine.js

@@ -1,213 +0,0 @@
-import { PersistentCodeChunkStore, } from "../../storage/code-chunk-store.js";
-import { PersistentCodeVectorStore, } from "../../storage/code-vector-store.js";
-import { SmartCodeChunker, } from "../chunks/smart-chunker.js";
-import { VectorStore, } from "../../retrieval/vector/vector-store.js";
-import { tokenize, } from "../../retrieval/tokenizer.js";
-import { codeSourceWeight, } from "./source-priority.js";
-export class SemanticCodeEngine {
-    options;
-    vectors = new VectorStore();
-    chunks = new Map();
-    constructor(options) {
-        this.options = options;
-    }
-    async initialize() {
-        const { projectId, rootPath, storage, embeddings, graph, model, } = this.options;
-        const chunkStore = new PersistentCodeChunkStore(storage);
-        const vectorStore = new PersistentCodeVectorStore(storage);
-        const chunks = await new SmartCodeChunker()
-            .build(projectId, rootPath, graph.allSymbols(projectId));
-        this.chunks.clear();
-        for (const chunk of chunks) {
-            this.chunks.set(chunk.id, chunk);
-        }
-        const previousChunks = await chunkStore.load(projectId);
-        const chunksChanged = !previousChunks ||
-            previousChunks.chunks.length !==
-                chunks.length ||
-            previousChunks.chunks.some((old) => !this.chunks.has(old.id));
-        if (chunksChanged) {
-            await chunkStore.save({
-                version: 1,
-                projectId,
-                updatedAt: new Date()
-                    .toISOString(),
-                chunks,
-            });
-        }
-        const previousVectors = await vectorStore.load(projectId);
-        let vectorsLoaded = 0;
-        if (previousVectors &&
-            previousVectors.model ===
-                model) {
-            vectorsLoaded =
-                this.vectors.importRecords(previousVectors.records);
-        }
-        const validIds = new Set(chunks.map((chunk) => chunk.id));
-        let vectorsRemoved = 0;
-        for (const record of this.vectors
-            .exportProject(projectId)) {
-            if (!validIds.has(record.id)) {
-                if (this.vectors.remove(record.id)) {
-                    vectorsRemoved++;
-                }
-            }
-        }
-        const pending = chunks.filter((chunk) => {
-            const existing = this.vectors.get(chunk.id);
-            if (!existing) {
-                return true;
-            }
-            return (existing.metadata
-                ?.contentHash !==
-                chunk.contentHash);
-        });
-        let vectorsIndexed = 0;
-        /*
-         * Batch nhỏ để tránh request embedding quá lớn.
-         */
-        const batchSize = 8;
-        let targetDimensions = this.vectors
-            .exportProject(projectId)[0]
-            ?.vector.length;
-        for (let offset = 0; offset < pending.length; offset += batchSize) {
-            const batch = pending.slice(offset, offset +
-                batchSize);
-            const texts = batch.map((chunk) => [
-                `file: ${chunk.filePath}`,
-                chunk.symbolName
-                    ? `symbol: ${chunk.symbolName}`
-                    : "",
-                chunk.symbolType
-                    ? `type: ${chunk.symbolType}`
-                    : "",
-                chunk.content,
-            ]
-                .filter(Boolean)
-                .join("\n"));
-            const vectors = await embeddings
-                .embedMany(texts);
-            for (let i = 0; i < batch.length; i++) {
-                const chunk = batch[i];
-                const vector = vectors[i];
-                if (!chunk ||
-                    !vector ||
-                    vector.length === 0) {
-                    continue;
-                }
-                if (targetDimensions ===
-                    undefined) {
-                    targetDimensions =
-                        vector.length;
-                }
-                /*
-                 * Không cho trộn vector 384d/128d
-                 * nếu provider fallback giữa chừng.
-                 */
-                if (vector.length !==
-                    targetDimensions) {
-                    console.warn(`[code-semantic] skip dimension mismatch: ${vector.length} != ${targetDimensions}`);
-                    continue;
-                }
-                this.vectors.upsert({
-                    id: chunk.id,
-                    projectId,
-                    vector,
-                    metadata: {
-                        contentHash: chunk.contentHash,
-                        filePath: chunk.filePath,
-                        symbolName: chunk.symbolName,
-                        symbolType: chunk.symbolType,
-                        startLine: chunk.startLine,
-                        endLine: chunk.endLine,
-                    },
-                });
-                vectorsIndexed++;
-            }
-        }
-        const records = this.vectors
-            .exportProject(projectId);
-        const shouldSaveVectors = vectorsIndexed > 0 ||
-            vectorsRemoved > 0 ||
-            !previousVectors ||
-            previousVectors.model !==
-                model;
-        if (shouldSaveVectors) {
-            await vectorStore.save({
-                version: 1,
-                projectId,
-                model,
-                dimensions: records[0]
-                    ?.vector.length ??
-                    targetDimensions ??
-                    0,
-                updatedAt: new Date()
-                    .toISOString(),
-                records,
-            });
-        }
-        return {
-            chunks: chunks.length,
-            vectorsLoaded,
-            vectorsIndexed,
-            vectorsRemoved,
-            totalVectors: records.length,
-            savedChunks: chunksChanged,
-            savedVectors: shouldSaveVectors,
-        };
-    }
-    async search(query, limit = 8) {
-        const queryVector = await this.options
-            .embeddings
-            .embed(query);
-        const vectorResults = this.vectors.search(this.options.projectId, queryVector, Math.max(limit * 4, 20));
-        const queryTokens = new Set(tokenize(query));
-        const output = [];
-        for (const result of vectorResults) {
-            const chunk = this.chunks.get(result.id);
-            if (!chunk) {
-                continue;
-            }
-            const documentTokens = new Set(tokenize([
-                chunk.filePath,
-                chunk.symbolName ??
-                    "",
-                chunk.symbolType ??
-                    "",
-                chunk.content,
-            ].join(" ")));
-            let matches = 0;
-            for (const token of queryTokens) {
-                if (documentTokens.has(token)) {
-                    matches++;
-                }
-            }
-            const lexicalScore = queryTokens.size === 0
-                ? 0
-                : matches /
-                    queryTokens.size;
-            const vectorScore = Math.max(0, result.score);
-            /*
-             * Semantic quan trọng hơn lexical.
-             */
-            const sourceWeight = codeSourceWeight(chunk.filePath, query);
-            const score = (vectorScore *
-                0.75 +
-                lexicalScore *
-                    0.25) *
-                sourceWeight;
-            output.push({
-                chunk,
-                score,
-                vectorScore,
-                lexicalScore,
-            });
-        }
-        return output
-            .sort((a, b) => b.score - a.score)
-            .slice(0, limit);
-    }
-    size() {
-        return this.chunks.size;
-    }
-}

package/dist/src/code-intelligence/semantic/source-priority.js

@@ -1,22 +0,0 @@
-export function codeSourceWeight(filePath, query) {
-    const path = filePath
-        .toLowerCase()
-        .replaceAll("\\", "/");
-    const q = query.toLowerCase();
-    const testIntent = /\b(test|tests|spec|fixture|mock)\b/
-        .test(q);
-    const isTest = path.startsWith("tests/") ||
-        path.includes("/tests/") ||
-        path.includes("__tests__") ||
-        /\.(test|spec)\.[^.]+$/
-            .test(path);
-    if (isTest &&
-        !testIntent) {
-        return 0.35;
-    }
-    if (path.includes("/generated/") ||
-        path.includes(".generated.")) {
-        return 0.25;
-    }
-    return 1;
-}

package/dist/src/code-intelligence/symbols/index.js

@@ -1 +0,0 @@
-export {};