toolnet-memory 0.2.0

package/dist/src/retrieval/vector/vector-store.js

@@ -0,0 +1,89 @@
+export class VectorStore {
+    records = new Map();
+    upsert(record) {
+        this.records.set(record.id, record);
+    }
+    get(id) {
+        return this.records.get(id);
+    }
+    has(id) {
+        return this.records.has(id);
+    }
+    remove(id) {
+        return this.records.delete(id);
+    }
+    clearProject(projectId) {
+        let deleted = 0;
+        for (const [id, record] of this.records) {
+            if (record.projectId ===
+                projectId) {
+                this.records.delete(id);
+                deleted++;
+            }
+        }
+        return deleted;
+    }
+    exportProject(projectId) {
+        return [...this.records.values()]
+            .filter((record) => record.projectId ===
+            projectId);
+    }
+    importRecords(records) {
+        let imported = 0;
+        for (const record of records) {
+            if (!record?.id ||
+                !record?.projectId ||
+                !Array.isArray(record.vector)) {
+                continue;
+            }
+            this.records.set(record.id, record);
+            imported++;
+        }
+        return imported;
+    }
+    search(projectId, queryVector, limit = 10) {
+        const results = [];
+        for (const record of this.records.values()) {
+            if (record.projectId !==
+                projectId) {
+                continue;
+            }
+            const score = cosineSimilarity(queryVector, record.vector);
+            results.push({
+                id: record.id,
+                score,
+                metadata: record.metadata,
+            });
+        }
+        return results
+            .sort((a, b) => b.score - a.score)
+            .slice(0, limit);
+    }
+    size() {
+        return this.records.size;
+    }
+}
+export function cosineSimilarity(a, b) {
+    if (a.length === 0 ||
+        b.length === 0 ||
+        a.length !== b.length) {
+        return 0;
+    }
+    let dot = 0;
+    let normA = 0;
+    let normB = 0;
+    for (let i = 0; i < a.length; i++) {
+        const av = a[i] ?? 0;
+        const bv = b[i] ?? 0;
+        dot += av * bv;
+        normA += av * av;
+        normB += bv * bv;
+    }
+    if (normA === 0 ||
+        normB === 0) {
+        return 0;
+    }
+    return (dot /
+        (Math.sqrt(normA) *
+            Math.sqrt(normB)));
+}

package/dist/src/retrieval/vector-hybrid-engine.js

@@ -0,0 +1,91 @@
+import { HybridSearch, } from "./hybrid-search.js";
+import { ContextBuilder, } from "./context-builder.js";
+import { VectorIndexer, } from "./vector/upsert.js";
+import { VectorSearch, } from "./vector/search.js";
+import { VectorStore, } from "./vector/vector-store.js";
+export class VectorHybridEngine {
+    memory;
+    lexical = new HybridSearch();
+    contextBuilder = new ContextBuilder();
+    store;
+    indexer;
+    vectorSearch;
+    constructor(memory, embeddings, store = new VectorStore()) {
+        this.memory = memory;
+        this.store =
+            store;
+        this.indexer =
+            new VectorIndexer(embeddings, this.store);
+        this.vectorSearch =
+            new VectorSearch(embeddings, this.store);
+    }
+    async rebuild(projectId) {
+        this.store.clearProject(projectId);
+        return this.indexer.index(this.memory.list(projectId));
+    }
+    async search(projectId, query, topK = 8) {
+        const memories = this.memory.list(projectId);
+        const byId = new Map(memories.map((memory) => [
+            memory.id,
+            memory,
+        ]));
+        const lexicalResults = this.lexical.search(query, memories, {
+            topK: Math.max(topK * 3, 20),
+        });
+        const vectorResults = await this.vectorSearch.search(projectId, query, Math.max(topK * 3, 20));
+        const merged = new Map();
+        for (const result of lexicalResults) {
+            merged.set(result.memory.id, {
+                lexicalScore: result.score,
+                vectorScore: 0,
+            });
+        }
+        for (const result of vectorResults) {
+            const current = merged.get(result.id) ?? {
+                lexicalScore: 0,
+                vectorScore: 0,
+            };
+            current.vectorScore =
+                Math.max(0, result.score);
+            merged.set(result.id, current);
+        }
+        const results = [];
+        for (const [id, scores] of merged) {
+            const memory = byId.get(id);
+            if (!memory) {
+                continue;
+            }
+            /*
+             * Hybrid:
+             * lexical/relevance 55%
+             * vector semantic   45%
+             */
+            const score = scores.lexicalScore *
+                0.55 +
+                scores.vectorScore *
+                    0.45;
+            results.push({
+                memory,
+                score,
+                lexicalScore: scores.lexicalScore,
+                vectorScore: scores.vectorScore,
+            });
+        }
+        return results
+            .sort((a, b) => b.score - a.score)
+            .slice(0, topK);
+    }
+    async context(projectId, query, topK = 5) {
+        const results = await this.search(projectId, query, topK);
+        return this.contextBuilder.build(results.map((result) => ({
+            memory: result.memory,
+            score: result.score,
+            scores: {
+                keyword: result.lexicalScore,
+                recency: 0,
+                importance: 0,
+                type: result.vectorScore,
+            },
+        })));
+    }
+}

package/dist/src/runtime/bootstrap.js

@@ -0,0 +1,28 @@
+import "dotenv/config";
+import { loadConfig, ProjectManager, } from "../core/index.js";
+import { createStorageProvider, withStorageRetry, ProjectScopedStorageProvider, } from "../storage/index.js";
+import { ToolNetMemoryRuntime, } from "./toolnet-memory-runtime.js";
+export function createToolNetMemoryRuntime(rootPath = process.cwd()) {
+    const config = loadConfig();
+    const project = new ProjectManager()
+        .detect(rootPath);
+    const rawStorage = createStorageProvider({
+        provider: config.storage.provider,
+        huggingface: config.storage.huggingface,
+        localRoot: config.storage.localRoot,
+    });
+    const retryStorage = withStorageRetry(rawStorage, {
+        attempts: Number(process.env
+            .TOOLNET_STORAGE_RETRIES ??
+            3),
+    });
+    const storage = new ProjectScopedStorageProvider(retryStorage, project.id, project.name, project.remote ?? project.name);
+    const embeddingModel = process.env
+        .HF_EMBEDDING_MODEL ??
+        "sentence-transformers/all-MiniLM-L6-v2";
+    return new ToolNetMemoryRuntime({
+        project,
+        storage,
+        embeddingModel,
+    });
+}

package/dist/src/runtime/graceful.js

@@ -0,0 +1,17 @@
+export function registerGracefulShutdown(runtime) {
+    let stopping = false;
+    const stop = async () => {
+        if (stopping) {
+            return;
+        }
+        stopping = true;
+        try {
+            await runtime.stop();
+        }
+        finally {
+            process.exit(0);
+        }
+    };
+    process.once("SIGINT", stop);
+    process.once("SIGTERM", stop);
+}

package/dist/src/runtime/index.js

@@ -0,0 +1,3 @@
+export * from "./toolnet-memory-runtime.js";
+export * from "./bootstrap.js";
+export * from "./graceful.js";

package/dist/src/runtime/toolnet-memory-runtime.js

@@ -0,0 +1,156 @@
+import { MemoryEngine, } from "../core/memory-engine.js";
+import { MemoryMaintenance, } from "../memory/maintenance.js";
+import { MemoryConsolidator, } from "../memory/consolidator.js";
+import { RetrievalEngine, } from "../retrieval/retrieval-engine.js";
+import { createEmbeddingProvider, } from "../embeddings/index.js";
+import { CodeGraphStore, ReferenceResolver, } from "../code-intelligence/index.js";
+import { IncrementalRepositoryIndexer, } from "../code-intelligence/incremental/incremental-indexer.js";
+import { ImpactGuard, } from "../code-intelligence/impact/impact-guard.js";
+import { VectorStore, VectorPersistenceManager, } from "../retrieval/vector/index.js";
+import { HookRuntime, AutoContextBuilder, AutoRetrieval, AutoImpactGuard, } from "../hooks/index.js";
+import { MemoryStore, PersistentCodeGraphStore, PersistentVectorStore, } from "../storage/index.js";
+import { reconcileSessionMemoryJournal, } from "../session/learner/index.js";
+import { SnapshotManager, } from "../snapshot/index.js";
+import { ProjectLock, } from "../production/project-lock.js";
+export class ToolNetMemoryRuntime {
+    options;
+    memory = new MemoryEngine();
+    graph = new CodeGraphStore();
+    vectors = new VectorStore();
+    retrieval;
+    references;
+    hooks;
+    autoRetrieval;
+    impactGuard;
+    autoImpactGuard;
+    memoryStore;
+    graphStore;
+    vectorStore;
+    processLock;
+    constructor(options) {
+        this.options = options;
+        this.memoryStore =
+            new MemoryStore(options.storage);
+        this.graphStore =
+            new PersistentCodeGraphStore(options.storage);
+        this.vectorStore =
+            new PersistentVectorStore(options.storage);
+        this.processLock =
+            new ProjectLock(options.project.id);
+        this.retrieval =
+            new RetrievalEngine(this.memory);
+        this.references =
+            new ReferenceResolver(this.graph);
+        this.hooks =
+            new HookRuntime({
+                projectId: options.project.id,
+                memory: this.memory,
+                memoryStore: this.memoryStore,
+                maxEventsBeforeFlush: 100,
+            });
+        const contextBuilder = new AutoContextBuilder(this.memory, this.retrieval, this.graph);
+        this.autoRetrieval =
+            new AutoRetrieval(this.hooks, contextBuilder, options.project.id);
+        this.impactGuard =
+            new ImpactGuard(this.graph);
+        this.autoImpactGuard =
+            new AutoImpactGuard(this.impactGuard, options.project.id);
+    }
+    async start() {
+        const { project, } = this.options;
+        await this.processLock
+            .acquire();
+        /*
+         * Session learners write immutable claims only.
+         * Reconcile them into current memory before runtime load.
+         */
+        await reconcileSessionMemoryJournal(project, this.options.storage);
+        // Load persistent memory.
+        const memories = await this.memoryStore.load(project.id);
+        this.memory.importRecords(memories);
+        const maintenance = new MemoryMaintenance(this.memory);
+        const maintenanceStats = maintenance.run(project.id);
+        if (maintenanceStats.expiredRemoved > 0 ||
+            maintenanceStats.supersededRemoved > 0) {
+            await this.memoryStore.save(project.id, this.memory.exportProject(project.id));
+        }
+        // Load code graph.
+        const graphSnapshot = await this.graphStore.load(project.id);
+        if (graphSnapshot) {
+            this.graph.import(graphSnapshot.symbols, graphSnapshot.edges);
+        }
+        // Load / update vectors.
+        const embeddings = createEmbeddingProvider();
+        const vectorManager = new VectorPersistenceManager(project.id, this.options.embeddingModel, embeddings, this.vectors, this.vectorStore);
+        const vectorStats = await vectorManager.initialize(this.memory.list(project.id));
+        await this.hooks
+            .sessionStart();
+        return {
+            memories: memories.length,
+            graphSymbols: this.graph
+                .allSymbols(project.id)
+                .length,
+            graphEdges: this.graph
+                .allEdges(project.id)
+                .length,
+            vectors: vectorStats.total,
+            vectorsIndexed: vectorStats.indexed,
+        };
+    }
+    async preparePrompt(prompt) {
+        return this.autoRetrieval
+            .prepare(prompt);
+    }
+    /*
+     * PHẢI gọi trước khi tool thật sự chạy.
+     * Edit/write/patch sẽ tự chạy Blast Radius.
+     */
+    async beforeTool(tool, input) {
+        const impact = await this.autoImpactGuard
+            .beforeTool(tool, input);
+        await this.hooks
+            .beforeTool(tool, input);
+        return impact;
+    }
+    async afterTool(tool, output) {
+        return this.hooks
+            .afterTool(tool, output);
+    }
+    async stop() {
+        const { project, storage, embeddingModel, } = this.options;
+        // Snapshot persistent state before changing it.
+        const snapshotManager = new SnapshotManager(storage);
+        const snapshot = await snapshotManager.create(project.id, "session-end-auto");
+        await snapshotManager.prune(project.id, 10);
+        // Flush pending memory events.
+        await this.hooks
+            .sessionEnd();
+        // Consolidate small activity memories.
+        const consolidation = new MemoryConsolidator(this.memory).consolidate(project.id);
+        if (consolidation.summaryCreated) {
+            await this.memoryStore.save(project.id, this.memory.exportProject(project.id));
+        }
+        // Incrementally update code graph.
+        const codeStats = await new IncrementalRepositoryIndexer(storage).index(project.id, project.rootPath);
+        // Reload latest graph snapshot.
+        const graphSnapshot = await this.graphStore.load(project.id);
+        if (graphSnapshot) {
+            this.graph.clearProject(project.id);
+            this.graph.import(graphSnapshot.symbols, graphSnapshot.edges);
+        }
+        // Persist only missing/new memory vectors.
+        const embeddings = createEmbeddingProvider();
+        const vectorManager = new VectorPersistenceManager(project.id, embeddingModel, embeddings, this.vectors, this.vectorStore);
+        const vectorStats = await vectorManager.initialize(this.memory.list(project.id));
+        const result = {
+            code: codeStats,
+            vectors: vectorStats,
+            consolidation,
+            snapshot: snapshot?.id,
+            memories: this.memory.list(project.id).length,
+        };
+        await this.processLock
+            .release();
+        return result;
+    }
+}

package/dist/src/security/encryption.js

@@ -0,0 +1 @@
+export {};

package/dist/src/security/file-filter.js

@@ -0,0 +1,30 @@
+import { basename, } from "node:path";
+const BLOCKED_NAMES = new Set([
+    ".env",
+    ".env.local",
+    ".env.production",
+    ".env.development",
+    "id_rsa",
+    "id_ed25519",
+    "credentials",
+    "credentials.json",
+    "service-account.json",
+]);
+const BLOCKED_EXTENSIONS = new Set([
+    ".pem",
+    ".key",
+    ".p12",
+    ".pfx",
+]);
+export function isSensitiveFile(filePath) {
+    const name = basename(filePath).toLowerCase();
+    if (BLOCKED_NAMES.has(name)) {
+        return true;
+    }
+    for (const extension of BLOCKED_EXTENSIONS) {
+        if (name.endsWith(extension)) {
+            return true;
+        }
+    }
+    return false;
+}

package/dist/src/security/index.js

@@ -0,0 +1,3 @@
+export * from "./secret-scanner.js";
+export * from "./sanitizer.js";
+export * from "./file-filter.js";

package/dist/src/security/sanitizer.js

@@ -0,0 +1,49 @@
+import { SecretScanner } from "./secret-scanner.js";
+export class Sanitizer {
+    scanner = new SecretScanner();
+    sanitize(text) {
+        let output = text;
+        const matches = this.scanner.scan(text);
+        const secretTypes = new Set();
+        for (const match of matches) {
+            secretTypes.add(match.type);
+            output = output
+                .split(match.value)
+                .join(`[REDACTED:${match.type}]`);
+        }
+        return {
+            text: output,
+            redacted: matches.length,
+            secretTypes: [...secretTypes],
+        };
+    }
+    sanitizeValue(value) {
+        if (typeof value === "string") {
+            return this.sanitize(value).text;
+        }
+        if (Array.isArray(value)) {
+            return value.map((item) => this.sanitizeValue(item));
+        }
+        if (value &&
+            typeof value === "object") {
+            const output = {};
+            for (const [key, item] of Object.entries(value)) {
+                const normalized = key.toLowerCase();
+                if (normalized.includes("password") ||
+                    normalized.includes("secret") ||
+                    normalized.includes("token") ||
+                    normalized.includes("cookie") ||
+                    normalized.includes("authorization")) {
+                    output[key] =
+                        "[REDACTED]";
+                }
+                else {
+                    output[key] =
+                        this.sanitizeValue(item);
+                }
+            }
+            return output;
+        }
+        return value;
+    }
+}

package/dist/src/security/secret-scanner.js

@@ -0,0 +1,56 @@
+const PATTERNS = [
+    {
+        type: "openai_key",
+        regex: /\bsk-[A-Za-z0-9_\-]{20,}\b/g,
+    },
+    {
+        type: "huggingface_token",
+        regex: /\bhf_[A-Za-z0-9]{20,}\b/g,
+    },
+    {
+        type: "hf_s3_access_key",
+        regex: /\bHFAK[A-Za-z0-9]{8,}\b/g,
+    },
+    {
+        type: "bearer_token",
+        regex: /\bBearer\s+[A-Za-z0-9._\-]{16,}\b/gi,
+    },
+    {
+        type: "jwt",
+        regex: /\beyJ[A-Za-z0-9_\-]{8,}\.[A-Za-z0-9_\-]{8,}\.[A-Za-z0-9_\-]{8,}\b/g,
+    },
+    {
+        type: "private_key",
+        regex: /-----BEGIN (?:RSA |EC |OPENSSH )?PRIVATE KEY-----[\s\S]*?-----END (?:RSA |EC |OPENSSH )?PRIVATE KEY-----/g,
+    },
+    {
+        type: "password_assignment",
+        regex: /\b(password|passwd|pwd)\s*[:=]\s*["']?[^"'\s]{6,}["']?/gi,
+    },
+    {
+        type: "secret_assignment",
+        regex: /\b(secret|api[_-]?key|access[_-]?token|refresh[_-]?token)\s*[:=]\s*["']?[^"'\s]{8,}["']?/gi,
+    },
+    {
+        type: "cookie",
+        regex: /\b(cookie|set-cookie)\s*[:=]\s*[^;\n]{8,}/gi,
+    },
+];
+export class SecretScanner {
+    scan(text) {
+        const matches = [];
+        for (const pattern of PATTERNS) {
+            const regex = new RegExp(pattern.regex.source, pattern.regex.flags);
+            for (const match of text.matchAll(regex)) {
+                matches.push({
+                    type: pattern.type,
+                    value: match[0],
+                });
+            }
+        }
+        return matches;
+    }
+    hasSecrets(text) {
+        return this.scan(text).length > 0;
+    }
+}

package/dist/src/session/agy/adapter.js

@@ -0,0 +1,159 @@
+import { existsSync, } from "node:fs";
+import { homedir, } from "node:os";
+import { join, resolve, } from "node:path";
+import { SessionCore, } from "../core.js";
+import { readAgyTranscript, } from "./transcript.js";
+function expandHome(value) {
+    if (value === "~") {
+        return homedir();
+    }
+    if (value.startsWith("~/")) {
+        return join(homedir(), value.slice(2));
+    }
+    return value;
+}
+export async function syncAgySession(options) {
+    const conversationId = options
+        .conversationId
+        .trim();
+    if (!conversationId) {
+        throw new Error("Agy conversationId is required");
+    }
+    const transcriptPath = resolve(expandHome(options
+        .transcriptPath));
+    const core = new SessionCore({
+        project: options.project,
+        storage: options.storage,
+        agent: "agy",
+        nativeSessionId: conversationId,
+        metadata: {
+            source: "antigravity-hook",
+            transcriptPath,
+            workspacePaths: options
+                .workspacePaths ??
+                [],
+            artifactDirectoryPath: options
+                .artifactDirectoryPath,
+            modelName: options.modelName,
+        },
+    });
+    const state = core.status();
+    const rawCursor = state
+        .sourceCursors["agy.transcript.offset"];
+    const offset = rawCursor
+        ? Number(rawCursor)
+        : 0;
+    const transcript = readAgyTranscript(transcriptPath, {
+        offset: Number.isFinite(offset)
+            ? offset
+            : 0,
+    });
+    const events = [];
+    if (state.lastSequence ===
+        0) {
+        events.push({
+            type: "session_start",
+            sourceEventId: `agy:${conversationId}:start`,
+            data: {
+                workspacePaths: options
+                    .workspacePaths ??
+                    [],
+                transcriptPath,
+                modelName: options.modelName,
+            },
+            provenance: {
+                source: "antigravity-hook",
+                sourcePath: transcriptPath,
+            },
+        });
+    }
+    if (transcript.reset) {
+        events.push({
+            type: "custom",
+            sourceEventId: `agy:${conversationId}:transcript-reset:${transcript.nextOffset}`,
+            data: {
+                event: "transcript_reset",
+                transcriptPath,
+            },
+            provenance: {
+                source: "agy-transcript",
+                sourcePath: transcriptPath,
+            },
+        });
+    }
+    events.push(...transcript.events);
+    if (options.phase ===
+        "stop") {
+        if (options.error) {
+            events.push({
+                type: "error",
+                sourceEventId: [
+                    "agy",
+                    conversationId,
+                    "stop-error",
+                    options
+                        .terminationReason ??
+                        "",
+                    transcript
+                        .nextOffset,
+                ].join(":"),
+                data: {
+                    error: options.error,
+                    terminationReason: options
+                        .terminationReason,
+                    fullyIdle: options
+                        .fullyIdle,
+                },
+                provenance: {
+                    source: "antigravity-stop-hook",
+                },
+            });
+        }
+        /*
+         * Stop is NOT permanent SessionEnd.
+         * The same conversation UUID may be resumed later.
+         */
+        if (options.fullyIdle) {
+            events.push({
+                type: "session_idle",
+                sourceEventId: [
+                    "agy",
+                    conversationId,
+                    "idle",
+                    transcript
+                        .nextOffset,
+                ].join(":"),
+                data: {
+                    terminationReason: options
+                        .terminationReason,
+                    fullyIdle: true,
+                },
+                provenance: {
+                    source: "antigravity-stop-hook",
+                },
+            });
+        }
+    }
+    const recorded = core.recordMany(events);
+    core.setSourceCursor("agy.transcript.offset", transcript.nextOffset);
+    if (options.phase) {
+        core.setSourceCursor("agy.last.phase", options.phase);
+    }
+    const flushed = await core.flush();
+    return {
+        conversationId,
+        imported: recorded.length,
+        eventCount: flushed.eventCount,
+        chunkCount: flushed.chunkCount,
+        status: flushed.status,
+        transcriptOffset: transcript
+            .nextOffset,
+        reset: transcript.reset,
+    };
+}
+export function defaultAgyTranscript(conversationId) {
+    return join(homedir(), ".gemini", "antigravity-cli", "brain", conversationId, ".system_generated", "logs", "transcript.jsonl");
+}
+export function agyTranscriptExists(conversationId) {
+    return existsSync(defaultAgyTranscript(conversationId));
+}