toolnet-memory 0.2.0

package/dist/src/production/config-check.js

@@ -0,0 +1,34 @@
+export function checkProductionConfig() {
+    const errors = [];
+    const warnings = [];
+    const provider = process.env
+        .MEMORY_STORAGE_PROVIDER ??
+        "local";
+    if (provider ===
+        "huggingface") {
+        const required = [
+            "HF_NAMESPACE",
+            "HF_BUCKET",
+            "HF_S3_ACCESS_KEY_ID",
+            "HF_S3_SECRET_ACCESS_KEY",
+        ];
+        for (const key of required) {
+            if (!process.env[key]) {
+                errors.push(`Missing ${key}`);
+            }
+        }
+        if (!process.env.HF_TOKEN) {
+            warnings.push("HF_TOKEN missing: semantic embedding will use fallback");
+        }
+    }
+    const model = process.env
+        .HF_EMBEDDING_MODEL;
+    if (!model) {
+        warnings.push("HF_EMBEDDING_MODEL not set; default model will be used");
+    }
+    return {
+        ok: errors.length === 0,
+        errors,
+        warnings,
+    };
+}

package/dist/src/production/doctor.js

@@ -0,0 +1,58 @@
+import "dotenv/config";
+import { loadConfig, ProjectManager, } from "../core/index.js";
+import { createStorageProvider, withStorageRetry, MemoryStore, PersistentCodeGraphStore, PersistentVectorStore, PersistentCodeChunkStore, PersistentCodeVectorStore, } from "../storage/index.js";
+import { createEmbeddingProvider, } from "../embeddings/index.js";
+import { SnapshotManager, } from "../snapshot/index.js";
+import { checkProductionConfig, } from "./config-check.js";
+import { ProductionHealth, } from "./health.js";
+async function main() {
+    const configCheck = checkProductionConfig();
+    if (!configCheck.ok) {
+        console.log({
+            ok: false,
+            config: configCheck,
+        });
+        process.exit(1);
+    }
+    const config = loadConfig();
+    const project = new ProjectManager()
+        .detect();
+    const storage = withStorageRetry(createStorageProvider({
+        provider: config.storage.provider,
+        huggingface: config.storage.huggingface,
+        localRoot: config.storage.localRoot,
+    }), {
+        attempts: 3,
+    });
+    const embeddings = createEmbeddingProvider();
+    const health = await new ProductionHealth(storage, embeddings).run();
+    const memories = await new MemoryStore(storage).load(project.id);
+    const graph = await new PersistentCodeGraphStore(storage).load(project.id);
+    const vectors = await new PersistentVectorStore(storage).load(project.id);
+    const chunks = await new PersistentCodeChunkStore(storage).load(project.id);
+    const codeVectors = await new PersistentCodeVectorStore(storage).load(project.id);
+    const snapshots = await new SnapshotManager(storage).list(project.id);
+    console.log({
+        ok: health.ok,
+        project: project.name,
+        storage: health.storage,
+        embedding: health.embedding,
+        memory: memories.length,
+        graphSymbols: graph?.symbols.length ?? 0,
+        graphEdges: graph?.edges.length ?? 0,
+        memoryVectors: vectors?.records.length ?? 0,
+        codeChunks: chunks?.chunks.length ?? 0,
+        codeVectors: codeVectors?.records.length ?? 0,
+        snapshots: snapshots.length,
+        warnings: configCheck.warnings,
+    });
+    if (!health.ok) {
+        process.exit(1);
+    }
+}
+main().catch((error) => {
+    console.error(error instanceof Error
+        ? error.message
+        : error);
+    process.exit(1);
+});

package/dist/src/production/full-index.js

@@ -0,0 +1,114 @@
+import { closeSync, mkdirSync, openSync, rmSync, writeFileSync, } from "node:fs";
+import { join, } from "node:path";
+import { ProjectManager, } from "../core/index.js";
+import { runProductionIndex, } from "./index-pipeline.js";
+const project = new ProjectManager()
+    .detect();
+const toolnetDir = join(project.rootPath, ".toolnet");
+const lockFile = join(toolnetDir, "index.lock");
+const statusFile = join(toolnetDir, "last-index.json");
+mkdirSync(toolnetDir, {
+    recursive: true,
+});
+function writeStatus(value) {
+    writeFileSync(statusFile, JSON.stringify(value, null, 2) + "\n", {
+        encoding: "utf8",
+        mode: 0o600,
+    });
+}
+let lockFd;
+try {
+    lockFd =
+        openSync(lockFile, "wx", 0o600);
+}
+catch {
+    console.error("❌ Index already running for this project.");
+    console.error(`Lock: ${lockFile}`);
+    process.exit(2);
+}
+const startedAt = new Date()
+    .toISOString();
+const completedStages = [];
+try {
+    console.log();
+    console.log("==============================================");
+    console.log(" ToolNet Production Index");
+    console.log("==============================================");
+    console.log(`Project : ${project.name}`);
+    console.log(`Remote  : projects/${project.remote ??
+        project.name}/`);
+    console.log("Runtime : production/dist");
+    console.log("Snapshot: disabled");
+    writeStatus({
+        version: 2,
+        state: "running",
+        startedAt,
+        completedStages,
+        project,
+    });
+    const result = await runProductionIndex(project, {
+        onStage: async (event) => {
+            if (event.state ===
+                "start") {
+                console.log();
+                console.log(`===== ${event.title} =====`);
+            }
+            else {
+                completedStages.push(event.id);
+                console.log(`✅ ${event.title} (${((event.durationMs ??
+                    0) /
+                    1000).toFixed(1)}s)`);
+            }
+            writeStatus({
+                version: 2,
+                state: "running",
+                startedAt,
+                currentStage: event.id,
+                completedStages: [
+                    ...completedStages,
+                ],
+                project,
+            });
+        },
+    });
+    writeStatus({
+        version: 2,
+        state: "complete",
+        startedAt,
+        finishedAt: new Date()
+            .toISOString(),
+        completedStages,
+        result,
+    });
+    console.log();
+    console.log("==============================================");
+    console.log(" FULL INDEX COMPLETE ✅");
+    console.log("==============================================");
+    console.log(JSON.stringify(result, null, 2));
+}
+catch (error) {
+    writeStatus({
+        version: 2,
+        state: "failed",
+        startedAt,
+        failedAt: new Date()
+            .toISOString(),
+        completedStages,
+        error: error instanceof Error
+            ? error.message
+            : String(error),
+    });
+    console.error("FULL INDEX FAILED ❌");
+    console.error(error);
+    process.exitCode =
+        1;
+}
+finally {
+    if (lockFd !==
+        undefined) {
+        closeSync(lockFd);
+    }
+    rmSync(lockFile, {
+        force: true,
+    });
+}

package/dist/src/production/health.js

@@ -0,0 +1,58 @@
+import { randomUUID, } from "node:crypto";
+export class ProductionHealth {
+    storage;
+    embeddings;
+    constructor(storage, embeddings) {
+        this.storage = storage;
+        this.embeddings = embeddings;
+    }
+    async run() {
+        const key = `_health/production-${randomUUID()}.txt`;
+        const payload = `toolnet-memory:${Date.now()}`;
+        let storageOk = false;
+        try {
+            await this.storage.put(key, payload, "text/plain");
+            const value = await this.storage.getText(key);
+            storageOk =
+                value === payload;
+        }
+        finally {
+            try {
+                await this.storage.delete(key);
+            }
+            catch {
+                // cleanup only
+            }
+        }
+        let embedding = null;
+        if (this.embeddings) {
+            try {
+                const vector = await this.embeddings.embed("ToolNet Memory production health check");
+                embedding = {
+                    ok: vector.length > 0,
+                    dimensions: vector.length,
+                    mode: vector.length === 384
+                        ? "huggingface"
+                        : "fallback-or-custom",
+                };
+            }
+            catch {
+                embedding = {
+                    ok: false,
+                    dimensions: 0,
+                    mode: "failed",
+                };
+            }
+        }
+        return {
+            ok: storageOk &&
+                (embedding === null ||
+                    embedding.ok),
+            storage: {
+                ok: storageOk,
+                provider: this.storage.name,
+            },
+            embedding,
+        };
+    }
+}

package/dist/src/production/index-pipeline.js

@@ -0,0 +1,151 @@
+import { loadConfig, } from "../core/index.js";
+import { createEmbeddingProvider, } from "../embeddings/index.js";
+import { ArchitectureEngine, CodeAnalysisEngine, RepositoryIndexer, RichGraphEnricher, SemanticCodeEngine, TypeScriptTypeResolver, VisualizationBuilder, } from "../code-intelligence/index.js";
+import { createStorageProvider, PersistentArchitectureStore, PersistentCodeAnalysisStore, PersistentCodeGraphStore, PersistentTypeResolutionStore, PersistentVisualizationStore, ProjectScopedStorageProvider, withStorageRetry, } from "../storage/index.js";
+export async function runProductionIndex(project, options = {}) {
+    const started = Date.now();
+    async function stage(id, title, run) {
+        await options.onStage?.({
+            id,
+            title,
+            state: "start",
+        });
+        const stageStarted = Date.now();
+        const result = await run();
+        await options.onStage?.({
+            id,
+            title,
+            state: "complete",
+            durationMs: Date.now() -
+                stageStarted,
+        });
+        return result;
+    }
+    const config = loadConfig();
+    const rawStorage = withStorageRetry(createStorageProvider({
+        provider: config.storage.provider,
+        huggingface: config.storage.huggingface,
+        localRoot: config.storage.localRoot,
+    }), {
+        attempts: 3,
+    });
+    const storage = new ProjectScopedStorageProvider(rawStorage, project.id, project.name, project.remote ??
+        project.name);
+    const graphStore = new PersistentCodeGraphStore(storage);
+    /*
+     * 1. SOURCE INDEX
+     */
+    const indexed = await stage("source-index", "Source Index", async () => {
+        const result = await new RepositoryIndexer()
+            .index(project.id, project.rootPath);
+        await graphStore.save({
+            version: 1,
+            projectId: project.id,
+            updatedAt: new Date()
+                .toISOString(),
+            files: result.files,
+            symbols: result.graph
+                .allSymbols(project.id),
+            edges: result.graph
+                .allEdges(project.id),
+        });
+        return result;
+    });
+    const graph = indexed.graph;
+    /*
+     * 2. TYPE RESOLUTION
+     */
+    const resolution = await stage("type-resolution", "Type Resolution", async () => {
+        const result = await new TypeScriptTypeResolver(graph).resolveProject(project.id, project.rootPath);
+        await new PersistentTypeResolutionStore(storage).save(result);
+        return result;
+    });
+    /*
+     * 3. RICH GRAPH
+     */
+    await stage("rich-graph", "Rich Graph", async () => {
+        const stats = new RichGraphEnricher(graph).enrich(project.id, project.rootPath, resolution);
+        const symbols = graph.allSymbols(project.id);
+        const edges = graph.allEdges(project.id);
+        await graphStore.save({
+            version: 1,
+            projectId: project.id,
+            updatedAt: new Date()
+                .toISOString(),
+            files: symbols.filter(item => item.type ===
+                "file").length,
+            symbols,
+            edges,
+        });
+        return stats;
+    });
+    /*
+     * 4. SEMANTIC INDEX
+     */
+    const semantic = await stage("semantic-index", "Semantic Code Index", async () => {
+        const engine = new SemanticCodeEngine({
+            projectId: project.id,
+            rootPath: project.rootPath,
+            model: process.env
+                .HF_EMBEDDING_MODEL ??
+                "sentence-transformers/all-MiniLM-L6-v2",
+            storage,
+            embeddings: createEmbeddingProvider(),
+            graph,
+        });
+        return engine.initialize();
+    });
+    /*
+     * 5. ARCHITECTURE
+     */
+    const architecture = await stage("architecture", "Architecture Intelligence", async () => {
+        const result = new ArchitectureEngine(graph).analyze(project.id);
+        await new PersistentArchitectureStore(storage).save(result);
+        return result;
+    });
+    /*
+     * 6. ANALYSIS
+     */
+    const analysis = await stage("analysis", "Graph Analysis", async () => {
+        const result = new CodeAnalysisEngine(graph).analyze(project.id);
+        await new PersistentCodeAnalysisStore(storage).save(result);
+        return result;
+    });
+    /*
+     * 7. VISUALIZATION
+     */
+    const visualization = await stage("visualization", "3D Visualization Dataset", async () => {
+        const result = new VisualizationBuilder(graph).build(project.id, architecture, analysis);
+        await new PersistentVisualizationStore(storage).save(result);
+        return result;
+    });
+    return {
+        project: {
+            id: project.id,
+            name: project.name,
+            remote: project.remote ??
+                project.name,
+        },
+        storage: storage.name,
+        durationMs: Date.now() -
+            started,
+        graph: {
+            symbols: graph
+                .allSymbols(project.id)
+                .length,
+            edges: graph
+                .allEdges(project.id)
+                .length,
+        },
+        resolution: {
+            total: resolution.total,
+            exact: resolution.exact,
+            high: resolution.high,
+            fallback: resolution.fallback,
+        },
+        semantic,
+        architecture: architecture.summary,
+        analysis: analysis.summary,
+        visualization: visualization.summary,
+    };
+}

package/dist/src/production/index.js

@@ -0,0 +1,4 @@
+export * from "./project-lock.js";
+export * from "./config-check.js";
+export * from "./health.js";
+export * from "./recovery.js";

package/dist/src/production/project-lock.js

@@ -0,0 +1,95 @@
+import { closeSync, mkdirSync, openSync, readFileSync, unlinkSync, writeFileSync, } from "node:fs";
+import { tmpdir, } from "node:os";
+import { join, } from "node:path";
+function processAlive(pid) {
+    try {
+        process.kill(pid, 0);
+        return true;
+    }
+    catch (error) {
+        return (error?.code ===
+            "EPERM");
+    }
+}
+export class ProjectLock {
+    acquired = false;
+    path;
+    exitHandler = () => {
+        this.releaseSync();
+    };
+    constructor(projectId) {
+        const root = join(tmpdir(), "toolnet-memory-locks");
+        mkdirSync(root, {
+            recursive: true,
+        });
+        const safeId = projectId.replace(/[^A-Za-z0-9_.-]/g, "_");
+        this.path =
+            join(root, `${safeId}.lock`);
+    }
+    async acquire() {
+        if (this.acquired) {
+            return;
+        }
+        for (let attempt = 0; attempt < 2; attempt++) {
+            try {
+                const fd = openSync(this.path, "wx");
+                writeFileSync(fd, JSON.stringify({
+                    pid: process.pid,
+                    createdAt: new Date()
+                        .toISOString(),
+                }));
+                closeSync(fd);
+                this.acquired =
+                    true;
+                process.once("exit", this.exitHandler);
+                return;
+            }
+            catch (error) {
+                if (error?.code !==
+                    "EEXIST") {
+                    throw error;
+                }
+                let stale = true;
+                try {
+                    const data = JSON.parse(readFileSync(this.path, "utf8"));
+                    if (Number.isInteger(data.pid) &&
+                        processAlive(data.pid)) {
+                        stale =
+                            false;
+                    }
+                }
+                catch {
+                    stale =
+                        true;
+                }
+                if (!stale) {
+                    throw new Error("ToolNet Memory is already running for this project");
+                }
+                try {
+                    unlinkSync(this.path);
+                }
+                catch {
+                    // retry once
+                }
+            }
+        }
+        throw new Error("Unable to acquire ToolNet Memory project lock");
+    }
+    async release() {
+        this.releaseSync();
+    }
+    releaseSync() {
+        if (!this.acquired) {
+            return;
+        }
+        try {
+            unlinkSync(this.path);
+        }
+        catch {
+            // already removed
+        }
+        this.acquired =
+            false;
+        process.off("exit", this.exitHandler);
+    }
+}

package/dist/src/production/project-status.js

@@ -0,0 +1,16 @@
+import { join, } from "node:path";
+import { ProjectManager, } from "../core/index.js";
+import { sanitizeProjectFolder, } from "../storage/project/folder.js";
+const project = new ProjectManager()
+    .detect();
+const remote = sanitizeProjectFolder(project.remote ??
+    project.name);
+console.log(JSON.stringify({
+    id: project.id,
+    name: project.name,
+    remote,
+    rootPath: project.rootPath,
+    localManifest: join(project.rootPath, ".toolnet", "project.json"),
+    remoteNamespace: `projects/${remote}/`,
+    isolation: "1 folder = 1 project = 1 remote namespace",
+}, null, 2));

package/dist/src/production/recovery.js

@@ -0,0 +1,15 @@
+import { SnapshotManager, } from "../snapshot/index.js";
+export async function restoreLatestSnapshot(storage, projectId) {
+    const manager = new SnapshotManager(storage);
+    const snapshots = await manager.list(projectId);
+    const latest = snapshots[0];
+    if (!latest) {
+        throw new Error("No snapshot available");
+    }
+    const result = await manager.restore(projectId, latest.id);
+    return {
+        snapshot: latest.id,
+        createdAt: latest.createdAt,
+        ...result,
+    };
+}

package/dist/src/production/snapshot-cli.js

@@ -0,0 +1,58 @@
+import "dotenv/config";
+import { loadConfig, ProjectManager, } from "../core/index.js";
+import { createStorageProvider, withStorageRetry, ProjectScopedStorageProvider, } from "../storage/index.js";
+import { SnapshotManager, } from "../snapshot/index.js";
+import { restoreLatestSnapshot, } from "./recovery.js";
+async function main() {
+    const command = process.argv[2];
+    const config = loadConfig();
+    const project = new ProjectManager()
+        .detect();
+    const rawStorage = withStorageRetry(createStorageProvider({
+        provider: config.storage.provider,
+        huggingface: config.storage.huggingface,
+        localRoot: config.storage.localRoot,
+    }), {
+        attempts: 3,
+    });
+    const storage = new ProjectScopedStorageProvider(rawStorage, project.id, project.name, project.remote ?? project.name);
+    const manager = new SnapshotManager(storage);
+    if (command === "list") {
+        const list = await manager.list(project.id);
+        console.log(list.map((snapshot) => ({
+            id: snapshot.id,
+            createdAt: snapshot.createdAt,
+            reason: snapshot.reason,
+        })));
+        return;
+    }
+    if (command === "create") {
+        const reason = process.argv
+            .slice(3)
+            .join(" ") ||
+            "manual-cli";
+        console.log(await manager.create(project.id, reason));
+        return;
+    }
+    if (command === "restore") {
+        const id = process.argv[3];
+        if (!id) {
+            throw new Error("snapshot id required");
+        }
+        await manager.create(project.id, "before-cli-restore");
+        console.log(await manager.restore(project.id, id));
+        return;
+    }
+    if (command ===
+        "recover-latest") {
+        console.log(await restoreLatestSnapshot(storage, project.id));
+        return;
+    }
+    throw new Error("Usage: snapshot-cli list|create|restore|recover-latest");
+}
+main().catch((error) => {
+    console.error(error instanceof Error
+        ? error.message
+        : error);
+    process.exit(1);
+});

package/dist/src/project-manual/cli.js

@@ -0,0 +1,77 @@
+import { readFileSync, } from "node:fs";
+import { loadConfig, ProjectManager, } from "../core/index.js";
+import { createStorageProvider, ProjectScopedStorageProvider, withStorageRetry, } from "../storage/index.js";
+import { ensureProjectManual, loadProjectManual, syncProjectManual, } from "./manager.js";
+function after(args, flag) {
+    const index = args.indexOf(flag);
+    return index >=
+        0
+        ? args[index + 1]
+        : undefined;
+}
+function storageFor(project) {
+    const config = loadConfig();
+    const raw = withStorageRetry(createStorageProvider({
+        provider: config.storage
+            .provider,
+        huggingface: config.storage
+            .huggingface,
+        localRoot: config.storage
+            .localRoot,
+    }), {
+        attempts: 3,
+    });
+    return new ProjectScopedStorageProvider(raw, project.id, project.name, project.remote ??
+        project.name);
+}
+async function main() {
+    const [command = "help", ...args] = process.argv.slice(2);
+    const project = new ProjectManager()
+        .detect(after(args, "--project") ??
+        process.cwd());
+    if (command ===
+        "init") {
+        const file = ensureProjectManual(project);
+        console.log(`✅ Project manual: ${file}`);
+        return;
+    }
+    if (command ===
+        "show") {
+        const manual = loadProjectManual(project, false);
+        if (!manual) {
+            console.log("No .toolnet/PROJECT.md yet.");
+            return;
+        }
+        process.stdout.write(readFileSync(manual.path, "utf8"));
+        return;
+    }
+    if (command ===
+        "sync") {
+        const manual = await syncProjectManual(project, storageFor(project));
+        console.log(JSON.stringify({
+            project: project.name,
+            path: manual.path,
+            digest: manual.digest,
+            bytes: manual.bytes,
+            rules: manual.rules.length,
+            enforce: manual.rules.filter(rule => rule.mode ===
+                "enforce").length,
+            advisory: manual.rules.filter(rule => rule.mode ===
+                "advisory").length,
+        }, null, 2));
+        return;
+    }
+    console.log(`Project Manual
+
+Commands:
+  init [--project PATH]
+  show [--project PATH]
+  sync [--project PATH]
+`);
+}
+main().catch(error => {
+    console.error(error instanceof Error
+        ? error.message
+        : error);
+    process.exit(1);
+});

package/dist/src/project-manual/index.js

@@ -0,0 +1,2 @@
+export * from "./types.js";
+export * from "./manager.js";