toolnet-memory 0.2.0

package/dist/src/storage/project/scoped-provider.js

@@ -0,0 +1,165 @@
+import { sanitizeProjectFolder, } from "./folder.js";
+import { mapProjectStoragePath, } from "./layout.js";
+export class ProjectScopedStorageProvider {
+    provider;
+    name;
+    /**
+     * Physical remote project folder.
+     */
+    folder;
+    sourcePrefix;
+    targetPrefix;
+    projectId;
+    projectName;
+    registryPromise;
+    constructor(provider, projectId, projectName, projectRemote) {
+        this.provider = provider;
+        this.name =
+            provider.name;
+        this.projectId =
+            projectId;
+        this.projectName =
+            projectName;
+        this.folder =
+            sanitizeProjectFolder(projectRemote ??
+                projectName);
+        /*
+         * Internal modules may still use the immutable project ID
+         * in logical paths.
+         */
+        this.sourcePrefix =
+            `projects/${projectId}`;
+        /*
+         * Human-readable physical namespace.
+         */
+        this.targetPrefix =
+            `projects/${this.folder}`;
+    }
+    /**
+     * Every real project gets:
+     *
+     * projects/<remote>/project.json
+     *
+     * This also acts as the remote project registry.
+     *
+     * IMPORTANT:
+     * If another project ID already owns the same remote folder,
+     * we fail instead of mixing their memory/graphs.
+     */
+    async registerProject() {
+        const key = `${this.targetPrefix}/project.json`;
+        const now = new Date()
+            .toISOString();
+        let createdAt = now;
+        const existingText = await this.provider.getText(key);
+        if (existingText) {
+            let existing;
+            try {
+                existing =
+                    JSON.parse(existingText);
+            }
+            catch (error) {
+                throw new Error(`Invalid remote ToolNet project manifest at ${key}: ${error instanceof Error
+                    ? error.message
+                    : String(error)}`);
+            }
+            if (typeof existing.id ===
+                "string" &&
+                existing.id !==
+                    this.projectId) {
+                throw new Error([
+                    "ToolNet remote project namespace collision.",
+                    `Remote folder: ${this.targetPrefix}`,
+                    `Existing project id: ${existing.id}`,
+                    `Current project id: ${this.projectId}`,
+                    "Refusing to mix data from two projects.",
+                ].join(" "));
+            }
+            if (typeof existing.createdAt ===
+                "string") {
+                createdAt =
+                    existing.createdAt;
+            }
+        }
+        const manifest = {
+            version: 1,
+            id: this.projectId,
+            name: this.projectName,
+            remote: this.folder,
+            createdAt,
+            updatedAt: now,
+        };
+        await this.provider.put(key, JSON.stringify(manifest, null, 2) + "\n", "application/json");
+    }
+    async ensureRegistered() {
+        if (!this.registryPromise) {
+            this.registryPromise =
+                this.registerProject();
+        }
+        return this.registryPromise;
+    }
+    key(value) {
+        value =
+            mapProjectStoragePath(value);
+        if (value ===
+            this.sourcePrefix) {
+            return this.targetPrefix;
+        }
+        if (value.startsWith(`${this.sourcePrefix}/`)) {
+            return (this.targetPrefix +
+                value.slice(this.sourcePrefix.length));
+        }
+        /*
+         * Accessing the already-resolved physical namespace
+         * of this same project is allowed.
+         */
+        if (value ===
+            this.targetPrefix ||
+            value.startsWith(`${this.targetPrefix}/`)) {
+            return value;
+        }
+        /*
+         * Critical isolation rule.
+         *
+         * A scoped project storage provider must never access:
+         *
+         * projects/<another-project>/...
+         */
+        if (value.startsWith("projects/")) {
+            throw new Error([
+                "Cross-project storage access denied.",
+                `Current project: ${this.targetPrefix}`,
+                `Requested key: ${value}`,
+            ].join(" "));
+        }
+        /*
+         * Preserve support for logical relative paths used
+         * internally by stores/snapshot components.
+         */
+        return value;
+    }
+    async put(key, data, contentType) {
+        await this.ensureRegistered();
+        return this.provider.put(this.key(key), data, contentType);
+    }
+    async get(key) {
+        await this.ensureRegistered();
+        return this.provider.get(this.key(key));
+    }
+    async getText(key) {
+        await this.ensureRegistered();
+        return this.provider.getText(this.key(key));
+    }
+    async delete(key) {
+        await this.ensureRegistered();
+        return this.provider.delete(this.key(key));
+    }
+    async exists(key) {
+        await this.ensureRegistered();
+        return this.provider.exists(this.key(key));
+    }
+    async list(prefix) {
+        await this.ensureRegistered();
+        return this.provider.list(this.key(prefix));
+    }
+}

package/dist/src/storage/provider.js

@@ -0,0 +1,26 @@
+import { homedir, } from "node:os";
+import { join, } from "node:path";
+import { HuggingFaceStorageProvider, } from "./huggingface/client.js";
+import { LocalStorageProvider, } from "./local/client.js";
+export function createStorageProvider(config) {
+    const localRoot = config.localRoot ??
+        join(homedir(), ".toolnet-memory", "storage");
+    if (config.provider ===
+        "huggingface") {
+        const hf = config.huggingface;
+        if (hf?.namespace &&
+            hf.bucket &&
+            hf.accessKeyId &&
+            hf.secretAccessKey) {
+            return new HuggingFaceStorageProvider({
+                namespace: hf.namespace,
+                bucket: hf.bucket,
+                accessKeyId: hf.accessKeyId,
+                secretAccessKey: hf.secretAccessKey,
+            });
+        }
+        console.warn("[storage] Hugging Face credentials missing. Using local fallback.");
+        return new LocalStorageProvider(localRoot);
+    }
+    return new LocalStorageProvider(localRoot);
+}

package/dist/src/storage/retrying-provider.js

@@ -0,0 +1,23 @@
+import { retry, } from "../utils/retry.js";
+const RETRY_METHODS = new Set([
+    "put",
+    "get",
+    "getText",
+    "delete",
+    "list",
+]);
+export function withStorageRetry(provider, options = {}) {
+    return new Proxy(provider, {
+        get(target, property) {
+            const value = Reflect.get(target, property, target);
+            if (typeof value !==
+                "function") {
+                return value;
+            }
+            if (!RETRY_METHODS.has(property)) {
+                return value.bind(target);
+            }
+            return (...args) => retry(() => Promise.resolve(value.apply(target, args)), options);
+        },
+    });
+}

package/dist/src/storage/test-connection.js

@@ -0,0 +1,32 @@
+import "dotenv/config";
+import { loadConfig, } from "../core/config.js";
+import { createStorageProvider, } from "./provider.js";
+async function main() {
+    const config = loadConfig();
+    const storage = createStorageProvider({
+        provider: config.storage.provider,
+        huggingface: config.storage
+            .huggingface,
+        localRoot: config.storage
+            .localRoot,
+    });
+    const key = `_health/${Date.now()}.txt`;
+    const value = `toolnet-memory:${new Date().toISOString()}`;
+    await storage.put(key, value, "text/plain");
+    const result = await storage.getText(key);
+    if (result !== value) {
+        throw new Error("Storage read/write mismatch");
+    }
+    await storage.delete(key);
+    console.log({
+        ok: true,
+        provider: storage.name,
+        write: "PASS",
+        read: "PASS",
+        delete: "PASS",
+    });
+}
+main().catch((error) => {
+    console.error(error);
+    process.exit(1);
+});

package/dist/src/storage/type-resolution-store.js

@@ -0,0 +1,26 @@
+export class PersistentTypeResolutionStore {
+    storage;
+    constructor(storage) {
+        this.storage = storage;
+    }
+    key(projectId) {
+        return [
+            "projects",
+            projectId,
+            "graph",
+            "resolution",
+            "current.json",
+        ].join("/");
+    }
+    async load(projectId) {
+        const text = await this.storage
+            .getText(this.key(projectId));
+        if (!text) {
+            return null;
+        }
+        return JSON.parse(text);
+    }
+    async save(snapshot) {
+        await this.storage.put(this.key(snapshot.projectId), JSON.stringify(snapshot), "application/json");
+    }
+}

package/dist/src/storage/types.js

@@ -0,0 +1 @@
+export {};

package/dist/src/storage/vector-store.js

@@ -0,0 +1,30 @@
+export class PersistentVectorStore {
+    storage;
+    constructor(storage) {
+        this.storage = storage;
+    }
+    key(projectId) {
+        return [
+            "projects",
+            projectId,
+            "vectors",
+            "current.json",
+        ].join("/");
+    }
+    async load(projectId) {
+        const text = await this.storage.getText(this.key(projectId));
+        if (!text) {
+            return null;
+        }
+        const parsed = JSON.parse(text);
+        if (parsed.version !== 1 ||
+            parsed.projectId !== projectId ||
+            !Array.isArray(parsed.records)) {
+            throw new Error("Invalid vector snapshot");
+        }
+        return parsed;
+    }
+    async save(snapshot) {
+        await this.storage.put(this.key(snapshot.projectId), JSON.stringify(snapshot), "application/json");
+    }
+}

package/dist/src/storage/visualization-store.js

@@ -0,0 +1,25 @@
+export class PersistentVisualizationStore {
+    storage;
+    constructor(storage) {
+        this.storage = storage;
+    }
+    key(projectId) {
+        return [
+            "projects",
+            projectId,
+            "code",
+            "visualization",
+            "graph.json",
+        ].join("/");
+    }
+    async load(projectId) {
+        const text = await this.storage.getText(this.key(projectId));
+        if (!text) {
+            return null;
+        }
+        return JSON.parse(text);
+    }
+    async save(graph) {
+        await this.storage.put(this.key(graph.projectId), JSON.stringify(graph, null, 2), "application/json");
+    }
+}

package/dist/src/sync/compressor.js

@@ -0,0 +1 @@
+export {};

package/dist/src/sync/conflict-resolver.js

@@ -0,0 +1 @@
+export {};

package/dist/src/sync/downloader.js

@@ -0,0 +1 @@
+export {};

package/dist/src/sync/index.js

@@ -0,0 +1 @@
+export {};

package/dist/src/sync/sync-engine.js

@@ -0,0 +1 @@
+export {};

package/dist/src/sync/upload-queue.js

@@ -0,0 +1 @@
+export {};

package/dist/src/utils/retry.js

@@ -0,0 +1,29 @@
+function sleep(ms) {
+    return new Promise((resolve) => setTimeout(resolve, ms));
+}
+export async function retry(operation, options = {}) {
+    const attempts = Math.max(1, options.attempts ?? 3);
+    const baseDelayMs = options.baseDelayMs ?? 150;
+    const maxDelayMs = options.maxDelayMs ?? 2000;
+    let lastError;
+    for (let attempt = 1; attempt <= attempts; attempt++) {
+        try {
+            return await operation();
+        }
+        catch (error) {
+            lastError =
+                error;
+            if (attempt >= attempts) {
+                break;
+            }
+            const exponential = Math.min(maxDelayMs, baseDelayMs *
+                2 **
+                    (attempt - 1));
+            const jitter = Math.floor(Math.random() *
+                Math.max(1, exponential * 0.2));
+            await sleep(exponential +
+                jitter);
+        }
+    }
+    throw lastError;
+}

package/dist/src/visualization/server.js

@@ -0,0 +1,270 @@
+import "dotenv/config";
+import { createReadStream, } from "node:fs";
+import { createServer, } from "node:http";
+import { dirname, resolve, } from "node:path";
+import { fileURLToPath, } from "node:url";
+import { loadConfig, ProjectManager, } from "../core/index.js";
+import { CodeGraphStore, VisualizationBuilder, } from "../code-intelligence/index.js";
+import { createStorageProvider, withStorageRetry, } from "../storage/index.js";
+const PORT = Number(process.env
+    .TOOLNET_GRAPH_PORT ??
+    9749);
+const HOST = process.env
+    .TOOLNET_GRAPH_HOST ??
+    "127.0.0.1";
+function extractKey(item) {
+    if (typeof item ===
+        "string") {
+        return item;
+    }
+    if (!item ||
+        typeof item !==
+            "object") {
+        return null;
+    }
+    const value = item;
+    for (const field of [
+        "key",
+        "Key",
+        "path",
+        "name",
+    ]) {
+        if (typeof value[field] ===
+            "string") {
+            return value[field];
+        }
+    }
+    return null;
+}
+async function listKeys(storage, prefix) {
+    const items = await storage.list(prefix);
+    return items
+        .map(extractKey)
+        .filter((value) => Boolean(value));
+}
+async function readJson(storage, key) {
+    const text = await storage.getText(key);
+    if (!text) {
+        return null;
+    }
+    try {
+        return JSON.parse(text);
+    }
+    catch {
+        return null;
+    }
+}
+async function discoverProjects(storage) {
+    const keys = await listKeys(storage, "projects/");
+    /*
+     * Registry rule:
+     *
+     * Only folders containing project.json are real projects.
+     *
+     * Backups / random directories never enter dashboard.
+     */
+    const remotes = [
+        ...new Set(keys
+            .filter(key => /^projects\/[^/]+\/project\.json$/
+            .test(key))
+            .map(key => key.split("/")[1])),
+    ];
+    const keySet = new Set(keys);
+    const projects = [];
+    for (const remote of remotes) {
+        const manifest = await readJson(storage, `projects/${remote}/project.json`);
+        if (!manifest ||
+            typeof manifest.id !==
+                "string") {
+            continue;
+        }
+        projects.push({
+            id: manifest.id,
+            name: typeof manifest.name ===
+                "string"
+                ? manifest.name
+                : remote,
+            remote,
+            hasGraph: keySet.has(`projects/${remote}/code/graph/current.json`),
+            hasArchitecture: keySet.has(`projects/${remote}/code/architecture/current.json`),
+            hasAnalysis: keySet.has(`projects/${remote}/code/analysis/current.json`),
+            hasVisualization: keySet.has(`projects/${remote}/code/visualization/graph.json`),
+        });
+    }
+    return projects.sort((a, b) => a.name.localeCompare(b.name));
+}
+async function main() {
+    const config = loadConfig();
+    const currentProject = new ProjectManager()
+        .detect();
+    const rawStorage = withStorageRetry(createStorageProvider({
+        provider: config.storage.provider,
+        huggingface: config.storage.huggingface,
+        localRoot: config.storage.localRoot,
+    }), {
+        attempts: 3,
+    });
+    if (rawStorage.name !==
+        "huggingface") {
+        console.warn("[graph-ui] Remote Hugging Face storage is not active.");
+    }
+    const moduleDir = dirname(fileURLToPath(import.meta.url));
+    const publicDir = resolve(moduleDir, "public");
+    const vendorFile = resolve(publicDir, "vendor", "3d-force-graph.min.js");
+    async function getCatalog() {
+        const projects = await discoverProjects(rawStorage);
+        const currentRemote = currentProject.remote ??
+            currentProject.name;
+        const current = projects.find(project => project.remote ===
+            currentRemote &&
+            project.hasGraph);
+        const firstGraph = projects.find(project => project.hasGraph);
+        return {
+            projects,
+            defaultProject: current?.remote ??
+                firstGraph?.remote ??
+                null,
+        };
+    }
+    async function buildProjectVisualization(project) {
+        const prefix = `projects/${project.remote}`;
+        const graphSnapshot = await readJson(rawStorage, `${prefix}/code/graph/current.json`);
+        if (!graphSnapshot ||
+            !Array.isArray(graphSnapshot.symbols) ||
+            !Array.isArray(graphSnapshot.edges)) {
+            throw new Error(`Code graph missing for project ${project.name}`);
+        }
+        const architecture = await readJson(rawStorage, `${prefix}/code/architecture/current.json`);
+        const analysis = await readJson(rawStorage, `${prefix}/code/analysis/current.json`);
+        const graph = new CodeGraphStore();
+        graph.import(graphSnapshot.symbols, graphSnapshot.edges);
+        const visualization = new VisualizationBuilder(graph).build(project.id, architecture, analysis);
+        /*
+         * Keep a cached visualization in that project's own
+         * namespace. It never crosses into another project.
+         */
+        await rawStorage.put(`${prefix}/code/visualization/graph.json`, JSON.stringify(visualization, null, 2), "application/json");
+        return {
+            ...visualization,
+            dashboard: {
+                id: project.id,
+                name: project.name,
+                remote: project.remote,
+                hasArchitecture: Boolean(architecture),
+                hasAnalysis: Boolean(analysis),
+            },
+        };
+    }
+    const server = createServer(async (req, res) => {
+        try {
+            const url = new URL(req.url ??
+                "/", `http://${req.headers.host ??
+                "localhost"}`);
+            if (url.pathname ===
+                "/api/health") {
+                const catalog = await getCatalog();
+                res.setHeader("content-type", "application/json; charset=utf-8");
+                res.setHeader("cache-control", "no-store");
+                res.end(JSON.stringify({
+                    ok: true,
+                    mode: "multi-project",
+                    projects: catalog.projects
+                        .length,
+                    indexedProjects: catalog.projects
+                        .filter(item => item.hasGraph)
+                        .length,
+                    defaultProject: catalog.defaultProject,
+                }));
+                return;
+            }
+            if (url.pathname ===
+                "/api/projects") {
+                const catalog = await getCatalog();
+                res.setHeader("content-type", "application/json; charset=utf-8");
+                res.setHeader("cache-control", "no-store");
+                res.end(JSON.stringify(catalog));
+                return;
+            }
+            if (url.pathname ===
+                "/api/graph") {
+                const catalog = await getCatalog();
+                const requested = url.searchParams.get("project") ??
+                    catalog.defaultProject;
+                if (!requested) {
+                    res.statusCode =
+                        404;
+                    res.setHeader("content-type", "application/json; charset=utf-8");
+                    res.end(JSON.stringify({
+                        error: "No indexed ToolNet project found.",
+                    }));
+                    return;
+                }
+                const project = catalog.projects
+                    .find(item => item.remote ===
+                    requested);
+                if (!project) {
+                    res.statusCode =
+                        404;
+                    res.setHeader("content-type", "application/json; charset=utf-8");
+                    res.end(JSON.stringify({
+                        error: "Project not found.",
+                        project: requested,
+                    }));
+                    return;
+                }
+                if (!project.hasGraph) {
+                    res.statusCode =
+                        409;
+                    res.setHeader("content-type", "application/json; charset=utf-8");
+                    res.end(JSON.stringify({
+                        error: "Project is registered but has not been indexed yet.",
+                        project,
+                    }));
+                    return;
+                }
+                const visualization = await buildProjectVisualization(project);
+                res.setHeader("content-type", "application/json; charset=utf-8");
+                res.setHeader("cache-control", "no-store");
+                res.end(JSON.stringify(visualization));
+                return;
+            }
+            if (url.pathname ===
+                "/vendor/3d-force-graph.min.js") {
+                res.setHeader("content-type", "application/javascript; charset=utf-8");
+                createReadStream(vendorFile).pipe(res);
+                return;
+            }
+            if (url.pathname ===
+                "/" ||
+                url.pathname ===
+                    "/index.html") {
+                res.setHeader("content-type", "text/html; charset=utf-8");
+                res.setHeader("cache-control", "no-store");
+                createReadStream(resolve(publicDir, "index.html")).pipe(res);
+                return;
+            }
+            res.statusCode =
+                404;
+            res.end("Not found");
+        }
+        catch (error) {
+            console.error(error);
+            res.statusCode =
+                500;
+            res.setHeader("content-type", "application/json; charset=utf-8");
+            res.end(JSON.stringify({
+                error: error instanceof Error
+                    ? error.message
+                    : String(error),
+            }));
+        }
+    });
+    server.listen(PORT, HOST, () => {
+        console.log("ToolNet Multi-Project 3D Graph");
+        console.log(`Listening: http://${HOST}:${PORT}`);
+    });
+}
+main().catch(error => {
+    console.error(error);
+    process.exit(1);
+});