toolnet-memory 0.2.0

package/dist/src/session/wal.js

@@ -0,0 +1,303 @@
+import { closeSync, existsSync, fsyncSync, mkdirSync, openSync, readSync, rmSync, statSync, writeSync, } from "node:fs";
+import { join, } from "node:path";
+import { readJsonFile, sha256, stableStringify, writeJsonAtomic, } from "./utils.js";
+const LOCK_STALE_MS = 120_000;
+const LOCK_ATTEMPTS = 80;
+const RECENT_EVENT_LIMIT = 2_000;
+function sleepSync(milliseconds) {
+    Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, milliseconds);
+}
+export class SessionWal {
+    identity;
+    eventsFile;
+    stateFile;
+    lockFile;
+    constructor(identity) {
+        this.identity = identity;
+        mkdirSync(identity.localDirectory, {
+            recursive: true,
+        });
+        this.eventsFile =
+            join(identity.localDirectory, "events.jsonl");
+        this.stateFile =
+            join(identity.localDirectory, "state.json");
+        this.lockFile =
+            join(identity.localDirectory, "journal.lock");
+    }
+    initialState() {
+        const now = new Date()
+            .toISOString();
+        return {
+            version: 1,
+            projectId: this.identity
+                .projectId,
+            agent: this.identity
+                .agent,
+            nativeSessionId: this.identity
+                .nativeSessionId,
+            status: "idle",
+            createdAt: now,
+            updatedAt: now,
+            lastSequence: 0,
+            lastRemoteSequence: 0,
+            remoteByteOffset: 0,
+            sourceCursors: {},
+            recentEventIds: [],
+        };
+    }
+    loadStateUnsafe() {
+        return (readJsonFile(this.stateFile) ??
+            this.initialState());
+    }
+    loadState() {
+        return this.withLock(() => this.loadStateUnsafe());
+    }
+    saveStateUnsafe(state) {
+        writeJsonAtomic(this.stateFile, state);
+    }
+    acquireLock() {
+        for (let attempt = 0; attempt <
+            LOCK_ATTEMPTS; attempt += 1) {
+            try {
+                return openSync(this.lockFile, "wx", 0o600);
+            }
+            catch (error) {
+                const code = error.code;
+                if (code !==
+                    "EEXIST") {
+                    throw error;
+                }
+                try {
+                    const lockAge = Date.now() -
+                        statSync(this.lockFile).mtimeMs;
+                    if (lockAge >
+                        LOCK_STALE_MS) {
+                        rmSync(this.lockFile, {
+                            force: true,
+                        });
+                        continue;
+                    }
+                }
+                catch {
+                    // Lock disappeared between checks.
+                }
+                sleepSync(25);
+            }
+        }
+        throw new Error(`Session journal is locked: ${this.lockFile}`);
+    }
+    withLock(run) {
+        const fd = this.acquireLock();
+        try {
+            return run();
+        }
+        finally {
+            closeSync(fd);
+            rmSync(this.lockFile, {
+                force: true,
+            });
+        }
+    }
+    append(inputs) {
+        if (inputs.length ===
+            0) {
+            return [];
+        }
+        return this.withLock(() => {
+            const state = this.loadStateUnsafe();
+            const recent = new Set(state.recentEventIds);
+            let nextSequence = state.lastSequence;
+            const normalized = [];
+            for (const input of inputs) {
+                const timestamp = input.timestamp ??
+                    new Date()
+                        .toISOString();
+                const data = input.data ??
+                    {};
+                const rawDigest = input.provenance
+                    ?.rawDigest ??
+                    sha256(stableStringify(data));
+                const idSeed = input.sourceEventId
+                    ? [
+                        this.identity
+                            .projectId,
+                        this.identity
+                            .agent,
+                        this.identity
+                            .nativeSessionId,
+                        input
+                            .sourceEventId,
+                    ].join("|")
+                    : [
+                        this.identity
+                            .projectId,
+                        this.identity
+                            .agent,
+                        this.identity
+                            .nativeSessionId,
+                        nextSequence +
+                            1,
+                        input.type,
+                        timestamp,
+                        rawDigest,
+                    ].join("|");
+                const eventId = sha256(idSeed).slice(0, 32);
+                /*
+                 * Fast bounded dedup.
+                 * Adapters will additionally use native source cursors.
+                 */
+                if (recent.has(eventId)) {
+                    continue;
+                }
+                nextSequence +=
+                    1;
+                const event = {
+                    version: 1,
+                    id: eventId,
+                    sequence: nextSequence,
+                    projectId: this.identity
+                        .projectId,
+                    agent: this.identity
+                        .agent,
+                    nativeSessionId: this.identity
+                        .nativeSessionId,
+                    type: input.type,
+                    timestamp,
+                    data,
+                    provenance: {
+                        ...input.provenance,
+                        rawDigest,
+                    },
+                };
+                if (input.role !==
+                    undefined) {
+                    event.role =
+                        input.role;
+                }
+                if (input.sourceEventId !==
+                    undefined) {
+                    event.sourceEventId =
+                        input.sourceEventId;
+                }
+                if (input.sourceSequence !==
+                    undefined) {
+                    event.sourceSequence =
+                        input.sourceSequence;
+                }
+                normalized.push(event);
+                recent.add(eventId);
+            }
+            if (normalized.length ===
+                0) {
+                return [];
+            }
+            const content = normalized
+                .map(item => JSON.stringify(item))
+                .join("\n") +
+                "\n";
+            const fd = openSync(this.eventsFile, "a", 0o600);
+            try {
+                writeSync(fd, content, null, "utf8");
+                fsyncSync(fd);
+            }
+            finally {
+                closeSync(fd);
+            }
+            const last = normalized[normalized.length -
+                1];
+            let status = "active";
+            if ((last.type ===
+                "session_end" ||
+                last.type ===
+                    "session_idle")) {
+                status =
+                    "idle";
+            }
+            else if (last.type ===
+                "error") {
+                status =
+                    "error";
+            }
+            const recentEventIds = Array.from(recent).slice(-RECENT_EVENT_LIMIT);
+            this.saveStateUnsafe({
+                ...state,
+                status,
+                updatedAt: last.timestamp,
+                lastSequence: last.sequence,
+                recentEventIds,
+            });
+            return normalized;
+        });
+    }
+    readPending() {
+        return this.withLock(() => {
+            const state = this.loadStateUnsafe();
+            if (!existsSync(this.eventsFile)) {
+                return {
+                    events: [],
+                    startOffset: state.remoteByteOffset,
+                    endOffset: state.remoteByteOffset,
+                };
+            }
+            const fileSize = statSync(this.eventsFile).size;
+            const startOffset = Math.min(state.remoteByteOffset, fileSize);
+            if (fileSize <=
+                startOffset) {
+                return {
+                    events: [],
+                    startOffset,
+                    endOffset: fileSize,
+                };
+            }
+            const length = fileSize -
+                startOffset;
+            const buffer = Buffer.alloc(length);
+            const fd = openSync(this.eventsFile, "r");
+            try {
+                readSync(fd, buffer, 0, length, startOffset);
+            }
+            finally {
+                closeSync(fd);
+            }
+            const text = buffer.toString("utf8");
+            const events = text
+                .split("\n")
+                .filter(Boolean)
+                .map(line => JSON.parse(line));
+            return {
+                events,
+                startOffset,
+                endOffset: fileSize,
+            };
+        });
+    }
+    markRemote(sequence, byteOffset) {
+        this.withLock(() => {
+            const state = this.loadStateUnsafe();
+            this.saveStateUnsafe({
+                ...state,
+                lastRemoteSequence: Math.max(state
+                    .lastRemoteSequence, sequence),
+                remoteByteOffset: Math.max(state
+                    .remoteByteOffset, byteOffset),
+                updatedAt: new Date()
+                    .toISOString(),
+            });
+        });
+    }
+    setSourceCursor(source, value) {
+        this.withLock(() => {
+            const state = this.loadStateUnsafe();
+            this.saveStateUnsafe({
+                ...state,
+                sourceCursors: {
+                    ...state
+                        .sourceCursors,
+                    [source]: String(value),
+                },
+                updatedAt: new Date()
+                    .toISOString(),
+            });
+        });
+    }
+}

package/dist/src/snapshot/index.js

@@ -0,0 +1,2 @@
+export * from "./types.js";
+export * from "./manager.js";

package/dist/src/snapshot/manager.js

@@ -0,0 +1,130 @@
+const CURRENT_OBJECTS = [
+    "memories/current.json",
+    "vectors/current.json",
+    "graph/current.json",
+    "graph/manifest.json",
+    "graph/resolution/current.json",
+    "code/chunks/current.json",
+    "code/vectors/current.json",
+    "code/architecture/current.json",
+    "code/architecture/entry-points.json",
+    "code/architecture/hotspots.json",
+    "code/architecture/layers.json",
+    "code/architecture/clusters.json",
+    "code/analysis/current.json",
+    "code/analysis/dead-code.json",
+    "code/analysis/dependencies.json",
+    "code/visualization/graph.json",
+];
+export class SnapshotManager {
+    storage;
+    constructor(storage) {
+        this.storage = storage;
+    }
+    projectPrefix(projectId) {
+        return `projects/${projectId}`;
+    }
+    snapshotPrefix(projectId, snapshotId) {
+        return [
+            this.projectPrefix(projectId),
+            "snapshots",
+            snapshotId,
+        ].join("/");
+    }
+    async create(projectId, reason = "automatic", metadata) {
+        const createdAt = new Date().toISOString();
+        const id = createdAt
+            .replace(/[:.]/g, "-");
+        const base = this.projectPrefix(projectId);
+        const snapshotBase = this.snapshotPrefix(projectId, id);
+        const savedObjects = [];
+        for (const relativeKey of CURRENT_OBJECTS) {
+            const sourceKey = `${base}/${relativeKey}`;
+            const data = await this.storage.get(sourceKey);
+            if (!data) {
+                continue;
+            }
+            await this.storage.put(`${snapshotBase}/${relativeKey}`, data, "application/json");
+            savedObjects.push(relativeKey);
+        }
+        if (savedObjects.length === 0) {
+            return null;
+        }
+        const manifest = {
+            version: 1,
+            id,
+            projectId,
+            createdAt,
+            reason,
+            objects: savedObjects,
+            metadata,
+        };
+        await this.storage.put(`${snapshotBase}/snapshot.json`, JSON.stringify(manifest, null, 2), "application/json");
+        return manifest;
+    }
+    async list(projectId) {
+        const prefix = [
+            this.projectPrefix(projectId),
+            "snapshots",
+        ].join("/");
+        const files = await this.storage.list(prefix);
+        const manifests = files.filter((file) => file.key.endsWith("/snapshot.json"));
+        const output = [];
+        for (const item of manifests) {
+            const text = await this.storage.getText(item.key);
+            if (!text) {
+                continue;
+            }
+            try {
+                output.push(JSON.parse(text));
+            }
+            catch {
+                // Ignore damaged snapshot manifests.
+            }
+        }
+        return output.sort((a, b) => b.createdAt.localeCompare(a.createdAt));
+    }
+    async restore(projectId, snapshotId) {
+        const snapshotBase = this.snapshotPrefix(projectId, snapshotId);
+        const manifestText = await this.storage.getText(`${snapshotBase}/snapshot.json`);
+        if (!manifestText) {
+            throw new Error(`Snapshot not found: ${snapshotId}`);
+        }
+        const manifest = JSON.parse(manifestText);
+        if (manifest.projectId !==
+            projectId) {
+            throw new Error("Snapshot project mismatch");
+        }
+        const base = this.projectPrefix(projectId);
+        const restored = [];
+        for (const relativeKey of manifest.objects) {
+            const data = await this.storage.get(`${snapshotBase}/${relativeKey}`);
+            if (!data) {
+                continue;
+            }
+            await this.storage.put(`${base}/${relativeKey}`, data, "application/json");
+            restored.push(relativeKey);
+        }
+        return {
+            restored,
+        };
+    }
+    async remove(projectId, snapshotId) {
+        const prefix = this.snapshotPrefix(projectId, snapshotId);
+        const objects = await this.storage.list(prefix);
+        for (const object of objects) {
+            await this.storage.delete(object.key);
+        }
+        return objects.length;
+    }
+    async prune(projectId, keep = 10) {
+        const snapshots = await this.list(projectId);
+        const remove = snapshots.slice(keep);
+        let deleted = 0;
+        for (const snapshot of remove) {
+            await this.remove(projectId, snapshot.id);
+            deleted++;
+        }
+        return deleted;
+    }
+}

package/dist/src/snapshot/test-snapshot.js

@@ -0,0 +1,28 @@
+import "dotenv/config";
+import { loadConfig, ProjectManager, } from "../core/index.js";
+import { createStorageProvider, ProjectScopedStorageProvider, } from "../storage/index.js";
+import { SnapshotManager, } from "./manager.js";
+async function main() {
+    const config = loadConfig();
+    const project = new ProjectManager()
+        .detect();
+    const rawStorage = createStorageProvider({
+        provider: config.storage.provider,
+        huggingface: config.storage.huggingface,
+        localRoot: config.storage.localRoot,
+    });
+    const storage = new ProjectScopedStorageProvider(rawStorage, project.id, project.name, project.remote ?? project.name);
+    const manager = new SnapshotManager(storage);
+    const created = await manager.create(project.id, "manual-test");
+    const list = await manager.list(project.id);
+    console.log({
+        ok: Boolean(created),
+        created: created?.id,
+        snapshots: list.length,
+        latest: list[0]?.id,
+    });
+}
+main().catch((error) => {
+    console.error(error);
+    process.exit(1);
+});

package/dist/src/snapshot/types.js

@@ -0,0 +1 @@
+export {};

package/dist/src/storage/architecture-store.js

@@ -0,0 +1,37 @@
+export class PersistentArchitectureStore {
+    storage;
+    constructor(storage) {
+        this.storage = storage;
+    }
+    base(projectId) {
+        return [
+            "projects",
+            projectId,
+            "code",
+            "architecture",
+        ].join("/");
+    }
+    async load(projectId) {
+        const text = await this.storage.getText(`${this.base(projectId)}/current.json`);
+        if (!text) {
+            return null;
+        }
+        return JSON.parse(text);
+    }
+    async save(snapshot) {
+        const base = this.base(snapshot.projectId);
+        const envelope = (data) => JSON.stringify({
+            version: snapshot.version,
+            projectId: snapshot.projectId,
+            updatedAt: snapshot.updatedAt,
+            data,
+        }, null, 2);
+        await Promise.all([
+            this.storage.put(`${base}/current.json`, JSON.stringify(snapshot, null, 2), "application/json"),
+            this.storage.put(`${base}/entry-points.json`, envelope(snapshot.entryPoints), "application/json"),
+            this.storage.put(`${base}/hotspots.json`, envelope(snapshot.hotspots), "application/json"),
+            this.storage.put(`${base}/layers.json`, envelope(snapshot.layers), "application/json"),
+            this.storage.put(`${base}/clusters.json`, envelope(snapshot.clusters), "application/json"),
+        ]);
+    }
+}

package/dist/src/storage/cleanup-health.js

@@ -0,0 +1,20 @@
+import "dotenv/config";
+import { loadConfig, } from "../core/index.js";
+import { createStorageProvider, withStorageRetry, } from "./index.js";
+const config = loadConfig();
+const storage = withStorageRetry(createStorageProvider({
+    provider: config.storage.provider,
+    huggingface: config.storage.huggingface,
+    localRoot: config.storage.localRoot,
+}), {
+    attempts: 3,
+});
+const objects = await storage.list("_health/");
+for (const object of objects) {
+    await storage.delete(object.key);
+    console.log(`deleted ${object.key}`);
+}
+console.log({
+    ok: true,
+    deleted: objects.length,
+});

package/dist/src/storage/code-analysis-store.js

@@ -0,0 +1,35 @@
+export class PersistentCodeAnalysisStore {
+    storage;
+    constructor(storage) {
+        this.storage = storage;
+    }
+    base(projectId) {
+        return [
+            "projects",
+            projectId,
+            "code",
+            "analysis",
+        ].join("/");
+    }
+    async load(projectId) {
+        const text = await this.storage.getText(`${this.base(projectId)}/current.json`);
+        if (!text) {
+            return null;
+        }
+        return JSON.parse(text);
+    }
+    async save(snapshot) {
+        const base = this.base(snapshot.projectId);
+        const envelope = (data) => JSON.stringify({
+            version: snapshot.version,
+            projectId: snapshot.projectId,
+            updatedAt: snapshot.updatedAt,
+            data,
+        }, null, 2);
+        await Promise.all([
+            this.storage.put(`${base}/current.json`, JSON.stringify(snapshot, null, 2), "application/json"),
+            this.storage.put(`${base}/dead-code.json`, envelope(snapshot.deadCode), "application/json"),
+            this.storage.put(`${base}/dependencies.json`, envelope(snapshot.dependencies), "application/json"),
+        ]);
+    }
+}

package/dist/src/storage/code-chunk-store.js

@@ -0,0 +1,25 @@
+export class PersistentCodeChunkStore {
+    storage;
+    constructor(storage) {
+        this.storage = storage;
+    }
+    key(projectId) {
+        return [
+            "projects",
+            projectId,
+            "code",
+            "chunks",
+            "current.json",
+        ].join("/");
+    }
+    async load(projectId) {
+        const text = await this.storage.getText(this.key(projectId));
+        if (!text) {
+            return null;
+        }
+        return JSON.parse(text);
+    }
+    async save(snapshot) {
+        await this.storage.put(this.key(snapshot.projectId), JSON.stringify(snapshot), "application/json");
+    }
+}

package/dist/src/storage/code-graph-store.js

@@ -0,0 +1,24 @@
+export class PersistentCodeGraphStore {
+    storage;
+    constructor(storage) {
+        this.storage = storage;
+    }
+    key(projectId) {
+        return [
+            "projects",
+            projectId,
+            "graph",
+            "current.json",
+        ].join("/");
+    }
+    async load(projectId) {
+        const text = await this.storage.getText(this.key(projectId));
+        if (!text) {
+            return null;
+        }
+        return JSON.parse(text);
+    }
+    async save(snapshot) {
+        await this.storage.put(this.key(snapshot.projectId), JSON.stringify(snapshot), "application/json");
+    }
+}

package/dist/src/storage/code-manifest-store.js

@@ -0,0 +1,24 @@
+export class PersistentCodeManifestStore {
+    storage;
+    constructor(storage) {
+        this.storage = storage;
+    }
+    key(projectId) {
+        return [
+            "projects",
+            projectId,
+            "graph",
+            "manifest.json",
+        ].join("/");
+    }
+    async load(projectId) {
+        const text = await this.storage.getText(this.key(projectId));
+        if (!text) {
+            return null;
+        }
+        return JSON.parse(text);
+    }
+    async save(manifest) {
+        await this.storage.put(this.key(manifest.projectId), JSON.stringify(manifest), "application/json");
+    }
+}

package/dist/src/storage/code-vector-store.js

@@ -0,0 +1,25 @@
+export class PersistentCodeVectorStore {
+    storage;
+    constructor(storage) {
+        this.storage = storage;
+    }
+    key(projectId) {
+        return [
+            "projects",
+            projectId,
+            "code",
+            "vectors",
+            "current.json",
+        ].join("/");
+    }
+    async load(projectId) {
+        const text = await this.storage.getText(this.key(projectId));
+        if (!text) {
+            return null;
+        }
+        return JSON.parse(text);
+    }
+    async save(snapshot) {
+        await this.storage.put(this.key(snapshot.projectId), JSON.stringify(snapshot), "application/json");
+    }
+}

package/dist/src/storage/github/client.js

@@ -0,0 +1 @@
+export {};

package/dist/src/storage/github/index.js

@@ -0,0 +1 @@
+export {};

package/dist/src/storage/google-drive/client.js

@@ -0,0 +1 @@
+export {};

package/dist/src/storage/google-drive/index.js

@@ -0,0 +1 @@
+export {};