toolnet-memory 0.2.0

package/dist/src/storage/huggingface/client.js

@@ -0,0 +1,116 @@
+import { DeleteObjectCommand, GetObjectCommand, HeadObjectCommand, ListObjectsV2Command, PutObjectCommand, S3Client, } from "@aws-sdk/client-s3";
+import { getSignedUrl } from "@aws-sdk/s3-request-presigner";
+export class HuggingFaceStorageProvider {
+    name = "huggingface";
+    client;
+    bucket;
+    constructor(options) {
+        this.bucket =
+            options.bucket;
+        this.client =
+            new S3Client({
+                region: "us-east-1",
+                endpoint: `https://s3.hf.co/${options.namespace}`,
+                forcePathStyle: true,
+                requestChecksumCalculation: "WHEN_REQUIRED",
+                responseChecksumValidation: "WHEN_REQUIRED",
+                credentials: {
+                    accessKeyId: options.accessKeyId,
+                    secretAccessKey: options.secretAccessKey,
+                },
+            });
+    }
+    async put(key, data, contentType = "application/octet-stream") {
+        const body = typeof data === "string"
+            ? Buffer.from(data, "utf8")
+            : data;
+        await this.client.send(new PutObjectCommand({
+            Bucket: this.bucket,
+            Key: key,
+            Body: body,
+            ContentType: contentType,
+        }));
+    }
+    async get(key) {
+        const signedUrl = await getSignedUrl(this.client, new GetObjectCommand({
+            Bucket: this.bucket,
+            Key: key,
+        }), {
+            expiresIn: 60,
+        });
+        const response = await fetch(signedUrl, {
+            redirect: "follow",
+        });
+        if (response.status === 404) {
+            return null;
+        }
+        if (!response.ok) {
+            throw new Error(`HF download failed: ${response.status} ${response.statusText}`);
+        }
+        return new Uint8Array(await response.arrayBuffer());
+    }
+    async getText(key) {
+        const data = await this.get(key);
+        if (!data) {
+            return null;
+        }
+        return Buffer
+            .from(data)
+            .toString("utf8");
+    }
+    async exists(key) {
+        try {
+            await this.client.send(new HeadObjectCommand({
+                Bucket: this.bucket,
+                Key: key,
+            }));
+            return true;
+        }
+        catch (error) {
+            if (typeof error === "object" &&
+                error !== null &&
+                "$metadata" in error) {
+                const status = error.$metadata
+                    ?.httpStatusCode;
+                if (status === 404) {
+                    return false;
+                }
+            }
+            throw error;
+        }
+    }
+    async delete(key) {
+        await this.client.send(new DeleteObjectCommand({
+            Bucket: this.bucket,
+            Key: key,
+        }));
+    }
+    async list(prefix = "") {
+        const output = [];
+        let continuationToken;
+        do {
+            const response = await this.client.send(new ListObjectsV2Command({
+                Bucket: this.bucket,
+                Prefix: prefix || undefined,
+                ContinuationToken: continuationToken,
+            }));
+            for (const item of response.Contents ?? []) {
+                if (!item.Key) {
+                    continue;
+                }
+                output.push({
+                    key: item.Key,
+                    size: item.Size,
+                    updatedAt: item.LastModified
+                        ?.toISOString(),
+                });
+            }
+            continuationToken =
+                response.IsTruncated
+                    ? response
+                        .NextContinuationToken
+                    : undefined;
+        } while (continuationToken);
+        return output;
+    }
+}

package/dist/src/storage/huggingface/index.js

@@ -0,0 +1 @@
+export * from "./client.js";

package/dist/src/storage/index.js

@@ -0,0 +1,16 @@
+export * from "./types.js";
+export * from "./provider.js";
+export * from "./memory-store.js";
+export * from "./local/index.js";
+export * from "./huggingface/index.js";
+export * from "./vector-store.js";
+export * from "./code-graph-store.js";
+export * from "./code-manifest-store.js";
+export * from "./code-chunk-store.js";
+export * from "./code-vector-store.js";
+export * from "./retrying-provider.js";
+export * from "./project/index.js";
+export * from "./type-resolution-store.js";
+export * from "./architecture-store.js";
+export * from "./code-analysis-store.js";
+export * from "./visualization-store.js";

package/dist/src/storage/list-bucket-raw.js

@@ -0,0 +1,14 @@
+import "dotenv/config";
+import { loadConfig } from "../core/index.js";
+import { createStorageProvider, withStorageRetry, } from "./index.js";
+const config = loadConfig();
+const storage = withStorageRetry(createStorageProvider({
+    provider: config.storage.provider,
+    huggingface: config.storage.huggingface,
+    localRoot: config.storage.localRoot,
+}), { attempts: 3 });
+const objects = await storage.list("");
+console.log(`TOTAL: ${objects.length}\n`);
+for (const object of objects.sort((a, b) => a.key.localeCompare(b.key))) {
+    console.log(object.key);
+}

package/dist/src/storage/list-bucket-tree.js

@@ -0,0 +1,38 @@
+import "dotenv/config";
+import { loadConfig, } from "../core/index.js";
+import { createStorageProvider, withStorageRetry, } from "./index.js";
+function tree(keys) {
+    const root = {};
+    for (const key of keys) {
+        let node = root;
+        for (const part of key.split("/").filter(Boolean)) {
+            node[part] ??= {};
+            node = node[part];
+        }
+    }
+    function print(node, prefix = "") {
+        const entries = Object.keys(node).sort();
+        entries.forEach((name, index) => {
+            const last = index === entries.length - 1;
+            console.log(`${prefix}${last ? "└── " : "├── "}${name}`);
+            print(node[name], prefix + (last ? "    " : "│   "));
+        });
+    }
+    print(root);
+}
+async function main() {
+    const config = loadConfig();
+    const storage = withStorageRetry(createStorageProvider({
+        provider: config.storage.provider,
+        huggingface: config.storage.huggingface,
+        localRoot: config.storage.localRoot,
+    }), { attempts: 3 });
+    const objects = await storage.list("");
+    console.log(`Bucket: ${process.env.HF_BUCKET}`);
+    console.log(`Objects: ${objects.length}\n`);
+    tree(objects.map((x) => x.key));
+}
+main().catch((err) => {
+    console.error(err);
+    process.exit(1);
+});

package/dist/src/storage/local/client.js

@@ -0,0 +1,99 @@
+import { access, mkdir, readFile, readdir, rm, stat, writeFile, } from "node:fs/promises";
+import { dirname, join, relative, resolve, } from "node:path";
+export class LocalStorageProvider {
+    root;
+    name = "local";
+    constructor(root) {
+        this.root = root;
+    }
+    path(key) {
+        const safeKey = key.replace(/^\/+/, "");
+        return resolve(this.root, safeKey);
+    }
+    async put(key, data) {
+        const path = this.path(key);
+        await mkdir(dirname(path), {
+            recursive: true,
+        });
+        await writeFile(path, data);
+    }
+    async get(key) {
+        try {
+            return await readFile(this.path(key));
+        }
+        catch (error) {
+            if (typeof error === "object" &&
+                error !== null &&
+                "code" in error &&
+                error.code === "ENOENT") {
+                return null;
+            }
+            throw error;
+        }
+    }
+    async getText(key) {
+        const data = await this.get(key);
+        if (!data) {
+            return null;
+        }
+        return Buffer
+            .from(data)
+            .toString("utf8");
+    }
+    async exists(key) {
+        try {
+            await access(this.path(key));
+            return true;
+        }
+        catch {
+            return false;
+        }
+    }
+    async delete(key) {
+        await rm(this.path(key), {
+            force: true,
+        });
+    }
+    async list(prefix = "") {
+        const start = this.path(prefix);
+        const output = [];
+        try {
+            await access(start);
+        }
+        catch {
+            return output;
+        }
+        const walk = async (directory) => {
+            const entries = await readdir(directory, {
+                withFileTypes: true,
+            });
+            for (const entry of entries) {
+                const fullPath = join(directory, entry.name);
+                if (entry.isDirectory()) {
+                    await walk(fullPath);
+                    continue;
+                }
+                const info = await stat(fullPath);
+                output.push({
+                    key: relative(this.root, fullPath),
+                    size: info.size,
+                    updatedAt: info.mtime
+                        .toISOString(),
+                });
+            }
+        };
+        const info = await stat(start);
+        if (info.isDirectory()) {
+            await walk(start);
+        }
+        else {
+            output.push({
+                key: relative(this.root, start),
+                size: info.size,
+                updatedAt: info.mtime
+                    .toISOString(),
+            });
+        }
+        return output;
+    }
+}

package/dist/src/storage/local/index.js

@@ -0,0 +1 @@
+export * from "./client.js";

package/dist/src/storage/memory-store.js

@@ -0,0 +1,30 @@
+export class MemoryStore {
+    storage;
+    constructor(storage) {
+        this.storage = storage;
+    }
+    key(projectId) {
+        return [
+            "projects",
+            projectId,
+            "memories",
+            "current.json",
+        ].join("/");
+    }
+    async load(projectId) {
+        const text = await this.storage
+            .getText(this.key(projectId));
+        if (!text) {
+            return [];
+        }
+        const parsed = JSON.parse(text);
+        if (!Array.isArray(parsed)) {
+            throw new Error("Invalid memory snapshot");
+        }
+        return parsed;
+    }
+    async save(projectId, memories) {
+        const payload = JSON.stringify(memories, null, 2);
+        await this.storage.put(this.key(projectId), payload, "application/json");
+    }
+}

package/dist/src/storage/project/cleanup-layout-v1.js

@@ -0,0 +1,72 @@
+import "dotenv/config";
+import { loadConfig, } from "../../core/index.js";
+import { createStorageProvider, withStorageRetry, } from "../index.js";
+function destination(key) {
+    let m;
+    m =
+        key.match(/^(projects\/[^/]+\/)memories\/(.+)$/);
+    if (m) {
+        return `${m[1]}memory/records/${m[2]}`;
+    }
+    m =
+        key.match(/^(projects\/[^/]+\/)vectors\/(.+)$/);
+    if (m) {
+        return `${m[1]}memory/vectors/${m[2]}`;
+    }
+    m =
+        key.match(/^(projects\/[^/]+\/)graph\/(.+)$/);
+    if (m) {
+        return `${m[1]}code/graph/${m[2]}`;
+    }
+    m =
+        key.match(/^(projects\/[^/]+\/snapshots\/[^/]+\/)memories\/(.+)$/);
+    if (m) {
+        return `${m[1]}memory/records/${m[2]}`;
+    }
+    m =
+        key.match(/^(projects\/[^/]+\/snapshots\/[^/]+\/)vectors\/(.+)$/);
+    if (m) {
+        return `${m[1]}memory/vectors/${m[2]}`;
+    }
+    m =
+        key.match(/^(projects\/[^/]+\/snapshots\/[^/]+\/)graph\/(.+)$/);
+    if (m) {
+        return `${m[1]}code/graph/${m[2]}`;
+    }
+    return null;
+}
+async function main() {
+    const config = loadConfig();
+    const storage = withStorageRetry(createStorageProvider({
+        provider: config.storage.provider,
+        huggingface: config.storage.huggingface,
+        localRoot: config.storage.localRoot,
+    }), {
+        attempts: 3,
+    });
+    const objects = await storage.list("projects/");
+    const oldObjects = objects
+        .map((item) => ({
+        from: item.key,
+        to: destination(item.key),
+    }))
+        .filter((item) => Boolean(item.to));
+    let deleted = 0;
+    for (const item of oldObjects) {
+        const newExists = await storage.exists(item.to);
+        if (!newExists) {
+            throw new Error(`REFUSE DELETE: destination missing: ${item.to}`);
+        }
+        console.log(`DELETE OLD ${item.from}`);
+        await storage.delete(item.from);
+        deleted++;
+    }
+    console.log({
+        ok: true,
+        deleted,
+    });
+}
+main().catch((error) => {
+    console.error(error);
+    process.exit(1);
+});

package/dist/src/storage/project/folder.js

@@ -0,0 +1,15 @@
+export function sanitizeProjectFolder(name) {
+    const value = name
+        .trim()
+        .replace(/\s+/g, "_")
+        .replace(/[^A-Za-z0-9._-]/g, "_")
+        .replace(/_+/g, "_")
+        .replace(/^\.+|\.+$/g, "")
+        .slice(0, 100);
+    if (!value ||
+        value === "." ||
+        value === "..") {
+        throw new Error("Invalid project storage folder");
+    }
+    return value;
+}

package/dist/src/storage/project/index.js

@@ -0,0 +1,3 @@
+export * from "./folder.js";
+export * from "./scoped-provider.js";
+export * from "./layout.js";

package/dist/src/storage/project/layout.js

@@ -0,0 +1,74 @@
+export function mapProjectStoragePath(key) {
+    const clean = key.replace(/^\/+/, "");
+    /*
+     * Relative project keys
+     */
+    if (clean.startsWith("memories/")) {
+        return ("memory/records/" +
+            clean.slice("memories/".length));
+    }
+    if (clean.startsWith("vectors/")) {
+        return ("memory/vectors/" +
+            clean.slice("vectors/".length));
+    }
+    if (clean.startsWith("graph/")) {
+        return ("code/graph/" +
+            clean.slice("graph/".length));
+    }
+    /*
+     * Snapshot logical keys
+     */
+    let snapshotMatch = clean.match(/^(snapshots\/[^/]+\/)memories\/(.+)$/);
+    if (snapshotMatch) {
+        return `${snapshotMatch[1]}memory/records/${snapshotMatch[2]}`;
+    }
+    snapshotMatch =
+        clean.match(/^(snapshots\/[^/]+\/)vectors\/(.+)$/);
+    if (snapshotMatch) {
+        return `${snapshotMatch[1]}memory/vectors/${snapshotMatch[2]}`;
+    }
+    snapshotMatch =
+        clean.match(/^(snapshots\/[^/]+\/)graph\/(.+)$/);
+    if (snapshotMatch) {
+        return `${snapshotMatch[1]}code/graph/${snapshotMatch[2]}`;
+    }
+    /*
+     * Full project snapshot keys
+     */
+    let fullSnapshotMatch = clean.match(/^(projects\/[^/]+\/snapshots\/[^/]+\/)memories\/(.+)$/);
+    if (fullSnapshotMatch) {
+        return `${fullSnapshotMatch[1]}memory/records/${fullSnapshotMatch[2]}`;
+    }
+    fullSnapshotMatch =
+        clean.match(/^(projects\/[^/]+\/snapshots\/[^/]+\/)vectors\/(.+)$/);
+    if (fullSnapshotMatch) {
+        return `${fullSnapshotMatch[1]}memory/vectors/${fullSnapshotMatch[2]}`;
+    }
+    fullSnapshotMatch =
+        clean.match(/^(projects\/[^/]+\/snapshots\/[^/]+\/)graph\/(.+)$/);
+    if (fullSnapshotMatch) {
+        return `${fullSnapshotMatch[1]}code/graph/${fullSnapshotMatch[2]}`;
+    }
+    /*
+     * Full project keys
+     *
+     * projects/<project>/memories/...
+     * projects/<project>/vectors/...
+     * projects/<project>/graph/...
+     */
+    let match = clean.match(/^(projects\/[^/]+\/)memories\/(.+)$/);
+    if (match) {
+        return (`${match[1]}memory/records/${match[2]}`);
+    }
+    match =
+        clean.match(/^(projects\/[^/]+\/)vectors\/(.+)$/);
+    if (match) {
+        return (`${match[1]}memory/vectors/${match[2]}`);
+    }
+    match =
+        clean.match(/^(projects\/[^/]+\/)graph\/(.+)$/);
+    if (match) {
+        return (`${match[1]}code/graph/${match[2]}`);
+    }
+    return clean;
+}

package/dist/src/storage/project/migrate-layout-v2.js

@@ -0,0 +1,101 @@
+import "dotenv/config";
+import { createHash, } from "node:crypto";
+import { loadConfig, } from "../../core/index.js";
+import { createStorageProvider, withStorageRetry, } from "../index.js";
+function sha(text) {
+    return createHash("sha256")
+        .update(text)
+        .digest("hex");
+}
+function destination(key) {
+    let match;
+    /*
+     * Current state
+     */
+    match =
+        key.match(/^(projects\/[^/]+\/)memories\/(.+)$/);
+    if (match) {
+        return (`${match[1]}memory/records/${match[2]}`);
+    }
+    match =
+        key.match(/^(projects\/[^/]+\/)vectors\/(.+)$/);
+    if (match) {
+        return (`${match[1]}memory/vectors/${match[2]}`);
+    }
+    match =
+        key.match(/^(projects\/[^/]+\/)graph\/(.+)$/);
+    if (match) {
+        return (`${match[1]}code/graph/${match[2]}`);
+    }
+    /*
+     * Historical snapshots
+     */
+    match =
+        key.match(/^(projects\/[^/]+\/snapshots\/[^/]+\/)memories\/(.+)$/);
+    if (match) {
+        return (`${match[1]}memory/records/${match[2]}`);
+    }
+    match =
+        key.match(/^(projects\/[^/]+\/snapshots\/[^/]+\/)vectors\/(.+)$/);
+    if (match) {
+        return (`${match[1]}memory/vectors/${match[2]}`);
+    }
+    match =
+        key.match(/^(projects\/[^/]+\/snapshots\/[^/]+\/)graph\/(.+)$/);
+    if (match) {
+        return (`${match[1]}code/graph/${match[2]}`);
+    }
+    return null;
+}
+async function main() {
+    const config = loadConfig();
+    const storage = withStorageRetry(createStorageProvider({
+        provider: config.storage.provider,
+        huggingface: config.storage.huggingface,
+        localRoot: config.storage.localRoot,
+    }), {
+        attempts: 3,
+    });
+    const objects = await storage.list("projects/");
+    const migrations = objects
+        .map((item) => ({
+        from: item.key,
+        to: destination(item.key),
+    }))
+        .filter((item) => Boolean(item.to) &&
+        item.from !== item.to);
+    console.log(`Objects to migrate: ${migrations.length}`);
+    let copied = 0;
+    let verified = 0;
+    for (const item of migrations) {
+        const source = await storage.getText(item.from);
+        if (source === null ||
+            source === undefined) {
+            throw new Error(`Cannot read ${item.from}`);
+        }
+        console.log(`COPY\n  ${item.from}\n  -> ${item.to}`);
+        await storage.put(item.to, source, "application/json");
+        copied++;
+        const target = await storage.getText(item.to);
+        if (target === null ||
+            target === undefined) {
+            throw new Error(`Cannot verify ${item.to}`);
+        }
+        if (sha(source) !==
+            sha(target)) {
+            throw new Error(`Checksum mismatch:\n${item.from}\n${item.to}`);
+        }
+        verified++;
+    }
+    console.log({
+        ok: true,
+        copied,
+        verified,
+        deleted: 0,
+        message: "Old objects intentionally preserved until build/tests pass.",
+    });
+}
+main().catch((error) => {
+    console.error(error);
+    process.exit(1);
+});

package/dist/src/storage/project/migrate.js

@@ -0,0 +1,73 @@
+import "dotenv/config";
+import { loadConfig, ProjectManager, } from "../../core/index.js";
+import { createStorageProvider, withStorageRetry, } from "../index.js";
+import { sanitizeProjectFolder, } from "./folder.js";
+async function main() {
+    const config = loadConfig();
+    const project = new ProjectManager()
+        .detect();
+    const storage = withStorageRetry(createStorageProvider({
+        provider: config.storage.provider,
+        huggingface: config.storage.huggingface,
+        localRoot: config.storage.localRoot,
+    }), {
+        attempts: 3,
+    });
+    const folder = sanitizeProjectFolder(project.name);
+    const oldPrefix = `projects/${project.id}`;
+    const newPrefix = `projects/${folder}`;
+    if (oldPrefix ===
+        newPrefix) {
+        console.log({
+            ok: true,
+            migrated: 0,
+            folder,
+            message: "Already using named project folder",
+        });
+        return;
+    }
+    const objects = await storage.list(oldPrefix);
+    let migrated = 0;
+    for (const object of objects) {
+        const data = await storage.get(object.key);
+        if (!data) {
+            continue;
+        }
+        const newKey = newPrefix +
+            object.key.slice(oldPrefix.length);
+        await storage.put(newKey, data, "application/json");
+        migrated++;
+    }
+    /*
+     * Project metadata dễ đọc.
+     */
+    await storage.put(`${newPrefix}/project.json`, JSON.stringify({
+        version: 1,
+        projectId: project.id,
+        projectName: project.name,
+        storageFolder: folder,
+        migratedAt: new Date()
+            .toISOString(),
+    }, null, 2), "application/json");
+    /*
+     * Chỉ xoá folder hash sau khi copy xong.
+     */
+    if (migrated > 0) {
+        for (const object of objects) {
+            await storage.delete(object.key);
+        }
+    }
+    console.log({
+        ok: true,
+        project: project.name,
+        projectId: project.id,
+        folder,
+        migrated,
+        from: oldPrefix,
+        to: newPrefix,
+    });
+}
+main().catch((error) => {
+    console.error(error);
+    process.exit(1);
+});