toolnet-memory 0.2.0 → 0.2.2

package/dist/src/storage/huggingface/client.js

@@ -1,116 +0,0 @@
-import { DeleteObjectCommand, GetObjectCommand, HeadObjectCommand, ListObjectsV2Command, PutObjectCommand, S3Client, } from "@aws-sdk/client-s3";
-import { getSignedUrl } from "@aws-sdk/s3-request-presigner";
-export class HuggingFaceStorageProvider {
-    name = "huggingface";
-    client;
-    bucket;
-    constructor(options) {
-        this.bucket =
-            options.bucket;
-        this.client =
-            new S3Client({
-                region: "us-east-1",
-                endpoint: `https://s3.hf.co/${options.namespace}`,
-                forcePathStyle: true,
-                requestChecksumCalculation: "WHEN_REQUIRED",
-                responseChecksumValidation: "WHEN_REQUIRED",
-                credentials: {
-                    accessKeyId: options.accessKeyId,
-                    secretAccessKey: options.secretAccessKey,
-                },
-            });
-    }
-    async put(key, data, contentType = "application/octet-stream") {
-        const body = typeof data === "string"
-            ? Buffer.from(data, "utf8")
-            : data;
-        await this.client.send(new PutObjectCommand({
-            Bucket: this.bucket,
-            Key: key,
-            Body: body,
-            ContentType: contentType,
-        }));
-    }
-    async get(key) {
-        const signedUrl = await getSignedUrl(this.client, new GetObjectCommand({
-            Bucket: this.bucket,
-            Key: key,
-        }), {
-            expiresIn: 60,
-        });
-        const response = await fetch(signedUrl, {
-            redirect: "follow",
-        });
-        if (response.status === 404) {
-            return null;
-        }
-        if (!response.ok) {
-            throw new Error(`HF download failed: ${response.status} ${response.statusText}`);
-        }
-        return new Uint8Array(await response.arrayBuffer());
-    }
-    async getText(key) {
-        const data = await this.get(key);
-        if (!data) {
-            return null;
-        }
-        return Buffer
-            .from(data)
-            .toString("utf8");
-    }
-    async exists(key) {
-        try {
-            await this.client.send(new HeadObjectCommand({
-                Bucket: this.bucket,
-                Key: key,
-            }));
-            return true;
-        }
-        catch (error) {
-            if (typeof error === "object" &&
-                error !== null &&
-                "$metadata" in error) {
-                const status = error.$metadata
-                    ?.httpStatusCode;
-                if (status === 404) {
-                    return false;
-                }
-            }
-            throw error;
-        }
-    }
-    async delete(key) {
-        await this.client.send(new DeleteObjectCommand({
-            Bucket: this.bucket,
-            Key: key,
-        }));
-    }
-    async list(prefix = "") {
-        const output = [];
-        let continuationToken;
-        do {
-            const response = await this.client.send(new ListObjectsV2Command({
-                Bucket: this.bucket,
-                Prefix: prefix || undefined,
-                ContinuationToken: continuationToken,
-            }));
-            for (const item of response.Contents ?? []) {
-                if (!item.Key) {
-                    continue;
-                }
-                output.push({
-                    key: item.Key,
-                    size: item.Size,
-                    updatedAt: item.LastModified
-                        ?.toISOString(),
-                });
-            }
-            continuationToken =
-                response.IsTruncated
-                    ? response
-                        .NextContinuationToken
-                    : undefined;
-        } while (continuationToken);
-        return output;
-    }
-}

package/dist/src/storage/huggingface/index.js

@@ -1 +0,0 @@
-export * from "./client.js";

package/dist/src/storage/index.js

@@ -1,16 +0,0 @@
-export * from "./types.js";
-export * from "./provider.js";
-export * from "./memory-store.js";
-export * from "./local/index.js";
-export * from "./huggingface/index.js";
-export * from "./vector-store.js";
-export * from "./code-graph-store.js";
-export * from "./code-manifest-store.js";
-export * from "./code-chunk-store.js";
-export * from "./code-vector-store.js";
-export * from "./retrying-provider.js";
-export * from "./project/index.js";
-export * from "./type-resolution-store.js";
-export * from "./architecture-store.js";
-export * from "./code-analysis-store.js";
-export * from "./visualization-store.js";

package/dist/src/storage/list-bucket-raw.js

@@ -1,14 +0,0 @@
-import "dotenv/config";
-import { loadConfig } from "../core/index.js";
-import { createStorageProvider, withStorageRetry, } from "./index.js";
-const config = loadConfig();
-const storage = withStorageRetry(createStorageProvider({
-    provider: config.storage.provider,
-    huggingface: config.storage.huggingface,
-    localRoot: config.storage.localRoot,
-}), { attempts: 3 });
-const objects = await storage.list("");
-console.log(`TOTAL: ${objects.length}\n`);
-for (const object of objects.sort((a, b) => a.key.localeCompare(b.key))) {
-    console.log(object.key);
-}

package/dist/src/storage/list-bucket-tree.js

@@ -1,38 +0,0 @@
-import "dotenv/config";
-import { loadConfig, } from "../core/index.js";
-import { createStorageProvider, withStorageRetry, } from "./index.js";
-function tree(keys) {
-    const root = {};
-    for (const key of keys) {
-        let node = root;
-        for (const part of key.split("/").filter(Boolean)) {
-            node[part] ??= {};
-            node = node[part];
-        }
-    }
-    function print(node, prefix = "") {
-        const entries = Object.keys(node).sort();
-        entries.forEach((name, index) => {
-            const last = index === entries.length - 1;
-            console.log(`${prefix}${last ? "└── " : "├── "}${name}`);
-            print(node[name], prefix + (last ? "    " : "│   "));
-        });
-    }
-    print(root);
-}
-async function main() {
-    const config = loadConfig();
-    const storage = withStorageRetry(createStorageProvider({
-        provider: config.storage.provider,
-        huggingface: config.storage.huggingface,
-        localRoot: config.storage.localRoot,
-    }), { attempts: 3 });
-    const objects = await storage.list("");
-    console.log(`Bucket: ${process.env.HF_BUCKET}`);
-    console.log(`Objects: ${objects.length}\n`);
-    tree(objects.map((x) => x.key));
-}
-main().catch((err) => {
-    console.error(err);
-    process.exit(1);
-});

package/dist/src/storage/local/client.js

@@ -1,99 +0,0 @@
-import { access, mkdir, readFile, readdir, rm, stat, writeFile, } from "node:fs/promises";
-import { dirname, join, relative, resolve, } from "node:path";
-export class LocalStorageProvider {
-    root;
-    name = "local";
-    constructor(root) {
-        this.root = root;
-    }
-    path(key) {
-        const safeKey = key.replace(/^\/+/, "");
-        return resolve(this.root, safeKey);
-    }
-    async put(key, data) {
-        const path = this.path(key);
-        await mkdir(dirname(path), {
-            recursive: true,
-        });
-        await writeFile(path, data);
-    }
-    async get(key) {
-        try {
-            return await readFile(this.path(key));
-        }
-        catch (error) {
-            if (typeof error === "object" &&
-                error !== null &&
-                "code" in error &&
-                error.code === "ENOENT") {
-                return null;
-            }
-            throw error;
-        }
-    }
-    async getText(key) {
-        const data = await this.get(key);
-        if (!data) {
-            return null;
-        }
-        return Buffer
-            .from(data)
-            .toString("utf8");
-    }
-    async exists(key) {
-        try {
-            await access(this.path(key));
-            return true;
-        }
-        catch {
-            return false;
-        }
-    }
-    async delete(key) {
-        await rm(this.path(key), {
-            force: true,
-        });
-    }
-    async list(prefix = "") {
-        const start = this.path(prefix);
-        const output = [];
-        try {
-            await access(start);
-        }
-        catch {
-            return output;
-        }
-        const walk = async (directory) => {
-            const entries = await readdir(directory, {
-                withFileTypes: true,
-            });
-            for (const entry of entries) {
-                const fullPath = join(directory, entry.name);
-                if (entry.isDirectory()) {
-                    await walk(fullPath);
-                    continue;
-                }
-                const info = await stat(fullPath);
-                output.push({
-                    key: relative(this.root, fullPath),
-                    size: info.size,
-                    updatedAt: info.mtime
-                        .toISOString(),
-                });
-            }
-        };
-        const info = await stat(start);
-        if (info.isDirectory()) {
-            await walk(start);
-        }
-        else {
-            output.push({
-                key: relative(this.root, start),
-                size: info.size,
-                updatedAt: info.mtime
-                    .toISOString(),
-            });
-        }
-        return output;
-    }
-}

package/dist/src/storage/local/index.js

@@ -1 +0,0 @@
-export * from "./client.js";

package/dist/src/storage/memory-store.js

@@ -1,30 +0,0 @@
-export class MemoryStore {
-    storage;
-    constructor(storage) {
-        this.storage = storage;
-    }
-    key(projectId) {
-        return [
-            "projects",
-            projectId,
-            "memories",
-            "current.json",
-        ].join("/");
-    }
-    async load(projectId) {
-        const text = await this.storage
-            .getText(this.key(projectId));
-        if (!text) {
-            return [];
-        }
-        const parsed = JSON.parse(text);
-        if (!Array.isArray(parsed)) {
-            throw new Error("Invalid memory snapshot");
-        }
-        return parsed;
-    }
-    async save(projectId, memories) {
-        const payload = JSON.stringify(memories, null, 2);
-        await this.storage.put(this.key(projectId), payload, "application/json");
-    }
-}

package/dist/src/storage/project/cleanup-layout-v1.js

@@ -1,72 +0,0 @@
-import "dotenv/config";
-import { loadConfig, } from "../../core/index.js";
-import { createStorageProvider, withStorageRetry, } from "../index.js";
-function destination(key) {
-    let m;
-    m =
-        key.match(/^(projects\/[^/]+\/)memories\/(.+)$/);
-    if (m) {
-        return `${m[1]}memory/records/${m[2]}`;
-    }
-    m =
-        key.match(/^(projects\/[^/]+\/)vectors\/(.+)$/);
-    if (m) {
-        return `${m[1]}memory/vectors/${m[2]}`;
-    }
-    m =
-        key.match(/^(projects\/[^/]+\/)graph\/(.+)$/);
-    if (m) {
-        return `${m[1]}code/graph/${m[2]}`;
-    }
-    m =
-        key.match(/^(projects\/[^/]+\/snapshots\/[^/]+\/)memories\/(.+)$/);
-    if (m) {
-        return `${m[1]}memory/records/${m[2]}`;
-    }
-    m =
-        key.match(/^(projects\/[^/]+\/snapshots\/[^/]+\/)vectors\/(.+)$/);
-    if (m) {
-        return `${m[1]}memory/vectors/${m[2]}`;
-    }
-    m =
-        key.match(/^(projects\/[^/]+\/snapshots\/[^/]+\/)graph\/(.+)$/);
-    if (m) {
-        return `${m[1]}code/graph/${m[2]}`;
-    }
-    return null;
-}
-async function main() {
-    const config = loadConfig();
-    const storage = withStorageRetry(createStorageProvider({
-        provider: config.storage.provider,
-        huggingface: config.storage.huggingface,
-        localRoot: config.storage.localRoot,
-    }), {
-        attempts: 3,
-    });
-    const objects = await storage.list("projects/");
-    const oldObjects = objects
-        .map((item) => ({
-        from: item.key,
-        to: destination(item.key),
-    }))
-        .filter((item) => Boolean(item.to));
-    let deleted = 0;
-    for (const item of oldObjects) {
-        const newExists = await storage.exists(item.to);
-        if (!newExists) {
-            throw new Error(`REFUSE DELETE: destination missing: ${item.to}`);
-        }
-        console.log(`DELETE OLD ${item.from}`);
-        await storage.delete(item.from);
-        deleted++;
-    }
-    console.log({
-        ok: true,
-        deleted,
-    });
-}
-main().catch((error) => {
-    console.error(error);
-    process.exit(1);
-});

package/dist/src/storage/project/folder.js

@@ -1,15 +0,0 @@
-export function sanitizeProjectFolder(name) {
-    const value = name
-        .trim()
-        .replace(/\s+/g, "_")
-        .replace(/[^A-Za-z0-9._-]/g, "_")
-        .replace(/_+/g, "_")
-        .replace(/^\.+|\.+$/g, "")
-        .slice(0, 100);
-    if (!value ||
-        value === "." ||
-        value === "..") {
-        throw new Error("Invalid project storage folder");
-    }
-    return value;
-}

package/dist/src/storage/project/index.js

@@ -1,3 +0,0 @@
-export * from "./folder.js";
-export * from "./scoped-provider.js";
-export * from "./layout.js";

package/dist/src/storage/project/layout.js

@@ -1,74 +0,0 @@
-export function mapProjectStoragePath(key) {
-    const clean = key.replace(/^\/+/, "");
-    /*
-     * Relative project keys
-     */
-    if (clean.startsWith("memories/")) {
-        return ("memory/records/" +
-            clean.slice("memories/".length));
-    }
-    if (clean.startsWith("vectors/")) {
-        return ("memory/vectors/" +
-            clean.slice("vectors/".length));
-    }
-    if (clean.startsWith("graph/")) {
-        return ("code/graph/" +
-            clean.slice("graph/".length));
-    }
-    /*
-     * Snapshot logical keys
-     */
-    let snapshotMatch = clean.match(/^(snapshots\/[^/]+\/)memories\/(.+)$/);
-    if (snapshotMatch) {
-        return `${snapshotMatch[1]}memory/records/${snapshotMatch[2]}`;
-    }
-    snapshotMatch =
-        clean.match(/^(snapshots\/[^/]+\/)vectors\/(.+)$/);
-    if (snapshotMatch) {
-        return `${snapshotMatch[1]}memory/vectors/${snapshotMatch[2]}`;
-    }
-    snapshotMatch =
-        clean.match(/^(snapshots\/[^/]+\/)graph\/(.+)$/);
-    if (snapshotMatch) {
-        return `${snapshotMatch[1]}code/graph/${snapshotMatch[2]}`;
-    }
-    /*
-     * Full project snapshot keys
-     */
-    let fullSnapshotMatch = clean.match(/^(projects\/[^/]+\/snapshots\/[^/]+\/)memories\/(.+)$/);
-    if (fullSnapshotMatch) {
-        return `${fullSnapshotMatch[1]}memory/records/${fullSnapshotMatch[2]}`;
-    }
-    fullSnapshotMatch =
-        clean.match(/^(projects\/[^/]+\/snapshots\/[^/]+\/)vectors\/(.+)$/);
-    if (fullSnapshotMatch) {
-        return `${fullSnapshotMatch[1]}memory/vectors/${fullSnapshotMatch[2]}`;
-    }
-    fullSnapshotMatch =
-        clean.match(/^(projects\/[^/]+\/snapshots\/[^/]+\/)graph\/(.+)$/);
-    if (fullSnapshotMatch) {
-        return `${fullSnapshotMatch[1]}code/graph/${fullSnapshotMatch[2]}`;
-    }
-    /*
-     * Full project keys
-     *
-     * projects/<project>/memories/...
-     * projects/<project>/vectors/...
-     * projects/<project>/graph/...
-     */
-    let match = clean.match(/^(projects\/[^/]+\/)memories\/(.+)$/);
-    if (match) {
-        return (`${match[1]}memory/records/${match[2]}`);
-    }
-    match =
-        clean.match(/^(projects\/[^/]+\/)vectors\/(.+)$/);
-    if (match) {
-        return (`${match[1]}memory/vectors/${match[2]}`);
-    }
-    match =
-        clean.match(/^(projects\/[^/]+\/)graph\/(.+)$/);
-    if (match) {
-        return (`${match[1]}code/graph/${match[2]}`);
-    }
-    return clean;
-}

package/dist/src/storage/project/migrate-layout-v2.js

@@ -1,101 +0,0 @@
-import "dotenv/config";
-import { createHash, } from "node:crypto";
-import { loadConfig, } from "../../core/index.js";
-import { createStorageProvider, withStorageRetry, } from "../index.js";
-function sha(text) {
-    return createHash("sha256")
-        .update(text)
-        .digest("hex");
-}
-function destination(key) {
-    let match;
-    /*
-     * Current state
-     */
-    match =
-        key.match(/^(projects\/[^/]+\/)memories\/(.+)$/);
-    if (match) {
-        return (`${match[1]}memory/records/${match[2]}`);
-    }
-    match =
-        key.match(/^(projects\/[^/]+\/)vectors\/(.+)$/);
-    if (match) {
-        return (`${match[1]}memory/vectors/${match[2]}`);
-    }
-    match =
-        key.match(/^(projects\/[^/]+\/)graph\/(.+)$/);
-    if (match) {
-        return (`${match[1]}code/graph/${match[2]}`);
-    }
-    /*
-     * Historical snapshots
-     */
-    match =
-        key.match(/^(projects\/[^/]+\/snapshots\/[^/]+\/)memories\/(.+)$/);
-    if (match) {
-        return (`${match[1]}memory/records/${match[2]}`);
-    }
-    match =
-        key.match(/^(projects\/[^/]+\/snapshots\/[^/]+\/)vectors\/(.+)$/);
-    if (match) {
-        return (`${match[1]}memory/vectors/${match[2]}`);
-    }
-    match =
-        key.match(/^(projects\/[^/]+\/snapshots\/[^/]+\/)graph\/(.+)$/);
-    if (match) {
-        return (`${match[1]}code/graph/${match[2]}`);
-    }
-    return null;
-}
-async function main() {
-    const config = loadConfig();
-    const storage = withStorageRetry(createStorageProvider({
-        provider: config.storage.provider,
-        huggingface: config.storage.huggingface,
-        localRoot: config.storage.localRoot,
-    }), {
-        attempts: 3,
-    });
-    const objects = await storage.list("projects/");
-    const migrations = objects
-        .map((item) => ({
-        from: item.key,
-        to: destination(item.key),
-    }))
-        .filter((item) => Boolean(item.to) &&
-        item.from !== item.to);
-    console.log(`Objects to migrate: ${migrations.length}`);
-    let copied = 0;
-    let verified = 0;
-    for (const item of migrations) {
-        const source = await storage.getText(item.from);
-        if (source === null ||
-            source === undefined) {
-            throw new Error(`Cannot read ${item.from}`);
-        }
-        console.log(`COPY\n  ${item.from}\n  -> ${item.to}`);
-        await storage.put(item.to, source, "application/json");
-        copied++;
-        const target = await storage.getText(item.to);
-        if (target === null ||
-            target === undefined) {
-            throw new Error(`Cannot verify ${item.to}`);
-        }
-        if (sha(source) !==
-            sha(target)) {
-            throw new Error(`Checksum mismatch:\n${item.from}\n${item.to}`);
-        }
-        verified++;
-    }
-    console.log({
-        ok: true,
-        copied,
-        verified,
-        deleted: 0,
-        message: "Old objects intentionally preserved until build/tests pass.",
-    });
-}
-main().catch((error) => {
-    console.error(error);
-    process.exit(1);
-});

package/dist/src/storage/project/migrate.js

@@ -1,73 +0,0 @@
-import "dotenv/config";
-import { loadConfig, ProjectManager, } from "../../core/index.js";
-import { createStorageProvider, withStorageRetry, } from "../index.js";
-import { sanitizeProjectFolder, } from "./folder.js";
-async function main() {
-    const config = loadConfig();
-    const project = new ProjectManager()
-        .detect();
-    const storage = withStorageRetry(createStorageProvider({
-        provider: config.storage.provider,
-        huggingface: config.storage.huggingface,
-        localRoot: config.storage.localRoot,
-    }), {
-        attempts: 3,
-    });
-    const folder = sanitizeProjectFolder(project.name);
-    const oldPrefix = `projects/${project.id}`;
-    const newPrefix = `projects/${folder}`;
-    if (oldPrefix ===
-        newPrefix) {
-        console.log({
-            ok: true,
-            migrated: 0,
-            folder,
-            message: "Already using named project folder",
-        });
-        return;
-    }
-    const objects = await storage.list(oldPrefix);
-    let migrated = 0;
-    for (const object of objects) {
-        const data = await storage.get(object.key);
-        if (!data) {
-            continue;
-        }
-        const newKey = newPrefix +
-            object.key.slice(oldPrefix.length);
-        await storage.put(newKey, data, "application/json");
-        migrated++;
-    }
-    /*
-     * Project metadata dễ đọc.
-     */
-    await storage.put(`${newPrefix}/project.json`, JSON.stringify({
-        version: 1,
-        projectId: project.id,
-        projectName: project.name,
-        storageFolder: folder,
-        migratedAt: new Date()
-            .toISOString(),
-    }, null, 2), "application/json");
-    /*
-     * Chỉ xoá folder hash sau khi copy xong.
-     */
-    if (migrated > 0) {
-        for (const object of objects) {
-            await storage.delete(object.key);
-        }
-    }
-    console.log({
-        ok: true,
-        project: project.name,
-        projectId: project.id,
-        folder,
-        migrated,
-        from: oldPrefix,
-        to: newPrefix,
-    });
-}
-main().catch((error) => {
-    console.error(error);
-    process.exit(1);
-});