yutia.db 1.3.2

package/dist/core/LRUCache.js

@@ -0,0 +1,36 @@
+// src/core/LRUCache.ts
+export class LRUCache {
+    maxEntries;
+    map = new Map();
+    constructor(maxEntries) {
+        this.maxEntries = maxEntries;
+    }
+    get(key) {
+        const v = this.map.get(key);
+        if (v === undefined)
+            return undefined;
+        this.map.delete(key);
+        this.map.set(key, v);
+        return v;
+    }
+    set(key, value) {
+        if (this.map.has(key))
+            this.map.delete(key);
+        this.map.set(key, value);
+        while (this.map.size > this.maxEntries) {
+            const oldest = this.map.keys().next().value;
+            if (oldest === undefined)
+                break;
+            this.map.delete(oldest);
+        }
+    }
+    delete(key) {
+        this.map.delete(key);
+    }
+    has(key) {
+        return this.map.has(key);
+    }
+    clear() {
+        this.map.clear();
+    }
+}

package/dist/core/Reader.d.ts

@@ -0,0 +1,11 @@
+import { Query } from "../types/index.js";
+import type { RecordPtr } from "./Writer.js";
+export declare class Reader {
+    private filename;
+    constructor(filename: string);
+    readByPointer<T>(ptr: RecordPtr): Promise<T | null>;
+    scan<T>(query: Query<T>): AsyncIterable<{
+        doc: T;
+        ptr: RecordPtr;
+    }>;
+}

package/dist/core/Reader.js

@@ -0,0 +1,66 @@
+// src/core/Reader.ts
+import fs from "fs";
+import { QueryMatcher } from "../utils/QueryMatcher.js";
+const MAGIC = Buffer.from("YDB3");
+export class Reader {
+    filename;
+    constructor(filename) {
+        this.filename = filename;
+    }
+    async readByPointer(ptr) {
+        const fd = fs.openSync(this.filename, "r");
+        try {
+            const buf = Buffer.allocUnsafe(ptr.length);
+            const got = fs.readSync(fd, buf, 0, ptr.length, ptr.offset);
+            if (got !== ptr.length)
+                return null;
+            return JSON.parse(buf.toString("utf8"));
+        }
+        finally {
+            fs.closeSync(fd);
+        }
+    }
+    async *scan(query) {
+        const fd = fs.openSync(this.filename, "r");
+        try {
+            const header = Buffer.allocUnsafe(5);
+            const hGot = fs.readSync(fd, header, 0, 5, 0);
+            if (hGot < 5)
+                return;
+            if (!header.subarray(0, 4).equals(MAGIC))
+                return;
+            let pos = 5;
+            const st = fs.fstatSync(fd);
+            const end = st.size;
+            const lenBuf = Buffer.allocUnsafe(4);
+            while (pos + 4 <= end) {
+                const gotLen = fs.readSync(fd, lenBuf, 0, 4, pos);
+                if (gotLen !== 4)
+                    break;
+                const len = lenBuf.readUInt32LE(0);
+                const jsonOffset = pos + 4;
+                const next = jsonOffset + len;
+                // recovery-safe: if record cut off, stop scanning
+                if (next > end)
+                    break;
+                const payload = Buffer.allocUnsafe(len);
+                const got = fs.readSync(fd, payload, 0, len, jsonOffset);
+                if (got !== len)
+                    break;
+                try {
+                    const doc = JSON.parse(payload.toString("utf8"));
+                    if (QueryMatcher.match(doc, query)) {
+                        yield { doc, ptr: { offset: jsonOffset, length: len } };
+                    }
+                }
+                catch {
+                    // skip corrupt record but continue
+                }
+                pos = next;
+            }
+        }
+        finally {
+            fs.closeSync(fd);
+        }
+    }
+}

package/dist/core/Recovery.d.ts

@@ -0,0 +1,6 @@
+export declare class Recovery {
+    static recoverFile(filename: string): Promise<{
+        truncated: boolean;
+        validBytes: number;
+    }>;
+}

package/dist/core/Recovery.js

@@ -0,0 +1,66 @@
+// src/core/Recovery.ts
+import { open } from "fs/promises";
+import { BlockCodec } from "./BlockCodec.js";
+import { decompress } from "./Compression.js";
+export class Recovery {
+    static async recoverFile(filename) {
+        try {
+            const fd = await open(filename, "r+");
+            try {
+                const stat = await fd.stat();
+                const size = stat.size;
+                if (size === 0)
+                    return { truncated: false, validBytes: 0 };
+                const CHUNK = 4 * 1024 * 1024;
+                let fileOffset = 0;
+                let carry = Buffer.alloc(0);
+                let lastGoodAbs = 0;
+                while (fileOffset < size) {
+                    const toRead = Math.min(CHUNK, size - fileOffset);
+                    const buf = Buffer.alloc(toRead);
+                    const { bytesRead } = await fd.read(buf, 0, toRead, fileOffset);
+                    if (bytesRead <= 0)
+                        break;
+                    let buffer = Buffer.concat([carry, buf.subarray(0, bytesRead)]);
+                    let off = 0;
+                    while (true) {
+                        const d = BlockCodec.tryDecodeHeader(buffer, off);
+                        if (!d)
+                            break;
+                        if (!d.ok) {
+                            off = d.nextOffset;
+                            continue;
+                        }
+                        const payload = buffer.subarray(d.payloadStart, d.payloadEnd);
+                        let raw;
+                        try {
+                            raw = decompress(d.header.algo, d.header.rawLen, payload);
+                        }
+                        catch {
+                            break;
+                        }
+                        if (!BlockCodec.verifyRaw(raw, d.header.crc))
+                            break;
+                        lastGoodAbs = fileOffset - carry.length + d.payloadEnd;
+                        off = d.payloadEnd;
+                    }
+                    carry = buffer.subarray(off);
+                    fileOffset += bytesRead;
+                }
+                if (lastGoodAbs < size) {
+                    await fd.truncate(lastGoodAbs);
+                    return { truncated: true, validBytes: lastGoodAbs };
+                }
+                return { truncated: false, validBytes: size };
+            }
+            finally {
+                await fd.close();
+            }
+        }
+        catch (err) {
+            if (err?.code === "ENOENT")
+                return { truncated: false, validBytes: 0 };
+            throw err;
+        }
+    }
+}

package/dist/core/SecondaryIndex.d.ts

@@ -0,0 +1,10 @@
+export declare class SecondaryIndex {
+    private fields;
+    private idx;
+    ensure(field: string): void;
+    addDoc(doc: any): void;
+    removeIdEverywhere(id: string): void;
+    getIds(field: string, value: any): Set<string> | null;
+    listFields(): string[];
+    clear(): void;
+}

package/dist/core/SecondaryIndex.js

@@ -0,0 +1,45 @@
+// src/core/SecondaryIndex.ts
+export class SecondaryIndex {
+    fields = new Set();
+    idx = new Map();
+    ensure(field) {
+        this.fields.add(field);
+        if (!this.idx.has(field))
+            this.idx.set(field, new Map());
+    }
+    addDoc(doc) {
+        const id = String(doc?._id ?? "");
+        if (!id)
+            return;
+        for (const f of this.fields) {
+            const v0 = doc[f];
+            if (v0 === undefined || v0 === null)
+                continue;
+            const v = String(v0);
+            const m = this.idx.get(f);
+            if (!m.has(v))
+                m.set(v, new Set());
+            m.get(v).add(id);
+        }
+    }
+    removeIdEverywhere(id) {
+        for (const [, m] of this.idx) {
+            for (const [, s] of m)
+                s.delete(id);
+        }
+    }
+    getIds(field, value) {
+        const m = this.idx.get(field);
+        if (!m)
+            return null;
+        const v = value === undefined || value === null ? "" : String(value);
+        return m.get(v) ?? null;
+    }
+    listFields() {
+        return [...this.fields];
+    }
+    clear() {
+        this.idx.clear();
+        this.fields.clear();
+    }
+}

package/dist/core/Writer.d.ts

@@ -0,0 +1,45 @@
+export type Durability = "none" | "batched" | "immediate";
+export type RecordPtr = {
+    offset: number;
+    length: number;
+};
+type WriterOptions = {
+    batchBytes?: number;
+    autoFlushMs?: number;
+    durability?: Durability;
+    fsyncEveryFlush?: number;
+};
+export declare class Writer {
+    private filename;
+    private fd;
+    private offset;
+    private queue;
+    private queuedBytes;
+    private batchBytes;
+    private autoFlushMs;
+    private durability;
+    private fsyncEveryFlush;
+    private flushTimer;
+    private flushing;
+    private closed;
+    private flushCount;
+    totalEnqueued: number;
+    totalFlushedRecords: number;
+    constructor(filename: string, opt?: WriterOptions);
+    open(): Promise<void>;
+    /**
+     * Super-fast enqueue. DOES NOT BLOCK on disk.
+     * Returns pointer info that will be valid after flush.
+     * (If you need pointer immediately, call drain/flush)
+     */
+    enqueue(json: string): {
+        ptr: RecordPtr;
+        recordBytes: number;
+    };
+    getPendingCount(): number;
+    getPendingBytes(): number;
+    flush(): Promise<void>;
+    drain(): Promise<void>;
+    close(): Promise<void>;
+}
+export {};

package/dist/core/Writer.js

@@ -0,0 +1,151 @@
+// src/core/Writer.ts
+import { promises as fsp } from "fs";
+import fs from "fs";
+const MAGIC = Buffer.from("YDB3"); // 4 bytes
+const VERSION = 1;
+function writeFsync(fd) {
+    return new Promise((resolve, reject) => {
+        fs.fsync(fd, (err) => (err ? reject(err) : resolve()));
+    });
+}
+export class Writer {
+    filename;
+    fd = null;
+    offset = 0;
+    queue = [];
+    queuedBytes = 0;
+    batchBytes;
+    autoFlushMs;
+    durability;
+    fsyncEveryFlush;
+    flushTimer = null;
+    flushing = false;
+    closed = false;
+    flushCount = 0;
+    // stats
+    totalEnqueued = 0;
+    totalFlushedRecords = 0;
+    constructor(filename, opt = {}) {
+        this.filename = filename;
+        this.batchBytes = opt.batchBytes ?? 4 * 1024 * 1024; // 4MB default
+        this.autoFlushMs = opt.autoFlushMs ?? 50;
+        this.durability = opt.durability ?? "batched";
+        this.fsyncEveryFlush = opt.fsyncEveryFlush ?? 1;
+        this.flushTimer = setInterval(() => {
+            void this.flush().catch(() => { });
+        }, this.autoFlushMs);
+    }
+    async open() {
+        if (this.fd !== null)
+            return;
+        await fsp
+            .mkdir(require("path").dirname(this.filename), { recursive: true })
+            .catch(() => { });
+        const exists = await fsp
+            .stat(this.filename)
+            .then(() => true)
+            .catch(() => false);
+        this.fd = fs.openSync(this.filename, "a+");
+        if (!exists) {
+            // write header
+            const header = Buffer.alloc(5);
+            MAGIC.copy(header, 0);
+            header.writeUInt8(VERSION, 4);
+            fs.writeSync(this.fd, header);
+            this.offset = header.length;
+        }
+        else {
+            const st = fs.fstatSync(this.fd);
+            this.offset = st.size;
+            // if file too small / invalid, you can handle here, but we keep it simple
+        }
+    }
+    /**
+     * Super-fast enqueue. DOES NOT BLOCK on disk.
+     * Returns pointer info that will be valid after flush.
+     * (If you need pointer immediately, call drain/flush)
+     */
+    enqueue(json) {
+        if (this.closed)
+            throw new Error("Writer closed");
+        const payload = Buffer.from(json, "utf8");
+        const len = payload.length;
+        const frame = Buffer.allocUnsafe(4 + len);
+        frame.writeUInt32LE(len, 0);
+        payload.copy(frame, 4);
+        // ptr offset points to JSON bytes (not including length prefix)
+        const ptr = {
+            offset: this.offset + this.queuedBytes + 4,
+            length: len,
+        };
+        this.queue.push(frame);
+        this.queuedBytes += frame.length;
+        this.totalEnqueued++;
+        // trigger flush without awaiting
+        if (this.queuedBytes >= this.batchBytes) {
+            void this.flush().catch(() => { });
+        }
+        return { ptr, recordBytes: frame.length };
+    }
+    getPendingCount() {
+        // pending records count
+        return this.queue.length;
+    }
+    getPendingBytes() {
+        return this.queuedBytes;
+    }
+    async flush() {
+        if (this.closed)
+            return;
+        if (this.flushing)
+            return;
+        if (this.queue.length === 0)
+            return;
+        this.flushing = true;
+        try {
+            await this.open();
+            const fd = this.fd;
+            const batch = this.queue;
+            this.queue = [];
+            const bytes = this.queuedBytes;
+            this.queuedBytes = 0;
+            // concat once per flush (big, but far less GC than per-doc concat)
+            const buf = Buffer.concat(batch, bytes);
+            // write at end (append)
+            fs.writeSync(fd, buf, 0, buf.length, this.offset);
+            this.offset += buf.length;
+            this.totalFlushedRecords += batch.length;
+            this.flushCount++;
+            if (this.durability === "immediate") {
+                await writeFsync(fd);
+            }
+            else if (this.durability === "batched") {
+                if (this.flushCount % this.fsyncEveryFlush === 0) {
+                    await writeFsync(fd);
+                }
+            }
+        }
+        finally {
+            this.flushing = false;
+        }
+    }
+    async drain() {
+        // ensure all queued data is on disk
+        while (this.queue.length > 0 || this.flushing) {
+            await this.flush();
+            await new Promise((r) => setTimeout(r, 5));
+        }
+    }
+    async close() {
+        if (this.closed)
+            return;
+        this.closed = true;
+        if (this.flushTimer)
+            clearInterval(this.flushTimer);
+        await this.drain();
+        if (this.fd !== null) {
+            fs.closeSync(this.fd);
+            this.fd = null;
+        }
+    }
+}

package/dist/index.d.ts

@@ -0,0 +1,3 @@
+export { Datastore } from "./core/Datastore.js";
+export { Compactor } from "./core/Compactor.js";
+export * from "./types/index.js";

package/dist/index.js

@@ -0,0 +1,3 @@
+export { Datastore } from "./core/Datastore.js";
+export { Compactor } from "./core/Compactor.js";
+export * from "./types/index.js";

package/dist/types/index.d.ts

@@ -0,0 +1,39 @@
+export type CompressionLevel = "off" | "fast" | "balanced" | "max";
+export interface YutiaOptions {
+    filename: string;
+    autoload?: boolean;
+    durability?: "immediate" | "batched";
+    compressionLevel?: CompressionLevel;
+    blockTargetBytes?: number;
+    maxCacheEntries?: number;
+    autoFlushMs?: number;
+    writeBatchSize?: number;
+    corruptAction?: "error" | "ignore" | "log";
+}
+export interface Doc {
+    _id?: string;
+    createdAt?: number;
+    updatedAt?: number;
+    _deleted?: boolean;
+    [key: string]: any;
+}
+export type Query<T> = Partial<T> & {
+    $gt?: any;
+    $lt?: any;
+    $gte?: any;
+    $lte?: any;
+    $ne?: any;
+    $in?: any[];
+    $nin?: any[];
+    $regex?: RegExp;
+    $exists?: boolean;
+};
+export interface UpdateQuery {
+    $set?: Partial<any>;
+    $inc?: {
+        [key: string]: number;
+    };
+    $unset?: {
+        [key: string]: 1;
+    };
+}

package/dist/types/index.js

@@ -0,0 +1 @@
+export {};

package/dist/utils/Checksum.d.ts

@@ -0,0 +1,4 @@
+export declare class Checksum {
+    static calculate(data: string): string;
+    static verify(data: string, hash: string): boolean;
+}

package/dist/utils/Checksum.js

@@ -0,0 +1,10 @@
+// src/utils/Checksum.ts
+import { createHash } from "crypto";
+export class Checksum {
+    static calculate(data) {
+        return createHash("sha256").update(data).digest("hex");
+    }
+    static verify(data, hash) {
+        return this.calculate(data) === hash;
+    }
+}

package/dist/utils/Crypto.d.ts

@@ -0,0 +1,6 @@
+export declare class CryptoManager {
+    private key;
+    constructor(secret: string);
+    encrypt(text: string): string;
+    decrypt(encryptedText: string): string;
+}

package/dist/utils/Crypto.js

@@ -0,0 +1,26 @@
+// src/utils/Crypto.ts
+import { randomBytes, createCipheriv, createDecipheriv } from "crypto";
+// Simplified placeholder for Encryption logic
+export class CryptoManager {
+    key;
+    constructor(secret) {
+        // In production, derive key properly (e.g., PBKDF2)
+        this.key = Buffer.from(secret.padEnd(32, "0").slice(0, 32));
+    }
+    encrypt(text) {
+        const iv = randomBytes(16);
+        const cipher = createCipheriv("aes-256-cbc", this.key, iv);
+        let encrypted = cipher.update(text, "utf8", "hex");
+        encrypted += cipher.final("hex");
+        return `${iv.toString("hex")}:${encrypted}`;
+    }
+    decrypt(encryptedText) {
+        const parts = encryptedText.split(":");
+        const iv = Buffer.from(parts[0], "hex");
+        const encrypted = parts[1];
+        const decipher = createDecipheriv("aes-256-cbc", this.key, iv);
+        let decrypted = decipher.update(encrypted, "hex", "utf8");
+        decrypted += decipher.final("utf8");
+        return decrypted;
+    }
+}

package/dist/utils/QueryMatcher.d.ts

@@ -0,0 +1,5 @@
+import { Query } from "../types/index.js";
+export declare class QueryMatcher {
+    static match<T extends any>(doc: T, query: Query<T>): boolean;
+    private static matchField;
+}

package/dist/utils/QueryMatcher.js

@@ -0,0 +1,55 @@
+const BAD_KEYS = new Set(["__proto__", "prototype", "constructor"]);
+const isBadKey = (k) => BAD_KEYS.has(k);
+export class QueryMatcher {
+    static match(doc, query) {
+        for (const key in query) {
+            if (isBadKey(key))
+                return false;
+            const qv = query[key];
+            const dv = doc?.[key];
+            if (!this.matchField(dv, qv))
+                return false;
+        }
+        return true;
+    }
+    static matchField(docVal, queryVal) {
+        if (queryVal instanceof RegExp) {
+            return typeof docVal === "string" && queryVal.test(docVal);
+        }
+        if (typeof queryVal === "object" &&
+            queryVal !== null &&
+            !Array.isArray(queryVal)) {
+            if ("$ne" in queryVal && docVal === queryVal.$ne)
+                return false;
+            if ("$gt" in queryVal && !(docVal > queryVal.$gt))
+                return false;
+            if ("$lt" in queryVal && !(docVal < queryVal.$lt))
+                return false;
+            if ("$gte" in queryVal && !(docVal >= queryVal.$gte))
+                return false;
+            if ("$lte" in queryVal && !(docVal <= queryVal.$lte))
+                return false;
+            if ("$in" in queryVal &&
+                Array.isArray(queryVal.$in) &&
+                !queryVal.$in.includes(docVal))
+                return false;
+            if ("$nin" in queryVal &&
+                Array.isArray(queryVal.$nin) &&
+                queryVal.$nin.includes(docVal))
+                return false;
+            if ("$exists" in queryVal) {
+                const exists = docVal !== undefined && docVal !== null;
+                if (exists !== Boolean(queryVal.$exists))
+                    return false;
+            }
+            if ("$regex" in queryVal) {
+                const rx = queryVal.$regex instanceof RegExp ? queryVal.$regex : null;
+                if (!rx)
+                    return false;
+                return typeof docVal === "string" && rx.test(docVal);
+            }
+            return true;
+        }
+        return docVal === queryVal;
+    }
+}

package/package.json

@@ -0,0 +1,63 @@
+{
+  "name": "yutia.db",
+  "version": "1.3.2",
+  "description": "Ultra-lightweight embedded database for Realtime Logs and AI Datasets.",
+  "author": "Clayza Aubert <https://clayza.is-a.dev> (https://github.com/clayzaaubert)",
+  "license": "MIT",
+  "type": "module",
+  "main": "./dist/index.js",
+  "types": "./dist/index.d.ts",
+  "exports": {
+    ".": {
+      "types": "./dist/index.d.ts",
+      "import": "./dist/index.js"
+    }
+  },
+  "files": [
+    "dist",
+    "README.md",
+    "LICENSE",
+    "package.json"
+  ],
+  "scripts": {
+    "build": "tsc",
+    "lint": "eslint src",
+    "prepublishOnly": "npm run build",
+    "stress": "node --enable-source-maps test/stress/stress.ts",
+    "stress-bun": "bun test/stress/stress.ts"
+  },
+  "engines": {
+    "node": ">=20.0.0"
+  },
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/clayzaaubert/yutia-db"
+  },
+  "homepage": "https://github.com/clayzaaubert/yutia-db#readme",
+  "bugs": {
+    "url": "https://github.com/clayzaaubert/yutia-db/issues"
+  },
+  "keywords": [
+    "embedded-database",
+    "lightweight-db",
+    "realtime-logs",
+    "ai-datasets",
+    "nedb-alternative",
+    "json-database",
+    "local-database",
+    "typescript",
+    "yutia-db",
+    "nodejs"
+  ],
+  "dependencies": {
+    "eventemitter3": "^5.0.1",
+    "p-queue": "^8.0.0"
+  },
+  "devDependencies": {
+    "@types/node": "^25.0.9",
+    "@types/p-queue": "^3.2.1",
+    "jest": "^29.7.0",
+    "ts-jest": "^29.1.1",
+    "typescript": "^5.3.0"
+  }
+}