yutia.db 1.3.2

package/README.md

@@ -0,0 +1,159 @@
+# YutiaDB (YDB3)
+
+Ultra-lightweight embedded database optimized for **realtime logs**, **event streams**, and **AI datasets** — with crash-safe append-only storage and fast by-id reads.
+
+> Storage format: **YDB3** = binary framing + JSON payload  
+> (`"YDB3" + version + [u32 len][json bytes]...`)
+
+## Why YutiaDB?
+
+YutiaDB is designed for workloads where **writes are massive & continuous**, and you need:
+- **Very fast ingestion** (append-only)
+- **Crash-safety** (power loss safe; last partial record is ignored on recovery)
+- **Small memory overhead** (streaming reads, small cache, optional pointer index)
+- **Easy export to JSON/JSONL** for analytics / AI training
+
+This is **not** a replacement for full-feature query databases (MongoDB/Postgres).  
+It’s a purpose-built embedded DB for **append-heavy** use cases.
+
+---
+
+## Features
+
+- ✅ Append-only realtime ingestion (framed JSON)
+- ✅ Crash-safe recovery (truncated tail-safe)
+- ✅ Fast `findOne({_id})` via pointer lookup (O(1) per read)
+- ✅ Streaming scan for analytics (no full file load)
+- ✅ Tombstone deletes (`_deleted: true`)
+- ✅ Lightweight LRU-ish cache for hot records
+- ✅ Works great for: logs, telemetry, dataset storage, user DB (by-id)
+
+---
+
+## Install (npm)
+
+```bash
+npm i yutiadb
+# or
+bun add yutiadb
+# or
+pnpm add yutiadb
+````
+
+---
+
+## Quick Start
+
+```ts
+import { Datastore } from "yutiadb";
+
+const db = new Datastore({
+  filename: "./data/app.ydb",
+  autoload: true,
+
+  // performance/durability tuning
+  durability: "batched",    // "none" | "batched" | "immediate"
+  batchBytes: 4 * 1024 * 1024,
+  autoFlushMs: 50,
+  fsyncEveryFlush: 1,
+
+  maxCacheEntries: 1000,
+  maxPendingDocs: 500_000,
+} as any);
+
+// write (realtime)
+await db.insert({ type: "req", path: "/api/v1/items", ts: Date.now() });
+
+// read by _id (fast)
+const one = await db.findOne({ _id: "..." });
+
+// scan query (stream scan)
+const items = await db.find({ type: "req" });
+
+// ensure durability (optional)
+await db.flush();  // flush buffered writes
+await db.drain();  // wait until all pending buffered writes are written
+
+await db.close();
+```
+
+---
+
+## Durability Modes
+
+* `durability: "none"`
+  Fastest. Data may be in OS buffers; power loss might lose last writes.
+
+* `durability: "batched"` (recommended)
+  Writes are buffered and fsync happens periodically (`fsyncEveryFlush`).
+
+* `durability: "immediate"`
+  Safest (fsync every flush). Slowest.
+
+**Tip for logs:** use `"batched"` and compact/export offline if needed.
+
+---
+
+## Storage Format (YDB3)
+
+File layout:
+
+* Header:
+
+  * `YDB3` (4 bytes)
+  * version (1 byte)
+
+* Records:
+
+  * `len` (UInt32LE, 4 bytes)
+  * `payload` (UTF-8 JSON bytes)
+
+Recovery behavior:
+
+* If a crash truncates the last record, reader stops safely at the last valid record.
+
+---
+
+## Use Cases
+
+### Realtime Logs (Web/API)
+
+* request logs
+* audit logs
+* webhook logs
+* notification logs
+
+### Dataset for AI
+
+* store training samples (JSON)
+* export to JSONL later
+* sequential scan for training
+
+### User DB (by-id)
+
+* fast `findOne({_id})`
+* tombstone deletes
+* optional compaction
+
+---
+
+## Limitations
+
+* Query engine is intentionally minimal (stream scan + by-id pointer)
+* Single-process embedded DB (not a network DB)
+* Multi-field secondary indexing is not included (yet)
+
+---
+
+## Roadmap (optional)
+
+* [ ] Time-window compaction for logs (keep last N days)
+* [ ] Index snapshot `.idx` for faster startup
+* [ ] Tail / live stream tool (`tail -f` for YDB)
+* [ ] Export JSONL tool
+
+---
+
+## License
+
+MIT

package/dist/core/BlockCodec.d.ts

@@ -0,0 +1,46 @@
+export type RecordPointer = {
+    blockOffset: number;
+    blockLength: number;
+    recOffset: number;
+    recLength: number;
+    deleted?: boolean;
+    updatedAt?: number;
+    createdAt?: number;
+};
+export declare class BlockCodec {
+    static HEADER_BYTES: number;
+    static encodeBlock(params: {
+        raw: Buffer;
+        algo: number;
+        flags: number;
+        payload: Buffer;
+    }): Buffer;
+    static tryDecodeHeader(buf: Buffer, offset: number): {
+        ok: false;
+        reason: string;
+        nextOffset: number;
+        header?: undefined;
+        payloadStart?: undefined;
+        payloadEnd?: undefined;
+    } | {
+        ok: true;
+        header: {
+            version: number;
+            flags: number;
+            algo: number;
+            blockLen: number;
+            rawLen: number;
+            crc: number;
+        };
+        payloadStart: number;
+        payloadEnd: number;
+        nextOffset: number;
+        reason?: undefined;
+    } | null;
+    static verifyRaw(raw: Buffer, crc: number): boolean;
+    static iterateRecords(raw: Buffer): Generator<{
+        recOffset: number;
+        recLength: number;
+    }, void, unknown>;
+    static buildRaw(recordsJson: string[]): Buffer<ArrayBuffer>;
+}

package/dist/core/BlockCodec.js

@@ -0,0 +1,84 @@
+// src/core/BlockCodec.ts
+import { createHash } from "crypto";
+const MAGIC = Buffer.from("YDB2");
+// magic(4) ver(1) flags(1) algo(1) reserved(1) blockLen(u32) rawLen(u32) crc(u32)
+const HEADER_BYTES = 4 + 1 + 1 + 1 + 1 + 4 + 4 + 4;
+function crc32ish(buf) {
+    const h = createHash("sha256").update(buf).digest();
+    return h.readUInt32LE(0);
+}
+export class BlockCodec {
+    static HEADER_BYTES = HEADER_BYTES;
+    static encodeBlock(params) {
+        const { raw, algo, flags, payload } = params;
+        const header = Buffer.alloc(HEADER_BYTES);
+        MAGIC.copy(header, 0);
+        header.writeUInt8(2, 4);
+        header.writeUInt8(flags, 5);
+        header.writeUInt8(algo, 6);
+        header.writeUInt8(0, 7);
+        header.writeUInt32LE(payload.length, 8);
+        header.writeUInt32LE(raw.length, 12);
+        header.writeUInt32LE(crc32ish(raw), 16);
+        return Buffer.concat([header, payload]);
+    }
+    static tryDecodeHeader(buf, offset) {
+        if (buf.length - offset < HEADER_BYTES)
+            return null;
+        if (buf[offset] !== MAGIC[0] ||
+            buf[offset + 1] !== MAGIC[1] ||
+            buf[offset + 2] !== MAGIC[2] ||
+            buf[offset + 3] !== MAGIC[3]) {
+            return {
+                ok: false,
+                reason: "bad_magic",
+                nextOffset: offset + 1,
+            };
+        }
+        const version = buf.readUInt8(offset + 4);
+        const flags = buf.readUInt8(offset + 5);
+        const algo = buf.readUInt8(offset + 6);
+        const blockLen = buf.readUInt32LE(offset + 8);
+        const rawLen = buf.readUInt32LE(offset + 12);
+        const crc = buf.readUInt32LE(offset + 16);
+        const payloadStart = offset + HEADER_BYTES;
+        const payloadEnd = payloadStart + blockLen;
+        if (payloadEnd > buf.length)
+            return null;
+        return {
+            ok: true,
+            header: { version, flags, algo, blockLen, rawLen, crc },
+            payloadStart,
+            payloadEnd,
+            nextOffset: payloadEnd,
+        };
+    }
+    static verifyRaw(raw, crc) {
+        return crc32ish(raw) === crc;
+    }
+    // records inside raw: len(u32) + json bytes
+    static *iterateRecords(raw) {
+        let off = 0;
+        while (off + 4 <= raw.length) {
+            const len = raw.readUInt32LE(off);
+            const start = off + 4;
+            const end = start + len;
+            if (end > raw.length)
+                break;
+            yield { recOffset: start, recLength: len };
+            off = end;
+        }
+    }
+    static buildRaw(recordsJson) {
+        const parts = [];
+        let total = 0;
+        for (const j of recordsJson) {
+            const payload = Buffer.from(j, "utf8");
+            const header = Buffer.alloc(4);
+            header.writeUInt32LE(payload.length, 0);
+            parts.push(header, payload);
+            total += 4 + payload.length;
+        }
+        return Buffer.concat(parts, total);
+    }
+}

package/dist/core/Compactor.d.ts

@@ -0,0 +1,12 @@
+export type CompactOptions = {
+    keepBackup?: boolean;
+    batchBytes?: number;
+    autoFlushMs?: number;
+    durability?: "none" | "batched" | "immediate";
+    fsyncEveryFlush?: number;
+};
+export declare class Compactor {
+    private filename;
+    constructor(filename: string);
+    compact(opt?: CompactOptions): Promise<void>;
+}

package/dist/core/Compactor.js

@@ -0,0 +1,68 @@
+// src/core/Compactor.ts
+import { promises as fs } from "fs";
+import { join, dirname } from "path";
+import { Reader } from "./Reader.js";
+import { Writer } from "./Writer.js";
+export class Compactor {
+    filename;
+    constructor(filename) {
+        this.filename = filename;
+    }
+    async compact(opt = {}) {
+        const keepBackup = opt.keepBackup ?? false;
+        const dir = dirname(this.filename);
+        const base = this.filename.split(/[\\/]/).pop() || "db.ydb";
+        const tmp = join(dir, base + ".tmp");
+        const bak = join(dir, base + ".bak");
+        await fs.mkdir(dir, { recursive: true }).catch(() => { });
+        await fs.rm(tmp, { force: true }).catch(() => { });
+        const reader = new Reader(this.filename);
+        // Dedupe: keep only latest version per _id
+        // NOTE: For massive DB, this Map will eat RAM.
+        // For logs use-case, you can disable dedupe (append-only) or compact by time window.
+        const latest = new Map();
+        for await (const { doc } of reader.scan({})) {
+            if (!doc || !doc._id)
+                continue;
+            if (doc._deleted) {
+                latest.delete(doc._id);
+                continue;
+            }
+            const id = String(doc._id);
+            const prev = latest.get(id);
+            if (!prev) {
+                latest.set(id, doc);
+                continue;
+            }
+            const prevTs = Number(prev.updatedAt ?? prev.createdAt ?? 0);
+            const curTs = Number(doc.updatedAt ?? doc.createdAt ?? 0);
+            if (curTs >= prevTs)
+                latest.set(id, doc);
+        }
+        // Write compacted file using NEW Writer (YDB3)
+        const writer = new Writer(tmp, {
+            batchBytes: opt.batchBytes ?? 4 * 1024 * 1024,
+            autoFlushMs: opt.autoFlushMs ?? 50,
+            durability: opt.durability ?? "batched",
+            fsyncEveryFlush: opt.fsyncEveryFlush ?? 1,
+        });
+        // Write all docs
+        for (const doc of latest.values()) {
+            writer.enqueue(JSON.stringify(doc));
+        }
+        await writer.drain();
+        await writer.close();
+        // Atomic replace
+        // Optionally keep backup
+        if (keepBackup) {
+            await fs.rm(bak, { force: true }).catch(() => { });
+            await fs.rename(this.filename, bak).catch(() => { });
+        }
+        else {
+            await fs.rm(this.filename, { force: true }).catch(() => { });
+        }
+        await fs.rename(tmp, this.filename);
+        // cleanup
+        await fs.rm(tmp, { force: true }).catch(() => { });
+    }
+}

package/dist/core/Compression.d.ts

@@ -0,0 +1,9 @@
+import { CompressionLevel } from "../types/index.js";
+export type Algo = 0 | 1 | 2;
+export declare function pickAlgo(level: CompressionLevel): Algo;
+export declare function compress(level: CompressionLevel, raw: Buffer): {
+    algo: Algo;
+    payload: Buffer;
+    flags: number;
+};
+export declare function decompress(algo: Algo, rawLen: number, payload: Buffer): Buffer;

package/dist/core/Compression.js

@@ -0,0 +1,38 @@
+// src/core/Compression.ts
+import zlib from "zlib";
+export function pickAlgo(level) {
+    if (level === "off")
+        return 0;
+    return 2; // brotli default (small file)
+}
+export function compress(level, raw) {
+    const algo = pickAlgo(level);
+    if (algo === 0)
+        return { algo, payload: raw, flags: 0 };
+    if (algo === 1) {
+        const gzLevel = level === "fast" ? 1 : level === "balanced" ? 6 : 9;
+        return { algo, payload: zlib.gzipSync(raw, { level: gzLevel }), flags: 1 };
+    }
+    const q = level === "fast" ? 3 : level === "balanced" ? 5 : 9;
+    return {
+        algo,
+        payload: zlib.brotliCompressSync(raw, {
+            params: {
+                [zlib.constants.BROTLI_PARAM_QUALITY]: q,
+                [zlib.constants.BROTLI_PARAM_SIZE_HINT]: raw.length,
+            },
+        }),
+        flags: 1,
+    };
+}
+export function decompress(algo, rawLen, payload) {
+    if (algo === 0)
+        return payload;
+    if (algo === 1)
+        return zlib.gunzipSync(payload);
+    const out = zlib.brotliDecompressSync(payload);
+    // rawLen mismatch not fatal; CRC will decide
+    if (rawLen > 0 && out.length !== rawLen) {
+    }
+    return out;
+}

package/dist/core/Datastore.d.ts

@@ -0,0 +1,37 @@
+import EventEmitter from "eventemitter3";
+import { Doc, Query, UpdateQuery, YutiaOptions } from "../types/index.js";
+import { Durability } from "./Writer.js";
+export declare class Datastore extends EventEmitter {
+    private filename;
+    private writer;
+    private reader;
+    private q;
+    private byId;
+    private cache;
+    private cacheLimit;
+    private maxPendingDocs;
+    private closed;
+    constructor(opt: YutiaOptions & {
+        durability?: Durability;
+        batchBytes?: number;
+        autoFlushMs?: number;
+        fsyncEveryFlush?: number;
+        maxCacheEntries?: number;
+        maxPendingDocs?: number;
+        autoload?: boolean;
+    });
+    private touchCache;
+    load(): Promise<void>;
+    /**
+     * Realtime insert: never blocks on disk.
+     * If you need guaranteed durability NOW, call await db.flush() or set durability=immediate.
+     */
+    insert(doc: Doc): Promise<Doc>;
+    flush(): Promise<void>;
+    drain(): Promise<void>;
+    findOne(query: Query<Doc>): Promise<Doc | null>;
+    find(query: Query<Doc>): Promise<Doc[]>;
+    update(query: Query<Doc>, update: UpdateQuery): Promise<number>;
+    remove(query: Query<Doc>): Promise<number>;
+    close(): Promise<void>;
+}

package/dist/core/Datastore.js

@@ -0,0 +1,184 @@
+// src/core/Datastore.ts
+import EventEmitter from "eventemitter3";
+import { randomUUID } from "crypto";
+import PQueue from "p-queue";
+import { Writer } from "./Writer.js";
+import { Reader } from "./Reader.js";
+export class Datastore extends EventEmitter {
+    filename;
+    writer;
+    reader;
+    // serialize only heavy ops
+    q = new PQueue({ concurrency: 1 });
+    // pointer index
+    byId = new Map();
+    // lightweight cache
+    cache = new Map();
+    cacheLimit;
+    // backpressure
+    maxPendingDocs;
+    closed = false;
+    constructor(opt) {
+        super();
+        this.filename = opt.filename;
+        this.cacheLimit = Math.max(100, Number(opt.maxCacheEntries ?? 5_000));
+        this.maxPendingDocs = Math.max(10_000, Number(opt.maxPendingDocs ?? 500_000));
+        this.writer = new Writer(this.filename, {
+            batchBytes: opt.batchBytes ?? 4 * 1024 * 1024,
+            autoFlushMs: opt.autoFlushMs ?? 50,
+            durability: opt.durability ?? "batched",
+            fsyncEveryFlush: opt.fsyncEveryFlush ?? 1,
+        });
+        this.reader = new Reader(this.filename);
+        const autoload = opt.autoload ?? true;
+        if (autoload)
+            void this.load();
+    }
+    touchCache(id, doc) {
+        if (this.cache.has(id))
+            this.cache.delete(id);
+        this.cache.set(id, doc);
+        if (this.cache.size > this.cacheLimit) {
+            const oldest = this.cache.keys().next().value;
+            if (oldest)
+                this.cache.delete(oldest);
+        }
+    }
+    async load() {
+        await this.q.add(async () => {
+            for await (const { doc, ptr } of this.reader.scan({})) {
+                if (!doc?._id)
+                    continue;
+                if (doc._deleted) {
+                    this.byId.delete(doc._id);
+                    this.cache.delete(doc._id);
+                    continue;
+                }
+                this.byId.set(doc._id, ptr);
+            }
+        });
+    }
+    /**
+     * Realtime insert: never blocks on disk.
+     * If you need guaranteed durability NOW, call await db.flush() or set durability=immediate.
+     */
+    async insert(doc) {
+        if (this.closed)
+            throw new Error("closed");
+        if (!doc._id)
+            doc._id = randomUUID();
+        if (!doc.createdAt)
+            doc.createdAt = Date.now();
+        doc.updatedAt = doc.updatedAt ?? doc.createdAt;
+        const json = JSON.stringify(doc);
+        const { ptr } = this.writer.enqueue(json);
+        // update pointer index immediately (valid after flush, but pointer is correct)
+        this.byId.set(doc._id, ptr);
+        this.touchCache(doc._id, doc);
+        this.emit("insert", doc);
+        // backpressure: if pending too high, force flush (but not deadlock)
+        if (this.writer.getPendingCount() > this.maxPendingDocs) {
+            void this.flush().catch(() => { });
+        }
+        return doc;
+    }
+    async flush() {
+        await this.q.add(async () => {
+            await this.writer.flush();
+        });
+    }
+    async drain() {
+        await this.q.add(async () => {
+            await this.writer.drain();
+        });
+    }
+    async findOne(query) {
+        if (this.closed)
+            throw new Error("closed");
+        if (query._id) {
+            const id = String(query._id);
+            const cached = this.cache.get(id);
+            if (cached && !cached._deleted)
+                return cached;
+            const ptr = this.byId.get(id);
+            if (!ptr)
+                return null;
+            const doc = await this.reader.readByPointer(ptr);
+            if (!doc || doc._deleted)
+                return null;
+            this.touchCache(id, doc);
+            return doc;
+        }
+        // fallback scan
+        let last = null;
+        for await (const { doc } of this.reader.scan(query)) {
+            if (doc._deleted)
+                continue;
+            last = doc;
+        }
+        if (last?._id)
+            this.touchCache(last._id, last);
+        return last;
+    }
+    async find(query) {
+        if (this.closed)
+            throw new Error("closed");
+        if (query._id) {
+            const one = await this.findOne(query);
+            return one ? [one] : [];
+        }
+        const out = [];
+        for await (const { doc } of this.reader.scan(query)) {
+            if (doc._deleted)
+                continue;
+            out.push(doc);
+        }
+        return out;
+    }
+    async update(query, update) {
+        const docs = await this.find(query);
+        let c = 0;
+        for (const d of docs) {
+            const n = { ...d };
+            if (update.$set)
+                Object.assign(n, update.$set);
+            if (update.$inc) {
+                for (const k of Object.keys(update.$inc))
+                    n[k] = (n[k] ?? 0) + update.$inc[k];
+            }
+            if (update.$unset) {
+                for (const k of Object.keys(update.$unset))
+                    delete n[k];
+            }
+            n.updatedAt = Date.now();
+            await this.insert(n);
+            c++;
+        }
+        return c;
+    }
+    async remove(query) {
+        const docs = await this.find(query);
+        let c = 0;
+        for (const d of docs) {
+            if (!d._id)
+                continue;
+            await this.insert({
+                _id: d._id,
+                _deleted: true,
+                updatedAt: Date.now(),
+            });
+            this.byId.delete(d._id);
+            this.cache.delete(d._id);
+            c++;
+        }
+        return c;
+    }
+    async close() {
+        if (this.closed)
+            return;
+        this.closed = true;
+        await this.drain();
+        await this.writer.close();
+        this.q.clear();
+    }
+}

package/dist/core/IndexSnapshot.d.ts

@@ -0,0 +1,6 @@
+import { RecordPointer } from "./BlockCodec.js";
+export declare class IndexSnapshot {
+    static idxName(dataFilename: string): string;
+    static load(dataFilename: string): Promise<Map<string, RecordPointer> | null>;
+    static saveAtomic(dataFilename: string, map: Map<string, RecordPointer>): Promise<void>;
+}

package/dist/core/IndexSnapshot.js

@@ -0,0 +1,33 @@
+// src/core/IndexSnapshot.ts
+import { readFile, writeFile, rename } from "fs/promises";
+export class IndexSnapshot {
+    static idxName(dataFilename) {
+        return dataFilename + ".idx";
+    }
+    static async load(dataFilename) {
+        const idx = this.idxName(dataFilename);
+        try {
+            const txt = await readFile(idx, "utf8");
+            const snap = JSON.parse(txt);
+            if (!snap || snap.version !== 1 || !snap.byId)
+                return null;
+            const m = new Map();
+            for (const [id, ptr] of Object.entries(snap.byId))
+                m.set(id, ptr);
+            return m;
+        }
+        catch {
+            return null;
+        }
+    }
+    static async saveAtomic(dataFilename, map) {
+        const idx = this.idxName(dataFilename);
+        const tmp = idx + ".tmp";
+        const byId = {};
+        for (const [id, ptr] of map.entries())
+            byId[id] = ptr;
+        const snap = { version: 1, createdAt: Date.now(), byId };
+        await writeFile(tmp, JSON.stringify(snap));
+        await rename(tmp, idx);
+    }
+}

package/dist/core/LRUCache.d.ts

@@ -0,0 +1,10 @@
+export declare class LRUCache<K, V> {
+    private maxEntries;
+    private map;
+    constructor(maxEntries: number);
+    get(key: K): V | undefined;
+    set(key: K, value: V): void;
+    delete(key: K): void;
+    has(key: K): boolean;
+    clear(): void;
+}