xitdb 0.1.0

package/example/README.md

@@ -0,0 +1,46 @@
+This is an example program that reads a xitdb file and prints out its most recent value (the last item of the top-level ArrayList). From the root of this repo, run:
+
+```
+bun run example/dump.ts tests/fixtures/test.db
+```
+
+...and it will print:
+
+```
+ArrayList[2]:
+    HashMap{7}:
+      (none):
+        LinkedArrayList[1]:
+          [0]:
+            "Wash the car"
+      (none):
+        HashSet{1}: ["a"]
+      (none):
+        ArrayList[2]:
+          [0]:
+            HashMap{2}:
+              (none):
+                26 (uint)
+              (none):
+                "Alice"
+          [1]:
+            HashMap{2}:
+              (none):
+                42 (uint)
+              (none):
+                "Bob"
+      (none):
+        "foo"
+      "fruits":
+        ArrayList[2]:
+          [0]:
+            "lemon"
+          [1]:
+            "pear"
+      (none):
+        CountedHashMap{1}:
+          "a":
+            2 (uint)
+      (none):
+        CountedHashSet{1}: ["a"]
+```

package/example/dump.ts

@@ -0,0 +1,201 @@
+#!/usr/bin/env bun
+import {
+  Database,
+  Hasher,
+  CoreBufferedFile,
+  ReadArrayList,
+  ReadCursor,
+  Tag,
+} from '../src';
+
+async function formatKey(cursor: ReadCursor): Promise<string> {
+  const tag = cursor.slotPtr.slot.tag;
+
+  switch (tag) {
+    case Tag.NONE:
+      return '(none)';
+    case Tag.BYTES:
+    case Tag.SHORT_BYTES: {
+      const bytes = await cursor.readBytes(null);
+      const text = new TextDecoder().decode(bytes);
+      return `"${text}"`;
+    }
+    case Tag.UINT:
+      return `${cursor.readUint()}`;
+    case Tag.INT:
+      return `${cursor.readInt()}`;
+    case Tag.FLOAT:
+      return `${cursor.readFloat()}`;
+    default:
+      return `<key tag: ${tag}>`;
+  }
+}
+
+async function printValue(cursor: ReadCursor, indent: string): Promise<void> {
+  const tag = cursor.slotPtr.slot.tag;
+
+  switch (tag) {
+    case Tag.NONE:
+      console.log(`${indent}(none)`);
+      break;
+
+    case Tag.ARRAY_LIST: {
+      const list = new ReadArrayList(cursor);
+      const count = await list.count();
+      console.log(`${indent}ArrayList[${count}]:`);
+      if (indent == '') {
+        const itemCursor = await list.getCursor(count - 1);
+        if (itemCursor) {
+          await printValue(itemCursor, indent + '    ');
+        }
+      } else {
+        for (let i = 0; i < count; i++) {
+          const itemCursor = await list.getCursor(i);
+          if (itemCursor) {
+            console.log(`${indent}  [${i}]:`);
+            await printValue(itemCursor, indent + '    ');
+          }
+        }
+      }
+      break;
+    }
+
+    case Tag.HASH_MAP:
+    case Tag.COUNTED_HASH_MAP: {
+      const iter = cursor.iterator();
+      await iter.init();
+      const entries: Array<{ key: string; valueCursor: ReadCursor }> = [];
+
+      while (await iter.hasNext()) {
+        const kvPairCursor = await iter.next();
+        if (kvPairCursor) {
+          const kvPair = await kvPairCursor.readKeyValuePair();
+          const key = await formatKey(kvPair.keyCursor);
+          entries.push({ key, valueCursor: kvPair.valueCursor });
+        }
+      }
+
+      const prefix = tag === Tag.COUNTED_HASH_MAP ? 'CountedHashMap' : 'HashMap';
+      console.log(`${indent}${prefix}{${entries.length}}:`);
+
+      for (const entry of entries) {
+        console.log(`${indent}  ${entry.key}:`);
+        await printValue(entry.valueCursor, indent + '    ');
+      }
+      break;
+    }
+
+    case Tag.HASH_SET:
+    case Tag.COUNTED_HASH_SET: {
+      const iter = cursor.iterator();
+      await iter.init();
+      const keys: string[] = [];
+
+      while (await iter.hasNext()) {
+        const kvPairCursor = await iter.next();
+        if (kvPairCursor) {
+          const kvPair = await kvPairCursor.readKeyValuePair();
+          const key = await formatKey(kvPair.keyCursor);
+          keys.push(key);
+        }
+      }
+
+      const prefix = tag === Tag.COUNTED_HASH_SET ? 'CountedHashSet' : 'HashSet';
+      console.log(`${indent}${prefix}{${keys.length}}: [${keys.join(', ')}]`);
+      break;
+    }
+
+    case Tag.LINKED_ARRAY_LIST: {
+      const count = await cursor.count();
+      console.log(`${indent}LinkedArrayList[${count}]:`);
+      const iter = cursor.iterator();
+      await iter.init();
+      let i = 0;
+      while (await iter.hasNext()) {
+        const itemCursor = await iter.next();
+        if (itemCursor) {
+          console.log(`${indent}  [${i}]:`);
+          await printValue(itemCursor, indent + '    ');
+        }
+        i++;
+      }
+      break;
+    }
+
+    case Tag.BYTES:
+    case Tag.SHORT_BYTES: {
+      const bytesObj = await cursor.readBytesObject(null);
+      const text = new TextDecoder().decode(bytesObj.value);
+      const isPrintable = /^[\x20-\x7E\n\r\t]*$/.test(text);
+
+      if (isPrintable && text.length <= 100) {
+        if (bytesObj.formatTag) {
+          const formatTag = new TextDecoder().decode(bytesObj.formatTag);
+          console.log(`${indent}"${text}" (format: ${formatTag})`);
+        } else {
+          console.log(`${indent}"${text}"`);
+        }
+      } else {
+        const preview = bytesObj.value.slice(0, 16);
+        const hex = Array.from(preview).map(b => b.toString(16).padStart(2, '0')).join(' ');
+        if (bytesObj.formatTag) {
+          const formatTag = new TextDecoder().decode(bytesObj.formatTag);
+          console.log(`${indent}<${bytesObj.value.length} bytes: ${hex}...> (format: ${formatTag})`);
+        } else {
+          console.log(`${indent}<${bytesObj.value.length} bytes: ${hex}...>`);
+        }
+      }
+      break;
+    }
+
+    case Tag.UINT: {
+      const value = cursor.readUint();
+      console.log(`${indent}${value} (uint)`);
+      break;
+    }
+
+    case Tag.INT: {
+      const value = cursor.readInt();
+      console.log(`${indent}${value} (int)`);
+      break;
+    }
+
+    case Tag.FLOAT: {
+      const value = cursor.readFloat();
+      console.log(`${indent}${value} (float)`);
+      break;
+    }
+
+    default:
+      console.log(`${indent}<unknown tag: ${tag}>`);
+  }
+}
+
+async function main() {
+  const args = process.argv.slice(2);
+
+  if (args.length < 1) {
+    console.error('Usage: bun run dump.ts <database-file>');
+    process.exit(1);
+  }
+
+  const filePath = args[0];
+
+  try {
+    const core = await CoreBufferedFile.create(filePath);
+    const hasher = new Hasher('SHA-1');
+    const db = await Database.create(core, hasher);
+
+    console.log(`Database: ${filePath}`);
+    console.log('---');
+
+    const rootCursor = await db.rootCursor();
+    await printValue(rootCursor, '');
+
+  } catch (error) {
+    console.error(`Error reading database: ${error}`);
+    process.exit(1);
+  }
+}
+
+main();

package/package.json

@@ -0,0 +1,17 @@
+{
+  "name": "xitdb",
+  "version": "0.1.0",
+  "type": "module",
+  "main": "dist/index.js",
+  "types": "dist/index.d.ts",
+  "scripts": {
+    "build": "bun build ./src/index.ts --outdir ./dist --target bun",
+    "build:types": "tsc --emitDeclarationOnly",
+    "test": "bun test",
+    "typecheck": "tsc --noEmit"
+  },
+  "devDependencies": {
+    "@types/bun": "latest",
+    "typescript": "^5.3.0"
+  }
+}

package/src/core-buffered-file.ts

@@ -0,0 +1,226 @@
+import type { Core, DataReader, DataWriter } from './core';
+import { CoreFile } from './core-file';
+import { CoreMemory } from './core-memory';
+
+export class CoreBufferedFile implements Core {
+  public file: RandomAccessBufferedFile;
+
+  constructor(file: RandomAccessBufferedFile) {
+    this.file = file;
+  }
+
+  static async create(filePath: string, bufferSize?: number): Promise<CoreBufferedFile> {
+    const file = await RandomAccessBufferedFile.create(filePath, bufferSize);
+    return new CoreBufferedFile(file);
+  }
+
+  reader(): DataReader {
+    return this.file;
+  }
+
+  writer(): DataWriter {
+    return this.file;
+  }
+
+  async length(): Promise<number> {
+    return await this.file.length();
+  }
+
+  async seek(pos: number): Promise<void> {
+    await this.file.seek(pos);
+  }
+
+  position(): number {
+    return this.file.position();
+  }
+
+  async setLength(len: number): Promise<void> {
+    await this.file.setLength(len);
+  }
+
+  async flush(): Promise<void> {
+    await this.file.flush();
+  }
+
+  async sync(): Promise<void> {
+    await this.file.sync();
+  }
+
+  [Symbol.dispose]() {
+    this.file.file[Symbol.dispose]();
+  }
+}
+
+const DEFAULT_BUFFER_SIZE = 8 * 1024 * 1024; // 8MB
+
+class RandomAccessBufferedFile implements DataReader, DataWriter {
+  public file: CoreFile;
+  private memory: CoreMemory;
+  private bufferSize: number; // flushes when the memory is >= this size
+  private filePos: number;
+  private memoryPos: number;
+
+  private constructor(file: CoreFile, bufferSize: number) {
+    this.file = file;
+    this.memory = new CoreMemory();
+    this.bufferSize = bufferSize;
+    this.filePos = 0;
+    this.memoryPos = 0;
+  }
+
+  static async create(filePath: string, bufferSize: number = DEFAULT_BUFFER_SIZE): Promise<RandomAccessBufferedFile> {
+    const file = await CoreFile.create(filePath);
+    return new RandomAccessBufferedFile(file, bufferSize);
+  }
+
+  async seek(pos: number): Promise<void> {
+    // flush if we are going past the end of the in-memory buffer
+    if (pos > this.memoryPos + await this.memory.length()) {
+      await this.flush();
+    }
+
+    this.filePos = pos;
+
+    // if the buffer is empty, set its position to this offset as well
+    if (await this.memory.length() === 0) {
+      this.memoryPos = pos;
+    }
+  }
+
+  async length(): Promise<number> {
+    return Math.max(this.memoryPos + await this.memory.length(), await this.file.length());
+  }
+
+  position(): number {
+    return this.filePos;
+  }
+
+  async setLength(len: number): Promise<void> {
+    await this.flush();
+    await this.file.setLength(len);
+    this.filePos = Math.min(len, this.filePos);
+  }
+
+  async flush(): Promise<void> {
+    if (await this.memory.length() > 0) {
+      await this.file.seek(this.memoryPos);
+      await this.file.writer().write(this.memory.memory.toByteArray());
+
+      this.memoryPos = 0;
+      this.memory.memory.reset();
+    }
+  }
+
+  async sync(): Promise<void> {
+    await this.flush();
+    await this.file.sync();
+  }
+
+  // DataWriter interface
+
+  async write(buffer: Uint8Array): Promise<void> {
+    if (await this.memory.length() + buffer.length > this.bufferSize) {
+      await this.flush();
+    }
+
+    if (this.filePos >= this.memoryPos && this.filePos <= this.memoryPos + await this.memory.length()) {
+      this.memory.seek(this.filePos - this.memoryPos);
+      await this.memory.memory.write(buffer);
+    } else {
+      // Write directly to file
+      await this.file.seek(this.filePos);
+      await this.file.writer().write(buffer);
+    }
+
+    this.filePos += buffer.length;
+  }
+
+  async writeByte(v: number): Promise<void> {
+    await this.write(new Uint8Array([v & 0xff]));
+  }
+
+  async writeShort(v: number): Promise<void> {
+    const buffer = new ArrayBuffer(2);
+    const view = new DataView(buffer);
+    view.setInt16(0, v & 0xffff, false); // big-endian
+    await this.write(new Uint8Array(buffer));
+  }
+
+  async writeLong(v: number): Promise<void> {
+    const buffer = new ArrayBuffer(8);
+    const view = new DataView(buffer);
+    view.setBigInt64(0, BigInt(v), false);
+    await this.write(new Uint8Array(buffer));
+  }
+
+  // DataReader interface
+
+  async readFully(buffer: Uint8Array): Promise<void> {
+    let pos = 0;
+
+    // read from the disk -- before the in-memory buffer
+    if (this.filePos < this.memoryPos) {
+      const sizeBeforeMem = Math.min(this.memoryPos - this.filePos, buffer.length);
+      const tempBuffer = new Uint8Array(sizeBeforeMem);
+      await this.file.seek(this.filePos);
+      await this.file.reader().readFully(tempBuffer);
+      buffer.set(tempBuffer, pos);
+      pos += sizeBeforeMem;
+      this.filePos += sizeBeforeMem;
+    }
+
+    if (pos === buffer.length) return;
+
+    // read from the in-memory buffer
+    if (this.filePos >= this.memoryPos && this.filePos < this.memoryPos + await this.memory.length()) {
+      const memPos = this.filePos - this.memoryPos;
+      const sizeInMem = Math.min(await this.memory.length() - memPos, buffer.length - pos);
+      this.memory.seek(memPos);
+      const memBuffer = new Uint8Array(sizeInMem);
+      await this.memory.memory.readFully(memBuffer);
+      buffer.set(memBuffer, pos);
+      pos += sizeInMem;
+      this.filePos += sizeInMem;
+    }
+
+    if (pos === buffer.length) return;
+
+    // read from the disk -- after the in-memory buffer
+    if (this.filePos >= this.memoryPos + await this.memory.length()) {
+      const sizeAfterMem = buffer.length - pos;
+      const tempBuffer = new Uint8Array(sizeAfterMem);
+      await this.file.seek(this.filePos);
+      await this.file.reader().readFully(tempBuffer);
+      buffer.set(tempBuffer, pos);
+      pos += sizeAfterMem;
+      this.filePos += sizeAfterMem;
+    }
+  }
+
+  async readByte(): Promise<number> {
+    const bytes = new Uint8Array(1);
+    await this.readFully(bytes);
+    return bytes[0];
+  }
+
+  async readShort(): Promise<number> {
+    const bytes = new Uint8Array(2);
+    await this.readFully(bytes);
+    const view = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength);
+    return view.getInt16(0, false); // big-endian
+  }
+
+  async readInt(): Promise<number> {
+    const bytes = new Uint8Array(4);
+    await this.readFully(bytes);
+    const view = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength);
+    return view.getInt32(0, false); // big-endian
+  }
+
+  async readLong(): Promise<number> {
+    const bytes = new Uint8Array(8);
+    await this.readFully(bytes);
+    const view = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength);
+    return Number(view.getBigInt64(0, false));
+  }
+}

package/src/core-file.ts

@@ -0,0 +1,137 @@
+import type { Core, DataReader, DataWriter } from './core';
+import * as fs from 'fs/promises';
+import type { FileHandle } from 'fs/promises';
+
+export class CoreFile implements Core {
+  public filePath: string;
+  private _position: number = 0;
+  public fileHandle: FileHandle;
+
+  private constructor(filePath: string, fileHandle: FileHandle) {
+    this.filePath = filePath;
+    this.fileHandle = fileHandle;
+  }
+
+  static async create(filePath: string): Promise<CoreFile> {
+    // Create file if it doesn't exist
+    try {
+      await fs.access(filePath);
+    } catch {
+      await fs.writeFile(filePath, new Uint8Array(0));
+    }
+    // Open file handle for reading and writing
+    const fileHandle = await fs.open(filePath, 'r+');
+    return new CoreFile(filePath, fileHandle);
+  }
+
+  reader(): DataReader {
+    return new FileDataReader(this);
+  }
+
+  writer(): DataWriter {
+    return new FileDataWriter(this);
+  }
+
+  async length(): Promise<number> {
+    const stats = await this.fileHandle.stat();
+    return stats.size;
+  }
+
+  async seek(pos: number): Promise<void> {
+    this._position = pos;
+  }
+
+  position(): number {
+    return this._position;
+  }
+
+  async setLength(len: number): Promise<void> {
+    await this.fileHandle.truncate(len);
+  }
+
+  async flush(): Promise<void> {
+  }
+
+  async sync(): Promise<void> {
+    await this.fileHandle.sync();
+  }
+
+  [Symbol.dispose]() {
+    import("fs").then(fs => {
+      fs.closeSync(this.fileHandle.fd);
+    });
+  }
+}
+
+class FileDataReader implements DataReader {
+  private core: CoreFile;
+
+  constructor(core: CoreFile) {
+    this.core = core;
+  }
+
+  async readFully(b: Uint8Array): Promise<void> {
+    const position = this.core.position();
+    await this.core.fileHandle.readv([b], position);
+    this.core.seek(position + b.length);
+  }
+
+  async readByte(): Promise<number> {
+    const bytes = new Uint8Array(1);
+    await this.readFully(bytes);
+    return bytes[0];
+  }
+
+  async readShort(): Promise<number> {
+    const bytes = new Uint8Array(2);
+    await this.readFully(bytes);
+    const view = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength);
+    return view.getInt16(0, false);
+  }
+
+  async readInt(): Promise<number> {
+    const bytes = new Uint8Array(4);
+    await this.readFully(bytes);
+    const view = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength);
+    return view.getInt32(0, false);
+  }
+
+  async readLong(): Promise<number> {
+    const bytes = new Uint8Array(8);
+    await this.readFully(bytes);
+    const view = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength);
+    return Number(view.getBigInt64(0, false));
+  }
+}
+
+class FileDataWriter implements DataWriter {
+  private core: CoreFile;
+
+  constructor(core: CoreFile) {
+    this.core = core;
+  }
+
+  async write(buffer: Uint8Array): Promise<void> {
+    const position = this.core.position();
+    await this.core.fileHandle.writev([buffer], position);
+    this.core.seek(position + buffer.length);
+  }
+
+  async writeByte(v: number): Promise<void> {
+    await this.write(new Uint8Array([v & 0xff]));
+  }
+
+  async writeShort(v: number): Promise<void> {
+    const buffer = new ArrayBuffer(2);
+    const view = new DataView(buffer);
+    view.setInt16(0, v, false);
+    await this.write(new Uint8Array(buffer));
+  }
+
+  async writeLong(v: number): Promise<void> {
+    const buffer = new ArrayBuffer(8);
+    const view = new DataView(buffer);
+    view.setBigInt64(0, BigInt(v), false);
+    await this.write(new Uint8Array(buffer));
+  }
+}