xitdb 0.1.0 → 0.3.0

package/example/dump.ts

@@ -1,201 +0,0 @@
-#!/usr/bin/env bun
-import {
-  Database,
-  Hasher,
-  CoreBufferedFile,
-  ReadArrayList,
-  ReadCursor,
-  Tag,
-} from '../src';
-
-async function formatKey(cursor: ReadCursor): Promise<string> {
-  const tag = cursor.slotPtr.slot.tag;
-
-  switch (tag) {
-    case Tag.NONE:
-      return '(none)';
-    case Tag.BYTES:
-    case Tag.SHORT_BYTES: {
-      const bytes = await cursor.readBytes(null);
-      const text = new TextDecoder().decode(bytes);
-      return `"${text}"`;
-    }
-    case Tag.UINT:
-      return `${cursor.readUint()}`;
-    case Tag.INT:
-      return `${cursor.readInt()}`;
-    case Tag.FLOAT:
-      return `${cursor.readFloat()}`;
-    default:
-      return `<key tag: ${tag}>`;
-  }
-}
-
-async function printValue(cursor: ReadCursor, indent: string): Promise<void> {
-  const tag = cursor.slotPtr.slot.tag;
-
-  switch (tag) {
-    case Tag.NONE:
-      console.log(`${indent}(none)`);
-      break;
-
-    case Tag.ARRAY_LIST: {
-      const list = new ReadArrayList(cursor);
-      const count = await list.count();
-      console.log(`${indent}ArrayList[${count}]:`);
-      if (indent == '') {
-        const itemCursor = await list.getCursor(count - 1);
-        if (itemCursor) {
-          await printValue(itemCursor, indent + '    ');
-        }
-      } else {
-        for (let i = 0; i < count; i++) {
-          const itemCursor = await list.getCursor(i);
-          if (itemCursor) {
-            console.log(`${indent}  [${i}]:`);
-            await printValue(itemCursor, indent + '    ');
-          }
-        }
-      }
-      break;
-    }
-
-    case Tag.HASH_MAP:
-    case Tag.COUNTED_HASH_MAP: {
-      const iter = cursor.iterator();
-      await iter.init();
-      const entries: Array<{ key: string; valueCursor: ReadCursor }> = [];
-
-      while (await iter.hasNext()) {
-        const kvPairCursor = await iter.next();
-        if (kvPairCursor) {
-          const kvPair = await kvPairCursor.readKeyValuePair();
-          const key = await formatKey(kvPair.keyCursor);
-          entries.push({ key, valueCursor: kvPair.valueCursor });
-        }
-      }
-
-      const prefix = tag === Tag.COUNTED_HASH_MAP ? 'CountedHashMap' : 'HashMap';
-      console.log(`${indent}${prefix}{${entries.length}}:`);
-
-      for (const entry of entries) {
-        console.log(`${indent}  ${entry.key}:`);
-        await printValue(entry.valueCursor, indent + '    ');
-      }
-      break;
-    }
-
-    case Tag.HASH_SET:
-    case Tag.COUNTED_HASH_SET: {
-      const iter = cursor.iterator();
-      await iter.init();
-      const keys: string[] = [];
-
-      while (await iter.hasNext()) {
-        const kvPairCursor = await iter.next();
-        if (kvPairCursor) {
-          const kvPair = await kvPairCursor.readKeyValuePair();
-          const key = await formatKey(kvPair.keyCursor);
-          keys.push(key);
-        }
-      }
-
-      const prefix = tag === Tag.COUNTED_HASH_SET ? 'CountedHashSet' : 'HashSet';
-      console.log(`${indent}${prefix}{${keys.length}}: [${keys.join(', ')}]`);
-      break;
-    }
-
-    case Tag.LINKED_ARRAY_LIST: {
-      const count = await cursor.count();
-      console.log(`${indent}LinkedArrayList[${count}]:`);
-      const iter = cursor.iterator();
-      await iter.init();
-      let i = 0;
-      while (await iter.hasNext()) {
-        const itemCursor = await iter.next();
-        if (itemCursor) {
-          console.log(`${indent}  [${i}]:`);
-          await printValue(itemCursor, indent + '    ');
-        }
-        i++;
-      }
-      break;
-    }
-
-    case Tag.BYTES:
-    case Tag.SHORT_BYTES: {
-      const bytesObj = await cursor.readBytesObject(null);
-      const text = new TextDecoder().decode(bytesObj.value);
-      const isPrintable = /^[\x20-\x7E\n\r\t]*$/.test(text);
-
-      if (isPrintable && text.length <= 100) {
-        if (bytesObj.formatTag) {
-          const formatTag = new TextDecoder().decode(bytesObj.formatTag);
-          console.log(`${indent}"${text}" (format: ${formatTag})`);
-        } else {
-          console.log(`${indent}"${text}"`);
-        }
-      } else {
-        const preview = bytesObj.value.slice(0, 16);
-        const hex = Array.from(preview).map(b => b.toString(16).padStart(2, '0')).join(' ');
-        if (bytesObj.formatTag) {
-          const formatTag = new TextDecoder().decode(bytesObj.formatTag);
-          console.log(`${indent}<${bytesObj.value.length} bytes: ${hex}...> (format: ${formatTag})`);
-        } else {
-          console.log(`${indent}<${bytesObj.value.length} bytes: ${hex}...>`);
-        }
-      }
-      break;
-    }
-
-    case Tag.UINT: {
-      const value = cursor.readUint();
-      console.log(`${indent}${value} (uint)`);
-      break;
-    }
-
-    case Tag.INT: {
-      const value = cursor.readInt();
-      console.log(`${indent}${value} (int)`);
-      break;
-    }
-
-    case Tag.FLOAT: {
-      const value = cursor.readFloat();
-      console.log(`${indent}${value} (float)`);
-      break;
-    }
-
-    default:
-      console.log(`${indent}<unknown tag: ${tag}>`);
-  }
-}
-
-async function main() {
-  const args = process.argv.slice(2);
-
-  if (args.length < 1) {
-    console.error('Usage: bun run dump.ts <database-file>');
-    process.exit(1);
-  }
-
-  const filePath = args[0];
-
-  try {
-    const core = await CoreBufferedFile.create(filePath);
-    const hasher = new Hasher('SHA-1');
-    const db = await Database.create(core, hasher);
-
-    console.log(`Database: ${filePath}`);
-    console.log('---');
-
-    const rootCursor = await db.rootCursor();
-    await printValue(rootCursor, '');
-
-  } catch (error) {
-    console.error(`Error reading database: ${error}`);
-    process.exit(1);
-  }
-}
-
-main();

package/src/core-buffered-file.ts

@@ -1,226 +0,0 @@
-import type { Core, DataReader, DataWriter } from './core';
-import { CoreFile } from './core-file';
-import { CoreMemory } from './core-memory';
-
-export class CoreBufferedFile implements Core {
-  public file: RandomAccessBufferedFile;
-
-  constructor(file: RandomAccessBufferedFile) {
-    this.file = file;
-  }
-
-  static async create(filePath: string, bufferSize?: number): Promise<CoreBufferedFile> {
-    const file = await RandomAccessBufferedFile.create(filePath, bufferSize);
-    return new CoreBufferedFile(file);
-  }
-
-  reader(): DataReader {
-    return this.file;
-  }
-
-  writer(): DataWriter {
-    return this.file;
-  }
-
-  async length(): Promise<number> {
-    return await this.file.length();
-  }
-
-  async seek(pos: number): Promise<void> {
-    await this.file.seek(pos);
-  }
-
-  position(): number {
-    return this.file.position();
-  }
-
-  async setLength(len: number): Promise<void> {
-    await this.file.setLength(len);
-  }
-
-  async flush(): Promise<void> {
-    await this.file.flush();
-  }
-
-  async sync(): Promise<void> {
-    await this.file.sync();
-  }
-
-  [Symbol.dispose]() {
-    this.file.file[Symbol.dispose]();
-  }
-}
-
-const DEFAULT_BUFFER_SIZE = 8 * 1024 * 1024; // 8MB
-
-class RandomAccessBufferedFile implements DataReader, DataWriter {
-  public file: CoreFile;
-  private memory: CoreMemory;
-  private bufferSize: number; // flushes when the memory is >= this size
-  private filePos: number;
-  private memoryPos: number;
-
-  private constructor(file: CoreFile, bufferSize: number) {
-    this.file = file;
-    this.memory = new CoreMemory();
-    this.bufferSize = bufferSize;
-    this.filePos = 0;
-    this.memoryPos = 0;
-  }
-
-  static async create(filePath: string, bufferSize: number = DEFAULT_BUFFER_SIZE): Promise<RandomAccessBufferedFile> {
-    const file = await CoreFile.create(filePath);
-    return new RandomAccessBufferedFile(file, bufferSize);
-  }
-
-  async seek(pos: number): Promise<void> {
-    // flush if we are going past the end of the in-memory buffer
-    if (pos > this.memoryPos + await this.memory.length()) {
-      await this.flush();
-    }
-
-    this.filePos = pos;
-
-    // if the buffer is empty, set its position to this offset as well
-    if (await this.memory.length() === 0) {
-      this.memoryPos = pos;
-    }
-  }
-
-  async length(): Promise<number> {
-    return Math.max(this.memoryPos + await this.memory.length(), await this.file.length());
-  }
-
-  position(): number {
-    return this.filePos;
-  }
-
-  async setLength(len: number): Promise<void> {
-    await this.flush();
-    await this.file.setLength(len);
-    this.filePos = Math.min(len, this.filePos);
-  }
-
-  async flush(): Promise<void> {
-    if (await this.memory.length() > 0) {
-      await this.file.seek(this.memoryPos);
-      await this.file.writer().write(this.memory.memory.toByteArray());
-
-      this.memoryPos = 0;
-      this.memory.memory.reset();
-    }
-  }
-
-  async sync(): Promise<void> {
-    await this.flush();
-    await this.file.sync();
-  }
-
-  // DataWriter interface
-
-  async write(buffer: Uint8Array): Promise<void> {
-    if (await this.memory.length() + buffer.length > this.bufferSize) {
-      await this.flush();
-    }
-
-    if (this.filePos >= this.memoryPos && this.filePos <= this.memoryPos + await this.memory.length()) {
-      this.memory.seek(this.filePos - this.memoryPos);
-      await this.memory.memory.write(buffer);
-    } else {
-      // Write directly to file
-      await this.file.seek(this.filePos);
-      await this.file.writer().write(buffer);
-    }
-
-    this.filePos += buffer.length;
-  }
-
-  async writeByte(v: number): Promise<void> {
-    await this.write(new Uint8Array([v & 0xff]));
-  }
-
-  async writeShort(v: number): Promise<void> {
-    const buffer = new ArrayBuffer(2);
-    const view = new DataView(buffer);
-    view.setInt16(0, v & 0xffff, false); // big-endian
-    await this.write(new Uint8Array(buffer));
-  }
-
-  async writeLong(v: number): Promise<void> {
-    const buffer = new ArrayBuffer(8);
-    const view = new DataView(buffer);
-    view.setBigInt64(0, BigInt(v), false);
-    await this.write(new Uint8Array(buffer));
-  }
-
-  // DataReader interface
-
-  async readFully(buffer: Uint8Array): Promise<void> {
-    let pos = 0;
-
-    // read from the disk -- before the in-memory buffer
-    if (this.filePos < this.memoryPos) {
-      const sizeBeforeMem = Math.min(this.memoryPos - this.filePos, buffer.length);
-      const tempBuffer = new Uint8Array(sizeBeforeMem);
-      await this.file.seek(this.filePos);
-      await this.file.reader().readFully(tempBuffer);
-      buffer.set(tempBuffer, pos);
-      pos += sizeBeforeMem;
-      this.filePos += sizeBeforeMem;
-    }
-
-    if (pos === buffer.length) return;
-
-    // read from the in-memory buffer
-    if (this.filePos >= this.memoryPos && this.filePos < this.memoryPos + await this.memory.length()) {
-      const memPos = this.filePos - this.memoryPos;
-      const sizeInMem = Math.min(await this.memory.length() - memPos, buffer.length - pos);
-      this.memory.seek(memPos);
-      const memBuffer = new Uint8Array(sizeInMem);
-      await this.memory.memory.readFully(memBuffer);
-      buffer.set(memBuffer, pos);
-      pos += sizeInMem;
-      this.filePos += sizeInMem;
-    }
-
-    if (pos === buffer.length) return;
-
-    // read from the disk -- after the in-memory buffer
-    if (this.filePos >= this.memoryPos + await this.memory.length()) {
-      const sizeAfterMem = buffer.length - pos;
-      const tempBuffer = new Uint8Array(sizeAfterMem);
-      await this.file.seek(this.filePos);
-      await this.file.reader().readFully(tempBuffer);
-      buffer.set(tempBuffer, pos);
-      pos += sizeAfterMem;
-      this.filePos += sizeAfterMem;
-    }
-  }
-
-  async readByte(): Promise<number> {
-    const bytes = new Uint8Array(1);
-    await this.readFully(bytes);
-    return bytes[0];
-  }
-
-  async readShort(): Promise<number> {
-    const bytes = new Uint8Array(2);
-    await this.readFully(bytes);
-    const view = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength);
-    return view.getInt16(0, false); // big-endian
-  }
-
-  async readInt(): Promise<number> {
-    const bytes = new Uint8Array(4);
-    await this.readFully(bytes);
-    const view = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength);
-    return view.getInt32(0, false); // big-endian
-  }
-
-  async readLong(): Promise<number> {
-    const bytes = new Uint8Array(8);
-    await this.readFully(bytes);
-    const view = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength);
-    return Number(view.getBigInt64(0, false));
-  }
-}

package/src/core-file.ts

@@ -1,137 +0,0 @@
-import type { Core, DataReader, DataWriter } from './core';
-import * as fs from 'fs/promises';
-import type { FileHandle } from 'fs/promises';
-
-export class CoreFile implements Core {
-  public filePath: string;
-  private _position: number = 0;
-  public fileHandle: FileHandle;
-
-  private constructor(filePath: string, fileHandle: FileHandle) {
-    this.filePath = filePath;
-    this.fileHandle = fileHandle;
-  }
-
-  static async create(filePath: string): Promise<CoreFile> {
-    // Create file if it doesn't exist
-    try {
-      await fs.access(filePath);
-    } catch {
-      await fs.writeFile(filePath, new Uint8Array(0));
-    }
-    // Open file handle for reading and writing
-    const fileHandle = await fs.open(filePath, 'r+');
-    return new CoreFile(filePath, fileHandle);
-  }
-
-  reader(): DataReader {
-    return new FileDataReader(this);
-  }
-
-  writer(): DataWriter {
-    return new FileDataWriter(this);
-  }
-
-  async length(): Promise<number> {
-    const stats = await this.fileHandle.stat();
-    return stats.size;
-  }
-
-  async seek(pos: number): Promise<void> {
-    this._position = pos;
-  }
-
-  position(): number {
-    return this._position;
-  }
-
-  async setLength(len: number): Promise<void> {
-    await this.fileHandle.truncate(len);
-  }
-
-  async flush(): Promise<void> {
-  }
-
-  async sync(): Promise<void> {
-    await this.fileHandle.sync();
-  }
-
-  [Symbol.dispose]() {
-    import("fs").then(fs => {
-      fs.closeSync(this.fileHandle.fd);
-    });
-  }
-}
-
-class FileDataReader implements DataReader {
-  private core: CoreFile;
-
-  constructor(core: CoreFile) {
-    this.core = core;
-  }
-
-  async readFully(b: Uint8Array): Promise<void> {
-    const position = this.core.position();
-    await this.core.fileHandle.readv([b], position);
-    this.core.seek(position + b.length);
-  }
-
-  async readByte(): Promise<number> {
-    const bytes = new Uint8Array(1);
-    await this.readFully(bytes);
-    return bytes[0];
-  }
-
-  async readShort(): Promise<number> {
-    const bytes = new Uint8Array(2);
-    await this.readFully(bytes);
-    const view = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength);
-    return view.getInt16(0, false);
-  }
-
-  async readInt(): Promise<number> {
-    const bytes = new Uint8Array(4);
-    await this.readFully(bytes);
-    const view = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength);
-    return view.getInt32(0, false);
-  }
-
-  async readLong(): Promise<number> {
-    const bytes = new Uint8Array(8);
-    await this.readFully(bytes);
-    const view = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength);
-    return Number(view.getBigInt64(0, false));
-  }
-}
-
-class FileDataWriter implements DataWriter {
-  private core: CoreFile;
-
-  constructor(core: CoreFile) {
-    this.core = core;
-  }
-
-  async write(buffer: Uint8Array): Promise<void> {
-    const position = this.core.position();
-    await this.core.fileHandle.writev([buffer], position);
-    this.core.seek(position + buffer.length);
-  }
-
-  async writeByte(v: number): Promise<void> {
-    await this.write(new Uint8Array([v & 0xff]));
-  }
-
-  async writeShort(v: number): Promise<void> {
-    const buffer = new ArrayBuffer(2);
-    const view = new DataView(buffer);
-    view.setInt16(0, v, false);
-    await this.write(new Uint8Array(buffer));
-  }
-
-  async writeLong(v: number): Promise<void> {
-    const buffer = new ArrayBuffer(8);
-    const view = new DataView(buffer);
-    view.setBigInt64(0, BigInt(v), false);
-    await this.write(new Uint8Array(buffer));
-  }
-}