xitdb 0.13.0 → 0.14.0

package/README.md

@@ -97,18 +97,18 @@ const moment = new ReadHashMap(momentCursor!);
 // the cursor to "foo" and then calling readBytes on it
 const fooCursor = moment.getCursor('foo');
 const fooValue = fooCursor!.readBytes(MAX_READ_BYTES);
-expect(new TextDecoder().decode(fooValue)).toBe('foo');
+assert.strictEqual(new TextDecoder().decode(fooValue), 'foo');
 
 // to get the "fruits" list, we get the cursor to it and
 // then pass it to the ReadArrayList constructor
 const fruitsCursor = moment.getCursor('fruits');
 const fruits = new ReadArrayList(fruitsCursor!);
-expect(fruits.count()).toBe(3);
+assert.strictEqual(fruits.count(), 3);
 
 // now we can get the first item from the fruits list and read it
 const appleCursor = fruits.getCursor(0);
 const appleValue = appleCursor!.readBytes(MAX_READ_BYTES);
-expect(new TextDecoder().decode(appleValue)).toBe('apple');
+assert.strictEqual(new TextDecoder().decode(appleValue), 'apple');
 ```
 
 ## Initializing a Database
@@ -132,8 +132,9 @@ In xitdb there are a variety of immutable data structures that you can nest arbi
 * `CountedHashMap` and `CountedHashSet` are just a `HashMap` and `HashSet` that maintain a count of their contents
 * `ArrayList` is a growable array
 * `LinkedArrayList` is like an `ArrayList` that can also be efficiently sliced and concatenated
+* `SortedMap` and `SortedSet` are like a `HashMap` and `HashSet` where the keys are byte arrays kept in lexicographic order
 
-All data structures use the hash array mapped trie, invented by Phil Bagwell. The `LinkedArrayList` is based on his later work on RRB trees. These data structures were originally made immutable and widely available by Rich Hickey in Clojure. To my knowledge, they haven't been available in any open source database until xitdb.
+The `Hash`-based data structures and the `Arraylist` use the hash array mapped trie, invented by Phil Bagwell (originally made immutable and widely available by Rich Hickey in Clojure). The `LinkedArrayList`, `SortedMap`, and `SortedSet` are based on a B-tree.
 
 There are also scalar types you can store in the above-mentioned data structures:
 
@@ -157,7 +158,7 @@ Then, you can read it like this:
 ```typescript
 const randomNumberCursor = moment.getCursor('random-number');
 const randomNumber = randomNumberCursor!.readBytesObject(MAX_READ_BYTES);
-expect(new TextDecoder().decode(randomNumber.formatTag!)).toBe('bi');
+assert.strictEqual(new TextDecoder().decode(randomNumber.formatTag!), 'bi');
 const randomBigInt = randomNumber.value;
 ```
 
@@ -191,12 +192,12 @@ const moment = new ReadHashMap(momentCursor!);
 // the food list includes the fruits
 const foodCursor = moment.getCursor('food');
 const food = new ReadArrayList(foodCursor!);
-expect(food.count()).toBe(6);
+assert.strictEqual(food.count(), 6);
 
 // ...but the fruits list hasn't been changed
 const fruitsCursor = moment.getCursor('fruits');
 const fruits = new ReadArrayList(fruitsCursor!);
-expect(fruits.count()).toBe(3);
+assert.strictEqual(fruits.count(), 3);
 ```
 
 Before we continue, let's save the latest history index, so we can revert back to this moment of the database later:
@@ -232,12 +233,12 @@ const moment = new ReadHashMap(momentCursor!);
 // the cities list contains all four
 const citiesCursor = moment.getCursor('cities');
 const cities = new ReadArrayList(citiesCursor!);
-expect(cities.count()).toBe(4);
+assert.strictEqual(cities.count(), 4);
 
 // ..but so does big-cities! we did not intend to mutate this
 const bigCitiesCursor = moment.getCursor('big-cities');
 const bigCities = new ReadArrayList(bigCitiesCursor!);
-expect(bigCities.count()).toBe(4);
+assert.strictEqual(bigCities.count(), 4);
 ```
 
 The reason that `big-cities` was mutated is because all data in a given transaction is temporarily mutable. This is a very important optimization, but in this case, it's not what we want.
@@ -278,12 +279,12 @@ const moment = new ReadHashMap(momentCursor!);
 // the cities list contains all four
 const citiesCursor = moment.getCursor('cities');
 const cities = new ReadArrayList(citiesCursor!);
-expect(cities.count()).toBe(4);
+assert.strictEqual(cities.count(), 4);
 
 // and big-cities only contains the original two
 const bigCitiesCursor = moment.getCursor('big-cities');
 const bigCities = new ReadArrayList(bigCitiesCursor!);
-expect(bigCities.count()).toBe(2);
+assert.strictEqual(bigCities.count(), 2);
 ```
 
 ## Large Byte Arrays
@@ -315,7 +316,7 @@ for (let n; (n = cursorReader.read(buf)) > 0; ) {
   }
 }
 if (line.length > 0) lineCount++;
-expect(lineCount).toBe(50);
+assert.strictEqual(lineCount, 50);
 ```
 
 ## Iterators
@@ -377,7 +378,7 @@ The size of the hash in bytes will be stored in the database's header. If you tr
 ```typescript
 core.seek(0);
 const header = Header.read(core);
-expect(header.hashSize).toBe(20);
+assert.strictEqual(header.hashSize, 20);
 ```
 
 The hash size alone does not disambiguate hashing algorithms, though. In addition, xitdb reserves four bytes in the header that you can use to put the name of the algorithm. You must provide it in the `Hasher` constructor:
@@ -391,7 +392,7 @@ The hash id is only written to the database header when it is first initialized.
 ```typescript
 core.seek(0);
 const header = Header.read(core);
-expect(Hasher.idToString(header.hashId)).toBe("sha1");
+assert.strictEqual(Hasher.idToString(header.hashId), "sha1");
 ```
 
 If you want to use SHA-256, I recommend using `sha2` as the hash id. You can then distinguish between SHA-256 and SHA-512 using the hash size, like this:
@@ -431,7 +432,7 @@ const compactDb = db.compact(compactCore);
 
 // read from the new compacted db
 const history = new ReadArrayList(compactDb.rootCursor());
-expect(history.count()).toBe(1);
+assert.strictEqual(history.count(), 1);
 ```
 
 This compacted database will be in a separate file. If you want to delete the original database and replace it with this one, you'll need to do that yourself. It is not possible to compact a database in-place (using the same file as the target database); doing so would fail and would render your original database unreadable.

package/dist/core-buffered-file.d.ts

@@ -1,5 +1,5 @@
-import type { Core, DataReader, DataWriter } from './core';
-import { CoreFile } from './core-file';
+import type { Core, DataReader, DataWriter } from './core.js';
+import { CoreFile } from './core-file.js';
 export declare class CoreBufferedFile implements Core {
     file: RandomAccessBufferedFile;
     constructor(filePath: string, bufferSize?: number);

package/dist/core-buffered-file.js

@@ -0,0 +1,182 @@
+import { CoreFile } from './core-file.js';
+import { CoreMemory } from './core-memory.js';
+export class CoreBufferedFile {
+    file;
+    constructor(filePath, bufferSize) {
+        this.file = new RandomAccessBufferedFile(filePath, bufferSize);
+    }
+    reader() {
+        return this.file;
+    }
+    writer() {
+        return this.file;
+    }
+    length() {
+        return this.file.length();
+    }
+    seek(pos) {
+        this.file.seek(pos);
+    }
+    position() {
+        return this.file.position();
+    }
+    setLength(len) {
+        this.file.setLength(len);
+    }
+    flush() {
+        this.file.flush();
+    }
+    sync() {
+        this.file.sync();
+    }
+    [Symbol.dispose]() {
+        this.file.file[Symbol.dispose]();
+    }
+}
+const DEFAULT_BUFFER_SIZE = 8 * 1024 * 1024; // 8MB
+class RandomAccessBufferedFile {
+    file;
+    memory;
+    bufferSize; // flushes when the memory is >= this size
+    filePos;
+    memoryPos;
+    constructor(filePath, bufferSize = DEFAULT_BUFFER_SIZE) {
+        this.file = new CoreFile(filePath);
+        this.memory = new CoreMemory();
+        this.bufferSize = bufferSize;
+        this.filePos = 0;
+        this.memoryPos = 0;
+    }
+    seek(pos) {
+        // flush if we are going past the end of the in-memory buffer
+        if (pos > this.memoryPos + this.memory.length()) {
+            this.flush();
+        }
+        this.filePos = pos;
+        // if the buffer is empty, set its position to this offset as well
+        if (this.memory.length() === 0) {
+            this.memoryPos = pos;
+        }
+    }
+    length() {
+        return Math.max(this.memoryPos + this.memory.length(), this.file.length());
+    }
+    position() {
+        return this.filePos;
+    }
+    setLength(len) {
+        this.flush();
+        this.file.setLength(len);
+        this.filePos = Math.min(len, this.filePos);
+    }
+    flush() {
+        if (this.memory.length() > 0) {
+            this.file.seek(this.memoryPos);
+            this.file.writer().write(this.memory.memory.toByteArray());
+            this.memoryPos = 0;
+            this.memory.memory.reset();
+        }
+    }
+    sync() {
+        this.flush();
+        this.file.sync();
+    }
+    // DataWriter interface
+    write(buffer) {
+        if (this.memory.length() + buffer.length > this.bufferSize) {
+            this.flush();
+        }
+        if (this.filePos >= this.memoryPos && this.filePos <= this.memoryPos + this.memory.length()) {
+            this.memory.seek(this.filePos - this.memoryPos);
+            this.memory.memory.write(buffer);
+        }
+        else {
+            // a direct disk write that overlaps the buffered region would be
+            // clobbered by a later flush of stale buffer bytes, so flush first
+            if (this.filePos < this.memoryPos + this.memory.length() && this.filePos + buffer.length > this.memoryPos) {
+                this.flush();
+            }
+            // Write directly to file
+            this.file.seek(this.filePos);
+            this.file.writer().write(buffer);
+        }
+        this.filePos += buffer.length;
+    }
+    writeByte(v) {
+        this.write(new Uint8Array([v & 0xff]));
+    }
+    writeShort(v) {
+        const buffer = new ArrayBuffer(2);
+        const view = new DataView(buffer);
+        view.setInt16(0, v & 0xffff, false); // big-endian
+        this.write(new Uint8Array(buffer));
+    }
+    writeLong(v) {
+        const buffer = new ArrayBuffer(8);
+        const view = new DataView(buffer);
+        view.setBigInt64(0, BigInt(v), false);
+        this.write(new Uint8Array(buffer));
+    }
+    // DataReader interface
+    readFully(buffer) {
+        let pos = 0;
+        // read from the disk -- before the in-memory buffer
+        if (this.filePos < this.memoryPos) {
+            const sizeBeforeMem = Math.min(this.memoryPos - this.filePos, buffer.length);
+            const tempBuffer = new Uint8Array(sizeBeforeMem);
+            this.file.seek(this.filePos);
+            this.file.reader().readFully(tempBuffer);
+            buffer.set(tempBuffer, pos);
+            pos += sizeBeforeMem;
+            this.filePos += sizeBeforeMem;
+        }
+        if (pos === buffer.length)
+            return;
+        // read from the in-memory buffer
+        if (this.filePos >= this.memoryPos && this.filePos < this.memoryPos + this.memory.length()) {
+            const memPos = this.filePos - this.memoryPos;
+            const sizeInMem = Math.min(this.memory.length() - memPos, buffer.length - pos);
+            this.memory.seek(memPos);
+            const memBuffer = new Uint8Array(sizeInMem);
+            this.memory.memory.readFully(memBuffer);
+            buffer.set(memBuffer, pos);
+            pos += sizeInMem;
+            this.filePos += sizeInMem;
+        }
+        if (pos === buffer.length)
+            return;
+        // read from the disk -- after the in-memory buffer
+        if (this.filePos >= this.memoryPos + this.memory.length()) {
+            const sizeAfterMem = buffer.length - pos;
+            const tempBuffer = new Uint8Array(sizeAfterMem);
+            this.file.seek(this.filePos);
+            this.file.reader().readFully(tempBuffer);
+            buffer.set(tempBuffer, pos);
+            pos += sizeAfterMem;
+            this.filePos += sizeAfterMem;
+        }
+    }
+    readByte() {
+        const bytes = new Uint8Array(1);
+        this.readFully(bytes);
+        return bytes[0];
+    }
+    readShort() {
+        const bytes = new Uint8Array(2);
+        this.readFully(bytes);
+        const view = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength);
+        return view.getInt16(0, false); // big-endian
+    }
+    readInt() {
+        const bytes = new Uint8Array(4);
+        this.readFully(bytes);
+        const view = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength);
+        return view.getInt32(0, false); // big-endian
+    }
+    readLong() {
+        const bytes = new Uint8Array(8);
+        this.readFully(bytes);
+        const view = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength);
+        return Number(view.getBigInt64(0, false));
+    }
+}

package/dist/core-file.d.ts

@@ -1,4 +1,4 @@
-import type { Core, DataReader, DataWriter } from './core';
+import type { Core, DataReader, DataWriter } from './core.js';
 export declare class CoreFile implements Core {
     filePath: string;
     private _position;

package/dist/core-file.js

@@ -0,0 +1,105 @@
+import * as fs from 'fs';
+export class CoreFile {
+    filePath;
+    _position = 0;
+    fd;
+    constructor(filePath) {
+        this.filePath = filePath;
+        // Create file if it doesn't exist
+        try {
+            fs.accessSync(filePath);
+        }
+        catch {
+            fs.writeFileSync(filePath, new Uint8Array(0));
+        }
+        // Open file for reading and writing
+        this.fd = fs.openSync(filePath, 'r+');
+    }
+    reader() {
+        return new FileDataReader(this);
+    }
+    writer() {
+        return new FileDataWriter(this);
+    }
+    length() {
+        const stats = fs.fstatSync(this.fd);
+        return stats.size;
+    }
+    seek(pos) {
+        this._position = pos;
+    }
+    position() {
+        return this._position;
+    }
+    setLength(len) {
+        fs.ftruncateSync(this.fd, len);
+    }
+    flush() {
+    }
+    sync() {
+        fs.fsyncSync(this.fd);
+    }
+    [Symbol.dispose]() {
+        fs.closeSync(this.fd);
+    }
+}
+class FileDataReader {
+    core;
+    constructor(core) {
+        this.core = core;
+    }
+    readFully(b) {
+        const position = this.core.position();
+        fs.readSync(this.core.fd, b, 0, b.length, position);
+        this.core.seek(position + b.length);
+    }
+    readByte() {
+        const bytes = new Uint8Array(1);
+        this.readFully(bytes);
+        return bytes[0];
+    }
+    readShort() {
+        const bytes = new Uint8Array(2);
+        this.readFully(bytes);
+        const view = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength);
+        return view.getInt16(0, false);
+    }
+    readInt() {
+        const bytes = new Uint8Array(4);
+        this.readFully(bytes);
+        const view = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength);
+        return view.getInt32(0, false);
+    }
+    readLong() {
+        const bytes = new Uint8Array(8);
+        this.readFully(bytes);
+        const view = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength);
+        return Number(view.getBigInt64(0, false));
+    }
+}
+class FileDataWriter {
+    core;
+    constructor(core) {
+        this.core = core;
+    }
+    write(buffer) {
+        const position = this.core.position();
+        fs.writeSync(this.core.fd, buffer, 0, buffer.length, position);
+        this.core.seek(position + buffer.length);
+    }
+    writeByte(v) {
+        this.write(new Uint8Array([v & 0xff]));
+    }
+    writeShort(v) {
+        const buffer = new ArrayBuffer(2);
+        const view = new DataView(buffer);
+        view.setInt16(0, v, false);
+        this.write(new Uint8Array(buffer));
+    }
+    writeLong(v) {
+        const buffer = new ArrayBuffer(8);
+        const view = new DataView(buffer);
+        view.setBigInt64(0, BigInt(v), false);
+        this.write(new Uint8Array(buffer));
+    }
+}

package/dist/core-memory.d.ts

@@ -1,4 +1,4 @@
-import type { Core, DataReader, DataWriter } from './core';
+import type { Core, DataReader, DataWriter } from './core.js';
 export declare class CoreMemory implements Core {
     memory: RandomAccessMemory;
     constructor();

package/dist/core-memory.js

@@ -0,0 +1,152 @@
+export class CoreMemory {
+    memory;
+    constructor() {
+        this.memory = new RandomAccessMemory();
+    }
+    reader() {
+        return this.memory;
+    }
+    writer() {
+        return this.memory;
+    }
+    length() {
+        return this.memory.size();
+    }
+    seek(pos) {
+        this.memory.seek(pos);
+    }
+    position() {
+        return this.memory.getPosition();
+    }
+    setLength(len) {
+        this.memory.setLength(len);
+    }
+    flush() {
+        // no-op for in-memory
+    }
+    sync() {
+        // no-op for in-memory
+    }
+}
+class RandomAccessMemory {
+    buffer;
+    _position = 0;
+    _count = 0;
+    constructor(initialSize = 1024) {
+        this.buffer = new Uint8Array(initialSize);
+    }
+    ensureCapacity(minCapacity) {
+        if (minCapacity > this.buffer.length) {
+            let newCapacity = this.buffer.length * 2;
+            if (newCapacity < minCapacity) {
+                newCapacity = minCapacity;
+            }
+            const newBuffer = new Uint8Array(newCapacity);
+            newBuffer.set(this.buffer.subarray(0, this._count));
+            this.buffer = newBuffer;
+        }
+    }
+    size() {
+        return this._count;
+    }
+    seek(pos) {
+        if (pos > this._count) {
+            this._position = this._count;
+        }
+        else {
+            this._position = pos;
+        }
+    }
+    getPosition() {
+        return this._position;
+    }
+    setLength(len) {
+        if (len === 0) {
+            this.reset();
+        }
+        else {
+            if (len > this._count)
+                throw new Error('Cannot extend length');
+            this._count = len;
+            if (this._position > len) {
+                this._position = len;
+            }
+        }
+    }
+    reset() {
+        this._count = 0;
+        this._position = 0;
+    }
+    toByteArray() {
+        return this.buffer.slice(0, this._count);
+    }
+    // DataWriter interface
+    write(data) {
+        const pos = this._position;
+        if (pos < this._count) {
+            const bytesBeforeEnd = Math.min(data.length, this._count - pos);
+            for (let i = 0; i < bytesBeforeEnd; i++) {
+                this.buffer[pos + i] = data[i];
+            }
+            if (bytesBeforeEnd < data.length) {
+                const bytesAfterEnd = data.length - bytesBeforeEnd;
+                this.ensureCapacity(this._count + bytesAfterEnd);
+                this.buffer.set(data.subarray(bytesBeforeEnd), this._count);
+                this._count += bytesAfterEnd;
+            }
+        }
+        else {
+            this.ensureCapacity(this._count + data.length);
+            this.buffer.set(data, this._count);
+            this._count += data.length;
+        }
+        this._position = pos + data.length;
+    }
+    writeByte(v) {
+        this.write(new Uint8Array([v & 0xff]));
+    }
+    writeShort(v) {
+        const buffer = new ArrayBuffer(2);
+        const view = new DataView(buffer);
+        view.setInt16(0, v, false); // big-endian
+        this.write(new Uint8Array(buffer));
+    }
+    writeLong(v) {
+        const buffer = new ArrayBuffer(8);
+        const view = new DataView(buffer);
+        view.setBigInt64(0, BigInt(v), false);
+        this.write(new Uint8Array(buffer));
+    }
+    // DataReader interface
+    readFully(b) {
+        const pos = this._position;
+        if (pos + b.length > this._count) {
+            throw new Error('End of stream');
+        }
+        b.set(this.buffer.subarray(pos, pos + b.length));
+        this._position = pos + b.length;
+    }
+    readByte() {
+        const bytes = new Uint8Array(1);
+        this.readFully(bytes);
+        return bytes[0];
+    }
+    readShort() {
+        const bytes = new Uint8Array(2);
+        this.readFully(bytes);
+        const view = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength);
+        return view.getInt16(0, false); // big-endian
+    }
+    readInt() {
+        const bytes = new Uint8Array(4);
+        this.readFully(bytes);
+        const view = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength);
+        return view.getInt32(0, false); // big-endian
+    }
+    readLong() {
+        const bytes = new Uint8Array(8);
+        this.readFully(bytes);
+        const view = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength);
+        return Number(view.getBigInt64(0, false));
+    }
+}

package/dist/core.js

@@ -0,0 +1 @@
+export {};