xitdb 0.12.0 → 0.14.0

package/README.md

@@ -17,6 +17,7 @@
 * Reads never block writes, and a database can be read from multiple threads/processes without locks.
 * No query engine of any kind. You just write data structures (primarily an `ArrayList` and `HashMap`) that can be nested arbitrarily.
 * No dependencies besides the JavaScript standard library.
+* Fully synchronous API — no async/await needed.
 * Available [on npm](https://www.npmjs.com/package/xitdb).
 
 This database was originally made for the [xit version control system](https://github.com/xit-vcs/xit), but I bet it has a lot of potential for other projects. The combination of being immutable and having an API similar to in-memory data structures is pretty powerful. Consider using it [instead of SQLite](https://gist.github.com/xeubie/03a0724484e1111ef4c05d72a935c42c) for your TypeScript projects: it's simpler, it's pure TypeScript, and it creates no impedance mismatch with your program the way SQL databases do.
@@ -36,13 +37,13 @@ In this example, we create a new database, write some data in a transaction, and
 
 ```typescript
 // init the db
-using core = await CoreBufferedFile.create('main.db');
+using core = new CoreBufferedFile('main.db');
 const hasher = new Hasher('SHA-1');
-const db = await Database.create(core, hasher);
+const db = new Database(core, hasher);
 
 // to get the benefits of immutability, the top-level data structure
 // must be an ArrayList, so each transaction is stored as an item in it
-const history = await WriteArrayList.create(db.rootCursor());
+const history = new WriteArrayList(db.rootCursor());
 
 // this is how a transaction is executed. we call history.appendContext,
 // providing it with the most recent copy of the db and a context
@@ -61,53 +62,53 @@ const history = await WriteArrayList.create(db.rootCursor());
 //    {"name": "Alice", "age": 25},
 //    {"name": "Bob", "age": 42}
 //  ]}
-await history.appendContext(await history.getSlot(-1), async (cursor) => {
-  const moment = await WriteHashMap.create(cursor);
-
-  await moment.put('foo', new Bytes('foo'));
-  await moment.put('bar', new Bytes('bar'));
-
-  const fruitsCursor = await moment.putCursor('fruits');
-  const fruits = await WriteArrayList.create(fruitsCursor);
-  await fruits.append(new Bytes('apple'));
-  await fruits.append(new Bytes('pear'));
-  await fruits.append(new Bytes('grape'));
-
-  const peopleCursor = await moment.putCursor('people');
-  const people = await WriteArrayList.create(peopleCursor);
-
-  const aliceCursor = await people.appendCursor();
-  const alice = await WriteHashMap.create(aliceCursor);
-  await alice.put('name', new Bytes('Alice'));
-  await alice.put('age', new Uint(25));
-
-  const bobCursor = await people.appendCursor();
-  const bob = await WriteHashMap.create(bobCursor);
-  await bob.put('name', new Bytes('Bob'));
-  await bob.put('age', new Uint(42));
+history.appendContext(history.getSlot(-1), (cursor) => {
+  const moment = new WriteHashMap(cursor);
+
+  moment.put('foo', new Bytes('foo'));
+  moment.put('bar', new Bytes('bar'));
+
+  const fruitsCursor = moment.putCursor('fruits');
+  const fruits = new WriteArrayList(fruitsCursor);
+  fruits.append(new Bytes('apple'));
+  fruits.append(new Bytes('pear'));
+  fruits.append(new Bytes('grape'));
+
+  const peopleCursor = moment.putCursor('people');
+  const people = new WriteArrayList(peopleCursor);
+
+  const aliceCursor = people.appendCursor();
+  const alice = new WriteHashMap(aliceCursor);
+  alice.put('name', new Bytes('Alice'));
+  alice.put('age', new Uint(25));
+
+  const bobCursor = people.appendCursor();
+  const bob = new WriteHashMap(bobCursor);
+  bob.put('name', new Bytes('Bob'));
+  bob.put('age', new Uint(42));
 });
 
 // get the most recent copy of the database, like a moment
 // in time. the -1 index will return the last index in the list.
-const momentCursor = await history.getCursor(-1);
+const momentCursor = history.getCursor(-1);
 const moment = new ReadHashMap(momentCursor!);
 
 // we can read the value of "foo" from the map by getting
 // the cursor to "foo" and then calling readBytes on it
-const fooCursor = await moment.getCursor('foo');
-const fooValue = await fooCursor!.readBytes(MAX_READ_BYTES);
-expect(new TextDecoder().decode(fooValue)).toBe('foo');
+const fooCursor = moment.getCursor('foo');
+const fooValue = fooCursor!.readBytes(MAX_READ_BYTES);
+assert.strictEqual(new TextDecoder().decode(fooValue), 'foo');
 
 // to get the "fruits" list, we get the cursor to it and
 // then pass it to the ReadArrayList constructor
-const fruitsCursor = await moment.getCursor('fruits');
+const fruitsCursor = moment.getCursor('fruits');
 const fruits = new ReadArrayList(fruitsCursor!);
-expect(await fruits.count()).toBe(3);
+assert.strictEqual(fruits.count(), 3);
 
 // now we can get the first item from the fruits list and read it
-const appleCursor = await fruits.getCursor(0);
-const appleValue = await appleCursor!.readBytes(MAX_READ_BYTES);
-expect(new TextDecoder().decode(appleValue)).toBe('apple');
+const appleCursor = fruits.getCursor(0);
+const appleValue = appleCursor!.readBytes(MAX_READ_BYTES);
+assert.strictEqual(new TextDecoder().decode(appleValue), 'apple');
 ```
 
 ## Initializing a Database
@@ -131,8 +132,9 @@ In xitdb there are a variety of immutable data structures that you can nest arbi
 * `CountedHashMap` and `CountedHashSet` are just a `HashMap` and `HashSet` that maintain a count of their contents
 * `ArrayList` is a growable array
 * `LinkedArrayList` is like an `ArrayList` that can also be efficiently sliced and concatenated
+* `SortedMap` and `SortedSet` are like a `HashMap` and `HashSet` where the keys are byte arrays kept in lexicographic order
 
-All data structures use the hash array mapped trie, invented by Phil Bagwell. The `LinkedArrayList` is based on his later work on RRB trees. These data structures were originally made immutable and widely available by Rich Hickey in Clojure. To my knowledge, they haven't been available in any open source database until xitdb.
+The `Hash`-based data structures and the `Arraylist` use the hash array mapped trie, invented by Phil Bagwell (originally made immutable and widely available by Rich Hickey in Clojure). The `LinkedArrayList`, `SortedMap`, and `SortedSet` are based on a B-tree.
 
 There are also scalar types you can store in the above-mentioned data structures:
 
@@ -148,15 +150,15 @@ In xitdb, you can optionally store a format tag with a byte array. A format tag
 ```typescript
 const randomBytes = new Uint8Array(32);
 crypto.getRandomValues(randomBytes);
-await moment.put('random-number', new Bytes(randomBytes, new TextEncoder().encode('bi')));
+moment.put('random-number', new Bytes(randomBytes, new TextEncoder().encode('bi')));
 ```
 
 Then, you can read it like this:
 
 ```typescript
-const randomNumberCursor = await moment.getCursor('random-number');
-const randomNumber = await randomNumberCursor!.readBytesObject(MAX_READ_BYTES);
-expect(new TextDecoder().decode(randomNumber.formatTag!)).toBe('bi');
+const randomNumberCursor = moment.getCursor('random-number');
+const randomNumber = randomNumberCursor!.readBytesObject(MAX_READ_BYTES);
+assert.strictEqual(new TextDecoder().decode(randomNumber.formatTag!), 'bi');
 const randomBigInt = randomNumber.value;
 ```
 
@@ -167,76 +169,76 @@ There are many types you may want to store this way. Maybe an ISO-8601 date like
 A powerful feature of immutable data is fast cloning. Any data structure can be instantly cloned and changed without affecting the original. Starting with the example code above, we can make a new transaction that creates a "food" list based on the existing "fruits" list:
 
 ```typescript
-await history.appendContext(await history.getSlot(-1), async (cursor) => {
-  const moment = await WriteHashMap.create(cursor);
+history.appendContext(history.getSlot(-1), (cursor) => {
+  const moment = new WriteHashMap(cursor);
 
-  const fruitsCursor = await moment.getCursor('fruits');
+  const fruitsCursor = moment.getCursor('fruits');
   const fruits = new ReadArrayList(fruitsCursor!);
 
   // create a new key called "food" whose initial value is
   // based on the "fruits" list
-  const foodCursor = await moment.putCursor('food');
-  await foodCursor.write(fruits.slot());
+  const foodCursor = moment.putCursor('food');
+  foodCursor.write(fruits.slot());
 
-  const food = await WriteArrayList.create(foodCursor);
-  await food.append(new Bytes('eggs'));
-  await food.append(new Bytes('rice'));
-  await food.append(new Bytes('fish'));
+  const food = new WriteArrayList(foodCursor);
+  food.append(new Bytes('eggs'));
+  food.append(new Bytes('rice'));
+  food.append(new Bytes('fish'));
 });
 
-const momentCursor = await history.getCursor(-1);
+const momentCursor = history.getCursor(-1);
 const moment = new ReadHashMap(momentCursor!);
 
 // the food list includes the fruits
-const foodCursor = await moment.getCursor('food');
+const foodCursor = moment.getCursor('food');
 const food = new ReadArrayList(foodCursor!);
-expect(await food.count()).toBe(6);
+assert.strictEqual(food.count(), 6);
 
 // ...but the fruits list hasn't been changed
-const fruitsCursor = await moment.getCursor('fruits');
+const fruitsCursor = moment.getCursor('fruits');
 const fruits = new ReadArrayList(fruitsCursor!);
-expect(await fruits.count()).toBe(3);
+assert.strictEqual(fruits.count(), 3);
 ```
 
 Before we continue, let's save the latest history index, so we can revert back to this moment of the database later:
 
 ```typescript
-const historyIndex = (await history.count()) - 1;
+const historyIndex = history.count() - 1;
 ```
 
 There's one catch you'll run into when cloning. If we try cloning a data structure that was created in the same transaction, it doesn't seem to work:
 
 ```typescript
-await history.appendContext(await history.getSlot(-1), async (cursor) => {
-  const moment = await WriteHashMap.create(cursor);
+history.appendContext(history.getSlot(-1), (cursor) => {
+  const moment = new WriteHashMap(cursor);
 
-  const bigCitiesCursor = await moment.putCursor('big-cities');
-  const bigCities = await WriteArrayList.create(bigCitiesCursor);
-  await bigCities.append(new Bytes('New York, NY'));
-  await bigCities.append(new Bytes('Los Angeles, CA'));
+  const bigCitiesCursor = moment.putCursor('big-cities');
+  const bigCities = new WriteArrayList(bigCitiesCursor);
+  bigCities.append(new Bytes('New York, NY'));
+  bigCities.append(new Bytes('Los Angeles, CA'));
 
   // create a new key called "cities" whose initial value is
   // based on the "big-cities" list
-  const citiesCursor = await moment.putCursor('cities');
-  await citiesCursor.write(bigCities.slot());
+  const citiesCursor = moment.putCursor('cities');
+  citiesCursor.write(bigCities.slot());
 
-  const cities = await WriteArrayList.create(citiesCursor);
-  await cities.append(new Bytes('Charleston, SC'));
-  await cities.append(new Bytes('Louisville, KY'));
+  const cities = new WriteArrayList(citiesCursor);
+  cities.append(new Bytes('Charleston, SC'));
+  cities.append(new Bytes('Louisville, KY'));
 });
 
-const momentCursor = await history.getCursor(-1);
+const momentCursor = history.getCursor(-1);
 const moment = new ReadHashMap(momentCursor!);
 
 // the cities list contains all four
-const citiesCursor = await moment.getCursor('cities');
+const citiesCursor = moment.getCursor('cities');
 const cities = new ReadArrayList(citiesCursor!);
-expect(await cities.count()).toBe(4);
+assert.strictEqual(cities.count(), 4);
 
 // ..but so does big-cities! we did not intend to mutate this
-const bigCitiesCursor = await moment.getCursor('big-cities');
+const bigCitiesCursor = moment.getCursor('big-cities');
 const bigCities = new ReadArrayList(bigCitiesCursor!);
-expect(await bigCities.count()).toBe(4);
+assert.strictEqual(bigCities.count(), 4);
 ```
 
 The reason that `big-cities` was mutated is because all data in a given transaction is temporarily mutable. This is a very important optimization, but in this case, it's not what we want.
@@ -244,45 +246,45 @@ The reason that `big-cities` was mutated is because all data in a given transact
 To show how to fix this, let's first undo the transaction we just made. Here we use the `historyIndex` we saved before to revert back to the older database moment:
 
 ```typescript
-await history.append((await history.getSlot(historyIndex))!);
+history.append(history.getSlot(historyIndex)!);
 ```
 
 This time, after making the "big cities" list, we call `freeze`, which tells xitdb to consider all data made so far in the transaction to be immutable. After that, we can clone it into the "cities" list and it will work the way we wanted:
 
 ```typescript
-await history.appendContext(await history.getSlot(-1), async (cursor) => {
-  const moment = await WriteHashMap.create(cursor);
+history.appendContext(history.getSlot(-1), (cursor) => {
+  const moment = new WriteHashMap(cursor);
 
-  const bigCitiesCursor = await moment.putCursor('big-cities');
-  const bigCities = await WriteArrayList.create(bigCitiesCursor);
-  await bigCities.append(new Bytes('New York, NY'));
-  await bigCities.append(new Bytes('Los Angeles, CA'));
+  const bigCitiesCursor = moment.putCursor('big-cities');
+  const bigCities = new WriteArrayList(bigCitiesCursor);
+  bigCities.append(new Bytes('New York, NY'));
+  bigCities.append(new Bytes('Los Angeles, CA'));
 
   // freeze here, so big-cities won't be mutated
   cursor.db.freeze();
 
   // create a new key called "cities" whose initial value is
   // based on the "big-cities" list
-  const citiesCursor = await moment.putCursor('cities');
-  await citiesCursor.write(bigCities.slot());
+  const citiesCursor = moment.putCursor('cities');
+  citiesCursor.write(bigCities.slot());
 
-  const cities = await WriteArrayList.create(citiesCursor);
-  await cities.append(new Bytes('Charleston, SC'));
-  await cities.append(new Bytes('Louisville, KY'));
+  const cities = new WriteArrayList(citiesCursor);
+  cities.append(new Bytes('Charleston, SC'));
+  cities.append(new Bytes('Louisville, KY'));
 });
 
-const momentCursor = await history.getCursor(-1);
+const momentCursor = history.getCursor(-1);
 const moment = new ReadHashMap(momentCursor!);
 
 // the cities list contains all four
-const citiesCursor = await moment.getCursor('cities');
+const citiesCursor = moment.getCursor('cities');
 const cities = new ReadArrayList(citiesCursor!);
-expect(await cities.count()).toBe(4);
+assert.strictEqual(cities.count(), 4);
 
 // and big-cities only contains the original two
-const bigCitiesCursor = await moment.getCursor('big-cities');
+const bigCitiesCursor = moment.getCursor('big-cities');
 const bigCities = new ReadArrayList(bigCitiesCursor!);
-expect(await bigCities.count()).toBe(2);
+assert.strictEqual(bigCities.count(), 2);
 ```
 
 ## Large Byte Arrays
@@ -290,12 +292,12 @@ expect(await bigCities.count()).toBe(2);
 When reading and writing large byte arrays, you probably don't want to have all of their contents in memory at once. To incrementally write to a byte array, just get a writer from a cursor:
 
 ```typescript
-const longTextCursor = await moment.putCursor('long-text');
-const cursorWriter = await longTextCursor.writer();
+const longTextCursor = moment.putCursor('long-text');
+const cursorWriter = longTextCursor.writer();
 for (let i = 0; i < 50; i++) {
-  await cursorWriter.write(new TextEncoder().encode('hello, world\n'));
+  cursorWriter.write(new TextEncoder().encode('hello, world\n'));
 }
-await cursorWriter.finish(); // remember to call this!
+cursorWriter.finish(); // remember to call this!
 ```
 
 If you need to set a format tag for the byte array, put it in the `formatTag` field of the writer before you call `finish`.
@@ -303,18 +305,18 @@ If you need to set a format tag for the byte array, put it in the `formatTag` fi
 To read a byte array incrementally, get a reader from a cursor:
 
 ```typescript
-const longTextCursor = await moment.getCursor('long-text');
-const cursorReader = await longTextCursor!.reader();
+const longTextCursor = moment.getCursor('long-text');
+const cursorReader = longTextCursor!.reader();
 let lineCount = 0, line: number[] = [];
 const buf = new Uint8Array(1024);
-for (let n; (n = await cursorReader.read(buf)) > 0; ) {
+for (let n; (n = cursorReader.read(buf)) > 0; ) {
   for (let i = 0; i < n; i++) {
     if (buf[i] === 0x0A) { lineCount++; line = []; }
     else line.push(buf[i]);
   }
 }
 if (line.length > 0) lineCount++;
-expect(lineCount).toBe(50);
+assert.strictEqual(lineCount, 50);
 ```
 
 ## Iterators
@@ -322,24 +324,24 @@ expect(lineCount).toBe(50);
 All data structures support iteration. Here's an example of iterating over an `ArrayList` and printing all of the keys and values of each `HashMap` contained in it:
 
 ```typescript
-const peopleCursor = await moment.getCursor('people');
+const peopleCursor = moment.getCursor('people');
 const people = new ReadArrayList(peopleCursor!);
 
-const peopleIter = await people.iterator();
-while (await peopleIter.hasNext()) {
-  const personCursor = await peopleIter.next();
+const peopleIter = people.iterator();
+while (peopleIter.hasNext()) {
+  const personCursor = peopleIter.next();
   const person = new ReadHashMap(personCursor!);
-  const personIter = await person.iterator();
-  while (await personIter.hasNext()) {
-    const kvPairCursor = await personIter.next();
-    const kvPair = await kvPairCursor!.readKeyValuePair();
+  const personIter = person.iterator();
+  while (personIter.hasNext()) {
+    const kvPairCursor = personIter.next();
+    const kvPair = kvPairCursor!.readKeyValuePair();
 
-    const key = new TextDecoder().decode(await kvPair.keyCursor.readBytes(MAX_READ_BYTES));
+    const key = new TextDecoder().decode(kvPair.keyCursor.readBytes(MAX_READ_BYTES));
 
     switch (kvPair.valueCursor.slot().tag) {
       case Tag.SHORT_BYTES:
       case Tag.BYTES:
-        console.log(`${key}: ${new TextDecoder().decode(await kvPair.valueCursor.readBytes(MAX_READ_BYTES))}`);
+        console.log(`${key}: ${new TextDecoder().decode(kvPair.valueCursor.readBytes(MAX_READ_BYTES))}`);
         break;
       case Tag.UINT:
         console.log(`${key}: ${kvPair.valueCursor.readUint()}`);
@@ -366,17 +368,17 @@ The hashing data structures will create the hash for you when you call methods l
 When initializing a database, you tell xitdb how to hash with the `Hasher`. If you're using SHA-1, it will look like this:
 
 ```typescript
-using core = await CoreBufferedFile.create('main.db');
+using core = new CoreBufferedFile('main.db');
 const hasher = new Hasher('SHA-1');
-const db = await Database.create(core, hasher);
+const db = new Database(core, hasher);
 ```
 
 The size of the hash in bytes will be stored in the database's header. If you try opening it later with a hashing algorithm that has the wrong hash size, it will throw an exception. If you are unsure what hash size the database uses, this creates a chicken-and-egg problem. You can read the header before initializing the database like this:
 
 ```typescript
-await core.seek(0);
-const header = await Header.read(core);
-expect(header.hashSize).toBe(20);
+core.seek(0);
+const header = Header.read(core);
+assert.strictEqual(header.hashSize, 20);
 ```
 
 The hash size alone does not disambiguate hashing algorithms, though. In addition, xitdb reserves four bytes in the header that you can use to put the name of the algorithm. You must provide it in the `Hasher` constructor:
@@ -388,9 +390,9 @@ const hasher = new Hasher('SHA-1', Hasher.stringToId('sha1'));
 The hash id is only written to the database header when it is first initialized. When you open it later, the hash id in the `Hasher` is ignored. You can read the hash id of an existing database like this:
 
 ```typescript
-await core.seek(0);
-const header = await Header.read(core);
-expect(Hasher.idToString(header.hashId)).toBe("sha1");
+core.seek(0);
+const header = Header.read(core);
+assert.strictEqual(Hasher.idToString(header.hashId), "sha1");
 ```
 
 If you want to use SHA-256, I recommend using `sha2` as the hash id. You can then distinguish between SHA-256 and SHA-512 using the hash size, like this:
@@ -425,12 +427,12 @@ switch (hashIdStr) {
 Normally, an immutable database grows forever, because old data is never deleted. To reclaim disk space and clear the history, xitdb supports compaction. This involves completely rebuilding the database file to only contain the data accessible from the latest copy (i.e., "moment") of the database.
 
 ```typescript
-using compactCore = await CoreBufferedFile.create('compact.db');
-const compactDb = await db.compact(compactCore);
+using compactCore = new CoreBufferedFile('compact.db');
+const compactDb = db.compact(compactCore);
 
 // read from the new compacted db
 const history = new ReadArrayList(compactDb.rootCursor());
-expect(await history.count()).toBe(1);
+assert.strictEqual(history.count(), 1);
 ```
 
 This compacted database will be in a separate file. If you want to delete the original database and replace it with this one, you'll need to do that yourself. It is not possible to compact a database in-place (using the same file as the target database); doing so would fail and would render your original database unreadable.

package/dist/core-buffered-file.d.ts

@@ -1,17 +1,16 @@
-import type { Core, DataReader, DataWriter } from './core';
-import { CoreFile } from './core-file';
+import type { Core, DataReader, DataWriter } from './core.js';
+import { CoreFile } from './core-file.js';
 export declare class CoreBufferedFile implements Core {
     file: RandomAccessBufferedFile;
-    constructor(file: RandomAccessBufferedFile);
-    static create(filePath: string, bufferSize?: number): Promise<CoreBufferedFile>;
+    constructor(filePath: string, bufferSize?: number);
     reader(): DataReader;
     writer(): DataWriter;
-    length(): Promise<number>;
-    seek(pos: number): Promise<void>;
+    length(): number;
+    seek(pos: number): void;
     position(): number;
-    setLength(len: number): Promise<void>;
-    flush(): Promise<void>;
-    sync(): Promise<void>;
+    setLength(len: number): void;
+    flush(): void;
+    sync(): void;
     [Symbol.dispose](): void;
 }
 declare class RandomAccessBufferedFile implements DataReader, DataWriter {
@@ -20,22 +19,21 @@ declare class RandomAccessBufferedFile implements DataReader, DataWriter {
     private bufferSize;
     private filePos;
     private memoryPos;
-    private constructor();
-    static create(filePath: string, bufferSize?: number): Promise<RandomAccessBufferedFile>;
-    seek(pos: number): Promise<void>;
-    length(): Promise<number>;
+    constructor(filePath: string, bufferSize?: number);
+    seek(pos: number): void;
+    length(): number;
     position(): number;
-    setLength(len: number): Promise<void>;
-    flush(): Promise<void>;
-    sync(): Promise<void>;
-    write(buffer: Uint8Array): Promise<void>;
-    writeByte(v: number): Promise<void>;
-    writeShort(v: number): Promise<void>;
-    writeLong(v: number): Promise<void>;
-    readFully(buffer: Uint8Array): Promise<void>;
-    readByte(): Promise<number>;
-    readShort(): Promise<number>;
-    readInt(): Promise<number>;
-    readLong(): Promise<number>;
+    setLength(len: number): void;
+    flush(): void;
+    sync(): void;
+    write(buffer: Uint8Array): void;
+    writeByte(v: number): void;
+    writeShort(v: number): void;
+    writeLong(v: number): void;
+    readFully(buffer: Uint8Array): void;
+    readByte(): number;
+    readShort(): number;
+    readInt(): number;
+    readLong(): number;
 }
 export {};