xitdb 0.10.0 → 0.12.0

package/README.md

@@ -3,19 +3,23 @@
   <br/>
   <br/>
   <b>Choose your flavor:</b>
-  <a href="https://github.com/radarroark/xitdb">Zig</a> |
-  <a href="https://github.com/radarroark/xitdb-java">Java</a> |
+  <a href="https://github.com/xit-vcs/xitdb">Zig</a> |
+  <a href="https://github.com/xit-vcs/xitdb-java">Java</a> |
   <a href="https://github.com/codeboost/xitdb-clj">Clojure</a> |
-  <a href="https://github.com/radarroark/xitdb-ts">TypeScript</a>
+  <a href="https://github.com/xit-vcs/xitdb-ts">TypeScript</a> |
+  <a href="https://github.com/xit-vcs/xitdb-go">Go</a>
 </p>
 
-* Each transaction efficiently creates a new "copy" of the database, and past copies can still be read from.
-* It supports writing to a file as well as purely in-memory use.
+* Each transaction efficiently creates a new "copy" of the database, and past copies can still be read from and reverted to.
+* Supports storing in a single file as well as purely in-memory use.
+* Runs as a library (embedded in process).
+* Incrementally reads and writes, so file-based databases can contain larger-than-memory datasets.
+* Reads never block writes, and a database can be read from multiple threads/processes without locks.
 * No query engine of any kind. You just write data structures (primarily an `ArrayList` and `HashMap`) that can be nested arbitrarily.
 * No dependencies besides the JavaScript standard library.
-* It is available [on npm](https://www.npmjs.com/package/xitdb).
+* Available [on npm](https://www.npmjs.com/package/xitdb).
 
-This database was originally made for the [xit version control system](https://github.com/radarroark/xit), but I bet it has a lot of potential for other projects. The combination of being immutable and having an API similar to in-memory data structures is pretty powerful. Consider using it [instead of SQLite](https://gist.github.com/radarroark/03a0724484e1111ef4c05d72a935c42c) for your TypeScript projects: it's simpler, it's pure TypeScript, and it creates no impedance mismatch with your program the way SQL databases do.
+This database was originally made for the [xit version control system](https://github.com/xit-vcs/xit), but I bet it has a lot of potential for other projects. The combination of being immutable and having an API similar to in-memory data structures is pretty powerful. Consider using it [instead of SQLite](https://gist.github.com/xeubie/03a0724484e1111ef4c05d72a935c42c) for your TypeScript projects: it's simpler, it's pure TypeScript, and it creates no impedance mismatch with your program the way SQL databases do.
 
 * [Example](#example)
 * [Initializing a Database](#initializing-a-database)
@@ -24,6 +28,7 @@ This database was originally made for the [xit version control system](https://g
 * [Large Byte Arrays](#large-byte-arrays)
 * [Iterators](#iterators)
 * [Hashing](#hashing)
+* [Compaction](#compaction)
 
 ## Example
 
@@ -414,3 +419,18 @@ switch (hashIdStr) {
     throw new Error('Invalid hash algorithm');
 }
 ```
+
+## Compaction
+
+Normally, an immutable database grows forever, because old data is never deleted. To reclaim disk space and clear the history, xitdb supports compaction. This involves completely rebuilding the database file to only contain the data accessible from the latest copy (i.e., "moment") of the database.
+
+```typescript
+using compactCore = await CoreBufferedFile.create('compact.db');
+const compactDb = await db.compact(compactCore);
+
+// read from the new compacted db
+const history = new ReadArrayList(compactDb.rootCursor());
+expect(await history.count()).toBe(1);
+```
+
+This compacted database will be in a separate file. If you want to delete the original database and replace it with this one, you'll need to do that yourself. It is not possible to compact a database in-place (using the same file as the target database); doing so would fail and would render your original database unreadable.

package/dist/database.d.ts

@@ -223,6 +223,7 @@ export declare class Database {
     static create(core: Core, hasher: Hasher): Promise<Database>;
     rootCursor(): WriteCursor;
     freeze(): Promise<void>;
+    compact(targetCore: Core): Promise<Database>;
     truncate(): Promise<void>;
     checkHashBytes(hash: Uint8Array): Uint8Array;
     checkHash(target: HashMapGetTarget): Uint8Array;

package/dist/index.js

@@ -2317,6 +2317,43 @@ class Database3 {
       throw new ExpectedTxStartException;
     }
   }
+  async compact(targetCore) {
+    const offsetMap = new Map;
+    const hasher = new Hasher(this.hasher.algorithm, this.header.hashId);
+    const target = await Database3.create(targetCore, hasher);
+    if (this.header.tag === 0 /* NONE */)
+      return target;
+    if (this.header.tag !== 2 /* ARRAY_LIST */)
+      throw new UnexpectedTagException;
+    await this.core.seek(Header.LENGTH);
+    const sourceReader = this.core.reader();
+    const headerBytes = new Uint8Array(ArrayListHeader.LENGTH);
+    await sourceReader.readFully(headerBytes);
+    const sourceHeader = ArrayListHeader.fromBytes(headerBytes);
+    if (sourceHeader.size === 0)
+      return target;
+    const lastKey = sourceHeader.size - 1;
+    const shift = lastKey < SLOT_COUNT ? 0 : Math.floor(Math.log(lastKey) / Math.log(SLOT_COUNT));
+    const lastSlotPtr = await this.readArrayListSlot(sourceHeader.ptr, lastKey, shift, 0 /* READ_ONLY */, true);
+    const momentSlot = lastSlotPtr.slot;
+    const targetWriter = targetCore.writer();
+    await targetCore.seek(Header.LENGTH);
+    const targetArrayListPtr = Header.LENGTH + TopLevelArrayListHeader.LENGTH;
+    await targetWriter.write(new TopLevelArrayListHeader(0, new ArrayListHeader(targetArrayListPtr, 1)).toBytes());
+    await targetWriter.write(new Uint8Array(INDEX_BLOCK_SIZE));
+    const remappedMoment = await remapSlot(this.core, targetCore, this.header.hashSize, offsetMap, momentSlot);
+    await targetCore.seek(targetArrayListPtr);
+    await targetWriter.write(remappedMoment.toBytes());
+    target.header = target.header.withTag(2 /* ARRAY_LIST */);
+    await targetCore.seek(0);
+    await target.header.write(targetCore);
+    await targetCore.flush();
+    const fileSize = await targetCore.length();
+    await targetCore.seek(Header.LENGTH + ArrayListHeader.LENGTH);
+    await targetWriter.writeLong(fileSize);
+    await targetCore.flush();
+    return target;
+  }
   async truncate() {
     if (this.header.tag !== 2 /* ARRAY_LIST */)
       return;
@@ -3141,6 +3178,216 @@ class Database3 {
     return new LinkedArrayListHeader(nextShift, rootPtr, headerA.size + headerB.size);
   }
 }
+async function remapSlot(sourceCore, targetCore, hashSize, offsetMap, slot) {
+  switch (slot.tag) {
+    case 0 /* NONE */:
+    case 8 /* UINT */:
+    case 9 /* INT */:
+    case 10 /* FLOAT */:
+    case 7 /* SHORT_BYTES */:
+      return slot;
+    case 6 /* BYTES */: {
+      const mapped = offsetMap.get(Number(slot.value));
+      if (mapped !== undefined)
+        return new Slot(mapped, slot.tag, slot.full);
+      const newOffset = await remapBytes(sourceCore, targetCore, slot);
+      offsetMap.set(Number(slot.value), newOffset);
+      return new Slot(newOffset, slot.tag, slot.full);
+    }
+    case 1 /* INDEX */: {
+      const mapped = offsetMap.get(Number(slot.value));
+      if (mapped !== undefined)
+        return new Slot(mapped, slot.tag, slot.full);
+      const newOffset = await remapIndex(sourceCore, targetCore, hashSize, offsetMap, slot);
+      offsetMap.set(Number(slot.value), newOffset);
+      return new Slot(newOffset, slot.tag, slot.full);
+    }
+    case 2 /* ARRAY_LIST */: {
+      const mapped = offsetMap.get(Number(slot.value));
+      if (mapped !== undefined)
+        return new Slot(mapped, slot.tag, slot.full);
+      const newOffset = await remapArrayList(sourceCore, targetCore, hashSize, offsetMap, slot);
+      offsetMap.set(Number(slot.value), newOffset);
+      return new Slot(newOffset, slot.tag, slot.full);
+    }
+    case 3 /* LINKED_ARRAY_LIST */: {
+      const mapped = offsetMap.get(Number(slot.value));
+      if (mapped !== undefined)
+        return new Slot(mapped, slot.tag, slot.full);
+      const newOffset = await remapLinkedArrayList(sourceCore, targetCore, hashSize, offsetMap, slot);
+      offsetMap.set(Number(slot.value), newOffset);
+      return new Slot(newOffset, slot.tag, slot.full);
+    }
+    case 4 /* HASH_MAP */:
+    case 11 /* HASH_SET */: {
+      const mapped = offsetMap.get(Number(slot.value));
+      if (mapped !== undefined)
+        return new Slot(mapped, slot.tag, slot.full);
+      const newOffset = await remapHashMapOrSet(sourceCore, targetCore, hashSize, offsetMap, slot, false);
+      offsetMap.set(Number(slot.value), newOffset);
+      return new Slot(newOffset, slot.tag, slot.full);
+    }
+    case 12 /* COUNTED_HASH_MAP */:
+    case 13 /* COUNTED_HASH_SET */: {
+      const mapped = offsetMap.get(Number(slot.value));
+      if (mapped !== undefined)
+        return new Slot(mapped, slot.tag, slot.full);
+      const newOffset = await remapHashMapOrSet(sourceCore, targetCore, hashSize, offsetMap, slot, true);
+      offsetMap.set(Number(slot.value), newOffset);
+      return new Slot(newOffset, slot.tag, slot.full);
+    }
+    case 5 /* KV_PAIR */: {
+      const mapped = offsetMap.get(Number(slot.value));
+      if (mapped !== undefined)
+        return new Slot(mapped, slot.tag, slot.full);
+      const newOffset = await remapKvPair(sourceCore, targetCore, hashSize, offsetMap, slot);
+      offsetMap.set(Number(slot.value), newOffset);
+      return new Slot(newOffset, slot.tag, slot.full);
+    }
+    default:
+      throw new UnexpectedTagException;
+  }
+}
+async function remapBytes(sourceCore, targetCore, slot) {
+  await sourceCore.seek(Number(slot.value));
+  const sourceReader = sourceCore.reader();
+  const length = await sourceReader.readLong();
+  const formatTagSize = slot.full ? 2 : 0;
+  const totalPayload = length + formatTagSize;
+  const newOffset = await targetCore.length();
+  await targetCore.seek(newOffset);
+  const targetWriter = targetCore.writer();
+  await targetWriter.writeLong(length);
+  let remaining = totalPayload;
+  while (remaining > 0) {
+    const chunk = Math.min(remaining, 4096);
+    const buf = new Uint8Array(chunk);
+    await sourceReader.readFully(buf);
+    await targetWriter.write(buf);
+    remaining -= chunk;
+  }
+  return newOffset;
+}
+async function remapIndex(sourceCore, targetCore, hashSize, offsetMap, slot) {
+  await sourceCore.seek(Number(slot.value));
+  const sourceReader = sourceCore.reader();
+  const blockBytes = new Uint8Array(INDEX_BLOCK_SIZE);
+  await sourceReader.readFully(blockBytes);
+  const remappedSlots = [];
+  for (let i = 0;i < SLOT_COUNT; i++) {
+    const slotBytes = blockBytes.slice(i * Slot.LENGTH, (i + 1) * Slot.LENGTH);
+    const childSlot = Slot.fromBytes(slotBytes);
+    remappedSlots.push(await remapSlot(sourceCore, targetCore, hashSize, offsetMap, childSlot));
+  }
+  const newOffset = await targetCore.length();
+  await targetCore.seek(newOffset);
+  const targetWriter = targetCore.writer();
+  for (const s of remappedSlots) {
+    await targetWriter.write(s.toBytes());
+  }
+  return newOffset;
+}
+async function remapArrayList(sourceCore, targetCore, hashSize, offsetMap, slot) {
+  await sourceCore.seek(Number(slot.value));
+  const sourceReader = sourceCore.reader();
+  const headerBytes = new Uint8Array(ArrayListHeader.LENGTH);
+  await sourceReader.readFully(headerBytes);
+  const header = ArrayListHeader.fromBytes(headerBytes);
+  const indexSlot = new Slot(header.ptr, 1 /* INDEX */);
+  const remappedIndex = await remapSlot(sourceCore, targetCore, hashSize, offsetMap, indexSlot);
+  const newOffset = await targetCore.length();
+  await targetCore.seek(newOffset);
+  const targetWriter = targetCore.writer();
+  await targetWriter.write(new ArrayListHeader(Number(remappedIndex.value), header.size).toBytes());
+  return newOffset;
+}
+async function remapLinkedArrayList(sourceCore, targetCore, hashSize, offsetMap, slot) {
+  await sourceCore.seek(Number(slot.value));
+  const sourceReader = sourceCore.reader();
+  const headerBytes = new Uint8Array(LinkedArrayListHeader.LENGTH);
+  await sourceReader.readFully(headerBytes);
+  const header = LinkedArrayListHeader.fromBytes(headerBytes);
+  const remappedPtr = await remapLinkedArrayListBlock(sourceCore, targetCore, hashSize, offsetMap, header.ptr);
+  const newOffset = await targetCore.length();
+  await targetCore.seek(newOffset);
+  const targetWriter = targetCore.writer();
+  await targetWriter.write(new LinkedArrayListHeader(header.shift, remappedPtr, header.size).toBytes());
+  return newOffset;
+}
+async function remapLinkedArrayListBlock(sourceCore, targetCore, hashSize, offsetMap, blockOffset) {
+  const mapped = offsetMap.get(blockOffset);
+  if (mapped !== undefined)
+    return mapped;
+  await sourceCore.seek(blockOffset);
+  const sourceReader = sourceCore.reader();
+  const blockBytes = new Uint8Array(LINKED_ARRAY_LIST_INDEX_BLOCK_SIZE);
+  await sourceReader.readFully(blockBytes);
+  const slots = [];
+  for (let i = 0;i < SLOT_COUNT; i++) {
+    const slotBytes = blockBytes.slice(i * LinkedArrayListSlot2.LENGTH, (i + 1) * LinkedArrayListSlot2.LENGTH);
+    slots.push(LinkedArrayListSlot2.fromBytes(slotBytes));
+  }
+  const remappedSlots = [];
+  for (const s of slots) {
+    if (s.slot.tag === 1 /* INDEX */) {
+      const remappedPtr = await remapLinkedArrayListBlock(sourceCore, targetCore, hashSize, offsetMap, Number(s.slot.value));
+      remappedSlots.push(new LinkedArrayListSlot2(s.size, new Slot(remappedPtr, 1 /* INDEX */, s.slot.full)));
+    } else if (s.slot.empty()) {
+      remappedSlots.push(s);
+    } else {
+      const remapped = await remapSlot(sourceCore, targetCore, hashSize, offsetMap, s.slot);
+      remappedSlots.push(new LinkedArrayListSlot2(s.size, remapped));
+    }
+  }
+  const newOffset = await targetCore.length();
+  await targetCore.seek(newOffset);
+  const targetWriter = targetCore.writer();
+  for (const s of remappedSlots) {
+    await targetWriter.write(s.toBytes());
+  }
+  offsetMap.set(blockOffset, newOffset);
+  return newOffset;
+}
+async function remapHashMapOrSet(sourceCore, targetCore, hashSize, offsetMap, slot, counted) {
+  await sourceCore.seek(Number(slot.value));
+  const sourceReader = sourceCore.reader();
+  let countValue = -1;
+  if (counted) {
+    countValue = await sourceReader.readLong();
+  }
+  const blockBytes = new Uint8Array(INDEX_BLOCK_SIZE);
+  await sourceReader.readFully(blockBytes);
+  const remappedSlots = [];
+  for (let i = 0;i < SLOT_COUNT; i++) {
+    const slotBytes = blockBytes.slice(i * Slot.LENGTH, (i + 1) * Slot.LENGTH);
+    const childSlot = Slot.fromBytes(slotBytes);
+    remappedSlots.push(await remapSlot(sourceCore, targetCore, hashSize, offsetMap, childSlot));
+  }
+  const newOffset = await targetCore.length();
+  await targetCore.seek(newOffset);
+  const targetWriter = targetCore.writer();
+  if (counted) {
+    await targetWriter.writeLong(countValue);
+  }
+  for (const s of remappedSlots) {
+    await targetWriter.write(s.toBytes());
+  }
+  return newOffset;
+}
+async function remapKvPair(sourceCore, targetCore, hashSize, offsetMap, slot) {
+  await sourceCore.seek(Number(slot.value));
+  const sourceReader = sourceCore.reader();
+  const kvPairBytes = new Uint8Array(KeyValuePair.length(hashSize));
+  await sourceReader.readFully(kvPairBytes);
+  const kvPair = KeyValuePair.fromBytes(kvPairBytes, hashSize);
+  const remappedKey = await remapSlot(sourceCore, targetCore, hashSize, offsetMap, kvPair.keySlot);
+  const remappedValue = await remapSlot(sourceCore, targetCore, hashSize, offsetMap, kvPair.valueSlot);
+  const newOffset = await targetCore.length();
+  await targetCore.seek(newOffset);
+  const targetWriter = targetCore.writer();
+  await targetWriter.write(new KeyValuePair(remappedValue, remappedKey, kvPair.hash).toBytes());
+  return newOffset;
+}
 // src/read-array-list.ts
 class ReadArrayList {
   cursor;

package/package.json

@@ -1,11 +1,11 @@
 {
   "name": "xitdb",
-  "version": "0.10.0",
+  "version": "0.12.0",
   "description": "An immutable database",
   "license": "MIT",
   "repository": {
     "type": "git",
-    "url": "https://github.com/radarroark/xitdb-ts.git"
+    "url": "https://github.com/xit-vcs/xitdb-ts.git"
   },
   "type": "module",
   "main": "dist/index.js",