tool-db 2.2.0 → 2.5.0

package/lib/crdt/listCrdt.ts

@@ -0,0 +1,190 @@
+import { BaseCrdt, CRDT_LIST } from "..";
+
+export type ListOperations = "INS" | "DEL";
+
+export interface ChangeListBase<T> {
+  t: ListOperations; // Operation type
+  i: string; // "index", author + n
+}
+
+export interface InsListChange<T> extends ChangeListBase<T> {
+  t: "INS";
+  v: T; // Value
+  p: string | undefined; // Previous index, if any
+  n: string | undefined; // Next index, if any
+}
+
+export interface DelListChange<T> extends ChangeListBase<T> {
+  t: "DEL";
+  v: string; // target index to tombstone
+}
+
+export type ListChanges<T> = InsListChange<T> | DelListChange<T>;
+
+interface ListTempCursor<T> {
+  value: T;
+  index: string;
+  tomb: boolean;
+  prev: string | undefined;
+  next: string | undefined;
+}
+
+export class ListCrdt<T> extends BaseCrdt<T, ListChanges<T>, T[]> {
+  public type = CRDT_LIST;
+
+  public _changes: ListChanges<T>[] = [];
+
+  private _author = "";
+
+  private _value: T[] = [];
+
+  public _tempValues: ListTempCursor<T>[] = [];
+
+  private _lastUpdateSize: number = 0;
+
+  constructor(author: string, changes?: ListChanges<T>[]) {
+    super();
+    this._author = author;
+    if (changes) {
+      this.mergeChanges(changes);
+    }
+
+    this.calculate();
+  }
+
+  changesSort(a: ListChanges<T>, b: ListChanges<T>) {
+    if (a.i > b.i) return 1;
+    if (a.i < b.i) return -1;
+    return 0; // Should never be equal!
+  }
+
+  calculate() {
+    const temp: ListTempCursor<T>[] = [];
+    // Only update if we have new changes
+    if (Object.values(this._changes).length !== this._lastUpdateSize) {
+      this._changes.sort(this.changesSort).forEach((change) => {
+        if (change.t === "INS") {
+          let poisitionToInsert = 0;
+          if (change.p) {
+            poisitionToInsert = temp.findIndex((v) => v.index === change.p) + 1;
+          } else if (change.n) {
+            poisitionToInsert = temp.findIndex((v) => v.index === change.n);
+          }
+
+          const cursorValue: ListTempCursor<T> = {
+            value: change.v,
+            index: change.i,
+            tomb: false,
+            prev: change.p,
+            next: change.n,
+          };
+
+          temp.splice(poisitionToInsert, 0, cursorValue);
+        } else if (change.t === "DEL") {
+          const poisitionToInsert = temp.findIndex((v) => v.index === change.v);
+          temp[poisitionToInsert].tomb = true;
+        }
+      });
+      this._lastUpdateSize = Object.values(temp).length;
+
+      this._tempValues = temp;
+
+      this._value = temp.filter((v) => !v.tomb).map((v) => v.value);
+    }
+  }
+
+  get value(): T[] {
+    this.calculate();
+    return this._value;
+  }
+
+  public mergeChanges(newChanges: ListChanges<T>[]) {
+    newChanges.forEach((change) => {
+      // Filter by author and index
+      const filtered = this._changes.filter(
+        (c) => c.i === change.i && c.t === change.t && c.v === change.v
+      );
+      // Only add if there are not matches
+      if (filtered.length === 0) {
+        this._changes.push(change);
+      }
+    });
+    this.calculate();
+  }
+
+  public getChanges(): ListChanges<T>[] {
+    return this._changes;
+  }
+
+  public INS(value: T, index: number) {
+    const filterTombs = this._tempValues.filter((v) => !v.tomb);
+
+    let currentPrev = undefined;
+    let currentNext = undefined;
+
+    if (filterTombs.length !== 0) {
+      const currentIndex = filterTombs[index].index;
+      const currentIndexPosition = this._tempValues.findIndex(
+        (v) => v.index === currentIndex
+      );
+      currentPrev = this._tempValues[currentIndexPosition - 1];
+      currentNext = this._tempValues[currentIndexPosition];
+    }
+
+    const ourChanges = this._changes.filter((c) =>
+      c.i.startsWith(this._author)
+    );
+
+    this._changes.push({
+      t: "INS",
+      p: currentPrev?.index,
+      n: currentNext?.index,
+      v: value,
+      i: this._author + "-" + `${ourChanges.length}`.padStart(8, "0"),
+    });
+    this.calculate();
+  }
+
+  public PUSH(value: T) {
+    const filterTombs = this._tempValues.filter((v) => !v.tomb);
+    const currentIndex = filterTombs[filterTombs.length - 1]?.index;
+    const currentIndexPosition = this._tempValues.findIndex(
+      (v) => v.index === currentIndex
+    );
+
+    const currentPrev = this._tempValues[currentIndexPosition];
+
+    const ourChanges = this._changes.filter((c) =>
+      c.i.startsWith(this._author)
+    );
+
+    this._changes.push({
+      t: "INS",
+      p: currentPrev?.index,
+      n: undefined,
+      v: value,
+      i: this._author + "-" + `${ourChanges.length}`.padStart(8, "0"),
+    });
+    this.calculate();
+  }
+
+  public DEL(index: number) {
+    const currentIndex = this._tempValues.filter((v) => !v.tomb)[index].index;
+    const currentIndexPosition = this._tempValues.findIndex(
+      (v) => v.index === currentIndex
+    );
+
+    const current = this._tempValues[currentIndexPosition];
+
+    const ourChanges = this._changes.filter((c) =>
+      c.i.startsWith(this._author)
+    );
+
+    this._changes.push({
+      t: "DEL",
+      v: current.index,
+      i: this._author + "-" + `${ourChanges.length}`.padStart(8, "0"),
+    });
+    this.calculate();
+  }
+}

package/lib/crdt/mapCrdt.ts

@@ -0,0 +1,119 @@
+import { BaseCrdt, CRDT_MAP } from "..";
+
+export type MapOperations = "SET" | "DEL";
+
+export interface ChangeMapBase<T> {
+  t: MapOperations;
+  a: string;
+  k: string;
+  i: number;
+}
+
+export interface SetMapChange<T> extends ChangeMapBase<T> {
+  t: "SET";
+  v: T;
+}
+
+export interface DelMapChange<T> extends ChangeMapBase<T> {
+  t: "DEL";
+}
+
+export type MapChanges<T> = SetMapChange<T> | DelMapChange<T>;
+
+export class MapCrdt<T> extends BaseCrdt<T, MapChanges<T>, Record<string, T>> {
+  public type = CRDT_MAP;
+
+  private _changes: MapChanges<T>[] = [];
+
+  private _author = "";
+
+  private _value: Record<string, T> = {};
+
+  private _keyIndex: Record<string, number> = {};
+
+  private _lastUpdateSize: number = 0;
+
+  constructor(author: string, changes?: MapChanges<T>[]) {
+    super();
+    this._author = author;
+    if (changes) {
+      this.mergeChanges(changes);
+    }
+
+    this.calculate();
+  }
+
+  changesSort(a: MapChanges<T>, b: MapChanges<T>) {
+    if (a.i < b.i) return -1;
+    if (a.i > b.i) return 1;
+    if (a.t === "SET" && b.t === "DEL") return -1;
+    if (a.t === "DEL" && b.t === "SET") return 1;
+    if (a.a < b.a) return -1;
+    if (a.a > b.a) return 1;
+    return 0; // Should never be equal!
+  }
+
+  calculate() {
+    const temp: Record<string, T> = {};
+    // Only update if we have new changes
+    if (Object.values(this._changes).length !== this._lastUpdateSize) {
+      this._changes.sort(this.changesSort).forEach((change) => {
+        // Here we apply the model properties
+        // Since this is a KV store we dont need much logic,
+        // except for the sorting of the changes. Then we just apply them.
+        if (change.t === "SET") {
+          temp[change.k] = change.v;
+        } else if (change.t === "DEL") {
+          delete temp[change.k];
+        }
+        this._keyIndex[change.k] = change.i;
+      });
+      this._lastUpdateSize = Object.values(temp).length;
+      this._value = temp;
+    }
+  }
+
+  get value(): Record<string, T> {
+    this.calculate();
+    return this._value;
+  }
+
+  public mergeChanges(newChanges: MapChanges<T>[]) {
+    newChanges.forEach((change) => {
+      // Filter by author and index
+      const filtered = this._changes.filter(
+        (c) => c.i === change.i && c.a === change.a && c.t === change.t
+      );
+      // Only add if there are not matches
+      if (filtered.length === 0) {
+        this._changes.push(change);
+      }
+    });
+    this.calculate();
+  }
+
+  public getChanges(): MapChanges<T>[] {
+    return this._changes;
+  }
+
+  public SET(key: string, value: T) {
+    this._keyIndex[key] = (this._keyIndex[key] || 0) + 1;
+    this._changes.push({
+      t: "SET",
+      k: key,
+      a: this._author,
+      v: value,
+      i: this._keyIndex[key],
+    });
+  }
+
+  public DEL(key: string) {
+    this._keyIndex[key] = (this._keyIndex[key] || 0) + 1;
+    this._changes.push({
+      t: "DEL",
+      k: key,
+      a: this._author,
+      i: this._keyIndex[key],
+    });
+  }
+}

package/{dist/index.d.ts → lib/index.ts}

@@ -1,36 +1,35 @@
-export * from "./types/tooldb";
-export * from "./types/message";
-export { default as BaseCrdt } from "./crdt/baseCrdt";
-export { default as CounterCrdt } from "./crdt/counterCrdt";
-export { default as ListCrdt } from "./crdt/listCrdt";
-export { default as MapCrdt } from "./crdt/mapCrdt";
 export { default as proofOfWork } from "./utils/proofOfWork";
 export { default as sha1 } from "./utils/sha1";
 export { default as sha256 } from "./utils/sha256";
-export { default as generateIv } from "./utils/generateIv";
+export { default as uniq } from "./utils/uniq";
+
 export { default as randomAnimal } from "./utils/randomAnimal";
-export { default as textRandom } from "./utils/textRandom";
-export { default as stringToArrayBuffer } from "./utils/encoding/stringToArrayBuffer";
 export { default as verifyMessage } from "./utils/verifyMessage";
+export { default as getPeerSignature } from "./utils/getPeerSignature";
 export { default as verifyPeer } from "./utils/verifyPeer";
+export { default as textRandom } from "./utils/textRandom";
+
+export { default as catchReturn } from "./utils/catchReturn";
+
+export { default as stringToArrayBuffer } from "./utils/encoding/stringToArrayBuffer";
 export { default as arrayBufferToString } from "./utils/encoding/arrayBufferToString";
+
 export { default as arrayBufferToHex } from "./utils/encoding/arrayBufferToHex";
 export { default as hexToArrayBuffer } from "./utils/encoding/hexToArrayBuffer";
 export { default as hexToString } from "./utils/encoding/hexToString";
 export { default as hexToUint8 } from "./utils/encoding/hexToUint8";
-export { default as generateKeyFromPassword } from "./utils/crypto/generateKeyFromPassword";
-export { default as decryptWithPass } from "./utils/crypto/decryptWithPass";
-export { default as encryptWithPass } from "./utils/crypto/encryptWithPass";
-export { default as ToolDbNetworkAdapter } from "./toolDbNetworkAdapter";
-export { default as ToolDbWebsocket } from "./toolDbWebsocket";
-export { default as ToolDbWebrtc } from "./toolDbWebrtc";
+
+export { default as ToolDbNetworkAdapter } from "./adapters-base/networkAdapter";
+export { default as ToolDbStorageAdapter } from "./adapters-base/storageAdapter";
+export { default as ToolDbUserAdapter } from "./adapters-base/userAdapter";
+
 export * from "./types/tooldb";
 export * from "./types/message";
 export * from "./crdt/baseCrdt";
 export * from "./crdt/counterCrdt";
 export * from "./crdt/listCrdt";
 export * from "./crdt/mapCrdt";
-export { default as ToolDbWeb3User } from "./toolDbWeb3User";
+
 export { default as handleSubscribe } from "./messageHandlers/handleSubscribe";
 export { default as handleCrdtGet } from "./messageHandlers/handleCrdtGet";
 export { default as handleCrdtPut } from "./messageHandlers/handleCrdtPut";
@@ -39,4 +38,5 @@ export { default as handlePing } from "./messageHandlers/handlePing";
 export { default as handlePong } from "./messageHandlers/handlePong";
 export { default as handleGet } from "./messageHandlers/handleGet";
 export { default as handlePut } from "./messageHandlers/handlePut";
+
 export { default as ToolDb } from "./tooldb";

package/lib/logger.ts

@@ -0,0 +1,30 @@
+import { ToolDb, sha1 } from ".";
+
+export default function logger(
+  this: ToolDb,
+  arg0: any,
+  arg1?: any,
+  arg2?: any,
+  arg3?: any,
+  arg4?: any,
+  arg5?: any
+) {
+  if (this.options.debug) {
+    const isNode = typeof window === "undefined";
+    const title = isNode
+      ? this.options.storageName
+      : "%c" + this.options.storageName;
+
+    const style = isNode
+      ? ""
+      : "background: #" +
+        sha1(this.options.storageName).slice(-6) +
+        "; padding: 2px; color: black";
+
+    console.log(
+      title,
+      style,
+      ...[arg0, arg1, arg2, arg3, arg4, arg4, arg5].slice(0, arguments.length)
+    );
+  }
+}

package/lib/messageHandlers/handleCrdtGet.ts

@@ -0,0 +1,29 @@
+import { ToolDb, CrdtGetMessage, CrdtPutMessage } from "..";
+
+export default function handleCrdtGet(
+  this: ToolDb,
+  message: CrdtGetMessage,
+  remotePeerId: string
+) {
+  this.store
+    .get(message.key)
+    .then((data) => {
+      try {
+        // Use the id of the get so the other client knows we are replying
+        const oldData = {
+          type: "crdtPut",
+          data: JSON.parse(data),
+          to: [],
+          id: message.id,
+        } as CrdtPutMessage;
+        this.network.sendToClientId(remotePeerId, oldData);
+      } catch (e) {
+        // socket.send(data);
+        // do nothing
+      }
+    })
+    .catch((e) => {
+      this.logger("Local key not found, relay", JSON.stringify(message));
+      this.network.sendToAll(message, false);
+    });
+}

package/lib/messageHandlers/handleCrdtPut.ts

@@ -0,0 +1,118 @@
+import {
+  ToolDb,
+  VerifyResult,
+  CrdtPutMessage,
+  VerificationData,
+  MapCrdt,
+  MapChanges,
+  CRDT_COUNTER,
+  CRDT_LIST,
+  CRDT_MAP,
+  ListCrdt,
+  ListChanges,
+  CounterCrdt,
+  CounterChanges,
+} from "..";
+
+import toolDbVerificationWrapper from "../toolDbVerificationWrapper";
+
+export default function handleCrdtPut(
+  this: ToolDb,
+  message: CrdtPutMessage,
+  remotePeerId: string
+) {
+  toolDbVerificationWrapper.call(this, message.data).then((value) => {
+    // this.logger("Verification wrapper result: ", value, message.k);
+    if (value === VerifyResult.Verified) {
+      this.emit("crdtput", message);
+      this.emit("data", message.data);
+      this.emit("verified", message);
+      // relay to other servers !!!
+      this.network.sendToAll(message, true);
+
+      this.store
+        .get(message.data.k)
+        .then((oldData) => {
+          try {
+            const parsedOldData: VerificationData<any> = JSON.parse(oldData);
+
+            let newMessage = message;
+
+            // Merge old document with new data incoming and save it
+            // Add handles for all kinds of CRDT we add
+            let oldDoc:
+              | MapCrdt<any>
+              | ListCrdt<any>
+              | CounterCrdt<any>
+              | undefined;
+
+            if (parsedOldData.c === CRDT_MAP) {
+              oldDoc = new MapCrdt(
+                this.userAccount.getAddress() || "",
+                parsedOldData.v
+              );
+            }
+
+            if (parsedOldData.c === CRDT_LIST) {
+              oldDoc = new ListCrdt(
+                this.userAccount.getAddress() || "",
+                parsedOldData.v
+              );
+            }
+
+            if (parsedOldData.c === CRDT_COUNTER) {
+              oldDoc = new CounterCrdt(
+                this.userAccount.getAddress() || "",
+                parsedOldData.v
+              );
+            }
+
+            let changesMerged:
+              | MapChanges<any>[]
+              | ListChanges<any>[]
+              | CounterChanges[] = [];
+
+            if (oldDoc) {
+              oldDoc.mergeChanges(message.data.v);
+              changesMerged = oldDoc.getChanges();
+            }
+            newMessage = {
+              ...message,
+            };
+            newMessage.data.v = changesMerged;
+
+            if (parsedOldData.t < message.data.t) {
+              const key = newMessage.data.k;
+              this.triggerKeyListener(key, newMessage.data);
+              this.store
+                .put(newMessage.data.k, JSON.stringify(newMessage.data))
+                .catch((e) => {
+                  // do nothing
+                });
+            } else {
+              const key = message.data.k;
+              this.triggerKeyListener(key, parsedOldData);
+            }
+            // } else {
+            //   this.logger(
+            //     `${message.k} has old data, but its newer. old ${parsedOldData.t} < new ${message.t}`
+            //   );
+            // }
+          } catch (e) {
+            this.logger("Couldnt parse crdt data", oldData, e);
+          }
+        })
+        .catch((e) => {
+          const key = message.data.k;
+          this.triggerKeyListener(key, message.data);
+          this.store
+            .put(message.data.k, JSON.stringify(message.data))
+            .catch((e) => {
+              // do nothing
+            });
+        });
+    } else {
+      this.logger("unverified message: ", value, message);
+    }
+  });
+}

package/lib/messageHandlers/handleGet.ts

@@ -0,0 +1,29 @@
+import { ToolDb, GetMessage, PutMessage } from "..";
+
+export default function handleGet(
+  this: ToolDb,
+  message: GetMessage,
+  remotePeerId: string
+) {
+  this.store
+    .get(message.key)
+    .then((data) => {
+      try {
+        // Use the id of the get so the other client knows we are replying
+        const oldData = {
+          type: "put",
+          data: JSON.parse(data),
+          to: [],
+          id: message.id,
+        } as PutMessage;
+        this.network.sendToClientId(remotePeerId, oldData);
+      } catch (e) {
+        // socket.send(data);
+        // do nothing
+      }
+    })
+    .catch((e) => {
+      this.logger("Local key not found, relay", JSON.stringify(message));
+      this.network.sendToAll(message, false);
+    });
+}

package/lib/messageHandlers/handlePing.ts

@@ -0,0 +1,40 @@
+import { ToolDb, verifyPeer, PingMessage, PongMessage } from "..";
+
+export default function handlePing(
+  this: ToolDb,
+  message: PingMessage,
+  remotePeerId: string
+) {
+  if (!this.isConnected) {
+    this.isConnected = true;
+    this.onConnect();
+  }
+
+  verifyPeer(this, message.peer).then((verified) => {
+    // Verify integrity and topic
+    if (verified && message.peer.topic === this.options.topic) {
+      // Add this peer to our list of peers
+      const filteredPeers = this.serverPeers.filter(
+        (p) => p.address === message.peer.address
+      );
+      if (filteredPeers.length === 0 && message.isServer) {
+        // Add this peer to the list
+        this.serverPeers.push(message.peer);
+      }
+
+      this.network.sendToClientId(remotePeerId, {
+        type: "pong",
+        isServer: this.options.server,
+        clientId: this.network.getClientAddress(),
+        to: [],
+        servers: this.serverPeers,
+        id: message.id,
+      } as PongMessage);
+
+      this.onPeerConnect(this.peerAccount.getAddress() || "");
+    } else {
+      this.logger("Blocked a remote peer from joining; ", message);
+      // Drop connection here!
+    }
+  });
+}

package/lib/messageHandlers/handlePong.ts

@@ -0,0 +1,30 @@
+import { ToolDb, verifyPeer, PongMessage } from "..";
+
+export default function handlePong(
+  this: ToolDb,
+  message: PongMessage,
+  remotePeerId: string
+) {
+  if (!this.isConnected) {
+    this.isConnected = true;
+    this.onConnect();
+  }
+
+  message.servers.forEach((peer) => {
+    verifyPeer(this, peer).then((verified) => {
+      // Verify integrity and topic
+      if (verified && peer.topic === this.options.topic) {
+        // Add this peer to our list of peers
+        const filteredPeers = this.serverPeers.filter(
+          (p) => p.address === peer.address
+        );
+        if (filteredPeers.length === 0 && peer.host && peer.port) {
+          // Add this peer to the list
+          this.serverPeers.push(peer);
+        }
+      }
+    });
+  });
+
+  this.onPeerConnect(this.peerAccount.getAddress() || "");
+}