tool-db 2.3.1 → 2.5.0

package/lib/crdt/mapCrdt.ts

@@ -0,0 +1,119 @@
+import { BaseCrdt, CRDT_MAP } from "..";
+
+export type MapOperations = "SET" | "DEL";
+
+export interface ChangeMapBase<T> {
+  t: MapOperations;
+  a: string;
+  k: string;
+  i: number;
+}
+
+export interface SetMapChange<T> extends ChangeMapBase<T> {
+  t: "SET";
+  v: T;
+}
+
+export interface DelMapChange<T> extends ChangeMapBase<T> {
+  t: "DEL";
+}
+
+export type MapChanges<T> = SetMapChange<T> | DelMapChange<T>;
+
+export class MapCrdt<T> extends BaseCrdt<T, MapChanges<T>, Record<string, T>> {
+  public type = CRDT_MAP;
+
+  private _changes: MapChanges<T>[] = [];
+
+  private _author = "";
+
+  private _value: Record<string, T> = {};
+
+  private _keyIndex: Record<string, number> = {};
+
+  private _lastUpdateSize: number = 0;
+
+  constructor(author: string, changes?: MapChanges<T>[]) {
+    super();
+    this._author = author;
+    if (changes) {
+      this.mergeChanges(changes);
+    }
+
+    this.calculate();
+  }
+
+  changesSort(a: MapChanges<T>, b: MapChanges<T>) {
+    if (a.i < b.i) return -1;
+    if (a.i > b.i) return 1;
+    if (a.t === "SET" && b.t === "DEL") return -1;
+    if (a.t === "DEL" && b.t === "SET") return 1;
+    if (a.a < b.a) return -1;
+    if (a.a > b.a) return 1;
+    return 0; // Should never be equal!
+  }
+
+  calculate() {
+    const temp: Record<string, T> = {};
+    // Only update if we have new changes
+    if (Object.values(this._changes).length !== this._lastUpdateSize) {
+      this._changes.sort(this.changesSort).forEach((change) => {
+        // Here we apply the model properties
+        // Since this is a KV store we dont need much logic,
+        // except for the sorting of the changes. Then we just apply them.
+        if (change.t === "SET") {
+          temp[change.k] = change.v;
+        } else if (change.t === "DEL") {
+          delete temp[change.k];
+        }
+        this._keyIndex[change.k] = change.i;
+      });
+      this._lastUpdateSize = Object.values(temp).length;
+      this._value = temp;
+    }
+  }
+
+  get value(): Record<string, T> {
+    this.calculate();
+    return this._value;
+  }
+
+  public mergeChanges(newChanges: MapChanges<T>[]) {
+    newChanges.forEach((change) => {
+      // Filter by author and index
+      const filtered = this._changes.filter(
+        (c) => c.i === change.i && c.a === change.a && c.t === change.t
+      );
+      // Only add if there are not matches
+      if (filtered.length === 0) {
+        this._changes.push(change);
+      }
+    });
+    this.calculate();
+  }
+
+  public getChanges(): MapChanges<T>[] {
+    return this._changes;
+  }
+
+  public SET(key: string, value: T) {
+    this._keyIndex[key] = (this._keyIndex[key] || 0) + 1;
+    this._changes.push({
+      t: "SET",
+      k: key,
+      a: this._author,
+      v: value,
+      i: this._keyIndex[key],
+    });
+  }
+
+  public DEL(key: string) {
+    this._keyIndex[key] = (this._keyIndex[key] || 0) + 1;
+    this._changes.push({
+      t: "DEL",
+      k: key,
+      a: this._author,
+      i: this._keyIndex[key],
+    });
+  }
+}

package/{dist/index.d.ts → lib/index.ts}

@@ -2,32 +2,34 @@ export { default as proofOfWork } from "./utils/proofOfWork";
 export { default as sha1 } from "./utils/sha1";
 export { default as sha256 } from "./utils/sha256";
 export { default as uniq } from "./utils/uniq";
+
 export { default as randomAnimal } from "./utils/randomAnimal";
 export { default as verifyMessage } from "./utils/verifyMessage";
 export { default as getPeerSignature } from "./utils/getPeerSignature";
 export { default as verifyPeer } from "./utils/verifyPeer";
 export { default as textRandom } from "./utils/textRandom";
+
 export { default as catchReturn } from "./utils/catchReturn";
+
 export { default as stringToArrayBuffer } from "./utils/encoding/stringToArrayBuffer";
 export { default as arrayBufferToString } from "./utils/encoding/arrayBufferToString";
+
 export { default as arrayBufferToHex } from "./utils/encoding/arrayBufferToHex";
 export { default as hexToArrayBuffer } from "./utils/encoding/hexToArrayBuffer";
 export { default as hexToString } from "./utils/encoding/hexToString";
 export { default as hexToUint8 } from "./utils/encoding/hexToUint8";
+
 export { default as ToolDbNetworkAdapter } from "./adapters-base/networkAdapter";
 export { default as ToolDbStorageAdapter } from "./adapters-base/storageAdapter";
 export { default as ToolDbUserAdapter } from "./adapters-base/userAdapter";
-export { default as ToolDbWebsocket } from "./adapters/toolDbWebsocket";
-export { default as ToolDbWebrtc } from "./adapters/toolDbWebrtc";
-export { default as ToolDbWeb3User } from "./adapters/toolDbWeb3User";
-export { default as ToolDbIndexedb } from "./adapters/toolDbIndexedb";
-export { default as ToolDbLeveldb } from "./adapters/toolDbLeveldb";
+
 export * from "./types/tooldb";
 export * from "./types/message";
 export * from "./crdt/baseCrdt";
 export * from "./crdt/counterCrdt";
 export * from "./crdt/listCrdt";
 export * from "./crdt/mapCrdt";
+
 export { default as handleSubscribe } from "./messageHandlers/handleSubscribe";
 export { default as handleCrdtGet } from "./messageHandlers/handleCrdtGet";
 export { default as handleCrdtPut } from "./messageHandlers/handleCrdtPut";
@@ -36,4 +38,5 @@ export { default as handlePing } from "./messageHandlers/handlePing";
 export { default as handlePong } from "./messageHandlers/handlePong";
 export { default as handleGet } from "./messageHandlers/handleGet";
 export { default as handlePut } from "./messageHandlers/handlePut";
+
 export { default as ToolDb } from "./tooldb";

package/lib/logger.ts

@@ -0,0 +1,30 @@
+import { ToolDb, sha1 } from ".";
+
+export default function logger(
+  this: ToolDb,
+  arg0: any,
+  arg1?: any,
+  arg2?: any,
+  arg3?: any,
+  arg4?: any,
+  arg5?: any
+) {
+  if (this.options.debug) {
+    const isNode = typeof window === "undefined";
+    const title = isNode
+      ? this.options.storageName
+      : "%c" + this.options.storageName;
+
+    const style = isNode
+      ? ""
+      : "background: #" +
+        sha1(this.options.storageName).slice(-6) +
+        "; padding: 2px; color: black";
+
+    console.log(
+      title,
+      style,
+      ...[arg0, arg1, arg2, arg3, arg4, arg4, arg5].slice(0, arguments.length)
+    );
+  }
+}

package/lib/messageHandlers/handleCrdtGet.ts

@@ -0,0 +1,29 @@
+import { ToolDb, CrdtGetMessage, CrdtPutMessage } from "..";
+
+export default function handleCrdtGet(
+  this: ToolDb,
+  message: CrdtGetMessage,
+  remotePeerId: string
+) {
+  this.store
+    .get(message.key)
+    .then((data) => {
+      try {
+        // Use the id of the get so the other client knows we are replying
+        const oldData = {
+          type: "crdtPut",
+          data: JSON.parse(data),
+          to: [],
+          id: message.id,
+        } as CrdtPutMessage;
+        this.network.sendToClientId(remotePeerId, oldData);
+      } catch (e) {
+        // socket.send(data);
+        // do nothing
+      }
+    })
+    .catch((e) => {
+      this.logger("Local key not found, relay", JSON.stringify(message));
+      this.network.sendToAll(message, false);
+    });
+}

package/lib/messageHandlers/handleCrdtPut.ts

@@ -0,0 +1,118 @@
+import {
+  ToolDb,
+  VerifyResult,
+  CrdtPutMessage,
+  VerificationData,
+  MapCrdt,
+  MapChanges,
+  CRDT_COUNTER,
+  CRDT_LIST,
+  CRDT_MAP,
+  ListCrdt,
+  ListChanges,
+  CounterCrdt,
+  CounterChanges,
+} from "..";
+
+import toolDbVerificationWrapper from "../toolDbVerificationWrapper";
+
+export default function handleCrdtPut(
+  this: ToolDb,
+  message: CrdtPutMessage,
+  remotePeerId: string
+) {
+  toolDbVerificationWrapper.call(this, message.data).then((value) => {
+    // this.logger("Verification wrapper result: ", value, message.k);
+    if (value === VerifyResult.Verified) {
+      this.emit("crdtput", message);
+      this.emit("data", message.data);
+      this.emit("verified", message);
+      // relay to other servers !!!
+      this.network.sendToAll(message, true);
+
+      this.store
+        .get(message.data.k)
+        .then((oldData) => {
+          try {
+            const parsedOldData: VerificationData<any> = JSON.parse(oldData);
+
+            let newMessage = message;
+
+            // Merge old document with new data incoming and save it
+            // Add handles for all kinds of CRDT we add
+            let oldDoc:
+              | MapCrdt<any>
+              | ListCrdt<any>
+              | CounterCrdt<any>
+              | undefined;
+
+            if (parsedOldData.c === CRDT_MAP) {
+              oldDoc = new MapCrdt(
+                this.userAccount.getAddress() || "",
+                parsedOldData.v
+              );
+            }
+
+            if (parsedOldData.c === CRDT_LIST) {
+              oldDoc = new ListCrdt(
+                this.userAccount.getAddress() || "",
+                parsedOldData.v
+              );
+            }
+
+            if (parsedOldData.c === CRDT_COUNTER) {
+              oldDoc = new CounterCrdt(
+                this.userAccount.getAddress() || "",
+                parsedOldData.v
+              );
+            }
+
+            let changesMerged:
+              | MapChanges<any>[]
+              | ListChanges<any>[]
+              | CounterChanges[] = [];
+
+            if (oldDoc) {
+              oldDoc.mergeChanges(message.data.v);
+              changesMerged = oldDoc.getChanges();
+            }
+            newMessage = {
+              ...message,
+            };
+            newMessage.data.v = changesMerged;
+
+            if (parsedOldData.t < message.data.t) {
+              const key = newMessage.data.k;
+              this.triggerKeyListener(key, newMessage.data);
+              this.store
+                .put(newMessage.data.k, JSON.stringify(newMessage.data))
+                .catch((e) => {
+                  // do nothing
+                });
+            } else {
+              const key = message.data.k;
+              this.triggerKeyListener(key, parsedOldData);
+            }
+            // } else {
+            //   this.logger(
+            //     `${message.k} has old data, but its newer. old ${parsedOldData.t} < new ${message.t}`
+            //   );
+            // }
+          } catch (e) {
+            this.logger("Couldnt parse crdt data", oldData, e);
+          }
+        })
+        .catch((e) => {
+          const key = message.data.k;
+          this.triggerKeyListener(key, message.data);
+          this.store
+            .put(message.data.k, JSON.stringify(message.data))
+            .catch((e) => {
+              // do nothing
+            });
+        });
+    } else {
+      this.logger("unverified message: ", value, message);
+    }
+  });
+}

package/lib/messageHandlers/handleGet.ts

@@ -0,0 +1,29 @@
+import { ToolDb, GetMessage, PutMessage } from "..";
+
+export default function handleGet(
+  this: ToolDb,
+  message: GetMessage,
+  remotePeerId: string
+) {
+  this.store
+    .get(message.key)
+    .then((data) => {
+      try {
+        // Use the id of the get so the other client knows we are replying
+        const oldData = {
+          type: "put",
+          data: JSON.parse(data),
+          to: [],
+          id: message.id,
+        } as PutMessage;
+        this.network.sendToClientId(remotePeerId, oldData);
+      } catch (e) {
+        // socket.send(data);
+        // do nothing
+      }
+    })
+    .catch((e) => {
+      this.logger("Local key not found, relay", JSON.stringify(message));
+      this.network.sendToAll(message, false);
+    });
+}

package/lib/messageHandlers/handlePing.ts

@@ -0,0 +1,40 @@
+import { ToolDb, verifyPeer, PingMessage, PongMessage } from "..";
+
+export default function handlePing(
+  this: ToolDb,
+  message: PingMessage,
+  remotePeerId: string
+) {
+  if (!this.isConnected) {
+    this.isConnected = true;
+    this.onConnect();
+  }
+
+  verifyPeer(this, message.peer).then((verified) => {
+    // Verify integrity and topic
+    if (verified && message.peer.topic === this.options.topic) {
+      // Add this peer to our list of peers
+      const filteredPeers = this.serverPeers.filter(
+        (p) => p.address === message.peer.address
+      );
+      if (filteredPeers.length === 0 && message.isServer) {
+        // Add this peer to the list
+        this.serverPeers.push(message.peer);
+      }
+
+      this.network.sendToClientId(remotePeerId, {
+        type: "pong",
+        isServer: this.options.server,
+        clientId: this.network.getClientAddress(),
+        to: [],
+        servers: this.serverPeers,
+        id: message.id,
+      } as PongMessage);
+
+      this.onPeerConnect(this.peerAccount.getAddress() || "");
+    } else {
+      this.logger("Blocked a remote peer from joining; ", message);
+      // Drop connection here!
+    }
+  });
+}

package/lib/messageHandlers/handlePong.ts

@@ -0,0 +1,30 @@
+import { ToolDb, verifyPeer, PongMessage } from "..";
+
+export default function handlePong(
+  this: ToolDb,
+  message: PongMessage,
+  remotePeerId: string
+) {
+  if (!this.isConnected) {
+    this.isConnected = true;
+    this.onConnect();
+  }
+
+  message.servers.forEach((peer) => {
+    verifyPeer(this, peer).then((verified) => {
+      // Verify integrity and topic
+      if (verified && peer.topic === this.options.topic) {
+        // Add this peer to our list of peers
+        const filteredPeers = this.serverPeers.filter(
+          (p) => p.address === peer.address
+        );
+        if (filteredPeers.length === 0 && peer.host && peer.port) {
+          // Add this peer to the list
+          this.serverPeers.push(peer);
+        }
+      }
+    });
+  });
+
+  this.onPeerConnect(this.peerAccount.getAddress() || "");
+}

package/lib/messageHandlers/handlePut.ts

@@ -0,0 +1,54 @@
+import { ToolDb, PutMessage, VerifyResult } from "..";
+
+import toolDbVerificationWrapper from "../toolDbVerificationWrapper";
+
+export default function handlePut(
+  this: ToolDb,
+  message: PutMessage,
+  remotePeerId: string
+) {
+  toolDbVerificationWrapper.call(this, message.data).then((value) => {
+    // this.logger("Verification wrapper result: ", value, message.k);
+    if (value === VerifyResult.Verified) {
+      this.emit("put", message);
+      this.emit("data", message.data);
+      this.emit("verified", message);
+      // relay to other servers !!!
+      this.network.sendToAll(message, true);
+
+      this.store
+        .get(message.data.k)
+        .then((oldData) => {
+          const parsedOldData = JSON.parse(oldData);
+          if (parsedOldData.t < message.data.t) {
+            const key = message.data.k;
+            this.triggerKeyListener(key, message.data);
+            this.store
+              .put(message.data.k, JSON.stringify(message.data))
+              .catch((e) => {
+                // do nothing
+              });
+          } else {
+            const key = message.data.k;
+            this.triggerKeyListener(key, parsedOldData);
+          }
+          // } else {
+          //   this.logger(
+          //     `${message.k} has old data, but its newer. old ${parsedOldData.t} < new ${message.t}`
+          //   );
+          // }
+        })
+        .catch((e) => {
+          const key = message.data.k;
+          this.triggerKeyListener(key, message.data);
+          this.store
+            .put(message.data.k, JSON.stringify(message.data))
+            .catch((e) => {
+              //
+            });
+        });
+    } else {
+      this.logger("unverified message: ", value, message);
+    }
+  });
+}

package/lib/messageHandlers/handleQuery.ts

@@ -0,0 +1,25 @@
+import { ToolDb, QueryAckMessage, QueryMessage } from "..";
+
+export default function handleQuery(
+  this: ToolDb,
+  message: QueryMessage,
+  remotePeerId: string
+) {
+  this.store
+    .query(message.key)
+    .then((keys) => {
+      this.network.sendToClientId(remotePeerId, {
+        type: "queryAck",
+        id: message.id,
+        to: [],
+        keys,
+      } as QueryAckMessage);
+    })
+    .catch((e) => {
+      // do nothing
+    });
+
+  if (this.options.server) {
+    this.network.sendToAll(message, true);
+  }
+}

package/lib/messageHandlers/handleSubscribe.ts

@@ -0,0 +1,46 @@
+import { ToolDb, PutMessage, SubscribeMessage } from "..";
+
+export default function handleSubscribe(
+  this: ToolDb,
+  message: SubscribeMessage,
+  remotePeerId: string
+) {
+  const subId = remotePeerId + "-" + message.key;
+  if (!this.subscriptions.includes(subId)) {
+    this.subscriptions.push(subId);
+
+    this.addKeyListener(message.key, (msg) => {
+      if (remotePeerId) {
+        // We do not reply to the socket directly
+        // instead we use the client id, in case the socket reconnects
+        const newMsg: PutMessage = {
+          data: msg,
+          id: message.id,
+          type: "put",
+          to: [],
+        };
+        this.network.sendToClientId(remotePeerId, newMsg);
+      }
+    });
+  }
+
+  // basically the exact same as GET, below
+  this.store
+    .get(message.key)
+    .then((data) => {
+      try {
+        const oldData: PutMessage = {
+          data: JSON.parse(data),
+          id: message.id,
+          type: "put",
+          to: [],
+        };
+        this.network.sendToClientId(remotePeerId, oldData);
+      } catch (e) {
+        // do nothing
+      }
+    })
+    .catch((e) => {
+      // do nothing
+    });
+}

package/lib/server.ts

@@ -0,0 +1,7 @@
+import { ToolDb } from ".";
+
+console.log("Starting server..");
+const server = new ToolDb({
+  port: 8080,
+  server: true,
+});

package/lib/shared.ts

@@ -0,0 +1,5 @@
+const shared = {
+  toolDb: undefined,
+} as any;
+
+export default shared;

package/lib/toolDbAnonSignIn.ts

@@ -0,0 +1,5 @@
+import { ToolDb } from ".";
+
+export default function toolDbAnonSignIn(this: ToolDb): void {
+  this.userAccount.anonUser();
+}

package/lib/toolDbClientOnMessage.ts

@@ -0,0 +1,75 @@
+import { ToolDb, ToolDbMessage } from ".";
+
+export default function toolDbClientOnMessage(
+  this: ToolDb,
+  message: ToolDbMessage,
+  remotePeerId: string
+) {
+  if (!this.processedIds[message.type]) {
+    this.processedIds[message.type] = [];
+  }
+  if (this.processedIds[message.type].includes(message.id)) {
+    // this.logger(
+    //   `Already processed this message > ${message.type} from ${remotePeerId}`
+    // );
+    return;
+  }
+
+  this.emit("message", message, remotePeerId);
+
+  this.processedIds[message.type].push(message.id);
+
+  this.logger(`Got message ${message.type} from ${remotePeerId}`);
+  this.logger(message);
+
+  // Check if we are listening for this ID
+  if (message.id) {
+    const msgId = message.id;
+    if (this._idListeners[msgId]) {
+      this._idListeners[msgId](message);
+      this.removeIdListener(msgId);
+    }
+  }
+
+  if (message === undefined || message.type === undefined) {
+    this.logger("Message is invalid!", message, typeof message);
+    return;
+  }
+
+  switch (message.type) {
+    case "ping":
+      this.handlePing(message, remotePeerId);
+      break;
+
+    case "pong":
+      this.handlePong(message, remotePeerId);
+      break;
+
+    case "subscribe":
+      this.handleSubscribe(message, remotePeerId);
+      break;
+
+    case "get":
+      this.handleGet(message, remotePeerId);
+      break;
+
+    case "put":
+      this.handlePut(message, remotePeerId);
+      break;
+
+    case "crdtPut":
+      this.handleCrdtPut(message, remotePeerId);
+      break;
+
+    case "crdtGet":
+      this.handleCrdtGet(message, remotePeerId);
+      break;
+
+    case "query":
+      this.handleQuery(message, remotePeerId);
+      break;
+
+    default:
+      break;
+  }
+}