tool-db 2.3.1 → 2.5.0

package/README.md

@@ -1,74 +1 @@
-# Tool Db
-
-ToolDb is a peer-to-peer model for a decentralized database, inspired by GundB
-
-
-It is the mix of a lot of cool concepts into one;
-- Cryptographically secure.
-- Offline first.
-- Fully decentralized.
-- Capable of providing realtime updates.
-- Key-value/document storage.
-- Works in the Browser and Nodejs seamlessly.
-
-For a more detailed documentation head here;
-
-[https://github.com/Manwe-777/tool-db-docs](https://github.com/Manwe-777/tool-db-docs)
-
-_(No live version just yet!, until the protocol is more finalized)_
-
-We push towards of the concept of federated servers, where we set up a p2p mesh network between any servers who want to join the swarm, and have those servers manage the connection and data sharing between them, while allowing client peers to connect to them to push updates to the database.
-This setup IS NOT a requirement! It is just what I think is the best and most reliable way of creating a p2p network for dApps using both desktop and web compatible technologies. Any peer has the capability of connecting to any other peer trough websockets, and modifiying the code to allow webRtc connections between web peers is absolutely possible.
-
-Since anyone can join a federated server swarm to help growing the network we use cryptography (basic public and private key authenthication and signature validation) to ensure all messages are coming from the real authors of the data that they intend to modify; this way by joining a swarm all peers in it can help validate without even having full data; even a new server peer can help, because all information is stored on each message and we dont rely on any centralized database to fetch users data.
-
-Please check the [chain-swarm](https://github.com/Manwe-777/chain-swarm) repository to see how a federated server swarm would look like, Tool Db only handles the connection and messaging between peers, but it does not have any logic for peer discovery.
-We do that using [discovery-channel](https://www.npmjs.com/package/discovery-channel), but you can use any DHT/networking solution you want! even WebRTC between browsers works using the `toolDbWebrtc` transport.
-
-# Future work
-
-While the database is currently functioning as expected, there are many things that could be added or improved to make it even better!
-
-- Allow data encryption (proably built in), add methods for ECC encryption, shared keys, etc.
-- Allow adding Noise/encryption to connections (probably based on the peer identity, requires initial keys exchange on connection)
-
-# Install
-
-You can install ToolDb via npm or yarn;
-
-```
-npm install tool-db
-```
-
-or use it via script in your html;
-
-```
-<script src="https://unpkg.com/tool-db/bundle.js"></script>
-```
-
-That will include all of ToolDb exports in `tooldb`, then you can use it like;
-
-```
-const { ToolDb, sha256 } = tooldb;
-```
-
-## Base usage
-
-Creating a webrtc peers network is as easy as;
-
-```
-import { ToolDb, ToolDbWebrtc } from "tool-db";
-
-const db = new ToolDb({
-  networkAdapter: ToolDbWebrtc,
-  debug: true,
-});
-```
-
-From there you can put and get data using the api;
-
-```
-db.putData("foo-key", "var").then(console.log);
-```
-
-There is a lot more you can do, like subscribe for updates, built-in users credentials validation based on ECC, create federated networks, run servers on Nodejs, auto-replicate data trough them, etc. Make sure more you read trough the [documentation](https://github.com/Manwe-777/tool-db-docs)!
+# `tool-db`

package/lib/adapters-base/networkAdapter.ts

@@ -0,0 +1,215 @@
+import {
+  ToolDb,
+  PingMessage,
+  ToolDbMessage,
+  Peer,
+  getPeerSignature,
+  textRandom,
+  uniq,
+} from "..";
+
+export default class ToolDbNetworkAdapter {
+  private _clientToSend: Record<string, (message: string) => void> = {};
+
+  private _isClientConnected: Record<string, () => boolean> = {};
+
+  private _clientIsServer: Record<string, boolean> = {};
+
+  private _tooldb: ToolDb;
+
+  constructor(db: ToolDb) {
+    this._tooldb = db;
+
+    if (this.tooldb.options.server) {
+      this.getMeAsPeer().then((meAsPeer) => {
+        this.tooldb.serverPeers.push(meAsPeer);
+      });
+    }
+  }
+
+  get clientToSend() {
+    return this._clientToSend;
+  }
+
+  get isClientConnected() {
+    return this._isClientConnected;
+  }
+
+  get tooldb() {
+    return this._tooldb;
+  }
+
+  public getMeAsPeer() {
+    const timestamp = new Date().getTime();
+    return getPeerSignature(
+      this.tooldb.peerAccount,
+      this.tooldb.options.topic,
+      timestamp,
+      this.tooldb.options.host,
+      this.tooldb.options.port
+    ).then((signature) => {
+      return {
+        topic: this.tooldb.options.topic,
+        timestamp: timestamp,
+        host: this.tooldb.options.host,
+        port: this.tooldb.options.port,
+        address: this.tooldb.peerAccount.getAddress() || "",
+        sig: signature,
+      } as Peer;
+    });
+  }
+
+  /**
+   * Check if the specified client is connected or not
+   * @param clientId Client ID
+   * @returns boolean
+   */
+  public isConnected(clientId: string) {
+    return this._isClientConnected[clientId]
+      ? this._isClientConnected[clientId]()
+      : false;
+  }
+
+  /**
+   * Check if the specified client is a server/relay
+   * @param clientId Client ID
+   * @returns boolean
+   */
+  public isServer(clientId: string) {
+    return this._clientIsServer[clientId] || false;
+  }
+
+  public craftPingMessage() {
+    return this.getMeAsPeer().then((meAsPeer) => {
+      return JSON.stringify({
+        type: "ping",
+        clientId: this.getClientAddress(),
+        to: [this.getClientAddress()],
+        isServer: this.tooldb.options.server,
+        id: textRandom(10),
+        peer: meAsPeer,
+      } as PingMessage);
+    });
+  }
+
+  /**
+   * Execute the function to send a message to the specified client ID
+   * @param clientId Client ID
+   * @param message Message
+   */
+  private executeSendToClient(clientId: string, message: string) {
+    if (this._clientToSend[clientId]) {
+      this._clientToSend[clientId](message);
+    }
+  }
+
+  public getClientAddress() {
+    // This is not a good idea to use on all adapters, so it should be replaced
+    // if its causing issues. The only reason we use the last 20 chars is to
+    // muse the same peer address as the webrtc adapter.
+    return (this.tooldb.peerAccount.getAddress() || "").slice(-20);
+  }
+
+  public onClientDisconnect(clientId: string) {
+    delete this._clientToSend[clientId];
+    delete this._clientIsServer[clientId];
+  }
+
+  /**
+   * Should be called as a message payload handler
+   * This function will take care of processing the messages and also making sure
+   * we use the correct response methods troughout the adapter.
+   * @param message message payload
+   * @param clientId Client ID (can be null for ping/pong)
+   * @param setClientId Callback to set the client id on the parent class
+   */
+  public onClientMessage(
+    message: string,
+    clientId: string | null,
+    setClientId: (clientId: string) => void
+  ) {
+    // this.tooldb.logger("onClientMessage", clientId);
+
+    if (clientId && !this.tooldb.processedOutHashes[clientId]) {
+      this.tooldb.processedOutHashes[clientId] = [];
+    }
+
+    try {
+      const parsedMessage = JSON.parse(message) as ToolDbMessage;
+      // We assume the first messages to arrive will always be ping or pong.
+      // Only after that we can get the client id for this socket.
+      if (parsedMessage.type === "ping" || parsedMessage.type === "pong") {
+        const cid = parsedMessage.clientId;
+        setClientId(cid);
+        this.tooldb.onPeerConnect(cid);
+        this._clientIsServer[cid] = parsedMessage.isServer;
+        this.tooldb.processedOutHashes[cid] = [];
+        this.tooldb.clientOnMessage(parsedMessage, cid);
+      } else if (clientId) {
+        this.tooldb.clientOnMessage(parsedMessage, clientId);
+      }
+    } catch (e) {
+      this.tooldb.logger("Got message ERR", message);
+      this.tooldb.logger(e);
+    }
+  }
+
+  /**
+   * Sends a message to all peers connected to us
+   * This function takes care of the message deduplication, making sure we dont send
+   * the same message twice to the same peer.
+   * @param msg message data
+   * @param crossServerOnly If this message should be send to server peers only
+   * @param isRelay if we should relay this message
+   */
+  public sendToAll(msg: ToolDbMessage, crossServerOnly = false) {
+    const to = uniq([...msg.to, this.getClientAddress()]);
+
+    const finalMessage = JSON.stringify({ ...msg, to });
+
+    const filteredConns = Object.keys(this.clientToSend)
+      .filter((id) => !to.includes(id))
+      .filter((clientId) => this.isConnected(clientId));
+
+    filteredConns.forEach((clientId) => {
+      if ((crossServerOnly && this.isServer(clientId)) || !crossServerOnly) {
+        this.tooldb.logger(to, "Sent out to:", clientId);
+
+        if (msg.type === "put" || msg.type === "crdtPut") {
+          if (!this.tooldb.processedOutHashes[clientId].includes(msg.data.h)) {
+            this.executeSendToClient(clientId, finalMessage);
+            this.tooldb.processedOutHashes[clientId].push(msg.data.h);
+          }
+        } else {
+          this.executeSendToClient(clientId, finalMessage);
+        }
+      }
+      // } else {
+      //   this.tooldb.logger("Fitlered out;", clientId);
+      // }
+    });
+  }
+
+  /**
+   * Sends a message to a single peer.
+   * This function also takes care of the message deduplication.
+   * @param clientId Peer/Client id we want to send to.
+   * @param msg message data
+   */
+  public sendToClientId(clientId: string, msg: ToolDbMessage) {
+    const to = uniq([...msg.to, this.getClientAddress()]);
+    const finalMessage = JSON.stringify({ ...msg, to });
+
+    if (msg.type === "put" || msg.type === "crdtPut") {
+      if (
+        clientId &&
+        !this.tooldb.processedOutHashes[clientId].includes(msg.data.h)
+      ) {
+        this.executeSendToClient(clientId, finalMessage);
+        this.tooldb.processedOutHashes[clientId].push(msg.data.h);
+      }
+    } else {
+      this.executeSendToClient(clientId, finalMessage);
+    }
+  }
+}

package/lib/adapters-base/storageAdapter.ts

@@ -0,0 +1,35 @@
+import { ToolDb } from "..";
+
+export default class ToolDbStorageAdapter {
+  private _tooldb: ToolDb;
+
+  private _forceStorageName: string | undefined;
+
+  get tooldb() {
+    return this._tooldb;
+  }
+
+  get storageName() {
+    return this._forceStorageName || this._tooldb.options.storageName;
+  }
+
+  constructor(db: ToolDb, forceStorageName?: string) {
+    this._tooldb = db;
+
+    if (forceStorageName) {
+      this._forceStorageName = forceStorageName;
+    }
+  }
+
+  put(key: string, data: string): Promise<unknown> {
+    return Promise.resolve();
+  }
+
+  get(key: string): Promise<string> {
+    return Promise.resolve("");
+  }
+
+  query(key: string): Promise<string[]> {
+    return Promise.resolve([]);
+  }
+}

package/lib/adapters-base/userAdapter.ts

@@ -0,0 +1,49 @@
+import { ToolDb, VerificationData } from "..";
+
+export default class ToolDbUserAdapter {
+  private _tooldb: ToolDb;
+
+  get tooldb() {
+    return this._tooldb;
+  }
+
+  constructor(db: ToolDb) {
+    this._tooldb = db;
+  }
+
+  public anonUser() {
+    return;
+  }
+
+  public setUser(account: unknown | undefined, name: string): void {}
+
+  public signData(data: string): Promise<string> {
+    return Promise.resolve("");
+  }
+
+  public verifySignature(
+    message: Partial<VerificationData<any>>
+  ): Promise<boolean> {
+    return Promise.resolve(false);
+  }
+
+  public getAccountFromPrivate(privateKey: string): Promise<unknown> {
+    return Promise.resolve(undefined);
+  }
+
+  public encryptAccount(password: string): Promise<unknown> {
+    return Promise.resolve(undefined);
+  }
+
+  public decryptAccount(acc: unknown, password: string): Promise<any> {
+    return Promise.resolve(undefined);
+  }
+
+  public getAddress(): string | undefined {
+    return "";
+  }
+
+  public getUsername(): string | undefined {
+    return "";
+  }
+}

package/lib/crdt/baseCrdt.ts

@@ -0,0 +1,21 @@
+export const CRDT_MAP = "MAP";
+
+export const CRDT_LIST = "LIST";
+
+export const CRDT_COUNTER = "COUNTER";
+
+export class BaseCrdt<T = any, Changes = any, Value = any> {
+  public type: string = "";
+
+  public mergeChanges(changes: Changes[]) {
+    //
+  }
+
+  public getChanges(): Changes[] {
+    return [];
+  }
+
+  get value(): Value {
+    return "" as any;
+  }
+}

package/lib/crdt/counterCrdt.ts

@@ -0,0 +1,111 @@
+import { BaseCrdt, CRDT_LIST } from "..";
+
+export type CounterOperations = "ADD" | "SUB";
+
+export interface ChangeCounterBase {
+  t: CounterOperations; // Operation type
+  v: number; // Value
+  a: string;
+  i: number;
+}
+
+export interface AddCounterChange extends ChangeCounterBase {
+  t: "ADD";
+}
+
+export interface SubCounterChange extends ChangeCounterBase {
+  t: "SUB";
+}
+
+export type CounterChanges = AddCounterChange | SubCounterChange;
+
+export class CounterCrdt<T> extends BaseCrdt<number, CounterChanges, number> {
+  public type = CRDT_LIST;
+
+  public _changes: CounterChanges[] = [];
+
+  private _author = "";
+
+  private _value: number = 0;
+
+  private _lastUpdateSize: number = 0;
+
+  constructor(author: string, changes?: CounterChanges[]) {
+    super();
+    this._author = author;
+    if (changes) {
+      this.mergeChanges(changes);
+    }
+
+    this.calculate();
+  }
+
+  changesSort(a: CounterChanges, b: CounterChanges) {
+    if (a.i > b.i) return 1;
+    if (a.i < b.i) return -1;
+    return 0; // Should never be equal!
+  }
+
+  calculate() {
+    let temp: number = 0;
+    // Only update if we have new changes
+    if (Object.values(this._changes).length !== this._lastUpdateSize) {
+      this._changes.sort(this.changesSort).forEach((change) => {
+        if (change.t === "ADD") {
+          temp += change.v;
+        } else if (change.t === "SUB") {
+          temp -= change.v;
+        }
+      });
+
+      this._value = temp;
+    }
+  }
+
+  get value(): number {
+    this.calculate();
+    return this._value;
+  }
+
+  public mergeChanges(newChanges: CounterChanges[]) {
+    newChanges.forEach((change) => {
+      // Filter by author and index
+      const filtered = this._changes.filter(
+        (c) => c.i === change.i && c.a === change.a && c.t === change.t
+      );
+      // Only add if there are not matches
+      if (filtered.length === 0) {
+        this._changes.push(change);
+      }
+    });
+    this.calculate();
+  }
+
+  public getChanges(): CounterChanges[] {
+    return this._changes;
+  }
+
+  public ADD(value: number) {
+    const ourChanges = this._changes.filter((c) => c.a === this._author);
+
+    this._changes.push({
+      t: "ADD",
+      v: value,
+      a: this._author,
+      i: ourChanges.length,
+    });
+    this.calculate();
+  }
+
+  public SUB(value: number) {
+    const ourChanges = this._changes.filter((c) => c.a === this._author);
+
+    this._changes.push({
+      t: "SUB",
+      v: value,
+      a: this._author,
+      i: ourChanges.length,
+    });
+    this.calculate();
+  }
+}

package/lib/crdt/listCrdt.ts

@@ -0,0 +1,190 @@
+import { BaseCrdt, CRDT_LIST } from "..";
+
+export type ListOperations = "INS" | "DEL";
+
+export interface ChangeListBase<T> {
+  t: ListOperations; // Operation type
+  i: string; // "index", author + n
+}
+
+export interface InsListChange<T> extends ChangeListBase<T> {
+  t: "INS";
+  v: T; // Value
+  p: string | undefined; // Previous index, if any
+  n: string | undefined; // Next index, if any
+}
+
+export interface DelListChange<T> extends ChangeListBase<T> {
+  t: "DEL";
+  v: string; // target index to tombstone
+}
+
+export type ListChanges<T> = InsListChange<T> | DelListChange<T>;
+
+interface ListTempCursor<T> {
+  value: T;
+  index: string;
+  tomb: boolean;
+  prev: string | undefined;
+  next: string | undefined;
+}
+
+export class ListCrdt<T> extends BaseCrdt<T, ListChanges<T>, T[]> {
+  public type = CRDT_LIST;
+
+  public _changes: ListChanges<T>[] = [];
+
+  private _author = "";
+
+  private _value: T[] = [];
+
+  public _tempValues: ListTempCursor<T>[] = [];
+
+  private _lastUpdateSize: number = 0;
+
+  constructor(author: string, changes?: ListChanges<T>[]) {
+    super();
+    this._author = author;
+    if (changes) {
+      this.mergeChanges(changes);
+    }
+
+    this.calculate();
+  }
+
+  changesSort(a: ListChanges<T>, b: ListChanges<T>) {
+    if (a.i > b.i) return 1;
+    if (a.i < b.i) return -1;
+    return 0; // Should never be equal!
+  }
+
+  calculate() {
+    const temp: ListTempCursor<T>[] = [];
+    // Only update if we have new changes
+    if (Object.values(this._changes).length !== this._lastUpdateSize) {
+      this._changes.sort(this.changesSort).forEach((change) => {
+        if (change.t === "INS") {
+          let poisitionToInsert = 0;
+          if (change.p) {
+            poisitionToInsert = temp.findIndex((v) => v.index === change.p) + 1;
+          } else if (change.n) {
+            poisitionToInsert = temp.findIndex((v) => v.index === change.n);
+          }
+
+          const cursorValue: ListTempCursor<T> = {
+            value: change.v,
+            index: change.i,
+            tomb: false,
+            prev: change.p,
+            next: change.n,
+          };
+
+          temp.splice(poisitionToInsert, 0, cursorValue);
+        } else if (change.t === "DEL") {
+          const poisitionToInsert = temp.findIndex((v) => v.index === change.v);
+          temp[poisitionToInsert].tomb = true;
+        }
+      });
+      this._lastUpdateSize = Object.values(temp).length;
+
+      this._tempValues = temp;
+
+      this._value = temp.filter((v) => !v.tomb).map((v) => v.value);
+    }
+  }
+
+  get value(): T[] {
+    this.calculate();
+    return this._value;
+  }
+
+  public mergeChanges(newChanges: ListChanges<T>[]) {
+    newChanges.forEach((change) => {
+      // Filter by author and index
+      const filtered = this._changes.filter(
+        (c) => c.i === change.i && c.t === change.t && c.v === change.v
+      );
+      // Only add if there are not matches
+      if (filtered.length === 0) {
+        this._changes.push(change);
+      }
+    });
+    this.calculate();
+  }
+
+  public getChanges(): ListChanges<T>[] {
+    return this._changes;
+  }
+
+  public INS(value: T, index: number) {
+    const filterTombs = this._tempValues.filter((v) => !v.tomb);
+
+    let currentPrev = undefined;
+    let currentNext = undefined;
+
+    if (filterTombs.length !== 0) {
+      const currentIndex = filterTombs[index].index;
+      const currentIndexPosition = this._tempValues.findIndex(
+        (v) => v.index === currentIndex
+      );
+      currentPrev = this._tempValues[currentIndexPosition - 1];
+      currentNext = this._tempValues[currentIndexPosition];
+    }
+
+    const ourChanges = this._changes.filter((c) =>
+      c.i.startsWith(this._author)
+    );
+
+    this._changes.push({
+      t: "INS",
+      p: currentPrev?.index,
+      n: currentNext?.index,
+      v: value,
+      i: this._author + "-" + `${ourChanges.length}`.padStart(8, "0"),
+    });
+    this.calculate();
+  }
+
+  public PUSH(value: T) {
+    const filterTombs = this._tempValues.filter((v) => !v.tomb);
+    const currentIndex = filterTombs[filterTombs.length - 1]?.index;
+    const currentIndexPosition = this._tempValues.findIndex(
+      (v) => v.index === currentIndex
+    );
+
+    const currentPrev = this._tempValues[currentIndexPosition];
+
+    const ourChanges = this._changes.filter((c) =>
+      c.i.startsWith(this._author)
+    );
+
+    this._changes.push({
+      t: "INS",
+      p: currentPrev?.index,
+      n: undefined,
+      v: value,
+      i: this._author + "-" + `${ourChanges.length}`.padStart(8, "0"),
+    });
+    this.calculate();
+  }
+
+  public DEL(index: number) {
+    const currentIndex = this._tempValues.filter((v) => !v.tomb)[index].index;
+    const currentIndexPosition = this._tempValues.findIndex(
+      (v) => v.index === currentIndex
+    );
+
+    const current = this._tempValues[currentIndexPosition];
+
+    const ourChanges = this._changes.filter((c) =>
+      c.i.startsWith(this._author)
+    );
+
+    this._changes.push({
+      t: "DEL",
+      v: current.index,
+      i: this._author + "-" + `${ourChanges.length}`.padStart(8, "0"),
+    });
+    this.calculate();
+  }
+}