reptree 0.1.1 → 0.1.3

package/README.md

@@ -8,11 +8,12 @@ A tree data structure using CRDTs for seamless replication between peers.
 
 ## Description
 
-RepTree is a tree data structure for storing vertices with properties.
-It uses 2 conflict-free replicated data types (CRDTs) to manage seamless replication between peers:
+RepTree uses 2 conflict-free replicated data types (CRDTs) to manage seamless replication between peers:
 - A move tree CRDT is used for the tree structure (https://martin.kleppmann.com/papers/move-op.pdf).
 - A last writer wins (LWW) CRDT is used for properties.
 
+RepTree can also be viewed as a hierarchical, distributed database. For more details on its database capabilities, see [RepTree as a Database](docs/database.md).
+
 ## Installation
 
 ```bash
@@ -28,18 +29,45 @@ import { RepTree } from 'reptree';
 const tree = new RepTree('peer1');
 
 // Root vertex is created automatically
-const rootVertex = tree.rootVertex;
-const rootId = rootVertex.id;
-
-// Add child vertices
-const childVertex = tree.newVertex(rootId);
-const childId = childVertex.id;
-
-// Set properties
-tree.setVertexProperty(childId, 'name', 'Child Node');
-
-// Move vertices
-tree.moveVertex(childId, anotherParentId);
+const rootVertex = tree.createRoot();
+rootVertex.name = 'Project';
+
+// Create a folder structure with properties
+const docsFolder = rootVertex.newNamedChild('Docs');
+docsFolder.setProperties({
+  type: 'folder',
+  icon: 'folder-icon'
+});
+
+const imagesFolder = rootVertex.newNamedChild('Images');
+imagesFolder.setProperties({
+  type: 'folder',
+  icon: 'image-icon'
+});
+
+// Add files to folders
+const readmeFile = docsFolder.newNamedChild('README.md');
+readmeFile.setProperties({
+  type: 'file',
+  size: 2048,
+  lastModified: '2023-10-15T14:22:10Z',
+  s3Path: 's3://my-bucket/docs/README.md'
+});
+
+const logoFile = imagesFolder.newNamedChild('logo.png');
+logoFile.setProperties({
+  type: 'file',
+  size: 15360,
+  dimensions: '512x512',
+  format: 'png',
+  s3Path: 's3://my-bucket/images/logo.png'
+});
+
+// Move a file to a different folder
+logoFile.moveTo(docsFolder);
+
+// Get children of a folder
+const docsFolderContents = docsFolder.children;
 
 // Syncing between trees
 const otherTree = new RepTree('peer2');

package/dist/index.cjs

@@ -22,6 +22,7 @@ var index_exports = {};
 __export(index_exports, {
   OpId: () => OpId,
   RepTree: () => RepTree,
+  StateVector: () => StateVector,
   TreeState: () => TreeState,
   Vertex: () => Vertex,
   VertexState: () => VertexState,
@@ -495,6 +496,191 @@ var Vertex = class {
   }
 };
 
+// src/StateVector.ts
+function subtractRanges(rangesA, rangesB) {
+  if (rangesB.length === 0) return rangesA.map((r) => [...r]);
+  if (rangesA.length === 0) return [];
+  const result = [];
+  let indexB = 0;
+  for (const rangeA of rangesA) {
+    let currentStart = rangeA[0];
+    const endA = rangeA[1];
+    while (indexB < rangesB.length && rangesB[indexB][1] < currentStart) {
+      indexB++;
+    }
+    while (indexB < rangesB.length && rangesB[indexB][0] <= endA) {
+      const startB = rangesB[indexB][0];
+      const endB = rangesB[indexB][1];
+      if (currentStart < startB) {
+        result.push([currentStart, Math.min(endA, startB - 1)]);
+      }
+      currentStart = Math.max(currentStart, endB + 1);
+      if (currentStart > endA) break;
+      if (endB >= endA) break;
+      if (endB < currentStart) {
+        indexB++;
+      } else if (startB >= currentStart) {
+        indexB++;
+      }
+    }
+    if (currentStart <= endA) {
+      result.push([currentStart, endA]);
+    }
+  }
+  return result;
+}
+var StateVector = class _StateVector {
+  /**
+   * Creates a new StateVector.
+   * @param initialState Optional initial state to copy from
+   */
+  constructor(initialState = {}) {
+    this.ranges = {};
+    for (const [peerId, peerRanges] of Object.entries(initialState)) {
+      this.ranges[peerId] = peerRanges.map((range) => [...range]);
+    }
+  }
+  /**
+   * Updates the state vector with a newly applied operation.
+   * Assumes ranges are sorted and non-overlapping.
+   * 
+   * @param peerId The peer ID of the operation
+   * @param counter The counter value of the operation
+   */
+  update(peerId, counter) {
+    if (!this.ranges[peerId]) {
+      this.ranges[peerId] = [];
+    }
+    const ranges = this.ranges[peerId];
+    if (ranges.length === 0) {
+      ranges.push([counter, counter]);
+      return;
+    }
+    let rangeExtendedOrMerged = false;
+    let insertIndex = -1;
+    for (let i = 0; i < ranges.length; i++) {
+      const range = ranges[i];
+      if (counter >= range[0] && counter <= range[1]) {
+        rangeExtendedOrMerged = true;
+        break;
+      }
+      if (counter === range[0] - 1) {
+        range[0] = counter;
+        rangeExtendedOrMerged = true;
+        if (i > 0 && range[0] === ranges[i - 1][1] + 1) {
+          ranges[i - 1][1] = range[1];
+          ranges.splice(i, 1);
+        }
+        break;
+      }
+      if (counter === range[1] + 1) {
+        range[1] = counter;
+        rangeExtendedOrMerged = true;
+        if (i < ranges.length - 1 && range[1] + 1 === ranges[i + 1][0]) {
+          range[1] = ranges[i + 1][1];
+          ranges.splice(i + 1, 1);
+        }
+        break;
+      }
+      if (counter < range[0] && insertIndex === -1) {
+        insertIndex = i;
+      }
+    }
+    if (!rangeExtendedOrMerged) {
+      if (insertIndex === -1) {
+        insertIndex = ranges.length;
+      }
+      ranges.splice(insertIndex, 0, [counter, counter]);
+      if (insertIndex > 0 && ranges[insertIndex][0] === ranges[insertIndex - 1][1] + 1) {
+        ranges[insertIndex - 1][1] = ranges[insertIndex][1];
+        ranges.splice(insertIndex, 1);
+        insertIndex--;
+      }
+      if (insertIndex < ranges.length - 1 && ranges[insertIndex][1] + 1 === ranges[insertIndex + 1][0]) {
+        ranges[insertIndex][1] = ranges[insertIndex + 1][1];
+        ranges.splice(insertIndex + 1, 1);
+      }
+    }
+  }
+  /**
+   * Updates the state vector with a newly applied operation.
+   * 
+   * @param op The operation that was just applied
+   */
+  updateFromOp(op) {
+    this.update(op.id.peerId, op.id.counter);
+  }
+  /**
+   * Returns the current state vector.
+   * Returns a readonly reference to the internal state.
+   */
+  getState() {
+    return this.ranges;
+  }
+  /**
+   * Calculates which operation ranges we have that the other state vector is missing
+   * by comparing state vectors.
+   * 
+   * @param other The other state vector to compare against
+   * @returns Array of operation ID ranges that we have but they don't
+   */
+  diff(other) {
+    const missingRanges = [];
+    const theirState = other.getState();
+    for (const [peerId, ourRanges] of Object.entries(this.ranges)) {
+      const theirRanges = theirState[peerId] || [];
+      const missing = subtractRanges(ourRanges, theirRanges);
+      for (const [start, end] of missing) {
+        if (start <= end) {
+          missingRanges.push({
+            peerId,
+            start,
+            end
+          });
+        }
+      }
+    }
+    return missingRanges;
+  }
+  /**
+   * Checks if the state vector contains the given operation ID
+   * 
+   * @param opId The operation ID to check
+   * @returns true if the operation is in the state vector, false otherwise
+   */
+  contains(opId) {
+    const peerId = opId.peerId;
+    const counter = opId.counter;
+    if (!this.ranges[peerId]) {
+      return false;
+    }
+    for (const [start, end] of this.ranges[peerId]) {
+      if (counter >= start && counter <= end) {
+        return true;
+      }
+    }
+    return false;
+  }
+  /**
+   * Creates a copy of this state vector
+   */
+  clone() {
+    return new _StateVector(this.ranges);
+  }
+  /**
+   * Builds a state vector from an array of operations
+   * @param operations The operations to build the state vector from
+   * @returns A new StateVector instance
+   */
+  static fromOperations(operations) {
+    const stateVector = new _StateVector();
+    for (const op of operations) {
+      stateVector.updateFromOp(op);
+    }
+    return stateVector;
+  }
+};
+
 // src/RepTree.ts
 var _RepTree = class _RepTree {
   /**
@@ -510,31 +696,40 @@ var _RepTree = class _RepTree {
     this.localOps = [];
     this.pendingMovesWithMissingParent = /* @__PURE__ */ new Map();
     this.pendingPropertiesWithMissingVertex = /* @__PURE__ */ new Map();
-    this.appliedOps = /* @__PURE__ */ new Set();
+    this.knownOps = /* @__PURE__ */ new Set();
     this.parentIdBeforeMove = /* @__PURE__ */ new Map();
     this.opAppliedCallbacks = [];
     this.maxDepth = _RepTree.DEFAULT_MAX_DEPTH;
+    this._stateVectorEnabled = true;
     this.peerId = peerId;
     this.state = new TreeState();
+    this.stateVector = new StateVector();
     if (ops != null && ops.length > 0) {
-      let rootMoveOp;
-      for (let i = 0; i < ops.length; i++) {
-        if (isMoveVertexOp(ops[i]) && ops[i].parentId === null) {
-          rootMoveOp = ops[i];
-          break;
-        }
-      }
-      if (rootMoveOp) {
-        this.rootVertexId = rootMoveOp.targetId;
-      } else {
-        throw new Error("The operations has to contain a move operation with a parentId as null to set the root vertex");
-      }
       this.applyOps(ops);
-      this.ensureTrashVertex();
+      const root = this.root;
+      if (!root) {
+        throw new Error("There has to be a root vertex in the operations");
+      }
     } else {
-      this.rootVertexId = this.newVertexInternalWithUUID(null);
-      this.ensureTrashVertex();
+      this.ensureNullVertex();
+    }
+  }
+  get root() {
+    if (!this.rootVertexId) {
+      const vertices = this.state.getAllVertices();
+      for (const vertex of vertices) {
+        if (vertex.parentId === null && vertex.id !== _RepTree.NULL_VERTEX_ID) {
+          this.rootVertexId = vertex.id;
+          return new Vertex(this, vertex);
+        }
+      }
+      return void 0;
+    }
+    const rootVertex = this.state.getVertex(this.rootVertexId);
+    if (!rootVertex) {
+      throw new Error("Root vertex not found");
     }
+    return new Vertex(this, rootVertex);
   }
   getMoveOps() {
     return this.moveOps;
@@ -546,13 +741,6 @@ var _RepTree = class _RepTree {
     const vertex = this.state.getVertex(vertexId);
     return vertex ? new Vertex(this, vertex) : void 0;
   }
-  get rootVertex() {
-    const rootVertex = this.state.getVertex(this.rootVertexId);
-    if (!rootVertex) {
-      throw new Error("Root vertex not found");
-    }
-    return new Vertex(this, rootVertex);
-  }
   getAllVertices() {
     return this.state.getAllVertices().map((v) => new Vertex(this, v));
   }
@@ -603,6 +791,17 @@ var _RepTree = class _RepTree {
   setMaxDepth(maxDepth) {
     this.maxDepth = maxDepth;
   }
+  createRoot() {
+    if (this.rootVertexId) {
+      throw new Error("Root vertex already exists");
+    }
+    this.rootVertexId = this.newVertexInternalWithUUID(null);
+    const rootVertex = this.state.getVertex(this.rootVertexId);
+    if (!rootVertex) {
+      throw new Error("Root vertex not found");
+    }
+    return new Vertex(this, rootVertex);
+  }
   newVertex(parentId, props = null) {
     const typedProps = props;
     const vertexId = this.newVertexInternalWithUUID(parentId);
@@ -635,7 +834,7 @@ var _RepTree = class _RepTree {
     this.applyMove(op);
   }
   deleteVertex(vertexId) {
-    this.moveVertex(vertexId, _RepTree.TRASH_VERTEX_ID);
+    this.moveVertex(vertexId, _RepTree.NULL_VERTEX_ID);
   }
   setTransientVertexProperty(vertexId, key, value) {
     this.lamportClock++;
@@ -659,6 +858,9 @@ var _RepTree = class _RepTree {
     path = path.replace(/^\/+/, "");
     path = path.replace(/\/+$/, "");
     const pathParts = path.split("/");
+    if (!this.rootVertexId) {
+      return void 0;
+    }
     const root = this.state.getVertex(this.rootVertexId);
     if (!root) {
       throw new Error("The root vertex is not found");
@@ -680,12 +882,38 @@ var _RepTree = class _RepTree {
     return void 0;
   }
   printTree() {
+    if (!this.rootVertexId) {
+      return "";
+    }
     return this.state.printTree(this.rootVertexId);
   }
   merge(ops) {
     this.applyOps(ops);
   }
+  /** Applies operations in an optimized way, sorting move ops by OpId to avoid undo-do-redo cycles */
+  applyOpsOptimizedForLotsOfMoves(ops) {
+    const newMoveOps = ops.filter((op) => isMoveVertexOp(op) && !this.knownOps.has(op.id.toString()));
+    if (newMoveOps.length > 0) {
+      const allMoveOps = [...this.moveOps, ...newMoveOps];
+      allMoveOps.sort((a, b) => OpId.compare(a.id, b.id));
+      for (let i = 0, len = allMoveOps.length; i < len; i++) {
+        const op = allMoveOps[i];
+        this.applyMove(op);
+      }
+    }
+    const propertyOps = ops.filter((op) => isSetPropertyOp(op) && !this.knownOps.has(op.id.toString()));
+    for (let i = 0, len = propertyOps.length; i < len; i++) {
+      const op = propertyOps[i];
+      this.applyProperty(op);
+    }
+  }
   compareStructure(other) {
+    if (this.root?.id !== other.root?.id) {
+      return false;
+    }
+    if (!this.rootVertexId) {
+      return true;
+    }
     return _RepTree.compareVertices(this.rootVertexId, this, other);
   }
   compareMoveOps(other) {
@@ -795,8 +1023,8 @@ var _RepTree = class _RepTree {
     const vertexId = uuid();
     return this.newVertexInternal(vertexId, parentId);
   }
-  ensureTrashVertex() {
-    const vertexId = _RepTree.TRASH_VERTEX_ID;
+  ensureNullVertex() {
+    const vertexId = _RepTree.NULL_VERTEX_ID;
     if (this.state.getVertex(vertexId)) {
       return;
     }
@@ -808,6 +1036,19 @@ var _RepTree = class _RepTree {
       this.lamportClock = operation.id.counter;
     }
   }
+  applyPendingMovesForParent(parentId) {
+    if (!this.state.getVertex(parentId)) {
+      return;
+    }
+    const pendingMoves = this.pendingMovesWithMissingParent.get(parentId);
+    if (!pendingMoves) {
+      return;
+    }
+    this.pendingMovesWithMissingParent.delete(parentId);
+    for (const pendingOp of pendingMoves) {
+      this.applyMove(pendingOp);
+    }
+  }
   applyMove(op) {
     if (op.parentId !== null && !this.state.getVertex(op.parentId)) {
       if (!this.pendingMovesWithMissingParent.has(op.parentId)) {
@@ -842,15 +1083,52 @@ var _RepTree = class _RepTree {
     }
     this.applyPendingMovesForParent(op.targetId);
   }
-  reportOpAsApplied(op) {
-    this.appliedOps.add(op.id.toString());
-    for (const callback of this.opAppliedCallbacks) {
-      callback(op);
+  setPropertyAndItsOpId(op) {
+    this.propertiesAndTheirOpIds.set(`${op.key}@${op.targetId}`, op.id);
+    this.state.setProperty(op.targetId, op.key, op.value);
+    this.reportOpAsApplied(op);
+  }
+  setTransientPropertyAndItsOpId(op) {
+    this.transientPropertiesAndTheirOpIds.set(`${op.key}@${op.targetId}`, op.id);
+    this.state.setTransientProperty(op.targetId, op.key, op.value);
+    this.reportOpAsApplied(op);
+  }
+  applyProperty(op) {
+    const targetVertex = this.state.getVertex(op.targetId);
+    if (!targetVertex) {
+      if (op.transient) {
+        return;
+      }
+      if (!this.pendingPropertiesWithMissingVertex.has(op.targetId)) {
+        this.pendingPropertiesWithMissingVertex.set(op.targetId, []);
+      }
+      this.pendingPropertiesWithMissingVertex.get(op.targetId).push(op);
+      return;
+    }
+    this.updateLamportClock(op);
+    const prevTransientOpId = this.transientPropertiesAndTheirOpIds.get(`${op.key}@${op.targetId}`);
+    const prevProp = targetVertex.getProperty(op.key);
+    const prevOpId = this.propertiesAndTheirOpIds.get(`${op.key}@${op.targetId}`);
+    if (!op.transient) {
+      this.setPropertyOps.push(op);
+      if (!prevProp || !prevOpId || op.id.isGreaterThan(prevOpId)) {
+        this.setPropertyAndItsOpId(op);
+      } else {
+        this.knownOps.add(op.id.toString());
+      }
+      if (prevTransientOpId && op.id.isGreaterThan(prevTransientOpId)) {
+        this.transientPropertiesAndTheirOpIds.delete(`${op.key}@${op.targetId}`);
+        targetVertex.removeTransientProperty(op.key);
+      }
+    } else {
+      if (!prevTransientOpId || op.id.isGreaterThan(prevTransientOpId)) {
+        this.setTransientPropertyAndItsOpId(op);
+      }
     }
   }
   applyOps(ops) {
     for (const op of ops) {
-      if (this.appliedOps.has(op.id.toString())) {
+      if (this.knownOps.has(op.id.toString())) {
         continue;
       }
       if (isMoveVertexOp(op)) {
@@ -860,34 +1138,13 @@ var _RepTree = class _RepTree {
       }
     }
   }
-  /** Applies operations in an optimized way, sorting move ops by OpId to avoid undo-do-redo cycles */
-  applyOpsOptimizedForLotsOfMoves(ops) {
-    const newMoveOps = ops.filter((op) => isMoveVertexOp(op) && !this.appliedOps.has(op.id.toString()));
-    if (newMoveOps.length > 0) {
-      const allMoveOps = [...this.moveOps, ...newMoveOps];
-      allMoveOps.sort((a, b) => OpId.compare(a.id, b.id));
-      for (let i = 0, len = allMoveOps.length; i < len; i++) {
-        const op = allMoveOps[i];
-        this.applyMove(op);
-      }
-    }
-    const propertyOps = ops.filter((op) => isSetPropertyOp(op) && !this.appliedOps.has(op.id.toString()));
-    for (let i = 0, len = propertyOps.length; i < len; i++) {
-      const op = propertyOps[i];
-      this.applyProperty(op);
-    }
-  }
-  applyPendingMovesForParent(parentId) {
-    if (!this.state.getVertex(parentId)) {
-      return;
-    }
-    const pendingMoves = this.pendingMovesWithMissingParent.get(parentId);
-    if (!pendingMoves) {
-      return;
+  reportOpAsApplied(op) {
+    this.knownOps.add(op.id.toString());
+    if (this._stateVectorEnabled) {
+      this.stateVector.updateFromOp(op);
     }
-    this.pendingMovesWithMissingParent.delete(parentId);
-    for (const pendingOp of pendingMoves) {
-      this.applyMove(pendingOp);
+    for (const callback of this.opAppliedCallbacks) {
+      callback(op);
     }
   }
   tryToMove(op) {
@@ -900,6 +1157,7 @@ var _RepTree = class _RepTree {
     this.state.moveVertex(op.targetId, op.parentId);
     if (!targetVertex) {
       const pendingProperties = this.pendingPropertiesWithMissingVertex.get(op.targetId) || [];
+      this.pendingPropertiesWithMissingVertex.delete(op.targetId);
       for (const prop of pendingProperties) {
         this.setPropertyAndItsOpId(prop);
       }
@@ -917,55 +1175,72 @@ var _RepTree = class _RepTree {
     }
     this.state.moveVertex(op.targetId, prevParentId);
   }
-  setPropertyAndItsOpId(op) {
-    this.propertiesAndTheirOpIds.set(`${op.key}@${op.targetId}`, op.id);
-    this.state.setProperty(op.targetId, op.key, op.value);
-    this.reportOpAsApplied(op);
-  }
-  setTransientPropertyAndItsOpId(op) {
-    this.transientPropertiesAndTheirOpIds.set(`${op.key}@${op.targetId}`, op.id);
-    this.state.setTransientProperty(op.targetId, op.key, op.value);
-    this.reportOpAsApplied(op);
+  // --- Range-Based State Vector Methods --- 
+  /**
+   * Returns the current state vector.
+   * Returns a readonly reference to the internal state vector.
+   */
+  getStateVector() {
+    if (!this._stateVectorEnabled) {
+      return null;
+    }
+    return this.stateVector.getState();
   }
-  applyProperty(op) {
-    const targetVertex = this.state.getVertex(op.targetId);
-    if (!targetVertex) {
-      if (op.transient) {
-        return;
-      }
-      if (!this.pendingPropertiesWithMissingVertex.has(op.targetId)) {
-        this.pendingPropertiesWithMissingVertex.set(op.targetId, []);
+  /**
+   * Determines which operations are needed to synchronize 
+   * with the provided state vector.
+   * 
+   * @param theirStateVector The state vector from another peer
+   * @returns Operations that should be sent to the other peer, sorted by OpId.
+   */
+  getMissingOps(theirStateVector) {
+    if (!this._stateVectorEnabled) {
+      return [...this.moveOps, ...this.setPropertyOps];
+    }
+    const otherStateVector = new StateVector(theirStateVector);
+    const missingRanges = this.stateVector.diff(otherStateVector);
+    const missingOps = [];
+    const allOps = [...this.moveOps, ...this.setPropertyOps];
+    for (const op of allOps) {
+      for (const range of missingRanges) {
+        if (op.id.peerId === range.peerId && op.id.counter >= range.start && op.id.counter <= range.end) {
+          missingOps.push(op);
+          break;
+        }
       }
-      this.pendingPropertiesWithMissingVertex.get(op.targetId).push(op);
-      return;
     }
-    this.updateLamportClock(op);
-    const prevTransientOpId = this.transientPropertiesAndTheirOpIds.get(`${op.key}@${op.targetId}`);
-    const prevProp = targetVertex.getProperty(op.key);
-    const prevOpId = this.propertiesAndTheirOpIds.get(`${op.key}@${op.targetId}`);
-    if (!op.transient) {
-      this.setPropertyOps.push(op);
-      if (!prevProp || !prevOpId || op.id.isGreaterThan(prevOpId)) {
-        this.setPropertyAndItsOpId(op);
-      }
-      if (prevTransientOpId && op.id.isGreaterThan(prevTransientOpId)) {
-        this.transientPropertiesAndTheirOpIds.delete(`${op.key}@${op.targetId}`);
-        targetVertex.removeTransientProperty(op.key);
-      }
+    missingOps.sort((a, b) => OpId.compare(a.id, b.id));
+    return missingOps;
+  }
+  /**
+   * Gets or sets whether state vector tracking is enabled
+   */
+  get stateVectorEnabled() {
+    return this._stateVectorEnabled;
+  }
+  /**
+   * Sets the state vector enabled status
+   * When enabled, rebuilds the state vector from existing operations if needed
+   */
+  set stateVectorEnabled(value) {
+    if (value === this._stateVectorEnabled) return;
+    if (value) {
+      this._stateVectorEnabled = true;
+      this.stateVector = StateVector.fromOperations([...this.moveOps, ...this.setPropertyOps]);
     } else {
-      if (!prevTransientOpId || op.id.isGreaterThan(prevTransientOpId)) {
-        this.setTransientPropertyAndItsOpId(op);
-      }
+      this._stateVectorEnabled = false;
+      this.stateVector = new StateVector();
     }
   }
 };
-_RepTree.TRASH_VERTEX_ID = "t";
+_RepTree.NULL_VERTEX_ID = "0";
 _RepTree.DEFAULT_MAX_DEPTH = 1e5;
 var RepTree = _RepTree;
 // Annotate the CommonJS export names for ESM import in node:
 0 && (module.exports = {
   OpId,
   RepTree,
+  StateVector,
   TreeState,
   Vertex,
   VertexState,