prosemirror-changeset 2.2.1 → 2.3.0

package/CHANGELOG.md

@@ -1,3 +1,9 @@
+## 2.3.0 (2025-05-05)
+
+### New features
+
+Change sets can now be passed a custom token encoder that controls the way changed content is diffed.
+
 ## 2.2.1 (2023-05-17)
 
 ### Bug fixes

package/README.md

@@ -41,7 +41,7 @@ A replaced range with metadata associated with it.
 
  * **`inserted`**`: readonly Span[]`\
    Data associated with the inserted content. Length adds up to
-   `this.toB - this.toA`.
+   `this.toB - this.fromB`.
 
  * `static `**`merge`**`<Data>(x: readonly Change[], y: readonly Change[], combine: fn(dataA: Data, dataB: Data) → Data) → readonly Change[]`\
    This merges two changesets (the end document of x should be the
@@ -96,12 +96,18 @@ partially undo themselves by comparing their content.
    make sure the method is called on the old set and passed the new
    set. The returned positions will be in new document coordinates.
 
- * `static `**`create`**`<Data = any>(doc: Node, combine?: fn(dataA: Data, dataB: Data) → Data = (a, b) => a === b ? a : null as any) → ChangeSet`\
+ * `static `**`create`**`<Data = any>(doc: Node, combine?: fn(dataA: Data, dataB: Data) → Data = (a, b) => a === b ? a : null as any, tokenEncoder?: TokenEncoder = DefaultEncoder) → ChangeSet`\
    Create a changeset with the given base object and configuration.
+
    The `combine` function is used to compare and combine metadata—it
    should return null when metadata isn't compatible, and a combined
    version for a merged range when it is.
 
+   When given, a token encoder determines how document tokens are
+   serialized and compared when diffing the content produced by
+   changes. The default is to just compare nodes by name and text
+   by character, ignoring marks and attributes.
+
 
  * **`simplifyChanges`**`(changes: readonly Change[], doc: Node) → Change[]`\
    Simplifies a set of changes for presentation. This makes the
@@ -111,3 +117,30 @@ partially undo themselves by comparing their content.
    words (in the new document) they touch. An exception is made for
    single-character replacements.
 
+
+### interface TokenEncoder`<T>`
+
+A token encoder can be passed when creating a `ChangeSet` in order
+to influence the way the library runs its diffing algorithm. The
+encoder determines how document tokens (such as nodes and
+characters) are encoded and compared.
+
+Note that both the encoding and the comparison may run a lot, and
+doing non-trivial work in these functions could impact
+performance.
+
+ * **`encodeCharacter`**`(char: number, marks: readonly Mark[]) → T`\
+   Encode a given character, with the given marks applied.
+
+ * **`encodeNodeStart`**`(node: Node) → T`\
+   Encode the start of a node or, if this is a leaf node, the
+   entire node.
+
+ * **`encodeNodeEnd`**`(node: Node) → T`\
+   Encode the end token for the given node. It is valid to encode
+   every end token in the same way.
+
+ * **`compareTokens`**`(a: T, b: T) → boolean`\
+   Compare the given tokens. Should return true when they count as
+   equal.
+

package/dist/index.cjs

@@ -1,113 +1,90 @@
 'use strict';
 
 function _toConsumableArray(arr) { return _arrayWithoutHoles(arr) || _iterableToArray(arr) || _unsupportedIterableToArray(arr) || _nonIterableSpread(); }
-
 function _nonIterableSpread() { throw new TypeError("Invalid attempt to spread non-iterable instance.\nIn order to be iterable, non-array objects must have a [Symbol.iterator]() method."); }
-
 function _unsupportedIterableToArray(o, minLen) { if (!o) return; if (typeof o === "string") return _arrayLikeToArray(o, minLen); var n = Object.prototype.toString.call(o).slice(8, -1); if (n === "Object" && o.constructor) n = o.constructor.name; if (n === "Map" || n === "Set") return Array.from(o); if (n === "Arguments" || /^(?:Ui|I)nt(?:8|16|32)(?:Clamped)?Array$/.test(n)) return _arrayLikeToArray(o, minLen); }
-
 function _iterableToArray(iter) { if (typeof Symbol !== "undefined" && iter[Symbol.iterator] != null || iter["@@iterator"] != null) return Array.from(iter); }
-
 function _arrayWithoutHoles(arr) { if (Array.isArray(arr)) return _arrayLikeToArray(arr); }
-
-function _arrayLikeToArray(arr, len) { if (len == null || len > arr.length) len = arr.length; for (var i = 0, arr2 = new Array(len); i < len; i++) { arr2[i] = arr[i]; } return arr2; }
-
+function _arrayLikeToArray(arr, len) { if (len == null || len > arr.length) len = arr.length; for (var i = 0, arr2 = new Array(len); i < len; i++) arr2[i] = arr[i]; return arr2; }
+function _typeof(o) { "@babel/helpers - typeof"; return _typeof = "function" == typeof Symbol && "symbol" == typeof Symbol.iterator ? function (o) { return typeof o; } : function (o) { return o && "function" == typeof Symbol && o.constructor === Symbol && o !== Symbol.prototype ? "symbol" : typeof o; }, _typeof(o); }
 function _classCallCheck(instance, Constructor) { if (!(instance instanceof Constructor)) { throw new TypeError("Cannot call a class as a function"); } }
-
-function _defineProperties(target, props) { for (var i = 0; i < props.length; i++) { var descriptor = props[i]; descriptor.enumerable = descriptor.enumerable || false; descriptor.configurable = true; if ("value" in descriptor) descriptor.writable = true; Object.defineProperty(target, descriptor.key, descriptor); } }
-
+function _defineProperties(target, props) { for (var i = 0; i < props.length; i++) { var descriptor = props[i]; descriptor.enumerable = descriptor.enumerable || false; descriptor.configurable = true; if ("value" in descriptor) descriptor.writable = true; Object.defineProperty(target, _toPropertyKey(descriptor.key), descriptor); } }
 function _createClass(Constructor, protoProps, staticProps) { if (protoProps) _defineProperties(Constructor.prototype, protoProps); if (staticProps) _defineProperties(Constructor, staticProps); Object.defineProperty(Constructor, "prototype", { writable: false }); return Constructor; }
-
-function _typeof(obj) { "@babel/helpers - typeof"; return _typeof = "function" == typeof Symbol && "symbol" == typeof Symbol.iterator ? function (obj) { return typeof obj; } : function (obj) { return obj && "function" == typeof Symbol && obj.constructor === Symbol && obj !== Symbol.prototype ? "symbol" : typeof obj; }, _typeof(obj); }
-
-Object.defineProperty(exports, '__esModule', {
-  value: true
-});
-
-function tokens(frag, start, end, target) {
+function _toPropertyKey(arg) { var key = _toPrimitive(arg, "string"); return _typeof(key) === "symbol" ? key : String(key); }
+function _toPrimitive(input, hint) { if (_typeof(input) !== "object" || input === null) return input; var prim = input[Symbol.toPrimitive]; if (prim !== undefined) { var res = prim.call(input, hint || "default"); if (_typeof(res) !== "object") return res; throw new TypeError("@@toPrimitive must return a primitive value."); } return (hint === "string" ? String : Number)(input); }
+var DefaultEncoder = {
+  encodeCharacter: function encodeCharacter(_char) {
+    return _char;
+  },
+  encodeNodeStart: function encodeNodeStart(node) {
+    return node.type.name;
+  },
+  encodeNodeEnd: function encodeNodeEnd() {
+    return -1;
+  },
+  compareTokens: function compareTokens(a, b) {
+    return a === b;
+  }
+};
+function tokens(frag, encoder, start, end, target) {
   for (var i = 0, off = 0; i < frag.childCount; i++) {
     var child = frag.child(i),
-        endOff = off + child.nodeSize;
+      endOff = off + child.nodeSize;
     var from = Math.max(off, start),
-        to = Math.min(endOff, end);
-
+      to = Math.min(endOff, end);
     if (from < to) {
       if (child.isText) {
-        for (var j = from; j < to; j++) {
-          target.push(child.text.charCodeAt(j - off));
-        }
+        for (var j = from; j < to; j++) target.push(encoder.encodeCharacter(child.text.charCodeAt(j - off), child.marks));
       } else if (child.isLeaf) {
-        target.push(child.type.name);
+        target.push(encoder.encodeNodeStart(child));
       } else {
-        if (from == off) target.push(child.type.name);
-        tokens(child.content, Math.max(off + 1, from) - off - 1, Math.min(endOff - 1, to) - off - 1, target);
-        if (to == endOff) target.push(-1);
+        if (from == off) target.push(encoder.encodeNodeStart(child));
+        tokens(child.content, encoder, Math.max(off + 1, from) - off - 1, Math.min(endOff - 1, to) - off - 1, target);
+        if (to == endOff) target.push(encoder.encodeNodeEnd(child));
       }
     }
-
     off = endOff;
   }
-
   return target;
 }
-
 var MAX_DIFF_SIZE = 5000;
-
 function minUnchanged(sizeA, sizeB) {
   return Math.min(15, Math.max(2, Math.floor(Math.max(sizeA, sizeB) / 10)));
 }
-
 function computeDiff(fragA, fragB, range) {
-  var tokA = tokens(fragA, range.fromA, range.toA, []);
-  var tokB = tokens(fragB, range.fromB, range.toB, []);
+  var encoder = arguments.length > 3 && arguments[3] !== undefined ? arguments[3] : DefaultEncoder;
+  var tokA = tokens(fragA, encoder, range.fromA, range.toA, []);
+  var tokB = tokens(fragB, encoder, range.fromB, range.toB, []);
   var start = 0,
-      endA = tokA.length,
-      endB = tokB.length;
-
-  while (start < tokA.length && start < tokB.length && tokA[start] === tokB[start]) {
-    start++;
-  }
-
+    endA = tokA.length,
+    endB = tokB.length;
+  var cmp = encoder.compareTokens;
+  while (start < tokA.length && start < tokB.length && cmp(tokA[start], tokB[start])) start++;
   if (start == tokA.length && start == tokB.length) return [];
-
-  while (endA > start && endB > start && tokA[endA - 1] === tokB[endB - 1]) {
-    endA--, endB--;
-  }
-
+  while (endA > start && endB > start && cmp(tokA[endA - 1], tokB[endB - 1])) endA--, endB--;
   if (endA == start || endB == start || endA == endB && endA == start + 1) return [range.slice(start, endA, start, endB)];
   var lenA = endA - start,
-      lenB = endB - start;
+    lenB = endB - start;
   var max = Math.min(MAX_DIFF_SIZE, lenA + lenB),
-      off = max + 1;
+    off = max + 1;
   var history = [];
   var frontier = [];
-
-  for (var len = off * 2, i = 0; i < len; i++) {
-    frontier[i] = -1;
-  }
-
+  for (var len = off * 2, i = 0; i < len; i++) frontier[i] = -1;
   for (var size = 0; size <= max; size++) {
-    for (var diag = -size; diag <= size; diag += 2) {
-      var next = frontier[diag + 1 + max],
-          prev = frontier[diag - 1 + max];
-      var x = next < prev ? prev : next + 1,
-          y = x + diag;
-
-      while (x < lenA && y < lenB && tokA[start + x] === tokB[start + y]) {
-        x++, y++;
-      }
-
-      frontier[diag + max] = x;
-
-      if (x >= lenA && y >= lenB) {
-        var _ret = function () {
+    var _loop = function _loop(_diag) {
+        var next = frontier[_diag + 1 + max],
+          prev = frontier[_diag - 1 + max];
+        var x = next < prev ? prev : next + 1,
+          y = x + _diag;
+        while (x < lenA && y < lenB && cmp(tokA[start + x], tokB[start + y])) x++, y++;
+        frontier[_diag + max] = x;
+        if (x >= lenA && y >= lenB) {
           var diff = [],
-              minSpan = minUnchanged(endA - start, endB - start);
+            minSpan = minUnchanged(endA - start, endB - start);
           var fromA = -1,
-              toA = -1,
-              fromB = -1,
-              toB = -1;
-
+            toA = -1,
+            fromB = -1,
+            toB = -1;
           var add = function add(fA, tA, fB, tB) {
             if (fromA > -1 && fromA < tA + minSpan) {
               fromA = fA;
@@ -120,50 +97,44 @@ function computeDiff(fragA, fragB, range) {
               toB = tB;
             }
           };
-
           for (var _i = size - 1; _i >= 0; _i--) {
-            var _next = frontier[diag + 1 + max],
-                _prev = frontier[diag - 1 + max];
-
+            var _next = frontier[_diag + 1 + max],
+              _prev = frontier[_diag - 1 + max];
             if (_next < _prev) {
-              diag--;
+              _diag--;
               x = _prev + start;
-              y = x + diag;
+              y = x + _diag;
               add(x, x, y, y + 1);
             } else {
-              diag++;
+              _diag++;
               x = _next + start;
-              y = x + diag;
+              y = x + _diag;
               add(x, x + 1, y, y);
             }
-
             frontier = history[_i >> 1];
           }
-
           if (fromA > -1) diff.push(range.slice(fromA, toA, fromB, toB));
           return {
             v: diff.reverse()
           };
-        }();
-
-        if (_typeof(_ret) === "object") return _ret.v;
-      }
+        }
+        diag = _diag;
+      },
+      _ret;
+    for (var diag = -size; diag <= size; diag += 2) {
+      _ret = _loop(diag);
+      if (_ret) return _ret.v;
     }
-
     if (size % 2 == 0) history.push(frontier.slice());
   }
-
   return [range.slice(start, endA, start, endB)];
 }
-
 var Span = function () {
   function Span(length, data) {
     _classCallCheck(this, Span);
-
     this.length = length;
     this.data = data;
   }
-
   _createClass(Span, [{
     key: "cut",
     value: function cut(length) {
@@ -175,15 +146,13 @@ var Span = function () {
       if (from == to) return Span.none;
       if (from == 0 && to == Span.len(spans)) return spans;
       var result = [];
-
       for (var i = 0, off = 0; off < to; i++) {
         var span = spans[i],
-            end = off + span.length;
+          end = off + span.length;
         var overlap = Math.min(to, end) - Math.max(from, off);
         if (overlap > 0) result.push(span.cut(overlap));
         off = end;
       }
-
       return result;
     }
   }, {
@@ -195,35 +164,23 @@ var Span = function () {
       if (combined == null) return a.concat(b);
       var result = a.slice(0, a.length - 1);
       result.push(new Span(a[a.length - 1].length + b[0].length, combined));
-
-      for (var i = 1; i < b.length; i++) {
-        result.push(b[i]);
-      }
-
+      for (var i = 1; i < b.length; i++) result.push(b[i]);
       return result;
     }
   }, {
     key: "len",
     value: function len(spans) {
       var len = 0;
-
-      for (var i = 0; i < spans.length; i++) {
-        len += spans[i].length;
-      }
-
+      for (var i = 0; i < spans.length; i++) len += spans[i].length;
       return len;
     }
   }]);
-
   return Span;
 }();
-
 Span.none = [];
-
 var Change = function () {
   function Change(fromA, toA, fromB, toB, deleted, inserted) {
     _classCallCheck(this, Change);
-
     this.fromA = fromA;
     this.toA = toA;
     this.fromB = fromB;
@@ -231,7 +188,6 @@ var Change = function () {
     this.deleted = deleted;
     this.inserted = inserted;
   }
-
   _createClass(Change, [{
     key: "lenA",
     get: function get() {
@@ -254,7 +210,6 @@ var Change = function () {
       if (x.length == 0) return y;
       if (y.length == 0) return x;
       var result = [];
-
       for (var iX = 0, iY = 0, curX = x[0], curY = y[0];;) {
         if (!curX && !curY) {
           return result;
@@ -264,97 +219,79 @@ var Change = function () {
           curX = iX++ == x.length ? null : x[iX];
         } else if (curY && (!curX || curY.toA < curX.fromB)) {
           var _off = iX ? x[iX - 1].toB - x[iX - 1].toA : 0;
-
           result.push(_off == 0 ? curY : new Change(curY.fromA - _off, curY.toA - _off, curY.fromB, curY.toB, curY.deleted, curY.inserted));
           curY = iY++ == y.length ? null : y[iY];
         } else {
           var pos = Math.min(curX.fromB, curY.fromA);
           var fromA = Math.min(curX.fromA, curY.fromA - (iX ? x[iX - 1].toB - x[iX - 1].toA : 0)),
-              toA = fromA;
+            toA = fromA;
           var fromB = Math.min(curY.fromB, curX.fromB + (iY ? y[iY - 1].toB - y[iY - 1].toA : 0)),
-              toB = fromB;
+            toB = fromB;
           var deleted = Span.none,
-              inserted = Span.none;
+            inserted = Span.none;
           var enteredX = false,
-              enteredY = false;
-
+            enteredY = false;
           for (;;) {
             var nextX = !curX ? 2e8 : pos >= curX.fromB ? curX.toB : curX.fromB;
             var nextY = !curY ? 2e8 : pos >= curY.fromA ? curY.toA : curY.fromA;
             var next = Math.min(nextX, nextY);
             var inX = curX && pos >= curX.fromB,
-                inY = curY && pos >= curY.fromA;
+              inY = curY && pos >= curY.fromA;
             if (!inX && !inY) break;
-
             if (inX && pos == curX.fromB && !enteredX) {
               deleted = Span.join(deleted, curX.deleted, combine);
               toA += curX.lenA;
               enteredX = true;
             }
-
             if (inX && !inY) {
               inserted = Span.join(inserted, Span.slice(curX.inserted, pos - curX.fromB, next - curX.fromB), combine);
               toB += next - pos;
             }
-
             if (inY && pos == curY.fromA && !enteredY) {
               inserted = Span.join(inserted, curY.inserted, combine);
               toB += curY.lenB;
               enteredY = true;
             }
-
             if (inY && !inX) {
               deleted = Span.join(deleted, Span.slice(curY.deleted, pos - curY.fromA, next - curY.fromA), combine);
               toA += next - pos;
             }
-
             if (inX && next == curX.toB) {
               curX = iX++ == x.length ? null : x[iX];
               enteredX = false;
             }
-
             if (inY && next == curY.toA) {
               curY = iY++ == y.length ? null : y[iY];
               enteredY = false;
             }
-
             pos = next;
           }
-
           if (fromA < toA || fromB < toB) result.push(new Change(fromA, toA, fromB, toB, deleted, inserted));
         }
       }
     }
   }]);
-
   return Change;
 }();
-
 var letter;
-
 try {
   letter = new RegExp("[\\p{Alphabetic}_]", "u");
 } catch (_) {}
-
 var nonASCIISingleCaseWordChar = /[\u00df\u0587\u0590-\u05f4\u0600-\u06ff\u3040-\u309f\u30a0-\u30ff\u3400-\u4db5\u4e00-\u9fcc\uac00-\ud7af]/;
-
 function isLetter(code) {
   if (code < 128) return code >= 48 && code <= 57 || code >= 65 && code <= 90 || code >= 79 && code <= 122;
   var ch = String.fromCharCode(code);
   if (letter) return letter.test(ch);
   return ch.toUpperCase() != ch.toLowerCase() || nonASCIISingleCaseWordChar.test(ch);
 }
-
 function getText(frag, start, end) {
   var out = "";
-
   function convert(frag, start, end) {
     for (var i = 0, off = 0; i < frag.childCount; i++) {
       var child = frag.child(i),
-          endOff = off + child.nodeSize;
+        endOff = off + child.nodeSize;
       var from = Math.max(off, start),
-          to = Math.min(endOff, end);
-
+        to = Math.min(endOff, end);
       if (from < to) {
         if (child.isText) {
           out += child.text.slice(Math.max(0, start - off), Math.min(child.text.length, end - off));
@@ -366,104 +303,76 @@ function getText(frag, start, end) {
           if (to == endOff) out += " ";
         }
       }
-
       off = endOff;
     }
   }
-
   convert(frag, start, end);
   return out;
 }
-
 var MAX_SIMPLIFY_DISTANCE = 30;
-
 function simplifyChanges(changes, doc) {
   var result = [];
-
   for (var i = 0; i < changes.length; i++) {
     var end = changes[i].toB,
-        start = i;
-
-    while (i < changes.length - 1 && changes[i + 1].fromB <= end + MAX_SIMPLIFY_DISTANCE) {
-      end = changes[++i].toB;
-    }
-
+      start = i;
+    while (i < changes.length - 1 && changes[i + 1].fromB <= end + MAX_SIMPLIFY_DISTANCE) end = changes[++i].toB;
     simplifyAdjacentChanges(changes, start, i + 1, doc, result);
   }
-
   return result;
 }
-
 function simplifyAdjacentChanges(changes, from, to, doc, target) {
   var start = Math.max(0, changes[from].fromB - MAX_SIMPLIFY_DISTANCE);
   var end = Math.min(doc.content.size, changes[to - 1].toB + MAX_SIMPLIFY_DISTANCE);
   var text = getText(doc.content, start, end);
-
   for (var i = from; i < to; i++) {
     var startI = i,
-        last = changes[i],
-        deleted = last.lenA,
-        inserted = last.lenB;
-
+      last = changes[i],
+      deleted = last.lenA,
+      inserted = last.lenB;
     while (i < to - 1) {
       var next = changes[i + 1],
-          boundary = false;
+        boundary = false;
       var prevLetter = last.toB == end ? false : isLetter(text.charCodeAt(last.toB - 1 - start));
-
       for (var pos = last.toB; !boundary && pos < next.fromB; pos++) {
         var nextLetter = pos == end ? false : isLetter(text.charCodeAt(pos - start));
         if ((!prevLetter || !nextLetter) && pos != changes[startI].fromB) boundary = true;
         prevLetter = nextLetter;
       }
-
       if (boundary) break;
       deleted += next.lenA;
       inserted += next.lenB;
       last = next;
       i++;
     }
-
     if (inserted > 0 && deleted > 0 && !(inserted == 1 && deleted == 1)) {
       var _from = changes[startI].fromB,
-          _to = changes[i].toB;
-      if (_from < end && isLetter(text.charCodeAt(_from - start))) while (_from > start && isLetter(text.charCodeAt(_from - 1 - start))) {
-        _from--;
-      }
-      if (_to > start && isLetter(text.charCodeAt(_to - 1 - start))) while (_to < end && isLetter(text.charCodeAt(_to - start))) {
-        _to++;
-      }
+        _to = changes[i].toB;
+      if (_from < end && isLetter(text.charCodeAt(_from - start))) while (_from > start && isLetter(text.charCodeAt(_from - 1 - start))) _from--;
+      if (_to > start && isLetter(text.charCodeAt(_to - 1 - start))) while (_to < end && isLetter(text.charCodeAt(_to - start))) _to++;
       var joined = fillChange(changes.slice(startI, i + 1), _from, _to);
-
       var _last = target.length ? target[target.length - 1] : null;
-
       if (_last && _last.toA == joined.fromA) target[target.length - 1] = new Change(_last.fromA, joined.toA, _last.fromB, joined.toB, _last.deleted.concat(joined.deleted), _last.inserted.concat(joined.inserted));else target.push(joined);
     } else {
-      for (var j = startI; j <= i; j++) {
-        target.push(changes[j]);
-      }
+      for (var j = startI; j <= i; j++) target.push(changes[j]);
     }
   }
-
   return changes;
 }
-
 function combine(a, b) {
   return a === b ? a : null;
 }
-
 function fillChange(changes, fromB, toB) {
   var fromA = changes[0].fromA - (changes[0].fromB - fromB);
   var last = changes[changes.length - 1];
   var toA = last.toA + (toB - last.toB);
   var deleted = Span.none,
-      inserted = Span.none;
+    inserted = Span.none;
   var delData = (changes[0].deleted.length ? changes[0].deleted : changes[0].inserted)[0].data;
   var insData = (changes[0].inserted.length ? changes[0].inserted : changes[0].deleted)[0].data;
-
   for (var posA = fromA, posB = fromB, i = 0;; i++) {
     var next = i == changes.length ? null : changes[i];
     var endA = next ? next.fromA : toA,
-        endB = next ? next.fromB : toB;
+      endB = next ? next.fromB : toB;
     if (endA > posA) deleted = Span.join(deleted, [new Span(endA - posA, delData)], combine);
     if (endB > posB) inserted = Span.join(inserted, [new Span(endB - posB, insData)], combine);
     if (!next) break;
@@ -474,26 +383,20 @@ function fillChange(changes, fromB, toB) {
     posA = next.toA;
     posB = next.toB;
   }
-
   return new Change(fromA, toA, fromB, toB, deleted, inserted);
 }
-
 var ChangeSet = function () {
   function ChangeSet(config, changes) {
     _classCallCheck(this, ChangeSet);
-
     this.config = config;
     this.changes = changes;
   }
-
   _createClass(ChangeSet, [{
     key: "addSteps",
     value: function addSteps(newDoc, maps, data) {
       var _this = this;
-
       var stepChanges = [];
-
-      var _loop = function _loop(i) {
+      var _loop2 = function _loop2() {
         var d = Array.isArray(data) ? data[i] : data;
         var off = 0;
         maps[i].forEach(function (fromA, toA, fromB, toB) {
@@ -501,54 +404,41 @@ var ChangeSet = function () {
           off = toB - fromB - (toA - fromA);
         });
       };
-
       for (var i = 0; i < maps.length; i++) {
-        _loop(i);
+        _loop2();
       }
-
       if (stepChanges.length == 0) return this;
       var newChanges = mergeAll(stepChanges, this.config.combine);
       var changes = Change.merge(this.changes, newChanges, this.config.combine);
       var updated = changes;
-
-      var _loop2 = function _loop2(_i3) {
-        var change = updated[_i3];
-
-        if (change.fromA == change.toA || change.fromB == change.toB || !newChanges.some(function (r) {
-          return r.toB > change.fromB && r.fromB < change.toB;
-        })) {
-          _i2 = _i3;
-          return "continue";
-        }
-
-        var diff = computeDiff(_this.config.doc.content, newDoc.content, change);
-
-        if (diff.length == 1 && diff[0].fromB == 0 && diff[0].toB == change.toB - change.fromB) {
+      var _loop3 = function _loop3(_i3) {
+          var change = updated[_i3];
+          if (change.fromA == change.toA || change.fromB == change.toB || !newChanges.some(function (r) {
+            return r.toB > change.fromB && r.fromB < change.toB;
+          })) {
+            _i2 = _i3;
+            return 0;
+          }
+          var diff = computeDiff(_this.config.doc.content, newDoc.content, change, _this.config.encoder);
+          if (diff.length == 1 && diff[0].fromB == 0 && diff[0].toB == change.toB - change.fromB) {
+            _i2 = _i3;
+            return 0;
+          }
+          if (updated == changes) updated = changes.slice();
+          if (diff.length == 1) {
+            updated[_i3] = diff[0];
+          } else {
+            var _updated;
+            (_updated = updated).splice.apply(_updated, [_i3, 1].concat(_toConsumableArray(diff)));
+            _i3 += diff.length - 1;
+          }
           _i2 = _i3;
-          return "continue";
-        }
-
-        if (updated == changes) updated = changes.slice();
-
-        if (diff.length == 1) {
-          updated[_i3] = diff[0];
-        } else {
-          var _updated;
-
-          (_updated = updated).splice.apply(_updated, [_i3, 1].concat(_toConsumableArray(diff)));
-
-          _i3 += diff.length - 1;
-        }
-
-        _i2 = _i3;
-      };
-
+        },
+        _ret2;
       for (var _i2 = 0; _i2 < updated.length; _i2++) {
-        var _ret2 = _loop2(_i2);
-
-        if (_ret2 === "continue") continue;
+        _ret2 = _loop3(_i2);
+        if (_ret2 === 0) continue;
       }
-
       return new ChangeSet(this.config, updated);
     }
   }, {
@@ -563,7 +453,6 @@ var ChangeSet = function () {
         var newData = f(span);
         return newData === span.data ? span : new Span(span.length, newData);
       };
-
       return new ChangeSet(this.config, this.changes.map(function (ch) {
         return new Change(ch.fromA, ch.toA, ch.fromB, ch.toB, ch.deleted.map(mapSpan), ch.inserted.map(mapSpan));
       }));
@@ -574,27 +463,21 @@ var ChangeSet = function () {
       if (b == this) return null;
       var touched = maps && touchedRange(maps);
       var moved = touched ? touched.toB - touched.fromB - (touched.toA - touched.fromA) : 0;
-
       function map(p) {
         return !touched || p <= touched.fromA ? p : p + moved;
       }
-
       var from = touched ? touched.fromB : 2e8,
-          to = touched ? touched.toB : -2e8;
-
+        to = touched ? touched.toB : -2e8;
       function add(start) {
         var end = arguments.length > 1 && arguments[1] !== undefined ? arguments[1] : start;
         from = Math.min(start, from);
         to = Math.max(end, to);
       }
-
       var rA = this.changes,
-          rB = b.changes;
-
+        rB = b.changes;
       for (var iA = 0, iB = 0; iA < rA.length && iB < rB.length;) {
         var rangeA = rA[iA],
-            rangeB = rB[iB];
-
+          rangeB = rB[iB];
         if (rangeA && rangeB && sameRanges(rangeA, rangeB, map)) {
           iA++;
           iB++;
@@ -606,7 +489,6 @@ var ChangeSet = function () {
           iA++;
         }
       }
-
       return from <= to ? {
         from: from,
         to: to
@@ -618,18 +500,17 @@ var ChangeSet = function () {
       var combine = arguments.length > 1 && arguments[1] !== undefined ? arguments[1] : function (a, b) {
         return a === b ? a : null;
       };
+      var tokenEncoder = arguments.length > 2 && arguments[2] !== undefined ? arguments[2] : DefaultEncoder;
       return new ChangeSet({
         combine: combine,
-        doc: doc
+        doc: doc,
+        encoder: tokenEncoder
       }, []);
     }
   }]);
-
   return ChangeSet;
 }();
-
 ChangeSet.computeDiff = computeDiff;
-
 function mergeAll(ranges, combine) {
   var start = arguments.length > 2 && arguments[2] !== undefined ? arguments[2] : 0;
   var end = arguments.length > 3 && arguments[3] !== undefined ? arguments[3] : ranges.length;
@@ -637,31 +518,25 @@ function mergeAll(ranges, combine) {
   var mid = start + end >> 1;
   return Change.merge(mergeAll(ranges, combine, start, mid), mergeAll(ranges, combine, mid, end), combine);
 }
-
 function endRange(maps) {
   var from = 2e8,
-      to = -2e8;
-
+    to = -2e8;
   for (var i = 0; i < maps.length; i++) {
     var map = maps[i];
-
     if (from != 2e8) {
       from = map.map(from, -1);
       to = map.map(to, 1);
     }
-
     map.forEach(function (_s, _e, start, end) {
       from = Math.min(from, start);
       to = Math.max(to, end);
     });
   }
-
   return from == 2e8 ? null : {
     from: from,
     to: to
   };
 }
-
 function touchedRange(maps) {
   var b = endRange(maps);
   if (!b) return null;
@@ -675,21 +550,14 @@ function touchedRange(maps) {
     toB: b.to
   };
 }
-
 function sameRanges(a, b, map) {
   return map(a.fromB) == b.fromB && map(a.toB) == b.toB && sameSpans(a.deleted, b.deleted) && sameSpans(a.inserted, b.inserted);
 }
-
 function sameSpans(a, b) {
   if (a.length != b.length) return false;
-
-  for (var i = 0; i < a.length; i++) {
-    if (a[i].length != b[i].length || a[i].data !== b[i].data) return false;
-  }
-
+  for (var i = 0; i < a.length; i++) if (a[i].length != b[i].length || a[i].data !== b[i].data) return false;
   return true;
 }
-
 exports.Change = Change;
 exports.ChangeSet = ChangeSet;
 exports.Span = Span;

package/dist/index.d.cts

@@ -1,4 +1,4 @@
-import { Node } from 'prosemirror-model';
+import { Mark, Node } from 'prosemirror-model';
 import { StepMap } from 'prosemirror-transform';
 
 /**
@@ -41,7 +41,7 @@ declare class Change<Data = any> {
     readonly deleted: readonly Span<Data>[];
     /**
     Data associated with the inserted content. Length adds up to
-    `this.toB - this.toA`.
+    `this.toB - this.fromB`.
     */
     readonly inserted: readonly Span<Data>[];
     /**
@@ -52,6 +52,38 @@ declare class Change<Data = any> {
     static merge<Data>(x: readonly Change<Data>[], y: readonly Change<Data>[], combine: (dataA: Data, dataB: Data) => Data): readonly Change<Data>[];
 }
 
+/**
+A token encoder can be passed when creating a `ChangeSet` in order
+to influence the way the library runs its diffing algorithm. The
+encoder determines how document tokens (such as nodes and
+characters) are encoded and compared.
+
+Note that both the encoding and the comparison may run a lot, and
+doing non-trivial work in these functions could impact
+performance.
+*/
+interface TokenEncoder<T> {
+    /**
+    Encode a given character, with the given marks applied.
+    */
+    encodeCharacter(char: number, marks: readonly Mark[]): T;
+    /**
+    Encode the start of a node or, if this is a leaf node, the
+    entire node.
+    */
+    encodeNodeStart(node: Node): T;
+    /**
+    Encode the end token for the given node. It is valid to encode
+    every end token in the same way.
+    */
+    encodeNodeEnd(node: Node): T;
+    /**
+    Compare the given tokens. Should return true when they count as
+    equal.
+    */
+    compareTokens(a: T, b: T): boolean;
+}
+
 /**
 Simplifies a set of changes for presentation. This makes the
 assumption that having both insertions and deletions within a word
@@ -108,11 +140,17 @@ declare class ChangeSet<Data = any> {
     } | null;
     /**
     Create a changeset with the given base object and configuration.
+    
     The `combine` function is used to compare and combine metadata—it
     should return null when metadata isn't compatible, and a combined
     version for a merged range when it is.
+    
+    When given, a token encoder determines how document tokens are
+    serialized and compared when diffing the content produced by
+    changes. The default is to just compare nodes by name and text
+    by character, ignoring marks and attributes.
     */
-    static create<Data = any>(doc: Node, combine?: (dataA: Data, dataB: Data) => Data): ChangeSet<Data>;
+    static create<Data = any>(doc: Node, combine?: (dataA: Data, dataB: Data) => Data, tokenEncoder?: TokenEncoder<any>): ChangeSet<Data>;
 }
 
-export { Change, ChangeSet, Span, simplifyChanges };
+export { Change, ChangeSet, Span, type TokenEncoder, simplifyChanges };

package/dist/index.d.ts

@@ -1,4 +1,4 @@
-import { Node } from 'prosemirror-model';
+import { Mark, Node } from 'prosemirror-model';
 import { StepMap } from 'prosemirror-transform';
 
 /**
@@ -41,7 +41,7 @@ declare class Change<Data = any> {
     readonly deleted: readonly Span<Data>[];
     /**
     Data associated with the inserted content. Length adds up to
-    `this.toB - this.toA`.
+    `this.toB - this.fromB`.
     */
     readonly inserted: readonly Span<Data>[];
     /**
@@ -52,6 +52,38 @@ declare class Change<Data = any> {
     static merge<Data>(x: readonly Change<Data>[], y: readonly Change<Data>[], combine: (dataA: Data, dataB: Data) => Data): readonly Change<Data>[];
 }
 
+/**
+A token encoder can be passed when creating a `ChangeSet` in order
+to influence the way the library runs its diffing algorithm. The
+encoder determines how document tokens (such as nodes and
+characters) are encoded and compared.
+
+Note that both the encoding and the comparison may run a lot, and
+doing non-trivial work in these functions could impact
+performance.
+*/
+interface TokenEncoder<T> {
+    /**
+    Encode a given character, with the given marks applied.
+    */
+    encodeCharacter(char: number, marks: readonly Mark[]): T;
+    /**
+    Encode the start of a node or, if this is a leaf node, the
+    entire node.
+    */
+    encodeNodeStart(node: Node): T;
+    /**
+    Encode the end token for the given node. It is valid to encode
+    every end token in the same way.
+    */
+    encodeNodeEnd(node: Node): T;
+    /**
+    Compare the given tokens. Should return true when they count as
+    equal.
+    */
+    compareTokens(a: T, b: T): boolean;
+}
+
 /**
 Simplifies a set of changes for presentation. This makes the
 assumption that having both insertions and deletions within a word
@@ -108,11 +140,17 @@ declare class ChangeSet<Data = any> {
     } | null;
     /**
     Create a changeset with the given base object and configuration.
+    
     The `combine` function is used to compare and combine metadata—it
     should return null when metadata isn't compatible, and a combined
     version for a merged range when it is.
+    
+    When given, a token encoder determines how document tokens are
+    serialized and compared when diffing the content produced by
+    changes. The default is to just compare nodes by name and text
+    by character, ignoring marks and attributes.
     */
-    static create<Data = any>(doc: Node, combine?: (dataA: Data, dataB: Data) => Data): ChangeSet<Data>;
+    static create<Data = any>(doc: Node, combine?: (dataA: Data, dataB: Data) => Data, tokenEncoder?: TokenEncoder<any>): ChangeSet<Data>;
 }
 
-export { Change, ChangeSet, Span, simplifyChanges };
+export { Change, ChangeSet, Span, type TokenEncoder, simplifyChanges };

package/dist/index.js

@@ -1,24 +1,30 @@
+const DefaultEncoder = {
+    encodeCharacter: char => char,
+    encodeNodeStart: node => node.type.name,
+    encodeNodeEnd: () => -1,
+    compareTokens: (a, b) => a === b
+};
 // Convert the given range of a fragment to tokens, where node open
 // tokens are encoded as strings holding the node name, characters as
 // their character code, and node close tokens as -1.
-function tokens(frag, start, end, target) {
+function tokens(frag, encoder, start, end, target) {
     for (let i = 0, off = 0; i < frag.childCount; i++) {
         let child = frag.child(i), endOff = off + child.nodeSize;
         let from = Math.max(off, start), to = Math.min(endOff, end);
         if (from < to) {
             if (child.isText) {
                 for (let j = from; j < to; j++)
-                    target.push(child.text.charCodeAt(j - off));
+                    target.push(encoder.encodeCharacter(child.text.charCodeAt(j - off), child.marks));
             }
             else if (child.isLeaf) {
-                target.push(child.type.name);
+                target.push(encoder.encodeNodeStart(child));
             }
             else {
                 if (from == off)
-                    target.push(child.type.name);
-                tokens(child.content, Math.max(off + 1, from) - off - 1, Math.min(endOff - 1, to) - off - 1, target);
+                    target.push(encoder.encodeNodeStart(child));
+                tokens(child.content, encoder, Math.max(off + 1, from) - off - 1, Math.min(endOff - 1, to) - off - 1, target);
                 if (to == endOff)
-                    target.push(-1);
+                    target.push(encoder.encodeNodeEnd(child));
             }
         }
         off = endOff;
@@ -37,16 +43,17 @@ const MAX_DIFF_SIZE = 5000;
 function minUnchanged(sizeA, sizeB) {
     return Math.min(15, Math.max(2, Math.floor(Math.max(sizeA, sizeB) / 10)));
 }
-function computeDiff(fragA, fragB, range) {
-    let tokA = tokens(fragA, range.fromA, range.toA, []);
-    let tokB = tokens(fragB, range.fromB, range.toB, []);
+function computeDiff(fragA, fragB, range, encoder = DefaultEncoder) {
+    let tokA = tokens(fragA, encoder, range.fromA, range.toA, []);
+    let tokB = tokens(fragB, encoder, range.fromB, range.toB, []);
     // Scan from both sides to cheaply eliminate work
     let start = 0, endA = tokA.length, endB = tokB.length;
-    while (start < tokA.length && start < tokB.length && tokA[start] === tokB[start])
+    let cmp = encoder.compareTokens;
+    while (start < tokA.length && start < tokB.length && cmp(tokA[start], tokB[start]))
         start++;
     if (start == tokA.length && start == tokB.length)
         return [];
-    while (endA > start && endB > start && tokA[endA - 1] === tokB[endB - 1])
+    while (endA > start && endB > start && cmp(tokA[endA - 1], tokB[endB - 1]))
         endA--, endB--;
     // If the result is simple _or_ too big to cheaply compute, return
     // the remaining region as the diff
@@ -65,7 +72,7 @@ function computeDiff(fragA, fragB, range) {
         for (let diag = -size; diag <= size; diag += 2) {
             let next = frontier[diag + 1 + max], prev = frontier[diag - 1 + max];
             let x = next < prev ? prev : next + 1, y = x + diag;
-            while (x < lenA && y < lenB && tokA[start + x] === tokB[start + y])
+            while (x < lenA && y < lenB && cmp(tokA[start + x], tokB[start + y]))
                 x++, y++;
             frontier[diag + max] = x;
             // Found a match
@@ -225,7 +232,7 @@ class Change {
     deleted, 
     /**
     Data associated with the inserted content. Length adds up to
-    `this.toB - this.toA`.
+    `this.toB - this.fromB`.
     */
     inserted) {
         this.fromA = fromA;
@@ -549,7 +556,7 @@ class ChangeSet {
                 // Only look at changes that touch newly added changed ranges
                 !newChanges.some(r => r.toB > change.fromB && r.fromB < change.toB))
                 continue;
-            let diff = computeDiff(this.config.doc.content, newDoc.content, change);
+            let diff = computeDiff(this.config.doc.content, newDoc.content, change, this.config.encoder);
             // Fast path: If they are completely different, don't do anything
             if (diff.length == 1 && diff[0].fromB == 0 && diff[0].toB == change.toB - change.fromB)
                 continue;
@@ -622,12 +629,18 @@ class ChangeSet {
     }
     /**
     Create a changeset with the given base object and configuration.
+    
     The `combine` function is used to compare and combine metadata—it
     should return null when metadata isn't compatible, and a combined
     version for a merged range when it is.
+    
+    When given, a token encoder determines how document tokens are
+    serialized and compared when diffing the content produced by
+    changes. The default is to just compare nodes by name and text
+    by character, ignoring marks and attributes.
     */
-    static create(doc, combine = (a, b) => a === b ? a : null) {
-        return new ChangeSet({ combine, doc }, []);
+    static create(doc, combine = (a, b) => a === b ? a : null, tokenEncoder = DefaultEncoder) {
+        return new ChangeSet({ combine, doc, encoder: tokenEncoder }, []);
     }
 }
 /**

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "prosemirror-changeset",
-  "version": "2.2.1",
+  "version": "2.3.0",
   "description": "Distills a series of editing steps into deleted and added ranges",
   "type": "module",
   "main": "dist/index.cjs",
@@ -29,11 +29,12 @@
   "devDependencies": {
     "@prosemirror/buildhelper": "^0.1.5",
     "prosemirror-model": "^1.0.0",
-    "prosemirror-test-builder": "^1.0.0"
+    "prosemirror-test-builder": "^1.0.0",
+    "builddocs": "^1.0.8"
   },
   "scripts": {
     "test": "pm-runtests",
     "prepare": "pm-buildhelper src/changeset.ts",
-    "build-readme": "FIXME"
+    "build-readme": "builddocs --format markdown --main src/README.md src/changeset.ts  > README.md"
   }
 }

package/src/README.md

@@ -25,4 +25,6 @@ it was made, or the step data necessary to invert it again.
 
 @ChangeSet
 
-@simplifyChanges
+@simplifyChanges
+
+@TokenEncoder

package/src/change.ts

@@ -66,7 +66,7 @@ export class Change<Data = any> {
     /// spans adds up to `this.toA - this.fromA`.
     readonly deleted: readonly Span<Data>[],
     /// Data associated with the inserted content. Length adds up to
-    /// `this.toB - this.toA`.
+    /// `this.toB - this.fromB`.
     readonly inserted: readonly Span<Data>[]
   ) {}
 

package/src/changeset.ts

@@ -1,9 +1,10 @@
 import {Node} from "prosemirror-model"
 import {StepMap} from "prosemirror-transform"
-import {computeDiff} from "./diff"
+import {computeDiff, TokenEncoder, DefaultEncoder} from "./diff"
 import {Change, Span} from "./change"
 export {Change, Span}
 export {simplifyChanges} from "./simplify"
+export {TokenEncoder}
 
 /// A change set tracks the changes to a document from a given point
 /// in the past. It condenses a number of step maps down to a flat
@@ -13,7 +14,11 @@ export class ChangeSet<Data = any> {
   /// @internal
   constructor(
     /// @internal
-    readonly config: {doc: Node, combine: (dataA: Data, dataB: Data) => Data},
+    readonly config: {
+      doc: Node,
+      combine: (dataA: Data, dataB: Data) => Data,
+      encoder: TokenEncoder<any>
+    },
     /// Replaced regions.
     readonly changes: readonly Change<Data>[]
   ) {}
@@ -69,7 +74,7 @@ export class ChangeSet<Data = any> {
       if (change.fromA == change.toA || change.fromB == change.toB ||
           // Only look at changes that touch newly added changed ranges
           !newChanges.some(r => r.toB > change.fromB && r.fromB < change.toB)) continue
-      let diff = computeDiff(this.config.doc.content, newDoc.content, change)
+      let diff = computeDiff(this.config.doc.content, newDoc.content, change, this.config.encoder)
 
       // Fast path: If they are completely different, don't do anything
       if (diff.length == 1 && diff[0].fromB == 0 && diff[0].toB == change.toB - change.fromB)
@@ -132,11 +137,21 @@ export class ChangeSet<Data = any> {
   }
 
   /// Create a changeset with the given base object and configuration.
+  ///
   /// The `combine` function is used to compare and combine metadata—it
   /// should return null when metadata isn't compatible, and a combined
   /// version for a merged range when it is.
-  static create<Data = any>(doc: Node, combine: (dataA: Data, dataB: Data) => Data = (a, b) => a === b ? a : null as any) {
-    return new ChangeSet({combine, doc}, [])
+  ///
+  /// When given, a token encoder determines how document tokens are
+  /// serialized and compared when diffing the content produced by
+  /// changes. The default is to just compare nodes by name and text
+  /// by character, ignoring marks and attributes.
+  static create<Data = any>(
+    doc: Node,
+    combine: (dataA: Data, dataB: Data) => Data = (a, b) => a === b ? a : null as any,
+    tokenEncoder: TokenEncoder<any> = DefaultEncoder
+  ) {
+    return new ChangeSet({combine, doc, encoder: tokenEncoder}, [])
   }
 
   /// Exported for testing @internal

package/src/diff.ts

@@ -1,22 +1,51 @@
-import {Fragment} from "prosemirror-model"
+import {Fragment, Node, Mark} from "prosemirror-model"
 import {Change} from "./change"
 
+/// A token encoder can be passed when creating a `ChangeSet` in order
+/// to influence the way the library runs its diffing algorithm. The
+/// encoder determines how document tokens (such as nodes and
+/// characters) are encoded and compared.
+///
+/// Note that both the encoding and the comparison may run a lot, and
+/// doing non-trivial work in these functions could impact
+/// performance.
+export interface TokenEncoder<T> {
+  /// Encode a given character, with the given marks applied.
+  encodeCharacter(char: number, marks: readonly Mark[]): T
+  /// Encode the start of a node or, if this is a leaf node, the
+  /// entire node.
+  encodeNodeStart(node: Node): T
+  /// Encode the end token for the given node. It is valid to encode
+  /// every end token in the same way.
+  encodeNodeEnd(node: Node): T
+  /// Compare the given tokens. Should return true when they count as
+  /// equal.
+  compareTokens(a: T, b: T): boolean
+}
+
+export const DefaultEncoder: TokenEncoder<number | string> = {
+  encodeCharacter: char => char,
+  encodeNodeStart: node => node.type.name,
+  encodeNodeEnd: () => -1,
+  compareTokens: (a, b) => a === b
+}
+
 // Convert the given range of a fragment to tokens, where node open
 // tokens are encoded as strings holding the node name, characters as
 // their character code, and node close tokens as -1.
-function tokens(frag: Fragment, start: number, end: number, target: (number | string)[]) {
+function tokens<T>(frag: Fragment, encoder: TokenEncoder<T>, start: number, end: number, target: T[]) {
   for (let i = 0, off = 0; i < frag.childCount; i++) {
     let child = frag.child(i), endOff = off + child.nodeSize
     let from = Math.max(off, start), to = Math.min(endOff, end)
     if (from < to) {
       if (child.isText) {
-        for (let j = from; j < to; j++) target.push(child.text!.charCodeAt(j - off))
+        for (let j = from; j < to; j++) target.push(encoder.encodeCharacter(child.text!.charCodeAt(j - off), child.marks))
       } else if (child.isLeaf) {
-        target.push(child.type.name)
+        target.push(encoder.encodeNodeStart(child))
       } else {
-        if (from == off) target.push(child.type.name)
-        tokens(child.content, Math.max(off + 1, from) - off - 1, Math.min(endOff - 1, to) - off - 1, target)
-        if (to == endOff) target.push(-1)
+        if (from == off) target.push(encoder.encodeNodeStart(child))
+        tokens(child.content, encoder, Math.max(off + 1, from) - off - 1, Math.min(endOff - 1, to) - off - 1, target)
+        if (to == endOff) target.push(encoder.encodeNodeEnd(child))
       }
     }
     off = endOff
@@ -38,15 +67,16 @@ function minUnchanged(sizeA: number, sizeB: number) {
   return Math.min(15, Math.max(2, Math.floor(Math.max(sizeA, sizeB) / 10)))
 }
 
-export function computeDiff(fragA: Fragment, fragB: Fragment, range: Change) {
-  let tokA = tokens(fragA, range.fromA, range.toA, [])
-  let tokB = tokens(fragB, range.fromB, range.toB, [])
+export function computeDiff(fragA: Fragment, fragB: Fragment, range: Change, encoder: TokenEncoder<any> = DefaultEncoder) {
+  let tokA = tokens(fragA, encoder, range.fromA, range.toA, [])
+  let tokB = tokens(fragB, encoder, range.fromB, range.toB, [])
 
   // Scan from both sides to cheaply eliminate work
   let start = 0, endA = tokA.length, endB = tokB.length
-  while (start < tokA.length && start < tokB.length && tokA[start] === tokB[start]) start++
+  let cmp = encoder.compareTokens
+  while (start < tokA.length && start < tokB.length && cmp(tokA[start], tokB[start])) start++
   if (start == tokA.length && start == tokB.length) return []
-  while (endA > start && endB > start && tokA[endA - 1] === tokB[endB - 1]) endA--, endB--
+  while (endA > start && endB > start && cmp(tokA[endA - 1], tokB[endB - 1])) endA--, endB--
   // If the result is simple _or_ too big to cheaply compute, return
   // the remaining region as the diff
   if (endA == start || endB == start || (endA == endB && endA == start + 1))
@@ -66,7 +96,7 @@ export function computeDiff(fragA: Fragment, fragB: Fragment, range: Change) {
     for (let diag = -size; diag <= size; diag += 2) {
       let next = frontier[diag + 1 + max], prev = frontier[diag - 1 + max]
       let x = next < prev ? prev : next + 1, y = x + diag
-      while (x < lenA && y < lenB && tokA[start + x] === tokB[start + y]) x++, y++
+      while (x < lenA && y < lenB && cmp(tokA[start + x], tokB[start + y])) x++, y++
       frontier[diag + max] = x
       // Found a match
       if (x >= lenA && y >= lenB) {