sharedb 4.0.2 → 4.1.1

package/lib/agent.js

@@ -453,6 +453,8 @@ Agent.prototype._handleMessage = function(request, callback) {
         return this._subscribePresence(request.ch, request.seq, callback);
       case ACTIONS.presenceUnsubscribe:
         return this._unsubscribePresence(request.ch, request.seq, callback);
+      case ACTIONS.presenceRequest:
+        return this._requestPresence(request.ch, callback);
       case ACTIONS.pingPong:
         return this._pingPong(callback);
       default:
@@ -776,6 +778,7 @@ Agent.prototype._broadcastPresence = function(presence, callback) {
     presence: presence,
     collection: presence.c
   };
+  var start = Date.now();
   backend.trigger(backend.MIDDLEWARE_ACTIONS.receivePresence, this, context, function(error) {
     if (error) return callback(error);
     backend.transformPresenceToLatestVersion(agent, presence, function(error, presence) {
@@ -783,6 +786,7 @@ Agent.prototype._broadcastPresence = function(presence, callback) {
       var channel = agent._getPresenceChannel(presence.ch);
       agent.backend.pubsub.publish([channel], presence, function(error) {
         if (error) return callback(error);
+        backend.emit('timing', 'presence.broadcast', Date.now() - start, context);
         callback(null, presence);
       });
     });
@@ -805,29 +809,40 @@ Agent.prototype._createPresence = function(request) {
   };
 };
 
-Agent.prototype._subscribePresence = function(channel, seq, callback) {
+Agent.prototype._subscribePresence = function(channel, seq, cb) {
   var agent = this;
+
+  function callback(error) {
+    cb(error, {ch: channel, seq: seq});
+  }
+
+  var existingStream = agent.subscribedPresences[channel];
+  if (existingStream) {
+    agent.presenceSubscriptionSeq[channel] = seq;
+    return callback();
+  }
+
   var presenceChannel = this._getPresenceChannel(channel);
   this.backend.pubsub.subscribe(presenceChannel, function(error, stream) {
     if (error) return callback(error);
     if (seq < agent.presenceSubscriptionSeq[channel]) {
       stream.destroy();
-      return callback(null, {ch: channel, seq: seq});
+      return callback();
     }
     agent.presenceSubscriptionSeq[channel] = seq;
     agent.subscribedPresences[channel] = stream;
     agent._subscribeToPresenceStream(channel, stream);
     agent._requestPresence(channel, function(error) {
-      callback(error, {ch: channel, seq: seq});
+      callback(error);
     });
   });
 };
 
 Agent.prototype._unsubscribePresence = function(channel, seq, callback) {
-  if (seq < this.presenceSubscriptionSeq[channel]) return;
   this.presenceSubscriptionSeq[channel] = seq;
   var stream = this.subscribedPresences[channel];
   if (stream) stream.destroy();
+  delete this.subscribedPresences[channel];
   callback(null, {ch: channel, seq: seq});
 };
 

package/lib/client/connection.js

@@ -809,6 +809,10 @@ Connection.prototype._addPresence = function(presence) {
   });
 };
 
+Connection.prototype._requestRemotePresence = function(channel) {
+  this.send({a: ACTIONS.presenceRequest, ch: channel});
+};
+
 Connection.prototype._handlePresenceSubscribe = function(error, message) {
   var presence = util.dig(this._presences, message.ch);
   if (presence) presence._handleSubscribe(error, message.seq);

package/lib/client/doc.js

@@ -329,10 +329,6 @@ Doc.prototype._handleSubscribe = function(error, snapshot) {
 Doc.prototype._handleOp = function(err, message) {
   if (err) {
     if (this.inflightOp) {
-      // The server has rejected submission of the current operation. If we get
-      // an "Op submit rejected" error, this was done intentionally
-      // and we should roll back but not return an error to the user.
-      if (err.code === ERROR_CODE.ERR_OP_SUBMIT_REJECTED) err = null;
       return this._rollback(err);
     }
     return this.emit('error', err);
@@ -984,38 +980,44 @@ Doc.prototype._rollback = function(err) {
   // working state, then call back
   var op = this.inflightOp;
 
-  if ('op' in op && op.type.invert) {
-    try {
-      op.op = op.type.invert(op.op);
-    } catch (error) {
-      // If the op doesn't support `.invert()`, we just reload the doc
-      // instead of trying to locally revert it.
-      return this._hardRollback(err);
-    }
+  if (!('op' in op && op.type.invert)) {
+    return this._hardRollback(err);
+  }
 
-    // Transform the undo operation by any pending ops.
-    for (var i = 0; i < this.pendingOps.length; i++) {
-      var transformErr = transformX(this.pendingOps[i], op);
-      if (transformErr) return this._hardRollback(transformErr);
-    }
+  try {
+    op.op = op.type.invert(op.op);
+  } catch (error) {
+    // If the op doesn't support `.invert()`, we just reload the doc
+    // instead of trying to locally revert it.
+    return this._hardRollback(err);
+  }
 
-    // ... and apply it locally, reverting the changes.
-    //
-    // This operation is applied to look like it comes from a remote source.
-    // I'm still not 100% sure about this functionality, because its really a
-    // local op. Basically, the problem is that if the client's op is rejected
-    // by the server, the editor window should update to reflect the undo.
-    try {
-      this._otApply(op, false);
-    } catch (error) {
-      return this._hardRollback(error);
-    }
+  // Transform the undo operation by any pending ops.
+  for (var i = 0; i < this.pendingOps.length; i++) {
+    var transformErr = transformX(this.pendingOps[i], op);
+    if (transformErr) return this._hardRollback(transformErr);
+  }
 
-    this._clearInflightOp(err);
-    return;
+  // ... and apply it locally, reverting the changes.
+  //
+  // This operation is applied to look like it comes from a remote source.
+  // I'm still not 100% sure about this functionality, because its really a
+  // local op. Basically, the problem is that if the client's op is rejected
+  // by the server, the editor window should update to reflect the undo.
+  try {
+    this._otApply(op, false);
+  } catch (error) {
+    return this._hardRollback(error);
   }
 
-  this._hardRollback(err);
+  // The server has rejected submission of the current operation. If we get
+  // an "Op submit rejected" error, this was done intentionally
+  // and we should roll back but not return an error to the user.
+  if (err.code === ERROR_CODE.ERR_OP_SUBMIT_REJECTED) {
+    return this._clearInflightOp(null);
+  }
+
+  this._clearInflightOp(err);
 };
 
 Doc.prototype._hardRollback = function(err) {
@@ -1024,9 +1026,8 @@ Doc.prototype._hardRollback = function(err) {
   // to hit a condition where we have no inflight op, but we do have pending
   // ops. This can happen when an invalid op is submitted, which causes us
   // to hard rollback before the pending op was flushed.
-  var pendingOps = [];
-  if (this.inflightOp) pendingOps.push(this.inflightOp);
-  pendingOps = pendingOps.concat(this.pendingOps);
+  var pendingOps = this.pendingOps;
+  var inflightOp = this.inflightOp;
 
   // Cancel all pending ops and reset if we can't invert
   this._setType(null);
@@ -1036,17 +1037,50 @@ Doc.prototype._hardRollback = function(err) {
 
   // Fetch the latest version from the server to get us back into a working state
   var doc = this;
-  this._fetch({force: true}, function() {
+  this._fetch({force: true}, function(fetchError) {
     // We want to check that no errors are swallowed, so we check that:
     // - there are callbacks to call, and
     // - that every single pending op called a callback
     // If there are no ops queued, or one of them didn't handle the error,
     // then we emit the error.
+
+    if (fetchError) {
+      // This is critical error as it means that our doc is not in usable state
+      // anymore, we should throw doc error.
+      logger.error('Hard rollback doc fetch failed.', fetchError, inflightOp);
+
+      doc.emit('error', new ShareDBError(
+        ERROR_CODE.ERR_HARD_ROLLBACK_FETCH_FAILED,
+        'Hard rollback fetch failed: ' + fetchError.message
+      ));
+    }
+
+    if (err.code === ERROR_CODE.ERR_OP_SUBMIT_REJECTED) {
+      /**
+       * Handle special case of ERR_OP_SUBMIT_REJECTED
+       * This ensures that we resolve the main op callback and reject
+       * all the pending ops. This is hard rollback so all the pending ops will be
+       * discarded. This will ensure that the user is at least informed about it.
+       * more info: https://github.com/share/sharedb/pull/626
+       */
+      if (inflightOp) {
+        util.callEach(inflightOp.callbacks);
+        inflightOp = null;
+      }
+
+      if (!pendingOps.length) return;
+      err = new ShareDBError(
+        ERROR_CODE.ERR_PENDING_OP_REMOVED_BY_OP_SUBMIT_REJECTED,
+        'Discarding pending op because of hard rollback during ERR_OP_SUBMIT_REJECTED'
+      );
+    }
+
+    if (inflightOp) pendingOps.unshift(inflightOp);
     var allOpsHadCallbacks = !!pendingOps.length;
     for (var i = 0; i < pendingOps.length; i++) {
       allOpsHadCallbacks = util.callEach(pendingOps[i].callbacks, err) && allOpsHadCallbacks;
     }
-    if (err && !allOpsHadCallbacks) return doc.emit('error', err);
+    if (err && !allOpsHadCallbacks) doc.emit('error', err);
   });
 };
 
@@ -1062,3 +1096,5 @@ Doc.prototype._clearInflightOp = function(err) {
 
   if (err && !called) return this.emit('error', err);
 };
+
+

package/lib/client/presence/presence.js

@@ -84,7 +84,15 @@ Presence.prototype.destroy = function(callback) {
 };
 
 Presence.prototype._sendSubscriptionAction = function(wantSubscribe, callback) {
-  this.wantSubscribe = !!wantSubscribe;
+  wantSubscribe = !!wantSubscribe;
+  if (wantSubscribe === this.wantSubscribe) {
+    if (!callback) return;
+    if (wantSubscribe === this.subscribed) return util.nextTick(callback);
+    if (Object.keys(this._subscriptionCallbacksBySeq).length) {
+      return this._combineSubscribeCallbackWithLastAdded(callback);
+    }
+  }
+  this.wantSubscribe = wantSubscribe;
   var action = this.wantSubscribe ? ACTIONS.presenceSubscribe : ACTIONS.presenceUnsubscribe;
   var seq = this.connection._presenceSeq++;
   this._subscriptionCallbacksBySeq[seq] = callback;
@@ -93,6 +101,10 @@ Presence.prototype._sendSubscriptionAction = function(wantSubscribe, callback) {
   }
 };
 
+Presence.prototype._requestRemotePresence = function() {
+  this.connection._requestRemotePresence(this.channel);
+};
+
 Presence.prototype._handleSubscribe = function(error, seq) {
   if (this.wantSubscribe) this.subscribed = true;
   var callback = this._subscriptionCallback(seq);
@@ -139,7 +151,10 @@ Presence.prototype._removeRemotePresence = function(id) {
 };
 
 Presence.prototype._onConnectionStateChanged = function() {
-  if (!this.connection.canSend) return;
+  if (!this.connection.canSend) {
+    this.subscribed = false;
+    return;
+  }
   this._resubscribe();
   for (var id in this.localPresences) {
     this.localPresences[id]._sendPending();
@@ -184,3 +199,14 @@ Presence.prototype._callEachOrEmit = function(callbacks, error) {
   var called = util.callEach(callbacks, error);
   if (!called && error) this.emit('error', error);
 };
+
+Presence.prototype._combineSubscribeCallbackWithLastAdded = function(callback) {
+  var seqs = Object.keys(this._subscriptionCallbacksBySeq);
+  var lastSeq = seqs[seqs.length - 1];
+  var originalCallback = this._subscriptionCallbacksBySeq[lastSeq];
+  if (!originalCallback) return this._subscriptionCallbacksBySeq[lastSeq] = callback;
+  this._subscriptionCallbacksBySeq[lastSeq] = function(error) {
+    originalCallback(error);
+    callback(error);
+  };
+};

package/lib/client/presence/remote-doc-presence.js

@@ -105,9 +105,7 @@ RemoteDocPresence.prototype._catchUpStalePresence = function() {
   if (!this._opCache) {
     this._startCachingOps();
     this._doc.fetch();
-    // We're already subscribed, but we send another subscribe message
-    // to force presence updates from other clients
-    this.presence.subscribe();
+    this.presence._requestRemotePresence();
     return false;
   }
 

package/lib/error.js

@@ -38,6 +38,8 @@ ShareDBError.CODES = {
   ERR_OP_ALREADY_SUBMITTED: 'ERR_OP_ALREADY_SUBMITTED',
   ERR_OP_NOT_ALLOWED_IN_PROJECTION: 'ERR_OP_NOT_ALLOWED_IN_PROJECTION',
   ERR_OP_SUBMIT_REJECTED: 'ERR_OP_SUBMIT_REJECTED',
+  ERR_PENDING_OP_REMOVED_BY_OP_SUBMIT_REJECTED: 'ERR_PENDING_OP_REMOVED_BY_OP_SUBMIT_REJECTED',
+  ERR_HARD_ROLLBACK_FETCH_FAILED: 'ERR_HARD_ROLLBACK_FETCH_FAILED',
   ERR_OP_VERSION_MISMATCH_AFTER_TRANSFORM: 'ERR_OP_VERSION_MISMATCH_AFTER_TRANSFORM',
   ERR_OP_VERSION_MISMATCH_DURING_TRANSFORM: 'ERR_OP_VERSION_MISMATCH_DURING_TRANSFORM',
   ERR_OP_VERSION_NEWER_THAN_CURRENT_SNAPSHOT: 'ERR_OP_VERSION_NEWER_THAN_CURRENT_SNAPSHOT',

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "sharedb",
-  "version": "4.0.2",
+  "version": "4.1.1",
   "description": "JSON OT database backend",
   "main": "lib/index.js",
   "dependencies": {

package/test/client/doc.js

@@ -6,6 +6,7 @@ var richText = require('rich-text').type;
 var ShareDBError = require('../../lib/error');
 var errorHandler = require('../util').errorHandler;
 var sinon = require('sinon');
+var types = require('../../lib/types');
 
 describe('Doc', function() {
   beforeEach(function() {
@@ -514,6 +515,38 @@ describe('Doc', function() {
       ], done);
     });
 
+    it('throws an error when hard rollback fetch failed', function(done) {
+      var backend = this.backend;
+      doc = this.connection.get('dogs', 'scrappy');
+      types.register(richText);
+
+      async.series([
+        doc.create.bind(doc, {ops: [{insert: 'Scrappy'}]}, 'rich-text'),
+        function(next) {
+          backend.use('reply', function(replyContext, cb) {
+            if (replyContext.request.a !== 'f') return cb();
+            cb({code: 'FETCH_ERROR'});
+          });
+          backend.use('submit', function(_context, cb) {
+            cb(new ShareDBError('SUBMIT_ERROR'));
+          });
+          var nonInvertibleOp = [{insert: 'e'}];
+
+          var count = 0;
+          function expectError(code) {
+            count++;
+            return function(error) {
+              expect(error.code).to.equal(code);
+              count--;
+              if (!count) next();
+            };
+          }
+
+          doc.on('error', expectError('ERR_HARD_ROLLBACK_FETCH_FAILED'));
+          doc.submitOp(nonInvertibleOp, expectError('SUBMIT_ERROR'));
+        }
+      ], done);
+    });
 
     it('rescues an irreversible op collision', function(done) {
       // This test case attempts to reconstruct the following corner case, with

package/test/client/presence/doc-presence.js

@@ -645,20 +645,14 @@ describe('DocPresence', function() {
       presence1.subscribe.bind(presence1),
       presence2.subscribe.bind(presence2),
       function(next) {
-        connection1._setState('disconnected');
-        next();
-      },
-      function(next) {
-        localPresence2.submit({index: 0}, errorHandler(done));
-        // We've not _actually_ disconnected the connection, so this
-        // event will still fire.
-        presence1.once('receive', function() {
+        connection1.once('closed', function() {
           next();
         });
+        connection1.close();
       },
+      localPresence2.submit.bind(localPresence2, {index: 0}),
       function(next) {
-        connection1._setState('connecting');
-        connection1._setState('connected');
+        backend.connect(connection1);
         presence1.once('receive', function(id, presence) {
           expect(presence).to.eql({index: 0});
           next();

package/test/client/presence/presence.js

@@ -344,9 +344,11 @@ describe('Presence', function() {
       presence1.subscribe.bind(presence1),
       presence2.subscribe.bind(presence2),
       function(next) {
-        connection2._setState('disconnected');
+        connection2.once('closed', function() {
+          next();
+        });
+        connection2.close();
         expect(connection2.canSend).to.be.false;
-        next();
       },
       localPresence1.submit.bind(localPresence1, {index: 1}),
       function(next) {
@@ -355,8 +357,7 @@ describe('Presence', function() {
           expect(presence).to.eql({index: 1});
           next();
         });
-        connection2._setState('connecting');
-        connection2._setState('connected');
+        backend.connect(connection2);
       }
     ], done);
   });
@@ -401,6 +402,64 @@ describe('Presence', function() {
     ], done);
   });
 
+  it('does not send another subscribe request if already subscribed', function(done) {
+    var sendPresenceAction = sinon.spy(connection1, '_sendPresenceAction');
+    async.series([
+      presence1.subscribe.bind(presence1),
+      presence1.subscribe.bind(presence1),
+      function(next) {
+        expect(sendPresenceAction).to.have.been.calledOnce;
+        next();
+      }
+    ], done);
+  });
+
+  it('only subscribes once when calling multiple in parallel', function(done) {
+    var sendPresenceAction = sinon.spy(connection1, '_sendPresenceAction');
+    async.series([
+      function(next) {
+        async.parallel([
+          presence1.subscribe.bind(presence1),
+          presence1.subscribe.bind(presence1)
+        ], next);
+      },
+      function(next) {
+        expect(sendPresenceAction).to.have.been.calledOnce;
+        next();
+      }
+    ], done);
+  });
+
+  it('subscribes once when calling again after no callback', function(done) {
+    var sendPresenceAction = sinon.spy(connection1, '_sendPresenceAction');
+    presence1.subscribe(); // no callback
+    async.series([
+      presence1.subscribe.bind(presence1),
+      function(next) {
+        expect(sendPresenceAction).to.have.been.calledOnce;
+        next();
+      }
+    ], done);
+  });
+
+  it('does not leak Streams when subscribing the same presence multiple times', function(done) {
+    var streamsCount;
+    async.series([
+      presence1.subscribe.bind(presence1),
+      function(next) {
+        streamsCount = backend.pubsub.streamsCount;
+        // Trick it into sending a duplicate request
+        presence1.wantSubscribe = false;
+        next();
+      },
+      presence1.subscribe.bind(presence1),
+      function(next) {
+        expect(backend.pubsub.streamsCount).to.equal(streamsCount);
+        next();
+      }
+    ], done);
+  });
+
   it('throws an error when trying to create a presence with a non-string ID', function() {
     expect(function() {
       presence1.create(123);

package/test/client/submit.js

@@ -5,9 +5,11 @@ var types = require('../../lib/types');
 var deserializedType = require('./deserialized-type');
 var numberType = require('./number-type');
 var errorHandler = require('../util').errorHandler;
+var richText = require('rich-text');
 types.register(deserializedType.type);
 types.register(deserializedType.type2);
 types.register(numberType.type);
+types.register(richText.type);
 
 module.exports = function() {
   describe('client submit', function() {
@@ -919,6 +921,27 @@ module.exports = function() {
       ], done);
     });
 
+
+    it('request.rejectedError() soft rejects main op and throws for pending ops on hard rollback', function(done) {
+      this.backend.use('submit', function(request, next) {
+        if (request.op.create) {
+          next(request.rejectedError());
+        }
+      });
+
+      var connection = this.backend.connect();
+      var doc = connection.get('dogs', 'fido');
+      doc.preventCompose = true;
+
+      doc.create({age: 3}, function(error) {
+        if (error) done(error);
+      });
+      doc.submitOp({p: ['age'], na: 1}, function(err) {
+        expect(err.code).to.be.equal('ERR_PENDING_OP_REMOVED_BY_OP_SUBMIT_REJECTED');
+        done();
+      });
+    });
+
     it('request.rejectedError() soft rejects a create', function(done) {
       this.backend.use('submit', function(request, next) {
         next(request.rejectedError());
@@ -949,6 +972,29 @@ module.exports = function() {
       });
     });
 
+    it(
+      'request.rejectedError() soft rejects main op and throws for pending ops on hard rollback without callback',
+      function(done) {
+        this.backend.use('submit', function(request, next) {
+          if (request.op.create) {
+            next(request.rejectedError());
+          }
+        });
+
+        var connection = this.backend.connect();
+        var doc = connection.get('dogs', 'fido');
+        doc.preventCompose = true;
+
+        doc.create({age: 3});
+        doc.submitOp({p: ['age'], na: 1});
+
+        doc.on('error', function(err) {
+          expect(err.code).to.be.equal('ERR_PENDING_OP_REMOVED_BY_OP_SUBMIT_REJECTED');
+          done();
+        });
+      }
+    );
+
     it('passing an error in submit middleware rejects an op and calls back with the erorr', function(done) {
       this.backend.use('submit', function(request, next) {
         if ('op' in request.op) return next({message: 'Custom error'});
@@ -1043,6 +1089,65 @@ module.exports = function() {
       });
     });
 
+    it('request.rejectedError() soft rejects main op and pending ops for invertible type', function(done) {
+      var rejectedOnce = false;
+      this.backend.use('submit', function(request, next) {
+        if ('op' in request.op && !rejectedOnce) {
+          rejectedOnce = true;
+          return next(request.rejectedError());
+        }
+        next();
+      });
+      var doc = this.backend.connect().get('dogs', 'fido');
+      doc.preventCompose = true;
+
+      doc.create({age: 3}, function(err) {
+        if (err) return done(err);
+        doc.submitOp({p: ['age'], na: 1}, function(err) {
+          if (err) return done(err);
+        });
+        doc.submitOp({p: ['age'], na: 3}, function(err) {
+          if (err) return done(err);
+          expect(doc.version).equal(2);
+          expect(doc.data).eql({age: 6});
+          done();
+        });
+        expect(doc.version).equal(1);
+        expect(doc.data).eql({age: 7});
+      });
+    });
+
+    it(
+      'request.rejectedError() soft rejects main op and throws for pending ops for non invertible type',
+      function(done) {
+        var rejectedOnce = false;
+        this.backend.use('submit', function(request, next) {
+          if ('op' in request.op && !rejectedOnce) {
+            rejectedOnce = true;
+            return next(request.rejectedError());
+          }
+          next();
+        });
+        var doc = this.backend.connect().get('dogs', 'fido');
+        doc.preventCompose = true;
+
+        doc.create({ops: [{insert: 'Scrappy'}]}, 'rich-text', function(err) {
+          if (err) return done(err);
+
+          var nonInvertibleOp = [{insert: 'a'}];
+          doc.submitOp(nonInvertibleOp, function(err) {
+            if (err) return done(err);
+          });
+          doc.submitOp([{insert: 'b'}], function(err) {
+            expect(err.code).to.be.equal('ERR_PENDING_OP_REMOVED_BY_OP_SUBMIT_REJECTED');
+            done();
+          });
+          expect(doc.version).equal(1);
+          expect(doc.data.ops).eql([{insert: 'baScrappy'}]);
+        });
+      }
+    );
+
     it('request.rejectedError() soft rejects an op without callback', function(done) {
       this.backend.use('submit', function(request, next) {
         if ('op' in request.op) return next(request.rejectedError());

package/CHANGELOG.md

@@ -1,44 +0,0 @@
-## v2.0
-
-### Breaking changes
-
-* Drop Node.js v10 support
-
-## v1.0-beta
-
-### Breaking changes
-
-* Add options argument to all public database adapter methods that read
-  or write from snapshots or ops.
-
-* DB methods that get snapshots or ops no longer return metadata unless
-  `{metadata: true}` option is passed.
-
-* Replace `source` argument with `options` in doc methods. Use `options.source`
-  instead.
-
-* Backend streams now write objects intead of strings.
-
-* MemoryDB.prototype._querySync now returns `{snapshots: ..., extra: ...}`
-  instead of just an array of snapshots.
-
-### Non-breaking changes
-
-* Add options argument to backend.submit.
-
-* Add error codes to all errors.
-
-* Add `'updated'` event on queries which fires on all query result changes.
-
-* In clients, wrap errors in Error objects to they get passed through event
-  emitters.
-
-* Sanitize stack traces when sending errors to client, but log them on the
-  server.
-
-## v0.11.37
-
-Beginning of changelog.
-
-If you're upgrading from ShareJS 0.7 or earlier,
-take a look at the [ShareJS upgrade guide](docs/upgrading-from-sharejs.md).