sharedb 5.0.3 → 5.1.0

package/lib/agent.js

@@ -4,6 +4,7 @@ var logger = require('./logger');
 var ACTIONS = require('./message-actions').ACTIONS;
 var types = require('./types');
 var util = require('./util');
+var protocol = require('./protocol');
 
 var ERROR_CODE = ShareDBError.CODES;
 
@@ -48,6 +49,10 @@ function Agent(backend, stream) {
   // request if the client disconnects ungracefully. This is a
   // map of channel -> id -> request
   this.presenceRequests = Object.create(null);
+  // Keep track of the latest known Doc version, so that we can avoid fetching
+  // ops to transform presence if not needed
+  this.latestDocVersionStreams = Object.create(null);
+  this.latestDocVersions = Object.create(null);
 
   // We need to track this manually to make sure we don't reply to messages
   // after the stream was closed.
@@ -58,6 +63,8 @@ function Agent(backend, stream) {
   // active, and it is passed to each middleware call
   this.custom = Object.create(null);
 
+  this.protocol = Object.create(null);
+
   // The first message received over the connection. Stored to warn if messages
   // are being sent before the handshake.
   this._firstReceivedMessage = null;
@@ -108,6 +115,12 @@ Agent.prototype._cleanup = function() {
     emitter.destroy();
   }
   this.subscribedQueries = Object.create(null);
+
+  for (var collection in this.latestDocVersionStreams) {
+    var streams = this.latestDocVersionStreams[collection];
+    for (var id in streams) streams[id].destroy();
+  }
+  this.latestDocVersionStreams = Object.create(null);
 };
 
 /**
@@ -115,17 +128,8 @@ Agent.prototype._cleanup = function() {
  * _sendOp()
  */
 Agent.prototype._subscribeToStream = function(collection, id, stream) {
-  if (this.closed) return stream.destroy();
-
-  var streams = this.subscribedDocs[collection] || (this.subscribedDocs[collection] = Object.create(null));
-
-  // If already subscribed to this document, destroy the previously subscribed stream
-  var previous = streams[id];
-  if (previous) previous.destroy();
-  streams[id] = stream;
-
   var agent = this;
-  stream.on('data', function(data) {
+  this._subscribeMapToStream(this.subscribedDocs, collection, id, stream, function(data) {
     if (data.error) {
       // Log then silently ignore errors in a subscription stream, since these
       // may not be the client's fault, and they were not the result of a
@@ -135,13 +139,26 @@ Agent.prototype._subscribeToStream = function(collection, id, stream) {
     }
     agent._onOp(collection, id, data);
   });
+};
+
+Agent.prototype._subscribeMapToStream = function(map, collection, id, stream, dataHandler) {
+  if (this.closed) return stream.destroy();
+
+  var streams = map[collection] || (map[collection] = Object.create(null));
+
+  // If already subscribed to this document, destroy the previously subscribed stream
+  var previous = streams[id];
+  if (previous) previous.destroy();
+  streams[id] = stream;
+
+  stream.on('data', dataHandler);
   stream.on('end', function() {
     // The op stream is done sending, so release its reference
-    var streams = agent.subscribedDocs[collection];
+    var streams = map[collection];
     if (!streams || streams[id] !== stream) return;
     delete streams[id];
     if (util.hasKeys(streams)) return;
-    delete agent.subscribedDocs[collection];
+    delete map[collection];
   });
 };
 
@@ -423,6 +440,7 @@ Agent.prototype._handleMessage = function(request, callback) {
     switch (request.a) {
       case ACTIONS.handshake:
         if (request.id) this.src = request.id;
+        this._setProtocol(request);
         return callback(null, this._initMessage(ACTIONS.handshake));
       case ACTIONS.queryFetch:
         return this._queryFetch(request.id, request.c, request.q, getQueryOptions(request), callback);
@@ -774,8 +792,8 @@ Agent.prototype._fetchSnapshotByTimestamp = function(collection, id, timestamp,
 Agent.prototype._initMessage = function(action) {
   return {
     a: action,
-    protocol: 1,
-    protocolMinor: 1,
+    protocol: protocol.major,
+    protocolMinor: protocol.minor,
     id: this._src(),
     type: types.defaultType.uri
   };
@@ -794,25 +812,74 @@ Agent.prototype._broadcastPresence = function(presence, callback) {
     collection: presence.c
   };
   var start = Date.now();
-  backend.trigger(backend.MIDDLEWARE_ACTIONS.receivePresence, this, context, function(error) {
+
+  var subscriptionUpdater = presence.p === null ?
+    this._unsubscribeDocVersion.bind(this) :
+    this._subscribeDocVersion.bind(this);
+
+  subscriptionUpdater(presence.c, presence.d, function(error) {
     if (error) return callback(error);
-    var requests = presenceRequests[presence.ch] || (presenceRequests[presence.ch] = Object.create(null));
-    var previousRequest = requests[presence.id];
-    if (!previousRequest || previousRequest.pv < presence.pv) {
-      presenceRequests[presence.ch][presence.id] = presence;
-    }
-    backend.transformPresenceToLatestVersion(agent, presence, function(error, presence) {
+    backend.trigger(backend.MIDDLEWARE_ACTIONS.receivePresence, agent, context, function(error) {
       if (error) return callback(error);
-      var channel = agent._getPresenceChannel(presence.ch);
-      agent.backend.pubsub.publish([channel], presence, function(error) {
-        if (error) return callback(error);
-        backend.emit('timing', 'presence.broadcast', Date.now() - start, context);
+      var requests = presenceRequests[presence.ch] || (presenceRequests[presence.ch] = Object.create(null));
+      var previousRequest = requests[presence.id];
+      if (!previousRequest || previousRequest.pv < presence.pv) {
+        presenceRequests[presence.ch][presence.id] = presence;
+      }
+
+      var transformer = function(agent, presence, callback) {
         callback(null, presence);
+      };
+
+      var latestDocVersion = util.dig(agent.latestDocVersions, presence.c, presence.d);
+      var presenceIsUpToDate = presence.v === latestDocVersion;
+      if (!presenceIsUpToDate) {
+        transformer = backend.transformPresenceToLatestVersion.bind(backend);
+      }
+
+      transformer(agent, presence, function(error, presence) {
+        if (error) return callback(error);
+        var channel = agent._getPresenceChannel(presence.ch);
+        agent.backend.pubsub.publish([channel], presence, function(error) {
+          if (error) return callback(error);
+          backend.emit('timing', 'presence.broadcast', Date.now() - start, context);
+          callback(null, presence);
+        });
       });
     });
   });
 };
 
+Agent.prototype._subscribeDocVersion = function(collection, id, callback) {
+  if (!collection || !id) return callback();
+
+  var latestDocVersions = this.latestDocVersions;
+  var isSubscribed = util.dig(latestDocVersions, collection, id) !== undefined;
+  if (isSubscribed) return callback();
+
+  var agent = this;
+  this.backend.subscribe(this, collection, id, null, function(error, stream, snapshot) {
+    if (error) return callback(error);
+
+    var versions = latestDocVersions[collection] || (latestDocVersions[collection] = Object.create(null));
+    versions[id] = snapshot.v;
+
+    agent._subscribeMapToStream(agent.latestDocVersionStreams, collection, id, stream, function(op) {
+      // op.v behind snapshot.v by 1
+      latestDocVersions[collection][id] = op.v + 1;
+    });
+
+    callback();
+  });
+};
+
+Agent.prototype._unsubscribeDocVersion = function(collection, id, callback) {
+  var stream = util.dig(this.latestDocVersionStreams, collection, id);
+  if (stream) stream.destroy();
+  util.digAndRemove(this.latestDocVersions, collection, id);
+  util.nextTick(callback);
+};
+
 Agent.prototype._createPresence = function(request) {
   return {
     a: ACTIONS.presence,
@@ -910,6 +977,11 @@ Agent.prototype._checkFirstMessage = function(request) {
   }
 };
 
+Agent.prototype._setProtocol = function(request) {
+  this.protocol.major = request.protocol;
+  this.protocol.minor = request.protocolMinor;
+};
+
 function createClientOp(request, clientId) {
   // src can be provided if it is not the same as the current agent,
   // such as a resubmission after a reconnect, but it usually isn't needed

package/lib/backend.js

@@ -43,6 +43,7 @@ function Backend(options) {
       'new Backend({doNotForwardSendPresenceErrorsToClient: true})\n\n'
     );
   }
+  this.doNotCommitNoOps = !!options.doNotCommitNoOps;
 
   // Map from event name to a list of middleware
   this.middleware = Object.create(null);

package/lib/client/connection.js

@@ -11,6 +11,7 @@ var types = require('../types');
 var util = require('../util');
 var logger = require('../logger');
 var DocPresenceEmitter = require('./presence/doc-presence-emitter');
+var protocol = require('../protocol');
 
 var ERROR_CODE = ShareDBError.CODES;
 
@@ -728,16 +729,21 @@ Connection.prototype._handleSnapshotFetch = function(error, message) {
 };
 
 Connection.prototype._handleLegacyInit = function(message) {
-  // If the minor protocol version has been set, we want to use the
+  // If the protocol is at least 1.1, we want to use the
   // new handshake protocol. Let's send a handshake initialize, because
   // we now know the server is ready. If we've already sent it, we'll
   // just ignore the response anyway.
-  if (message.protocolMinor) return this._initializeHandshake();
+  if (protocol.checkAtLeast(message, '1.1')) return this._initializeHandshake();
   this._initialize(message);
 };
 
 Connection.prototype._initializeHandshake = function() {
-  this.send({a: ACTIONS.handshake, id: this.id});
+  this.send({
+    a: ACTIONS.handshake,
+    id: this.id,
+    protocol: protocol.major,
+    protocolMinor: protocol.minor
+  });
 };
 
 Connection.prototype._handleHandshake = function(error, message) {
@@ -753,7 +759,7 @@ Connection.prototype._handlePingPong = function(error) {
 Connection.prototype._initialize = function(message) {
   if (this.state !== 'connecting') return;
 
-  if (message.protocol !== 1) {
+  if (message.protocol !== protocol.major) {
     return this.emit('error', new ShareDBError(
       ERROR_CODE.ERR_PROTOCOL_VERSION_NOT_SUPPORTED,
       'Unsupported protocol version: ' + message.protocol

package/lib/client/doc.js

@@ -328,6 +328,15 @@ Doc.prototype._handleSubscribe = function(error, snapshot) {
 
 Doc.prototype._handleOp = function(err, message) {
   if (err) {
+    if (err.code === ERROR_CODE.ERR_NO_OP && message.seq === this.inflightOp.seq) {
+      // Our op was a no-op, either because we submitted a no-op, or - more
+      // likely - because our op was transformed into a no-op by the server
+      // because of a similar remote op. In this case, the server has avoided
+      // committing the op to the database, and we should just clear the in-flight
+      // op and call the callbacks. However, let's first catch ourselves up to
+      // the remote, so that we're in a nice consistent state
+      return this.fetch(this._clearInflightOp.bind(this));
+    }
     if (this.inflightOp) {
       return this._rollback(err);
     }

package/lib/error.js

@@ -35,6 +35,7 @@ ShareDBError.CODES = {
   ERR_MAX_SUBMIT_RETRIES_EXCEEDED: 'ERR_MAX_SUBMIT_RETRIES_EXCEEDED',
   ERR_MESSAGE_BADLY_FORMED: 'ERR_MESSAGE_BADLY_FORMED',
   ERR_MILESTONE_ARGUMENT_INVALID: 'ERR_MILESTONE_ARGUMENT_INVALID',
+  ERR_NO_OP: 'ERR_NO_OP',
   ERR_OP_ALREADY_SUBMITTED: 'ERR_OP_ALREADY_SUBMITTED',
   ERR_OP_NOT_ALLOWED_IN_PROJECTION: 'ERR_OP_NOT_ALLOWED_IN_PROJECTION',
   ERR_OP_SUBMIT_REJECTED: 'ERR_OP_SUBMIT_REJECTED',

package/lib/protocol.js

@@ -0,0 +1,28 @@
+module.exports = {
+  major: 1,
+  minor: 2,
+  checkAtLeast: checkAtLeast
+};
+
+function checkAtLeast(toCheck, checkAgainst) {
+  toCheck = normalizedProtocol(toCheck);
+  checkAgainst = normalizedProtocol(checkAgainst);
+  if (toCheck.major > checkAgainst.major) return true;
+  return toCheck.major === checkAgainst.major &&
+    toCheck.minor >= checkAgainst.minor;
+}
+
+function normalizedProtocol(protocol) {
+  if (typeof protocol === 'string') {
+    var segments = protocol.split('.');
+    protocol = {
+      major: segments[0],
+      minor: segments[1]
+    };
+  }
+
+  return {
+    major: +(protocol.protocol || protocol.major || 0),
+    minor: +(protocol.protocolMinor || protocol.minor || 0)
+  };
+}

package/lib/submit-request.js

@@ -2,6 +2,7 @@ var ot = require('./ot');
 var projections = require('./projections');
 var ShareDBError = require('./error');
 var types = require('./types');
+var protocol = require('./protocol');
 
 var ERROR_CODE = ShareDBError.CODES;
 
@@ -152,6 +153,18 @@ SubmitRequest.prototype.submit = function(callback) {
       err = request._transformOp(ops);
       if (err) return callback(err);
 
+      var skipNoOp = backend.doNotCommitNoOps &&
+        protocol.checkAtLeast(request.agent.protocol, '1.2') &&
+        request.op.op &&
+        request.op.op.length === 0;
+
+      if (skipNoOp) {
+        // The op is a no-op, either because it was submitted as such, or - more
+        // likely - because it was transformed into one. Let's avoid committing it
+        // and tell the client.
+        return callback(request.noOpError());
+      }
+
       if (op.v !== snapshot.v) {
         // This shouldn't happen, but is just a final sanity check to make
         // sure we have transformed the op to the current snapshot version
@@ -361,3 +374,9 @@ SubmitRequest.prototype.maxRetriesError = function() {
     'Op submit failed. Exceeded max submit retries of ' + this.maxRetries
   );
 };
+SubmitRequest.prototype.noOpError = function() {
+  return new ShareDBError(
+    ERROR_CODE.ERR_NO_OP,
+    'Op is a no-op. Skipping apply.'
+  );
+};

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "sharedb",
-  "version": "5.0.3",
+  "version": "5.1.0",
   "description": "JSON OT database backend",
   "main": "lib/index.js",
   "dependencies": {

package/test/agent.js

@@ -5,6 +5,7 @@ var StreamSocket = require('../lib/stream-socket');
 var expect = require('chai').expect;
 var ACTIONS = require('../lib/message-actions').ACTIONS;
 var Connection = require('../lib/client/connection');
+var protocol = require('../lib/protocol');
 var LegacyConnection = require('sharedb-legacy/lib/client').Connection;
 
 describe('Agent', function() {
@@ -70,5 +71,16 @@ describe('Agent', function() {
         done();
       });
     });
+
+    it('records the client protocol on the agent', function(done) {
+      var connection = backend.connect();
+      connection.once('connected', function() {
+        expect(connection.agent.protocol).to.eql({
+          major: protocol.major,
+          minor: protocol.minor
+        });
+        done();
+      });
+    });
   });
 });

package/test/client/presence/doc-presence.js

@@ -5,6 +5,7 @@ var types = require('../../../lib/types');
 var presenceTestType = require('./presence-test-type');
 var errorHandler = require('../../util').errorHandler;
 var PresencePauser = require('./presence-pauser');
+var sinon = require('sinon');
 types.register(presenceTestType.type);
 
 describe('DocPresence', function() {
@@ -297,6 +298,23 @@ describe('DocPresence', function() {
     ], done);
   });
 
+  it('does not call getOps() when presence is already up-to-date', function(done) {
+    var localPresence1 = presence1.create('presence-1');
+
+    async.series([
+      doc1.fetch.bind(doc1), // Ensure up-to-date
+      function(next) {
+        sinon.spy(Backend.prototype, 'getOps');
+        next();
+      },
+      localPresence1.submit.bind(localPresence1, {index: 1}),
+      function(next) {
+        expect(Backend.prototype.getOps).not.to.have.been.called;
+        next();
+      }
+    ], done);
+  });
+
   // This test case attempts to force us into a tight race condition corner case:
   // 1. doc1 sends presence, as well as submits an op
   // 2. doc2 receives the op first, followed by the presence, which is now out-of-date

package/test/client/submit.js

@@ -1290,6 +1290,119 @@ module.exports = function() {
       });
     });
 
+    it('commits both of two identical ops submitted from different clients by default', function(done) {
+      var backend = this.backend;
+      backend.doNotCommitNoOps = false;
+      var doc1 = backend.connect().get('dogs', id);
+      var doc2 = backend.connect().get('dogs', id);
+      var op = [{p: ['tricks', 0], ld: 'fetch'}];
+
+      async.series([
+        doc1.create.bind(doc1, {tricks: ['fetch', 'sit']}),
+        doc2.fetch.bind(doc2),
+        async.parallel.bind(async.parallel, [
+          doc1.submitOp.bind(doc1, op),
+          doc2.submitOp.bind(doc2, op)
+        ]),
+        function(next) {
+          expect(doc1.data).to.eql({tricks: ['sit']});
+          expect(doc2.data).to.eql({tricks: ['sit']});
+          next();
+        },
+        function(next) {
+          backend.db.getOps('dogs', id, 0, null, {}, function(error, ops) {
+            if (error) return next(error);
+            // Expect:
+            // v0: create
+            // v1: update
+            // v2: duplicate update
+            expect(ops).to.have.length(3);
+            next();
+          });
+        }
+      ], done);
+    });
+
+    it('only commits one of two identical ops submitted from different clients', function(done) {
+      var backend = this.backend;
+      backend.doNotCommitNoOps = true;
+      var doc1 = backend.connect().get('dogs', id);
+      var doc2 = backend.connect().get('dogs', id);
+      var op = [{p: ['tricks', 0], ld: 'fetch'}];
+
+      async.series([
+        doc1.create.bind(doc1, {tricks: ['fetch', 'sit']}),
+        doc2.fetch.bind(doc2),
+        async.parallel.bind(async.parallel, [
+          doc1.submitOp.bind(doc1, op),
+          doc2.submitOp.bind(doc2, op)
+        ]),
+        function(next) {
+          expect(doc1.data).to.eql({tricks: ['sit']});
+          expect(doc2.data).to.eql({tricks: ['sit']});
+          next();
+        },
+        function(next) {
+          backend.db.getOps('dogs', id, 0, null, {}, function(error, ops) {
+            if (error) return next(error);
+            // Expect:
+            // v0: create
+            // v1: update
+            // no duplicate update
+            expect(ops).to.have.length(2);
+            next();
+          });
+        }
+      ], done);
+    });
+
+    it('can submit a new op after getting a no-op', function(done) {
+      var backend = this.backend;
+      backend.doNotCommitNoOps = true;
+      var doc1 = backend.connect().get('dogs', id);
+      var doc2 = backend.connect().get('dogs', id);
+      var op = [{p: ['tricks', 0], ld: 'fetch'}];
+
+      async.series([
+        doc1.create.bind(doc1, {tricks: ['fetch', 'sit']}),
+        doc2.fetch.bind(doc2),
+        async.parallel.bind(async.parallel, [
+          doc1.submitOp.bind(doc1, op),
+          doc2.submitOp.bind(doc2, op)
+        ]),
+        function(next) {
+          expect(doc1.data).to.eql({tricks: ['sit']});
+          expect(doc2.data).to.eql({tricks: ['sit']});
+          next();
+        },
+        doc1.submitOp.bind(doc1, [{p: ['tricks', 0], li: 'play dead'}]),
+        function(next) {
+          expect(doc1.data.tricks).to.eql(['play dead', 'sit']);
+          next();
+        }
+      ], done);
+    });
+
+    it('fixes up even if an op is fixed up to become a no-op', function(done) {
+      var backend = this.backend;
+      backend.doNotCommitNoOps = true;
+      var doc = backend.connect().get('dogs', id);
+
+      backend.use('apply', function(req, next) {
+        req.$fixup([{p: ['fixme'], od: true}]);
+        next();
+      });
+
+      async.series([
+        doc.create.bind(doc, {}),
+        doc.submitOp.bind(doc, [{p: ['fixme'], oi: true}]),
+        function(next) {
+          expect(doc.data).to.eql({});
+          next();
+        }
+      ], done);
+    });
+
     describe('type.deserialize', function() {
       it('can create a new doc', function(done) {
         var doc = this.backend.connect().get('dogs', id);

package/test/protocol.js

@@ -0,0 +1,31 @@
+var protocol = require('../lib/protocol');
+var expect = require('chai').expect;
+
+describe('protocol', function() {
+  describe('checkAtLeast', function() {
+    var FIXTURES = [
+      ['1.0', '1.0', true],
+      ['1.1', '1.0', true],
+      ['1.0', '1.1', false],
+      ['1.0', '1', true],
+      ['1.10', '1.3', true],
+      ['2.0', '1.3', true],
+      [{major: 1, minor: 0}, {major: 1, minor: 0}, true],
+      [{major: 1, minor: 1}, {major: 1, minor: 0}, true],
+      [{major: 1, minor: 0}, {major: 1, minor: 1}, false],
+      [{protocol: 1, protocolMinor: 0}, {protocol: 1, protocolMinor: 0}, true],
+      [{protocol: 1, protocolMinor: 1}, {protocol: 1, protocolMinor: 0}, true],
+      [{protocol: 1, protocolMinor: 0}, {protocol: 1, protocolMinor: 1}, false],
+      [{}, '1.0', false],
+      ['', '1.0', false]
+    ];
+
+    FIXTURES.forEach(function(fixture) {
+      var is = fixture[2] ? ' is ' : ' is not ';
+      var name = 'checks ' + JSON.stringify(fixture[0]) + is + 'at least ' + JSON.stringify(fixture[1]);
+      it(name, function() {
+        expect(protocol.checkAtLeast(fixture[0], fixture[1])).to.equal(fixture[2]);
+      });
+    });
+  });
+});