sharedb 7.0.0 → 7.0.2

package/lib/submit-request.js

@@ -267,8 +267,15 @@ SubmitRequest.prototype.retry = function(callback) {
     return callback(this.maxRetriesError());
   }
   this._resetFixups();
+  // Unset the failed attempt's results, which the next attempt replaces
+  this.snapshot = null;
+  this.channels = null;
   this.backend.emit('timing', 'submit.retry', Date.now() - this.start, this);
-  this.submit(callback);
+  var request = this;
+  this.backend.trigger(this.backend.MIDDLEWARE_ACTIONS.retry, this.agent, this, function(err) {
+    if (err) return callback(err);
+    request.submit(callback);
+  });
 };
 
 // The fixups must be undone before the op is transformed forward again

package/lib/util.js

@@ -105,6 +105,16 @@ exports.clone = function(obj) {
   return (obj === undefined) ? undefined : JSON.parse(JSON.stringify(obj));
 };
 
+exports.shallowCopy = function(object) {
+  var out = {};
+  for (var key in object) {
+    if (exports.hasOwn(object, key)) {
+      out[key] = object[key];
+    }
+  }
+  return out;
+};
+
 // A null prototype makes '__proto__' an ordinary own key, so it can be in the map
 var objectProtoPropNames = Object.create(null);
 Object.getOwnPropertyNames(Object.prototype).forEach(function(prop) {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "sharedb",
-  "version": "7.0.0",
+  "version": "7.0.2",
   "description": "JSON OT database backend",
   "main": "lib/index.js",
   "dependencies": {

package/test/agent.js

@@ -8,6 +8,11 @@ var ACTIONS = require('../lib/message-actions').ACTIONS;
 var Connection = require('../lib/client/connection');
 var protocol = require('../lib/protocol');
 var LegacyConnection = require('sharedb-legacy/lib/client').Connection;
+var async = require('async');
+var types = require('../lib/types');
+var presenceTestType = require('./client/presence/presence-test-type');
+var util = require('./util');
+types.register(presenceTestType.type);
 
 describe('Agent', function() {
   var backend;
@@ -144,6 +149,20 @@ describe('Agent', function() {
       });
     });
 
+    [unconvertible, ['dogs'], '__proto__'].forEach(function(badCollection) {
+      it('rejects doc presence with collection ' + JSON.stringify(badCollection), function(done) {
+        var message = {a: ACTIONS.presence, ch: 'dogs.fido', c: badCollection, d: 'fido', id: 'rex', pv: 0, v: 0};
+        expectRejected(message, 'Invalid collection', done);
+      });
+    });
+
+    [unconvertible, '__proto__'].forEach(function(badChannel) {
+      it('rejects doc presence with channel ' + JSON.stringify(badChannel), function(done) {
+        var message = {a: ACTIONS.presence, ch: badChannel, c: 'dogs', d: 'fido', id: 'rex', pv: 0, v: 0};
+        expectRejected(message, 'Invalid presence channel', done);
+      });
+    });
+
     [unconvertible, true].forEach(function(badProtocol) {
       it('rejects a handshake with protocol ' + JSON.stringify(badProtocol), function(done) {
         expectRejected({a: ACTIONS.handshake, protocol: badProtocol, protocolMinor: 2}, 'Invalid protocol', done);
@@ -322,5 +341,401 @@ describe('Agent', function() {
         });
       });
     });
+
+    describe('with presence', function() {
+      beforeEach(function(done) {
+        backend.close(function(error) {
+          if (error) return done(error);
+          backend = new Backend({presence: true});
+          writer = backend.connect();
+          subscriber = backend.connect();
+          done();
+        });
+      });
+
+      function expectOnlyPresence(presence, callback) {
+        var messages = [];
+        subscriber.on('receive', function(request) {
+          messages.push(request.data);
+        });
+        presence.once('receive', function() {
+          expect(messages).to.have.length(1);
+          expect(messages[0].a).to.equal(ACTIONS.presence);
+          callback();
+        });
+      }
+
+      it('does not deliver ops from a collection whose channel matches the presence channel', function(done) {
+        var presence = subscriber.getPresence('secrets');
+        presence.subscribe(function(error) {
+          if (error) return done(error);
+          expectOnlyPresence(presence, done);
+          writer.get('$presence.secrets', 'alice').create({pin: '1234'}, function(error) {
+            if (error) return done(error);
+            writer.getPresence('secrets').create('writer').submit({name: 'Writer'}, function(error) {
+              if (error) done(error);
+            });
+          });
+        });
+      });
+
+      it('does not deliver ops from a doc whose channel matches the doc presence channel', function(done) {
+        var writerDoc = writer.get('notes', 'alice');
+        var subscriberDoc = subscriber.get('notes', 'alice');
+        var presence = subscriber.getDocPresence('notes', 'alice');
+        async.series([
+          writerDoc.create.bind(writerDoc, 'Hello', presenceTestType.type.name),
+          subscriberDoc.fetch.bind(subscriberDoc),
+          presence.subscribe.bind(presence),
+          function(next) {
+            expectOnlyPresence(presence, next);
+            writer.get('$presence', 'notes.alice').create({pin: '1234'}, function(error) {
+              if (error) return next(error);
+              writer.getDocPresence('notes', 'alice').create('writer').submit({index: 1}, function(error) {
+                if (error) next(error);
+              });
+            });
+          }
+        ], done);
+      });
+    });
+  });
+
+  describe('cleanup', function() {
+    var connection;
+
+    beforeEach(function(done) {
+      backend.close(function(error) {
+        if (error) return done(error);
+        backend = new Backend({presence: true});
+        connection = backend.connect();
+        connection.get('books', 'northern-lights').create('North Lights', presenceTestType.type.name, done);
+      });
+    });
+
+    function closeConnection(callback) {
+      connection.agent.stream.once('end', callback);
+      connection.close();
+    }
+
+    function opensStreams(count, step) {
+      return function(next) {
+        var streamsCount = backend.pubsub.streamsCount;
+        step(function(error) {
+          if (error) return next(error);
+          expect(backend.pubsub.streamsCount - streamsCount).to.equal(count);
+          next();
+        });
+      };
+    }
+
+    it('destroys a stream that is missing from its subscription map', function(done) {
+      connection.get('dogs', 'fido').subscribe(function(error) {
+        if (error) return done(error);
+        delete connection.agent.subscribedDocs.dogs;
+        closeConnection(function() {
+          expect(backend.pubsub.streamsCount).to.equal(0);
+          done();
+        });
+      });
+    });
+
+    it('stops tracking streams once they close', function(done) {
+      var doc = connection.get('dogs', 'fido');
+      var presence = connection.getDocPresence('books', 'northern-lights');
+      var localPresence = presence.create('presence-1');
+      async.series([
+        doc.subscribe.bind(doc),
+        doc.unsubscribe.bind(doc),
+        presence.subscribe.bind(presence),
+        presence.unsubscribe.bind(presence),
+        localPresence.submit.bind(localPresence, {index: 0}),
+        localPresence.destroy.bind(localPresence),
+        function(next) {
+          expect(connection.agent._streams).to.be.empty;
+          next();
+        }
+      ], done);
+    });
+
+    it('empties its stream maps as soon as the connection closes', function(done) {
+      var presence = connection.getDocPresence('books', 'northern-lights');
+      var agent = connection.agent;
+      async.series([
+        function(next) {
+          connection.get('dogs', 'fido').subscribe(next);
+        },
+        presence.subscribe.bind(presence),
+        function(next) {
+          presence.create('presence-1').submit({index: 0}, next);
+        },
+        function(next) {
+          agent.stream.once('end', function() {
+            expect(agent.subscribedDocs).to.be.empty;
+            expect(agent.subscribedPresences).to.be.empty;
+            expect(agent.latestDocVersionStreams).to.be.empty;
+            next();
+          });
+          connection.close();
+        }
+      ], done);
+    });
+
+    it('does not track a stream that closed before reaching the agent', function(done) {
+      backend.use(backend.MIDDLEWARE_ACTIONS.readSnapshots, function(context, next) {
+        backend.pubsub.close();
+        next();
+      });
+      connection.get('dogs', 'fido').subscribe(function(error) {
+        if (error) return done(error);
+        expect(connection.agent._streams).to.be.empty;
+        done();
+      });
+    });
+
+    it('destroys every pub/sub stream when the connection closes', function(done) {
+      var presence = connection.getDocPresence('books', 'northern-lights');
+      async.series([
+        opensStreams(1, function(next) {
+          connection.get('dogs', 'fido').subscribe(next);
+        }),
+        opensStreams(2, function(next) {
+          connection.startBulk();
+          connection.get('dogs', 'rex').subscribe();
+          connection.get('dogs', 'spot').subscribe(next);
+          connection.endBulk();
+        }),
+        opensStreams(1, presence.subscribe.bind(presence)),
+        opensStreams(1, function(next) {
+          presence.create('presence-1').submit({index: 0}, next);
+        }),
+        opensStreams(1, function(next) {
+          connection.createSubscribeQuery('dogs', {}, null, next);
+        }),
+        closeConnection,
+        function(next) {
+          expect(backend.pubsub.streamsCount).to.equal(0);
+          next();
+        }
+      ], done);
+    });
+  });
+
+  describe('query options', function() {
+    var connection;
+
+    beforeEach(function() {
+      connection = backend.connect();
+    });
+
+    [ACTIONS.queryFetch, ACTIONS.querySubscribe].forEach(function(action) {
+      it('does not pass a client\'s fields to ' + action + ' middleware', function(done) {
+        var queryOptions;
+        backend.use('query', function(context, next) {
+          queryOptions = context.options;
+          next();
+        });
+        connection.on('receive', function(request) {
+          if (request.data.a !== action) return;
+          expect(request.data).not.to.have.property('error');
+          expect(queryOptions).not.to.have.property('fields');
+          done();
+        });
+        connection.send({a: action, id: 1, c: 'dogs', q: {}, o: {fields: {secret: true}}});
+      });
+    });
+  });
+
+  describe('query subscribe with known results', function() {
+    var connection;
+
+    beforeEach(function(done) {
+      connection = backend.connect();
+      connection.get('dogs', 'fido').create({name: 'Fido'}, done);
+    });
+
+    function resubscribe(knownResults, callback) {
+      connection.on('receive', function(request) {
+        if (request.data.a === ACTIONS.querySubscribe) callback(request.data);
+      });
+      connection.send({
+        a: ACTIONS.querySubscribe,
+        id: 1,
+        c: 'dogs',
+        q: {},
+        r: knownResults
+      });
+    }
+
+    it('does not read known results when the query middleware rejects it', function(done) {
+      backend.use('query', function(context, next) {
+        next(new Error('Forbidden'));
+      });
+      sinon.spy(backend.db, 'getOpsBulk');
+      sinon.spy(backend.db, 'getSnapshotBulk');
+      resubscribe([['fido', 0], ['spot', null]], function(reply) {
+        expect(reply.error).to.have.property('message', 'Forbidden');
+        expect(backend.db.getOpsBulk).not.to.have.been.called;
+        expect(backend.db.getSnapshotBulk).not.to.have.been.called;
+        done();
+      });
+    });
+
+    it('does not stay subscribed if fetching known results fails', function(done) {
+      backend.use('op', function(context, next) {
+        next(new Error('Forbidden'));
+      });
+      resubscribe([['fido', 0], ['spot', null]], function(reply) {
+        expect(reply.error).to.have.property('message', 'Forbidden');
+        expect(connection.agent.subscribedQueries).to.be.empty;
+        expect(backend.pubsub.streamsCount).to.equal(0);
+        done();
+      });
+    });
+
+    it('does not stay subscribed if one of its known snapshots is rejected', function(done) {
+      backend.use('readSnapshots', function(context, next) {
+        context.snapshots.forEach(function(snapshot) {
+          if (snapshot.id === 'spot') context.rejectSnapshotRead(snapshot, new Error('Forbidden'));
+        });
+        next();
+      });
+      resubscribe([['fido', null], ['spot', null]], function(reply) {
+        expect(reply.error).to.have.property('code', 'ERR_SNAPSHOT_READS_REJECTED');
+        expect(connection.agent.subscribedQueries).to.be.empty;
+        expect(backend.pubsub.streamsCount).to.equal(0);
+        done();
+      });
+    });
+
+    it('does not send known results\' ops if reading their snapshots fails', function(done) {
+      backend.use('readSnapshots', function(context, next) {
+        next(new Error('Forbidden'));
+      });
+      var sentOps = [];
+      backend.on('send', function(agent, message) {
+        if (message.a === ACTIONS.op) sentOps.push(message);
+      });
+      var getOpsBulk = backend.getOpsBulk;
+      var finishGettingOps;
+      sinon.stub(backend, 'getOpsBulk').callsFake(function(agent, collection, fromMap, toMap, callback) {
+        finishGettingOps = function() {
+          getOpsBulk.call(backend, agent, collection, fromMap, toMap, function(error, opsMap) {
+            callback(error, opsMap);
+            expect(sentOps).to.be.empty;
+            done();
+          });
+        };
+      });
+      resubscribe([['fido', 0], ['spot', null]], function(reply) {
+        expect(reply.error).to.have.property('message', 'Forbidden');
+        finishGettingOps();
+      });
+    });
+
+    it('does not stay subscribed if re-polling the query fails', function(done) {
+      var clock = util.useFakeTimers();
+      backend.db.pollInterval = 50;
+      sinon.stub(backend.db, 'queryPoll').yieldsAsync(new Error('Forbidden'));
+      resubscribe([['fido', 0], ['spot', null]], function(reply) {
+        expect(reply.error).to.have.property('message', 'Forbidden');
+        expect(connection.agent.subscribedQueries).to.be.empty;
+        expect(backend.pubsub.streamsCount).to.equal(0);
+        clock.tick(1000);
+        expect(backend.db.queryPoll).to.have.been.calledOnce;
+        done();
+      });
+    });
+
+    [
+      {collection: 'dog_names', fields: {name: true}},
+      {collection: 'dogs', fields: undefined}
+    ].forEach(function(test) {
+      it('re-polls ' + test.collection + ' with its projection\'s fields, not the client\'s', function(done) {
+        backend.addProjection('dog_names', 'dogs', {name: true});
+        sinon.spy(backend.db, 'queryPoll');
+        connection.on('receive', function(request) {
+          if (request.data.a !== ACTIONS.querySubscribe) return;
+          expect(request.data).not.to.have.property('error');
+          var pollOptions = backend.db.queryPoll.firstCall.args[2];
+          expect(pollOptions.fields).to.eql(test.fields);
+          done();
+        });
+        connection.send({
+          a: ACTIONS.querySubscribe,
+          id: 1,
+          c: test.collection,
+          q: {},
+          o: {fields: {secret: true}},
+          r: [['fido', 0]]
+        });
+      });
+    });
+
+    it('re-polls each query with its own fields without writing them to shared options', function(done) {
+      backend.addProjection('dog_names', 'dogs', {name: true});
+      sinon.spy(backend.db, 'queryPoll');
+      var options = {ids: ['fido']};
+      async.eachSeries(['dog_names', 'dogs'], function(index, next) {
+        backend.querySubscribe(connection.agent, index, {}, options, function(err, emitter) {
+          if (err) return next(err);
+          emitter.queryPoll(function(err) {
+            emitter.destroy();
+            next(err);
+          });
+        });
+      }, function(err) {
+        if (err) return done(err);
+        var pollFields = backend.db.queryPoll.args.map(function(args) {
+          return args[2].fields;
+        });
+        expect(pollFields).to.eql([{name: true}, undefined]);
+        expect(options).not.to.have.property('fields');
+        done();
+      });
+    });
+
+    it('keeps a newer subscription with the same id if re-polling the query fails', function(done) {
+      var failRepoll;
+      sinon.stub(backend.db, 'queryPoll').callsFake(function(collection, query, options, callback) {
+        failRepoll = callback.bind(null, new Error('Forbidden'));
+        connection.send({a: ACTIONS.querySubscribe, id: 1, c: 'dogs', q: {}});
+      });
+      var replies = [];
+      resubscribe([['fido', 0]], function(reply) {
+        replies.push(reply);
+        if (replies.length === 1) return failRepoll();
+        expect(replies[0]).not.to.have.property('error');
+        expect(replies[1].error).to.have.property('message', 'Forbidden');
+        expect(connection.agent.subscribedQueries).to.have.property('1');
+        done();
+      });
+    });
+
+    it('delivers an op committed while fetching known results', function(done) {
+      var writer = backend.connect().get('dogs', 'fido');
+      connection.createSubscribeQuery('dogs', {}, null, function(error, results) {
+        if (error) return done(error);
+        var fido = results[0];
+        fido.on('op', function() {
+          expect(fido.data).to.eql({name: 'Rex'});
+          done();
+        });
+        var getOpsBulk = backend.db.getOpsBulk;
+        sinon.stub(backend.db, 'getOpsBulk').callsFake(function(collection, fromMap, toMap, options, callback) {
+          backend.db.getOpsBulk.restore();
+          getOpsBulk.call(backend.db, collection, fromMap, toMap, options, function(error, opsMap) {
+            writer.fetch(function(error) {
+              if (error) return done(error);
+              writer.submitOp({p: ['name'], od: 'Fido', oi: 'Rex'}, function(error) {
+                callback(error, opsMap);
+              });
+            });
+          });
+        });
+        connection.close();
+        backend.connect(connection);
+      });
+    });
   });
 });