sharedb 6.0.4 → 7.0.0

package/lib/agent.js

@@ -25,8 +25,7 @@ function Agent(backend, stream) {
   // src is a client-configurable "id" which the client will set in its handshake,
   // and attach to its ops. This should take precedence over clientId if set.
   // Only legacy clients, or new clients connecting for the first time will use the
-  // Agent-provided clientId. Ideally we'll deprecate clientId in favour of src
-  // in the next breaking change.
+  // Agent-provided clientId.
   this.src = null;
   this.connectTime = Date.now();
 
@@ -229,6 +228,7 @@ Agent.prototype._subscribeToQuery = function(emitter, queryId, collection, query
 
 Agent.prototype._onOp = function(collection, id, op) {
   if (this._isOwnOp(collection, op)) return;
+  if (this._isOpFromCollidingChannel(collection, id, op)) return;
 
   // Ops emitted here are coming directly from pubsub, which emits the same op
   // object to listeners without making a copy. The pattern in middleware is to
@@ -263,6 +263,12 @@ Agent.prototype._isOwnOp = function(collection, op) {
   return (this._src() === op.src) && (collection === (op.i || op.c));
 };
 
+Agent.prototype._isOpFromCollidingChannel = function(collection, id, op) {
+  var projection = this.backend.projections[collection];
+  var target = (projection) ? projection.target : collection;
+  return op.c !== target || op.d !== id;
+};
+
 Agent.prototype.send = function(message) {
   // Quietly drop replies if the stream was closed
   if (this.closed) return;
@@ -375,6 +381,42 @@ Agent.prototype._open = function() {
   this.stream.on('close', cleanup);
 };
 
+function isInvalidName(name) {
+  return typeof name !== 'string' || util.isDangerousProperty(name);
+}
+
+function isInvalidVersion(version) {
+  return version != null && (typeof version !== 'number' || version < 0);
+}
+
+function isInvalidProtocol(value) {
+  return value != null && typeof value !== 'number' && typeof value !== 'string';
+}
+
+function checkBulkData(data) {
+  if (!data || typeof data !== 'object') return 'Invalid bulk subscribe data';
+  if (Array.isArray(data)) {
+    if (data.some(isInvalidName)) return 'Invalid id';
+    return;
+  }
+  for (var id in data) {
+    if (isInvalidName(id)) return 'Invalid id';
+    if (isInvalidVersion(data[id])) return 'Invalid version';
+  }
+}
+
+function checkQueryResults(results) {
+  if (!Array.isArray(results)) return 'Invalid query results';
+  for (var i = 0; i < results.length; i++) {
+    var result = results[i];
+    if (!Array.isArray(result)) return 'Invalid query results';
+    var id = result[0];
+    var version = result[1];
+    if (isInvalidName(id)) return 'Invalid id';
+    if (isInvalidVersion(version)) return 'Invalid version';
+  }
+}
+
 // Check a request to see if its valid. Returns an error if there's a problem.
 Agent.prototype._checkRequest = function(request) {
   if (
@@ -384,6 +426,10 @@ Agent.prototype._checkRequest = function(request) {
   ) {
     // Query messages need an ID property.
     if (typeof request.id !== 'number') return 'Missing query ID';
+    if (request.r != null) {
+      var resultsError = checkQueryResults(request.r);
+      if (resultsError) return resultsError;
+    }
   } else if (request.a === ACTIONS.op ||
       request.a === ACTIONS.fetch ||
       request.a === ACTIONS.subscribe ||
@@ -392,30 +438,19 @@ Agent.prototype._checkRequest = function(request) {
       request.a === ACTIONS.snapshotFetchByTimestamp ||
       request.a === ACTIONS.presence) {
     // Doc-based request.
-    if (request.c != null) {
-      if (typeof request.c !== 'string' || util.isDangerousProperty(request.c)) {
-        return 'Invalid collection';
-      }
-    }
-    if (request.d != null) {
-      if (typeof request.d !== 'string' || util.isDangerousProperty(request.d)) {
-        return 'Invalid id';
-      }
-    }
+    if (request.d != null && isInvalidName(request.d)) return 'Invalid id';
 
     if (request.a === ACTIONS.snapshotFetch || request.a === ACTIONS.snapshotFetchByTimestamp) {
       if (request.c == null) return 'Invalid collection';
       if (request.d == null) return 'Invalid id';
     }
 
-    if (request.a === ACTIONS.op || request.a === ACTIONS.presence) {
-      if (request.v != null && (typeof request.v !== 'number' || request.v < 0)) return 'Invalid version';
-    }
+    if (isInvalidVersion(request.v)) return 'Invalid version';
+    if (request.ts != null && typeof request.ts !== 'number') return 'Invalid timestamp';
 
     if (request.a === ACTIONS.presence) {
-      if (typeof request.id !== 'string' || util.isDangerousProperty(request.id)) {
-        return 'Invalid presence ID';
-      }
+      if (isInvalidName(request.id)) return 'Invalid presence ID';
+      if (isInvalidVersion(request.pv)) return 'Invalid presence version';
     }
   } else if (
     request.a === ACTIONS.bulkFetch ||
@@ -423,18 +458,14 @@ Agent.prototype._checkRequest = function(request) {
     request.a === ACTIONS.bulkUnsubscribe
   ) {
     // Bulk request
-    if (request.c != null) {
-      if (typeof request.c !== 'string' || util.isDangerousProperty(request.c)) {
-        return 'Invalid collection';
-      }
-    }
-    if (typeof request.b !== 'object') return 'Invalid bulk subscribe data';
-  }
-  if (request.ch != null) {
-    if (typeof request.ch !== 'string' || util.isDangerousProperty(request.ch)) {
-      return 'Invalid presence channel';
-    }
+    var bulkError = checkBulkData(request.b);
+    if (bulkError) return bulkError;
+  } else if (request.a === ACTIONS.handshake) {
+    if (isInvalidProtocol(request.protocol) || isInvalidProtocol(request.protocolMinor)) return 'Invalid protocol';
   }
+  if (request.c != null && isInvalidName(request.c)) return 'Invalid collection';
+  if (request.ch != null && isInvalidName(request.ch)) return 'Invalid presence channel';
+  if (request.seq != null && typeof request.seq !== 'number') return 'Invalid seq';
 };
 
 // Handle an incoming message from the client
@@ -533,6 +564,8 @@ function getQueryOptions(request) {
     }
   }
   var options = request.o || {};
+  delete options.pollInterval;
+  delete options.pollDebounce;
   options.ids = ids;
   options.fetch = fetch;
   options.fetchOps = fetchOps;
@@ -924,6 +957,10 @@ Agent.prototype._subscribePresence = function(channel, seq, cb) {
       return callback();
     }
     agent.presenceSubscriptionSeq[channel] = seq;
+    if (agent.subscribedPresences[channel]) {
+      stream.destroy();
+      return callback();
+    }
     agent.subscribedPresences[channel] = stream;
     agent._subscribeToPresenceStream(channel, stream);
     agent._requestPresence(channel, function(error) {

package/lib/backend.js

@@ -569,22 +569,25 @@ Backend.prototype.subscribeBulk = function(agent, index, versions, callback) {
   var backend = this;
   var streams = Object.create(null);
   var doFetch = Array.isArray(versions);
-  var ids = (doFetch) ? versions : Object.keys(versions);
+  var ids = (doFetch) ? uniqueIds(versions) : Object.keys(versions);
   var request = {
     agent: agent,
     index: index,
     collection: collection,
     versions: versions
   };
+  var subscribeFailed = false;
   async.each(ids, function(id, eachCb) {
     var channel = backend.getDocChannel(collection, id);
     backend.pubsub.subscribe(channel, function(err, stream) {
       if (err) return eachCb(err);
+      if (subscribeFailed) return stream.destroy();
       streams[id] = stream;
       eachCb();
     });
   }, function(err) {
     if (err) {
+      subscribeFailed = true;
       destroyStreams(streams);
       return callback(err);
     }
@@ -626,6 +629,14 @@ function destroyStreams(streams) {
     streams[id].destroy();
   }
 }
+function uniqueIds(ids) {
+  var seen = Object.create(null);
+  return ids.filter(function(id) {
+    if (seen[id]) return false;
+    seen[id] = true;
+    return true;
+  });
+}
 
 Backend.prototype.queryFetch = function(agent, index, query, options, callback) {
   var start = Date.now();
@@ -739,12 +750,16 @@ Backend.prototype._triggerQuery = function(agent, index, query, options, callbac
     if (err) return callback(err);
     // Set the DB reference for the request after the middleware trigger so
     // that the db option can be changed in middleware
-    request.db = (options.db) ? backend.extraDbs[options.db] : backend.db;
+    request.db = (options.db) ? getExtraDb(backend, options.db) : backend.db;
     if (!request.db) return callback(new ShareDBError(ERROR_CODE.ERR_DATABASE_ADAPTER_NOT_FOUND, 'DB not found'));
     request.snapshotProjection = backend._getSnapshotProjection(request.db, projection);
     callback(null, request);
   });
 };
+function getExtraDb(backend, name) {
+  if (typeof name !== 'string' || !util.hasOwn(backend.extraDbs, name)) return null;
+  return backend.extraDbs[name];
+}
 
 Backend.prototype._query = function(agent, request, callback) {
   var backend = this;
@@ -875,8 +890,6 @@ Backend.prototype._fetchSnapshotByTimestamp = function(collection, id, timestamp
   var backend = this;
 
   var milestoneSnapshot;
-  var from = 0;
-  var to = null;
 
   // Always fetch the current snapshot first. We request its metadata so that we
   // can read its mtime, which lets us serve the current snapshot directly when
@@ -898,11 +911,12 @@ Backend.prototype._fetchSnapshotByTimestamp = function(collection, id, timestamp
     milestoneDb.getMilestoneSnapshotAtOrBeforeTime(collection, id, timestamp, function(error, snapshot) {
       if (error) return callback(error);
       milestoneSnapshot = snapshot;
-      if (snapshot) from = snapshot.v;
 
       milestoneDb.getMilestoneSnapshotAtOrAfterTime(collection, id, timestamp, function(error, snapshot) {
         if (error) return callback(error);
-        if (snapshot) to = snapshot.v;
+        var from = milestoneSnapshot ? milestoneSnapshot.v : 0;
+        var to = snapshot ? snapshot.v : currentSnapshot.v;
+        if (from >= to) return backend._buildSnapshotFromOps(id, milestoneSnapshot, [], callback);
 
         var options = {metadata: true};
         db.getOps(collection, id, from, to, options, function(error, ops) {

package/lib/client/doc.js

@@ -762,7 +762,20 @@ Doc.prototype._submit = function(op, source, callback) {
       return util.callbackOrEmit(this, err, callback);
     }
     // Try to normalize the op. This removes trailing skip:0's and things like that.
-    if (this.type.normalize) op.op = this.type.normalize(op.op);
+    if (this.type.normalize) {
+      try {
+        op.op = this.type.normalize(op.op);
+      } catch (error) {
+        // Otherwise a type that throws on a badly formed op throws out of
+        // submitOp() rather than calling back
+        var normalizeError = error instanceof ShareDBError ? error : new ShareDBError(
+          ERROR_CODE.ERR_OT_OP_BADLY_FORMED,
+          (error && error.message) || String(error)
+        );
+        if (callback) return callback(normalizeError);
+        return this.emit('error', normalizeError);
+      }
+    }
 
     // This has to happen before _pushOp(), because _tryCompose() applies the op
     // to a pending create, well before _otApply() gets a chance to check it

package/lib/client/presence/remote-doc-presence.js

@@ -120,7 +120,13 @@ RemoteDocPresence.prototype._catchUpStalePresence = function() {
       this._pending.p = null;
       this._pending.v++;
     } else {
-      ot.transformPresence(this._pending, op, isOwnOp);
+      var error = ot.transformPresence(this._pending, op, isOwnOp);
+      if (error) {
+        this._pending = null;
+        this._stopCachingOps();
+        this.presence.emit('error', error);
+        return false;
+      }
     }
   }
 

package/lib/ot.js

@@ -120,32 +120,71 @@ function applyOpEdit(snapshot, edit) {
   }
 }
 
+function isJson0OpComponent(component) {
+  return !!component && typeof component === 'object' && Array.isArray(component.p);
+}
+
 // ot-json0 walks ops with .length and numeric indexing, so it treats array-like
 // objects as ops too, and it coerces .length. This traversal has to match it
-// exactly: anything ot-json0 will apply has to be checked here.
+// exactly: anything ot-json0 will apply has to be checked here. Coercing
+// .length or a path segment throws for some objects; ot-json0 throws on those
+// too, so they are rejected rather than applied.
 // See GHSA-9rqw-j2q5-gg2g
 function checkJson0OpPaths(op) {
   if (op == null) return;
-  for (var i = 0; i < op.length; i++) {
-    var component = op[i];
-    // ot-json0's checkValidOp() rejects the whole op before applying any of it,
-    // so there is nothing beyond this component left to check
-    if (!component || typeof component !== 'object' || !Array.isArray(component.p)) return;
-    for (var j = 0; j < component.p.length; j++) {
-      if (util.isDangerousProperty(component.p[j])) {
-        return new ShareDBError(ERROR_CODE.ERR_OT_OP_NOT_APPLIED, 'Invalid path segment');
+  try {
+    for (var i = 0; i < op.length; i++) {
+      var component = op[i];
+      // ot-json0's checkValidOp() rejects the whole op before applying any of it,
+      // so there is nothing beyond this component left to check
+      if (!isJson0OpComponent(component)) return;
+      for (var j = 0; j < component.p.length; j++) {
+        if (util.isDangerousProperty(component.p[j])) {
+          return new ShareDBError(ERROR_CODE.ERR_OT_OP_NOT_APPLIED, 'Invalid path segment');
+        }
       }
     }
+  } catch (error) {
+    return new ShareDBError(ERROR_CODE.ERR_OT_OP_NOT_APPLIED, error.message);
   }
 }
 
+// Only for ops arriving at submit-request, which reach ot-json0 exactly as
+// they came off the wire, with nothing to normalize them on the way. Ops that
+// are already committed can only have their paths checked, since ops that
+// ot-json0 quietly treats as no-ops were committable by older versions of
+// ShareDB
+function checkSubmittedJson0Op(op) {
+  if (!Array.isArray(op)) {
+    return new ShareDBError(ERROR_CODE.ERR_OT_OP_BADLY_FORMED, 'json0 op must be an array');
+  }
+
+  for (var i = 0; i < op.length; i++) {
+    if (!isJson0OpComponent(op[i])) {
+      return new ShareDBError(ERROR_CODE.ERR_OT_OP_BADLY_FORMED, 'Missing path');
+    }
+  }
+
+  return checkJson0OpPaths(op);
+}
+
+function isJson0(type) {
+  if (typeof type === 'string') type = types.map[type];
+  return !!type && type.name === 'json0';
+}
+
 exports.checkOpPathsForType = function(type, op) {
   if (!('op' in op)) return;
-  if (typeof type === 'string') type = types.map[type];
-  if (!type || type.name !== 'json0') return;
+  if (!isJson0(type)) return;
   return checkJson0OpPaths(op.op);
 };
 
+exports.checkSubmittedOpForType = function(type, op) {
+  if (!('op' in op)) return;
+  if (!isJson0(type)) return;
+  return checkSubmittedJson0Op(op.op);
+};
+
 exports.transform = function(type, op, appliedOp) {
   // There are 16 cases this function needs to deal with - which are all the
   // combinations of create/delete/op/noop from both op and appliedOp
@@ -223,9 +262,9 @@ exports.transformPresence = function(presence, op, isOwnOp) {
   if (typeof type === 'string') {
     type = types.map[type];
   }
-  if (!type) return {code: ERROR_CODE.ERR_DOC_TYPE_NOT_RECOGNIZED, message: 'Unknown type'};
+  if (!type) return new ShareDBError(ERROR_CODE.ERR_DOC_TYPE_NOT_RECOGNIZED, 'Unknown type');
   if (!util.supportsPresence(type)) {
-    return {code: ERROR_CODE.ERR_TYPE_DOES_NOT_SUPPORT_PRESENCE, message: 'Type does not support presence'};
+    return new ShareDBError(ERROR_CODE.ERR_TYPE_DOES_NOT_SUPPORT_PRESENCE, 'Type does not support presence');
   }
 
   if (op.create || op.del) {
@@ -239,7 +278,7 @@ exports.transformPresence = function(presence, op, isOwnOp) {
       null :
       type.transformPresence(presence.p, op.op, isOwnOp);
   } catch (error) {
-    return {code: ERROR_CODE.ERR_PRESENCE_TRANSFORM_FAILED, message: error.message || error};
+    return new ShareDBError(ERROR_CODE.ERR_PRESENCE_TRANSFORM_FAILED, error.message || error);
   }
 
   presence.v++;

package/lib/projections.js

@@ -97,13 +97,13 @@ function isSnapshotAllowed(fields, snapshot) {
 }
 
 function isEditAllowed(fields, op) {
+  if (!Array.isArray(op)) return false;
   for (var i = 0; i < op.length; i++) {
-    var c = op[i];
-    if (c.p.length === 0) {
-      return false;
-    } else if (!isFieldAllowed(fields, c.p[0])) {
-      return false;
-    }
+    var path = op[i] && op[i].p;
+    if (!Array.isArray(path)) return false;
+    var field = path[0];
+    if (typeof field !== 'string' && typeof field !== 'number') return false;
+    if (!isFieldAllowed(fields, field)) return false;
   }
   return true;
 }

package/lib/pubsub/index.js

@@ -19,6 +19,8 @@ function PubSub(options) {
   // isn't complete until the callback returns from Redis
   // Maps channel -> true
   this.subscribed = Object.create(null);
+  // Maps channel -> callbacks waiting on _subscribe
+  this._subscribing = Object.create(null);
 
   var pubsub = this;
   this._defaultCallback = function(err) {
@@ -76,11 +78,21 @@ PubSub.prototype.subscribe = function(channel, callback) {
     });
     return;
   }
+  if (this._subscribing[channel]) {
+    this._subscribing[channel].push(callback);
+    return;
+  }
+  var callbacks = this._subscribing[channel] = [callback];
   this._subscribe(channel, function(err) {
-    if (err) return callback(err);
+    delete pubsub._subscribing[channel];
+    if (err) return util.callEach(callbacks, err);
     pubsub.subscribed[channel] = true;
-    var stream = pubsub._createStream(channel);
-    callback(null, stream);
+    var streams = callbacks.map(function() {
+      return pubsub._createStream(channel);
+    });
+    callbacks.forEach(function(callback, i) {
+      callback(null, streams[i]);
+    });
   });
 };
 

package/lib/submit-request.js

@@ -109,6 +109,12 @@ SubmitRequest.prototype.submit = function(callback) {
     request.snapshot = snapshot;
     request._addSnapshotMeta();
 
+    // The type is only known once we have the snapshot, so this is the earliest
+    // we can validate the op against it. It has to happen before the op reaches
+    // any type function, including through $fixup() in the apply middleware
+    var opError = ot.checkSubmittedOpForType(snapshot.type, op);
+    if (opError) return callback(opError);
+
     if (op.v == null) {
       if (op.create && snapshot.type && op.src) {
         // If the document was already created by another op, we will return a

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "sharedb",
-  "version": "6.0.4",
+  "version": "7.0.0",
   "description": "JSON OT database backend",
   "main": "lib/index.js",
   "dependencies": {