sorla-db 1.1.0 → 1.3.0

package/src/query_language/manipulation.js

@@ -1,8 +1,11 @@
 const documents = require('../documents/document.js');
 const operatorsCheck = require('./operators.js');
+const emitter = require('../events.js');
+const { applyUpdate } = require('./update.js');
 
-class manipulation {
+class manipulation extends emitter {
     constructor(validationSchema = null) {
+        super();
         this.validationSchema = validationSchema;
         this._index = {};
         this._prohibitedCharacters = ['$', '.'];
@@ -52,106 +55,374 @@ class manipulation {
     }
 
     /**
-     * Verify if exists some document that is not object
-     * @param array documents
+     * Whether a value is a document: an object that is not null nor an array
+     * @param mixed value
      * @return boolean
      */
-    _validateDocumentsType(documents)
+    _isDocument(value)
     {
-        const count = documents.reduce((acc, doc) => {
-            return typeof doc !== 'object' ? ++acc : acc;
-        }, 0);
-
-        return count;
+        return value !== null && typeof value === 'object' && !Array.isArray(value);
     }
 
     /**
-     * Verify if keys has prohibited characters
+     * Verify that no key, at any depth, has prohibited characters
      * @param object document
      * @return boolean
      */
     _validateKeys(document)
     {
-        const keys = Object.keys(document);
         const operators = new operatorsCheck();
-        for(let key of keys)
+        for(const key of Object.keys(document))
         {
-            const hasProhibitedCharacters = this._prohibitedCharacters.reduce((acc, char) => {
-                return key.indexOf(char) > -1 ? ++acc : acc;
-            }, 0);
-
-            if(hasProhibitedCharacters > 0 && !operators.isOperator(key))
+            const hasProhibitedCharacters = this._prohibitedCharacters.some(char => key.indexOf(char) > -1);
+            if(hasProhibitedCharacters && !operators.isOperator(key))
                 return false;
 
-            if(typeof document[key] === 'object')
-                return this._validateKeys(document[key]);
-
-            if(Array.isArray(document[key]))
+            const value = document[key];
+            const nested = Array.isArray(value) ? value : [value];
+            for(const item of nested)
             {
-                for(let value of document[key])
-                {
-                    if(typeof value === 'object')
-                        return this._validateKeys(value);
-                }
+                // null is a valid value, only objects have keys to check
+                if(item !== null && typeof item === 'object' && !this._validateKeys(item))
+                    return false;
             }
         }
 
         return true;
     }
 
+    /**
+     * Set of the _id values of the collection. It is rebuilt when its size no longer matches the
+     * documents, so removals done directly on the documents array are handled too.
+     * @return Set
+     */
+    _ids()
+    {
+        if(!this._idSet || this._idSet.size !== this._documents.length)
+            this._idSet = new Set(this._documents.map(doc => doc._data._id));
+
+        return this._idSet;
+    }
+
+    /**
+     * Copies a document, so the object given by the caller is never changed
+     * @param object document
+     * @return object
+     */
+    _copy(document)
+    {
+        try {
+            return structuredClone(document);
+        } catch(error) {
+            throw new Error('Documents can only contain cloneable values (no functions or DOM nodes)');
+        }
+    }
+
     /**
      * Inserts one document to the collection
      * @author Anderson Arruda < anderson@sysborg.com.br >
-     * @version 1.0.0
+     * @version 1.2.0
      * @param object document
-     * @param function callback - under construction
-     * @return void
+     * @param function callback - receives the inserted _id
+     * @return mixed the _id of the inserted document
     **/
     insertOne(document, callback) 
     {
-        if(typeof document !== 'object' || Array.isArray(document))
+        const id = this._insertDocument(document);
+        this._emitChange('insert', { ids: [id], documents: [this._copy(this._documents.at(-1)._data)] });
+
+        if(typeof callback === 'function')
+            callback(id);
+
+        return id;
+    }
+
+    /**
+     * Validates and stores one document, without events
+     * @param object document
+     * @return mixed the _id
+     */
+    _insertDocument(document)
+    {
+        if(!this._isDocument(document))
             throw new Error('Invalid document type');
 
         if(!this._validateKeys(document))
             throw new Error('Invalid document keys');
 
-        if(typeof document._id === 'undefined')
+        const data = this._copy(document);
+        if(typeof data._id === 'undefined')
         {
-            let _idKey = '';
             do {
-                _idKey = this._generateKey();
-            } while(this._documents.find(doc => doc._id === document._id));
-
-            document._id = _idKey;
+                data._id = this._generateKey();
+            } while(this._ids().has(data._id));
+        }
+        else if(this._ids().has(data._id))
+        {
+            throw new Error(`Duplicate _id "${data._id}"`);
         }
 
-        if(!this._validate(document))
+        if(!this._validate(data))
             throw new Error('Invalid document');
 
-        this._documents.push(new documents(document));
-
-        if(typeof callback === 'function')
-            callback();
+        this._documents.push(new documents(data));
+        this._ids().add(data._id);
+        return data._id;
     }
 
     /**
-     * Insert many documents
+     * Insert many documents. Every document is validated before the first one is inserted,
+     * so an invalid document never leaves the collection with only part of the batch.
      * @author Anderson Arruda < anderson@sysborg.com.br >
-     * @version 1.0.0
+     * @version 1.2.0
      * @param array documents 
-     * @param function callback 
-     * @return void
+     * @param function callback - receives the inserted ids
+     * @return array the _id of each inserted document
      */
     insertMany(documents, callback) {
-        console.info(this._validateDocumentsType(documents));
-        if(!Array.isArray(documents) || this._validateDocumentsType(documents) > 0)
+        if(!Array.isArray(documents) || !documents.every(document => this._isDocument(document)))
             throw new Error('Documents must be an array of objects');
 
-        for(let document of documents)
-            this.insertOne(document);
+        const given = new Set();
+        for(const document of documents)
+        {
+            if(!this._validateKeys(document))
+                throw new Error('Invalid document keys');
+
+            if(typeof document._id === 'undefined')
+                continue;
+
+            if(given.has(document._id) || this._ids().has(document._id))
+                throw new Error(`Duplicate _id "${document._id}"`);
+
+            given.add(document._id);
+        }
+
+        const start = this._documents.length;
+        const ids = documents.map(document => this._insertDocument(document));
+        this._emitChange('insert', { ids, documents: this._documents.slice(start).map(doc => this._copy(doc._data)) });
 
         if(typeof callback === 'function')
-            callback();
+            callback(ids);
+
+        return ids;
+    }
+
+    /**
+     * Helpers the update operators need
+     * @return object
+     */
+    _updateHelpers()
+    {
+        return {
+            copy: value => this._copy(value),
+            validateKeys: value => this._validateKeys(value),
+            operators: this._operators
+        };
+    }
+
+    /**
+     * Document for an upsert: the equality conditions of the filter, then the update
+     * @param object filter
+     * @return object
+     */
+    _upsertBase(filter)
+    {
+        const base = {};
+        for(const [key, value] of Object.entries(filter))
+        {
+            // only top level equality conditions describe the new document
+            if(key.startsWith('$') || key.includes('.') || this._operators.isRegExp(value))
+                continue;
+
+            if(this._operators.isOperatorObject(value))
+            {
+                if('$eq' in value)
+                    base[key] = value.$eq;
+
+                continue;
+            }
+
+            base[key] = value;
+        }
+        return base;
+    }
+
+    /**
+     * Applies an update to the matching documents
+     * @param object filter
+     * @param object update
+     * @param object options {upsert}
+     * @param boolean many
+     * @return object {matchedCount, modifiedCount, upsertedId}
+     */
+    _update(filter, update, options, many)
+    {
+        this._requireFilter(filter);
+        const helpers = this._updateHelpers();
+        const changes = [];
+
+        // compute every new version first: one invalid update leaves all documents untouched
+        for(let i = 0; i < this._documents.length; i++)
+        {
+            const doc = this._documents[i];
+            if(!this._operators.matches(doc, filter))
+                continue;
+
+            changes.push({ index: i, previous: doc._data, next: applyUpdate(doc._data, update, helpers) });
+            if(!many) break;
+        }
+
+        if(!changes.length && options.upsert)
+        {
+            const data = applyUpdate(this._upsertBase(filter), update, helpers);
+            const id = this._insertDocument(data);
+            this._emitChange('insert', { ids: [id], documents: [this._copy(this._documents.at(-1)._data)] });
+            return { matchedCount: 0, modifiedCount: 0, upsertedId: id };
+        }
+
+        const modified = changes.filter(change => JSON.stringify(change.previous) !== JSON.stringify(change.next));
+        for(const change of modified)
+            this._documents[change.index] = new documents(change.next);
+
+        if(modified.length)
+        {
+            this._emitChange('update', {
+                ids: modified.map(change => change.next._id),
+                documents: modified.map(change => this._copy(change.next)),
+                previous: modified.map(change => this._copy(change.previous))
+            });
+        }
+
+        return { matchedCount: changes.length, modifiedCount: modified.length, upsertedId: null };
+    }
+
+    /**
+     * Updates the first matching document: updateOne({name: 'Ana'}, {$set: {age: 30}, $inc: {visits: 1}})
+     * Operators: $set, $unset, $inc, $push (with $each), $pull. The _id cannot change.
+     * @param object filter
+     * @param object update
+     * @param object options {upsert: insert when nothing matches}
+     * @return object {matchedCount, modifiedCount, upsertedId}
+     */
+    updateOne(filter, update, options = {})
+    {
+        return this._update(filter, update, options, false);
+    }
+
+    /**
+     * Updates every matching document, see updateOne()
+     * @param object filter
+     * @param object update
+     * @param object options {upsert}
+     * @return object {matchedCount, modifiedCount, upsertedId}
+     */
+    updateMany(filter, update, options = {})
+    {
+        return this._update(filter, update, options, true);
+    }
+
+    /**
+     * Replaces the first matching document, keeping its _id
+     * @param object filter
+     * @param object replacement a document without update operators
+     * @param object options {upsert}
+     * @return object {matchedCount, modifiedCount, upsertedId}
+     */
+    replaceOne(filter, replacement, options = {})
+    {
+        this._requireFilter(filter);
+        if(!this._isDocument(replacement))
+            throw new Error('The replacement must be a document');
+
+        if(Object.keys(replacement).some(key => key.startsWith('$')))
+            throw new Error('The replacement cannot contain update operators, use updateOne()');
+
+        if(!this._validateKeys(replacement))
+            throw new Error('Invalid document keys');
+
+        const index = this._documents.findIndex(doc => this._operators.matches(doc, filter));
+        if(index === -1)
+        {
+            if(!options.upsert)
+                return { matchedCount: 0, modifiedCount: 0, upsertedId: null };
+
+            const id = this._insertDocument(replacement);
+            this._emitChange('insert', { ids: [id], documents: [this._copy(this._documents.at(-1)._data)] });
+            return { matchedCount: 0, modifiedCount: 0, upsertedId: id };
+        }
+
+        const previous = this._documents[index]._data;
+        if(typeof replacement._id !== 'undefined' && replacement._id !== previous._id)
+            throw new Error('The _id field cannot be changed');
+
+        const next = { ...this._copy(replacement), _id: previous._id };
+        const modified = JSON.stringify(previous) !== JSON.stringify(next);
+        if(modified)
+        {
+            this._documents[index] = new documents(next);
+            this._emitChange('replace', { ids: [next._id], documents: [this._copy(next)], previous: [this._copy(previous)] });
+        }
+
+        return { matchedCount: 1, modifiedCount: modified ? 1 : 0, upsertedId: null };
+    }
+
+    /**
+     * Removes matching documents, keeping the documents array in place
+     * @param object filter
+     * @param boolean many
+     * @return object {deletedCount}
+     */
+    _delete(filter, many)
+    {
+        this._requireFilter(filter);
+        const removed = [];
+        let write = 0;
+        for(let read = 0; read < this._documents.length; read++)
+        {
+            const doc = this._documents[read];
+            if((many || !removed.length) && this._operators.matches(doc, filter))
+                removed.push(doc._data);
+            else
+                this._documents[write++] = doc;
+        }
+        this._documents.length = write;
+
+        if(removed.length)
+        {
+            for(const data of removed)
+                this._idSet?.delete(data._id);
+
+            this._emitChange('delete', { ids: removed.map(data => data._id), documents: removed });
+        }
+
+        return { deletedCount: removed.length };
+    }
+
+    /**
+     * Deletes the first matching document
+     * @param object filter
+     * @return object {deletedCount}
+     */
+    deleteOne(filter)
+    {
+        return this._delete(filter, false);
+    }
+
+    /**
+     * Deletes every matching document; deleteMany({}) empties the collection
+     * @param object filter
+     * @return object {deletedCount}
+     */
+    deleteMany(filter)
+    {
+        return this._delete(filter, true);
+    }
+
+    _requireFilter(filter)
+    {
+        if(!this._isDocument(filter))
+            throw new Error('The filter must be a query object, use {} to match every document');
     }
 
     //create index

package/src/query_language/operators.js

@@ -4,7 +4,7 @@ class operators {
      */
     constructor() {
         this._logicalOperators = ['$or', '$and', '$not', '$nor'];
-        this._comparisonOperators = ['$lt', '$lte', '$gt', '$gte', '$eq', '$ne', '$in', '$nin'];
+        this._comparisonOperators = ['$lt', '$lte', '$gt', '$gte', '$eq', '$ne', '$in', '$nin', '$exists', '$regex', '$options'];
     }
 
     /**
@@ -94,9 +94,15 @@ class operators {
      * @return boolean
      */
     matchesCondition(value, condition) {
+        if(this.isRegExp(condition))
+            return this.matchesRegex(value, condition);
+
         if(!this.isOperatorObject(condition))
             return this.$eq(value, condition);
 
+        if('$options' in condition && !('$regex' in condition))
+            throw new Error('The $options operator must be used with $regex');
+
         for(const operator of Object.keys(condition)) {
             if(operator === '$not') {
                 if(this.matchesCondition(value, condition[operator]))
@@ -105,6 +111,16 @@ class operators {
                 continue;
             }
 
+            if(operator === '$options')
+                continue;
+
+            if(operator === '$regex') {
+                if(!this.matchesRegex(value, this.toRegExp(condition.$regex, condition.$options)))
+                    return false;
+
+                continue;
+            }
+
             if(!this.isOperator(operator))
                 throw new Error(`Unknown operator ${operator}`);
 
@@ -115,6 +131,45 @@ class operators {
         return true;
     }
 
+    /**
+     * Whether a value is a RegExp (also from another realm: iframes, workers, vm contexts)
+     * @param mixed value
+     * @return boolean
+     */
+    isRegExp(value) {
+        return Object.prototype.toString.call(value) === '[object RegExp]';
+    }
+
+    /**
+     * Builds the RegExp of a $regex condition
+     * @param string|RegExp pattern
+     * @param string options i (case insensitive), m (multiline), s (dot matches new lines)
+     * @return RegExp
+     */
+    toRegExp(pattern, options = '') {
+        if(typeof options !== 'string' || /[^ims]/.test(options))
+            throw new Error('The $options operator accepts only the flags i, m and s');
+
+        if(this.isRegExp(pattern))
+            return new RegExp(pattern.source, [...new Set((pattern.flags + options).replace(/[gy]/g, ''))].join(''));
+
+        if(typeof pattern !== 'string')
+            throw new Error('The $regex operator must be a string or a RegExp');
+
+        return new RegExp(pattern, options);
+    }
+
+    /**
+     * Strings match when the pattern finds them, arrays when one of their strings does
+     * @param mixed value
+     * @param RegExp regex
+     * @return boolean
+     */
+    matchesRegex(value, regex) {
+        const test = item => typeof item === 'string' && (regex.lastIndex = 0, regex.test(item));
+        return Array.isArray(value) ? value.some(test) : test(value);
+    }
+
     /**
      * Whether a condition is an object made of operators, like {$gt: 5}
      * @param mixed condition
@@ -128,6 +183,16 @@ class operators {
         return keys.length > 0 && keys.every(key => key.startsWith('$'));
     }
 
+    /**
+     * Exists operator: {field: {$exists: true}} matches documents that have the field (even null)
+     * @param mixed doc_value
+     * @param boolean exists
+     * @return boolean
+     */
+    get $exists() {
+        return (doc_value, exists) => (doc_value !== undefined) === !!exists;
+    }
+
     /**
      * Not in operator
      * @param mixed doc_value

package/src/query_language/query.js

@@ -14,9 +14,13 @@ class query extends manipulation
      * Count documents
      * @return int
      */
-    count()
+    count(query)
     {
-        return this._documents.length;
+        if(typeof query === 'undefined')
+            return this._documents.length;
+
+        // counts without copying the documents
+        return this._documents.reduce((total, doc) => total + (this._operators.matches(doc, query) ? 1 : 0), 0);
     }
 
     /**