wingbot-mongodb 3.2.0 → 3.2.2

package/.nyc_output/processinfo/{cb79cd4d-6c45-40e4-b29a-e38af10c7c7c.json → 98a862df-b2a1-4d14-9176-6b0bada108f7.json}

@@ -1 +1 @@
-{"parent":null,"pid":24990,"argv":["/usr/local/bin/node","/Users/david/Development/wingbot-mongodb/node_modules/.bin/mocha","./test"],"execArgv":[],"cwd":"/Users/david/Development/wingbot-mongodb","time":1676028268542,"ppid":24989,"coverageFilename":"/Users/david/Development/wingbot-mongodb/.nyc_output/cb79cd4d-6c45-40e4-b29a-e38af10c7c7c.json","externalId":"","uuid":"cb79cd4d-6c45-40e4-b29a-e38af10c7c7c","files":["/Users/david/Development/wingbot-mongodb/src/BaseStorage.js","/Users/david/Development/wingbot-mongodb/src/defaultLogger.js","/Users/david/Development/wingbot-mongodb/src/AttachmentCache.js","/Users/david/Development/wingbot-mongodb/src/AuditLogStorage.js","/Users/david/Development/wingbot-mongodb/src/BotConfigStorage.js","/Users/david/Development/wingbot-mongodb/src/BotTokenStorage.js","/Users/david/Development/wingbot-mongodb/src/tokenFactory.js","/Users/david/Development/wingbot-mongodb/src/ChatLogStorage.js","/Users/david/Development/wingbot-mongodb/src/NotificationsStorage.js","/Users/david/Development/wingbot-mongodb/src/StateStorage.js"]}
+{"parent":null,"pid":81134,"argv":["/usr/local/bin/node","/Users/david/Development/wingbot-mongodb/node_modules/.bin/mocha","./test"],"execArgv":[],"cwd":"/Users/david/Development/wingbot-mongodb","time":1676533760202,"ppid":81133,"coverageFilename":"/Users/david/Development/wingbot-mongodb/.nyc_output/98a862df-b2a1-4d14-9176-6b0bada108f7.json","externalId":"","uuid":"98a862df-b2a1-4d14-9176-6b0bada108f7","files":["/Users/david/Development/wingbot-mongodb/src/BaseStorage.js","/Users/david/Development/wingbot-mongodb/src/defaultLogger.js","/Users/david/Development/wingbot-mongodb/src/AttachmentCache.js","/Users/david/Development/wingbot-mongodb/src/AuditLogStorage.js","/Users/david/Development/wingbot-mongodb/src/BotConfigStorage.js","/Users/david/Development/wingbot-mongodb/src/BotTokenStorage.js","/Users/david/Development/wingbot-mongodb/src/tokenFactory.js","/Users/david/Development/wingbot-mongodb/src/ChatLogStorage.js","/Users/david/Development/wingbot-mongodb/src/NotificationsStorage.js","/Users/david/Development/wingbot-mongodb/src/StateStorage.js"]}

package/.nyc_output/processinfo/index.json

@@ -1 +1 @@
-{"processes":{"cb79cd4d-6c45-40e4-b29a-e38af10c7c7c":{"parent":null,"children":[]}},"files":{"/Users/david/Development/wingbot-mongodb/src/BaseStorage.js":["cb79cd4d-6c45-40e4-b29a-e38af10c7c7c"],"/Users/david/Development/wingbot-mongodb/src/defaultLogger.js":["cb79cd4d-6c45-40e4-b29a-e38af10c7c7c"],"/Users/david/Development/wingbot-mongodb/src/AttachmentCache.js":["cb79cd4d-6c45-40e4-b29a-e38af10c7c7c"],"/Users/david/Development/wingbot-mongodb/src/AuditLogStorage.js":["cb79cd4d-6c45-40e4-b29a-e38af10c7c7c"],"/Users/david/Development/wingbot-mongodb/src/BotConfigStorage.js":["cb79cd4d-6c45-40e4-b29a-e38af10c7c7c"],"/Users/david/Development/wingbot-mongodb/src/BotTokenStorage.js":["cb79cd4d-6c45-40e4-b29a-e38af10c7c7c"],"/Users/david/Development/wingbot-mongodb/src/tokenFactory.js":["cb79cd4d-6c45-40e4-b29a-e38af10c7c7c"],"/Users/david/Development/wingbot-mongodb/src/ChatLogStorage.js":["cb79cd4d-6c45-40e4-b29a-e38af10c7c7c"],"/Users/david/Development/wingbot-mongodb/src/NotificationsStorage.js":["cb79cd4d-6c45-40e4-b29a-e38af10c7c7c"],"/Users/david/Development/wingbot-mongodb/src/StateStorage.js":["cb79cd4d-6c45-40e4-b29a-e38af10c7c7c"]},"externalIds":{}}
+{"processes":{"98a862df-b2a1-4d14-9176-6b0bada108f7":{"parent":null,"children":[]}},"files":{"/Users/david/Development/wingbot-mongodb/src/BaseStorage.js":["98a862df-b2a1-4d14-9176-6b0bada108f7"],"/Users/david/Development/wingbot-mongodb/src/defaultLogger.js":["98a862df-b2a1-4d14-9176-6b0bada108f7"],"/Users/david/Development/wingbot-mongodb/src/AttachmentCache.js":["98a862df-b2a1-4d14-9176-6b0bada108f7"],"/Users/david/Development/wingbot-mongodb/src/AuditLogStorage.js":["98a862df-b2a1-4d14-9176-6b0bada108f7"],"/Users/david/Development/wingbot-mongodb/src/BotConfigStorage.js":["98a862df-b2a1-4d14-9176-6b0bada108f7"],"/Users/david/Development/wingbot-mongodb/src/BotTokenStorage.js":["98a862df-b2a1-4d14-9176-6b0bada108f7"],"/Users/david/Development/wingbot-mongodb/src/tokenFactory.js":["98a862df-b2a1-4d14-9176-6b0bada108f7"],"/Users/david/Development/wingbot-mongodb/src/ChatLogStorage.js":["98a862df-b2a1-4d14-9176-6b0bada108f7"],"/Users/david/Development/wingbot-mongodb/src/NotificationsStorage.js":["98a862df-b2a1-4d14-9176-6b0bada108f7"],"/Users/david/Development/wingbot-mongodb/src/StateStorage.js":["98a862df-b2a1-4d14-9176-6b0bada108f7"]},"externalIds":{}}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "wingbot-mongodb",
-  "version": "3.2.0",
+  "version": "3.2.2",
   "description": "MongoDB storage for wingbot.ai",
   "main": "src/main.js",
   "scripts": {

package/src/BaseStorage.js

@@ -33,12 +33,31 @@ function getNestedObjects (obj, nested, attr = null, ret = {}) {
 
 class BaseStorage {
 
+    static netFailuresIntervalMs = 600000; // 10 minutes
+
+    static netFailuresCount = 0;
+
+    static _killing = null;
+
+    static _failStack = [];
+
+    static killer = () => setTimeout(() => {
+        process.exit(1);
+    }, 10000);
+
+    static networkFailureErrorNames = [
+        'MongoServerSelectionError',
+        'MongoNetworkError',
+        'MongoNetworkTimeoutError',
+        'MongoTopologyClosedError'
+    ];
+
     /**
      *
      * @param {Db|{():Promise<Db>}} mongoDb
      * @param {string} collectionName
      * @param {{error:Function,log:Function}} [log] - console like logger
-     * @param {boolean} [isCosmo]
+     * @param {boolean|number} [isCosmo]
      * @example
      *
      * const { BaseStorage } = require('winbot-mongodb');
@@ -67,7 +86,7 @@ class BaseStorage {
     constructor (mongoDb, collectionName, log = defaultLogger, isCosmo = false) {
         this._mongoDb = mongoDb;
         this._collectionName = collectionName;
-        this._isCosmo = isCosmo;
+        this._isCosmo = typeof isCosmo === 'number' || isCosmo;
         this._log = log;
 
         /**
@@ -84,7 +103,11 @@ class BaseStorage {
 
         this._fixtures = [];
 
-        if (isCosmo && !process.argv.some((a) => a.endsWith('mocha'))) {
+        if (typeof isCosmo === 'number') {
+            BaseStorage.netFailuresCount = isCosmo;
+        }
+
+        if (this._isCosmo && !process.argv.some((a) => a.endsWith('mocha'))) {
             process.nextTick(() => {
                 const hasUniqueIndex = this._indexes.some((i) => i.options.unique);
 
@@ -96,6 +119,45 @@ class BaseStorage {
         }
     }
 
+    /**
+     *
+     * @template T
+     * @param {{():T}} fn
+     * @returns {Promise<T>}
+     */
+    _catchNetworkIssues (fn) {
+        return Promise.resolve(fn())
+            .catch((e) => {
+                this._detectNetworkIssueException(e);
+                throw e;
+            });
+    }
+
+    /**
+     * @template {Error} T
+     * @param {T} e
+     * @returns {T}
+     */
+    _detectNetworkIssueException (e) {
+        if (BaseStorage.netFailuresCount !== 0
+            && BaseStorage.networkFailureErrorNames.includes(e.name)) {
+
+            const now = Date.now();
+            BaseStorage._failStack.push(now);
+
+            if (BaseStorage._failStack.length >= BaseStorage.netFailuresCount
+                && BaseStorage._failStack.shift() > (now - BaseStorage.netFailuresIntervalMs)
+                && BaseStorage._killing === null) {
+
+                this._log.error(`${this._collectionName}: KILLING APP DUE FREQUENT NETWORK ERRORS ${BaseStorage._failStack.length}`, e);
+
+                // let it alive for following ten seconds to process all logs
+                BaseStorage._killing = BaseStorage.killer() || null;
+            }
+        }
+        return e;
+    }
+
     preHeat () {
         return this._getCollection();
     }
@@ -201,6 +263,7 @@ class BaseStorage {
                 this._collection = c;
             } catch (e) {
                 this._collection = null;
+                this._detectNetworkIssueException(e);
                 throw e;
             }
         }

package/src/NotificationsStorage.js

@@ -303,13 +303,14 @@ class NotificationsStorage {
 
         const findMissingIds = tasks
             .reduce((arr, {
-                campaignId, senderId, pageId, sent
+                campaignId, senderId, pageId, sent, enqueue
             }, i) => {
                 if (typeof res.upsertedIds[i] !== 'undefined') {
                     return arr;
                 }
                 arr.push({
                     i,
+                    enqueue,
                     filter: {
                         campaignId, senderId, pageId, sent
                     }
@@ -321,23 +322,31 @@ class NotificationsStorage {
 
         if (findMissingIds.length > 0) {
             await Promise.all(findMissingIds
-                .map(({ filter, i }) => c.findOne(filter, {
+                .map(({ filter, i, enqueue }) => c.findOne(filter, {
                     projection: {
                         _id: 1, insEnqueue: 1, enqueue: 1, ups: 1
                     }
                 })
                     .then((found) => {
-                        const id = typeof found._id === 'string'
-                            ? found._id
-                            : found._id.toHexString();
-                        missingIds.set(i, {
-                            id,
-                            insEnqueue: found.insEnqueue,
-                            enqueue: found.insEnqueue === found.enqueue
-                                && found.enqueue !== MAX_TS && found.ups !== 1
-                                ? found.enqueue + 1
-                                : found.enqueue
-                        });
+                        if (!found) { // race condition occurred
+                            missingIds.set(i, {
+                                id: null,
+                                insEnqueue: -1,
+                                enqueue
+                            });
+                        } else {
+                            const id = typeof found._id === 'string'
+                                ? found._id
+                                : found._id.toHexString();
+                            missingIds.set(i, {
+                                id,
+                                insEnqueue: found.insEnqueue,
+                                enqueue: found.insEnqueue === found.enqueue
+                                    && found.enqueue !== MAX_TS && found.ups !== 1
+                                    ? found.enqueue + 1
+                                    : found.enqueue
+                            });
+                        }
                     })));
         }
 

package/src/StateStorage.js

@@ -40,7 +40,7 @@ class StateStorage extends BaseStorage {
      * @param {boolean|number} isCosmo - boolean or number of failures in last 10min to kill app
      */
     constructor (mongoDb, collectionName = 'states', log = defaultLogger, isCosmo = false) {
-        super(mongoDb, collectionName, log, typeof isCosmo === 'number' || isCosmo);
+        super(mongoDb, collectionName, log, isCosmo);
 
         this.addIndex(
             { senderId: 1, pageId: 1 },
@@ -51,10 +51,6 @@ class StateStorage extends BaseStorage {
             { name: LAST_INTERACTION_INDEX }
         );
 
-        this._failStack = [];
-
-        this._failures = typeof isCosmo === 'number' ? isCosmo : 0;
-
         if (isCosmo) {
             this.addIndex(
                 { name: 1 },
@@ -66,21 +62,6 @@ class StateStorage extends BaseStorage {
                 { name: SEARCH }
             );
         }
-
-        this.failuresIntervalMs = 600000; // 10 minutes
-
-        this._killing = null;
-
-        this.networkFailureErrorNames = [
-            'MongoServerSelectionError',
-            'MongoNetworkError',
-            'MongoNetworkTimeoutError',
-            'MongoTopologyClosedError'
-        ];
-
-        this.killer = () => setTimeout(() => {
-            process.exit(1);
-        }, 10000);
     }
 
     /**
@@ -118,59 +99,37 @@ class StateStorage extends BaseStorage {
      * @returns {Promise<object>} - conversation state
      */
     async getOrCreateAndLock (senderId, pageId, defaultState = {}, timeout = 300) {
-        let now = Date.now();
-        try {
-            const c = await this._getCollection();
-
-            const $setOnInsert = {
-                state: defaultState,
-                lastSendError: null,
-                off: false
-            };
-
-            const $set = {
-                lock: now
-            };
-
-            const $lt = now - timeout;
-
-            const res = await c.findOneAndUpdate({
-                senderId,
-                pageId,
-                lock: { $lt }
-            }, {
-                $setOnInsert,
-                $set
-            }, {
-                upsert: true,
-                returnDocument: 'after',
-                projection: {
-                    _id: 0
-                }
-            });
-
-            return res.value;
-        } catch (e) {
-            if (this._failures !== 0
-                && this.networkFailureErrorNames.includes(e.name)) {
-
-                now = Date.now();
-                this._failStack.push(now);
-
-                if (this._failStack.length >= this._failures
-                    && this._failStack.shift() > (now - this.failuresIntervalMs)
-                    && this._killing === null) {
-
-                    this._log.error(`StateStorage: KILLING APP DUE FREQUENT NETWORK ERRORS ${this._failStack.length}`, e);
-
-                    // let it alive for following ten seconds to process all logs
-                    this._killing = this.killer() || null;
-                }
-            }
+        const now = Date.now();
+        const c = await this._getCollection();
 
-            throw e;
-        }
+        const $setOnInsert = {
+            state: defaultState,
+            lastSendError: null,
+            off: false
+        };
+
+        const $set = {
+            lock: now
+        };
+
+        const $lt = now - timeout;
+
+        const res = await this._catchNetworkIssues(() => c.findOneAndUpdate({
+            senderId,
+            pageId,
+            lock: { $lt }
+        }, {
+            $setOnInsert,
+            $set
+        }, {
+            upsert: true,
+            returnDocument: 'after',
+            projection: {
+                _id: 0
+            }
+        }));
 
+        return res.value;
     }
 
     /**