wingbot-mongodb 2.18.0 → 2.19.0

package/src/AuditLogStorage.js

@@ -0,0 +1,364 @@
+/**
+ * @author wingbot.ai
+ */
+'use strict';
+
+const jsonwebtoken = require('jsonwebtoken');
+const BaseStorage = require('./BaseStorage');
+
+/** @typedef {import('mongodb/lib/db')} Db */
+
+const LEVEL_CRITICAL = 'Critical';
+const LEVEL_IMPORTANT = 'Important';
+const LEVEL_DEBUG = 'Debug';
+
+const TYPE_ERROR = 'Error';
+const TYPE_WARN = 'Warn';
+const TYPE_INFO = 'Info';
+
+/**
+ * @typedef {object} TrackingEvent
+ * @prop {string} [type='audit']
+ * @prop {string} category
+ * @prop {string} action
+ * @prop {string} [label]
+ * @prop {object} [payload]
+ */
+
+/**
+ * @typedef {object} User
+ * @prop {string} [id]
+ * @prop {string} [senderId]
+ * @prop {string} [pageId]
+ * @prop {string} [jwt] - jwt to check the authorship
+ */
+
+/**
+ * @typedef {object} Meta
+ * @prop {string} [ip]
+ * @prop {string} [ua]
+ * @prop {string} [ro] - referrer || origin
+ */
+
+/**
+ * @typedef {object} LogEntry
+ * @prop {string} date - ISO date
+ * @prop {number} delta - time skew in ms if there was a write conflict
+ * @prop {string} [eventType='audit']
+ * @prop {string} category
+ * @prop {string} action
+ * @prop {string} [label]
+ * @prop {object} [payload]
+ * @prop {string} level - (Critical|Important|Debug)
+ * @prop {boolean} ok - signature matches
+ * @prop {number} seq - sequence number
+ * @prop {string} type - (Error|Warn|Info)
+ * @prop {User} user
+ * @prop {string} wid - workspace id
+ * @prop {Meta} meta
+ */
+
+/**
+ * JWT Verifier
+ *
+ * @callback JwtVerifier
+ * @param {string} token
+ * @param {string} userId
+ * @param {User} [user]
+ * @returns {Promise<boolean>}
+ */
+
+/**
+ * @typedef {object} AuditLogEntry
+ * @prop {string} date - ISO date
+ * @prop {string} [eventType='audit']
+ * @prop {string} category
+ * @prop {string} action
+ * @prop {string} [label]
+ * @prop {object} [payload]
+ * @prop {string} level - (Critical|Important|Debug)
+ * @prop {string} type - (Error|Warn|Info)
+ * @prop {User} user
+ * @prop {string} wid - workspace id
+ * @prop {Meta} meta
+ */
+
+/**
+ * Audit Log Callback
+ *
+ * @callback AuditLogCallback
+ * @param {AuditLogEntry} entry
+ * @returns {Promise}
+ */
+
+/**
+ * Storage for audit logs with signatures chain
+ */
+class AuditLogStorage extends BaseStorage {
+
+    /**
+     *
+     * @param {Db|{():Promise<Db>}} mongoDb
+     * @param {string} collectionName
+     * @param {{error:Function,log:Function}} [log] - console like logger
+     * @param {boolean} [isCosmo]
+     * @param {string|Promise<string>} [secret]
+     * @param {string|Promise<string>} [jwtVerifier]
+     */
+    constructor (mongoDb, collectionName = 'auditlog', log = console, isCosmo = false, secret = null, jwtVerifier = null) {
+        super(mongoDb, collectionName, log, isCosmo);
+
+        this.addIndex({
+            wid: 1,
+            seq: -1
+        }, {
+            unique: true,
+            name: 'wid_1_seq_-1'
+        });
+
+        if (isCosmo) {
+            this.addIndex({
+                wid: 1
+            }, {
+                name: 'wid_1'
+            });
+
+            this.addIndex({
+                seq: -1
+            }, {
+                name: 'seq_-1'
+            });
+        } else {
+            this.addIndex({
+                wid: 1, date: -1
+            }, {
+                name: 'wid_1_date_-1'
+            });
+        }
+
+        this.defaultWid = '0';
+
+        this.muteErrors = true;
+        this.maxRetries = 4;
+        this._secret = secret;
+
+        this.LEVEL_CRITICAL = LEVEL_CRITICAL;
+        this.LEVEL_IMPORTANT = LEVEL_IMPORTANT;
+        this.LEVEL_DEBUG = LEVEL_DEBUG;
+
+        this.TYPE_ERROR = TYPE_ERROR;
+        this.TYPE_WARN = TYPE_WARN;
+        this.TYPE_INFO = TYPE_INFO;
+
+        /**
+         * @type {JwtVerifier}
+         */
+        // @ts-ignore
+        this._jwtVerify = typeof jwtVerifier === 'function' || jwtVerifier === null
+            ? jwtVerifier
+            : async (token, userId) => {
+                const jwtSec = await Promise.resolve(jwtVerifier);
+                const decoded = await new Promise((resolve) => {
+                    jsonwebtoken.verify(token, jwtSec, (err, res) => {
+                        resolve(res || {});
+                    });
+                });
+                return decoded.id === userId;
+            };
+
+        /** @type {AuditLogCallback} */
+        this.callback = null;
+    }
+
+    /**
+     * Add a log
+     *
+     * @param {TrackingEvent} event
+     * @param {User} user
+     * @param {Meta} [meta]
+     * @param {string} [wid] - workspace ID
+     * @param {string} [type]
+     * @param {string} [level]
+     * @param {Date} [date]
+     * @returns {Promise}
+     */
+    async log (
+        event,
+        user = {},
+        meta = {},
+        wid = this.defaultWid,
+        type = TYPE_INFO,
+        level = LEVEL_IMPORTANT,
+        date = new Date()
+    ) {
+        const {
+            type: eventType = 'audit',
+            ...rest
+        } = event;
+        const entry = {
+            date,
+            eventType,
+            ...rest,
+            level,
+            meta,
+            type,
+            user,
+            wid
+        };
+
+        const secret = await Promise.resolve(this._secret);
+        const stored = await this._storeWithRetry(secret, entry);
+        if (!this.callback) {
+            return;
+        }
+        try {
+            await this.callback(stored);
+        } catch (e) {
+            this._log.error('Failed to send AuditLog', e, entry);
+        }
+
+        // logEvent(level, type, workflowType, workflowInstance, eventData, account)
+        // level is the criticality of the event ('Critical','Important','Debug').
+        // type is the type of the event ('Error','Warn','Info').
+        /**
+         *  - log (N/A)
+            - report (Debug)
+            - conversation (N/A)
+            - audit (Important)
+            - user (N/A)
+         */
+    }
+
+    /**
+     *
+     * @param {string} [wid] - workspace id
+     * @param {number} [fromSeq] - for paging
+     * @param {number} [limit]
+     * @returns {Promise<LogEntry[]>}
+     */
+    async list (wid = this.defaultWid, fromSeq = 0, limit = 40) {
+        const c = await this._getCollection();
+
+        const cond = { wid };
+
+        if (fromSeq) {
+            cond.seq = { $lt: fromSeq };
+        }
+
+        const data = await c.find({ wid })
+            .limit(limit + 1)
+            .sort({ seq: -1 })
+            .project({ _id: 0 })
+            .toArray();
+
+        const secret = await Promise.resolve(this._secret);
+
+        const len = data.length === limit + 1
+            ? data.length - 1
+            : data.length;
+        const ret = new Array(len);
+
+        let verifyUser = false;
+        for (let i = 0; i < len; i++) {
+            const {
+                sign,
+                ...log
+            } = data[i];
+
+            verifyUser = verifyUser || (log.user.id && log.user.jwt);
+
+            if (secret) {
+                const previous = data[i + 1] || { sign: null };
+                const objToSign = this._objectToSign(log);
+                const compare = this._signWithSecret(objToSign, secret, previous.sign);
+                log.ok = compare === sign;
+                if (!log.ok) {
+                    this._log.error(`AuditLog: found wrong signature at wid: "${log.wid}", seq: "${log.seq}"`, log);
+                }
+            } else {
+                log.ok = null;
+            }
+
+            ret[i] = log;
+        }
+
+        if (verifyUser && this._jwtVerify) {
+            return Promise.all(
+                ret.map(async (log) => {
+                    if (!log.user.id || !log.user.jwt) {
+                        return log;
+                    }
+                    const userChecked = await this._jwtVerify(log.user.jwt, log.user.id, log.user);
+                    return Object.assign(log, {
+                        // it's ok, when there was null
+                        ok: log.ok !== false && userChecked
+                    });
+                })
+            );
+        }
+
+        return ret;
+    }
+
+    _wait (ms) {
+        return new Promise((r) => setTimeout(r, ms));
+    }
+
+    async _storeWithRetry (secret, entry, delta = 0, i = 1) {
+        const start = Date.now();
+        Object.assign(entry, { delta });
+        try {
+            await this._store(entry, secret);
+            return entry;
+        } catch (e) {
+            if (e.code === 11000) {
+                // duplicate key
+                if (i >= this.maxRetries) {
+                    throw new Error('AuditLog: cannot store log due to max-retries');
+                } else {
+                    await this._wait((i * 50) + (Math.random() * 100));
+                    const add = Date.now() - start;
+                    return this._storeWithRetry(secret, entry, delta + add, i + 1);
+                }
+            } else if (this.muteErrors) {
+                this._log.error('Audit log store error', e, entry);
+                return entry;
+            } else {
+                throw e;
+            }
+        }
+    }
+
+    async _store (entry, secret) {
+        const c = await this._getCollection();
+
+        const previous = await c.findOne({
+            wid: entry.wid
+        }, {
+            sort: { seq: -1 },
+            projection: { seq: 1, _id: 0, sign: 1 }
+        });
+
+        Object.assign(entry, {
+            seq: previous ? previous.seq + 1 : 0
+        });
+
+        let insert;
+        if (secret) {
+            insert = this._objectToSign(entry);
+            const sign = this._signWithSecret(insert, secret, previous ? previous.sign : null);
+            Object.assign(insert, { sign });
+        } else {
+            insert = {
+                ...entry,
+                sign: null
+            };
+        }
+
+        // @ts-ignore
+        await c.insertOne(insert);
+    }
+
+}
+
+module.exports = AuditLogStorage;

package/src/BaseStorage.js

@@ -4,15 +4,19 @@
 'use strict';
 
 const mongodb = require('mongodb'); // eslint-disable-line no-unused-vars
+const crypto = require('crypto');
+
+/** @typedef {import('mongodb/lib/db')} Db */
+/** @typedef {import('mongodb/lib/collection')} Collection */
 
 class BaseStorage {
 
     /**
      *
-     * @param {mongodb.Db|{():Promise<mongodb.Db>}} mongoDb
+     * @param {Db|{():Promise<Db>}} mongoDb
      * @param {string} collectionName
      * @param {{error:Function,log:Function}} [log] - console like logger
-     * @param {boolean} isCosmo
+     * @param {boolean} [isCosmo]
      * @example
      *
      * const { BaseStorage } = require('winbot-mongodb');
@@ -45,11 +49,14 @@ class BaseStorage {
         this._log = log;
 
         /**
-         * @type {Promise<mongodb.Collection>}
+         * @type {Collection|Promise<Collection>}
          */
         this._collection = null;
 
         this._indexes = [];
+
+        this.ignoredSignatureKeys = ['_id', 'sign'];
+        this._secret = null;
     }
 
     /**
@@ -101,7 +108,7 @@ class BaseStorage {
     /**
      * Returns the collection to operate with
      *
-     * @returns {Promise<mongodb.Collection>}
+     * @returns {Promise<Collection>}
      */
     async _getCollection () {
         if (this._collection === null) {
@@ -152,6 +159,44 @@ class BaseStorage {
             }, Promise.resolve());
     }
 
+    async _sign (object) {
+        if (!this._secret) {
+            return object;
+        }
+        const secret = await Promise.resolve(this._secret);
+        const objToSign = this._objectToSign(object);
+        const sign = this._signWithSecret(objToSign, secret);
+
+        return Object.assign(objToSign, {
+            sign
+        });
+    }
+
+    _objectToSign (object) {
+        const entries = Object.keys(object)
+            .filter((key) => !this.ignoredSignatureKeys.includes(key));
+
+        entries.sort();
+
+        return entries.reduce((o, key) => {
+            let val = object[key];
+            if (val instanceof Date) {
+                val = val.toISOString();
+            }
+            return Object.assign(o, { [key]: val });
+        }, {});
+    }
+
+    _signWithSecret (objToSign, secret, previous = null) {
+        const h = crypto.createHmac('sha3-224', secret)
+            .update(JSON.stringify(objToSign));
+
+        if (previous) {
+            h.update(previous);
+        }
+
+        return h.digest('base64');
+    }
 }
 
 module.exports = BaseStorage;

package/src/BotTokenStorage.js

@@ -131,7 +131,7 @@ class BotTokenStorage {
             }
         }, {
             upsert: true,
-            returnOriginal: false
+            returnDocument: 'after'
         });
 
         res = res.value;

package/src/ChatLogStorage.js

@@ -3,12 +3,13 @@
 */
 'use strict';
 
-const mongodb = require('mongodb'); // eslint-disable-line no-unused-vars
 const BaseStorage = require('./BaseStorage');
 
 const PAGE_SENDER_TIMESTAMP = 'pageId_1_senderId_1_timestamp_-1';
 const TIMESTAMP = 'timestamp_1';
 
+/** @typedef {import('mongodb/lib/db')} Db */
+
 /**
  * Storage for conversation logs
  *
@@ -18,12 +19,13 @@ class ChatLogStorage extends BaseStorage {
 
     /**
      *
-     * @param {mongodb.Db|{():Promise<mongodb.Db>}} mongoDb
+     * @param {Db|{():Promise<Db>}} mongoDb
      * @param {string} collectionName
      * @param {{error:Function,log:Function}} [log] - console like logger
-     * @param {boolean} isCosmo
+     * @param {boolean} [isCosmo]
+     * @param {string|Promise<string>} [secret]
      */
-    constructor (mongoDb, collectionName = 'chatlogs', log = console, isCosmo = false) {
+    constructor (mongoDb, collectionName = 'chatlogs', log = console, isCosmo = false, secret = null) {
         super(mongoDb, collectionName, log, isCosmo);
 
         this.addIndex({
@@ -43,6 +45,7 @@ class ChatLogStorage extends BaseStorage {
         }
 
         this.muteErrors = true;
+        this._secret = secret;
     }
 
     /**
@@ -54,6 +57,7 @@ class ChatLogStorage extends BaseStorage {
      * @param {number} [limit]
      * @param {number} [endAt] - iterate backwards to history
      * @param {number} [startAt] - iterate forward to last interaction
+     * @returns {Promise<object[]>}
      */
     async getInteractions (senderId, pageId, limit = 10, endAt = null, startAt = null) {
         const c = await this._getCollection();
@@ -80,14 +84,33 @@ class ChatLogStorage extends BaseStorage {
         const res = await c.find(q)
             .limit(limit)
             .sort({ timestamp: orderBackwards ? 1 : -1 })
-            .project({ _id: 0, time: 0 })
+            .project({ _id: 0 })
             .toArray();
 
         if (!orderBackwards) {
             res.reverse();
         }
 
-        return res;
+        if (!this._secret) {
+            return res.map((r) => Object.assign(r, { ok: null }));
+        }
+
+        const secret = await Promise.resolve(this._secret);
+
+        return res.map((r) => {
+            const {
+                sign,
+                ...log
+            } = r;
+            const objToSign = this._objectToSign(log);
+            const compare = this._signWithSecret(objToSign, secret);
+            const ok = compare === sign;
+            if (!ok) {
+                this._log.error(`ChatLog: found wrong signature at pageId: "${r.pageId}", senderId: "${r.senderId}", at: ${r.timestamp}`, r);
+            }
+
+            return Object.assign(log, { ok });
+        });
     }
 
     /**
@@ -102,22 +125,38 @@ class ChatLogStorage extends BaseStorage {
     log (senderId, responses = [], request = {}, metadata = {}) {
         const log = {
             senderId,
-            time: new Date(request.timestamp || Date.now()),
             request,
-            responses
+            responses,
+            ...metadata
         };
 
-        Object.assign(log, metadata);
-
-        return this._getCollection()
-            .then((c) => c.insertOne(log))
-            .catch((err) => {
-                this._log.error('Failed to store chat log', err, log);
+        return this._storeLog(log);
+    }
 
-                if (!this.muteErrors) {
-                    throw err;
-                }
+    async _storeLog (event) {
+        let log = event;
+        if (!event.timestamp) {
+            Object.assign(event, {
+                timestamp: event.request.timestamp || Date.now()
+            });
+        }
+        if (typeof event.pageId === 'undefined') {
+            Object.assign(event, {
+                pageId: null
             });
+        }
+        try {
+            const c = await this._getCollection();
+            log = await this._sign(log);
+            // @ts-ignore
+            await c.insertOne(log);
+        } catch (e) {
+            this._log.error('Failed to store chat log', e, log);
+
+            if (!this.muteErrors) {
+                throw e;
+            }
+        }
     }
 
     /**
@@ -135,23 +174,15 @@ class ChatLogStorage extends BaseStorage {
     error (err, senderId, responses = [], request = {}, metadata = {}) {
         const log = {
             senderId,
-            time: new Date(request.timestamp || Date.now()),
             request,
             responses,
-            err: `${err}`
+            err: `${err}`,
+            ...metadata
         };
 
         Object.assign(log, metadata);
 
-        return this._getCollection()
-            .then((c) => c.insertOne(log))
-            .catch((storeError) => {
-                this._log.error('Failed to store chat log', storeError, log);
-
-                if (!this.muteErrors) {
-                    throw storeError;
-                }
-            });
+        return this._storeLog(log);
     }
 
 }

package/src/NotificationsStorage.js

@@ -381,7 +381,7 @@ class NotificationsStorage {
                 }
             }, {
                 sort: { enqueue: 1 },
-                returnOriginal: false
+                returnDocument: 'after'
             });
             if (found.value) {
                 pop.push(this._mapGenericObject(found.value));
@@ -475,7 +475,7 @@ class NotificationsStorage {
         }, {
             $set: data
         }, {
-            returnOriginal: false
+            returnDocument: 'after'
         });
 
         return this._mapGenericObject(res.value);
@@ -565,7 +565,7 @@ class NotificationsStorage {
                     [eventType]: ts
                 }
             }, {
-                returnOriginal: false
+                returnDocument: 'after'
             }))
         );
 
@@ -602,7 +602,7 @@ class NotificationsStorage {
                 id: campaign.id
             }, update, {
                 upsert: true,
-                returnOriginal: false
+                returnDocument: 'after'
             });
             ret = this._mapCampaign(res.value);
         } else {
@@ -661,7 +661,7 @@ class NotificationsStorage {
         }, {
             $set: data
         }, {
-            returnOriginal: false
+            returnDocument: 'after'
         });
 
         return this._mapCampaign(res.value);
@@ -681,7 +681,7 @@ class NotificationsStorage {
         }, {
             $set: { startAt: null }
         }, {
-            returnOriginal: true
+            returnDocument: 'before'
         });
 
         return this._mapCampaign(res.value);
@@ -807,7 +807,7 @@ class NotificationsStorage {
             }, {
                 $pull: { subs: tag }
             }, {
-                returnOriginal: false
+                returnDocument: 'after'
             });
 
             if (res.value) {

package/src/StateStorage.js

@@ -115,7 +115,7 @@ class StateStorage extends BaseStorage {
             $set
         }, {
             upsert: true,
-            returnOriginal: false,
+            returnDocument: 'after',
             projection: {
                 _id: 0
             }

package/src/main.js

@@ -9,6 +9,7 @@ const BotTokenStorage = require('./BotTokenStorage');
 const ChatLogStorage = require('./ChatLogStorage');
 const BotConfigStorage = require('./BotConfigStorage');
 const AttachmentCache = require('./AttachmentCache');
+const AuditLogStorage = require('./AuditLogStorage');
 const NotificationsStorage = require('./NotificationsStorage');
 
 module.exports = {
@@ -18,5 +19,6 @@ module.exports = {
     ChatLogStorage,
     BotConfigStorage,
     AttachmentCache,
-    NotificationsStorage
+    NotificationsStorage,
+    AuditLogStorage
 };