wingbot-mongodb 2.18.0 → 2.20.1

package/src/AuditLogStorage.js

@@ -0,0 +1,364 @@
+/**
+ * @author wingbot.ai
+ */
+'use strict';
+
+const jsonwebtoken = require('jsonwebtoken');
+const BaseStorage = require('./BaseStorage');
+
+/** @typedef {import('mongodb/lib/db')} Db */
+
+const LEVEL_CRITICAL = 'Critical';
+const LEVEL_IMPORTANT = 'Important';
+const LEVEL_DEBUG = 'Debug';
+
+const TYPE_ERROR = 'Error';
+const TYPE_WARN = 'Warn';
+const TYPE_INFO = 'Info';
+
+/**
+ * @typedef {object} TrackingEvent
+ * @prop {string} [type='audit']
+ * @prop {string} category
+ * @prop {string} action
+ * @prop {string} [label]
+ * @prop {object} [payload]
+ */
+
+/**
+ * @typedef {object} User
+ * @prop {string} [id]
+ * @prop {string} [senderId]
+ * @prop {string} [pageId]
+ * @prop {string} [jwt] - jwt to check the authorship
+ */
+
+/**
+ * @typedef {object} Meta
+ * @prop {string} [ip]
+ * @prop {string} [ua]
+ * @prop {string} [ro] - referrer || origin
+ */
+
+/**
+ * @typedef {object} LogEntry
+ * @prop {string} date - ISO date
+ * @prop {number} delta - time skew in ms if there was a write conflict
+ * @prop {string} [eventType='audit']
+ * @prop {string} category
+ * @prop {string} action
+ * @prop {string} [label]
+ * @prop {object} [payload]
+ * @prop {string} level - (Critical|Important|Debug)
+ * @prop {boolean} ok - signature matches
+ * @prop {number} seq - sequence number
+ * @prop {string} type - (Error|Warn|Info)
+ * @prop {User} user
+ * @prop {string} wid - workspace id
+ * @prop {Meta} meta
+ */
+
+/**
+ * JWT Verifier
+ *
+ * @callback JwtVerifier
+ * @param {string} token
+ * @param {string} userId
+ * @param {User} [user]
+ * @returns {Promise<boolean>}
+ */
+
+/**
+ * @typedef {object} AuditLogEntry
+ * @prop {string} date - ISO date
+ * @prop {string} [eventType='audit']
+ * @prop {string} category
+ * @prop {string} action
+ * @prop {string} [label]
+ * @prop {object} [payload]
+ * @prop {string} level - (Critical|Important|Debug)
+ * @prop {string} type - (Error|Warn|Info)
+ * @prop {User} user
+ * @prop {string} wid - workspace id
+ * @prop {Meta} meta
+ */
+
+/**
+ * Audit Log Callback
+ *
+ * @callback AuditLogCallback
+ * @param {AuditLogEntry} entry
+ * @returns {Promise}
+ */
+
+/**
+ * Storage for audit logs with signatures chain
+ */
+class AuditLogStorage extends BaseStorage {
+
+    /**
+     *
+     * @param {Db|{():Promise<Db>}} mongoDb
+     * @param {string} collectionName
+     * @param {{error:Function,log:Function}} [log] - console like logger
+     * @param {boolean} [isCosmo]
+     * @param {string|Promise<string>} [secret]
+     * @param {string|Promise<string>} [jwtVerifier]
+     */
+    constructor (mongoDb, collectionName = 'auditlog', log = console, isCosmo = false, secret = null, jwtVerifier = null) {
+        super(mongoDb, collectionName, log, isCosmo);
+
+        this.addIndex({
+            wid: 1,
+            seq: -1
+        }, {
+            unique: true,
+            name: 'wid_1_seq_-1'
+        });
+
+        if (isCosmo) {
+            this.addIndex({
+                wid: 1
+            }, {
+                name: 'wid_1'
+            });
+
+            this.addIndex({
+                seq: -1
+            }, {
+                name: 'seq_-1'
+            });
+        } else {
+            this.addIndex({
+                wid: 1, date: -1
+            }, {
+                name: 'wid_1_date_-1'
+            });
+        }
+
+        this.defaultWid = '0';
+
+        this.muteErrors = true;
+        this.maxRetries = 4;
+        this._secret = secret;
+
+        this.LEVEL_CRITICAL = LEVEL_CRITICAL;
+        this.LEVEL_IMPORTANT = LEVEL_IMPORTANT;
+        this.LEVEL_DEBUG = LEVEL_DEBUG;
+
+        this.TYPE_ERROR = TYPE_ERROR;
+        this.TYPE_WARN = TYPE_WARN;
+        this.TYPE_INFO = TYPE_INFO;
+
+        /**
+         * @type {JwtVerifier}
+         */
+        // @ts-ignore
+        this._jwtVerify = typeof jwtVerifier === 'function' || jwtVerifier === null
+            ? jwtVerifier
+            : async (token, userId) => {
+                const jwtSec = await Promise.resolve(jwtVerifier);
+                const decoded = await new Promise((resolve) => {
+                    jsonwebtoken.verify(token, jwtSec, (err, res) => {
+                        resolve(res || {});
+                    });
+                });
+                return decoded.id === userId;
+            };
+
+        /** @type {AuditLogCallback} */
+        this.callback = null;
+    }
+
+    /**
+     * Add a log
+     *
+     * @param {TrackingEvent} event
+     * @param {User} user
+     * @param {Meta} [meta]
+     * @param {string} [wid] - workspace ID
+     * @param {string} [type]
+     * @param {string} [level]
+     * @param {Date} [date]
+     * @returns {Promise}
+     */
+    async log (
+        event,
+        user = {},
+        meta = {},
+        wid = this.defaultWid,
+        type = TYPE_INFO,
+        level = LEVEL_IMPORTANT,
+        date = new Date()
+    ) {
+        const {
+            type: eventType = 'audit',
+            ...rest
+        } = event;
+        const entry = {
+            date,
+            eventType,
+            ...rest,
+            level,
+            meta,
+            type,
+            user,
+            wid
+        };
+
+        const secret = await Promise.resolve(this._secret);
+        const stored = await this._storeWithRetry(secret, entry);
+        if (!this.callback) {
+            return;
+        }
+        try {
+            await this.callback(stored);
+        } catch (e) {
+            this._log.error('Failed to send AuditLog', e, entry);
+        }
+
+        // logEvent(level, type, workflowType, workflowInstance, eventData, account)
+        // level is the criticality of the event ('Critical','Important','Debug').
+        // type is the type of the event ('Error','Warn','Info').
+        /**
+         *  - log (N/A)
+            - report (Debug)
+            - conversation (N/A)
+            - audit (Important)
+            - user (N/A)
+         */
+    }
+
+    /**
+     *
+     * @param {string} [wid] - workspace id
+     * @param {number} [fromSeq] - for paging
+     * @param {number} [limit]
+     * @returns {Promise<LogEntry[]>}
+     */
+    async list (wid = this.defaultWid, fromSeq = 0, limit = 40) {
+        const c = await this._getCollection();
+
+        const cond = { wid };
+
+        if (fromSeq) {
+            cond.seq = { $lt: fromSeq };
+        }
+
+        const data = await c.find({ wid })
+            .limit(limit + 1)
+            .sort({ seq: -1 })
+            .project({ _id: 0 })
+            .toArray();
+
+        const secret = await Promise.resolve(this._secret);
+
+        const len = data.length === limit + 1
+            ? data.length - 1
+            : data.length;
+        const ret = new Array(len);
+
+        let verifyUser = false;
+        for (let i = 0; i < len; i++) {
+            const {
+                sign,
+                ...log
+            } = data[i];
+
+            verifyUser = verifyUser || (log.user.id && log.user.jwt);
+
+            if (secret) {
+                const previous = data[i + 1] || { sign: null };
+                const objToSign = this._objectToSign(log);
+                const compare = this._signWithSecret(objToSign, secret, previous.sign);
+                log.ok = compare === sign;
+                if (!log.ok) {
+                    this._log.error(`AuditLog: found wrong signature at wid: "${log.wid}", seq: "${log.seq}"`, log);
+                }
+            } else {
+                log.ok = null;
+            }
+
+            ret[i] = log;
+        }
+
+        if (verifyUser && this._jwtVerify) {
+            return Promise.all(
+                ret.map(async (log) => {
+                    if (!log.user.id || !log.user.jwt) {
+                        return log;
+                    }
+                    const userChecked = await this._jwtVerify(log.user.jwt, log.user.id, log.user);
+                    return Object.assign(log, {
+                        // it's ok, when there was null
+                        ok: log.ok !== false && userChecked
+                    });
+                })
+            );
+        }
+
+        return ret;
+    }
+
+    _wait (ms) {
+        return new Promise((r) => setTimeout(r, ms));
+    }
+
+    async _storeWithRetry (secret, entry, delta = 0, i = 1) {
+        const start = Date.now();
+        Object.assign(entry, { delta });
+        try {
+            await this._store(entry, secret);
+            return entry;
+        } catch (e) {
+            if (e.code === 11000) {
+                // duplicate key
+                if (i >= this.maxRetries) {
+                    throw new Error('AuditLog: cannot store log due to max-retries');
+                } else {
+                    await this._wait((i * 50) + (Math.random() * 100));
+                    const add = Date.now() - start;
+                    return this._storeWithRetry(secret, entry, delta + add, i + 1);
+                }
+            } else if (this.muteErrors) {
+                this._log.error('Audit log store error', e, entry);
+                return entry;
+            } else {
+                throw e;
+            }
+        }
+    }
+
+    async _store (entry, secret) {
+        const c = await this._getCollection();
+
+        const previous = await c.findOne({
+            wid: entry.wid
+        }, {
+            sort: { seq: -1 },
+            projection: { seq: 1, _id: 0, sign: 1 }
+        });
+
+        Object.assign(entry, {
+            seq: previous ? previous.seq + 1 : 0
+        });
+
+        let insert;
+        if (secret) {
+            insert = this._objectToSign(entry);
+            const sign = this._signWithSecret(insert, secret, previous ? previous.sign : null);
+            Object.assign(insert, { sign });
+        } else {
+            insert = {
+                ...entry,
+                sign: null
+            };
+        }
+
+        // @ts-ignore
+        await c.insertOne(insert);
+    }
+
+}
+
+module.exports = AuditLogStorage;

package/src/BaseStorage.js

@@ -4,15 +4,19 @@
 'use strict';
 
 const mongodb = require('mongodb'); // eslint-disable-line no-unused-vars
+const crypto = require('crypto');
+
+/** @typedef {import('mongodb/lib/db')} Db */
+/** @typedef {import('mongodb/lib/collection')} Collection */
 
 class BaseStorage {
 
     /**
      *
-     * @param {mongodb.Db|{():Promise<mongodb.Db>}} mongoDb
+     * @param {Db|{():Promise<Db>}} mongoDb
      * @param {string} collectionName
      * @param {{error:Function,log:Function}} [log] - console like logger
-     * @param {boolean} isCosmo
+     * @param {boolean} [isCosmo]
      * @example
      *
      * const { BaseStorage } = require('winbot-mongodb');
@@ -45,11 +49,27 @@ class BaseStorage {
         this._log = log;
 
         /**
-         * @type {Promise<mongodb.Collection>}
+         * @type {Collection|Promise<Collection>}
          */
         this._collection = null;
 
         this._indexes = [];
+
+        this.ignoredSignatureKeys = ['_id', 'sign'];
+        this._secret = null;
+
+        this.systemIndexes = ['_id_', '_id'];
+
+        this._fixtures = [];
+    }
+
+    /**
+     * Insert defalt document to DB
+     *
+     * @param  {...any} objects
+     */
+    addFixtureDoc (...objects) {
+        this._fixtures.push(...objects);
     }
 
     /**
@@ -76,6 +96,7 @@ class BaseStorage {
         let collection;
 
         if (this._isCosmo) {
+            // @ts-ignore
             const collections = await db.collections();
 
             collection = collections
@@ -101,7 +122,7 @@ class BaseStorage {
     /**
      * Returns the collection to operate with
      *
-     * @returns {Promise<mongodb.Collection>}
+     * @returns {Promise<Collection>}
      */
     async _getCollection () {
         if (this._collection === null) {
@@ -127,31 +148,82 @@ class BaseStorage {
         }
 
         await existing
-            .filter((e) => !['_id_', '_id'].includes(e.name)
+            .filter((e) => !this.systemIndexes.includes(e.name)
                 && !indexes.some((i) => e.name === i.options.name))
             .reduce((p, e) => {
-                // eslint-disable-next-line no-console
-                this._log.log(`dropping index ${e.name}`);
+                this._log.log(`DB.${this._collectionName} dropping index ${e.name}`);
                 return p
                     .then(() => collection.dropIndex(e.name))
                     .catch((err) => {
-                        // eslint-disable-next-line no-console
-                        this._log.error(`dropping index ${e.name} FAILED`, err);
+                        this._log.error(`DB.${this._collectionName} dropping index ${e.name} FAILED`, err);
                     });
             }, Promise.resolve());
 
-        await indexes
+        const updated = await indexes
             .filter((i) => !existing.some((e) => e.name === i.options.name))
             .reduce((p, i) => {
-                this._log.log(`creating index ${i.name}`);
+                this._log.log(`DB.${this._collectionName} creating index ${i.options.name}`);
                 return p
                     .then(() => collection.createIndex(i.index, i.options))
                     .catch((e) => {
-                        this._log.error(`failed to create index ${i.options.name} on ${collection.collectionName}`, e);
-                    });
-            }, Promise.resolve());
+                        this._log.error(`DB.${this._collectionName} failed to create index ${i.options.name} on ${collection.collectionName}`, e);
+                    })
+                    .then(() => true);
+            }, Promise.resolve(false));
+
+        if (updated || existing.every((i) => this.systemIndexes.includes(i.name))) {
+            // upsert fixtures
+
+            await this._fixtures.reduce((p, o) => p
+                .then(() => collection.insertOne(o))
+                .then(() => this._log.log(`DB.${this._collectionName} Inserted fixture doc to "${this._collectionName}"`))
+                .catch((e) => {
+                    if (e.code !== 11000) {
+                        this._log.error(`DB.${this._collectionName} failed to insert fixture doc to "${this._collectionName}"`, e);
+                    }
+                }),
+            Promise.resolve());
+        }
+    }
+
+    async _sign (object) {
+        if (!this._secret) {
+            return object;
+        }
+        const secret = await Promise.resolve(this._secret);
+        const objToSign = this._objectToSign(object);
+        const sign = this._signWithSecret(objToSign, secret);
+
+        return Object.assign(objToSign, {
+            sign
+        });
     }
 
+    _objectToSign (object) {
+        const entries = Object.keys(object)
+            .filter((key) => !this.ignoredSignatureKeys.includes(key));
+
+        entries.sort();
+
+        return entries.reduce((o, key) => {
+            let val = object[key];
+            if (val instanceof Date) {
+                val = val.toISOString();
+            }
+            return Object.assign(o, { [key]: val });
+        }, {});
+    }
+
+    _signWithSecret (objToSign, secret, previous = null) {
+        const h = crypto.createHmac('sha3-224', secret)
+            .update(JSON.stringify(objToSign));
+
+        if (previous) {
+            h.update(previous);
+        }
+
+        return h.digest('base64');
+    }
 }
 
 module.exports = BaseStorage;

package/src/BotConfigStorage.js

@@ -3,8 +3,16 @@
  */
 'use strict';
 
-const mongodb = require('mongodb'); // eslint-disable-line no-unused-vars
-const { apiAuthorizer } = require('wingbot');
+let apiAuthorizer = () => false;
+try {
+    // @ts-ignore
+    ({ apiAuthorizer } = module.require('wingbot'));
+} catch (e) {
+    // noop
+}
+
+/** @typedef {import('mongodb/lib/db')} Db */
+/** @typedef {import('mongodb/lib/collection')} Collection */
 
 const CONFIG_ID = 'config';
 
@@ -17,7 +25,7 @@ class BotConfigStorage {
 
     /**
      *
-     * @param {mongodb.Db|{():Promise<mongodb.Db>}} mongoDb
+     * @param {Db|{():Promise<Db>}} mongoDb
      * @param {string} collectionName
      */
     constructor (mongoDb, collectionName = 'botconfig') {
@@ -25,13 +33,13 @@ class BotConfigStorage {
         this._collectionName = collectionName;
 
         /**
-         * @type {mongodb.Collection}
+         * @type {Collection}
          */
         this._collection = null;
     }
 
     /**
-     * @returns {Promise<mongodb.Collection>}
+     * @returns {Promise<Collection>}
      */
     async _getCollection () {
         if (this._collection === null) {
@@ -56,6 +64,7 @@ class BotConfigStorage {
         const storage = this;
         return {
             async updateBot (args, ctx) {
+                // @ts-ignore
                 if (!apiAuthorizer(args, ctx, acl)) {
                     return null;
                 }
@@ -74,6 +83,7 @@ class BotConfigStorage {
     async invalidateConfig () {
         const c = await this._getCollection();
 
+        // @ts-ignore
         return c.deleteOne({ _id: CONFIG_ID });
     }
 

package/src/BotTokenStorage.js

@@ -131,7 +131,7 @@ class BotTokenStorage {
             }
         }, {
             upsert: true,
-            returnOriginal: false
+            returnDocument: 'after'
         });
 
         res = res.value;