wingbot 3.24.5 → 3.26.0-alpha.2

package/index.js

@@ -21,6 +21,7 @@ const { asserts } = require('./src/testTools');
 const BuildRouter = require('./src/BuildRouter');
 const MockAiModel = require('./src/MockAiModel');
 const ReturnSender = require('./src/ReturnSender');
+const CallbackAuditLog = require('./src/CallbackAuditLog');
 const Plugins = require('./src/Plugins');
 const NotificationsStorage = require('./src/notifications/NotificationsStorage');
 const Notifications = require('./src/notifications/Notifications');
@@ -70,6 +71,7 @@ module.exports = {
     bufferloader,
     asserts,
     MemoryStateStorage,
+    CallbackAuditLog,
     MemoryBotConfigStorage,
     CachedModel,
     CustomEntityDetectionModel,

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "wingbot",
-  "version": "3.24.5",
+  "version": "3.26.0-alpha.2",
   "description": "Enterprise Messaging Bot Conversation Engine",
   "main": "index.js",
   "scripts": {

package/src/BotApp.js

@@ -9,6 +9,7 @@ const crypto = require('crypto');
 const BotAppSender = require('./BotAppSender');
 const Processor = require('./Processor');
 const ReturnSender = require('./ReturnSender');
+const headersToAuditMeta = require('./utils/headersToAuditMeta');
 
 const DEFAULT_API_URL = 'https://orchestrator-api.wingbot.ai';
 
@@ -20,6 +21,9 @@ const DEFAULT_API_URL = 'https://orchestrator-api.wingbot.ai';
 /** @typedef {import('./Responder')} Responder */
 /** @typedef {import('./Processor').Plugin} Plugin */
 
+/** @typedef {import('./CallbackAuditLog')} AuditLog */
+/** @typedef {import('./BotAppSender').TlsOptions} TlsOptions */
+
 /**
  * @typedef {object} BotAppOptions
  * @prop {string|Promise<string>} secret
@@ -28,6 +32,8 @@ const DEFAULT_API_URL = 'https://orchestrator-api.wingbot.ai';
  * @prop {Function} [fetch]
  * @prop {ChatLogStorage} [chatLogStorage]
  * @prop {boolean} [preferSynchronousResponse]
+ * @prop {AuditLog} [auditLog]
+ * @prop {TlsOptions} [tls]
  *
  * @typedef {ProcessorOptions & BotAppOptions} Options
  */
@@ -58,12 +64,16 @@ class BotApp {
             fetch = null,
             appId = null,
             preferSynchronousResponse = false,
+            auditLog = null,
+            tls = null,
             ...processorOptions
         } = options;
 
         this._secret = Promise.resolve(secret);
         this._fetch = fetch; // mock
         this._appId = appId;
+        this._auditLog = auditLog;
+        this._tls = tls;
 
         let { apiUrl } = options;
 
@@ -90,6 +100,18 @@ class BotApp {
         this._preferSynchronousResponse = preferSynchronousResponse;
     }
 
+    /**
+     * Get authorization token for wingbot orchestrator
+     *
+     * @param {string} body
+     * @param {string} secret
+     * @param {string} appId
+     * @returns {Promise<string>}
+     */
+    static signBody (body, secret, appId) {
+        return BotApp.signBody(body, secret, appId);
+    }
+
     /**
      * Returns processor plugin, which updates thread context automatically
      *
@@ -140,13 +162,44 @@ class BotApp {
         };
     }
 
-    async _processIncommingMessage (message, senderId, pageId, appId, secret, sync = false) {
+    /**
+     * @param {ReturnSender} sender
+     * @param {string} senderId
+     * @param {string} pageId
+     * @param {object} headers
+     */
+    async _processSenderResponses (sender, senderId, pageId, headers) {
+        if (!this._auditLog) {
+            return;
+        }
+        await Promise.all(
+            sender.tracking.events.map(async (event) => {
+                if (!['audit', 'report'].includes(event.type)) {
+                    return;
+                }
+
+                await this._auditLog.log(
+                    event,
+                    { senderId, pageId },
+                    headersToAuditMeta(headers),
+                    this._auditLog.defaultWid,
+                    'Info',
+                    event.type === 'audit' ? 'Important' : 'Debug'
+                );
+            })
+        );
+    }
+
+    async _processIncommingMessage (
+        message, senderId, pageId, appId, secret, sync = false, headers = {}
+    ) {
         const { mid = null } = message;
 
         if (sync || this._preferSynchronousResponse) {
             const sender = new ReturnSender({}, senderId, message, this._senderLogger);
             sender.propagatesWaitEvent = true;
             const res = await this._processor.processMessage(message, pageId, sender, { appId });
+            await this._processSenderResponses(sender, senderId, pageId, headers);
 
             return {
                 status: res.status,
@@ -174,11 +227,13 @@ class BotApp {
             appId,
             secret,
             mid,
-            fetch: this._fetch
+            fetch: this._fetch,
+            tls: this._tls
         };
 
         const sender = new BotAppSender(options, senderId, message, this._senderLogger);
         const res = await this._processor.processMessage(message, pageId, sender, { appId });
+        await this._processSenderResponses(sender, senderId, pageId, headers);
 
         return {
             status: res.status,
@@ -280,7 +335,7 @@ class BotApp {
 
         const body = JSON.parse(rawBody);
 
-        const entry = await this._processEntries(appId, secret, body.entry);
+        const entry = await this._processEntries(appId, secret, body.entry, rawHeaders);
 
         return {
             statusCode: 200,
@@ -293,7 +348,7 @@ class BotApp {
         };
     }
 
-    async _processEntries (appId, secret, entry = []) {
+    async _processEntries (appId, secret, entry = [], headers = {}) {
         return Promise.all(
             entry.map(async (event) => {
                 const {
@@ -308,7 +363,8 @@ class BotApp {
                     ...standby.map((e) => ({ ...e, isStandby: true }))
                 ];
 
-                const responses = await this._processMessaging(events, pageId, appId, secret, sync);
+                const responses = await this
+                    ._processMessaging(events, pageId, appId, secret, sync, headers);
 
                 return {
                     id: pageId,
@@ -318,7 +374,7 @@ class BotApp {
         );
     }
 
-    async _processMessaging (process, pageId, appId, secret, sync) {
+    async _processMessaging (process, pageId, appId, secret, sync, headers) {
         const eventsBySenderId = new Map();
         process.forEach((e) => this._processMessagingArrayItem(e, eventsBySenderId));
 
@@ -329,7 +385,7 @@ class BotApp {
 
                     for (const message of messaging) {
                         const res = await this._processIncommingMessage(
-                            message, senderId, pageId, appId, secret, sync
+                            message, senderId, pageId, appId, secret, sync, headers
                         );
 
                         responses.push(res);

package/src/BotAppSender.js

@@ -6,17 +6,25 @@
 const jwt = require('jsonwebtoken');
 const { default: fetch, Headers } = require('node-fetch');
 const crypto = require('crypto');
+const { Agent } = require('https');
 const { promisify } = require('util');
 const ReturnSender = require('./ReturnSender');
 
 /** @typedef {import('./ReturnSender').ReturnSenderOptions} ReturnSenderOptions */
 /** @typedef {import('./ReturnSender').ChatLogStorage} ChatLogStorage */
 
+/**
+ * @typedef {object} TlsOptions
+ * @prop {string|Promise<string>} key
+ * @prop {string|Promise<string>} cert
+ */
+
 /**
  * @typedef {object} BotAppSenderOptions
  * @prop {string} apiUrl
  * @prop {string} pageId
  * @prop {string} appId
+ * @prop {TlsOptions} [tls]
  * @prop {string} [mid]
  * @prop {Function} [fetch]
  * @prop {string|Promise<string>} secret
@@ -47,6 +55,9 @@ class BotAppSender extends ReturnSender {
 
         this._secret = Promise.resolve(options.secret);
 
+        this._tls = options.tls;
+        this._agent = null;
+
         this._fetch = options.fetch || fetch;
     }
 
@@ -65,6 +76,21 @@ class BotAppSender extends ReturnSender {
         }, goodSecret);
     }
 
+    async _getAgent () {
+        if (!this._tls) {
+            return null;
+        }
+        if (!this._agent) {
+            const [key, cert] = await Promise.all([
+                Promise.resolve(this._tls.key), Promise.resolve(this._tls.cert)
+            ]);
+
+            this._agent = new Agent({ cert, key });
+        }
+
+        return this._agent;
+    }
+
     async _send (payload) {
         // attach sender
         if (typeof payload.sender === 'undefined') {
@@ -78,14 +104,19 @@ class BotAppSender extends ReturnSender {
 
         const body = JSON.stringify(payload);
 
-        const token = await BotAppSender.signBody(body, this._secret, this._appId);
+        const [token, agent] = await Promise.all([
+            BotAppSender.signBody(body, this._secret, this._appId),
+            this._getAgent()
+        ]);
 
         const headers = new Headers();
 
         headers.set('Authorization', token);
         headers.set('Content-Type', 'application/json');
 
-        const response = await this._fetch(this._apiUrl, { headers, body, method: 'POST' })
+        const response = await this._fetch(this._apiUrl, {
+            headers, body, agent, method: 'POST'
+        })
             .then((r) => r.json());
 
         const { request, errors = null } = response;

package/src/CallbackAuditLog.js

@@ -0,0 +1,145 @@
+/**
+ * @author wingbot.ai
+ */
+'use strict';
+
+const LEVEL_CRITICAL = 'Critical';
+const LEVEL_IMPORTANT = 'Important';
+const LEVEL_DEBUG = 'Debug';
+
+const TYPE_ERROR = 'Error';
+const TYPE_WARN = 'Warn';
+const TYPE_INFO = 'Info';
+
+/**
+ * @typedef {object} TrackingEvent
+ * @prop {string} [type='audit']
+ * @prop {string} category
+ * @prop {string} action
+ * @prop {string} [label]
+ * @prop {object} [payload]
+ */
+
+/**
+ * @typedef {object} User
+ * @prop {string} [id]
+ * @prop {string} [senderId]
+ * @prop {string} [pageId]
+ * @prop {string} [jwt] - jwt to check the authorship
+ */
+
+/**
+ * @typedef {object} Meta
+ * @prop {string} [ip]
+ * @prop {string} [ua]
+ * @prop {string} [ro] - referrer || origin
+ */
+
+/**
+ * @typedef {object} Logger
+ * @prop {Function} log
+ * @prop {Function} error
+ */
+
+/**
+ * @typedef {object} AuditLogEntry
+ * @prop {string} date - ISO date
+ * @prop {string} [eventType='audit']
+ * @prop {string} category
+ * @prop {string} action
+ * @prop {string} [label]
+ * @prop {object} [payload]
+ * @prop {string} level - (Critical|Important|Debug)
+ * @prop {string} type - (Error|Warn|Info)
+ * @prop {User} user
+ * @prop {string} wid - workspace id
+ * @prop {Meta} meta
+ */
+
+/**
+ * Audit Log Callback
+ *
+ * @callback AuditLogCallback
+ * @param {AuditLogEntry} entry
+ * @returns {Promise}
+ */
+
+/**
+ * Just a simple auditlog interface
+ */
+class CallbackAuditLog {
+
+    /**
+     *
+     * @param {AuditLogCallback} callback
+     * @param {Logger} log
+     */
+    constructor (callback = null, log = console) {
+        this._log = log;
+        this.defaultWid = '0';
+
+        this.LEVEL_CRITICAL = LEVEL_CRITICAL;
+        this.LEVEL_IMPORTANT = LEVEL_IMPORTANT;
+        this.LEVEL_DEBUG = LEVEL_DEBUG;
+
+        this.TYPE_ERROR = TYPE_ERROR;
+        this.TYPE_WARN = TYPE_WARN;
+        this.TYPE_INFO = TYPE_INFO;
+
+        /** @type {AuditLogCallback} */
+        this.callback = callback || (async (e) => {
+            const {
+                category,
+                action,
+                ...rest
+            } = e;
+            log.log(`AuditLog: ${category} - ${action}`, rest);
+        });
+    }
+
+    /**
+     * Add a log
+     *
+     * @param {TrackingEvent} event
+     * @param {User} user
+     * @param {Meta} [meta]
+     * @param {string} [wid] - workspace ID
+     * @param {string} [type]
+     * @param {string} [level]
+     * @param {Date} [date]
+     * @returns {Promise}
+     */
+    async log (
+        event,
+        user = {},
+        meta = {},
+        wid = this.defaultWid,
+        type = TYPE_INFO,
+        level = LEVEL_IMPORTANT,
+        date = new Date()
+    ) {
+        const {
+            type: eventType = 'audit',
+            ...rest
+        } = event;
+        const entry = {
+            date: date.toISOString(),
+            eventType,
+            ...rest,
+            level,
+            meta,
+            type,
+            user,
+            wid
+        };
+
+        try {
+            await this.callback(entry);
+        } catch (e) {
+            this._log.error('Failed to send AuditLog', e, entry);
+        }
+    }
+
+}
+
+module.exports = CallbackAuditLog;

package/src/Responder.js

@@ -190,7 +190,7 @@ class Responder {
      * Events are aggregated within ReturnSender and can be caught
      * within Processor's `interaction` event (event.tracking.events)
      *
-     * @param {string} type
+     * @param {string} type - (log,report,conversation,audit,user)
      * @param {string} category
      * @param {string} [action]
      * @param {string} [label]

package/src/graphApi/GraphApi.js

@@ -8,6 +8,7 @@ const { graphql, buildSchema } = require('graphql');
 const WingbotApiConnector = require('./WingbotApiConnector');
 // @ts-ignore
 const packageJson = require('../../package.json');
+const headersToAuditMeta = require('../utils/headersToAuditMeta');
 
 const DEFAULT_GROUPS = ['botEditor', 'botAdmin', 'appToken'];
 const KEYS_URL = 'https://api.wingbot.ai/keys';
@@ -19,6 +20,8 @@ const DEFAULT_CACHE = 86400000; // 24 hours
  * @param {object[]} [errors]
  */
 
+/** @typedef {import('../CallbackAuditLog')} AuditLog */
+
 /**
  * Experimental chatbot API
  */
@@ -32,6 +35,7 @@ class GraphApi {
      * @param {string} [options.appToken] - public token
      * @param {string[]} [options.groups] - list of allowed bot groups
      * @param {boolean} [options.useBundledGql] - uses library bundled graphql definition
+     * @param {AuditLog} [options.auditLog]
      */
     constructor (apis, options) {
         this._root = {
@@ -44,7 +48,16 @@ class GraphApi {
             token: null,
             groups: DEFAULT_GROUPS,
             keysUrl: KEYS_URL,
-            cacheKeys: DEFAULT_CACHE
+            cacheKeys: DEFAULT_CACHE,
+            auditLog: {
+                async callback () {
+                    // noop
+                },
+                defaultWid: '0',
+                async log () {
+                    // noop
+                }
+            }
         };
 
         Object.assign(opts, options);
@@ -57,6 +70,10 @@ class GraphApi {
 
         this._defaultGroups = opts.groups;
 
+        /** @type {AuditLog} */
+        // @ts-ignore
+        this.auditLog = opts.auditLog;
+
         this._apiConnector = new WingbotApiConnector({
             token: opts.token,
             appToken: opts.appToken,
@@ -75,6 +92,10 @@ class GraphApi {
      * @param {object} headers
      * @param {string} [headers.Authorization]
      * @param {string} [headers.authorization]
+     * @param {string} [headers.Origin]
+     * @param {string} [headers.origin]
+     * @param {string} [headers.Referer]
+     * @param {string} [headers.referer]
      * @param {string} [wingbotToken]
      * @returns {Promise<GraphQlResponse>}
      */
@@ -83,13 +104,39 @@ class GraphApi {
         assert.equal(typeof body.query, 'string', 'GraphQL request should contain a query property');
 
         const authHeader = headers.Authorization || headers.authorization;
-        const token = await this._apiConnector.verifyToken(authHeader, wingbotToken);
+        let token = {};
+
+        const audit = async (action, payload = {}, important = false, warn = false) => {
+            await this.auditLog.log(
+                {
+                    category: 'API',
+                    action,
+                    payload
+                },
+                {
+                    id: token.id,
+                    jwt: token.id && authHeader.replace(/^bearer\s/i, '')
+                },
+                headersToAuditMeta(headers),
+                this.auditLog.defaultWid,
+                warn ? 'Warn' : 'Info',
+                important ? 'Important' : 'Debug'
+            );
+        };
+
+        try {
+            token = await this._apiConnector.verifyToken(authHeader, wingbotToken);
+        } catch (e) {
+            await audit('authorization failed', { message: e.message, authHeader }, true, true);
+            throw e;
+        }
 
         const schema = await this._schema();
 
         const ctx = {
             token,
-            groups: this._defaultGroups
+            groups: this._defaultGroups,
+            audit
         };
 
         // @ts-ignore

package/src/graphApi/conversationTestApi.js

@@ -42,6 +42,8 @@ function conversationTestApi (testsSource, botFactory, options, acl) {
 
             const { bot: validationRequestBody, step = null, lang = null } = args;
 
+            await ctx.audit('conversationTest', args);
+
             return test.test(validationRequestBody, step, lang);
         }
     };

package/src/graphApi/conversationsApi.js

@@ -131,6 +131,8 @@ function conversationsApi (
                 condition
             } = args;
 
+            await ctx.audit('conversations', args);
+
             const { data, lastKey: nextKey } = await stateStorage
                 .getStates(condition || {}, limit, lastKey);
 
@@ -154,6 +156,8 @@ function conversationsApi (
                 senderId
             } = args;
 
+            await ctx.audit('conversation', args);
+
             const state = await stateStorage.getState(senderId, pageId);
 
             if (!state) {

package/src/graphApi/postBackApi.js

@@ -35,6 +35,7 @@ function postBackApi (processor, acl) {
 
             const event = Request.postBack(args.senderId, args.action, args.data);
 
+            await ctx.audit('postBack', args);
             return processor.processMessage(event, args.senderId, args.pageId);
         }
     };

package/src/graphApi/validateBotApi.js

@@ -99,6 +99,7 @@ function validateBotApi (botFactory, postBackTest = null, textTest = null, acl =
 
             const bot = botFactory();
 
+            await ctx.audit('validateBot');
             return validate(bot, validationRequestBody, postBackTest, textTest);
         }
     };

package/src/utils/getUpdate.js

@@ -179,7 +179,7 @@ function getSetState (setState = {}, req, res = null, useState = null) {
                         const useValue = typeof value === 'string'
                             ? handlebars.compile(value)(stateData(req, res))
                                 .split(/(?<!\\),/g)
-                                .map((v) => v.trim())
+                                .map((v) => v.replace(/\\,/g, ',').trim())
                             : value;
                         values = toArray(useValue);
                     }

package/src/utils/headersToAuditMeta.js

@@ -0,0 +1,12 @@
+/**
+ * @author wingbot.ai
+ */
+'use strict';
+
+module.exports = function headersToAuditMeta (headers) {
+    return {
+        ua: headers['User-Agent'] || headers['user-agent'] || headers['User-agent'],
+        ro: headers.Referer || headers.referer || headers.Origin || headers.origin,
+        ip: headers['X-Forwarded-For'] || headers['x-forwarded-for'] || headers['X-Azure-ClientIP'] || headers['x-azure-clientip']
+    };
+};