wingbot 3.46.0-alpha.1 → 3.46.0-alpha.11

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "wingbot",
-  "version": "3.46.0-alpha.1",
+  "version": "3.46.0-alpha.11",
   "description": "Enterprise Messaging Bot Conversation Engine",
   "main": "index.js",
   "scripts": {
@@ -9,7 +9,7 @@
     "doc": "npm run doc:gql && node ./bin/makeApiDoc.js && cpy ./CHANGELOG.md ./doc && gitbook install ./doc && gitbook build ./doc && rimraf -rf ./docs && rimraf --rf ./doc/CHANGELOG.md && move-cli ./doc/_book ./docs",
     "test": "npm run test:lint && npm run test:coverage && npm run test:coverage:threshold",
     "test:coverage": "nyc --reporter=html mocha ./test && nyc report",
-    "test:coverage:threshold": "nyc check-coverage --lines 90 --functions 90 --branches 80",
+    "test:coverage:threshold": "nyc check-coverage --lines 90 --functions 89 --branches 80",
     "test:backend": "mocha ./test",
     "test:lint": "eslint --ext .js src test *.js plugins"
   },

package/src/Processor.js

@@ -62,6 +62,7 @@ const { mergeState, isUserInteraction } = require('./utils/stateVariables');
  * @prop {TrackingEvent[]} events
  * @prop {ResponseFlag|null} flag
  * @prop {boolean} nonInteractive
+ * @prop {string[]} responseTexts
  */
 
 /**
@@ -434,6 +435,7 @@ class Processor extends EventEmitter {
         const { events = [] } = messageSender.tracking;
 
         const event = {
+            responseTexts: messageSender.responseTexts,
             req,
             actions,
             lastAction,

package/src/ReturnSender.js

@@ -6,6 +6,7 @@
 const ai = require('./Ai');
 const { FEATURE_PHRASES, FEATURE_TRACKING } = require('./features');
 const { ResponseFlag } = require('./analytics/consts');
+const extractText = require('./transcript/extractText');
 
 /** @typedef {import('./Request')} Request */
 /** @typedef {import('./Responder')} Responder */
@@ -122,8 +123,12 @@ class ReturnSender {
             events: []
         };
 
+        this._responseTexts = [];
+
         this._intentsAndEntities = [];
 
+        this._confidentInput = false;
+
         /**
          * @type {Function}
          * @private
@@ -147,6 +152,18 @@ class ReturnSender {
         return this._simulatesOptIn;
     }
 
+    /**
+     * @returns {string[]}
+     */
+    get responseTexts () {
+        const filter = this._confidentInput
+            ? this.confidentInputFilter
+            : this.textFilter;
+
+        return this._responseTexts
+            .map((t) => filter(t));
+    }
+
     _gotAnotherEvent () {
         if (this._gotAnotherEventDefer) {
             this._gotAnotherEventDefer();
@@ -379,6 +396,10 @@ class ReturnSender {
             return;
         }
 
+        const text = extractText(payload);
+        if (text) {
+            this._responseTexts.push(text);
+        }
         this._queue.push(payload);
         this._gotAnotherEvent();
 
@@ -509,7 +530,7 @@ class ReturnSender {
     async finished (req = null, res = null, err = null, reportError = console.error) {
         this._finish(req);
         const meta = this._createMeta(req, res);
-        const confidentInput = req && req.isConfidentInput();
+        this._confidentInput = !!req && req.isConfidentInput();
         let error = err;
         try {
             await this._promise;
@@ -526,7 +547,7 @@ class ReturnSender {
             const processedEvent = req
                 ? req.event
                 : this._incommingMessage;
-            let incomming = this._filterMessage(processedEvent, confidentInput, req);
+            let incomming = this._filterMessage(processedEvent, this._confidentInput, req);
 
             if (processedEvent !== this._incommingMessage) {
                 incomming = {

package/src/analytics/consts.js

@@ -42,6 +42,10 @@ const TrackingType = { // max length 12
  * @enum {string} TrackingCategory
  */
 const TrackingCategory = { // max length 3
+    // PAGE_VIEW: 'page_view'
+    PAGE_VIEW_FIRST: 'pf',
+    PAGE_VIEW_SUBSEQUENT: 'pp',
+
     // CONVERSATION_EVENT: 'conversation'
     STICKER: 'sti',
     IMAGE: 'img',
@@ -58,18 +62,22 @@ const TrackingCategory = { // max length 3
 
     // TRAINING: 'train'
     INTENT_DETECTION: 'int',
-    HANDOVER_OCCURRED: 'hum',
     DISAMBIGUATION_SELECTED: 'dis',
     DISAMBIGUATION_OFFERED: 'dio',
 
     // REPORT: 'report'
-    REPORT_FEEDBACK: 'fdb'
+    REPORT_FEEDBACK: 'fdb',
+    HANDOVER_OCCURRED: 'hum'
 };
 
 /**
  * @type {Object<TrackingCategory,string>}
  */
 const CATEGORY_LABELS = {
+    // PAGE_VIEW: 'page_view'
+    [TrackingCategory.PAGE_VIEW_FIRST]: 'Bot: Interaction',
+    [TrackingCategory.PAGE_VIEW_SUBSEQUENT]: 'Bot: Sub-interaction',
+
     // CONVERSATION_EVENT: 'conversation'
     [TrackingCategory.STICKER]: 'User: Sticker',
     [TrackingCategory.IMAGE]: 'User: Image',
@@ -86,12 +94,12 @@ const CATEGORY_LABELS = {
 
     // TRAINING: 'train'
     [TrackingCategory.INTENT_DETECTION]: 'Intent: Detection',
-    [TrackingCategory.HANDOVER_OCCURRED]: 'Bot: Handover out',
     [TrackingCategory.DISAMBIGUATION_SELECTED]: 'Disambiguation: selected',
     [TrackingCategory.DISAMBIGUATION_OFFERED]: 'Disambiguation: offered',
 
     // REPORT: 'report'
-    [TrackingCategory.REPORT_FEEDBACK]: 'User: Feedback'
+    [TrackingCategory.REPORT_FEEDBACK]: 'User: Feedback',
+    [TrackingCategory.HANDOVER_OCCURRED]: 'Bot: Handover out'
 };
 
 module.exports = {

package/src/analytics/onInteractionHandler.js

@@ -31,6 +31,56 @@ const {
  * @prop {string} [action]
  * @prop {string} [label]
  * @prop {number} [value]
+ * @prop {string} [lang]
+ */
+
+/**
+ * @typedef {object} ConversationEventExtension
+ * @prop {string} [lastAction]
+ * @prop {string} [skill]
+ * @prop {string} [text]
+ * @prop {string} [expected]
+ * @prop {boolean} expectedTaken
+ * @prop {boolean} isContextUpdate
+ * @prop {boolean} isAttachment
+ * @prop {boolean} isNotification
+ * @prop {boolean} isQuickReply
+ * @prop {boolean} isPassThread
+ * @prop {boolean} isPostback
+ * @prop {boolean} isText
+ * @prop {boolean} didHandover
+ * @prop {boolean} withUser
+ * @prop {string} [userId]
+ * @prop {number} [feedback]
+ * @prop {string} [winnerAction]
+ * @prop {string} [winnerIntent]
+ * @prop {string[]|string} [winnerEntities]
+ * @prop {number} [winnerScore]
+ * @prop {boolean} [winnerTaken]
+ * @prop {string} [intent]
+ * @prop {number} [intentScore]
+ * @prop {string[]|string} [entities]
+ * @prop {string[]|string} allActions
+ * @prop {boolean} nonInteractive
+ *
+ * @typedef {Event & ConversationEventExtension} ConversationEvent
+ */
+
+/**
+ * @typedef {object} PageViewEventExtension
+ * @prop {string} [lastAction]
+ * @prop {string} [prevAction]
+ * @prop {string} [skill]
+ * @prop {string[]|string} allActions
+ * @prop {boolean} nonInteractive
+ * @prop {boolean} isGoto
+ * @prop {boolean} withUser
+ *
+ * @typedef {Event & PageViewEventExtension} PageViewEvent
+ */
+
+/**
+ * @typedef {ConversationEvent | Event | PageViewEvent} TrackingEvent
  */
 
 /**
@@ -40,6 +90,12 @@ const {
  * @prop {string} [action]
  * @prop {string} [snapshot]
  * @prop {string} [botId]
+ * @prop {boolean} [didHandover]
+ * @prop {number|null} [feedback]
+ * @prop {string} [timeZone]
+ * @prop {number} [sessionStart]
+ * @prop {number} [sessionDuration]
+ * @prop {string[]} [responseTexts]
  */
 
 /**
@@ -50,7 +106,6 @@ const {
  * @param {SessionMetadata} [metadata]
  * @param {number} [ts]
  * @param {boolean} [nonInteractive]
- * @param {string} [timeZone]
  * @returns {Promise}
  */
 
@@ -59,12 +114,12 @@ const {
  * @param {string} pageId
  * @param {string} senderId
  * @param {string} sessionId
- * @param {Event[]} events
+ * @param {TrackingEvent[]} events
  * @param {GAUser} [user]
  * @param {number} [ts]
  * @param {boolean} [nonInteractive]
  * @param {boolean} [sessionStarted]
- * @param {string} [timeZone]
+ * @param {SessionMetadata} [metadata]
  * @returns {Promise}
  */
 
@@ -82,6 +137,7 @@ const {
  * @prop {boolean} [supportsArrays]
  * @prop {boolean} [useDescriptiveCategories]
  * @prop {boolean} [useExtendedScalars]
+ * @prop {boolean} [parallelSessionInsert]
  */
 
 /**
@@ -98,7 +154,7 @@ const {
 
 /**
  * @typedef {object} TrackingEvents
- * @prop {Event[]} events
+ * @prop {TrackingEvent[]} events
  */
 
 /**
@@ -160,13 +216,14 @@ function onInteractionHandler (
         supportsArrays = false,
         useExtendedScalars = false,
         hasExtendedEvents = false,
-        useDescriptiveCategories = true
+        useDescriptiveCategories = true,
+        parallelSessionInsert = false
     } = analyticsStorage;
 
     const asArray = (data = []) => (supportsArrays ? data : data.join(','));
     const asCategory = (cat) => (useDescriptiveCategories && CATEGORY_LABELS[cat]) || cat;
     const noneAction = useExtendedScalars ? null : '(none)';
-    const noneValue = useExtendedScalars ? -1 : 0;
+    const noneValue = useExtendedScalars ? null : 0;
 
     /**
      * @param {InteractionEvent} params
@@ -180,7 +237,8 @@ function onInteractionHandler (
         skill,
         events,
         flag,
-        nonInteractive
+        nonInteractive,
+        responseTexts
     }) {
         if (!enabled) {
             return;
@@ -201,26 +259,60 @@ function onInteractionHandler (
                 lang
             } = req.state;
 
+            const trackEvents = [];
+
             const [action = noneAction, ...otherActions] = actions;
 
-            if (createSession) {
-                const metadata = {
-                    sessionCount,
-                    lang,
-                    action,
-                    snapshot,
-                    botId
-                };
+            const feedbackEvent = events.find((e) => e.type === TrackingType.REPORT
+                && e.category === TrackingCategory.REPORT_FEEDBACK);
+            const feedback = feedbackEvent
+                ? feedbackEvent.value
+                : noneValue;
+            let didHandover = flag === ResponseFlag.HANDOVER;
+            const hasHandoverEvent = events
+                .some((e) => e.category === TrackingCategory.HANDOVER_OCCURRED);
+
+            if (didHandover && !hasHandoverEvent) {
+                trackEvents.push({
+                    type: TrackingType.REPORT,
+                    category: asCategory(TrackingCategory.HANDOVER_OCCURRED),
+                    action: null,
+                    label: null,
+                    value: noneValue
+                });
+            } else if (hasHandoverEvent) {
+                didHandover = true;
+            }
+
+            const metadata = {
+                sessionCount,
+                lang,
+                action,
+                snapshot,
+                botId,
+                didHandover,
+                feedback,
+                timeZone,
+                sessionStart,
+                responseTexts,
+                sessionDuration: sessionTs - sessionStart
+            };
 
-                await analyticsStorage.createUserSession(
+            let sessionPromise;
+            if (createSession) {
+                sessionPromise = analyticsStorage.createUserSession(
                     pageId,
                     senderId,
                     sessionId,
                     metadata,
                     timestamp,
-                    nonInteractive,
-                    timeZone
+                    nonInteractive
                 );
+
+                if (!parallelSessionInsert) {
+                    await sessionPromise;
+                    sessionPromise = null;
+                }
             }
 
             const [{
@@ -233,6 +325,7 @@ function onInteractionHandler (
                 : anonymize(
                     replaceDiacritics(req.text()).replace(/\s+/g, ' ').toLowerCase().trim()
                 );
+            const useSkill = (skill && webalize(skill)) || noneAction;
 
             let winnerAction = '';
             let winnerScore = 0;
@@ -253,14 +346,6 @@ function onInteractionHandler (
             }
 
             const expected = req.expected() ? req.expected().action : '';
-
-            const feedbackEvent = events.find((e) => e.type === TrackingType.REPORT
-                && e.category === TrackingCategory.REPORT_FEEDBACK);
-            const feedback = feedbackEvent
-                ? feedbackEvent.value
-                : noneValue;
-            const didHandover = flag === ResponseFlag.HANDOVER;
-
             const user = userExtractor(req.state);
 
             const isContextUpdate = req.isSetContext();
@@ -274,12 +359,12 @@ function onInteractionHandler (
             const allActions = asArray(actions);
             const requestAction = req.action();
 
-            const trackEvents = [];
-
             const langsExtension = hasExtendedEvents
                 ? { lang }
                 : { cd1: lang };
 
+            const withUser = user !== null && !!user.id;
+
             const actionMeta = {
                 requestAction: req.action() || noneAction,
                 expected,
@@ -292,13 +377,9 @@ function onInteractionHandler (
                 isText,
                 isPostback,
                 didHandover,
-                withUser: user !== null && !!user.id,
-                sessionStart,
-                sessionDuration: sessionTs - sessionStart,
+                withUser,
                 feedback,
-                skill: webalize(skill),
-                snapshot,
-                botId,
+                skill: useSkill,
                 winnerAction,
                 winnerIntent,
                 winnerEntities: asArray(winnerEntities.map((e) => e.entity)),
@@ -311,16 +392,24 @@ function onInteractionHandler (
                 allActions
             };
 
+            const notHandled = actions.some((a) => a.match(/\*$/)) && !req.isQuickReply();
+            const value = notHandled ? 1 : 0;
+
             trackEvents.push({
                 type: TrackingType.PAGE_VIEW,
+                category: asCategory(TrackingCategory.PAGE_VIEW_FIRST),
                 action,
+                label: (isText || isQuickReply ? text : null),
+                value,
                 allActions,
                 nonInteractive,
                 lastAction,
+                // @ts-ignore
                 prevAction: lastAction,
-                skill,
-                lang,
-                cd1: req.state.lang,
+                skill: useSkill,
+                isGoto: false,
+                withUser,
+                ...langsExtension,
                 ...(hasExtendedEvents ? {} : actionMeta)
             });
 
@@ -330,12 +419,14 @@ function onInteractionHandler (
                 ...otherActions.map((a) => {
                     const r = {
                         type: TrackingType.PAGE_VIEW,
+                        category: asCategory(TrackingCategory.PAGE_VIEW_SUBSEQUENT),
                         action: a,
+                        value: 0,
                         allActions,
                         nonInteractive: false,
                         lastAction,
                         prevAction,
-                        skill,
+                        skill: useSkill,
                         isGoto: true,
                         ...langsExtension
                     };
@@ -347,14 +438,14 @@ function onInteractionHandler (
 
             trackEvents.push(
                 ...events.map(({
-                    type, category, action: eventAction, label, value
+                    type, category, action: eventAction, label, value: eVal
                 }) => ({
                     lastAction,
                     type,
-                    category,
+                    category: asCategory(category),
                     action: eventAction,
                     label,
-                    value,
+                    value: eVal,
                     ...langsExtension
                 }))
             );
@@ -375,11 +466,8 @@ function onInteractionHandler (
                     });
                 }
 
-                const notHandled = actions.some((a) => a.match(/\*$/)) && !req.isQuickReply();
-
                 let actionCategory;
                 let label = noneAction;
-                const value = notHandled ? 1 : 0;
 
                 if (isPassThread) {
                     actionCategory = TrackingCategory.HANDOVER_TO_BOT;
@@ -408,7 +496,7 @@ function onInteractionHandler (
                     actionCategory = TrackingCategory.REFERRAL;
                 } else if (isPostback) {
                     actionCategory = TrackingCategory.POSTBACK_BUTTON;
-                    label = req.data.postback.title || '(unknown)';
+                    label = req.event.postback.title || (useExtendedScalars ? null : '(unknown)');
                 } else {
                     actionCategory = TrackingCategory.OTHER;
                 }
@@ -416,7 +504,6 @@ function onInteractionHandler (
                 trackEvents.push({
                     ...(analyticsStorage.hasExtendedEvents ? actionMeta : {}),
                     type: TrackingType.CONVERSATION_EVENT,
-                    // @ts-ignore
                     lastAction,
                     category: asCategory(actionCategory),
                     action,
@@ -426,18 +513,21 @@ function onInteractionHandler (
                 });
             }
 
-            await analyticsStorage.storeEvents(
-                pageId,
-                senderId,
-                sessionId,
-                // @ts-ignore
-                trackEvents,
-                user,
-                timestamp,
-                nonInteractive,
-                createSession,
-                timeZone
-            );
+            await Promise.all([
+                analyticsStorage.storeEvents(
+                    pageId,
+                    senderId,
+                    sessionId,
+                    // @ts-ignore
+                    trackEvents,
+                    user,
+                    timestamp,
+                    nonInteractive,
+                    createSession,
+                    metadata
+                ),
+                sessionPromise
+            ]);
         } catch (e) {
             if (throwException) {
                 throw e;

package/src/tools/MemoryChatLogStorage.js

@@ -0,0 +1,70 @@
+/**
+ * @author David Menger
+ */
+'use strict';
+
+/**
+ * @class MemoryChatLogStorage
+ */
+class MemoryChatLogStorage {
+
+    constructor () {
+        this._logs = new Map();
+    }
+
+    /**
+     * Interate history
+     * all limits are inclusive
+     *
+     * @param {string} senderId
+     * @param {string} pageId
+     * @param {number} [limit]
+     * @param {number} [endAt] - iterate backwards to history
+     * @param {number} [startAt] - iterate forward to last interaction
+     * @returns {Promise<object[]>}
+     */
+    async getInteractions (senderId, pageId, limit = 10, endAt = null, startAt = null) { // eslint-disable-line max-len,no-unused-vars
+        const events = this._getEvents(senderId);
+        return events.slice(-limit);
+    }
+
+    /**
+     *
+     * @param {string} senderId
+     * @returns {object[]}
+     */
+    _getEvents (senderId) {
+        let events = this._logs.get(senderId);
+        if (!events) {
+            events = [];
+            this._logs.set(senderId, events);
+        }
+        return events;
+    }
+
+    /**
+     * Log single event
+     *
+     * @param {string} senderId
+     * @param {object[]} responses - list of sent responses
+     * @param {object} request - event request
+     * @param {object} [metadata] - request metadata
+     * @returns {void}
+     */
+    log (senderId, responses = [], request = {}, metadata = {}) {
+        const events = this._getEvents(senderId);
+        events.push({
+            senderId,
+            request,
+            responses,
+            ...metadata
+        });
+    }
+
+    error (error, senderId, sent, incomming, meta) {
+        return this.log(senderId, sent, incomming, { ...meta, error });
+    }
+
+}
+
+module.exports = MemoryChatLogStorage;

package/src/transcript/extractText.js

@@ -0,0 +1,54 @@
+/**
+ * @author David Menger
+ */
+'use strict';
+
+/**
+ * Extracts text from conversational event
+ *
+ * @param {object} payload
+ * @returns {string|null}
+ */
+function extractText (payload) {
+
+    // text message
+    if (payload.message && payload.message.text) {
+        return payload.message.text;
+    }
+
+    // button message
+    if (payload.message && payload.message.attachment
+        && payload.message.attachment.type === 'template'
+        && payload.message.attachment.payload
+        && payload.message.attachment.payload.text) {
+
+        return payload.message.attachment.payload.text;
+    }
+
+    if (!payload.postback) {
+        return null;
+    }
+
+    // postback with title
+    if (payload.postback.title) {
+        return payload.postback.title;
+    }
+
+    if (payload.postback.payload && payload.postback.payload.action) {
+        return payload.postback.payload.action;
+    }
+
+    if (typeof payload.postback.payload !== 'string') {
+        return null;
+    }
+
+    if (payload.postback.payload[0] === '{') {
+        const pl = JSON.parse(payload.postback.payload);
+
+        return pl.action;
+    }
+
+    return payload.postback.payload;
+}
+
+module.exports = extractText;

package/src/transcript/htmlBodyFromTranscript.js

@@ -0,0 +1,21 @@
+/**
+ * @author David Menger
+ */
+'use strict';
+
+/** @typedef {import('./transcriptFromHistory').Transcript} Transcript */
+
+/**
+ * @param {Transcript[]} transcript
+ * @param {string} [userSide]
+ * @param {string} [botSide]
+ * @returns {string}
+ */
+function htmlBodyFromTranscript (transcript, userSide = 'User', botSide = 'Bot') {
+
+    return transcript
+        .map((msg) => `<b>${msg.fromBot ? botSide : userSide}:</b> ${msg.text}`)
+        .join('<br />');
+}
+
+module.exports = htmlBodyFromTranscript;

package/src/transcript/textBodyFromTranscript.js

@@ -0,0 +1,21 @@
+/**
+ * @author David Menger
+ */
+'use strict';
+
+/** @typedef {import('./transcriptFromHistory').Transcript} Transcript */
+
+/**
+ * @param {Transcript[]} transcript
+ * @param {string} [userSide]
+ * @param {string} [botSide]
+ * @returns {string}
+ */
+function textBodyFromTranscript (transcript, userSide = 'User', botSide = 'Bot') {
+
+    return transcript
+        .map((msg, i) => `${msg.fromBot ? '  <' : `${i > 0 ? '\n' : ''}# >`} ${msg.fromBot ? botSide : userSide}: ${msg.text}`)
+        .join('\n');
+}
+
+module.exports = textBodyFromTranscript;

package/src/transcript/transcriptFromHistory.js

@@ -0,0 +1,52 @@
+/**
+ * @author David Menger
+ */
+'use strict';
+
+const extractText = require('./extractText');
+
+/**
+ * @callback GetInteractions
+ * @param {string} senderId
+ * @param {string} pageId
+ * @param {number} limit
+ * @returns {Promise<object[]>}
+ */
+
+/**
+ * @typedef {object} IChatStorage
+ * @prop {GetInteractions} getInteractions
+ */
+
+/**
+ * @typedef {object} Transcript
+ * @prop {string} text
+ * @prop {boolean} fromBot
+ */
+
+/**
+ *
+ * @param {IChatStorage} chatLogStorage
+ * @param {string} senderId
+ * @param {string} pageId
+ * @param {number} limit
+ * @returns {Promise<Transcript[]>}
+ */
+async function transcriptFromHistory (chatLogStorage, senderId, pageId, limit = 20) {
+    const data = await chatLogStorage.getInteractions(senderId, pageId, limit);
+
+    return data
+        .map((turn) => {
+            const { request, responses = [] } = turn;
+
+            return [
+                { fromBot: false, text: extractText(request) },
+                ...responses
+                    .map((response) => ({ fromBot: true, text: extractText(response) }))
+            ];
+        })
+        .reduce((ret, arr) => [...ret, ...arr], [])
+        .filter((ret) => !!ret.text);
+}
+
+module.exports = transcriptFromHistory;