wingbot 3.44.0-alpha.1 → 3.44.1

package/index.js

@@ -45,6 +45,8 @@ const { disambiguationQuickReply, quickReplyAction } = require('./src/utils/quic
 const { getUpdate, getValue, getSetState } = require('./src/utils/getUpdate');
 const { vars } = require('./src/utils/stateVariables');
 const compileWithState = require('./src/utils/compileWithState');
+const onInteractionHandler = require('./src/analytics/onInteractionHandler');
+const GA4 = require('./src/analytics/GA4');
 const plugins = require('./plugins/plugins.json');
 const {
     bufferloader,
@@ -123,5 +125,9 @@ module.exports = {
     // flags
     ...flags,
 
-    wingbotVersion
+    wingbotVersion,
+
+    // ANALYTICS
+    onInteractionHandler,
+    GA4
 };

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "wingbot",
-  "version": "3.44.0-alpha.1",
+  "version": "3.44.1",
   "description": "Enterprise Messaging Bot Conversation Engine",
   "main": "index.js",
   "scripts": {

package/src/BotApp.js

@@ -10,6 +10,7 @@ const BotAppSender = require('./BotAppSender');
 const Processor = require('./Processor');
 const ReturnSender = require('./ReturnSender');
 const headersToAuditMeta = require('./utils/headersToAuditMeta');
+const onInteractionHandler = require('./analytics/onInteractionHandler');
 
 const DEFAULT_API_URL = 'https://orchestrator-api.wingbot.ai';
 
@@ -25,6 +26,9 @@ const DEFAULT_API_URL = 'https://orchestrator-api.wingbot.ai';
 /** @typedef {import('./BotAppSender').TlsOptions} TlsOptions */
 /** @typedef {import('./ReturnSender').ReturnSenderOptions} ReturnSenderOptions */
 
+/** @typedef {import('./analytics/onInteractionHandler').IAnalyticsStorage} IAnalyticsStorage */
+/** @typedef {import('./analytics/onInteractionHandler').HandlerConfig} HandlerConfig */
+
 /**
  * @typedef {object} BotAppOptions
  * @prop {string|Promise<string>} secret
@@ -95,6 +99,8 @@ class BotApp {
         this._appId = appId;
         this._auditLog = auditLog;
         this._tls = tls;
+        this._logger = options.log || console;
+        this._textFilter = options.textFilter;
 
         let { apiUrl } = options;
 
@@ -171,6 +177,34 @@ class BotApp {
         return this._processor;
     }
 
+    /**
+     *
+     * @param {IAnalyticsStorage} analyticsStorage
+     * @param {HandlerConfig} [options]
+     * @returns {this}
+     * @example
+     * const { GA4 } = require('wingbot');
+     *
+     * botApp.registerAnalyticsStorage(new GA4({
+     *     measurementId: 'G-123456,
+     *     apiSecret: 'apisecret'
+     * }))
+     */
+    registerAnalyticsStorage (analyticsStorage, options = {}) {
+        const log = this._logger || options.log;
+
+        analyticsStorage.setDefaultLogger(log);
+
+        const handler = onInteractionHandler({
+            log,
+            anonymize: this._textFilter,
+            ...options
+        }, analyticsStorage);
+
+        this.processor.on('interaction', handler);
+        return this;
+    }
+
     _errorResponse (message, status) {
         return {
             statusCode: status,

package/src/Processor.js

@@ -65,11 +65,18 @@ const { mergeState, isUserInteraction } = require('./utils/stateVariables');
  * @type {InteractionEvent}
  */
 
+/**
+ * @typedef {object} ILogger
+ * @prop {Function} log
+ * @prop {Function} warn
+ * @prop {Function} error
+ */
+
 /**
  *
  * @typedef {object} ProcessorOptions
  * @prop {string} [appUrl] - url basepath for relative links
- * @prop {object} [stateStorage] - chatbot state storage
+ * @prop {IStateStorage} [stateStorage] - chatbot state storage
  * @prop {object} [tokenStorage] - frontend token storage
  * @prop {Function} [translator] - text translate function
  * @prop {number} [timeout] - chat sesstion lock duration (30000)
@@ -78,7 +85,7 @@ const { mergeState, isUserInteraction } = require('./utils/stateVariables');
  * @prop {number} [retriesWhenWaiting] - number of attampts (6)
  * @prop {Function} [nameFromState] - override the name translator
  * @prop {boolean|AutoTypingConfig} [autoTyping] - enable or disable automatic typing
- * @prop {Function} [log] - console like error logger
+ * @prop {ILogger} [log] - console like error logger
  * @prop {object} [defaultState] - default chat state
  * @prop {boolean} [autoSeen] - send seen automatically
  * @prop {number} [redirectLimit] - maximum number of redirects at single request
@@ -103,6 +110,13 @@ const { mergeState, isUserInteraction } = require('./utils/stateVariables');
  * @prop {string|null} [meta.targetAction]
  */
 
+/**
+ * @typedef {object} IStateStorage
+ * @prop {Function} saveState
+ * @prop {Function} getState
+ * @prop {Function} getOrCreateAndLock
+ */
+
 function NAME_FROM_STATE (state) {
     if (state.user && state.user.firstName) {
         return `${state.user.firstName} ${state.user.lastName}`;
@@ -157,7 +171,7 @@ class Processor extends EventEmitter {
         this.reducer = reducer;
 
         /**
-         * @type {StateStorage}
+         * @type {IStateStorage}
          */
         this.stateStorage = this.options.stateStorage;
 
@@ -236,7 +250,7 @@ class Processor extends EventEmitter {
 
     async _reportError (pageId, err, event, senderId = null) {
         if (err.code === 204) {
-            this.options.log.info(`nothing sent: ${err.message}`, event);
+            this.options.log.log(`nothing sent: ${err.message}`, event);
             return;
         }
         if (err.code !== 403) {

package/src/analytics/GA4.js

@@ -0,0 +1,272 @@
+/**
+ * @author David Menger
+ */
+'use strict';
+
+const fetch = require('node-fetch').default;
+
+/** @typedef {import('./onInteractionHandler').Event} Event */
+/** @typedef {import('./onInteractionHandler').IAnalyticsStorage} IAnalyticsStorage */
+/** @typedef {import('./onInteractionHandler').GAUser} GAUser */
+/** @typedef {import('./onInteractionHandler').SessionMetadata} SessionMetadata */
+/** @typedef {import('./onInteractionHandler').IGALogger} IGALogger */
+
+/** @typedef {import('node-fetch').RequestInit} RequestInit */
+
+/**
+ * @typedef {object} FetchResult
+ * @param {number} status
+ * @param {string} [statusText]
+ * @param {Promise<object>} json
+ */
+
+/**
+ * @callback MockFetch
+ * @param {string} url
+ * @param {RequestInit} [options]
+ * @returns {Promise<FetchResult>}
+ */
+
+/**
+ * @typedef {object} GAOptions
+ * @prop {string} measurementId
+ * @prop {string} apiSecret
+ * @prop {boolean} [debug]
+ * @prop {IGALogger} [log]
+ * @prop {MockFetch} [fetch]
+ */
+/**
+ * @class GA4
+ * @implements {IAnalyticsStorage}
+ */
+class GA4 {
+
+    /**
+     *
+     * @param {GAOptions} options
+     */
+    constructor (options) {
+        this._options = options;
+
+        /** @type {IGALogger} */
+        this._logger = options.log || console;
+
+        this._urlQuery = `measurement_id=${encodeURIComponent(options.measurementId)}&api_secret=${encodeURIComponent(options.apiSecret)}`;
+        this._url = `https://www.google-analytics.com/mp/collect?${this._urlQuery}`;
+
+        this.hasExtendedEvents = true;
+
+        this._fetch = options.fetch || fetch;
+    }
+
+    /**
+     * @param {IGALogger} logger
+     */
+    setDefaultLogger (logger) {
+        if (this._logger === console) {
+            this._logger = logger;
+        }
+    }
+
+    /**
+     *
+     * @param {string} pageId
+     * @param {string} senderId
+     * @param {string} sessionId
+     * @param {SessionMetadata} [metadata]
+     * @param {number} [ts]
+     * @param {boolean} [nonInteractive]
+     * @returns {Promise}
+     */
+    async createUserSession (
+        pageId,
+        senderId,
+        sessionId,
+        metadata = {},
+        ts = Date.now(),
+        nonInteractive = false
+    ) {
+        const uafvl = 'wingbot';
+
+        const { lang = '', sessionCount = 1, action = '/' } = metadata;
+
+        const event = {
+            v: 2,
+            tid: this._options.measurementId,
+            _p: Math.round(2147483647 * Math.random()),
+            sr: '1x1',
+            _dbg: this._options.debug ? 1 : 0,
+            ul: lang, // language
+            cid: this._conversationId(pageId, senderId),
+
+            // dl: 'https://wingbot-web-staging.flyto.cloud/dp',
+            dp: action,
+            dr: '', // referral (url)
+            dt: action === '/' ? '(none)' : action.replace(/-/g, ' '),
+
+            // en: 'page_view', // event name
+            en: 'scroll',
+            'epn.percent_scrolled': 100,
+
+            uafvl, // must have
+
+            sct: sessionCount, // session count (int)
+            seg: nonInteractive ? 0 : 1, // session engagement (boolean)
+            sid: sessionId, // session id (string)
+
+            // this was sent during the first visit
+            _fv: sessionCount === 1, // first visit (bool)
+            _nsi: 1, // new session id (bool)
+            _ss: 1, // session start (bool)
+
+            _ee: 1, // ? page_view event parameter (??? event engagement ??)
+
+            _s: 1, // session hit count (was 1 every request),
+            _et: ts // event time (number)
+        };
+
+        if (this._options.debug) {
+            this._logger.log('GA4: starting session', event);
+        }
+
+        const query = Object.entries(event)
+            .map(([k, v]) => `${encodeURIComponent(k)}=${encodeURIComponent(v)}`)
+            .join('&');
+
+        // const url = 'https://region1.google-analytics.com/g/collect';
+        const url = 'https://www.google-analytics.com/g/collect';
+
+        const res = await this._fetch(`${url}?${query}`, {
+            method: 'POST',
+            headers: {
+                'user-agent': uafvl
+            }
+        });
+
+        if (res.status >= 400) {
+            this._logger.error('GA4: failed to create session', {
+                url,
+                query,
+                status: res.status
+            });
+        }
+    }
+
+    _conversationId (pageId, senderId) {
+        return `${pageId}.${senderId}`;
+    }
+
+    /**
+     *
+     * @param {string} pageId
+     * @param {string} senderId
+     * @param {string} sessionId
+     * @param {Event[]} events
+     * @param {GAUser} [user]
+     * @param {number} [ts]
+     * @returns {Promise}
+     */
+    async storeEvents (
+        pageId,
+        senderId,
+        sessionId,
+        events,
+        user = null,
+        ts = Date.now()
+    ) {
+        if (events.length === 0) {
+            return;
+        }
+
+        const body = {
+            client_id: this._conversationId(pageId, senderId),
+            timestamp_micros: ts * 1000,
+            non_personalized_ads: false,
+            events: events.map((e) => {
+                const { type: name, ...params } = e;
+                Object.entries(params)
+                    .forEach(([k, v]) => {
+                        if (v === null) {
+                            params[k] = '(none)';
+                        }
+                    });
+                // Object.assign(params, { session_id: sessionId });
+                switch (name) {
+                    case 'page_view':
+                        return {
+                            name,
+                            params: {
+                                page_path: e.action,
+                                page_title: e.action
+                                    .replace(/^\/+/, '')
+                                    .replace(/[-]+/g, ' ')
+                                    .replace(/[/]+/g, ' - '),
+                                ...params
+                            }
+                        };
+                    default:
+                        return { name, params };
+                }
+            })
+        };
+
+        if (user) {
+            const { id, ...other } = user;
+
+            Object.assign(body, {
+                user_id: id,
+                user_properties: Object.fromEntries(
+                    Object.entries(other)
+                        .map(([key, value]) => [key, { value }])
+                )
+            });
+        }
+
+        const params = {
+            method: 'POST',
+            body: JSON.stringify(body)
+        };
+
+        let err;
+        let res;
+        try {
+            res = await this._fetch(this._url, params);
+
+            if (res.status >= 400) {
+                throw new Error(`${res.statusText} [${res.status}]`);
+            }
+            if (!this._options.debug) {
+                return;
+            }
+        } catch (e) {
+            err = e;
+        }
+
+        let { message = 'GENERIC FAIL' } = err || { message: null };
+        let validationMessages = [];
+        try {
+            const dbg = await this._fetch(`https://www.google-analytics.com/debug/mp/collect?${this._urlQuery}`, params);
+
+            if (dbg.status >= 300) {
+                throw new Error(`${dbg.statusText} [${dbg.status}]`);
+            }
+
+            ({ validationMessages = [] } = await dbg.json());
+
+            message = (validationMessages[0] || { description: message }).description;
+        } catch (e) {
+            this._logger.log('GA4 debug failed', e);
+            message += ` +(${e.message})`;
+        }
+
+        if (validationMessages.length || message) {
+            this._logger.log('GA4: validationMessages', validationMessages);
+            this._logger.error(`GA4: fail: ${message} [${res ? res.status : 0}]`, params.body);
+        } else {
+            this._logger.log(`GA4: debug [${res ? res.status : 0}]`, params.body);
+        }
+    }
+
+}
+
+module.exports = GA4;

package/src/analytics/onInteractionHandler.js

@@ -0,0 +1,393 @@
+/**
+ * @author wingbot.ai
+ */
+'use strict';
+
+const { replaceDiacritics } = require('webalize');
+const Ai = require('../Ai');
+
+/** @typedef {import('../Processor').InteractionEvent} InteractionEvent */
+/** @typedef {import('../Request')} Request */
+
+/**
+ * @typedef {object} GAUser
+ * @prop {string} [id]
+ */
+
+/**
+ * @typedef IGALogger
+ * @prop {Function} log
+ * @prop {Function} error
+ */
+
+/**
+ * @typedef {object} Event
+ * @prop {'conversation'|'page_view'|string} type
+ * @prop {string} [category]
+ * @prop {string} [action]
+ * @prop {string} [label]
+ * @prop {number} [value]
+ */
+
+/**
+ * @typedef {object} SessionMetadata
+ * @prop {number} [sessionCount]
+ * @prop {string} [lang]
+ * @prop {string} [action]
+ */
+
+/**
+ * @callback CreateUserSession
+ * @param {string} pageId
+ * @param {string} senderId
+ * @param {string} sessionId
+ * @param {SessionMetadata} [metadata]
+ * @param {number} [ts]
+ * @param {boolean} [nonInteractive]
+ * @returns {Promise}
+ */
+
+/**
+ * @callback StoreEvents
+ * @param {string} pageId
+ * @param {string} senderId
+ * @param {string} sessionId
+ * @param {Event[]} events
+ * @param {GAUser} [user]
+ * @param {number} [ts]
+ * @param {boolean} [nonInteractive]
+ * @param {boolean} [sessionStarted]
+ * @returns {Promise}
+ */
+
+/**
+ * @callback LoggerSetter
+ * @param {IGALogger} logger
+ */
+
+/**
+ * @typedef {object} IAnalyticsStorage
+ * @prop {LoggerSetter} setDefaultLogger - console like logger
+ * @prop {StoreEvents} storeEvents
+ * @prop {CreateUserSession} createUserSession
+ * @prop {boolean} [hasExtendedEvents]
+ */
+
+/**
+ * @callback UserExtractor
+ * @param {Request} req
+ * @returns {object & GAUser}
+ */
+
+/**
+ * @callback Anonymizer
+ * @param {string} text
+ * @returns {string}
+ */
+
+/**
+ * @typedef {object} TrackingEvents
+ * @prop {Event[]} events
+ */
+
+/**
+ * @typedef {object} IConfidenceProvider
+ * @prop {number} confidence
+ */
+
+/**
+ * @typedef {object} HandlerConfig
+ * @prop {boolean} [enabled] - default true
+ * @prop {boolean} [throwException] - default false
+ * @prop {IGALogger} [log] - console like logger
+ * @prop {Anonymizer} [anonymize] - text anonymization function
+ * @prop {UserExtractor} [userExtractor] - text anonymization function
+ */
+
+/**
+ * @callback IInteractionHandler
+ * @param {InteractionEvent} params
+ * @returns {Promise}
+ */
+
+/**
+ *
+ * @param {HandlerConfig} config
+ * @param {IAnalyticsStorage} analyticsStorage
+ * @param {IConfidenceProvider} [ai]
+ * @returns {IInteractionHandler}
+ */
+function onInteractionHandler (
+    {
+        enabled = true,
+        throwException = false,
+        log = console,
+        anonymize = (x) => x,
+        userExtractor = (req) => null // eslint-disable-line no-unused-vars
+    },
+    analyticsStorage,
+    ai = Ai.ai
+) {
+
+    /**
+     * @param {InteractionEvent} params
+     */
+    async function onInteraction ({
+        req,
+        actions,
+        lastAction,
+        // state,
+        // data,
+        skill,
+        tracking
+    }) {
+        if (!enabled) {
+            return;
+        }
+        try {
+            const nonInteractive = !!req.campaign;
+            const {
+                pageId,
+                senderId,
+                timestamp
+            } = req;
+
+            const {
+                _snew: createSession,
+                _sct: sessionCount,
+                _sid: sessionId,
+                lang
+            } = req.state;
+
+            const [action = '(none)', ...otherActions] = actions;
+
+            if (createSession) {
+                const metadata = {
+                    sessionCount,
+                    lang,
+                    action
+                };
+
+                await analyticsStorage.createUserSession(
+                    pageId,
+                    senderId,
+                    sessionId,
+                    metadata,
+                    timestamp,
+                    nonInteractive
+                );
+            }
+
+            const [{
+                intent = '',
+                score = 0
+            } = {}] = req.intents;
+
+            const text = req.isConfidentInput()
+                ? '*****'
+                : anonymize(
+                    replaceDiacritics(req.text()).replace(/\s+/g, ' ').toLowerCase().trim()
+                );
+
+            let winnerAction = '';
+            let winnerScore = 0;
+            let winnerIntent = '';
+            let winnerEntities = [];
+            let winnerTaken = false;
+
+            const winners = req.aiActions();
+
+            if (winners.length > 0) {
+                [{
+                    action: winnerAction = '(none)',
+                    sort: winnerScore = 0,
+                    intent: { intent: winnerIntent, entities: winnerEntities = [] }
+                }] = winners;
+
+                winnerTaken = action === winnerAction;
+            }
+
+            const expected = req.expected() ? req.expected().action : '';
+
+            const isContextUpdate = req.isSetContext();
+            const isNotification = !!req.campaign;
+            const isAttachment = req.isAttachment();
+            const isQuickReply = req.isQuickReply();
+            const isPassThread = !!req.event.pass_thread_control;
+            const isText = !isQuickReply && req.isText();
+            const isPostback = req.isPostBack();
+
+            const allActions = actions.join(',');
+            const requestAction = req.action();
+
+            const events = [];
+
+            const actionMeta = {
+                requestAction: req.action() || '(none)',
+                expected,
+                expectedTaken: requestAction === expected,
+                isContextUpdate,
+                isAttachment,
+                isNotification,
+                isQuickReply,
+                isPassThread,
+                isText,
+                isPostback,
+                winnerAction,
+                winnerIntent,
+                winnerEntities: winnerEntities.map((e) => e.entity).join(','),
+                winnerScore,
+                winnerTaken,
+                intent,
+                intentScore: score,
+                entities: req.entities.map((e) => e.entity).join(','),
+                text,
+                allActions
+            };
+
+            events.push({
+                type: 'page_view',
+                action,
+                allActions,
+                nonInteractive,
+                lastAction,
+                prevAction: lastAction,
+                skill,
+                lang,
+                cd1: req.state.lang,
+                ...(analyticsStorage.hasExtendedEvents ? {} : actionMeta)
+            });
+
+            let prevAction = action;
+
+            events.push(
+                ...otherActions.map((a) => {
+                    const r = {
+                        type: 'page_view',
+                        action: a,
+                        allActions,
+                        nonInteractive: false,
+                        lastAction,
+                        prevAction,
+                        skill,
+                        isGoto: true,
+                        ...(analyticsStorage.hasExtendedEvents
+                            ? { lang }
+                            : { cd1: lang })
+                    };
+
+                    prevAction = a;
+                    return r;
+                })
+            );
+
+            events.push(
+                ...tracking.events.map(({
+                    type, category, action: eventAction, label, value
+                }) => ({
+                    lastAction,
+                    type,
+                    category,
+                    action: eventAction,
+                    label,
+                    value,
+                    ...(analyticsStorage.hasExtendedEvents
+                        ? { lang }
+                        : { cd1: lang })
+                }))
+            );
+
+            if (!nonInteractive) {
+
+                if (req.isText()) {
+                    events.push({
+                        type: 'ai',
+                        // @ts-ignore
+                        lastAction,
+                        category: 'Intent: Detection',
+                        intent,
+                        action,
+                        label: text,
+                        value: score >= ai.confidence ? 0 : 1,
+                        ...(analyticsStorage.hasExtendedEvents
+                            ? { lang }
+                            : { cd1: lang })
+                    });
+                }
+
+                const notHandled = actions.some((a) => a.match(/\*$/)) && !req.isQuickReply();
+
+                let actionCategory = 'User: ';
+                let label = '(none)';
+                const value = notHandled ? 1 : 0;
+
+                if (req.isSticker()) {
+                    actionCategory += 'Sticker';
+                    label = req.attachmentUrl(0);
+                } else if (req.isImage()) {
+                    actionCategory += 'Image';
+                    label = req.attachmentUrl(0);
+                } else if (req.hasLocation()) {
+                    actionCategory += 'Location';
+                    const { lat, long } = req.getLocation();
+                    label = `${lat}, ${long}`;
+                } else if (isAttachment) {
+                    actionCategory += 'Attachement';
+                    label = req.attachment(0).type;
+                } else if (isText) {
+                    actionCategory += 'Text';
+                    label = text;
+                } else if (isQuickReply) {
+                    actionCategory += 'Quick reply';
+                    label = text;
+                } else if (req.isReferral() || req.isOptin()) {
+                    actionCategory = req.isOptin()
+                        ? 'Entry: Optin'
+                        : 'Entry: Referral';
+                } else if (isPostback) {
+                    actionCategory += 'Button - bot';
+                    label = req.data.postback.title || '(unknown)';
+                } else {
+                    actionCategory += 'Other';
+                }
+
+                events.push({
+                    ...(analyticsStorage.hasExtendedEvents ? actionMeta : {}),
+                    type: 'conversation',
+                    // @ts-ignore
+                    lastAction,
+                    category: actionCategory,
+                    action,
+                    label,
+                    value,
+                    ...(analyticsStorage.hasExtendedEvents
+                        ? { lang }
+                        : { cd1: lang })
+                });
+            }
+
+            const user = userExtractor(req);
+
+            await analyticsStorage.storeEvents(
+                pageId,
+                senderId,
+                sessionId,
+                // @ts-ignore
+                events,
+                user,
+                timestamp,
+                nonInteractive,
+                createSession
+            );
+        } catch (e) {
+            if (throwException) {
+                throw e;
+            }
+            log.error('failed sending logs', e);
+        }
+    }
+
+    return onInteraction;
+}
+
+module.exports = onInteractionHandler;