wingbot 3.71.6 → 3.72.0

package/src/Tester.js

@@ -19,8 +19,12 @@ const Router = require('./Router'); // eslint-disable-line no-unused-vars
 const ReducerWrapper = require('./ReducerWrapper'); // eslint-disable-line no-unused-vars
 const { FEATURE_TEXT } = require('./features');
 const LLMMockProvider = require('./LLMMockProvider');
+const PromptAssert = require('./testTools/PromptAssert');
 
 /** @typedef {import('./Processor').ProcessorOptions<Router>} ProcessorOptions */
+/** @typedef {import('./LLM').PromptInfo} PromptInfo */
+/** @typedef {import('./LLM').LLMRole} LLMRole */
+/** @typedef {import('./LLM').LLMMessage} LLMMessage */
 
 /**
  * Utility for testing requests
@@ -128,6 +132,9 @@ class Tester {
         this.responses = [];
         this.actions = [];
 
+        /** @type {PromptInfo[]} */
+        this.prompts = [];
+
         /**
          * @prop {object} predefined test data to use
          */
@@ -201,6 +208,7 @@ class Tester {
         this._actionsCollector = [];
         this._pluginBlocksCollector = [];
         this._responsesMock = [];
+        this.prompts = [];
     }
 
     /**
@@ -259,6 +267,7 @@ class Tester {
             throw Object.assign(new Error(`Processor failed with status ${res.status}`), { code: res.status });
         }
         this.responses = messageSender.responses;
+        this.prompts = messageSender.prompts;
         this.pluginBlocks = this._pluginBlocksCollector;
         this.actions = this._actionsCollector;
         this._actionsCollector = [];
@@ -388,6 +397,36 @@ class Tester {
         this.storage.saveState(stateObj);
     }
 
+    /**
+     *
+     * @returns {PromptAssert}
+     */
+    anyPrompt () {
+        return new PromptAssert(this.prompts);
+    }
+
+    /**
+     *
+     * @returns {PromptAssert}
+     */
+    lastPrompt () {
+        return new PromptAssert(
+            this.prompts.length
+                ? [this.prompts[this.prompts.length - 1]]
+                : []
+        );
+    }
+
+    /**
+     *
+     * @returns {LLMMessage}
+     */
+    getLastPromptResult () {
+        return this.prompts.length
+            ? this.prompts[this.prompts.length - 1].result
+            : null;
+    }
+
     /**
      * Assert, that state contains a subset of provided value
      *
@@ -606,9 +645,10 @@ class Tester {
     /**
      * Prints last conversation turnaround
      *
-     * @param {boolean} [showPrivateKeys]
+     * @param {boolean} [full=false]
+     * @param {boolean} [showPrivateKeys=false]
      */
-    debug (showPrivateKeys = false) {
+    debug (full = false, showPrivateKeys = false) {
         // eslint-disable-next-line no-console
         console.log(
             '\n===== actions =====\n',
@@ -625,6 +665,8 @@ class Tester {
                 Object.entries(this.getState().state)
                     .filter((e) => showPrivateKeys || !e[0].startsWith('_'))
             ),
+            '\n------- LLM -------\n',
+            ...PromptAssert.debug(this.prompts, full, true),
             '\n===================\n'
         );
     }

package/src/resolvers/contextMessage.js

@@ -8,30 +8,47 @@ const Router = require('../Router');
 const Responder = require('../Responder');
 const { getLanguageText } = require('./utils');
 const compileWithState = require('../utils/compileWithState');
+const LLM = require('../LLM');
 
 /** @typedef {import('../BuildRouter').BotContext<any>} BotContext */
 /** @typedef {import('../Router').Resolver<any>} Resolver */
 /** @typedef {import('./utils').Translations} Translations */
+/** @typedef {import('../LLM').EvaluationRule} EvaluationRule */
+
+/**
+ * @typedef {object} ContextMessageParams
+ * @property {Translations} context
+ * @property {string} [type]
+ * @property {EvaluationRule[]} [rules]
+ */
 
 /**
  *
- * @param {object} params
- * @param {Translations} params.context
- * @param {string} [params.type]
+ * @param {ContextMessageParams} params
  * @param {BotContext} context
  * @returns {Resolver}
  */
 // eslint-disable-next-line no-unused-vars
 function contextMessage (params, context) {
 
+    const { rules = [] } = params;
+    const preprocessedRules = LLM.preprocessEvaluationRules(rules, context);
+
     return async (req, res) => {
-        res.llmAddSystemPrompt((r) => {
+        res.llmAddInstructions((r) => {
             const translated = getLanguageText(params.context, req.state.lang);
             const translatedText = Array.isArray(translated) ? translated[0] : translated;
             const statefulPrompt = compileWithState(req, r, translatedText);
             return statefulPrompt;
         }, params.type);
 
+        preprocessedRules.forEach((rule) => res.llmAddResultRule(
+            rule,
+            undefined,
+            undefined,
+            params.type
+        ));
+
         return Router.CONTINUE;
     };
 }

package/src/resolvers/message.js

@@ -23,8 +23,11 @@ const { vars, VAR_TYPES } = require('../utils/stateVariables');
 const LLM = require('../LLM');
 
 /** @typedef {import('../Responder').VoiceControl} VoiceControl */
+/** @typedef {import('../BuildRouter').LinksMap} LinksMap */
 /** @typedef {import('./utils').Translations} Translations */
 /** @typedef {import('./utils').TextObject} TextObject */
+/** @typedef {import('../utils/getCondition').ConditionDefinition} ConditionDefinition */
+/** @typedef {import('../utils/getCondition').ConditionContext} ConditionContext */
 /**
  * Returns voice control props from params
  *
@@ -68,6 +71,30 @@ function getVoiceControlFromParams (params, lang = null) {
     return Object.keys(voiceControl).length > 0 ? voiceControl : null;
 }
 
+/**
+ * @typedef {object} QuickReplyData
+ * @prop {boolean} [isLocation]
+ * @prop {boolean} [isEmail]
+ * @prop {boolean} [isPhone]
+ * @prop {boolean} [trackAsNegative]
+ * @prop {string} [action]
+ * @prop {string} [targetRouteId]
+ * @prop {{l:string,t:string[]}[] | string} [title]
+ * @prop {object} [setState]
+ * @prop {string[]} [aiTags]
+ * @prop {{l:string,t:string[]}[] | string} [aiTitle]
+ */
+
+/**
+ * @typedef {ConditionDefinition & QuickReplyData} QuickReply
+ */
+
+/**
+ *
+ * @param {QuickReply[]} replies
+ * @param {LinksMap} linksMap
+ * @param {ConditionContext} context
+ */
 function parseReplies (replies, linksMap, context) {
     return replies.map((reply) => {
 
@@ -314,8 +341,9 @@ function message (params, context = {}) {
     /**
      * @param {Request} req
      * @param {Responder} res
+     * @param {Function } postBack
      */
-    return async (req, res) => {
+    return async (req, res, postBack) => {
         if (condition === undefined) {
             condition = getCondition(params, context, 'Message condition');
         }
@@ -425,6 +453,20 @@ function message (params, context = {}) {
             await session.systemPrompt(text)
                 .generate();
 
+            const evaluation = await res.llmEvaluate(session, params.llmContextType);
+
+            if (evaluation.discard) {
+                if (isLastMessage && !req.actionData()._resolverTag) {
+                    res.finalMessageSent = true;
+                }
+                return ret;
+            }
+
+            if (evaluation.action) {
+                postBack(evaluation.action);
+                return Router.END;
+            }
+
             // if (!response.content) {
             //    // no response?
             // }

package/src/resolvers/utils.js

@@ -31,17 +31,26 @@ const WEBVIEW_COMPACT = 'compact';
  * @prop {string|null} p - purpose
  */
 
+/**
+ * null = text+voice
+ * t = text only
+ * v = voice only
+ * s = ssml
+ *
+ * @typedef {"t"|"v"|"s"|null} Purpose
+ */
+
 /**
  * @typedef Translation
  * @property {string|string[]} t - text alternatives
  * @property {string | null} l - language
- * @property {string|string[]} [p] - purposes
- * null = default + voice
- * t = text
- * v = voice
- * s = ssml
+ * @property {Purpose | Purpose[]} [p] - purposes
  */
 
+/**
+ * @param {string | string[] | Translation | Translation[]} translations
+ * @returns {translations is Translation[]}
+ */
 function isArrayOfObjects (translations) {
     return Array.isArray(translations)
         && typeof translations[0] === 'object'

package/src/testTools/PromptAssert.js

@@ -0,0 +1,184 @@
+/**
+ * @author David Menger
+ */
+'use strict';
+
+const assert = require('assert');
+const asserts = require('./asserts');
+
+const { ex } = asserts;
+
+/** @typedef {import('../LLM').PromptInfo} PromptInfo */
+/** @typedef {import('../LLM').LLMMessage} LLMMessage */
+/** @typedef {import('../LLM').LLMRole} LLMRole */
+
+/**
+ * @class PromptAssert
+ */
+class PromptAssert {
+
+    /**
+     *
+     * @param {PromptInfo[]} prompts
+     */
+    constructor (prompts) {
+        this._prompts = prompts;
+    }
+
+    /**
+     * Check if recent instruction contains selected string
+     *
+     * @param {string} search
+     * @returns {this}
+     */
+    instructionContains (search) {
+        const messages = this._flatPrompt((m) => m.role === 'system');
+        this._promptContains(search, messages, false, 'of role "system"');
+        return this;
+    }
+
+    /**
+     * Check if recent prompt input contains selected string
+     *
+     * @param {string} search
+     * @returns {this}
+     */
+    promptContains (search) {
+        this._promptContains(search, this._flatPrompt());
+        return this;
+    }
+
+    /**
+     * Check if recent results contains selected string
+     *
+     * @param {string} search
+     * @returns {this}
+     */
+    resultContains (search) {
+        this._promptContains(search, this._flatResults());
+        return this;
+    }
+
+    /**
+     *
+     * @param {(value: LLMMessage, index: number, array: LLMMessage[]) => unknown} filter
+     * @returns {LLMMessage[]}
+     */
+    _flatPrompt (filter = () => true) {
+        return this._prompts
+            .flatMap((prompt) => prompt.prompt
+                .filter(filter));
+    }
+
+    /**
+     *
+     * @returns {LLMMessage[]}
+     */
+    _flatResults () {
+        return this._prompts
+            .map((prompt) => prompt.result);
+    }
+
+    _promptContains (search, messages, notContains = false, addMessage = 'No LLM message found') {
+        if (messages.length === 0) {
+            PromptAssert.debug(this._prompts, true);
+            assert.fail(ex(`No LLM message ${addMessage}`, search));
+        }
+
+        const found = messages.some((m) => asserts.llmContains(m, search, false));
+        // console.log({ found, messages, notContains });
+        if (notContains === found) {
+            PromptAssert.debug(this._prompts, true);
+            assert.fail(ex(
+                `Text${notContains ? '' : ' not'} found in LLM messages ${addMessage}`,
+                search
+            ));
+        }
+    }
+
+    /**
+     *
+     * @param {LLMMessage[]} prompt
+     * @returns {string}
+     */
+    static promptStats (prompt) {
+        const stats = prompt.reduce((o, m) => Object.assign(o, {
+            [m.role]: (o[m.role] || 0) + 1
+        }), {
+            system: 0, assistant: 0, user: 0, tool: 0
+        });
+
+        return `[ ${Object.entries(stats).map(([k, v]) => `${k.toUpperCase()}: ${v}`).join(' ')} ]`;
+    }
+
+    /**
+     *
+     * @param {LLMRole} role
+     * @return {string}
+     */
+    static _marker (role) {
+        switch (role) {
+            case 'user':
+                return '>';
+            case 'system':
+                return '*';
+            case 'assistant':
+                return '<';
+            default:
+                return '-';
+        }
+    }
+
+    static _content (msg, full) {
+        if (typeof msg.content !== 'string') {
+            return msg.content === null ? '<null>' : typeof msg.content;
+        }
+        const txt = msg.content.replace(/\n+/g, ' ').replace(/\s\s+/g, ' ');
+        return `'${txt.length <= 100 || full ? txt : `${txt.substring(0, 100)}...`}'`;
+    }
+
+    /**
+     *
+     * @param {PromptInfo[]} prompts
+     * @param {boolean} [full=false]
+     * @param {boolean} [silent=false]
+     * @returns {string[]}
+     */
+    static debug (prompts, full = false, silent = false) {
+        let out;
+        if (prompts.length === 0) {
+            out = ['no LLM prompts occured'];
+        } else {
+            out = prompts.map((p, i) => {
+                const lastIndexOfUser = full
+                    ? 0
+                    : p.prompt.reduce((li, m, c) => (m.role === 'user' ? c : li), 0);
+
+                const prompt = p.prompt.reduce((a, m, c) => {
+                    if (['user', 'assistant'].includes(m.role) && c < lastIndexOfUser) {
+                        const prev = a[a.length - 1];
+                        if (prev === '...') {
+                            return a;
+                        }
+                        return [...a, '...'];
+                    }
+                    return [...a, `${PromptAssert._marker(m.role)} ${PromptAssert._content(m, full)}`];
+                }, []);
+
+                return [
+                    `${i + 1}) ${PromptAssert.promptStats(p.prompt)}`,
+                    `${prompt.join('\n   ')}`,
+                    `# ${PromptAssert._content(p.result)}`
+                ].join('\n   ');
+            });
+        }
+        if (!silent) {
+            // eslint-disable-next-line no-console
+            console.log(...out);
+        }
+        return out;
+    }
+
+}
+
+module.exports = PromptAssert;

package/src/testTools/asserts.js

@@ -7,6 +7,9 @@ const assert = require('assert');
 const deepExtend = require('deep-extend');
 const { actionMatches, parseActionPayload } = require('../utils');
 
+/** @typedef {import('../LLM').PromptInfo} PromptInfo */
+/** @typedef {import('../LLM').LLMMessage} LLMMessage */
+
 /**
  * Format message
  *
@@ -29,9 +32,21 @@ function m (text, actual = null, expected = null) {
     return `${text}${result}`;
 }
 
-function ex (message, expected, actual) {
-    const actuals = Array.isArray(actual) ? actual : [actual];
-    return `${message}\n  + expected: "${expected}"\n  - actual:   ${actuals
+function ex (message, expected, actual = null) {
+    let actuals;
+    if (Array.isArray(actual)) {
+        actuals = actual;
+    } else {
+        actuals = actual === null ? [] : [actual];
+    }
+
+    const msg = `${message}\n  + expected: "${expected}"`;
+
+    if (actuals.length === 0) {
+        return msg;
+    }
+
+    return `${msg}\n  - actual:   ${actuals
         .map((a) => `"${a}"`).join('\n              ')}`;
 }
 
@@ -144,6 +159,28 @@ function contains (response, search, message = 'Should contain a text') {
     return true;
 }
 
+/**
+ *
+ * @param {LLMMessage} llmMsg
+ * @param {string} search
+ * @param {string|false} message
+ */
+function llmContains (llmMsg, search, message = 'Should contain a text') {
+    const text = llmMsg.content;
+
+    const typeIsText = typeof text === 'string';
+    if (message === false && !typeIsText) {
+        return false;
+    }
+    assert.ok(typeIsText, m(message, search, 'not a text message'));
+    const match = searchMatchesText(search, text);
+    if (message === false) {
+        return match;
+    }
+    assert.ok(match, m(message, text, search));
+    return true;
+}
+
 /**
  * Checks quick response action
  *
@@ -374,5 +411,6 @@ module.exports = {
     quickReplyText,
     getText,
     parseActionPayload,
-    ex
+    ex,
+    llmContains
 };