wingbot 3.74.7 → 3.75.9-alpha.1

package/src/LLMRouter.js

@@ -0,0 +1,325 @@
+/**
+ * @author David Menger
+ */
+'use strict';
+
+const GlobalIntents = require('./GlobalIntents');
+const LLMType = require('./LLMType');
+
+/** @typedef {import('./GlobalIntents').GlobalIntentsMap} GlobalIntentsMap */
+/** @typedef {import('./GlobalIntents').ResolvedGlobalIntentsMap} ResolvedGlobalIntentsMap */
+/** @typedef {import('./GlobalIntents').GlobalIntentResolved} GlobalIntentResolved */
+/** @typedef {import('./GlobalIntents').MatchingResolver} MatchingResolver */
+/** @typedef {import('./GlobalIntents').MatcherResult} MatcherResult */
+/** @typedef {import('./LLMDispatcher').ILLMRouter} ILLMRouter */
+/** @typedef {import('./Request')} Request */
+/** @typedef {import('./LLM')} LLM */
+/** @typedef {import('./prompt').Prompt} Prompt */
+/** @typedef {import('./Router').RoutingInstruction} RoutingInstruction */
+
+/**
+ * @typedef {object} NLPActionInfo
+ * @prop {MatcherResult} intent
+ * @prop {boolean} aboveConfidence
+ * @prop {boolean} winner
+ * @prop {string} [title]
+ * @prop {string|string[]} [match]
+ */
+
+/**
+ * @typedef {object} BaseRouteAction
+ * @prop {string} action
+ * @prop {object} [data]
+ * @prop {object} [setState]
+ * @prop {object} [_aiKeys]
+ */
+
+/**
+ * @typedef {object} LLMRouting
+ * @prop {LLMRoutingCache} globals
+ * @prop {ILLMRouter|null} router
+ * @prop {Map<string, LLMRoutingCache>} locals
+ */
+
+/**
+ * @typedef {Pick<GlobalIntentResolved, 'action'|'classification'>} LLMAction
+ */
+
+/**
+ * @typedef {BaseRouteAction & Partial<NLPActionInfo>} RoutingAction
+ */
+
+/**
+ * @callback RouterResolver
+ * @param {LLM} llm
+ * @param {Request} req
+ * @param {LLMRoutingInput} routing
+ *
+ * @returns {Promise<RoutingAction|null>}
+ */
+
+/**
+ * @typedef {Pick<LLMRoutingCache, 'actionList'|'actionListString'|'byAction'>} LLMRoutingInput
+ */
+
+/**
+ * @typedef {object} LLMRoutingCache
+ * @prop {boolean} [keepUserInInteractionsWithBounceAllowed]
+ * @prop {string} actionListString
+ * @prop {LLMAction[]} actionList
+ * @prop {ILLMRouter|null} router
+ * @prop {Map<string, GlobalIntentResolved>} byAction
+ */
+
+/**
+ * @typedef {object} LLMRouterOptions
+ * @prop {boolean} [keepUserInInteractionsWithBounceAllowed]
+ * @prop {boolean} [global]
+ */
+
+/**
+ * @typedef {Omit<LLMRouterOptions, 'global'>} GILLMExtension
+ */
+
+/**
+ * @class LLMRouter
+ */
+class LLMRouter {
+
+    // lets give it "reduce" method? - can forward that info to dispatcher
+
+    /**
+     *
+     * @param {RouterResolver} routeResolver
+     * @param {Omit<LLMRouterOptions, 'global'>} [options]
+     */
+    static global (routeResolver, options = {}) {
+        return new LLMRouter(routeResolver, {
+            ...options,
+            global: true
+        });
+    }
+
+    /**
+     *
+     * @param {RouterResolver} [routeResolver]
+     * @param {LLMRouterOptions} [options]
+     */
+    constructor (routeResolver = null, options = {}) {
+        const { global = false, ...rest } = options;
+        this._route = routeResolver;
+        this._global = global;
+
+        /** @typedef {GlobalIntentsMap} */
+        this.globalIntents = new Map();
+
+        if (global) {
+            const gi = GlobalIntents.create({
+                router: this._route ? this : null,
+                ...rest
+            });
+            this.globalIntents.set(gi.id, gi);
+        } else if (Object.keys(rest).length) {
+            const gi = GlobalIntents.create({
+                local: true,
+                ...rest
+            });
+            this.globalIntents.set(gi.id, gi);
+        }
+    }
+
+    /**
+     * @type {object}
+     */
+    static _cachedStructuredOutput = null;
+
+    /**
+     *
+     * @returns {object}
+     */
+    static structuredOutput () {
+        if (LLMRouter._cachedStructuredOutput === null) {
+            LLMRouter._cachedStructuredOutput = LLMType
+                .object({
+                    action: LLMType.string()
+                        .description('recommended action')
+                }, 'conversation routing information')
+                .toJSON();
+        }
+        return LLMRouter._cachedStructuredOutput;
+    }
+
+    /**
+     *
+     * prompt wi
+     *
+     * @param {Prompt} prompt
+     * @returns {RouterResolver}
+     */
+    static defaultRoute (prompt) {
+        /** @type {RouterResolver} */
+        const route = async (llm, req, routing) => {
+            const res = await llm.session('routing')
+                .setData(routing)
+                .systemPrompt(prompt)
+                .generateStructured(LLMRouter.structuredOutput());
+
+            if (!routing.byAction.has(LLMRouter._normByAction(res.action))) {
+                return null;
+            }
+
+            return res;
+        };
+
+        return route;
+    }
+
+    /**
+     *
+     * @returns {RoutingInstruction}
+     */
+    reduce () {
+        return false;
+    }
+
+    on () {
+        // acually does nothing too
+    }
+
+    /**
+     *
+     * @param {LLM} llm
+     * @param {Request} req
+     * @param {LLMRoutingInput} routing
+     * @returns {Promise<RoutingAction|null>}
+     */
+    async resolve (llm, req, routing) { // eslint-disable-line no-unused-vars
+        if (!this._route) {
+            return null;
+        }
+        return this._route(llm, req, routing);
+    }
+
+    /**
+     *
+     * @param {string} path
+     * @param {string} classification
+     * @returns {MatchingResolver}
+     */
+    route (path, classification) {
+        return GlobalIntents.resolver(path, {
+            classification,
+            router: this._route ? this : null,
+            local: !this._global
+        });
+    }
+
+    /**
+     *
+     * @param {string} path
+     * @param {string} classification
+     * @returns {MatchingResolver}
+     */
+    static route (path, classification) {
+        return GlobalIntents.resolver(path, {
+            classification
+        });
+    }
+
+    static _normByAction (act) {
+        return act.replace(/^\/|\/$/g, '');
+    }
+
+    /**
+     *
+     * @param {GlobalIntentResolved[]} resolvedLlm
+     * @returns {LLMRoutingCache}
+     */
+    static _prepareRoutingInput (resolvedLlm) {
+        const byAction = new Map();
+        let router = null;
+        let keepUserInInteractionsWithBounceAllowed;
+
+        const actionList = resolvedLlm.map((gi) => {
+            const { action, classification } = gi;
+            byAction.set(LLMRouter._normByAction(gi.action), gi);
+
+            if (typeof gi.keepUserInInteractionsWithBounceAllowed === 'boolean') {
+                keepUserInInteractionsWithBounceAllowed = true;
+            }
+
+            if (router && router !== gi.router) {
+                throw new Error(`Multiple local routers on "${gi.localPath}"`);
+            } else {
+                router = gi.router;
+            }
+
+            return { action, classification };
+        });
+
+        const actionListString = actionList
+            .map((gi) => `- \`${gi.action}\`: ${gi.classification}`)
+            .join('\n');
+
+        return {
+            ...(typeof keepUserInInteractionsWithBounceAllowed === 'boolean'
+                ? { keepUserInInteractionsWithBounceAllowed }
+                : {}),
+            actionListString,
+            actionList,
+            byAction,
+            router
+        };
+    }
+
+    /**
+     *
+     * @param {ResolvedGlobalIntentsMap} globalIntents
+     * @returns {LLMRouting}
+     */
+    static prepareRouting (globalIntents) {
+        /** @type {Map<string, GlobalIntentResolved[]>} */
+        const locals = new Map();
+        /** @type {GlobalIntentResolved[]} */
+        const globals = [];
+        /** @type {ILLMRouter} */
+        let router = null;
+
+        for (const gi of globalIntents.values()) {
+            if (!gi.classification
+                && !gi.router
+                && typeof gi.keepUserInInteractionsWithBounceAllowed === 'undefined') continue;
+
+            if (gi.local) {
+                let local = locals.get(gi.localPath);
+                if (!local) {
+                    local = [];
+                    locals.set(gi.localPath, local);
+                }
+                local.push(gi);
+            } else if (gi.classification) {
+                globals.push(gi);
+            } else if (gi.router) { // public default router without classification
+                if (router !== null) {
+                    throw new Error(`Detected more than one global router on '${gi.localPath}'. Only single global router allowed.`);
+                }
+                router = gi.router;
+            }
+        }
+
+        return {
+            router,
+            locals: new Map(
+                Array.from(locals)
+                    .map(([localPath, locs]) => [
+                        localPath,
+                        LLMRouter._prepareRoutingInput(locs)
+                    ])
+            ),
+            globals: LLMRouter._prepareRoutingInput(globals)
+        };
+    }
+
+}
+
+module.exports = LLMRouter;

package/src/LLMSession.js

@@ -6,12 +6,16 @@
 const {
     FILTER_SCOPE_CONVERSATION, ROLE_ASSISTANT, ROLE_USER, ROLE_SYSTEM
 } = require('./LLMConsts');
+const stateData = require('./utils/stateData');
 
 /** @typedef {import('./Responder').QuickReply} QuickReply */
 /** @typedef {import('./LLM').LLMCallPreset} LLMCallPreset */
 /** @typedef {import('./LLM')} LLM */
 /** @typedef {import('./LLM').LLMLogOptions} LLMLogOptions */
 
+/** @typedef {import('./Request')} Request */
+/** @typedef {import('./Responder')} Responder */
+
 /** @typedef {'user'|'assistant'} LLMChatRole */
 /** @typedef {'system'} LLMSystemRole */
 /** @typedef {'tool'} LLMToolRole */
@@ -29,7 +33,9 @@ const {
  * @prop {string} args - JSON string
  */
 
-/** @typedef {string|Promise<string>} PossiblyAsyncContent */
+/** @typedef {import('./prompt').Renderer} Renderer */
+
+/** @typedef {string|Promise<string>|Renderer} PossiblyAsyncContent */
 
 /**
  * @template {LLMRole} [R=LLMRole]
@@ -155,6 +161,16 @@ const {
  *      - toJson
  */
 
+/**
+ * @typedef {object} SessionOptions
+ * @prop {LLMFilter[]} [filters]
+ * @prop {SendCallback} [onSend]
+ * @prop {Request} [req]
+ * @prop {Responder} [res]
+ * @prop {object} [data]
+ * @prop {LLMCallPreset} [preset]
+ */
+
 /**
  * @class LLMSession
  * @implements {PromiseLike<LLMMessage<any>>}
@@ -165,10 +181,18 @@ class LLMSession {
      *
      * @param {LLM} llm
      * @param {(PossiblyAsyncLLMMessage|AsyncLLMMessage)[]} [chat]
-     * @param {SendCallback} [onSend]
-     * @param {LLMFilter[]} [filters=[]]
+     * @param {SessionOptions} [options]
      */
-    constructor (llm, chat = [], onSend = null, filters = []) {
+    constructor (llm, chat = [], options = {}) {
+        const {
+            onSend = null,
+            filters = [],
+            data = {},
+            req = null,
+            res = null,
+            preset = undefined
+        } = options;
+
         this._llm = llm;
 
         this._onSend = onSend;
@@ -194,6 +218,36 @@ class LLMSession {
 
         /** @type {Map<string,ObjectTool>} */
         this._tools = new Map();
+
+        this._data = { ...data };
+
+        this._req = req || llm?.req;
+        this._res = res || llm?.res;
+
+        this._preset = preset;
+    }
+
+    /**
+     *
+     * @param {object} data
+     * @returns {this}
+     */
+    setData (data) {
+        this._job(() => {
+            Object.assign(this._data, data);
+        }, true);
+        return this;
+    }
+
+    _resolveData () {
+        const dataFromReqRes = this._req || this._res
+            ? stateData(this._req, this._res)
+            : {};
+
+        return {
+            ...dataFromReqRes,
+            ...this._data
+        };
     }
 
     _job (task, runNowAndSyncWhenQueueIsEmpty = false) {
@@ -447,11 +501,15 @@ class LLMSession {
                 return;
             }
             if ('content' in m
+                && typeof m.content !== 'function'
                 && (m.content instanceof Promise
                     || (typeof m.content !== 'string' && m.content && 'then' in m.content))) {
                 return;
             }
-            if (filtered && this._filters.length >= 0 && 'content' in m && typeof m.content === 'string') {
+            if (filtered
+                && this._filters.length >= 0
+                && 'content' in m
+                && (typeof m.content === 'string' || typeof m.content === 'function')) {
                 const content = this._filters.reduce((text, filter) => {
                     if (!text) {
                         return text;
@@ -463,7 +521,7 @@ class LLMSession {
                     }
                     const res = filter.filter(text, m.role);
                     return res === true ? text : res;
-                }, m.content);
+                }, typeof m.content === 'function' ? m.content.toString() : m.content);
 
                 if (typeof content === 'string') {
                     // @ts-ignore
@@ -472,6 +530,11 @@ class LLMSession {
                         content
                     });
                 }
+            } else if ('content' in m && typeof m.content === 'function') {
+                ret.push({
+                    ...m,
+                    content: m.content.toString()
+                });
             } else {
                 // @ts-ignore
                 ret.push(m);
@@ -520,6 +583,9 @@ class LLMSession {
         if (typeof m.content === 'string') {
             return m.content;
         }
+        if (typeof m.content === 'function') {
+            return m.content.toString();
+        }
         return '<Promise>';
     }
 
@@ -627,12 +693,17 @@ class LLMSession {
                 })();
                 return wrapped;
             }
-            if (!('content' in msg) || !this._contentIsPromise(msg.content)) {
+            if (!('content' in msg) || (!this._contentIsPromise(msg.content) && typeof msg.content !== 'function')) {
                 return msg;
             }
+
+            const contentPromise = typeof msg.content === 'function'
+                ? msg.content(this._resolveData())
+                : msg.content;
+
             const ret = {
                 ...msg,
-                content: Promise.resolve(msg.content)
+                content: Promise.resolve(contentPromise)
                     .then((r) => {
                         // @ts-ignore
                         ret.content = r;
@@ -688,7 +759,7 @@ class LLMSession {
 
     /**
      *
-     * @param {string|Promise<string>} content
+     * @param {string|Promise<string>|Renderer} content
      * @returns {this}
      */
     user (content) {
@@ -701,7 +772,7 @@ class LLMSession {
 
     /**
      *
-     * @param {string|Promise<string>} content
+     * @param {string|Promise<string>|Renderer} content
      * @returns {this}
      */
     assistant (content) {
@@ -714,7 +785,7 @@ class LLMSession {
 
     /**
      *
-     * @param {string|Promise<string>} content
+     * @param {string|Promise<string>|Renderer} content
      * @returns {this}
      */
     systemPrompt (content) {
@@ -747,7 +818,7 @@ class LLMSession {
      * @param {LLMLogOptions} [logOptions]
      * @returns {this}
      */
-    generate (providerOptions = undefined, logOptions = {}) {
+    generate (providerOptions = this._preset, logOptions = {}) {
         this._job(() => this._generate(providerOptions, logOptions));
         return this;
     }
@@ -759,7 +830,7 @@ class LLMSession {
      * @param {LLMLogOptions} [logOptions]
      * @returns {this}
      */
-    generateStructured (output, providerOptions = undefined, logOptions = {}) {
+    generateStructured (output, providerOptions = this._preset, logOptions = {}) {
 
         const responseFormat = 'toJSON' in output && typeof output.toJSON === 'function'
             ? output.toJSON()
@@ -787,7 +858,7 @@ class LLMSession {
      * @param {LLMLogOptions} [logOptions]
      * @returns {Promise<LLMMessage<any>>}
      */
-    async _generate (providerOptions = undefined, logOptions = {}) {
+    async _generate (providerOptions = this._preset, logOptions = {}) {
         let result = await this._llm.generate(this, providerOptions, logOptions);
 
         if (result.toolCalls?.length) {

package/src/Processor.js

@@ -13,6 +13,7 @@ const ReturnSender = require('./ReturnSender');
 const { prepareState, mergeState, isUserInteraction } = require('./utils/stateVariables');
 const LLM = require('./LLM');
 const LLMMockProvider = require('./LLMMockProvider');
+const LLMDispatcher = require('./LLMDispatcher');
 
 /** @typedef {import('./wingbot/CustomEntityDetectionModel').Intent} Intent */
 /** @typedef {import('./ReducerWrapper')} ReducerWrapper */
@@ -140,9 +141,9 @@ const LLMMockProvider = require('./LLMMockProvider');
  */
 
 /**
- * @template {ReducerWrapper|Router|BuildRouter} T
+ * @template {ReducerWrapper|Router|BuildRouter} R
  * @callback Preloader
- * @param {T} router
+ * @param {R} router
  * @param {Ai} ai
  * @returns {Promise<void>}
  */
@@ -268,12 +269,11 @@ class Processor extends EventEmitter {
             }
 
             if (dispatchSync) {
-                let previousAction;
                 return Promise.resolve(data)
                     .then((resolvedData) => {
                         let reduceResult;
                         Object.assign(resolvedData, { _localpostback: true });
-                        previousAction = req.setAction(action, resolvedData);
+                        req.dispatcher.overrideAction({ action, data: resolvedData });
                         if (typeof this.reducer === 'function') {
                             // @ts-ignore
                             reduceResult = this.reducer(req, res, postBack);
@@ -283,7 +283,7 @@ class Processor extends EventEmitter {
                         return reduceResult;
                     })
                     .then((reduceResult) => {
-                        req.setAction(previousAction);
+                        req.dispatcher.overrideAction(null);
                         return reduceResult;
                     });
             }
@@ -407,7 +407,7 @@ class Processor extends EventEmitter {
 
         /** @type {LLMGlobalConfig} */
         const llmConfiguration = {
-            provider: new LLMMockProvider(),
+            provider: new LLMMockProvider(message.llmMocks),
             ...this.options.llm
         };
 
@@ -533,10 +533,11 @@ class Processor extends EventEmitter {
                 ? Request.text('none', text)
                 : text;
             // @ts-ignore
-            const req = new Request(request, { lang }, pageId, this.reducer.globalIntents);
-
-            await Ai.ai.preloadAi(req);
-
+            const dispatcher = new LLMDispatcher(Ai.ai, this.reducer.routing, {
+                log: this.options.log
+            });
+            const req = new Request(request, { lang }, pageId, dispatcher);
+            await dispatcher.dispatch(req);
             const actions = req.aiActions();
 
             if (actions.length === 0 && allowEmptyAction && req.intents.length > 0) {
@@ -555,6 +556,7 @@ class Processor extends EventEmitter {
                 ];
             }
 
+            // @ts-ignore
             return actions
                 .map((a) => ({
                     ...a,
@@ -638,13 +640,14 @@ class Processor extends EventEmitter {
                 }
             }
 
-            // @ts-ignore
+            const dispatcher = new LLMDispatcher(Ai.ai, this.reducer.routing, llm, {
+                log: this.options.log
+            });
             req = new Request(
                 message,
                 state,
                 pageId,
-                // @ts-ignore
-                this.reducer.globalIntents,
+                dispatcher,
                 {
                     apiUrl: this.options.apiUrl,
                     secret: this.options.secret,
@@ -719,6 +722,7 @@ class Processor extends EventEmitter {
                 senderMeta,
                 llm
             );
+            llm.setReqRes(req, res);
             const postBack = this._createPostBack(postbackAcumulator, req, res, features);
 
             let continueDispatching = true;
@@ -738,7 +742,7 @@ class Processor extends EventEmitter {
 
             await Promise.all([
                 preloadPromise,
-                Ai.ai.preloadAi(req, res)
+                dispatcher.dispatch(req, res)
             ]);
 
             // @deprecated backward compatibility

package/src/ReducerWrapper.js

@@ -4,6 +4,11 @@
 'use strict';
 
 const EventEmitter = require('events');
+const LLMDispatcher = require('./LLMDispatcher');
+
+/** @typedef {import('./Responder')} Responder */
+/** @typedef {import('./Request')} Request */
+/** @typedef {import('./LLMDispatcher').Routing} Routing */
 
 /**
  * Solution for catching events. This is useful for analytics.
@@ -39,6 +44,25 @@ class ReducerWrapper extends EventEmitter {
         this.processMessage = null;
 
         this.setMaxListeners(Infinity);
+
+        this.globalIntents = new Map();
+
+        /** @type {Routing} */
+        this._routing = null;
+    }
+
+    resetRouter () {
+        this._routing = null;
+    }
+
+    /**
+     * @returns {Routing}
+     */
+    get routing () {
+        if (this._routing === null) {
+            this._routing = LLMDispatcher.prepareRouting(this.globalIntents);
+        }
+        return this._routing;
     }
 
     /**