wingbot 3.77.0-alpha.2 → 3.77.0-alpha.3

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "wingbot",
-  "version": "3.77.0-alpha.2",
+  "version": "3.77.0-alpha.3",
   "description": "Enterprise Messaging Bot Conversation Engine",
   "main": "index.js",
   "type": "commonjs",

package/src/LLMSession.js

@@ -170,6 +170,14 @@ const stateData = require('./utils/stateData');
  * @prop {Responder} [res]
  * @prop {object} [data]
  * @prop {LLMCallPreset} [preset]
+ * @prop {SystemSource} [system] - the system prompt of the session - compiled again,
+ *   when recompileSystemPrompt() is called (e.g. a tool changed the state)
+ */
+
+/**
+ * @callback SystemSource
+ * @param {boolean} recompile - false at the start, true to compile the instructions again
+ * @returns {Promise<LLMMessage[]>}
  */
 
 // max number of consecutive tool-call rounds resolved within a single generate()
@@ -219,11 +227,20 @@ class LLMSession {
             data = {},
             req = null,
             res = null,
-            preset = undefined
+            preset = undefined,
+            system = null
         } = options;
 
         this._llm = llm;
 
+        /** @type {SystemSource|null} */
+        this._systemSource = system;
+
+        /** @type {Set<object>} - the system messages compiled from the system source */
+        this._compiledSystem = new Set();
+
+        this._systemRecompileRequested = false;
+
         this._onSend = onSend;
 
         this._inExecution = false;
@@ -234,6 +251,9 @@ class LLMSession {
 
         /** @type {LLMMessageSrc[]} */
         this._chat = [];
+        if (system) {
+            this.push(this._compileSystem(false));
+        }
         this.push(...chat);
 
         this._generatedIndex = null;
@@ -822,6 +842,55 @@ class LLMSession {
         return this;
     }
 
+    /**
+     * @param {boolean} recompile
+     * @returns {Promise<LLMMessage[]>}
+     */
+    async _compileSystem (recompile) {
+        const messages = await this._systemSource(recompile);
+        messages.forEach((m) => this._compiledSystem.add(m));
+        return messages;
+    }
+
+    /**
+     * The system prompt (from the system source of the session) is compiled again
+     * before the next request to the LLM - e.g. a tool changed the language or the buttons
+     * under the answer. The other system messages stay.
+     *
+     * @returns {this}
+     */
+    recompileSystemPrompt () {
+        this._systemRecompileRequested = !!this._systemSource;
+        return this;
+    }
+
+    async _recompileSystemIfRequested () {
+        if (!this._systemRecompileRequested) {
+            return;
+        }
+        this._systemRecompileRequested = false;
+
+        const previous = this._compiledSystem;
+        this._compiledSystem = new Set();
+        const messages = await this._compileSystem(true);
+
+        // the new messages at the place of the previous ones
+        const at = this._chat.findIndex((m) => previous.has(m));
+        const chat = this._chat.filter((m) => !previous.has(m));
+        chat.splice(at === -1 ? 0 : at, 0, ...messages);
+        this._chat = chat;
+    }
+
+    /**
+     * @param {LLMCallPreset} providerOptions
+     * @param {LLMLogOptions} logOptions
+     * @returns {Promise<LLMMessage<any>>}
+     */
+    async _requestLlm (providerOptions, logOptions) {
+        await this._recompileSystemIfRequested();
+        return this._llm.generate(this, providerOptions, logOptions);
+    }
+
     /**
      *
      * @param {string|Promise<string>|Renderer} content
@@ -1032,7 +1101,7 @@ class LLMSession {
      */
     async _generate (providerOptions = this._preset, logOptions = {}) {
         const { maxToolCallRounds = MAX_TOOL_CALL_ROUNDS } = this._llm.llmOptions(providerOptions);
-        let result = await this._llm.generate(this, providerOptions, logOptions);
+        let result = await this._requestLlm(providerOptions, logOptions);
 
         // the model may chain several rounds of tool calls before it produces
         // a final text answer - keep resolving them until it stops (bounded)
@@ -1068,7 +1137,7 @@ class LLMSession {
             }
 
             if (toolCalls.length) {
-                result = await this._llm.generate(this, providerOptions, logOptions);
+                result = await this._requestLlm(providerOptions, logOptions);
             } else {
                 // everything failed - force a final text answer without tools
                 this._llm.log.error(
@@ -1115,7 +1184,7 @@ class LLMSession {
                 ...providerOptions,
                 toolChoice: 'none'
             };
-        return this._llm.generate(this, overrideChoice, logOptions);
+        return this._requestLlm(overrideChoice, logOptions);
     }
 
     /**

package/src/LLMToolRunner.js

@@ -463,6 +463,14 @@ class LLMToolRunner {
         // only the buttons of the last situation are offered (the earlier ones are outdated)
         this.quickReplies = [];
         this._addButtons(sitDef, situation, values);
+
+        // the answer of the LLM follows - its system prompt compiled again: the tool could change
+        // the state (e.g. the language) and these buttons go under the answer ({{$qrTexts}})
+        res.setData({ _qRepliesToSend: [...this.quickReplies] });
+        if (session && typeof session.recompileSystemPrompt === 'function') {
+            session.recompileSystemPrompt();
+        }
+
         return this._instructionText(sitDef, situation, values);
     }
 

package/src/Responder.js

@@ -262,6 +262,17 @@ class Responder {
             [this.LLM_ERROR, []]
         ]);
 
+        /**
+         * the instructions as added - _llmContext keeps them compiled (once in the request),
+         * these compile them again (e.g. after a tool changed the state)
+         *
+         * @type {Map<string,PromptContextItem[]>}
+         */
+        this._llmContextSource = new Map([
+            [this.LLM_CTX_DEFAULT, []],
+            [this.LLM_ERROR, []]
+        ]);
+
         /** @type {Map<string,PreprocessedRule[]>} */
         this._llmResultRules = new Map([
             [this.LLM_CTX_DEFAULT, []]
@@ -298,8 +309,10 @@ class Responder {
         if (!this._llmContext.has(contextType)) {
             // @todo make it array of messages / maybe keep it in a single array
             this._llmContext.set(contextType, []);
+            this._llmContextSource.set(contextType, []);
         }
         this._llmContext.get(contextType).push(systemPrompt);
+        this._llmContextSource.get(contextType).push(systemPrompt);
 
         return this;
     }
@@ -390,10 +403,11 @@ class Responder {
     /**
      *
      * @param {string} contextType
+     * @param {boolean} [recompile] - compile the instructions again (the state could change)
      * @returns {AsyncLLMMessage}
      */
-    async _getSystemMessagesForType (contextType) {
-        const system = await this._getSystemContentForType(contextType);
+    async _getSystemMessagesForType (contextType, recompile = false) {
+        const system = await this._getSystemContentForType(contextType, [], recompile);
         return system.map((content) => ({ role: LLM.ROLE_SYSTEM, content }));
     }
 
@@ -405,14 +419,14 @@ class Responder {
      * @returns {LLMSession}
      */
     llmSession (contextType = this.LLM_CTX_DEFAULT, callPreset = undefined, errorPrompt = undefined) { // eslint-disable-line max-len
-        const system = this._getSystemMessagesForType(contextType);
-
         const filters = this._filtersForContext(contextType);
-        const session = new LLMSession(this.llm, [system], {
+        const session = new LLMSession(this.llm, [], {
             onSend: this._llmSend.bind(this),
             filters,
             preset: callPreset,
-            res: this
+            res: this,
+            // compiled again on demand (session.recompileSystemPrompt())
+            system: (recompile) => this._getSystemMessagesForType(contextType, recompile)
         });
         session.onDelegatedError(
             errorPrompt ?? (() => this._getSystemContentForType(this.LLM_ERROR)
@@ -483,9 +497,10 @@ class Responder {
      *
      * @param {string} contextType
      * @param {string[]} [callStack]
+     * @param {boolean} [recompile] - compile the instructions again (the state could change)
      * @returns {Promise<string[]>}
      */
-    async _getSystemContentForType (contextType, callStack = []) {
+    async _getSystemContentForType (contextType, callStack = [], recompile = false) {
         if (new Set(callStack).size < callStack.length) {
             throw new Error(`Circular reference detected: contextType -> ${callStack}`);
         }
@@ -494,9 +509,14 @@ class Responder {
             return [];
         }
 
+        // compiled once in the request - again only on demand (from the instructions as added)
+        const parts = recompile && this._llmContextSource.has(contextType)
+            ? this._llmContextSource.get(contextType)
+            : this._llmContext.get(contextType);
+
         /** @typedef {Promise<string>} PromisedString */
         /** @type {PromisedString[]} */
-        const promiseStrings = this._llmContext.get(contextType)
+        const promiseStrings = parts
             .map(async (p) => (typeof p === 'function' ? p(this) : p));
         this._llmContext.set(contextType, promiseStrings);
 
@@ -506,8 +526,11 @@ class Responder {
                     const s = await promiseString;
 
                     const replaced = await this._replaceAsync(s.trim(), /\$\{([a-zA-Z0-9\s]+)\}/g, async (str, reqType) => {
-                        const nested = await this
-                            ._getSystemContentForType(reqType, [...callStack, contextType]);
+                        const nested = await this._getSystemContentForType(
+                            reqType,
+                            [...callStack, contextType],
+                            recompile
+                        );
 
                         return nested.join('\n\n');
                     });
@@ -539,13 +562,14 @@ class Responder {
 
         const filters = this._filtersForContext(contextType);
         const session = new LLMSession(this.llm, [
-            this._getSystemMessagesForType(contextType),
             this._getTranscriptMessages(transcriptLength, transcriptFlag, transcriptAnonymize)
         ], {
             onSend: this._llmSend.bind(this),
             filters,
             preset: callPreset,
-            res: this
+            res: this,
+            // compiled again on demand (session.recompileSystemPrompt())
+            system: (recompile) => this._getSystemMessagesForType(contextType, recompile)
         });
         session.onDelegatedError(
             errorPrompt ?? (() => this._getSystemContentForType(this.LLM_ERROR)