wingbot 3.74.0 → 3.74.2

package/src/LLMSession.js

@@ -3,10 +3,13 @@
  */
 'use strict';
 
-const LLM = require('./LLM');
+const {
+    FILTER_SCOPE_CONVERSATION, ROLE_ASSISTANT, ROLE_USER, ROLE_SYSTEM
+} = require('./LLMConsts');
 
 /** @typedef {import('./Responder').QuickReply} QuickReply */
-/** @typedef {import('./LLM').LLMProviderOptions} LLMProviderOptions */
+/** @typedef {import('./LLM').LLMCallPreset} LLMCallPreset */
+/** @typedef {import('./LLM')} LLM */
 /** @typedef {import('./LLM').LLMLogOptions} LLMLogOptions */
 
 /** @typedef {'user'|'assistant'} LLMChatRole */
@@ -21,20 +24,42 @@ const LLM = require('./LLM');
 /**
  * @typedef {object} ToolCall
  * @prop {string} id
+ * @prop {string|'function'} type
  * @prop {string} name
  * @prop {string} args - JSON string
  */
 
+/** @typedef {string|Promise<string>} PossiblyAsyncContent */
+
 /**
  * @template {LLMRole} [R=LLMRole]
+ * @template {PossiblyAsyncContent} [C=string]
  * @typedef {object} LLMMessage
  * @prop {R} role
- * @prop {string} content
- * @prop {string} [toolCallId]
+ * @prop {C} [content]
  * @prop {LLMFinishReason} [finishReason]
  * @prop {ToolCall[]} [toolCalls]
+ * @prop {string} [toolCallId]
+ */
+
+/**
+ * @template {LLMRole} [R=LLMRole]
+ * @typedef {LLMMessage<R, PossiblyAsyncContent>} PossiblyAsyncLLMMessage
+ */
+
+/**
+ * @typedef {Promise<LLMMessage<LLMRole,string>|LLMMessage<LLMRole,string>[]>} AsyncLLMMessage
+ */
+
+/**
+ * @typedef {object} FailedLLMAsync
+ * @prop {'error'|string} role
+ * @prop {Error} error
  */
 
+/** @typedef {FailedLLMAsync|LLMMessage} SyncLLMSrc */
+/** @typedef {FailedLLMAsync|AsyncLLMMessage|PossiblyAsyncLLMMessage} LLMMessageSrc */
+
 /**
  * @callback SendCallback
  * @param {LLMMessage[]} messages
@@ -54,55 +79,313 @@ const LLM = require('./LLM');
  * @prop {FilterScope} scope
  */
 
+/**
+ * @typedef {object} JsonSchemaProp
+ * @prop {string} [name]
+ * @prop {string|'string'|'number'|'boolean'|'array'} type - The data type
+ * @prop {string} [description] - Description of the parameter
+ * @prop {string[]} [enum] - Allowed values for this parameter
+ * @prop {number} [minimum] - Minimum value for numeric parameters
+ * @prop {number} [maximum] - Maximum value for numeric parameters
+ */
+
+/** @typedef {{ [key: string]: JsonSchemaProp }} SimpleJsonSchema */
+
+/**
+ * @callback ToolFnCallback
+ * @param {{[key: string]: any}} input
+ * @returns {string|Promise<string>}
+ */
+
+/**
+ * @typedef {object} FnParamsObject
+ * @prop {string} [type] - Schema type ('object')
+ * @prop {string} [name] - Schema name (required for structured output)
+ * @prop {SimpleJsonSchema} properties - Parameter definitions
+ * @prop {string[]} [required] - Required parameters
+ * @prop {boolean} [additionalProperties]
+ *
+ */
+
+/**
+ * @typedef {object} ToolFunction
+ * @prop {string} name - The function name
+ * @prop {string} [description] - What the function does
+ * @prop {FnParamsObject} [parameters] - Parameter schema
+ * @prop {boolean} [strict]
+ */
+
+/**
+ * @typedef {object} ToolInputWithFactory
+ * @prop {string} [name] - The function name
+ * @prop {FnParamsObject|ParametersFactory} [parameters] - Parameter schema
+ */
+
+/** @typedef {Omit<ToolFunction, 'parameters'|'name'> & ToolInputWithFactory} ToolFunctionInput */
+
+/**
+ * @typedef {object} ToolFnObject
+ * @prop {ToolFnCallback} fn
+ */
+
+/** @typedef {ToolFnCallback & ToolFunction} CallbackTool */
+/** @typedef {ToolFnObject & ToolFunction} ObjectTool */
+
+/** @typedef {CallbackTool | ObjectTool} Tool */
+
+/**
+ * @typedef {object} ParametersFactory
+ * @prop {Function} toJSON()
+ */
+
+/** @typedef {Omit<Tool, 'parameters'> & Omit<ToolInputWithFactory, 'name'>} ToolInput */
+
+/**
+ * CONSIDERATION:
+ * - spustit joby až na await, nebo hned? - na await je bezpečnější
+ * - joby si musí umět předat výsledek (což je dané jen implementací, nikoliv nutností)
+ *
+ * CONCEPT
+ * - asynchronní metody vracející this vždy přidávají job (job vrací jen LLM výsledek)
+ * - synchronní metody vracející this, kde záleží na pořadí volání,
+ *   přidávají job s "runNowAndSyncWhenQueueIsEmpty=true" parametrem
+ * - asynchronní metody vracející data, by měly awaitovat this (aby byla data aktuální)
+ * - synchronní metody vracející data NEČEKAJÍ A MAJÍ V NÁZVU "Sync" s výjimkou:
+ *      - toString
+ *      - toJson
+ */
+
 /**
  * @class LLMSession
+ * @implements {PromiseLike<LLMMessage<any>>}
  */
 class LLMSession {
 
     /**
      *
      * @param {LLM} llm
-     * @param {LLMMessage<any>[]} [chat]
+     * @param {(PossiblyAsyncLLMMessage|AsyncLLMMessage)[]} [chat]
      * @param {SendCallback} [onSend]
      * @param {LLMFilter[]} [filters=[]]
      */
-    constructor (llm, chat = [], onSend = () => {}, filters = []) {
+    constructor (llm, chat = [], onSend = null, filters = []) {
         this._llm = llm;
 
         this._onSend = onSend;
 
-        /** @type {LLMMessage<any>[]} */
-        this._chat = chat;
+        this._inExecution = false;
 
-        this._generatedIndex = null;
+        this._jobQ = [];
+
+        this._worker = null;
 
-        this._sort();
+        /** @type {LLMMessageSrc[]} */
+        this._chat = [];
+        this.push(...chat);
+
+        this._generatedIndex = null;
 
         /** @type {LLMFilter[]} */
         this._filters = filters;
 
         this._SCOPE_CONVERSATION_ROLES = [
-            LLM.ROLE_ASSISTANT, LLM.ROLE_USER
+            ROLE_ASSISTANT, ROLE_USER
         ];
+
+        /** @type {Map<string,ObjectTool>} */
+        this._tools = new Map();
+    }
+
+    _job (task, runNowAndSyncWhenQueueIsEmpty = false) {
+        if (runNowAndSyncWhenQueueIsEmpty && this._jobQ.length === 0) {
+            task(null);
+        } else {
+            // @todo remove stack
+            this._jobQ.push(task);
+        }
+    }
+
+    // IDEA: then === defered taskl
+
+    async _runWorker () {
+        let fail = null;
+        let res = null;
+        let lastWasAwait = false;
+
+        const isAwait = (job) => 'onDone' in job;
+        while (this._jobQ.some((job) => isAwait(job))) {
+            const job = this._jobQ.shift();
+            if (isAwait(job)) {
+                lastWasAwait = true;
+                if (fail) {
+                    job.onDone(fail);
+                } else {
+                    job.onDone(null, res);
+                }
+            } else {
+                if (lastWasAwait) {
+                    lastWasAwait = false;
+                    fail = null;
+                    res = null;
+                }
+                try {
+                    this._inExecution = true;
+                    Error.stackTraceLimit = 50;
+                    const result = await Promise.resolve(job(res));
+                    if (typeof result !== 'undefined') {
+                        res = result;
+                    }
+                } catch (e) {
+                    fail = e;
+                } finally {
+                    this._inExecution = false;
+                }
+            }
+        }
+        if (fail) {
+            throw fail;
+        }
+        return res;
+    }
+
+    async _awaitIfNotNestedCall () {
+        if (this._inExecution) return;
+        await this;
     }
 
-    _sort (what = this._chat) {
-        what.sort((a, z) => {
-            if (a.role === z.role
-                || (a.role !== LLM.ROLE_SYSTEM && z.role !== LLM.ROLE_SYSTEM)) {
-                return 0;
+    /**
+     *
+     * @template TResult1
+     * @template TResult2
+     * @param {(value: any) => TResult1 | PromiseLike<TResult1>} [onFulfilled]
+     * @param {(reason: any) => TResult2 | PromiseLike<TResult2>} [onRejected]
+     * @returns {PromiseLike<TResult1 | TResult2>}
+     */
+    then (onFulfilled, onRejected) {
+        let error;
+        let result = null;
+        this._jobQ.push({
+            onDone: (err, res) => {
+                if (err) {
+                    error = err;
+                } else {
+                    result = res;
+                }
             }
-            return a.role === LLM.ROLE_SYSTEM ? -1 : 1;
         });
+        if (this._worker === null) {
+            this._worker = this._runWorker()
+                .finally(() => {
+                    this._worker = null;
+                });
+        }
+        return this._worker
+            .then(() => {
+                if (error) {
+                    throw error;
+                }
+                return result;
+            })
+            .then(onFulfilled, onRejected);
+    }
+
+    /**
+     * @returns {ToolFunction[]}
+     */
+    get tools () {
+        return Array.from(this._tools.values())
+            .map(({
+                name,
+                description = null,
+                parameters = {},
+                strict = true
+            }) => ({
+                name,
+                ...(description && { description }),
+                ...(typeof strict === 'boolean' ? { strict } : {}),
+                parameters: {
+                    type: 'object',
+                    properties: {},
+                    additionalProperties: false,
+                    required: 'properties' in parameters
+                        ? Object.keys(parameters.properties)
+                        : [],
+                    ...parameters
+                }
+            }));
+    }
+
+    /**
+     * @returns {Promise<LLMMessage[]>}
+     */
+    async _resolveMessages () {
+        await Promise.all(
+            this._chat.map(async (msg) => {
+                if ('then' in msg && typeof msg.then === 'function') {
+                    return msg;
+                }
+                if ('content' in msg && typeof msg.content !== 'string' && 'then' in msg.content) {
+                    return msg.content;
+                }
+                return null;
+            })
+        );
+
+        // @ts-ignore
+        return this._moveSystemToTop(this._chat);
+    }
+
+    /**
+     *
+     * @param {LLMMessageSrc[]} messages
+     */
+    _throwAsyncError (messages = this._chat) {
+        const errors = messages.filter((c) => 'role' in c && c.role === 'error');
+
+        if (errors.length === 1) {
+            // @ts-ignore
+            throw errors[0].error;
+        } else if (errors.length) {
+            // @ts-ignore
+            const errs = errors.map((e) => e.error);
+            this._llm.log.log('LLMSession failures', errs);
+            throw new Error(errs.map((e) => e.message).join(', '));
+        }
+    }
+
+    /**
+     *
+     * @template {LLMMessageSrc} T
+     * @param {T[]} what
+     * @returns {T[]}
+     */
+    _moveSystemToTop (what) {
+        let nextSystem = 0;
+        for (let i = 0; i < what.length; i++) {
+            const el = what[i];
+            const isSystem = 'role' in el && el.role === 'system';
+            if (isSystem && i > nextSystem) {
+                what.splice(i, 1);
+                what.splice(nextSystem, 0, el);
+            }
+            if (isSystem && i >= nextSystem) {
+                nextSystem++;
+            }
+        }
         return what;
     }
 
-    _mergeSystem () {
+    /**
+     *
+     * @param {SyncLLMSrc[]} chat
+     * @returns {SyncLLMSrc[]}
+     */
+    _mergeSystem (chat) {
         /** @type {LLMMessage<any>[]} */
         const sysMessages = [];
 
-        const otherMessages = this._chat.filter((message) => {
-            if (message.role !== LLM.ROLE_SYSTEM) {
+        const otherMessages = chat.filter((message) => {
+            if (message.role !== ROLE_SYSTEM) {
                 return true;
             }
             sysMessages.push(message);
@@ -131,7 +414,7 @@ class LLMSession {
 
         return [
             {
-                role: LLM.ROLE_SYSTEM, content
+                role: ROLE_SYSTEM, content
             },
             ...otherMessages
         ];
@@ -140,83 +423,155 @@ class LLMSession {
     /**
      *
      * @param {boolean} [filtered=false]
-     * @returns {LLMMessage[]}
+     * @returns {SyncLLMSrc[]}
      */
-    toArray (filtered = false) {
-        const messages = this._mergeSystem();
-        if (!filtered || this._filters.length === 0) {
-            return messages;
-        }
-        return messages
-            .map((message) => {
-                if (!message.content) {
-                    return message;
-                }
+    toArraySync (filtered = false) {
+        this._moveSystemToTop(this._chat);
+        this._throwAsyncError(this._chat);
+        const sync = this._processSyncMessages(this._chat, filtered);
+        return this._mergeSystem(sync);
+    }
+
+    /**
+     *
+     * @param {LLMMessageSrc[]} messages
+     * @param {boolean} [filtered=false]
+     * @returns {SyncLLMSrc[]}
+     */
+    _processSyncMessages (messages, filtered = false) {
+        /** @type {SyncLLMSrc[]} */
+        const ret = [];
+
+        messages.forEach((m) => {
+            if (!('role' in m)) {
+                return;
+            }
+            if ('content' in m
+                && (m.content instanceof Promise
+                    || (typeof m.content !== 'string' && m.content && 'then' in m.content))) {
+                return;
+            }
+            if (filtered && this._filters.length >= 0 && 'content' in m && typeof m.content === 'string') {
                 const content = this._filters.reduce((text, filter) => {
                     if (!text) {
                         return text;
                     }
-                    if (filter.scope !== message.role
-                        && (filter.scope !== LLM.FILTER_SCOPE_CONVERSATION
-                            || !this._SCOPE_CONVERSATION_ROLES.includes(message.role))) {
+                    if (filter.scope !== m.role
+                        && (filter.scope !== FILTER_SCOPE_CONVERSATION
+                            || !this._SCOPE_CONVERSATION_ROLES.includes(m.role))) {
                         return text;
                     }
-                    const res = filter.filter(text, message.role);
+                    const res = filter.filter(text, m.role);
                     return res === true ? text : res;
-                }, message.content);
-
-                if (!content) {
-                    return null;
+                }, m.content);
+
+                if (typeof content === 'string') {
+                    // @ts-ignore
+                    ret.push({
+                        ...m,
+                        content
+                    });
                 }
+            } else {
+                // @ts-ignore
+                ret.push(m);
+            }
 
-                return {
-                    ...message,
-                    content
-                };
-            })
-            .filter((message) => message !== null);
+        });
+
+        return ret;
+    }
+
+    /**
+     *
+     * @param {boolean} [filtered=false]
+     * @returns {Promise<LLMMessage[]>}
+     */
+    async toArray (filtered = false) {
+        await this._awaitIfNotNestedCall();
+        const messages = await this._resolveMessages();
+        this._throwAsyncError(messages);
+        const sync = this._processSyncMessages(messages, filtered);
+        return this._mergeSystem(sync);
+    }
+
+    /**
+     *
+     * @param {PossiblyAsyncContent} content
+     * @returns {boolean}
+     */
+    _contentIsPromise (content) {
+        return !!content && typeof content !== 'string';
     }
 
     /**
      *
-     * @param {LLMMessage} msg
+     * @param {PossiblyAsyncLLMMessage} m
      * @returns {string}
      */
-    _msgPrefix (msg) {
-        switch (msg.role) {
-            case LLM.ROLE_SYSTEM:
-                return '-';
-            case LLM.ROLE_ASSISTANT:
-                return msg.content ? '<' : '#';
-            case LLM.ROLE_USER:
-                return '>';
-            default:
-                return '*';
+    _contentToString (m) {
+        if (m.toolCalls) {
+            return `{ REQUESTED TOOLS:\n${m.toolCalls
+                .map((t) => `   .${t.name}(${t.args})`).join('\n')} }`;
+        }
+        if (!m.content) {
+            return '[-no-content-]';
         }
+        if (typeof m.content === 'string') {
+            return m.content;
+        }
+        return '<Promise>';
     }
 
     /**
      *
-     * @param {LLMMessage[]} [messages]
+     * @param {LLMMessageSrc[]} [messages]
      * @returns {string}
      */
     toString (messages = this._chat) {
         if (messages.length === 0) {
-            return '[<empty>]';
+            return '-[<empty>]';
         }
         return messages.map((m) => {
+            if ('then' in m) {
+                return '-{ <Promise> }';
+            }
+            if (!('role' in m)) {
+                return '-!- unknown message -!';
+            }
+            if ('error' in m) {
+                return `-<Error: ${m.error.message}>`;
+            }
             switch (m.role) {
-                case LLM.ROLE_SYSTEM:
-                    return `--- system ---\n${m.content}\n--------------`;
+                case ROLE_SYSTEM:
+                    return `- -- system ---\n${m.content}\n--------------`;
                 default:
-                    return `${this._msgPrefix(m)} ${m.content}`;
+                    return `${this._msgPrefix(m)} ${this._contentToString(m)}`;
             }
         })
             .join('\n');
     }
 
+    /**
+     *
+     * @param {PossiblyAsyncLLMMessage} msg
+     * @returns {string}
+     */
+    _msgPrefix (msg) {
+        switch (msg.role) {
+            case ROLE_SYSTEM:
+                return '--';
+            case ROLE_ASSISTANT:
+                return msg.content ? '-<' : '-#';
+            case ROLE_USER:
+                return '->';
+            default:
+                return '-():';
+        }
+    }
+
     toJSON () {
-        return this.toArray();
+        return this.toArraySync();
     }
 
     /**
@@ -225,35 +580,148 @@ class LLMSession {
      * @returns {this}
      */
     debug (needRaw = false) {
-        // eslint-disable-next-line no-console
-        console.log('LLMSession#debug\n', this.toString(
-            needRaw ? this._chat : this.toArray()
-        ));
+        this._job(() => {
+            // eslint-disable-next-line no-console
+            console.log(`LLMSession#debug\n${this.toString(
+                needRaw
+                    ? this._chat
+                    : this.toArraySync(false)
+            )}`);
+        }, true);
         return this;
     }
 
     /**
      *
-     * @param {LLMMessage} message
+     * @param {...(PossiblyAsyncLLMMessage|AsyncLLMMessage)} messages
      * @returns {this}
      */
-    push (message) {
-        // if its system, append it on top
-        this._chat.push(message);
+    push (...messages) {
+        this._job(() => this._pushNow(...messages), true);
+        return this;
+    }
+
+    /**
+     *
+     * @param {...(PossiblyAsyncLLMMessage|AsyncLLMMessage)} messages
+     * @returns {void}
+     */
+    _pushNow (...messages) {
+        this._chat.push(...messages.map((msg) => {
+            if ('then' in msg && typeof msg.then === 'function') {
+                const wrapped = (async () => {
+                    /** @type {SyncLLMSrc[]} */
+                    let expand;
+                    try {
+                        const ret = await msg;
+                        expand = Array.isArray(ret) ? ret : [ret];
+                    } catch (e) {
+                        expand = [{
+                            role: 'error',
+                            error: e
+                        }];
+                    }
+                    const index = this._chat.indexOf(wrapped);
+                    this._chat.splice(index, 1, ...expand);
+                    return expand;
+                })();
+                return wrapped;
+            }
+            if (!('content' in msg) || !this._contentIsPromise(msg.content)) {
+                return msg;
+            }
+            const ret = {
+                ...msg,
+                content: Promise.resolve(msg.content)
+                    .then((r) => {
+                        // @ts-ignore
+                        ret.content = r;
+                        return r;
+                    })
+                    .catch((e) => {
+                        const index = this._chat.indexOf(ret);
+                        this._chat.splice(index, 1, { role: 'error', error: e });
+                        return null;
+                    })
+            };
+
+            return ret;
+        }));
+    }
+
+    /**
+     *
+     * @param  {...ToolInput} addedTools
+     * @returns {this}
+     */
+    tool (...addedTools) {
+        addedTools.forEach((tool) => {
+            if (!tool.name) {
+                throw new Error(`Tool is missing .name: ${tool}`);
+            }
+        });
+        this._job(() => {
+            for (const input of addedTools) {
+                // @ts-ignore
+                // eslint-disable-next-line prefer-const, object-curly-newline
+                let { fn, parameters = {}, name, ...rest } = input;
+
+                if (typeof input === 'function') {
+                    fn = input;
+                }
+
+                if ('toJSON' in parameters && typeof parameters.toJSON === 'function') {
+                    parameters = parameters.toJSON();
+                }
+
+                this._tools.set(name, {
+                    fn,
+                    name,
+                    // @ts-ignore
+                    parameters,
+                    ...rest
+                });
+            }
+        }, true);
+        return this;
+    }
+
+    /**
+     *
+     * @param {string|Promise<string>} content
+     * @returns {this}
+     */
+    user (content) {
+        this.push({
+            role: ROLE_USER,
+            content
+        });
+        return this;
+    }
+
+    /**
+     *
+     * @param {string|Promise<string>} content
+     * @returns {this}
+     */
+    assistant (content) {
+        this.push({
+            role: ROLE_ASSISTANT,
+            content
+        });
         return this;
     }
 
     /**
      *
-     * @param {string} content
+     * @param {string|Promise<string>} content
      * @returns {this}
      */
     systemPrompt (content) {
         this.push({
-            role: LLM.ROLE_SYSTEM,
+            role: ROLE_SYSTEM,
             content
         });
-        this._sort();
         return this;
     }
 
@@ -263,22 +731,101 @@ class LLMSession {
      * @returns {this}
      */
     addFilter (filter) {
-        if (Array.isArray(filter)) {
-            this._filters.push(...filter);
-        } else {
-            this._filters.push(filter);
+        this._job(() => {
+            if (Array.isArray(filter)) {
+                this._filters.push(...filter);
+            } else {
+                this._filters.push(filter);
+            }
+        }, true);
+        return this;
+    }
+
+    /**
+     *
+     * @param {LLMCallPreset} [providerOptions]
+     * @param {LLMLogOptions} [logOptions]
+     * @returns {this}
+     */
+    generate (providerOptions = undefined, logOptions = {}) {
+        this._job(() => this._generate(providerOptions, logOptions));
+        return this;
+    }
+
+    /**
+     *
+     * @param {FnParamsObject|ParametersFactory} output
+     * @param {LLMCallPreset} [providerOptions]
+     * @param {LLMLogOptions} [logOptions]
+     * @returns {this}
+     */
+    generateStructured (output, providerOptions = undefined, logOptions = {}) {
+
+        const responseFormat = 'toJSON' in output && typeof output.toJSON === 'function'
+            ? output.toJSON()
+            : output;
+
+        if (!responseFormat.name) {
+            throw new Error('Missing root object name for LLM structured output');
         }
+
+        this._job(async () => {
+            const result = await this._generate({
+                ...(typeof providerOptions === 'object' ? providerOptions : {}),
+                ...(typeof providerOptions === 'string' ? { preset: providerOptions } : {}),
+                responseFormat
+            }, logOptions);
+
+            return JSON.parse(result.content);
+        });
         return this;
     }
 
     /**
      *
-     * @param {LLMProviderOptions} [providerOptions={}]
+     * @param {LLMCallPreset} [providerOptions]
      * @param {LLMLogOptions} [logOptions]
      * @returns {Promise<LLMMessage<any>>}
      */
-    async generate (providerOptions = {}, logOptions = {}) {
-        const result = await this._llm.generate(this, providerOptions, logOptions);
+    async _generate (providerOptions = undefined, logOptions = {}) {
+        let result = await this._llm.generate(this, providerOptions, logOptions);
+
+        if (result.toolCalls?.length) {
+            const toolCalls = [];
+            const results = await Promise.all(
+                result.toolCalls.map(async (tc) => {
+                    const msg = await this._executeToolCall(tc);
+                    if (msg) {
+                        toolCalls.push(tc);
+                    }
+                    return msg;
+                })
+            );
+
+            if (toolCalls.length) {
+                this._pushNow(
+                    {
+                        role: ROLE_ASSISTANT,
+                        toolCalls
+                    },
+                    ...results.filter((r) => !!r)
+                );
+                result = await this._llm.generate(this, providerOptions, logOptions);
+            } else {
+                // everything failed
+                /** @type {LLMCallPreset} */
+                const overrideChoice = typeof providerOptions === 'string'
+                    ? {
+                        preset: providerOptions,
+                        toolChoice: 'none'
+                    }
+                    : {
+                        ...providerOptions,
+                        toolChoice: 'none'
+                    };
+                result = await this._llm.generate(this, overrideChoice, logOptions);
+            }
+        }
 
         this._generatedIndex = this._chat.length;
         this._chat.push(result);
@@ -286,16 +833,76 @@ class LLMSession {
         return result;
     }
 
+    /**
+     *
+     * @param {ToolCall} toolCall
+     * @returns {Promise<LLMMessage>}
+     */
+    async _executeToolCall (toolCall) {
+
+        const tool = this._tools.get(toolCall.name);
+
+        if (!tool) {
+            this._llm.log.error(`LLM tool "${toolCall.name}": NOT FOUND`, {
+                toolCall
+            });
+            return null;
+        }
+
+        let args;
+        try {
+            args = JSON.parse(toolCall.args);
+            const fnResult = await Promise.resolve(tool.fn(args));
+
+            /**
+             * {
+                "role": "assistant",
+                "tool_calls": [
+                    {
+                    "id": "call_123",
+                    "type": "function",
+                    "function": {
+                        "name": "get_weather",
+                        "arguments": "{\"city\": \"Prague\"}"
+                    }
+                    }
+                ]
+                },
+             */
+            return {
+                content: typeof fnResult === 'string' ? fnResult : JSON.stringify(fnResult),
+                role: 'tool',
+                toolCallId: toolCall.id
+            };
+        } catch (e) {
+            this._llm.log.error(`LLM tool ${toolCall.name}: ${e.message}`, e, {
+                args, toolCall
+            });
+            return null;
+        }
+    }
+
+    /**
+     *
+     * @returns {Promise<string>}
+     */
+    async lastResponse () {
+        await this._awaitIfNotNestedCall();
+        return this.lastResponseSync();
+    }
+
     /**
      *
      * @returns {string}
      */
-    lastResponse () {
+    lastResponseSync () {
         const messages = [];
         for (let i = this._chat.length - 1; i >= 0; i--) {
             const message = this._chat[i];
-
-            if (message.role !== LLM.ROLE_ASSISTANT || !message.content) {
+            if (!('role' in message) || !('content' in message)) {
+                break;
+            }
+            if (message.role !== ROLE_ASSISTANT || !message.content) {
                 break;
             }
             messages.unshift(message.content);
@@ -303,19 +910,31 @@ class LLMSession {
         return messages.join('\n\n');
     }
 
+    /**
+     *
+     * @param {boolean} [dontMarkAsSent=false]
+     * @returns {Promise<LLMMessage[]>}
+     */
+    async messagesToSend (dontMarkAsSent = false) {
+        await this._awaitIfNotNestedCall();
+        return this.messagesToSendSync(dontMarkAsSent);
+    }
+
     /**
      *
      * @param {boolean} [dontMarkAsSent=false]
      * @returns {LLMMessage[]}
      */
-    messagesToSend (dontMarkAsSent = false) {
+    messagesToSendSync (dontMarkAsSent = false) {
         if (!this._generatedIndex) {
             return [];
         }
 
-        let messages = this._chat.splice(this._generatedIndex);
-        messages = messages.flatMap((msg) => LLM.toMessages(msg));
+        const allMessages = this._chat.splice(this._generatedIndex);
+        let messages = this._processSyncMessages(allMessages);
+        messages = messages.flatMap((msg) => LLMSession.toMessages(msg));
 
+        // probably issue - generated messages are now not in the chat
         if (dontMarkAsSent) {
             return messages;
         }
@@ -332,13 +951,37 @@ class LLMSession {
      * @returns {this}
      */
     send (quickReplies = undefined) {
-        const messages = this.messagesToSend();
-
-        this._onSend(messages, quickReplies);
+        this._job(() => {
+            const messages = this.messagesToSendSync();
+            this._onSend(messages, quickReplies);
+        }, true);
 
         return this;
     }
 
+    /**
+     *
+     * @param {LLMMessage} result
+     * @returns {LLMMessage[]}
+     */
+    static toMessages (result) {
+        let filtered = result.content
+            .replace(/\n\n\n+/g, '\n\n')
+            .split(/\n\n+(?!\s*-)/g)
+            .map((t) => t.replace(/\s*\n\s+/g, '\n')
+                .trim())
+            .filter((t) => !!t);
+
+        if (result.finishReason === 'length' && filtered.length <= 0) {
+            filtered = filtered.slice(0, filtered.length - 1);
+        }
+
+        return filtered.map((content) => ({
+            content,
+            role: result.role
+        }));
+    }
+
 }
 
 module.exports = LLMSession;