letta-nightly 0.1.7.dev20240924104148__py3-none-any.whl

letta/local_llm/llm_chat_completion_wrappers/zephyr.py

@@ -0,0 +1,345 @@
+from letta.utils import json_dumps, json_loads
+
+from ...errors import LLMJSONParsingError
+from ..json_parser import clean_json
+from .wrapper_base import LLMChatCompletionWrapper
+
+
+class ZephyrMistralWrapper(LLMChatCompletionWrapper):
+    """
+    Wrapper for Zephyr Alpha and Beta, Mistral 7B:
+    https://huggingface.co/HuggingFaceH4/zephyr-7b-alpha
+    https://huggingface.co/HuggingFaceH4/zephyr-7b-beta
+    Note: this wrapper formats a prompt that only generates JSON, no inner thoughts
+    """
+
+    def __init__(
+        self,
+        simplify_json_content=True,
+        clean_function_args=True,
+        include_assistant_prefix=True,
+        include_opening_brace_in_prefix=True,
+        include_section_separators=False,
+    ):
+        self.simplify_json_content = simplify_json_content
+        self.clean_func_args = clean_function_args
+        self.include_assistant_prefix = include_assistant_prefix
+        self.include_opening_brance_in_prefix = include_opening_brace_in_prefix
+        self.include_section_separators = include_section_separators
+
+    def chat_completion_to_prompt(self, messages, functions, function_documentation=None):
+        """
+        Zephyr prompt format:
+            <|system|>
+            </s>
+            <|user|>
+            {prompt}</s>
+            <|assistant|>
+        (source: https://huggingface.co/TheBloke/zephyr-7B-beta-GGUF#prompt-template-zephyr)
+        """
+
+        prompt = ""
+
+        IM_END_TOKEN = "</s>"
+
+        # System instructions go first
+        assert messages[0]["role"] == "system"
+        prompt += f"<|system|>"
+        prompt += f"\n{messages[0]['content']}"
+
+        # Next is the functions preamble
+        def create_function_description(schema):
+            # airorobos style
+            func_str = ""
+            func_str += f"{schema['name']}:"
+            func_str += f"\n  description: {schema['description']}"
+            func_str += f"\n  params:"
+            for param_k, param_v in schema["parameters"]["properties"].items():
+                # TODO we're ignoring type
+                func_str += f"\n    {param_k}: {param_v['description']}"
+            # TODO we're ignoring schema['parameters']['required']
+            return func_str
+
+        # prompt += f"\nPlease select the most suitable function and parameters from the list of available functions below, based on the user's input. Provide your response in JSON format."
+        prompt += f"\nPlease select the most suitable function and parameters from the list of available functions below, based on the ongoing conversation. Provide your response in JSON format."
+        prompt += f"\nAvailable functions:"
+        if function_documentation is not None:
+            prompt += f"\n{function_documentation}"
+        else:
+            for function_dict in functions:
+                prompt += f"\n{create_function_description(function_dict)}"
+
+        # Put functions INSIDE system message (TODO experiment with this)
+        prompt += IM_END_TOKEN
+
+        def create_function_call(function_call):
+            airo_func_call = {
+                "function": function_call["name"],
+                "params": json_loads(function_call["arguments"]),
+            }
+            return json_dumps(airo_func_call, indent=2)
+
+        for message in messages[1:]:
+            assert message["role"] in ["user", "assistant", "function", "tool"], message
+
+            if message["role"] == "user":
+                if self.simplify_json_content:
+                    try:
+                        content_json = json_loads(message["content"])
+                        content_simple = content_json["message"]
+                        prompt += f"\n<|user|>\n{content_simple}{IM_END_TOKEN}"
+                        # prompt += f"\nUSER: {content_simple}"
+                    except:
+                        prompt += f"\n<|user|>\n{message['content']}{IM_END_TOKEN}"
+                        # prompt += f"\nUSER: {message['content']}"
+            elif message["role"] == "assistant":
+                prompt += f"\n<|assistant|>"
+                if message["content"] is not None:
+                    prompt += f"\n{message['content']}"
+                # prompt += f"\nASSISTANT: {message['content']}"
+                # need to add the function call if there was one
+                if "function_call" in message and message["function_call"]:
+                    prompt += f"\n{create_function_call(message['function_call'])}"
+                prompt += f"{IM_END_TOKEN}"
+            elif message["role"] in ["function", "tool"]:
+                # TODO find a good way to add this
+                # prompt += f"\nASSISTANT: (function return) {message['content']}"
+                prompt += f"\n<|assistant|>"
+                prompt += f"\nFUNCTION RETURN: {message['content']}"
+                # prompt += f"\nFUNCTION RETURN: {message['content']}"
+                continue
+            else:
+                raise ValueError(message)
+
+        # Add a sep for the response
+        # if self.include_section_separators:
+        # prompt += "\n### RESPONSE"
+
+        if self.include_assistant_prefix:
+            # prompt += f"\nASSISTANT:"
+            prompt += f"\n<|assistant|>"
+            if self.include_opening_brance_in_prefix:
+                prompt += "\n{"
+
+        return prompt
+
+    def clean_function_args(self, function_name, function_args):
+        """Some basic Letta-specific cleaning of function args"""
+        cleaned_function_name = function_name
+        cleaned_function_args = function_args.copy() if function_args is not None else {}
+
+        if function_name == "send_message":
+            # strip request_heartbeat
+            cleaned_function_args.pop("request_heartbeat", None)
+
+        # TODO more cleaning to fix errors LLM makes
+        return cleaned_function_name, cleaned_function_args
+
+    def output_to_chat_completion_response(self, raw_llm_output):
+        """Turn raw LLM output into a ChatCompletion style response with:
+        "message" = {
+            "role": "assistant",
+            "content": ...,
+            "function_call": {
+                "name": ...
+                "arguments": {
+                    "arg1": val1,
+                    ...
+                }
+            }
+        }
+        """
+        if self.include_opening_brance_in_prefix and raw_llm_output[0] != "{":
+            raw_llm_output = "{" + raw_llm_output
+
+        try:
+            function_json_output = clean_json(raw_llm_output)
+        except Exception as e:
+            raise Exception(f"Failed to decode JSON from LLM output:\n{raw_llm_output} - error\n{str(e)}")
+        try:
+            function_name = function_json_output["function"]
+            function_parameters = function_json_output["params"]
+        except KeyError as e:
+            raise LLMJSONParsingError(f"Received valid JSON from LLM, but JSON was missing fields: {str(e)}")
+
+        if self.clean_func_args:
+            function_name, function_parameters = self.clean_function_args(function_name, function_parameters)
+
+        message = {
+            "role": "assistant",
+            "content": None,
+            "function_call": {
+                "name": function_name,
+                "arguments": json_dumps(function_parameters),
+            },
+        }
+        return message
+
+
+class ZephyrMistralInnerMonologueWrapper(ZephyrMistralWrapper):
+    """Still expect only JSON outputs from model, but add inner monologue as a field"""
+
+    """
+    Wrapper for Zephyr Alpha and Beta, Mistral 7B:
+    https://huggingface.co/HuggingFaceH4/zephyr-7b-alpha
+    https://huggingface.co/HuggingFaceH4/zephyr-7b-beta
+    Note: this wrapper formats a prompt with inner thoughts included
+    """
+
+    def __init__(
+        self,
+        simplify_json_content=True,
+        clean_function_args=True,
+        include_assistant_prefix=True,
+        include_opening_brace_in_prefix=True,
+        include_section_separators=True,
+    ):
+        self.simplify_json_content = simplify_json_content
+        self.clean_func_args = clean_function_args
+        self.include_assistant_prefix = include_assistant_prefix
+        self.include_opening_brance_in_prefix = include_opening_brace_in_prefix
+        self.include_section_separators = include_section_separators
+
+    def chat_completion_to_prompt(self, messages, functions, function_documentation=None):
+        prompt = ""
+
+        IM_END_TOKEN = "</s>"
+
+        # System insturctions go first
+        assert messages[0]["role"] == "system"
+        prompt += messages[0]["content"]
+
+        # Next is the functions preamble
+        def create_function_description(schema, add_inner_thoughts=True):
+            # airorobos style
+            func_str = ""
+            func_str += f"{schema['name']}:"
+            func_str += f"\n  description: {schema['description']}"
+            func_str += f"\n  params:"
+            if add_inner_thoughts:
+                func_str += f"\n    inner_thoughts: Deep inner monologue private to you only."
+            for param_k, param_v in schema["parameters"]["properties"].items():
+                # TODO we're ignoring type
+                func_str += f"\n    {param_k}: {param_v['description']}"
+            # TODO we're ignoring schema['parameters']['required']
+            return func_str
+
+        # prompt += f"\nPlease select the most suitable function and parameters from the list of available functions below, based on the user's input. Provide your response in JSON format."
+        prompt += f"\nPlease select the most suitable function and parameters from the list of available functions below, based on the ongoing conversation. Provide your response in JSON format."
+        prompt += f"\nAvailable functions:"
+        if function_documentation is not None:
+            prompt += f"\n{function_documentation}"
+        else:
+            for function_dict in functions:
+                prompt += f"\n{create_function_description(function_dict)}"
+
+        def create_function_call(function_call, inner_thoughts=None):
+            airo_func_call = {
+                "function": function_call["name"],
+                "params": {
+                    "inner_thoughts": inner_thoughts,
+                    **json_loads(function_call["arguments"]),
+                },
+            }
+            return json_dumps(airo_func_call, indent=2)
+
+        # Add a sep for the conversation
+        if self.include_section_separators:
+            prompt += "\n<|user|>"
+
+        # Last are the user/assistant messages
+        for message in messages[1:]:
+            assert message["role"] in ["user", "assistant", "function", "tool"], message
+
+            if message["role"] == "user":
+                if self.simplify_json_content:
+                    try:
+                        content_json = json_loads(message["content"])
+                        content_simple = content_json["message"]
+                        prompt += f"\n<|user|>\n{content_simple}{IM_END_TOKEN}"
+                    except:
+                        prompt += f"\n<|user|>\n{message['content']}{IM_END_TOKEN}"
+            elif message["role"] == "assistant":
+                prompt += f"\n<|assistant|>"
+                # need to add the function call if there was one
+                inner_thoughts = message["content"]
+                if "function_call" in message and message["function_call"]:
+                    prompt += f"\n{create_function_call(message['function_call'], inner_thoughts=inner_thoughts)}"
+            elif message["role"] in ["function", "tool"]:
+                # TODO find a good way to add this
+                # prompt += f"\nASSISTANT: (function return) {message['content']}"
+                prompt += f"\nFUNCTION RETURN: {message['content']}"
+                continue
+            else:
+                raise ValueError(message)
+
+        # Add a sep for the response
+        # if self.include_section_separators:
+        #    prompt += "\n### RESPONSE"
+
+        if self.include_assistant_prefix:
+            prompt += f"\n<|assistant|>"
+            if self.include_opening_brance_in_prefix:
+                prompt += "\n{"
+
+        return prompt
+
+    def clean_function_args(self, function_name, function_args):
+        """Some basic Letta-specific cleaning of function args"""
+        cleaned_function_name = function_name
+        cleaned_function_args = function_args.copy() if function_args is not None else {}
+
+        if function_name == "send_message":
+            # strip request_heartbeat
+            cleaned_function_args.pop("request_heartbeat", None)
+
+        inner_thoughts = None
+        if "inner_thoughts" in function_args:
+            inner_thoughts = cleaned_function_args.pop("inner_thoughts")
+
+        # TODO more cleaning to fix errors LLM makes
+        return inner_thoughts, cleaned_function_name, cleaned_function_args
+
+    def output_to_chat_completion_response(self, raw_llm_output):
+        """Turn raw LLM output into a ChatCompletion style response with:
+        "message" = {
+            "role": "assistant",
+            "content": ...,
+            "function_call": {
+                "name": ...
+                "arguments": {
+                    "arg1": val1,
+                    ...
+                }
+            }
+        }
+        """
+        if self.include_opening_brance_in_prefix and raw_llm_output[0] != "{":
+            raw_llm_output = "{" + raw_llm_output
+
+        try:
+            function_json_output = clean_json(raw_llm_output)
+        except Exception as e:
+            raise Exception(f"Failed to decode JSON from LLM output:\n{raw_llm_output} - error\n{str(e)}")
+        try:
+            function_name = function_json_output["function"]
+            function_parameters = function_json_output["params"]
+        except KeyError as e:
+            raise LLMJSONParsingError(f"Received valid JSON from LLM, but JSON was missing fields: {str(e)}")
+
+        if self.clean_func_args:
+            (
+                inner_thoughts,
+                function_name,
+                function_parameters,
+            ) = self.clean_function_args(function_name, function_parameters)
+
+        message = {
+            "role": "assistant",
+            "content": inner_thoughts,
+            "function_call": {
+                "name": function_name,
+                "arguments": json_dumps(function_parameters),
+            },
+        }
+        return message

letta/local_llm/lmstudio/api.py

@@ -0,0 +1,100 @@
+from urllib.parse import urljoin
+
+from letta.local_llm.settings.settings import get_completions_settings
+from letta.local_llm.utils import post_json_auth_request
+from letta.utils import count_tokens
+
+LMSTUDIO_API_CHAT_SUFFIX = "/v1/chat/completions"
+LMSTUDIO_API_COMPLETIONS_SUFFIX = "/v1/completions"
+
+
+def get_lmstudio_completion(endpoint, auth_type, auth_key, prompt, context_window, api="completions"):
+    """Based on the example for using LM Studio as a backend from https://github.com/lmstudio-ai/examples/tree/main/Hello%2C%20world%20-%20OpenAI%20python%20client"""
+    from letta.utils import printd
+
+    prompt_tokens = count_tokens(prompt)
+    if prompt_tokens > context_window:
+        raise Exception(f"Request exceeds maximum context length ({prompt_tokens} > {context_window} tokens)")
+
+    settings = get_completions_settings()
+    settings.update(
+        {
+            "input_prefix": "",
+            "input_suffix": "",
+            # This controls how LM studio handles context overflow
+            # In Letta we handle this ourselves, so this should be disabled
+            # "context_overflow_policy": 0,
+            "lmstudio": {"context_overflow_policy": 0},  # 0 = stop at limit
+            "stream": False,
+            "model": "local model",
+        }
+    )
+
+    # Uses the ChatCompletions API style
+    # Seems to work better, probably because it's applying some extra settings under-the-hood?
+    if api == "chat":
+        URI = urljoin(endpoint.strip("/") + "/", LMSTUDIO_API_CHAT_SUFFIX.strip("/"))
+
+        # Settings for the generation, includes the prompt + stop tokens, max length, etc
+        request = settings
+        request["max_tokens"] = context_window
+
+        # Put the entire completion string inside the first message
+        message_structure = [{"role": "user", "content": prompt}]
+        request["messages"] = message_structure
+
+    # Uses basic string completions (string in, string out)
+    # Does not work as well as ChatCompletions for some reason
+    elif api == "completions":
+        URI = urljoin(endpoint.strip("/") + "/", LMSTUDIO_API_COMPLETIONS_SUFFIX.strip("/"))
+
+        # Settings for the generation, includes the prompt + stop tokens, max length, etc
+        request = settings
+        request["max_tokens"] = context_window
+
+        # Standard completions format, formatted string goes in prompt
+        request["prompt"] = prompt
+
+    else:
+        raise ValueError(api)
+
+    if not endpoint.startswith(("http://", "https://")):
+        raise ValueError(f"Provided OPENAI_API_BASE value ({endpoint}) must begin with http:// or https://")
+
+    try:
+        response = post_json_auth_request(uri=URI, json_payload=request, auth_type=auth_type, auth_key=auth_key)
+        if response.status_code == 200:
+            result_full = response.json()
+            printd(f"JSON API response:\n{result_full}")
+            if api == "chat":
+                result = result_full["choices"][0]["message"]["content"]
+                usage = result_full.get("usage", None)
+            elif api == "completions":
+                result = result_full["choices"][0]["text"]
+                usage = result_full.get("usage", None)
+        else:
+            # Example error: msg={"error":"Context length exceeded. Tokens in context: 8000, Context length: 8000"}
+            if "context length" in str(response.text).lower():
+                # "exceeds context length" is what appears in the LM Studio error message
+                # raise an alternate exception that matches OpenAI's message, which is "maximum context length"
+                raise Exception(f"Request exceeds maximum context length (code={response.status_code}, msg={response.text}, URI={URI})")
+            else:
+                raise Exception(
+                    f"API call got non-200 response code (code={response.status_code}, msg={response.text}) for address: {URI}."
+                    + f" Make sure that the LM Studio local inference server is running and reachable at {URI}."
+                )
+    except:
+        # TODO handle gracefully
+        raise
+
+    # Pass usage statistics back to main thread
+    # These are used to compute memory warning messages
+    completion_tokens = usage.get("completion_tokens", None) if usage is not None else None
+    total_tokens = prompt_tokens + completion_tokens if completion_tokens is not None else None
+    usage = {
+        "prompt_tokens": prompt_tokens,  # can grab from usage dict, but it's usually wrong (set to 0)
+        "completion_tokens": completion_tokens,
+        "total_tokens": total_tokens,
+    }
+
+    return result, usage

letta/local_llm/lmstudio/settings.py

@@ -0,0 +1,29 @@
+SIMPLE = {
+    "stop": [
+        "\nUSER:",
+        "\nASSISTANT:",
+        "\nFUNCTION RETURN:",
+        "\nUSER",
+        "\nASSISTANT",
+        "\nFUNCTION RETURN",
+        "\nFUNCTION",
+        "\nFUNC",
+        "<|im_start|>",
+        "<|im_end|>",
+        "<|im_sep|>",
+        # '\n' +
+        # '</s>',
+        # '<|',
+        # '\n#',
+        # '\n\n\n',
+    ],
+    # This controls the maximum number of tokens that the model can generate
+    # Cap this at the model context length (assuming 8k for Mistral 7B)
+    # "max_tokens": 8000,
+    # "max_tokens": LLM_MAX_TOKENS,
+    # This controls how LM studio handles context overflow
+    # In Letta we handle this ourselves, so this should be commented out
+    # "lmstudio": {"context_overflow_policy": 2},
+    "stream": False,
+    "model": "local model",
+}

letta/local_llm/ollama/api.py

@@ -0,0 +1,88 @@
+from urllib.parse import urljoin
+
+from letta.errors import LocalLLMError
+from letta.local_llm.settings.settings import get_completions_settings
+from letta.local_llm.utils import post_json_auth_request
+from letta.utils import count_tokens
+
+OLLAMA_API_SUFFIX = "/api/generate"
+
+
+def get_ollama_completion(endpoint, auth_type, auth_key, model, prompt, context_window, grammar=None):
+    """See https://github.com/jmorganca/ollama/blob/main/docs/api.md for instructions on how to run the LLM web server"""
+    from letta.utils import printd
+
+    prompt_tokens = count_tokens(prompt)
+    if prompt_tokens > context_window:
+        raise Exception(f"Request exceeds maximum context length ({prompt_tokens} > {context_window} tokens)")
+
+    if model is None:
+        raise LocalLLMError(
+            f"Error: model name not specified. Set model in your config to the model you want to run (e.g. 'dolphin2.2-mistral')"
+        )
+
+    # Settings for the generation, includes the prompt + stop tokens, max length, etc
+    # https://github.com/jmorganca/ollama/blob/main/docs/modelfile.md#valid-parameters-and-values
+    settings = get_completions_settings()
+    settings.update(
+        {
+            # specific naming for context length
+            "num_ctx": context_window,
+        }
+    )
+
+    # https://github.com/jmorganca/ollama/blob/main/docs/api.md#generate-a-completion
+    request = {
+        ## base parameters
+        "model": model,
+        "prompt": prompt,
+        # "images": [],  # TODO eventually support
+        ## advanced parameters
+        # "format": "json",  # TODO eventually support
+        "stream": False,
+        "options": settings,
+        "raw": True,  # no prompt formatting
+        # "raw mode does not support template, system, or context"
+        # "system": "",  # no prompt formatting
+        # "template": "{{ .Prompt }}",  # no prompt formatting
+        # "context": None,  # no memory via prompt formatting
+    }
+
+    # Set grammar
+    if grammar is not None:
+        # request["grammar_string"] = load_grammar_file(grammar)
+        raise NotImplementedError(f"Ollama does not support grammars")
+
+    if not endpoint.startswith(("http://", "https://")):
+        raise ValueError(f"Provided OPENAI_API_BASE value ({endpoint}) must begin with http:// or https://")
+
+    try:
+        URI = urljoin(endpoint.strip("/") + "/", OLLAMA_API_SUFFIX.strip("/"))
+        response = post_json_auth_request(uri=URI, json_payload=request, auth_type=auth_type, auth_key=auth_key)
+        if response.status_code == 200:
+            # https://github.com/jmorganca/ollama/blob/main/docs/api.md
+            result_full = response.json()
+            printd(f"JSON API response:\n{result_full}")
+            result = result_full["response"]
+        else:
+            raise Exception(
+                f"API call got non-200 response code (code={response.status_code}, msg={response.text}) for address: {URI}."
+                + f" Make sure that the ollama API server is running and reachable at {URI}."
+            )
+
+    except:
+        # TODO handle gracefully
+        raise
+
+    # Pass usage statistics back to main thread
+    # These are used to compute memory warning messages
+    # https://github.com/jmorganca/ollama/blob/main/docs/api.md#response
+    completion_tokens = result_full.get("eval_count", None)
+    total_tokens = prompt_tokens + completion_tokens if completion_tokens is not None else None
+    usage = {
+        "prompt_tokens": prompt_tokens,  # can also grab from "prompt_eval_count"
+        "completion_tokens": completion_tokens,
+        "total_tokens": total_tokens,
+    }
+
+    return result, usage

letta/local_llm/ollama/settings.py

@@ -0,0 +1,32 @@
+# see https://github.com/jmorganca/ollama/blob/main/docs/api.md
+# and https://github.com/jmorganca/ollama/blob/main/docs/modelfile.md#valid-parameters-and-values
+SIMPLE = {
+    "options": {
+        "stop": [
+            "\nUSER:",
+            "\nASSISTANT:",
+            "\nFUNCTION RETURN:",
+            "\nUSER",
+            "\nASSISTANT",
+            "\nFUNCTION RETURN",
+            "\nFUNCTION",
+            "\nFUNC",
+            "<|im_start|>",
+            "<|im_end|>",
+            "<|im_sep|>",
+            # '\n' +
+            # '</s>',
+            # '<|',
+            # '\n#',
+            # '\n\n\n',
+        ],
+        # "num_ctx": LLM_MAX_TOKENS,
+    },
+    "stream": False,
+    # turn off Ollama's own prompt formatting
+    "system": "",
+    "template": "{{ .Prompt }}",
+    # "system": None,
+    # "template": None,
+    "context": None,
+}

letta/local_llm/settings/deterministic_mirostat.py

@@ -0,0 +1,45 @@
+from letta.local_llm.settings.simple import settings as simple_settings
+
+settings = {
+    "max_new_tokens": 250,
+    "do_sample": False,
+    "temperature": 0,
+    "top_p": 0,
+    "typical_p": 1,
+    "repetition_penalty": 1.18,
+    "repetition_penalty_range": 0,
+    "encoder_repetition_penalty": 1,
+    "top_k": 1,
+    "min_length": 0,
+    "no_repeat_ngram_size": 0,
+    "num_beams": 1,
+    "penalty_alpha": 0,
+    "length_penalty": 1,
+    "early_stopping": False,
+    "guidance_scale": 1,
+    "negative_prompt": "",
+    "seed": -1,
+    "add_bos_token": True,
+    # NOTE: important - these are the BASE stopping strings, and should be combined with {{user}}/{{char}}-based stopping strings
+    "stopping_strings": [
+        simple_settings["stop"]
+        # '### Response (JSON only, engaging, natural, authentic, descriptive, creative):',
+        # "</s>",
+        # "<|",
+        # "\n#",
+        # "\n*{{user}} ",
+        # "\n\n\n",
+        # "\n{",
+        # ",\n{",
+    ],
+    "truncation_length": 4096,
+    "ban_eos_token": False,
+    "skip_special_tokens": True,
+    "top_a": 0,
+    "tfs": 1,
+    "epsilon_cutoff": 0,
+    "eta_cutoff": 0,
+    "mirostat_mode": 2,
+    "mirostat_tau": 4,
+    "mirostat_eta": 0.1,
+}