letta-nightly 0.1.7.dev20240924104148__py3-none-any.whl

letta/local_llm/grammars/json.gbnf

@@ -0,0 +1,26 @@
+# https://github.com/ggerganov/llama.cpp/blob/master/grammars/json.gbnf
+root   ::= object
+value  ::= object | array | string | number | ("true" | "false" | "null") ws
+
+object ::=
+  "{" ws (
+            string ":" ws value
+    ("," ws string ":" ws value)*
+  )? "}" ws
+
+array  ::=
+  "[" ws (
+            value
+    ("," ws value)*
+  )? "]" ws
+
+string ::=
+  "\"" (
+    [^"\\] |
+    "\\" (["\\/bfnrt] | "u" [0-9a-fA-F] [0-9a-fA-F] [0-9a-fA-F] [0-9a-fA-F]) # escapes
+  )* "\"" ws
+
+number ::= ("-"? ([0-9] | [1-9] [0-9]*)) ("." [0-9]+)? ([eE] [-+]? [0-9]+)? ws
+
+# Optional space: by convention, applied in this grammar after literal chars when allowed
+ws ::= ([ \t\n] ws)?

letta/local_llm/grammars/json_func_calls_with_inner_thoughts.gbnf

@@ -0,0 +1,32 @@
+root ::= Function
+Function ::= SendMessage | PauseHeartbeats | CoreMemoryAppend | CoreMemoryReplace | ConversationSearch | ConversationSearchDate | ArchivalMemoryInsert | ArchivalMemorySearch
+SendMessage ::= "{"   ws   "\"function\":"   ws   "\"send_message\","   ws   "\"params\":"   ws   SendMessageParams   "}"
+PauseHeartbeats ::= "{"   ws   "\"function\":"   ws   "\"pause_heartbeats\","   ws   "\"params\":"   ws   PauseHeartbeatsParams   "}"
+CoreMemoryAppend ::= "{"   ws   "\"function\":"   ws   "\"core_memory_append\","   ws   "\"params\":"   ws   CoreMemoryAppendParams   "}"
+CoreMemoryReplace ::= "{"   ws   "\"function\":"   ws   "\"core_memory_replace\","   ws   "\"params\":"   ws   CoreMemoryReplaceParams   "}"
+ConversationSearch  ::= "{"   ws   "\"function\":"   ws   "\"conversation_search\","   ws   "\"params\":"   ws   ConversationSearchParams   "}"
+ConversationSearchDate  ::= "{"   ws   "\"function\":"   ws   "\"conversation_search_date\","   ws   "\"params\":"   ws   ConversationSearchDateParams   "}"
+ArchivalMemoryInsert  ::= "{"   ws   "\"function\":"   ws   "\"archival_memory_insert\","   ws   "\"params\":"   ws   ArchivalMemoryInsertParams   "}"
+ArchivalMemorySearch  ::= "{"   ws   "\"function\":"   ws   "\"archival_memory_search\","   ws   "\"params\":"   ws   ArchivalMemorySearchParams   "}"
+SendMessageParams ::= "{"   ws   InnerThoughtsParam   ","   ws   "\"message\":"   ws   string   ws   "}"
+PauseHeartbeatsParams ::= "{"   ws   InnerThoughtsParam   ","      ws   "\"minutes\":"   ws   number   ws   "}"
+CoreMemoryAppendParams ::= "{"   ws   InnerThoughtsParam   ","      ws   "\"name\":"   ws   namestring   ","   ws   "\"content\":"   ws   string   ws   ","   ws   RequestHeartbeatParam   ws   "}"
+CoreMemoryReplaceParams ::= "{"   ws   InnerThoughtsParam   ","      ws   "\"name\":"   ws   namestring   ","   ws   "\"old_content\":"   ws   string   ","   ws   "\"new_content\":"   ws   string   ws   ","   ws   RequestHeartbeatParam   ws   "}"
+ConversationSearchParams ::= "{"   ws   InnerThoughtsParam   ","      ws   "\"query\":"   ws   string   ws   ","   ws   "\"page\":"   ws   number   ws   ","   ws   RequestHeartbeatParam   ws   "}"
+ConversationSearchDateParams ::= "{"   ws   InnerThoughtsParam   ","      ws   "\"start_date\":"   ws   string   ws   ","      ws   "\"end_date\":"   ws   string   ws   ","   ws   "\"page\":"   ws   number   ws   ","   ws   RequestHeartbeatParam   ws   "}"
+ArchivalMemoryInsertParams ::= "{"   ws   InnerThoughtsParam    ","   ws   "\"content\":"   ws   string   ws   ","   ws   RequestHeartbeatParam   ws   "}"
+ArchivalMemorySearchParams ::= "{"   ws   InnerThoughtsParam   ","  ws   "\"query\":"   ws   string   ws   ","   ws   "\"page\":"   ws   number   ws   ","   ws   RequestHeartbeatParam   ws   "}"
+InnerThoughtsParam ::= "\"inner_thoughts\":"   ws   string
+RequestHeartbeatParam ::= "\"request_heartbeat\":"   ws   boolean
+namestring ::= "\"human\"" | "\"persona\""
+boolean ::= "true" | "false"
+number ::= [0-9]+
+
+string ::=
+  "\"" (
+    [^"\\] |
+    "\\" (["\\/bfnrt] | "u" [0-9a-fA-F] [0-9a-fA-F] [0-9a-fA-F] [0-9a-fA-F]) # escapes
+  )* "\"" ws
+
+# Optional space: by convention, applied in this grammar after literal chars when allowed
+ws ::= ([ \t\n] ws)?

letta/local_llm/groq/api.py

@@ -0,0 +1,97 @@
+from typing import Tuple
+from urllib.parse import urljoin
+
+from letta.local_llm.settings.settings import get_completions_settings
+from letta.local_llm.utils import post_json_auth_request
+from letta.utils import count_tokens
+
+API_CHAT_SUFFIX = "/v1/chat/completions"
+# LMSTUDIO_API_COMPLETIONS_SUFFIX = "/v1/completions"
+
+
+def get_groq_completion(endpoint: str, auth_type: str, auth_key: str, model: str, prompt: str, context_window: int) -> Tuple[str, dict]:
+    """TODO no support for function calling OR raw completions, so we need to route the request into /chat/completions instead"""
+    from letta.utils import printd
+
+    prompt_tokens = count_tokens(prompt)
+    if prompt_tokens > context_window:
+        raise Exception(f"Request exceeds maximum context length ({prompt_tokens} > {context_window} tokens)")
+
+    settings = get_completions_settings()
+    settings.update(
+        {
+            # see https://console.groq.com/docs/text-chat, supports:
+            # "temperature": ,
+            # "max_tokens": ,
+            # "top_p",
+            # "stream",
+            # "stop",
+            # Groq only allows 4 stop tokens
+            "stop": [
+                "\nUSER",
+                "\nASSISTANT",
+                "\nFUNCTION",
+                # "\nFUNCTION RETURN",
+                # "<|im_start|>",
+                # "<|im_end|>",
+                # "<|im_sep|>",
+                # # airoboros specific
+                # "\n### ",
+                # # '\n' +
+                # # '</s>',
+                # # '<|',
+                # "\n#",
+                # # "\n\n\n",
+                # # prevent chaining function calls / multi json objects / run-on generations
+                # # NOTE: this requires the ability to patch the extra '}}' back into the prompt
+                "  }\n}\n",
+            ]
+        }
+    )
+
+    URI = urljoin(endpoint.strip("/") + "/", API_CHAT_SUFFIX.strip("/"))
+
+    # Settings for the generation, includes the prompt + stop tokens, max length, etc
+    request = settings
+    request["model"] = model
+    request["max_tokens"] = context_window
+    # NOTE: Hack for chat/completion-only endpoints: put the entire completion string inside the first message
+    message_structure = [{"role": "user", "content": prompt}]
+    request["messages"] = message_structure
+
+    if not endpoint.startswith(("http://", "https://")):
+        raise ValueError(f"Provided OPENAI_API_BASE value ({endpoint}) must begin with http:// or https://")
+
+    try:
+        response = post_json_auth_request(uri=URI, json_payload=request, auth_type=auth_type, auth_key=auth_key)
+        if response.status_code == 200:
+            result_full = response.json()
+            printd(f"JSON API response:\n{result_full}")
+            result = result_full["choices"][0]["message"]["content"]
+            usage = result_full.get("usage", None)
+        else:
+            # Example error: msg={"error":"Context length exceeded. Tokens in context: 8000, Context length: 8000"}
+            if "context length" in str(response.text).lower():
+                # "exceeds context length" is what appears in the LM Studio error message
+                # raise an alternate exception that matches OpenAI's message, which is "maximum context length"
+                raise Exception(f"Request exceeds maximum context length (code={response.status_code}, msg={response.text}, URI={URI})")
+            else:
+                raise Exception(
+                    f"API call got non-200 response code (code={response.status_code}, msg={response.text}) for address: {URI}."
+                    + f" Make sure that the inference server is running and reachable at {URI}."
+                )
+    except:
+        # TODO handle gracefully
+        raise
+
+    # Pass usage statistics back to main thread
+    # These are used to compute memory warning messages
+    completion_tokens = usage.get("completion_tokens", None) if usage is not None else None
+    total_tokens = prompt_tokens + completion_tokens if completion_tokens is not None else None
+    usage = {
+        "prompt_tokens": prompt_tokens,  # can grab from usage dict, but it's usually wrong (set to 0)
+        "completion_tokens": completion_tokens,
+        "total_tokens": total_tokens,
+    }
+
+    return result, usage

letta/local_llm/json_parser.py

@@ -0,0 +1,202 @@
+import json
+import re
+
+from letta.errors import LLMJSONParsingError
+from letta.utils import json_loads
+
+
+def clean_json_string_extra_backslash(s):
+    """Clean extra backslashes out from stringified JSON
+
+    NOTE: Google AI Gemini API likes to include these
+    """
+    # Strip slashes that are used to escape single quotes and other backslashes
+    # Use json.loads to parse it correctly
+    while "\\\\" in s:
+        s = s.replace("\\\\", "\\")
+    return s
+
+
+def replace_escaped_underscores(string: str):
+    r"""Handles the case of escaped underscores, e.g.:
+
+    {
+      "function":"send\_message",
+      "params": {
+        "inner\_thoughts": "User is asking for information about themselves. Retrieving data from core memory.",
+        "message": "I know that you are Chad. Is there something specific you would like to know or talk about regarding yourself?"
+    """
+    return string.replace(r"\_", "_")
+
+
+def extract_first_json(string: str):
+    """Handles the case of two JSON objects back-to-back"""
+    from letta.utils import printd
+
+    depth = 0
+    start_index = None
+
+    for i, char in enumerate(string):
+        if char == "{":
+            if depth == 0:
+                start_index = i
+            depth += 1
+        elif char == "}":
+            depth -= 1
+            if depth == 0 and start_index is not None:
+                try:
+                    return json_loads(string[start_index : i + 1])
+                except json.JSONDecodeError as e:
+                    raise LLMJSONParsingError(f"Matched closing bracket, but decode failed with error: {str(e)}")
+    printd("No valid JSON object found.")
+    raise LLMJSONParsingError("Couldn't find starting bracket")
+
+
+def add_missing_heartbeat(llm_json):
+    """Manually insert heartbeat requests into messages that should have them
+
+    Use the following heuristic:
+      - if (function call is not send_message && prev message['role'] == user): insert heartbeat
+
+    Basically, if Letta is calling a function (not send_message) immediately after the user sending a message,
+    it probably is a retriever or insertion call, in which case we likely want to eventually reply with send_message
+
+            "message" = {
+            "role": "assistant",
+            "content": ...,
+            "function_call": {
+                "name": ...
+                "arguments": {
+                    "arg1": val1,
+                    ...
+                }
+            }
+        }
+    """
+    raise NotImplementedError
+
+
+def clean_and_interpret_send_message_json(json_string):
+    # If normal parsing fails, attempt to clean and extract manually
+    cleaned_json_string = re.sub(r"[^\x00-\x7F]+", "", json_string)  # Remove non-ASCII characters
+    function_match = re.search(r'"function":\s*"send_message"', cleaned_json_string)
+    inner_thoughts_match = re.search(r'"inner_thoughts":\s*"([^"]+)"', cleaned_json_string)
+    message_match = re.search(r'"message":\s*"([^"]+)"', cleaned_json_string)
+
+    if function_match and inner_thoughts_match and message_match:
+        return {
+            "function": "send_message",
+            "params": {
+                "inner_thoughts": inner_thoughts_match.group(1),
+                "message": message_match.group(1),
+            },
+        }
+    else:
+        raise LLMJSONParsingError(f"Couldn't manually extract send_message pattern from:\n{json_string}")
+
+
+def repair_json_string(json_string):
+    """
+    This function repairs a JSON string where line feeds were accidentally added
+    within string literals. The line feeds are replaced with the escaped line
+    feed sequence '\\n'.
+    """
+    new_string = ""
+    in_string = False
+    escape = False
+
+    for char in json_string:
+        if char == '"' and not escape:
+            in_string = not in_string
+        if char == "\\" and not escape:
+            escape = True
+        else:
+            escape = False
+        if char == "\n" and in_string:
+            new_string += "\\n"
+        else:
+            new_string += char
+
+    return new_string
+
+
+def repair_even_worse_json(json_string):
+    """
+    This function repairs a malformed JSON string where string literals are broken up and
+    not properly enclosed in quotes. It aims to consolidate everything between 'message': and
+    the two ending curly braces into one string for the 'message' field.
+    """
+    # State flags
+    in_message = False
+    in_string = False
+    escape = False
+    message_content = []
+
+    # Storage for the new JSON
+    new_json_parts = []
+
+    # Iterating through each character
+    for char in json_string:
+        if char == '"' and not escape:
+            in_string = not in_string
+            if not in_message:
+                # If we encounter a quote and are not in message, append normally
+                new_json_parts.append(char)
+        elif char == "\\" and not escape:
+            escape = True
+            new_json_parts.append(char)
+        else:
+            if escape:
+                escape = False
+            if in_message:
+                if char == "}":
+                    # Append the consolidated message and the closing characters then reset the flag
+                    new_json_parts.append('"{}"'.format("".join(message_content).replace("\n", " ")))
+                    new_json_parts.append(char)
+                    in_message = False
+                elif in_string or char.isalnum() or char.isspace() or char in ".',;:!":
+                    # Collect the message content, excluding structural characters
+                    message_content.append(char)
+            else:
+                # If we're not in message mode, append character to the output as is
+                new_json_parts.append(char)
+                if '"message":' in "".join(new_json_parts[-10:]):
+                    # If we detect "message": pattern, switch to message mode
+                    in_message = True
+                    message_content = []
+
+    # Joining everything to form the new JSON
+    repaired_json = "".join(new_json_parts)
+    return repaired_json
+
+
+def clean_json(raw_llm_output, messages=None, functions=None):
+    from letta.utils import printd
+
+    strategies = [
+        lambda output: json_loads(output),
+        lambda output: json_loads(output + "}"),
+        lambda output: json_loads(output + "}}"),
+        lambda output: json_loads(output + '"}}'),
+        # with strip and strip comma
+        lambda output: json_loads(output.strip().rstrip(",") + "}"),
+        lambda output: json_loads(output.strip().rstrip(",") + "}}"),
+        lambda output: json_loads(output.strip().rstrip(",") + '"}}'),
+        # more complex patchers
+        lambda output: json_loads(repair_json_string(output)),
+        lambda output: json_loads(repair_even_worse_json(output)),
+        lambda output: extract_first_json(output + "}}"),
+        lambda output: clean_and_interpret_send_message_json(output),
+        # replace underscores
+        lambda output: json_loads(replace_escaped_underscores(output)),
+        lambda output: extract_first_json(replace_escaped_underscores(output) + "}}"),
+    ]
+
+    for strategy in strategies:
+        try:
+            printd(f"Trying strategy: {strategy.__name__}")
+            return strategy(raw_llm_output)
+        except (json.JSONDecodeError, LLMJSONParsingError) as e:
+            printd(f"Strategy {strategy.__name__} failed with error: {e}")
+
+    raise LLMJSONParsingError(f"Failed to decode valid Letta JSON from LLM output:\n=====\n{raw_llm_output}\n=====")

letta/local_llm/koboldcpp/api.py

@@ -0,0 +1,62 @@
+from urllib.parse import urljoin
+
+from letta.local_llm.settings.settings import get_completions_settings
+from letta.local_llm.utils import count_tokens, post_json_auth_request
+
+KOBOLDCPP_API_SUFFIX = "/api/v1/generate"
+
+
+def get_koboldcpp_completion(endpoint, auth_type, auth_key, prompt, context_window, grammar=None):
+    """See https://lite.koboldai.net/koboldcpp_api for API spec"""
+    from letta.utils import printd
+
+    prompt_tokens = count_tokens(prompt)
+    if prompt_tokens > context_window:
+        raise Exception(f"Request exceeds maximum context length ({prompt_tokens} > {context_window} tokens)")
+
+    # Settings for the generation, includes the prompt + stop tokens, max length, etc
+    settings = get_completions_settings()
+    request = settings
+    request["prompt"] = prompt
+    request["max_context_length"] = context_window
+    request["max_length"] = 400  # if we don't set this, it'll default to 100 which is quite short
+
+    # Set grammar
+    if grammar is not None:
+        request["grammar"] = grammar
+
+    if not endpoint.startswith(("http://", "https://")):
+        raise ValueError(f"Provided OPENAI_API_BASE value ({endpoint}) must begin with http:// or https://")
+
+    try:
+        # NOTE: llama.cpp server returns the following when it's out of context
+        # curl: (52) Empty reply from server
+        URI = urljoin(endpoint.strip("/") + "/", KOBOLDCPP_API_SUFFIX.strip("/"))
+        response = post_json_auth_request(uri=URI, json_payload=request, auth_type=auth_type, auth_key=auth_key)
+        if response.status_code == 200:
+            result_full = response.json()
+            printd(f"JSON API response:\n{result_full}")
+            result = result_full["results"][0]["text"]
+        else:
+            raise Exception(
+                f"API call got non-200 response code (code={response.status_code}, msg={response.text}) for address: {URI}."
+                + f" Make sure that the koboldcpp server is running and reachable at {URI}."
+            )
+
+    except:
+        # TODO handle gracefully
+        raise
+
+    # Pass usage statistics back to main thread
+    # These are used to compute memory warning messages
+    # KoboldCpp doesn't return anything?
+    # https://lite.koboldai.net/koboldcpp_api#/v1/post_v1_generate
+    completion_tokens = None
+    total_tokens = prompt_tokens + completion_tokens if completion_tokens is not None else None
+    usage = {
+        "prompt_tokens": prompt_tokens,
+        "completion_tokens": completion_tokens,
+        "total_tokens": total_tokens,
+    }
+
+    return result, usage

letta/local_llm/koboldcpp/settings.py

@@ -0,0 +1,23 @@
+# see https://lite.koboldai.net/koboldcpp_api#/v1/post_v1_generate
+SIMPLE = {
+    "stop_sequence": [
+        "\nUSER:",
+        "\nASSISTANT:",
+        "\nFUNCTION RETURN:",
+        "\nUSER",
+        "\nASSISTANT",
+        "\nFUNCTION RETURN",
+        "\nFUNCTION",
+        "\nFUNC",
+        "<|im_start|>",
+        "<|im_end|>",
+        "<|im_sep|>",
+        # '\n' +
+        # '</s>',
+        # '<|',
+        # '\n#',
+        # '\n\n\n',
+    ],
+    # "max_context_length": LLM_MAX_TOKENS,
+    "max_length": 512,
+}

letta/local_llm/llamacpp/api.py

@@ -0,0 +1,58 @@
+from urllib.parse import urljoin
+
+from letta.local_llm.settings.settings import get_completions_settings
+from letta.local_llm.utils import count_tokens, post_json_auth_request
+
+LLAMACPP_API_SUFFIX = "/completion"
+
+
+def get_llamacpp_completion(endpoint, auth_type, auth_key, prompt, context_window, grammar=None):
+    """See https://github.com/ggerganov/llama.cpp/blob/master/examples/server/README.md for instructions on how to run the LLM web server"""
+    from letta.utils import printd
+
+    prompt_tokens = count_tokens(prompt)
+    if prompt_tokens > context_window:
+        raise Exception(f"Request exceeds maximum context length ({prompt_tokens} > {context_window} tokens)")
+
+    # Settings for the generation, includes the prompt + stop tokens, max length, etc
+    settings = get_completions_settings()
+    request = settings
+    request["prompt"] = prompt
+
+    # Set grammar
+    if grammar is not None:
+        request["grammar"] = grammar
+
+    if not endpoint.startswith(("http://", "https://")):
+        raise ValueError(f"Provided OPENAI_API_BASE value ({endpoint}) must begin with http:// or https://")
+
+    try:
+        # NOTE: llama.cpp server returns the following when it's out of context
+        # curl: (52) Empty reply from server
+        URI = urljoin(endpoint.strip("/") + "/", LLAMACPP_API_SUFFIX.strip("/"))
+        response = post_json_auth_request(uri=URI, json_payload=request, auth_type=auth_type, auth_key=auth_key)
+        if response.status_code == 200:
+            result_full = response.json()
+            printd(f"JSON API response:\n{result_full}")
+            result = result_full["content"]
+        else:
+            raise Exception(
+                f"API call got non-200 response code (code={response.status_code}, msg={response.text}) for address: {URI}."
+                + f" Make sure that the llama.cpp server is running and reachable at {URI}."
+            )
+
+    except:
+        # TODO handle gracefully
+        raise
+
+    # Pass usage statistics back to main thread
+    # These are used to compute memory warning messages
+    completion_tokens = result_full.get("tokens_predicted", None)
+    total_tokens = prompt_tokens + completion_tokens if completion_tokens is not None else None
+    usage = {
+        "prompt_tokens": prompt_tokens,  # can grab from "tokens_evaluated", but it's usually wrong (set to 0)
+        "completion_tokens": completion_tokens,
+        "total_tokens": total_tokens,
+    }
+
+    return result, usage

letta/local_llm/llamacpp/settings.py

@@ -0,0 +1,22 @@
+# see https://github.com/ggerganov/llama.cpp/blob/master/examples/server/README.md#api-endpoints for options
+SIMPLE = {
+    "stop": [
+        "\nUSER:",
+        "\nASSISTANT:",
+        "\nFUNCTION RETURN:",
+        "\nUSER",
+        "\nASSISTANT",
+        "\nFUNCTION RETURN",
+        "\nFUNCTION",
+        "\nFUNC",
+        "<|im_start|>",
+        "<|im_end|>",
+        "<|im_sep|>",
+        # '\n' +
+        # '</s>',
+        # '<|',
+        # '\n#',
+        # '\n\n\n',
+    ],
+    # "n_predict": 3072,
+}