chatlas 0.9.1__py3-none-any.whl → 0.10.0__py3-none-any.whl

chatlas/__init__.py

@@ -1,20 +1,26 @@
 from . import types
-from ._anthropic import ChatAnthropic, ChatBedrockAnthropic
 from ._auto import ChatAuto
 from ._chat import Chat
 from ._content import ContentToolRequest, ContentToolResult, ContentToolResultImage
 from ._content_image import content_image_file, content_image_plot, content_image_url
 from ._content_pdf import content_pdf_file, content_pdf_url
-from ._databricks import ChatDatabricks
-from ._github import ChatGithub
-from ._google import ChatGoogle, ChatVertex
-from ._groq import ChatGroq
 from ._interpolate import interpolate, interpolate_file
-from ._ollama import ChatOllama
-from ._openai import ChatAzureOpenAI, ChatOpenAI
-from ._perplexity import ChatPerplexity
 from ._provider import Provider
-from ._snowflake import ChatSnowflake
+from ._provider_anthropic import ChatAnthropic, ChatBedrockAnthropic
+from ._provider_cloudflare import ChatCloudflare
+from ._provider_databricks import ChatDatabricks
+from ._provider_deepseek import ChatDeepSeek
+from ._provider_github import ChatGithub
+from ._provider_google import ChatGoogle, ChatVertex
+from ._provider_groq import ChatGroq
+from ._provider_huggingface import ChatHuggingFace
+from ._provider_mistral import ChatMistral
+from ._provider_ollama import ChatOllama
+from ._provider_openai import ChatAzureOpenAI, ChatOpenAI
+from ._provider_openrouter import ChatOpenRouter
+from ._provider_perplexity import ChatPerplexity
+from ._provider_portkey import ChatPortkey
+from ._provider_snowflake import ChatSnowflake
 from ._tokens import token_usage
 from ._tools import Tool, ToolRejectError
 from ._turn import Turn
@@ -28,14 +34,20 @@ __all__ = (
     "ChatAnthropic",
     "ChatAuto",
     "ChatBedrockAnthropic",
+    "ChatCloudflare",
     "ChatDatabricks",
+    "ChatDeepSeek",
     "ChatGithub",
     "ChatGoogle",
     "ChatGroq",
+    "ChatHuggingFace",
+    "ChatMistral",
     "ChatOllama",
     "ChatOpenAI",
+    "ChatOpenRouter",
     "ChatAzureOpenAI",
     "ChatPerplexity",
+    "ChatPortkey",
     "ChatSnowflake",
     "ChatVertex",
     "Chat",

chatlas/_auto.py

@@ -5,16 +5,16 @@ from typing import Callable, Literal, Optional
 
 import orjson
 
-from ._anthropic import ChatAnthropic, ChatBedrockAnthropic
 from ._chat import Chat
-from ._databricks import ChatDatabricks
-from ._github import ChatGithub
-from ._google import ChatGoogle, ChatVertex
-from ._groq import ChatGroq
-from ._ollama import ChatOllama
-from ._openai import ChatAzureOpenAI, ChatOpenAI
-from ._perplexity import ChatPerplexity
-from ._snowflake import ChatSnowflake
+from ._provider_anthropic import ChatAnthropic, ChatBedrockAnthropic
+from ._provider_databricks import ChatDatabricks
+from ._provider_github import ChatGithub
+from ._provider_google import ChatGoogle, ChatVertex
+from ._provider_groq import ChatGroq
+from ._provider_ollama import ChatOllama
+from ._provider_openai import ChatAzureOpenAI, ChatOpenAI
+from ._provider_perplexity import ChatPerplexity
+from ._provider_snowflake import ChatSnowflake
 
 AutoProviders = Literal[
     "anthropic",

chatlas/_chat.py

@@ -65,6 +65,7 @@ class TokensDict(TypedDict):
     role: Literal["user", "assistant"]
     tokens: int
     tokens_total: int
+    tokens_cached: int
 
 
 CompletionT = TypeVar("CompletionT")
@@ -293,12 +294,15 @@ class Chat(Generic[SubmitInputArgsT, CompletionT]):
             {
                 "role": "user",
                 "tokens": turns[1].tokens[0],
+                # Number of tokens currently cached (reduces input token usage)
+                "tokens_cached": turns[1].tokens[2],
                 "tokens_total": turns[1].tokens[0],
             },
             # The token count for the 1st assistant response
             {
                 "role": "assistant",
                 "tokens": turns[1].tokens[1],
+                "tokens_cached": 0,
                 "tokens_total": turns[1].tokens[1],
             },
         ]
@@ -319,8 +323,11 @@ class Chat(Generic[SubmitInputArgsT, CompletionT]):
                 [
                     {
                         "role": "user",
-                        # Implied token count for the user input
+                        # Implied new token count for the user input (input tokens - context - cached reads)
+                        # Cached reads are only subtracted for particular providers
                         "tokens": tj.tokens[0] - sum(ti.tokens),
+                        # Number of tokens currently cached (reduces input token usage depending on provider's API)
+                        "tokens_cached": tj.tokens[2],
                         # Total tokens = Total User Tokens for the Turn = Distinct new tokens + context sent
                         "tokens_total": tj.tokens[0],
                     },
@@ -329,6 +336,7 @@ class Chat(Generic[SubmitInputArgsT, CompletionT]):
                         # The token count for the assistant response
                         "tokens": tj.tokens[1],
                         # Total tokens = Total Assistant tokens used in the turn
+                        "tokens_cached": 0,
                         "tokens_total": tj.tokens[1],
                     },
                 ]
@@ -339,7 +347,7 @@ class Chat(Generic[SubmitInputArgsT, CompletionT]):
     def get_cost(
         self,
         options: Literal["all", "last"] = "all",
-        token_price: Optional[tuple[float, float]] = None,
+        token_price: Optional[tuple[float, float, float]] = None,
     ) -> float:
         """
         Estimate the cost of the chat.
@@ -357,11 +365,13 @@ class Chat(Generic[SubmitInputArgsT, CompletionT]):
               - `"last"`: Return the cost of the last turn in the chat.
         token_price
             An optional tuple in the format of (input_token_cost,
-            output_token_cost) for bringing your own cost information.
+            output_token_cost, cached_token_cost) for bringing your own cost information.
                  - `"input_token_cost"`: The cost per user token in USD per
                    million tokens.
                  - `"output_token_cost"`: The cost per assistant token in USD
                    per million tokens.
+                - `"cached_token_cost"`: The cost per cached token read in USD
+                   per million tokens.
 
         Returns
         -------
@@ -374,15 +384,19 @@ class Chat(Generic[SubmitInputArgsT, CompletionT]):
         if token_price:
             input_token_price = token_price[0] / 1e6
             output_token_price = token_price[1] / 1e6
+            cached_token_price = token_price[2] / 1e6
         else:
             price_token = get_token_pricing(self.provider.name, self.provider.model)
             if not price_token:
                 raise KeyError(
-                    f"We could not locate pricing information for model '{self.provider.model}' from provider '{self.provider.name}'. "
+                    f"We could not locate pricing information for model '{self.provider.model}'"
+                    f" from provider '{self.provider.name}'. "
                     "If you know the pricing for this model, specify it in `token_price`."
                 )
+
             input_token_price = price_token["input"] / 1e6
-            output_token_price = price_token["output"] / 1e6
+            output_token_price = price_token.get("output", 0) / 1e6
+            cached_token_price = price_token.get("cached_input", 0) / 1e6
 
         if len(turns_tokens) == 0:
             return 0.0
@@ -399,8 +413,16 @@ class Chat(Generic[SubmitInputArgsT, CompletionT]):
             user_tokens = sum(
                 u["tokens_total"] for u in turns_tokens if u["role"] == "user"
             )
-            cost = (asst_tokens * output_token_price) + (
-                user_tokens * input_token_price
+            # We add the cached tokens here because for relevant providers they have already been subtracted
+            # from the user tokens. This assumes the provider uses (reads) the cache each time.
+            cached_token_reads = sum(
+                u["tokens_cached"] for u in turns_tokens if u["role"] == "user"
+            )
+
+            cost = (
+                (asst_tokens * output_token_price)
+                + (user_tokens * input_token_price)
+                + (cached_token_reads * cached_token_price)
             )
             return cost
 
@@ -408,7 +430,9 @@ class Chat(Generic[SubmitInputArgsT, CompletionT]):
         if last_turn["role"] == "assistant":
             return last_turn["tokens"] * output_token_price
         if last_turn["role"] == "user":
-            return last_turn["tokens_total"] * input_token_price
+            return (last_turn["tokens_total"] * input_token_price) + (
+                last_turn["tokens_cached"] * cached_token_price
+            )
         raise ValueError(
             f"Expected last turn to have a role of 'user' or `'assistant'`, not '{last_turn['role']}'"
         )
@@ -2224,8 +2248,12 @@ class Chat(Generic[SubmitInputArgsT, CompletionT]):
         tokens = self.get_tokens()
         tokens_asst = sum(u["tokens_total"] for u in tokens if u["role"] == "assistant")
         tokens_user = sum(u["tokens_total"] for u in tokens if u["role"] == "user")
+        tokens_cached = sum(u["tokens_cached"] for u in tokens if u["role"] == "user")
 
-        res = f"<Chat {self.provider.name}/{self.provider.model} turns={len(turns)} tokens={tokens_user}/{tokens_asst}"
+        res = (
+            f"<Chat {self.provider.name}/{self.provider.model} turns={len(turns)}"
+            f" tokens={tokens_user + tokens_cached}/{tokens_asst}"
+        )
 
         # Add cost info only if we can compute it
         cost = compute_cost(
@@ -2233,6 +2261,7 @@ class Chat(Generic[SubmitInputArgsT, CompletionT]):
             self.provider.model,
             tokens_user,
             tokens_asst,
+            tokens_cached,
         )
         if cost is not None:
             res += f" ${round(cost, ndigits=2)}"

chatlas/{_anthropic.py → _provider_anthropic.py}

@@ -163,7 +163,7 @@ def ChatAnthropic(
     """
 
     if model is None:
-        model = log_model_default("claude-3-7-sonnet-latest")
+        model = log_model_default("claude-sonnet-4-0")
 
     return Chat(
         provider=AnthropicProvider(
@@ -586,7 +586,16 @@ class AnthropicProvider(
                         )
                     )
 
-        tokens = completion.usage.input_tokens, completion.usage.output_tokens
+        usage = completion.usage
+        # N.B. Currently, Anthropic doesn't cache by default and we currently do not support
+        # manual caching in chatlas. Note also that this only tracks reads, NOT writes, which
+        # have their own cost. To track that properly, we would need another caching category and per-token cost.
+
+        tokens = (
+            completion.usage.input_tokens,
+            completion.usage.output_tokens,
+            usage.cache_read_input_tokens if usage.cache_read_input_tokens else 0,
+        )
 
         tokens_log(self, tokens)
 
@@ -733,8 +742,7 @@ def ChatBedrockAnthropic(
     """
 
     if model is None:
-        # Default model from https://github.com/anthropics/anthropic-sdk-python?tab=readme-ov-file#aws-bedrock
-        model = log_model_default("anthropic.claude-3-5-sonnet-20241022-v2:0")
+        model = log_model_default("us.anthropic.claude-sonnet-4-20250514-v1:0")
 
     return Chat(
         provider=AnthropicBedrockProvider(
@@ -764,7 +772,7 @@ class AnthropicBedrockProvider(AnthropicProvider):
         aws_session_token: str | None,
         max_tokens: int = 4096,
         base_url: str | None,
-        name: str = "AnthropicBedrock",
+        name: str = "AWS/Bedrock",
         kwargs: Optional["ChatBedrockClientArgs"] = None,
     ):
         super().__init__(name=name, model=model, max_tokens=max_tokens)

chatlas/_provider_cloudflare.py

@@ -0,0 +1,165 @@
+from __future__ import annotations
+
+import os
+from typing import TYPE_CHECKING, Optional
+
+from ._chat import Chat
+from ._logging import log_model_default
+from ._provider_openai import OpenAIProvider
+from ._utils import MISSING, MISSING_TYPE, is_testing
+
+if TYPE_CHECKING:
+    from ._provider_openai import ChatCompletion
+    from .types.openai import ChatClientArgs, SubmitInputArgs
+
+
+def ChatCloudflare(
+    *,
+    account: Optional[str] = None,
+    system_prompt: Optional[str] = None,
+    model: Optional[str] = None,
+    api_key: Optional[str] = None,
+    seed: Optional[int] | MISSING_TYPE = MISSING,
+    kwargs: Optional["ChatClientArgs"] = None,
+) -> Chat["SubmitInputArgs", ChatCompletion]:
+    """
+    Chat with a model hosted on Cloudflare Workers AI.
+
+    Cloudflare Workers AI hosts a variety of open-source AI models.
+
+    Prerequisites
+    -------------
+
+    ::: {.callout-note}
+    ## API credentials
+
+    To use the Cloudflare API, you must have an Account ID and an Access Token,
+    which you can obtain by following the instructions at
+    <https://developers.cloudflare.com/workers-ai/get-started/rest-api/>.
+    :::
+
+    Examples
+    --------
+
+    ```python
+    import os
+    from chatlas import ChatCloudflare
+
+    chat = ChatCloudflare(
+        api_key=os.getenv("CLOUDFLARE_API_KEY"),
+        account=os.getenv("CLOUDFLARE_ACCOUNT_ID"),
+    )
+    chat.chat("What is the capital of France?")
+    ```
+
+    Known limitations
+    -----------------
+
+    - Tool calling does not appear to work.
+    - Images don't appear to work.
+
+    Parameters
+    ----------
+    account
+        The Cloudflare account ID. You generally should not supply this directly,
+        but instead set the `CLOUDFLARE_ACCOUNT_ID` environment variable.
+    system_prompt
+        A system prompt to set the behavior of the assistant.
+    model
+        The model to use for the chat. The default, None, will pick a reasonable
+        default, and warn you about it. We strongly recommend explicitly choosing
+        a model for all but the most casual use.
+    api_key
+        The API key to use for authentication. You generally should not supply
+        this directly, but instead set the `CLOUDFLARE_API_KEY` environment
+        variable.
+    seed
+        Optional integer seed that ChatGPT uses to try and make output more
+        reproducible.
+    kwargs
+        Additional arguments to pass to the `openai.OpenAI()` client constructor.
+
+    Returns
+    -------
+    Chat
+        A chat object that retains the state of the conversation.
+
+    Note
+    ----
+    This function is a lightweight wrapper around [](`~chatlas.ChatOpenAI`) with
+    the defaults tweaked for Cloudflare.
+
+    Note
+    ----
+    Pasting credentials into a chat constructor (e.g.,
+    `ChatCloudflare(api_key="...", account="...")`) is the simplest way to get
+    started, and is fine for interactive use, but is problematic for code that
+    may be shared with others.
+
+    Instead, consider using environment variables or a configuration file to manage
+    your credentials. One popular way to manage credentials is to use a `.env` file
+    to store your credentials, and then use the `python-dotenv` package to load them
+    into your environment.
+
+    ```shell
+    pip install python-dotenv
+    ```
+
+    ```shell
+    # .env
+    CLOUDFLARE_API_KEY=...
+    CLOUDFLARE_ACCOUNT_ID=...
+    ```
+
+    ```python
+    from chatlas import ChatCloudflare
+    from dotenv import load_dotenv
+
+    load_dotenv()
+    chat = ChatCloudflare()
+    chat.console()
+    ```
+
+    Another, more general, solution is to load your environment variables into the shell
+    before starting Python (maybe in a `.bashrc`, `.zshrc`, etc. file):
+
+    ```shell
+    export CLOUDFLARE_API_KEY=...
+    export CLOUDFLARE_ACCOUNT_ID=...
+    ```
+    """
+    # List at https://developers.cloudflare.com/workers-ai/models/
+    # `@cf` appears to be part of the model name
+    if model is None:
+        model = log_model_default("@cf/meta/llama-3.3-70b-instruct-fp8-fast")
+
+    if api_key is None:
+        api_key = os.getenv("CLOUDFLARE_API_KEY")
+
+    if account is None:
+        account = os.getenv("CLOUDFLARE_ACCOUNT_ID")
+
+    if account is None:
+        raise ValueError(
+            "Cloudflare account ID is required. Set the CLOUDFLARE_ACCOUNT_ID "
+            "environment variable or pass the `account` parameter."
+        )
+
+    if isinstance(seed, MISSING_TYPE):
+        seed = 1014 if is_testing() else None
+
+    # https://developers.cloudflare.com/workers-ai/configuration/open-ai-compatibility/
+    cloudflare_api = "https://api.cloudflare.com/client/v4/accounts"
+    base_url = f"{cloudflare_api}/{account}/ai/v1/"
+
+    return Chat(
+        provider=OpenAIProvider(
+            api_key=api_key,
+            model=model,
+            base_url=base_url,
+            seed=seed,
+            name="Cloudflare",
+            kwargs=kwargs,
+        ),
+        system_prompt=system_prompt,
+    )

chatlas/{_databricks.py → _provider_databricks.py}

@@ -4,12 +4,12 @@ from typing import TYPE_CHECKING, Optional
 
 from ._chat import Chat
 from ._logging import log_model_default
-from ._openai import OpenAIProvider
+from ._provider_openai import OpenAIProvider
 
 if TYPE_CHECKING:
     from databricks.sdk import WorkspaceClient
 
-    from ._openai import ChatCompletion
+    from ._provider_openai import ChatCompletion
     from .types.openai import SubmitInputArgs
 
 
@@ -127,3 +127,14 @@ class DatabricksProvider(OpenAIProvider):
             api_key="no-token",  # A placeholder to pass validations, this will not be used
             http_client=httpx.AsyncClient(auth=client._client.auth),
         )
+
+    # Databricks doesn't support stream_options
+    def _chat_perform_args(
+        self, stream, turns, tools, data_model=None, kwargs=None
+    ) -> "SubmitInputArgs":
+        kwargs2 = super()._chat_perform_args(stream, turns, tools, data_model, kwargs)
+
+        if "stream_options" in kwargs2:
+            del kwargs2["stream_options"]
+
+        return kwargs2

chatlas/_provider_deepseek.py

@@ -0,0 +1,171 @@
+from __future__ import annotations
+
+import os
+from typing import TYPE_CHECKING, Optional, cast
+
+from ._chat import Chat
+from ._logging import log_model_default
+from ._provider_openai import OpenAIProvider
+from ._turn import Turn
+from ._utils import MISSING, MISSING_TYPE, is_testing
+
+if TYPE_CHECKING:
+    from openai.types.chat import ChatCompletion, ChatCompletionMessageParam
+
+    from .types.openai import ChatClientArgs, SubmitInputArgs
+
+
+def ChatDeepSeek(
+    *,
+    system_prompt: Optional[str] = None,
+    model: Optional[str] = None,
+    api_key: Optional[str] = None,
+    base_url: str = "https://api.deepseek.com",
+    seed: Optional[int] | MISSING_TYPE = MISSING,
+    kwargs: Optional["ChatClientArgs"] = None,
+) -> Chat["SubmitInputArgs", ChatCompletion]:
+    """
+    Chat with a model hosted on DeepSeek.
+
+    DeepSeek is a platform for AI inference with competitive pricing
+    and performance.
+
+    Prerequisites
+    -------------
+
+    ::: {.callout-note}
+    ## API key
+
+    Sign up at <https://platform.deepseek.com> to get an API key.
+    :::
+
+    Examples
+    --------
+
+    ```python
+    import os
+    from chatlas import ChatDeepSeek
+
+    chat = ChatDeepSeek(api_key=os.getenv("DEEPSEEK_API_KEY"))
+    chat.chat("What is the capital of France?")
+    ```
+
+    Known limitations
+    --------------
+
+    * Structured data extraction is not supported.
+    * Images are not supported.
+
+    Parameters
+    ----------
+    system_prompt
+        A system prompt to set the behavior of the assistant.
+    model
+        The model to use for the chat. The default, None, will pick a reasonable
+        default, and warn you about it. We strongly recommend explicitly choosing
+        a model for all but the most casual use.
+    api_key
+        The API key to use for authentication. You generally should not supply
+        this directly, but instead set the `DEEPSEEK_API_KEY` environment variable.
+    base_url
+        The base URL to the endpoint; the default uses DeepSeek's API.
+    seed
+        Optional integer seed that DeepSeek uses to try and make output more
+        reproducible.
+    kwargs
+        Additional arguments to pass to the `openai.OpenAI()` client constructor.
+
+    Returns
+    -------
+    Chat
+        A chat object that retains the state of the conversation.
+
+    Note
+    ----
+    This function is a lightweight wrapper around [](`~chatlas.ChatOpenAI`) with
+    the defaults tweaked for DeepSeek.
+
+    Note
+    ----
+    Pasting an API key into a chat constructor (e.g., `ChatDeepSeek(api_key="...")`)
+    is the simplest way to get started, and is fine for interactive use, but is
+    problematic for code that may be shared with others.
+
+    Instead, consider using environment variables or a configuration file to manage
+    your credentials. One popular way to manage credentials is to use a `.env` file
+    to store your credentials, and then use the `python-dotenv` package to load them
+    into your environment.
+
+    ```shell
+    pip install python-dotenv
+    ```
+
+    ```shell
+    # .env
+    DEEPSEEK_API_KEY=...
+    ```
+
+    ```python
+    from chatlas import ChatDeepSeek
+    from dotenv import load_dotenv
+
+    load_dotenv()
+    chat = ChatDeepSeek()
+    chat.console()
+    ```
+
+    Another, more general, solution is to load your environment variables into the shell
+    before starting Python (maybe in a `.bashrc`, `.zshrc`, etc. file):
+
+    ```shell
+    export DEEPSEEK_API_KEY=...
+    ```
+    """
+    if model is None:
+        model = log_model_default("deepseek-chat")
+
+    if api_key is None:
+        api_key = os.getenv("DEEPSEEK_API_KEY")
+
+    if isinstance(seed, MISSING_TYPE):
+        seed = 1014 if is_testing() else None
+
+    return Chat(
+        provider=DeepSeekProvider(
+            api_key=api_key,
+            model=model,
+            base_url=base_url,
+            seed=seed,
+            name="DeepSeek",
+            kwargs=kwargs,
+        ),
+        system_prompt=system_prompt,
+    )
+
+
+class DeepSeekProvider(OpenAIProvider):
+    @staticmethod
+    def _as_message_param(turns: list[Turn]) -> list["ChatCompletionMessageParam"]:
+        from openai.types.chat import (
+            ChatCompletionAssistantMessageParam,
+            ChatCompletionUserMessageParam,
+        )
+
+        params = OpenAIProvider._as_message_param(turns)
+
+        # Content must be a string
+        for i, param in enumerate(params):
+            if param["role"] in ["assistant", "user"]:
+                param = cast(
+                    ChatCompletionAssistantMessageParam
+                    | ChatCompletionUserMessageParam,
+                    param,
+                )
+                contents = param.get("content", None)
+                if not isinstance(contents, list):
+                    continue
+                params[i]["content"] = "".join(
+                    content.get("text", "") for content in contents
+                )
+
+        return params

chatlas/{_github.py → _provider_github.py}

@@ -5,11 +5,11 @@ from typing import TYPE_CHECKING, Optional
 
 from ._chat import Chat
 from ._logging import log_model_default
-from ._openai import OpenAIProvider
+from ._provider_openai import OpenAIProvider
 from ._utils import MISSING, MISSING_TYPE, is_testing
 
 if TYPE_CHECKING:
-    from ._openai import ChatCompletion
+    from ._provider_openai import ChatCompletion
     from .types.openai import ChatClientArgs, SubmitInputArgs
 
 

chatlas/{_google.py → _provider_google.py}

@@ -426,9 +426,7 @@ class GoogleProvider(
                 )
             )
         elif isinstance(content, ContentToolResult):
-            if isinstance(
-                content, (ContentToolResultImage, ContentToolResultResource)
-            ):
+            if isinstance(content, (ContentToolResultImage, ContentToolResultResource)):
                 raise NotImplementedError(
                     "Tool results with images or resources aren't supported by Google (Gemini). "
                 )
@@ -507,11 +505,13 @@ class GoogleProvider(
                     )
 
         usage = message.get("usage_metadata")
-        tokens = (0, 0)
+        tokens = (0, 0, 0)
         if usage:
+            cached = usage.get("cached_content_token_count") or 0
             tokens = (
-                usage.get("prompt_token_count") or 0,
+                (usage.get("prompt_token_count") or 0) - cached,
                 usage.get("candidates_token_count") or 0,
+                usage.get("cached_content_token_count") or 0,
             )
 
         tokens_log(self, tokens)