mistral_common 1.0.0__tar.gz

mistral_common-1.0.0/PKG-INFO

@@ -0,0 +1,110 @@
+Metadata-Version: 2.1
+Name: mistral_common
+Version: 1.0.0
+Summary: 
+Author: bam4d
+Author-email: bam4d@mistral.ai
+Requires-Python: >=3.8.10,<4.0.0
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.9
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Requires-Dist: jsonschema (==4.21.1)
+Requires-Dist: pydantic (==2.6.1)
+Requires-Dist: sentencepiece (==0.1.99)
+Requires-Dist: typing-extensions (>=4.11.0,<5.0.0)
+Description-Content-Type: text/markdown
+
+# Mistral Common
+
+## What is it? 
+mistral-common is a set of tools to help you work with Mistral models. 
+
+Our first release contains tokenization. Our tokenizers go beyond  the usual text <-> tokens, adding parsing of tools and structured conversation. We also release the validation and normalization code that is used in our API. 
+
+We are releasing three versions of our tokenizer powering different sets of models. 
+
+- v1: open-mistral-7b, open-mixtral-8x7b, mistral-embed
+- v2: mistral-small-latest, mistral-large-latest
+- v3: open-mixtral-8x22b
+
+## Installation 
+
+### pip 
+You can install `mistral-common` via pip: 
+```
+pip install mistral-common
+```
+
+### From Source
+Alternatively, you can install from source directly. This repo uses poetry as a dependency and virtual environment manager.
+
+You can install poetry with
+```
+pip install poetry
+```
+
+poetry will set up a virtual environment and install dependencies with the following command:
+```
+poetry install
+```
+
+## Examples 
+<a target="_blank" href="https://colab.research.google.com/github/mistralai/mistral-common/blob/main/examples/tokenizer.ipynb">
+  <img src="https://colab.research.google.com/assets/colab-badge.svg" alt="Open In Colab"/>
+</a>
+
+
+
+```py
+# Import needed packages:
+from mistral_common.protocol.instruct.messages import UserMessage
+from mistral_common.protocol.instruct.request import ChatCompletionRequest
+from mistral_common.protocol.instruct.tool_calls import Function, Tool
+from mistral_common.tokens.tokenizers.mistral import MistralTokenizer
+
+# Load Mistral tokenizer
+
+model_name = "open-mixtral-8x22B"
+
+tokenizer = MistralTokenizer.from_model(model_name)
+
+# Tokenize a list of messages
+tokenized = tokenizer.encode_chat_completion(
+    ChatCompletionRequest(
+        tools=[
+            Tool(
+                function=Function(
+                    name="get_current_weather",
+                    description="Get the current weather",
+                    parameters={
+                        "type": "object",
+                        "properties": {
+                            "location": {
+                                "type": "string",
+                                "description": "The city and state, e.g. San Francisco, CA",
+                            },
+                            "format": {
+                                "type": "string",
+                                "enum": ["celsius", "fahrenheit"],
+                                "description": "The temperature unit to use. Infer this from the users location.",
+                            },
+                        },
+                        "required": ["location", "format"],
+                    },
+                )
+            )
+        ],
+        messages=[
+            UserMessage(content="What's the weather like today in Paris"),
+        ],
+        model=model_name,
+    )
+)
+tokens, text = tokenized.tokens, tokenized.text
+
+# Count the number of tokens
+print(len(tokens))
+```
+

mistral_common-1.0.0/README.md

@@ -0,0 +1,91 @@
+# Mistral Common
+
+## What is it? 
+mistral-common is a set of tools to help you work with Mistral models. 
+
+Our first release contains tokenization. Our tokenizers go beyond  the usual text <-> tokens, adding parsing of tools and structured conversation. We also release the validation and normalization code that is used in our API. 
+
+We are releasing three versions of our tokenizer powering different sets of models. 
+
+- v1: open-mistral-7b, open-mixtral-8x7b, mistral-embed
+- v2: mistral-small-latest, mistral-large-latest
+- v3: open-mixtral-8x22b
+
+## Installation 
+
+### pip 
+You can install `mistral-common` via pip: 
+```
+pip install mistral-common
+```
+
+### From Source
+Alternatively, you can install from source directly. This repo uses poetry as a dependency and virtual environment manager.
+
+You can install poetry with
+```
+pip install poetry
+```
+
+poetry will set up a virtual environment and install dependencies with the following command:
+```
+poetry install
+```
+
+## Examples 
+<a target="_blank" href="https://colab.research.google.com/github/mistralai/mistral-common/blob/main/examples/tokenizer.ipynb">
+  <img src="https://colab.research.google.com/assets/colab-badge.svg" alt="Open In Colab"/>
+</a>
+
+
+
+```py
+# Import needed packages:
+from mistral_common.protocol.instruct.messages import UserMessage
+from mistral_common.protocol.instruct.request import ChatCompletionRequest
+from mistral_common.protocol.instruct.tool_calls import Function, Tool
+from mistral_common.tokens.tokenizers.mistral import MistralTokenizer
+
+# Load Mistral tokenizer
+
+model_name = "open-mixtral-8x22B"
+
+tokenizer = MistralTokenizer.from_model(model_name)
+
+# Tokenize a list of messages
+tokenized = tokenizer.encode_chat_completion(
+    ChatCompletionRequest(
+        tools=[
+            Tool(
+                function=Function(
+                    name="get_current_weather",
+                    description="Get the current weather",
+                    parameters={
+                        "type": "object",
+                        "properties": {
+                            "location": {
+                                "type": "string",
+                                "description": "The city and state, e.g. San Francisco, CA",
+                            },
+                            "format": {
+                                "type": "string",
+                                "enum": ["celsius", "fahrenheit"],
+                                "description": "The temperature unit to use. Infer this from the users location.",
+                            },
+                        },
+                        "required": ["location", "format"],
+                    },
+                )
+            )
+        ],
+        messages=[
+            UserMessage(content="What's the weather like today in Paris"),
+        ],
+        model=model_name,
+    )
+)
+tokens, text = tokenized.tokens, tokenized.text
+
+# Count the number of tokens
+print(len(tokens))
+```

mistral_common-1.0.0/pyproject.toml

@@ -0,0 +1,58 @@
+[tool.poetry]
+name = "mistral_common"
+version = "v1.0.0"
+description = ""
+authors = ["bam4d <bam4d@mistral.ai>"]
+readme = "README.md"
+packages = [{ include = "mistral_common", from = "src" }]
+
+[tool.ruff]
+lint.select = ["E", "F", "W", "Q", "I"]
+lint.ignore = ["E203"]
+lint.fixable = ["ALL"]
+lint.unfixable = []
+line-length = 120
+exclude = ["docs", "build"]
+
+[tool.mypy]
+disallow_untyped_defs = true
+show_error_codes = true
+no_implicit_optional = true
+warn_return_any = true
+warn_unused_ignores = true
+exclude = ["docs", "tools", "build"]
+
+[[tool.mypy.overrides]]
+module = "sentencepiece.*"
+ignore_missing_imports = true
+
+[tool.poetry.dependencies]
+python = "^3.8.10"
+pydantic = "2.6.1"
+jsonschema = "4.21.1"
+sentencepiece = "0.1.99"
+typing-extensions = "^4.11.0"
+
+[tool.poetry.group.dev.dependencies]
+types-jsonschema = "4.21.0.20240118"
+types-protobuf = "4.24.0.20240129"
+mypy-protobuf = "^3.5.0"
+pytest = "7.4.4"
+ruff = "^0.2.2"
+mypy = "^1.8.0"
+pytest-cov = "^4.1.0"
+diff-cover = "^8.0.3"
+coverage = {extras = ["toml"], version = "^7.4.4"}
+
+[build-system]
+requires = ["poetry-core"]
+build-backend = "poetry.core.masonry.api"
+
+[tool.pytest.ini_options]
+testpaths = ["./tests"]
+
+[tool.coverage.run]
+omit = ["tests", "*src/mistral_common/data*"]
+
+[tool.coverage.report]
+skip_covered = true

mistral_common-1.0.0/src/mistral_common/base.py

@@ -0,0 +1,9 @@
+from pydantic import BaseModel, ConfigDict
+
+
+class MistralBase(BaseModel):
+    """
+    Base class for all Mistral Pydantic models.
+    """
+
+    model_config = ConfigDict(extra="forbid", strict=True, validate_default=True, use_enum_values=True)

mistral_common-1.0.0/src/mistral_common/exceptions.py

@@ -0,0 +1,67 @@
+from typing import Optional
+
+
+class MistralCommonException(Exception):
+    message: str = "Internal server error"
+
+    def __init__(
+        self,
+        message: Optional[str] = None,
+    ) -> None:
+        if message:
+            self.message = message
+
+
+class TokenizerException(MistralCommonException):
+    def __init__(self, message: str) -> None:
+        super().__init__(message)
+
+
+class UnsupportedTokenizerFeatureException(MistralCommonException):
+    def __init__(self, message: str) -> None:
+        super().__init__(message)
+
+
+class InvalidRequestException(MistralCommonException):
+    def __init__(self, message: str) -> None:
+        super().__init__(message)
+
+
+class InvalidSystemPromptException(MistralCommonException):
+    def __init__(self, message: str) -> None:
+        super().__init__(message)
+
+
+class InvalidMessageStructureException(MistralCommonException):
+    def __init__(self, message: str) -> None:
+        super().__init__(message)
+
+
+class InvalidAssistantMessageException(MistralCommonException):
+    def __init__(self, message: str) -> None:
+        super().__init__(message)
+
+
+class InvalidToolMessageException(MistralCommonException):
+    def __init__(self, message: str) -> None:
+        super().__init__(message)
+
+
+class InvalidToolSchemaException(MistralCommonException):
+    def __init__(self, message: str) -> None:
+        super().__init__(message)
+
+
+class InvalidUserMessageException(MistralCommonException):
+    def __init__(self, message: str) -> None:
+        super().__init__(message)
+
+
+class InvalidFunctionCallException(MistralCommonException):
+    def __init__(self, message: str) -> None:
+        super().__init__(message)
+
+
+class InvalidToolException(MistralCommonException):
+    def __init__(self, message: str) -> None:
+        super().__init__(message)

mistral_common-1.0.0/src/mistral_common/protocol/base.py

@@ -0,0 +1,9 @@
+from typing import Optional
+
+from mistral_common.base import MistralBase
+
+
+class UsageInfo(MistralBase):
+    prompt_tokens: int = 0
+    total_tokens: int = 0
+    completion_tokens: Optional[int] = 0

mistral_common-1.0.0/src/mistral_common/protocol/embedding/request.py

@@ -0,0 +1,10 @@
+from typing import List, Optional, Union
+
+from mistral_common.base import MistralBase
+from pydantic import Field
+
+
+class EmbeddingRequest(MistralBase):
+    input: Union[str, List[str]] = Field(description="Text to embed.")
+    model: str = Field(description="ID of the model to use.")
+    encoding_format: Optional[str] = Field(default="float", description="The format to return the embeddings in.")

mistral_common-1.0.0/src/mistral_common/protocol/embedding/response.py

@@ -0,0 +1,20 @@
+from typing import List
+
+from mistral_common.base import MistralBase
+from mistral_common.protocol.base import UsageInfo
+from mistral_common.protocol.utils import random_uuid
+from pydantic import Field
+
+
+class EmbeddingObject(MistralBase):
+    object: str = Field(default="embedding", description="The type of the object returned.")
+    embedding: List[float] = Field(description="The type of the object returned.")
+    index: int = Field(description="The index of the embedding in the input text.")
+
+
+class EmbeddingResponse(MistralBase):
+    id: str = Field(default_factory=lambda: f"embd-{random_uuid()}")
+    object: str = Field(default="list", description="The type of the object returned.")
+    data: List[EmbeddingObject] = Field(description="List of embeddings.")
+    model: str = Field(description="The model used to generate the embeddings.")
+    usage: UsageInfo

mistral_common-1.0.0/src/mistral_common/protocol/instruct/messages.py

@@ -0,0 +1,56 @@
+from enum import Enum
+from typing import List, Literal, Optional, Union
+
+from pydantic import Field
+from typing_extensions import Annotated  # compatibility with 3.8
+
+from mistral_common.base import MistralBase
+from mistral_common.protocol.instruct.tool_calls import ToolCall
+
+
+class Roles(str, Enum):
+    system = "system"
+    user = "user"
+    assistant = "assistant"
+    tool = "tool"
+
+
+class ChunkTypes(str, Enum):
+    text = "text"
+
+
+class ContentChunk(MistralBase):
+    type: ChunkTypes = ChunkTypes.text
+    text: str
+
+
+class BaseMessage(MistralBase):
+    role: Literal[Roles.system, Roles.user, Roles.assistant, Roles.tool]
+
+
+class UserMessage(BaseMessage):
+    role: Literal[Roles.user] = Roles.user
+    content: Union[str, List[ContentChunk]]
+
+
+class SystemMessage(BaseMessage):
+    role: Literal[Roles.system] = Roles.system
+    content: Union[str, List[ContentChunk]]
+
+
+class AssistantMessage(BaseMessage):
+    role: Literal[Roles.assistant] = Roles.assistant
+    content: Optional[str] = None
+    tool_calls: Optional[List[ToolCall]] = None
+
+
+class ToolMessage(BaseMessage):
+    content: str
+    role: Literal[Roles.tool] = Roles.tool
+    tool_call_id: Optional[str] = None
+
+    # Deprecated in V3 tokenization
+    name: Optional[str] = None
+
+
+ChatMessage = Annotated[Union[SystemMessage, UserMessage, AssistantMessage, ToolMessage], Field(discriminator="role")]

mistral_common-1.0.0/src/mistral_common/protocol/instruct/request.py

@@ -0,0 +1,29 @@
+from enum import Enum
+from typing import List, Optional
+
+from pydantic import Field
+
+from mistral_common.base import MistralBase
+from mistral_common.protocol.instruct.messages import ChatMessage
+from mistral_common.protocol.instruct.tool_calls import Tool, ToolChoice
+
+
+class ResponseFormats(str, Enum):
+    text: str = "text"
+    json: str = "json_object"
+
+
+class ResponseFormat(MistralBase):
+    type: ResponseFormats = ResponseFormats.text
+
+
+class ChatCompletionRequest(MistralBase):
+    model: Optional[str] = None
+    messages: List[ChatMessage]
+    response_format: ResponseFormat = Field(default_factory=ResponseFormat)
+    tools: List[Tool] = Field(default_factory=list)
+    tool_choice: ToolChoice = ToolChoice.auto
+    temperature: float = Field(default=0.7, ge=0.0, le=1.0)
+    top_p: float = Field(default=1.0, ge=0.0, le=1.0)
+    max_tokens: Optional[int] = Field(default=None, ge=0)
+    random_seed: Optional[int] = Field(default=None, ge=0)

mistral_common-1.0.0/src/mistral_common/protocol/instruct/response.py

@@ -0,0 +1,66 @@
+import time
+from enum import Enum
+from typing import List, Optional
+
+from pydantic import Field
+
+from mistral_common.base import MistralBase
+from mistral_common.protocol.base import UsageInfo
+from mistral_common.protocol.instruct.tool_calls import ToolCall
+from mistral_common.protocol.utils import random_uuid
+
+
+class FinishReason(str, Enum):
+    stop: str = "stop"
+    length: str = "length"
+    model_length: str = "model_length"
+    error: str = "error"
+    tool_call: str = "tool_calls"
+
+
+class ChatCompletionTokenLogprobs(MistralBase):
+    token: str
+    logprob: float
+    bytes: List[int]
+
+
+class ChatCompletionResponseChoiceLogprobs(MistralBase):
+    content: List[ChatCompletionTokenLogprobs]
+
+
+class DeltaMessage(MistralBase):
+    role: Optional[str] = None
+    content: Optional[str] = None
+    tool_calls: Optional[List[ToolCall]] = None
+
+
+class ChatCompletionResponseChoice(MistralBase):
+    index: int
+    message: DeltaMessage
+    finish_reason: Optional[FinishReason] = None
+    logprobs: Optional[ChatCompletionResponseChoiceLogprobs] = None
+
+
+class ChatCompletionResponse(MistralBase):
+    id: str = Field(default_factory=lambda: f"chatcmpl-{random_uuid()}")
+    object: str = "chat.completion"
+    created: int = Field(default_factory=lambda: int(time.time()))
+    model: str
+    choices: List[ChatCompletionResponseChoice]
+    usage: UsageInfo
+
+
+class ChatCompletionResponseStreamChoice(MistralBase):
+    index: int
+    delta: DeltaMessage
+    finish_reason: Optional[FinishReason] = None
+    logprobs: Optional[ChatCompletionResponseChoiceLogprobs] = None
+
+
+class ChatCompletionStreamResponse(MistralBase):
+    id: str = Field(default_factory=lambda: f"chatcmpl-{random_uuid()}")
+    object: str = "chat.completion.chunk"
+    created: int = Field(default_factory=lambda: int(time.time()))
+    model: str
+    choices: List[ChatCompletionResponseStreamChoice]
+    usage: Optional[UsageInfo] = None

mistral_common-1.0.0/src/mistral_common/protocol/instruct/tool_calls.py

@@ -0,0 +1,36 @@
+from enum import Enum
+from typing import Any, Dict
+
+from mistral_common.base import MistralBase
+
+
+class Function(MistralBase):
+    name: str
+    description: str = ""
+    parameters: Dict[str, Any]
+
+
+class ToolType(str, Enum):
+    function = "function"
+
+
+class ToolChoice(str, Enum):
+    auto: str = "auto"
+    none: str = "none"
+    any: str = "any"
+
+
+class Tool(MistralBase):
+    type: ToolType = ToolType.function
+    function: Function
+
+
+class FunctionCall(MistralBase):
+    name: str
+    arguments: str
+
+
+class ToolCall(MistralBase):
+    id: str = "null"  # required for V3 tokenization
+    type: ToolType = ToolType.function
+    function: FunctionCall