unique_toolkit 0.8.14__py3-none-any.whl → 0.8.15__py3-none-any.whl

unique_toolkit/_common/default_language_model.py

@@ -0,0 +1,6 @@
+from unique_toolkit.language_model.infos import LanguageModelName
+
+DEFAULT_GPT_35_TURBO = LanguageModelName.AZURE_GPT_35_TURBO_0125
+DEFAULT_GPT_4o = LanguageModelName.AZURE_GPT_4o_2024_1120
+DEFAULT_GPT_4o_STRUCTURED_OUTPUT = LanguageModelName.AZURE_GPT_4o_2024_0806
+DEFAULT_GPT_4o_MINI = LanguageModelName.AZURE_GPT_4o_MINI_2024_0718

unique_toolkit/_common/token/image_token_counting.py

@@ -0,0 +1,67 @@
+import base64
+import math
+import re
+from enum import Enum
+from io import BytesIO
+
+from PIL import Image
+
+
+class DetailLevel(Enum):
+    LOW = "low"
+    HIGH = "high"
+
+
+# https://platform.openai.com/docs/guides/vision/calculating-costs#calculating-costs
+def calculate_image_tokens(width, height, detail: DetailLevel):
+    """
+    Calculate the token cost of an image based on its dimensions and detail level.
+    NOTE: While we followed the documentation provided by openai to calculate image token cost, in practice,
+    we notice that this function overestimate the number of tokens consumed by the model.
+
+    Parameters:
+    - width (int): The width of the image in pixels.
+    - height (int): The height of the image in pixels.
+    - detail (str): The detail level, either "low" or "high".
+
+    Returns:
+    - int: The token cost of the image.
+    """
+    # Base cost for low detail
+    if detail == DetailLevel.LOW:
+        return 85
+
+    # Scaling for high detail
+    # Scale down to fit within 2048x2048 square
+    max_long_dim = 2048
+    long_dim = max(width, height)
+    if long_dim > max_long_dim:
+        scale_factor = long_dim / max_long_dim
+        width = int(width / scale_factor)
+        height = int(height / scale_factor)
+
+    # Scale down the shortest side to 768
+    max_short_dim = 768
+    short_dim = min(width, height)
+    if short_dim > max_short_dim:
+        scale_factor = short_dim / max_short_dim
+        width = int(width / scale_factor)
+        height = int(height / scale_factor)
+
+    # Step 3: Calculate the number of 512x512 tiles
+    tiles = math.ceil(width / 512) * math.ceil(height / 512)
+    # Step 4: Compute token cost
+    token_cost = (tiles * 170) + 85
+    return token_cost
+
+
+def calculate_image_tokens_from_base64(base64_string: str):
+    base64_string = remove_base64_header(base64_string)
+    image = Image.open(BytesIO(base64.b64decode(base64_string)))
+    # DETAIL LEVEL HIGH IS THE DEFAULT TO BE ON THE SAFE SIDE
+    return calculate_image_tokens(image.width, image.height, DetailLevel.HIGH)
+
+
+def remove_base64_header(base64_string: str):
+    header_pattern = r"^data:image/\w+;base64,"
+    return re.sub(header_pattern, "", base64_string)

unique_toolkit/_common/token/token_counting.py

@@ -0,0 +1,196 @@
+# Original source
+# https://github.com/openai/openai-cookbook/blob/main/examples/How_to_count_tokens_with_tiktoken.ipynb
+
+import json
+from typing import Any, Callable
+
+from pydantic import BaseModel
+from unique_toolkit.language_model import (
+    LanguageModelMessage,
+    LanguageModelMessages,
+    LanguageModelName,
+)
+
+from _common.utils.token.image_token_counting import (
+    calculate_image_tokens_from_base64,
+)
+
+
+class SpecialToolCallingTokens(BaseModel):
+    func_init: int = 0
+    prop_init: int = 0
+    prop_key: int = 0
+    enum_init: int = 0
+    enum_item: int = 0
+    func_end: int = 0
+
+
+def get_special_token(model: LanguageModelName) -> SpecialToolCallingTokens:
+    special_token = SpecialToolCallingTokens()
+
+    match model:
+        case (
+            LanguageModelName.AZURE_GPT_4o_2024_0513
+            | LanguageModelName.AZURE_GPT_4o_2024_0806
+            | LanguageModelName.AZURE_GPT_4o_MINI_2024_0718
+            | LanguageModelName.AZURE_GPT_4o_2024_1120
+        ):
+            special_token.func_init = 7
+            special_token.prop_init = 3
+            special_token.prop_key = 3
+            special_token.enum_init = -3
+            special_token.enum_item = 3
+            special_token.func_end = 12
+
+        case (
+            LanguageModelName.AZURE_GPT_35_TURBO_0125
+            | LanguageModelName.AZURE_GPT_4_0613
+            | LanguageModelName.AZURE_GPT_4_32K_0613
+            | LanguageModelName.AZURE_GPT_4_TURBO_2024_0409
+        ):
+            special_token.func_init = 10
+            special_token.prop_init = 3
+            special_token.prop_key = 3
+            special_token.enum_init = -3
+            special_token.enum_item = 3
+            special_token.func_end = 12
+
+        case _:
+            raise NotImplementedError(
+                f"""num_tokens_for_tools() is not implemented for model {model}."""
+            )
+    return special_token
+
+
+def num_tokens_per_messages(
+    messages: list[dict[str, str]], encode: Callable[[str], list[int]]
+) -> list[int]:
+    """Return the number of tokens used by a list of messages."""
+
+    num_token_per_message = []
+    for message in messages:
+        num_tokens = 3  # extra_tokens_per_message
+        for key, value in message.items():
+            if isinstance(value, str):
+                num_tokens += len(encode(value))
+            elif isinstance(value, list):
+                # NOTE: The result returned by the function below is not 100% accurate.
+                num_tokens += handle_message_with_images(value, encode)
+            if key == "name":
+                num_tokens += 1  # extra_tokens_per_name
+
+        num_token_per_message.append(num_tokens)
+
+    return num_token_per_message
+
+
+def num_tokens_from_messages(
+    messages: list[dict[str, str]], encode: Callable[[str], list[int]]
+) -> int:
+    """Return the number of tokens used by a list of messages."""
+
+    num_tokens_per_message = num_tokens_per_messages(messages, encode)
+    num_tokens = sum(num_tokens_per_message) + 3
+
+    return num_tokens
+
+
+def num_tokens_for_tools(
+    functions: list[dict[str, Any]],
+    special_token: SpecialToolCallingTokens,
+    encode: Callable[[str], list[int]],
+):
+    def num_token_function_enum(
+        properties: dict[str, Any], encode: Callable[[str], list[int]]
+    ):
+        enum_token_count = 0
+        enum_token_count += special_token.enum_init
+        for item in properties[key]["enum"]:
+            enum_token_count += special_token.enum_item
+            enum_token_count += len(encode(item))
+
+        return enum_token_count
+
+    func_token_count = 0
+    if len(functions) > 0:
+        for func in functions:
+            func_token_count += special_token.func_init
+            function = func.get("function", {})
+            func_token_count += len(
+                encode(
+                    function.get("name", "")
+                    + ":"
+                    + function.get("description", "").rstrip(".").rstrip()
+                )
+            )
+            if len(function.get("parameters", {}).get("properties", "")) > 0:
+                properties = function.get("parameters", {}).get(
+                    "properties", ""
+                )
+                func_token_count += special_token.prop_init
+
+                for key in list(properties.keys()):
+                    func_token_count += special_token.prop_key
+
+                    if "enum" in properties[key].keys():
+                        func_token_count += num_token_function_enum(
+                            properties, encode
+                        )
+
+                    func_token_count += len(
+                        encode(
+                            f"{key}:{properties[key]['type']}:{properties[key]['description'].rstrip('.').rstrip()}"
+                        )
+                    )
+
+        func_token_count += special_token.func_end
+
+    return func_token_count
+
+
+def handle_message_with_images(
+    message: list[dict], encode: Callable[[str], list[int]]
+):
+    token_count = 0
+    for item in message:
+        if item.get("type") == "image_url":
+            image_url = item.get("imageUrl", {}).get("url")
+            if image_url:
+                token_count += calculate_image_tokens_from_base64(image_url)
+        elif item.get("type") == "text":
+            token_count += len(encode(item.get("text", "")))
+    return token_count
+
+
+def messages_to_openai_messages(
+    messages: LanguageModelMessages | list[LanguageModelMessage],
+):
+    if isinstance(messages, list):
+        messages = LanguageModelMessages(messages)
+
+    return [
+        {
+            k: v
+            for k, v in m.items()
+            if (k in ["content", "role"] and v is not None)
+        }
+        for m in json.loads(messages.model_dump_json())
+    ]
+
+
+def num_tokens_per_language_model_message(
+    messages: LanguageModelMessages | list[LanguageModelMessage],
+    encode: Callable[[str], list[int]],
+) -> list[int]:
+    return num_tokens_per_messages(
+        messages=messages_to_openai_messages(messages), encode=encode
+    )
+
+
+def num_token_for_language_model_messages(
+    messages: LanguageModelMessages | list[LanguageModelMessage],
+    encode: Callable[[str], list[int]],
+) -> int:
+    return num_tokens_from_messages(
+        messages_to_openai_messages(messages), encode
+    )

unique_toolkit/history_manager/history_construction_with_contents.py

@@ -0,0 +1,307 @@
+import base64
+import mimetypes
+
+from datetime import datetime
+from enum import StrEnum
+
+import numpy as np
+import tiktoken
+
+from pydantic import RootModel
+
+from _common.token.token_counting import num_tokens_per_language_model_message
+from chat.service import ChatService
+from content.service import ContentService
+from language_model.schemas import LanguageModelMessages
+from unique_toolkit.app import ChatEventUserMessage
+from unique_toolkit.chat.schemas import ChatMessage
+from unique_toolkit.chat.schemas import ChatMessageRole as ChatRole
+from unique_toolkit.content.schemas import Content
+from unique_toolkit.language_model import LanguageModelMessageRole as LLMRole
+from unique_toolkit.language_model.infos import EncoderName
+
+
+
+# TODO: Test this once it moves into the unique toolkit
+
+map_chat_llm_message_role = {
+    ChatRole.USER: LLMRole.USER,
+    ChatRole.ASSISTANT: LLMRole.ASSISTANT,
+}
+
+
+class ImageMimeType(StrEnum):
+    JPEG = "image/jpeg"
+    PNG = "image/png"
+    GIF = "image/gif"
+    BMP = "image/bmp"
+    WEBP = "image/webp"
+    TIFF = "image/tiff"
+    SVG = "image/svg+xml"
+
+
+class FileMimeType(StrEnum):
+    PDF = "application/pdf"
+    DOCX = "application/vnd.openxmlformats-officedocument.wordprocessingml.document"
+    DOC = "application/msword"
+    XLSX = "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet"
+    XLS = "application/vnd.ms-excel"
+    PPTX = "application/vnd.openxmlformats-officedocument.presentationml.presentation"
+    CSV = "text/csv"
+    HTML = "text/html"
+    MD = "text/markdown"
+    TXT = "text/plain"
+
+
+class ChatMessageWithContents(ChatMessage):
+    contents: list[Content] = []
+
+
+class ChatHistoryWithContent(RootModel):
+    root: list[ChatMessageWithContents]
+
+    @classmethod
+    def from_chat_history_and_contents(
+        cls,
+        chat_history: list[ChatMessage],
+        chat_contents: list[Content],
+    ):
+        combined = chat_contents + chat_history
+        combined.sort(key=lambda x: x.created_at or datetime.min)
+
+        grouped_elements = []
+        content_container = []
+
+        # Content is collected and added to the next chat message
+        for c in combined:
+            if isinstance(c, ChatMessage):
+                grouped_elements.append(
+                    ChatMessageWithContents(
+                        contents=content_container.copy(),
+                        **c.model_dump(),
+                    ),
+                )
+                content_container.clear()
+            else:
+                content_container.append(c)
+
+        return cls(root=grouped_elements)
+
+    def __iter__(self):
+        return iter(self.root)
+
+    def __getitem__(self, item):
+        return self.root[item]
+
+
+def is_image_content(filename: str) -> bool:
+    mimetype, _ = mimetypes.guess_type(filename)
+
+    if not mimetype:
+        return False
+
+    return mimetype in ImageMimeType.__members__.values()
+
+
+def is_file_content(filename: str) -> bool:
+    mimetype, _ = mimetypes.guess_type(filename)
+
+    if not mimetype:
+        return False
+
+    return mimetype in FileMimeType.__members__.values()
+
+
+def get_chat_history_with_contents(
+    user_message: ChatEventUserMessage,
+    chat_id: str,
+    chat_history: list[ChatMessage],
+    content_service: ContentService,
+) -> ChatHistoryWithContent:
+    last_user_message = ChatMessage(
+        id=user_message.id,
+        chat_id=chat_id,
+        text=user_message.text,
+        originalText=user_message.original_text,
+        role=ChatRole.USER,
+        gpt_request=None,
+        created_at=datetime.fromisoformat(user_message.created_at),
+    )
+    if len(chat_history) > 0 and last_user_message.id == chat_history[-1].id:
+        pass
+    else:
+        chat_history.append(last_user_message)
+
+    chat_contents = content_service.search_contents(
+        where={
+            "ownerId": {
+                "equals": chat_id,
+            },
+        },
+    )
+
+    return ChatHistoryWithContent.from_chat_history_and_contents(
+        chat_history,
+        chat_contents,
+    )
+
+
+def download_encoded_images(
+    contents: list[Content],
+    content_service: ContentService,
+    chat_id: str,
+) -> list[str]:
+    base64_encoded_images = []
+    for im in contents:
+        if is_image_content(im.key):
+            try:
+                file_bytes = content_service.download_content_to_bytes(
+                    content_id=im.id,
+                    chat_id=chat_id,
+                )
+
+                mime_type, _ = mimetypes.guess_type(im.key)
+                encoded_string = base64.b64encode(file_bytes).decode("utf-8")
+                image_string = f"data:{mime_type};base64," + encoded_string
+                base64_encoded_images.append(image_string)
+            except Exception as e:
+                print(e)
+    return base64_encoded_images
+
+
+class FileContentSerialization(StrEnum):
+    NONE = "none"
+    FILE_NAME = "file_name"
+
+
+class ImageContentInclusion(StrEnum):
+    NONE = "none"
+    ALL = "all"
+
+
+def file_content_serialization(
+    file_contents: list[Content],
+    file_content_serialization: FileContentSerialization,
+) -> str:
+    match file_content_serialization:
+        case FileContentSerialization.NONE:
+            return ""
+        case FileContentSerialization.FILE_NAME:
+            file_names = [
+                f"- Uploaded file: {f.key} at {f.created_at}"
+                for f in file_contents
+            ]
+            return "\n".join(
+                [
+                    "Files Uploaded to Chat can be accessed by internal search tool if available:\n",
+                ]
+                + file_names,
+            )
+
+
+def get_full_history_with_contents(
+    user_message: ChatEventUserMessage,
+    chat_id: str,
+    chat_service: ChatService,
+    content_service: ContentService,
+    include_images: ImageContentInclusion = ImageContentInclusion.ALL,
+    file_content_serialization_type: FileContentSerialization = FileContentSerialization.FILE_NAME,
+) -> LanguageModelMessages:
+    grouped_elements = get_chat_history_with_contents(
+        user_message=user_message,
+        chat_id=chat_id,
+        chat_history=chat_service.get_full_history(),
+        content_service=content_service,
+    )
+
+    builder = LanguageModelMessages([]).builder()
+    for c in grouped_elements:
+        # LanguageModelUserMessage has not field original content
+        text = c.original_content if c.original_content else c.content
+        if text is None:
+            if c.role == ChatRole.USER:
+                raise ValueError(
+                    "Content or original_content of LanguageModelMessages should exist.",
+                )
+            text = ""
+
+        if len(c.contents) > 0:
+            file_contents = [
+                co for co in c.contents if is_file_content(co.key)
+            ]
+            image_contents = [
+                co for co in c.contents if is_image_content(co.key)
+            ]
+
+            content = (
+                text
+                + "\n\n"
+                + file_content_serialization(
+                    file_contents,
+                    file_content_serialization_type,
+                )
+            )
+            content = content.strip()
+
+            if include_images and len(image_contents) > 0:
+                builder.image_message_append(
+                    content=content,
+                    images=download_encoded_images(
+                        contents=image_contents,
+                        content_service=content_service,
+                        chat_id=chat_id,
+                    ),
+                    role=map_chat_llm_message_role[c.role],
+                )
+            else:
+                builder.message_append(
+                    role=map_chat_llm_message_role[c.role],
+                    content=content,
+                )
+        else:
+            builder.message_append(
+                role=map_chat_llm_message_role[c.role],
+                content=text,
+            )
+    return builder.build()
+
+
+def get_full_history_as_llm_messages(
+    chat_service: ChatService,
+) -> LanguageModelMessages:
+    chat_history = chat_service.get_full_history()
+
+    map_chat_llm_message_role = {
+        ChatRole.USER: LLMRole.USER,
+        ChatRole.ASSISTANT: LLMRole.ASSISTANT,
+    }
+
+    builder = LanguageModelMessages([]).builder()
+    for c in chat_history:
+        builder.message_append(
+            role=map_chat_llm_message_role[c.role],
+            content=c.content or "",
+        )
+    return builder.build()
+
+
+
+def limit_to_token_window(
+    messages: LanguageModelMessages,
+    token_limit: int,
+    encoding_name: EncoderName = EncoderName.O200K_BASE,
+) -> LanguageModelMessages:
+    encoder = tiktoken.get_encoding(encoding_name)
+    token_per_message_reversed = num_tokens_per_language_model_message(
+        messages,
+        encode=encoder.encode,
+    )
+
+    to_take: list[bool] = (
+        np.cumsum(token_per_message_reversed) < token_limit
+    ).tolist()
+    to_take.reverse()
+
+    return LanguageModelMessages(
+        root=[m for m, tt in zip(messages, to_take, strict=False) if tt],
+    )

unique_toolkit/history_manager/history_manager.py

@@ -1,9 +1,10 @@
 from datetime import datetime
 from logging import Logger
-from typing import Awaitable, Callable
+from typing import Annotated, Awaitable, Callable
 
 from pydantic import BaseModel, Field
 
+import tiktoken
 from unique_toolkit.app.schemas import ChatEvent
 
 
@@ -17,17 +18,54 @@ from unique_toolkit.language_model.schemas import (
     LanguageModelFunction,
     LanguageModelMessage,
     LanguageModelMessageRole,
+    LanguageModelSystemMessage,
     LanguageModelToolMessage,
     LanguageModelUserMessage,
 )
 
 from unique_toolkit.tools.schemas import ToolCallResponse
-from unique_toolkit.content.utils import count_tokens
 from unique_toolkit.history_manager.utils import transform_chunks_to_string
 
+from _common.validators import LMI
+from history_manager.loop_token_reducer import LoopTokenReducer
+from reference_manager.reference_manager import ReferenceManager
+from tools.config import get_configuration_dict
+
+DeactivatedNone = Annotated[
+    None,
+    Field(title="Deactivated", description="None"),
+]
 
 class HistoryManagerConfig(BaseModel):
 
+    class InputTokenDistributionConfig(BaseModel):
+        model_config = get_configuration_dict(frozen=True)
+
+        percent_for_history: float = Field(
+            default=0.6,
+            ge=0.0,
+            lt=1.0,
+            description="The fraction of the max input tokens that will be reserved for the history.",
+        )
+
+        def max_history_tokens(self, max_input_token: int) -> int:
+            return int(self.percent_for_history * max_input_token)
+    
+    class UploadedContentConfig(BaseModel):
+        model_config = get_configuration_dict()
+
+        user_context_window_limit_warning: str = Field(
+            default="The uploaded content is too large to fit into the ai model. "
+            "Unique AI will search for relevant sections in the material and if needed combine the data with knowledge base content",
+            description="Message to show when using the Internal Search instead of upload and chat tool due to context window limit. Jinja template.",
+        )
+        percent_for_uploaded_content: float = Field(
+            default=0.6,
+            ge=0.0,
+            lt=1.0,
+            description="The fraction of the max input tokens that will be reserved for the uploaded content.",
+        )
+
     class ExperimentalFeatures(BaseModel):
         def __init__(self, full_sources_serialize_dump: bool = False):
             self.full_sources_serialize_dump = full_sources_serialize_dump
@@ -48,6 +86,20 @@ class HistoryManagerConfig(BaseModel):
         description="The maximum number of tokens to keep in the history.",
     )
 
+    uploaded_content_config: (
+        Annotated[
+            UploadedContentConfig,
+            Field(title="Active"),
+        ]
+        | DeactivatedNone
+    ) = UploadedContentConfig()
+
+
+    input_token_distribution: InputTokenDistributionConfig = Field(
+        default=InputTokenDistributionConfig(),
+        description="Configuration for the input token distribution.",
+    )
+
 
 class HistoryManager:
     """
@@ -78,11 +130,20 @@ class HistoryManager:
         logger: Logger,
         event: ChatEvent,
         config: HistoryManagerConfig,
+        language_model: LMI,
+        reference_manager: ReferenceManager,
     ):
         self._config = config
         self._logger = logger
-        self._chat_service = ChatService(event)
-        self._content_service = ContentService.from_event(event)
+        self._language_model = language_model
+        self._token_reducer = LoopTokenReducer(
+            logger=self._logger,
+            event=event,
+            config=self._config,
+            language_model=self._language_model,
+            reference_manager=reference_manager,
+        )
+
 
     def has_no_loop_messages(self) -> bool:
         return len(self._loop_history) == 0
@@ -150,112 +211,25 @@ class HistoryManager:
     def add_assistant_message(self, message: LanguageModelAssistantMessage) -> None:
         self._loop_history.append(message)
 
-    async def get_history(
+        
+    async def get_history_for_model_call(
         self,
-        postprocessing_step: Callable[
-            [list[LanguageModelMessage]], list[LanguageModelMessage]
-        ]
-        | None = None,
-    ) -> list[LanguageModelMessage]:
-        """
-        Get the history of the conversation. The function will retrieve a subset of the full history based on the configuration.
-
-        Returns:
-            list[LanguageModelMessage]: The history
-        """
-        # Get uploaded files
-        uploaded_files = self._content_service.search_content_on_chat(
-            chat_id=self._chat_service.chat_id
+        original_user_message: str,
+        rendered_user_message_string: str,
+        rendered_system_message_string: str,
+        remove_from_text: Callable[[str], Awaitable[str]]
+    ) -> list[
+        LanguageModelMessage
+        | LanguageModelToolMessage
+        | LanguageModelAssistantMessage
+        | LanguageModelSystemMessage
+        | LanguageModelUserMessage
+    ]:
+        messages = await self._token_reducer.get_history_for_model_call(
+            original_user_message=original_user_message,
+            rendered_user_message_string=rendered_user_message_string,
+            rendered_system_message_string=rendered_system_message_string,
+            loop_history=self._loop_history,
+            remove_from_text=remove_from_text,
         )
-        # Get all message history
-        full_history = await self._chat_service.get_full_history_async()
-
-        merged_history = self._merge_history_and_uploads(full_history, uploaded_files)
-
-        if postprocessing_step is not None:
-            merged_history = postprocessing_step(merged_history)
-
-        limited_history = self._limit_to_token_window(
-            merged_history, self._config.max_history_tokens
-        )
-
-        # Add current user message if not already in history
-        # we grab it fresh from the db so it must contain all the messages this code is not needed anymore below currently it's left in for explainability
-        # current_user_msg = LanguageModelUserMessage(
-        #     content=self.event.payload.user_message.text
-        # )
-        # if not any(
-        #     msg.role == LanguageModelMessageRole.USER
-        #     and msg.content == current_user_msg.content
-        #     for msg in complete_history
-        # ):
-        #     complete_history.append(current_user_msg)
-
-        # # Add final assistant response - this should be available when this method is called
-        # if (
-        #     hasattr(self, "loop_response")
-        #     and self.loop_response
-        #     and self.loop_response.message.text
-        # ):
-        #     complete_history.append(
-        #         LanguageModelAssistantMessage(
-        #             content=self.loop_response.message.text
-        #         )
-        #     )
-        # else:
-        #     self.logger.warning(
-        #         "Called get_complete_conversation_history_after_streaming_no_tool_calls but no loop_response.message.text is available"
-        #     )
-
-        return limited_history
-
-    def _merge_history_and_uploads(
-        self, history: list[ChatMessage], uploads: list[Content]
-    ) -> list[LanguageModelMessage]:
-        # Assert that all content have a created_at
-        content_with_created_at = [content for content in uploads if content.created_at]
-        sorted_history = sorted(
-            history + content_with_created_at,
-            key=lambda x: x.created_at or datetime.min,
-        )
-
-        msg_builder = MessagesBuilder()
-        for msg in sorted_history:
-            if isinstance(msg, Content):
-                msg_builder.user_message_append(
-                    f"Uploaded file: {msg.key}, ContentId: {msg.id}"
-                )
-            else:
-                msg_builder.messages.append(
-                    LanguageModelMessage(
-                        role=LanguageModelMessageRole(msg.role),
-                        content=msg.content,
-                    )
-                )
-        return msg_builder.messages
-
-    def _limit_to_token_window(
-        self, messages: list[LanguageModelMessage], token_limit: int
-    ) -> list[LanguageModelMessage]:
-        selected_messages = []
-        token_count = 0
-        for msg in messages[::-1]:
-            msg_token_count = count_tokens(str(msg.content))
-            if token_count + msg_token_count > token_limit:
-                break
-            selected_messages.append(msg)
-            token_count += msg_token_count
-        return selected_messages[::-1]
-
-    async def remove_post_processing_manipulations(
-        self, remove_from_text: Callable[[str], Awaitable[str]]
-    ) -> list[LanguageModelMessage]:
-        messages = await self.get_history()
-        for message in messages:
-            if isinstance(message.content, str):
-                message.content = await remove_from_text(message.content)
-            else:
-                self._logger.warning(
-                    f"Skipping message with unsupported content type: {type(message.content)}"
-                )
-        return messages
+        return messages.root

unique_toolkit/history_manager/loop_token_reducer.py

@@ -0,0 +1,457 @@
+
+import json
+from logging import Logger
+from typing import Awaitable, Callable
+
+from pydantic import BaseModel
+import tiktoken
+from _common.token.token_counting import num_token_for_language_model_messages
+from _common.validators import LMI
+from app.schemas import ChatEvent
+from chat.service import ChatService
+from content.schemas import ContentChunk
+from content.service import ContentService
+from history_manager.history_construction_with_contents import FileContentSerialization, get_full_history_with_contents
+from history_manager.history_manager import HistoryManagerConfig
+from language_model.schemas import LanguageModelAssistantMessage, LanguageModelMessage, LanguageModelMessageRole, LanguageModelMessages, LanguageModelSystemMessage, LanguageModelToolMessage, LanguageModelUserMessage
+from reference_manager.reference_manager import ReferenceManager
+
+
+class SourceReductionResult(BaseModel):
+    message: LanguageModelToolMessage
+    reduced_chunks: list[ContentChunk]
+    chunk_offset: int
+    source_offset: int
+
+    class Config:
+        arbitrary_types_allowed = True
+
+
+class LoopTokenReducer():
+    
+    def __init__(
+        self,
+        logger: Logger,
+        event: ChatEvent,
+        config: HistoryManagerConfig,
+        reference_manager: ReferenceManager,
+        language_model: LMI
+    ):
+        self._config = config
+        self._logger = logger
+        self._reference_manager = reference_manager
+        self._language_model = language_model
+        self._encoder = self._get_encoder(language_model)
+        self._chat_service = ChatService(event)
+        self._content_service = ContentService.from_event(event)
+        self._user_message = event.payload.user_message
+        self._chat_id = event.payload.chat_id
+    
+
+    def _get_encoder(self, language_model: LMI) -> tiktoken.Encoding:
+        name = language_model.name or "cl100k_base"
+        return tiktoken.get_encoding(name)
+
+    async def get_history_for_model_call( self,
+        original_user_message: str,
+        rendered_user_message_string: str,
+        rendered_system_message_string: str,
+        loop_history: list[LanguageModelMessage],
+        remove_from_text: Callable[[str], Awaitable[str]]
+        ) -> LanguageModelMessages:
+        """Compose the system and user messages for the plan execution step, which is evaluating if any further tool calls are required."""
+
+
+        messages = await self._construct_history(
+            original_user_message,
+            rendered_user_message_string,
+            rendered_system_message_string,
+            loop_history,
+            remove_from_text
+            )
+
+        token_count = self._count_message_tokens(messages)
+        self._log_token_usage(token_count)
+
+        while self._exceeds_token_limit(token_count):
+            token_count_before_reduction = token_count
+            loop_history = self._handle_token_limit_exceeded(loop_history)
+            messages = await self._construct_history(
+              original_user_message,
+              rendered_user_message_string,
+              rendered_system_message_string,
+              loop_history,
+              remove_from_text
+            )
+            token_count = self._count_message_tokens(messages)
+            self._log_token_usage(token_count)
+            token_count_after_reduction = token_count
+            if token_count_after_reduction >= token_count_before_reduction:
+                break
+
+        return messages
+    
+    def _exceeds_token_limit(self, token_count: int) -> bool:
+        """Check if token count exceeds the maximum allowed limit and if at least one tool call has more than one source."""
+        # At least one tool call should have more than one chunk as answer
+        has_multiple_chunks_for_a_tool_call = any(
+            len(chunks) > 1
+            for chunks in self._reference_manager.get_chunks_of_all_tools()
+        )
+
+        # TODO: This is not fully correct at the moment as the token_count
+        # include system_prompt and user question already
+        # TODO: There is a problem if we exceed but only have one chunk per tool call
+        exceeds_limit = (
+            token_count
+            > self._language_model.token_limits.token_limit_input
+        )
+
+        return has_multiple_chunks_for_a_tool_call and exceeds_limit
+
+
+    def _count_message_tokens(self, messages: LanguageModelMessages) -> int:
+        """Count tokens in messages using the configured encoding model."""
+        return num_token_for_language_model_messages(
+            messages=messages,
+            encode=self._encoder.encode,
+        )
+
+    def _log_token_usage(self, token_count: int) -> None:
+        """Log token usage and update debug info."""
+        self._logger.info(f"Token messages: {token_count}")
+        # self.agent_debug_info.add("token_messages", token_count)
+
+    async def _construct_history(
+        self,
+        original_user_message: str,
+        rendered_user_message_string: str,
+        rendered_system_message_string: str,
+        loop_history: list[LanguageModelMessage],
+        remove_from_text: Callable[[str], Awaitable[str]]
+    ) -> LanguageModelMessages:
+        history_from_db = await self._get_history_from_db(remove_from_text)
+        history_from_db = self._replace_user_message(history_from_db, original_user_message, rendered_user_message_string)
+        system_message = LanguageModelSystemMessage(content=rendered_system_message_string)
+
+        constructed_history = LanguageModelMessages(
+            [system_message] + history_from_db + loop_history,
+        )
+
+
+        return constructed_history
+    
+
+    def _handle_token_limit_exceeded(self,loop_history: list[LanguageModelMessage]) -> list[LanguageModelMessage]:
+        """Handle case where token limit is exceeded by reducing sources in tool responses."""
+        self._logger.warning(
+            f"Length of messages is exceeds limit of {self._language_model.token_limits.token_limit_input} tokens. "
+            "Reducing number of sources per tool call.",
+        )
+       
+        return self._reduce_message_length_by_reducing_sources_in_tool_response(loop_history)
+
+    def _replace_user_message(
+        self,
+        history: list[LanguageModelMessage],
+        original_user_message: str,
+        rendered_user_message_string: str,
+    ) -> list[LanguageModelMessage]:
+        """
+        Replaces the original user message in the history with the rendered user message string.
+        """
+        if history[-1].role == LanguageModelMessageRole.USER:
+            m = history[-1]
+
+            if isinstance(m.content, list):
+                # Replace the last text element but be careful not to delete data added when merging with contents
+                for t in reversed(m.content):
+                    field = t.get("type", "")
+                    if field == "text" and isinstance(field, dict):
+                        inner_field = field.get("text", "")
+                        if isinstance(inner_field, str):
+                            added_to_message_by_history = inner_field.replace(
+                                original_user_message,
+                                "",
+                            )
+                            t["text"] = rendered_user_message_string + added_to_message_by_history
+                        break
+            elif m.content:
+                added_to_message_by_history = m.content.replace(original_user_message, "")
+                m.content = rendered_user_message_string + added_to_message_by_history
+        else:
+            history = history + [
+                LanguageModelUserMessage(content=rendered_user_message_string),
+            ]
+        return history
+    
+
+    async def _get_history_from_db(
+        self,
+        remove_from_text: Callable[[str], Awaitable[str]]
+    ) -> list[LanguageModelMessage]:
+        """
+        Get the history of the conversation. The function will retrieve a subset of the full history based on the configuration.
+
+        Returns:
+            list[LanguageModelMessage]: The history
+        """
+        full_history = get_full_history_with_contents(
+                user_message=self._user_message,
+                chat_id=self._chat_id,
+                chat_service=self._chat_service,
+                content_service=self._content_service,
+                file_content_serialization_type=(
+                    FileContentSerialization.NONE
+                    if self._config.uploaded_content_config
+                    else FileContentSerialization.FILE_NAME
+                ),
+            )
+        
+        full_history.root = await self._clean_messages(full_history.root, remove_from_text)
+        
+        limited_history_messages = self._limit_to_token_window(
+            full_history.root,
+            self._config.input_token_distribution.max_history_tokens(
+                self._language_model.token_limits.token_limit_input,
+            )
+        )
+        
+        
+        if len(limited_history_messages) == 0:
+            limited_history_messages = full_history.root[-1:]
+
+        self._logger.info(
+            f"Reduced history to {len(limited_history_messages)} messages from {len(full_history.root)}",
+        )
+
+        return self.ensure_last_message_is_user_message(limited_history_messages)
+
+    def _limit_to_token_window(
+        self, messages: list[LanguageModelMessage], token_limit: int
+    ) -> list[LanguageModelMessage]:
+        selected_messages = []
+        token_count = 0
+        for msg in messages[::-1]:
+            msg_token_count = self._count_tokens(str(msg.content))
+            if token_count + msg_token_count > token_limit:
+                break
+            selected_messages.append(msg)
+            token_count += msg_token_count
+        return selected_messages[::-1]
+
+    async def _clean_messages(
+        self, 
+        messages: list[LanguageModelMessage | LanguageModelToolMessage | LanguageModelAssistantMessage | LanguageModelSystemMessage | LanguageModelUserMessage],
+        remove_from_text: Callable[[str], Awaitable[str]]
+    ) -> list[LanguageModelMessage]:
+        for message in messages:
+            if isinstance(message.content, str):
+                message.content = await remove_from_text(message.content)
+            else:
+                self._logger.warning(
+                    f"Skipping message with unsupported content type: {type(message.content)}"
+                )
+        return messages
+
+    def _count_tokens(self,text:str) -> int:
+
+        return len(self._encoder.encode(text))
+
+    def ensure_last_message_is_user_message(self, limited_history_messages):
+        """
+        As the token limit can be reached in the middle of a gpt_request, 
+        we move forward to the next user message,to avoid confusing messages for the LLM
+        """
+        idx = 0
+        for idx, message in enumerate(limited_history_messages):
+            if message.role == LanguageModelMessageRole.USER:
+                break
+
+        # FIXME: This might reduce the history by a lot if we have a lot of tool calls / references in the history. Could make sense to summarize the messages and include
+        # FIXME: We should remove chunks no longer in history from handler
+        return limited_history_messages[idx:]
+    
+
+    def _reduce_message_length_by_reducing_sources_in_tool_response(
+      self,
+      history: list[LanguageModelMessage],
+    ) -> list[LanguageModelMessage]:
+      """
+      Reduce the message length by removing the last source result of each tool call.
+      If there is only one source for a tool call, the tool call message is returned unchanged.
+      """
+      history_reduced: list[LanguageModelMessage] = []
+      content_chunks_reduced: list[ContentChunk] = []
+      chunk_offset = 0
+      source_offset = 0
+
+      for message in history:
+          if self._should_reduce_message(message):
+              result = self._reduce_sources_in_tool_message(
+                  message,  # type: ignore
+                  chunk_offset,
+                  source_offset,
+              )
+              content_chunks_reduced.extend(result.reduced_chunks)
+              history_reduced.append(result.message)
+              chunk_offset = result.chunk_offset
+              source_offset = result.source_offset
+          else:
+              history_reduced.append(message)
+
+      self._reference_manager.replace(chunks=content_chunks_reduced)
+      return history_reduced
+    
+    def _should_reduce_message(self, message: LanguageModelMessage) -> bool:
+      """Determine if a message should have its sources reduced."""
+      return (
+          message.role == LanguageModelMessageRole.TOOL
+          and isinstance(message, LanguageModelToolMessage)
+      )
+  
+
+    def _reduce_sources_in_tool_message(
+        self,
+        message: LanguageModelToolMessage,
+        chunk_offset: int,
+        source_offset: int,
+    ) -> SourceReductionResult:
+        """
+        Reduce the sources in the tool message by removing the last source.
+        If there is only one source, the message is returned unchanged.
+        """
+        tool_chunks = self._reference_manager.get_chunks_of_tool(message.tool_call_id)
+        num_sources = len(tool_chunks)
+
+        if num_sources == 0:
+            return SourceReductionResult(
+                message=message,
+                reduced_chunks=[],
+                chunk_offset=chunk_offset,
+                source_offset=source_offset,
+            )
+
+        # Reduce chunks, keeping all but the last one if multiple exist
+        if num_sources == 1:
+            reduced_chunks = tool_chunks
+            content_chunks_reduced = self._reference_manager.get_chunks()[
+                chunk_offset : chunk_offset + num_sources
+            ]
+        else:
+            reduced_chunks = tool_chunks[:-1]
+            content_chunks_reduced = self._reference_manager.get_chunks()[
+                chunk_offset : chunk_offset + num_sources - 1
+            ]
+            self._reference_manager.replace_chunks_of_tool(
+                message.tool_call_id,
+                reduced_chunks
+            )
+
+        # Create new message with reduced sources
+        new_message = self._create_tool_call_message_with_reduced_sources(
+            message=message,
+            content_chunks=reduced_chunks,
+            source_offset=source_offset,
+        )
+
+        return SourceReductionResult(
+            message=new_message,
+            reduced_chunks=content_chunks_reduced,
+            chunk_offset=chunk_offset + num_sources,
+            source_offset=source_offset
+            + num_sources
+            - (1 if num_sources != 1 else 0),
+        )
+    
+    def _create_tool_call_message_with_reduced_sources(
+        self,
+        message: LanguageModelToolMessage,
+        content_chunks: list[ContentChunk] | None = None,
+        source_offset: int = 0,
+    ) -> LanguageModelToolMessage:
+        # Handle special case for TableSearch tool
+        if message.name == "TableSearch":
+            return self._create_reduced_table_search_message(
+                message, content_chunks, source_offset
+            )
+
+        # Handle empty content case
+        if not content_chunks:
+            return self._create_reduced_empty_sources_message(message)
+
+        # Handle standard content chunks
+        return self._create_reduced_standard_sources_message(
+            message, content_chunks, source_offset
+        )
+    
+    def _create_reduced_table_search_message(
+        self,
+        message: LanguageModelToolMessage,
+        content_chunks: list[ContentChunk] | None,
+        source_offset: int,
+    ) -> LanguageModelToolMessage:
+        """
+        Create a message for TableSearch tool.
+
+        Note: TableSearch content consists of a single result with SQL results,
+        not content chunks.
+        """
+        if not content_chunks:
+            content = message.content
+        else:
+            if isinstance(message.content, str):
+                content_dict = json.loads(message.content)
+            elif isinstance(message.content, dict):
+                content_dict = message.content
+            else:
+                raise ValueError(
+                    f"Unexpected content type: {type(message.content)}"
+                )
+
+            content = json.dumps(
+                {
+                    "source_number": source_offset,
+                    "content": content_dict.get("content"),
+                }
+            )
+
+        return LanguageModelToolMessage(
+            content=content,
+            tool_call_id=message.tool_call_id,
+            name=message.name,
+        )
+
+
+    def _create_reduced_empty_sources_message(
+        self,
+        message: LanguageModelToolMessage,
+    ) -> LanguageModelToolMessage:
+        """Create a message when no content chunks are available."""
+        return LanguageModelToolMessage(
+            content="No relevant sources found.",
+            tool_call_id=message.tool_call_id,
+            name=message.name,
+        )
+
+
+    def _create_reduced_standard_sources_message(
+        self,
+        message: LanguageModelToolMessage,
+        content_chunks: list[ContentChunk],
+        source_offset: int,
+    ) -> LanguageModelToolMessage:
+        """Create a message with standard content chunks."""
+        sources = [
+            {
+                "source_number": source_offset + i,
+                "content": chunk.text,
+            }
+            for i, chunk in enumerate(content_chunks)
+        ]
+
+        return LanguageModelToolMessage(
+            content=str(sources),
+            tool_call_id=message.tool_call_id,
+            name=message.name,
+        )

unique_toolkit/reference_manager/reference_manager.py

@@ -3,7 +3,7 @@ from unique_toolkit.tools.schemas import ToolCallResponse
 
 
 class tool_chunks:
-    def __init__(self, name: str, chunks: list) -> None:
+    def __init__(self, name: str, chunks: list[ContentChunk]) -> None:
         self.name = name
         self.chunks = chunks
 
@@ -47,8 +47,21 @@ class ReferenceManager:
     def get_chunks(self) -> list[ContentChunk]:
         return self._chunks
 
-    def get_tool_chunks(self) -> dict:
+    def get_tool_chunks(self) -> dict[str, tool_chunks]:
         return self._tool_chunks
+    
+
+    def get_chunks_of_all_tools(self) -> list[list[ContentChunk]]:
+        return [tool_chunks.chunks for tool_chunks in self._tool_chunks.values()]
+
+    def get_chunks_of_tool(self, tool_call_id: str) -> list[ContentChunk]:
+        return self._tool_chunks.get(tool_call_id, tool_chunks("", [])).chunks
+
+    
+    def replace_chunks_of_tool(self, tool_call_id: str,chunks: list[ContentChunk]) -> None:
+        if tool_call_id in self._tool_chunks:
+            self._tool_chunks[tool_call_id].chunks = chunks
+
 
     def replace(self, chunks: list[ContentChunk]):
         self._chunks = chunks

{unique_toolkit-0.8.14.dist-info → unique_toolkit-0.8.15.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: unique_toolkit
-Version: 0.8.14
+Version: 0.8.15
 Summary: 
 License: Proprietary
 Author: Martin Fadler
@@ -114,6 +114,9 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), 
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## [0.8.15] - 2025-08-19
+- Added history loading from database for History Manager
+
 ## [0.8.14] - 2025-08-19
 - Including GPT-5 series deployed via LiteLLM into language model info
 

{unique_toolkit-0.8.14.dist-info → unique_toolkit-0.8.15.dist-info}/RECORD

@@ -1,7 +1,10 @@
 unique_toolkit/__init__.py,sha256=waK7W0EK3v2RJ26hawccwVz1i3yHGvHIIu5qgGjEGHQ,583
 unique_toolkit/_common/_base_service.py,sha256=S8H0rAebx7GsOldA7xInLp3aQJt9yEPDQdsGSFRJsGg,276
 unique_toolkit/_common/_time_utils.py,sha256=ztmTovTvr-3w71Ns2VwXC65OKUUh-sQlzbHdKTQWm-w,135
+unique_toolkit/_common/default_language_model.py,sha256=M6OiVfpi21CixfgYFigOcJGqG8r987f2rxHnn0NZ2dc,333
 unique_toolkit/_common/exception.py,sha256=caQIE1btsQnpKCHqL2cgWUSbHup06enQu_Pt7uGUTTE,727
+unique_toolkit/_common/token/image_token_counting.py,sha256=VpFfZyY0GIH27q_Wy4YNjk2algqvbCtJyzuuROoFQPw,2189
+unique_toolkit/_common/token/token_counting.py,sha256=l8tDo5EaD5FIlKz7Zd6CTNYwMhF-UZ2S3Hb-pU5z2UY,6281
 unique_toolkit/_common/validate_required_values.py,sha256=Y_M1ub9gIKP9qZ45F6Zq3ZHtuIqhmOjl8Z2Vd3avg8w,588
 unique_toolkit/_common/validators.py,sha256=uPGPkeygNi3KimWZxKOKYFxwpCxTkhhYBAn-b_5TS_M,2584
 unique_toolkit/app/__init__.py,sha256=ETxYDpEizg_PKmi4JPX_P76ySq-us-xypfAIdKQ1QZU,1284
@@ -54,7 +57,9 @@ unique_toolkit/framework_utilities/langchain/history.py,sha256=R9RuCeSFNaUO3OZ0G
 unique_toolkit/framework_utilities/openai/client.py,sha256=IasxPXlVJHIsZdXHin7yq-5tO4RNLUu9cEuhrgb4ghE,1205
 unique_toolkit/framework_utilities/openai/message_builder.py,sha256=VU6mJm_upLcarJQKFft_t1RlLRncWDxDuLC5LIJ5lQQ,4339
 unique_toolkit/framework_utilities/utils.py,sha256=JK7g2yMfEx3eMprug26769xqNpS5WJcizf8n2zWMBng,789
-unique_toolkit/history_manager/history_manager.py,sha256=ScC9CrkJCNbxFbWqTO-vpPlCUAUADpJCX2AqUd4Ff04,10104
+unique_toolkit/history_manager/history_construction_with_contents.py,sha256=xKUVnJ4ZJq4-nnO2_35dbDh9d-zfCJfRzuj7v9hXUdM,9049
+unique_toolkit/history_manager/history_manager.py,sha256=ULtsC7cGl92G2fXKIkEajH3tIy_qqWKIK8FudpNKhu4,8834
+unique_toolkit/history_manager/loop_token_reducer.py,sha256=-7Ezk3OLUsrU0Jd9Qc73_PBJZIayz7bVE3awc-q6Se0,17624
 unique_toolkit/history_manager/utils.py,sha256=3GT53SfOQ7g-dN3PHFIPaAab74sUfV28hbUtGMdX-bY,5607
 unique_toolkit/language_model/__init__.py,sha256=lRQyLlbwHbNFf4-0foBU13UGb09lwEeodbVsfsSgaCk,1971
 unique_toolkit/language_model/builder.py,sha256=4OKfwJfj3TrgO1ezc_ewIue6W7BCQ2ZYQXUckWVPPTA,3369
@@ -68,7 +73,7 @@ unique_toolkit/language_model/service.py,sha256=N_I3VtK5B0G8s5c6TcBVWM7CcLGqakDh
 unique_toolkit/language_model/utils.py,sha256=bPQ4l6_YO71w-zaIPanUUmtbXC1_hCvLK0tAFc3VCRc,1902
 unique_toolkit/postprocessor/postprocessor_manager.py,sha256=68TAcXMU_ohWOtzo91LntY950HV9I9gGU92-V0Mxmr8,4239
 unique_toolkit/protocols/support.py,sha256=V15WEIFKVMyF1QCnR8vIi4GrJy4dfTCB6d6JlqPZ58o,2341
-unique_toolkit/reference_manager/reference_manager.py,sha256=JofRoTcnB-Azj9X10kmhqqTUUyPp4GRRWBmaiybzaUo,3446
+unique_toolkit/reference_manager/reference_manager.py,sha256=WIvZkRgQztkY0zNTM_KIPSqJFT22HIGNexJ4yG3aj5E,3993
 unique_toolkit/short_term_memory/__init__.py,sha256=2mI3AUrffgH7Yt-xS57EGqnHf7jnn6xquoKEhJqk3Wg,185
 unique_toolkit/short_term_memory/constants.py,sha256=698CL6-wjup2MvU19RxSmQk3gX7aqW_OOpZB7sbz_Xg,34
 unique_toolkit/short_term_memory/functions.py,sha256=3WiK-xatY5nh4Dr5zlDUye1k3E6kr41RiscwtTplw5k,4484
@@ -90,7 +95,7 @@ unique_toolkit/tools/utils/execution/execution.py,sha256=vjG2Y6awsGNtlvyQAGCTthQ
 unique_toolkit/tools/utils/source_handling/schema.py,sha256=pvNhtL2daDLpCVIQpfdn6R35GvKmITVLXjZNLAwpgUE,871
 unique_toolkit/tools/utils/source_handling/source_formatting.py,sha256=C7uayNbdkNVJdEARA5CENnHtNY1SU6etlaqbgHNyxaQ,9152
 unique_toolkit/tools/utils/source_handling/tests/test_source_formatting.py,sha256=zu3AJnYH9CMqZPrxKEH3IgI-fM3nlvIBuspJG6W6B18,6978
-unique_toolkit-0.8.14.dist-info/LICENSE,sha256=GlN8wHNdh53xwOPg44URnwag6TEolCjoq3YD_KrWgss,193
-unique_toolkit-0.8.14.dist-info/METADATA,sha256=5QadKjdIsrfsgxMGglGAN_qfn916APhtTcQWRc9S52c,27642
-unique_toolkit-0.8.14.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
-unique_toolkit-0.8.14.dist-info/RECORD,,
+unique_toolkit-0.8.15.dist-info/LICENSE,sha256=GlN8wHNdh53xwOPg44URnwag6TEolCjoq3YD_KrWgss,193
+unique_toolkit-0.8.15.dist-info/METADATA,sha256=HC0kUwceqxR_BLx4SUd7-mFMLZ4roX2mIxpTRy19xnE,27726
+unique_toolkit-0.8.15.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
+unique_toolkit-0.8.15.dist-info/RECORD,,