unique_toolkit 0.8.14__tar.gz → 0.8.16__tar.gz

{unique_toolkit-0.8.14 → unique_toolkit-0.8.16}/CHANGELOG.md

@@ -5,6 +5,12 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), 
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## [0.8.16] - 2025-08-19
+- moved Hallucination evaluator into toolkit
+
+## [0.8.15] - 2025-08-19
+- Added history loading from database for History Manager
+
 ## [0.8.14] - 2025-08-19
 - Including GPT-5 series deployed via LiteLLM into language model info
 

{unique_toolkit-0.8.14 → unique_toolkit-0.8.16}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: unique_toolkit
-Version: 0.8.14
+Version: 0.8.16
 Summary: 
 License: Proprietary
 Author: Martin Fadler
@@ -114,6 +114,12 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), 
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## [0.8.16] - 2025-08-19
+- moved Hallucination evaluator into toolkit
+
+## [0.8.15] - 2025-08-19
+- Added history loading from database for History Manager
+
 ## [0.8.14] - 2025-08-19
 - Including GPT-5 series deployed via LiteLLM into language model info
 

{unique_toolkit-0.8.14 → unique_toolkit-0.8.16}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "unique_toolkit"
-version = "0.8.14"
+version = "0.8.16"
 description = ""
 authors = [
     "Martin Fadler <martin.fadler@unique.ch>",

unique_toolkit-0.8.16/unique_toolkit/_common/default_language_model.py

@@ -0,0 +1,6 @@
+from unique_toolkit.language_model.infos import LanguageModelName
+
+DEFAULT_GPT_35_TURBO = LanguageModelName.AZURE_GPT_35_TURBO_0125
+DEFAULT_GPT_4o = LanguageModelName.AZURE_GPT_4o_2024_1120
+DEFAULT_GPT_4o_STRUCTURED_OUTPUT = LanguageModelName.AZURE_GPT_4o_2024_0806
+DEFAULT_GPT_4o_MINI = LanguageModelName.AZURE_GPT_4o_MINI_2024_0718

unique_toolkit-0.8.16/unique_toolkit/_common/token/image_token_counting.py

@@ -0,0 +1,67 @@
+import base64
+import math
+import re
+from enum import Enum
+from io import BytesIO
+
+from PIL import Image
+
+
+class DetailLevel(Enum):
+    LOW = "low"
+    HIGH = "high"
+
+
+# https://platform.openai.com/docs/guides/vision/calculating-costs#calculating-costs
+def calculate_image_tokens(width, height, detail: DetailLevel):
+    """
+    Calculate the token cost of an image based on its dimensions and detail level.
+    NOTE: While we followed the documentation provided by openai to calculate image token cost, in practice,
+    we notice that this function overestimate the number of tokens consumed by the model.
+
+    Parameters:
+    - width (int): The width of the image in pixels.
+    - height (int): The height of the image in pixels.
+    - detail (str): The detail level, either "low" or "high".
+
+    Returns:
+    - int: The token cost of the image.
+    """
+    # Base cost for low detail
+    if detail == DetailLevel.LOW:
+        return 85
+
+    # Scaling for high detail
+    # Scale down to fit within 2048x2048 square
+    max_long_dim = 2048
+    long_dim = max(width, height)
+    if long_dim > max_long_dim:
+        scale_factor = long_dim / max_long_dim
+        width = int(width / scale_factor)
+        height = int(height / scale_factor)
+
+    # Scale down the shortest side to 768
+    max_short_dim = 768
+    short_dim = min(width, height)
+    if short_dim > max_short_dim:
+        scale_factor = short_dim / max_short_dim
+        width = int(width / scale_factor)
+        height = int(height / scale_factor)
+
+    # Step 3: Calculate the number of 512x512 tiles
+    tiles = math.ceil(width / 512) * math.ceil(height / 512)
+    # Step 4: Compute token cost
+    token_cost = (tiles * 170) + 85
+    return token_cost
+
+
+def calculate_image_tokens_from_base64(base64_string: str):
+    base64_string = remove_base64_header(base64_string)
+    image = Image.open(BytesIO(base64.b64decode(base64_string)))
+    # DETAIL LEVEL HIGH IS THE DEFAULT TO BE ON THE SAFE SIDE
+    return calculate_image_tokens(image.width, image.height, DetailLevel.HIGH)
+
+
+def remove_base64_header(base64_string: str):
+    header_pattern = r"^data:image/\w+;base64,"
+    return re.sub(header_pattern, "", base64_string)

unique_toolkit-0.8.16/unique_toolkit/_common/token/token_counting.py

@@ -0,0 +1,196 @@
+# Original source
+# https://github.com/openai/openai-cookbook/blob/main/examples/How_to_count_tokens_with_tiktoken.ipynb
+
+import json
+from typing import Any, Callable
+
+from pydantic import BaseModel
+from unique_toolkit.language_model import (
+    LanguageModelMessage,
+    LanguageModelMessages,
+    LanguageModelName,
+)
+
+from _common.utils.token.image_token_counting import (
+    calculate_image_tokens_from_base64,
+)
+
+
+class SpecialToolCallingTokens(BaseModel):
+    func_init: int = 0
+    prop_init: int = 0
+    prop_key: int = 0
+    enum_init: int = 0
+    enum_item: int = 0
+    func_end: int = 0
+
+
+def get_special_token(model: LanguageModelName) -> SpecialToolCallingTokens:
+    special_token = SpecialToolCallingTokens()
+
+    match model:
+        case (
+            LanguageModelName.AZURE_GPT_4o_2024_0513
+            | LanguageModelName.AZURE_GPT_4o_2024_0806
+            | LanguageModelName.AZURE_GPT_4o_MINI_2024_0718
+            | LanguageModelName.AZURE_GPT_4o_2024_1120
+        ):
+            special_token.func_init = 7
+            special_token.prop_init = 3
+            special_token.prop_key = 3
+            special_token.enum_init = -3
+            special_token.enum_item = 3
+            special_token.func_end = 12
+
+        case (
+            LanguageModelName.AZURE_GPT_35_TURBO_0125
+            | LanguageModelName.AZURE_GPT_4_0613
+            | LanguageModelName.AZURE_GPT_4_32K_0613
+            | LanguageModelName.AZURE_GPT_4_TURBO_2024_0409
+        ):
+            special_token.func_init = 10
+            special_token.prop_init = 3
+            special_token.prop_key = 3
+            special_token.enum_init = -3
+            special_token.enum_item = 3
+            special_token.func_end = 12
+
+        case _:
+            raise NotImplementedError(
+                f"""num_tokens_for_tools() is not implemented for model {model}."""
+            )
+    return special_token
+
+
+def num_tokens_per_messages(
+    messages: list[dict[str, str]], encode: Callable[[str], list[int]]
+) -> list[int]:
+    """Return the number of tokens used by a list of messages."""
+
+    num_token_per_message = []
+    for message in messages:
+        num_tokens = 3  # extra_tokens_per_message
+        for key, value in message.items():
+            if isinstance(value, str):
+                num_tokens += len(encode(value))
+            elif isinstance(value, list):
+                # NOTE: The result returned by the function below is not 100% accurate.
+                num_tokens += handle_message_with_images(value, encode)
+            if key == "name":
+                num_tokens += 1  # extra_tokens_per_name
+
+        num_token_per_message.append(num_tokens)
+
+    return num_token_per_message
+
+
+def num_tokens_from_messages(
+    messages: list[dict[str, str]], encode: Callable[[str], list[int]]
+) -> int:
+    """Return the number of tokens used by a list of messages."""
+
+    num_tokens_per_message = num_tokens_per_messages(messages, encode)
+    num_tokens = sum(num_tokens_per_message) + 3
+
+    return num_tokens
+
+
+def num_tokens_for_tools(
+    functions: list[dict[str, Any]],
+    special_token: SpecialToolCallingTokens,
+    encode: Callable[[str], list[int]],
+):
+    def num_token_function_enum(
+        properties: dict[str, Any], encode: Callable[[str], list[int]]
+    ):
+        enum_token_count = 0
+        enum_token_count += special_token.enum_init
+        for item in properties[key]["enum"]:
+            enum_token_count += special_token.enum_item
+            enum_token_count += len(encode(item))
+
+        return enum_token_count
+
+    func_token_count = 0
+    if len(functions) > 0:
+        for func in functions:
+            func_token_count += special_token.func_init
+            function = func.get("function", {})
+            func_token_count += len(
+                encode(
+                    function.get("name", "")
+                    + ":"
+                    + function.get("description", "").rstrip(".").rstrip()
+                )
+            )
+            if len(function.get("parameters", {}).get("properties", "")) > 0:
+                properties = function.get("parameters", {}).get(
+                    "properties", ""
+                )
+                func_token_count += special_token.prop_init
+
+                for key in list(properties.keys()):
+                    func_token_count += special_token.prop_key
+
+                    if "enum" in properties[key].keys():
+                        func_token_count += num_token_function_enum(
+                            properties, encode
+                        )
+
+                    func_token_count += len(
+                        encode(
+                            f"{key}:{properties[key]['type']}:{properties[key]['description'].rstrip('.').rstrip()}"
+                        )
+                    )
+
+        func_token_count += special_token.func_end
+
+    return func_token_count
+
+
+def handle_message_with_images(
+    message: list[dict], encode: Callable[[str], list[int]]
+):
+    token_count = 0
+    for item in message:
+        if item.get("type") == "image_url":
+            image_url = item.get("imageUrl", {}).get("url")
+            if image_url:
+                token_count += calculate_image_tokens_from_base64(image_url)
+        elif item.get("type") == "text":
+            token_count += len(encode(item.get("text", "")))
+    return token_count
+
+
+def messages_to_openai_messages(
+    messages: LanguageModelMessages | list[LanguageModelMessage],
+):
+    if isinstance(messages, list):
+        messages = LanguageModelMessages(messages)
+
+    return [
+        {
+            k: v
+            for k, v in m.items()
+            if (k in ["content", "role"] and v is not None)
+        }
+        for m in json.loads(messages.model_dump_json())
+    ]
+
+
+def num_tokens_per_language_model_message(
+    messages: LanguageModelMessages | list[LanguageModelMessage],
+    encode: Callable[[str], list[int]],
+) -> list[int]:
+    return num_tokens_per_messages(
+        messages=messages_to_openai_messages(messages), encode=encode
+    )
+
+
+def num_token_for_language_model_messages(
+    messages: LanguageModelMessages | list[LanguageModelMessage],
+    encode: Callable[[str], list[int]],
+) -> int:
+    return num_tokens_from_messages(
+        messages_to_openai_messages(messages), encode
+    )

unique_toolkit-0.8.16/unique_toolkit/evals/config.py

@@ -0,0 +1,36 @@
+from typing import Any
+
+from humps import camelize
+from pydantic import BaseModel, ConfigDict, Field
+
+from unique_toolkit._common.validators import LMI
+from unique_toolkit.language_model.infos import LanguageModelInfo, LanguageModelName
+
+
+from .schemas import (
+    EvaluationMetricName,
+)
+
+model_config = ConfigDict(
+    alias_generator=camelize,
+    populate_by_name=True,
+    arbitrary_types_allowed=True,
+    validate_default=True,
+)
+
+
+class EvaluationMetricConfig(BaseModel):
+    model_config = model_config
+
+    enabled: bool = False
+    name: EvaluationMetricName
+    language_model: LMI = LanguageModelInfo.from_name(
+        LanguageModelName.AZURE_GPT_35_TURBO_0125,
+    )
+    additional_llm_options: dict[str, Any] = Field(
+        default={},
+        description="Additional options to pass to the language model.",
+    )
+    custom_prompts: dict[str, str] = {}
+    score_to_label: dict[str, str] = {}
+    score_to_title: dict[str, str] = {}

unique_toolkit-0.8.16/unique_toolkit/evals/context_relevancy/prompts.py

@@ -0,0 +1,56 @@
+CONTEXT_RELEVANCY_METRIC_SYSTEM_MSG = """
+You will receive an input and a set of contexts.
+Your task is to evaluate how relevant the contexts are to the input text.
+
+Use the following rating scale to generate a score:
+[low] - The contexts are not relevant to the input.
+[medium] - The contexts are somewhat relevant to the input.
+[high] - The contexts are highly relevant to the input.
+
+Your answer must be in JSON format:
+{
+ "reason": Your explanation of your judgement of the evaluation,
+ "value": decision, must be one of the following ["low", "medium", "high"]
+}
+"""
+
+CONTEXT_RELEVANCY_METRIC_SYSTEM_MSG_STRUCTURED_OUTPUT = """
+You will receive an input and a set of contexts.
+Your task is to evaluate how relevant the contexts are to the input text. 
+Further you should extract relevant facts from the contexts.
+
+# Output Format
+- Generate data according to the provided data schema.
+- Ensure the output adheres to the format required by the pydantic object.
+- All necessary fields should be populated as per the data schema guidelines.
+"""
+
+CONTEXT_RELEVANCY_METRIC_USER_MSG = """
+Here is the data:
+
+Input:
+'''
+$input_text
+'''
+
+Contexts:
+'''
+$context_texts
+'''
+
+Answer as JSON:
+"""
+
+CONTEXT_RELEVANCY_METRIC_USER_MSG_STRUCTURED_OUTPUT = """
+Here is the data:
+
+Input:
+'''
+$input_text
+'''
+
+Contexts:
+'''
+$context_texts
+'''
+"""

unique_toolkit-0.8.16/unique_toolkit/evals/context_relevancy/schema.py

@@ -0,0 +1,88 @@
+from pydantic import BaseModel, Field, create_model
+from pydantic.json_schema import SkipJsonSchema
+
+
+
+from pydantic import BaseModel, ConfigDict
+
+from unique_toolkit.tools.config import get_configuration_dict
+
+
+class StructuredOutputModel(BaseModel):
+    model_config = ConfigDict(extra="forbid")
+
+
+
+class StructuredOutputConfig(BaseModel):
+    model_config = get_configuration_dict()
+
+    enabled: bool = Field(
+        default=False,
+        description="Whether to use structured output for the evaluation.",
+    )
+    extract_fact_list: bool = Field(
+        default=False,
+        description="Whether to extract a list of relevant facts from context chunks with structured output.",
+    )
+    reason_description: str = Field(
+        default="A brief explanation justifying your evaluation decision.",
+        description="The description of the reason field for structured output.",
+    )
+    value_description: str = Field(
+        default="Assessment of how relevant the facts are to the query. Must be one of: ['low', 'medium', 'high'].",
+        description="The description of the value field for structured output.",
+    )
+
+    fact_description: str = Field(
+        default="A fact is an information that is directly answers the user's query. Make sure to emphasize the important information from the fact with bold text.",
+        description="The description of the fact field for structured output.",
+    )
+    fact_list_description: str = Field(
+        default="A list of relevant facts extracted from the source that supports or answers the user's query.",
+        description="The description of the fact list field for structured output.",
+    )
+
+
+class Fact(StructuredOutputModel):
+    fact: str
+
+
+class EvaluationSchemaStructuredOutput(StructuredOutputModel):
+    reason: str
+    value: str
+
+    fact_list: list[Fact] = Field(default_factory=list[Fact])
+
+    @classmethod
+    def get_with_descriptions(cls, config: StructuredOutputConfig):
+        if config.extract_fact_list:
+            FactWithDescription = create_model(
+                "Fact",
+                fact=(str, Field(..., description=config.fact_description)),
+                __base__=Fact,
+            )
+            fact_list_field = (
+                list[FactWithDescription],
+                Field(
+                    description=config.fact_list_description,
+                ),
+            )
+        else:
+            fact_list_field = (
+                SkipJsonSchema[list[Fact]],
+                Field(default_factory=list[Fact]),
+            )
+
+        return create_model(
+            "EvaluationSchemaStructuredOutputWithDescription",
+            reason=(
+                str,
+                Field(..., description=config.reason_description),
+            ),
+            value=(
+                str,
+                Field(..., description=config.value_description),
+            ),
+            fact_list=fact_list_field,
+            __base__=cls,
+        )