unique_toolkit 0.8.46__tar.gz → 0.8.48__tar.gz

{unique_toolkit-0.8.46 → unique_toolkit-0.8.48}/CHANGELOG.md

@@ -5,6 +5,13 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), 
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## [0.8.48] - 2025-09-05
+- Add documentation on language models to markdown
+
+## [0.8.47] - 2025-09-05
+- Removed old code
+- Fixed small bugs in history manager & set the hallucination to use gpt4o as default.
+
 ## [0.8.46] - 2025-09-04
 - Bugfix for hostname identification inside Unique cluster in `unique_settings.py`
 

{unique_toolkit-0.8.46 → unique_toolkit-0.8.48}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: unique_toolkit
-Version: 0.8.46
+Version: 0.8.48
 Summary: 
 License: Proprietary
 Author: Cedric Klinkert
@@ -117,6 +117,13 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), 
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## [0.8.48] - 2025-09-05
+- Add documentation on language models to markdown
+
+## [0.8.47] - 2025-09-05
+- Removed old code
+- Fixed small bugs in history manager & set the hallucination to use gpt4o as default.
+
 ## [0.8.46] - 2025-09-04
 - Bugfix for hostname identification inside Unique cluster in `unique_settings.py`
 

{unique_toolkit-0.8.46 → unique_toolkit-0.8.48}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "unique_toolkit"
-version = "0.8.46"
+version = "0.8.48"
 description = ""
 authors = [
     "Cedric Klinkert <cedric.klinkert@unique.ch>",

{unique_toolkit-0.8.46 → unique_toolkit-0.8.48}/unique_toolkit/evals/hallucination/constants.py

@@ -2,6 +2,7 @@ from typing import Any
 
 from pydantic import Field
 
+from unique_toolkit._common.default_language_model import DEFAULT_GPT_4o
 from unique_toolkit._common.validators import LMI
 from unique_toolkit.evals.config import EvaluationMetricConfig
 from unique_toolkit.evals.hallucination.prompts import (
@@ -14,7 +15,7 @@ from unique_toolkit.evals.schemas import (
     EvaluationMetricInputFieldName,
     EvaluationMetricName,
 )
-from unique_toolkit.language_model.infos import LanguageModelInfo, LanguageModelName
+from unique_toolkit.language_model.infos import LanguageModelInfo
 
 SYSTEM_MSG_KEY = "systemPrompt"
 USER_MSG_KEY = "userPrompt"
@@ -26,7 +27,7 @@ class HallucinationConfig(EvaluationMetricConfig):
     enabled: bool = False
     name: EvaluationMetricName = EvaluationMetricName.HALLUCINATION
     language_model: LMI = LanguageModelInfo.from_name(
-        LanguageModelName.AZURE_GPT_35_TURBO_0125,
+        DEFAULT_GPT_4o,
     )
     additional_llm_options: dict[str, Any] = Field(
         default={},

{unique_toolkit-0.8.46 → unique_toolkit-0.8.48}/unique_toolkit/history_manager/history_manager.py

@@ -36,7 +36,7 @@ class UploadedContentConfig(BaseModel):
     percent_for_uploaded_content: float = Field(
         default=0.6,
         ge=0.0,
-        lt=1.0,
+        le=1.0,
         description="The fraction of the max input tokens that will be reserved for the uploaded content.",
     )
 

{unique_toolkit-0.8.46 → unique_toolkit-0.8.48}/unique_toolkit/language_model/functions.py

@@ -10,7 +10,6 @@ from pydantic import BaseModel
 
 from unique_toolkit.chat.schemas import ChatMessage, ChatMessageRole
 from unique_toolkit.content.schemas import ContentChunk, ContentReference
-from unique_toolkit.evaluators import DOMAIN_NAME
 from unique_toolkit.language_model import (
     LanguageModelMessageRole,
     LanguageModelMessages,
@@ -34,7 +33,7 @@ from .constants import (
     DEFAULT_COMPLETE_TIMEOUT,
 )
 
-logger = logging.getLogger(f"toolkit.{DOMAIN_NAME}.{__name__}")
+logger = logging.getLogger(f"toolkit.language_model.{__name__}")
 
 
 def complete(

unique_toolkit-0.8.46/unique_toolkit/evaluators/__init__.py

@@ -1 +0,0 @@
-from .constants import DOMAIN_NAME as DOMAIN_NAME

unique_toolkit-0.8.46/unique_toolkit/evaluators/config.py

@@ -1,26 +0,0 @@
-from humps import camelize
-from pydantic import BaseModel, ConfigDict
-
-from unique_toolkit._common.validators import LMI, LanguageModelInfo
-from unique_toolkit.evaluators.schemas import (
-    EvaluationMetricName,
-)
-from unique_toolkit.language_model.infos import (
-    LanguageModelName,
-)
-
-
-class EvaluationMetricConfig(BaseModel):
-    model_config = ConfigDict(
-        alias_generator=camelize,
-        populate_by_name=True,
-        validate_default=True,
-    )
-
-    enabled: bool = False
-    name: EvaluationMetricName
-    language_model: LMI = LanguageModelInfo.from_name(
-        LanguageModelName.AZURE_GPT_35_TURBO_0125,
-    )
-    custom_prompts: dict[str, str] = {}
-    score_to_emoji: dict[str, str] = {}

unique_toolkit-0.8.46/unique_toolkit/evaluators/constants.py

@@ -1 +0,0 @@
-DOMAIN_NAME = "evaluators"

unique_toolkit-0.8.46/unique_toolkit/evaluators/context_relevancy/constants.py

@@ -1,34 +0,0 @@
-from unique_toolkit.evaluators.config import EvaluationMetricConfig
-from unique_toolkit.evaluators.context_relevancy.prompts import (
-    CONTEXT_RELEVANCY_METRIC_SYSTEM_MSG,
-    CONTEXT_RELEVANCY_METRIC_USER_MSG,
-)
-from unique_toolkit.evaluators.schemas import (
-    EvaluationMetricInputFieldName,
-    EvaluationMetricName,
-)
-from unique_toolkit.language_model.infos import LanguageModelInfo
-from unique_toolkit.language_model.service import LanguageModelName
-
-SYSTEM_MSG_KEY = "systemPrompt"
-USER_MSG_KEY = "userPrompt"
-
-# Required input fields for context relevancy evaluation
-context_relevancy_required_input_fields = [
-    EvaluationMetricInputFieldName.INPUT_TEXT,
-    EvaluationMetricInputFieldName.CONTEXT_TEXTS,
-]
-
-
-default_config = EvaluationMetricConfig(
-    enabled=False,
-    name=EvaluationMetricName.CONTEXT_RELEVANCY,
-    language_model=LanguageModelInfo.from_name(
-        LanguageModelName.AZURE_GPT_35_TURBO_0125
-    ),
-    score_to_emoji={"LOW": "🟢", "MEDIUM": "🟡", "HIGH": "🔴"},
-    custom_prompts={
-        SYSTEM_MSG_KEY: CONTEXT_RELEVANCY_METRIC_SYSTEM_MSG,
-        USER_MSG_KEY: CONTEXT_RELEVANCY_METRIC_USER_MSG,
-    },
-)

unique_toolkit-0.8.46/unique_toolkit/evaluators/context_relevancy/prompts.py

@@ -1,31 +0,0 @@
-CONTEXT_RELEVANCY_METRIC_SYSTEM_MSG = """
-You will receive an input and a set of contexts.
-Your task is to evaluate how relevant the contexts are to the input text.
-
-Use the following rating scale to generate a score:
-[low] - The contexts are not relevant to the input.
-[medium] - The contexts are somewhat relevant to the input.
-[high] - The contexts are highly relevant to the input.
-
-Your answer must be in JSON format:
-{
- "reason": Your explanation of your judgement of the evaluation,
- "value": decision, must be one of the following ["low", "medium", "high"]
-}
-"""
-
-CONTEXT_RELEVANCY_METRIC_USER_MSG = """
-Here is the data:
-
-Input:
-'''
-$input_text
-'''
-
-Contexts:
-'''
-$context_texts
-'''
-
-Answer as JSON:
-"""

unique_toolkit-0.8.46/unique_toolkit/evaluators/context_relevancy/service.py

@@ -1,53 +0,0 @@
-from logging import Logger
-
-from unique_toolkit.app.schemas import Event
-from unique_toolkit.evaluators.config import EvaluationMetricConfig
-from unique_toolkit.evaluators.context_relevancy.constants import default_config
-from unique_toolkit.evaluators.context_relevancy.utils import (
-    check_context_relevancy_async,
-)
-from unique_toolkit.evaluators.schemas import (
-    EvaluationMetricInput,
-    EvaluationMetricResult,
-)
-
-
-class ContextRelevancyEvaluator:
-    def __init__(
-        self,
-        event: Event,
-        logger: Logger,
-    ):
-        self.event = event
-        self.logger = logger
-
-    async def run(
-        self,
-        input: EvaluationMetricInput,
-        config: EvaluationMetricConfig = default_config,
-    ) -> EvaluationMetricResult | None:
-        """
-        Analyzes the level of relevancy of a context by comparing
-        it with the input text.
-
-        Args:
-            input (EvaluationMetricInput): The input for the metric.
-            config (EvaluationMetricConfig): The configuration for the metric.
-
-        Returns:
-            EvaluationMetricResult | None: The result of the evaluation, indicating the level of context relevancy.
-                                           Returns None if the metric is not enabled.
-
-        Raises:
-            EvaluatorException: If required fields are missing or an error occurs during evaluation.
-        """
-        if config.enabled is False:
-            self.logger.info("Context relevancy metric is not enabled.")
-            return None
-
-        return await check_context_relevancy_async(
-            company_id=self.event.company_id,
-            input=input,
-            config=config,
-            logger=self.logger,
-        )

unique_toolkit-0.8.46/unique_toolkit/evaluators/context_relevancy/utils.py

@@ -1,156 +0,0 @@
-import logging
-from string import Template
-
-from unique_toolkit.evaluators.config import (
-    EvaluationMetricConfig,
-)
-from unique_toolkit.evaluators.context_relevancy.constants import (
-    SYSTEM_MSG_KEY,
-    USER_MSG_KEY,
-    context_relevancy_required_input_fields,
-)
-from unique_toolkit.evaluators.context_relevancy.prompts import (
-    CONTEXT_RELEVANCY_METRIC_SYSTEM_MSG,
-    CONTEXT_RELEVANCY_METRIC_USER_MSG,
-)
-from unique_toolkit.evaluators.exception import EvaluatorException
-from unique_toolkit.evaluators.output_parser import (
-    parse_eval_metric_result,
-)
-from unique_toolkit.evaluators.schemas import (
-    EvaluationMetricInput,
-    EvaluationMetricName,
-    EvaluationMetricResult,
-)
-from unique_toolkit.language_model import LanguageModelName
-from unique_toolkit.language_model.schemas import (
-    LanguageModelMessages,
-    LanguageModelSystemMessage,
-    LanguageModelUserMessage,
-)
-from unique_toolkit.language_model.service import LanguageModelService
-
-logger = logging.getLogger(__name__)
-
-
-async def check_context_relevancy_async(
-    company_id: str,
-    evaluation_metric_input: EvaluationMetricInput,
-    config: EvaluationMetricConfig,
-    logger: logging.Logger = logger,
-) -> EvaluationMetricResult | None:
-    """Analyzes the relevancy of the context provided for the given evaluation_metric_input and output.
-
-    The analysis classifies the context relevancy level as:
-    - low
-    - medium
-    - high
-
-    This method performs the following steps:
-    1. Logs the start of the analysis using the provided `logger`.
-    2. Validates the required fields in the `evaluation_metric_input` data.
-    3. Retrieves the messages using the `_get_msgs` method.
-    4. Calls `LanguageModelService.complete_async_util` to get a completion result.
-    5. Parses and returns the evaluation metric result based on the content of the completion result.
-
-    Args:
-        company_id (str): The company ID for the analysis.
-        evaluation_metric_input (EvaluationMetricInput): The evaluation_metric_input data used for evaluation, including the generated output and reference information.
-        config (EvaluationMetricConfig): Configuration settings for the evaluation.
-        logger (Optional[logging.Logger], optional): The logger used for logging information and errors. Defaults to the logger for the current module.
-
-    Returns:
-        EvaluationMetricResult | None: The result of the evaluation, indicating the level of context relevancy. Returns `None` if an error occurs.
-
-    Raises:
-        EvaluatorException: If required fields are missing or an error occurs during the evaluation.
-
-    """
-    model_group_name = (
-        config.language_model.name.value
-        if isinstance(config.language_model.name, LanguageModelName)
-        else config.language_model.name
-    )
-    logger.info(f"Analyzing context relevancy with {model_group_name}.")
-
-    evaluation_metric_input.validate_required_fields(
-        context_relevancy_required_input_fields,
-    )
-
-    if (
-        evaluation_metric_input.context_texts
-        and len(evaluation_metric_input.context_texts) == 0
-    ):
-        error_message = "No context texts provided."
-        raise EvaluatorException(
-            user_message=error_message,
-            error_message=error_message,
-        )
-
-    try:
-        msgs = _get_msgs(evaluation_metric_input, config)
-        result = await LanguageModelService.complete_async_util(
-            company_id=company_id,
-            messages=msgs,
-            model_name=model_group_name,
-        )
-        result_content = result.choices[0].message.content
-        if not result_content:
-            error_message = "Context relevancy evaluation did not return a result."
-            raise EvaluatorException(
-                error_message=error_message,
-                user_message=error_message,
-            )
-        return parse_eval_metric_result(
-            result_content,  # type: ignore
-            EvaluationMetricName.CONTEXT_RELEVANCY,
-        )
-    except Exception as e:
-        error_message = "Error occurred during context relevancy metric analysis"
-        raise EvaluatorException(
-            error_message=f"{error_message}: {e}",
-            user_message=error_message,
-            exception=e,
-        )
-
-
-def _get_msgs(
-    evaluation_metric_input: EvaluationMetricInput,
-    config: EvaluationMetricConfig,
-) -> LanguageModelMessages:
-    """Composes the messages for context relevancy analysis.
-
-    The messages are based on the provided evaluation_metric_input and configuration.
-
-    Args:
-        evaluation_metric_input (EvaluationMetricInput): The evaluation_metric_input data that includes context texts for the analysis.
-        config (EvaluationMetricConfig): The configuration settings for composing messages.
-
-    Returns:
-        LanguageModelMessages: The composed messages as per the provided evaluation_metric_input and configuration.
-
-    """
-    system_msg_content = _get_system_prompt(config)
-    system_msg = LanguageModelSystemMessage(content=system_msg_content)
-
-    user_msg_templ = Template(_get_user_prompt(config))
-    user_msg_content = user_msg_templ.substitute(
-        evaluation_metric_input_text=evaluation_metric_input.evaluation_metric_input_text,
-        contexts_text=evaluation_metric_input.get_joined_context_texts(),
-    )
-    user_msg = LanguageModelUserMessage(content=user_msg_content)
-    return LanguageModelMessages([system_msg, user_msg])
-
-
-def _get_system_prompt(config: EvaluationMetricConfig):
-    return config.custom_prompts.setdefault(
-        SYSTEM_MSG_KEY,
-        CONTEXT_RELEVANCY_METRIC_SYSTEM_MSG,
-    )
-
-
-def _get_user_prompt(config: EvaluationMetricConfig):
-    return config.custom_prompts.setdefault(
-        USER_MSG_KEY,
-        CONTEXT_RELEVANCY_METRIC_USER_MSG,
-    )

unique_toolkit-0.8.46/unique_toolkit/evaluators/exception.py

@@ -1,5 +0,0 @@
-from unique_toolkit._common.exception import CommonException
-
-
-class EvaluatorException(CommonException):
-    pass

unique_toolkit-0.8.46/unique_toolkit/evaluators/hallucination/constants.py

@@ -1,41 +0,0 @@
-from unique_toolkit.evaluators.config import EvaluationMetricConfig
-from unique_toolkit.evaluators.hallucination.prompts import (
-    HALLUCINATION_METRIC_SYSTEM_MSG,
-    HALLUCINATION_METRIC_SYSTEM_MSG_DEFAULT,
-    HALLUCINATION_METRIC_USER_MSG,
-    HALLUCINATION_METRIC_USER_MSG_DEFAULT,
-)
-from unique_toolkit.evaluators.schemas import (
-    EvaluationMetricInputFieldName,
-    EvaluationMetricName,
-)
-from unique_toolkit.language_model.infos import (
-    LanguageModelInfo,
-    LanguageModelName,
-)
-
-SYSTEM_MSG_KEY = "systemPrompt"
-USER_MSG_KEY = "userPrompt"
-SYSTEM_MSG_DEFAULT_KEY = "systemPromptDefault"
-USER_MSG_DEFAULT_KEY = "userPromptDefault"
-
-
-hallucination_metric_default_config = EvaluationMetricConfig(
-    enabled=False,
-    name=EvaluationMetricName.HALLUCINATION,
-    language_model=LanguageModelInfo.from_name(LanguageModelName.AZURE_GPT_4_0613),
-    score_to_emoji={"LOW": "🟢", "MEDIUM": "🟡", "HIGH": "🔴"},
-    custom_prompts={
-        SYSTEM_MSG_KEY: HALLUCINATION_METRIC_SYSTEM_MSG,
-        USER_MSG_KEY: HALLUCINATION_METRIC_USER_MSG,
-        SYSTEM_MSG_DEFAULT_KEY: HALLUCINATION_METRIC_SYSTEM_MSG_DEFAULT,
-        USER_MSG_DEFAULT_KEY: HALLUCINATION_METRIC_USER_MSG_DEFAULT,
-    },
-)
-
-hallucination_required_input_fields = [
-    EvaluationMetricInputFieldName.INPUT_TEXT,
-    EvaluationMetricInputFieldName.CONTEXT_TEXTS,
-    EvaluationMetricInputFieldName.HISTORY_MESSAGES,
-    EvaluationMetricInputFieldName.OUTPUT_TEXT,
-]

unique_toolkit-0.8.46/unique_toolkit/evaluators/hallucination/prompts.py

@@ -1,79 +0,0 @@
-HALLUCINATION_METRIC_SYSTEM_MSG = """
-You will receive a question, references, a conversation between a user and an agent, and an output. 
-The output is the answer to the question. 
-Your task is to evaluate if the output is fully supported by the information provided in the references and conversation, and provide explanations on your judgement in 2 sentences.
-
-Use the following entailment scale to generate a score:
-[low] - All information in output is supported by the references/conversation, or extractions from the references/conversation.
-[medium] - The output is supported by the references/conversation to some extent, but there is at least some information in the output that is not discussed in the references/conversation. For example, if an instruction asks about two concepts and the references/conversation only discusses either of them, it should be considered a [medium] hallucination level.
-[high] - The output contains information that is not part of the references/conversation, is unrelated to the references/conversation, or contradicts the references/conversation.
-
-Make sure to not use any external information/knowledge to judge whether the output is true or not. Only check whether the output is supported by the references/conversation, and not whether the output is correct or not. Also do not evaluate if the references/conversation contain further information that is not part of the output but could be relevant to the qestion.
-
-Your answer must be in JSON format:
-{
- "reason": Your explanation of your judgement of the evaluation,
- "value": decision, must be one of the following: ["high", "medium", "low"]
-}                                                  
-"""
-
-HALLUCINATION_METRIC_USER_MSG = """
-Here is the data:
-
-Input:
-'''
-$input_text
-'''
-
-References:
-'''
-$contexts_text
-'''
-
-Conversation:
-'''
-$history_messages_text
-'''
-
-Output:
-'''
-$output_text
-'''
-
-Answer as JSON:
-"""
-
-HALLUCINATION_METRIC_SYSTEM_MSG_DEFAULT = """
-You will receive a question and an output. 
-The output is the answer to the question. 
-The situation is that no references could be found to answer the question. Your task is to evaluate if the output contains any information to answer the question,
-and provide a short explanations of your reasoning in 2 sentences. Also mention in your explanation that no references were provided to answer the question.
-
-Use the following entailment scale to generate a score:
-[low] - The output does not contain any information to answer the question.
-[medium] - The output contains some information to answer the question, but does not answer the question entirely. 
-[high] - The output answers the question.
-
-It is not considered an answer when the output relates to the questions subject. Make sure to not use any external information/knowledge to judge whether the output is true or not. Only check that the output does not answer the question, and not whether the output is correct or not.
-Your answer must be in JSON format:
-{
- "reason": Your explanation of your reasoning of the evaluation,
- "value": decision, must be one of the following: ["low", "medium", "high"]
-}
-"""
-
-HALLUCINATION_METRIC_USER_MSG_DEFAULT = """                                                  
-Here is the data:
-
-Input:
-'''
-$input_text
-'''
-
-Output:
-'''
-$output_text
-'''
-
-Answer as JSON:
-"""

unique_toolkit-0.8.46/unique_toolkit/evaluators/hallucination/service.py

@@ -1,58 +0,0 @@
-import logging
-
-from unique_toolkit.app.schemas import Event
-from unique_toolkit.evaluators.config import (
-    EvaluationMetricConfig,
-)
-from unique_toolkit.evaluators.hallucination.constants import (
-    hallucination_metric_default_config,
-)
-from unique_toolkit.evaluators.hallucination.utils import check_hallucination_async
-from unique_toolkit.evaluators.schemas import (
-    EvaluationMetricInput,
-    EvaluationMetricResult,
-)
-
-logger = logging.getLogger(__name__)
-
-
-class HallucinationEvaluator:
-    def __init__(self, event: Event, logger: logging.Logger = logger):
-        self.event = event
-        self.logger = logger
-
-    async def run(
-        self,
-        input: EvaluationMetricInput,
-        config: EvaluationMetricConfig = hallucination_metric_default_config,
-    ) -> EvaluationMetricResult | None:
-        """
-        Analyzes the level of hallucination in the generated output by comparing it with the input
-        and the provided contexts or history. The analysis classifies the hallucination level as:
-        - low
-        - medium
-        - high
-
-        If no contexts or history are referenced in the generated output, the method verifies
-        that the output does not contain any relevant information to answer the question.
-
-        This method calls `check_hallucination_async` to perform the actual analysis. The `check_hallucination_async`
-        function handles the evaluation using the company ID from the event, the provided input, and the configuration.
-
-        Args:
-            input (EvaluationMetricInput): The input data used for evaluation, including the generated output and reference information.
-            config (EvaluationMetricConfig, optional): Configuration settings for the evaluation. Defaults to `hallucination_metric_default_config`.
-
-        Returns:
-            EvaluationMetricResult | None: The result of the evaluation, indicating the level of hallucination. Returns `None` if the analysis cannot be performed.
-
-        Raises:
-            EvaluatorException: If the context texts are empty, required fields are missing, or an error occurs during the evaluation.
-        """
-        if config.enabled is False:
-            self.logger.info("Hallucination metric is not enabled.")
-            return None
-
-        return await check_hallucination_async(
-            company_id=self.event.company_id, input=input, config=config
-        )