unique_toolkit 0.5.24__py3-none-any.whl → 0.5.28__py3-none-any.whl

unique_toolkit/_common/exception.py

@@ -0,0 +1,33 @@
+from typing import Optional
+
+
+class CommonException(Exception):
+    def __init__(
+        self,
+        user_message: str,
+        error_message: str,
+        exception: Optional[Exception] = None,
+    ):
+        super().__init__(error_message)
+        self._user_message = user_message
+        self._error_message = error_message
+        self._exception = exception
+
+    @property
+    def user_message(self):
+        return self._user_message
+
+    @property
+    def error_message(self):
+        return self._error_message
+
+    @property
+    def name(self):
+        return self.__class__.__name__
+
+    @property
+    def exception(self):
+        return self._exception
+
+    def __str__(self):
+        return self._error_message

unique_toolkit/_common/validators.py

@@ -0,0 +1,8 @@
+from unique_toolkit.language_model import LanguageModel, LanguageModelName
+
+
+def validate_and_init_language_model(value: LanguageModelName | LanguageModel | str):
+    if isinstance(value, LanguageModel):
+        return value
+
+    return LanguageModel(value)

unique_toolkit/content/service.py

@@ -204,6 +204,13 @@ class ContentService(BaseService):
 
         return self._map_contents(contents)
 
+    def search_content_on_chat(
+        self,
+    ) -> list[Content]:
+        where = {"ownerId": {"equals": self.event.payload.chat_id}}
+
+        return self.search_contents(where)
+
     @staticmethod
     def _map_content_chunk(content_chunk: dict):
         return ContentChunk(

unique_toolkit/evaluators/config.py

@@ -0,0 +1,35 @@
+from humps import camelize
+from pydantic import BaseModel, ConfigDict, field_validator
+
+from unique_toolkit._common.validators import validate_and_init_language_model
+from unique_toolkit.evaluators.schemas import (
+    EvaluationMetricName,
+)
+from unique_toolkit.language_model.infos import (
+    LanguageModel,
+    LanguageModelName,
+)
+
+model_config = ConfigDict(
+    alias_generator=camelize,
+    populate_by_name=True,
+    arbitrary_types_allowed=True,
+    validate_default=True,
+    json_encoders={LanguageModel: lambda v: v.display_name},
+)
+
+
+class EvaluationMetricConfig(BaseModel):
+    model_config = model_config
+
+    enabled: bool = False
+    name: EvaluationMetricName
+    language_model: LanguageModel = LanguageModel(
+        LanguageModelName.AZURE_GPT_35_TURBO_0613
+    )
+    custom_prompts: dict[str, str] = {}
+    score_to_emoji: dict[str, str] = {}
+
+    @field_validator("language_model", mode="before")
+    def validate_language_model(cls, value: LanguageModelName | LanguageModel):
+        return validate_and_init_language_model(value)

unique_toolkit/evaluators/context_relevancy/constants.py

@@ -0,0 +1,32 @@
+from unique_toolkit.evaluators.config import EvaluationMetricConfig
+from unique_toolkit.evaluators.context_relevancy.prompts import (
+    CONTEXT_RELEVANCY_METRIC_SYSTEM_MSG,
+    CONTEXT_RELEVANCY_METRIC_USER_MSG,
+)
+from unique_toolkit.evaluators.schemas import (
+    EvaluationMetricName,
+)
+from unique_toolkit.language_model.infos import LanguageModel
+from unique_toolkit.language_model.service import LanguageModelName
+
+SYSTEM_MSG_KEY = "systemPrompt"
+USER_MSG_KEY = "userPrompt"
+
+# Required input fields for context relevancy evaluation
+context_relevancy_required_input_fields = [
+    "input_text",
+    "output_text",
+    "context_texts",
+]
+
+
+default_config = EvaluationMetricConfig(
+    enabled=False,
+    name=EvaluationMetricName.CONTEXT_RELEVANCY,
+    language_model=LanguageModel(LanguageModelName.AZURE_GPT_35_TURBO_0613),
+    score_to_emoji={"LOW": "🟢", "MEDIUM": "🟡", "HIGH": "🔴"},
+    custom_prompts={
+        SYSTEM_MSG_KEY: CONTEXT_RELEVANCY_METRIC_SYSTEM_MSG,
+        USER_MSG_KEY: CONTEXT_RELEVANCY_METRIC_USER_MSG,
+    },
+)

unique_toolkit/evaluators/context_relevancy/prompts.py

@@ -0,0 +1,31 @@
+CONTEXT_RELEVANCY_METRIC_SYSTEM_MSG = """
+You will receive an input and a set of contexts.
+Your task is to evaluate how relevant the contexts are to the input text.
+
+Use the following rating scale to generate a score:
+[low] - The contexts are not relevant to the input.
+[medium] - The contexts are somewhat relevant to the input.
+[high] - The contexts are highly relevant to the input.
+
+Your answer must be in JSON format:
+{
+ "reason": Your explanation of your judgement of the evaluation,
+ "value": decision, must be one of the following ["low", "medium", "high"]
+}
+"""
+
+CONTEXT_RELEVANCY_METRIC_USER_MSG = """
+Here is the data:
+
+Input:
+'''
+$input_text
+'''
+
+Contexts:
+'''
+$context_texts
+'''
+
+Answer as JSON:
+"""

unique_toolkit/evaluators/context_relevancy/service.py

@@ -0,0 +1,53 @@
+from logging import Logger
+
+from unique_toolkit.app.schemas import Event
+from unique_toolkit.evaluators.config import EvaluationMetricConfig
+from unique_toolkit.evaluators.context_relevancy.constants import default_config
+from unique_toolkit.evaluators.context_relevancy.utils import (
+    check_context_relevancy_async,
+)
+from unique_toolkit.evaluators.schemas import (
+    EvaluationMetricInput,
+    EvaluationMetricResult,
+)
+
+
+class ContextRelevancyEvaluator:
+    def __init__(
+        self,
+        event: Event,
+        logger: Logger,
+    ):
+        self.event = event
+        self.logger = logger
+
+    async def run(
+        self,
+        input: EvaluationMetricInput,
+        config: EvaluationMetricConfig = default_config,
+    ) -> EvaluationMetricResult | None:
+        """
+        Analyzes the level of relevancy of a context by comparing
+        it with the input text.
+
+        Args:
+            input (EvaluationMetricInput): The input for the metric.
+            config (EvaluationMetricConfig): The configuration for the metric.
+
+        Returns:
+            EvaluationMetricResult | None: The result of the evaluation, indicating the level of context relevancy.
+                                           Returns None if the metric is not enabled.
+
+        Raises:
+            EvaluatorException: If required fields are missing or an error occurs during evaluation.
+        """
+        if config.enabled is False:
+            self.logger.info("Context relevancy metric is not enabled.")
+            return None
+
+        return await check_context_relevancy_async(
+            company_id=self.event.company_id,
+            input=input,
+            config=config,
+            logger=self.logger,
+        )

unique_toolkit/evaluators/context_relevancy/utils.py

@@ -0,0 +1,139 @@
+import logging
+from string import Template
+
+from unique_toolkit.evaluators.config import (
+    EvaluationMetricConfig,
+)
+from unique_toolkit.evaluators.context_relevancy.constants import (
+    SYSTEM_MSG_KEY,
+    USER_MSG_KEY,
+    context_relevancy_required_input_fields,
+)
+from unique_toolkit.evaluators.context_relevancy.prompts import (
+    CONTEXT_RELEVANCY_METRIC_SYSTEM_MSG,
+    CONTEXT_RELEVANCY_METRIC_USER_MSG,
+)
+from unique_toolkit.evaluators.exception import EvaluatorException
+from unique_toolkit.evaluators.output_parser import (
+    parse_eval_metric_result,
+)
+from unique_toolkit.evaluators.schemas import (
+    EvaluationMetricInput,
+    EvaluationMetricName,
+    EvaluationMetricResult,
+)
+from unique_toolkit.language_model.schemas import (
+    LanguageModelMessages,
+    LanguageModelSystemMessage,
+    LanguageModelUserMessage,
+)
+from unique_toolkit.language_model.service import LanguageModelService
+
+logger = logging.getLogger(__name__)
+
+
+async def check_context_relevancy_async(
+    company_id: str,
+    input: EvaluationMetricInput,
+    config: EvaluationMetricConfig,
+    logger: logging.Logger = logger,
+) -> EvaluationMetricResult | None:
+    """
+    Analyzes the relevancy of the context provided for the given input and output.
+    The analysis classifies the context relevancy level as:
+    - low
+    - medium
+    - high
+
+    This method performs the following steps:
+    1. Logs the start of the analysis using the provided `logger`.
+    2. Validates the required fields in the `input` data.
+    3. Retrieves the messages using the `_get_msgs` method.
+    4. Calls `LanguageModelService.complete_async_util` to get a completion result.
+    5. Parses and returns the evaluation metric result based on the content of the completion result.
+
+    Args:
+        company_id (str): The company ID for the analysis.
+        input (EvaluationMetricInput): The input data used for evaluation, including the generated output and reference information.
+        config (EvaluationMetricConfig): Configuration settings for the evaluation.
+        logger (Optional[logging.Logger], optional): The logger used for logging information and errors. Defaults to the logger for the current module.
+
+    Returns:
+        EvaluationMetricResult | None: The result of the evaluation, indicating the level of context relevancy. Returns `None` if an error occurs.
+
+    Raises:
+        EvaluatorException: If required fields are missing or an error occurs during the evaluation.
+    """
+    model_name = config.language_model.name
+    logger.info(f"Analyzing context relevancy with {model_name}.")
+
+    input.validate_required_fields(context_relevancy_required_input_fields)
+
+    if input.context_texts and len(input.context_texts) == 0:
+        error_message = "No context texts provided."
+        raise EvaluatorException(
+            user_message=error_message,
+            error_message=error_message,
+        )
+
+    try:
+        msgs = _get_msgs(input, config)
+        result = await LanguageModelService.complete_async_util(
+            company_id=company_id, messages=msgs, model_name=model_name
+        )
+        result_content = result.choices[0].message.content
+        if not result_content:
+            error_message = "Context relevancy evaluation did not return a result."
+            raise EvaluatorException(
+                error_message=error_message,
+                user_message=error_message,
+            )
+        return parse_eval_metric_result(
+            result_content, EvaluationMetricName.CONTEXT_RELEVANCY
+        )
+    except Exception as e:
+        error_message = "Error occurred during context relevancy metric analysis"
+        raise EvaluatorException(
+            error_message=f"{error_message}: {e}",
+            user_message=error_message,
+            exception=e,
+        )
+
+
+def _get_msgs(
+    input: EvaluationMetricInput,
+    config: EvaluationMetricConfig,
+):
+    """
+    Composes the messages for context relevancy analysis based on the provided input and configuration.
+
+    Args:
+        input (EvaluationMetricInput): The input data that includes context texts for the analysis.
+        config (EvaluationMetricConfig): The configuration settings for composing messages.
+
+    Returns:
+        LanguageModelMessages: The composed messages as per the provided input and configuration.
+    """
+    system_msg_content = _get_system_prompt(config)
+    system_msg = LanguageModelSystemMessage(content=system_msg_content)
+
+    user_msg_templ = Template(_get_user_prompt(config))
+    user_msg_content = user_msg_templ.substitute(
+        input_text=input.input_text, contexts_text=input.get_joined_context_texts()
+    )
+    user_msg = LanguageModelUserMessage(content=user_msg_content)
+    return LanguageModelMessages([system_msg, user_msg])
+
+
+def _get_system_prompt(config: EvaluationMetricConfig):
+    return config.custom_prompts.setdefault(
+        SYSTEM_MSG_KEY,
+        CONTEXT_RELEVANCY_METRIC_SYSTEM_MSG,
+    )
+
+
+def _get_user_prompt(config: EvaluationMetricConfig):
+    return config.custom_prompts.setdefault(
+        USER_MSG_KEY,
+        CONTEXT_RELEVANCY_METRIC_USER_MSG,
+    )

unique_toolkit/evaluators/exception.py

@@ -0,0 +1,5 @@
+from unique_toolkit._common.exception import CommonException
+
+
+class EvaluatorException(CommonException):
+    pass

unique_toolkit/evaluators/hallucination/constants.py

@@ -0,0 +1,41 @@
+from unique_toolkit.evaluators.config import EvaluationMetricConfig
+from unique_toolkit.evaluators.hallucination.prompts import (
+    HALLUCINATION_METRIC_SYSTEM_MSG,
+    HALLUCINATION_METRIC_SYSTEM_MSG_DEFAULT,
+    HALLUCINATION_METRIC_USER_MSG,
+    HALLUCINATION_METRIC_USER_MSG_DEFAULT,
+)
+from unique_toolkit.evaluators.schemas import (
+    EvaluationMetricInputFieldName,
+    EvaluationMetricName,
+)
+from unique_toolkit.language_model.infos import (
+    LanguageModel,
+    LanguageModelName,
+)
+
+SYSTEM_MSG_KEY = "systemPrompt"
+USER_MSG_KEY = "userPrompt"
+SYSTEM_MSG_DEFAULT_KEY = "systemPromptDefault"
+USER_MSG_DEFAULT_KEY = "userPromptDefault"
+
+
+hallucination_metric_default_config = EvaluationMetricConfig(
+    enabled=False,
+    name=EvaluationMetricName.HALLUCINATION,
+    language_model=LanguageModel(LanguageModelName.AZURE_GPT_4_0613),
+    score_to_emoji={"LOW": "🟢", "MEDIUM": "🟡", "HIGH": "🔴"},
+    custom_prompts={
+        SYSTEM_MSG_KEY: HALLUCINATION_METRIC_SYSTEM_MSG,
+        USER_MSG_KEY: HALLUCINATION_METRIC_USER_MSG,
+        SYSTEM_MSG_DEFAULT_KEY: HALLUCINATION_METRIC_SYSTEM_MSG_DEFAULT,
+        USER_MSG_DEFAULT_KEY: HALLUCINATION_METRIC_USER_MSG_DEFAULT,
+    },
+)
+
+hallucination_required_input_fields = [
+    EvaluationMetricInputFieldName.INPUT_TEXT,
+    EvaluationMetricInputFieldName.CONTEXT_TEXTS,
+    EvaluationMetricInputFieldName.HISTORY_MESSAGES,
+    EvaluationMetricInputFieldName.OUTPUT_TEXT,
+]

unique_toolkit/evaluators/hallucination/prompts.py

@@ -0,0 +1,79 @@
+HALLUCINATION_METRIC_SYSTEM_MSG = """
+You will receive a question, references, a conversation between a user and an agent, and an output. 
+The output is the answer to the question. 
+Your task is to evaluate if the output is fully supported by the information provided in the references and conversation, and provide explanations on your judgement in 2 sentences.
+
+Use the following entailment scale to generate a score:
+[low] - All information in output is supported by the references/conversation, or extractions from the references/conversation.
+[medium] - The output is supported by the references/conversation to some extent, but there is at least some information in the output that is not discussed in the references/conversation. For example, if an instruction asks about two concepts and the references/conversation only discusses either of them, it should be considered a [medium] hallucination level.
+[high] - The output contains information that is not part of the references/conversation, is unrelated to the references/conversation, or contradicts the references/conversation.
+
+Make sure to not use any external information/knowledge to judge whether the output is true or not. Only check whether the output is supported by the references/conversation, and not whether the output is correct or not. Also do not evaluate if the references/conversation contain further information that is not part of the output but could be relevant to the qestion.
+
+Your answer must be in JSON format:
+{
+ "reason": Your explanation of your judgement of the evaluation,
+ "value": decision, must be one of the following: ["high", "medium", "low"]
+}                                                  
+"""
+
+HALLUCINATION_METRIC_USER_MSG = """
+Here is the data:
+
+Input:
+'''
+$input_text
+'''
+
+References:
+'''
+$contexts_text
+'''
+
+Conversation:
+'''
+$history_messages_text
+'''
+
+Output:
+'''
+$output_text
+'''
+
+Answer as JSON:
+"""
+
+HALLUCINATION_METRIC_SYSTEM_MSG_DEFAULT = """
+You will receive a question and an output. 
+The output is the answer to the question. 
+The situation is that no references could be found to answer the question. Your task is to evaluate if the output contains any information to answer the question,
+and provide a short explanations of your reasoning in 2 sentences. Also mention in your explanation that no references were provided to answer the question.
+
+Use the following entailment scale to generate a score:
+[low] - The output does not contain any information to answer the question.
+[medium] - The output contains some information to answer the question, but does not answer the question entirely. 
+[high] - The output answers the question.
+
+It is not considered an answer when the output relates to the questions subject. Make sure to not use any external information/knowledge to judge whether the output is true or not. Only check that the output does not answer the question, and not whether the output is correct or not.
+Your answer must be in JSON format:
+{
+ "reason": Your explanation of your reasoning of the evaluation,
+ "value": decision, must be one of the following: ["low", "medium", "high"]
+}
+"""
+
+HALLUCINATION_METRIC_USER_MSG_DEFAULT = """                                                  
+Here is the data:
+
+Input:
+'''
+$input_text
+'''
+
+Output:
+'''
+$output_text
+'''
+
+Answer as JSON:
+"""

unique_toolkit/evaluators/hallucination/service.py

@@ -0,0 +1,58 @@
+import logging
+
+from unique_toolkit.app.schemas import Event
+from unique_toolkit.evaluators.config import (
+    EvaluationMetricConfig,
+)
+from unique_toolkit.evaluators.hallucination.constants import (
+    hallucination_metric_default_config,
+)
+from unique_toolkit.evaluators.hallucination.utils import check_hallucination_async
+from unique_toolkit.evaluators.schemas import (
+    EvaluationMetricInput,
+    EvaluationMetricResult,
+)
+
+logger = logging.getLogger(__name__)
+
+
+class HallucinationEvaluator:
+    def __init__(self, event: Event, logger: logging.Logger = logger):
+        self.event = event
+        self.logger = logger
+
+    async def run(
+        self,
+        input: EvaluationMetricInput,
+        config: EvaluationMetricConfig = hallucination_metric_default_config,
+    ) -> EvaluationMetricResult | None:
+        """
+        Analyzes the level of hallucination in the generated output by comparing it with the input
+        and the provided contexts or history. The analysis classifies the hallucination level as:
+        - low
+        - medium
+        - high
+
+        If no contexts or history are referenced in the generated output, the method verifies
+        that the output does not contain any relevant information to answer the question.
+
+        This method calls `check_hallucination_async` to perform the actual analysis. The `check_hallucination_async`
+        function handles the evaluation using the company ID from the event, the provided input, and the configuration.
+
+        Args:
+            input (EvaluationMetricInput): The input data used for evaluation, including the generated output and reference information.
+            config (EvaluationMetricConfig, optional): Configuration settings for the evaluation. Defaults to `hallucination_metric_default_config`.
+
+        Returns:
+            EvaluationMetricResult | None: The result of the evaluation, indicating the level of hallucination. Returns `None` if the analysis cannot be performed.
+
+        Raises:
+            EvaluatorException: If the context texts are empty, required fields are missing, or an error occurs during the evaluation.
+        """
+        if config.enabled is False:
+            self.logger.info("Hallucination metric is not enabled.")
+            return None
+
+        return await check_hallucination_async(
+            company_id=self.event.company_id, input=input, config=config
+        )

unique_toolkit/evaluators/hallucination/utils.py

@@ -0,0 +1,201 @@
+import logging
+from string import Template
+
+from unique_toolkit.evaluators.config import (
+    EvaluationMetricConfig,
+)
+from unique_toolkit.evaluators.exception import EvaluatorException
+from unique_toolkit.evaluators.hallucination.constants import (
+    SYSTEM_MSG_DEFAULT_KEY,
+    SYSTEM_MSG_KEY,
+    USER_MSG_DEFAULT_KEY,
+    USER_MSG_KEY,
+    hallucination_required_input_fields,
+)
+from unique_toolkit.evaluators.output_parser import (
+    parse_eval_metric_result,
+)
+from unique_toolkit.evaluators.schemas import (
+    EvaluationMetricInput,
+    EvaluationMetricName,
+    EvaluationMetricResult,
+)
+from unique_toolkit.language_model.schemas import (
+    LanguageModelMessages,
+    LanguageModelSystemMessage,
+    LanguageModelUserMessage,
+)
+from unique_toolkit.language_model.service import LanguageModelService
+
+from .prompts import (
+    HALLUCINATION_METRIC_SYSTEM_MSG,
+    HALLUCINATION_METRIC_SYSTEM_MSG_DEFAULT,
+    HALLUCINATION_METRIC_USER_MSG,
+    HALLUCINATION_METRIC_USER_MSG_DEFAULT,
+)
+
+logger = logging.getLogger(__name__)
+
+
+async def check_hallucination_async(
+    company_id: str,
+    input: EvaluationMetricInput,
+    config: EvaluationMetricConfig,
+    logger: logging.Logger = logger,
+) -> EvaluationMetricResult | None:
+    """
+    Analyzes the level of hallucination in the generated output by comparing it with the provided input
+    and the contexts or history. The analysis classifies the hallucination level as:
+    - low
+    - medium
+    - high
+
+    If no contexts or history are referenced in the generated output, the method checks that the output
+    does not contain any relevant information to answer the question.
+
+    This method performs the following steps:
+    1. Checks if the hallucination metric is enabled using the provided `config`.
+    2. Logs the start of the analysis using the provided `logger`.
+    3. Validates the required fields in the `input` data.
+    4. Retrieves the messages using the `_get_msgs` method.
+    5. Calls `LanguageModelService.complete_async_util` to get a completion result.
+    6. Parses and returns the evaluation metric result based on the content of the completion result.
+
+    Args:
+        company_id (str): The company ID for the analysis.
+        input (EvaluationMetricInput): The input data used for evaluation, including the generated output and reference information.
+        config (EvaluationMetricConfig, optional): Configuration settings for the evaluation. Defaults to `hallucination_metric_default_config`.
+        logger (Optional[logging.Logger], optional): The logger used for logging information and errors. Defaults to the logger for the current module.
+
+    Returns:
+        EvaluationMetricResult | None: The result of the evaluation, indicating the level of hallucination. Returns `None` if the metric is not enabled or if an error occurs.
+
+    Raises:
+        EvaluatorException: If the context texts are empty, required fields are missing, or an error occurs during the evaluation.
+    """
+    model_name = config.language_model.name
+    logger.info(f"Analyzing level of hallucination with {model_name}.")
+
+    input.validate_required_fields(hallucination_required_input_fields)
+
+    try:
+        msgs = _get_msgs(input, config, logger)
+        result = await LanguageModelService.complete_async_util(
+            company_id=company_id, messages=msgs, model_name=model_name
+        )
+        result_content = result.choices[0].message.content
+        if not result_content:
+            error_message = "Hallucination evaluation did not return a result."
+            raise EvaluatorException(
+                error_message=error_message,
+                user_message=error_message,
+            )
+        return parse_eval_metric_result(
+            result_content, EvaluationMetricName.HALLUCINATION
+        )
+    except Exception as e:
+        error_message = "Error occurred during hallucination metric analysis"
+        raise EvaluatorException(
+            error_message=f"{error_message}: {e}",
+            user_message=error_message,
+            exception=e,
+        )
+
+
+def _get_msgs(
+    input: EvaluationMetricInput,
+    config: EvaluationMetricConfig,
+    logger: logging.Logger,
+):
+    """
+    Composes the messages for hallucination analysis based on the provided input and configuration.
+
+    This method decides how to compose the messages based on the availability of context texts and history
+    message texts in the `input`
+
+    Args:
+        input (EvaluationMetricInput): The input data that includes context texts and history message texts
+                                      for the analysis.
+        config (EvaluationMetricConfig): The configuration settings for composing messages.
+        logger (Optional[logging.Logger], optional): The logger used for logging debug information.
+                                                     Defaults to the logger for the current module.
+
+    Returns:
+        The composed messages as per the provided input and configuration. The exact type and structure
+        depend on the implementation of the `compose_msgs` and `compose_msgs_default` methods.
+
+    """
+    if input.context_texts or input.history_messages:
+        logger.debug("Using context / history for hallucination evaluation.")
+        return _compose_msgs(input, config)
+    else:
+        logger.debug("No contexts and history provided for hallucination evaluation.")
+        return _compose_msgs_default(input, config)
+
+
+def _compose_msgs(
+    input: EvaluationMetricInput,
+    config: EvaluationMetricConfig,
+):
+    """
+    Composes the hallucination analysis messages.
+    """
+    system_msg_content = _get_system_prompt_with_contexts(config)
+    system_msg = LanguageModelSystemMessage(content=system_msg_content)
+
+    user_msg_templ = Template(_get_user_prompt_with_contexts(config))
+    user_msg_content = user_msg_templ.substitute(
+        input_text=input.input_text,
+        contexts_text=input.get_joined_context_texts(tag_name="reference"),
+        history_messages_text=input.get_joined_history_texts(tag_name="conversation"),
+        output_text=input.output_text,
+    )
+    user_msg = LanguageModelUserMessage(content=user_msg_content)
+    return LanguageModelMessages([system_msg, user_msg])
+
+
+def _compose_msgs_default(
+    input: EvaluationMetricInput,
+    config: EvaluationMetricConfig,
+):
+    """
+    Composes the hallucination analysis prompt without messages.
+    """
+    system_msg_content = _get_system_prompt_default(config)
+    system_msg = LanguageModelSystemMessage(content=system_msg_content)
+
+    user_msg_templ = Template(_get_user_prompt_default(config))
+    user_msg_content = user_msg_templ.substitute(
+        input_text=input.input_text,
+        output_text=input.output_text,
+    )
+    user_msg = LanguageModelUserMessage(content=user_msg_content)
+    return LanguageModelMessages([system_msg, user_msg])
+
+
+def _get_system_prompt_with_contexts(config: EvaluationMetricConfig):
+    return config.custom_prompts.setdefault(
+        SYSTEM_MSG_KEY,
+        HALLUCINATION_METRIC_SYSTEM_MSG,
+    )
+
+
+def _get_user_prompt_with_contexts(config: EvaluationMetricConfig):
+    return config.custom_prompts.setdefault(
+        USER_MSG_KEY,
+        HALLUCINATION_METRIC_USER_MSG,
+    )
+
+
+def _get_system_prompt_default(config: EvaluationMetricConfig):
+    return config.custom_prompts.setdefault(
+        SYSTEM_MSG_DEFAULT_KEY,
+        HALLUCINATION_METRIC_SYSTEM_MSG_DEFAULT,
+    )
+
+
+def _get_user_prompt_default(config: EvaluationMetricConfig):
+    return config.custom_prompts.setdefault(
+        USER_MSG_DEFAULT_KEY,
+        HALLUCINATION_METRIC_USER_MSG_DEFAULT,
+    )

unique_toolkit/evaluators/output_parser.py

@@ -0,0 +1,30 @@
+from unique_toolkit.evaluators.exception import EvaluatorException
+from unique_toolkit.evaluators.schemas import (
+    EvaluationMetricName,
+    EvaluationMetricResult,
+)
+from unique_toolkit.language_model.utils import convert_string_to_json
+
+
+def parse_eval_metric_result(
+    result: str,
+    metric_name: EvaluationMetricName,
+):
+    """
+    Parses the evaluation metric result.
+    """
+
+    try:
+        parsed_result = convert_string_to_json(result)
+    except Exception as e:
+        error_message = "Error occurred during parsing the evaluation metric result"
+        raise EvaluatorException(
+            user_message=f"{error_message}.",
+            error_message=f"{error_message}: {str(e)}",
+        )
+
+    return EvaluationMetricResult(
+        name=metric_name,
+        value=parsed_result.get("value", "None"),
+        reason=parsed_result.get("reason", "None"),
+    )

unique_toolkit/evaluators/schemas.py

@@ -0,0 +1,82 @@
+from enum import Enum
+from typing import Optional
+
+from pydantic import BaseModel
+
+from unique_toolkit.chat import ChatMessage
+from unique_toolkit.evaluators.exception import EvaluatorException
+
+
+class EvaluationMetricName(Enum):
+    HALLUCINATION = "hallucination"
+    CONTEXT_RELEVANCY = "relevancy"
+
+
+class EvaluationMetricInputFieldName(str, Enum):
+    INPUT_TEXT = "input_text"
+    CONTEXT_TEXTS = "context_texts"
+    HISTORY_MESSAGES = "history_messages"
+    OUTPUT_TEXT = "output_text"
+
+
+class EvaluationMetricInput(BaseModel):
+    """
+    Input for any metric evaluation. Depending on the metric, the input can be different.
+    """
+
+    input_text: Optional[str] = None
+    context_texts: Optional[list[str]] = None
+    history_messages: Optional[list[ChatMessage]] = None
+    output_text: Optional[str] = None
+
+    def get_joined_context_texts(self, tag_name: str = "reference") -> str:
+        """
+        Concatenates context_texts.
+        """
+        if not self.context_texts:
+            return f"<No {tag_name} texts provided>"
+
+        return "\n".join(
+            [
+                f"<{tag_name}-{index}>{text}</{tag_name}-{index}>"
+                for index, text in enumerate(self.context_texts)
+            ]
+        )
+
+    def get_history_message_text(self, chat_message: ChatMessage):
+        return f"{chat_message.role.value}: {chat_message.content}"
+
+    def get_history_message_texts(self):
+        if not self.history_messages:
+            return []
+        return [self.get_history_message_text(msg) for msg in self.history_messages]
+
+    def get_joined_history_texts(self, tag_name: str = "conversation") -> str:
+        """
+        Concatenates history message texts.
+        """
+        if not self.history_messages:
+            return f"<No {tag_name} texts provided>"
+
+        return "\n".join(self.get_history_message_texts())
+
+    def validate_required_fields(
+        self, required_fields: list[EvaluationMetricInputFieldName]
+    ):
+        """
+        Validates the input fields for the hallucination metric.
+        """
+        for field in required_fields:
+            value = getattr(self, field)
+            if value is None:
+                error_message = f"Missing required input field: {field}"
+                raise EvaluatorException(
+                    user_message=error_message,
+                    error_message=error_message,
+                )
+
+
+class EvaluationMetricResult(BaseModel):
+    name: EvaluationMetricName
+    value: str
+    reason: str

unique_toolkit/language_model/infos.py

@@ -20,6 +20,35 @@ class LanguageModelName(StrEnum):
     AZURE_GPT_4o_MINI_2024_0718 = "AZURE_GPT_4o_MINI_2024_0718"
 
 
+class EncoderName(StrEnum):
+    O200K_BASE = "o200k_base"
+    CL100K_BASE = "cl100k_base"
+
+
+def get_encoder_name(model_name: LanguageModelName) -> Optional[EncoderName]:
+    LMN = LanguageModelName
+    match model_name:
+        case (
+            LMN.AZURE_GPT_35_TURBO
+            | LMN.AZURE_GPT_35_TURBO_16K
+            | LMN.AZURE_GPT_35_TURBO_0613
+        ):
+            return EncoderName.CL100K_BASE
+        case (
+            LMN.AZURE_GPT_4_0613
+            | LMN.AZURE_GPT_4_TURBO_1106
+            | LMN.AZURE_GPT_4_VISION_PREVIEW
+            | LMN.AZURE_GPT_4_32K_0613
+            | LMN.AZURE_GPT_4_TURBO_2024_0409
+        ):
+            return EncoderName.CL100K_BASE
+        case LMN.AZURE_GPT_4o_2024_0513 | LMN.AZURE_GPT_4o_MINI_2024_0718:
+            return EncoderName.O200K_BASE
+        case _:
+            print(f"{model_name} is not supported. Please add encoder information.")
+            return None
+
+
 class LanguageModelProvider(StrEnum):
     AZURE = "AZURE"
     CUSTOM = "CUSTOM"
@@ -30,6 +59,7 @@ class LanguageModelInfo(BaseModel):
     version: str
     provider: LanguageModelProvider
 
+    encoder_name: Optional[EncoderName] = None
     token_limits: Optional[LanguageModelTokenLimits] = None
 
     info_cutoff_at: Optional[date] = None
@@ -53,6 +83,7 @@ class LanguageModel:
         - name
         - version
         - provider
+        - encoder_name
         - token_limits
         - info_cutoff_at
         - published_at
@@ -86,6 +117,13 @@ class LanguageModel:
         """
         return self._model_info.version
 
+    @property
+    def encoder_name(self) -> Optional[EncoderName]:
+        """
+        Returns the encoder_name used for the model.
+        """
+        return self._model_info.encoder_name
+
     @property
     def token_limit(self) -> Optional[int]:
         """
@@ -191,6 +229,7 @@ def create_language_model(
     provider: LanguageModelProvider,
     info_cutoff_at: date,
     published_at: date,
+    encoder_name: Optional[EncoderName] = None,
     token_limit: Optional[int] = None,
     token_limit_input: Optional[int] = None,
     token_limit_output: Optional[int] = None,
@@ -202,6 +241,7 @@ def create_language_model(
         name=name,
         version=version,
         provider=provider,
+        encoder_name=encoder_name,
         token_limits=LanguageModelTokenLimits(
             token_limit=token_limit,
             token_limit_input=token_limit_input,
@@ -229,6 +269,7 @@ AzureGpt35Turbo0613 = create_language_model(
     name=LanguageModelName.AZURE_GPT_35_TURBO_0613,
     provider=LanguageModelProvider.AZURE,
     version="0613",
+    encoder_name=get_encoder_name(LanguageModelName.AZURE_GPT_35_TURBO_0613),
     token_limit=8192,
     info_cutoff_at=date(2021, 9, 1),
     published_at=date(2023, 6, 13),
@@ -239,6 +280,7 @@ AzureGpt35Turbo = create_language_model(
     name=LanguageModelName.AZURE_GPT_35_TURBO,
     provider=LanguageModelProvider.AZURE,
     version="0301",
+    encoder_name=get_encoder_name(LanguageModelName.AZURE_GPT_35_TURBO),
     token_limit=4096,
     info_cutoff_at=date(2021, 9, 1),
     published_at=date(2023, 3, 1),
@@ -249,6 +291,7 @@ AzureGpt35Turbo16k = create_language_model(
     name=LanguageModelName.AZURE_GPT_35_TURBO_16K,
     provider=LanguageModelProvider.AZURE,
     version="0613",
+    encoder_name=get_encoder_name(LanguageModelName.AZURE_GPT_35_TURBO_16K),
     token_limit=16382,
     info_cutoff_at=date(2021, 9, 1),
     published_at=date(2023, 6, 13),
@@ -260,6 +303,7 @@ AzureGpt40613 = create_language_model(
     name=LanguageModelName.AZURE_GPT_4_0613,
     provider=LanguageModelProvider.AZURE,
     version="0613",
+    encoder_name=get_encoder_name(LanguageModelName.AZURE_GPT_4_0613),
     token_limit=8192,
     info_cutoff_at=date(2021, 9, 1),
     published_at=date(2023, 6, 13),
@@ -272,6 +316,7 @@ AzureGpt4Turbo1106 = create_language_model(
     name=LanguageModelName.AZURE_GPT_4_TURBO_1106,
     provider=LanguageModelProvider.AZURE,
     version="1106-preview",
+    encoder_name=get_encoder_name(LanguageModelName.AZURE_GPT_4_TURBO_1106),
     token_limit_input=128000,
     token_limit_output=4096,
     info_cutoff_at=date(2023, 4, 1),
@@ -283,6 +328,7 @@ AzureGpt4VisionPreview = create_language_model(
     name=LanguageModelName.AZURE_GPT_4_VISION_PREVIEW,
     provider=LanguageModelProvider.AZURE,
     version="vision-preview",
+    encoder_name=get_encoder_name(LanguageModelName.AZURE_GPT_4_VISION_PREVIEW),
     token_limit_input=128000,
     token_limit_output=4096,
     info_cutoff_at=date(2023, 4, 1),
@@ -293,6 +339,7 @@ AzureGpt432k0613 = create_language_model(
     name=LanguageModelName.AZURE_GPT_4_32K_0613,
     provider=LanguageModelProvider.AZURE,
     version="1106-preview",
+    encoder_name=get_encoder_name(LanguageModelName.AZURE_GPT_4_32K_0613),
     token_limit=32768,
     info_cutoff_at=date(2021, 9, 1),
     published_at=date(2023, 6, 13),
@@ -302,6 +349,7 @@ AzureGpt432k0613 = create_language_model(
 
 AzureGpt4Turbo20240409 = create_language_model(
     name=LanguageModelName.AZURE_GPT_4_TURBO_2024_0409,
+    encoder_name=get_encoder_name(LanguageModelName.AZURE_GPT_4_TURBO_2024_0409),
     provider=LanguageModelProvider.AZURE,
     version="turbo-2024-04-09",
     token_limit_input=128000,
@@ -312,6 +360,7 @@ AzureGpt4Turbo20240409 = create_language_model(
 
 AzureGpt4o20240513 = create_language_model(
     name=LanguageModelName.AZURE_GPT_4o_2024_0513,
+    encoder_name=get_encoder_name(LanguageModelName.AZURE_GPT_4o_2024_0513),
     provider=LanguageModelProvider.AZURE,
     version="2024-05-13",
     token_limit_input=128000,
@@ -324,6 +373,7 @@ AzureGpt4oMini20240718 = create_language_model(
     name=LanguageModelName.AZURE_GPT_4o_MINI_2024_0718,
     provider=LanguageModelProvider.AZURE,
     version="2024-07-18",
+    encoder_name=get_encoder_name(LanguageModelName.AZURE_GPT_4o_MINI_2024_0718),
     token_limit_input=128000,
     token_limit_output=16384,
     info_cutoff_at=date(2023, 10, 1),

unique_toolkit/language_model/schemas.py

@@ -3,7 +3,14 @@ from enum import StrEnum
 from typing import Any, Optional, Self
 
 from humps import camelize
-from pydantic import BaseModel, ConfigDict, RootModel, field_validator, model_validator
+from pydantic import (
+    BaseModel,
+    ConfigDict,
+    Field,
+    RootModel,
+    field_validator,
+    model_validator,
+)
 
 # set config to convert camelCase to snake_case
 model_config = ConfigDict(
@@ -173,7 +180,11 @@ class LanguageModelToolParameters(BaseModel):
 
 
 class LanguageModelTool(BaseModel):
-    name: str
+    name: str = Field(
+        ...,
+        pattern=r"^[a-zA-Z_-]+$",
+        description="Name must adhere to the pattern ^[a-zA-Z_-]+$",
+    )
     description: str
     parameters: LanguageModelToolParameters
     returns: LanguageModelToolParameterProperty | LanguageModelToolParameters | None = (

unique_toolkit/language_model/service.py

@@ -196,6 +196,7 @@ class LanguageModelService(BaseService):
             The LanguageModelStreamResponse object once the stream has finished.
         """
         options = self._add_tools_to_options({}, tools)
+        options["temperature"] = temperature
         search_context = self._to_search_context(content_chunks)
         messages = messages.model_dump(exclude_none=True)
         model = (
@@ -217,7 +218,6 @@ class LanguageModelService(BaseService):
                 # TODO change or extend types in unique_sdk
                 model=model,
                 timeout=timeout,
-                temperature=temperature,
                 assistantId=self.event.payload.assistant_id,
                 debugInfo=debug_info,
                 options=options,  # type: ignore
@@ -257,6 +257,7 @@ class LanguageModelService(BaseService):
         """
 
         options = self._add_tools_to_options({}, tools)
+        options["temperature"] = temperature
         search_context = self._to_search_context(content_chunks)
         messages = messages.model_dump(exclude_none=True, exclude=["tool_calls"])
         model = (
@@ -277,7 +278,6 @@ class LanguageModelService(BaseService):
                 searchContext=search_context,
                 model=model,
                 timeout=timeout,
-                temperature=temperature,
                 assistantId=self.event.payload.assistant_id,
                 debugInfo=debug_info,
                 # TODO change or extend types in unique_sdk

{unique_toolkit-0.5.24.dist-info → unique_toolkit-0.5.28.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: unique_toolkit
-Version: 0.5.24
+Version: 0.5.28
 Summary: 
 License: Proprietary
 Author: Martin Fadler
@@ -17,7 +17,7 @@ Requires-Dist: python-dotenv (>=1.0.1,<2.0.0)
 Requires-Dist: regex (>=2024.5.15,<2025.0.0)
 Requires-Dist: tiktoken (>=0.7.0,<0.8.0)
 Requires-Dist: typing-extensions (>=4.9.0,<5.0.0)
-Requires-Dist: unique-sdk (>=0.9.7,<0.10.0)
+Requires-Dist: unique-sdk (>=0.9.8,<0.10.0)
 Description-Content-Type: text/markdown
 
 # Unique Toolkit
@@ -100,6 +100,20 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), 
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## [0.5.28] - 2024-10-23
+- Correctly use `temperature` parameter in `LanguageModelService.complete`, `LanguageModelService.complete_async`, `LanguageModelService.stream_complete` and `LanguageModelService.stream_complete_async` methods
+
+## [0.5.27] - 2024-10-22
+- Add encoder_name to to language model info
+- Verify tool name for `LanguageModelTool` to conform with frontent requirements.
+- Add `search_on_chat` to `ContentService`
+
+## [0.5.26] - 2024-10-16
+- Bump `unique_sdk` version
+
+## [0.5.25] - 2024-09-26
+- Add `evaluators` for hallucination and context relevancy evaluation
+
 ## [0.5.24] - 2024-09-26
 - Add `originalText` to `_construct_message_modify_params` and `_construct_message_create_params`. This addition makes sure that the `originalText` on the database is populated with the `text`
 

{unique_toolkit-0.5.24.dist-info → unique_toolkit-0.5.28.dist-info}/RECORD

@@ -1,6 +1,8 @@
 unique_toolkit/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 unique_toolkit/_common/_base_service.py,sha256=S8H0rAebx7GsOldA7xInLp3aQJt9yEPDQdsGSFRJsGg,276
 unique_toolkit/_common/_time_utils.py,sha256=ztmTovTvr-3w71Ns2VwXC65OKUUh-sQlzbHdKTQWm-w,135
+unique_toolkit/_common/exception.py,sha256=caQIE1btsQnpKCHqL2cgWUSbHup06enQu_Pt7uGUTTE,727
+unique_toolkit/_common/validators.py,sha256=w5lzvRxl0sBTvv0CXLF9UwtJyKmmS2lez0KXaqapgBE,258
 unique_toolkit/app/__init__.py,sha256=sZyGrz74jBlAjv6OcHgcp6VtP6-AKKpaVYjakr1Xk60,735
 unique_toolkit/app/init_logging.py,sha256=Sh26SRxOj8i8dzobKhYha2lLrkrMTHfB1V4jR3h23gQ,678
 unique_toolkit/app/init_sdk.py,sha256=Nv4Now4pMfM0AgRhbtatLpm_39rKxn0WmRLwmPhRl-8,1285
@@ -15,18 +17,30 @@ unique_toolkit/chat/state.py,sha256=Cjgwv_2vhDFbV69xxsn7SefhaoIAEqLx3ferdVFCnOg,
 unique_toolkit/chat/utils.py,sha256=ihm-wQykBWhB4liR3LnwPVPt_qGW6ETq21Mw4HY0THE,854
 unique_toolkit/content/__init__.py,sha256=MSH2sxjQyKD2Sef92fzE5Dt9SihdzivB6yliSwJfTmQ,890
 unique_toolkit/content/schemas.py,sha256=zks_Pkki2VhxICJJgHZyc-LPmRuj5dLbw3pgcUT7SW8,2362
-unique_toolkit/content/service.py,sha256=AHyMJTXm5IpYbg1uINzjGqvSL_5aJwEHwSH7Y5pkXBg,14028
+unique_toolkit/content/service.py,sha256=ZGYWYTphXpcByXyMqr1VOVUHdmdnsR-XIS_YRX0Wyv4,14211
 unique_toolkit/content/utils.py,sha256=Lake671plRsqNvO3pN_rmyVcpwbdED_KQpLcCnc4lv4,6902
 unique_toolkit/embedding/__init__.py,sha256=dr8M9jvslQTxPpxgaGwzxY0FildiWf-DidN_cahPAWw,191
 unique_toolkit/embedding/schemas.py,sha256=1GvKCaSk4jixzVQ2PKq8yDqwGEVY_hWclYtoAr6CC2g,96
 unique_toolkit/embedding/service.py,sha256=Iiw-sbdkjuWlWMfLM9qyC4GNTJOotQAaVjkYvh5Su4Y,2370
 unique_toolkit/embedding/utils.py,sha256=v86lo__bCJbxZBQ3OcLu5SuwT6NbFfWlcq8iyk6BuzQ,279
+unique_toolkit/evaluators/config.py,sha256=JRSHJvIjioXDMgd9hodK10J-52j3LMgJFvG0Vy7ePa8,1056
+unique_toolkit/evaluators/context_relevancy/constants.py,sha256=YErC92sqsY31cmBUG3dFQw78mUjbcpjMG7TLfYuLYmw,1051
+unique_toolkit/evaluators/context_relevancy/prompts.py,sha256=gTlWP7fDuxhrXhCYNCqXMbCey_DalZMdi5l-a6RHgk0,713
+unique_toolkit/evaluators/context_relevancy/service.py,sha256=9hzdMuF4A4T97-3X3zcXgrDISLn1bleZ6tTL1bHa9dQ,1722
+unique_toolkit/evaluators/context_relevancy/utils.py,sha256=DCFaoxZT_qDMKirjy3hTo1DIE7HpZ7-XR5P-rHuAoHQ,5137
+unique_toolkit/evaluators/exception.py,sha256=7lcVbCyoN4Md1chNJDFxpUYyWbVrcr9dcc3TxWykJTc,115
+unique_toolkit/evaluators/hallucination/constants.py,sha256=DEycXlxY9h01D0iF3aU5LIdPrDJ-5OkF0VdXDLn_tSs,1440
+unique_toolkit/evaluators/hallucination/prompts.py,sha256=9yCpO_WGLDvYfPWKL1VuRA-jt0P_-A-qvLUOmuv-Nks,3320
+unique_toolkit/evaluators/hallucination/service.py,sha256=k8qro5Lw4Ak58m4HYp3G4HPLIaexeFySIIVvW6fAdeA,2408
+unique_toolkit/evaluators/hallucination/utils.py,sha256=507BsX1mFTEne1-LdRCNMgBj-IXSFvBj1t3BPe1UkGs,7639
+unique_toolkit/evaluators/output_parser.py,sha256=eI72qkzK1dZyUvnfP2SOAQCGBj_-PwX5wy_aLPMsJMY,883
+unique_toolkit/evaluators/schemas.py,sha256=Jaue6Uhx75X1CyHKWj8sT3RE1JZXTqoLtfLt2xQNCX8,2507
 unique_toolkit/language_model/__init__.py,sha256=YuhyczGPj6w9xX-sOVUhmozvzIFxcckHFEkeMBecr5s,1784
-unique_toolkit/language_model/infos.py,sha256=ETAUV0YTs6BjwuiTdhKz247CtL0W8Jwo3-c0ZQ2HdXs,9962
-unique_toolkit/language_model/schemas.py,sha256=sLpE29Ks0zEfhZUQrYOt1Cak2xzQcr9fpTXFDHkfURA,4868
-unique_toolkit/language_model/service.py,sha256=CvVo5CBa5Ia_fQD3DtJRsVChybuUfGFV5ml2_78_p1I,13395
+unique_toolkit/language_model/infos.py,sha256=Oxkr9_6s8gFubxjox-iCm1GSs1RCAQQ5t8oh20izlC0,12002
+unique_toolkit/language_model/schemas.py,sha256=LO3QHsyFuJXG3HxXWFf44QV28JJzW8YW5TeIYhVzZTI,5035
+unique_toolkit/language_model/service.py,sha256=R8j2cr-lDbR96Vl5LVQIdtscS0gfscezKMXNMM2AZHM,13403
 unique_toolkit/language_model/utils.py,sha256=WBPj1XKkDgxy_-T8HCZvsfkkSzj_1w4UZzNmyvdbBLY,1081
-unique_toolkit-0.5.24.dist-info/LICENSE,sha256=GlN8wHNdh53xwOPg44URnwag6TEolCjoq3YD_KrWgss,193
-unique_toolkit-0.5.24.dist-info/METADATA,sha256=N63Q5_PfoinaFgt94EqZ3lfCOdKRCEcCbz1vivjBRyg,11938
-unique_toolkit-0.5.24.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
-unique_toolkit-0.5.24.dist-info/RECORD,,
+unique_toolkit-0.5.28.dist-info/LICENSE,sha256=GlN8wHNdh53xwOPg44URnwag6TEolCjoq3YD_KrWgss,193
+unique_toolkit-0.5.28.dist-info/METADATA,sha256=mvFUKA2gYyPWx19cex2Y6YIdoZ-WSwx0LeTGymenwVo,12521
+unique_toolkit-0.5.28.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
+unique_toolkit-0.5.28.dist-info/RECORD,,