unique_toolkit 0.8.15__py3-none-any.whl → 0.8.17__py3-none-any.whl

unique_toolkit/_common/token/token_counting.py

@@ -5,15 +5,14 @@ import json
 from typing import Any, Callable
 
 from pydantic import BaseModel
+from unique_toolkit._common.token.image_token_counting import calculate_image_tokens_from_base64
 from unique_toolkit.language_model import (
     LanguageModelMessage,
     LanguageModelMessages,
     LanguageModelName,
 )
 
-from _common.utils.token.image_token_counting import (
-    calculate_image_tokens_from_base64,
-)
+
 
 
 class SpecialToolCallingTokens(BaseModel):

unique_toolkit/_common/validators.py

@@ -28,6 +28,13 @@ LMI = Annotated[
     ),
 ]
 
+def get_LMI_default_field(llm_name: LanguageModelName, **kwargs) -> Any:
+    return Field(
+        default=LanguageModelInfo.from_name(llm_name),
+        json_schema_extra={"default": llm_name},
+        **kwargs,
+    )
+
 
 def serialize_lmi(model: LanguageModelInfo) -> str | LanguageModelInfo:
     if model.provider == LanguageModelProvider.CUSTOM:

unique_toolkit/debug_info_manager/debug_info_manager.py

@@ -0,0 +1,19 @@
+from unique_toolkit.content.schemas import ContentChunk, ContentReference
+from unique_toolkit.tools.schemas import ToolCallResponse
+
+
+class DebugInfoManager:
+    def __init__(self):
+        self.debug_info = {"tools": []}
+
+    def extract_tool_debug_info(self, tool_call_responses: list[ToolCallResponse]):
+        for tool_call_response in tool_call_responses:
+            self.debug_info["tools"].append(
+                {"name": tool_call_response.name, "data": tool_call_response.debug_info}
+            )
+
+    def add(self, key, value):
+        self.debug_info = self.debug_info | {key: value}
+
+    def get(self):
+        return self.debug_info

unique_toolkit/evals/config.py

@@ -0,0 +1,36 @@
+from typing import Any
+
+from humps import camelize
+from pydantic import BaseModel, ConfigDict, Field
+
+from unique_toolkit._common.validators import LMI
+from unique_toolkit.language_model.infos import LanguageModelInfo, LanguageModelName
+
+
+from .schemas import (
+    EvaluationMetricName,
+)
+
+model_config = ConfigDict(
+    alias_generator=camelize,
+    populate_by_name=True,
+    arbitrary_types_allowed=True,
+    validate_default=True,
+)
+
+
+class EvaluationMetricConfig(BaseModel):
+    model_config = model_config
+
+    enabled: bool = False
+    name: EvaluationMetricName
+    language_model: LMI = LanguageModelInfo.from_name(
+        LanguageModelName.AZURE_GPT_35_TURBO_0125,
+    )
+    additional_llm_options: dict[str, Any] = Field(
+        default={},
+        description="Additional options to pass to the language model.",
+    )
+    custom_prompts: dict[str, str] = {}
+    score_to_label: dict[str, str] = {}
+    score_to_title: dict[str, str] = {}

unique_toolkit/evals/context_relevancy/prompts.py

@@ -0,0 +1,56 @@
+CONTEXT_RELEVANCY_METRIC_SYSTEM_MSG = """
+You will receive an input and a set of contexts.
+Your task is to evaluate how relevant the contexts are to the input text.
+
+Use the following rating scale to generate a score:
+[low] - The contexts are not relevant to the input.
+[medium] - The contexts are somewhat relevant to the input.
+[high] - The contexts are highly relevant to the input.
+
+Your answer must be in JSON format:
+{
+ "reason": Your explanation of your judgement of the evaluation,
+ "value": decision, must be one of the following ["low", "medium", "high"]
+}
+"""
+
+CONTEXT_RELEVANCY_METRIC_SYSTEM_MSG_STRUCTURED_OUTPUT = """
+You will receive an input and a set of contexts.
+Your task is to evaluate how relevant the contexts are to the input text. 
+Further you should extract relevant facts from the contexts.
+
+# Output Format
+- Generate data according to the provided data schema.
+- Ensure the output adheres to the format required by the pydantic object.
+- All necessary fields should be populated as per the data schema guidelines.
+"""
+
+CONTEXT_RELEVANCY_METRIC_USER_MSG = """
+Here is the data:
+
+Input:
+'''
+$input_text
+'''
+
+Contexts:
+'''
+$context_texts
+'''
+
+Answer as JSON:
+"""
+
+CONTEXT_RELEVANCY_METRIC_USER_MSG_STRUCTURED_OUTPUT = """
+Here is the data:
+
+Input:
+'''
+$input_text
+'''
+
+Contexts:
+'''
+$context_texts
+'''
+"""

unique_toolkit/evals/context_relevancy/schema.py

@@ -0,0 +1,88 @@
+from pydantic import BaseModel, Field, create_model
+from pydantic.json_schema import SkipJsonSchema
+
+
+
+from pydantic import BaseModel, ConfigDict
+
+from unique_toolkit.tools.config import get_configuration_dict
+
+
+class StructuredOutputModel(BaseModel):
+    model_config = ConfigDict(extra="forbid")
+
+
+
+class StructuredOutputConfig(BaseModel):
+    model_config = get_configuration_dict()
+
+    enabled: bool = Field(
+        default=False,
+        description="Whether to use structured output for the evaluation.",
+    )
+    extract_fact_list: bool = Field(
+        default=False,
+        description="Whether to extract a list of relevant facts from context chunks with structured output.",
+    )
+    reason_description: str = Field(
+        default="A brief explanation justifying your evaluation decision.",
+        description="The description of the reason field for structured output.",
+    )
+    value_description: str = Field(
+        default="Assessment of how relevant the facts are to the query. Must be one of: ['low', 'medium', 'high'].",
+        description="The description of the value field for structured output.",
+    )
+
+    fact_description: str = Field(
+        default="A fact is an information that is directly answers the user's query. Make sure to emphasize the important information from the fact with bold text.",
+        description="The description of the fact field for structured output.",
+    )
+    fact_list_description: str = Field(
+        default="A list of relevant facts extracted from the source that supports or answers the user's query.",
+        description="The description of the fact list field for structured output.",
+    )
+
+
+class Fact(StructuredOutputModel):
+    fact: str
+
+
+class EvaluationSchemaStructuredOutput(StructuredOutputModel):
+    reason: str
+    value: str
+
+    fact_list: list[Fact] = Field(default_factory=list[Fact])
+
+    @classmethod
+    def get_with_descriptions(cls, config: StructuredOutputConfig):
+        if config.extract_fact_list:
+            FactWithDescription = create_model(
+                "Fact",
+                fact=(str, Field(..., description=config.fact_description)),
+                __base__=Fact,
+            )
+            fact_list_field = (
+                list[FactWithDescription],
+                Field(
+                    description=config.fact_list_description,
+                ),
+            )
+        else:
+            fact_list_field = (
+                SkipJsonSchema[list[Fact]],
+                Field(default_factory=list[Fact]),
+            )
+
+        return create_model(
+            "EvaluationSchemaStructuredOutputWithDescription",
+            reason=(
+                str,
+                Field(..., description=config.reason_description),
+            ),
+            value=(
+                str,
+                Field(..., description=config.value_description),
+            ),
+            fact_list=fact_list_field,
+            __base__=cls,
+        )

unique_toolkit/evals/context_relevancy/service.py

@@ -0,0 +1,241 @@
+import logging
+
+from pydantic import BaseModel, ValidationError
+from unique_toolkit.app.schemas import ChatEvent
+from unique_toolkit.chat.service import ChatService
+from unique_toolkit.language_model.infos import (
+    LanguageModelInfo,
+    LanguageModelName,
+    ModelCapabilities,
+)
+from unique_toolkit.language_model.prompt import Prompt
+from unique_toolkit.language_model.schemas import (
+    LanguageModelMessages,
+)
+from unique_toolkit.language_model.service import (
+    LanguageModelService,
+)
+from unique_toolkit.evals.config import EvaluationMetricConfig
+from unique_toolkit.evals.context_relevancy.schema import (
+    EvaluationSchemaStructuredOutput,
+)
+from unique_toolkit.evals.exception import EvaluatorException
+from unique_toolkit.evals.output_parser import (
+    parse_eval_metric_result,
+    parse_eval_metric_result_structured_output,
+)
+from unique_toolkit.evals.schemas import (
+    EvaluationMetricInput,
+    EvaluationMetricInputFieldName,
+    EvaluationMetricName,
+    EvaluationMetricResult,
+)
+
+
+from .prompts import (
+    CONTEXT_RELEVANCY_METRIC_SYSTEM_MSG,
+    CONTEXT_RELEVANCY_METRIC_SYSTEM_MSG_STRUCTURED_OUTPUT,
+    CONTEXT_RELEVANCY_METRIC_USER_MSG,
+    CONTEXT_RELEVANCY_METRIC_USER_MSG_STRUCTURED_OUTPUT,
+)
+
+SYSTEM_MSG_KEY = "systemPrompt"
+USER_MSG_KEY = "userPrompt"
+
+default_config = EvaluationMetricConfig(
+    enabled=False,
+    name=EvaluationMetricName.CONTEXT_RELEVANCY,
+    language_model=LanguageModelInfo.from_name(
+        LanguageModelName.AZURE_GPT_4o_2024_1120
+    ),
+    custom_prompts={
+        SYSTEM_MSG_KEY: CONTEXT_RELEVANCY_METRIC_SYSTEM_MSG,
+        USER_MSG_KEY: CONTEXT_RELEVANCY_METRIC_USER_MSG,
+    },
+)
+
+relevancy_required_input_fields = [
+    EvaluationMetricInputFieldName.INPUT_TEXT,
+    EvaluationMetricInputFieldName.CONTEXT_TEXTS,
+]
+
+
+class ContextRelevancyEvaluator:
+    def __init__(
+        self,
+        event: ChatEvent,
+    ):
+        self.chat_service = ChatService(event)
+        self.language_model_service = LanguageModelService(event)
+        self.logger = logging.getLogger(f"ContextRelevancyEvaluator.{__name__}")
+
+    async def analyze(
+        self,
+        input: EvaluationMetricInput,
+        config: EvaluationMetricConfig = default_config,
+        structured_output_schema: type[BaseModel] | None = None,
+    ) -> EvaluationMetricResult | None:
+        """
+        Analyzes the level of relevancy of a context by comparing
+        it with the input text.
+
+        Args:
+            input (EvaluationMetricInput): The input for the metric.
+            config (EvaluationMetricConfig): The configuration for the metric.
+
+        Returns:
+            EvaluationMetricResult | None
+
+        Raises:
+            EvaluatorException: If the context texts are empty or required fields are missing or error occurred during evaluation.
+        """
+        if config.enabled is False:
+            self.logger.info("Hallucination metric is not enabled.")
+            return None
+
+        input.validate_required_fields(relevancy_required_input_fields)
+
+        if len(input.context_texts) == 0:  # type: ignore
+            error_message = "No context texts provided."
+            raise EvaluatorException(
+                user_message=error_message,
+                error_message=error_message,
+            )
+
+        try:
+            # Handle structured output if enabled and supported by the model
+            if (
+                structured_output_schema
+                and ModelCapabilities.STRUCTURED_OUTPUT
+                in config.language_model.capabilities
+            ):
+                return await self._handle_structured_output(
+                    input, config, structured_output_schema
+                )
+
+            # Handle regular output
+            return await self._handle_regular_output(input, config)
+
+        except Exception as e:
+            error_message = (
+                "Unknown error occurred during context relevancy metric analysis"
+            )
+            raise EvaluatorException(
+                error_message=f"{error_message}: {e}",
+                user_message=error_message,
+                exception=e,
+            )
+
+    async def _handle_structured_output(
+        self,
+        input: EvaluationMetricInput,
+        config: EvaluationMetricConfig,
+        structured_output_schema: type[BaseModel],
+    ) -> EvaluationMetricResult:
+        """Handle the structured output case for context relevancy evaluation."""
+        self.logger.info("Using structured output for context relevancy evaluation.")
+        msgs = self._compose_msgs(input, config, enable_structured_output=True)
+        result = await self.language_model_service.complete_async(
+            messages=msgs,
+            model_name=config.language_model.name,
+            structured_output_model=structured_output_schema,
+            structured_output_enforce_schema=True,
+            other_options=config.additional_llm_options,
+        )
+
+        try:
+            result_content = EvaluationSchemaStructuredOutput.model_validate(
+                result.choices[0].message.parsed
+            )
+        except ValidationError as e:
+            error_message = "Error occurred during structured output validation of the context relevancy evaluation."
+            raise EvaluatorException(
+                error_message=error_message,
+                user_message=error_message,
+                exception=e,
+            )
+
+        return parse_eval_metric_result_structured_output(
+            result_content, EvaluationMetricName.CONTEXT_RELEVANCY
+        )
+
+    async def _handle_regular_output(
+        self,
+        input: EvaluationMetricInput,
+        config: EvaluationMetricConfig,
+    ) -> EvaluationMetricResult:
+        """Handle the regular output case for context relevancy evaluation."""
+        msgs = self._compose_msgs(input, config, enable_structured_output=False)
+        result = await self.language_model_service.complete_async(
+            messages=msgs,
+            model_name=config.language_model.name,
+            other_options=config.additional_llm_options,
+        )
+
+        result_content = result.choices[0].message.content
+        if not result_content or not isinstance(result_content, str):
+            error_message = "Context relevancy evaluation did not return a result."
+            raise EvaluatorException(
+                error_message=error_message,
+                user_message=error_message,
+            )
+
+        return parse_eval_metric_result(
+            result_content, EvaluationMetricName.CONTEXT_RELEVANCY
+        )
+
+    def _compose_msgs(
+        self,
+        input: EvaluationMetricInput,
+        config: EvaluationMetricConfig,
+        enable_structured_output: bool,
+    ) -> LanguageModelMessages:
+        """
+        Composes the messages for the relevancy metric.
+        """
+        system_msg_content = self._get_system_prompt(config, enable_structured_output)
+        system_msg = Prompt(system_msg_content).to_system_msg()
+
+        user_msg = Prompt(
+            self._get_user_prompt(config, enable_structured_output),
+            input_text=input.input_text,
+            context_texts=input.get_joined_context_texts(),
+        ).to_user_msg()
+
+        return LanguageModelMessages([system_msg, user_msg])
+
+    def _get_system_prompt(
+        self,
+        config: EvaluationMetricConfig,
+        enable_structured_output: bool,
+    ):
+        if (
+            enable_structured_output
+            and ModelCapabilities.STRUCTURED_OUTPUT
+            in config.language_model.capabilities
+        ):
+            return config.custom_prompts.setdefault(
+                SYSTEM_MSG_KEY,
+                CONTEXT_RELEVANCY_METRIC_SYSTEM_MSG_STRUCTURED_OUTPUT,
+            )
+        else:
+            return config.custom_prompts.setdefault(
+                SYSTEM_MSG_KEY,
+                CONTEXT_RELEVANCY_METRIC_SYSTEM_MSG,
+            )
+
+    def _get_user_prompt(
+        self,
+        config: EvaluationMetricConfig,
+        enable_structured_output: bool,
+    ):
+        if enable_structured_output:
+            return config.custom_prompts.setdefault(
+                USER_MSG_KEY,
+                CONTEXT_RELEVANCY_METRIC_USER_MSG_STRUCTURED_OUTPUT,
+            )
+        else:
+            return config.custom_prompts.setdefault(
+                USER_MSG_KEY,
+                CONTEXT_RELEVANCY_METRIC_USER_MSG,
+            )

unique_toolkit/evals/hallucination/constants.py

@@ -0,0 +1,61 @@
+from typing import Any
+
+from pydantic import Field
+
+from unique_toolkit._common.validators import LMI
+from unique_toolkit.evals.config import EvaluationMetricConfig
+from unique_toolkit.evals.hallucination.prompts import (
+    HALLUCINATION_METRIC_SYSTEM_MSG,
+    HALLUCINATION_METRIC_SYSTEM_MSG_DEFAULT,
+    HALLUCINATION_METRIC_USER_MSG,
+    HALLUCINATION_METRIC_USER_MSG_DEFAULT,
+)
+from unique_toolkit.evals.schemas import (
+    EvaluationMetricInputFieldName,
+    EvaluationMetricName,
+)
+from unique_toolkit.language_model.infos import LanguageModelInfo, LanguageModelName
+
+
+SYSTEM_MSG_KEY = "systemPrompt"
+USER_MSG_KEY = "userPrompt"
+SYSTEM_MSG_DEFAULT_KEY = "systemPromptDefault"
+USER_MSG_DEFAULT_KEY = "userPromptDefault"
+
+
+class HallucinationConfig(EvaluationMetricConfig):
+    enabled: bool = False
+    name: EvaluationMetricName = EvaluationMetricName.HALLUCINATION
+    language_model: LMI = LanguageModelInfo.from_name(
+        LanguageModelName.AZURE_GPT_35_TURBO_0125,
+    )
+    additional_llm_options: dict[str, Any] = Field(
+        default={},
+        description="Additional options to pass to the language model.",
+    )
+    custom_prompts: dict = {
+        SYSTEM_MSG_KEY: HALLUCINATION_METRIC_SYSTEM_MSG,
+        USER_MSG_KEY: HALLUCINATION_METRIC_USER_MSG,
+        SYSTEM_MSG_DEFAULT_KEY: HALLUCINATION_METRIC_SYSTEM_MSG_DEFAULT,
+        USER_MSG_DEFAULT_KEY: HALLUCINATION_METRIC_USER_MSG_DEFAULT,
+    }
+    score_to_label: dict = {
+        "LOW": "GREEN",
+        "MEDIUM": "YELLOW",
+        "HIGH": "RED",
+    }
+    score_to_title: dict = {
+        "LOW": "No Hallucination Detected",
+        "MEDIUM": "Hallucination Warning",
+        "HIGH": "High Hallucination",
+    }
+
+
+hallucination_metric_default_config = HallucinationConfig()
+
+hallucination_required_input_fields = [
+    EvaluationMetricInputFieldName.INPUT_TEXT,
+    EvaluationMetricInputFieldName.CONTEXT_TEXTS,
+    EvaluationMetricInputFieldName.HISTORY_MESSAGES,
+    EvaluationMetricInputFieldName.OUTPUT_TEXT,
+]

unique_toolkit/evals/hallucination/hallucination_evaluation.py

@@ -0,0 +1,91 @@
+from typing import Any
+
+from unique_toolkit.app.schemas import ChatEvent
+from unique_toolkit.chat.schemas import (
+    ChatMessageAssessmentLabel,
+    ChatMessageAssessmentStatus,
+    ChatMessageAssessmentType,
+)
+from unique_toolkit.evals.evaluation_manager import Evaluation
+from unique_toolkit.evals.hallucination.utils import check_hallucination
+from unique_toolkit.evals.schemas import (
+    EvaluationAssessmentMessage,
+    EvaluationMetricInput,
+    EvaluationMetricName,
+    EvaluationMetricResult,
+)
+from unique_toolkit.evals.hallucination.constants import (
+    HallucinationConfig,
+)
+from unique_toolkit.reference_manager.reference_manager import (
+    ReferenceManager,
+)
+
+from unique_toolkit.language_model.schemas import (
+    LanguageModelStreamResponse,
+)
+
+
+class HallucinationEvaluation(Evaluation):
+    def __init__(
+        self,
+        config: HallucinationConfig,
+        event: ChatEvent,
+        reference_manager: ReferenceManager,
+    ):
+        self.config = config
+        self._company_id = event.company_id
+        self._user_id = event.user_id
+        self._reference_manager = reference_manager
+        self._user_message = event.payload.user_message.text
+        super().__init__(EvaluationMetricName.HALLUCINATION)
+
+    async def run(
+        self, loop_response: LanguageModelStreamResponse
+    ) -> EvaluationMetricResult:  # type: ignore
+        chunks = self._reference_manager.get_chunks()
+
+        evaluation_result: EvaluationMetricResult = await check_hallucination(
+            company_id=self._company_id,
+            input=EvaluationMetricInput(
+                input_text=self._user_message,
+                context_texts=[context.text for context in chunks],
+                history_messages=[],  # TODO include loop_history messages
+                output_text=loop_response.message.text,
+            ),
+            config=self.config,
+        )
+
+        score_to_label = self.config.score_to_label
+        evaluation_result.is_positive = (
+            score_to_label.get(evaluation_result.value.upper(), "RED") != "RED"
+        )
+        return evaluation_result
+
+    def get_assessment_type(self) -> ChatMessageAssessmentType:
+        return ChatMessageAssessmentType.HALLUCINATION
+
+    async def evaluation_metric_to_assessment(
+        self, evaluation_result: EvaluationMetricResult
+    ) -> EvaluationAssessmentMessage:
+        title = self.config.score_to_title.get(
+            evaluation_result.value.upper(), evaluation_result.value
+        )
+        label = ChatMessageAssessmentLabel(
+            self.config.score_to_label.get(
+                evaluation_result.value.upper(), evaluation_result.value.upper()
+            )
+        )
+        status = (
+            ChatMessageAssessmentStatus.DONE
+            if not evaluation_result.error
+            else ChatMessageAssessmentStatus.ERROR
+        )
+
+        return EvaluationAssessmentMessage(
+            status=status,
+            title=title,
+            explanation=evaluation_result.reason,
+            label=label,
+            type=self.get_assessment_type(),
+        )