unique_toolkit 0.8.12__py3-none-any.whl → 0.8.14__py3-none-any.whl

unique_toolkit/_common/validators.py

@@ -1,6 +1,7 @@
-from typing import Annotated
+import logging
+from typing import Annotated, Any
 
-from pydantic import BeforeValidator, PlainSerializer
+from pydantic import BeforeValidator, Field, PlainSerializer, ValidationInfo
 
 from unique_toolkit.language_model import LanguageModelName
 from unique_toolkit.language_model.infos import (
@@ -8,6 +9,11 @@ from unique_toolkit.language_model.infos import (
     LanguageModelProvider,
 )
 
+from pydantic.fields import FieldInfo
+
+
+logger = logging.getLogger(__name__)
+
 # TODO @klcd: Inform on deprecation of str as input
 LMI = Annotated[
     LanguageModelInfo,
@@ -55,3 +61,32 @@ def validate_and_init_language_model_info(
         )
 
     return v
+
+
+def ClipInt(*, min_value: int, max_value: int) -> tuple[BeforeValidator, FieldInfo]:
+    def _validator(value: Any, info: ValidationInfo) -> Any:
+        if not isinstance(value, int):
+            value = int(value)
+
+        field_name = info.field_name
+        if value < min_value:
+            logger.warning(
+                "Field %s is below the allowed minimum of %s. It will be set to %s.",
+                field_name,
+                min_value,
+                min_value,
+            )
+            return min_value
+
+        if value > max_value:
+            logger.warning(
+                "Field %s is above the allowed maximum of %s. It will be set to %s.",
+                field_name,
+                max_value,
+                max_value,
+            )
+            return max_value
+
+        return value
+
+    return (BeforeValidator(_validator), Field(ge=min_value, le=max_value))

unique_toolkit/evals/evaluation_manager.py

@@ -0,0 +1,206 @@
+from abc import ABC
+import asyncio
+
+
+from unique_toolkit.tools.utils.execution.execution import Result, SafeTaskExecutor
+from logging import Logger
+from unique_toolkit.evals.schemas import (
+    EvaluationAssessmentMessage,
+    EvaluationMetricName,
+    EvaluationMetricResult,
+)
+from unique_toolkit.chat.schemas import (
+    ChatMessageAssessmentStatus,
+    ChatMessageAssessmentType,
+)
+from unique_toolkit.chat.service import ChatService
+from unique_toolkit.language_model.schemas import (
+    LanguageModelStreamResponse,
+)
+
+
+class Evaluation(ABC):
+    """
+    Abstract base class for evaluation metrics.
+    
+    typical use cases include:
+    - Hallucination checking
+    - compliance checking
+    """
+   
+
+    def __init__(self, name: EvaluationMetricName):
+        self.name = name
+
+    def get_name(self) -> EvaluationMetricName:
+        return self.name
+
+    def get_assessment_type(self) -> ChatMessageAssessmentType:
+        raise NotImplementedError(
+            "Subclasses must implement this method to return the assessment type."
+        )
+
+    async def run(
+        self, loop_response: LanguageModelStreamResponse
+    ) -> EvaluationMetricResult:
+        raise NotImplementedError("Subclasses must implement this method.")
+
+    async def evaluation_metric_to_assessment(
+        self, evaluation_result: EvaluationMetricResult
+    ) -> EvaluationAssessmentMessage:
+        raise NotImplementedError(
+            "Subclasses must implement this method to convert evaluation results to assessment messages."
+        )
+
+
+class EvaluationManager:
+    """
+    Manages the evaluation metrics and executes evaluation calls.
+
+    This class is responsible for:
+    - Storing and managing evaluation instances, identified by their unique names.
+    - Executing selected evaluations asynchronously and processing their results.
+    - Tracking the overall success or failure of evaluations.
+    - Integrating with external services like logging and chat systems to display evaluation statuses and results.
+    - Handling errors gracefully, including missing evaluations or failed executions.
+
+    Key Features:
+    - Evaluation Storage: Maintains a dictionary of evaluation instances for quick retrieval.
+    - Asynchronous Execution: Supports concurrent execution of multiple evaluations for efficiency.
+    - Result Processing: Tracks evaluation outcomes and updates the internal state based on results.
+    - Chat Integration: Updates the chat interface with evaluation statuses and detailed assessments.
+    - Error Handling: Provides robust error messages and fallback mechanisms for missing or failed evaluations.
+
+    The EvaluationManager serves as the central hub for managing and executing evaluations.
+    """
+    # a hashmap to hold evaluations by their names
+    _evaluations: dict[EvaluationMetricName, Evaluation] = {}
+    _evaluation_passed: bool = True
+
+    def __init__(
+        self,
+        logger: Logger,
+        chat_service: ChatService,
+        assistant_message_id: str,
+    ):
+        self._logger = logger
+        self._chat_service = chat_service
+        self._assistant_message_id = assistant_message_id
+
+    def add_evaluation(self, evaluation: Evaluation):
+        self._evaluations[evaluation.get_name()] = evaluation
+
+    def get_evaluation_by_name(self, name: EvaluationMetricName) -> Evaluation | None:
+        return self._evaluations.get(name)
+
+    async def run_evaluations(
+        self,
+        selected_evaluation_names: list[EvaluationMetricName],
+        loop_response: LanguageModelStreamResponse,
+    ) -> list[EvaluationMetricResult]:
+        task_executor = SafeTaskExecutor(
+            logger=self._logger,
+        )
+
+        tasks = [
+            task_executor.execute_async(
+                self.execute_evaluation_call,
+                loop_response=loop_response,
+                evaluation_name=evaluation_name,
+            )
+            for evaluation_name in selected_evaluation_names
+        ]
+        evaluation_results = await asyncio.gather(*tasks)
+        evaluation_results_unpacked: list[EvaluationMetricResult] = []
+
+        for i, result in enumerate(evaluation_results):
+            unpacked_evaluation_result = self._create_evaluation_metric_result(
+                result, selected_evaluation_names[i]
+            )
+            if not unpacked_evaluation_result.is_positive:
+                self._evaluation_passed = False
+            evaluation_results_unpacked.append(unpacked_evaluation_result)
+
+        return evaluation_results_unpacked
+
+    async def execute_evaluation_call(
+        self,
+        evaluation_name: EvaluationMetricName,
+        loop_response: LanguageModelStreamResponse,
+    ) -> EvaluationMetricResult:
+        self._logger.info(f"Processing tool call: {evaluation_name}")
+
+        evaluation_instance = self.get_evaluation_by_name(evaluation_name)
+
+        if evaluation_instance:
+            # Execute the evaluation
+            await self._create_assistant_message(evaluation_instance)
+            evaluation_metric_result: EvaluationMetricResult = (
+                await evaluation_instance.run(loop_response)
+            )
+            # show results to the user
+            await self._show_message_assessment(
+                evaluation_instance, evaluation_metric_result
+            )
+
+            return evaluation_metric_result
+
+        return EvaluationMetricResult(
+            name=evaluation_name,
+            is_positive=True,
+            value="RED",
+            reason=f"Evaluation named {evaluation_name} not found",
+            error=Exception("Evaluation named {evaluation_name} not found"),
+        )
+
+    def _create_evaluation_metric_result(
+        self,
+        result: Result[EvaluationMetricResult],
+        evaluation_name: EvaluationMetricName,
+    ) -> EvaluationMetricResult:
+        if not result.success:
+            return EvaluationMetricResult(
+                name=evaluation_name,
+                is_positive=True,
+                value="RED",
+                reason=str(result.exception),
+                error=Exception("Evaluation result is not successful"),
+            )
+        unpacked = result.unpack()
+        if not isinstance(unpacked, EvaluationMetricResult):
+            return EvaluationMetricResult(
+                name=evaluation_name,
+                is_positive=True,
+                value="RED",
+                reason="Evaluation result is not of type EvaluationMetricResult",
+                error=Exception(
+                    "Evaluation result is not of type EvaluationMetricResult"
+                ),
+            )
+        return unpacked
+
+    async def _show_message_assessment(
+        self,
+        evaluation_instance: Evaluation,
+        evaluation_metric_result: EvaluationMetricResult,
+    ) -> None:
+        evaluation_assessment_message = (
+            await evaluation_instance.evaluation_metric_to_assessment(
+                evaluation_metric_result
+            )
+        )
+        await self._chat_service.modify_message_assessment_async(
+            assistant_message_id=self._assistant_message_id,
+            status=evaluation_assessment_message.status,
+            title=evaluation_assessment_message.title,
+            explanation=evaluation_assessment_message.explanation,
+            label=evaluation_assessment_message.label,
+            type=evaluation_assessment_message.type,
+        )
+
+    async def _create_assistant_message(self, evaluation_instance: Evaluation):
+        await self._chat_service.create_message_assessment_async(
+            assistant_message_id=self._assistant_message_id,
+            status=ChatMessageAssessmentStatus.PENDING,
+            type=evaluation_instance.get_assessment_type(),
+        )

unique_toolkit/evals/exception.py

@@ -0,0 +1,5 @@
+from unique_toolkit._common.exception import CommonException
+
+
+class EvaluatorException(CommonException):
+    pass

unique_toolkit/evals/schemas.py

@@ -0,0 +1,100 @@
+from enum import StrEnum
+from typing import Optional
+
+from pydantic import BaseModel, ConfigDict, Field
+from unique_toolkit.chat import ChatMessage
+from unique_toolkit.evals.exception import EvaluatorException
+from unique_toolkit.chat.schemas import (
+    ChatMessageAssessmentLabel,
+    ChatMessageAssessmentStatus,
+    ChatMessageAssessmentType,
+)
+
+
+class EvaluationMetricName(StrEnum):
+    HALLUCINATION = "hallucination"
+    CONTEXT_RELEVANCY = "relevancy"
+
+
+class EvaluationMetricInputFieldName(StrEnum):
+    INPUT_TEXT = "input_text"
+    CONTEXT_TEXTS = "context_texts"
+    HISTORY_MESSAGES = "history_messages"
+    OUTPUT_TEXT = "output_text"
+
+
+class EvaluationMetricInput(BaseModel):
+    """
+    Input for any metric evaluation. Depending on the metric, the input can be different.
+    """
+
+    input_text: Optional[str] = None
+    context_texts: Optional[list[str]] = None
+    history_messages: Optional[list[ChatMessage]] = None
+    output_text: Optional[str] = None
+
+    def get_joined_context_texts(self, tag_name: str = "reference") -> str:
+        """
+        Concatenates context_texts.
+        """
+        if not self.context_texts:
+            return f"<No {tag_name} texts provided>"
+
+        return "\n".join(
+            [
+                f"<{tag_name}-{index + 1}>{text}</{tag_name}-{index + 1}>"
+                for index, text in enumerate(self.context_texts)
+            ]
+        )
+
+    def get_history_message_text(self, chat_message: ChatMessage):
+        return f"{chat_message.role.value}: {chat_message.content}"
+
+    def get_history_message_texts(self) -> list[str]:
+        if not self.history_messages:
+            return []
+        return [self.get_history_message_text(msg) for msg in self.history_messages]
+
+    def get_joined_history_texts(self, tag_name: str = "conversation") -> str:
+        """
+        Concatenates history message texts.
+        """
+        if not self.history_messages:
+            return f"<No {tag_name} texts provided>"
+
+        return "\n".join(self.get_history_message_texts())
+
+    def validate_required_fields(
+        self, required_fields: list[EvaluationMetricInputFieldName]
+    ):
+        """
+        Validates the input fields for the hallucination metric.
+        """
+        for field in required_fields:
+            value = getattr(self, field)
+            if value is None:
+                error_message = f"Missing required input field: {field}"
+                raise EvaluatorException(
+                    user_message=error_message,
+                    error_message=error_message,
+                )
+
+
+class EvaluationMetricResult(BaseModel):
+    model_config = ConfigDict(arbitrary_types_allowed=True)
+
+    name: EvaluationMetricName
+    value: str
+    reason: str
+    is_positive: Optional[bool] = None
+    user_info: Optional[str] = None
+    error: Exception | None = None
+    fact_list: list[str] = Field(default_factory=list[str])
+
+
+class EvaluationAssessmentMessage(BaseModel):
+    status: ChatMessageAssessmentStatus
+    explanation: str
+    title: str
+    label: ChatMessageAssessmentLabel
+    type: ChatMessageAssessmentType

unique_toolkit/history_manager/history_manager.py

@@ -0,0 +1,261 @@
+from datetime import datetime
+from logging import Logger
+from typing import Awaitable, Callable
+
+from pydantic import BaseModel, Field
+
+from unique_toolkit.app.schemas import ChatEvent
+
+
+from unique_toolkit.chat.schemas import ChatMessage
+from unique_toolkit.chat.service import ChatService
+from unique_toolkit.content.schemas import Content
+from unique_toolkit.content.service import ContentService
+from unique_toolkit.language_model.builder import MessagesBuilder
+from unique_toolkit.language_model.schemas import (
+    LanguageModelAssistantMessage,
+    LanguageModelFunction,
+    LanguageModelMessage,
+    LanguageModelMessageRole,
+    LanguageModelToolMessage,
+    LanguageModelUserMessage,
+)
+
+from unique_toolkit.tools.schemas import ToolCallResponse
+from unique_toolkit.content.utils import count_tokens
+from unique_toolkit.history_manager.utils import transform_chunks_to_string
+
+
+class HistoryManagerConfig(BaseModel):
+
+    class ExperimentalFeatures(BaseModel):
+        def __init__(self, full_sources_serialize_dump: bool = False):
+            self.full_sources_serialize_dump = full_sources_serialize_dump
+
+        full_sources_serialize_dump: bool = Field(
+            default=False,
+            description="If True, the sources will be serialized in full, otherwise only the content will be serialized.",
+        )
+
+    experimental_features: ExperimentalFeatures = Field(
+        default=ExperimentalFeatures(),
+        description="Experimental features for the history manager.",
+    )
+
+    max_history_tokens: int = Field(
+        default=8000,
+        ge=0,
+        description="The maximum number of tokens to keep in the history.",
+    )
+
+
+class HistoryManager:
+    """
+    Manages the history of tool calls and conversation loops.
+
+    This class is responsible for:
+    - Storing and maintaining the history of tool call results and conversation messages.
+    - Merging uploaded content with the conversation history for a unified view.
+    - Limiting the history to fit within a configurable token window for efficient processing.
+    - Providing methods to retrieve, manipulate, and append to the conversation history.
+    - Handling post-processing steps to clean or modify the history as needed.
+
+    Key Features:
+    - Tool Call History: Tracks the results of tool calls and appends them to the conversation history.
+    - Loop History: Maintains a record of conversation loops, including assistant and user messages.
+    - History Merging: Combines uploaded files and chat messages into a cohesive history.
+    - Token Window Management: Ensures the history stays within a specified token limit for optimal performance.
+    - Post-Processing Support: Allows for custom transformations or cleanup of the conversation history.
+
+    The HistoryManager serves as the backbone for managing and retrieving conversation history in a structured and efficient manner.
+    """
+    _tool_call_result_history: list[ToolCallResponse] = []
+    _loop_history: list[LanguageModelMessage] = []
+    _source_enumerator = 0
+
+    def __init__(
+        self,
+        logger: Logger,
+        event: ChatEvent,
+        config: HistoryManagerConfig,
+    ):
+        self._config = config
+        self._logger = logger
+        self._chat_service = ChatService(event)
+        self._content_service = ContentService.from_event(event)
+
+    def has_no_loop_messages(self) -> bool:
+        return len(self._loop_history) == 0
+
+    def add_tool_call_results(self, tool_call_results: list[ToolCallResponse]):
+        for tool_response in tool_call_results:
+            if not tool_response.successful:
+                self._loop_history.append(
+                    LanguageModelToolMessage(
+                        name=tool_response.name,
+                        tool_call_id=tool_response.id,
+                        content=f"Tool call {tool_response.name} failed with error: {tool_response.error_message}",
+                    )
+                )
+                continue
+            self._append_tool_call_result_to_history(tool_response)
+
+    def _append_tool_call_result_to_history(
+        self,
+        tool_response: ToolCallResponse,
+    ) -> None:
+        tool_call_result_for_history = self._get_tool_call_result_for_loop_history(
+            tool_response=tool_response
+        )
+        self._loop_history.append(tool_call_result_for_history)
+
+    def _get_tool_call_result_for_loop_history(
+        self,
+        tool_response: ToolCallResponse,
+    ) -> LanguageModelMessage:
+        self._logger.debug(
+            f"Appending tool call result to history: {tool_response.name}"
+        )
+
+        content_chunks = (
+            tool_response.content_chunks or []
+        )  # it can be that the tool response does not have content chunks
+
+        # Transform content chunks into sources to be appended to tool result
+        sources = transform_chunks_to_string(
+            content_chunks,
+            self._source_enumerator,
+            None,  # Use None for SourceFormatConfig
+            self._config.experimental_features.full_sources_serialize_dump,
+        )
+
+        self._source_enumerator += len(
+            sources
+        )  # To make sure all sources have unique source numbers
+
+        # Append the result to the history
+        return LanguageModelToolMessage(
+            content=sources,
+            tool_call_id=tool_response.id,  # type: ignore
+            name=tool_response.name,
+        )
+
+    def _append_tool_calls_to_history(
+        self, tool_calls: list[LanguageModelFunction]
+    ) -> None:
+        self._loop_history.append(
+            LanguageModelAssistantMessage.from_functions(tool_calls=tool_calls)
+        )
+
+    def add_assistant_message(self, message: LanguageModelAssistantMessage) -> None:
+        self._loop_history.append(message)
+
+    async def get_history(
+        self,
+        postprocessing_step: Callable[
+            [list[LanguageModelMessage]], list[LanguageModelMessage]
+        ]
+        | None = None,
+    ) -> list[LanguageModelMessage]:
+        """
+        Get the history of the conversation. The function will retrieve a subset of the full history based on the configuration.
+
+        Returns:
+            list[LanguageModelMessage]: The history
+        """
+        # Get uploaded files
+        uploaded_files = self._content_service.search_content_on_chat(
+            chat_id=self._chat_service.chat_id
+        )
+        # Get all message history
+        full_history = await self._chat_service.get_full_history_async()
+
+        merged_history = self._merge_history_and_uploads(full_history, uploaded_files)
+
+        if postprocessing_step is not None:
+            merged_history = postprocessing_step(merged_history)
+
+        limited_history = self._limit_to_token_window(
+            merged_history, self._config.max_history_tokens
+        )
+
+        # Add current user message if not already in history
+        # we grab it fresh from the db so it must contain all the messages this code is not needed anymore below currently it's left in for explainability
+        # current_user_msg = LanguageModelUserMessage(
+        #     content=self.event.payload.user_message.text
+        # )
+        # if not any(
+        #     msg.role == LanguageModelMessageRole.USER
+        #     and msg.content == current_user_msg.content
+        #     for msg in complete_history
+        # ):
+        #     complete_history.append(current_user_msg)
+
+        # # Add final assistant response - this should be available when this method is called
+        # if (
+        #     hasattr(self, "loop_response")
+        #     and self.loop_response
+        #     and self.loop_response.message.text
+        # ):
+        #     complete_history.append(
+        #         LanguageModelAssistantMessage(
+        #             content=self.loop_response.message.text
+        #         )
+        #     )
+        # else:
+        #     self.logger.warning(
+        #         "Called get_complete_conversation_history_after_streaming_no_tool_calls but no loop_response.message.text is available"
+        #     )
+
+        return limited_history
+
+    def _merge_history_and_uploads(
+        self, history: list[ChatMessage], uploads: list[Content]
+    ) -> list[LanguageModelMessage]:
+        # Assert that all content have a created_at
+        content_with_created_at = [content for content in uploads if content.created_at]
+        sorted_history = sorted(
+            history + content_with_created_at,
+            key=lambda x: x.created_at or datetime.min,
+        )
+
+        msg_builder = MessagesBuilder()
+        for msg in sorted_history:
+            if isinstance(msg, Content):
+                msg_builder.user_message_append(
+                    f"Uploaded file: {msg.key}, ContentId: {msg.id}"
+                )
+            else:
+                msg_builder.messages.append(
+                    LanguageModelMessage(
+                        role=LanguageModelMessageRole(msg.role),
+                        content=msg.content,
+                    )
+                )
+        return msg_builder.messages
+
+    def _limit_to_token_window(
+        self, messages: list[LanguageModelMessage], token_limit: int
+    ) -> list[LanguageModelMessage]:
+        selected_messages = []
+        token_count = 0
+        for msg in messages[::-1]:
+            msg_token_count = count_tokens(str(msg.content))
+            if token_count + msg_token_count > token_limit:
+                break
+            selected_messages.append(msg)
+            token_count += msg_token_count
+        return selected_messages[::-1]
+
+    async def remove_post_processing_manipulations(
+        self, remove_from_text: Callable[[str], Awaitable[str]]
+    ) -> list[LanguageModelMessage]:
+        messages = await self.get_history()
+        for message in messages:
+            if isinstance(message.content, str):
+                message.content = await remove_from_text(message.content)
+            else:
+                self._logger.warning(
+                    f"Skipping message with unsupported content type: {type(message.content)}"
+                )
+        return messages