unique_toolkit 0.8.11__tar.gz → 0.8.13__tar.gz

{unique_toolkit-0.8.11 → unique_toolkit-0.8.13}/CHANGELOG.md

@@ -5,6 +5,18 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), 
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## [0.8.12] - 2025-08-18
+- Adding initial versions of
+  - Evaluation Manager
+  - History Manager
+  - Postprocessor Manager
+  - Thinking Manager
+- Updated tool manager
+
+## [0.8.13] - 2025-08-18
+- Fix no tool call respoonse in ChatMessage -> Open Ai messages translation
+- Add simple append method to OpenAIMessageBuilder
+
 ## [0.8.11] - 2025-08-15
 - Fix no tool call respoonse in ChatMessage -> Open Ai messages translation
 - Add simple append method to OpenAIMessageBuilder

{unique_toolkit-0.8.11 → unique_toolkit-0.8.13}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: unique_toolkit
-Version: 0.8.11
+Version: 0.8.13
 Summary: 
 License: Proprietary
 Author: Martin Fadler
@@ -114,6 +114,18 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), 
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## [0.8.12] - 2025-08-18
+- Adding initial versions of
+  - Evaluation Manager
+  - History Manager
+  - Postprocessor Manager
+  - Thinking Manager
+- Updated tool manager
+
+## [0.8.13] - 2025-08-18
+- Fix no tool call respoonse in ChatMessage -> Open Ai messages translation
+- Add simple append method to OpenAIMessageBuilder
+
 ## [0.8.11] - 2025-08-15
 - Fix no tool call respoonse in ChatMessage -> Open Ai messages translation
 - Add simple append method to OpenAIMessageBuilder

{unique_toolkit-0.8.11 → unique_toolkit-0.8.13}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "unique_toolkit"
-version = "0.8.11"
+version = "0.8.13"
 description = ""
 authors = [
     "Martin Fadler <martin.fadler@unique.ch>",

{unique_toolkit-0.8.11 → unique_toolkit-0.8.13}/unique_toolkit/_common/validators.py

@@ -1,6 +1,7 @@
-from typing import Annotated
+import logging
+from typing import Annotated, Any
 
-from pydantic import BeforeValidator, PlainSerializer
+from pydantic import BeforeValidator, Field, PlainSerializer, ValidationInfo
 
 from unique_toolkit.language_model import LanguageModelName
 from unique_toolkit.language_model.infos import (
@@ -8,6 +9,11 @@ from unique_toolkit.language_model.infos import (
     LanguageModelProvider,
 )
 
+from pydantic.fields import FieldInfo
+
+
+logger = logging.getLogger(__name__)
+
 # TODO @klcd: Inform on deprecation of str as input
 LMI = Annotated[
     LanguageModelInfo,
@@ -55,3 +61,32 @@ def validate_and_init_language_model_info(
         )
 
     return v
+
+
+def ClipInt(*, min_value: int, max_value: int) -> tuple[BeforeValidator, FieldInfo]:
+    def _validator(value: Any, info: ValidationInfo) -> Any:
+        if not isinstance(value, int):
+            value = int(value)
+
+        field_name = info.field_name
+        if value < min_value:
+            logger.warning(
+                "Field %s is below the allowed minimum of %s. It will be set to %s.",
+                field_name,
+                min_value,
+                min_value,
+            )
+            return min_value
+
+        if value > max_value:
+            logger.warning(
+                "Field %s is above the allowed maximum of %s. It will be set to %s.",
+                field_name,
+                max_value,
+                max_value,
+            )
+            return max_value
+
+        return value
+
+    return (BeforeValidator(_validator), Field(ge=min_value, le=max_value))

{unique_toolkit-0.8.11 → unique_toolkit-0.8.13}/unique_toolkit/content/service.py

@@ -8,6 +8,7 @@ from typing_extensions import deprecated
 
 from unique_toolkit._common.validate_required_values import validate_required_values
 from unique_toolkit.app.schemas import BaseEvent, ChatEvent, Event
+from unique_toolkit.app.unique_settings import UniqueSettings
 from unique_toolkit.content import DOMAIN_NAME
 from unique_toolkit.content.constants import DEFAULT_SEARCH_LANGUAGE
 from unique_toolkit.content.functions import (
@@ -53,7 +54,7 @@ class ContentService:
         *,
         company_id: str,
         user_id: str,
-        chat_id: str | None,
+        chat_id: str | None = None,
         metadata_filter: dict | None = None,
     ): ...
 
@@ -107,6 +108,19 @@ class ContentService:
             metadata_filter=metadata_filter,
         )
 
+    @classmethod
+    def from_settings(
+        cls, settings: UniqueSettings, metadata_filter: dict | None = None
+    ):
+        """
+        Initialize the ContentService with a settings object.
+        """
+        return cls(
+            company_id=settings.auth.company_id.get_secret_value(),
+            user_id=settings.auth.user_id.get_secret_value(),
+            metadata_filter=metadata_filter,
+        )
+
     @property
     @deprecated(
         "The event property is deprecated and will be removed in a future version."

{unique_toolkit-0.8.11 → unique_toolkit-0.8.13}/unique_toolkit/embedding/service.py

@@ -5,6 +5,7 @@ from typing_extensions import deprecated
 from unique_toolkit._common._base_service import BaseService
 from unique_toolkit._common.validate_required_values import validate_required_values
 from unique_toolkit.app.schemas import BaseEvent, Event
+from unique_toolkit.app.unique_settings import UniqueSettings
 from unique_toolkit.embedding.constants import DEFAULT_TIMEOUT
 from unique_toolkit.embedding.functions import embed_texts, embed_texts_async
 from unique_toolkit.embedding.schemas import Embeddings
@@ -54,6 +55,16 @@ class EmbeddingService(BaseService):
         """
         return cls(company_id=event.company_id, user_id=event.user_id)
 
+    @classmethod
+    def from_settings(cls, settings: UniqueSettings):
+        """
+        Initialize the EmbeddingService with a settings object.
+        """
+        return cls(
+            company_id=settings.auth.company_id.get_secret_value(),
+            user_id=settings.auth.user_id.get_secret_value(),
+        )
+
     @property
     @deprecated(
         "The event property is deprecated and will be removed in a future version."

unique_toolkit-0.8.13/unique_toolkit/evals/evaluation_manager.py

@@ -0,0 +1,206 @@
+from abc import ABC
+import asyncio
+
+
+from unique_toolkit.tools.utils.execution.execution import Result, SafeTaskExecutor
+from logging import Logger
+from unique_toolkit.evals.schemas import (
+    EvaluationAssessmentMessage,
+    EvaluationMetricName,
+    EvaluationMetricResult,
+)
+from unique_toolkit.chat.schemas import (
+    ChatMessageAssessmentStatus,
+    ChatMessageAssessmentType,
+)
+from unique_toolkit.chat.service import ChatService
+from unique_toolkit.language_model.schemas import (
+    LanguageModelStreamResponse,
+)
+
+
+class Evaluation(ABC):
+    """
+    Abstract base class for evaluation metrics.
+    
+    typical use cases include:
+    - Hallucination checking
+    - compliance checking
+    """
+   
+
+    def __init__(self, name: EvaluationMetricName):
+        self.name = name
+
+    def get_name(self) -> EvaluationMetricName:
+        return self.name
+
+    def get_assessment_type(self) -> ChatMessageAssessmentType:
+        raise NotImplementedError(
+            "Subclasses must implement this method to return the assessment type."
+        )
+
+    async def run(
+        self, loop_response: LanguageModelStreamResponse
+    ) -> EvaluationMetricResult:
+        raise NotImplementedError("Subclasses must implement this method.")
+
+    async def evaluation_metric_to_assessment(
+        self, evaluation_result: EvaluationMetricResult
+    ) -> EvaluationAssessmentMessage:
+        raise NotImplementedError(
+            "Subclasses must implement this method to convert evaluation results to assessment messages."
+        )
+
+
+class EvaluationManager:
+    """
+    Manages the evaluation metrics and executes evaluation calls.
+
+    This class is responsible for:
+    - Storing and managing evaluation instances, identified by their unique names.
+    - Executing selected evaluations asynchronously and processing their results.
+    - Tracking the overall success or failure of evaluations.
+    - Integrating with external services like logging and chat systems to display evaluation statuses and results.
+    - Handling errors gracefully, including missing evaluations or failed executions.
+
+    Key Features:
+    - Evaluation Storage: Maintains a dictionary of evaluation instances for quick retrieval.
+    - Asynchronous Execution: Supports concurrent execution of multiple evaluations for efficiency.
+    - Result Processing: Tracks evaluation outcomes and updates the internal state based on results.
+    - Chat Integration: Updates the chat interface with evaluation statuses and detailed assessments.
+    - Error Handling: Provides robust error messages and fallback mechanisms for missing or failed evaluations.
+
+    The EvaluationManager serves as the central hub for managing and executing evaluations.
+    """
+    # a hashmap to hold evaluations by their names
+    _evaluations: dict[EvaluationMetricName, Evaluation] = {}
+    _evaluation_passed: bool = True
+
+    def __init__(
+        self,
+        logger: Logger,
+        chat_service: ChatService,
+        assistant_message_id: str,
+    ):
+        self._logger = logger
+        self._chat_service = chat_service
+        self._assistant_message_id = assistant_message_id
+
+    def add_evaluation(self, evaluation: Evaluation):
+        self._evaluations[evaluation.get_name()] = evaluation
+
+    def get_evaluation_by_name(self, name: EvaluationMetricName) -> Evaluation | None:
+        return self._evaluations.get(name)
+
+    async def run_evaluations(
+        self,
+        selected_evaluation_names: list[EvaluationMetricName],
+        loop_response: LanguageModelStreamResponse,
+    ) -> list[EvaluationMetricResult]:
+        task_executor = SafeTaskExecutor(
+            logger=self._logger,
+        )
+
+        tasks = [
+            task_executor.execute_async(
+                self.execute_evaluation_call,
+                loop_response=loop_response,
+                evaluation_name=evaluation_name,
+            )
+            for evaluation_name in selected_evaluation_names
+        ]
+        evaluation_results = await asyncio.gather(*tasks)
+        evaluation_results_unpacked: list[EvaluationMetricResult] = []
+
+        for i, result in enumerate(evaluation_results):
+            unpacked_evaluation_result = self._create_evaluation_metric_result(
+                result, selected_evaluation_names[i]
+            )
+            if not unpacked_evaluation_result.is_positive:
+                self._evaluation_passed = False
+            evaluation_results_unpacked.append(unpacked_evaluation_result)
+
+        return evaluation_results_unpacked
+
+    async def execute_evaluation_call(
+        self,
+        evaluation_name: EvaluationMetricName,
+        loop_response: LanguageModelStreamResponse,
+    ) -> EvaluationMetricResult:
+        self._logger.info(f"Processing tool call: {evaluation_name}")
+
+        evaluation_instance = self.get_evaluation_by_name(evaluation_name)
+
+        if evaluation_instance:
+            # Execute the evaluation
+            await self._create_assistant_message(evaluation_instance)
+            evaluation_metric_result: EvaluationMetricResult = (
+                await evaluation_instance.run(loop_response)
+            )
+            # show results to the user
+            await self._show_message_assessment(
+                evaluation_instance, evaluation_metric_result
+            )
+
+            return evaluation_metric_result
+
+        return EvaluationMetricResult(
+            name=evaluation_name,
+            is_positive=True,
+            value="RED",
+            reason=f"Evaluation named {evaluation_name} not found",
+            error=Exception("Evaluation named {evaluation_name} not found"),
+        )
+
+    def _create_evaluation_metric_result(
+        self,
+        result: Result[EvaluationMetricResult],
+        evaluation_name: EvaluationMetricName,
+    ) -> EvaluationMetricResult:
+        if not result.success:
+            return EvaluationMetricResult(
+                name=evaluation_name,
+                is_positive=True,
+                value="RED",
+                reason=str(result.exception),
+                error=Exception("Evaluation result is not successful"),
+            )
+        unpacked = result.unpack()
+        if not isinstance(unpacked, EvaluationMetricResult):
+            return EvaluationMetricResult(
+                name=evaluation_name,
+                is_positive=True,
+                value="RED",
+                reason="Evaluation result is not of type EvaluationMetricResult",
+                error=Exception(
+                    "Evaluation result is not of type EvaluationMetricResult"
+                ),
+            )
+        return unpacked
+
+    async def _show_message_assessment(
+        self,
+        evaluation_instance: Evaluation,
+        evaluation_metric_result: EvaluationMetricResult,
+    ) -> None:
+        evaluation_assessment_message = (
+            await evaluation_instance.evaluation_metric_to_assessment(
+                evaluation_metric_result
+            )
+        )
+        await self._chat_service.modify_message_assessment_async(
+            assistant_message_id=self._assistant_message_id,
+            status=evaluation_assessment_message.status,
+            title=evaluation_assessment_message.title,
+            explanation=evaluation_assessment_message.explanation,
+            label=evaluation_assessment_message.label,
+            type=evaluation_assessment_message.type,
+        )
+
+    async def _create_assistant_message(self, evaluation_instance: Evaluation):
+        await self._chat_service.create_message_assessment_async(
+            assistant_message_id=self._assistant_message_id,
+            status=ChatMessageAssessmentStatus.PENDING,
+            type=evaluation_instance.get_assessment_type(),
+        )

unique_toolkit-0.8.13/unique_toolkit/evals/schemas.py

@@ -0,0 +1,100 @@
+from enum import StrEnum
+from typing import Optional
+
+from pydantic import BaseModel, ConfigDict, Field
+from unique_toolkit.chat import ChatMessage
+from unique_toolkit.evals.exception import EvaluatorException
+from unique_toolkit.chat.schemas import (
+    ChatMessageAssessmentLabel,
+    ChatMessageAssessmentStatus,
+    ChatMessageAssessmentType,
+)
+
+
+class EvaluationMetricName(StrEnum):
+    HALLUCINATION = "hallucination"
+    CONTEXT_RELEVANCY = "relevancy"
+
+
+class EvaluationMetricInputFieldName(StrEnum):
+    INPUT_TEXT = "input_text"
+    CONTEXT_TEXTS = "context_texts"
+    HISTORY_MESSAGES = "history_messages"
+    OUTPUT_TEXT = "output_text"
+
+
+class EvaluationMetricInput(BaseModel):
+    """
+    Input for any metric evaluation. Depending on the metric, the input can be different.
+    """
+
+    input_text: Optional[str] = None
+    context_texts: Optional[list[str]] = None
+    history_messages: Optional[list[ChatMessage]] = None
+    output_text: Optional[str] = None
+
+    def get_joined_context_texts(self, tag_name: str = "reference") -> str:
+        """
+        Concatenates context_texts.
+        """
+        if not self.context_texts:
+            return f"<No {tag_name} texts provided>"
+
+        return "\n".join(
+            [
+                f"<{tag_name}-{index + 1}>{text}</{tag_name}-{index + 1}>"
+                for index, text in enumerate(self.context_texts)
+            ]
+        )
+
+    def get_history_message_text(self, chat_message: ChatMessage):
+        return f"{chat_message.role.value}: {chat_message.content}"
+
+    def get_history_message_texts(self) -> list[str]:
+        if not self.history_messages:
+            return []
+        return [self.get_history_message_text(msg) for msg in self.history_messages]
+
+    def get_joined_history_texts(self, tag_name: str = "conversation") -> str:
+        """
+        Concatenates history message texts.
+        """
+        if not self.history_messages:
+            return f"<No {tag_name} texts provided>"
+
+        return "\n".join(self.get_history_message_texts())
+
+    def validate_required_fields(
+        self, required_fields: list[EvaluationMetricInputFieldName]
+    ):
+        """
+        Validates the input fields for the hallucination metric.
+        """
+        for field in required_fields:
+            value = getattr(self, field)
+            if value is None:
+                error_message = f"Missing required input field: {field}"
+                raise EvaluatorException(
+                    user_message=error_message,
+                    error_message=error_message,
+                )
+
+
+class EvaluationMetricResult(BaseModel):
+    model_config = ConfigDict(arbitrary_types_allowed=True)
+
+    name: EvaluationMetricName
+    value: str
+    reason: str
+    is_positive: Optional[bool] = None
+    user_info: Optional[str] = None
+    error: Exception | None = None
+    fact_list: list[str] = Field(default_factory=list[str])
+
+
+class EvaluationAssessmentMessage(BaseModel):
+    status: ChatMessageAssessmentStatus
+    explanation: str
+    title: str
+    label: ChatMessageAssessmentLabel
+    type: ChatMessageAssessmentType

unique_toolkit-0.8.13/unique_toolkit/evaluators/exception.py

@@ -0,0 +1,5 @@
+from unique_toolkit._common.exception import CommonException
+
+
+class EvaluatorException(CommonException):
+    pass