unique_toolkit 0.7.7__py3-none-any.whl → 1.23.0__py3-none-any.whl

unique_toolkit/{evaluators → agentic/evaluation}/hallucination/service.py

@@ -1,27 +1,28 @@
 import logging
 
-from unique_toolkit.app.schemas import Event
-from unique_toolkit.evaluators.config import (
-    EvaluationMetricConfig,
-)
-from unique_toolkit.evaluators.hallucination.constants import (
-    hallucination_metric_default_config,
-)
-from unique_toolkit.evaluators.hallucination.utils import check_hallucination_async
-from unique_toolkit.evaluators.schemas import (
+from unique_toolkit.agentic.evaluation.config import EvaluationMetricConfig
+from unique_toolkit.agentic.evaluation.schemas import (
     EvaluationMetricInput,
     EvaluationMetricResult,
 )
+from unique_toolkit.app.schemas import ChatEvent
 
-logger = logging.getLogger(__name__)
+from .constants import hallucination_metric_default_config
+from .utils import check_hallucination
+
+SYSTEM_MSG_KEY = "systemPrompt"
+USER_MSG_KEY = "userPrompt"
+SYSTEM_MSG_DEFAULT_KEY = "systemPromptDefault"
+USER_MSG_DEFAULT_KEY = "userPromptDefault"
 
 
 class HallucinationEvaluator:
-    def __init__(self, event: Event, logger: logging.Logger = logger):
+    def __init__(self, event: ChatEvent):
         self.event = event
-        self.logger = logger
 
-    async def run(
+        self.logger = logging.getLogger(f"HallucinationEvaluator.{__name__}")
+
+    async def analyze(
         self,
         input: EvaluationMetricInput,
         config: EvaluationMetricConfig = hallucination_metric_default_config,
@@ -36,7 +37,7 @@ class HallucinationEvaluator:
         If no contexts or history are referenced in the generated output, the method verifies
         that the output does not contain any relevant information to answer the question.
 
-        This method calls `check_hallucination_async` to perform the actual analysis. The `check_hallucination_async`
+        This method calls `check_hallucination` to perform the actual analysis. The `check_hallucination`
         function handles the evaluation using the company ID from the event, the provided input, and the configuration.
 
         Args:
@@ -53,6 +54,6 @@ class HallucinationEvaluator:
             self.logger.info("Hallucination metric is not enabled.")
             return None
 
-        return await check_hallucination_async(
+        return await check_hallucination(
             company_id=self.event.company_id, input=input, config=config
         )

unique_toolkit/{evaluators → agentic/evaluation}/hallucination/utils.py

@@ -1,32 +1,30 @@
 import logging
 from string import Template
 
-from unique_toolkit.evaluators.config import (
-    EvaluationMetricConfig,
-)
-from unique_toolkit.evaluators.exception import EvaluatorException
-from unique_toolkit.evaluators.hallucination.constants import (
-    SYSTEM_MSG_DEFAULT_KEY,
-    SYSTEM_MSG_KEY,
-    USER_MSG_DEFAULT_KEY,
-    USER_MSG_KEY,
-    hallucination_required_input_fields,
-)
-from unique_toolkit.evaluators.output_parser import (
-    parse_eval_metric_result,
-)
-from unique_toolkit.evaluators.schemas import (
+from unique_toolkit.agentic.evaluation.config import EvaluationMetricConfig
+from unique_toolkit.agentic.evaluation.exception import EvaluatorException
+from unique_toolkit.agentic.evaluation.output_parser import parse_eval_metric_result
+from unique_toolkit.agentic.evaluation.schemas import (
     EvaluationMetricInput,
     EvaluationMetricName,
     EvaluationMetricResult,
 )
+from unique_toolkit.content.schemas import ContentChunk
 from unique_toolkit.language_model.schemas import (
     LanguageModelMessages,
+    LanguageModelStreamResponse,
     LanguageModelSystemMessage,
     LanguageModelUserMessage,
 )
 from unique_toolkit.language_model.service import LanguageModelService
 
+from .constants import (
+    SYSTEM_MSG_DEFAULT_KEY,
+    SYSTEM_MSG_KEY,
+    USER_MSG_DEFAULT_KEY,
+    USER_MSG_KEY,
+    hallucination_required_input_fields,
+)
 from .prompts import (
     HALLUCINATION_METRIC_SYSTEM_MSG,
     HALLUCINATION_METRIC_SYSTEM_MSG_DEFAULT,
@@ -34,15 +32,12 @@ from .prompts import (
     HALLUCINATION_METRIC_USER_MSG_DEFAULT,
 )
 
-logger = logging.getLogger(__name__)
-
 
-async def check_hallucination_async(
+async def check_hallucination(
     company_id: str,
     input: EvaluationMetricInput,
     config: EvaluationMetricConfig,
-    logger: logging.Logger = logger,
-) -> EvaluationMetricResult | None:
+) -> EvaluationMetricResult:
     """
     Analyzes the level of hallucination in the generated output by comparing it with the provided input
     and the contexts or history. The analysis classifies the hallucination level as:
@@ -73,6 +68,9 @@ async def check_hallucination_async(
     Raises:
         EvaluatorException: If the context texts are empty, required fields are missing, or an error occurs during the evaluation.
     """
+
+    logger = logging.getLogger(f"check_hallucination.{__name__}")
+
     model_name = config.language_model.name
     logger.info(f"Analyzing level of hallucination with {model_name}.")
 
@@ -200,3 +198,15 @@ def _get_user_prompt_default(config: EvaluationMetricConfig):
         USER_MSG_DEFAULT_KEY,
         HALLUCINATION_METRIC_USER_MSG_DEFAULT,
     )
+
+
+def context_text_from_stream_response(
+    response: LanguageModelStreamResponse, selected_chunks: list[ContentChunk]
+):
+    response_references = response.message.references
+    reference_ids = [reference.source_id for reference in response_references]
+    filtered_contexts: list[str] = []
+    for chunk in selected_chunks:
+        if f"{chunk.id}_{chunk.chunk_id}" in reference_ids:
+            filtered_contexts.append(chunk.text)
+    return filtered_contexts

unique_toolkit/{evaluators → agentic/evaluation}/output_parser.py

@@ -1,5 +1,8 @@
-from unique_toolkit.evaluators.exception import EvaluatorException
-from unique_toolkit.evaluators.schemas import (
+from unique_toolkit.agentic.evaluation.context_relevancy.schema import (
+    EvaluationSchemaStructuredOutput,
+)
+from unique_toolkit.agentic.evaluation.exception import EvaluatorException
+from unique_toolkit.agentic.evaluation.schemas import (
     EvaluationMetricName,
     EvaluationMetricResult,
 )
@@ -28,3 +31,18 @@ def parse_eval_metric_result(
         value=parsed_result.get("value", "None"),
         reason=parsed_result.get("reason", "None"),
     )
+
+
+def parse_eval_metric_result_structured_output(
+    result: EvaluationSchemaStructuredOutput,
+    metric_name: EvaluationMetricName,
+) -> EvaluationMetricResult:
+    """
+    Parses the evaluation metric result.
+    """
+    return EvaluationMetricResult(
+        name=metric_name,
+        value=result.value,
+        reason=result.reason,
+        fact_list=[item.fact for item in result.fact_list],
+    )

unique_toolkit/{evaluators → agentic/evaluation}/schemas.py

@@ -1,18 +1,24 @@
-from enum import Enum
+from enum import StrEnum
 from typing import Optional
 
-from pydantic import BaseModel
+from pydantic import BaseModel, ConfigDict, Field
 
+from unique_toolkit.agentic.evaluation.exception import EvaluatorException
 from unique_toolkit.chat import ChatMessage
-from unique_toolkit.evaluators.exception import EvaluatorException
+from unique_toolkit.chat.schemas import (
+    ChatMessageAssessmentLabel,
+    ChatMessageAssessmentStatus,
+    ChatMessageAssessmentType,
+)
 
 
-class EvaluationMetricName(Enum):
+class EvaluationMetricName(StrEnum):
     HALLUCINATION = "hallucination"
     CONTEXT_RELEVANCY = "relevancy"
+    SUB_AGENT = "sub_agent"
 
 
-class EvaluationMetricInputFieldName(str, Enum):
+class EvaluationMetricInputFieldName(StrEnum):
     INPUT_TEXT = "input_text"
     CONTEXT_TEXTS = "context_texts"
     HISTORY_MESSAGES = "history_messages"
@@ -38,7 +44,7 @@ class EvaluationMetricInput(BaseModel):
 
         return "\n".join(
             [
-                f"<{tag_name}-{index}>{text}</{tag_name}-{index}>"
+                f"<{tag_name}-{index + 1}>{text}</{tag_name}-{index + 1}>"
                 for index, text in enumerate(self.context_texts)
             ]
         )
@@ -46,7 +52,7 @@ class EvaluationMetricInput(BaseModel):
     def get_history_message_text(self, chat_message: ChatMessage):
         return f"{chat_message.role.value}: {chat_message.content}"
 
-    def get_history_message_texts(self):
+    def get_history_message_texts(self) -> list[str]:
         if not self.history_messages:
             return []
         return [self.get_history_message_text(msg) for msg in self.history_messages]
@@ -77,6 +83,20 @@ class EvaluationMetricInput(BaseModel):
 
 
 class EvaluationMetricResult(BaseModel):
+    model_config = ConfigDict(arbitrary_types_allowed=True)
+
     name: EvaluationMetricName
     value: str
     reason: str
+    is_positive: Optional[bool] = None
+    user_info: Optional[str] = None
+    error: Exception | None = None
+    fact_list: list[str] = Field(default_factory=list[str])
+
+
+class EvaluationAssessmentMessage(BaseModel):
+    status: ChatMessageAssessmentStatus
+    explanation: str
+    title: str
+    label: ChatMessageAssessmentLabel
+    type: ChatMessageAssessmentType

unique_toolkit/agentic/evaluation/tests/test_context_relevancy_service.py

@@ -0,0 +1,253 @@
+from unittest.mock import MagicMock, patch
+
+import pytest
+
+from unique_toolkit.agentic.evaluation.config import EvaluationMetricConfig
+from unique_toolkit.agentic.evaluation.context_relevancy.prompts import (
+    CONTEXT_RELEVANCY_METRIC_SYSTEM_MSG,
+)
+from unique_toolkit.agentic.evaluation.context_relevancy.schema import (
+    EvaluationSchemaStructuredOutput,
+)
+from unique_toolkit.agentic.evaluation.context_relevancy.service import (
+    ContextRelevancyEvaluator,
+)
+from unique_toolkit.agentic.evaluation.exception import EvaluatorException
+from unique_toolkit.agentic.evaluation.schemas import (
+    EvaluationMetricInput,
+    EvaluationMetricName,
+    EvaluationMetricResult,
+)
+from unique_toolkit.app.schemas import ChatEvent
+from unique_toolkit.chat.service import LanguageModelName
+from unique_toolkit.language_model.infos import (
+    LanguageModelInfo,
+)
+from unique_toolkit.language_model.schemas import (
+    LanguageModelAssistantMessage,
+    LanguageModelCompletionChoice,
+    LanguageModelMessages,
+)
+from unique_toolkit.language_model.service import LanguageModelResponse
+
+
+@pytest.fixture
+def event():
+    event = MagicMock(spec=ChatEvent)
+    event.payload = MagicMock()
+    event.payload.user_message = MagicMock()
+    event.payload.user_message.text = "Test query"
+    event.user_id = "user_0"
+    event.company_id = "company_0"
+    return event
+
+
+@pytest.fixture
+def evaluator(event):
+    return ContextRelevancyEvaluator(event)
+
+
+@pytest.fixture
+def basic_config():
+    return EvaluationMetricConfig(
+        enabled=True,
+        name=EvaluationMetricName.CONTEXT_RELEVANCY,
+        language_model=LanguageModelInfo.from_name(
+            LanguageModelName.AZURE_GPT_4o_2024_0806
+        ),
+    )
+
+
+@pytest.fixture
+def structured_config(basic_config):
+    model_info = LanguageModelInfo.from_name(LanguageModelName.AZURE_GPT_4o_2024_0806)
+    return EvaluationMetricConfig(
+        enabled=True,
+        name=EvaluationMetricName.CONTEXT_RELEVANCY,
+        language_model=model_info,
+    )
+
+
+@pytest.fixture
+def sample_input():
+    return EvaluationMetricInput(
+        input_text="test query",
+        context_texts=["test context 1", "test context 2"],
+    )
+
+
+@pytest.mark.asyncio
+async def test_analyze_disabled(evaluator, sample_input, basic_config):
+    basic_config.enabled = False
+    result = await evaluator.analyze(sample_input, basic_config)
+    assert result is None
+
+
+@pytest.mark.asyncio
+async def test_analyze_empty_context(evaluator, basic_config):
+    input_with_empty_context = EvaluationMetricInput(
+        input_text="test query", context_texts=[]
+    )
+
+    with pytest.raises(EvaluatorException) as exc_info:
+        await evaluator.analyze(input_with_empty_context, basic_config)
+
+    assert "No context texts provided." in str(exc_info.value)
+
+
+@pytest.mark.asyncio
+async def test_analyze_regular_output(evaluator, sample_input, basic_config):
+    mock_result = LanguageModelResponse(
+        choices=[
+            LanguageModelCompletionChoice(
+                index=0,
+                message=LanguageModelAssistantMessage(
+                    content="""{
+                        "value": "high",
+                        "reason": "Test reason"
+                    }"""
+                ),
+                finish_reason="stop",
+            )
+        ]
+    )
+
+    with patch.object(
+        evaluator.language_model_service,
+        "complete_async",
+        return_value=mock_result,
+    ) as mock_complete:
+        result = await evaluator.analyze(sample_input, basic_config)
+
+        assert isinstance(result, EvaluationMetricResult)
+        assert result.value.lower() == "high"
+        mock_complete.assert_called_once()
+
+
+@pytest.mark.asyncio
+async def test_analyze_structured_output(evaluator, sample_input, structured_config):
+    mock_result = LanguageModelResponse(
+        choices=[
+            LanguageModelCompletionChoice(
+                index=0,
+                message=LanguageModelAssistantMessage(
+                    content="HIGH",
+                    parsed={"value": "high", "reason": "Test reason"},
+                ),
+                finish_reason="stop",
+            )
+        ]
+    )
+
+    structured_output_schema = EvaluationSchemaStructuredOutput
+
+    with patch.object(
+        evaluator.language_model_service,
+        "complete_async",
+        return_value=mock_result,
+    ) as mock_complete:
+        result = await evaluator.analyze(
+            sample_input, structured_config, structured_output_schema
+        )
+        assert isinstance(result, EvaluationMetricResult)
+        assert result.value.lower() == "high"
+        mock_complete.assert_called_once()
+
+
+@pytest.mark.asyncio
+async def test_analyze_structured_output_validation_error(
+    evaluator, sample_input, structured_config
+):
+    mock_result = LanguageModelResponse(
+        choices=[
+            LanguageModelCompletionChoice(
+                index=0,
+                message=LanguageModelAssistantMessage(
+                    content="HIGH", parsed={"invalid": "data"}
+                ),
+                finish_reason="stop",
+            )
+        ]
+    )
+
+    structured_output_schema = EvaluationSchemaStructuredOutput
+
+    with patch.object(
+        evaluator.language_model_service,
+        "complete_async",
+        return_value=mock_result,
+    ):
+        with pytest.raises(EvaluatorException) as exc_info:
+            await evaluator.analyze(
+                sample_input, structured_config, structured_output_schema
+            )
+        assert "Error occurred during structured output validation" in str(
+            exc_info.value
+        )
+
+
+@pytest.mark.asyncio
+async def test_analyze_regular_output_empty_response(
+    evaluator, sample_input, basic_config
+):
+    mock_result = LanguageModelResponse(
+        choices=[
+            LanguageModelCompletionChoice(
+                index=0,
+                message=LanguageModelAssistantMessage(content=""),
+                finish_reason="stop",
+            )
+        ]
+    )
+
+    with patch.object(
+        evaluator.language_model_service,
+        "complete_async",
+        return_value=mock_result,
+    ):
+        with pytest.raises(EvaluatorException) as exc_info:
+            await evaluator.analyze(sample_input, basic_config)
+        assert "did not return a result" in str(exc_info.value)
+
+
+def test_compose_msgs_regular(evaluator, sample_input, basic_config):
+    messages = evaluator._compose_msgs(
+        sample_input, basic_config, enable_structured_output=False
+    )
+
+    assert isinstance(messages, LanguageModelMessages)
+    assert messages.root[0].content == CONTEXT_RELEVANCY_METRIC_SYSTEM_MSG
+    assert isinstance(messages.root[1].content, str)
+    assert "test query" in messages.root[1].content
+    assert "test context 1" in messages.root[1].content
+    assert "test context 2" in messages.root[1].content
+
+
+def test_compose_msgs_structured(evaluator, sample_input, structured_config):
+    messages = evaluator._compose_msgs(
+        sample_input, structured_config, enable_structured_output=True
+    )
+
+    assert isinstance(messages, LanguageModelMessages)
+    assert len(messages.root) == 2
+    assert (
+        messages.root[0].content != CONTEXT_RELEVANCY_METRIC_SYSTEM_MSG
+    )  # Should use structured output prompt
+    assert isinstance(messages.root[1].content, str)
+    assert "test query" in messages.root[1].content
+    assert "test context 1" in messages.root[1].content
+    assert "test context 2" in messages.root[1].content
+
+
+@pytest.mark.asyncio
+async def test_analyze_unknown_error(evaluator, sample_input, basic_config):
+    with patch.object(
+        evaluator.language_model_service,
+        "complete_async",
+        side_effect=Exception("Unknown error"),
+    ):
+        with pytest.raises(EvaluatorException) as exc_info:
+            await evaluator.analyze(sample_input, basic_config)
+        assert "Unknown error occurred during context relevancy metric analysis" in str(
+            exc_info.value
+        )

unique_toolkit/agentic/evaluation/tests/test_output_parser.py

@@ -0,0 +1,87 @@
+import pytest
+
+from unique_toolkit.agentic.evaluation.context_relevancy.schema import (
+    EvaluationSchemaStructuredOutput,
+    Fact,
+)
+from unique_toolkit.agentic.evaluation.exception import EvaluatorException
+from unique_toolkit.agentic.evaluation.output_parser import (
+    parse_eval_metric_result,
+    parse_eval_metric_result_structured_output,
+)
+from unique_toolkit.agentic.evaluation.schemas import (
+    EvaluationMetricName,
+    EvaluationMetricResult,
+)
+
+
+def test_parse_eval_metric_result_success():
+    # Test successful parsing with all fields
+    result = '{"value": "high", "reason": "Test reason"}'
+    parsed = parse_eval_metric_result(result, EvaluationMetricName.CONTEXT_RELEVANCY)
+
+    assert isinstance(parsed, EvaluationMetricResult)
+    assert parsed.name == EvaluationMetricName.CONTEXT_RELEVANCY
+    assert parsed.value == "high"
+    assert parsed.reason == "Test reason"
+    assert parsed.fact_list == []
+
+
+def test_parse_eval_metric_result_missing_fields():
+    # Test parsing with missing fields (should use default "None")
+    result = '{"value": "high"}'
+    parsed = parse_eval_metric_result(result, EvaluationMetricName.CONTEXT_RELEVANCY)
+
+    assert isinstance(parsed, EvaluationMetricResult)
+    assert parsed.name == EvaluationMetricName.CONTEXT_RELEVANCY
+    assert parsed.value == "high"
+    assert parsed.reason == "None"
+    assert parsed.fact_list == []
+
+
+def test_parse_eval_metric_result_invalid_json():
+    # Test parsing with invalid JSON
+    result = "invalid json"
+    with pytest.raises(EvaluatorException) as exc_info:
+        parse_eval_metric_result(result, EvaluationMetricName.CONTEXT_RELEVANCY)
+
+    assert "Error occurred during parsing the evaluation metric result" in str(
+        exc_info.value
+    )
+
+
+def test_parse_eval_metric_result_structured_output_basic():
+    # Test basic structured output without fact list
+    result = EvaluationSchemaStructuredOutput(value="high", reason="Test reason")
+    parsed = parse_eval_metric_result_structured_output(
+        result, EvaluationMetricName.CONTEXT_RELEVANCY
+    )
+
+    assert isinstance(parsed, EvaluationMetricResult)
+    assert parsed.name == EvaluationMetricName.CONTEXT_RELEVANCY
+    assert parsed.value == "high"
+    assert parsed.reason == "Test reason"
+    assert parsed.fact_list == []
+
+
+def test_parse_eval_metric_result_structured_output_with_facts():
+    # Test structured output with fact list
+    result = EvaluationSchemaStructuredOutput(
+        value="high",
+        reason="Test reason",
+        fact_list=[
+            Fact(fact="Fact 1"),
+            Fact(fact="Fact 2"),
+        ],
+    )
+    parsed = parse_eval_metric_result_structured_output(
+        result, EvaluationMetricName.CONTEXT_RELEVANCY
+    )
+
+    assert isinstance(parsed, EvaluationMetricResult)
+    assert parsed.name == EvaluationMetricName.CONTEXT_RELEVANCY
+    assert parsed.value == "high"
+    assert parsed.reason == "Test reason"
+    assert parsed.fact_list == ["Fact 1", "Fact 2"]
+    assert isinstance(parsed.fact_list, list)
+    assert len(parsed.fact_list) == 2  # None fact should be filtered out