unique_toolkit 1.45.5__py3-none-any.whl → 1.45.6__py3-none-any.whl

unique_toolkit/agentic/evaluation/tests/test_output_parser.py

@@ -1,3 +1,5 @@
+"""Tests for evaluation metric output parsers."""
+
 import pytest
 
 from unique_toolkit.agentic.evaluation.context_relevancy.schema import (
@@ -15,11 +17,21 @@ from unique_toolkit.agentic.evaluation.schemas import (
 )
 
 
-def test_parse_eval_metric_result_success():
-    # Test successful parsing with all fields
-    result = '{"value": "high", "reason": "Test reason"}'
-    parsed = parse_eval_metric_result(result, EvaluationMetricName.CONTEXT_RELEVANCY)
+@pytest.mark.ai
+def test_parse_eval_metric_result__succeeds__with_all_fields() -> None:
+    """
+    Purpose: Verify parsing of complete evaluation metric JSON result with all fields.
+    Why this matters: Core parsing functionality for evaluation results from LLM.
+    Setup summary: Provide valid JSON with all fields, assert correct parsing and field values.
+    """
+    # Arrange
+    result_json: str = '{"value": "high", "reason": "Test reason"}'
+    metric_name: EvaluationMetricName = EvaluationMetricName.CONTEXT_RELEVANCY
+
+    # Act
+    parsed: EvaluationMetricResult = parse_eval_metric_result(result_json, metric_name)
 
+    # Assert
     assert isinstance(parsed, EvaluationMetricResult)
     assert parsed.name == EvaluationMetricName.CONTEXT_RELEVANCY
     assert parsed.value == "high"
@@ -27,11 +39,21 @@ def test_parse_eval_metric_result_success():
     assert parsed.fact_list == []
 
 
-def test_parse_eval_metric_result_missing_fields():
-    # Test parsing with missing fields (should use default "None")
-    result = '{"value": "high"}'
-    parsed = parse_eval_metric_result(result, EvaluationMetricName.CONTEXT_RELEVANCY)
+@pytest.mark.ai
+def test_parse_eval_metric_result__uses_default_reason__with_missing_field() -> None:
+    """
+    Purpose: Verify parsing handles missing optional fields by using defaults.
+    Why this matters: Ensures robustness when LLM returns incomplete JSON responses.
+    Setup summary: Provide JSON with only required field, assert default value for reason.
+    """
+    # Arrange
+    result_json: str = '{"value": "high"}'
+    metric_name: EvaluationMetricName = EvaluationMetricName.CONTEXT_RELEVANCY
+
+    # Act
+    parsed: EvaluationMetricResult = parse_eval_metric_result(result_json, metric_name)
 
+    # Assert
     assert isinstance(parsed, EvaluationMetricResult)
     assert parsed.name == EvaluationMetricName.CONTEXT_RELEVANCY
     assert parsed.value == "high"
@@ -39,24 +61,49 @@ def test_parse_eval_metric_result_missing_fields():
     assert parsed.fact_list == []
 
 
-def test_parse_eval_metric_result_invalid_json():
-    # Test parsing with invalid JSON
-    result = "invalid json"
+@pytest.mark.ai
+def test_parse_eval_metric_result__raises_evaluator_exception__with_invalid_json() -> (
+    None
+):
+    """
+    Purpose: Verify parser raises appropriate exception for malformed JSON.
+    Why this matters: Provides clear error handling for invalid LLM responses.
+    Setup summary: Provide invalid JSON string, assert EvaluatorException with descriptive message.
+    """
+    # Arrange
+    result_json: str = "invalid json"
+    metric_name: EvaluationMetricName = EvaluationMetricName.CONTEXT_RELEVANCY
+
+    # Act & Assert
     with pytest.raises(EvaluatorException) as exc_info:
-        parse_eval_metric_result(result, EvaluationMetricName.CONTEXT_RELEVANCY)
+        parse_eval_metric_result(result_json, metric_name)
 
     assert "Error occurred during parsing the evaluation metric result" in str(
         exc_info.value
     )
 
 
-def test_parse_eval_metric_result_structured_output_basic():
-    # Test basic structured output without fact list
-    result = EvaluationSchemaStructuredOutput(value="high", reason="Test reason")
-    parsed = parse_eval_metric_result_structured_output(
-        result, EvaluationMetricName.CONTEXT_RELEVANCY
+@pytest.mark.ai
+def test_parse_eval_metric_result_structured_output__succeeds__without_fact_list() -> (
+    None
+):
+    """
+    Purpose: Verify parsing of structured output without optional fact list.
+    Why this matters: Ensures structured output parsing works for basic evaluations.
+    Setup summary: Create structured output object without facts, assert correct parsing.
+    """
+    # Arrange
+    result: EvaluationSchemaStructuredOutput = EvaluationSchemaStructuredOutput(
+        value="high", reason="Test reason"
     )
+    metric_name: EvaluationMetricName = EvaluationMetricName.CONTEXT_RELEVANCY
 
+    # Act
+    parsed: EvaluationMetricResult = parse_eval_metric_result_structured_output(
+        result, metric_name
+    )
+
+    # Assert
     assert isinstance(parsed, EvaluationMetricResult)
     assert parsed.name == EvaluationMetricName.CONTEXT_RELEVANCY
     assert parsed.value == "high"
@@ -64,9 +111,17 @@ def test_parse_eval_metric_result_structured_output_basic():
     assert parsed.fact_list == []
 
 
-def test_parse_eval_metric_result_structured_output_with_facts():
-    # Test structured output with fact list
-    result = EvaluationSchemaStructuredOutput(
+@pytest.mark.ai
+def test_parse_eval_metric_result_structured_output__includes_facts__with_fact_list() -> (
+    None
+):
+    """
+    Purpose: Verify parsing of structured output with fact list extracts all facts.
+    Why this matters: Fact extraction is critical for detailed evaluation feedback.
+    Setup summary: Create structured output with multiple facts, assert all facts extracted.
+    """
+    # Arrange
+    result: EvaluationSchemaStructuredOutput = EvaluationSchemaStructuredOutput(
         value="high",
         reason="Test reason",
         fact_list=[
@@ -74,14 +129,18 @@ def test_parse_eval_metric_result_structured_output_with_facts():
             Fact(fact="Fact 2"),
         ],
     )
-    parsed = parse_eval_metric_result_structured_output(
-        result, EvaluationMetricName.CONTEXT_RELEVANCY
+    metric_name: EvaluationMetricName = EvaluationMetricName.CONTEXT_RELEVANCY
+
+    # Act
+    parsed: EvaluationMetricResult = parse_eval_metric_result_structured_output(
+        result, metric_name
     )
 
+    # Assert
     assert isinstance(parsed, EvaluationMetricResult)
     assert parsed.name == EvaluationMetricName.CONTEXT_RELEVANCY
     assert parsed.value == "high"
     assert parsed.reason == "Test reason"
     assert parsed.fact_list == ["Fact 1", "Fact 2"]
     assert isinstance(parsed.fact_list, list)
-    assert len(parsed.fact_list) == 2  # None fact should be filtered out
+    assert len(parsed.fact_list) == 2

unique_toolkit/agentic/evaluation/tests/test_prompt_loaders.py

@@ -0,0 +1,348 @@
+"""Tests for prompt loader functions."""
+
+import pytest
+
+from unique_toolkit.agentic.evaluation.context_relevancy.prompts import (
+    system_prompt_loader as context_system_prompt_loader,
+)
+from unique_toolkit.agentic.evaluation.context_relevancy.prompts import (
+    user_prompt_loader as context_user_prompt_loader,
+)
+from unique_toolkit.agentic.evaluation.hallucination.prompts import (
+    system_prompt_loader as hallucination_system_prompt_loader,
+)
+from unique_toolkit.agentic.evaluation.hallucination.prompts import (
+    user_prompt_loader as hallucination_user_prompt_loader,
+)
+
+
+@pytest.mark.ai
+def test_context_system_prompt_loader__returns_non_empty_string__on_call() -> None:
+    """
+    Purpose: Verify that the context relevancy system prompt loader returns a valid string.
+    Why this matters: The system prompt is critical for guiding the evaluation LLM's behavior.
+    Setup summary: Call the loader function and assert type and non-empty content.
+    """
+    # Arrange - No setup needed for this test
+
+    # Act
+    prompt: str = context_system_prompt_loader()
+
+    # Assert
+    assert isinstance(prompt, str)
+    assert len(prompt) > 0
+
+
+@pytest.mark.ai
+def test_context_user_prompt_loader__returns_non_empty_string__on_call() -> None:
+    """
+    Purpose: Verify that the context relevancy user prompt loader returns a valid string.
+    Why this matters: The user prompt template must be valid for evaluation requests.
+    Setup summary: Call the loader function and assert type and non-empty content.
+    """
+    # Arrange - No setup needed for this test
+
+    # Act
+    prompt: str = context_user_prompt_loader()
+
+    # Assert
+    assert isinstance(prompt, str)
+    assert len(prompt) > 0
+
+
+@pytest.mark.ai
+def test_context_system_prompt__contains_jinja_syntax__for_structured_output() -> None:
+    """
+    Purpose: Verify that system prompt contains Jinja2 template syntax for structured output control.
+    Why this matters: Template must support conditional rendering based on structured_output flag.
+    Setup summary: Load system prompt and check for Jinja2 conditional blocks.
+    """
+    # Arrange - No setup needed
+
+    # Act
+    prompt: str = context_system_prompt_loader()
+
+    # Assert
+    assert "{% if structured_output %}" in prompt or "{%" in prompt
+
+
+@pytest.mark.ai
+def test_context_user_prompt__contains_jinja_variables__for_input_and_context() -> None:
+    """
+    Purpose: Verify that user prompt contains required Jinja2 variable placeholders.
+    Why this matters: Template must support dynamic insertion of input text and context texts.
+    Setup summary: Load user prompt and check for expected variable placeholders.
+    """
+    # Arrange - No setup needed
+
+    # Act
+    prompt: str = context_user_prompt_loader()
+
+    # Assert
+    assert "{{ input_text }}" in prompt
+    assert "{{ context_texts }}" in prompt
+
+
+@pytest.mark.ai
+def test_context_system_prompt__has_both_structured_and_regular_modes__in_template() -> (
+    None
+):
+    """
+    Purpose: Verify that system prompt template supports both structured and regular output modes.
+    Why this matters: Template must handle both evaluation output formats correctly.
+    Setup summary: Load system prompt and check for conditional blocks for both modes.
+    """
+    # Arrange - No setup needed
+
+    # Act
+    prompt: str = context_system_prompt_loader()
+
+    # Assert
+    assert "{% if structured_output %}" in prompt
+    assert "{% else %}" in prompt or "{% endif %}" in prompt
+
+
+@pytest.mark.ai
+def test_context_user_prompt__has_conditional_json_instruction__for_unstructured_mode() -> (
+    None
+):
+    """
+    Purpose: Verify that user prompt has conditional JSON instruction for unstructured mode.
+    Why this matters: Non-structured mode requires explicit JSON formatting instructions.
+    Setup summary: Load user prompt and check for conditional JSON instruction block.
+    """
+    # Arrange - No setup needed
+
+    # Act
+    prompt: str = context_user_prompt_loader()
+
+    # Assert
+    assert "{% if not structured_output %}" in prompt or "{%" in prompt
+
+
+@pytest.mark.ai
+def test_context_prompts__are_consistent_between_calls__for_determinism() -> None:
+    """
+    Purpose: Verify that prompt loaders return consistent content across multiple invocations.
+    Why this matters: Ensures deterministic behavior and no hidden state in loaders.
+    Setup summary: Call loaders twice and compare results for equality.
+    """
+    # Arrange - No setup needed
+
+    # Act
+    system_prompt_1: str = context_system_prompt_loader()
+    system_prompt_2: str = context_system_prompt_loader()
+    user_prompt_1: str = context_user_prompt_loader()
+    user_prompt_2: str = context_user_prompt_loader()
+
+    # Assert
+    assert system_prompt_1 == system_prompt_2
+    assert user_prompt_1 == user_prompt_2
+
+
+@pytest.mark.ai
+def test_hallucination_system_prompt_loader__returns_non_empty_string__on_call() -> (
+    None
+):
+    """
+    Purpose: Verify that the hallucination system prompt loader returns a valid string.
+    Why this matters: The system prompt is critical for hallucination detection behavior.
+    Setup summary: Call the loader function and assert type and non-empty content.
+    """
+    # Arrange - No setup needed
+
+    # Act
+    prompt: str = hallucination_system_prompt_loader()
+
+    # Assert
+    assert isinstance(prompt, str)
+    assert len(prompt) > 0
+
+
+@pytest.mark.ai
+def test_hallucination_user_prompt_loader__returns_non_empty_string__on_call() -> None:
+    """
+    Purpose: Verify that the hallucination user prompt loader returns a valid string.
+    Why this matters: The user prompt template must be valid for hallucination evaluation.
+    Setup summary: Call the loader function and assert type and non-empty content.
+    """
+    # Arrange - No setup needed
+
+    # Act
+    prompt: str = hallucination_user_prompt_loader()
+
+    # Assert
+    assert isinstance(prompt, str)
+    assert len(prompt) > 0
+
+
+@pytest.mark.ai
+def test_hallucination_system_prompt__contains_jinja_syntax__for_has_context() -> None:
+    """
+    Purpose: Verify that system prompt contains Jinja2 template syntax for context handling.
+    Why this matters: Template must support conditional rendering based on has_context flag.
+    Setup summary: Load system prompt and check for Jinja2 conditional blocks.
+    """
+    # Arrange - No setup needed
+
+    # Act
+    prompt: str = hallucination_system_prompt_loader()
+
+    # Assert
+    assert "{% if has_context %}" in prompt or "{%" in prompt
+
+
+@pytest.mark.ai
+def test_hallucination_user_prompt__contains_jinja_variables__for_input_and_output() -> (
+    None
+):
+    """
+    Purpose: Verify that user prompt contains required Jinja2 variable placeholders.
+    Why this matters: Template must support dynamic insertion of input and output texts.
+    Setup summary: Load user prompt and check for expected variable placeholders.
+    """
+    # Arrange - No setup needed
+
+    # Act
+    prompt: str = hallucination_user_prompt_loader()
+
+    # Assert
+    assert "{{ input_text }}" in prompt
+    assert "{{ output_text }}" in prompt
+
+
+@pytest.mark.ai
+def test_hallucination_system_prompt__has_context_conditional__in_template() -> None:
+    """
+    Purpose: Verify that system prompt template has conditional logic for has_context.
+    Why this matters: Template must handle both context and non-context evaluation scenarios.
+    Setup summary: Load system prompt and check for conditional blocks with else/endif.
+    """
+    # Arrange - No setup needed
+
+    # Act
+    prompt: str = hallucination_system_prompt_loader()
+
+    # Assert
+    assert "{% if has_context %}" in prompt
+    assert "{% else %}" in prompt or "{% endif %}" in prompt
+
+
+@pytest.mark.ai
+def test_hallucination_user_prompt__has_optional_context_fields__in_template() -> None:
+    """
+    Purpose: Verify that user prompt has conditional blocks for optional context fields.
+    Why this matters: Template must support optional contexts_text and history_messages_text.
+    Setup summary: Load user prompt and check for conditional blocks or variable placeholders.
+    """
+    # Arrange - No setup needed
+
+    # Act
+    prompt: str = hallucination_user_prompt_loader()
+
+    # Assert
+    assert "{% if contexts_text %}" in prompt or "{{ contexts_text }}" in prompt
+    assert (
+        "{% if history_messages_text %}" in prompt
+        or "{{ history_messages_text }}" in prompt
+    )
+
+
+@pytest.mark.ai
+def test_hallucination_system_prompt__mentions_hallucination_concepts__in_content() -> (
+    None
+):
+    """
+    Purpose: Verify that system prompt mentions hallucination-related concepts.
+    Why this matters: Ensures prompt properly guides model to detect hallucinations.
+    Setup summary: Load system prompt and check for hallucination-related keywords.
+    """
+    # Arrange - No setup needed
+
+    # Act
+    prompt: str = hallucination_system_prompt_loader()
+    prompt_lower: str = prompt.lower()
+
+    # Assert
+    assert (
+        "hallucination" in prompt_lower
+        or "grounded" in prompt_lower
+        or "supported" in prompt_lower
+    )
+
+
+@pytest.mark.ai
+def test_hallucination_user_prompt__contains_data_sections__for_input_and_output() -> (
+    None
+):
+    """
+    Purpose: Verify that user prompt has sections for input and output data.
+    Why this matters: Template must clearly separate input and output for evaluation.
+    Setup summary: Load user prompt and check for input/output section markers.
+    """
+    # Arrange - No setup needed
+
+    # Act
+    prompt: str = hallucination_user_prompt_loader()
+
+    # Assert
+    assert "Input:" in prompt or "input" in prompt.lower()
+    assert "Output:" in prompt or "output" in prompt.lower()
+
+
+@pytest.mark.ai
+def test_hallucination_prompts__are_consistent_between_calls__for_determinism() -> None:
+    """
+    Purpose: Verify that hallucination prompt loaders return consistent content.
+    Why this matters: Ensures deterministic behavior and no hidden state in loaders.
+    Setup summary: Call loaders twice and compare results for equality.
+    """
+    # Arrange - No setup needed
+
+    # Act
+    system_prompt_1: str = hallucination_system_prompt_loader()
+    system_prompt_2: str = hallucination_system_prompt_loader()
+    user_prompt_1: str = hallucination_user_prompt_loader()
+    user_prompt_2: str = hallucination_user_prompt_loader()
+
+    # Assert
+    assert system_prompt_1 == system_prompt_2
+    assert user_prompt_1 == user_prompt_2
+
+
+@pytest.mark.ai
+def test_context_relevancy_loaders__can_access_template_files__without_errors() -> None:
+    """
+    Purpose: Verify that context relevancy loaders can successfully access template files.
+    Why this matters: Ensures template files exist and are readable at runtime.
+    Setup summary: Call loaders and assert no FileNotFoundError is raised.
+    """
+    # Arrange - No setup needed
+
+    # Act & Assert
+    try:
+        system_prompt: str = context_system_prompt_loader()
+        user_prompt: str = context_user_prompt_loader()
+        assert system_prompt is not None
+        assert user_prompt is not None
+    except FileNotFoundError as e:
+        pytest.fail(f"Prompt loader failed to access template file: {e}")
+
+
+@pytest.mark.ai
+def test_hallucination_loaders__can_access_template_files__without_errors() -> None:
+    """
+    Purpose: Verify that hallucination loaders can successfully access template files.
+    Why this matters: Ensures template files exist and are readable at runtime.
+    Setup summary: Call loaders and assert no FileNotFoundError is raised.
+    """
+    # Arrange - No setup needed
+
+    # Act & Assert
+    try:
+        system_prompt: str = hallucination_system_prompt_loader()
+        user_prompt: str = hallucination_user_prompt_loader()
+        assert system_prompt is not None
+        assert user_prompt is not None
+    except FileNotFoundError as e:
+        pytest.fail(f"Prompt loader failed to access template file: {e}")

unique_toolkit/agentic/evaluation/utils.py

@@ -0,0 +1,8 @@
+from pathlib import Path
+
+
+def load_template(parent_dir: Path, filename: str) -> str:
+    """Load a Jinja2 template file from the hallucination directory."""
+    template_path = parent_dir / filename
+    with open(template_path, "r") as f:
+        return f.read()

{unique_toolkit-1.45.5.dist-info → unique_toolkit-1.45.6.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: unique_toolkit
-Version: 1.45.5
+Version: 1.45.6
 Summary: 
 License: Proprietary
 Author: Cedric Klinkert
@@ -125,6 +125,9 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), 
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## [1.45.6] - 2026-01-30
+- hallucination evaluator: Use original response to retrieve referenced chunk 
+
 ## [1.45.5] - 2026-01-29
 - Add HTML rendering support for code interpreter generated files
 

{unique_toolkit-1.45.5.dist-info → unique_toolkit-1.45.6.dist-info}/RECORD

@@ -69,21 +69,31 @@ unique_toolkit/_common/validators.py,sha256=ElnkMsyEY24TfzfTVHvireyT39EnZgW5N40T
 unique_toolkit/agentic/__init__.py,sha256=AbpHGcgLb-kRsJGnwFEktk7uzpZOCcBY74-YBdrKVGs,1
 unique_toolkit/agentic/debug_info_manager/debug_info_manager.py,sha256=30ZZaw0vffjZjiu9AYdO1Sm8G9FN6XR2ehdOEUCKqh0,891
 unique_toolkit/agentic/debug_info_manager/test/test_debug_info_manager.py,sha256=_fIS6_DHA8A3AB64-LPgHgUGa1w0CFUWwtgV-ZbhkzA,10535
-unique_toolkit/agentic/evaluation/config.py,sha256=Xbti9Tx7vgyUySed9Ovwx0uKL_tFWakMGll5MfOkvBM,905
-unique_toolkit/agentic/evaluation/context_relevancy/prompts.py,sha256=EdHFUOB581yVxcOL8482KUv_LzaRjuiem71EF8udYMc,1331
+unique_toolkit/agentic/evaluation/config.py,sha256=u5-iuT-4mfA2_9UZjZ-TN3YutVf35aR4EsF9l61Odnk,1696
+unique_toolkit/agentic/evaluation/context_relevancy/prompts/__init__.py,sha256=IKGvHW2viBylTUjEJNKvIXO4YrYwdJXBHSxA9rKtiRI,355
+unique_toolkit/agentic/evaluation/context_relevancy/prompts/system_prompt.j2,sha256=PFT9Y7QQ85h-ronwK8sq6zJJrxfrZhvZkYHXspMdDHU,985
+unique_toolkit/agentic/evaluation/context_relevancy/prompts/user_prompt.j2,sha256=9HXm0Qg0xAPIE5mnaUiHrscolD_qO7f83qlIjXiP1n4,150
 unique_toolkit/agentic/evaluation/context_relevancy/schema.py,sha256=lZd0TPzH43ifgWWGg3WO6b1AQX8aK2R9y51yH0d1DHM,2919
-unique_toolkit/agentic/evaluation/context_relevancy/service.py,sha256=2NM1_PCP6fXsRm0r-MGrUg5Z3WO00FBCqmiU8f5Kagg,9661
+unique_toolkit/agentic/evaluation/context_relevancy/service.py,sha256=dsgpfKRSg9B4kjLhHJD_Kath4GVhHE-ZOVAGRkiCz20,8729
 unique_toolkit/agentic/evaluation/evaluation_manager.py,sha256=wDN_Uuut9kEGek8JY3QeInKpF-ukbvOSKOVd7DHFT3Q,8121
 unique_toolkit/agentic/evaluation/exception.py,sha256=7lcVbCyoN4Md1chNJDFxpUYyWbVrcr9dcc3TxWykJTc,115
-unique_toolkit/agentic/evaluation/hallucination/constants.py,sha256=XyRTAlL9nVWuHXTe6MP97fOkAtnbEhtw7KCl08QJydU,2150
+unique_toolkit/agentic/evaluation/hallucination/constants.py,sha256=-PnZ3N9VpwgbIe6hcUye40nvJa-JIRuTidCZAQwZ3GA,2473
 unique_toolkit/agentic/evaluation/hallucination/hallucination_evaluation.py,sha256=x5ta2Fum4fE5ySgIXPKlnbTtmV140z0IazSATd0-REg,4092
-unique_toolkit/agentic/evaluation/hallucination/prompts.py,sha256=O3Hi_rOzZlujvnO2wn2jhoPmrYLjzVtRWwxn5Q81m9Y,3405
+unique_toolkit/agentic/evaluation/hallucination/prompts/__init__.py,sha256=4KFYMZsB3fJUKzoiUJE1npZ0gueWgvceB32EUrN-v7A,343
+unique_toolkit/agentic/evaluation/hallucination/prompts/system_prompt.j2,sha256=sDUX6G645Ba40D_qKu4cUI8g-sJOfG8JpZreTNFgf7M,2616
+unique_toolkit/agentic/evaluation/hallucination/prompts/user_prompt.j2,sha256=mD_qE9fOkyc1XXrebFt097ddx8bTlA6lbY04hKSQmWs,273
 unique_toolkit/agentic/evaluation/hallucination/service.py,sha256=WJF1f45uHnYLx1S4TW31bSFobFpV-YlOS3G_zMhuBVU,2512
-unique_toolkit/agentic/evaluation/hallucination/utils.py,sha256=uHKTJw4kJyq0_Gi-EOhbocBAij4_Vzn3dW1wTxAuFg4,9706
+unique_toolkit/agentic/evaluation/hallucination/utils.py,sha256=fxT7H1PQ6xANNvtViuhhR_9ac5ggDmFx-YfjcKUZRcg,12013
 unique_toolkit/agentic/evaluation/output_parser.py,sha256=0FDo8YY_Dc4qlTNeYyQkznzIFj9aX9wMrLOTbhhTl6g,1418
 unique_toolkit/agentic/evaluation/schemas.py,sha256=m9JMCUmeqP8KhsJOVEzsz6dRXUe1uKw-bxRDtn5qwvM,3156
-unique_toolkit/agentic/evaluation/tests/test_context_relevancy_service.py,sha256=4tDxHTApbaTMxN1sNS8WCqj2BweRk6YqZ5_zHP45jto,7977
-unique_toolkit/agentic/evaluation/tests/test_output_parser.py,sha256=RN_HcBbU6qy_e_PoYyUFcjWnp3ymJ6-gLj6TgEOupAI,3107
+unique_toolkit/agentic/evaluation/tests/fixtures.py,sha256=Q-ughTfDiAdsMKbBVGzFiBucFdAx-FXgJ9iqp5xMyPs,2801
+unique_toolkit/agentic/evaluation/tests/test_config.py,sha256=p7xFQ7KE_yU8jGpqYA7ntAYe5Vln33wd6nwv3FM9XfI,8327
+unique_toolkit/agentic/evaluation/tests/test_context_relevancy_service.py,sha256=NcSOyBJ_lqYehtlraZPo9RLutCitTP76kvkuyogSD2A,9477
+unique_toolkit/agentic/evaluation/tests/test_hallucination_constants.py,sha256=jT61WxKic-jDUJT1BeVjzhck02EnaMi1ng2H82-Aq_Q,19348
+unique_toolkit/agentic/evaluation/tests/test_hallucination_utils.py,sha256=PKyGR073HxT0J_g8626kCURbMSlrMgkg-xPP7dPHD-0,31838
+unique_toolkit/agentic/evaluation/tests/test_output_parser.py,sha256=KfltytmvqnPWLhmZpBXqcRmnlYorw_USwM5rkLVv8so,5179
+unique_toolkit/agentic/evaluation/tests/test_prompt_loaders.py,sha256=zBREdlKf5tdDyB8XSaNgpQv3-tuZJoYteeJrp6WMWDM,11897
+unique_toolkit/agentic/evaluation/utils.py,sha256=HmyPaDV8wdW-_gOjjW-wDaMKgdrsP5-SHP7OqTmGI_A,264
 unique_toolkit/agentic/feature_flags/__init__.py,sha256=LhE2cHoa9AYBOR7TjiIToOn46sttm9paKcrzE7gnDPM,149
 unique_toolkit/agentic/feature_flags/feature_flags.py,sha256=4jPH0GGGt5-tQ6PJWNpMBIlYzNrQIIqBLx8W02lwxD0,1140
 unique_toolkit/agentic/history_manager/history_construction_with_contents.py,sha256=TwamOOnYTYZMQdY1mAzj6_MZOe3T5RsjFDarT1tCtYo,8150
@@ -244,7 +254,7 @@ unique_toolkit/short_term_memory/service.py,sha256=5PeVBu1ZCAfyDb2HLVvlmqSbyzBBu
 unique_toolkit/smart_rules/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 unique_toolkit/smart_rules/compile.py,sha256=Ozhh70qCn2yOzRWr9d8WmJeTo7AQurwd3tStgBMPFLA,1246
 unique_toolkit/test_utilities/events.py,sha256=_mwV2bs5iLjxS1ynDCjaIq-gjjKhXYCK-iy3dRfvO3g,6410
-unique_toolkit-1.45.5.dist-info/LICENSE,sha256=GlN8wHNdh53xwOPg44URnwag6TEolCjoq3YD_KrWgss,193
-unique_toolkit-1.45.5.dist-info/METADATA,sha256=qOQh95f58XhyiINOIVryMeJiaLEIXX_it-uWUgalJt8,49138
-unique_toolkit-1.45.5.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
-unique_toolkit-1.45.5.dist-info/RECORD,,
+unique_toolkit-1.45.6.dist-info/LICENSE,sha256=GlN8wHNdh53xwOPg44URnwag6TEolCjoq3YD_KrWgss,193
+unique_toolkit-1.45.6.dist-info/METADATA,sha256=Ojrf6ABO88IO5h892iywEweJuc9p67MWzHMXa--40gE,49243
+unique_toolkit-1.45.6.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
+unique_toolkit-1.45.6.dist-info/RECORD,,

unique_toolkit/agentic/evaluation/hallucination/prompts.py

@@ -1,79 +0,0 @@
-HALLUCINATION_METRIC_SYSTEM_MSG = """
-You will receive a question, references, a conversation between a user and an agent, and an output. 
-The output is the answer to the question. 
-Your task is to evaluate if the output is fully supported by the information provided in the references and conversation, and provide explanations on your judgement in 2 sentences.
-
-Use the following entailment scale to generate a score:
-[low] - All information in output is supported by the references/conversation, or extractions from the references/conversation.
-[medium] - The output is supported by the references/conversation to some extent, but there is at least some information in the output that is not discussed in the references/conversation. For example, if an instruction asks about two concepts and the references/conversation only discusses either of them, it should be considered a [medium] hallucination level.
-[high] - The output contains information that is not part of the references/conversation, is unrelated to the references/conversation, or contradicts the references/conversation.
-
-Make sure to not use any external information/knowledge to judge whether the output is true or not. Only check whether the output is supported by the references/conversation, and not whether the output is correct or not. Also do not evaluate if the references/conversation contain further information that is not part of the output but could be relevant to the question. If the output mentions a plot or chart, ignore this information in your evaluation.
-
-Your answer must be in JSON format:
-{
- "reason": Your explanation of your judgement of the evaluation,
- "value": decision, must be one of the following: ["high", "medium", "low"]
-}                                                  
-"""
-
-HALLUCINATION_METRIC_USER_MSG = """
-Here is the data:
-
-Input:
-'''
-$input_text
-'''
-
-References:
-'''
-$contexts_text
-'''
-
-Conversation:
-'''
-$history_messages_text
-'''
-
-Output:
-'''
-$output_text
-'''
-
-Answer as JSON:
-"""
-
-HALLUCINATION_METRIC_SYSTEM_MSG_DEFAULT = """
-You will receive a question and an output. 
-The output is the answer to the question. 
-The situation is that no references could be found to answer the question. Your task is to evaluate if the output contains any information to answer the question,
-and provide a short explanations of your reasoning in 2 sentences. Also mention in your explanation that no references were provided to answer the question.
-
-Use the following entailment scale to generate a score:
-[low] - The output does not contain any information to answer the question.
-[medium] - The output contains some information to answer the question, but does not answer the question entirely. 
-[high] - The output answers the question.
-
-It is not considered an answer when the output relates to the questions subject. Make sure to not use any external information/knowledge to judge whether the output is true or not. Only check that the output does not answer the question, and not whether the output is correct or not.
-Your answer must be in JSON format:
-{
- "reason": Your explanation of your reasoning of the evaluation,
- "value": decision, must be one of the following: ["low", "medium", "high"]
-}
-"""
-
-HALLUCINATION_METRIC_USER_MSG_DEFAULT = """                                                  
-Here is the data:
-
-Input:
-'''
-$input_text
-'''
-
-Output:
-'''
-$output_text
-'''
-
-Answer as JSON:
-"""