unique_toolkit 1.45.5__py3-none-any.whl → 1.45.7__py3-none-any.whl

unique_toolkit/agentic/evaluation/tests/test_output_parser.py

@@ -1,3 +1,5 @@
+"""Tests for evaluation metric output parsers."""
+
 import pytest
 
 from unique_toolkit.agentic.evaluation.context_relevancy.schema import (
@@ -15,11 +17,21 @@ from unique_toolkit.agentic.evaluation.schemas import (
 )
 
 
-def test_parse_eval_metric_result_success():
-    # Test successful parsing with all fields
-    result = '{"value": "high", "reason": "Test reason"}'
-    parsed = parse_eval_metric_result(result, EvaluationMetricName.CONTEXT_RELEVANCY)
+@pytest.mark.ai
+def test_parse_eval_metric_result__succeeds__with_all_fields() -> None:
+    """
+    Purpose: Verify parsing of complete evaluation metric JSON result with all fields.
+    Why this matters: Core parsing functionality for evaluation results from LLM.
+    Setup summary: Provide valid JSON with all fields, assert correct parsing and field values.
+    """
+    # Arrange
+    result_json: str = '{"value": "high", "reason": "Test reason"}'
+    metric_name: EvaluationMetricName = EvaluationMetricName.CONTEXT_RELEVANCY
+
+    # Act
+    parsed: EvaluationMetricResult = parse_eval_metric_result(result_json, metric_name)
 
+    # Assert
     assert isinstance(parsed, EvaluationMetricResult)
     assert parsed.name == EvaluationMetricName.CONTEXT_RELEVANCY
     assert parsed.value == "high"
@@ -27,11 +39,21 @@ def test_parse_eval_metric_result_success():
     assert parsed.fact_list == []
 
 
-def test_parse_eval_metric_result_missing_fields():
-    # Test parsing with missing fields (should use default "None")
-    result = '{"value": "high"}'
-    parsed = parse_eval_metric_result(result, EvaluationMetricName.CONTEXT_RELEVANCY)
+@pytest.mark.ai
+def test_parse_eval_metric_result__uses_default_reason__with_missing_field() -> None:
+    """
+    Purpose: Verify parsing handles missing optional fields by using defaults.
+    Why this matters: Ensures robustness when LLM returns incomplete JSON responses.
+    Setup summary: Provide JSON with only required field, assert default value for reason.
+    """
+    # Arrange
+    result_json: str = '{"value": "high"}'
+    metric_name: EvaluationMetricName = EvaluationMetricName.CONTEXT_RELEVANCY
+
+    # Act
+    parsed: EvaluationMetricResult = parse_eval_metric_result(result_json, metric_name)
 
+    # Assert
     assert isinstance(parsed, EvaluationMetricResult)
     assert parsed.name == EvaluationMetricName.CONTEXT_RELEVANCY
     assert parsed.value == "high"
@@ -39,24 +61,49 @@ def test_parse_eval_metric_result_missing_fields():
     assert parsed.fact_list == []
 
 
-def test_parse_eval_metric_result_invalid_json():
-    # Test parsing with invalid JSON
-    result = "invalid json"
+@pytest.mark.ai
+def test_parse_eval_metric_result__raises_evaluator_exception__with_invalid_json() -> (
+    None
+):
+    """
+    Purpose: Verify parser raises appropriate exception for malformed JSON.
+    Why this matters: Provides clear error handling for invalid LLM responses.
+    Setup summary: Provide invalid JSON string, assert EvaluatorException with descriptive message.
+    """
+    # Arrange
+    result_json: str = "invalid json"
+    metric_name: EvaluationMetricName = EvaluationMetricName.CONTEXT_RELEVANCY
+
+    # Act & Assert
     with pytest.raises(EvaluatorException) as exc_info:
-        parse_eval_metric_result(result, EvaluationMetricName.CONTEXT_RELEVANCY)
+        parse_eval_metric_result(result_json, metric_name)
 
     assert "Error occurred during parsing the evaluation metric result" in str(
         exc_info.value
     )
 
 
-def test_parse_eval_metric_result_structured_output_basic():
-    # Test basic structured output without fact list
-    result = EvaluationSchemaStructuredOutput(value="high", reason="Test reason")
-    parsed = parse_eval_metric_result_structured_output(
-        result, EvaluationMetricName.CONTEXT_RELEVANCY
+@pytest.mark.ai
+def test_parse_eval_metric_result_structured_output__succeeds__without_fact_list() -> (
+    None
+):
+    """
+    Purpose: Verify parsing of structured output without optional fact list.
+    Why this matters: Ensures structured output parsing works for basic evaluations.
+    Setup summary: Create structured output object without facts, assert correct parsing.
+    """
+    # Arrange
+    result: EvaluationSchemaStructuredOutput = EvaluationSchemaStructuredOutput(
+        value="high", reason="Test reason"
     )
+    metric_name: EvaluationMetricName = EvaluationMetricName.CONTEXT_RELEVANCY
 
+    # Act
+    parsed: EvaluationMetricResult = parse_eval_metric_result_structured_output(
+        result, metric_name
+    )
+
+    # Assert
     assert isinstance(parsed, EvaluationMetricResult)
     assert parsed.name == EvaluationMetricName.CONTEXT_RELEVANCY
     assert parsed.value == "high"
@@ -64,9 +111,17 @@ def test_parse_eval_metric_result_structured_output_basic():
     assert parsed.fact_list == []
 
 
-def test_parse_eval_metric_result_structured_output_with_facts():
-    # Test structured output with fact list
-    result = EvaluationSchemaStructuredOutput(
+@pytest.mark.ai
+def test_parse_eval_metric_result_structured_output__includes_facts__with_fact_list() -> (
+    None
+):
+    """
+    Purpose: Verify parsing of structured output with fact list extracts all facts.
+    Why this matters: Fact extraction is critical for detailed evaluation feedback.
+    Setup summary: Create structured output with multiple facts, assert all facts extracted.
+    """
+    # Arrange
+    result: EvaluationSchemaStructuredOutput = EvaluationSchemaStructuredOutput(
         value="high",
         reason="Test reason",
         fact_list=[
@@ -74,14 +129,18 @@ def test_parse_eval_metric_result_structured_output_with_facts():
             Fact(fact="Fact 2"),
         ],
     )
-    parsed = parse_eval_metric_result_structured_output(
-        result, EvaluationMetricName.CONTEXT_RELEVANCY
+    metric_name: EvaluationMetricName = EvaluationMetricName.CONTEXT_RELEVANCY
+
+    # Act
+    parsed: EvaluationMetricResult = parse_eval_metric_result_structured_output(
+        result, metric_name
     )
 
+    # Assert
     assert isinstance(parsed, EvaluationMetricResult)
     assert parsed.name == EvaluationMetricName.CONTEXT_RELEVANCY
     assert parsed.value == "high"
     assert parsed.reason == "Test reason"
     assert parsed.fact_list == ["Fact 1", "Fact 2"]
     assert isinstance(parsed.fact_list, list)
-    assert len(parsed.fact_list) == 2  # None fact should be filtered out
+    assert len(parsed.fact_list) == 2

unique_toolkit/agentic/evaluation/tests/test_prompt_loaders.py

@@ -0,0 +1,348 @@
+"""Tests for prompt loader functions."""
+
+import pytest
+
+from unique_toolkit.agentic.evaluation.context_relevancy.prompts import (
+    system_prompt_loader as context_system_prompt_loader,
+)
+from unique_toolkit.agentic.evaluation.context_relevancy.prompts import (
+    user_prompt_loader as context_user_prompt_loader,
+)
+from unique_toolkit.agentic.evaluation.hallucination.prompts import (
+    system_prompt_loader as hallucination_system_prompt_loader,
+)
+from unique_toolkit.agentic.evaluation.hallucination.prompts import (
+    user_prompt_loader as hallucination_user_prompt_loader,
+)
+
+
+@pytest.mark.ai
+def test_context_system_prompt_loader__returns_non_empty_string__on_call() -> None:
+    """
+    Purpose: Verify that the context relevancy system prompt loader returns a valid string.
+    Why this matters: The system prompt is critical for guiding the evaluation LLM's behavior.
+    Setup summary: Call the loader function and assert type and non-empty content.
+    """
+    # Arrange - No setup needed for this test
+
+    # Act
+    prompt: str = context_system_prompt_loader()
+
+    # Assert
+    assert isinstance(prompt, str)
+    assert len(prompt) > 0
+
+
+@pytest.mark.ai
+def test_context_user_prompt_loader__returns_non_empty_string__on_call() -> None:
+    """
+    Purpose: Verify that the context relevancy user prompt loader returns a valid string.
+    Why this matters: The user prompt template must be valid for evaluation requests.
+    Setup summary: Call the loader function and assert type and non-empty content.
+    """
+    # Arrange - No setup needed for this test
+
+    # Act
+    prompt: str = context_user_prompt_loader()
+
+    # Assert
+    assert isinstance(prompt, str)
+    assert len(prompt) > 0
+
+
+@pytest.mark.ai
+def test_context_system_prompt__contains_jinja_syntax__for_structured_output() -> None:
+    """
+    Purpose: Verify that system prompt contains Jinja2 template syntax for structured output control.
+    Why this matters: Template must support conditional rendering based on structured_output flag.
+    Setup summary: Load system prompt and check for Jinja2 conditional blocks.
+    """
+    # Arrange - No setup needed
+
+    # Act
+    prompt: str = context_system_prompt_loader()
+
+    # Assert
+    assert "{% if structured_output %}" in prompt or "{%" in prompt
+
+
+@pytest.mark.ai
+def test_context_user_prompt__contains_jinja_variables__for_input_and_context() -> None:
+    """
+    Purpose: Verify that user prompt contains required Jinja2 variable placeholders.
+    Why this matters: Template must support dynamic insertion of input text and context texts.
+    Setup summary: Load user prompt and check for expected variable placeholders.
+    """
+    # Arrange - No setup needed
+
+    # Act
+    prompt: str = context_user_prompt_loader()
+
+    # Assert
+    assert "{{ input_text }}" in prompt
+    assert "{{ context_texts }}" in prompt
+
+
+@pytest.mark.ai
+def test_context_system_prompt__has_both_structured_and_regular_modes__in_template() -> (
+    None
+):
+    """
+    Purpose: Verify that system prompt template supports both structured and regular output modes.
+    Why this matters: Template must handle both evaluation output formats correctly.
+    Setup summary: Load system prompt and check for conditional blocks for both modes.
+    """
+    # Arrange - No setup needed
+
+    # Act
+    prompt: str = context_system_prompt_loader()
+
+    # Assert
+    assert "{% if structured_output %}" in prompt
+    assert "{% else %}" in prompt or "{% endif %}" in prompt
+
+
+@pytest.mark.ai
+def test_context_user_prompt__has_conditional_json_instruction__for_unstructured_mode() -> (
+    None
+):
+    """
+    Purpose: Verify that user prompt has conditional JSON instruction for unstructured mode.
+    Why this matters: Non-structured mode requires explicit JSON formatting instructions.
+    Setup summary: Load user prompt and check for conditional JSON instruction block.
+    """
+    # Arrange - No setup needed
+
+    # Act
+    prompt: str = context_user_prompt_loader()
+
+    # Assert
+    assert "{% if not structured_output %}" in prompt or "{%" in prompt
+
+
+@pytest.mark.ai
+def test_context_prompts__are_consistent_between_calls__for_determinism() -> None:
+    """
+    Purpose: Verify that prompt loaders return consistent content across multiple invocations.
+    Why this matters: Ensures deterministic behavior and no hidden state in loaders.
+    Setup summary: Call loaders twice and compare results for equality.
+    """
+    # Arrange - No setup needed
+
+    # Act
+    system_prompt_1: str = context_system_prompt_loader()
+    system_prompt_2: str = context_system_prompt_loader()
+    user_prompt_1: str = context_user_prompt_loader()
+    user_prompt_2: str = context_user_prompt_loader()
+
+    # Assert
+    assert system_prompt_1 == system_prompt_2
+    assert user_prompt_1 == user_prompt_2
+
+
+@pytest.mark.ai
+def test_hallucination_system_prompt_loader__returns_non_empty_string__on_call() -> (
+    None
+):
+    """
+    Purpose: Verify that the hallucination system prompt loader returns a valid string.
+    Why this matters: The system prompt is critical for hallucination detection behavior.
+    Setup summary: Call the loader function and assert type and non-empty content.
+    """
+    # Arrange - No setup needed
+
+    # Act
+    prompt: str = hallucination_system_prompt_loader()
+
+    # Assert
+    assert isinstance(prompt, str)
+    assert len(prompt) > 0
+
+
+@pytest.mark.ai
+def test_hallucination_user_prompt_loader__returns_non_empty_string__on_call() -> None:
+    """
+    Purpose: Verify that the hallucination user prompt loader returns a valid string.
+    Why this matters: The user prompt template must be valid for hallucination evaluation.
+    Setup summary: Call the loader function and assert type and non-empty content.
+    """
+    # Arrange - No setup needed
+
+    # Act
+    prompt: str = hallucination_user_prompt_loader()
+
+    # Assert
+    assert isinstance(prompt, str)
+    assert len(prompt) > 0
+
+
+@pytest.mark.ai
+def test_hallucination_system_prompt__contains_jinja_syntax__for_has_context() -> None:
+    """
+    Purpose: Verify that system prompt contains Jinja2 template syntax for context handling.
+    Why this matters: Template must support conditional rendering based on has_context flag.
+    Setup summary: Load system prompt and check for Jinja2 conditional blocks.
+    """
+    # Arrange - No setup needed
+
+    # Act
+    prompt: str = hallucination_system_prompt_loader()
+
+    # Assert
+    assert "{% if has_context %}" in prompt or "{%" in prompt
+
+
+@pytest.mark.ai
+def test_hallucination_user_prompt__contains_jinja_variables__for_input_and_output() -> (
+    None
+):
+    """
+    Purpose: Verify that user prompt contains required Jinja2 variable placeholders.
+    Why this matters: Template must support dynamic insertion of input and output texts.
+    Setup summary: Load user prompt and check for expected variable placeholders.
+    """
+    # Arrange - No setup needed
+
+    # Act
+    prompt: str = hallucination_user_prompt_loader()
+
+    # Assert
+    assert "{{ input_text }}" in prompt
+    assert "{{ output_text }}" in prompt
+
+
+@pytest.mark.ai
+def test_hallucination_system_prompt__has_context_conditional__in_template() -> None:
+    """
+    Purpose: Verify that system prompt template has conditional logic for has_context.
+    Why this matters: Template must handle both context and non-context evaluation scenarios.
+    Setup summary: Load system prompt and check for conditional blocks with else/endif.
+    """
+    # Arrange - No setup needed
+
+    # Act
+    prompt: str = hallucination_system_prompt_loader()
+
+    # Assert
+    assert "{% if has_context %}" in prompt
+    assert "{% else %}" in prompt or "{% endif %}" in prompt
+
+
+@pytest.mark.ai
+def test_hallucination_user_prompt__has_optional_context_fields__in_template() -> None:
+    """
+    Purpose: Verify that user prompt has conditional blocks for optional context fields.
+    Why this matters: Template must support optional contexts_text and history_messages_text.
+    Setup summary: Load user prompt and check for conditional blocks or variable placeholders.
+    """
+    # Arrange - No setup needed
+
+    # Act
+    prompt: str = hallucination_user_prompt_loader()
+
+    # Assert
+    assert "{% if contexts_text %}" in prompt or "{{ contexts_text }}" in prompt
+    assert (
+        "{% if history_messages_text %}" in prompt
+        or "{{ history_messages_text }}" in prompt
+    )
+
+
+@pytest.mark.ai
+def test_hallucination_system_prompt__mentions_hallucination_concepts__in_content() -> (
+    None
+):
+    """
+    Purpose: Verify that system prompt mentions hallucination-related concepts.
+    Why this matters: Ensures prompt properly guides model to detect hallucinations.
+    Setup summary: Load system prompt and check for hallucination-related keywords.
+    """
+    # Arrange - No setup needed
+
+    # Act
+    prompt: str = hallucination_system_prompt_loader()
+    prompt_lower: str = prompt.lower()
+
+    # Assert
+    assert (
+        "hallucination" in prompt_lower
+        or "grounded" in prompt_lower
+        or "supported" in prompt_lower
+    )
+
+
+@pytest.mark.ai
+def test_hallucination_user_prompt__contains_data_sections__for_input_and_output() -> (
+    None
+):
+    """
+    Purpose: Verify that user prompt has sections for input and output data.
+    Why this matters: Template must clearly separate input and output for evaluation.
+    Setup summary: Load user prompt and check for input/output section markers.
+    """
+    # Arrange - No setup needed
+
+    # Act
+    prompt: str = hallucination_user_prompt_loader()
+
+    # Assert
+    assert "Input:" in prompt or "input" in prompt.lower()
+    assert "Output:" in prompt or "output" in prompt.lower()
+
+
+@pytest.mark.ai
+def test_hallucination_prompts__are_consistent_between_calls__for_determinism() -> None:
+    """
+    Purpose: Verify that hallucination prompt loaders return consistent content.
+    Why this matters: Ensures deterministic behavior and no hidden state in loaders.
+    Setup summary: Call loaders twice and compare results for equality.
+    """
+    # Arrange - No setup needed
+
+    # Act
+    system_prompt_1: str = hallucination_system_prompt_loader()
+    system_prompt_2: str = hallucination_system_prompt_loader()
+    user_prompt_1: str = hallucination_user_prompt_loader()
+    user_prompt_2: str = hallucination_user_prompt_loader()
+
+    # Assert
+    assert system_prompt_1 == system_prompt_2
+    assert user_prompt_1 == user_prompt_2
+
+
+@pytest.mark.ai
+def test_context_relevancy_loaders__can_access_template_files__without_errors() -> None:
+    """
+    Purpose: Verify that context relevancy loaders can successfully access template files.
+    Why this matters: Ensures template files exist and are readable at runtime.
+    Setup summary: Call loaders and assert no FileNotFoundError is raised.
+    """
+    # Arrange - No setup needed
+
+    # Act & Assert
+    try:
+        system_prompt: str = context_system_prompt_loader()
+        user_prompt: str = context_user_prompt_loader()
+        assert system_prompt is not None
+        assert user_prompt is not None
+    except FileNotFoundError as e:
+        pytest.fail(f"Prompt loader failed to access template file: {e}")
+
+
+@pytest.mark.ai
+def test_hallucination_loaders__can_access_template_files__without_errors() -> None:
+    """
+    Purpose: Verify that hallucination loaders can successfully access template files.
+    Why this matters: Ensures template files exist and are readable at runtime.
+    Setup summary: Call loaders and assert no FileNotFoundError is raised.
+    """
+    # Arrange - No setup needed
+
+    # Act & Assert
+    try:
+        system_prompt: str = hallucination_system_prompt_loader()
+        user_prompt: str = hallucination_user_prompt_loader()
+        assert system_prompt is not None
+        assert user_prompt is not None
+    except FileNotFoundError as e:
+        pytest.fail(f"Prompt loader failed to access template file: {e}")

unique_toolkit/agentic/evaluation/utils.py

@@ -0,0 +1,8 @@
+from pathlib import Path
+
+
+def load_template(parent_dir: Path, filename: str) -> str:
+    """Load a Jinja2 template file from the hallucination directory."""
+    template_path = parent_dir / filename
+    with open(template_path, "r") as f:
+        return f.read()

unique_toolkit/chat/responses_api.py

@@ -1,3 +1,4 @@
+import json
 import logging
 from typing import Any, NamedTuple, Sequence
 
@@ -196,12 +197,23 @@ def _prepare_responses_params_util(
     log_exc_info=False,
     logger=logger,
 )
-def _attempt_extract_reasoning_from_options(options: dict) -> Reasoning | None:
-    reasoning = None
+def _attempt_extract_reasoning_from_options(
+    options: dict[str, Any],
+) -> Reasoning | None:
+    reasoning: dict[str, Any] | str | None = None
 
     # Responses API
     if "reasoning" in options:
         reasoning = options["reasoning"]
+        # Handle case where reasoning is stored as JSON string (UI limitation)
+        if isinstance(reasoning, str):
+            try:
+                reasoning = json.loads(reasoning)
+            except (json.JSONDecodeError, TypeError):
+                logger.warning(
+                    f"Failed to parse reasoning as JSON string: {reasoning}. "
+                    "Continuing with raw value."
+                )
 
     # Completions API
     elif "reasoning_effort" in options:
@@ -222,20 +234,28 @@ def _attempt_extract_reasoning_from_options(options: dict) -> Reasoning | None:
     logger=logger,
 )
 def _attempt_extract_verbosity_from_options(
-    options: dict,
+    options: dict[str, Any],
 ) -> ResponseTextConfigParam | None:
-    reasoning = None
+    if "verbosity" in options:
+        return TypeAdapter(ResponseTextConfigParam).validate_python(
+            {"verbosity": options["verbosity"]}
+        )
 
     # Responses API
     if "text" in options:
-        reasoning = options["text"]
-
-    # Completions API
-    elif "verbosity" in options:
-        reasoning = {"verbosity": options["verbosity"]}
-
-    if reasoning is not None:
-        return TypeAdapter(ResponseTextConfigParam).validate_python(reasoning)
+        text_config: dict[str, Any] | str = options["text"]
+        # Handle case where text is stored as JSON string (UI limitation)
+        if isinstance(text_config, str):
+            try:
+                text_config = json.loads(text_config)
+                return TypeAdapter(ResponseTextConfigParam).validate_python(text_config)
+            except (json.JSONDecodeError, TypeError):
+                logger.warning(
+                    f"Failed to parse text as JSON string: {text_config}. "
+                    "Continuing with raw value."
+                )
+        if isinstance(text_config, dict):
+            return TypeAdapter(ResponseTextConfigParam).validate_python(text_config)
 
     return None
 
@@ -283,42 +303,26 @@ def _prepare_responses_args(
 
     openai_options: unique_sdk.Integrated.CreateStreamResponsesOpenaiParams = {}
 
-    if params.temperature is not None:
-        openai_options["temperature"] = params.temperature
-
-    if params.reasoning is not None:
-        openai_options["reasoning"] = params.reasoning
-
-    if params.text is not None:
-        openai_options["text"] = params.text
-
-    if include is not None:
-        openai_options["include"] = include
-
-    if instructions is not None:
-        openai_options["instructions"] = instructions
-
-    if max_output_tokens is not None:
-        openai_options["max_output_tokens"] = max_output_tokens
-
-    if metadata is not None:
-        openai_options["metadata"] = metadata
-
-    if parallel_tool_calls is not None:
-        openai_options["parallel_tool_calls"] = parallel_tool_calls
-
-    if tool_choice is not None:
-        openai_options["tool_choice"] = tool_choice
-
-    if params.tools is not None:
-        openai_options["tools"] = params.tools
-
-    if top_p is not None:
-        openai_options["top_p"] = top_p
+    explicit_options = {
+        "temperature": params.temperature,
+        "reasoning": params.reasoning,
+        "text": params.text,
+        "include": include,
+        "instructions": instructions,
+        "max_output_tokens": max_output_tokens,
+        "metadata": metadata,
+        "parallel_tool_calls": parallel_tool_calls,
+        "tool_choice": tool_choice,
+        "tools": params.tools,
+        "top_p": top_p,
+    }
+
+    openai_options.update({k: v for k, v in explicit_options.items() if v is not None})  # type: ignore[arg-type]
 
     # allow any other openai.resources.responses.Response.create options
     if other_options is not None:
-        openai_options.update(other_options)  # type: ignore
+        for k, v in other_options.items():
+            openai_options.setdefault(k, v)  # type: ignore
 
     options["options"] = openai_options
 

{unique_toolkit-1.45.5.dist-info → unique_toolkit-1.45.7.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: unique_toolkit
-Version: 1.45.5
+Version: 1.45.7
 Summary: 
 License: Proprietary
 Author: Cedric Klinkert
@@ -125,6 +125,14 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), 
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## [1.45.7] - 2026-01-30
+- Add JSON string parsing support for reasoning and text parameters in responses API (UI compatibility)
+- Fix variable name bug in `_attempt_extract_verbosity_from_options` function
+- Improve `other_options` handling to prevent overwriting explicitly set parameters
+
+## [1.45.6] - 2026-01-30
+- hallucination evaluator: Use original response to retrieve referenced chunk
+
 ## [1.45.5] - 2026-01-29
 - Add HTML rendering support for code interpreter generated files