kiln-ai 0.17.0__py3-none-any.whl → 0.18.0__py3-none-any.whl

kiln_ai/adapters/eval/test_g_eval.py

@@ -274,6 +274,36 @@ def test_token_case():
         assert token.lower() == token
 
 
+def test_generate_run_description(test_eval_config, test_run_config, test_task_run):
+    """Test that generate_run_description correctly uses task_run.output.output (the string) rather than task_run.output (the object)."""
+    # Create G-Eval instance
+    g_eval = GEval(test_eval_config, test_run_config)
+
+    # Call generate_run_description
+    description = g_eval.generate_run_description(
+        test_task_run.input, test_task_run.output.output
+    )
+
+    # Verify that the actual string output is in the description
+    expected_output = "Why did the chicken cross the road? To get to the other side!"
+    assert expected_output in description
+
+    # Verify that the input is also in the description
+    assert "Tell me a chicken joke" in description
+
+    # Verify the description has the expected structure
+    assert "<eval_data>" in description
+    assert description.count("<eval_data>") == 2  # 2 opening tags
+    assert description.count("</eval_data>") == 2  # 2 closing tags
+    assert "The model was given the following input for the task:" in description
+    assert "The model produced the following output for the task:" in description
+
+    # Verify that we're getting the actual string value, not a Python object representation
+    # The string should not contain 'TaskOutput' or other object indicators
+    assert "TaskOutput" not in description
+    assert "output=" not in description  # Would appear if object __repr__ was used
+
+
 def test_metric_offsets_and_search_ranges(
     test_eval_config, test_run_config, test_task_run
 ):
@@ -401,7 +431,7 @@ def test_rating_token_to_score(test_eval_config, test_run_config):
 
     # Test single token case
     token_logprob = MockTokenLogprob("5", [("5", 0.0)], logprob=1e-8)  # log(1) = 0
-    score = g_eval.rating_token_to_score(token_logprob)
+    score = g_eval.rating_token_to_score(token_logprob)  # type: ignore
     assert score == 5.0
 
     # Test weighted average case
@@ -413,20 +443,62 @@ def test_rating_token_to_score(test_eval_config, test_run_config):
         ],
         logprob=math.log(0.6),
     )
-    score = g_eval.rating_token_to_score(token_logprob)
+    score = g_eval.rating_token_to_score(token_logprob)  # type: ignore
     assert pytest.approx(score) == 4.4  # (4 * 0.6 + 5 * 0.4)
 
     # Test invalid token
     token_logprob = MockTokenLogprob(":", [(":", 0.0)], logprob=1e-8)
-    assert g_eval.rating_token_to_score(token_logprob) is None
+    assert g_eval.rating_token_to_score(token_logprob) is None  # type: ignore
 
     # Test missing from top logprobs
     token_logprob = MockTokenLogprob("5", [], logprob=1e-8)
-    assert pytest.approx(g_eval.rating_token_to_score(token_logprob)) == 5.0
+    assert pytest.approx(g_eval.rating_token_to_score(token_logprob)) == 5.0  # type: ignore
 
     # Test missing from top logprobs, with special case logprob
     token_logprob = MockTokenLogprob("5", [], logprob=-9999)
-    assert pytest.approx(g_eval.rating_token_to_score(token_logprob)) == 5.0
+    assert pytest.approx(g_eval.rating_token_to_score(token_logprob)) == 5.0  # type: ignore
+
+
+def test_rating_token_to_score_zero_score_bug_fix(test_eval_config, test_run_config):
+    """Test that rating_token_to_score correctly handles 0.0 scores (like 'fail') and doesn't return None.
+
+    This test verifies the fix for the bug where 'if not primary_token_score:' would incorrectly
+    treat 0.0 as falsy and return None, when it should only return None for actual None values.
+    """
+    g_eval = GEval(test_eval_config, test_run_config)
+
+    class MockTopLogprob:
+        def __init__(self, token, logprob):
+            self.token = token
+            self.logprob = logprob
+
+    class MockTokenLogprob:
+        def __init__(self, token, top_logprobs, logprob):
+            self.token = token
+            self.top_logprobs = [MockTopLogprob(t, lp) for t, lp in top_logprobs]
+            self.logprob = logprob
+
+    # Test that "fail" token (which maps to 0.0) is handled correctly
+    token_logprob = MockTokenLogprob("fail", [("fail", 0.0)], logprob=1e-8)
+    score = g_eval.rating_token_to_score(token_logprob)  # type: ignore
+    assert score == 0.0, f"Expected 0.0 for 'fail' token, got {score}"
+
+    # Test that "0" token (which maps to None) still returns None
+    token_logprob = MockTokenLogprob("0", [("0", 0.0)], logprob=1e-8)
+    score = g_eval.rating_token_to_score(token_logprob)  # type: ignore
+    assert score is None, f"Expected None for '0' token, got {score}"
+
+    # Test weighted average case with fail token
+    token_logprob = MockTokenLogprob(
+        "fail",
+        [
+            ("fail", math.log(0.7)),  # 70% probability for fail (0.0)
+            ("pass", math.log(0.3)),  # 30% probability for pass (1.0)
+        ],
+        logprob=math.log(0.7),
+    )
+    score = g_eval.rating_token_to_score(token_logprob)  # type: ignore
+    assert pytest.approx(score) == 0.3  # (0.0 * 0.7 + 1.0 * 0.3)
 
 
 def test_g_eval_system_instruction():
@@ -502,3 +574,41 @@ async def test_all_built_in_models_logprobs_geval(
         model_name,
         provider_name.value,
     )
+
+
+def check_supports_llm_as_judge(model_name: str, provider_name: str):
+    for model in built_in_models:
+        if model.name != model_name:
+            continue
+        for provider in model.providers:
+            if provider.name != provider_name:
+                continue
+            if not provider.supports_structured_output:
+                pytest.skip(
+                    f"Skipping {model.name} {provider.name} because it does not support llm_as_judge (structured_output_mode)"
+                )
+            return
+    raise RuntimeError(f"No model {model_name} {provider_name} found")
+
+
+@pytest.mark.paid
+@pytest.mark.ollama
+@pytest.mark.parametrize("model_name,provider_name", get_all_models_and_providers())
+async def test_all_built_in_models_llm_as_judge(
+    model_name,
+    provider_name,
+    test_task,
+    test_eval_config,
+    test_task_run,
+    test_run_config,
+):
+    check_supports_llm_as_judge(model_name, provider_name)
+    await run_g_eval_test(
+        test_task,
+        test_eval_config,
+        test_task_run,
+        EvalConfigType.llm_as_judge,
+        test_run_config,
+        model_name,
+        provider_name.value,
+    )

kiln_ai/adapters/fine_tune/base_finetune.py

@@ -3,12 +3,7 @@ from typing import Literal
 
 from pydantic import BaseModel
 
-from kiln_ai.adapters.ml_model_list import built_in_models
-from kiln_ai.datamodel import (
-    DatasetSplit,
-    FineTuneStatusType,
-    Task,
-)
+from kiln_ai.datamodel import DatasetSplit, FineTuneStatusType, Task
 from kiln_ai.datamodel import Finetune as FinetuneModel
 from kiln_ai.datamodel.datamodel_enums import ChatStrategy
 from kiln_ai.utils.name_generator import generate_memorable_name

kiln_ai/adapters/fine_tune/dataset_formatter.py

@@ -1,15 +1,11 @@
 import json
 import tempfile
-from dataclasses import dataclass
 from enum import Enum
 from pathlib import Path
 from typing import Any, Dict, Protocol
 from uuid import uuid4
 
-from kiln_ai.adapters.chat.chat_formatter import (
-    ChatMessage,
-    get_chat_formatter,
-)
+from kiln_ai.adapters.chat.chat_formatter import ChatMessage, get_chat_formatter
 from kiln_ai.datamodel import DatasetSplit, TaskRun
 from kiln_ai.datamodel.datamodel_enums import THINKING_DATA_STRATEGIES, ChatStrategy
 from kiln_ai.utils.exhaustive_error import raise_exhaustive_enum_error

kiln_ai/adapters/fine_tune/test_dataset_formatter.py

@@ -857,7 +857,7 @@ def test_serialize_r1_style_message_missing_thinking(thinking, final_output):
 
 def test_vertex_gemini_role_map_coverage():
     """Test that VERTEX_GEMINI_ROLE_MAP covers all possible ChatMessage.role values"""
-    from typing import Literal, get_type_hints
+    from typing import get_type_hints
 
     # Get the Literal type from ChatMessage.role
     role_type = get_type_hints(ChatMessage)["role"]

kiln_ai/adapters/fine_tune/test_vertex_finetune.py

@@ -1,6 +1,5 @@
-import time
 from pathlib import Path
-from unittest.mock import AsyncMock, MagicMock, patch
+from unittest.mock import MagicMock, patch
 
 import pytest
 from google.cloud import storage
@@ -10,11 +9,7 @@ from vertexai.tuning import sft
 from kiln_ai.adapters.fine_tune.base_finetune import FineTuneStatusType
 from kiln_ai.adapters.fine_tune.dataset_formatter import DatasetFormat, DatasetFormatter
 from kiln_ai.adapters.fine_tune.vertex_finetune import VertexFinetune
-from kiln_ai.datamodel import (
-    DatasetSplit,
-    StructuredOutputMode,
-    Task,
-)
+from kiln_ai.datamodel import DatasetSplit, StructuredOutputMode, Task
 from kiln_ai.datamodel import Finetune as FinetuneModel
 from kiln_ai.datamodel.datamodel_enums import ChatStrategy
 from kiln_ai.datamodel.dataset_split import Train80Test20SplitDefinition

kiln_ai/adapters/fine_tune/together_finetune.py

@@ -1,4 +1,4 @@
-from typing import Literal, Tuple
+from typing import Tuple
 
 from together import Together
 from together.types.files import FilePurpose