kiln-ai 0.16.0__py3-none-any.whl → 0.18.0__py3-none-any.whl

kiln_ai/adapters/eval/g_eval.py

@@ -5,11 +5,14 @@ from litellm.types.utils import ChatCompletionTokenLogprob
 
 from kiln_ai.adapters.adapter_registry import adapter_for_task
 from kiln_ai.adapters.eval.base_eval import BaseEval
+from kiln_ai.adapters.ml_model_list import (
+    default_structured_output_mode_for_model_provider,
+)
 from kiln_ai.adapters.model_adapters.base_adapter import AdapterConfig, RunOutput
 from kiln_ai.adapters.prompt_builders import PromptGenerators
 from kiln_ai.datamodel import Project, Task, TaskRun
 from kiln_ai.datamodel.eval import EvalConfig, EvalConfigType, EvalScores
-from kiln_ai.datamodel.task import RunConfig
+from kiln_ai.datamodel.task import RunConfig, RunConfigProperties, StructuredOutputMode
 
 # all the tokens we score for, and their float scores.
 TOKEN_TO_SCORE_MAP: Dict[str, float] = {
@@ -99,6 +102,18 @@ class GEval(BaseEval):
 
         self.geval_task = GEvalTask(eval_config)
 
+    def generate_run_description(self, eval_input: str, eval_output: str) -> str:
+        return f"""The model was given the following input for the task: 
+<eval_data>
+{eval_input}
+</eval_data>
+
+The model produced the following output for the task:
+<eval_data>
+{eval_output}
+</eval_data>
+"""
+
     async def run_eval(
         self, task_run: TaskRun
     ) -> tuple[EvalScores, Dict[str, str] | None]:
@@ -114,12 +129,27 @@ class GEval(BaseEval):
             10 if self.eval_config.config_type == EvalConfigType.g_eval else None
         )
 
-        adapter = adapter_for_task(
-            self.geval_task,
+        # We don't expose setting this manually in the UI, so pull a recommended mode from ml_model_list
+        structured_output_mode = default_structured_output_mode_for_model_provider(
             model_name,
             provider,
-            # We always use Simple COT for G-Eval and LLM as Judge
-            prompt_id=PromptGenerators.SIMPLE_CHAIN_OF_THOUGHT,
+            default=StructuredOutputMode.json_schema,
+            # G-eval expects JSON, so don't allow function calling modes
+            disallowed_modes=[
+                StructuredOutputMode.function_calling,
+                StructuredOutputMode.function_calling_weak,
+            ],
+        )
+
+        adapter = adapter_for_task(
+            self.geval_task,
+            run_config_properties=RunConfigProperties(
+                model_name=model_name,
+                model_provider_name=provider,
+                # We always use Simple COT for G-Eval and LLM as Judge
+                prompt_id=PromptGenerators.SIMPLE_CHAIN_OF_THOUGHT,
+                structured_output_mode=structured_output_mode,
+            ),
             base_adapter_config=AdapterConfig(
                 # Don't save this run into the task_runs. It will be saved into an eval_run where it belongs
                 allow_saving=False,
@@ -127,19 +157,12 @@ class GEval(BaseEval):
             ),
         )
 
-        input = f"""The model was given the following input for the task: 
-<eval_data>
-{task_run.input}
-</eval_data>
-
-The model produced the following output for the task:
-<eval_data>
-{task_run.output}
-</eval_data>
-"""
+        run_description = self.generate_run_description(
+            task_run.input, task_run.output.output
+        )
 
         # We don't need the run, but invoke_returning_run_output() runs validations for us over _run()
-        _, run_output = await adapter.invoke_returning_run_output(input)
+        _, run_output = await adapter.invoke_returning_run_output(run_description)
 
         if self.eval_config.config_type == EvalConfigType.llm_as_judge:
             return self.build_llm_as_judge_score(
@@ -292,7 +315,7 @@ The model produced the following output for the task:
         """
         primary_token_score = self.score_from_token_string(token_logprob.token)
         # check this is a real rating token, it could just be the ": ", "," or whitespace
-        if not primary_token_score:
+        if primary_token_score is None:
             return None
 
         total_score = 0.0

kiln_ai/adapters/eval/test_base_eval.py

@@ -1,9 +1,9 @@
 import json
+from unittest.mock import AsyncMock, MagicMock, patch
 
 import pytest
 
 from kiln_ai.adapters.eval.base_eval import BaseEval
-from kiln_ai.datamodel import BasePrompt, DataSource, DataSourceType
 from kiln_ai.datamodel.eval import Eval, EvalConfig, EvalOutputScore
 from kiln_ai.datamodel.task import (
     RunConfigProperties,
@@ -43,7 +43,9 @@ def test_score_schema_five_star():
 
     # Check score property, and that it's an enum of 1-5
     score_prop = schema["properties"]["quality_score"]
-    assert score_prop["enum"] == [1, 2, 3, 4, 5]
+    assert score_prop["type"] == "integer"
+    assert score_prop["minimum"] == 1
+    assert score_prop["maximum"] == 5
     assert "Quality Score" in score_prop["title"]
     assert "Rate the quality" in score_prop["description"]
     assert "between 1 and 5" in score_prop["description"]
@@ -51,7 +53,9 @@ def test_score_schema_five_star():
     # Check overall rating property, and that it's an enum of 1-5
     assert "overall_rating" in schema["properties"]
     overall = schema["properties"]["overall_rating"]
-    assert overall["enum"] == [1, 2, 3, 4, 5]
+    assert overall["type"] == "integer"
+    assert overall["minimum"] == 1
+    assert overall["maximum"] == 5
     assert "Overall Rating" in overall["title"]
     assert "The overall rating for the task output" in overall["description"]
     assert "between 1 and 5" in overall["description"]
@@ -127,6 +131,7 @@ def test_score_schema_pass_fail():
     schema = json.loads(schema_str)
 
     score_prop = schema["properties"]["pass_fail_test"]
+    assert score_prop["type"] == "string"
     assert score_prop["enum"] == ["pass", "fail"]
     assert "Pass Fail Test" in score_prop["title"]
     assert "Check if it passes" in score_prop["description"]
@@ -173,6 +178,7 @@ def test_score_schema_pass_fail_critical():
     score_prop = schema["properties"]["critical_test"]
     assert "enum" in score_prop
     assert score_prop["enum"] == ["pass", "fail", "critical"]
+    assert score_prop["type"] == "string"
     assert "'pass', 'fail', or 'critical'" in score_prop["description"]
 
     assert schema["properties"]["overall_rating"] is not None
@@ -245,7 +251,7 @@ class EvalTester(BaseEval):
     """Test implementation of BaseEval"""
 
     async def run_eval(self, task_run):
-        return {"overall_rating": 5, "quality": 4}
+        return {"overall_rating": 5, "quality": 4}, None
 
 
 @pytest.mark.paid
@@ -265,14 +271,8 @@ async def test_run_method():
 
     eval_config = EvalConfig(
         name="Test Eval Config",
-        model=DataSource(
-            type=DataSourceType.synthetic,
-            properties={
-                "model_name": "gpt-4o",
-                "model_provider": "openai",
-                "adapter_name": "test",
-            },
-        ),
+        model_name="gpt-4o",
+        model_provider="openai",
         parent=Eval(
             name="Test Eval",
             parent=task,
@@ -291,10 +291,6 @@ async def test_run_method():
                 ),
             ],
         ),
-        prompt=BasePrompt(
-            name="Test Prompt",
-            prompt="Test prompt",
-        ),
         properties={"eval_steps": ["test_step"]},
     )
 
@@ -311,7 +307,9 @@ async def test_run_method():
     evaluator = EvalTester(eval_config, run_config.run_config())
 
     # Run the evaluation
-    task_run, eval_scores = await evaluator.run("test input")
+    task_run, eval_scores, intermediate_outputs = await evaluator.run_task_and_eval(
+        "test input"
+    )
 
     # Verify task run was created
     assert task_run.input == "test input"
@@ -323,3 +321,162 @@ async def test_run_method():
 
     # Verify schema validation worked (these keys should exist per schema)
     assert set(eval_scores.keys()) == {"overall_rating", "quality"}
+
+
+@pytest.mark.asyncio
+async def test_run_task_and_eval():
+    """Test run_task_and_eval method with mocked dependencies"""
+    # Create test data
+    task = Task(
+        name="Test Task",
+        instruction="Test instruction",
+        requirements=[
+            TaskRequirement(
+                name="Quality",
+                instruction="Rate quality",
+                type=TaskOutputRatingType.five_star,
+            ),
+        ],
+    )
+
+    eval_config = EvalConfig(
+        name="Test Eval Config",
+        model_name="gpt-4o",
+        model_provider="openai",
+        parent=Eval(
+            name="Test Eval",
+            parent=task,
+            eval_set_filter_id="all",
+            eval_configs_filter_id="all",
+            output_scores=[
+                EvalOutputScore(
+                    name="Quality",
+                    instruction="Rate quality",
+                    type=TaskOutputRatingType.five_star,
+                ),
+                EvalOutputScore(
+                    name="Overall Rating",
+                    instruction="The overall rating for the task output",
+                    type=TaskOutputRatingType.five_star,
+                ),
+            ],
+        ),
+        properties={"eval_steps": ["test_step"]},
+    )
+
+    run_config = TaskRunConfig(
+        name="Test Run Config",
+        run_config_properties=RunConfigProperties(
+            model_name="llama_3_1_8b",
+            model_provider_name="groq",
+            prompt_id="simple_prompt_builder",
+            structured_output_mode="json_schema",
+        ),
+        parent=task,
+    )
+
+    # Create evaluator instance
+    class MockEval(BaseEval):
+        async def run_eval(self, task_run):
+            return {"overall_rating": 5, "quality": 4}, {"thinking": "test thinking"}
+
+    evaluator = MockEval(eval_config, run_config.run_config())
+
+    # Mock dependencies
+    mock_adapter = AsyncMock()
+    mock_task_run = MagicMock()
+    mock_task_run.input = "test input"
+    mock_task_run.output.output = "test output"
+    mock_adapter.invoke.return_value = mock_task_run
+
+    with (
+        patch(
+            "kiln_ai.adapters.eval.base_eval.adapter_for_task"
+        ) as mock_adapter_for_task,
+        patch(
+            "kiln_ai.adapters.eval.base_eval.validate_schema_with_value_error"
+        ) as mock_validate,
+    ):
+        mock_adapter_for_task.return_value = mock_adapter
+
+        # Test with string input
+        result = await evaluator.run_task_and_eval("test input")
+
+        # Verify adapter_for_task was called with correct parameters
+        mock_adapter_for_task.assert_called_once()
+        assert mock_adapter_for_task.call_args[0][0] == evaluator.target_task
+        props = mock_adapter_for_task.call_args[0][1]
+        assert props.model_name == "llama_3_1_8b"
+        assert props.model_provider_name == "groq"
+        assert props.prompt_id == "simple_prompt_builder"
+        bac = mock_adapter_for_task.call_args[1]
+        assert bac["base_adapter_config"].allow_saving is False
+
+        # Verify the base_adapter_config has allow_saving=False
+        adapter_config = mock_adapter_for_task.call_args[1]["base_adapter_config"]
+        assert adapter_config.allow_saving is False
+
+        # Verify adapter.invoke was called with correct input
+        mock_adapter.invoke.assert_called_once_with("test input")
+
+        # Verify validate_schema_with_value_error was called
+        mock_validate.assert_called_once_with(
+            {"overall_rating": 5, "quality": 4},
+            evaluator.score_schema,
+            "Eval output does not match score schema.",
+        )
+
+        # Verify return values
+        task_run, eval_scores, intermediate_outputs = result
+        assert task_run == mock_task_run
+        assert eval_scores == {"overall_rating": 5, "quality": 4}
+        assert intermediate_outputs == {"thinking": "test thinking"}
+
+
+@pytest.mark.asyncio
+async def test_run_task_and_eval_no_run_config():
+    """Test run_task_and_eval raises error when run_config is None"""
+    task = Task(
+        name="Test Task",
+        instruction="Test instruction",
+        requirements=[
+            TaskRequirement(
+                name="Quality",
+                instruction="Rate quality",
+                type=TaskOutputRatingType.five_star,
+            ),
+        ],
+    )
+
+    eval_config = EvalConfig(
+        name="Test Eval Config",
+        model_name="gpt-4o",
+        model_provider="openai",
+        parent=Eval(
+            name="Test Eval",
+            parent=task,
+            eval_set_filter_id="all",
+            eval_configs_filter_id="all",
+            output_scores=[
+                EvalOutputScore(
+                    name="Quality",
+                    instruction="Rate quality",
+                    type=TaskOutputRatingType.five_star,
+                ),
+            ],
+        ),
+        properties={"eval_steps": ["test_step"]},
+    )
+
+    # Create evaluator instance with no run_config
+    class MockEval(BaseEval):
+        async def run_eval(self, task_run):
+            return {"quality": 4}, None
+
+    evaluator = MockEval(eval_config, None)
+
+    # Test that it raises ValueError
+    with pytest.raises(
+        ValueError, match="Run config is required for run_task_and_eval"
+    ):
+        await evaluator.run_task_and_eval("test input")

kiln_ai/adapters/eval/test_eval_runner.py

@@ -94,6 +94,7 @@ def mock_run_config(
             model_name="gpt-4",
             model_provider_name="openai",
             prompt_id="simple_prompt_builder",
+            structured_output_mode="json_schema",
         ),
         parent=mock_task,
     )
@@ -209,6 +210,7 @@ def test_collect_tasks_filtering(
             model_name="gpt-4",
             model_provider_name="openai",
             prompt_id="simple_prompt_builder",
+            structured_output_mode="json_schema",
         ),
         parent=mock_task,
     )
@@ -416,6 +418,7 @@ def test_collect_tasks_multiple_run_configs(
             model_name="gpt-3.5",
             model_provider_name="openai",
             prompt_id="simple_prompt_builder",
+            structured_output_mode="json_schema",
         ),
         parent=mock_task,
     )

kiln_ai/adapters/eval/test_g_eval.py

@@ -99,6 +99,7 @@ def test_run_config(test_task):
         model_provider_name="groq",
         task=test_task,
         prompt_id="simple_prompt_builder",
+        structured_output_mode="json_schema",
     )
 
 
@@ -273,6 +274,36 @@ def test_token_case():
         assert token.lower() == token
 
 
+def test_generate_run_description(test_eval_config, test_run_config, test_task_run):
+    """Test that generate_run_description correctly uses task_run.output.output (the string) rather than task_run.output (the object)."""
+    # Create G-Eval instance
+    g_eval = GEval(test_eval_config, test_run_config)
+
+    # Call generate_run_description
+    description = g_eval.generate_run_description(
+        test_task_run.input, test_task_run.output.output
+    )
+
+    # Verify that the actual string output is in the description
+    expected_output = "Why did the chicken cross the road? To get to the other side!"
+    assert expected_output in description
+
+    # Verify that the input is also in the description
+    assert "Tell me a chicken joke" in description
+
+    # Verify the description has the expected structure
+    assert "<eval_data>" in description
+    assert description.count("<eval_data>") == 2  # 2 opening tags
+    assert description.count("</eval_data>") == 2  # 2 closing tags
+    assert "The model was given the following input for the task:" in description
+    assert "The model produced the following output for the task:" in description
+
+    # Verify that we're getting the actual string value, not a Python object representation
+    # The string should not contain 'TaskOutput' or other object indicators
+    assert "TaskOutput" not in description
+    assert "output=" not in description  # Would appear if object __repr__ was used
+
+
 def test_metric_offsets_and_search_ranges(
     test_eval_config, test_run_config, test_task_run
 ):
@@ -400,7 +431,7 @@ def test_rating_token_to_score(test_eval_config, test_run_config):
 
     # Test single token case
     token_logprob = MockTokenLogprob("5", [("5", 0.0)], logprob=1e-8)  # log(1) = 0
-    score = g_eval.rating_token_to_score(token_logprob)
+    score = g_eval.rating_token_to_score(token_logprob)  # type: ignore
     assert score == 5.0
 
     # Test weighted average case
@@ -412,20 +443,62 @@ def test_rating_token_to_score(test_eval_config, test_run_config):
         ],
         logprob=math.log(0.6),
     )
-    score = g_eval.rating_token_to_score(token_logprob)
+    score = g_eval.rating_token_to_score(token_logprob)  # type: ignore
     assert pytest.approx(score) == 4.4  # (4 * 0.6 + 5 * 0.4)
 
     # Test invalid token
     token_logprob = MockTokenLogprob(":", [(":", 0.0)], logprob=1e-8)
-    assert g_eval.rating_token_to_score(token_logprob) is None
+    assert g_eval.rating_token_to_score(token_logprob) is None  # type: ignore
 
     # Test missing from top logprobs
     token_logprob = MockTokenLogprob("5", [], logprob=1e-8)
-    assert pytest.approx(g_eval.rating_token_to_score(token_logprob)) == 5.0
+    assert pytest.approx(g_eval.rating_token_to_score(token_logprob)) == 5.0  # type: ignore
 
     # Test missing from top logprobs, with special case logprob
     token_logprob = MockTokenLogprob("5", [], logprob=-9999)
-    assert pytest.approx(g_eval.rating_token_to_score(token_logprob)) == 5.0
+    assert pytest.approx(g_eval.rating_token_to_score(token_logprob)) == 5.0  # type: ignore
+
+
+def test_rating_token_to_score_zero_score_bug_fix(test_eval_config, test_run_config):
+    """Test that rating_token_to_score correctly handles 0.0 scores (like 'fail') and doesn't return None.
+
+    This test verifies the fix for the bug where 'if not primary_token_score:' would incorrectly
+    treat 0.0 as falsy and return None, when it should only return None for actual None values.
+    """
+    g_eval = GEval(test_eval_config, test_run_config)
+
+    class MockTopLogprob:
+        def __init__(self, token, logprob):
+            self.token = token
+            self.logprob = logprob
+
+    class MockTokenLogprob:
+        def __init__(self, token, top_logprobs, logprob):
+            self.token = token
+            self.top_logprobs = [MockTopLogprob(t, lp) for t, lp in top_logprobs]
+            self.logprob = logprob
+
+    # Test that "fail" token (which maps to 0.0) is handled correctly
+    token_logprob = MockTokenLogprob("fail", [("fail", 0.0)], logprob=1e-8)
+    score = g_eval.rating_token_to_score(token_logprob)  # type: ignore
+    assert score == 0.0, f"Expected 0.0 for 'fail' token, got {score}"
+
+    # Test that "0" token (which maps to None) still returns None
+    token_logprob = MockTokenLogprob("0", [("0", 0.0)], logprob=1e-8)
+    score = g_eval.rating_token_to_score(token_logprob)  # type: ignore
+    assert score is None, f"Expected None for '0' token, got {score}"
+
+    # Test weighted average case with fail token
+    token_logprob = MockTokenLogprob(
+        "fail",
+        [
+            ("fail", math.log(0.7)),  # 70% probability for fail (0.0)
+            ("pass", math.log(0.3)),  # 30% probability for pass (1.0)
+        ],
+        logprob=math.log(0.7),
+    )
+    score = g_eval.rating_token_to_score(token_logprob)  # type: ignore
+    assert pytest.approx(score) == 0.3  # (0.0 * 0.7 + 1.0 * 0.3)
 
 
 def test_g_eval_system_instruction():
@@ -501,3 +574,41 @@ async def test_all_built_in_models_logprobs_geval(
         model_name,
         provider_name.value,
     )
+
+
+def check_supports_llm_as_judge(model_name: str, provider_name: str):
+    for model in built_in_models:
+        if model.name != model_name:
+            continue
+        for provider in model.providers:
+            if provider.name != provider_name:
+                continue
+            if not provider.supports_structured_output:
+                pytest.skip(
+                    f"Skipping {model.name} {provider.name} because it does not support llm_as_judge (structured_output_mode)"
+                )
+            return
+    raise RuntimeError(f"No model {model_name} {provider_name} found")
+
+
+@pytest.mark.paid
+@pytest.mark.ollama
+@pytest.mark.parametrize("model_name,provider_name", get_all_models_and_providers())
+async def test_all_built_in_models_llm_as_judge(
+    model_name,
+    provider_name,
+    test_task,
+    test_eval_config,
+    test_task_run,
+    test_run_config,
+):
+    check_supports_llm_as_judge(model_name, provider_name)
+    await run_g_eval_test(
+        test_task,
+        test_eval_config,
+        test_task_run,
+        EvalConfigType.llm_as_judge,
+        test_run_config,
+        model_name,
+        provider_name.value,
+    )

kiln_ai/adapters/fine_tune/base_finetune.py

@@ -3,14 +3,9 @@ from typing import Literal
 
 from pydantic import BaseModel
 
-from kiln_ai.adapters.ml_model_list import built_in_models
-from kiln_ai.datamodel import (
-    DatasetSplit,
-    FinetuneDataStrategy,
-    FineTuneStatusType,
-    Task,
-)
+from kiln_ai.datamodel import DatasetSplit, FineTuneStatusType, Task
 from kiln_ai.datamodel import Finetune as FinetuneModel
+from kiln_ai.datamodel.datamodel_enums import ChatStrategy
 from kiln_ai.utils.name_generator import generate_memorable_name
 
 
@@ -62,7 +57,7 @@ class BaseFinetuneAdapter(ABC):
         train_split_name: str,
         system_message: str,
         thinking_instructions: str | None,
-        data_strategy: FinetuneDataStrategy,
+        data_strategy: ChatStrategy,
         parameters: dict[str, str | int | float | bool] = {},
         name: str | None = None,
         description: str | None = None,