kiln-ai 0.15.0__py3-none-any.whl → 0.17.0__py3-none-any.whl

kiln_ai/adapters/eval/eval_runner.py

@@ -1,4 +1,3 @@
-import asyncio
 import logging
 from dataclasses import dataclass
 from typing import AsyncGenerator, Dict, List, Literal, Set
@@ -10,6 +9,7 @@ from kiln_ai.datamodel.dataset_filters import dataset_filter_from_id
 from kiln_ai.datamodel.eval import EvalConfig, EvalRun, EvalScores
 from kiln_ai.datamodel.task import TaskRunConfig
 from kiln_ai.datamodel.task_run import TaskRun
+from kiln_ai.utils.async_job_runner import AsyncJobRunner, Progress
 
 logger = logging.getLogger(__name__)
 
@@ -23,13 +23,6 @@ class EvalJob:
     task_run_config: TaskRunConfig | None = None
 
 
-@dataclass
-class EvalProgress:
-    complete: int | None = None
-    total: int | None = None
-    errors: int | None = None
-
-
 class EvalRunner:
     """
     Runs an eval. Async execution is supported to make it faster when using remote/fast model providers.
@@ -161,67 +154,15 @@ class EvalRunner:
             if task_run.id not in already_run[eval_config.id][run_config.id]
         ]
 
-    async def run(self, concurrency: int = 25) -> AsyncGenerator[EvalProgress, None]:
+    async def run(self, concurrency: int = 25) -> AsyncGenerator[Progress, None]:
         """
         Runs the configured eval run with parallel workers and yields progress updates.
         """
         jobs = self.collect_tasks()
 
-        complete = 0
-        errors = 0
-        total = len(jobs)
-
-        # Send initial status
-        yield EvalProgress(complete=complete, total=total, errors=errors)
-
-        worker_queue: asyncio.Queue[EvalJob] = asyncio.Queue()
-        for job in jobs:
-            worker_queue.put_nowait(job)
-
-        # simple status queue to return progress. True=success, False=error
-        status_queue: asyncio.Queue[bool] = asyncio.Queue()
-
-        workers = []
-        for i in range(concurrency):
-            task = asyncio.create_task(self.run_worker(worker_queue, status_queue))
-            workers.append(task)
-
-        # Send status updates until workers are done, and they are all sent
-        while not status_queue.empty() or not all(worker.done() for worker in workers):
-            try:
-                # Use timeout to prevent hanging if all workers complete
-                # between our while condition check and get()
-                success = await asyncio.wait_for(status_queue.get(), timeout=0.1)
-                if success:
-                    complete += 1
-                else:
-                    errors += 1
-
-                yield EvalProgress(complete=complete, total=total, errors=errors)
-            except asyncio.TimeoutError:
-                # Timeout is expected, just continue to recheck worker status
-                # Don't love this but beats sentinels for reliability
-                continue
-
-        # These are redundant, but keeping them will catch async errors
-        await asyncio.gather(*workers)
-        await worker_queue.join()
-
-    async def run_worker(
-        self, worker_queue: asyncio.Queue[EvalJob], status_queue: asyncio.Queue[bool]
-    ):
-        while True:
-            try:
-                job = worker_queue.get_nowait()
-            except asyncio.QueueEmpty:
-                # worker can end when the queue is empty
-                break
-            try:
-                success = await self.run_job(job)
-                await status_queue.put(success)
-            finally:
-                # Always mark the dequeued task as done, even on exceptions
-                worker_queue.task_done()
+        runner = AsyncJobRunner(concurrency=concurrency)
+        async for progress in runner.run(jobs, self.run_job):
+            yield progress
 
     async def run_job(self, job: EvalJob) -> bool:
         try:
@@ -266,5 +207,8 @@ class EvalRunner:
 
             return True
         except Exception as e:
-            logger.error(f"Error running eval job for dataset item {job.item.id}: {e}")
+            logger.error(
+                f"Error running eval job for dataset item {job.item.id}: {e}",
+                exc_info=True,
+            )
             return False

kiln_ai/adapters/eval/g_eval.py

@@ -5,11 +5,14 @@ from litellm.types.utils import ChatCompletionTokenLogprob
 
 from kiln_ai.adapters.adapter_registry import adapter_for_task
 from kiln_ai.adapters.eval.base_eval import BaseEval
+from kiln_ai.adapters.ml_model_list import (
+    default_structured_output_mode_for_model_provider,
+)
 from kiln_ai.adapters.model_adapters.base_adapter import AdapterConfig, RunOutput
 from kiln_ai.adapters.prompt_builders import PromptGenerators
 from kiln_ai.datamodel import Project, Task, TaskRun
 from kiln_ai.datamodel.eval import EvalConfig, EvalConfigType, EvalScores
-from kiln_ai.datamodel.task import RunConfig
+from kiln_ai.datamodel.task import RunConfig, RunConfigProperties, StructuredOutputMode
 
 # all the tokens we score for, and their float scores.
 TOKEN_TO_SCORE_MAP: Dict[str, float] = {
@@ -43,9 +46,9 @@ class GEvalTask(Task, parent_of={}):
 
         # Build the COT eval instructions
         cot_instructions = "First, think step by step about the model's performance following these evaluation steps:\n\n"
-        steps = eval_config.properties.get("eval_steps", None)
-        if not steps or not isinstance(steps, list):
-            raise ValueError("eval_steps must be a list")
+        steps = eval_config.properties.get("eval_steps", [])
+        if not isinstance(steps, list):
+            raise ValueError("eval_steps must be a list.")
         for i, step in enumerate(steps):
             cot_instructions += f"{i + 1}) {step}\n"
 
@@ -114,12 +117,27 @@ class GEval(BaseEval):
             10 if self.eval_config.config_type == EvalConfigType.g_eval else None
         )
 
-        adapter = adapter_for_task(
-            self.geval_task,
+        # We don't expose setting this manually in the UI, so pull a recommended mode from ml_model_list
+        structured_output_mode = default_structured_output_mode_for_model_provider(
             model_name,
             provider,
-            # We always use Simple COT for G-Eval and LLM as Judge
-            prompt_id=PromptGenerators.SIMPLE_CHAIN_OF_THOUGHT,
+            default=StructuredOutputMode.json_schema,
+            # G-eval expects JSON, so don't allow function calling modes
+            disallowed_modes=[
+                StructuredOutputMode.function_calling,
+                StructuredOutputMode.function_calling_weak,
+            ],
+        )
+
+        adapter = adapter_for_task(
+            self.geval_task,
+            run_config_properties=RunConfigProperties(
+                model_name=model_name,
+                model_provider_name=provider,
+                # We always use Simple COT for G-Eval and LLM as Judge
+                prompt_id=PromptGenerators.SIMPLE_CHAIN_OF_THOUGHT,
+                structured_output_mode=structured_output_mode,
+            ),
             base_adapter_config=AdapterConfig(
                 # Don't save this run into the task_runs. It will be saved into an eval_run where it belongs
                 allow_saving=False,

kiln_ai/adapters/eval/test_base_eval.py

@@ -1,9 +1,9 @@
 import json
+from unittest.mock import AsyncMock, MagicMock, patch
 
 import pytest
 
 from kiln_ai.adapters.eval.base_eval import BaseEval
-from kiln_ai.datamodel import BasePrompt, DataSource, DataSourceType
 from kiln_ai.datamodel.eval import Eval, EvalConfig, EvalOutputScore
 from kiln_ai.datamodel.task import (
     RunConfigProperties,
@@ -245,7 +245,7 @@ class EvalTester(BaseEval):
     """Test implementation of BaseEval"""
 
     async def run_eval(self, task_run):
-        return {"overall_rating": 5, "quality": 4}
+        return {"overall_rating": 5, "quality": 4}, None
 
 
 @pytest.mark.paid
@@ -265,14 +265,8 @@ async def test_run_method():
 
     eval_config = EvalConfig(
         name="Test Eval Config",
-        model=DataSource(
-            type=DataSourceType.synthetic,
-            properties={
-                "model_name": "gpt-4o",
-                "model_provider": "openai",
-                "adapter_name": "test",
-            },
-        ),
+        model_name="gpt-4o",
+        model_provider="openai",
         parent=Eval(
             name="Test Eval",
             parent=task,
@@ -291,10 +285,6 @@ async def test_run_method():
                 ),
             ],
         ),
-        prompt=BasePrompt(
-            name="Test Prompt",
-            prompt="Test prompt",
-        ),
         properties={"eval_steps": ["test_step"]},
     )
 
@@ -311,7 +301,9 @@ async def test_run_method():
     evaluator = EvalTester(eval_config, run_config.run_config())
 
     # Run the evaluation
-    task_run, eval_scores = await evaluator.run("test input")
+    task_run, eval_scores, intermediate_outputs = await evaluator.run_task_and_eval(
+        "test input"
+    )
 
     # Verify task run was created
     assert task_run.input == "test input"
@@ -323,3 +315,162 @@ async def test_run_method():
 
     # Verify schema validation worked (these keys should exist per schema)
     assert set(eval_scores.keys()) == {"overall_rating", "quality"}
+
+
+@pytest.mark.asyncio
+async def test_run_task_and_eval():
+    """Test run_task_and_eval method with mocked dependencies"""
+    # Create test data
+    task = Task(
+        name="Test Task",
+        instruction="Test instruction",
+        requirements=[
+            TaskRequirement(
+                name="Quality",
+                instruction="Rate quality",
+                type=TaskOutputRatingType.five_star,
+            ),
+        ],
+    )
+
+    eval_config = EvalConfig(
+        name="Test Eval Config",
+        model_name="gpt-4o",
+        model_provider="openai",
+        parent=Eval(
+            name="Test Eval",
+            parent=task,
+            eval_set_filter_id="all",
+            eval_configs_filter_id="all",
+            output_scores=[
+                EvalOutputScore(
+                    name="Quality",
+                    instruction="Rate quality",
+                    type=TaskOutputRatingType.five_star,
+                ),
+                EvalOutputScore(
+                    name="Overall Rating",
+                    instruction="The overall rating for the task output",
+                    type=TaskOutputRatingType.five_star,
+                ),
+            ],
+        ),
+        properties={"eval_steps": ["test_step"]},
+    )
+
+    run_config = TaskRunConfig(
+        name="Test Run Config",
+        run_config_properties=RunConfigProperties(
+            model_name="llama_3_1_8b",
+            model_provider_name="groq",
+            prompt_id="simple_prompt_builder",
+            structured_output_mode="json_schema",
+        ),
+        parent=task,
+    )
+
+    # Create evaluator instance
+    class MockEval(BaseEval):
+        async def run_eval(self, task_run):
+            return {"overall_rating": 5, "quality": 4}, {"thinking": "test thinking"}
+
+    evaluator = MockEval(eval_config, run_config.run_config())
+
+    # Mock dependencies
+    mock_adapter = AsyncMock()
+    mock_task_run = MagicMock()
+    mock_task_run.input = "test input"
+    mock_task_run.output.output = "test output"
+    mock_adapter.invoke.return_value = mock_task_run
+
+    with (
+        patch(
+            "kiln_ai.adapters.eval.base_eval.adapter_for_task"
+        ) as mock_adapter_for_task,
+        patch(
+            "kiln_ai.adapters.eval.base_eval.validate_schema_with_value_error"
+        ) as mock_validate,
+    ):
+        mock_adapter_for_task.return_value = mock_adapter
+
+        # Test with string input
+        result = await evaluator.run_task_and_eval("test input")
+
+        # Verify adapter_for_task was called with correct parameters
+        mock_adapter_for_task.assert_called_once()
+        assert mock_adapter_for_task.call_args[0][0] == evaluator.target_task
+        props = mock_adapter_for_task.call_args[0][1]
+        assert props.model_name == "llama_3_1_8b"
+        assert props.model_provider_name == "groq"
+        assert props.prompt_id == "simple_prompt_builder"
+        bac = mock_adapter_for_task.call_args[1]
+        assert bac["base_adapter_config"].allow_saving is False
+
+        # Verify the base_adapter_config has allow_saving=False
+        adapter_config = mock_adapter_for_task.call_args[1]["base_adapter_config"]
+        assert adapter_config.allow_saving is False
+
+        # Verify adapter.invoke was called with correct input
+        mock_adapter.invoke.assert_called_once_with("test input")
+
+        # Verify validate_schema_with_value_error was called
+        mock_validate.assert_called_once_with(
+            {"overall_rating": 5, "quality": 4},
+            evaluator.score_schema,
+            "Eval output does not match score schema.",
+        )
+
+        # Verify return values
+        task_run, eval_scores, intermediate_outputs = result
+        assert task_run == mock_task_run
+        assert eval_scores == {"overall_rating": 5, "quality": 4}
+        assert intermediate_outputs == {"thinking": "test thinking"}
+
+
+@pytest.mark.asyncio
+async def test_run_task_and_eval_no_run_config():
+    """Test run_task_and_eval raises error when run_config is None"""
+    task = Task(
+        name="Test Task",
+        instruction="Test instruction",
+        requirements=[
+            TaskRequirement(
+                name="Quality",
+                instruction="Rate quality",
+                type=TaskOutputRatingType.five_star,
+            ),
+        ],
+    )
+
+    eval_config = EvalConfig(
+        name="Test Eval Config",
+        model_name="gpt-4o",
+        model_provider="openai",
+        parent=Eval(
+            name="Test Eval",
+            parent=task,
+            eval_set_filter_id="all",
+            eval_configs_filter_id="all",
+            output_scores=[
+                EvalOutputScore(
+                    name="Quality",
+                    instruction="Rate quality",
+                    type=TaskOutputRatingType.five_star,
+                ),
+            ],
+        ),
+        properties={"eval_steps": ["test_step"]},
+    )
+
+    # Create evaluator instance with no run_config
+    class MockEval(BaseEval):
+        async def run_eval(self, task_run):
+            return {"quality": 4}, None
+
+    evaluator = MockEval(eval_config, None)
+
+    # Test that it raises ValueError
+    with pytest.raises(
+        ValueError, match="Run config is required for run_task_and_eval"
+    ):
+        await evaluator.run_task_and_eval("test input")

kiln_ai/adapters/eval/test_eval_runner.py

@@ -94,6 +94,7 @@ def mock_run_config(
             model_name="gpt-4",
             model_provider_name="openai",
             prompt_id="simple_prompt_builder",
+            structured_output_mode="json_schema",
         ),
         parent=mock_task,
     )
@@ -209,6 +210,7 @@ def test_collect_tasks_filtering(
             model_name="gpt-4",
             model_provider_name="openai",
             prompt_id="simple_prompt_builder",
+            structured_output_mode="json_schema",
         ),
         parent=mock_task,
     )
@@ -416,6 +418,7 @@ def test_collect_tasks_multiple_run_configs(
             model_name="gpt-3.5",
             model_provider_name="openai",
             prompt_id="simple_prompt_builder",
+            structured_output_mode="json_schema",
         ),
         parent=mock_task,
     )

kiln_ai/adapters/eval/test_g_eval.py

@@ -99,6 +99,7 @@ def test_run_config(test_task):
         model_provider_name="groq",
         task=test_task,
         prompt_id="simple_prompt_builder",
+        structured_output_mode="json_schema",
     )
 
 

kiln_ai/adapters/fine_tune/base_finetune.py

@@ -6,11 +6,11 @@ from pydantic import BaseModel
 from kiln_ai.adapters.ml_model_list import built_in_models
 from kiln_ai.datamodel import (
     DatasetSplit,
-    FinetuneDataStrategy,
     FineTuneStatusType,
     Task,
 )
 from kiln_ai.datamodel import Finetune as FinetuneModel
+from kiln_ai.datamodel.datamodel_enums import ChatStrategy
 from kiln_ai.utils.name_generator import generate_memorable_name
 
 
@@ -62,7 +62,7 @@ class BaseFinetuneAdapter(ABC):
         train_split_name: str,
         system_message: str,
         thinking_instructions: str | None,
-        data_strategy: FinetuneDataStrategy,
+        data_strategy: ChatStrategy,
         parameters: dict[str, str | int | float | bool] = {},
         name: str | None = None,
         description: str | None = None,