kiln-ai 0.11.1__py3-none-any.whl → 0.13.0__py3-none-any.whl

kiln_ai/adapters/eval/eval_runner.py

@@ -0,0 +1,270 @@
+import asyncio
+import logging
+from dataclasses import dataclass
+from typing import AsyncGenerator, Dict, List, Literal, Set
+
+from kiln_ai.adapters.eval.base_eval import BaseEval
+from kiln_ai.adapters.eval.registry import eval_adapter_from_type
+from kiln_ai.datamodel.basemodel import ID_TYPE
+from kiln_ai.datamodel.dataset_filters import dataset_filter_from_id
+from kiln_ai.datamodel.eval import EvalConfig, EvalRun, EvalScores
+from kiln_ai.datamodel.task import TaskRunConfig
+from kiln_ai.datamodel.task_run import TaskRun
+
+logger = logging.getLogger(__name__)
+
+
+@dataclass
+class EvalJob:
+    item: TaskRun
+    type: Literal["task_run_eval", "eval_config_eval"]
+    # If type == "task_run_eval", both of these should be set. If type == "eval_config_eval", only eval_config should be set.
+    eval_config: EvalConfig
+    task_run_config: TaskRunConfig | None = None
+
+
+@dataclass
+class EvalProgress:
+    complete: int | None = None
+    total: int | None = None
+    errors: int | None = None
+
+
+class EvalRunner:
+    """
+    Runs an eval. Async execution is supported to make it faster when using remote/fast model providers.
+
+    Can run an eval in 2 modes:
+    1) eval_config_eval: evaluate an eval config using existing dataset items.
+    2) task_run_eval: evaluate a range of task run configs, generating new run output using existing dataset item input.
+    """
+
+    def __init__(
+        self,
+        eval_configs: List[EvalConfig],
+        run_configs: List[TaskRunConfig] | None,
+        eval_run_type: Literal["eval_config_eval", "task_run_eval"],
+    ):
+        if len(eval_configs) == 0:
+            raise ValueError("Eval runner requires at least one eval config")
+        target_eval = eval_configs[0].parent_eval()
+        if target_eval is None:
+            raise ValueError("Eval config requires a parent eval")
+        for eval_config in eval_configs:
+            parent_eval = eval_config.parent_eval()
+            if parent_eval is None:
+                raise ValueError("Eval config requires a parent eval")
+            if parent_eval.id != target_eval.id:
+                raise ValueError("All eval configs must have the same parent eval")
+
+        target_task = target_eval.parent_task()
+        if target_task is None:
+            raise ValueError("Eval config requires a (grand)parent task")
+
+        # Check that run_configs is compatible
+        if eval_run_type == "task_run_eval":
+            if run_configs is None or len(run_configs) == 0:
+                raise ValueError("Task run eval requires run configs")
+            for run_config in run_configs:
+                parent_task = run_config.parent_task()
+                if parent_task is None:
+                    raise ValueError("All run configs must have a parent task")
+                if parent_task.id != target_task.id:
+                    raise ValueError(
+                        "Run config is not for the same task as the eval configs"
+                    )
+        else:
+            if run_configs is not None:
+                raise ValueError("Mode 'eval_config_eval' does not support run configs")
+
+        self.eval_run_type = eval_run_type
+        self.eval_configs = eval_configs
+        self.run_configs = run_configs
+        self.task = target_task
+        self.eval = target_eval
+
+    def collect_tasks(self) -> List[EvalJob]:
+        if self.eval_run_type == "eval_config_eval":
+            return self.collect_tasks_for_eval_config_eval()
+        else:
+            return self.collect_tasks_for_task_run_eval()
+
+    def collect_tasks_for_eval_config_eval(self) -> List[EvalJob]:
+        """
+        Collect all jobs for this run, excluding any that have already been run.
+
+        This variant is used for mode "eval_config_eval", using existing dataset run data (input/output).
+
+        The tasks:
+        - should be in the eval config set filter
+        - should not have already been run for this eval config + dataset item pair
+        """
+        filter = dataset_filter_from_id(self.eval.eval_configs_filter_id)
+
+        # already_run[eval_config_id][dataset_id]
+        already_run: Dict[ID_TYPE, Set[ID_TYPE]] = {}
+        for eval_config in self.eval_configs:
+            already_run[eval_config.id] = set()
+            for run in eval_config.runs(readonly=True):
+                already_run[eval_config.id].add(run.dataset_id)
+
+        return [
+            EvalJob(
+                item=task_run,
+                eval_config=eval_config,
+                type="eval_config_eval",
+            )
+            for task_run in self.task.runs(readonly=True)
+            if filter(task_run)
+            for eval_config in self.eval_configs
+            if task_run.id not in already_run[eval_config.id]
+        ]
+
+    def collect_tasks_for_task_run_eval(self) -> List[EvalJob]:
+        """
+        Collect all jobs for this run, excluding any that have already been run.
+
+        This variant is used for mode "task_run_eval", generating new run output using existing dataset item input.
+
+        The tasks:
+        - should be in the eval set filter
+        - should not have already been run for this eval config + run config + dataset item
+        """
+        filter = dataset_filter_from_id(self.eval.eval_set_filter_id)
+
+        # already_run[eval_config_id][run_config_id][dataset_id]
+        already_run: Dict[ID_TYPE, Dict[ID_TYPE, Set[ID_TYPE]]] = {}
+        for eval_config in self.eval_configs:
+            already_run[eval_config.id] = {}
+            for run_config in self.run_configs or []:
+                already_run[eval_config.id][run_config.id] = set()
+            for run in eval_config.runs(readonly=True):
+                if (
+                    run.task_run_config_id is not None
+                    and run.task_run_config_id in already_run[eval_config.id]
+                ):
+                    already_run[eval_config.id][run.task_run_config_id].add(
+                        run.dataset_id
+                    )
+
+        return [
+            EvalJob(
+                item=task_run,
+                task_run_config=run_config,
+                type="task_run_eval",
+                eval_config=eval_config,
+            )
+            for task_run in self.task.runs(readonly=True)
+            if filter(task_run)
+            for eval_config in self.eval_configs
+            for run_config in self.run_configs or []
+            if task_run.id not in already_run[eval_config.id][run_config.id]
+        ]
+
+    async def run(self, concurrency: int = 25) -> AsyncGenerator[EvalProgress, None]:
+        """
+        Runs the configured eval run with parallel workers and yields progress updates.
+        """
+        jobs = self.collect_tasks()
+
+        complete = 0
+        errors = 0
+        total = len(jobs)
+
+        # Send initial status
+        yield EvalProgress(complete=complete, total=total, errors=errors)
+
+        worker_queue: asyncio.Queue[EvalJob] = asyncio.Queue()
+        for job in jobs:
+            worker_queue.put_nowait(job)
+
+        # simple status queue to return progress. True=success, False=error
+        status_queue: asyncio.Queue[bool] = asyncio.Queue()
+
+        workers = []
+        for i in range(concurrency):
+            task = asyncio.create_task(self.run_worker(worker_queue, status_queue))
+            workers.append(task)
+
+        # Send status updates until workers are done, and they are all sent
+        while not status_queue.empty() or not all(worker.done() for worker in workers):
+            try:
+                # Use timeout to prevent hanging if all workers complete
+                # between our while condition check and get()
+                success = await asyncio.wait_for(status_queue.get(), timeout=0.1)
+                if success:
+                    complete += 1
+                else:
+                    errors += 1
+
+                yield EvalProgress(complete=complete, total=total, errors=errors)
+            except asyncio.TimeoutError:
+                # Timeout is expected, just continue to recheck worker status
+                # Don't love this but beats sentinels for reliability
+                continue
+
+        # These are redundant, but keeping them will catch async errors
+        await asyncio.gather(*workers)
+        await worker_queue.join()
+
+    async def run_worker(
+        self, worker_queue: asyncio.Queue[EvalJob], status_queue: asyncio.Queue[bool]
+    ):
+        while True:
+            try:
+                job = worker_queue.get_nowait()
+            except asyncio.QueueEmpty:
+                # worker can end when the queue is empty
+                break
+            try:
+                success = await self.run_job(job)
+                await status_queue.put(success)
+            finally:
+                # Always mark the dequeued task as done, even on exceptions
+                worker_queue.task_done()
+
+    async def run_job(self, job: EvalJob) -> bool:
+        try:
+            # Create the evaluator for this eval config/run config pair
+            evaluator = eval_adapter_from_type(job.eval_config.config_type)(
+                job.eval_config,
+                job.task_run_config.run_config() if job.task_run_config else None,
+            )
+            if not isinstance(evaluator, BaseEval):
+                raise ValueError("Not able to create evaluator from eval config")
+
+            task_output: str | None = None
+            scores: EvalScores | None = None
+            intermediate_outputs: Dict[str, str] | None = None
+            if job.type == "eval_config_eval":
+                # Eval config eval, we use the saved input from the task run, not invoking the task again
+                scores, intermediate_outputs = await evaluator.run_eval(job.item)
+                task_output = job.item.output.output
+            else:
+                # Task run eval, we invoke the task again to get a fresh output
+                (
+                    result_task_run,
+                    scores,
+                    intermediate_outputs,
+                ) = await evaluator.run_task_and_eval(job.item.input)
+                task_output = result_task_run.output.output
+
+            # Save the job result
+            eval_run = EvalRun(
+                parent=job.eval_config,
+                task_run_config_id=job.task_run_config.id
+                if job.task_run_config
+                else None,
+                dataset_id=job.item.id,
+                eval_config_eval=job.type == "eval_config_eval",
+                scores=scores,
+                input=job.item.input,
+                output=task_output,
+                intermediate_outputs=intermediate_outputs,
+            )
+            eval_run.save_to_file()
+
+            return True
+        except Exception as e:
+            logger.error(f"Error running eval job for dataset item {job.item.id}: {e}")
+            return False

kiln_ai/adapters/eval/g_eval.py

@@ -0,0 +1,368 @@
+import math
+from typing import Dict, List, Tuple
+
+from litellm.types.utils import ChatCompletionTokenLogprob
+
+from kiln_ai.adapters.adapter_registry import adapter_for_task
+from kiln_ai.adapters.eval.base_eval import BaseEval
+from kiln_ai.adapters.model_adapters.base_adapter import AdapterConfig, RunOutput
+from kiln_ai.adapters.prompt_builders import PromptGenerators
+from kiln_ai.datamodel import Project, Task, TaskRun
+from kiln_ai.datamodel.eval import EvalConfig, EvalConfigType, EvalScores
+from kiln_ai.datamodel.task import RunConfig
+
+# all the tokens we score for, and their float scores.
+TOKEN_TO_SCORE_MAP: Dict[str, float] = {
+    "1": 1.0,
+    "2": 2.0,
+    "3": 3.0,
+    "4": 4.0,
+    "5": 5.0,
+    "pass": 1.0,
+    "fail": 0.0,
+    "critical": -1.0,
+}
+
+
+class GEvalTask(Task, parent_of={}):
+    """
+    Kiln task for executing a G-Eval. Can be run on any Kiln adapter which supports logprobs.
+
+    Note G-Eval implements both G-Eval and LLM as Judge as they are very similar.
+    """
+
+    def __init__(self, eval_config: EvalConfig):
+        tmp_project = Project(name="GEval")
+
+        # Build a simple LLM as Judge system instruction
+        system_instruction = "Your job to evaluate a model's performance on a task. Blocks will be marked with <eval_data> tags.\n"
+        # Optionally add a short task description
+        task_description = eval_config.properties.get("task_description", None)
+        if task_description:
+            system_instruction += f"\nThe task the model was given is as follows:\n<eval_data>\n{task_description}\n</eval_data>\n"
+
+        # Build the COT eval instructions
+        cot_instructions = "First, think step by step about the model's performance following these evaluation steps:\n\n"
+        steps = eval_config.properties.get("eval_steps", None)
+        if not steps or not isinstance(steps, list):
+            raise ValueError("eval_steps must be a list")
+        for i, step in enumerate(steps):
+            cot_instructions += f"{i + 1}) {step}\n"
+
+        eval = eval_config.parent_eval()
+        if not eval:
+            raise ValueError("Eval config must have a parent eval")
+
+        # Build the output schema from the eval's target output scores.
+        # We restrict the LLM's output scoring schema to discrete scores (pass/fail/critical/1-5) - allow_float_scores=False
+        # However, the final scores from the evaluator can be a float (see later logprob calculation, which requires discrete token outputs)
+        output_schema = BaseEval.build_score_schema(eval, allow_float_scores=False)
+
+        super().__init__(
+            name="GEval Task",
+            parent=tmp_project,
+            instruction=system_instruction,
+            thinking_instruction=cot_instructions,
+            output_json_schema=output_schema,
+        )
+
+
+class GEval(BaseEval):
+    """
+    A evaluator which implements G-Eval and LLM as Judge.
+
+    G-Eval is a method of evaluating the quality of a model's output. It is a weighted average of the scores of the tokens in the output. The weights are the log probabilities of the tokens in the output. https://arxiv.org/abs/2303.16634
+
+    LLM as Judge is a method of evaluating the quality of a model's output. It simply asks the LLM to score, and uses the returned output (no logprobs needed). Also called direct evaluation.
+
+    @misc{liu2023gevalnlgevaluationusing,
+        title={G-Eval: NLG Evaluation using GPT-4 with Better Human Alignment},
+        author={Yang Liu and Dan Iter and Yichong Xu and Shuohang Wang and Ruochen Xu and Chenguang Zhu},
+        year={2023},
+        eprint={2303.16634},
+        archivePrefix={arXiv},
+        primaryClass={cs.CL},
+        url={https://arxiv.org/abs/2303.16634},
+    }
+    """
+
+    def __init__(self, eval_config: EvalConfig, run_config: RunConfig | None):
+        if (
+            eval_config.config_type != EvalConfigType.g_eval
+            and eval_config.config_type != EvalConfigType.llm_as_judge
+        ):
+            raise ValueError(
+                f"GEval must be initialized with a GEval or LLM as Judge config_type. Got {eval_config.config_type}"
+            )
+
+        super().__init__(eval_config, run_config)
+
+        self.geval_task = GEvalTask(eval_config)
+
+    async def run_eval(
+        self, task_run: TaskRun
+    ) -> tuple[EvalScores, Dict[str, str] | None]:
+        """
+        Run this eval on the given task run.
+        """
+
+        model_name, provider = self.model_and_provider()
+
+        # Only fetch logprobs for G-Eval
+        # There are at most 5 valid rating tokens per rating type (five_star being largest), so 10 is more than enough to get to the very very unlikely
+        top_logprobs = (
+            10 if self.eval_config.config_type == EvalConfigType.g_eval else None
+        )
+
+        adapter = adapter_for_task(
+            self.geval_task,
+            model_name,
+            provider,
+            # We always use Simple COT for G-Eval and LLM as Judge
+            prompt_id=PromptGenerators.SIMPLE_CHAIN_OF_THOUGHT,
+            base_adapter_config=AdapterConfig(
+                # Don't save this run into the task_runs. It will be saved into an eval_run where it belongs
+                allow_saving=False,
+                top_logprobs=top_logprobs,
+            ),
+        )
+
+        input = f"""The model was given the following input for the task: 
+<eval_data>
+{task_run.input}
+</eval_data>
+
+The model produced the following output for the task:
+<eval_data>
+{task_run.output}
+</eval_data>
+"""
+
+        # We don't need the run, but invoke_returning_run_output() runs validations for us over _run()
+        _, run_output = await adapter.invoke_returning_run_output(input)
+
+        if self.eval_config.config_type == EvalConfigType.llm_as_judge:
+            return self.build_llm_as_judge_score(
+                run_output
+            ), run_output.intermediate_outputs
+        else:
+            return self.build_g_eval_score(run_output), run_output.intermediate_outputs
+
+    def build_llm_as_judge_score(self, run_output: RunOutput) -> EvalScores:
+        """
+        Build the LLM as Judge score for the given run and run output.
+        """
+        # Convert the output format we asked for (discreet values) to our float scores
+        scores: EvalScores = {}
+        if not isinstance(run_output.output, dict):
+            raise ValueError("LLM as Judge output must be a dictionary")
+
+        for metric, score in run_output.output.items():
+            token_score = self.score_from_token_string(f"{score}")
+            if token_score is None:
+                raise ValueError(
+                    f"No score found for metric: {metric}. The LLM failed to follow the scoring rubric/instructions/schema."
+                )
+            scores[metric] = token_score
+        return scores
+
+    def build_g_eval_score(self, run_output: RunOutput) -> EvalScores:
+        """
+        Build the G-Eval score for the given run and run output.
+
+        We create a weighted average of each rating using the logprobs.
+
+        @misc{liu2023gevalnlgevaluationusing,
+            title={G-Eval: NLG Evaluation using GPT-4 with Better Human Alignment},
+            author={Yang Liu and Dan Iter and Yichong Xu and Shuohang Wang and Ruochen Xu and Chenguang Zhu},
+            year={2023},
+            eprint={2303.16634},
+            archivePrefix={arXiv},
+            primaryClass={cs.CL},
+            url={https://arxiv.org/abs/2303.16634},
+        }
+        """
+        # We use structured output
+        outputs = run_output.output
+        assert isinstance(outputs, dict)
+
+        # Build raw string output from the logprobs, which is easier to work with than Dict for the next bit
+        raw_output = self.raw_output_from_logprobs(run_output)
+
+        # find the offset the start of each metric in the raw output json
+        metrics: List[str] = list(outputs.keys())
+        metric_offsets = self.metric_offsets(raw_output, metrics)
+
+        final_scores: EvalScores = {}
+        for metric in metrics:
+            score = self.g_eval_single_metric(
+                run_output, metric, metric_offsets, raw_output
+            )
+            if score is None:
+                raise ValueError(
+                    f"No score found for metric: {metric}. The LLM failed to follow the scoring rubric/instructions/schema."
+                )
+            final_scores[metric] = score
+
+        return final_scores
+
+    def g_eval_single_metric(
+        self,
+        run_output: RunOutput,
+        metric: str,
+        metric_offsets: Dict[str, int],
+        raw_output: str,
+    ) -> float | None:
+        """
+        Run the G-Eval for a single metric.
+
+        Scan the logprobs for the metric and return the weighted score of the rating token.
+        """
+
+        start_offset, end_offset = self.token_search_range(
+            raw_output, metric, metric_offsets
+        )
+
+        offset = 0
+
+        if (
+            run_output.output_logprobs is None
+            or run_output.output_logprobs.content is None
+        ):
+            raise RuntimeError(
+                "No logprobs found for output - can not calculate g-eval"
+            )
+
+        # scan the tokens in the range, looking for the rating token
+        for _, chat_logprob in enumerate(run_output.output_logprobs.content):
+            if offset >= end_offset:
+                break
+            if offset >= start_offset:
+                score = self.rating_token_to_score(chat_logprob)
+                if score is not None:
+                    return score
+            offset += len(chat_logprob.token)
+
+        return None
+
+    def raw_output_from_logprobs(self, run_output: RunOutput) -> str:
+        """
+        Build the raw output string from the logprobs. Generate from logprobs so it's guaranteed to match the logprobs offsets
+        """
+        if (
+            run_output.output_logprobs is None
+            or run_output.output_logprobs.content is None
+        ):
+            raise RuntimeError(
+                "No logprobs found for output - can not calculate g-eval"
+            )
+
+        raw = ""
+        for chat_logprob in run_output.output_logprobs.content:
+            raw += chat_logprob.token
+        return raw
+
+    def token_search_range(
+        self, raw_output: str, metric: str, metric_offsets: Dict[str, int]
+    ) -> Tuple[int, int]:
+        """
+        Find the start and end offsets of the metric in the raw output.
+
+        Start searching after the end of the target metric json entry ("overall_rating":), and before the start of the next metric ("some_other_score").
+        """
+        start_offset = metric_offsets[metric] + len(metric)
+
+        # Find the lowest end offset that is greater than the start offset
+        end_offset = len(raw_output)
+        for v in list(metric_offsets.values()):
+            if v < end_offset and v > start_offset:
+                end_offset = v
+
+        return start_offset, end_offset
+
+    def rating_token_to_score(
+        self, token_logprob: ChatCompletionTokenLogprob
+    ) -> float | None:
+        """
+        Convert a rating token to a score using weighted average of top logprobs.
+
+        Only includes tokens that have valid scores.
+
+        Some cleanup for upper case, whitespace and quotes. LLMs aren't always consistent.
+        """
+        primary_token_score = self.score_from_token_string(token_logprob.token)
+        # check this is a real rating token, it could just be the ": ", "," or whitespace
+        if not primary_token_score:
+            return None
+
+        total_score = 0.0
+        total_probability = 0.0
+
+        # Process all valid scoring tokens
+        for top_logprob in token_logprob.top_logprobs:
+            token_score = self.score_from_token_string(top_logprob.token)
+            if token_score is not None:
+                # Convert logprob to probability
+                probability = math.exp(top_logprob.logprob)
+                total_score += token_score * probability
+                total_probability += probability
+
+        if total_probability <= 0.0:
+            raise RuntimeError(
+                f"No valid scoring tokens found for {token_logprob.token}. This should never happen. Please file a bug if you see this."
+            )
+
+        # Normalize by total probability of valid tokens (LLM may have wanted to generate other non-rating tokens, these shouldn't lower score of rating tokens)
+        weighted_score = total_score / total_probability
+
+        return weighted_score
+
+    def score_from_token_string(self, token: str) -> float | None:
+        if token in TOKEN_TO_SCORE_MAP:
+            return TOKEN_TO_SCORE_MAP[token]
+
+        # handle more token variations like '"1"' and '"pass"' and ' paSS' and 'PASS'
+        unquoted_token = token.strip().strip('"').lower()
+        if unquoted_token in TOKEN_TO_SCORE_MAP:
+            return TOKEN_TO_SCORE_MAP[unquoted_token]
+
+        # handle numeric tokens like "1.0"
+        try:
+            float_value = float(token)
+            if float_value.is_integer():
+                str_token = str(int(float_value))
+                if str_token in TOKEN_TO_SCORE_MAP:
+                    return TOKEN_TO_SCORE_MAP[str_token]
+        except ValueError:
+            pass
+
+        return None
+
+    def metric_offsets(self, raw_output: str, metrics: List[str]) -> Dict[str, int]:
+        """
+        Find the offset to the start of each metric in the raw output json
+
+        For the example json: `{"overall_rating": 1}` == 1
+
+        should return:
+        {
+            "overall_rating": 1 # it's 1 character into the json string
+        }
+        """
+        metric_offsets: Dict[str, int] = {}
+        for metric in metrics:
+            # the quoted metric name is expected in the json: `{"overall_rating": 1}` == 1
+            metric_name = f'"{metric}"'
+
+            # we expect it exactly once
+            count = raw_output.count(metric_name)
+            if count != 1:
+                raise ValueError(
+                    f"Metric {metric} should appear exactly once in the output. Found {count} times"
+                )
+
+            offset = raw_output.find(metric_name)
+            if offset == -1:
+                raise ValueError(f"Metric {metric} not found in raw output")
+            metric_offsets[metric] = offset
+        return metric_offsets

kiln_ai/adapters/eval/registry.py

@@ -0,0 +1,16 @@
+from kiln_ai.adapters.eval.base_eval import BaseEval
+from kiln_ai.adapters.eval.g_eval import GEval
+from kiln_ai.datamodel.eval import EvalConfigType
+from kiln_ai.utils.exhaustive_error import raise_exhaustive_enum_error
+
+
+def eval_adapter_from_type(eval_config_type: EvalConfigType) -> type[BaseEval]:
+    match eval_config_type:
+        case EvalConfigType.g_eval:
+            return GEval
+        case EvalConfigType.llm_as_judge:
+            # Also implemented by GEval
+            return GEval
+        case _:
+            # type checking will catch missing cases
+            raise_exhaustive_enum_error(eval_config_type)