kiln-ai 0.11.1__py3-none-any.whl → 0.13.0__py3-none-any.whl

kiln_ai/adapters/__init__.py

@@ -12,10 +12,13 @@ The prompt_builders submodule contains classes that build prompts for use with t
 The repair submodule contains an adapter for the repair task.
 
 The parser submodule contains parsers for the output of the AI models.
+
+The eval submodule contains the code for evaluating the performance of a model.
 """
 
 from . import (
     data_gen,
+    eval,
     fine_tune,
     ml_model_list,
     model_adapters,
@@ -30,4 +33,5 @@ __all__ = [
     "ml_model_list",
     "prompt_builders",
     "repair",
+    "eval",
 ]

kiln_ai/adapters/adapter_registry.py

@@ -2,14 +2,13 @@ from os import getenv
 
 from kiln_ai import datamodel
 from kiln_ai.adapters.ml_model_list import ModelProviderName
-from kiln_ai.adapters.model_adapters.base_adapter import BaseAdapter
-from kiln_ai.adapters.model_adapters.langchain_adapters import LangchainAdapter
-from kiln_ai.adapters.model_adapters.openai_model_adapter import (
-    OpenAICompatibleAdapter,
-    OpenAICompatibleConfig,
+from kiln_ai.adapters.model_adapters.base_adapter import AdapterConfig, BaseAdapter
+from kiln_ai.adapters.model_adapters.litellm_adapter import (
+    LiteLlmAdapter,
+    LiteLlmConfig,
 )
-from kiln_ai.adapters.prompt_builders import BasePromptBuilder
-from kiln_ai.adapters.provider_tools import core_provider, openai_compatible_config
+from kiln_ai.adapters.provider_tools import core_provider, lite_llm_config
+from kiln_ai.datamodel import PromptId
 from kiln_ai.utils.config import Config
 from kiln_ai.utils.exhaustive_error import raise_exhaustive_enum_error
 
@@ -18,59 +17,193 @@ def adapter_for_task(
     kiln_task: datamodel.Task,
     model_name: str,
     provider: ModelProviderName,
-    prompt_builder: BasePromptBuilder | None = None,
-    tags: list[str] | None = None,
+    prompt_id: PromptId | None = None,
+    base_adapter_config: AdapterConfig | None = None,
 ) -> BaseAdapter:
     # Get the provider to run. For things like the fine-tune provider, we want to run the underlying provider
     core_provider_name = core_provider(model_name, provider)
 
     match core_provider_name:
         case ModelProviderName.openrouter:
-            return OpenAICompatibleAdapter(
+            return LiteLlmAdapter(
                 kiln_task=kiln_task,
-                config=OpenAICompatibleConfig(
+                config=LiteLlmConfig(
+                    model_name=model_name,
                     base_url=getenv("OPENROUTER_BASE_URL")
                     or "https://openrouter.ai/api/v1",
-                    api_key=Config.shared().open_router_api_key,
-                    model_name=model_name,
                     provider_name=provider,
-                    openrouter_style_reasoning=True,
                     default_headers={
                         "HTTP-Referer": "https://getkiln.ai/openrouter",
                         "X-Title": "KilnAI",
                     },
+                    additional_body_options={
+                        "api_key": Config.shared().open_router_api_key,
+                    },
                 ),
-                prompt_builder=prompt_builder,
-                tags=tags,
+                prompt_id=prompt_id,
+                base_adapter_config=base_adapter_config,
             )
         case ModelProviderName.openai:
-            return OpenAICompatibleAdapter(
+            return LiteLlmAdapter(
                 kiln_task=kiln_task,
-                config=OpenAICompatibleConfig(
-                    api_key=Config.shared().open_ai_api_key,
+                config=LiteLlmConfig(
                     model_name=model_name,
                     provider_name=provider,
+                    additional_body_options={
+                        "api_key": Config.shared().open_ai_api_key,
+                    },
                 ),
-                prompt_builder=prompt_builder,
-                tags=tags,
+                prompt_id=prompt_id,
+                base_adapter_config=base_adapter_config,
             )
         case ModelProviderName.openai_compatible:
-            config = openai_compatible_config(model_name)
-            return OpenAICompatibleAdapter(
+            config = lite_llm_config(model_name)
+            return LiteLlmAdapter(
                 kiln_task=kiln_task,
                 config=config,
-                prompt_builder=prompt_builder,
-                tags=tags,
+                prompt_id=prompt_id,
+                base_adapter_config=base_adapter_config,
             )
-        # Use LangchainAdapter for the rest
         case ModelProviderName.groq:
-            pass
+            return LiteLlmAdapter(
+                kiln_task=kiln_task,
+                prompt_id=prompt_id,
+                base_adapter_config=base_adapter_config,
+                config=LiteLlmConfig(
+                    model_name=model_name,
+                    provider_name=provider,
+                    additional_body_options={
+                        "api_key": Config.shared().groq_api_key,
+                    },
+                ),
+            )
         case ModelProviderName.amazon_bedrock:
-            pass
+            return LiteLlmAdapter(
+                kiln_task=kiln_task,
+                prompt_id=prompt_id,
+                base_adapter_config=base_adapter_config,
+                config=LiteLlmConfig(
+                    model_name=model_name,
+                    provider_name=provider,
+                    additional_body_options={
+                        "aws_access_key_id": Config.shared().bedrock_access_key,
+                        "aws_secret_access_key": Config.shared().bedrock_secret_key,
+                        # The only region that's widely supported for bedrock
+                        "aws_region_name": "us-west-2",
+                    },
+                ),
+            )
         case ModelProviderName.ollama:
-            pass
+            ollama_base_url = (
+                Config.shared().ollama_base_url or "http://localhost:11434"
+            )
+            return LiteLlmAdapter(
+                kiln_task=kiln_task,
+                prompt_id=prompt_id,
+                base_adapter_config=base_adapter_config,
+                config=LiteLlmConfig(
+                    model_name=model_name,
+                    provider_name=provider,
+                    # Set the Ollama base URL for 2 reasons:
+                    # 1. To use the correct base URL
+                    # 2. We use Ollama's OpenAI compatible API (/v1), and don't just let litellm use the Ollama API. We use more advanced features like json_schema.
+                    base_url=ollama_base_url + "/v1",
+                ),
+            )
         case ModelProviderName.fireworks_ai:
-            pass
+            return LiteLlmAdapter(
+                kiln_task=kiln_task,
+                prompt_id=prompt_id,
+                base_adapter_config=base_adapter_config,
+                config=LiteLlmConfig(
+                    model_name=model_name,
+                    provider_name=provider,
+                    additional_body_options={
+                        "api_key": Config.shared().fireworks_api_key,
+                    },
+                ),
+            )
+        case ModelProviderName.anthropic:
+            return LiteLlmAdapter(
+                kiln_task=kiln_task,
+                prompt_id=prompt_id,
+                base_adapter_config=base_adapter_config,
+                config=LiteLlmConfig(
+                    model_name=model_name,
+                    provider_name=provider,
+                    additional_body_options={
+                        "api_key": Config.shared().anthropic_api_key,
+                    },
+                ),
+            )
+        case ModelProviderName.gemini_api:
+            return LiteLlmAdapter(
+                kiln_task=kiln_task,
+                prompt_id=prompt_id,
+                base_adapter_config=base_adapter_config,
+                config=LiteLlmConfig(
+                    model_name=model_name,
+                    provider_name=provider,
+                    additional_body_options={
+                        "api_key": Config.shared().gemini_api_key,
+                    },
+                ),
+            )
+        case ModelProviderName.vertex:
+            return LiteLlmAdapter(
+                kiln_task=kiln_task,
+                prompt_id=prompt_id,
+                base_adapter_config=base_adapter_config,
+                config=LiteLlmConfig(
+                    model_name=model_name,
+                    provider_name=provider,
+                    additional_body_options={
+                        "vertex_project": Config.shared().vertex_project_id,
+                        "vertex_location": Config.shared().vertex_location,
+                    },
+                ),
+            )
+        case ModelProviderName.together_ai:
+            return LiteLlmAdapter(
+                kiln_task=kiln_task,
+                prompt_id=prompt_id,
+                base_adapter_config=base_adapter_config,
+                config=LiteLlmConfig(
+                    model_name=model_name,
+                    provider_name=provider,
+                    additional_body_options={
+                        "api_key": Config.shared().together_api_key,
+                    },
+                ),
+            )
+        case ModelProviderName.azure_openai:
+            return LiteLlmAdapter(
+                kiln_task=kiln_task,
+                prompt_id=prompt_id,
+                base_adapter_config=base_adapter_config,
+                config=LiteLlmConfig(
+                    base_url=Config.shared().azure_openai_endpoint,
+                    model_name=model_name,
+                    provider_name=provider,
+                    additional_body_options={
+                        "api_key": Config.shared().azure_openai_api_key,
+                        "api_version": "2025-02-01-preview",
+                    },
+                ),
+            )
+        case ModelProviderName.huggingface:
+            return LiteLlmAdapter(
+                kiln_task=kiln_task,
+                prompt_id=prompt_id,
+                base_adapter_config=base_adapter_config,
+                config=LiteLlmConfig(
+                    model_name=model_name,
+                    provider_name=provider,
+                    additional_body_options={
+                        "api_key": Config.shared().huggingface_api_key,
+                    },
+                ),
+            )
         # These are virtual providers that should have mapped to an actual provider in core_provider
         case ModelProviderName.kiln_fine_tune:
             raise ValueError(
@@ -82,12 +215,3 @@ def adapter_for_task(
             )
         case _:
             raise_exhaustive_enum_error(core_provider_name)
-
-    # We use langchain for all others right now, but moving off it as we touch anything.
-    return LangchainAdapter(
-        kiln_task,
-        model_name=model_name,
-        provider=provider,
-        prompt_builder=prompt_builder,
-        tags=tags,
-    )

kiln_ai/adapters/data_gen/data_gen_task.py

@@ -183,3 +183,21 @@ class DataGenSampleTask(Task, parent_of={}):
             input_json_schema=json.dumps(DataGenSampleTaskInput.model_json_schema()),
             output_json_schema=list_json_schema_for_task(target_task),
         )
+
+
+def wrap_task_with_guidance(original_instruction: str, guidance: str) -> str:
+    """Wrap the original instruction with human guidance.
+
+    Args:
+        original_instruction: The original instruction to wrap
+        guidance: The human guidance to wrap the instruction with
+    """
+    return f"""{original_instruction}
+
+# Special Instructions
+
+The above instructions are the original instructions for this task. For this execution, we've been given additional instructions. Follow both, but prioritize the additional instructions when they conflict. The additional instructions are:
+<additional_instructions>
+{guidance}
+</additional_instructions>
+"""

kiln_ai/adapters/eval/__init__.py

@@ -0,0 +1,28 @@
+"""
+# Evals
+
+This module contains the code for evaluating the performance of a model.
+
+The submodules contain:
+
+- BaseEval: each eval technique implements this interface.
+- G-Eval: an eval implementation, that implements G-Eval and LLM as Judge.
+- EvalRunner: a class that runs an full evaluation (many smaller evals jobs). Includes async parallel processing, and the ability to restart where it left off.
+- EvalRegistry: a registry for all eval implementations.
+
+The datamodel for Evals is in the `kiln_ai.datamodel.eval` module.
+"""
+
+from . import (
+    base_eval,
+    eval_runner,
+    g_eval,
+    registry,
+)
+
+__all__ = [
+    "base_eval",
+    "eval_runner",
+    "g_eval",
+    "registry",
+]

kiln_ai/adapters/eval/base_eval.py

@@ -0,0 +1,164 @@
+import json
+from abc import abstractmethod
+from typing import Dict
+
+from kiln_ai.adapters.adapter_registry import adapter_for_task
+from kiln_ai.adapters.ml_model_list import ModelProviderName
+from kiln_ai.adapters.model_adapters.base_adapter import AdapterConfig
+from kiln_ai.datamodel.eval import Eval, EvalConfig, EvalScores
+from kiln_ai.datamodel.json_schema import validate_schema
+from kiln_ai.datamodel.task import RunConfig, TaskOutputRatingType, TaskRun
+from kiln_ai.utils.exhaustive_error import raise_exhaustive_enum_error
+
+
+class BaseEval:
+    """
+    Base class for all evals/evaluators.
+
+    Should be subclassed, and the run_eval method implemented.
+    """
+
+    def __init__(self, eval_config: EvalConfig, run_config: RunConfig | None):
+        self.eval_config = eval_config
+        eval = eval_config.parent_eval()
+        if not eval:
+            raise ValueError("Eval config must have a parent eval")
+        self.eval = eval
+        task = self.eval.parent_task()
+        if not task:
+            raise ValueError("Eval must have a parent task")
+        self.target_task = task
+        self.score_schema = BaseEval.build_score_schema(eval, allow_float_scores=True)
+        self.run_config = run_config
+
+    def model_and_provider(self) -> tuple[str, ModelProviderName]:
+        model_name = self.eval_config.model_name
+        provider = self.eval_config.model_provider
+        if (
+            not model_name
+            or not provider
+            or not isinstance(model_name, str)
+            or not isinstance(provider, str)
+            or provider not in ModelProviderName.__members__
+        ):
+            raise ValueError(
+                "Model name and provider must be set in the eval config model properties"
+            )
+
+        return model_name, ModelProviderName(provider)
+
+    async def run_task_and_eval(
+        self, input: str
+    ) -> tuple[TaskRun, EvalScores, Dict[str, str] | None]:
+        """
+        Runs the task on the provided run_config to generate fresh output, then runs the eval on that output.
+        """
+        if self.run_config is None:
+            raise ValueError("Run config is required for run_task_and_eval")
+
+        run_adapter = adapter_for_task(
+            self.target_task,
+            self.run_config.model_name,
+            ModelProviderName(self.run_config.model_provider_name),
+            base_adapter_config=AdapterConfig(allow_saving=False),
+        )
+
+        # Parse structured input if needed
+        parsed_input = input
+        if self.target_task.output_json_schema is not None:
+            parsed_input = json.loads(input)
+
+        # we don't save by default here. We'll save manually after validating the output
+        run_output = await run_adapter.invoke(parsed_input)
+
+        eval_output, intermediate_outputs = await self.run_eval(run_output)
+        validate_schema(eval_output, self.score_schema)
+
+        return run_output, eval_output, intermediate_outputs
+
+    @abstractmethod
+    async def run_eval(
+        self, task_run: TaskRun
+    ) -> tuple[EvalScores, Dict[str, str] | None]:
+        """
+        Runs the eval on the given task run.
+
+        Returns a dictionary of scores which should conform to the score schema, and a dictionary of intermediate outputs (eval thinking).
+        """
+        pass
+
+    @classmethod
+    def build_score_schema(cls, eval: Eval, allow_float_scores: bool = False) -> str:
+        """
+        Build a JSON schema for the scoring output of the task requirements
+
+        We allow 2 modes: allow_float_scores=True and allow_float_scores=False.
+
+        allow_float_scores=False is used for the call to the model, and forces the model into selecting into discrete rating options (int 1-5, pass-fail, etc).
+        allow_float_scores=True is used for final score output (for example, after we take a g-eval weighting of the model's logprobs). A pass/fail rating might return 0.75 for likely pass (as opposed to 0.99 for near certain pass), or a 1-5 score might return 3.75.
+        """
+
+        # Note: python maintains order, which is good as we want the user defined order, and overall last
+        properties = {}
+        for output_score in eval.output_scores:
+            output_score_json_key = output_score.json_key()
+
+            if len(output_score_json_key) == 0:
+                raise ValueError(
+                    f"Invalid output score name: {output_score.name}. Can not be used as JSON schema key."
+                )
+            property: dict[str, str | int | float | list[str] | list[int]] = {
+                "title": output_score.name,
+            }
+            match output_score.type:
+                case TaskOutputRatingType.five_star:
+                    if allow_float_scores:
+                        property["type"] = "number"
+                        property["minimum"] = 1
+                        property["maximum"] = 5
+                    else:
+                        property["enum"] = [1, 2, 3, 4, 5]
+
+                    property["description"] = (
+                        f"{output_score.instruction}\n\nThe rating should be between 1 and 5, with 1 being the worst and 5 being the best."
+                    )
+                case TaskOutputRatingType.pass_fail:
+                    if allow_float_scores:
+                        property["type"] = "number"
+                        property["minimum"] = 0
+                        property["maximum"] = 1
+                        property["description"] = (
+                            f"{output_score.instruction}\n\nThe rating should be between 0 and 1, with 0 being a failure and 1 being a pass."
+                        )
+                    else:
+                        property["enum"] = ["pass", "fail"]
+                        property["description"] = (
+                            f"{output_score.instruction}\n\nThe rating should be either 'pass' or 'fail'."
+                        )
+                case TaskOutputRatingType.pass_fail_critical:
+                    if allow_float_scores:
+                        property["type"] = "number"
+                        property["minimum"] = -1
+                        property["maximum"] = 1
+                        property["description"] = (
+                            f"{output_score.instruction}\n\nThe rating should be between -1 and 1, with 1 being a pass, 0 being a failure, and -1 being a critical failure (very severe failure)."
+                        )
+                    else:
+                        property["enum"] = ["pass", "fail", "critical"]
+                        property["description"] = (
+                            f"{output_score.instruction}\n\nThe rating should be either 'pass', 'fail', or 'critical' where critical a very severe failure."
+                        )
+                case TaskOutputRatingType.custom:
+                    # Skip custom rating types in evals
+                    continue
+                case _:
+                    raise_exhaustive_enum_error(output_score.type)
+
+            properties[output_score_json_key] = property
+
+        schema = {
+            "type": "object",
+            "properties": properties,
+            "required": list(properties.keys()),
+        }
+        return json.dumps(schema, ensure_ascii=False)