kiln-ai 0.11.1__py3-none-any.whl → 0.12.0__py3-none-any.whl

kiln_ai/adapters/ml_model_list.py

@@ -43,6 +43,8 @@ class ModelFamily(str, Enum):
     mixtral = "mixtral"
     qwen = "qwen"
     deepseek = "deepseek"
+    dolphin = "dolphin"
+    grok = "grok"
 
 
 # Where models have instruct and raw versions, instruct is default and raw is specified
@@ -71,6 +73,8 @@ class ModelName(str, Enum):
     gemma_2_27b = "gemma_2_27b"
     claude_3_5_haiku = "claude_3_5_haiku"
     claude_3_5_sonnet = "claude_3_5_sonnet"
+    claude_3_7_sonnet = "claude_3_7_sonnet"
+    claude_3_7_sonnet_thinking = "claude_3_7_sonnet_thinking"
     gemini_1_5_flash = "gemini_1_5_flash"
     gemini_1_5_flash_8b = "gemini_1_5_flash_8b"
     gemini_1_5_pro = "gemini_1_5_pro"
@@ -88,6 +92,8 @@ class ModelName(str, Enum):
     deepseek_r1_distill_qwen_1p5b = "deepseek_r1_distill_qwen_1p5b"
     deepseek_r1_distill_qwen_7b = "deepseek_r1_distill_qwen_7b"
     deepseek_r1_distill_llama_8b = "deepseek_r1_distill_llama_8b"
+    dolphin_2_9_8x22b = "dolphin_2_9_8x22b"
+    grok_2 = "grok_2"
 
 
 class ModelParserID(str, Enum):
@@ -123,6 +129,15 @@ class KilnModelProvider(BaseModel):
     structured_output_mode: StructuredOutputMode = StructuredOutputMode.default
     parser: ModelParserID | None = None
     reasoning_capable: bool = False
+    supports_logprobs: bool = False
+
+    # TODO P1: Need a more generalized way to handle custom provider parameters.
+    # Making them quite declarative here for now, isolating provider specific logic
+    # to this file. Later I should be able to override anything in this file via config.
+    r1_openrouter_options: bool = False
+    require_openrouter_reasoning: bool = False
+    logprobs_openrouter_options: bool = False
+    openrouter_skip_required_parameters: bool = False
 
 
 class KilnModel(BaseModel):
@@ -155,11 +170,14 @@ built_in_models: List[KilnModel] = [
                 provider_options={"model": "gpt-4o-mini"},
                 provider_finetune_id="gpt-4o-mini-2024-07-18",
                 structured_output_mode=StructuredOutputMode.json_schema,
+                supports_logprobs=True,
             ),
             KilnModelProvider(
                 name=ModelProviderName.openrouter,
                 provider_options={"model": "openai/gpt-4o-mini"},
                 structured_output_mode=StructuredOutputMode.json_schema,
+                supports_logprobs=True,
+                logprobs_openrouter_options=True,
             ),
         ],
     ),
@@ -174,11 +192,14 @@ built_in_models: List[KilnModel] = [
                 provider_options={"model": "gpt-4o"},
                 provider_finetune_id="gpt-4o-2024-08-06",
                 structured_output_mode=StructuredOutputMode.json_schema,
+                supports_logprobs=True,
             ),
             KilnModelProvider(
                 name=ModelProviderName.openrouter,
                 provider_options={"model": "openai/gpt-4o"},
                 structured_output_mode=StructuredOutputMode.json_schema,
+                supports_logprobs=True,
+                logprobs_openrouter_options=True,
             ),
         ],
     ),
@@ -190,7 +211,7 @@ built_in_models: List[KilnModel] = [
         providers=[
             KilnModelProvider(
                 name=ModelProviderName.openrouter,
-                structured_output_mode=StructuredOutputMode.function_calling,
+                structured_output_mode=StructuredOutputMode.json_instruction_and_object,
                 provider_options={"model": "anthropic/claude-3-5-haiku"},
             ),
         ],
@@ -203,51 +224,37 @@ built_in_models: List[KilnModel] = [
         providers=[
             KilnModelProvider(
                 name=ModelProviderName.openrouter,
-                structured_output_mode=StructuredOutputMode.function_calling,
+                structured_output_mode=StructuredOutputMode.json_instruction_and_object,
                 provider_options={"model": "anthropic/claude-3.5-sonnet"},
             ),
         ],
     ),
-    # DeepSeek 3
+    # Claude 3.7 Sonnet
     KilnModel(
-        family=ModelFamily.deepseek,
-        name=ModelName.deepseek_3,
-        friendly_name="DeepSeek v3",
+        family=ModelFamily.claude,
+        name=ModelName.claude_3_7_sonnet,
+        friendly_name="Claude 3.7 Sonnet",
         providers=[
             KilnModelProvider(
                 name=ModelProviderName.openrouter,
-                provider_options={"model": "deepseek/deepseek-chat"},
                 structured_output_mode=StructuredOutputMode.function_calling,
+                provider_options={"model": "anthropic/claude-3.7-sonnet"},
             ),
         ],
     ),
-    # DeepSeek R1
+    # Claude 3.7 Sonnet Thinking
     KilnModel(
-        family=ModelFamily.deepseek,
-        name=ModelName.deepseek_r1,
-        friendly_name="DeepSeek R1",
+        family=ModelFamily.claude,
+        name=ModelName.claude_3_7_sonnet_thinking,
+        friendly_name="Claude 3.7 Sonnet Thinking",
         providers=[
             KilnModelProvider(
                 name=ModelProviderName.openrouter,
-                provider_options={"model": "deepseek/deepseek-r1"},
-                # No custom parser -- openrouter implemented it themselves
-                structured_output_mode=StructuredOutputMode.json_instructions,
+                provider_options={"model": "anthropic/claude-3.7-sonnet:thinking"},
                 reasoning_capable=True,
-            ),
-            KilnModelProvider(
-                name=ModelProviderName.fireworks_ai,
-                provider_options={"model": "accounts/fireworks/models/deepseek-r1"},
-                parser=ModelParserID.r1_thinking,
-                structured_output_mode=StructuredOutputMode.json_instructions,
-                reasoning_capable=True,
-            ),
-            KilnModelProvider(
-                # I want your RAM
-                name=ModelProviderName.ollama,
-                provider_options={"model": "deepseek-r1:671b"},
-                parser=ModelParserID.r1_thinking,
+                # For reasoning models, we need to use json_instructions with OpenRouter
                 structured_output_mode=StructuredOutputMode.json_instructions,
-                reasoning_capable=True,
+                require_openrouter_reasoning=True,
             ),
         ],
     ),
@@ -379,8 +386,11 @@ built_in_models: List[KilnModel] = [
             KilnModelProvider(
                 name=ModelProviderName.openrouter,
                 supports_data_gen=False,
-                structured_output_mode=StructuredOutputMode.function_calling,
+                # Need to not pass "strict=True" to the function call to get this to work with logprobs for some reason. Openrouter issue.
+                structured_output_mode=StructuredOutputMode.function_calling_weak,
                 provider_options={"model": "meta-llama/llama-3.1-70b-instruct"},
+                supports_logprobs=True,
+                logprobs_openrouter_options=True,
             ),
             KilnModelProvider(
                 name=ModelProviderName.ollama,
@@ -819,6 +829,58 @@ built_in_models: List[KilnModel] = [
             ),
         ],
     ),
+    # DeepSeek 3
+    KilnModel(
+        family=ModelFamily.deepseek,
+        name=ModelName.deepseek_3,
+        friendly_name="DeepSeek V3",
+        providers=[
+            KilnModelProvider(
+                name=ModelProviderName.openrouter,
+                provider_options={"model": "deepseek/deepseek-chat"},
+                structured_output_mode=StructuredOutputMode.function_calling,
+            ),
+            KilnModelProvider(
+                name=ModelProviderName.fireworks_ai,
+                provider_options={"model": "accounts/fireworks/models/deepseek-v3"},
+                structured_output_mode=StructuredOutputMode.json_mode,
+                supports_structured_output=True,
+                supports_data_gen=False,
+            ),
+        ],
+    ),
+    # DeepSeek R1
+    KilnModel(
+        family=ModelFamily.deepseek,
+        name=ModelName.deepseek_r1,
+        friendly_name="DeepSeek R1",
+        providers=[
+            KilnModelProvider(
+                name=ModelProviderName.openrouter,
+                provider_options={"model": "deepseek/deepseek-r1"},
+                # No custom parser -- openrouter implemented it themselves
+                structured_output_mode=StructuredOutputMode.json_instructions,
+                reasoning_capable=True,
+                r1_openrouter_options=True,
+                require_openrouter_reasoning=True,
+            ),
+            KilnModelProvider(
+                name=ModelProviderName.fireworks_ai,
+                provider_options={"model": "accounts/fireworks/models/deepseek-r1"},
+                parser=ModelParserID.r1_thinking,
+                structured_output_mode=StructuredOutputMode.json_instructions,
+                reasoning_capable=True,
+            ),
+            KilnModelProvider(
+                # I want your RAM
+                name=ModelProviderName.ollama,
+                provider_options={"model": "deepseek-r1:671b"},
+                parser=ModelParserID.r1_thinking,
+                structured_output_mode=StructuredOutputMode.json_instructions,
+                reasoning_capable=True,
+            ),
+        ],
+    ),
     # DeepSeek R1 Distill Qwen 32B
     KilnModel(
         family=ModelFamily.deepseek,
@@ -830,6 +892,8 @@ built_in_models: List[KilnModel] = [
                 reasoning_capable=True,
                 structured_output_mode=StructuredOutputMode.json_instructions,
                 provider_options={"model": "deepseek/deepseek-r1-distill-qwen-32b"},
+                r1_openrouter_options=True,
+                require_openrouter_reasoning=True,
             ),
             KilnModelProvider(
                 name=ModelProviderName.ollama,
@@ -851,6 +915,8 @@ built_in_models: List[KilnModel] = [
                 reasoning_capable=True,
                 structured_output_mode=StructuredOutputMode.json_instructions,
                 provider_options={"model": "deepseek/deepseek-r1-distill-llama-70b"},
+                r1_openrouter_options=True,
+                require_openrouter_reasoning=True,
             ),
             KilnModelProvider(
                 name=ModelProviderName.ollama,
@@ -874,6 +940,9 @@ built_in_models: List[KilnModel] = [
                 reasoning_capable=True,
                 structured_output_mode=StructuredOutputMode.json_instructions,
                 provider_options={"model": "deepseek/deepseek-r1-distill-qwen-14b"},
+                r1_openrouter_options=True,
+                require_openrouter_reasoning=True,
+                openrouter_skip_required_parameters=True,
             ),
             KilnModelProvider(
                 name=ModelProviderName.ollama,
@@ -897,6 +966,9 @@ built_in_models: List[KilnModel] = [
                 reasoning_capable=True,
                 structured_output_mode=StructuredOutputMode.json_instructions,
                 provider_options={"model": "deepseek/deepseek-r1-distill-llama-8b"},
+                r1_openrouter_options=True,
+                require_openrouter_reasoning=True,
+                openrouter_skip_required_parameters=True,
             ),
             KilnModelProvider(
                 name=ModelProviderName.ollama,
@@ -937,6 +1009,9 @@ built_in_models: List[KilnModel] = [
                 reasoning_capable=True,
                 structured_output_mode=StructuredOutputMode.json_instructions,
                 provider_options={"model": "deepseek/deepseek-r1-distill-qwen-1.5b"},
+                r1_openrouter_options=True,
+                require_openrouter_reasoning=True,
+                openrouter_skip_required_parameters=True,
             ),
             KilnModelProvider(
                 name=ModelProviderName.ollama,
@@ -948,4 +1023,41 @@ built_in_models: List[KilnModel] = [
             ),
         ],
     ),
+    # Dolphin 2.9 Mixtral 8x22B
+    KilnModel(
+        family=ModelFamily.dolphin,
+        name=ModelName.dolphin_2_9_8x22b,
+        friendly_name="Dolphin 2.9 8x22B",
+        providers=[
+            KilnModelProvider(
+                name=ModelProviderName.ollama,
+                structured_output_mode=StructuredOutputMode.json_schema,
+                supports_data_gen=True,
+                provider_options={"model": "dolphin-mixtral:8x22b"},
+            ),
+            KilnModelProvider(
+                name=ModelProviderName.openrouter,
+                provider_options={
+                    "model": "cognitivecomputations/dolphin-mixtral-8x22b"
+                },
+                supports_data_gen=True,
+                structured_output_mode=StructuredOutputMode.json_instruction_and_object,
+            ),
+        ],
+    ),
+    # Grok 2
+    KilnModel(
+        family=ModelFamily.grok,
+        name=ModelName.grok_2,
+        friendly_name="Grok 2",
+        providers=[
+            KilnModelProvider(
+                name=ModelProviderName.openrouter,
+                provider_options={"model": "x-ai/grok-2-1212"},
+                supports_structured_output=True,
+                supports_data_gen=True,
+                structured_output_mode=StructuredOutputMode.json_schema,
+            ),
+        ],
+    ),
 ]

kiln_ai/adapters/model_adapters/base_adapter.py

@@ -5,7 +5,7 @@ from typing import Dict, Literal, Tuple
 
 from kiln_ai.adapters.ml_model_list import KilnModelProvider, StructuredOutputMode
 from kiln_ai.adapters.parsers.parser_registry import model_parser_from_id
-from kiln_ai.adapters.prompt_builders import BasePromptBuilder, SimplePromptBuilder
+from kiln_ai.adapters.prompt_builders import prompt_builder_from_id
 from kiln_ai.adapters.provider_tools import kiln_model_provider_from
 from kiln_ai.adapters.run_output import RunOutput
 from kiln_ai.datamodel import (
@@ -16,16 +16,21 @@ from kiln_ai.datamodel import (
     TaskRun,
 )
 from kiln_ai.datamodel.json_schema import validate_schema
+from kiln_ai.datamodel.task import RunConfig
 from kiln_ai.utils.config import Config
 
 
 @dataclass
-class AdapterInfo:
-    adapter_name: str
-    model_name: str
-    model_provider: str
-    prompt_builder_name: str
-    prompt_id: str | None = None
+class AdapterConfig:
+    """
+    An adapter config is config options that do NOT impact the output of the model.
+
+    For example: if it's saved, of if we request additional data like logprobs.
+    """
+
+    allow_saving: bool = True
+    top_logprobs: int | None = None
+    default_tags: list[str] | None = None
 
 
 COT_FINAL_ANSWER_PROMPT = "Considering the above, return a final result."
@@ -47,35 +52,36 @@ class BaseAdapter(metaclass=ABCMeta):
 
     def __init__(
         self,
-        kiln_task: Task,
-        model_name: str,
-        model_provider_name: str,
-        prompt_builder: BasePromptBuilder | None = None,
-        tags: list[str] | None = None,
+        run_config: RunConfig,
+        config: AdapterConfig | None = None,
     ):
-        self.prompt_builder = prompt_builder or SimplePromptBuilder(kiln_task)
-        self.kiln_task = kiln_task
-        self.output_schema = self.kiln_task.output_json_schema
-        self.input_schema = self.kiln_task.input_json_schema
-        self.default_tags = tags
-        self.model_name = model_name
-        self.model_provider_name = model_provider_name
+        self.run_config = run_config
+        self.prompt_builder = prompt_builder_from_id(
+            run_config.prompt_id, run_config.task
+        )
         self._model_provider: KilnModelProvider | None = None
 
+        self.output_schema = self.task().output_json_schema
+        self.input_schema = self.task().input_json_schema
+        self.base_adapter_config = config or AdapterConfig()
+
+    def task(self) -> Task:
+        return self.run_config.task
+
     def model_provider(self) -> KilnModelProvider:
         """
         Lazy load the model provider for this adapter.
         """
         if self._model_provider is not None:
             return self._model_provider
-        if not self.model_name or not self.model_provider_name:
+        if not self.run_config.model_name or not self.run_config.model_provider_name:
             raise ValueError("model_name and model_provider_name must be provided")
         self._model_provider = kiln_model_provider_from(
-            self.model_name, self.model_provider_name
+            self.run_config.model_name, self.run_config.model_provider_name
         )
         if not self._model_provider:
             raise ValueError(
-                f"model_provider_name {self.model_provider_name} not found for model {self.model_name}"
+                f"model_provider_name {self.run_config.model_provider_name} not found for model {self.run_config.model_name}"
             )
         return self._model_provider
 
@@ -85,7 +91,7 @@ class BaseAdapter(metaclass=ABCMeta):
         input_source: DataSource | None = None,
     ) -> Dict | str:
         result = await self.invoke(input, input_source)
-        if self.kiln_task.output_json_schema is None:
+        if self.task().output_json_schema is None:
             return result.output.output
         else:
             return json.loads(result.output.output)
@@ -95,6 +101,14 @@ class BaseAdapter(metaclass=ABCMeta):
         input: Dict | str,
         input_source: DataSource | None = None,
     ) -> TaskRun:
+        run_output, _ = await self.invoke_returning_run_output(input, input_source)
+        return run_output
+
+    async def invoke_returning_run_output(
+        self,
+        input: Dict | str,
+        input_source: DataSource | None = None,
+    ) -> Tuple[TaskRun, RunOutput]:
         # validate input
         if self.input_schema is not None:
             if not isinstance(input, dict):
@@ -128,19 +142,23 @@ class BaseAdapter(metaclass=ABCMeta):
         run = self.generate_run(input, input_source, parsed_output)
 
         # Save the run if configured to do so, and we have a path to save to
-        if Config.shared().autosave_runs and self.kiln_task.path is not None:
+        if (
+            self.base_adapter_config.allow_saving
+            and Config.shared().autosave_runs
+            and self.task().path is not None
+        ):
             run.save_to_file()
         else:
             # Clear the ID to indicate it's not persisted
             run.id = None
 
-        return run
+        return run, run_output
 
     def has_structured_output(self) -> bool:
         return self.output_schema is not None
 
     @abstractmethod
-    def adapter_info(self) -> AdapterInfo:
+    def adapter_name(self) -> str:
         pass
 
     @abstractmethod
@@ -203,7 +221,7 @@ class BaseAdapter(metaclass=ABCMeta):
             )
 
         new_task_run = TaskRun(
-            parent=self.kiln_task,
+            parent=self.task(),
             input=input_str,
             input_source=input_source,
             output=TaskOutput(
@@ -215,7 +233,7 @@ class BaseAdapter(metaclass=ABCMeta):
                 ),
             ),
             intermediate_outputs=run_output.intermediate_outputs,
-            tags=self.default_tags or [],
+            tags=self.base_adapter_config.default_tags or [],
         )
 
         return new_task_run
@@ -224,12 +242,9 @@ class BaseAdapter(metaclass=ABCMeta):
         props = {}
 
         # adapter info
-        adapter_info = self.adapter_info()
-        props["adapter_name"] = adapter_info.adapter_name
-        props["model_name"] = adapter_info.model_name
-        props["model_provider"] = adapter_info.model_provider
-        props["prompt_builder_name"] = adapter_info.prompt_builder_name
-        if adapter_info.prompt_id is not None:
-            props["prompt_id"] = adapter_info.prompt_id
+        props["adapter_name"] = self.adapter_name()
+        props["model_name"] = self.run_config.model_name
+        props["model_provider"] = self.run_config.model_provider_name
+        props["prompt_id"] = self.run_config.prompt_id
 
         return props

kiln_ai/adapters/model_adapters/langchain_adapters.py

@@ -20,9 +20,8 @@ from kiln_ai.adapters.ml_model_list import (
 )
 from kiln_ai.adapters.model_adapters.base_adapter import (
     COT_FINAL_ANSWER_PROMPT,
-    AdapterInfo,
+    AdapterConfig,
     BaseAdapter,
-    BasePromptBuilder,
     RunOutput,
 )
 from kiln_ai.adapters.ollama_tools import (
@@ -30,6 +29,8 @@ from kiln_ai.adapters.ollama_tools import (
     ollama_base_url,
     ollama_model_installed,
 )
+from kiln_ai.datamodel import PromptId
+from kiln_ai.datamodel.task import RunConfig
 from kiln_ai.utils.config import Config
 from kiln_ai.utils.exhaustive_error import raise_exhaustive_enum_error
 
@@ -45,8 +46,8 @@ class LangchainAdapter(BaseAdapter):
         custom_model: BaseChatModel | None = None,
         model_name: str | None = None,
         provider: str | None = None,
-        prompt_builder: BasePromptBuilder | None = None,
-        tags: list[str] | None = None,
+        prompt_id: PromptId | None = None,
+        base_adapter_config: AdapterConfig | None = None,
     ):
         if custom_model is not None:
             self._model = custom_model
@@ -78,12 +79,16 @@ class LangchainAdapter(BaseAdapter):
         if model_name is None:
             raise ValueError("model_name must be provided")
 
-        super().__init__(
-            kiln_task,
+        run_config = RunConfig(
+            task=kiln_task,
             model_name=model_name,
             model_provider_name=provider,
-            prompt_builder=prompt_builder,
-            tags=tags,
+            prompt_id=prompt_id or datamodel.PromptGenerators.SIMPLE,
+        )
+
+        super().__init__(
+            run_config=run_config,
+            config=base_adapter_config,
         )
 
     async def model(self) -> LangChainModelType:
@@ -111,15 +116,15 @@ class LangchainAdapter(BaseAdapter):
                     f"model {self._model} does not support structured output, cannot use output_json_schema"
                 )
             # Langchain expects title/description to be at top level, on top of json schema
-            output_schema = self.kiln_task.output_schema()
+            output_schema = self.task().output_schema()
             if output_schema is None:
                 raise ValueError(
-                    f"output_json_schema is not valid json: {self.kiln_task.output_json_schema}"
+                    f"output_json_schema is not valid json: {self.task().output_json_schema}"
                 )
             output_schema["title"] = "task_response"
             output_schema["description"] = "A response from the task"
             with_structured_output_options = self.get_structured_output_options(
-                self.model_name, self.model_provider_name
+                self.run_config.model_name, self.run_config.model_provider_name
             )
             self._model = self._model.with_structured_output(
                 output_schema,
@@ -129,6 +134,11 @@ class LangchainAdapter(BaseAdapter):
         return self._model
 
     async def _run(self, input: Dict | str) -> RunOutput:
+        if self.base_adapter_config.top_logprobs is not None:
+            raise ValueError(
+                "Kiln's Langchain adapter does not support logprobs/top_logprobs. Select a model from an OpenAI compatible provider (openai, openrouter, etc) instead."
+            )
+
         provider = self.model_provider()
         model = await self.model()
         chain = model
@@ -191,14 +201,8 @@ class LangchainAdapter(BaseAdapter):
             intermediate_outputs=intermediate_outputs,
         )
 
-    def adapter_info(self) -> AdapterInfo:
-        return AdapterInfo(
-            model_name=self.model_name,
-            model_provider=self.model_provider_name,
-            adapter_name="kiln_langchain_adapter",
-            prompt_builder_name=self.prompt_builder.__class__.prompt_builder_name(),
-            prompt_id=self.prompt_builder.prompt_id(),
-        )
+    def adapter_name(self) -> str:
+        return "kiln_langchain_adapter"
 
     def _munge_response(self, response: Dict) -> Dict:
         # Mistral Large tool calling format is a bit different. Convert to standard format.
@@ -220,6 +224,9 @@ class LangchainAdapter(BaseAdapter):
         options = {}
         # We may need to add some provider specific logic here if providers use different names for the same mode, but everyone is copying openai for now
         match provider.structured_output_mode:
+            case StructuredOutputMode.function_calling_weak:
+                # Langchaing doesn't handle weak/strict separately
+                options["method"] = "function_calling"
             case StructuredOutputMode.function_calling:
                 options["method"] = "function_calling"
             case StructuredOutputMode.json_mode:
@@ -246,7 +253,7 @@ class LangchainAdapter(BaseAdapter):
 
     async def langchain_model_from(self) -> BaseChatModel:
         provider = self.model_provider()
-        return await langchain_model_from_provider(provider, self.model_name)
+        return await langchain_model_from_provider(provider, self.run_config.model_name)
 
 
 async def langchain_model_from_provider(

kiln_ai/adapters/model_adapters/openai_compatible_config.py

@@ -8,4 +8,3 @@ class OpenAICompatibleConfig:
     provider_name: str
     base_url: str | None = None  # Defaults to OpenAI
     default_headers: dict[str, str] | None = None
-    openrouter_style_reasoning: bool = False