kiln-ai 0.13.2__py3-none-any.whl → 0.15.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of kiln-ai might be problematic. Click here for more details.
- kiln_ai/adapters/eval/base_eval.py +7 -2
- kiln_ai/adapters/fine_tune/base_finetune.py +6 -23
- kiln_ai/adapters/fine_tune/dataset_formatter.py +4 -4
- kiln_ai/adapters/fine_tune/finetune_registry.py +2 -0
- kiln_ai/adapters/fine_tune/fireworks_finetune.py +163 -15
- kiln_ai/adapters/fine_tune/test_base_finetune.py +7 -9
- kiln_ai/adapters/fine_tune/test_dataset_formatter.py +3 -3
- kiln_ai/adapters/fine_tune/test_fireworks_tinetune.py +495 -9
- kiln_ai/adapters/fine_tune/test_vertex_finetune.py +586 -0
- kiln_ai/adapters/fine_tune/vertex_finetune.py +217 -0
- kiln_ai/adapters/ml_model_list.py +319 -43
- kiln_ai/adapters/model_adapters/base_adapter.py +15 -10
- kiln_ai/adapters/model_adapters/litellm_adapter.py +10 -5
- kiln_ai/adapters/provider_tools.py +7 -0
- kiln_ai/adapters/test_provider_tools.py +16 -0
- kiln_ai/datamodel/json_schema.py +24 -7
- kiln_ai/datamodel/task_output.py +9 -5
- kiln_ai/datamodel/task_run.py +29 -5
- kiln_ai/datamodel/test_example_models.py +104 -3
- kiln_ai/datamodel/test_json_schema.py +22 -3
- kiln_ai/datamodel/test_model_perf.py +3 -2
- {kiln_ai-0.13.2.dist-info → kiln_ai-0.15.0.dist-info}/METADATA +3 -2
- {kiln_ai-0.13.2.dist-info → kiln_ai-0.15.0.dist-info}/RECORD +25 -24
- kiln_ai/adapters/test_generate_docs.py +0 -69
- {kiln_ai-0.13.2.dist-info → kiln_ai-0.15.0.dist-info}/WHEEL +0 -0
- {kiln_ai-0.13.2.dist-info → kiln_ai-0.15.0.dist-info}/licenses/LICENSE.txt +0 -0
|
@@ -1,69 +0,0 @@
|
|
|
1
|
-
import logging
|
|
2
|
-
from typing import List
|
|
3
|
-
|
|
4
|
-
import pytest
|
|
5
|
-
|
|
6
|
-
from libs.core.kiln_ai.adapters.ml_model_list import KilnModelProvider, built_in_models
|
|
7
|
-
from libs.core.kiln_ai.adapters.provider_tools import provider_name_from_id
|
|
8
|
-
|
|
9
|
-
logger = logging.getLogger(__name__)
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
def _all_providers_support(providers: List[KilnModelProvider], attribute: str) -> bool:
|
|
13
|
-
"""Check if all providers support a given feature"""
|
|
14
|
-
return all(getattr(provider, attribute) for provider in providers)
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
def _any_providers_support(providers: List[KilnModelProvider], attribute: str) -> bool:
|
|
18
|
-
"""Check if any providers support a given feature"""
|
|
19
|
-
return any(getattr(provider, attribute) for provider in providers)
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
def _get_support_status(providers: List[KilnModelProvider], attribute: str) -> str:
|
|
23
|
-
"""Get the support status for a feature"""
|
|
24
|
-
if _all_providers_support(providers, attribute):
|
|
25
|
-
return "✅︎"
|
|
26
|
-
elif _any_providers_support(providers, attribute):
|
|
27
|
-
return "✅︎ (some providers)"
|
|
28
|
-
return ""
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
def _has_finetune_support(providers: List[KilnModelProvider]) -> str:
|
|
32
|
-
"""Check if any provider supports fine-tuning"""
|
|
33
|
-
return "✅︎" if any(p.provider_finetune_id for p in providers) else ""
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
@pytest.mark.paid(reason="Marking as paid so it isn't run by default")
|
|
37
|
-
def test_generate_model_table():
|
|
38
|
-
"""Generate a markdown table of all models and their capabilities"""
|
|
39
|
-
|
|
40
|
-
# Table header
|
|
41
|
-
table = [
|
|
42
|
-
"| Model Name | Providers | Structured Output | Reasoning | Synthetic Data | API Fine-Tuneable |",
|
|
43
|
-
"|------------|-----------|-------------------|-----------|----------------|-------------------|",
|
|
44
|
-
]
|
|
45
|
-
|
|
46
|
-
for model in built_in_models:
|
|
47
|
-
provider_names = ", ".join(
|
|
48
|
-
sorted(provider_name_from_id(p.name.value) for p in model.providers)
|
|
49
|
-
)
|
|
50
|
-
structured_output = _get_support_status(
|
|
51
|
-
model.providers, "supports_structured_output"
|
|
52
|
-
)
|
|
53
|
-
reasoning = _get_support_status(model.providers, "reasoning_capable")
|
|
54
|
-
data_gen = _get_support_status(model.providers, "supports_data_gen")
|
|
55
|
-
finetune = _has_finetune_support(model.providers)
|
|
56
|
-
|
|
57
|
-
row = f"| {model.friendly_name} | {provider_names} | {structured_output} | {reasoning} | {data_gen} | {finetune} |"
|
|
58
|
-
table.append(row)
|
|
59
|
-
|
|
60
|
-
# Print the table (useful for documentation)
|
|
61
|
-
print("\nModel Capability Matrix:\n")
|
|
62
|
-
print("\n".join(table))
|
|
63
|
-
|
|
64
|
-
# Basic assertions to ensure the table is well-formed
|
|
65
|
-
assert len(table) > 2, "Table should have header and at least one row"
|
|
66
|
-
assert all("|" in row for row in table), "All rows should be properly formatted"
|
|
67
|
-
assert len(table[0].split("|")) == len(table[1].split("|")), (
|
|
68
|
-
"Header and separator should have same number of columns"
|
|
69
|
-
)
|
|
File without changes
|
|
File without changes
|