kiln-ai 0.5.4__py3-none-any.whl → 0.6.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of kiln-ai might be problematic. Click here for more details.
- kiln_ai/adapters/base_adapter.py +24 -35
- kiln_ai/adapters/data_gen/data_gen_prompts.py +73 -0
- kiln_ai/adapters/data_gen/data_gen_task.py +117 -0
- kiln_ai/adapters/data_gen/test_data_gen_task.py +292 -0
- kiln_ai/adapters/langchain_adapters.py +39 -7
- kiln_ai/adapters/ml_model_list.py +68 -1
- kiln_ai/adapters/prompt_builders.py +66 -0
- kiln_ai/adapters/repair/test_repair_task.py +4 -1
- kiln_ai/adapters/test_langchain_adapter.py +73 -0
- kiln_ai/adapters/test_ml_model_list.py +56 -0
- kiln_ai/adapters/test_prompt_adaptors.py +54 -18
- kiln_ai/adapters/test_prompt_builders.py +97 -7
- kiln_ai/adapters/test_saving_adapter_results.py +16 -6
- kiln_ai/adapters/test_structured_output.py +33 -5
- kiln_ai/datamodel/__init__.py +28 -7
- kiln_ai/datamodel/json_schema.py +1 -0
- kiln_ai/datamodel/test_models.py +44 -8
- kiln_ai/utils/config.py +3 -2
- kiln_ai/utils/test_config.py +7 -0
- {kiln_ai-0.5.4.dist-info → kiln_ai-0.6.0.dist-info}/METADATA +41 -7
- kiln_ai-0.6.0.dist-info/RECORD +36 -0
- {kiln_ai-0.5.4.dist-info → kiln_ai-0.6.0.dist-info}/WHEEL +1 -1
- kiln_ai-0.5.4.dist-info/RECORD +0 -33
- {kiln_ai-0.5.4.dist-info → kiln_ai-0.6.0.dist-info}/licenses/LICENSE.txt +0 -0
|
@@ -4,10 +4,14 @@ import pytest
|
|
|
4
4
|
|
|
5
5
|
from kiln_ai.adapters.base_adapter import AdapterInfo, BaseAdapter
|
|
6
6
|
from kiln_ai.adapters.prompt_builders import (
|
|
7
|
+
FewShotChainOfThoughtPromptBuilder,
|
|
7
8
|
FewShotPromptBuilder,
|
|
9
|
+
MultiShotChainOfThoughtPromptBuilder,
|
|
8
10
|
MultiShotPromptBuilder,
|
|
9
11
|
RepairsPromptBuilder,
|
|
12
|
+
SimpleChainOfThoughtPromptBuilder,
|
|
10
13
|
SimplePromptBuilder,
|
|
14
|
+
chain_of_thought_prompt,
|
|
11
15
|
prompt_builder_from_ui_name,
|
|
12
16
|
)
|
|
13
17
|
from kiln_ai.adapters.test_prompt_adaptors import build_test_task
|
|
@@ -43,9 +47,6 @@ def test_simple_prompt_builder(tmp_path):
|
|
|
43
47
|
|
|
44
48
|
|
|
45
49
|
class MockAdapter(BaseAdapter):
|
|
46
|
-
def adapter_specific_instructions(self) -> str | None:
|
|
47
|
-
return "You are a mock, send me the response!"
|
|
48
|
-
|
|
49
50
|
def _run(self, input: str) -> str:
|
|
50
51
|
return "mock response"
|
|
51
52
|
|
|
@@ -64,10 +65,6 @@ def test_simple_prompt_builder_structured_output(tmp_path):
|
|
|
64
65
|
prompt = builder.build_prompt()
|
|
65
66
|
assert "You are an assistant which tells a joke, given a subject." in prompt
|
|
66
67
|
|
|
67
|
-
# check adapter instructions are included
|
|
68
|
-
run_adapter = MockAdapter(task, prompt_builder=builder)
|
|
69
|
-
assert "You are a mock, send me the response!" in run_adapter.build_prompt()
|
|
70
|
-
|
|
71
68
|
user_msg = builder.build_user_message(input)
|
|
72
69
|
assert input in user_msg
|
|
73
70
|
assert input not in prompt
|
|
@@ -313,6 +310,18 @@ def test_prompt_builder_from_ui_name():
|
|
|
313
310
|
assert prompt_builder_from_ui_name("few_shot") == FewShotPromptBuilder
|
|
314
311
|
assert prompt_builder_from_ui_name("many_shot") == MultiShotPromptBuilder
|
|
315
312
|
assert prompt_builder_from_ui_name("repairs") == RepairsPromptBuilder
|
|
313
|
+
assert (
|
|
314
|
+
prompt_builder_from_ui_name("simple_chain_of_thought")
|
|
315
|
+
== SimpleChainOfThoughtPromptBuilder
|
|
316
|
+
)
|
|
317
|
+
assert (
|
|
318
|
+
prompt_builder_from_ui_name("few_shot_chain_of_thought")
|
|
319
|
+
== FewShotChainOfThoughtPromptBuilder
|
|
320
|
+
)
|
|
321
|
+
assert (
|
|
322
|
+
prompt_builder_from_ui_name("multi_shot_chain_of_thought")
|
|
323
|
+
== MultiShotChainOfThoughtPromptBuilder
|
|
324
|
+
)
|
|
316
325
|
|
|
317
326
|
with pytest.raises(ValueError, match="Unknown prompt builder: invalid_name"):
|
|
318
327
|
prompt_builder_from_ui_name("invalid_name")
|
|
@@ -336,3 +345,84 @@ def test_repair_multi_shot_prompt_builder(task_with_examples):
|
|
|
336
345
|
'Initial Output Which Was Insufficient: {"joke": "Moo I am a cow joke."}'
|
|
337
346
|
in prompt
|
|
338
347
|
)
|
|
348
|
+
|
|
349
|
+
|
|
350
|
+
def test_chain_of_thought_prompt(tmp_path):
|
|
351
|
+
# Test with default thinking instruction
|
|
352
|
+
task = Task(
|
|
353
|
+
name="Test Task",
|
|
354
|
+
instruction="Test instruction",
|
|
355
|
+
parent=None,
|
|
356
|
+
thinking_instruction=None,
|
|
357
|
+
)
|
|
358
|
+
assert (
|
|
359
|
+
chain_of_thought_prompt(task)
|
|
360
|
+
== "Think step by step, explaining your reasoning."
|
|
361
|
+
)
|
|
362
|
+
|
|
363
|
+
# Test with custom thinking instruction
|
|
364
|
+
custom_instruction = "First analyze the problem, then break it down into steps."
|
|
365
|
+
task = Task(
|
|
366
|
+
name="Test Task",
|
|
367
|
+
instruction="Test instruction",
|
|
368
|
+
parent=None,
|
|
369
|
+
thinking_instruction=custom_instruction,
|
|
370
|
+
)
|
|
371
|
+
assert chain_of_thought_prompt(task) == custom_instruction
|
|
372
|
+
|
|
373
|
+
|
|
374
|
+
@pytest.mark.parametrize(
|
|
375
|
+
"builder_class",
|
|
376
|
+
[
|
|
377
|
+
SimpleChainOfThoughtPromptBuilder,
|
|
378
|
+
FewShotChainOfThoughtPromptBuilder,
|
|
379
|
+
MultiShotChainOfThoughtPromptBuilder,
|
|
380
|
+
],
|
|
381
|
+
)
|
|
382
|
+
def test_chain_of_thought_prompt_builders(builder_class, task_with_examples):
|
|
383
|
+
# Test with default thinking instruction
|
|
384
|
+
builder = builder_class(task=task_with_examples)
|
|
385
|
+
assert (
|
|
386
|
+
builder.chain_of_thought_prompt()
|
|
387
|
+
== "Think step by step, explaining your reasoning."
|
|
388
|
+
)
|
|
389
|
+
|
|
390
|
+
# Test with custom thinking instruction
|
|
391
|
+
custom_instruction = "First analyze the problem, then break it down into steps."
|
|
392
|
+
task_with_custom = task_with_examples.model_copy(
|
|
393
|
+
update={"thinking_instruction": custom_instruction}
|
|
394
|
+
)
|
|
395
|
+
builder = builder_class(task=task_with_custom)
|
|
396
|
+
assert builder.chain_of_thought_prompt() == custom_instruction
|
|
397
|
+
|
|
398
|
+
|
|
399
|
+
def test_build_prompt_for_ui(tmp_path):
|
|
400
|
+
# Test regular prompt builder
|
|
401
|
+
task = build_test_task(tmp_path)
|
|
402
|
+
simple_builder = SimplePromptBuilder(task=task)
|
|
403
|
+
ui_prompt = simple_builder.build_prompt_for_ui()
|
|
404
|
+
|
|
405
|
+
# Should match regular prompt since no chain of thought
|
|
406
|
+
assert ui_prompt == simple_builder.build_prompt()
|
|
407
|
+
assert "# Thinking Instructions" not in ui_prompt
|
|
408
|
+
|
|
409
|
+
# Test chain of thought prompt builder
|
|
410
|
+
cot_builder = SimpleChainOfThoughtPromptBuilder(task=task)
|
|
411
|
+
ui_prompt_cot = cot_builder.build_prompt_for_ui()
|
|
412
|
+
|
|
413
|
+
# Should include both base prompt and thinking instructions
|
|
414
|
+
assert cot_builder.build_prompt() in ui_prompt_cot
|
|
415
|
+
assert "# Thinking Instructions" in ui_prompt_cot
|
|
416
|
+
assert "Think step by step" in ui_prompt_cot
|
|
417
|
+
|
|
418
|
+
# Test with custom thinking instruction
|
|
419
|
+
custom_instruction = "First analyze the problem, then solve it."
|
|
420
|
+
task_with_custom = task.model_copy(
|
|
421
|
+
update={"thinking_instruction": custom_instruction}
|
|
422
|
+
)
|
|
423
|
+
custom_cot_builder = SimpleChainOfThoughtPromptBuilder(task=task_with_custom)
|
|
424
|
+
ui_prompt_custom = custom_cot_builder.build_prompt_for_ui()
|
|
425
|
+
|
|
426
|
+
assert custom_cot_builder.build_prompt() in ui_prompt_custom
|
|
427
|
+
assert "# Thinking Instructions" in ui_prompt_custom
|
|
428
|
+
assert custom_instruction in ui_prompt_custom
|
|
@@ -2,7 +2,7 @@ from unittest.mock import patch
|
|
|
2
2
|
|
|
3
3
|
import pytest
|
|
4
4
|
|
|
5
|
-
from kiln_ai.adapters.base_adapter import AdapterInfo, BaseAdapter
|
|
5
|
+
from kiln_ai.adapters.base_adapter import AdapterInfo, BaseAdapter, RunOutput
|
|
6
6
|
from kiln_ai.datamodel import (
|
|
7
7
|
DataSource,
|
|
8
8
|
DataSourceType,
|
|
@@ -14,7 +14,7 @@ from kiln_ai.utils.config import Config
|
|
|
14
14
|
|
|
15
15
|
class MockAdapter(BaseAdapter):
|
|
16
16
|
async def _run(self, input: dict | str) -> dict | str:
|
|
17
|
-
return "Test output"
|
|
17
|
+
return RunOutput(output="Test output", intermediate_outputs=None)
|
|
18
18
|
|
|
19
19
|
def adapter_info(self) -> AdapterInfo:
|
|
20
20
|
return AdapterInfo(
|
|
@@ -42,9 +42,13 @@ def test_save_run_isolation(test_task):
|
|
|
42
42
|
adapter = MockAdapter(test_task)
|
|
43
43
|
input_data = "Test input"
|
|
44
44
|
output_data = "Test output"
|
|
45
|
+
run_output = RunOutput(
|
|
46
|
+
output=output_data,
|
|
47
|
+
intermediate_outputs={"chain_of_thought": "Test chain of thought"},
|
|
48
|
+
)
|
|
45
49
|
|
|
46
50
|
task_run = adapter.generate_run(
|
|
47
|
-
input=input_data, input_source=None,
|
|
51
|
+
input=input_data, input_source=None, run_output=run_output
|
|
48
52
|
)
|
|
49
53
|
task_run.save_to_file()
|
|
50
54
|
|
|
@@ -52,6 +56,9 @@ def test_save_run_isolation(test_task):
|
|
|
52
56
|
assert task_run.parent == test_task
|
|
53
57
|
assert task_run.input == input_data
|
|
54
58
|
assert task_run.input_source.type == DataSourceType.human
|
|
59
|
+
assert task_run.intermediate_outputs == {
|
|
60
|
+
"chain_of_thought": "Test chain of thought"
|
|
61
|
+
}
|
|
55
62
|
created_by = Config.shared().user_id
|
|
56
63
|
if created_by and created_by != "":
|
|
57
64
|
assert task_run.input_source.properties["created_by"] == created_by
|
|
@@ -86,13 +93,16 @@ def test_save_run_isolation(test_task):
|
|
|
86
93
|
)
|
|
87
94
|
|
|
88
95
|
# Run again, with same input and different output. Should create a new TaskRun.
|
|
89
|
-
|
|
96
|
+
different_run_output = RunOutput(
|
|
97
|
+
output="Different output", intermediate_outputs=None
|
|
98
|
+
)
|
|
99
|
+
task_output = adapter.generate_run(input_data, None, different_run_output)
|
|
90
100
|
task_output.save_to_file()
|
|
91
101
|
assert len(test_task.runs()) == 2
|
|
92
102
|
assert "Different output" in set(run.output.output for run in test_task.runs())
|
|
93
103
|
|
|
94
104
|
# run again with same input and same output. Should not create a new TaskRun.
|
|
95
|
-
task_output = adapter.generate_run(input_data, None,
|
|
105
|
+
task_output = adapter.generate_run(input_data, None, run_output)
|
|
96
106
|
task_output.save_to_file()
|
|
97
107
|
assert len(test_task.runs()) == 2
|
|
98
108
|
assert "Different output" in set(run.output.output for run in test_task.runs())
|
|
@@ -110,7 +120,7 @@ def test_save_run_isolation(test_task):
|
|
|
110
120
|
"adapter_name": "mock_adapter",
|
|
111
121
|
},
|
|
112
122
|
),
|
|
113
|
-
|
|
123
|
+
run_output,
|
|
114
124
|
)
|
|
115
125
|
task_output.save_to_file()
|
|
116
126
|
assert len(test_task.runs()) == 3
|
|
@@ -6,12 +6,17 @@ import jsonschema.exceptions
|
|
|
6
6
|
import pytest
|
|
7
7
|
|
|
8
8
|
import kiln_ai.datamodel as datamodel
|
|
9
|
-
from kiln_ai.adapters.base_adapter import AdapterInfo, BaseAdapter
|
|
9
|
+
from kiln_ai.adapters.base_adapter import AdapterInfo, BaseAdapter, RunOutput
|
|
10
10
|
from kiln_ai.adapters.langchain_adapters import LangChainPromptAdapter
|
|
11
11
|
from kiln_ai.adapters.ml_model_list import (
|
|
12
12
|
built_in_models,
|
|
13
13
|
ollama_online,
|
|
14
14
|
)
|
|
15
|
+
from kiln_ai.adapters.prompt_builders import (
|
|
16
|
+
BasePromptBuilder,
|
|
17
|
+
SimpleChainOfThoughtPromptBuilder,
|
|
18
|
+
)
|
|
19
|
+
from kiln_ai.adapters.test_prompt_adaptors import get_all_models_and_providers
|
|
15
20
|
from kiln_ai.datamodel.test_json_schema import json_joke_schema, json_triangle_schema
|
|
16
21
|
|
|
17
22
|
|
|
@@ -59,8 +64,8 @@ class MockAdapter(BaseAdapter):
|
|
|
59
64
|
super().__init__(kiln_task)
|
|
60
65
|
self.response = response
|
|
61
66
|
|
|
62
|
-
async def _run(self, input: str) ->
|
|
63
|
-
return self.response
|
|
67
|
+
async def _run(self, input: str) -> RunOutput:
|
|
68
|
+
return RunOutput(output=self.response, intermediate_outputs=None)
|
|
64
69
|
|
|
65
70
|
def adapter_info(self) -> AdapterInfo:
|
|
66
71
|
return AdapterInfo(
|
|
@@ -190,7 +195,18 @@ def build_structured_input_test_task(tmp_path: Path):
|
|
|
190
195
|
|
|
191
196
|
async def run_structured_input_test(tmp_path: Path, model_name: str, provider: str):
|
|
192
197
|
task = build_structured_input_test_task(tmp_path)
|
|
193
|
-
|
|
198
|
+
await run_structured_input_task(task, model_name, provider)
|
|
199
|
+
|
|
200
|
+
|
|
201
|
+
async def run_structured_input_task(
|
|
202
|
+
task: datamodel.Task,
|
|
203
|
+
model_name: str,
|
|
204
|
+
provider: str,
|
|
205
|
+
pb: BasePromptBuilder | None = None,
|
|
206
|
+
):
|
|
207
|
+
a = LangChainPromptAdapter(
|
|
208
|
+
task, model_name=model_name, provider=provider, prompt_builder=pb
|
|
209
|
+
)
|
|
194
210
|
with pytest.raises(ValueError):
|
|
195
211
|
# not structured input in dictionary
|
|
196
212
|
await a.invoke("a=1, b=2, c=3")
|
|
@@ -203,7 +219,10 @@ async def run_structured_input_test(tmp_path: Path, model_name: str, provider: s
|
|
|
203
219
|
assert isinstance(response, str)
|
|
204
220
|
assert "[[equilateral]]" in response
|
|
205
221
|
adapter_info = a.adapter_info()
|
|
206
|
-
|
|
222
|
+
expected_pb_name = "simple_prompt_builder"
|
|
223
|
+
if pb is not None:
|
|
224
|
+
expected_pb_name = pb.__class__.prompt_builder_name()
|
|
225
|
+
assert adapter_info.prompt_builder_name == expected_pb_name
|
|
207
226
|
assert adapter_info.model_name == model_name
|
|
208
227
|
assert adapter_info.model_provider == provider
|
|
209
228
|
assert adapter_info.adapter_name == "kiln_langchain_adapter"
|
|
@@ -224,3 +243,12 @@ async def test_all_built_in_models_structured_input(tmp_path):
|
|
|
224
243
|
await run_structured_input_test(tmp_path, model.name, provider.name)
|
|
225
244
|
except Exception as e:
|
|
226
245
|
raise RuntimeError(f"Error running {model.name} {provider}") from e
|
|
246
|
+
|
|
247
|
+
|
|
248
|
+
@pytest.mark.paid
|
|
249
|
+
@pytest.mark.ollama
|
|
250
|
+
@pytest.mark.parametrize("model_name,provider_name", get_all_models_and_providers())
|
|
251
|
+
async def test_structured_cot_prompt_builder(tmp_path, model_name, provider_name):
|
|
252
|
+
task = build_structured_input_test_task(tmp_path)
|
|
253
|
+
pb = SimpleChainOfThoughtPromptBuilder(task)
|
|
254
|
+
await run_structured_input_task(task, model_name, provider_name, pb)
|
kiln_ai/datamodel/__init__.py
CHANGED
|
@@ -48,8 +48,18 @@ __all__ = [
|
|
|
48
48
|
|
|
49
49
|
# Filename compatible names
|
|
50
50
|
NAME_REGEX = r"^[A-Za-z0-9 _-]+$"
|
|
51
|
-
NAME_FIELD = Field(
|
|
52
|
-
|
|
51
|
+
NAME_FIELD = Field(
|
|
52
|
+
min_length=1,
|
|
53
|
+
max_length=120,
|
|
54
|
+
pattern=NAME_REGEX,
|
|
55
|
+
description="A name for this entity.",
|
|
56
|
+
)
|
|
57
|
+
SHORT_NAME_FIELD = Field(
|
|
58
|
+
min_length=1,
|
|
59
|
+
max_length=32,
|
|
60
|
+
pattern=NAME_REGEX,
|
|
61
|
+
description="A name for this entity",
|
|
62
|
+
)
|
|
53
63
|
|
|
54
64
|
|
|
55
65
|
class Priority(IntEnum):
|
|
@@ -280,6 +290,10 @@ class TaskRun(KilnParentedModel):
|
|
|
280
290
|
default=None,
|
|
281
291
|
description="An version of the output with issues fixed. This must be a 'fixed' version of the existing output, and not an entirely new output. If you wish to generate an ideal curatorial output for this task unrelated to this output, generate a new TaskOutput with type 'human' instead of using this field.",
|
|
282
292
|
)
|
|
293
|
+
intermediate_outputs: Dict[str, str] | None = Field(
|
|
294
|
+
default=None,
|
|
295
|
+
description="Intermediate outputs from the task run. Keys are the names of the intermediate output steps (cot=chain of thought, etc), values are the output data.",
|
|
296
|
+
)
|
|
283
297
|
|
|
284
298
|
def parent_task(self) -> Task | None:
|
|
285
299
|
if not isinstance(self.parent, Task):
|
|
@@ -372,14 +386,21 @@ class Task(
|
|
|
372
386
|
"""
|
|
373
387
|
|
|
374
388
|
name: str = NAME_FIELD
|
|
375
|
-
description: str = Field(
|
|
376
|
-
|
|
377
|
-
|
|
378
|
-
|
|
389
|
+
description: str | None = Field(
|
|
390
|
+
default=None,
|
|
391
|
+
description="A description of the task for you and your team. Will not be used in prompts/training/validation.",
|
|
392
|
+
)
|
|
393
|
+
instruction: str = Field(
|
|
394
|
+
min_length=1,
|
|
395
|
+
description="The instructions for the task. Will be used in prompts/training/validation.",
|
|
396
|
+
)
|
|
379
397
|
requirements: List[TaskRequirement] = Field(default=[])
|
|
380
|
-
# TODO: make this required, or formalize the default message output schema
|
|
381
398
|
output_json_schema: JsonObjectSchema | None = None
|
|
382
399
|
input_json_schema: JsonObjectSchema | None = None
|
|
400
|
+
thinking_instruction: str | None = Field(
|
|
401
|
+
default=None,
|
|
402
|
+
description="Instructions for the model 'thinking' about the requirement prior to answering. Used for chain of thought style prompting.",
|
|
403
|
+
)
|
|
383
404
|
|
|
384
405
|
def output_schema(self) -> Dict | None:
|
|
385
406
|
if self.output_json_schema is None:
|
kiln_ai/datamodel/json_schema.py
CHANGED
|
@@ -64,6 +64,7 @@ def schema_from_json_str(v: str) -> Dict:
|
|
|
64
64
|
jsonschema.Draft202012Validator.check_schema(parsed)
|
|
65
65
|
if not isinstance(parsed, dict):
|
|
66
66
|
raise ValueError(f"JSON schema must be a dict, not {type(parsed)}")
|
|
67
|
+
# Top level arrays are valid JSON schemas, but we don't want to allow them here as they often cause issues
|
|
67
68
|
if (
|
|
68
69
|
"type" not in parsed
|
|
69
70
|
or parsed["type"] != "object"
|
kiln_ai/datamodel/test_models.py
CHANGED
|
@@ -3,7 +3,14 @@ import json
|
|
|
3
3
|
import pytest
|
|
4
4
|
from pydantic import ValidationError
|
|
5
5
|
|
|
6
|
-
from kiln_ai.datamodel import
|
|
6
|
+
from kiln_ai.datamodel import (
|
|
7
|
+
DataSource,
|
|
8
|
+
DataSourceType,
|
|
9
|
+
Project,
|
|
10
|
+
Task,
|
|
11
|
+
TaskOutput,
|
|
12
|
+
TaskRun,
|
|
13
|
+
)
|
|
7
14
|
from kiln_ai.datamodel.test_json_schema import json_joke_schema
|
|
8
15
|
|
|
9
16
|
|
|
@@ -62,9 +69,7 @@ def test_save_to_file(test_project_file):
|
|
|
62
69
|
|
|
63
70
|
def test_task_defaults():
|
|
64
71
|
task = Task(name="Test Task", instruction="Test Instruction")
|
|
65
|
-
assert task.description
|
|
66
|
-
assert task.priority == Priority.p2
|
|
67
|
-
assert task.determinism == TaskDeterminism.flexible
|
|
72
|
+
assert task.description is None
|
|
68
73
|
|
|
69
74
|
|
|
70
75
|
def test_task_serialization(test_project_file):
|
|
@@ -73,9 +78,8 @@ def test_task_serialization(test_project_file):
|
|
|
73
78
|
parent=project,
|
|
74
79
|
name="Test Task",
|
|
75
80
|
description="Test Description",
|
|
76
|
-
determinism=TaskDeterminism.semantic_match,
|
|
77
|
-
priority=Priority.p0,
|
|
78
81
|
instruction="Test Base Task Instruction",
|
|
82
|
+
thinking_instruction="Test Thinking Instruction",
|
|
79
83
|
)
|
|
80
84
|
|
|
81
85
|
task.save_to_file()
|
|
@@ -84,8 +88,7 @@ def test_task_serialization(test_project_file):
|
|
|
84
88
|
assert parsed_task.name == "Test Task"
|
|
85
89
|
assert parsed_task.description == "Test Description"
|
|
86
90
|
assert parsed_task.instruction == "Test Base Task Instruction"
|
|
87
|
-
assert parsed_task.
|
|
88
|
-
assert parsed_task.priority == Priority.p0
|
|
91
|
+
assert parsed_task.thinking_instruction == "Test Thinking Instruction"
|
|
89
92
|
|
|
90
93
|
|
|
91
94
|
def test_save_to_file_without_path():
|
|
@@ -189,3 +192,36 @@ def test_task_output_schema(tmp_path):
|
|
|
189
192
|
task = Task(name="Test Task", output_json_schema="{'asdf':{}}", path=path)
|
|
190
193
|
with pytest.raises(ValidationError):
|
|
191
194
|
task = Task(name="Test Task", input_json_schema="{asdf", path=path)
|
|
195
|
+
|
|
196
|
+
|
|
197
|
+
def test_task_run_intermediate_outputs():
|
|
198
|
+
# Create a basic task output
|
|
199
|
+
output = TaskOutput(
|
|
200
|
+
output="test output",
|
|
201
|
+
source=DataSource(
|
|
202
|
+
type=DataSourceType.synthetic,
|
|
203
|
+
properties={
|
|
204
|
+
"model_name": "test-model",
|
|
205
|
+
"model_provider": "test-provider",
|
|
206
|
+
"adapter_name": "test-adapter",
|
|
207
|
+
},
|
|
208
|
+
),
|
|
209
|
+
)
|
|
210
|
+
|
|
211
|
+
# Test valid intermediate outputs
|
|
212
|
+
task_run = TaskRun(
|
|
213
|
+
input="test input",
|
|
214
|
+
input_source=DataSource(
|
|
215
|
+
type=DataSourceType.human,
|
|
216
|
+
properties={"created_by": "test-user"},
|
|
217
|
+
),
|
|
218
|
+
output=output,
|
|
219
|
+
intermediate_outputs={
|
|
220
|
+
"cot": "chain of thought output",
|
|
221
|
+
"draft": "draft output",
|
|
222
|
+
},
|
|
223
|
+
)
|
|
224
|
+
assert task_run.intermediate_outputs == {
|
|
225
|
+
"cot": "chain of thought output",
|
|
226
|
+
"draft": "draft output",
|
|
227
|
+
}
|
kiln_ai/utils/config.py
CHANGED
|
@@ -97,7 +97,8 @@ class Config:
|
|
|
97
97
|
|
|
98
98
|
# Check if the value is in settings
|
|
99
99
|
if name in self._settings:
|
|
100
|
-
|
|
100
|
+
value = self._settings[name]
|
|
101
|
+
return value if value is None else property_config.type(value)
|
|
101
102
|
|
|
102
103
|
# Check environment variable
|
|
103
104
|
if property_config.env_var and property_config.env_var in os.environ:
|
|
@@ -110,7 +111,7 @@ class Config:
|
|
|
110
111
|
else:
|
|
111
112
|
value = property_config.default
|
|
112
113
|
|
|
113
|
-
return property_config.type(value)
|
|
114
|
+
return None if value is None else property_config.type(value)
|
|
114
115
|
|
|
115
116
|
def __setattr__(self, name, value):
|
|
116
117
|
if name in ("_properties", "_settings"):
|
kiln_ai/utils/test_config.py
CHANGED
|
@@ -26,6 +26,7 @@ def config_with_yaml(mock_yaml_file):
|
|
|
26
26
|
str, default="default_value", env_var="EXAMPLE_PROPERTY"
|
|
27
27
|
),
|
|
28
28
|
"int_property": ConfigProperty(int, default=0),
|
|
29
|
+
"empty_property": ConfigProperty(str),
|
|
29
30
|
}
|
|
30
31
|
)
|
|
31
32
|
|
|
@@ -67,6 +68,12 @@ def test_nonexistent_property(config_with_yaml):
|
|
|
67
68
|
config.nonexistent_property
|
|
68
69
|
|
|
69
70
|
|
|
71
|
+
def test_nonexistent_property_get_value(config_with_yaml):
|
|
72
|
+
config = config_with_yaml
|
|
73
|
+
assert config.get_value("nonexistent_property") is None
|
|
74
|
+
assert config.get_value("empty_property") is None
|
|
75
|
+
|
|
76
|
+
|
|
70
77
|
def test_property_type_conversion(config_with_yaml):
|
|
71
78
|
config = config_with_yaml
|
|
72
79
|
config = Config(properties={"int_property": ConfigProperty(int, default="42")})
|
|
@@ -1,13 +1,12 @@
|
|
|
1
1
|
Metadata-Version: 2.3
|
|
2
2
|
Name: kiln-ai
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.6.0
|
|
4
4
|
Summary: Kiln AI
|
|
5
5
|
Project-URL: Homepage, https://getkiln.ai
|
|
6
6
|
Project-URL: Repository, https://github.com/Kiln-AI/kiln
|
|
7
7
|
Project-URL: Documentation, https://kiln-ai.github.io/Kiln/kiln_core_docs/kiln_ai.html
|
|
8
8
|
Project-URL: Issues, https://github.com/Kiln-AI/kiln/issues
|
|
9
9
|
Author-email: "Steve Cosman, Chesterfield Laboratories Inc" <scosman@users.noreply.github.com>
|
|
10
|
-
License-File: LICENSE.txt
|
|
11
10
|
Classifier: Intended Audience :: Developers
|
|
12
11
|
Classifier: License :: OSI Approved :: MIT License
|
|
13
12
|
Classifier: Programming Language :: Python :: 3.10
|
|
@@ -31,6 +30,12 @@ Description-Content-Type: text/markdown
|
|
|
31
30
|
|
|
32
31
|
# kiln_ai
|
|
33
32
|
|
|
33
|
+
<p align="center">
|
|
34
|
+
<picture>
|
|
35
|
+
<img width="205" alt="Kiln AI Logo" src="https://github.com/user-attachments/assets/5fbcbdf7-1feb-45c9-bd73-99a46dd0a47f">
|
|
36
|
+
</picture>
|
|
37
|
+
</p>
|
|
38
|
+
|
|
34
39
|
[](https://pypi.org/project/kiln-ai)
|
|
35
40
|
[](https://pypi.org/project/kiln-ai)
|
|
36
41
|
[](https://kiln-ai.github.io/Kiln/kiln_core_docs/index.html)
|
|
@@ -43,12 +48,41 @@ Description-Content-Type: text/markdown
|
|
|
43
48
|
pip install kiln_ai
|
|
44
49
|
```
|
|
45
50
|
|
|
46
|
-
## About
|
|
51
|
+
## About
|
|
52
|
+
|
|
53
|
+
This package is the Kiln AI core library. There is also a separate desktop application and server package. Learn more about Kiln AI at [getkiln.ai](https://getkiln.ai)
|
|
54
|
+
|
|
55
|
+
- Github: [github.com/Kiln-AI/kiln](https://github.com/Kiln-AI/kiln)
|
|
56
|
+
- Core Library Docs: [https://kiln-ai.github.io/Kiln/kiln_core_docs/index.html](https://kiln-ai.github.io/Kiln/kiln_core_docs/index.html)
|
|
57
|
+
|
|
58
|
+
## Quick Start
|
|
47
59
|
|
|
48
|
-
|
|
60
|
+
```python
|
|
61
|
+
from kiln_ai.datamodel import Project
|
|
49
62
|
|
|
50
|
-
|
|
63
|
+
print("Reading Kiln project")
|
|
64
|
+
project = Project.load_from_file("path/to/project.kiln")
|
|
65
|
+
print("Project: ", project.name, " - ", project.description)
|
|
51
66
|
|
|
52
|
-
|
|
67
|
+
task = project.tasks()[0]
|
|
68
|
+
print("Task: ", task.name, " - ", task.description)
|
|
69
|
+
print("Total dataset size:", len(task.runs()))
|
|
53
70
|
|
|
54
|
-
|
|
71
|
+
# ... app specific code using the typed kiln datamodel
|
|
72
|
+
|
|
73
|
+
# Alternatively, load data into pandas or a similar tool:
|
|
74
|
+
import glob
|
|
75
|
+
import json
|
|
76
|
+
import pandas as pd
|
|
77
|
+
from pathlib import Path
|
|
78
|
+
|
|
79
|
+
dataitem_glob = str(task.path.parent) + "/runs/*/task_run.kiln"
|
|
80
|
+
|
|
81
|
+
dfs = []
|
|
82
|
+
for file in glob.glob(dataitem_glob):
|
|
83
|
+
js = json.loads(Path(file).read_text())
|
|
84
|
+
df = pd.json_normalize(js)
|
|
85
|
+
dfs.append(df)
|
|
86
|
+
final_df = pd.concat(dfs, ignore_index=True)
|
|
87
|
+
print(final_df)
|
|
88
|
+
```
|
|
@@ -0,0 +1,36 @@
|
|
|
1
|
+
kiln_ai/__init__.py,sha256=Sc4z8LRVFMwJUoc_DPVUriSXTZ6PO9MaJ80PhRbKyB8,34
|
|
2
|
+
kiln_ai/adapters/__init__.py,sha256=3NC1lE_Sg1bF4IsKCoUgje2GL0IwTd1atw1BcDLI8IA,883
|
|
3
|
+
kiln_ai/adapters/base_adapter.py,sha256=E_RfXxzEhW-i066xOhZdPuTM7OPKQv70hDpfMsxfYEs,6145
|
|
4
|
+
kiln_ai/adapters/langchain_adapters.py,sha256=iN26w-BTM14AtnaTU_maq57j7Lm2uKfrOlVronyHsno,6712
|
|
5
|
+
kiln_ai/adapters/ml_model_list.py,sha256=maEawe9BQ3T-2qSssbQ6tkB7mUzi0YhHP5R7uwmF6iA,26084
|
|
6
|
+
kiln_ai/adapters/prompt_builders.py,sha256=Mdu-f1mC9hWIDwoF7Qwd9F99GDx6oNGvtEZN-SrOsNM,10325
|
|
7
|
+
kiln_ai/adapters/test_langchain_adapter.py,sha256=sQU_ZDF2p022wU9eYg4QFKQva8yQvw8QRb6DoCvfywk,4703
|
|
8
|
+
kiln_ai/adapters/test_ml_model_list.py,sha256=3O0T86yLOXDF1XTc9BxdK-pYPRN0eRV_rgB_AfFCwbM,6375
|
|
9
|
+
kiln_ai/adapters/test_prompt_adaptors.py,sha256=T8ecAzQ9HMcEkxJIAKvF-Om6d0eIHT_FTIgZAJ_r9x0,7414
|
|
10
|
+
kiln_ai/adapters/test_prompt_builders.py,sha256=sU0bSBZa9Y4Q-mmkDf3HbQ0MNSWk5o9bC9sNgtnBokk,14598
|
|
11
|
+
kiln_ai/adapters/test_saving_adapter_results.py,sha256=SYYh2xY1zmeKhFHfWAuEY4pEiLd8SitSV5ewGOTmaOI,6447
|
|
12
|
+
kiln_ai/adapters/test_structured_output.py,sha256=o3w7r7pSjFxVUh4AIRflkbLRByLAG7A20M6nvdTwHkI,10035
|
|
13
|
+
kiln_ai/adapters/data_gen/data_gen_prompts.py,sha256=kudjHnAz7L3q0k_NLyTlaIV7M0uRFrxXNcfcnjOE2uc,5810
|
|
14
|
+
kiln_ai/adapters/data_gen/data_gen_task.py,sha256=ALe6byYp870CNFR08Vg9ti9vE94TzyGTGBPMuCTPL4o,3613
|
|
15
|
+
kiln_ai/adapters/data_gen/test_data_gen_task.py,sha256=rWcVqs8or5mXVbjQfRZz9K9IeRUMUedIeNTopj3RJQU,9428
|
|
16
|
+
kiln_ai/adapters/repair/__init__.py,sha256=dOO9MEpEhjiwzDVFg3MNfA2bKMPlax9iekDatpTkX8E,217
|
|
17
|
+
kiln_ai/adapters/repair/repair_task.py,sha256=VXvX1l9AYDE_GV0i3S_vPThltJoQlCFVCCHV9m-QA7k,3297
|
|
18
|
+
kiln_ai/adapters/repair/test_repair_task.py,sha256=kYCEIgRj_-b0AUqXokzheg7RzloWsRHWbUU_J6EQWDM,7938
|
|
19
|
+
kiln_ai/datamodel/__init__.py,sha256=bWuJnJl6whe-kTJEp9yQorzCbweEOjkHnIQHYBEARpY,15406
|
|
20
|
+
kiln_ai/datamodel/basemodel.py,sha256=lNSxZQO2isNP1JhDpc-bK6VqKWz1oY_Cc9RUT3T-Pz8,17535
|
|
21
|
+
kiln_ai/datamodel/json_schema.py,sha256=l4BIq1ItLHgcSHqsqDOchegLLHY48U4yR0SP2aMb4i0,2449
|
|
22
|
+
kiln_ai/datamodel/test_basemodel.py,sha256=679aT4UplglHtTVYmHGgT4v5uyJd7J2w8dBicZlLomk,9439
|
|
23
|
+
kiln_ai/datamodel/test_datasource.py,sha256=GAiZz31qezVVPwFqnt8wHMu15WvtlV89jw8C1Ue6YNI,3165
|
|
24
|
+
kiln_ai/datamodel/test_example_models.py,sha256=PxDtC8H3EfZem4kAQ-waFlRMlqnV4YlLEFKzz0Pq4tM,20013
|
|
25
|
+
kiln_ai/datamodel/test_json_schema.py,sha256=vdLnTQxxrcmuSrf6iOmkrmpfh7JnxqIw4B4dbDAAcZ4,3199
|
|
26
|
+
kiln_ai/datamodel/test_models.py,sha256=4VwggmkHEpTi8yHN48guVwtRUuBmDo7D_OQpA6DDXMo,6731
|
|
27
|
+
kiln_ai/datamodel/test_nested_save.py,sha256=xciCddqvPyKyoyjC5Lx_3Kh1t4LJv1xYRAPazR3SRcs,5588
|
|
28
|
+
kiln_ai/datamodel/test_output_rating.py,sha256=iw7fVUAPORA-0-VFiikZV3NDycGFaFMHSX1a38t_aQA,2647
|
|
29
|
+
kiln_ai/utils/__init__.py,sha256=PTD0MwBCKAMIOGsTAwsFaJOusTJJoRFTfOGqRvCaU-E,142
|
|
30
|
+
kiln_ai/utils/config.py,sha256=hjfshIWKLNMvNfpmgLJfermC3G6bgEgCXOSFogHG9FQ,5505
|
|
31
|
+
kiln_ai/utils/formatting.py,sha256=VtB9oag0lOGv17dwT7OPX_3HzBfaU9GsLH-iLete0yM,97
|
|
32
|
+
kiln_ai/utils/test_config.py,sha256=pTYItz5WD15rTRdxKE7vszXF_mb-dik2qrFWzkVemEY,7671
|
|
33
|
+
kiln_ai-0.6.0.dist-info/METADATA,sha256=l_PLws0QnE2TM9wbPCcqIAY0JhBcriRKbujTMFhQCO8,2979
|
|
34
|
+
kiln_ai-0.6.0.dist-info/WHEEL,sha256=C2FUgwZgiLbznR-k0b_5k3Ai_1aASOXDss3lzCUsUug,87
|
|
35
|
+
kiln_ai-0.6.0.dist-info/licenses/LICENSE.txt,sha256=_NA5pnTYgRRr4qH6lE3X-TuZJ8iRcMUi5ASoGr-lEx8,1209
|
|
36
|
+
kiln_ai-0.6.0.dist-info/RECORD,,
|
kiln_ai-0.5.4.dist-info/RECORD
DELETED
|
@@ -1,33 +0,0 @@
|
|
|
1
|
-
kiln_ai/__init__.py,sha256=Sc4z8LRVFMwJUoc_DPVUriSXTZ6PO9MaJ80PhRbKyB8,34
|
|
2
|
-
kiln_ai/adapters/__init__.py,sha256=3NC1lE_Sg1bF4IsKCoUgje2GL0IwTd1atw1BcDLI8IA,883
|
|
3
|
-
kiln_ai/adapters/base_adapter.py,sha256=xXCISAJHaPCYHad28CS0wZEUlx711FZ_6AwW4rJx4jk,6688
|
|
4
|
-
kiln_ai/adapters/langchain_adapters.py,sha256=WNxhuTdjGCsCyqmXJNLe7HJ-MzJ08yagGV-eAHPZF-E,5411
|
|
5
|
-
kiln_ai/adapters/ml_model_list.py,sha256=mqtFCav-m4m4DzxtoWHBktxrTB7haj0vUPx3xa8CL2U,23414
|
|
6
|
-
kiln_ai/adapters/prompt_builders.py,sha256=nfZnEr1E30ZweQhEzIP21rNrL2Or1ILajyX8gU3B7w0,7796
|
|
7
|
-
kiln_ai/adapters/test_langchain_adapter.py,sha256=_xHpVAkkoGh0PRO3BFFqvVj95SVtYZPOdFbYGYfzvQ0,1876
|
|
8
|
-
kiln_ai/adapters/test_ml_model_list.py,sha256=XHbwEFFb7WvZ6UkArqIiQ_yhS_urezHtgvJOSnaricY,4660
|
|
9
|
-
kiln_ai/adapters/test_prompt_adaptors.py,sha256=z_X-REnWmTai23Ay_xSEB1qRKpp3HAeqgs4K6TQneb0,6332
|
|
10
|
-
kiln_ai/adapters/test_prompt_builders.py,sha256=WmTR59tnKnKQ5gnX1X9EqvEUdQr0PQ8OvadYtRQR5sQ,11483
|
|
11
|
-
kiln_ai/adapters/test_saving_adapter_results.py,sha256=tQvpLawo8mR2scPwmRCIz9Sp0ZkerS3kVJKBzlcjwRE,6041
|
|
12
|
-
kiln_ai/adapters/test_structured_output.py,sha256=Z9A2R-TC-2atsdr8sGVGDlJhfa7uytW8Xi8PKBdEEAw,9033
|
|
13
|
-
kiln_ai/adapters/repair/__init__.py,sha256=dOO9MEpEhjiwzDVFg3MNfA2bKMPlax9iekDatpTkX8E,217
|
|
14
|
-
kiln_ai/adapters/repair/repair_task.py,sha256=VXvX1l9AYDE_GV0i3S_vPThltJoQlCFVCCHV9m-QA7k,3297
|
|
15
|
-
kiln_ai/adapters/repair/test_repair_task.py,sha256=12PHb4SgBvVdLUzjZz31M8OTa8D8QjHD0Du4s7ij-i8,7819
|
|
16
|
-
kiln_ai/datamodel/__init__.py,sha256=zkqabC2tdK3ia7qp-pur5lo3LFRYeszsA6SdjCdxerE,14761
|
|
17
|
-
kiln_ai/datamodel/basemodel.py,sha256=lNSxZQO2isNP1JhDpc-bK6VqKWz1oY_Cc9RUT3T-Pz8,17535
|
|
18
|
-
kiln_ai/datamodel/json_schema.py,sha256=mrb_oXf6gT-ahjBlzfIw2lT7iTmmowmMYxHnmVqPTgE,2334
|
|
19
|
-
kiln_ai/datamodel/test_basemodel.py,sha256=679aT4UplglHtTVYmHGgT4v5uyJd7J2w8dBicZlLomk,9439
|
|
20
|
-
kiln_ai/datamodel/test_datasource.py,sha256=GAiZz31qezVVPwFqnt8wHMu15WvtlV89jw8C1Ue6YNI,3165
|
|
21
|
-
kiln_ai/datamodel/test_example_models.py,sha256=PxDtC8H3EfZem4kAQ-waFlRMlqnV4YlLEFKzz0Pq4tM,20013
|
|
22
|
-
kiln_ai/datamodel/test_json_schema.py,sha256=vdLnTQxxrcmuSrf6iOmkrmpfh7JnxqIw4B4dbDAAcZ4,3199
|
|
23
|
-
kiln_ai/datamodel/test_models.py,sha256=FS0G4HRrE9SqX98ZO4SnEwire4AcXw5V96JIPu_Wi9Q,5930
|
|
24
|
-
kiln_ai/datamodel/test_nested_save.py,sha256=xciCddqvPyKyoyjC5Lx_3Kh1t4LJv1xYRAPazR3SRcs,5588
|
|
25
|
-
kiln_ai/datamodel/test_output_rating.py,sha256=iw7fVUAPORA-0-VFiikZV3NDycGFaFMHSX1a38t_aQA,2647
|
|
26
|
-
kiln_ai/utils/__init__.py,sha256=PTD0MwBCKAMIOGsTAwsFaJOusTJJoRFTfOGqRvCaU-E,142
|
|
27
|
-
kiln_ai/utils/config.py,sha256=jXUB8lwFkxLNEaizwIsoeFLg1BwjWr39-5KdEGF37Bg,5424
|
|
28
|
-
kiln_ai/utils/formatting.py,sha256=VtB9oag0lOGv17dwT7OPX_3HzBfaU9GsLH-iLete0yM,97
|
|
29
|
-
kiln_ai/utils/test_config.py,sha256=lbN0NhgKPEZ0idaS-zTn6mWsSAV6omo32JcIy05h2-M,7411
|
|
30
|
-
kiln_ai-0.5.4.dist-info/METADATA,sha256=UGIFG4gewCNOmUQpxYq_3g_dzCfq3tLuLLTVCnrhbOY,2017
|
|
31
|
-
kiln_ai-0.5.4.dist-info/WHEEL,sha256=1yFddiXMmvYK7QYTqtRNtX66WJ0Mz8PYEiEUoOUUxRY,87
|
|
32
|
-
kiln_ai-0.5.4.dist-info/licenses/LICENSE.txt,sha256=_NA5pnTYgRRr4qH6lE3X-TuZJ8iRcMUi5ASoGr-lEx8,1209
|
|
33
|
-
kiln_ai-0.5.4.dist-info/RECORD,,
|
|
File without changes
|