kiln-ai 0.7.0__py3-none-any.whl → 0.7.1__py3-none-any.whl

kiln_ai/datamodel/test_model_cache.py

@@ -0,0 +1,244 @@
+from pathlib import Path
+from unittest import mock
+
+import pytest
+from pydantic import BaseModel
+
+from libs.core.kiln_ai.datamodel.model_cache import ModelCache
+
+
+# Define a simple Pydantic model for testing
+class ModelTest(BaseModel):
+    name: str
+    value: int
+
+
+@pytest.fixture
+def model_cache():
+    return ModelCache()
+
+
+def should_skip_test(model_cache):
+    return not model_cache._enabled
+
+
+@pytest.fixture
+def test_path(tmp_path):
+    # Create a temporary file path for testing
+    test_file = tmp_path / "test_model.kiln"
+    test_file.touch()  # Create the file
+    return test_file
+
+
+def test_set_and_get_model(model_cache, test_path):
+    if not model_cache._enabled:
+        pytest.skip("Cache is disabled on this fs")
+
+    model = ModelTest(name="test", value=123)
+    mtime_ns = test_path.stat().st_mtime_ns
+
+    model_cache.set_model(test_path, model, mtime_ns)
+    cached_model = model_cache.get_model(test_path, ModelTest)
+
+    assert cached_model is not None
+    assert cached_model.name == "test"
+    assert cached_model.value == 123
+
+
+def test_invalidate_model(model_cache, test_path):
+    model = ModelTest(name="test", value=123)
+    mtime = test_path.stat().st_mtime
+
+    model_cache.set_model(test_path, model, mtime)
+    model_cache.invalidate(test_path)
+    cached_model = model_cache.get_model(test_path, ModelTest)
+
+    assert cached_model is None
+
+
+def test_clear_cache(model_cache, test_path):
+    model = ModelTest(name="test", value=123)
+    mtime = test_path.stat().st_mtime
+
+    model_cache.set_model(test_path, model, mtime)
+    model_cache.clear()
+    cached_model = model_cache.get_model(test_path, ModelTest)
+
+    assert cached_model is None
+
+
+def test_cache_invalid_due_to_mtime_change(model_cache, test_path):
+    model = ModelTest(name="test", value=123)
+    mtime = test_path.stat().st_mtime
+
+    model_cache.set_model(test_path, model, mtime)
+
+    # Simulate a file modification by updating the mtime
+    test_path.touch()
+    cached_model = model_cache.get_model(test_path, ModelTest)
+
+    assert cached_model is None
+
+
+def test_get_model_wrong_type(model_cache, test_path):
+    if not model_cache._enabled:
+        pytest.skip("Cache is disabled on this fs")
+
+    class AnotherModel(BaseModel):
+        other_field: str
+
+    model = ModelTest(name="test", value=123)
+    mtime_ns = test_path.stat().st_mtime_ns
+
+    model_cache.set_model(test_path, model, mtime_ns)
+
+    with pytest.raises(ValueError):
+        model_cache.get_model(test_path, AnotherModel)
+
+    # Test that the cache invalidates
+    cached_model = model_cache.get_model(test_path, ModelTest)
+    assert cached_model is None
+
+
+def test_is_cache_valid_true(model_cache, test_path):
+    mtime_ns = test_path.stat().st_mtime_ns
+    assert model_cache._is_cache_valid(test_path, mtime_ns) is True
+
+
+def test_is_cache_valid_false_due_to_mtime_change(model_cache, test_path):
+    if not model_cache._enabled:
+        pytest.skip("Cache is disabled on this fs")
+
+    mtime_ns = test_path.stat().st_mtime_ns
+    # Simulate a file modification by updating the mtime
+    test_path.touch()
+    assert model_cache._is_cache_valid(test_path, mtime_ns) is False
+
+
+def test_is_cache_valid_false_due_to_missing_file(model_cache):
+    non_existent_path = Path("/non/existent/path")
+    assert model_cache._is_cache_valid(non_existent_path, 0) is False
+
+
+def test_benchmark_get_model(benchmark, model_cache, test_path):
+    model = ModelTest(name="test", value=123)
+    mtime = test_path.stat().st_mtime
+
+    # Set the model in the cache
+    model_cache.set_model(test_path, model, mtime)
+
+    # Benchmark the get_model method
+    def get_model():
+        return model_cache.get_model(test_path, ModelTest)
+
+    benchmark(get_model)
+    stats = benchmark.stats.stats
+
+    # 25k ops per second is the target. Getting 250k on Macbook, but CI will be slower
+    target = 1 / 25000
+    if stats.mean > target:
+        pytest.fail(
+            f"Average time per iteration: {stats.mean}, expected less than {target}"
+        )
+
+
+def test_get_model_returns_copy(model_cache, test_path):
+    if not model_cache._enabled:
+        pytest.skip("Cache is disabled on this fs")
+
+    model = ModelTest(name="test", value=123)
+    mtime_ns = test_path.stat().st_mtime_ns
+
+    # Set the model in the cache
+    model_cache.set_model(test_path, model, mtime_ns)
+
+    # Get a copy of the model from the cache
+    cached_model = model_cache.get_model(test_path, ModelTest)
+
+    # Different instance (is), same data (==)
+    assert cached_model is not model
+    assert cached_model == model
+
+    # Mutate the cached model
+    cached_model.name = "mutated"
+
+    # Get the model again from the cache
+    new_cached_model = model_cache.get_model(test_path, ModelTest)
+
+    # Assert that the new cached model has the original values
+    assert new_cached_model == model
+    assert new_cached_model.name == "test"
+
+    # Save the mutated model back to the cache
+    model_cache.set_model(test_path, cached_model, mtime_ns)
+
+    # Get the model again from the cache
+    updated_cached_model = model_cache.get_model(test_path, ModelTest)
+
+    # Assert that the updated cached model has the mutated values
+    assert updated_cached_model.name == "mutated"
+    assert updated_cached_model.value == 123
+
+
+def test_no_cache_when_no_fine_granularity(model_cache, test_path):
+    model = ModelTest(name="test", value=123)
+    mtime_ns = test_path.stat().st_mtime_ns
+
+    model_cache._enabled = False
+    model_cache.set_model(test_path, model, mtime_ns)
+    cached_model = model_cache.get_model(test_path, ModelTest)
+
+    # Assert that the model is not cached
+    assert cached_model is None
+    assert model_cache.model_cache == {}
+    assert model_cache._enabled is False
+
+
+def test_check_timestamp_granularity_macos():
+    with mock.patch("sys.platform", "darwin"):
+        cache = ModelCache()
+        assert cache._check_timestamp_granularity() is True
+        assert cache._enabled is True
+
+
+def test_check_timestamp_granularity_windows():
+    with mock.patch("sys.platform", "win32"):
+        cache = ModelCache()
+        assert cache._check_timestamp_granularity() is True
+        assert cache._enabled is True
+
+
+def test_check_timestamp_granularity_linux_good():
+    mock_stats = mock.Mock()
+    mock_stats.f_timespec = 9  # nanosecond precision
+
+    with (
+        mock.patch("sys.platform", "linux"),
+        mock.patch("os.statvfs", return_value=mock_stats),
+    ):
+        cache = ModelCache()
+        assert cache._check_timestamp_granularity() is True
+        assert cache._enabled is True
+
+
+def test_check_timestamp_granularity_linux_poor():
+    mock_stats = mock.Mock()
+    mock_stats.f_timespec = 3  # millisecond precision
+
+    with (
+        mock.patch("sys.platform", "linux"),
+        mock.patch("os.statvfs", return_value=mock_stats),
+    ):
+        cache = ModelCache()
+        assert cache._check_timestamp_granularity() is False
+        assert cache._enabled is False
+
+
+def test_check_timestamp_granularity_linux_error():
+    with (
+        mock.patch("sys.platform", "linux"),
+        mock.patch("os.statvfs", side_effect=OSError("Mock filesystem error")),
+    ):
+        cache = ModelCache()
+        assert cache._check_timestamp_granularity() is False
+        assert cache._enabled is False

kiln_ai/datamodel/test_models.py

@@ -1,4 +1,6 @@
 import json
+import os
+from unittest.mock import patch
 
 import pytest
 from pydantic import ValidationError
@@ -82,6 +84,7 @@ def test_task_serialization(test_project_file):
         instruction="Test Base Task Instruction",
         thinking_instruction="Test Thinking Instruction",
     )
+    assert task._loaded_from_file is False
 
     task.save_to_file()
 
@@ -90,6 +93,11 @@ def test_task_serialization(test_project_file):
     assert parsed_task.description == "Test Description"
     assert parsed_task.instruction == "Test Base Task Instruction"
     assert parsed_task.thinking_instruction == "Test Thinking Instruction"
+    assert parsed_task._loaded_from_file is True
+
+    # Confirm the local property is not persisted to disk
+    json_data = json.loads(parsed_task.path.read_text())
+    assert "_loaded_from_file" not in json_data
 
 
 def test_save_to_file_without_path():
@@ -315,3 +323,119 @@ def test_finetune_parameters_validation():
             base_model_id="gpt-3.5-turbo",
             parameters={"invalid": [1, 2, 3]},  # Lists are not allowed
         )
+
+
+def test_task_run_input_source_validation(tmp_path):
+    # Setup basic output for TaskRun creation
+    output = TaskOutput(
+        output="test output",
+        source=DataSource(
+            type=DataSourceType.synthetic,
+            properties={
+                "model_name": "test-model",
+                "model_provider": "test-provider",
+                "adapter_name": "test-adapter",
+            },
+        ),
+    )
+
+    project_path = tmp_path / "project.kiln"
+    project = Project(name="Test Project", path=project_path)
+    project.save_to_file()
+    task = Task(name="Test Task", instruction="Test Instruction", parent=project)
+    task.save_to_file()
+
+    # Test 1: Creating without input_source should work when strict mode is off
+    task_run = TaskRun(
+        input="test input",
+        output=output,
+    )
+    task_run.parent = task
+    assert task_run.input_source is None
+
+    # Save for later usage
+    task_run.save_to_file()
+    task_missing_input_source = task_run.path
+
+    # Test 2: Creating with input_source should work when strict mode is off
+    task_run = TaskRun(
+        input="test input 2",
+        input_source=DataSource(
+            type=DataSourceType.human,
+            properties={"created_by": "test-user"},
+        ),
+        output=output,
+    )
+    assert task_run.input_source is not None
+
+    # Test 3: Creating without input_source should fail when strict mode is on
+    with patch("kiln_ai.datamodel.strict_mode", return_value=True):
+        with pytest.raises(ValueError) as exc_info:
+            task_run = TaskRun(
+                input="test input 3",
+                output=output,
+            )
+        assert "input_source is required when strict mode is enabled" in str(
+            exc_info.value
+        )
+
+        # Test 4: Loading from disk should work without input_source, even with strict mode on
+        assert os.path.exists(task_missing_input_source)
+        task_run = TaskRun.load_from_file(task_missing_input_source)
+        assert task_run.input_source is None
+
+
+def test_task_output_source_validation(tmp_path):
+    # Setup basic output source for validation
+    output_source = DataSource(
+        type=DataSourceType.synthetic,
+        properties={
+            "model_name": "test-model",
+            "model_provider": "test-provider",
+            "adapter_name": "test-adapter",
+        },
+    )
+
+    project_path = tmp_path / "project.kiln"
+    project = Project(name="Test Project", path=project_path)
+    project.save_to_file()
+    task = Task(name="Test Task", instruction="Test Instruction", parent=project)
+    task.save_to_file()
+
+    # Test 1: Creating without source should work when strict mode is off
+    task_output = TaskOutput(
+        output="test output",
+    )
+    assert task_output.source is None
+
+    # Save for later usage
+    task_run = TaskRun(
+        input="test input",
+        input_source=output_source,
+        output=task_output,
+    )
+    task_run.parent = task
+    task_run.save_to_file()
+    task_missing_output_source = task_run.path
+
+    # Test 2: Creating with source should work when strict mode is off
+    task_output = TaskOutput(
+        output="test output 2",
+        source=output_source,
+    )
+    assert task_output.source is not None
+
+    # Test 3: Creating without source should fail when strict mode is on
+    with patch("kiln_ai.datamodel.strict_mode", return_value=True):
+        with pytest.raises(ValueError) as exc_info:
+            task_output = TaskOutput(
+                output="test output 3",
+            )
+        assert "Output source is required when strict mode is enabled" in str(
+            exc_info.value
+        )
+
+        # Test 4: Loading from disk should work without source, even with strict mode on
+        assert os.path.exists(task_missing_output_source)
+        task_run = TaskRun.load_from_file(task_missing_output_source)
+        assert task_run.output.source is None

kiln_ai/utils/config.py

@@ -80,6 +80,10 @@ class Config:
                 list,
                 default_lambda=lambda: [],
             ),
+            "custom_models": ConfigProperty(
+                list,
+                default_lambda=lambda: [],
+            ),
         }
         self._settings = self.load_settings()
 
@@ -145,7 +149,7 @@ class Config:
             settings = yaml.safe_load(f.read()) or {}
         return settings
 
-    def settings(self, hide_sensitive=False):
+    def settings(self, hide_sensitive=False) -> Dict[str, Any]:
         if hide_sensitive:
             return {
                 k: "[hidden]"

kiln_ai-0.7.1.dist-info/METADATA

@@ -0,0 +1,237 @@
+Metadata-Version: 2.4
+Name: kiln-ai
+Version: 0.7.1
+Summary: Kiln AI
+Project-URL: Homepage, https://getkiln.ai
+Project-URL: Repository, https://github.com/Kiln-AI/kiln
+Project-URL: Documentation, https://kiln-ai.github.io/Kiln/kiln_core_docs/kiln_ai.html
+Project-URL: Issues, https://github.com/Kiln-AI/kiln/issues
+Author-email: "Steve Cosman, Chesterfield Laboratories Inc" <scosman@users.noreply.github.com>
+License-File: LICENSE.txt
+Classifier: Intended Audience :: Developers
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Requires-Python: >=3.10
+Requires-Dist: coverage>=7.6.4
+Requires-Dist: jsonschema>=4.23.0
+Requires-Dist: langchain-aws>=0.2.4
+Requires-Dist: langchain-fireworks>=0.2.5
+Requires-Dist: langchain-groq>=0.2.0
+Requires-Dist: langchain-ollama>=0.2.0
+Requires-Dist: langchain-openai>=0.2.4
+Requires-Dist: langchain>=0.3.5
+Requires-Dist: openai>=1.53.0
+Requires-Dist: pdoc>=15.0.0
+Requires-Dist: pydantic>=2.9.2
+Requires-Dist: pytest-benchmark>=5.1.0
+Requires-Dist: pytest-cov>=6.0.0
+Requires-Dist: pyyaml>=6.0.2
+Requires-Dist: typing-extensions>=4.12.2
+Description-Content-Type: text/markdown
+
+# Kiln AI Core Library
+
+<p align="center">
+    <picture>
+        <img width="205" alt="Kiln AI Logo" src="https://github.com/user-attachments/assets/5fbcbdf7-1feb-45c9-bd73-99a46dd0a47f">
+    </picture>
+</p>
+
+[![PyPI - Version](https://img.shields.io/pypi/v/kiln-ai.svg?logo=pypi&label=PyPI&logoColor=gold)](https://pypi.org/project/kiln-ai)
+[![PyPI - Python Version](https://img.shields.io/pypi/pyversions/kiln-ai.svg)](https://pypi.org/project/kiln-ai)
+[![Docs](https://img.shields.io/badge/docs-pdoc-blue)](https://kiln-ai.github.io/Kiln/kiln_core_docs/index.html)
+
+---
+
+## Installation
+
+```console
+pip install kiln_ai
+```
+
+## About
+
+This package is the Kiln AI core library. There is also a separate desktop application and server package. Learn more about Kiln AI at [getkiln.ai](https://getkiln.ai) and on Github: [github.com/Kiln-AI/kiln](https://github.com/Kiln-AI/kiln).
+
+# Guide: Using the Kiln Python Library
+
+In this guide we'll walk common examples of how to use the library.
+
+## Documentation
+
+The library has a [comprehensive set of docs](https://kiln-ai.github.io/Kiln/kiln_core_docs/index.html).
+
+## Table of Contents
+
+- [Using the Kiln Data Model](#using-the-kiln-data-model)
+  - [Understanding the Kiln Data Model](#understanding-the-kiln-data-model)
+  - [Datamodel Overview](#datamodel-overview)
+  - [Load a Project](#load-a-project)
+  - [Load an Existing Dataset into a Kiln Task Dataset](#load-an-existing-dataset-into-a-kiln-task-dataset)
+  - [Using your Kiln Dataset in a Notebook or Project](#using-your-kiln-dataset-in-a-notebook-or-project)
+  - [Using Kiln Dataset in Pandas](#using-kiln-dataset-in-pandas)
+- [Advanced Usage](#advanced-usage)
+
+## Installation
+
+```bash
+pip install kiln-ai
+```
+
+## Using the Kiln Data Model
+
+### Understanding the Kiln Data Model
+
+Kiln projects are simply a directory of files (mostly JSON files with the extension `.kiln`) that describe your project, including tasks, runs, and other data.
+
+This dataset design was chosen for several reasons:
+
+- Git compatibility: Kiln project folders are easy to collaborate on in git. The filenames use unique IDs to avoid conflicts and allow many people to work in parallel. The files are small and easy to compare using standard diff tools.
+- JSON allows you to easily load and manipulate the data using standard tools (pandas, polars, etc)
+
+The Kiln Python library provides a set of Python classes that which help you easily interact with your Kiln dataset. Using the library to load and manipulate your dataset is the fastest way to get started, and will guarantees you don't insert any invalid data into your dataset. There's extensive validation when using the library, so we recommend using it to load and manipulate your dataset over direct JSON manipulation.
+
+### Datamodel Overview
+
+- Project: a Kiln Project that organizes related tasks
+  - Task: a specific task including prompt instructions, input/output schemas, and requirements
+    - TaskRun: a sample (run) of a task including input, output and human rating information
+    - DatasetSplit: a frozen collection of task runs divided into train/test/validation splits
+    - Finetune: configuration and status tracking for fine-tuning models on task data
+
+### Load a Project
+
+Assuming you've created a project in the Kiln UI, you'll have a `project.kiln` file in your `~/Kiln Projects/Project Name` directory.
+
+```python
+from kiln_ai.datamodel import Project
+
+project = Project.load_from_file("path/to/your/project.kiln")
+print("Project: ", project.name, " - ", project.description)
+
+# List all tasks in the project, and their dataset sizes
+tasks = project.tasks()
+for task in tasks:
+    print("Task: ", task.name, " - ", task.description)
+    print("Total dataset size:", len(task.runs()))
+```
+
+### Load an Existing Dataset into a Kiln Task Dataset
+
+If you already have a dataset in a file, you can load it into a Kiln project.
+
+**Important**: Kiln will validate the input and output schemas, and ensure that each datapoint in the dataset is valid for this task.
+
+- Plaintext input/output: ensure "output_json_schema" and "input_json_schema" not set in your Task definition.
+- JSON input/output: ensure "output_json_schema" and "input_json_schema" are valid JSON schemas in your Task definition. Every datapoint in the dataset must be valid JSON fitting the schema.
+
+Here's a simple example of how to load a dataset into a Kiln task:
+
+```python
+
+import kiln_ai
+import kiln_ai.datamodel
+
+# Created a project and task via the UI and put its path here
+task_path = "/Users/youruser/Kiln Projects/test project/tasks/632780983478 - Joke Generator/task.kiln"
+task = kiln_ai.datamodel.Task.load_from_file(task_path)
+
+# Add data to the task - loop over you dataset and run this for each item
+item = kiln_ai.datamodel.TaskRun(
+    parent=task,
+    input='{"topic": "AI"}',
+    output=kiln_ai.datamodel.TaskOutput(
+        output='{"setup": "What is AI?", "punchline": "content_here"}',
+    ),
+)
+item.save_to_file()
+print("Saved item to file: ", item.path)
+```
+
+And here's a more complex example of how to load a dataset into a Kiln task. This example sets the source of the data (human in this case, but you can also set it be be synthetic), the created_by property, and a 5-star rating.
+
+```python
+import kiln_ai
+import kiln_ai.datamodel
+
+# Created a project and task via the UI and put its path here
+task_path = "/Users/youruser/Kiln Projects/test project/tasks/632780983478 - Joke Generator/task.kiln"
+task = kiln_ai.datamodel.Task.load_from_file(task_path)
+
+# Add data to the task - loop over you dataset and run this for each item
+item = kiln_ai.datamodel.TaskRun(
+    parent=task,
+    input='{"topic": "AI"}',
+    input_source=kiln_ai.datamodel.DataSource(
+        type=kiln_ai.datamodel.DataSourceType.human,
+        properties={"created_by": "John Doe"},
+    ),
+    output=kiln_ai.datamodel.TaskOutput(
+        output='{"setup": "What is AI?", "punchline": "content_here"}',
+        source=kiln_ai.datamodel.DataSource(
+            type=kiln_ai.datamodel.DataSourceType.human,
+            properties={"created_by": "Jane Doe"},
+        ),
+        rating=kiln_ai.datamodel.TaskOutputRating(score=5,type="five_star"),
+    ),
+)
+item.save_to_file()
+print("Saved item to file: ", item.path)
+```
+
+### Using your Kiln Dataset in a Notebook or Project
+
+You can use your Kiln dataset in a notebook or project by loading the dataset into a pandas dataframe.
+
+```python
+import kiln_ai
+import kiln_ai.datamodel
+
+# Created a project and task via the UI and put its path here
+task_path = "/Users/youruser/Kiln Projects/test project/tasks/632780983478 - Joke Generator/task.kiln"
+task = kiln_ai.datamodel.Task.load_from_file(task_path)
+
+runs = task.runs()
+for run in runs:
+    print(f"Input: {run.input}")
+    print(f"Output: {run.output.output}")
+
+print(f"Total runs: {len(runs)}")
+```
+
+### Using Kiln Dataset in Pandas
+
+You can also use your Kiln dataset in a pandas dataframe, or a similar script for other tools like polars.
+
+```python
+import glob
+import json
+import pandas as pd
+from pathlib import Path
+
+task_dir = "/Users/youruser/Kiln Projects/test project/tasks/632780983478 - Joke Generator"
+dataitem_glob = task_dir + "/runs/*/task_run.kiln"
+
+dfs = []
+for file in glob.glob(dataitem_glob):
+    js = json.loads(Path(file).read_text())
+
+    df = pd.DataFrame([{
+        "input": js["input"],
+        "output": js["output"]["output"],
+    }])
+
+    # Alternatively: you can use pd.json_normalize(js) to get the full json structure
+    # df = pd.json_normalize(js)
+    dfs.append(df)
+final_df = pd.concat(dfs, ignore_index=True)
+print(final_df)
+```
+
+### Advanced Usage
+
+The library can do a lot more than the examples we've shown here.
+
+See the [docs](https://kiln-ai.github.io/Kiln/kiln_core_docs/index.html) for more information.