kiln-ai 0.7.0__py3-none-any.whl → 0.8.0__py3-none-any.whl

kiln_ai/datamodel/test_model_cache.py

@@ -0,0 +1,244 @@
+from pathlib import Path
+from unittest import mock
+
+import pytest
+from pydantic import BaseModel
+
+from libs.core.kiln_ai.datamodel.model_cache import ModelCache
+
+
+# Define a simple Pydantic model for testing
+class ModelTest(BaseModel):
+    name: str
+    value: int
+
+
+@pytest.fixture
+def model_cache():
+    return ModelCache()
+
+
+def should_skip_test(model_cache):
+    return not model_cache._enabled
+
+
+@pytest.fixture
+def test_path(tmp_path):
+    # Create a temporary file path for testing
+    test_file = tmp_path / "test_model.kiln"
+    test_file.touch()  # Create the file
+    return test_file
+
+
+def test_set_and_get_model(model_cache, test_path):
+    if not model_cache._enabled:
+        pytest.skip("Cache is disabled on this fs")
+
+    model = ModelTest(name="test", value=123)
+    mtime_ns = test_path.stat().st_mtime_ns
+
+    model_cache.set_model(test_path, model, mtime_ns)
+    cached_model = model_cache.get_model(test_path, ModelTest)
+
+    assert cached_model is not None
+    assert cached_model.name == "test"
+    assert cached_model.value == 123
+
+
+def test_invalidate_model(model_cache, test_path):
+    model = ModelTest(name="test", value=123)
+    mtime = test_path.stat().st_mtime
+
+    model_cache.set_model(test_path, model, mtime)
+    model_cache.invalidate(test_path)
+    cached_model = model_cache.get_model(test_path, ModelTest)
+
+    assert cached_model is None
+
+
+def test_clear_cache(model_cache, test_path):
+    model = ModelTest(name="test", value=123)
+    mtime = test_path.stat().st_mtime
+
+    model_cache.set_model(test_path, model, mtime)
+    model_cache.clear()
+    cached_model = model_cache.get_model(test_path, ModelTest)
+
+    assert cached_model is None
+
+
+def test_cache_invalid_due_to_mtime_change(model_cache, test_path):
+    model = ModelTest(name="test", value=123)
+    mtime = test_path.stat().st_mtime
+
+    model_cache.set_model(test_path, model, mtime)
+
+    # Simulate a file modification by updating the mtime
+    test_path.touch()
+    cached_model = model_cache.get_model(test_path, ModelTest)
+
+    assert cached_model is None
+
+
+def test_get_model_wrong_type(model_cache, test_path):
+    if not model_cache._enabled:
+        pytest.skip("Cache is disabled on this fs")
+
+    class AnotherModel(BaseModel):
+        other_field: str
+
+    model = ModelTest(name="test", value=123)
+    mtime_ns = test_path.stat().st_mtime_ns
+
+    model_cache.set_model(test_path, model, mtime_ns)
+
+    with pytest.raises(ValueError):
+        model_cache.get_model(test_path, AnotherModel)
+
+    # Test that the cache invalidates
+    cached_model = model_cache.get_model(test_path, ModelTest)
+    assert cached_model is None
+
+
+def test_is_cache_valid_true(model_cache, test_path):
+    mtime_ns = test_path.stat().st_mtime_ns
+    assert model_cache._is_cache_valid(test_path, mtime_ns) is True
+
+
+def test_is_cache_valid_false_due_to_mtime_change(model_cache, test_path):
+    if not model_cache._enabled:
+        pytest.skip("Cache is disabled on this fs")
+
+    mtime_ns = test_path.stat().st_mtime_ns
+    # Simulate a file modification by updating the mtime
+    test_path.touch()
+    assert model_cache._is_cache_valid(test_path, mtime_ns) is False
+
+
+def test_is_cache_valid_false_due_to_missing_file(model_cache):
+    non_existent_path = Path("/non/existent/path")
+    assert model_cache._is_cache_valid(non_existent_path, 0) is False
+
+
+def test_benchmark_get_model(benchmark, model_cache, test_path):
+    model = ModelTest(name="test", value=123)
+    mtime = test_path.stat().st_mtime
+
+    # Set the model in the cache
+    model_cache.set_model(test_path, model, mtime)
+
+    # Benchmark the get_model method
+    def get_model():
+        return model_cache.get_model(test_path, ModelTest)
+
+    benchmark(get_model)
+    stats = benchmark.stats.stats
+
+    # 25k ops per second is the target. Getting 250k on Macbook, but CI will be slower
+    target = 1 / 25000
+    if stats.mean > target:
+        pytest.fail(
+            f"Average time per iteration: {stats.mean}, expected less than {target}"
+        )
+
+
+def test_get_model_returns_copy(model_cache, test_path):
+    if not model_cache._enabled:
+        pytest.skip("Cache is disabled on this fs")
+
+    model = ModelTest(name="test", value=123)
+    mtime_ns = test_path.stat().st_mtime_ns
+
+    # Set the model in the cache
+    model_cache.set_model(test_path, model, mtime_ns)
+
+    # Get a copy of the model from the cache
+    cached_model = model_cache.get_model(test_path, ModelTest)
+
+    # Different instance (is), same data (==)
+    assert cached_model is not model
+    assert cached_model == model
+
+    # Mutate the cached model
+    cached_model.name = "mutated"
+
+    # Get the model again from the cache
+    new_cached_model = model_cache.get_model(test_path, ModelTest)
+
+    # Assert that the new cached model has the original values
+    assert new_cached_model == model
+    assert new_cached_model.name == "test"
+
+    # Save the mutated model back to the cache
+    model_cache.set_model(test_path, cached_model, mtime_ns)
+
+    # Get the model again from the cache
+    updated_cached_model = model_cache.get_model(test_path, ModelTest)
+
+    # Assert that the updated cached model has the mutated values
+    assert updated_cached_model.name == "mutated"
+    assert updated_cached_model.value == 123
+
+
+def test_no_cache_when_no_fine_granularity(model_cache, test_path):
+    model = ModelTest(name="test", value=123)
+    mtime_ns = test_path.stat().st_mtime_ns
+
+    model_cache._enabled = False
+    model_cache.set_model(test_path, model, mtime_ns)
+    cached_model = model_cache.get_model(test_path, ModelTest)
+
+    # Assert that the model is not cached
+    assert cached_model is None
+    assert model_cache.model_cache == {}
+    assert model_cache._enabled is False
+
+
+def test_check_timestamp_granularity_macos():
+    with mock.patch("sys.platform", "darwin"):
+        cache = ModelCache()
+        assert cache._check_timestamp_granularity() is True
+        assert cache._enabled is True
+
+
+def test_check_timestamp_granularity_windows():
+    with mock.patch("sys.platform", "win32"):
+        cache = ModelCache()
+        assert cache._check_timestamp_granularity() is True
+        assert cache._enabled is True
+
+
+def test_check_timestamp_granularity_linux_good():
+    mock_stats = mock.Mock()
+    mock_stats.f_timespec = 9  # nanosecond precision
+
+    with (
+        mock.patch("sys.platform", "linux"),
+        mock.patch("os.statvfs", return_value=mock_stats),
+    ):
+        cache = ModelCache()
+        assert cache._check_timestamp_granularity() is True
+        assert cache._enabled is True
+
+
+def test_check_timestamp_granularity_linux_poor():
+    mock_stats = mock.Mock()
+    mock_stats.f_timespec = 3  # millisecond precision
+
+    with (
+        mock.patch("sys.platform", "linux"),
+        mock.patch("os.statvfs", return_value=mock_stats),
+    ):
+        cache = ModelCache()
+        assert cache._check_timestamp_granularity() is False
+        assert cache._enabled is False
+
+
+def test_check_timestamp_granularity_linux_error():
+    with (
+        mock.patch("sys.platform", "linux"),
+        mock.patch("os.statvfs", side_effect=OSError("Mock filesystem error")),
+    ):
+        cache = ModelCache()
+        assert cache._check_timestamp_granularity() is False
+        assert cache._enabled is False

kiln_ai/datamodel/test_models.py

@@ -1,4 +1,6 @@
 import json
+import os
+from unittest.mock import patch
 
 import pytest
 from pydantic import ValidationError
@@ -82,6 +84,7 @@ def test_task_serialization(test_project_file):
         instruction="Test Base Task Instruction",
         thinking_instruction="Test Thinking Instruction",
     )
+    assert task._loaded_from_file is False
 
     task.save_to_file()
 
@@ -90,6 +93,11 @@ def test_task_serialization(test_project_file):
     assert parsed_task.description == "Test Description"
     assert parsed_task.instruction == "Test Base Task Instruction"
     assert parsed_task.thinking_instruction == "Test Thinking Instruction"
+    assert parsed_task._loaded_from_file is True
+
+    # Confirm the local property is not persisted to disk
+    json_data = json.loads(parsed_task.path.read_text())
+    assert "_loaded_from_file" not in json_data
 
 
 def test_save_to_file_without_path():
@@ -315,3 +323,168 @@ def test_finetune_parameters_validation():
             base_model_id="gpt-3.5-turbo",
             parameters={"invalid": [1, 2, 3]},  # Lists are not allowed
         )
+
+
+def test_task_run_input_source_validation(tmp_path):
+    # Setup basic output for TaskRun creation
+    output = TaskOutput(
+        output="test output",
+        source=DataSource(
+            type=DataSourceType.synthetic,
+            properties={
+                "model_name": "test-model",
+                "model_provider": "test-provider",
+                "adapter_name": "test-adapter",
+            },
+        ),
+    )
+
+    project_path = tmp_path / "project.kiln"
+    project = Project(name="Test Project", path=project_path)
+    project.save_to_file()
+    task = Task(name="Test Task", instruction="Test Instruction", parent=project)
+    task.save_to_file()
+
+    # Test 1: Creating without input_source should work when strict mode is off
+    task_run = TaskRun(
+        input="test input",
+        output=output,
+    )
+    task_run.parent = task
+    assert task_run.input_source is None
+
+    # Save for later usage
+    task_run.save_to_file()
+    task_missing_input_source = task_run.path
+
+    # Test 2: Creating with input_source should work when strict mode is off
+    task_run = TaskRun(
+        input="test input 2",
+        input_source=DataSource(
+            type=DataSourceType.human,
+            properties={"created_by": "test-user"},
+        ),
+        output=output,
+    )
+    assert task_run.input_source is not None
+
+    # Test 3: Creating without input_source should fail when strict mode is on
+    with patch("kiln_ai.datamodel.strict_mode", return_value=True):
+        with pytest.raises(ValueError) as exc_info:
+            task_run = TaskRun(
+                input="test input 3",
+                output=output,
+            )
+        assert "input_source is required when strict mode is enabled" in str(
+            exc_info.value
+        )
+
+        # Test 4: Loading from disk should work without input_source, even with strict mode on
+        assert os.path.exists(task_missing_input_source)
+        task_run = TaskRun.load_from_file(task_missing_input_source)
+        assert task_run.input_source is None
+
+
+def test_task_output_source_validation(tmp_path):
+    # Setup basic output source for validation
+    output_source = DataSource(
+        type=DataSourceType.synthetic,
+        properties={
+            "model_name": "test-model",
+            "model_provider": "test-provider",
+            "adapter_name": "test-adapter",
+        },
+    )
+
+    project_path = tmp_path / "project.kiln"
+    project = Project(name="Test Project", path=project_path)
+    project.save_to_file()
+    task = Task(name="Test Task", instruction="Test Instruction", parent=project)
+    task.save_to_file()
+
+    # Test 1: Creating without source should work when strict mode is off
+    task_output = TaskOutput(
+        output="test output",
+    )
+    assert task_output.source is None
+
+    # Save for later usage
+    task_run = TaskRun(
+        input="test input",
+        input_source=output_source,
+        output=task_output,
+    )
+    task_run.parent = task
+    task_run.save_to_file()
+    task_missing_output_source = task_run.path
+
+    # Test 2: Creating with source should work when strict mode is off
+    task_output = TaskOutput(
+        output="test output 2",
+        source=output_source,
+    )
+    assert task_output.source is not None
+
+    # Test 3: Creating without source should fail when strict mode is on
+    with patch("kiln_ai.datamodel.strict_mode", return_value=True):
+        with pytest.raises(ValueError) as exc_info:
+            task_output = TaskOutput(
+                output="test output 3",
+            )
+        assert "Output source is required when strict mode is enabled" in str(
+            exc_info.value
+        )
+
+        # Test 4: Loading from disk should work without source, even with strict mode on
+        assert os.path.exists(task_missing_output_source)
+        task_run = TaskRun.load_from_file(task_missing_output_source)
+        assert task_run.output.source is None
+
+
+def test_task_run_tags_validation():
+    # Setup basic output for TaskRun creation
+    output = TaskOutput(
+        output="test output",
+        source=DataSource(
+            type=DataSourceType.synthetic,
+            properties={
+                "model_name": "test-model",
+                "model_provider": "test-provider",
+                "adapter_name": "test-adapter",
+            },
+        ),
+    )
+
+    # Test 1: Valid tags should work
+    task_run = TaskRun(
+        input="test input",
+        output=output,
+        tags=["test_tag", "another_tag", "tag123"],
+    )
+    assert task_run.tags == ["test_tag", "another_tag", "tag123"]
+
+    # Test 2: Empty list of tags should work
+    task_run = TaskRun(
+        input="test input",
+        output=output,
+        tags=[],
+    )
+    assert task_run.tags == []
+
+    # Test 3: Empty string tag should fail
+    with pytest.raises(ValueError) as exc_info:
+        TaskRun(
+            input="test input",
+            output=output,
+            tags=["valid_tag", ""],
+        )
+    assert "Tags cannot be empty strings" in str(exc_info.value)
+
+    # Test 4: Tag with spaces should fail
+    with pytest.raises(ValueError) as exc_info:
+        TaskRun(
+            input="test input",
+            output=output,
+            tags=["valid_tag", "invalid tag"],
+        )
+    assert "Tags cannot contain spaces. Try underscores." in str(exc_info.value)