data-designer 0.3.7__tar.gz → 0.3.8rc1__tar.gz

{data_designer-0.3.7 → data_designer-0.3.8rc1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: data-designer
-Version: 0.3.7
+Version: 0.3.8rc1
 Summary: General framework for synthetic data generation
 License-Expression: Apache-2.0
 License-File: LICENSE

{data_designer-0.3.7 → data_designer-0.3.8rc1}/src/data_designer/_version.py

@@ -28,7 +28,7 @@ version_tuple: VERSION_TUPLE
 commit_id: COMMIT_ID
 __commit_id__: COMMIT_ID
 
-__version__ = version = '0.3.7'
-__version_tuple__ = version_tuple = (0, 3, 7)
+__version__ = version = '0.3.8rc1'
+__version_tuple__ = version_tuple = (0, 3, 8, 'rc1')
 
 __commit_id__ = commit_id = None

{data_designer-0.3.7 → data_designer-0.3.8rc1}/src/data_designer/engine/column_generators/generators/llm_completion.py

@@ -55,6 +55,9 @@ class ColumnGeneratorWithModelChatCompletion(ColumnGeneratorWithModel[TaskConfig
         )
 
     def generate(self, data: dict) -> dict:
+        # Deserialize input data from previous columns so Jinja2 templates can access nested fields
+        # Example: If prev column stored '{"key": "value"}', templates can use {{ prev_column.key }}
+        # Note: This creates a new dict and doesn't mutate the original `data` argument
         deserialized_record = deserialize_json_values(data)
 
         multi_modal_context = None
@@ -81,13 +84,18 @@ class ColumnGeneratorWithModelChatCompletion(ColumnGeneratorWithModel[TaskConfig
             purpose=f"running generation for column '{self.config.name}'",
         )
 
-        data[self.config.name] = deserialize_json_values(self.response_recipe.serialize_output(response))
+        serialized_output = self.response_recipe.serialize_output(response)
+        data[self.config.name] = self._process_serialized_output(serialized_output)
 
         if reasoning_trace:
             data[self.config.name + REASONING_TRACE_COLUMN_POSTFIX] = reasoning_trace
 
         return data
 
+    def _process_serialized_output(self, serialized_output: str) -> str | dict | list:
+        """Process the serialized output from the model. Subclasses can override to customize deserialization."""
+        return serialized_output
+
 
 class LLMTextCellGenerator(ColumnGeneratorWithModelChatCompletion[LLMTextColumnConfig]): ...
 
@@ -95,7 +103,11 @@ class LLMTextCellGenerator(ColumnGeneratorWithModelChatCompletion[LLMTextColumnC
 class LLMCodeCellGenerator(ColumnGeneratorWithModelChatCompletion[LLMCodeColumnConfig]): ...
 
 
-class LLMStructuredCellGenerator(ColumnGeneratorWithModelChatCompletion[LLMStructuredColumnConfig]): ...
+class LLMStructuredCellGenerator(ColumnGeneratorWithModelChatCompletion[LLMStructuredColumnConfig]):
+    def _process_serialized_output(self, serialized_output: str) -> dict | list:
+        return deserialize_json_values(serialized_output)
 
 
-class LLMJudgeCellGenerator(ColumnGeneratorWithModelChatCompletion[LLMJudgeColumnConfig]): ...
+class LLMJudgeCellGenerator(ColumnGeneratorWithModelChatCompletion[LLMJudgeColumnConfig]):
+    def _process_serialized_output(self, serialized_output: str) -> dict | list:
+        return deserialize_json_values(serialized_output)

{data_designer-0.3.7 → data_designer-0.3.8rc1}/src/data_designer/engine/compiler.py

@@ -5,9 +5,10 @@ from __future__ import annotations
 
 import logging
 
-from data_designer.config.column_configs import SeedDatasetColumnConfig
+from data_designer.config.column_configs import SamplerColumnConfig, SeedDatasetColumnConfig
 from data_designer.config.data_designer_config import DataDesignerConfig
 from data_designer.config.errors import InvalidConfigError
+from data_designer.config.sampler_params import UUIDSamplerParams
 from data_designer.engine.resources.resource_provider import ResourceProvider
 from data_designer.engine.resources.seed_reader import SeedReader
 from data_designer.engine.validation import ViolationLevel, rich_print_violations, validate_data_designer_config
@@ -17,6 +18,7 @@ logger = logging.getLogger(__name__)
 
 def compile_data_designer_config(config: DataDesignerConfig, resource_provider: ResourceProvider) -> DataDesignerConfig:
     _resolve_and_add_seed_columns(config, resource_provider.seed_reader)
+    _add_internal_row_id_column_if_needed(config)
     _validate(config)
     return config
 
@@ -41,6 +43,35 @@ def _resolve_and_add_seed_columns(config: DataDesignerConfig, seed_reader: SeedR
     config.columns.extend([SeedDatasetColumnConfig(name=col_name) for col_name in seed_col_names])
 
 
+def _add_internal_row_id_column_if_needed(config: DataDesignerConfig) -> None:
+    """Adds a UUID sampler column named '_internal_row_id' (set to drop) if needed to enable generation.
+
+    Generation requires either:
+    - At least one sampler column (which can generate data from scratch), OR
+    - A seed dataset (which provides initial data rows)
+
+    If neither exists, a UUID sampler column '_internal_row_id' is automatically added and marked for drop
+    to enable the generation process to start.
+
+    Args:
+        config: The DataDesigner configuration to potentially modify.
+    """
+    has_sampler_column = any(isinstance(col, SamplerColumnConfig) for col in config.columns)
+    has_seed_dataset_column = any(isinstance(col, SeedDatasetColumnConfig) for col in config.columns)
+
+    if not has_sampler_column and not has_seed_dataset_column:
+        logger.warning(
+            "🔔 No sampler column or seed dataset detected. Adding UUID column '_internal_row_id' (marked for drop) to enable generation."
+        )
+        id_column = SamplerColumnConfig(
+            name="_internal_row_id",
+            sampler_type="uuid",
+            params=UUIDSamplerParams(),
+            drop=True,
+        )
+        config.columns.insert(0, id_column)
+
+
 def _validate(config: DataDesignerConfig) -> None:
     allowed_references = _get_allowed_references(config)
     violations = validate_data_designer_config(

{data_designer-0.3.7 → data_designer-0.3.8rc1}/src/data_designer/engine/processing/utils.py

@@ -4,6 +4,7 @@
 from __future__ import annotations
 
 import ast
+import copy
 import json
 import logging
 import re
@@ -50,6 +51,8 @@ def deserialize_json_values(data: T) -> T: ...
 def deserialize_json_values(data):
     """De-serialize JSON strings in various input formats.
 
+    This function creates a deep copy of the input data and does not mutate the original.
+
     Args:
         data: Input data in one of four formats:
             - Single string (JSON string to deserialize)
@@ -63,18 +66,22 @@ def deserialize_json_values(data):
             - List of dictionaries (when input is a list of strings)
             - Dictionary (when input is a dictionary, with nested JSON strings deserialized)
             - The original object (if there is no deserialization to perform)
+
     """
+    # Create a deep copy to avoid mutating the original data
+    data_copy = copy.deepcopy(data)
+
     # Case 1: Single string input
-    if isinstance(data, str):
+    if isinstance(data_copy, str):
         try:
-            return json.loads(data)
+            return json.loads(data_copy)
         except json.JSONDecodeError:
-            return data
+            return data_copy
 
     # Case 2: List of strings input
-    elif isinstance(data, list):
+    elif isinstance(data_copy, list):
         result = []
-        for item in data:
+        for item in data_copy:
             if isinstance(item, str):
                 try:
                     result.append(json.loads(item))
@@ -86,9 +93,9 @@ def deserialize_json_values(data):
         return result
 
     # Case 3: Dictionary input with potential nested JSON strings
-    elif isinstance(data, dict):
+    elif isinstance(data_copy, dict):
         result = {}
-        for key, value in data.items():
+        for key, value in data_copy.items():
             if isinstance(value, str):
                 try:
                     result[key] = json.loads(value)
@@ -103,7 +110,7 @@ def deserialize_json_values(data):
 
     # Fallback for other data types
     else:
-        return data
+        return data_copy
 
 
 def parse_list_string(text: str) -> list[str]:

{data_designer-0.3.7 → data_designer-0.3.8rc1}/tests/engine/column_generators/generators/test_llm_completion_generators.py

@@ -269,3 +269,82 @@ def test_generate_with_json_deserialization():
     result = generator.generate(data)
 
     assert result["test_column"] == {"result": "json_output"}
+
+
+@pytest.mark.parametrize(
+    "generator_class,config_class,config_kwargs,serialized_output,expected_output",
+    [
+        (
+            LLMTextCellGenerator,
+            LLMTextColumnConfig,
+            {"name": "text_col", "prompt": "Generate text", "model_alias": "test_model"},
+            '["plain", "text", "output"]',
+            '["plain", "text", "output"]',
+        ),
+        (
+            LLMCodeCellGenerator,
+            LLMCodeColumnConfig,
+            {"name": "code_col", "prompt": "Generate code", "model_alias": "test_model", "code_lang": "python"},
+            "def hello(): pass",
+            "def hello(): pass",
+        ),
+        (
+            LLMStructuredCellGenerator,
+            LLMStructuredColumnConfig,
+            {
+                "name": "struct_col",
+                "prompt": "Generate struct",
+                "model_alias": "test_model",
+                "output_format": {"type": "object", "properties": {"field": {"type": "string"}}},
+            },
+            '{"field": "value", "nested": {"key": "val"}}',
+            {"field": "value", "nested": {"key": "val"}},
+        ),
+        (
+            LLMJudgeCellGenerator,
+            LLMJudgeColumnConfig,
+            {
+                "name": "judge_col",
+                "prompt": "Judge this",
+                "model_alias": "test_model",
+                "scores": [{"name": "quality", "description": "Quality", "options": {1: "good", 0: "bad"}}],
+            },
+            '{"quality": 1, "reasoning": "Good quality"}',
+            {"quality": 1, "reasoning": "Good quality"},
+        ),
+    ],
+)
+def test_generator_output_type_handling(
+    stub_resource_provider: Mock,
+    generator_class: type,
+    config_class: type,
+    config_kwargs: dict,
+    serialized_output: str,
+    expected_output: str | dict,
+) -> None:
+    """Test that each generator type correctly handles its output format via polymorphism.
+
+    - Text/Code generators return plain strings
+    - Structured/Judge generators deserialize JSON to Python objects
+    """
+    config = config_class(**config_kwargs)
+    generator = generator_class(config=config, resource_provider=stub_resource_provider)
+
+    # Mock the prompt renderer and response recipe
+    mock_prompt_renderer = Mock()
+    mock_response_recipe = Mock()
+    generator.prompt_renderer = mock_prompt_renderer
+    generator.response_recipe = mock_response_recipe
+
+    # Setup mocks
+    mock_prompt_renderer.render.side_effect = ["rendered_user_prompt", "rendered_system_prompt"]
+    mock_response_recipe.serialize_output.return_value = serialized_output
+    stub_resource_provider.model_registry.get_model.return_value.generate.return_value = (
+        {"result": "raw_output"},
+        None,
+    )
+
+    data = {"input": "test_input"}
+    result = generator.generate(data)
+
+    assert result[config.name] == expected_output

{data_designer-0.3.7 → data_designer-0.3.8rc1}/tests/engine/dataset_builders/test_column_wise_builder.py

@@ -13,6 +13,7 @@ from data_designer.config.config_builder import DataDesignerConfigBuilder
 from data_designer.config.dataset_builders import BuildStage
 from data_designer.config.processors import DropColumnsProcessorConfig
 from data_designer.config.run_config import RunConfig
+from data_designer.config.sampler_params import SamplerType, UUIDSamplerParams
 from data_designer.engine.column_generators.generators.base import GenerationStrategy
 from data_designer.engine.dataset_builders.column_wise_builder import (
     MAX_CONCURRENCY_PER_NON_LLM_GENERATOR,
@@ -31,6 +32,7 @@ if TYPE_CHECKING:
 @pytest.fixture
 def stub_test_column_configs():
     return [
+        SamplerColumnConfig(name="some_id", sampler_type=SamplerType.UUID, params=UUIDSamplerParams()),
         LLMTextColumnConfig(name="test_column", prompt="Test prompt", model_alias="test_model"),
         LLMTextColumnConfig(name="column_to_drop", prompt="Test prompt", model_alias="test_model"),
     ]
@@ -92,8 +94,7 @@ def test_column_wise_dataset_builder_creation(stub_resource_provider, stub_test_
         data_designer_config=stub_test_config_builder.build(),
         resource_provider=stub_resource_provider,
     )
-
-    assert len(builder._column_configs) == 2
+    assert len(builder._column_configs) == 3
     assert builder._resource_provider == stub_resource_provider
     assert isinstance(builder._registry, DataDesignerRegistry)
 
@@ -136,26 +137,33 @@ def test_column_wise_dataset_builder_batch_manager_initialization(stub_column_wi
 def test_column_wise_dataset_builder_single_column_configs_property(
     stub_resource_provider, stub_model_configs, config_type, expected_single_configs
 ):
+    config_builder = DataDesignerConfigBuilder(model_configs=stub_model_configs)
+
     if config_type == "single":
-        single_config = LLMTextColumnConfig(name="test_column", prompt="Test prompt", model_alias="test_model")
-        config_builder = DataDesignerConfigBuilder(model_configs=stub_model_configs)
+        # Add an LLM text column - these don't get grouped into MultiColumnConfigs
+        single_config = expected_single_configs[0]
         config_builder.add_column(single_config)
+
         builder = ColumnWiseDatasetBuilder(
             data_designer_config=config_builder.build(),
             resource_provider=stub_resource_provider,
         )
-        assert builder.single_column_configs == [single_config]
+
+        # Since there's no sampler, _internal_row_id is auto-added, plus the LLM column
+        configs = builder.single_column_configs
+        assert len(configs) == 2
+        assert configs[0].name == "_internal_row_id"
+        assert configs[1] == single_config
+
     else:
-        sampler_config = SamplerColumnConfig(
-            name="sampler_col", sampler_type="category", params={"values": ["A", "B", "C"]}
-        )
-        config_builder = DataDesignerConfigBuilder(model_configs=stub_model_configs)
+        sampler_config = expected_single_configs[0]
         config_builder.add_column(sampler_config)
+
         builder = ColumnWiseDatasetBuilder(
             data_designer_config=config_builder.build(),
             resource_provider=stub_resource_provider,
         )
-        assert builder.single_column_configs == [sampler_config]
+        assert builder.single_column_configs == expected_single_configs
 
 
 def test_column_wise_dataset_builder_build_method_basic_flow(

{data_designer-0.3.7 → data_designer-0.3.8rc1}/tests/engine/test_compiler.py

@@ -5,10 +5,10 @@ from unittest.mock import patch
 
 import pytest
 
-from data_designer.config.column_configs import SamplerColumnConfig
+from data_designer.config.column_configs import ExpressionColumnConfig, SamplerColumnConfig
 from data_designer.config.config_builder import DataDesignerConfigBuilder
 from data_designer.config.errors import InvalidConfigError
-from data_designer.config.sampler_params import CategorySamplerParams, SamplerType
+from data_designer.config.sampler_params import CategorySamplerParams, SamplerType, UUIDSamplerParams
 from data_designer.config.seed_source import HuggingFaceSeedSource
 from data_designer.engine.compiler import compile_data_designer_config
 from data_designer.engine.resources.resource_provider import ResourceProvider
@@ -78,3 +78,69 @@ def test_validation_errors(resource_provider: ResourceProvider):
             compile_data_designer_config(builder.build(), resource_provider)
 
     assert "validation errors" in str(excinfo)
+
+
+def test_adds_id_column_when_no_sampler_and_no_seed_dataset(stub_resource_provider: ResourceProvider):
+    """Test that a UUID '_internal_row_id' column is automatically added when there's no sampler column or seed dataset."""
+    builder = DataDesignerConfigBuilder()
+    builder.add_column(
+        ExpressionColumnConfig(
+            name="derived_value",
+            expr="'constant_value'",
+        )
+    )
+    stub_resource_provider.seed_reader = None
+
+    config = compile_data_designer_config(builder.build(), stub_resource_provider)
+
+    assert len(config.columns) == 2
+    assert config.columns[0].name == "_internal_row_id"
+    assert isinstance(config.columns[0], SamplerColumnConfig)
+    assert config.columns[0].sampler_type == "uuid"
+    assert isinstance(config.columns[0].params, UUIDSamplerParams)
+    assert config.columns[0].drop is True
+
+
+def test_does_not_add_id_column_when_sampler_exists(stub_resource_provider: ResourceProvider):
+    """Test that no '_internal_row_id' column is added when a sampler column already exists."""
+    builder = DataDesignerConfigBuilder()
+    builder.add_column(
+        SamplerColumnConfig(
+            name="category",
+            sampler_type=SamplerType.CATEGORY,
+            params=CategorySamplerParams(values=["a", "b", "c"]),
+        )
+    )
+    builder.add_column(
+        ExpressionColumnConfig(
+            name="derived_value",
+            expr="{{ category }}_suffix",
+        )
+    )
+    stub_resource_provider.seed_reader = None
+
+    config = compile_data_designer_config(builder.build(), stub_resource_provider)
+
+    assert len(config.columns) == 2
+    assert config.columns[0].name == "category"
+    assert config.columns[1].name == "derived_value"
+    assert not any(col.name == "_internal_row_id" for col in config.columns)
+
+
+def test_does_not_add_id_column_when_seed_dataset_exists(resource_provider: ResourceProvider):
+    """Test that no '_internal_row_id' column is added when a seed dataset is configured."""
+    builder = DataDesignerConfigBuilder()
+    builder.add_column(
+        ExpressionColumnConfig(
+            name="derived_value",
+            expr="{{ city }}_derived",
+        )
+    )
+    builder.with_seed_dataset(HuggingFaceSeedSource(path="hf://datasets/test/data.csv"))
+
+    config = compile_data_designer_config(builder.build(), resource_provider)
+
+    # Should have the expression column + 2 seed columns (city, country) from the fixture
+    assert len(config.columns) == 3
+    assert config.columns[0].name == "derived_value"
+    assert not any(col.name == "_internal_row_id" for col in config.columns)