data-designer 0.2.1__tar.gz → 0.2.2__tar.gz

{data_designer-0.2.1 → data_designer-0.2.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: data-designer
-Version: 0.2.1
+Version: 0.2.2
 Summary: General framework for synthetic data generation
 License-Expression: Apache-2.0
 License-File: LICENSE
@@ -15,36 +15,34 @@ Classifier: Programming Language :: Python :: 3.13
 Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
 Classifier: Topic :: Software Development
 Requires-Python: >=3.10
-Requires-Dist: anyascii<1.0,>=0.3.3
-Requires-Dist: datasets>=4.0.0
-Requires-Dist: duckdb==1.1.3
-Requires-Dist: faker==20.1.0
-Requires-Dist: httpx-retries>=0.4.2
-Requires-Dist: httpx>=0.27.2
-Requires-Dist: huggingface-hub>=0.34.4
+Requires-Dist: anyascii<1,>=0.3.3
+Requires-Dist: duckdb<2,>=1.1.3
+Requires-Dist: faker<21,>=20.1.0
+Requires-Dist: httpx-retries<1,>=0.4.2
+Requires-Dist: httpx<1,>=0.27.2
+Requires-Dist: huggingface-hub<2,>=1.0.1
 Requires-Dist: jinja2<4,>=3.1.6
-Requires-Dist: json-repair==0.48.0
-Requires-Dist: jsonpath-rust-bindings>=1.0
-Requires-Dist: litellm==1.73.6
-Requires-Dist: lxml>=6.0.2
-Requires-Dist: marko==2.1.2
-Requires-Dist: networkx==3.0
-Requires-Dist: numpy>=1.23.5
-Requires-Dist: pandas>=1.5.3
-Requires-Dist: prompt-toolkit>=3.0.0
-Requires-Dist: pyarrow>=19.0.1
-Requires-Dist: pydantic>=2.9.2
-Requires-Dist: pydantic[email]>=2.9.2
-Requires-Dist: pygments>=2.19.2
-Requires-Dist: python-json-logger==2.0.7
-Requires-Dist: pyyaml>=6.0.1
+Requires-Dist: json-repair<1,>=0.48.0
+Requires-Dist: jsonpath-rust-bindings<2,>=1.0
+Requires-Dist: litellm<2,>=1.73.6
+Requires-Dist: lxml<7,>=6.0.2
+Requires-Dist: marko<3,>=2.1.2
+Requires-Dist: networkx<4,>=3.0
+Requires-Dist: numpy<3,>=1.23.5
+Requires-Dist: pandas<3,>=2.3.3
+Requires-Dist: prompt-toolkit<4,>=3.0.0
+Requires-Dist: pyarrow<20,>=19.0.1
+Requires-Dist: pydantic[email]<3,>=2.9.2
+Requires-Dist: pygments<3,>=2.19.2
+Requires-Dist: python-json-logger<4,>=3
+Requires-Dist: pyyaml<7,>=6.0.1
 Requires-Dist: requests<3,>=2.32.2
-Requires-Dist: rich>=13.7.1
-Requires-Dist: ruff==0.12.3
-Requires-Dist: scipy>=1.11.0
-Requires-Dist: sqlfluff==3.2.0
-Requires-Dist: tiktoken>=0.8.0
-Requires-Dist: typer>=0.12.0
+Requires-Dist: rich<14,>=13.7.1
+Requires-Dist: ruff<1,>=0.14.10
+Requires-Dist: scipy<2,>=1.11.0
+Requires-Dist: sqlfluff<4,>=3.2.0
+Requires-Dist: tiktoken<1,>=0.8.0
+Requires-Dist: typer<1,>=0.12.0
 Description-Content-Type: text/markdown
 
 # 🎨 NeMo Data Designer

{data_designer-0.2.1 → data_designer-0.2.2}/pyproject.toml

@@ -20,36 +20,34 @@ classifiers = [
 ]
 
 dependencies = [
-  "huggingface-hub>=0.34.4",
+  "anyascii>=0.3.3,<1",
+  "duckdb>=1.1.3,<2",
+  "faker>=20.1.0,<21",
+  "httpx-retries>=0.4.2,<1",
+  "httpx>=0.27.2,<1",
+  "huggingface-hub>=1.0.1,<2",
   "jinja2>=3.1.6,<4",
-  "numpy>=1.23.5",
-  "pandas>=1.5.3",
-  "pyarrow>=19.0.1",
-  "pydantic>=2.9.2",
-  "pygments>=2.19.2",
-  "pyyaml>=6.0.1",
-  "python-json-logger==2.0.7",
-  "prompt-toolkit>=3.0.0",
+  "json-repair>=0.48.0,<1",
+  "jsonpath-rust-bindings>=1.0,<2",
+  "litellm>=1.73.6,<2",
+  "lxml>=6.0.2,<7",
+  "marko>=2.1.2,<3",
+  "networkx>=3.0,<4",
+  "numpy>=1.23.5,<3",
+  "pandas>=2.3.3,<3",
+  "prompt-toolkit>=3.0.0,<4",
+  "pyarrow>=19.0.1,<20",
+  "pydantic[email]>=2.9.2,<3",
+  "pygments>=2.19.2,<3",
+  "python-json-logger>=3,<4",
+  "pyyaml>=6.0.1,<7",
   "requests<3,>=2.32.2",
-  "rich>=13.7.1",
-  "typer>=0.12.0",
-  "anyascii>=0.3.3,<1.0",
-  "datasets>=4.0.0",
-  "duckdb==1.1.3",
-  "faker==20.1.0",
-  "httpx>=0.27.2",
-  "httpx-retries>=0.4.2",
-  "json-repair==0.48.0",
-  "jsonpath-rust-bindings>=1.0",
-  "litellm==1.73.6",
-  "lxml>=6.0.2",
-  "marko==2.1.2",
-  "networkx==3.0",
-  "pydantic[email]>=2.9.2",
-  "scipy>=1.11.0",
-  "sqlfluff==3.2.0",
-  "tiktoken>=0.8.0",
-  "ruff==0.12.3",
+  "rich>=13.7.1,<14",
+  "ruff>=0.14.10,<1",
+  "scipy>=1.11.0,<2",
+  "sqlfluff>=3.2.0,<4",
+  "tiktoken>=0.8.0,<1",
+  "typer>=0.12.0,<1",
 ]
 
 [project.scripts]
@@ -57,27 +55,28 @@ data-designer = "data_designer.cli:main"
 
 [dependency-groups]
 dev = [
-  "jsonpath-ng==1.5.3",
-  "pre-commit>=4.0.0",
-  "pytest>=8.3.3",
-  "pytest-asyncio>=0.24.0",
-  "pytest-cov>=7.0.0",
-  "pytest-env>=1.2.0",
-  "pytest-httpx>=0.35.0",
+  "jsonpath-ng>=1.5.3,<2",
+  "pre-commit>=4.0.0,<5",
+  "pytest-asyncio>=0.24.0,<1",
+  "pytest-cov>=7.0.0,<8",
+  "pytest-env>=1.2.0,<2",
+  "pytest-httpx>=0.35.0,<1",
+  "pytest>=8.3.3,<9",
 ]
 docs = [
-  "mkdocs>=1.6.1",
-  "mkdocstrings>=0.30.1",
-  "mkdocstrings-python>=1.18.2",
-  "mkdocs-material>=9.6.22",
-  "mkdocs-jupyter>=0.25.1",
-  "mkdocs-redirects>=1.2.2",
-  "mike>=2.1.3",
+  "datasets>=4.0.0,<5",
+  "mike>=2.1.3,<3",
+  "mkdocs-jupyter>=0.25.1,<1",
+  "mkdocs-material>=9.6.22,<10",
+  "mkdocs-redirects>=1.2.2,<2",
+  "mkdocs>=1.6.1,<2",
+  "mkdocstrings-python>=1.18.2,<2",
+  "mkdocstrings>=0.30.1,<1",
 ]
 notebooks = [
-  "jupyter>=1.0.0",
-  "ipykernel>=6.29.0",
-  "pillow>=12.0.0",
+  "ipykernel>=6.29.0,<7",
+  "jupyter>=1.0.0,<2",
+  "pillow>=12.0.0,<13",
 ]
 
 [build-system]

{data_designer-0.2.1 → data_designer-0.2.2}/src/data_designer/_version.py

@@ -28,7 +28,7 @@ version_tuple: VERSION_TUPLE
 commit_id: COMMIT_ID
 __commit_id__: COMMIT_ID
 
-__version__ = version = '0.2.1'
-__version_tuple__ = version_tuple = (0, 2, 1)
+__version__ = version = '0.2.2'
+__version_tuple__ = version_tuple = (0, 2, 2)
 
 __commit_id__ = commit_id = None

{data_designer-0.2.1 → data_designer-0.2.2}/src/data_designer/config/config_builder.py

@@ -224,6 +224,9 @@ class DataDesignerConfigBuilder:
 
         Returns:
             The current Data Designer config builder instance.
+
+        Raises:
+            BuilderConfigurationError: If the column name collides with an existing seed dataset column.
         """
         if column_config is None:
             if name is None or column_type is None:
@@ -240,6 +243,13 @@ class DataDesignerConfigBuilder:
                 f"{', '.join([t.__name__ for t in allowed_column_configs])}"
             )
 
+        existing_config = self._column_configs.get(column_config.name)
+        if existing_config is not None and isinstance(existing_config, SeedDatasetColumnConfig):
+            raise BuilderConfigurationError(
+                f"🛑 Column {column_config.name!r} already exists as a seed dataset column. "
+                "Please use a different column name or update the seed dataset."
+            )
+
         self._column_configs[column_config.name] = column_config
         return self
 
@@ -578,7 +588,18 @@ class DataDesignerConfigBuilder:
 
         Returns:
             The current Data Designer config builder instance.
+
+        Raises:
+            BuilderConfigurationError: If any seed dataset column name collides with an existing column.
         """
+        seed_column_names = fetch_seed_dataset_column_names(dataset_reference)
+        colliding_columns = [name for name in seed_column_names if name in self._column_configs]
+        if colliding_columns:
+            raise BuilderConfigurationError(
+                f"🛑 Seed dataset column(s) {colliding_columns} collide with existing column(s). "
+                "Please remove the conflicting columns or use a seed dataset with different column names."
+            )
+
         self._seed_config = SeedConfig(
             dataset=dataset_reference.dataset,
             sampling_strategy=sampling_strategy,
@@ -587,7 +608,7 @@ class DataDesignerConfigBuilder:
         self.set_seed_datastore_settings(
             dataset_reference.datastore_settings if hasattr(dataset_reference, "datastore_settings") else None
         )
-        for column_name in fetch_seed_dataset_column_names(dataset_reference):
+        for column_name in seed_column_names:
             self._column_configs[column_name] = SeedDatasetColumnConfig(name=column_name)
         return self
 

{data_designer-0.2.1 → data_designer-0.2.2}/src/data_designer/engine/column_generators/generators/base.py

@@ -1,23 +1,27 @@
 # SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
 # SPDX-License-Identifier: Apache-2.0
 
+from __future__ import annotations
+
 import functools
 import logging
 from abc import ABC, abstractmethod
-from typing import overload
+from enum import Enum
+from typing import TYPE_CHECKING, overload
 
 import pandas as pd
 
-from data_designer.config.column_types import COLUMN_TYPE_EMOJI_MAP
-from data_designer.config.models import BaseInferenceParams, ModelConfig
-from data_designer.config.utils.type_helpers import StrEnum
 from data_designer.engine.configurable_task import ConfigurableTask, ConfigurableTaskMetadata, DataT, TaskConfigT
-from data_designer.engine.models.facade import ModelFacade
+
+if TYPE_CHECKING:
+    from data_designer.config.models import BaseInferenceParams, ModelConfig
+    from data_designer.engine.models.facade import ModelFacade
+
 
 logger = logging.getLogger(__name__)
 
 
-class GenerationStrategy(StrEnum):
+class GenerationStrategy(str, Enum):
     CELL_BY_CELL = "cell_by_cell"
     FULL_COLUMN = "full_column"
 
@@ -82,8 +86,7 @@ class WithModelGeneration:
         return self.model_config.inference_parameters
 
     def log_pre_generation(self) -> None:
-        emoji = COLUMN_TYPE_EMOJI_MAP[self.config.column_type]
-        logger.info(f"{emoji} Preparing {self.config.column_type} column generation")
+        logger.info(f"Preparing {self.config.column_type} column generation")
         logger.info(f"  |-- column name: {self.config.name!r}")
         logger.info(f"  |-- model config:\n{self.model_config.model_dump_json(indent=4)}")
         if self.model_config.provider is None:

{data_designer-0.2.1 → data_designer-0.2.2}/src/data_designer/engine/validators/python.py

@@ -2,8 +2,8 @@
 # SPDX-License-Identifier: Apache-2.0
 
 import ast
+import json
 import logging
-import re
 import subprocess
 import tempfile
 from collections import defaultdict
@@ -179,9 +179,8 @@ class PythonValidator(BaseValidator):
         for file in Path(codebase_path).glob("*.py"):
             processed[file.stem] = PythonLinterMessages()
 
-        # Run ruff linter
+        # Run ruff linter with JSON output
         ruff_bin = find_ruff_bin()
-        env = {"NO_COLOR": "1"}
 
         ruff_exec = subprocess.run(
             [
@@ -189,9 +188,9 @@ class PythonValidator(BaseValidator):
                 "check",
                 "--select",
                 "E,F6,F7,F8,SIM,PLC,PLE,PLR,PLW",
+                "--output-format=json",
                 codebase_path,
             ],
-            env=env,
             text=True,
             capture_output=True,
             check=False,
@@ -199,30 +198,34 @@ class PythonValidator(BaseValidator):
         )
         ruff_output = ruff_exec.stdout
 
-        # Parse ruff output
-        if "All checks passed!" in ruff_output:
-            return processed  # no errors or warnings
-
-        pattern = r"(.*):([0-9]*):([0-9]*): ([A-Za-z0-9]*):? (?:\[\*\] )?(.*)\n"
-        errors = re.findall(pattern, ruff_output)
+        # Parse JSON output
+        try:
+            diagnostics = json.loads(ruff_output)
+        except json.JSONDecodeError as e:
+            raise RuntimeError(f"Failed to parse ruff JSON output: {e}")
 
-        if errors == []:  # output could not be parsed
-            raise RuntimeError("ruff's output could not be parsed")
+        if not diagnostics:
+            return processed  # no errors or warnings
 
-        try:
-            for error in errors:
-                filename, line, column, symbol, message = error
-                processed[Path(filename).stem].add(
-                    PythonLinterMessage(
-                        type=TYPE_FROM_SYMBOL[re.sub(r"[^A-Za-z]+", "", symbol)],
-                        symbol=symbol,
-                        line=int(line),
-                        column=int(column),
-                        message=message,
-                    )
+        for diagnostic in diagnostics:
+            filename = diagnostic["filename"]
+            code = diagnostic["code"]
+            location = diagnostic["location"]
+            message = diagnostic["message"]
+
+            # Extract alphabetic prefix from code for type mapping
+            alpha_prefix = "".join(c for c in code if c.isalpha())
+            error_type = TYPE_FROM_SYMBOL.get(alpha_prefix, "warning")
+
+            processed[Path(filename).stem].add(
+                PythonLinterMessage(
+                    type=error_type,
+                    symbol=code,
+                    line=location["row"],
+                    column=location["column"],
+                    message=message,
                 )
-        except Exception:  # output not in expected format
-            raise RuntimeError("ruff's output not in expected format")
+            )
 
         return processed
 

{data_designer-0.2.1 → data_designer-0.2.2}/tests/config/test_config_builder.py

@@ -42,14 +42,14 @@ class DummyStructuredModel(BaseModel):
 @pytest.fixture
 def mock_fetch_seed_dataset_column_names():
     with patch("data_designer.config.config_builder.fetch_seed_dataset_column_names") as mock_fetch_seed:
-        mock_fetch_seed.return_value = ["id", "name", "age", "city"]
+        mock_fetch_seed.return_value = ["id", "name", "city", "country"]
         yield mock_fetch_seed
 
 
 @pytest.fixture
 def stub_data_designer_builder(stub_data_designer_builder_config_str):
     with patch("data_designer.config.config_builder.fetch_seed_dataset_column_names") as mock_fetch_seed:
-        mock_fetch_seed.return_value = ["id", "name", "age", "city"]
+        mock_fetch_seed.return_value = ["id", "name", "city", "country"]
         yield DataDesignerConfigBuilder.from_config(config=stub_data_designer_builder_config_str)
 
 
@@ -404,25 +404,25 @@ def test_delete_constraints(stub_data_designer_builder):
 
 
 def test_delete_column(stub_data_designer_builder):
-    assert len(stub_data_designer_builder.get_columns_of_type(DataDesignerColumnType.SAMPLER)) == 3
+    assert len(stub_data_designer_builder.get_columns_of_type(DataDesignerColumnType.SAMPLER)) == 4
     stub_data_designer_builder.delete_column(column_name="code_id")
-    assert len(stub_data_designer_builder.get_columns_of_type(DataDesignerColumnType.SAMPLER)) == 2
+    assert len(stub_data_designer_builder.get_columns_of_type(DataDesignerColumnType.SAMPLER)) == 3
 
     with pytest.raises(
         BuilderConfigurationError, match="Seed columns cannot be deleted. Please update the seed dataset instead."
     ):
-        stub_data_designer_builder.delete_column(column_name="age")
+        stub_data_designer_builder.delete_column(column_name="id")
 
 
 def test_getters(stub_data_designer_builder):
-    assert len(stub_data_designer_builder.get_column_configs()) == 11
+    assert len(stub_data_designer_builder.get_column_configs()) == 12
     assert stub_data_designer_builder.get_column_config(name="code_id").name == "code_id"
     assert len(stub_data_designer_builder.get_constraints(target_column="age")) == 1
     assert len(stub_data_designer_builder.get_llm_gen_columns()) == 3
-    assert len(stub_data_designer_builder.get_columns_of_type(DataDesignerColumnType.SAMPLER)) == 3
+    assert len(stub_data_designer_builder.get_columns_of_type(DataDesignerColumnType.SAMPLER)) == 4
     assert len(stub_data_designer_builder.get_columns_excluding_type(DataDesignerColumnType.SAMPLER)) == 8
     assert stub_data_designer_builder.get_seed_config().dataset == "test-repo/testing/data.csv"
-    assert stub_data_designer_builder.num_columns_of_type(DataDesignerColumnType.SAMPLER) == 3
+    assert stub_data_designer_builder.num_columns_of_type(DataDesignerColumnType.SAMPLER) == 4
 
 
 def test_write_config(stub_data_designer_builder):
@@ -759,3 +759,90 @@ def test_delete_model_config(stub_empty_builder):
 
     assert result is stub_empty_builder
     assert len(stub_empty_builder.model_configs) == 2
+
+
+def test_add_column_collision_with_seed_dataset(stub_empty_builder: DataDesignerConfigBuilder) -> None:
+    """Test that adding a column that collides with a seed dataset column raises an error."""
+    datastore_settings = DatastoreSettings(endpoint="https://huggingface.co", token="test-token")
+
+    with patch("data_designer.config.config_builder.fetch_seed_dataset_column_names") as mock_fetch:
+        mock_fetch.return_value = ["id", "name", "age"]
+        stub_empty_builder.with_seed_dataset(
+            DatastoreSeedDatasetReference(dataset="test-repo/test-data.parquet", datastore_settings=datastore_settings)
+        )
+
+    with pytest.raises(
+        BuilderConfigurationError,
+        match="Column 'id' already exists as a seed dataset column",
+    ):
+        stub_empty_builder.add_column(
+            name="id",
+            column_type=DataDesignerColumnType.SAMPLER,
+            sampler_type=SamplerType.UUID,
+        )
+
+    with pytest.raises(
+        BuilderConfigurationError,
+        match="Column 'name' already exists as a seed dataset column",
+    ):
+        stub_empty_builder.add_column(
+            LLMTextColumnConfig(
+                name="name",
+                prompt="Write a name",
+                model_alias="stub-model",
+            )
+        )
+
+
+def test_with_seed_dataset_collision_with_existing_columns(stub_empty_builder: DataDesignerConfigBuilder) -> None:
+    """Test that adding a seed dataset with columns that collide with existing columns raises an error."""
+    stub_empty_builder.add_column(
+        name="name",
+        column_type=DataDesignerColumnType.LLM_TEXT,
+        prompt="Write a name",
+        model_alias="stub-model",
+    )
+    stub_empty_builder.add_column(
+        name="age",
+        column_type=DataDesignerColumnType.SAMPLER,
+        sampler_type=SamplerType.UNIFORM,
+        params={"low": 1, "high": 100},
+    )
+
+    datastore_settings = DatastoreSettings(endpoint="https://huggingface.co", token="test-token")
+
+    with patch("data_designer.config.config_builder.fetch_seed_dataset_column_names") as mock_fetch:
+        mock_fetch.return_value = ["id", "name", "age", "city"]
+        with pytest.raises(
+            BuilderConfigurationError,
+            match=r"Seed dataset column\(s\) \['name', 'age'\] collide with existing column\(s\)",
+        ):
+            stub_empty_builder.with_seed_dataset(
+                DatastoreSeedDatasetReference(
+                    dataset="test-repo/test-data.parquet", datastore_settings=datastore_settings
+                )
+            )
+
+    assert stub_empty_builder.get_seed_config() is None
+    assert len(stub_empty_builder.get_columns_of_type(DataDesignerColumnType.SEED_DATASET)) == 0
+
+
+def test_with_seed_dataset_no_collision(stub_empty_builder: DataDesignerConfigBuilder) -> None:
+    """Test that adding a seed dataset with non-colliding columns works fine."""
+    stub_empty_builder.add_column(
+        name="unique_column",
+        column_type=DataDesignerColumnType.SAMPLER,
+        sampler_type=SamplerType.UUID,
+    )
+
+    datastore_settings = DatastoreSettings(endpoint="https://huggingface.co", token="test-token")
+
+    with patch("data_designer.config.config_builder.fetch_seed_dataset_column_names") as mock_fetch:
+        mock_fetch.return_value = ["id", "name", "age"]
+        stub_empty_builder.with_seed_dataset(
+            DatastoreSeedDatasetReference(dataset="test-repo/test-data.parquet", datastore_settings=datastore_settings)
+        )
+
+    assert stub_empty_builder.get_seed_config() is not None
+    assert len(stub_empty_builder.get_columns_of_type(DataDesignerColumnType.SEED_DATASET)) == 3
+    assert len(stub_empty_builder.get_columns_of_type(DataDesignerColumnType.SAMPLER)) == 1

{data_designer-0.2.1 → data_designer-0.2.2}/tests/conftest.py

@@ -164,7 +164,7 @@ def stub_empty_builder(stub_model_configs: list[ModelConfig]) -> DataDesignerCon
 @pytest.fixture
 def stub_complete_builder(stub_data_designer_builder_config_str: str) -> DataDesignerConfigBuilder:
     with patch("data_designer.config.config_builder.fetch_seed_dataset_column_names") as mock_fetch:
-        mock_fetch.return_value = ["id", "name", "age", "city"]
+        mock_fetch.return_value = ["id", "name", "city", "country"]
         return DataDesignerConfigBuilder.from_config(config=stub_data_designer_builder_config_str)
 
 

{data_designer-0.2.1 → data_designer-0.2.2}/tests/engine/column_generators/generators/test_llm_completion_generators.py

@@ -102,7 +102,7 @@ def test_log_pre_generation(mock_logger):
     generator.log_pre_generation()
 
     assert mock_logger.info.call_count == 3
-    mock_logger.info.assert_any_call("📝 Preparing llm-text column generation")
+    mock_logger.info.assert_any_call("Preparing llm-text column generation")
     mock_logger.info.assert_any_call("  |-- column name: 'test_column'")
     mock_logger.info.assert_any_call('  |-- model config:\n{"test": "config"}')