data-designer 0.3.8rc2__tar.gz → 0.4.0__tar.gz

{data_designer-0.3.8rc2 → data_designer-0.4.0}/.gitignore

@@ -79,8 +79,8 @@ venv.bak/
 *.tar.gz
 *.zip
 
-# Auto-generated version file
-src/data_designer/_version.py
+# Auto-generated version files
+**/_version.py
 
 # Local scratch space
 .scratch/
@@ -94,3 +94,7 @@ tests_e2e/uv.lock
 
 # Performance profiling
 perf_*.txt
+NOTEPAD.md
+
+# Build-time copy of README for data-designer package (copied from top-level during build)
+packages/data-designer/README.md

{data_designer-0.3.8rc2 → data_designer-0.4.0}/PKG-INFO

@@ -1,9 +1,8 @@
 Metadata-Version: 2.4
 Name: data-designer
-Version: 0.3.8rc2
+Version: 0.4.0
 Summary: General framework for synthetic data generation
 License-Expression: Apache-2.0
-License-File: LICENSE
 Classifier: Development Status :: 4 - Beta
 Classifier: Intended Audience :: Developers
 Classifier: Intended Audience :: Science/Research
@@ -15,33 +14,9 @@ Classifier: Programming Language :: Python :: 3.13
 Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
 Classifier: Topic :: Software Development
 Requires-Python: >=3.10
-Requires-Dist: anyascii<1,>=0.3.3
-Requires-Dist: duckdb<2,>=1.1.3
-Requires-Dist: faker<21,>=20.1.0
-Requires-Dist: httpx-retries<1,>=0.4.2
-Requires-Dist: httpx<1,>=0.27.2
-Requires-Dist: huggingface-hub<2,>=1.0.1
-Requires-Dist: jinja2<4,>=3.1.6
-Requires-Dist: json-repair<1,>=0.48.0
-Requires-Dist: jsonpath-rust-bindings<2,>=1.0
-Requires-Dist: litellm<1.80.12,>=1.73.6
-Requires-Dist: lxml<7,>=6.0.2
-Requires-Dist: marko<3,>=2.1.2
-Requires-Dist: networkx<4,>=3.0
-Requires-Dist: numpy<3,>=1.23.5
-Requires-Dist: pandas<3,>=2.3.3
+Requires-Dist: data-designer-config
+Requires-Dist: data-designer-engine
 Requires-Dist: prompt-toolkit<4,>=3.0.0
-Requires-Dist: pyarrow<20,>=19.0.1
-Requires-Dist: pydantic[email]<3,>=2.9.2
-Requires-Dist: pygments<3,>=2.19.2
-Requires-Dist: python-json-logger<4,>=3
-Requires-Dist: pyyaml<7,>=6.0.1
-Requires-Dist: requests<3,>=2.32.2
-Requires-Dist: rich<15,>=13.7.1
-Requires-Dist: ruff<1,>=0.14.10
-Requires-Dist: scipy<2,>=1.11.0
-Requires-Dist: sqlfluff<4,>=3.2.0
-Requires-Dist: tiktoken<1,>=0.8.0
 Requires-Dist: typer<1,>=0.12.0
 Description-Content-Type: text/markdown
 
@@ -104,26 +79,19 @@ export OPENROUTER_API_KEY="your-openrouter-api-key-here"
 
 ### 3. Start generating data!
 ```python
-from data_designer.essentials import (
-    CategorySamplerParams,
-    DataDesigner,
-    DataDesignerConfigBuilder,
-    LLMTextColumnConfig,
-    PersonSamplerParams,
-    SamplerColumnConfig,
-    SamplerType,
-)
+import data_designer.config as dd
+from data_designer.interface import DataDesigner
 
 # Initialize with default settings
 data_designer = DataDesigner()
-config_builder = DataDesignerConfigBuilder()
+config_builder = dd.DataDesignerConfigBuilder()
 
 # Add a product category
 config_builder.add_column(
-    SamplerColumnConfig(
+    dd.SamplerColumnConfig(
         name="product_category",
-        sampler_type=SamplerType.CATEGORY,
-        params=CategorySamplerParams(
+        sampler_type=dd.SamplerType.CATEGORY,
+        params=dd.CategorySamplerParams(
             values=["Electronics", "Clothing", "Home & Kitchen", "Books"],
         ),
     )
@@ -131,7 +99,7 @@ config_builder.add_column(
 
 # Generate personalized customer reviews
 config_builder.add_column(
-    LLMTextColumnConfig(
+    dd.LLMTextColumnConfig(
         name="review",
         model_alias="nvidia-text",
         prompt="Write a brief product review for a {{ product_category }} item you recently purchased.",

{data_designer-0.3.8rc2 → data_designer-0.4.0}/README.md

@@ -57,26 +57,19 @@ export OPENROUTER_API_KEY="your-openrouter-api-key-here"
 
 ### 3. Start generating data!
 ```python
-from data_designer.essentials import (
-    CategorySamplerParams,
-    DataDesigner,
-    DataDesignerConfigBuilder,
-    LLMTextColumnConfig,
-    PersonSamplerParams,
-    SamplerColumnConfig,
-    SamplerType,
-)
+import data_designer.config as dd
+from data_designer.interface import DataDesigner
 
 # Initialize with default settings
 data_designer = DataDesigner()
-config_builder = DataDesignerConfigBuilder()
+config_builder = dd.DataDesignerConfigBuilder()
 
 # Add a product category
 config_builder.add_column(
-    SamplerColumnConfig(
+    dd.SamplerColumnConfig(
         name="product_category",
-        sampler_type=SamplerType.CATEGORY,
-        params=CategorySamplerParams(
+        sampler_type=dd.SamplerType.CATEGORY,
+        params=dd.CategorySamplerParams(
             values=["Electronics", "Clothing", "Home & Kitchen", "Books"],
         ),
     )
@@ -84,7 +77,7 @@ config_builder.add_column(
 
 # Generate personalized customer reviews
 config_builder.add_column(
-    LLMTextColumnConfig(
+    dd.LLMTextColumnConfig(
         name="review",
         model_alias="nvidia-text",
         prompt="Write a brief product review for a {{ product_category }} item you recently purchased.",

data_designer-0.4.0/dev-tools/hatch_build.py

@@ -0,0 +1,31 @@
+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+"""Custom hatch metadata hook to sync README from root.
+
+This hook runs during metadata resolution (before build hooks) to ensure
+the README.md from the repository root is copied before hatchling validates
+that the readme file exists.
+"""
+
+from __future__ import annotations
+
+import shutil
+from pathlib import Path
+from typing import Any
+
+from hatchling.metadata.plugin.interface import MetadataHookInterface
+
+
+class ReadmeSyncHook(MetadataHookInterface):
+    """Metadata hook that copies README.md from repository root before building."""
+
+    PLUGIN_NAME = "readme-sync"
+
+    def update(self, metadata: dict[str, Any]) -> None:
+        """Copy README.md from repository root to package directory."""
+        root_readme = Path(self.root) / ".." / ".." / "README.md"
+        package_readme = Path(self.root) / "README.md"
+
+        if root_readme.exists():
+            shutil.copy2(root_readme, package_readme)

data_designer-0.4.0/pyproject.toml

@@ -0,0 +1,58 @@
+[project]
+name = "data-designer"
+dynamic = ["version"]
+description = "General framework for synthetic data generation"
+readme = "README.md"
+requires-python = ">=3.10"
+license = "Apache-2.0"
+
+classifiers = [
+    "Development Status :: 4 - Beta",
+    "Intended Audience :: Developers",
+    "Intended Audience :: Science/Research",
+    "Topic :: Scientific/Engineering :: Artificial Intelligence",
+    "Topic :: Software Development",
+    "License :: OSI Approved :: Apache Software License",
+    "Programming Language :: Python :: 3.10",
+    "Programming Language :: Python :: 3.11",
+    "Programming Language :: Python :: 3.12",
+    "Programming Language :: Python :: 3.13",
+]
+
+dependencies = [
+    "data-designer-config",
+    "data-designer-engine",
+    "prompt-toolkit>=3.0.0,<4",
+    "typer>=0.12.0,<1",
+]
+
+[project.scripts]
+data-designer = "data_designer.cli:main"
+
+[build-system]
+requires = ["hatchling", "hatch-vcs"]
+build-backend = "hatchling.build"
+
+[tool.hatch.version]
+source = "vcs"
+fallback-version = "0.1.0.dev0"
+raw-options = { root = "../.." }
+
+[tool.hatch.build.hooks.vcs]
+version-file = "src/data_designer/interface/_version.py"
+
+[tool.hatch.metadata.hooks.custom]
+path = "dev-tools/hatch_build.py"
+
+[tool.hatch.build.targets.wheel]
+packages = ["src/data_designer"]
+
+[tool.ruff]
+extend = "../../pyproject.toml"
+
+[tool.uv]
+package = true
+
+[tool.uv.sources]
+data-designer-config = { workspace = true }
+data-designer-engine = { workspace = true }

data_designer-0.4.0/src/data_designer/cli/commands/__init__.py

@@ -0,0 +1,2 @@
+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0

data_designer-0.4.0/src/data_designer/interface/__init__.py

@@ -0,0 +1,22 @@
+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+from data_designer.config.default_model_settings import resolve_seed_default_model_settings
+from data_designer.interface.data_designer import DataDesigner
+from data_designer.interface.errors import (
+    DataDesignerGenerationError,
+    DataDesignerProfilingError,
+)
+from data_designer.interface.results import DatasetCreationResults
+from data_designer.logging import configure_logging
+
+configure_logging()
+resolve_seed_default_model_settings()
+
+
+__all__ = [
+    "DataDesigner",
+    "DataDesignerGenerationError",
+    "DataDesignerProfilingError",
+    "DatasetCreationResults",
+]

{data_designer-0.3.8rc2/src/data_designer → data_designer-0.4.0/src/data_designer/interface}/_version.py

@@ -28,7 +28,7 @@ version_tuple: VERSION_TUPLE
 commit_id: COMMIT_ID
 __commit_id__: COMMIT_ID
 
-__version__ = version = '0.3.8rc2'
-__version_tuple__ = version_tuple = (0, 3, 8, 'rc2')
+__version__ = version = '0.4.0'
+__version_tuple__ = version_tuple = (0, 4, 0)
 
 __commit_id__ = commit_id = None

{data_designer-0.3.8rc2 → data_designer-0.4.0}/src/data_designer/interface/data_designer.py

@@ -317,13 +317,7 @@ class DataDesigner(DataDesignerInterface[DatasetCreationResults]):
         Args:
             run_config: A RunConfig instance containing runtime settings such as
                 early shutdown behavior, batch sizing via `buffer_size`, and non-inference worker
-                concurrency via `non_inference_max_parallel_workers`. Import RunConfig from
-                data_designer.essentials.
-
-        Example:
-            >>> from data_designer.essentials import DataDesigner, RunConfig
-            >>> dd = DataDesigner()
-            >>> dd.set_run_config(RunConfig(disable_early_shutdown=True))
+                concurrency via `non_inference_max_parallel_workers`.
 
         Notes:
             When `disable_early_shutdown=True`, DataDesigner will never terminate generation early

{data_designer-0.3.8rc2 → data_designer-0.4.0}/tests/cli/controllers/test_download_controller.py

@@ -85,16 +85,18 @@ def test_run_personas_with_all_flag(
     # Verify NGC check was called
     mock_check_ngc.assert_called_once()
 
-    # Verify all 5 locales were downloaded
-    assert mock_download.call_count == 5
+    # Verify all 7 locales were downloaded
+    assert mock_download.call_count == 7
 
     # Verify each locale was downloaded
     downloaded_locales = [call[0][0] for call in mock_download.call_args_list]
     assert "en_US" in downloaded_locales
     assert "en_IN" in downloaded_locales
+    assert "en_SG" in downloaded_locales
     assert "hi_Deva_IN" in downloaded_locales
     assert "hi_Latn_IN" in downloaded_locales
     assert "ja_JP" in downloaded_locales
+    assert "pt_BR" in downloaded_locales
 
 
 @patch.object(DownloadController, "_download_locale", return_value=True)
@@ -217,12 +219,14 @@ def test_determine_locales_with_all_flag(controller: DownloadController) -> None
     """Test _determine_locales returns all locales when all_locales=True."""
     result = controller._determine_locales(locales=None, all_locales=True)
 
-    assert len(result) == 5
+    assert len(result) == 7
     assert "en_US" in result
     assert "en_IN" in result
+    assert "en_SG" in result
     assert "hi_Deva_IN" in result
     assert "hi_Latn_IN" in result
     assert "ja_JP" in result
+    assert "pt_BR" in result
 
 
 def test_determine_locales_with_valid_locale_flags(controller: DownloadController) -> None:

{data_designer-0.3.8rc2 → data_designer-0.4.0}/tests/cli/repositories/test_persona_repository.py

@@ -15,7 +15,7 @@ def repository() -> PersonaRepository:
 def test_init(repository: PersonaRepository) -> None:
     """Test repository initialization creates registry."""
     assert repository._registry is not None
-    assert len(repository._registry.locales) == 5
+    assert len(repository._registry.locales) == 7
     assert repository._registry.dataset_prefix == "nemotron-personas-dataset-"
 
 
@@ -24,11 +24,11 @@ def test_list_all(repository: PersonaRepository) -> None:
     locales = repository.list_all()
 
     assert isinstance(locales, list)
-    assert len(locales) == 5
+    assert len(locales) == 7
 
     # Verify all expected locales are present
     locale_codes = {locale.code for locale in locales}
-    assert locale_codes == {"en_US", "en_IN", "hi_Deva_IN", "hi_Latn_IN", "ja_JP"}
+    assert locale_codes == {"en_US", "en_IN", "en_SG", "hi_Deva_IN", "hi_Latn_IN", "ja_JP", "pt_BR"}
 
     # Verify each locale has required fields
     for locale in locales:

{data_designer-0.3.8rc2 → data_designer-0.4.0}/tests/cli/services/test_download_service.py

@@ -51,12 +51,14 @@ def test_get_available_locales(service: DownloadService) -> None:
     locales = service.get_available_locales()
 
     assert isinstance(locales, dict)
-    assert len(locales) == 5
+    assert len(locales) == 7
     assert "en_US" in locales
     assert "en_IN" in locales
+    assert "en_SG" in locales
     assert "hi_Deva_IN" in locales
     assert "hi_Latn_IN" in locales
     assert "ja_JP" in locales
+    assert "pt_BR" in locales
 
     # Verify values are locale codes (not descriptions)
     assert locales["en_US"] == "en_US"

data_designer-0.4.0/tests/conftest.py

@@ -0,0 +1,4 @@
+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+pytest_plugins = ["data_designer.config.testing.fixtures"]

{data_designer-0.3.8rc2 → data_designer-0.4.0}/tests/interface/test_data_designer.py

@@ -20,11 +20,9 @@ from data_designer.config.run_config import RunConfig
 from data_designer.config.sampler_params import CategorySamplerParams, SamplerType
 from data_designer.config.seed_source import HuggingFaceSeedSource
 from data_designer.engine.secret_resolver import CompositeResolver, EnvironmentResolver, PlaintextResolver
+from data_designer.engine.testing.stubs import StubHuggingFaceSeedReader
 from data_designer.interface.data_designer import DataDesigner
-from data_designer.interface.errors import (
-    DataDesignerGenerationError,
-    DataDesignerProfilingError,
-)
+from data_designer.interface.errors import DataDesignerGenerationError, DataDesignerProfilingError
 from data_designer.lazy_heavy_imports import pd
 
 if TYPE_CHECKING:
@@ -56,6 +54,11 @@ def stub_model_providers():
     ]
 
 
+@pytest.fixture
+def stub_seed_reader():
+    return StubHuggingFaceSeedReader()
+
+
 def test_init_with_custom_secret_resolver(stub_artifact_path, stub_model_providers):
     """Test DataDesigner initialization with custom secret resolver."""
     designer = DataDesigner(

{data_designer-0.3.8rc2 → data_designer-0.4.0}/tests/test_import_perf.py

@@ -16,7 +16,8 @@ PERF_TEST_TIMEOUT_SECONDS = 30.0
 def test_import_performance():
     """Test that average import time never exceeds 6 seconds (1 cold start + 4 warm cache runs)."""
     # Get the project root (where Makefile is located)
-    project_root = Path(__file__).parent.parent
+    # For workspace packages, need to go up to the workspace root
+    project_root = Path(__file__).parent.parent.parent.parent
 
     num_runs = 5
     import_times = []

data_designer-0.3.8rc2/.claude/agents/docs-searcher.md

@@ -1,74 +0,0 @@
----
-name: docs-searcher
-description: Search local documentation in the docs/ folder for content related to a topic. Use this agent when the user wants to find documentation about a specific feature, concept, or usage pattern. Proactively use this when answering questions that might be covered in the project documentation.
-tools: Glob, Grep, Read
-model: haiku
-permissionMode: bypassPermissions
----
-
-# Documentation Search Agent
-
-You are a documentation search specialist. Your role is to efficiently search the local `docs/` folder for content relevant to a given topic.
-
-## Instructions
-
-When given a search topic, perform the following searches:
-
-1. **Find all documentation files** in the docs/ folder:
-   ```
-   Glob pattern: "docs/**/*.md"
-   ```
-
-2. **Search for topic keywords** across all markdown files:
-   ```
-   Grep pattern: "<topic keywords>" in path: "docs/"
-   ```
-   - Try multiple variations of the search terms (singular/plural, related terms)
-   - Use case-insensitive search (`-i: true`)
-
-3. **Read relevant sections** from files with matches:
-   - Read the matched files to get full context
-   - Extract the most relevant sections around the matches
-
-4. **Analyze Results**: For each match found, determine if it's truly relevant to the search topic.
-
-5. **Output Format**: Return a structured markdown summary with:
-   - Links to relevant documentation files
-   - Brief excerpts showing the relevant content
-   - A sentence explaining why each result is pertinent
-
-## Output Template
-
-```markdown
-## Documentation Search Results for "<topic>"
-
-### Relevant Documentation
-
-- **[docs/path/to/file.md](docs/path/to/file.md)**
-  > Brief excerpt showing relevant content...
-
-  Explanation of why this is relevant to the search topic.
-
-- **[docs/another/file.md](docs/another/file.md)**
-  > Another relevant excerpt...
-
-  Explanation of relevance.
-
-### Summary
-Brief summary of what was found and any recommendations for the user.
-```
-
-## Important Notes
-
-- Only include results that are actually relevant to the search topic
-- If no relevant documentation is found, clearly state that
-- Keep excerpts concise but include enough context to be useful
-- Prioritize user guides and examples over API reference when both exist
-- If the docs/ folder doesn't exist or is empty, report that clearly
-
-## Search Strategy
-
-1. Start with exact keyword matches
-2. If few results, try related terms or partial matches
-3. Check file names for topic-related terms (e.g., searching "models" should check files named `models.md`, `model-config.md`, etc.)
-4. Look at section headings within files for topic mentions

data_designer-0.3.8rc2/.claude/agents/github-searcher.md

@@ -1,81 +0,0 @@
----
-name: github-searcher
-description: Search GitHub issues, discussions, and PRs for content related to a topic. Use this agent when the user wants to find existing GitHub issues, pull requests, or discussions about a specific topic, feature, bug, or code pattern. Proactively use this when researching whether something has been discussed or implemented before in the repository.
-tools: Bash
-model: haiku
-permissionMode: bypassPermissions
----
-
-# GitHub Content Search Agent
-
-You are a GitHub search specialist. Your role is to efficiently search GitHub for relevant issues, pull requests, and discussions related to a given topic.
-
-## Instructions
-
-When given a search topic, perform the following searches:
-
-1. **Search Issues** using the `gh` CLI:
-   ```bash
-   gh issue list --search "<topic>" --limit 20 --json number,title,url,body,state
-   ```
-
-2. **Search Pull Requests** using the `gh` CLI:
-   ```bash
-   gh pr list --search "<topic>" --limit 20 --json number,title,url,body,state
-   ```
-
-3. **Search Discussions** using the `gh` CLI (if the repository has discussions enabled):
-   ```bash
-   gh api graphql -f query='
-     query($search: String!) {
-       search(query: $search, type: DISCUSSION, first: 20) {
-         nodes {
-           ... on Discussion {
-             title
-             url
-             body
-             category { name }
-           }
-         }
-       }
-     }
-   ' -f search="repo:{owner}/{repo} <topic>"
-   ```
-   Note: Get the owner/repo from `gh repo view --json nameWithOwner -q .nameWithOwner`
-
-4. **Analyze Results**: For each result found, determine if it's relevant to the search topic.
-
-5. **Output Format**: Return a markdown list with:
-   - A link to each relevant item (issue, PR, or discussion)
-   - A *single* sentence explaining why that link is pertinent to the search topic
-
-## Output Template
-
-```markdown
-## GitHub Search Results for "<topic>"
-
-### Issues
-- [Issue #123: Title](url) - Brief explanation of relevance.
-- [Issue #456: Title](url) - Brief explanation of relevance.
-
-### Pull Requests
-- [PR #789: Title](url) - Brief explanation of relevance.
-
-### Discussions
-- [Discussion: Title](url) - Brief explanation of relevance.
-```
-
-## Important Notes
-
-- Only include results that are actually relevant to the search topic
-- If a category (issues, PRs, discussions) has no relevant results, note "No relevant items found"
-- Keep descriptions to a single sentence
-- If discussions search fails (repository doesn't have discussions), skip that section
-- Prioritize open items over closed ones, but include relevant closed items too
-
-## Command Guidelines
-
-- **NEVER use pipes or shell fallbacks** like `|| echo "..."` or `| grep ...` in your commands
-- Run each `gh` command directly without any error handling wrappers
-- If a command returns an error or empty result, handle it in your analysis logic, not with shell constructs
-- Run the three searches (issues, PRs, discussions) as separate Bash commands

data_designer-0.3.8rc2/.claude/settings.json

@@ -1 +0,0 @@
-{}

data_designer-0.3.8rc2/.claude/settings.local.json

@@ -1,22 +0,0 @@
-{
-  "permissions": {
-    "allow": [
-      "Bash(uv run:*)",
-      "Bash(make install-dev-notebooks:*)",
-      "Bash(make lint:*)",
-      "Bash(curl:*)",
-      "Bash(uv pip:*)",
-      "Bash(make lint-fix:*)",
-      "Bash(mv:*)",
-      "Bash(make test:*)",
-      "Bash(make serve-docs-locally:*)",
-      "Bash(rm:*)",
-      "Bash(ls:*)",
-      "Bash(find:*)",
-      "Bash(git -C /Users/johnnygreco/projects/nvidia/DataDesigner diff --stat)",
-      "Bash(git cherry-pick:*)"
-    ],
-    "deny": [],
-    "ask": []
-  }
-}