data-designer 0.1.0__tar.gz

data_designer-0.1.0/.github/workflows/build-docs.yml

@@ -0,0 +1,18 @@
+name: build_docs
+on:
+  push:
+    branches:
+      - main
+jobs:
+  deploy:
+    runs-on: ubuntu-latest
+    permissions:
+      contents: write
+    steps:
+      - uses: actions/checkout@v2
+      - uses: astral-sh/setup-uv@v6
+        with:
+          version: "0.9.5"
+      - run: uv python install
+      - run: uv sync --group docs
+      - run: uv run mkdocs gh-deploy --force --clean --verbose

data_designer-0.1.0/.github/workflows/ci.yml

@@ -0,0 +1,87 @@
+name: CI
+
+on:
+  push:
+    branches: [ main ]
+  pull_request:
+    branches: [ main ]
+  workflow_dispatch:
+
+jobs:
+  test:
+    name: Test (Python ${{ matrix.python-version }} on ${{ matrix.os }})
+    runs-on: ${{ matrix.os }}
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [ubuntu-latest, macos-latest]
+        python-version: ["3.10", "3.11", "3.12", "3.13"]
+
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Install uv
+        uses: astral-sh/setup-uv@v5
+        with:
+          version: "latest"
+          python-version: ${{ matrix.python-version }}
+          enable-cache: true
+
+      - name: Install dependencies
+        run: |
+          uv sync --group dev
+
+      - name: Run tests with coverage
+        run: |
+          uv run pytest -v --cov=data_designer --cov-report=term-missing --cov-report=xml --cov-fail-under=90
+
+  lint:
+    name: Lint and Format Check
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Install uv
+        uses: astral-sh/setup-uv@v5
+        with:
+          version: "latest"
+          python-version: "3.11"
+          enable-cache: true
+
+      - name: Install dependencies
+        run: |
+          uv sync --group dev
+
+      - name: Check formatting
+        run: |
+          uv run ruff format --check
+
+      - name: Run linter
+        run: |
+          uv run ruff check
+
+  license-headers:
+    name: Check License Headers
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Install uv
+        uses: astral-sh/setup-uv@v5
+        with:
+          version: "latest"
+          python-version: "3.11"
+          enable-cache: true
+
+      - name: Install dependencies
+        run: |
+          uv sync --group dev
+
+      - name: Check license headers
+        run: |
+          uv run python scripts/update_license_headers.py --check

data_designer-0.1.0/.github/workflows/dco-assistant.yml

@@ -0,0 +1,44 @@
+name: "DCO Assistant"
+on:
+  issue_comment:
+    types: [created]
+  pull_request_target:
+    types: [opened,closed,synchronize]
+
+permissions:
+  actions: write
+  checks: none
+  contents: write
+  deployments: none
+  id-token: none
+  issues: none
+  discussions: none
+  packages: none
+  pages: none
+  pull-requests: write
+  repository-projects: none
+  security-events: none
+  statuses: write
+
+jobs:
+  DCOAssistant:
+    if: github.repository_owner == 'NVIDIA-NeMo'
+    runs-on: ubuntu-latest
+    steps:
+      - name: "DCO Assistant"
+        if: (github.event.comment.body == 'recheck' || github.event.comment.body == 'I have read the Contributor Agreement including DCO and I hereby sign the Contributor Agreement and DCO') || github.event_name == 'pull_request_target'
+        uses: contributor-assistant/github-action@ca4a40a7d1004f18d9960b404b97e5f30a505a08
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          PERSONAL_ACCESS_TOKEN: ${{ secrets.DCO_ASSISTANT_TOKEN }}
+        with:
+          path-to-signatures: "dco-signatures.json"
+          path-to-document: 'https://github.com/NVIDIA-NeMo/DataDesigner/blob/main/DCO'
+          branch: 'signatures'
+          allowlist: dependabot
+          create-file-commit-message: "chore: create file to store dco signatures"
+          signed-commit-message: "chore: $contributorName has signed the dco in #$pullRequestNo"
+          custom-notsigned-prcomment: "Thank you for your submission! We ask that $you sign our [Developer Certificate of Origin](https://github.com/NVIDIA-NeMo/DataDesigner/blob/main/DCO) before we can accept your contribution. You can sign the DCO by adding a comment below using this text:"
+          custom-pr-sign-comment: "I have read the DCO document and I hereby sign the DCO."
+          lock-pullrequest-aftermerge: false
+          use-dco-flag: true

data_designer-0.1.0/.github/workflows/semantic-pull-requests.yml

@@ -0,0 +1,26 @@
+name: Validate PR title
+
+on:
+  pull_request_target:
+    types:
+      - opened
+      - edited
+      - synchronize
+      - reopened
+  pull_request:
+    types:
+      - opened
+      - edited
+      - synchronize
+      - reopened
+
+defaults:
+  run:
+    shell: bash -x -e -u -o pipefail {0}
+
+permissions:
+  pull-requests: read
+
+jobs:
+  semantic-pull-request:
+    uses: NVIDIA-NeMo/FW-CI-templates/.github/workflows/_semantic_pull_request.yml@v0.65.12

data_designer-0.1.0/.gitignore

@@ -0,0 +1,90 @@
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+share/python-wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.coverage
+.coverage.*
+.cache
+coverage.xml
+.pytest_cache/
+
+# mkdocs documentation
+/site
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# IPython
+profile_default/
+ipython_config.py
+
+# pyenv
+.python-version
+
+# uv
+.venv/
+.uv/
+
+# Environments
+.env
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+
+# Ruff
+.ruff_cache/
+
+# IDEs
+.vscode/
+.idea/
+*.swp
+*.swo
+*~
+.DS_Store
+
+# Build artifacts
+*.whl
+*.tar.gz
+*.zip
+
+# Auto-generated version file
+src/data_designer/_version.py
+
+# Local scratch space
+.scratch/
+
+.claude/
+
+docs/examples/artifacts/

data_designer-0.1.0/.pre-commit-config.yaml

@@ -0,0 +1,24 @@
+repos:
+  # General file checks
+  - repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: v5.0.0
+    hooks:
+      - id: trailing-whitespace
+      - id: end-of-file-fixer
+      - id: check-yaml
+      - id: check-added-large-files
+      - id: check-json
+      - id: check-toml
+      - id: check-merge-conflict
+      - id: debug-statements
+      - id: mixed-line-ending
+
+  # Ruff - linting and formatting (using your existing ruff config)
+  - repo: https://github.com/astral-sh/ruff-pre-commit
+    rev: v0.12.3
+    hooks:
+      # Run the linter
+      - id: ruff
+        args: [--fix]
+      # Run the formatter
+      - id: ruff-format

data_designer-0.1.0/AGENTS.md

@@ -0,0 +1,417 @@
+# AGENTS.md
+
+This file provides guidance to agents when working with code in this repository.
+
+## Project Overview
+
+**DataDesigner** is an NVIDIA NeMo project for creating synthetic datasets from scratch. It's a comprehensive framework that generates structured data using multiple generation strategies:
+
+- **Sampled data**: Built-in generators (UUID, DateTime, etc.) and Faker integration
+- **LLM-generated content**: Text, code, and structured data via LiteLLM
+- **Expression-based columns**: Derived columns using Jinja2 templates
+- **Validation & scoring**: Python, SQL, and remote validators; LLM-based judge scoring
+- **Seed dataset-based generation**: Generate from existing datasets
+
+### Architecture
+
+The project follows a layered architecture:
+
+1. **Config Layer** ([src/data_designer/config/](src/data_designer/config/)): User-facing configuration API
+   - `config_builder.py`: Main builder API for constructing configurations
+   - `columns.py`: Column configuration types (Sampler, LLMText, LLMCode, LLMStructured, LLMJudge, Expression, Validation, SeedDataset)
+   - `models.py`: Model configurations and inference parameters
+   - `sampler_params.py`: Parametrized samplers (Uniform, Category, Person, DateTime, etc.)
+
+2. **Engine Layer** ([src/data_designer/engine/](src/data_designer/engine/)): Internal generation and processing
+   - `column_generators/`: Generates individual columns from configs
+   - `dataset_builders/`: Orchestrates full dataset generation with DAG-based dependency management
+   - `models/`: LLM integration via LiteLLM with response parsing
+   - `validators/`: Column validation (Python, SQL, Code, Remote)
+   - `sampling_gen/`: Sophisticated person/entity sampling
+
+3. **Interface Layer** ([src/data_designer/interface/](src/data_designer/interface/)): Public API
+   - `data_designer.py`: Main `DataDesigner` class (primary entry point)
+   - `results.py`: Result containers
+   - `errors.py`: Public error types
+
+4. **Essentials** ([src/data_designer/essentials/](src/data_designer/essentials/)): Convenience module re-exporting key classes for users
+
+### Key Design Patterns
+
+- **Builder pattern**: Configuration construction via `DataDesignerConfigBuilder`
+- **Registry pattern**: Plugin system for column generators, validators, and profilers
+- **Strategy pattern**: Multiple generation approaches (sampled, LLM, expression, seed)
+- **DAG-based execution**: Column dependencies managed as directed acyclic graph
+
+## Development Workflow
+
+This project uses `uv` for dependency management and `make` for common tasks:
+
+```bash
+# Install dependencies
+uv sync
+
+# Install with dev dependencies
+uv sync --all-extras
+
+# Run the main module (if applicable)
+uv run python -m data_designer
+```
+
+### Code Quality
+
+```bash
+# Using Make (recommended)
+make lint           # Run ruff linter
+make lint-fix       # Fix linting issues automatically
+make format         # Format code with ruff
+make format-check   # Check code formatting without changes
+make check-all      # Run all checks (format-check + lint)
+make check-all-fix  # Run all checks with autofix (format + lint-fix)
+
+# Direct commands
+uv run ruff check                # Lint all files
+uv run ruff check --fix          # Lint with autofix
+uv run ruff format               # Format all files
+uv run ruff format --check       # Check formatting
+```
+
+### Running Tests
+
+```bash
+# Run all tests
+uv run pytest
+
+# Run tests with verbose output
+uv run pytest -v
+
+# Run a specific test file
+uv run pytest tests/config/test_sampler_constraints.py
+
+# Run tests with coverage
+uv run pytest --cov=data_designer --cov-report=term-missing --cov-report=html
+
+# Using Make
+make test           # Run all tests
+make coverage       # Run tests with coverage report
+```
+
+## Key Files
+
+- [src/data_designer/interface/data_designer.py](src/data_designer/interface/data_designer.py) - Main entry point (`DataDesigner` class)
+- [src/data_designer/config/config_builder.py](src/data_designer/config/config_builder.py) - Configuration API (`DataDesignerConfigBuilder`)
+- [src/data_designer/engine/dataset_builders/column_wise_builder.py](src/data_designer/engine/dataset_builders/column_wise_builder.py) - Generation orchestrator
+- [src/data_designer/essentials/\_\_init\_\_.py](src/data_designer/essentials/__init__.py) - User-facing API exports
+- [pyproject.toml](pyproject.toml) - Project dependencies and tool configurations
+- [Makefile](Makefile) - Common development commands
+
+## Working Guidelines
+
+- **Comments**: Only insert comments when code is especially important to understand. For basic code blocks, comments aren't necessary. We want readable code without vacuous comments.
+- **License headers**: All Python files must include the NVIDIA SPDX license header:
+  ```python
+  # SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+  # SPDX-License-Identifier: Apache-2.0
+  ```
+  Use `make update-license-headers` to add headers to all files automatically.
+- **Imports**: Avoid importing Python modules inside method definitions. Prefer module-level imports for better performance and clarity.
+- **Type annotations**: ALWAYS add type annotations to all functions, methods, and class attributes (including tests).
+
+## Code Style
+
+This project uses `ruff` (v0.12.3) for linting and formatting. Follow these guidelines to avoid linter errors:
+
+### General Formatting
+
+- **Line length**: Maximum 120 characters per line
+- **Quote style**: Always use double quotes (`"`) for strings
+- **Indentation**: Use 4 spaces (never tabs)
+- **Target version**: Python 3.11+
+
+### Type Annotations
+
+Type annotations are REQUIRED for all code in this project. This is strictly enforced for code quality and maintainability.
+
+- **ALWAYS** add type annotations to all functions, methods, and class attributes (including tests)
+- Use primitive types when possible: `list` not `List`, `dict` not `Dict`, `set` not `Set`, `tuple` not `Tuple`
+- Use modern union syntax with `|` for optional and union types (Python 3.10+):
+  - `str | None` not `Optional[str]`
+  - `int | str` not `Union[int, str]`
+- Only import from `typing` when absolutely necessary for complex generic types
+- For Pydantic models, use field-level type annotations
+
+  ```python
+  # Good
+  def process_items(items: list[str], max_count: int | None = None) -> dict[str, int]:
+      return {item: len(item) for item in items}
+
+  # Avoid - missing type annotations
+  def process_items(items, max_count=None):
+      return {item: len(item) for item in items}
+
+  # Avoid - old-style typing
+  from typing import List, Dict, Optional
+  def process_items(items: List[str], max_count: Optional[int] = None) -> Dict[str, int]:
+      return {item: len(item) for item in items}
+  ```
+
+### Import Style
+
+- **ALWAYS** use absolute imports, never relative imports
+- Place imports at module level, not inside functions
+- Import sorting is handled by `ruff`'s `isort` - imports should be grouped and sorted:
+  1. Standard library imports
+  2. Third-party imports
+  3. First-party imports (`data_designer`)
+- Use standard import conventions (enforced by `ICN`)
+
+  ```python
+  # Good
+  from data_designer.config.config_builder import DataDesignerConfigBuilder
+
+  # Bad - relative import (will cause linter errors)
+  from .config_builder import DataDesignerConfigBuilder
+
+  # Good - imports at module level
+  from pathlib import Path
+
+  def process_file(filename: str) -> None:
+      path = Path(filename)
+
+  # Bad - import inside function
+  def process_file(filename: str) -> None:
+      from pathlib import Path
+      path = Path(filename)
+  ```
+
+### Naming Conventions (PEP 8)
+
+Follow PEP 8 naming conventions:
+
+- **Functions and variables**: `snake_case`
+- **Classes**: `PascalCase`
+- **Constants**: `UPPER_SNAKE_CASE`
+- **Private attributes**: prefix with single underscore `_private_var`
+
+  ```python
+  # Good
+  class DatasetGenerator:
+      MAX_RETRIES = 3
+
+      def __init__(self) -> None:
+          self._cache: dict[str, str] = {}
+
+      def generate_dataset(self, config: dict[str, str]) -> pd.DataFrame:
+          pass
+
+  # Bad
+  class dataset_generator:  # Should be PascalCase
+      maxRetries = 3        # Should be UPPER_SNAKE_CASE
+
+      def GenerateDataset(self, Config):  # Should be snake_case
+          pass
+  ```
+
+### Common Pitfalls to Avoid
+
+1. **Mutable default arguments**:
+
+   ```python
+   # Bad - mutable default argument
+   def add_item(item: str, items: list[str] = []) -> list[str]:
+       items.append(item)
+       return items
+
+   # Good
+   def add_item(item: str, items: list[str] | None = None) -> list[str]:
+       if items is None:
+           items = []
+       items.append(item)
+       return items
+   ```
+
+2. **Unused imports and variables**:
+
+   ```python
+   # Bad - unused import
+   from pathlib import Path
+   from typing import Any  # Not used
+
+   def process() -> None:
+       pass
+
+   # Good - only import what you use
+   from pathlib import Path
+
+   def process() -> None:
+       pass
+   ```
+
+3. **Simplify code where possible** (enforced by `SIM`):
+
+   ```python
+   # Bad
+   if condition:
+       return True
+   else:
+       return False
+
+   # Good
+   return condition
+
+   # Bad
+   if key in my_dict:
+       value = my_dict[key]
+   else:
+       value = default
+
+   # Good
+   value = my_dict.get(key, default)
+   ```
+
+4. **Use comprehensions properly**:
+
+   ```python
+   # Bad
+   list([x for x in items])  # Unnecessary list() call
+
+   # Good
+   [x for x in items]
+
+   # Bad
+   dict([(k, v) for k, v in items])
+
+   # Good
+   {k: v for k, v in items}
+   ```
+
+5. **Proper return statements**:
+
+   ```python
+   # Bad - unnecessary else after return
+   def get_value(condition: bool) -> str:
+       if condition:
+           return "yes"
+       else:
+           return "no"
+
+   # Good
+   def get_value(condition: bool) -> str:
+       if condition:
+           return "yes"
+       return "no"
+   ```
+
+### Active Linter Rules
+
+The following ruff linter rules are currently enabled (see [pyproject.toml](pyproject.toml)):
+
+- `W`: pycodestyle warnings
+- `F`: pyflakes (unused imports, undefined names)
+- `I`: isort (import sorting)
+- `ICN`: flake8-import-conventions (standard import names)
+- `PIE`: flake8-pie (miscellaneous lints)
+
+**Note**: Additional rules (E, N, UP, ANN, B, C4, DTZ, RET, SIM, PTH) are commented out but may be enabled in the future. Write code that would pass these checks for future-proofing.
+
+## Testing Patterns
+
+The project uses `pytest` with the following patterns:
+
+- **Fixtures**: Shared test data and configurations in [tests/conftest.py](tests/conftest.py)
+- **Stub configs**: YAML-based configuration stubs for testing (see `stub_data_designer_config_str` fixture)
+- **Mocking**: Use `unittest.mock.patch` for external services and dependencies
+- **Async support**: pytest-asyncio for async tests (`asyncio_default_fixture_loop_scope = "session"`)
+- **HTTP mocking**: pytest-httpx for mocking HTTP requests
+- **Coverage**: Track test coverage with pytest-cov
+
+Example test structure:
+
+```python
+import pytest
+from data_designer.config.config_builder import DataDesignerConfigBuilder
+
+def test_something(stub_model_configs):
+    """Test description."""
+    builder = DataDesignerConfigBuilder(model_configs=stub_model_configs)
+    # ... test implementation
+    assert expected == actual
+```
+
+## Column Configuration Types
+
+When working with column configurations, understand these key types:
+
+- **`SamplerColumnConfig`**: Built-in samplers (UUID, Category, Uniform, Gaussian, Person, DateTime, etc.)
+- **`LLMTextColumnConfig`**: LLM text generation with Jinja2 templating
+- **`LLMCodeColumnConfig`**: Code generation with language specification
+- **`LLMStructuredColumnConfig`**: Structured JSON generation with schema
+- **`LLMJudgeColumnConfig`**: Judge/scoring columns for quality assessment
+- **`ExpressionColumnConfig`**: Expression-based derived columns (Python eval or Jinja2)
+- **`ValidationColumnConfig`**: Validation results (Python, SQL, Code, Remote validators)
+- **`SeedDatasetColumnConfig`**: Data from seed datasets
+
+See [src/data_designer/config/columns.py](src/data_designer/config/columns.py) for detailed schemas.
+
+## Model Configuration
+
+Models are configured via `ModelConfig` with:
+
+- `alias`: User-defined alias for the model
+- `model`: Model ID (e.g., from build.nvidia.com)
+- `inference_parameters`: Temperature, top_p, max_tokens (can be distribution-based)
+- `system_prompt`: Optional system prompt
+- `image_modality`: Support for image inputs
+
+See [src/data_designer/config/models.py](src/data_designer/config/models.py) for details.
+
+## Registry System
+
+The project uses a registry pattern for extensibility. Key registries:
+
+- **Column generators**: [src/data_designer/engine/column_generators/registry.py](src/data_designer/engine/column_generators/registry.py)
+- **Validators**: [src/data_designer/engine/validators/](src/data_designer/engine/validators/)
+- **Column profilers**: [src/data_designer/engine/analysis/column_profilers/registry.py](src/data_designer/engine/analysis/column_profilers/registry.py)
+- **Models**: [src/data_designer/engine/models/registry.py](src/data_designer/engine/models/registry.py)
+
+When adding new generators or validators, register them appropriately.
+
+## Pre-commit Hooks
+
+The project uses pre-commit hooks to enforce code quality. Install them with:
+
+```bash
+uv run pre-commit install
+```
+
+Hooks include:
+- Trailing whitespace removal
+- End-of-file fixer
+- YAML/JSON/TOML validation
+- Merge conflict detection
+- Debug statement detection
+- Ruff linting and formatting
+
+## Common Development Tasks
+
+```bash
+# Clean up generated files
+make clean
+
+# Update license headers
+make update-license-headers
+
+# Run all checks before committing
+make check-all-fix
+make test
+
+# Generate coverage report
+make coverage
+# View htmlcov/index.html in browser
+```
+
+## Additional Resources
+
+- **README.md**: Installation and basic usage examples
+- **src/data_designer/config/**: Configuration API documentation
+- **tests/**: Comprehensive test suite with usage examples