data-designer 0.1.5__py3-none-any.whl → 0.2.0__py3-none-any.whl

data_designer/_version.py

@@ -28,7 +28,7 @@ version_tuple: VERSION_TUPLE
 commit_id: COMMIT_ID
 __commit_id__: COMMIT_ID
 
-__version__ = version = '0.1.5'
-__version_tuple__ = version_tuple = (0, 1, 5)
+__version__ = version = '0.2.0'
+__version_tuple__ = version_tuple = (0, 2, 0)
 
 __commit_id__ = commit_id = None

data_designer/cli/README.md

@@ -129,8 +129,10 @@ class ConfigRepository(ABC, Generic[T]):
 - Field-level validation
 - Auto-completion support
 - History navigation (arrow keys)
-- Default value handling
+- Current value display when editing (`(current value: X)` instead of `(default: X)`)
+- Value clearing support (type `'clear'` to remove optional parameter values)
 - Back navigation support
+- Empty input handling (Enter key keeps current value or skips optional fields)
 
 #### 6. **UI Utilities** (`ui.py`)
 - **Purpose**: User interface utilities for terminal output and input
@@ -179,17 +181,29 @@ model_configs:
     model: meta/llama-3.1-70b-instruct
     provider: nvidia
     inference_parameters:
+      generation_type: chat-completion
       temperature: 0.7
       top_p: 0.9
       max_tokens: 2048
       max_parallel_requests: 4
+      timeout: 60
   - alias: gpt-4
     model: gpt-4-turbo
     provider: openai
     inference_parameters:
+      generation_type: chat-completion
       temperature: 0.8
       top_p: 0.95
       max_tokens: 4096
+      max_parallel_requests: 4
+  - alias: embedder
+    model: text-embedding-3-large
+    provider: openai
+    inference_parameters:
+      generation_type: embedding
+      encoding_format: float
+      dimensions: 1024
+      max_parallel_requests: 4
 ```
 
 ## Usage Examples

data_designer/cli/commands/download.py

@@ -0,0 +1,56 @@
+# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+import typer
+
+from data_designer.cli.controllers.download_controller import DownloadController
+from data_designer.config.utils.constants import DATA_DESIGNER_HOME
+
+
+def personas_command(
+    locales: list[str] = typer.Option(
+        None,
+        "--locale",
+        "-l",
+        help="Locales to download (en_US, en_IN, hi_Deva_IN, hi_Latn_IN, ja_JP). Can be specified multiple times.",
+    ),
+    all_locales: bool = typer.Option(
+        False,
+        "--all",
+        help="Download all available locales",
+    ),
+    dry_run: bool = typer.Option(
+        False,
+        "--dry-run",
+        help="Show what would be downloaded without actually downloading",
+    ),
+    list_available: bool = typer.Option(
+        False,
+        "--list",
+        help="List available persona datasets and their sizes",
+    ),
+) -> None:
+    """Download Nemotron-Personas datasets for synthetic data generation.
+
+    Examples:
+        # List available datasets
+        data-designer download personas --list
+
+        # Interactive selection
+        data-designer download personas
+
+        # Download specific locales
+        data-designer download personas --locale en_US --locale ja_JP
+
+        # Download all available locales
+        data-designer download personas --all
+
+        # Preview what would be downloaded
+        data-designer download personas --all --dry-run
+    """
+    controller = DownloadController(DATA_DESIGNER_HOME)
+
+    if list_available:
+        controller.list_personas()
+    else:
+        controller.run_personas(locales=locales, all_locales=all_locales, dry_run=dry_run)

data_designer/cli/commands/list.py

@@ -95,32 +95,18 @@ def display_models(model_repo: ModelRepository) -> None:
         # Display as table
         table = Table(title="Model Configurations", border_style=NordColor.NORD8.value)
         table.add_column("Alias", style=NordColor.NORD14.value, no_wrap=True)
-        table.add_column("Model ID", style=NordColor.NORD4.value)
+        table.add_column("Model", style=NordColor.NORD4.value)
         table.add_column("Provider", style=NordColor.NORD9.value, no_wrap=True)
-        table.add_column("Temperature", style=NordColor.NORD15.value, justify="right")
-        table.add_column("Top P", style=NordColor.NORD15.value, justify="right")
-        table.add_column("Max Tokens", style=NordColor.NORD15.value, justify="right")
+        table.add_column("Inference Parameters", style=NordColor.NORD15.value)
 
         for mc in registry.model_configs:
-            # Handle distribution-based parameters
-            temp_display = (
-                f"{mc.inference_parameters.temperature:.2f}"
-                if isinstance(mc.inference_parameters.temperature, (int, float))
-                else "dist"
-            )
-            top_p_display = (
-                f"{mc.inference_parameters.top_p:.2f}"
-                if isinstance(mc.inference_parameters.top_p, (int, float))
-                else "dist"
-            )
+            params_display = mc.inference_parameters.format_for_display()
 
             table.add_row(
                 mc.alias,
                 mc.model,
                 mc.provider or "(default)",
-                temp_display,
-                top_p_display,
-                str(mc.inference_parameters.max_tokens) if mc.inference_parameters.max_tokens else "(none)",
+                params_display,
             )
 
         console.print(table)

data_designer/cli/controllers/__init__.py

@@ -1,7 +1,8 @@
 # SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
 # SPDX-License-Identifier: Apache-2.0
 
+from data_designer.cli.controllers.download_controller import DownloadController
 from data_designer.cli.controllers.model_controller import ModelController
 from data_designer.cli.controllers.provider_controller import ProviderController
 
-__all__ = ["ModelController", "ProviderController"]
+__all__ = ["DownloadController", "ModelController", "ProviderController"]

data_designer/cli/controllers/download_controller.py

@@ -0,0 +1,217 @@
+# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+import subprocess
+from pathlib import Path
+
+from data_designer.cli.repositories.persona_repository import PersonaRepository
+from data_designer.cli.services.download_service import DownloadService
+from data_designer.cli.ui import (
+    confirm_action,
+    console,
+    print_error,
+    print_header,
+    print_info,
+    print_success,
+    print_text,
+    select_multiple_with_arrows,
+)
+from data_designer.cli.utils import check_ngc_cli_available, get_ngc_version
+
+NGC_URL = "https://catalog.ngc.nvidia.com/"
+NGC_CLI_INSTALL_URL = "https://org.ngc.nvidia.com/setup/installers/cli"
+
+
+class DownloadController:
+    """Controller for asset download workflows."""
+
+    def __init__(self, config_dir: Path):
+        self.config_dir = config_dir
+        self.persona_repository = PersonaRepository()
+        self.service = DownloadService(config_dir, self.persona_repository)
+
+    def list_personas(self) -> None:
+        """List available persona datasets and their sizes."""
+        print_header("Available Nemotron-Persona Datasets")
+        console.print()
+
+        available_locales = self.persona_repository.list_all()
+
+        print_text("📦 Available locales:")
+        console.print()
+
+        for locale in available_locales:
+            already_downloaded = self.service.is_locale_downloaded(locale.code)
+            status = " (downloaded)" if already_downloaded else ""
+            print_text(f"  • {locale.code}: {locale.size}{status}")
+
+        console.print()
+        print_info(f"Total: {len(available_locales)} datasets available")
+
+    def run_personas(self, locales: list[str] | None, all_locales: bool, dry_run: bool = False) -> None:
+        """Main entry point for persona dataset downloads.
+
+        Args:
+            locales: List of locale codes to download (if provided via CLI flags)
+            all_locales: If True, download all available locales
+            dry_run: If True, only show what would be downloaded without actually downloading
+        """
+        header = "Download Nemotron-Persona Datasets (Dry Run)" if dry_run else "Download Nemotron-Persona Datasets"
+        print_header(header)
+        print_info(f"Datasets will be saved to: {self.service.get_managed_assets_directory()}")
+        console.print()
+
+        # Check NGC CLI availability (skip checking in dry run mode)
+        if not dry_run and not check_ngc_cli_with_instructions():
+            return
+
+        # Determine which locales to download
+        selected_locales = self._determine_locales(locales, all_locales)
+
+        if not selected_locales:
+            print_info("No locales selected")
+            return
+
+        # Show what will be downloaded
+        console.print()
+        action = "Would download" if dry_run else "Will download"
+        print_text(f"📦 {action} {len(selected_locales)} Nemotron-Persona dataset(s):")
+        for locale_code in selected_locales:
+            locale = self.persona_repository.get_by_code(locale_code)
+            already_downloaded = self.service.is_locale_downloaded(locale_code)
+            status = " - already exists, will update" if already_downloaded else ""
+            size = locale.size if locale else "unknown"
+            print_text(f"  • {locale_code} ({size}){status}")
+
+        console.print()
+
+        # In dry run mode, exit here
+        if dry_run:
+            print_info("Dry run complete - no files were downloaded")
+            return
+
+        # Confirm download
+        if not confirm_action("Proceed with download?", default=True):
+            print_info("Download cancelled")
+            return
+
+        # Download each locale
+        console.print()
+        successful = []
+        failed = []
+
+        for locale in selected_locales:
+            if self._download_locale(locale):
+                successful.append(locale)
+            else:
+                failed.append(locale)
+
+        # Summary
+        console.print()
+        if successful:
+            print_success(f"Successfully downloaded {len(successful)} dataset(s): {', '.join(successful)}")
+            print_info(f"Saved datasets to: {self.service.get_managed_assets_directory()}")
+
+        if failed:
+            print_error(f"Failed to download {len(failed)} dataset(s): {', '.join(failed)}")
+
+    def _determine_locales(self, locales: list[str] | None, all_locales: bool) -> list[str]:
+        """Determine which locales to download based on user input.
+
+        Args:
+            locales: List of locales from CLI flags (may be None)
+            all_locales: Whether to download all locales
+
+        Returns:
+            List of locale codes to download
+        """
+        available_locales = self.service.get_available_locales()
+
+        # If --all flag is set, return all locales
+        if all_locales:
+            return list(available_locales.keys())
+
+        # If locales specified via flags, validate and return them
+        if locales:
+            invalid_locales = [loc for loc in locales if loc not in available_locales]
+            if invalid_locales:
+                print_error(f"Invalid locale(s): {', '.join(invalid_locales)}")
+                print_info(f"Available locales: {', '.join(available_locales.keys())}")
+                return []
+            return locales
+
+        # Interactive multi-select
+        return self._select_locales_interactive(available_locales)
+
+    def _select_locales_interactive(self, available_locales: dict[str, str]) -> list[str]:
+        """Interactive multi-select for locales.
+
+        Args:
+            available_locales: Dictionary of {locale_code: description}
+
+        Returns:
+            List of selected locale codes
+        """
+        console.print()
+        print_text("Select locales you want to download:")
+        console.print()
+
+        selected = select_multiple_with_arrows(
+            options=available_locales,
+            prompt_text="Use ↑/↓ to navigate, Space to toggle ✓, Enter to confirm:",
+            default_keys=None,
+            allow_empty=False,
+        )
+
+        return selected if selected else []
+
+    def _download_locale(self, locale: str) -> bool:
+        """Download a single locale using NGC CLI.
+
+        Args:
+            locale: Locale code to download
+
+        Returns:
+            True if download succeeded, False otherwise
+        """
+        # Print header before download (NGC CLI will show its own progress)
+        print_text(f"📦 Downloading Nemotron-Persona dataset for {locale}...")
+        console.print()
+
+        try:
+            self.service.download_persona_dataset(locale)
+            console.print()
+            print_success(f"✓ Downloaded Nemotron-Persona dataset for {locale}")
+            return True
+
+        except subprocess.CalledProcessError as e:
+            console.print()
+            print_error(f"✗ Failed to download Nemotron-Persona dataset for {locale}")
+            print_error(f"NGC CLI error: {e}")
+            return False
+
+        except Exception as e:
+            console.print()
+            print_error(f"✗ Failed to download Nemotron-Persona dataset for {locale}")
+            print_error(f"Unexpected error: {e}")
+            return False
+
+
+def check_ngc_cli_with_instructions() -> bool:
+    """Check if NGC CLI is installed and guide user if not."""
+    if check_ngc_cli_available():
+        version = get_ngc_version()
+        if version:
+            print_info(version)
+        return True
+
+    print_error("NGC CLI not found!")
+    console.print()
+    print_text("The NGC CLI is required to download the Nemotron-Personas datasets.")
+    console.print()
+    print_text("To download the Nemotron-Personas datasets, follow these steps:")
+    print_text(f"    1. Create an NVIDIA NGC account: {NGC_URL}")
+    print_text(f"    2. Install the NGC CLI: {NGC_CLI_INSTALL_URL}")
+    print_text("    3. Following the install instructions to set up the NGC CLI")
+    print_text("    4. Run 'data-designer download personas'")
+    return False

data_designer/cli/controllers/model_controller.py

@@ -160,9 +160,10 @@ class ModelController:
             return
 
         # Check if model has distribution-based parameters
-        if hasattr(model.inference_parameters.temperature, "sample") or hasattr(
-            model.inference_parameters.top_p, "sample"
-        ):
+        params_dict = model.inference_parameters.model_dump(mode="json", exclude_none=True)
+        has_distribution = any(isinstance(v, dict) and "distribution_type" in v for v in params_dict.values())
+
+        if has_distribution:
             print_warning(
                 "This model uses distribution-based inference parameters, "
                 "which cannot be edited via the CLI. Please edit the configuration file directly."

data_designer/cli/forms/field.py

@@ -5,6 +5,7 @@ from abc import ABC, abstractmethod
 from collections.abc import Callable
 from typing import Any, Generic, TypeVar
 
+from data_designer.cli.ui import BACK, prompt_text_input, select_with_arrows
 from data_designer.cli.utils import validate_numeric_range
 
 T = TypeVar("T")
@@ -40,8 +41,14 @@ class Field(ABC, Generic[T]):
         return self._value
 
     @value.setter
-    def value(self, val: T) -> None:
-        """Set and validate the field value."""
+    def value(self, val: T | str) -> None:
+        """Set and validate the field value. Converts empty strings to None for optional fields."""
+        # Handle empty string for optional fields (clearing the value)
+        if val == "" and not self.required:
+            self._value = None
+            return
+
+        # Standard validation for non-empty values
         if self.validator:
             # For string validators, convert to string first if needed
             val_str = str(val) if not isinstance(val, str) else val
@@ -50,6 +57,40 @@ class Field(ABC, Generic[T]):
                 raise ValidationError(error_msg or "Invalid value")
         self._value = val
 
+    def _build_prompt_text(self) -> str:
+        """Build prompt text with current value information."""
+        has_current_value = self.default is not None
+
+        if has_current_value:
+            # Show as "current" instead of "default" with dimmed styling
+            if not self.required:
+                return f"{self.prompt} <dim>(current value: {self.default}, type 'clear' to remove)</dim>"
+            return f"{self.prompt} <dim>(current value: {self.default})</dim>"
+
+        return self.prompt
+
+    def _handle_prompt_result(self, result: str | None | Any) -> str | None | Any:
+        """Handle common prompt result logic (BACK, None, clear keywords, empty input)."""
+        if result is BACK:
+            return BACK
+
+        if result is None:
+            # User cancelled (ESC)
+            return None
+
+        # Check for special keywords to clear the value
+        if result and result.lower() in ("clear", "none", "default"):
+            return ""
+
+        if not result:
+            # Empty input: return current value if exists
+            has_current_value = self.default is not None
+            if has_current_value:
+                return self.default
+            return ""
+
+        return result
+
     @abstractmethod
     def prompt_user(self, allow_back: bool = False) -> T | None | Any:
         """Prompt user for input."""
@@ -75,21 +116,19 @@ class TextField(Field[str]):
 
     def prompt_user(self, allow_back: bool = False) -> str | None | Any:
         """Prompt user for text input."""
-        from data_designer.cli.ui import BACK, prompt_text_input
+        prompt_text = self._build_prompt_text()
 
+        # Don't pass default to prompt_text_input to avoid duplicate "(default: X)" text
         result = prompt_text_input(
-            self.prompt,
-            default=self.default,
+            prompt_text,
+            default=None,
             validator=self.validator,
             mask=self.mask,
             completions=self.completions,
             allow_back=allow_back,
         )
 
-        if result is BACK:
-            return BACK
-
-        return result
+        return self._handle_prompt_result(result)
 
 
 class SelectField(Field[str]):
@@ -109,8 +148,6 @@ class SelectField(Field[str]):
 
     def prompt_user(self, allow_back: bool = False) -> str | None | Any:
         """Prompt user for selection."""
-        from data_designer.cli.ui import BACK, select_with_arrows
-
         result = select_with_arrows(
             self.options,
             self.prompt,
@@ -144,6 +181,9 @@ class NumericField(Field[float]):
         def range_validator(value: str) -> tuple[bool, str | None]:
             if not value and not required:
                 return True, None
+            # Allow special keywords to clear the value
+            if value and value.lower() in ("clear", "none", "default"):
+                return True, None
             if min_value is not None and max_value is not None:
                 is_valid, parsed = validate_numeric_range(value, min_value, max_value)
                 if not is_valid:
@@ -163,18 +203,24 @@ class NumericField(Field[float]):
 
     def prompt_user(self, allow_back: bool = False) -> float | None | Any:
         """Prompt user for numeric input."""
-        from data_designer.cli.ui import BACK, prompt_text_input
-
-        default_str = str(self.default) if self.default is not None else None
+        prompt_text = self._build_prompt_text()
 
+        # Don't pass default to prompt_text_input to avoid duplicate "(default: X)" text
         result = prompt_text_input(
-            self.prompt,
-            default=default_str,
+            prompt_text,
+            default=None,
             validator=self.validator,
             allow_back=allow_back,
         )
 
-        if result is BACK:
-            return BACK
+        result = self._handle_prompt_result(result)
 
-        return float(result) if result else None
+        # Return special values (BACK, None, empty string, defaults) as-is
+        if result is BACK or result is None or result == "":
+            return result
+
+        # Convert numeric strings to float (but not if it's already a float from default)
+        if isinstance(result, str):
+            return float(result)
+
+        return result