data-designer 0.1.3__py3-none-any.whl → 0.1.5__py3-none-any.whl

data_designer/essentials/__init__.py

@@ -1,137 +1,29 @@
 # SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
 # SPDX-License-Identifier: Apache-2.0
-from ..logging import LoggingConfig, configure_logging
+
+from data_designer.config.default_model_settings import resolve_seed_default_model_settings
+from data_designer.config.exports import *  # noqa: F403
+from data_designer.config.validator_params import LocalCallableValidatorParams
+from data_designer.interface.data_designer import DataDesigner
+from data_designer.logging import LoggingConfig, configure_logging
 
 configure_logging(LoggingConfig.default())
 
-from ..config.analysis.column_profilers import JudgeScoreProfilerConfig
-from ..config.column_configs import (
-    ExpressionColumnConfig,
-    LLMCodeColumnConfig,
-    LLMJudgeColumnConfig,
-    LLMStructuredColumnConfig,
-    LLMTextColumnConfig,
-    SamplerColumnConfig,
-    Score,
-    SeedDatasetColumnConfig,
-    ValidationColumnConfig,
-)
-from ..config.column_types import DataDesignerColumnType
-from ..config.config_builder import DataDesignerConfigBuilder
-from ..config.data_designer_config import DataDesignerConfig
-from ..config.dataset_builders import BuildStage
-from ..config.datastore import DatastoreSettings
-from ..config.models import (
-    ImageContext,
-    ImageFormat,
-    InferenceParameters,
-    ManualDistribution,
-    ManualDistributionParams,
-    Modality,
-    ModalityContext,
-    ModalityDataType,
-    ModelConfig,
-    UniformDistribution,
-    UniformDistributionParams,
-)
-from ..config.processors import DropColumnsProcessorConfig, ProcessorType
-from ..config.sampler_constraints import ColumnInequalityConstraint, ScalarInequalityConstraint
-from ..config.sampler_params import (
-    BernoulliMixtureSamplerParams,
-    BernoulliSamplerParams,
-    BinomialSamplerParams,
-    CategorySamplerParams,
-    DatetimeSamplerParams,
-    GaussianSamplerParams,
-    PersonFromFakerSamplerParams,
-    PersonSamplerParams,
-    PoissonSamplerParams,
-    SamplerType,
-    ScipySamplerParams,
-    SubcategorySamplerParams,
-    TimeDeltaSamplerParams,
-    UniformSamplerParams,
-    UUIDSamplerParams,
-)
-from ..config.seed import DatastoreSeedDatasetReference, IndexRange, PartitionBlock, SamplingStrategy, SeedConfig
-from ..config.utils.code_lang import CodeLang
-from ..config.utils.info import InfoType
-from ..config.utils.misc import can_run_data_designer_locally
-from ..config.validator_params import (
-    CodeValidatorParams,
-    RemoteValidatorParams,
-    ValidatorType,
-)
+# Resolve default model settings on import to ensure they are available when the library is used.
+resolve_seed_default_model_settings()
+
 
-local_library_imports = []
-try:
-    if can_run_data_designer_locally():
-        from ..config.validator_params import LocalCallableValidatorParams  # noqa: F401
-        from ..engine.model_provider import ModelProvider  # noqa: F401
-        from ..interface.data_designer import DataDesigner  # noqa: F401
+def get_essentials_exports() -> list[str]:
+    logging = [
+        configure_logging.__name__,
+        LoggingConfig.__name__,
+    ]
+    local = [
+        DataDesigner.__name__,
+        LocalCallableValidatorParams.__name__,
+    ]
 
-        local_library_imports = ["DataDesigner", "LocalCallableValidatorParams", "ModelProvider"]
-except ModuleNotFoundError:
-    pass
+    return logging + local + get_config_exports()  # noqa: F405
 
-__all__ = [
-    "BernoulliMixtureSamplerParams",
-    "BernoulliSamplerParams",
-    "BinomialSamplerParams",
-    "CategorySamplerParams",
-    "CodeLang",
-    "CodeValidatorParams",
-    "ColumnInequalityConstraint",
-    "configure_logging",
-    "DataDesignerColumnType",
-    "DataDesignerConfig",
-    "DataDesignerConfigBuilder",
-    "BuildStage",
-    "DatastoreSeedDatasetReference",
-    "DatastoreSettings",
-    "DatetimeSamplerParams",
-    "DropColumnsProcessorConfig",
-    "ExpressionColumnConfig",
-    "GaussianSamplerParams",
-    "IndexRange",
-    "InfoType",
-    "ImageContext",
-    "ImageFormat",
-    "InferenceParameters",
-    "JudgeScoreProfilerConfig",
-    "LLMCodeColumnConfig",
-    "LLMJudgeColumnConfig",
-    "LLMStructuredColumnConfig",
-    "LLMTextColumnConfig",
-    "LoggingConfig",
-    "ManualDistribution",
-    "ManualDistributionParams",
-    "Modality",
-    "ModalityContext",
-    "ModalityDataType",
-    "ModelConfig",
-    "PartitionBlock",
-    "PersonSamplerParams",
-    "PersonFromFakerSamplerParams",
-    "PoissonSamplerParams",
-    "ProcessorType",
-    "RemoteValidatorParams",
-    "SamplerColumnConfig",
-    "SamplerType",
-    "SamplingStrategy",
-    "ScalarInequalityConstraint",
-    "ScipySamplerParams",
-    "Score",
-    "SeedConfig",
-    "SeedDatasetColumnConfig",
-    "SubcategorySamplerParams",
-    "TimeDeltaSamplerParams",
-    "UniformDistribution",
-    "UniformDistributionParams",
-    "UniformSamplerParams",
-    "UUIDSamplerParams",
-    "ValidationColumnConfig",
-    "ValidatorType",
-]
 
-__all__.extend(local_library_imports)
+__all__ = get_essentials_exports()

data_designer/interface/data_designer.py

@@ -13,7 +13,6 @@ from data_designer.config.default_model_settings import (
     get_default_model_providers_missing_api_keys,
     get_default_provider_name,
     get_default_providers,
-    resolve_seed_default_model_settings,
 )
 from data_designer.config.interface import DataDesignerInterface
 from data_designer.config.models import (
@@ -31,7 +30,6 @@ from data_designer.config.utils.constants import (
 )
 from data_designer.config.utils.info import InfoType, InterfaceInfo
 from data_designer.config.utils.io_helpers import write_seed_dataset
-from data_designer.config.utils.misc import can_run_data_designer_locally
 from data_designer.engine.analysis.dataset_profiler import (
     DataDesignerDatasetProfiler,
     DatasetProfilerConfig,
@@ -66,11 +64,6 @@ DEFAULT_BUFFER_SIZE = 1000
 logger = logging.getLogger(__name__)
 
 
-# Resolve default model settings on import to ensure they are available when the library is used.
-if can_run_data_designer_locally():
-    resolve_seed_default_model_settings()
-
-
 class DataDesigner(DataDesignerInterface[DatasetCreationResults]):
     """Main interface for creating datasets with Data Designer.
 
@@ -256,6 +249,17 @@ class DataDesigner(DataDesignerInterface[DatasetCreationResults]):
         except Exception as e:
             raise DataDesignerProfilingError(f"🛑 Error profiling preview dataset: {e}")
 
+        if builder.artifact_storage.processors_outputs_path.exists():
+            processor_artifacts = {
+                processor_config.name: pd.read_parquet(
+                    builder.artifact_storage.processors_outputs_path / f"{processor_config.name}.parquet",
+                    dtype_backend="pyarrow",
+                ).to_dict(orient="records")
+                for processor_config in config_builder.get_processor_configs()
+            }
+        else:
+            processor_artifacts = {}
+
         if (
             len(processed_dataset) > 0
             and isinstance(analysis, DatasetProfilerResults)
@@ -266,6 +270,7 @@ class DataDesigner(DataDesignerInterface[DatasetCreationResults]):
         return PreviewResults(
             dataset=processed_dataset,
             analysis=analysis,
+            processor_artifacts=processor_artifacts,
             config_builder=config_builder,
         )
 
@@ -315,18 +320,17 @@ class DataDesigner(DataDesignerInterface[DatasetCreationResults]):
 
     def _resolve_model_providers(self, model_providers: list[ModelProvider] | None) -> list[ModelProvider]:
         if model_providers is None:
-            if can_run_data_designer_locally():
-                model_providers = get_default_providers()
-                missing_api_keys = get_default_model_providers_missing_api_keys()
-                if len(missing_api_keys) == len(PREDEFINED_PROVIDERS):
-                    logger.warning(
-                        "🚨 You are trying to use a default model provider but your API keys are missing."
-                        "\n\t\t\tSet the API key for the default providers you intend to use and re-initialize the Data Designer object."
-                        "\n\t\t\tAlternatively, you can provide your own model providers during Data Designer object initialization."
-                        "\n\t\t\tSee https://nvidia-nemo.github.io/DataDesigner/models/model-providers/ for more information."
-                    )
-                    self._get_interface_info(model_providers).display(InfoType.MODEL_PROVIDERS)
-                return model_providers
+            model_providers = get_default_providers()
+            missing_api_keys = get_default_model_providers_missing_api_keys()
+            if len(missing_api_keys) == len(PREDEFINED_PROVIDERS):
+                logger.warning(
+                    "🚨 You are trying to use a default model provider but your API keys are missing."
+                    "\n\t\t\tSet the API key for the default providers you intend to use and re-initialize the Data Designer object."
+                    "\n\t\t\tAlternatively, you can provide your own model providers during Data Designer object initialization."
+                    "\n\t\t\tSee https://nvidia-nemo.github.io/DataDesigner/concepts/models/model-providers/ for more information."
+                )
+                self._get_interface_info(model_providers).display(InfoType.MODEL_PROVIDERS)
+            return model_providers
         return model_providers or []
 
     def _create_dataset_builder(

data_designer/interface/results.py

@@ -3,12 +3,15 @@
 
 from __future__ import annotations
 
+from pathlib import Path
+
 import pandas as pd
 
 from data_designer.config.analysis.dataset_profiler import DatasetProfilerResults
 from data_designer.config.config_builder import DataDesignerConfigBuilder
 from data_designer.config.utils.visualization import WithRecordSamplerMixin
 from data_designer.engine.dataset_builders.artifact_storage import ArtifactStorage
+from data_designer.engine.dataset_builders.errors import ArtifactStorageError
 
 
 class DatasetCreationResults(WithRecordSamplerMixin):
@@ -53,3 +56,36 @@ class DatasetCreationResults(WithRecordSamplerMixin):
             A pandas DataFrame containing the full generated dataset.
         """
         return self.artifact_storage.load_dataset()
+
+    def load_processor_dataset(self, processor_name: str) -> pd.DataFrame:
+        """Load the dataset generated by a processor.
+
+        This only works for processors that write their artifacts in Parquet format.
+
+        Args:
+            processor_name: The name of the processor to load the dataset from.
+
+        Returns:
+            A pandas DataFrame containing the dataset generated by the processor.
+        """
+        try:
+            dataset = self.artifact_storage.read_parquet_files(
+                self.artifact_storage.processors_outputs_path / processor_name
+            )
+        except Exception as e:
+            raise ArtifactStorageError(f"Failed to load dataset for processor {processor_name}: {e}")
+
+        return dataset
+
+    def get_path_to_processor_artifacts(self, processor_name: str) -> Path:
+        """Get the path to the artifacts generated by a processor.
+
+        Args:
+            processor_name: The name of the processor to load the artifact from.
+
+        Returns:
+            The path to the artifacts.
+        """
+        if not self.artifact_storage.processors_outputs_path.exists():
+            raise ArtifactStorageError(f"Processor {processor_name} has no artifacts.")
+        return self.artifact_storage.processors_outputs_path / processor_name

data_designer/logging.py

@@ -1,11 +1,11 @@
 # SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
 # SPDX-License-Identifier: Apache-2.0
 
-from dataclasses import dataclass, field
 import logging
-from pathlib import Path
 import random
 import sys
+from dataclasses import dataclass, field
+from pathlib import Path
 from typing import TextIO, Union
 
 from pythonjsonlogger import jsonlogger

data_designer/plugin_manager.py

@@ -6,25 +6,16 @@ from __future__ import annotations
 from enum import Enum
 from typing import TYPE_CHECKING, Type, TypeAlias
 
-from .config.utils.misc import can_run_data_designer_locally
+from data_designer.plugins.plugin import PluginType
+from data_designer.plugins.registry import PluginRegistry
 
 if TYPE_CHECKING:
     from data_designer.plugins.plugin import Plugin
 
 
-if can_run_data_designer_locally():
-    from data_designer.plugins.plugin import PluginType
-    from data_designer.plugins.registry import PluginRegistry
-
-
 class PluginManager:
     def __init__(self):
-        if can_run_data_designer_locally():
-            self._plugins_supported = True
-            self._plugin_registry = PluginRegistry()
-        else:
-            self._plugins_supported = False
-            self._plugin_registry = None
+        self._plugin_registry = PluginRegistry()
 
     def get_column_generator_plugins(self) -> list[Plugin]:
         """Get all column generator plugins.
@@ -32,7 +23,7 @@ class PluginManager:
         Returns:
             A list of all column generator plugins.
         """
-        return self._plugin_registry.get_plugins(PluginType.COLUMN_GENERATOR) if self._plugins_supported else []
+        return self._plugin_registry.get_plugins(PluginType.COLUMN_GENERATOR)
 
     def get_column_generator_plugin_if_exists(self, plugin_name: str) -> Plugin | None:
         """Get a column generator plugin by name if it exists.
@@ -43,9 +34,8 @@ class PluginManager:
         Returns:
             The plugin if found, otherwise None.
         """
-        if self._plugins_supported and self._plugin_registry.plugin_exists(plugin_name):
+        if self._plugin_registry.plugin_exists(plugin_name):
             return self._plugin_registry.get_plugin(plugin_name)
-        return None
 
     def get_plugin_column_types(self, enum_type: Type[Enum], required_resources: list[str] | None = None) -> list[Enum]:
         """Get a list of plugin column types.
@@ -58,13 +48,12 @@ class PluginManager:
             A list of plugin column types.
         """
         type_list = []
-        if self._plugins_supported:
-            for plugin in self._plugin_registry.get_plugins(PluginType.COLUMN_GENERATOR):
-                if required_resources:
-                    task_required_resources = plugin.task_cls.metadata().required_resources or []
-                    if not all(resource in task_required_resources for resource in required_resources):
-                        continue
-                type_list.append(enum_type(plugin.name))
+        for plugin in self._plugin_registry.get_plugins(PluginType.COLUMN_GENERATOR):
+            if required_resources:
+                task_required_resources = plugin.task_cls.metadata().required_resources or []
+                if not all(resource in task_required_resources for resource in required_resources):
+                    continue
+            type_list.append(enum_type(plugin.name))
         return type_list
 
     def inject_into_column_config_type_union(self, column_config_type: Type[TypeAlias]) -> Type[TypeAlias]:
@@ -76,8 +65,7 @@ class PluginManager:
         Returns:
             The column config type with plugins injected.
         """
-        if self._plugins_supported:
-            column_config_type = self._plugin_registry.add_plugin_types_to_union(
-                column_config_type, PluginType.COLUMN_GENERATOR
-            )
+        column_config_type = self._plugin_registry.add_plugin_types_to_union(
+            column_config_type, PluginType.COLUMN_GENERATOR
+        )
         return column_config_type

data_designer/plugins/registry.py

@@ -1,10 +1,10 @@
 # SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
 # SPDX-License-Identifier: Apache-2.0
 
-from importlib.metadata import entry_points
 import logging
 import os
 import threading
+from importlib.metadata import entry_points
 from typing import Type, TypeAlias
 
 from typing_extensions import Self

{data_designer-0.1.3.dist-info → data_designer-0.1.5.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: data-designer
-Version: 0.1.3
+Version: 0.1.5
 Summary: General framework for synthetic data generation
 License-Expression: Apache-2.0
 License-File: LICENSE
@@ -144,12 +144,12 @@ preview.display_sample_record()
 
 ### 📚 Learn more
 
-- **[Quick Start Guide](https://nvidia-nemo.github.io/DataDesigner/quick-start/)** – Detailed walkthrough with more examples
-- **[Tutorial Notebooks](https://nvidia-nemo.github.io/DataDesigner/notebooks/)** – Step-by-step interactive tutorials
-- **[Column Types](https://nvidia-nemo.github.io/DataDesigner/concepts/columns/)** – Explore samplers, LLM columns, validators, and more
-- **[Validators](https://nvidia-nemo.github.io/DataDesigner/concepts/validators/)** – Learn how to validate generated data with Python, SQL, and remote validators
-- **[Model Configuration](https://nvidia-nemo.github.io/DataDesigner/models/model-configs/)** – Configure custom models and providers
-- **[Person Sampling](https://nvidia-nemo.github.io/DataDesigner/concepts/person_sampling/)** – Learn how to sample realistic person data with demographic attributes
+- **[Quick Start Guide](https://nvidia-nemo.github.io/DataDesigner/latest/quick-start/)** – Detailed walkthrough with more examples
+- **[Tutorial Notebooks](https://nvidia-nemo.github.io/DataDesigner/latest/notebooks/)** – Step-by-step interactive tutorials
+- **[Column Types](https://nvidia-nemo.github.io/DataDesigner/latest/concepts/columns/)** – Explore samplers, LLM columns, validators, and more
+- **[Validators](https://nvidia-nemo.github.io/DataDesigner/latest/concepts/validators/)** – Learn how to validate generated data with Python, SQL, and remote validators
+- **[Model Configuration](https://nvidia-nemo.github.io/DataDesigner/latest/concepts/models/model-configs/)** – Configure custom models and providers
+- **[Person Sampling](https://nvidia-nemo.github.io/DataDesigner/latest/concepts/person_sampling/)** – Learn how to sample realistic person data with demographic attributes
 
 ### 🔧 Configure models via CLI
 
@@ -161,7 +161,7 @@ data-designer config list      # View current settings
 
 ### 🤝 Get involved
 
-- **[Contributing Guide](https://nvidia-nemo.github.io/DataDesigner/CONTRIBUTING)** – Help improve Data Designer
+- **[Contributing Guide](https://nvidia-nemo.github.io/DataDesigner/latest/CONTRIBUTING)** – Help improve Data Designer
 - **[GitHub Issues](https://github.com/NVIDIA-NeMo/DataDesigner/issues)** – Report bugs or make a feature request
 
 ---
@@ -178,7 +178,7 @@ If you use NeMo Data Designer in your research, please cite it using the followi
 
 ```bibtex
 @misc{nemo-data-designer,
-  author = {The NeMo Data Designer Team},
+  author = {The NeMo Data Designer Team, NVIDIA},
   title = {NeMo Data Designer: A framework for generating synthetic data from scratch or based on your own seed data},
   howpublished = {\url{https://github.com/NVIDIA-NeMo/DataDesigner}},
   year = {2025},