data-designer 0.2.3__py3-none-any.whl → 0.3.0__py3-none-any.whl

data_designer/_version.py

@@ -28,7 +28,7 @@ version_tuple: VERSION_TUPLE
 commit_id: COMMIT_ID
 __commit_id__: COMMIT_ID
 
-__version__ = version = '0.2.3'
-__version_tuple__ = version_tuple = (0, 2, 3)
+__version__ = version = '0.3.0'
+__version_tuple__ = version_tuple = (0, 3, 0)
 
 __commit_id__ = commit_id = None

data_designer/cli/forms/model_builder.py

@@ -125,11 +125,11 @@ class ModelFormBuilder(FormBuilder[ModelConfig]):
             fields.append(
                 NumericField(
                     "max_tokens",
-                    "Max tokens <dim>(maximum total tokens including input and output)</dim>",
+                    "Max tokens <dim>(maximum tokens to generate in response)</dim>",
                     default=initial_params.get("max_tokens"),
                     min_value=1.0,
                     required=False,
-                    help_text="Maximum number of tokens including both input prompt and generated response",
+                    help_text="Maximum number of tokens to generate in the response",
                 )
             )
 

data_designer/config/config_builder.py

@@ -24,9 +24,8 @@ from data_designer.config.column_types import (
 )
 from data_designer.config.data_designer_config import DataDesignerConfig
 from data_designer.config.dataset_builders import BuildStage
-from data_designer.config.datastore import DatastoreSettings, fetch_seed_dataset_column_names
 from data_designer.config.default_model_settings import get_default_model_configs
-from data_designer.config.errors import BuilderConfigurationError, InvalidColumnTypeError, InvalidConfigError
+from data_designer.config.errors import BuilderConfigurationError, BuilderSerializationError, InvalidColumnTypeError
 from data_designer.config.models import ModelConfig, load_model_configs
 from data_designer.config.processors import ProcessorConfigT, ProcessorType, get_processor_config_from_kwargs
 from data_designer.config.sampler_constraints import (
@@ -36,20 +35,17 @@ from data_designer.config.sampler_constraints import (
     ScalarInequalityConstraint,
 )
 from data_designer.config.seed import (
-    DatastoreSeedDatasetReference,
     IndexRange,
-    LocalSeedDatasetReference,
     PartitionBlock,
     SamplingStrategy,
     SeedConfig,
-    SeedDatasetReference,
 )
+from data_designer.config.seed_source import DataFrameSeedSource, SeedSource
 from data_designer.config.utils.constants import DEFAULT_REPR_HTML_STYLE, REPR_HTML_TEMPLATE
 from data_designer.config.utils.info import ConfigBuilderInfo
 from data_designer.config.utils.io_helpers import serialize_data, smart_load_yaml
 from data_designer.config.utils.misc import can_run_data_designer_locally, json_indent_list_of_strings, kebab_to_snake
 from data_designer.config.utils.type_helpers import resolve_string_enum
-from data_designer.config.utils.validation import ViolationLevel, rich_print_violations, validate_data_designer_config
 
 logger = logging.getLogger(__name__)
 
@@ -63,12 +59,9 @@ class BuilderConfig(ExportableConfigBase):
     Attributes:
         data_designer: The main Data Designer configuration containing columns,
             constraints, profilers, and other settings.
-        datastore_settings: Optional datastore settings for accessing external
-            datasets.
     """
 
     data_designer: DataDesignerConfig
-    datastore_settings: DatastoreSettings | None
 
 
 class DataDesignerConfigBuilder:
@@ -101,30 +94,19 @@ class DataDesignerConfigBuilder:
             builder_config = BuilderConfig.model_validate(json_config)
 
         builder = cls(model_configs=builder_config.data_designer.model_configs)
-        config = builder_config.data_designer
+        data_designer_config = builder_config.data_designer
 
-        for col in config.columns:
+        for col in data_designer_config.columns:
             builder.add_column(col)
 
-        for constraint in config.constraints or []:
+        for constraint in data_designer_config.constraints or []:
             builder.add_constraint(constraint=constraint)
 
-        if config.seed_config:
-            if builder_config.datastore_settings is None:
-                if can_run_data_designer_locally():
-                    seed_dataset_reference = LocalSeedDatasetReference(dataset=config.seed_config.dataset)
-                else:
-                    raise BuilderConfigurationError("🛑 Datastore settings are required.")
-            else:
-                seed_dataset_reference = DatastoreSeedDatasetReference(
-                    dataset=config.seed_config.dataset,
-                    datastore_settings=builder_config.datastore_settings,
-                )
-                builder.set_seed_datastore_settings(builder_config.datastore_settings)
+        if (seed_config := data_designer_config.seed_config) is not None:
             builder.with_seed_dataset(
-                seed_dataset_reference,
-                sampling_strategy=config.seed_config.sampling_strategy,
-                selection_strategy=config.seed_config.selection_strategy,
+                seed_config.source,
+                sampling_strategy=seed_config.sampling_strategy,
+                selection_strategy=seed_config.selection_strategy,
             )
 
         return builder
@@ -144,7 +126,6 @@ class DataDesignerConfigBuilder:
         self._seed_config: SeedConfig | None = None
         self._constraints: list[ColumnConstraintT] = []
         self._profilers: list[ColumnProfilerConfigT] = []
-        self._datastore_settings: DatastoreSettings | None = None
 
     @property
     def model_configs(self) -> list[ModelConfig]:
@@ -243,13 +224,6 @@ class DataDesignerConfigBuilder:
                 f"{', '.join([t.__name__ for t in allowed_column_configs])}"
             )
 
-        existing_config = self._column_configs.get(column_config.name)
-        if existing_config is not None and isinstance(existing_config, SeedDatasetColumnConfig):
-            raise BuilderConfigurationError(
-                f"🛑 Column {column_config.name!r} already exists as a seed dataset column. "
-                "Please use a different column name or update the seed dataset."
-            )
-
         self._column_configs[column_config.name] = column_config
         return self
 
@@ -371,19 +345,12 @@ class DataDesignerConfigBuilder:
         """
         return self._profilers
 
-    def build(self, *, skip_validation: bool = False, raise_exceptions: bool = False) -> DataDesignerConfig:
+    def build(self) -> DataDesignerConfig:
         """Build a DataDesignerConfig instance based on the current builder configuration.
 
-        Args:
-            skip_validation: Whether to skip validation of the configuration.
-            raise_exceptions: Whether to raise an exception if the configuration is invalid.
-
         Returns:
             The current Data Designer config object.
         """
-        if not skip_validation:
-            self.validate(raise_exceptions=raise_exceptions)
-
         return DataDesignerConfig(
             model_configs=self._model_configs,
             seed_config=self._seed_config,
@@ -512,14 +479,6 @@ class DataDesignerConfigBuilder:
         """
         return self._seed_config
 
-    def get_seed_datastore_settings(self) -> DatastoreSettings | None:
-        """Get most recent datastore settings for the current Data Designer configuration.
-
-        Returns:
-            The datastore settings if configured, None otherwise.
-        """
-        return None if not self._datastore_settings else DatastoreSettings.model_validate(self._datastore_settings)
-
     def num_columns_of_type(self, column_type: DataDesignerColumnType) -> int:
         """Get the count of columns of the specified type.
 
@@ -531,85 +490,33 @@ class DataDesignerConfigBuilder:
         """
         return len(self.get_columns_of_type(column_type))
 
-    def set_seed_datastore_settings(self, datastore_settings: DatastoreSettings | None) -> Self:
-        """Set the datastore settings for the seed dataset.
-
-        Args:
-            datastore_settings: The datastore settings to use for the seed dataset.
-        """
-        self._datastore_settings = datastore_settings
-        return self
-
-    def validate(self, *, raise_exceptions: bool = False) -> Self:
-        """Validate the current Data Designer configuration.
-
-        Args:
-            raise_exceptions: Whether to raise an exception if the configuration is invalid.
-
-        Returns:
-            The current Data Designer config builder instance.
-
-        Raises:
-            InvalidConfigError: If the configuration is invalid and raise_exceptions is True.
-        """
-
-        violations = validate_data_designer_config(
-            columns=list(self._column_configs.values()),
-            processor_configs=self._processor_configs,
-            allowed_references=self.allowed_references,
-        )
-        rich_print_violations(violations)
-        if raise_exceptions and len([v for v in violations if v.level == ViolationLevel.ERROR]) > 0:
-            raise InvalidConfigError(
-                "🛑 Your configuration contains validation errors. Please address the indicated issues and try again."
-            )
-        if len(violations) == 0:
-            logger.info("✅ Validation passed")
-        return self
-
     def with_seed_dataset(
         self,
-        dataset_reference: SeedDatasetReference,
+        seed_source: SeedSource,
         *,
         sampling_strategy: SamplingStrategy = SamplingStrategy.ORDERED,
         selection_strategy: IndexRange | PartitionBlock | None = None,
     ) -> Self:
         """Add a seed dataset to the current Data Designer configuration.
 
-        This method sets the seed dataset for the configuration and automatically creates
-        SeedDatasetColumnConfig objects for each column found in the dataset. The column
-        names are fetched from the dataset source, which can be the Hugging Face Hub, the
-        NeMo Microservices Datastore, or in the case of direct library usage, a local file.
+        This method sets the seed dataset for the configuration, but columns are not resolved until
+        compilation (including validation) is performed by the engine using a SeedReader.
 
         Args:
-            dataset_reference: Seed dataset reference for fetching from the datastore.
+            seed_source: The pointer to the seed dataset.
             sampling_strategy: The sampling strategy to use when generating data from the seed dataset.
                 Defaults to ORDERED sampling.
+            selection_strategy: An optional selection strategy to use when generating data from the seed dataset.
+                Defaults to None.
 
         Returns:
             The current Data Designer config builder instance.
-
-        Raises:
-            BuilderConfigurationError: If any seed dataset column name collides with an existing column.
         """
-        seed_column_names = fetch_seed_dataset_column_names(dataset_reference)
-        colliding_columns = [name for name in seed_column_names if name in self._column_configs]
-        if colliding_columns:
-            raise BuilderConfigurationError(
-                f"🛑 Seed dataset column(s) {colliding_columns} collide with existing column(s). "
-                "Please remove the conflicting columns or use a seed dataset with different column names."
-            )
-
         self._seed_config = SeedConfig(
-            dataset=dataset_reference.dataset,
+            source=seed_source,
             sampling_strategy=sampling_strategy,
             selection_strategy=selection_strategy,
         )
-        self.set_seed_datastore_settings(
-            dataset_reference.datastore_settings if hasattr(dataset_reference, "datastore_settings") else None
-        )
-        for column_name in seed_column_names:
-            self._column_configs[column_name] = SeedDatasetColumnConfig(name=column_name)
         return self
 
     def write_config(self, path: str | Path, indent: int | None = 2, **kwargs) -> None:
@@ -622,7 +529,17 @@ class DataDesignerConfigBuilder:
 
         Raises:
             BuilderConfigurationError: If the file format is unsupported.
-        """
+            BuilderSerializationError: If the configuration cannot be serialized.
+        """
+        if (seed_config := self.get_seed_config()) is not None and isinstance(seed_config.source, DataFrameSeedSource):
+            raise BuilderSerializationError(
+                "This builder was configured with a DataFrame seed dataset. "
+                "DataFrame seeds cannot be serialized to config files. "
+                "To serialize this configuration, change your seed dataset to a more persistent, serializable source format. "
+                "For example, you could make a local file seed source from the dataframe:\n\n"
+                "LocalFileSeedSource.from_dataframe(my_dataframe, '/path/to/data.parquet')"
+            )
+
         cfg = self.get_builder_config()
         suffix = Path(path).suffix
         if suffix in {".yaml", ".yml"}:
@@ -638,7 +555,7 @@ class DataDesignerConfigBuilder:
         Returns:
             The builder config.
         """
-        return BuilderConfig(data_designer=self.build(), datastore_settings=self._datastore_settings)
+        return BuilderConfig(data_designer=self.build())
 
     def __repr__(self) -> str:
         """Generates a string representation of the DataDesignerConfigBuilder instance.
@@ -650,7 +567,7 @@ class DataDesignerConfigBuilder:
             return f"{self.__class__.__name__}()"
 
         props_to_repr = {
-            "seed_dataset": (None if self._seed_config is None else f"'{self._seed_config.dataset}'"),
+            "seed_dataset": (None if self._seed_config is None else f"{self._seed_config.source.seed_type} seed"),
         }
 
         for column_type in get_column_display_order():

data_designer/config/errors.py

@@ -7,6 +7,9 @@ from data_designer.errors import DataDesignerError
 class BuilderConfigurationError(DataDesignerError): ...
 
 
+class BuilderSerializationError(DataDesignerError): ...
+
+
 class InvalidColumnTypeError(DataDesignerError): ...
 
 

data_designer/config/exports.py

@@ -18,14 +18,12 @@ from data_designer.config.column_types import DataDesignerColumnType
 from data_designer.config.config_builder import DataDesignerConfigBuilder
 from data_designer.config.data_designer_config import DataDesignerConfig
 from data_designer.config.dataset_builders import BuildStage
-from data_designer.config.datastore import DatastoreSettings
 from data_designer.config.models import (
     ChatCompletionInferenceParams,
     EmbeddingInferenceParams,
     GenerationType,
     ImageContext,
     ImageFormat,
-    InferenceParameters,
     ManualDistribution,
     ManualDistributionParams,
     Modality,
@@ -60,12 +58,16 @@ from data_designer.config.sampler_params import (
     UUIDSamplerParams,
 )
 from data_designer.config.seed import (
-    DatastoreSeedDatasetReference,
     IndexRange,
     PartitionBlock,
     SamplingStrategy,
     SeedConfig,
 )
+from data_designer.config.seed_source import (
+    DataFrameSeedSource,
+    HuggingFaceSeedSource,
+    LocalFileSeedSource,
+)
 from data_designer.config.utils.code_lang import CodeLang
 from data_designer.config.utils.info import InfoType
 from data_designer.config.validator_params import (
@@ -89,9 +91,8 @@ def get_config_exports() -> list[str]:
         DataDesignerColumnType.__name__,
         DataDesignerConfig.__name__,
         DataDesignerConfigBuilder.__name__,
+        DataFrameSeedSource.__name__,
         BuildStage.__name__,
-        DatastoreSeedDatasetReference.__name__,
-        DatastoreSettings.__name__,
         DatetimeSamplerParams.__name__,
         DropColumnsProcessorConfig.__name__,
         EmbeddingColumnConfig.__name__,
@@ -99,16 +100,17 @@ def get_config_exports() -> list[str]:
         ExpressionColumnConfig.__name__,
         GaussianSamplerParams.__name__,
         GenerationType.__name__,
+        HuggingFaceSeedSource.__name__,
         IndexRange.__name__,
         InfoType.__name__,
         ImageContext.__name__,
         ImageFormat.__name__,
-        InferenceParameters.__name__,
         JudgeScoreProfilerConfig.__name__,
         LLMCodeColumnConfig.__name__,
         LLMJudgeColumnConfig.__name__,
         LLMStructuredColumnConfig.__name__,
         LLMTextColumnConfig.__name__,
+        LocalFileSeedSource.__name__,
         ManualDistribution.__name__,
         ManualDistributionParams.__name__,
         Modality.__name__,

data_designer/config/models.py

@@ -5,7 +5,7 @@ import logging
 from abc import ABC, abstractmethod
 from enum import Enum
 from pathlib import Path
-from typing import Any, Generic, Literal, TypeVar
+from typing import Annotated, Any, Generic, Literal, TypeVar
 
 import numpy as np
 from pydantic import BaseModel, Field, field_validator, model_validator
@@ -278,7 +278,7 @@ class ChatCompletionInferenceParams(BaseInferenceParams):
         generation_type: Type of generation, always "chat-completion" for this class.
         temperature: Sampling temperature (0.0-2.0). Can be a fixed value or a distribution for dynamic sampling.
         top_p: Nucleus sampling probability (0.0-1.0). Can be a fixed value or a distribution for dynamic sampling.
-        max_tokens: Maximum number of tokens (includes both input and output tokens).
+        max_tokens: Maximum number of tokens to generate in the response.
     """
 
     generation_type: Literal[GenerationType.CHAT_COMPLETION] = GenerationType.CHAT_COMPLETION
@@ -357,21 +357,6 @@ class ChatCompletionInferenceParams(BaseInferenceParams):
         return super()._format_value(key, value)
 
 
-# Maintain backwards compatibility with a deprecation warning
-class InferenceParameters(ChatCompletionInferenceParams):
-    """
-    Deprecated: Use ChatCompletionInferenceParams instead.
-    This alias will be removed in a future version.
-    """
-
-    def __init__(self, *args: Any, **kwargs: Any) -> None:
-        logger.warning(
-            "InferenceParameters is deprecated and will be removed in a future version. "
-            "Use ChatCompletionInferenceParams instead."
-        )
-        super().__init__(*args, **kwargs)
-
-
 class EmbeddingInferenceParams(BaseInferenceParams):
     """Configuration for embedding generation parameters.
 
@@ -395,7 +380,9 @@ class EmbeddingInferenceParams(BaseInferenceParams):
         return result
 
 
-InferenceParamsT: TypeAlias = ChatCompletionInferenceParams | EmbeddingInferenceParams | InferenceParameters
+InferenceParamsT: TypeAlias = Annotated[
+    ChatCompletionInferenceParams | EmbeddingInferenceParams, Field(discriminator="generation_type")
+]
 
 
 class ModelConfig(ConfigBase):
@@ -441,6 +428,7 @@ class ModelProvider(ConfigBase):
         provider_type: Provider type (default: "openai"). Determines the API format to use.
         api_key: Optional API key for authentication.
         extra_body: Additional parameters to pass in API requests.
+        extra_headers: Additional headers to pass in API requests.
     """
 
     name: str
@@ -448,6 +436,7 @@ class ModelProvider(ConfigBase):
     provider_type: str = "openai"
     api_key: str | None = None
     extra_body: dict[str, Any] | None = None
+    extra_headers: dict[str, str] | None = None
 
 
 def load_model_configs(model_configs: list[ModelConfig] | str | Path) -> list[ModelConfig]:

data_designer/config/run_config.py

@@ -0,0 +1,34 @@
+# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+from pydantic import Field, model_validator
+from typing_extensions import Self
+
+from data_designer.config.base import ConfigBase
+
+
+class RunConfig(ConfigBase):
+    """Runtime configuration for dataset generation.
+
+    Groups configuration options that control generation behavior but aren't
+    part of the dataset configuration itself.
+
+    Attributes:
+        disable_early_shutdown: If True, disables early shutdown entirely. Generation
+            will continue regardless of error rate. Default is False.
+        shutdown_error_rate: Error rate threshold (0.0-1.0) that triggers early shutdown.
+            When early shutdown is disabled, this value is normalized to 1.0. Default is 0.5.
+        shutdown_error_window: Minimum number of completed tasks before error rate
+            monitoring begins. Must be >= 0. Default is 10.
+    """
+
+    disable_early_shutdown: bool = False
+    shutdown_error_rate: float = Field(default=0.5, ge=0.0, le=1.0)
+    shutdown_error_window: int = Field(default=10, ge=0)
+
+    @model_validator(mode="after")
+    def normalize_shutdown_settings(self) -> Self:
+        """Set shutdown_error_rate to 1.0 when early shutdown is disabled."""
+        if self.disable_early_shutdown:
+            self.shutdown_error_rate = 1.0
+        return self

data_designer/config/seed.py

@@ -1,19 +1,13 @@
 # SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
 # SPDX-License-Identifier: Apache-2.0
 
-from abc import ABC
 from enum import Enum
 
-from pydantic import Field, field_validator, model_validator
+from pydantic import Field, model_validator
 from typing_extensions import Self
 
 from data_designer.config.base import ConfigBase
-from data_designer.config.datastore import DatastoreSettings
-from data_designer.config.utils.io_helpers import (
-    VALID_DATASET_FILE_EXTENSIONS,
-    validate_dataset_file_path,
-    validate_path_contains_files_of_type,
-)
+from data_designer.config.seed_source import SeedSourceT
 
 
 class SamplingStrategy(str, Enum):
@@ -62,7 +56,7 @@ class SeedConfig(ConfigBase):
     """Configuration for sampling data from a seed dataset.
 
     Args:
-        dataset: Path or identifier for the seed dataset.
+        source: A SeedSource defining where the seed data exists
         sampling_strategy: Strategy for how to sample rows from the dataset.
             - ORDERED: Read rows sequentially in their original order.
             - SHUFFLE: Randomly shuffle rows before sampling. When used with
@@ -75,70 +69,46 @@ class SeedConfig(ConfigBase):
 
     Examples:
         Read rows sequentially from start to end:
-            SeedConfig(dataset="my_data.parquet", sampling_strategy=SamplingStrategy.ORDERED)
+            SeedConfig(
+                source=LocalFileSeedSource(path="my_data.parquet"),
+                sampling_strategy=SamplingStrategy.ORDERED
+            )
 
         Read rows in random order:
-            SeedConfig(dataset="my_data.parquet", sampling_strategy=SamplingStrategy.SHUFFLE)
+            SeedConfig(
+                source=LocalFileSeedSource(path="my_data.parquet"),
+                sampling_strategy=SamplingStrategy.SHUFFLE
+            )
 
         Read specific index range (rows 100-199):
             SeedConfig(
-                dataset="my_data.parquet",
+                source=LocalFileSeedSource(path="my_data.parquet"),
                 sampling_strategy=SamplingStrategy.ORDERED,
                 selection_strategy=IndexRange(start=100, end=199)
             )
 
         Read random rows from a specific index range (shuffles within rows 100-199):
             SeedConfig(
-                dataset="my_data.parquet",
+                source=LocalFileSeedSource(path="my_data.parquet"),
                 sampling_strategy=SamplingStrategy.SHUFFLE,
                 selection_strategy=IndexRange(start=100, end=199)
             )
 
         Read from partition 2 (3rd partition, zero-based) of 5 partitions (20% of dataset):
             SeedConfig(
-                dataset="my_data.parquet",
+                source=LocalFileSeedSource(path="my_data.parquet"),
                 sampling_strategy=SamplingStrategy.ORDERED,
                 selection_strategy=PartitionBlock(index=2, num_partitions=5)
             )
 
         Read shuffled rows from partition 0 of 10 partitions (shuffles within the partition):
             SeedConfig(
-                dataset="my_data.parquet",
+                source=LocalFileSeedSource(path="my_data.parquet"),
                 sampling_strategy=SamplingStrategy.SHUFFLE,
                 selection_strategy=PartitionBlock(index=0, num_partitions=10)
             )
     """
 
-    dataset: str
+    source: SeedSourceT
     sampling_strategy: SamplingStrategy = SamplingStrategy.ORDERED
     selection_strategy: IndexRange | PartitionBlock | None = None
-
-
-class SeedDatasetReference(ABC, ConfigBase):
-    dataset: str
-
-
-class DatastoreSeedDatasetReference(SeedDatasetReference):
-    datastore_settings: DatastoreSettings
-
-    @property
-    def repo_id(self) -> str:
-        return "/".join(self.dataset.split("/")[:-1])
-
-    @property
-    def filename(self) -> str:
-        return self.dataset.split("/")[-1]
-
-
-class LocalSeedDatasetReference(SeedDatasetReference):
-    @field_validator("dataset", mode="after")
-    def validate_dataset_is_file(cls, v: str) -> str:
-        valid_wild_card_versions = {f"*{ext}" for ext in VALID_DATASET_FILE_EXTENSIONS}
-        if any(v.endswith(wildcard) for wildcard in valid_wild_card_versions):
-            parts = v.split("*.")
-            file_path = parts[0]
-            file_extension = parts[-1]
-            validate_path_contains_files_of_type(file_path, file_extension)
-        else:
-            validate_dataset_file_path(v)
-        return v

data_designer/config/seed_source.py

@@ -0,0 +1,73 @@
+# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+from abc import ABC
+from typing import Annotated, Literal
+
+import pandas as pd
+from pydantic import BaseModel, ConfigDict, Field, field_validator
+from typing_extensions import Self
+
+from data_designer.config.utils.io_helpers import (
+    VALID_DATASET_FILE_EXTENSIONS,
+    validate_dataset_file_path,
+    validate_path_contains_files_of_type,
+)
+
+
+class SeedSource(BaseModel, ABC):
+    """Base class for seed dataset configurations.
+
+    All subclasses must define a `seed_type` field with a Literal value.
+    This serves as a discriminated union discriminator.
+    """
+
+    seed_type: str
+
+
+class LocalFileSeedSource(SeedSource):
+    seed_type: Literal["local"] = "local"
+
+    path: str
+
+    @field_validator("path", mode="after")
+    def validate_path(cls, v: str) -> str:
+        valid_wild_card_versions = {f"*{ext}" for ext in VALID_DATASET_FILE_EXTENSIONS}
+        if any(v.endswith(wildcard) for wildcard in valid_wild_card_versions):
+            parts = v.split("*.")
+            file_path = parts[0]
+            file_extension = parts[-1]
+            validate_path_contains_files_of_type(file_path, file_extension)
+        else:
+            validate_dataset_file_path(v)
+        return v
+
+    @classmethod
+    def from_dataframe(cls, df: pd.DataFrame, path: str) -> Self:
+        df.to_parquet(path, index=False)
+        return cls(path=path)
+
+
+class HuggingFaceSeedSource(SeedSource):
+    seed_type: Literal["hf"] = "hf"
+
+    path: str = Field(
+        ...,
+        description="Path to the seed data in HuggingFace. Wildcards are allowed. Examples include 'datasets/my-username/my-dataset/data/000_00000.parquet', 'datasets/my-username/my-dataset/data/*.parquet', 'datasets/my-username/my-dataset/**/*.parquet'",
+    )
+    token: str | None = None
+    endpoint: str = "https://huggingface.co"
+
+
+class DataFrameSeedSource(SeedSource):
+    seed_type: Literal["df"] = "df"
+
+    model_config = ConfigDict(arbitrary_types_allowed=True)
+
+    df: pd.DataFrame
+
+
+SeedSourceT = Annotated[
+    LocalFileSeedSource | HuggingFaceSeedSource | DataFrameSeedSource,
+    Field(discriminator="seed_type"),
+]