data-designer 0.1.0__py3-none-any.whl → 0.1.1__py3-none-any.whl

data_designer/_version.py

@@ -28,7 +28,7 @@ version_tuple: VERSION_TUPLE
 commit_id: COMMIT_ID
 __commit_id__: COMMIT_ID
 
-__version__ = version = '0.1.0'
-__version_tuple__ = version_tuple = (0, 1, 0)
+__version__ = version = '0.1.1'
+__version_tuple__ = version_tuple = (0, 1, 1)
 
 __commit_id__ = commit_id = None

data_designer/config/datastore.py

@@ -124,7 +124,7 @@ def _fetch_seed_dataset_column_names_from_datastore(
         raise InvalidFileFormatError(f"🛑 Unsupported file type: {filename!r}")
 
     datastore_settings = resolve_datastore_settings(datastore_settings)
-    fs = HfFileSystem(endpoint=datastore_settings.endpoint, token=datastore_settings.token)
+    fs = HfFileSystem(endpoint=datastore_settings.endpoint, token=datastore_settings.token, skip_instance_cache=True)
 
     with fs.open(f"datasets/{repo_id}/{filename}") as f:
         return get_file_column_names(f, file_type)

data_designer/config/default_model_settings.py

@@ -4,6 +4,7 @@
 
 from functools import lru_cache
 import logging
+import os
 from pathlib import Path
 from typing import Any, Literal, Optional
 
@@ -15,7 +16,6 @@ from .utils.constants import (
     PREDEFINED_PROVIDERS,
     PREDEFINED_PROVIDERS_MODEL_MAP,
 )
-from .utils.info import ConfigBuilderInfo, InfoType, InterfaceInfo
 from .utils.io_helpers import load_config_file, save_config_file
 
 logger = logging.getLogger(__name__)
@@ -75,7 +75,15 @@ def get_default_model_configs() -> list[ModelConfig]:
         config_dict = load_config_file(MODEL_CONFIGS_FILE_PATH)
         if "model_configs" in config_dict:
             return [ModelConfig.model_validate(mc) for mc in config_dict["model_configs"]]
-    raise FileNotFoundError(f"Default model configs file not found at {str(MODEL_CONFIGS_FILE_PATH)!r}")
+    return []
+
+
+def get_defaul_model_providers_missing_api_keys() -> list[str]:
+    missing_api_keys = []
+    for predefined_provider in PREDEFINED_PROVIDERS:
+        if os.environ.get(predefined_provider["api_key"]) is None:
+            missing_api_keys.append(predefined_provider["api_key"])
+    return missing_api_keys
 
 
 def get_default_providers() -> list[ModelProvider]:
@@ -91,21 +99,17 @@ def get_default_provider_name() -> Optional[str]:
 
 def resolve_seed_default_model_settings() -> None:
     if not MODEL_CONFIGS_FILE_PATH.exists():
-        logger.info(
+        logger.debug(
             f"🍾 Default model configs were not found, so writing the following to {str(MODEL_CONFIGS_FILE_PATH)!r}"
         )
-        config_builder_info = ConfigBuilderInfo(model_configs=get_builtin_model_configs())
-        config_builder_info.display(info_type=InfoType.MODEL_CONFIGS)
         save_config_file(
             MODEL_CONFIGS_FILE_PATH, {"model_configs": [mc.model_dump() for mc in get_builtin_model_configs()]}
         )
 
     if not MODEL_PROVIDERS_FILE_PATH.exists():
-        logger.info(
+        logger.debug(
             f"🪄  Default model providers were not found, so writing the following to {str(MODEL_PROVIDERS_FILE_PATH)!r}"
         )
-        interface_info = InterfaceInfo(model_providers=get_builtin_model_providers())
-        interface_info.display(info_type=InfoType.MODEL_PROVIDERS)
         save_config_file(
             MODEL_PROVIDERS_FILE_PATH, {"providers": [p.model_dump() for p in get_builtin_model_providers()]}
         )

data_designer/engine/resources/seed_dataset_data_store.py

@@ -42,11 +42,29 @@ class HfHubSeedDatasetDataStore(SeedDatasetDataStore):
 
     def __init__(self, endpoint: str, token: str | None):
         self.hfapi = HfApi(endpoint=endpoint, token=token)
-        self.hffs = HfFileSystem(endpoint=endpoint, token=token)
+        self.endpoint = endpoint
+        self.token = token
 
     def create_duckdb_connection(self) -> duckdb.DuckDBPyConnection:
+        """Create a DuckDB connection with a fresh HfFileSystem registered.
+
+        Creates a new HfFileSystem instance for each connection to ensure file metadata
+        is fetched fresh from the datastore, avoiding cache-related issues when reading
+        recently updated parquet files.
+
+        Returns:
+            A DuckDB connection with the HfFileSystem registered for hf:// URI support.
+        """
+        # Use skip_instance_cache to avoid fsspec-level caching
+        hffs = HfFileSystem(endpoint=self.endpoint, token=self.token, skip_instance_cache=True)
+
+        # Clear all internal caches to avoid stale metadata issues
+        # HfFileSystem caches file metadata (size, etc.) which can become stale when files are re-uploaded
+        if hasattr(hffs, "dircache"):
+            hffs.dircache.clear()
+
         conn = duckdb.connect()
-        conn.register_filesystem(self.hffs)
+        conn.register_filesystem(hffs)
         return conn
 
     def get_dataset_uri(self, file_id: str) -> str:

data_designer/interface/data_designer.py

@@ -9,6 +9,7 @@ import pandas as pd
 from data_designer.config.analysis.dataset_profiler import DatasetProfilerResults
 from data_designer.config.config_builder import DataDesignerConfigBuilder
 from data_designer.config.default_model_settings import (
+    get_defaul_model_providers_missing_api_keys,
     get_default_model_configs,
     get_default_provider_name,
     get_default_providers,
@@ -26,8 +27,9 @@ from data_designer.config.utils.constants import (
     MANAGED_ASSETS_PATH,
     MODEL_CONFIGS_FILE_PATH,
     MODEL_PROVIDERS_FILE_PATH,
+    PREDEFINED_PROVIDERS,
 )
-from data_designer.config.utils.info import InterfaceInfo
+from data_designer.config.utils.info import InfoType, InterfaceInfo
 from data_designer.config.utils.io_helpers import write_seed_dataset
 from data_designer.config.utils.misc import can_run_data_designer_locally
 from data_designer.engine.analysis.dataset_profiler import (
@@ -103,7 +105,7 @@ class DataDesigner(DataDesignerInterface[DatasetCreationResults]):
         self._artifact_path = Path(artifact_path) if artifact_path is not None else Path.cwd() / "artifacts"
         self._buffer_size = DEFAULT_BUFFER_SIZE
         self._managed_assets_path = Path(managed_assets_path or MANAGED_ASSETS_PATH)
-        self._model_providers = model_providers or self.get_default_model_providers()
+        self._model_providers = self._resolve_model_providers(model_providers)
         self._model_provider_registry = resolve_model_provider_registry(
             self._model_providers, get_default_provider_name()
         )
@@ -151,7 +153,7 @@ class DataDesigner(DataDesignerInterface[DatasetCreationResults]):
         Returns:
             InterfaceInfo object with information about the Data Designer interface.
         """
-        return InterfaceInfo(model_providers=self._model_providers)
+        return self._get_interface_info(self._model_providers)
 
     def create(
         self,
@@ -307,6 +309,22 @@ class DataDesigner(DataDesignerInterface[DatasetCreationResults]):
             raise InvalidBufferValueError("Buffer size must be greater than 0.")
         self._buffer_size = buffer_size
 
+    def _resolve_model_providers(self, model_providers: list[ModelProvider] | None) -> list[ModelProvider]:
+        if model_providers is None:
+            if can_run_data_designer_locally():
+                model_providers = get_default_providers()
+                missing_api_keys = get_defaul_model_providers_missing_api_keys()
+                if len(missing_api_keys) == len(PREDEFINED_PROVIDERS):
+                    logger.warning(
+                        "🚨 You are trying to use a default model provider but your API keys are missing."
+                        "\n\t\t\tSet the API key for the default providers you intend to use and re-initialize the Data Designer object."
+                        "\n\t\t\tAlternatively, you can provide your own model providers during Data Designer object initialization."
+                        "\n\t\t\tSee https://nvidia-nemo.github.io/DataDesigner/models/model-providers/ for more information."
+                    )
+                    self._get_interface_info(model_providers).display(InfoType.MODEL_PROVIDERS)
+                return model_providers
+        return model_providers or []
+
     def _create_dataset_builder(
         self, config_builder: DataDesignerConfigBuilder, resource_provider: ResourceProvider
     ) -> ColumnWiseDatasetBuilder:
@@ -349,3 +367,6 @@ class DataDesigner(DataDesignerInterface[DatasetCreationResults]):
                 )
             ),
         )
+
+    def _get_interface_info(self, model_providers: list[ModelProvider]) -> InterfaceInfo:
+        return InterfaceInfo(model_providers=model_providers)

{data_designer-0.1.0.dist-info → data_designer-0.1.1.dist-info}/METADATA

@@ -1,18 +1,18 @@
 Metadata-Version: 2.4
 Name: data-designer
-Version: 0.1.0
+Version: 0.1.1
 Summary: General framework for synthetic data generation
+License-Expression: Apache-2.0
 License-File: LICENSE
 Classifier: Development Status :: 4 - Beta
 Classifier: Intended Audience :: Developers
 Classifier: Intended Audience :: Science/Research
-Classifier: License :: Other/Proprietary License
+Classifier: License :: OSI Approved :: Apache Software License
 Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
 Classifier: Programming Language :: Python :: 3.12
 Classifier: Programming Language :: Python :: 3.13
 Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
-Classifier: Topic :: Scientific/Engineering :: Human Machine Interfaces
 Classifier: Topic :: Software Development
 Requires-Python: >=3.10
 Requires-Dist: anyascii<1.0,>=0.3.3
@@ -51,7 +51,7 @@ Description-Content-Type: text/markdown
 
 [![CI](https://github.com/NVIDIA-NeMo/DataDesigner/actions/workflows/ci.yml/badge.svg)](https://github.com/NVIDIA-NeMo/DataDesigner/actions/workflows/ci.yml)
 [![License](https://img.shields.io/badge/License-Apache_2.0-blue.svg)](https://opensource.org/licenses/Apache-2.0)
-[![Python 3.10+](https://img.shields.io/badge/python-3.10+-blue.svg)](https://www.python.org/downloads/) [![NeMo Microservices](https://img.shields.io/badge/NeMo-Microservices-76b900)](https://docs.nvidia.com/nemo/microservices/latest/index.html)
+[![Python 3.10 - 3.13](https://img.shields.io/badge/🐍_Python-3.10_|_3.11_|_3.12_|_3.13-blue.svg)](https://www.python.org/downloads/) [![NeMo Microservices](https://img.shields.io/badge/NeMo-Microservices-76b900)](https://docs.nvidia.com/nemo/microservices/latest/index.html) [![Code](https://img.shields.io/badge/Code-Documentation-8A2BE2.svg)](https://nvidia-nemo.github.io/DataDesigner/)
 
 **Generate high-quality synthetic datasets from scratch or using your own seed data.**
 
@@ -147,10 +147,12 @@ preview.display_sample_record()
 
 ### 📚 Learn more
 
-- **[Quick Start Guide](https://nvidia-nemo.github.io/DataDesigner)** – Detailed walkthrough with more examples
-- **[Tutorial Notebooks](https://nvidia-nemo.github.io/DataDesigner/notebooks/1-the-basics/)** – Step-by-step interactive tutorials
+- **[Quick Start Guide](https://nvidia-nemo.github.io/DataDesigner/quick-start/)** – Detailed walkthrough with more examples
+- **[Tutorial Notebooks](https://nvidia-nemo.github.io/DataDesigner/notebooks/intro/)** – Step-by-step interactive tutorials
 - **[Column Types](https://nvidia-nemo.github.io/DataDesigner/concepts/columns/)** – Explore samplers, LLM columns, validators, and more
+- **[Validators](https://nvidia-nemo.github.io/DataDesigner/concepts/validators/)** – Learn how to validate generated data with Python, SQL, and remote validators
 - **[Model Configuration](https://nvidia-nemo.github.io/DataDesigner/models/model-configs/)** – Configure custom models and providers
+- **[Person Sampling](https://nvidia-nemo.github.io/DataDesigner/concepts/person_sampling/)** – Learn how to sample realistic person data with demographic attributes
 
 ### 🔧 Configure models via CLI
 
@@ -162,12 +164,27 @@ data-designer config list      # View current settings
 
 ### 🤝 Get involved
 
-- **[Contributing Guide](https://nvidia-nemo.github.io/DataDesigner/CONTRIBUTING.md)** – Help improve Data Designer
-- **[GitHub Issues](https://github.com/NVIDIA-NeMo/DataDesigner/issues)** – Report bugs or request features
-- **[GitHub Discussions](https://github.com/NVIDIA-NeMo/DataDesigner/discussions)** – Ask questions and share ideas
+- **[Contributing Guide](https://nvidia-nemo.github.io/DataDesigner/CONTRIBUTING)** – Help improve Data Designer
+- **[GitHub Issues](https://github.com/NVIDIA-NeMo/DataDesigner/issues)** – Report bugs or make a feature request
 
 ---
 
 ## License
 
 Apache License 2.0 – see [LICENSE](LICENSE) for details.
+
+---
+
+## Citation
+
+If you use NeMo Data Designer in your research, please cite it using the following BibTeX entry:
+
+```bibtex
+@misc{nemo-data-designer,
+  author = {The NeMo Data Designer Team},
+  title = {NeMo Data Designer: A framework for generating synthetic data from scratch or based on your own seed data},
+  howpublished = {\url{https://github.com/NVIDIA-NeMo/DataDesigner}},
+  year = {2025},
+  note = {GitHub Repository},
+}
+```

{data_designer-0.1.0.dist-info → data_designer-0.1.1.dist-info}/RECORD

@@ -1,5 +1,5 @@
 data_designer/__init__.py,sha256=iCeqRnb640RrL2QpA630GY5Ng7JiDt83Vq0DwLnNugU,461
-data_designer/_version.py,sha256=5jwwVncvCiTnhOedfkzzxmxsggwmTBORdFL_4wq0ZeY,704
+data_designer/_version.py,sha256=m8HxkqoKGw_wAJtc4ZokpJKNLXqp4zwnNhbnfDtro7w,704
 data_designer/errors.py,sha256=Z4eN9XwzZvGRdBluSNoSqQYkPPzNQIDf0ET_OqWRZh8,179
 data_designer/logging.py,sha256=O6LlQRj4IdkvEEYiMkKfMb_ZDgN1YpkGQUCqcp7nY6w,5354
 data_designer/plugin_manager.py,sha256=jWoo80x0oCiOIJMA43t-vK-_hVv9_xt4WhBcurYoDqw,3098
@@ -36,8 +36,8 @@ data_designer/config/column_types.py,sha256=V0Ijwb-asYOX-GQyG9W-X_A-FIbFSajKuus5
 data_designer/config/config_builder.py,sha256=NlAe6cwN6IAE90A8uPLsOdABmmYyUt6UnGYZwgmf_xE,27288
 data_designer/config/data_designer_config.py,sha256=cvIXMVQzYn9vC4GINPz972pDBmt-HrV5dvw1568LVmE,1719
 data_designer/config/dataset_builders.py,sha256=1pNFy_pkQ5lJ6AVZ43AeTuSbz6yC_l7Ndcyp5yaT8hQ,327
-data_designer/config/datastore.py,sha256=oPC4jeupalPexhe8K2BkMSlPvDaOZWAyoDuaq9m-Uo4,6272
-data_designer/config/default_model_settings.py,sha256=TMnxGQNAE7ipTmPF1R0qJBEUX199FWdTnjNiy5oR1Bo,4668
+data_designer/config/datastore.py,sha256=okuwUz-M5bSThvp_a9erKRoG4ej0bey1HUQBA7hgL98,6298
+data_designer/config/default_model_settings.py,sha256=b_oWsD350rb43009kcRxuPNgCZegB_noohURR4n1ZR0,4516
 data_designer/config/errors.py,sha256=XneHH6tKHG2sZ71HzmPr7k3UBZ_psnSANknT30n-aa8,449
 data_designer/config/interface.py,sha256=2_tHvxtKAv0C5L7K4ztm-Xa1A-u9Njlwo2drdPa2qmk,1499
 data_designer/config/models.py,sha256=5Cy55BnKYyr-I1UHLUTqZxe6Ca9uVQWpUiwt9X0ZlrU,7521
@@ -133,7 +133,7 @@ data_designer/engine/resources/managed_dataset_generator.py,sha256=KXrWdgod-NFaC
 data_designer/engine/resources/managed_dataset_repository.py,sha256=lqVxuoCxc07QTrhnAR1mgDiHFkzjjkx2IwcrxrdbloY,7547
 data_designer/engine/resources/managed_storage.py,sha256=jRnGeCTGlu6FxC6tOCssPiSpbHEf0mbqFfm3mM0utdA,2079
 data_designer/engine/resources/resource_provider.py,sha256=CbB2D538ECGkvyHF1V63_TDn-wStCoklV7bF0y4mabY,1859
-data_designer/engine/resources/seed_dataset_data_store.py,sha256=uD8g_7dmVvGmOIG5NMnkMok_0zSdEHVQ1kQcfFqWIG4,2226
+data_designer/engine/resources/seed_dataset_data_store.py,sha256=dM2HgfyUgbF7MidN8dn5S-LAR0GVPJfjqXpDPTP2XoA,3035
 data_designer/engine/sampling_gen/column.py,sha256=gDIPth7vK2797rGtLhf_kVGMAC-khefKHodeeDoqV-I,3946
 data_designer/engine/sampling_gen/constraints.py,sha256=RyhRF9KeUOwEiHr_TN3QwLWOVLTpuCFpCI_3Qr-9Whs,3028
 data_designer/engine/sampling_gen/errors.py,sha256=UBZBtosD07EisCdeo8r-Uq4h0QL3tYS1qwtEmca8_jM,828
@@ -163,15 +163,15 @@ data_designer/engine/validators/remote.py,sha256=jtDIvWzfHh17m2ac_Fp93p49Th8RlkB
 data_designer/engine/validators/sql.py,sha256=bxbyxPxDT9yuwjhABVEY40iR1pzWRFi65WU4tPgG2bE,2250
 data_designer/essentials/__init__.py,sha256=zrDZ7hahOmOhCPdfoj0z9ALN10lXIesfwd2qXRqTcdY,4125
 data_designer/interface/__init__.py,sha256=9eG4WHKyrJcNoK4GEz6BCw_E0Ewo9elQoDN4TLMbAog,137
-data_designer/interface/data_designer.py,sha256=yh_lqEvL0LoqXX-KYDflVjVp8yGFkhSUe_yzZxtV__Q,14904
+data_designer/interface/data_designer.py,sha256=MSzT9OFd3V6saZID0vfQxx0oB6Fth8GmEFcnmFNXOVo,16271
 data_designer/interface/errors.py,sha256=jagKT3tPUnYq4e3e6AkTnBkcayHyEfxjPMBzx-GEKe4,565
 data_designer/interface/results.py,sha256=qFxa8SuCXeADiRpaCMBwJcExkJBCfUPeGCdcJSTjoTc,2111
 data_designer/plugins/__init__.py,sha256=c_V7q4QhfVoNf_uc9UwmXCsWqwtyWogI7YoN_0PzzE4,234
 data_designer/plugins/errors.py,sha256=yPIHpSddEr-o9ZcNVibb2hI-73O15Kg_Od8SlmQlnRs,297
 data_designer/plugins/plugin.py,sha256=7ErdUyrTdOb5PCBE3msdhTOrvQpldjOQw90-Bu4Bosc,2522
 data_designer/plugins/registry.py,sha256=iPDTh4duV1cKt7H1fXkj1bKLG6SyUKmzQ9xh-vjEoaM,3018
-data_designer-0.1.0.dist-info/METADATA,sha256=pW_EXcja79dhuYz8nL5RuenZqpBSEnS8r85TY6B87dc,5918
-data_designer-0.1.0.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
-data_designer-0.1.0.dist-info/entry_points.txt,sha256=NWWWidyDxN6CYX6y664PhBYMhbaYTQTyprqfYAgkyCg,57
-data_designer-0.1.0.dist-info/licenses/LICENSE,sha256=cSWJDwVqHyQgly8Zmt3pqXJ2eQbZVYwN9qd0NMssxXY,11336
-data_designer-0.1.0.dist-info/RECORD,,
+data_designer-0.1.1.dist-info/METADATA,sha256=MeR9kVPEkyXH8I-qiYdZpTiZ1yM2FWWi3PiPYsGsX9c,6698
+data_designer-0.1.1.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+data_designer-0.1.1.dist-info/entry_points.txt,sha256=NWWWidyDxN6CYX6y664PhBYMhbaYTQTyprqfYAgkyCg,57
+data_designer-0.1.1.dist-info/licenses/LICENSE,sha256=cSWJDwVqHyQgly8Zmt3pqXJ2eQbZVYwN9qd0NMssxXY,11336
+data_designer-0.1.1.dist-info/RECORD,,