data-designer 0.3.8__py3-none-any.whl → 0.3.8rc1__py3-none-any.whl

data_designer/_version.py

@@ -28,7 +28,7 @@ version_tuple: VERSION_TUPLE
 commit_id: COMMIT_ID
 __commit_id__: COMMIT_ID
 
-__version__ = version = '0.3.8'
-__version_tuple__ = version_tuple = (0, 3, 8)
+__version__ = version = '0.3.8rc1'
+__version_tuple__ = version_tuple = (0, 3, 8, 'rc1')
 
 __commit_id__ = commit_id = None

data_designer/config/default_model_settings.py

@@ -71,20 +71,12 @@ def get_default_model_configs() -> list[ModelConfig]:
     return []
 
 
-def get_providers_with_missing_api_keys(providers: list[ModelProvider]) -> list[ModelProvider]:
-    providers_with_missing_keys = []
-
-    for provider in providers:
-        if provider.api_key is None:
-            # No API key specified at all
-            providers_with_missing_keys.append(provider)
-        elif provider.api_key.isupper() and "_" in provider.api_key:
-            # Looks like an environment variable name, check if it's set
-            if os.environ.get(provider.api_key) is None:
-                providers_with_missing_keys.append(provider)
-        # else: It's an actual API key value (not an env var), so it's valid
-
-    return providers_with_missing_keys
+def get_default_model_providers_missing_api_keys() -> list[str]:
+    missing_api_keys = []
+    for predefined_provider in PREDEFINED_PROVIDERS:
+        if os.environ.get(predefined_provider["api_key"]) is None:
+            missing_api_keys.append(predefined_provider["api_key"])
+    return missing_api_keys
 
 
 def get_default_providers() -> list[ModelProvider]:

data_designer/config/run_config.py

@@ -26,8 +26,6 @@ class RunConfig(ConfigBase):
         buffer_size: Number of records to process in each batch during dataset generation.
             A batch is processed end-to-end (column generation, post-batch processors, and writing the batch
             to artifact storage) before moving on to the next batch. Must be > 0. Default is 1000.
-        non_inference_max_parallel_workers: Maximum number of worker threads used for non-inference
-            cell-by-cell generators. Must be >= 1. Default is 4.
         max_conversation_restarts: Maximum number of full conversation restarts permitted when
             generation tasks call `ModelFacade.generate(...)`. Must be >= 0. Default is 5.
         max_conversation_correction_steps: Maximum number of correction rounds permitted within a
@@ -39,7 +37,6 @@ class RunConfig(ConfigBase):
     shutdown_error_rate: float = Field(default=0.5, ge=0.0, le=1.0)
     shutdown_error_window: int = Field(default=10, ge=0)
     buffer_size: int = Field(default=1000, gt=0)
-    non_inference_max_parallel_workers: int = Field(default=4, ge=1)
     max_conversation_restarts: int = Field(default=5, ge=0)
     max_conversation_correction_steps: int = Field(default=0, ge=0)
 

data_designer/config/utils/constants.py

@@ -353,11 +353,9 @@ PREDEFINED_PROVIDERS_MODEL_MAP = {
 NEMOTRON_PERSONAS_DATASET_SIZES = {
     "en_US": "1.24 GB",
     "en_IN": "2.39 GB",
-    "en_SG": "0.30 GB",
     "hi_Deva_IN": "4.14 GB",
     "hi_Latn_IN": "2.7 GB",
     "ja_JP": "1.69 GB",
-    "pt_BR": "2.33 GB",
 }
 
 LOCALES_WITH_MANAGED_DATASETS = list[str](NEMOTRON_PERSONAS_DATASET_SIZES.keys())

data_designer/engine/dataset_builders/column_wise_builder.py

@@ -31,7 +31,10 @@ from data_designer.engine.compiler import compile_data_designer_config
 from data_designer.engine.dataset_builders.artifact_storage import SDG_CONFIG_FILENAME, ArtifactStorage
 from data_designer.engine.dataset_builders.errors import DatasetGenerationError, DatasetProcessingError
 from data_designer.engine.dataset_builders.multi_column_configs import MultiColumnConfig
-from data_designer.engine.dataset_builders.utils.concurrency import ConcurrentThreadExecutor
+from data_designer.engine.dataset_builders.utils.concurrency import (
+    MAX_CONCURRENCY_PER_NON_LLM_GENERATOR,
+    ConcurrentThreadExecutor,
+)
 from data_designer.engine.dataset_builders.utils.config_compiler import compile_dataset_builder_column_configs
 from data_designer.engine.dataset_builders.utils.dataset_batch_manager import DatasetBatchManager
 from data_designer.engine.models.telemetry import InferenceEvent, NemoSourceEnum, TaskStatusEnum, TelemetryHandler
@@ -199,7 +202,7 @@ class ColumnWiseDatasetBuilder:
         self.batch_manager.add_records(df.to_dict(orient="records"))
 
     def _run_cell_by_cell_generator(self, generator: ColumnGenerator) -> None:
-        max_workers = self._resource_provider.run_config.non_inference_max_parallel_workers
+        max_workers = MAX_CONCURRENCY_PER_NON_LLM_GENERATOR
         if isinstance(generator, ColumnGeneratorWithModel):
             max_workers = generator.inference_parameters.max_parallel_requests
         self._fan_out_with_threads(generator, max_workers=max_workers)

data_designer/engine/dataset_builders/utils/concurrency.py

@@ -16,6 +16,9 @@ from data_designer.engine.errors import DataDesignerRuntimeError, ErrorTrap
 
 logger = logging.getLogger(__name__)
 
+# Constants
+MAX_CONCURRENCY_PER_NON_LLM_GENERATOR = 4
+
 
 class ExecutorResults(BaseModel):
     failure_threshold: float = 0.0  # Error rate threshold

data_designer/engine/sampling_gen/entities/dataset_based_person_fields.py

@@ -40,16 +40,13 @@ PII_FIELDS = [
     "state",
     "email_address",
     "phone_number",
-    # Brazil-specific fields
-    "race",
     # Japan-specific fields
     "area",
     "prefecture",
     "zone",
-    # Brazil and India shared fields
-    "religion",
     # India-specific fields
     "district",
+    "religion",
     "education_degree",
     "first_language",
     "second_language",
@@ -81,10 +78,9 @@ PERSONA_FIELDS = [
     # Japan-specific persona fields
     "aspects",
     "digital_skills",
-    # Brazil and India shared persona fields
-    "religious_persona",
-    "religious_background",
     # India-specific persona fields
     "linguistic_persona",
+    "religious_persona",
     "linguistic_background",
+    "religious_background",
 ]

data_designer/interface/data_designer.py

@@ -12,9 +12,9 @@ from data_designer.config.config_builder import DataDesignerConfigBuilder
 from data_designer.config.data_designer_config import DataDesignerConfig
 from data_designer.config.default_model_settings import (
     get_default_model_configs,
+    get_default_model_providers_missing_api_keys,
     get_default_provider_name,
     get_default_providers,
-    get_providers_with_missing_api_keys,
 )
 from data_designer.config.interface import DataDesignerInterface
 from data_designer.config.models import (
@@ -28,6 +28,7 @@ from data_designer.config.utils.constants import (
     MANAGED_ASSETS_PATH,
     MODEL_CONFIGS_FILE_PATH,
     MODEL_PROVIDERS_FILE_PATH,
+    PREDEFINED_PROVIDERS,
 )
 from data_designer.config.utils.info import InfoType, InterfaceInfo
 from data_designer.engine.analysis.dataset_profiler import DataDesignerDatasetProfiler, DatasetProfilerConfig
@@ -316,8 +317,7 @@ class DataDesigner(DataDesignerInterface[DatasetCreationResults]):
 
         Args:
             run_config: A RunConfig instance containing runtime settings such as
-                early shutdown behavior, batch sizing via `buffer_size`, and non-inference worker
-                concurrency via `non_inference_max_parallel_workers`. Import RunConfig from
+                early shutdown behavior and batch sizing via `buffer_size`. Import RunConfig from
                 data_designer.essentials.
 
         Example:
@@ -334,11 +334,8 @@ class DataDesigner(DataDesignerInterface[DatasetCreationResults]):
     def _resolve_model_providers(self, model_providers: list[ModelProvider] | None) -> list[ModelProvider]:
         if model_providers is None:
             model_providers = get_default_providers()
-            # Check which providers have missing API keys (from YAML file or env vars)
-            providers_with_missing_keys = get_providers_with_missing_api_keys(model_providers)
-
-            if len(providers_with_missing_keys) == len(model_providers):
-                # All providers have missing API keys
+            missing_api_keys = get_default_model_providers_missing_api_keys()
+            if len(missing_api_keys) == len(PREDEFINED_PROVIDERS):
                 logger.warning(
                     "🚨 You are trying to use a default model provider but your API keys are missing."
                     "\n\t\t\tSet the API key for the default providers you intend to use and re-initialize the Data Designer object."

{data_designer-0.3.8.dist-info → data_designer-0.3.8rc1.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: data-designer
-Version: 0.3.8
+Version: 0.3.8rc1
 Summary: General framework for synthetic data generation
 License-Expression: Apache-2.0
 License-File: LICENSE

{data_designer-0.3.8.dist-info → data_designer-0.3.8rc1.dist-info}/RECORD

@@ -1,5 +1,5 @@
 data_designer/__init__.py,sha256=iLr6FpW41-DFbGexuXCJ6gN1xBMNUZ2jfj9XxySmQhk,502
-data_designer/_version.py,sha256=iWEYFk8Ve-UDYmpiH5KfwfwhQTYtkxZlaGPwKE2xDy0,704
+data_designer/_version.py,sha256=NQlgQnitduzFb4wWkqWxIRTi5E5gw_xAEogbRRBMSzA,714
 data_designer/errors.py,sha256=r1pBvmvRBAsPmb7oF_veubhkxZ2uPo9cGEDwykLziX4,220
 data_designer/lazy_heavy_imports.py,sha256=wULSEPQRUOZXvOnb0tdf6wNbRBpaaczYfAjY-pstCBM,1512
 data_designer/logging.py,sha256=gRi9BOqm95UC1-u4pn6n-G4EySy9HhwKVyKLRO4aqm4,5382
@@ -42,14 +42,14 @@ data_designer/config/config_builder.py,sha256=vuPibkodbJxbCXdaI1tt1Uyo1SVCnAOfLB
 data_designer/config/data_designer_config.py,sha256=qOojviug05vHR2S4800sjd4OmxhSVi6kB8SAFXLlPog,1891
 data_designer/config/dataset_builders.py,sha256=jdCujJYFlKAiSkPNX2Qeyrs683GrRcCDv_m8ZZhtg64,368
 data_designer/config/dataset_metadata.py,sha256=UTlEgnHWgjwPuc7bP95T7gaKmcr7pIhFMy9vvbUwMV4,647
-data_designer/config/default_model_settings.py,sha256=c-llH2otfG0tMCMsxoz3ZcS1nFxIQQPfRedFXAydDbc,4868
+data_designer/config/default_model_settings.py,sha256=d9ZuTDGMtS1rZpIDqoSQjCiD5tcHrUOr22X0-mGQspc,4497
 data_designer/config/errors.py,sha256=JhvUYecfLmP0gZjQzqA3OmfaSs9TRlC5E-ubnV_-3gs,560
 data_designer/config/exports.py,sha256=lNwteK4djETKXrMKh5PPeHeZvPAZ5RpnJt2otpoaUz0,4756
 data_designer/config/interface.py,sha256=ikmpm_KwencTpM-yg0auo7XMgcmMSa67S75IqdpFLfk,1676
 data_designer/config/models.py,sha256=OekrXEVnI9WdHzEVk-8fO0NtxLZtjKVtCL03RY8qwYs,15457
 data_designer/config/preview_results.py,sha256=WnPlDcHElIHNfjV_P-nLu_Dpul8D3Eyb5qyi3E173Gs,1744
 data_designer/config/processors.py,sha256=lnyUZA1EhO9NWjjVFFioYxSgeYpoAaM1J7UzwOYkvms,6028
-data_designer/config/run_config.py,sha256=oJ163DpHXu9PzST5Hn9px-bIP9DYjIkCO7UGB93J7bI,2663
+data_designer/config/run_config.py,sha256=5TA1PSmZ3Ca5V0GA1KTds3xrEwGPFZY9C35Vf_1cAs0,2429
 data_designer/config/sampler_constraints.py,sha256=tQI1XLF5bS4TnyKMLo0nArvefnXI8dWCzov38r4qNCQ,1197
 data_designer/config/sampler_params.py,sha256=Gio-53vjSYOdPhF2CEq4HSWCXCaZMy4WpGPbuFVcWOM,27965
 data_designer/config/seed.py,sha256=eShSqOcSUzfCEZBnqY-rB0qZpRGxjeOE3fSaJAwacec,4668
@@ -63,7 +63,7 @@ data_designer/config/analysis/dataset_profiler.py,sha256=-5eX55IXivwUBMg2pI-d_3e
 data_designer/config/analysis/utils/errors.py,sha256=pvmdQ_YuIlWW4NFw-cX_rOoQf-GG8y_FiQzNctB__DQ,331
 data_designer/config/analysis/utils/reporting.py,sha256=teTzd1OHtpI4vbIinGOGsKXyNldO3F5eqbNdAztF0_s,7066
 data_designer/config/utils/code_lang.py,sha256=EqMJh1GL5ysUZIoyqx_6vmqenUKHm4J-RQtKXiA4EPg,2354
-data_designer/config/utils/constants.py,sha256=eqDQ57b8B0v5qRSO0He45LEjSxtfxlsPtHRvBu1xkw0,8973
+data_designer/config/utils/constants.py,sha256=KU4ZCIe18gXdBp2N_BgZlRW90FIqjFPYmJtqgVY3Ink,8925
 data_designer/config/utils/errors.py,sha256=HCjer0YrF0bMn5j8gmgWaLb0395LAr_hxMD1ftOsOc8,520
 data_designer/config/utils/info.py,sha256=yOa4U8kI_CY4OfCKZxCm2okU8klAiThvyjKM5tG-F0A,3469
 data_designer/config/utils/io_helpers.py,sha256=kzvOR7QgqijkqU-O2enIlpCWwHvzc3oRaEl4Lsjh1Do,8466
@@ -101,11 +101,11 @@ data_designer/engine/column_generators/utils/generator_classification.py,sha256=
 data_designer/engine/column_generators/utils/judge_score_factory.py,sha256=gESiqMrQzbbcFpZas0sAAAkrH2DL0Z4Nq5ywBO-pQ6k,2141
 data_designer/engine/column_generators/utils/prompt_renderer.py,sha256=LATVAlDYwL7HyM7Nogd6n9XTTk-j9s64o4z0LpKHMhQ,4819
 data_designer/engine/dataset_builders/artifact_storage.py,sha256=CKpTBtJTde7OQvsFZQa1v1autVz5yUxlBHkIKeATFnE,10999
-data_designer/engine/dataset_builders/column_wise_builder.py,sha256=lzCSk3dFmdZvKLPAVIRNp9oJQsiilthHRW7mB4dUUB4,15716
+data_designer/engine/dataset_builders/column_wise_builder.py,sha256=_3_JJJG-tA9qLhNiEKbHxl1EHYBbMVAGUtaAdqO_wsc,15736
 data_designer/engine/dataset_builders/errors.py,sha256=gLXtPcGSMBG10PzQ85dOXskdA0mKbBQrHa_VtP9sbVY,400
 data_designer/engine/dataset_builders/multi_column_configs.py,sha256=U4Pg0ETCBq5phRhb2zt8IFa4fRx-aTMakomKOBnrs0U,1660
 data_designer/engine/dataset_builders/utils/__init__.py,sha256=XLO09Ei8g0lU7hYlzKCvhvQhLFBe5CBwE4v2PqK9xWY,142
-data_designer/engine/dataset_builders/utils/concurrency.py,sha256=Lga_xd8i3ZAPqJlKCB4GHG7uxWxws1m-UGAz9UeqU_8,8283
+data_designer/engine/dataset_builders/utils/concurrency.py,sha256=Q0ro9UY-3-FFzfi3MZ29nMTSiDZgg1Um6y_HQFztDhk,8338
 data_designer/engine/dataset_builders/utils/config_compiler.py,sha256=NGI6U0vgG88d5YKj7oW_SIJ4-_fhA6VFhPbjqGRHea4,2441
 data_designer/engine/dataset_builders/utils/dag.py,sha256=RIEI75OtiphkuDl1vfI_MQC1xMiiIg29s-0C_fNZkWQ,2613
 data_designer/engine/dataset_builders/utils/dataset_batch_manager.py,sha256=IfWd_HcfEzIPhgFp2dJaxNIKRlrPsHqYATFXauvCfaw,8133
@@ -164,7 +164,7 @@ data_designer/engine/sampling_gen/data_sources/base.py,sha256=zUG5XTplD5pgHh4ytC
 data_designer/engine/sampling_gen/data_sources/errors.py,sha256=_9rbwUpaz0Pd2Ods4AVDQ7Uq4JvPyfHhTp51BdtJDto,367
 data_designer/engine/sampling_gen/data_sources/sources.py,sha256=53KVPp7REjNKA0rajGmT_tBkxwQqwrcIKhcijBGcfcs,13647
 data_designer/engine/sampling_gen/entities/__init__.py,sha256=XLO09Ei8g0lU7hYlzKCvhvQhLFBe5CBwE4v2PqK9xWY,142
-data_designer/engine/sampling_gen/entities/dataset_based_person_fields.py,sha256=r8qXWe8EquJognihPNGzma4fFuSQAAZHlkYVsGmcX2w,2006
+data_designer/engine/sampling_gen/entities/dataset_based_person_fields.py,sha256=0_eUTtrWFGxTfTfqlz9ig9bJEtYeckb50J7w5LhYTr8,1883
 data_designer/engine/sampling_gen/entities/email_address_utils.py,sha256=THfD7muq5tMHkRWOATN-N3iSFgkKjT4e8hKquDFMTlU,5272
 data_designer/engine/sampling_gen/entities/errors.py,sha256=SbtwwG6JgoY4k6pq2-y-lD60nX_pqjf5QftmwgXt0us,352
 data_designer/engine/sampling_gen/entities/national_id_utils.py,sha256=XUFB6RhfLGFQUNyy0B6BSgtrG9NdEnIjfSALBwJplho,2652
@@ -179,7 +179,7 @@ data_designer/engine/validators/remote.py,sha256=rythhIrH2GvqncMQeF3FiJa9Om0KZWe
 data_designer/engine/validators/sql.py,sha256=AMaEdA-gj9j0zwVp809x3ycKltd51wVEhI8mMYGyxd4,2408
 data_designer/essentials/__init__.py,sha256=dIGYH9s0_VQJ1lG8S-ElZiISz59LHo9v7Y5upizcA1M,1135
 data_designer/interface/__init__.py,sha256=XLO09Ei8g0lU7hYlzKCvhvQhLFBe5CBwE4v2PqK9xWY,142
-data_designer/interface/data_designer.py,sha256=nqsl2Et1wSF3TQzYx3gC1drJzrKlm4FtRE7QJYilbys,17544
+data_designer/interface/data_designer.py,sha256=aX3Etg1qGpjivQQdplQ0Oi3aB7_6jzClk_4yrpcE5fY,17322
 data_designer/interface/errors.py,sha256=Ft9GMeIrOHJv_PC_1rU6hWcNyq1GHdsFYZSc9HnUrxU,606
 data_designer/interface/results.py,sha256=3fGwlhif4ufqUGh-EgsGccrob4S6a7WZ6BgFiszTo_A,3871
 data_designer/plugins/__init__.py,sha256=qe1alcTEtnMSMdzknjb57vvjqKgFE5cEHXxBj8tPWMI,275
@@ -189,8 +189,8 @@ data_designer/plugins/registry.py,sha256=Cnt33Q25o9bS2v2YDbV3QPM57VNrtIBKAb4ERQR
 data_designer/plugins/testing/__init__.py,sha256=yyxrrH_i3q0Xb56QO9Ma35WtHlQ5PJF1b2pQoKa16xU,296
 data_designer/plugins/testing/stubs.py,sha256=9tUF209ayZR6f0Q1LsRDW4kEOTgPoIxV8jlq4QoWuW0,3498
 data_designer/plugins/testing/utils.py,sha256=a9LEgK827cnIzHEkgXOdgywrKDLBE36cyttrpG1ctT4,973
-data_designer-0.3.8.dist-info/METADATA,sha256=n9jXs34c2_rOL-Tme5Y6xrb4fMo0-GNNEVHlCl5WfLY,8119
-data_designer-0.3.8.dist-info/WHEEL,sha256=WLgqFyCfm_KASv4WHyYy0P3pM_m7J5L9k2skdKLirC8,87
-data_designer-0.3.8.dist-info/entry_points.txt,sha256=NWWWidyDxN6CYX6y664PhBYMhbaYTQTyprqfYAgkyCg,57
-data_designer-0.3.8.dist-info/licenses/LICENSE,sha256=cSWJDwVqHyQgly8Zmt3pqXJ2eQbZVYwN9qd0NMssxXY,11336
-data_designer-0.3.8.dist-info/RECORD,,
+data_designer-0.3.8rc1.dist-info/METADATA,sha256=YrRgO4uKxDznFADcGj2TqIucRNvbYIGk5_4R9Pqq2Qc,8122
+data_designer-0.3.8rc1.dist-info/WHEEL,sha256=WLgqFyCfm_KASv4WHyYy0P3pM_m7J5L9k2skdKLirC8,87
+data_designer-0.3.8rc1.dist-info/entry_points.txt,sha256=NWWWidyDxN6CYX6y664PhBYMhbaYTQTyprqfYAgkyCg,57
+data_designer-0.3.8rc1.dist-info/licenses/LICENSE,sha256=cSWJDwVqHyQgly8Zmt3pqXJ2eQbZVYwN9qd0NMssxXY,11336
+data_designer-0.3.8rc1.dist-info/RECORD,,