data-designer 0.3.8rc1__py3-none-any.whl → 0.3.8rc2__py3-none-any.whl

data_designer/_version.py

@@ -28,7 +28,7 @@ version_tuple: VERSION_TUPLE
 commit_id: COMMIT_ID
 __commit_id__: COMMIT_ID
 
-__version__ = version = '0.3.8rc1'
-__version_tuple__ = version_tuple = (0, 3, 8, 'rc1')
+__version__ = version = '0.3.8rc2'
+__version_tuple__ = version_tuple = (0, 3, 8, 'rc2')
 
 __commit_id__ = commit_id = None

data_designer/config/default_model_settings.py

@@ -71,12 +71,20 @@ def get_default_model_configs() -> list[ModelConfig]:
     return []
 
 
-def get_default_model_providers_missing_api_keys() -> list[str]:
-    missing_api_keys = []
-    for predefined_provider in PREDEFINED_PROVIDERS:
-        if os.environ.get(predefined_provider["api_key"]) is None:
-            missing_api_keys.append(predefined_provider["api_key"])
-    return missing_api_keys
+def get_providers_with_missing_api_keys(providers: list[ModelProvider]) -> list[ModelProvider]:
+    providers_with_missing_keys = []
+
+    for provider in providers:
+        if provider.api_key is None:
+            # No API key specified at all
+            providers_with_missing_keys.append(provider)
+        elif provider.api_key.isupper() and "_" in provider.api_key:
+            # Looks like an environment variable name, check if it's set
+            if os.environ.get(provider.api_key) is None:
+                providers_with_missing_keys.append(provider)
+        # else: It's an actual API key value (not an env var), so it's valid
+
+    return providers_with_missing_keys
 
 
 def get_default_providers() -> list[ModelProvider]:

data_designer/config/run_config.py

@@ -26,6 +26,8 @@ class RunConfig(ConfigBase):
         buffer_size: Number of records to process in each batch during dataset generation.
             A batch is processed end-to-end (column generation, post-batch processors, and writing the batch
             to artifact storage) before moving on to the next batch. Must be > 0. Default is 1000.
+        non_inference_max_parallel_workers: Maximum number of worker threads used for non-inference
+            cell-by-cell generators. Must be >= 1. Default is 4.
         max_conversation_restarts: Maximum number of full conversation restarts permitted when
             generation tasks call `ModelFacade.generate(...)`. Must be >= 0. Default is 5.
         max_conversation_correction_steps: Maximum number of correction rounds permitted within a
@@ -37,6 +39,7 @@ class RunConfig(ConfigBase):
     shutdown_error_rate: float = Field(default=0.5, ge=0.0, le=1.0)
     shutdown_error_window: int = Field(default=10, ge=0)
     buffer_size: int = Field(default=1000, gt=0)
+    non_inference_max_parallel_workers: int = Field(default=4, ge=1)
     max_conversation_restarts: int = Field(default=5, ge=0)
     max_conversation_correction_steps: int = Field(default=0, ge=0)
 

data_designer/engine/dataset_builders/column_wise_builder.py

@@ -31,10 +31,7 @@ from data_designer.engine.compiler import compile_data_designer_config
 from data_designer.engine.dataset_builders.artifact_storage import SDG_CONFIG_FILENAME, ArtifactStorage
 from data_designer.engine.dataset_builders.errors import DatasetGenerationError, DatasetProcessingError
 from data_designer.engine.dataset_builders.multi_column_configs import MultiColumnConfig
-from data_designer.engine.dataset_builders.utils.concurrency import (
-    MAX_CONCURRENCY_PER_NON_LLM_GENERATOR,
-    ConcurrentThreadExecutor,
-)
+from data_designer.engine.dataset_builders.utils.concurrency import ConcurrentThreadExecutor
 from data_designer.engine.dataset_builders.utils.config_compiler import compile_dataset_builder_column_configs
 from data_designer.engine.dataset_builders.utils.dataset_batch_manager import DatasetBatchManager
 from data_designer.engine.models.telemetry import InferenceEvent, NemoSourceEnum, TaskStatusEnum, TelemetryHandler
@@ -202,7 +199,7 @@ class ColumnWiseDatasetBuilder:
         self.batch_manager.add_records(df.to_dict(orient="records"))
 
     def _run_cell_by_cell_generator(self, generator: ColumnGenerator) -> None:
-        max_workers = MAX_CONCURRENCY_PER_NON_LLM_GENERATOR
+        max_workers = self._resource_provider.run_config.non_inference_max_parallel_workers
         if isinstance(generator, ColumnGeneratorWithModel):
             max_workers = generator.inference_parameters.max_parallel_requests
         self._fan_out_with_threads(generator, max_workers=max_workers)

data_designer/engine/dataset_builders/utils/concurrency.py

@@ -16,9 +16,6 @@ from data_designer.engine.errors import DataDesignerRuntimeError, ErrorTrap
 
 logger = logging.getLogger(__name__)
 
-# Constants
-MAX_CONCURRENCY_PER_NON_LLM_GENERATOR = 4
-
 
 class ExecutorResults(BaseModel):
     failure_threshold: float = 0.0  # Error rate threshold

data_designer/interface/data_designer.py

@@ -12,9 +12,9 @@ from data_designer.config.config_builder import DataDesignerConfigBuilder
 from data_designer.config.data_designer_config import DataDesignerConfig
 from data_designer.config.default_model_settings import (
     get_default_model_configs,
-    get_default_model_providers_missing_api_keys,
     get_default_provider_name,
     get_default_providers,
+    get_providers_with_missing_api_keys,
 )
 from data_designer.config.interface import DataDesignerInterface
 from data_designer.config.models import (
@@ -28,7 +28,6 @@ from data_designer.config.utils.constants import (
     MANAGED_ASSETS_PATH,
     MODEL_CONFIGS_FILE_PATH,
     MODEL_PROVIDERS_FILE_PATH,
-    PREDEFINED_PROVIDERS,
 )
 from data_designer.config.utils.info import InfoType, InterfaceInfo
 from data_designer.engine.analysis.dataset_profiler import DataDesignerDatasetProfiler, DatasetProfilerConfig
@@ -317,7 +316,8 @@ class DataDesigner(DataDesignerInterface[DatasetCreationResults]):
 
         Args:
             run_config: A RunConfig instance containing runtime settings such as
-                early shutdown behavior and batch sizing via `buffer_size`. Import RunConfig from
+                early shutdown behavior, batch sizing via `buffer_size`, and non-inference worker
+                concurrency via `non_inference_max_parallel_workers`. Import RunConfig from
                 data_designer.essentials.
 
         Example:
@@ -334,8 +334,11 @@ class DataDesigner(DataDesignerInterface[DatasetCreationResults]):
     def _resolve_model_providers(self, model_providers: list[ModelProvider] | None) -> list[ModelProvider]:
         if model_providers is None:
             model_providers = get_default_providers()
-            missing_api_keys = get_default_model_providers_missing_api_keys()
-            if len(missing_api_keys) == len(PREDEFINED_PROVIDERS):
+            # Check which providers have missing API keys (from YAML file or env vars)
+            providers_with_missing_keys = get_providers_with_missing_api_keys(model_providers)
+
+            if len(providers_with_missing_keys) == len(model_providers):
+                # All providers have missing API keys
                 logger.warning(
                     "🚨 You are trying to use a default model provider but your API keys are missing."
                     "\n\t\t\tSet the API key for the default providers you intend to use and re-initialize the Data Designer object."

{data_designer-0.3.8rc1.dist-info → data_designer-0.3.8rc2.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: data-designer
-Version: 0.3.8rc1
+Version: 0.3.8rc2
 Summary: General framework for synthetic data generation
 License-Expression: Apache-2.0
 License-File: LICENSE

{data_designer-0.3.8rc1.dist-info → data_designer-0.3.8rc2.dist-info}/RECORD

@@ -1,5 +1,5 @@
 data_designer/__init__.py,sha256=iLr6FpW41-DFbGexuXCJ6gN1xBMNUZ2jfj9XxySmQhk,502
-data_designer/_version.py,sha256=NQlgQnitduzFb4wWkqWxIRTi5E5gw_xAEogbRRBMSzA,714
+data_designer/_version.py,sha256=wP4Vt8OKUu98a4RThXFwdZlfYV_E8tf8D24DFaCFjqg,714
 data_designer/errors.py,sha256=r1pBvmvRBAsPmb7oF_veubhkxZ2uPo9cGEDwykLziX4,220
 data_designer/lazy_heavy_imports.py,sha256=wULSEPQRUOZXvOnb0tdf6wNbRBpaaczYfAjY-pstCBM,1512
 data_designer/logging.py,sha256=gRi9BOqm95UC1-u4pn6n-G4EySy9HhwKVyKLRO4aqm4,5382
@@ -42,14 +42,14 @@ data_designer/config/config_builder.py,sha256=vuPibkodbJxbCXdaI1tt1Uyo1SVCnAOfLB
 data_designer/config/data_designer_config.py,sha256=qOojviug05vHR2S4800sjd4OmxhSVi6kB8SAFXLlPog,1891
 data_designer/config/dataset_builders.py,sha256=jdCujJYFlKAiSkPNX2Qeyrs683GrRcCDv_m8ZZhtg64,368
 data_designer/config/dataset_metadata.py,sha256=UTlEgnHWgjwPuc7bP95T7gaKmcr7pIhFMy9vvbUwMV4,647
-data_designer/config/default_model_settings.py,sha256=d9ZuTDGMtS1rZpIDqoSQjCiD5tcHrUOr22X0-mGQspc,4497
+data_designer/config/default_model_settings.py,sha256=c-llH2otfG0tMCMsxoz3ZcS1nFxIQQPfRedFXAydDbc,4868
 data_designer/config/errors.py,sha256=JhvUYecfLmP0gZjQzqA3OmfaSs9TRlC5E-ubnV_-3gs,560
 data_designer/config/exports.py,sha256=lNwteK4djETKXrMKh5PPeHeZvPAZ5RpnJt2otpoaUz0,4756
 data_designer/config/interface.py,sha256=ikmpm_KwencTpM-yg0auo7XMgcmMSa67S75IqdpFLfk,1676
 data_designer/config/models.py,sha256=OekrXEVnI9WdHzEVk-8fO0NtxLZtjKVtCL03RY8qwYs,15457
 data_designer/config/preview_results.py,sha256=WnPlDcHElIHNfjV_P-nLu_Dpul8D3Eyb5qyi3E173Gs,1744
 data_designer/config/processors.py,sha256=lnyUZA1EhO9NWjjVFFioYxSgeYpoAaM1J7UzwOYkvms,6028
-data_designer/config/run_config.py,sha256=5TA1PSmZ3Ca5V0GA1KTds3xrEwGPFZY9C35Vf_1cAs0,2429
+data_designer/config/run_config.py,sha256=oJ163DpHXu9PzST5Hn9px-bIP9DYjIkCO7UGB93J7bI,2663
 data_designer/config/sampler_constraints.py,sha256=tQI1XLF5bS4TnyKMLo0nArvefnXI8dWCzov38r4qNCQ,1197
 data_designer/config/sampler_params.py,sha256=Gio-53vjSYOdPhF2CEq4HSWCXCaZMy4WpGPbuFVcWOM,27965
 data_designer/config/seed.py,sha256=eShSqOcSUzfCEZBnqY-rB0qZpRGxjeOE3fSaJAwacec,4668
@@ -101,11 +101,11 @@ data_designer/engine/column_generators/utils/generator_classification.py,sha256=
 data_designer/engine/column_generators/utils/judge_score_factory.py,sha256=gESiqMrQzbbcFpZas0sAAAkrH2DL0Z4Nq5ywBO-pQ6k,2141
 data_designer/engine/column_generators/utils/prompt_renderer.py,sha256=LATVAlDYwL7HyM7Nogd6n9XTTk-j9s64o4z0LpKHMhQ,4819
 data_designer/engine/dataset_builders/artifact_storage.py,sha256=CKpTBtJTde7OQvsFZQa1v1autVz5yUxlBHkIKeATFnE,10999
-data_designer/engine/dataset_builders/column_wise_builder.py,sha256=_3_JJJG-tA9qLhNiEKbHxl1EHYBbMVAGUtaAdqO_wsc,15736
+data_designer/engine/dataset_builders/column_wise_builder.py,sha256=lzCSk3dFmdZvKLPAVIRNp9oJQsiilthHRW7mB4dUUB4,15716
 data_designer/engine/dataset_builders/errors.py,sha256=gLXtPcGSMBG10PzQ85dOXskdA0mKbBQrHa_VtP9sbVY,400
 data_designer/engine/dataset_builders/multi_column_configs.py,sha256=U4Pg0ETCBq5phRhb2zt8IFa4fRx-aTMakomKOBnrs0U,1660
 data_designer/engine/dataset_builders/utils/__init__.py,sha256=XLO09Ei8g0lU7hYlzKCvhvQhLFBe5CBwE4v2PqK9xWY,142
-data_designer/engine/dataset_builders/utils/concurrency.py,sha256=Q0ro9UY-3-FFzfi3MZ29nMTSiDZgg1Um6y_HQFztDhk,8338
+data_designer/engine/dataset_builders/utils/concurrency.py,sha256=Lga_xd8i3ZAPqJlKCB4GHG7uxWxws1m-UGAz9UeqU_8,8283
 data_designer/engine/dataset_builders/utils/config_compiler.py,sha256=NGI6U0vgG88d5YKj7oW_SIJ4-_fhA6VFhPbjqGRHea4,2441
 data_designer/engine/dataset_builders/utils/dag.py,sha256=RIEI75OtiphkuDl1vfI_MQC1xMiiIg29s-0C_fNZkWQ,2613
 data_designer/engine/dataset_builders/utils/dataset_batch_manager.py,sha256=IfWd_HcfEzIPhgFp2dJaxNIKRlrPsHqYATFXauvCfaw,8133
@@ -179,7 +179,7 @@ data_designer/engine/validators/remote.py,sha256=rythhIrH2GvqncMQeF3FiJa9Om0KZWe
 data_designer/engine/validators/sql.py,sha256=AMaEdA-gj9j0zwVp809x3ycKltd51wVEhI8mMYGyxd4,2408
 data_designer/essentials/__init__.py,sha256=dIGYH9s0_VQJ1lG8S-ElZiISz59LHo9v7Y5upizcA1M,1135
 data_designer/interface/__init__.py,sha256=XLO09Ei8g0lU7hYlzKCvhvQhLFBe5CBwE4v2PqK9xWY,142
-data_designer/interface/data_designer.py,sha256=aX3Etg1qGpjivQQdplQ0Oi3aB7_6jzClk_4yrpcE5fY,17322
+data_designer/interface/data_designer.py,sha256=nqsl2Et1wSF3TQzYx3gC1drJzrKlm4FtRE7QJYilbys,17544
 data_designer/interface/errors.py,sha256=Ft9GMeIrOHJv_PC_1rU6hWcNyq1GHdsFYZSc9HnUrxU,606
 data_designer/interface/results.py,sha256=3fGwlhif4ufqUGh-EgsGccrob4S6a7WZ6BgFiszTo_A,3871
 data_designer/plugins/__init__.py,sha256=qe1alcTEtnMSMdzknjb57vvjqKgFE5cEHXxBj8tPWMI,275
@@ -189,8 +189,8 @@ data_designer/plugins/registry.py,sha256=Cnt33Q25o9bS2v2YDbV3QPM57VNrtIBKAb4ERQR
 data_designer/plugins/testing/__init__.py,sha256=yyxrrH_i3q0Xb56QO9Ma35WtHlQ5PJF1b2pQoKa16xU,296
 data_designer/plugins/testing/stubs.py,sha256=9tUF209ayZR6f0Q1LsRDW4kEOTgPoIxV8jlq4QoWuW0,3498
 data_designer/plugins/testing/utils.py,sha256=a9LEgK827cnIzHEkgXOdgywrKDLBE36cyttrpG1ctT4,973
-data_designer-0.3.8rc1.dist-info/METADATA,sha256=YrRgO4uKxDznFADcGj2TqIucRNvbYIGk5_4R9Pqq2Qc,8122
-data_designer-0.3.8rc1.dist-info/WHEEL,sha256=WLgqFyCfm_KASv4WHyYy0P3pM_m7J5L9k2skdKLirC8,87
-data_designer-0.3.8rc1.dist-info/entry_points.txt,sha256=NWWWidyDxN6CYX6y664PhBYMhbaYTQTyprqfYAgkyCg,57
-data_designer-0.3.8rc1.dist-info/licenses/LICENSE,sha256=cSWJDwVqHyQgly8Zmt3pqXJ2eQbZVYwN9qd0NMssxXY,11336
-data_designer-0.3.8rc1.dist-info/RECORD,,
+data_designer-0.3.8rc2.dist-info/METADATA,sha256=-gMHOKpWpc0HRy8QfQb4KQkEqeYJaHbxlL_l9M1PKmk,8122
+data_designer-0.3.8rc2.dist-info/WHEEL,sha256=WLgqFyCfm_KASv4WHyYy0P3pM_m7J5L9k2skdKLirC8,87
+data_designer-0.3.8rc2.dist-info/entry_points.txt,sha256=NWWWidyDxN6CYX6y664PhBYMhbaYTQTyprqfYAgkyCg,57
+data_designer-0.3.8rc2.dist-info/licenses/LICENSE,sha256=cSWJDwVqHyQgly8Zmt3pqXJ2eQbZVYwN9qd0NMssxXY,11336
+data_designer-0.3.8rc2.dist-info/RECORD,,