data-designer-engine 0.6.1rc2__tar.gz → 0.7.0__tar.gz

{data_designer_engine-0.6.1rc2 → data_designer_engine-0.7.0}/.gitignore

@@ -38,9 +38,6 @@ htmlcov/
 coverage.xml
 .pytest_cache/
 
-# mkdocs documentation
-/site
-
 # Jupyter Notebook
 .ipynb_checkpoints
 

{data_designer_engine-0.6.1rc2 → data_designer_engine-0.7.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: data-designer-engine
-Version: 0.6.1rc2
+Version: 0.7.0
 Summary: Generation engine for DataDesigner synthetic data generation
 License-Expression: Apache-2.0
 Classifier: Development Status :: 4 - Beta
@@ -17,7 +17,7 @@ Requires-Python: >=3.10
 Requires-Dist: anyascii<1,>=0.3.3
 Requires-Dist: chardet<6,>=3.0.2
 Requires-Dist: cryptography<47,>=46.0.7
-Requires-Dist: data-designer-config==0.6.1rc2
+Requires-Dist: data-designer-config==0.7.0
 Requires-Dist: duckdb<2,>=1.5.0
 Requires-Dist: faker<21,>=20.1.0
 Requires-Dist: fsspec<2026,>=2025.3.0
@@ -32,10 +32,12 @@ Requires-Dist: marko<3,>=2.1.2
 Requires-Dist: mcp<2,>=1.26.0
 Requires-Dist: networkx<4,>=3.0
 Requires-Dist: numpy<3,>=1.23.5
+Requires-Dist: pyjwt[crypto]<3,>=2.13.0
 Requires-Dist: python-multipart<1,>=0.0.29
 Requires-Dist: ruff<1,>=0.14.10
 Requires-Dist: scipy<2,>=1.11.0
 Requires-Dist: sqlfluff<5,>=4.1.0
+Requires-Dist: starlette<2,>=1.2.0
 Requires-Dist: tiktoken<1,>=0.8.0
 Description-Content-Type: text/markdown
 

{data_designer_engine-0.6.1rc2 → data_designer_engine-0.7.0}/pyproject.toml

@@ -51,10 +51,12 @@ dependencies = [
     "mcp>=1.26.0,<2",
     "networkx>=3.0,<4",
     "numpy>=1.23.5,<3",
+    "pyjwt[crypto]>=2.13.0,<3", # 2.13.0 fixes PyJWT security advisories pulled in by mcp
     "python-multipart>=0.0.29,<1", # 0.0.29 addresses multipart security advisories pulled in by mcp
     "ruff>=0.14.10,<1",
     "scipy>=1.11.0,<2",
     "sqlfluff>=4.1.0,<5",
+    "starlette>=1.2.0,<2", # 1.2.0 fixes security advisory pulled in by mcp
     "tiktoken>=0.8.0,<1",
 ]
 

{data_designer_engine-0.6.1rc2 → data_designer_engine-0.7.0}/src/data_designer/engine/analysis/column_profilers/base.py

@@ -12,6 +12,7 @@ from typing_extensions import Self
 
 from data_designer.config.base import ConfigBase, SingleColumnConfig
 from data_designer.config.column_types import DataDesignerColumnType
+from data_designer.config.run_config import JinjaRenderingEngine
 from data_designer.engine.configurable_task import ConfigurableTask, TaskConfigT
 
 logger = logging.getLogger(__name__)
@@ -20,6 +21,7 @@ logger = logging.getLogger(__name__)
 class ColumnConfigWithDataFrame(ConfigBase):
     column_config: SingleColumnConfig
     df: pd.DataFrame
+    jinja_rendering_engine: JinjaRenderingEngine = JinjaRenderingEngine.SECURE
 
     @model_validator(mode="after")
     def validate_column_exists(self) -> Self:

{data_designer_engine-0.6.1rc2 → data_designer_engine-0.7.0}/src/data_designer/engine/analysis/column_statistics.py

@@ -73,7 +73,11 @@ class GeneralColumnStatisticsCalculator(BaseModel):
 
 class LLMTextColumnStatisticsCalculator(GeneralColumnStatisticsCalculator):
     def calculate_token_stats(self) -> dict[str, Any]:
-        return calculate_token_stats(self.column_config, self.df)
+        return calculate_token_stats(
+            self.column_config,
+            self.df,
+            jinja_rendering_engine=self.column_config_with_df.jinja_rendering_engine,
+        )
 
 
 class LLMCodeColumnStatisticsCalculator(LLMTextColumnStatisticsCalculator): ...

{data_designer_engine-0.6.1rc2 → data_designer_engine-0.7.0}/src/data_designer/engine/analysis/dataset_profiler.py

@@ -76,7 +76,7 @@ class DataDesignerDatasetProfiler:
             logger.info(f"{LOG_INDENT}{c.get_column_emoji()} column: '{c.name}'")
             column_statistics.append(
                 get_column_statistics_calculator(c.column_type)(
-                    column_config_with_df=ColumnConfigWithDataFrame(column_config=c, df=dataset)
+                    column_config_with_df=self._create_column_config_with_df(c, dataset)
                 ).calculate()
             )
 
@@ -86,7 +86,7 @@ class DataDesignerDatasetProfiler:
             applicable_column_types = profiler.get_applicable_column_types()
             for c in self.config.column_configs:
                 if c.column_type in applicable_column_types:
-                    params = ColumnConfigWithDataFrame(column_config=c, df=dataset)
+                    params = self._create_column_config_with_df(c, dataset)
                     column_profiles.append(profiler.profile(params))
             if len(column_profiles) == 0:
                 logger.warning(
@@ -128,6 +128,17 @@ class DataDesignerDatasetProfiler:
             config=profiler_config, resource_provider=self.resource_provider
         )
 
+    def _create_column_config_with_df(
+        self,
+        column_config: ColumnConfigT,
+        dataset: pd.DataFrame,
+    ) -> ColumnConfigWithDataFrame:
+        return ColumnConfigWithDataFrame(
+            column_config=column_config,
+            df=dataset,
+            jinja_rendering_engine=self.resource_provider.run_config.jinja_rendering_engine,
+        )
+
     def _validate_column_profiler_configs(self) -> None:
         if self.config.column_profiler_configs:
             if self.resource_provider.model_registry is None:

{data_designer_engine-0.6.1rc2 → data_designer_engine-0.7.0}/src/data_designer/engine/analysis/utils/column_statistics_calculations.py

@@ -17,6 +17,7 @@ from data_designer.config.analysis.column_statistics import (
 from data_designer.config.column_configs import (
     LLMTextColumnConfig,
 )
+from data_designer.config.run_config import JinjaRenderingEngine
 from data_designer.engine.column_generators.utils.prompt_renderer import (
     PromptType,
     RecordBasedPromptRenderer,
@@ -95,12 +96,18 @@ def calculate_general_column_info(column_name: str, df: pd.DataFrame) -> dict[st
 
 
 def calculate_input_token_stats(
-    column_config: LLMTextColumnConfig, df: pd.DataFrame
+    column_config: LLMTextColumnConfig,
+    df: pd.DataFrame,
+    *,
+    jinja_rendering_engine: JinjaRenderingEngine = JinjaRenderingEngine.SECURE,
 ) -> dict[str, float | MissingValue]:
     try:
         num_tokens = []
         num_samples = min(MAX_PROMPT_SAMPLE_SIZE, len(df))
-        renderer = RecordBasedPromptRenderer(response_recipe=create_response_recipe(column_config))
+        renderer = RecordBasedPromptRenderer(
+            response_recipe=create_response_recipe(column_config),
+            jinja_rendering_engine=jinja_rendering_engine,
+        )
         for record in df.sample(num_samples, random_state=RANDOM_SEED).to_dict(orient="records"):
             system_prompt = renderer.render(
                 prompt_template=column_config.system_prompt, record=record, prompt_type=PromptType.SYSTEM_PROMPT
@@ -143,9 +150,14 @@ def calculate_output_token_stats(
         }
 
 
-def calculate_token_stats(column_config: LLMTextColumnConfig, df: pd.DataFrame) -> dict[str, float | MissingValue]:
+def calculate_token_stats(
+    column_config: LLMTextColumnConfig,
+    df: pd.DataFrame,
+    *,
+    jinja_rendering_engine: JinjaRenderingEngine = JinjaRenderingEngine.SECURE,
+) -> dict[str, float | MissingValue]:
     return {
-        **calculate_input_token_stats(column_config, df),
+        **calculate_input_token_stats(column_config, df, jinja_rendering_engine=jinja_rendering_engine),
         **calculate_output_token_stats(column_config, df),
     }
 

{data_designer_engine-0.6.1rc2 → data_designer_engine-0.7.0}/src/data_designer/engine/column_generators/generators/base.py

@@ -363,6 +363,5 @@ class ColumnGeneratorFullColumn(ColumnGenerator[TaskConfigT], ABC):
 
         Returns:
             DataFrame containing the input columns plus the new column and any side-effect
-            columns. When ``config.allow_resize`` is ``False``, the row count must match
-            the input; when it is ``True``, the row count may change.
+            columns. The row count must match the input.
         """

{data_designer_engine-0.6.1rc2 → data_designer_engine-0.7.0}/src/data_designer/engine/column_generators/generators/custom.py

@@ -50,12 +50,12 @@ def _compute_bridge_timeout(
 
 
 class _AsyncBridgedModelFacade:
-    """Proxy that bridges ``model.generate()`` to ``model.agenerate()`` in async engine mode.
+    """Proxy that bridges ``model.generate()`` to ``model.agenerate()``.
 
-    When a sync custom column runs inside ``asyncio.to_thread`` under the async engine,
-    the sync HTTP client is unavailable. This proxy intercepts the resulting
-    ``SyncClientUnavailableError`` and schedules ``agenerate()`` on the engine's persistent
-    event loop via ``run_coroutine_threadsafe``.
+    When a synchronous custom column runs inside ``asyncio.to_thread``, the sync
+    HTTP client is unavailable. This proxy intercepts the resulting
+    ``SyncClientUnavailableError`` and schedules ``agenerate()`` on the engine's
+    persistent event loop via ``run_coroutine_threadsafe``.
 
     All other attributes are forwarded to the underlying facade unchanged.
     """
@@ -82,7 +82,7 @@ class _AsyncBridgedModelFacade:
             pass  # No running loop - safe to bridge
         else:
             raise RuntimeError(
-                "model.generate() is not available in async engine mode from the event loop. "
+                "model.generate() is not available from the event loop. "
                 "Use 'await model.agenerate()' in async custom columns."
             )
 
@@ -153,13 +153,10 @@ class CustomColumnGenerator(ColumnGenerator[CustomColumnConfig]):
 
     def get_generation_strategy(self) -> GenerationStrategy:
         """Return strategy based on config."""
-        return self.config.generation_strategy
+        return GenerationStrategy(self.config.generation_strategy)
 
-    def generate(self, data: dict | pd.DataFrame) -> dict | pd.DataFrame | list[dict]:
-        """Generate column value(s) for a row (dict) or batch (DataFrame).
-
-        For cell_by_cell with allow_resize=True, may return dict or list[dict] (0, 1, or N rows).
-        """
+    def generate(self, data: dict | pd.DataFrame) -> dict | pd.DataFrame:
+        """Generate column value(s) for a row (dict) or batch (DataFrame)."""
         is_full_column = self.config.generation_strategy == GenerationStrategy.FULL_COLUMN
         is_dataframe = not isinstance(data, dict)
 
@@ -177,7 +174,7 @@ class CustomColumnGenerator(ColumnGenerator[CustomColumnConfig]):
 
         return self._generate(data, is_dataframe)
 
-    async def agenerate(self, data: dict | pd.DataFrame) -> dict | pd.DataFrame | list[dict]:
+    async def agenerate(self, data: dict | pd.DataFrame) -> dict | pd.DataFrame:
         """Async generate — branches on strategy and detects coroutine functions."""
         is_full_column = self.config.generation_strategy == GenerationStrategy.FULL_COLUMN
         if is_full_column:
@@ -229,7 +226,7 @@ class CustomColumnGenerator(ColumnGenerator[CustomColumnConfig]):
             models = self._build_models_dict()
             return await fn(data, self.config.generator_params, models)
 
-    def _generate(self, data: dict | pd.DataFrame, is_dataframe: bool) -> dict | pd.DataFrame | list[dict]:
+    def _generate(self, data: dict | pd.DataFrame, is_dataframe: bool) -> dict | pd.DataFrame:
         """Unified generation logic for both strategies."""
         get_keys = (lambda d: set(d.columns)) if is_dataframe else (lambda d: set(d.keys()))
 
@@ -265,28 +262,11 @@ class CustomColumnGenerator(ColumnGenerator[CustomColumnConfig]):
 
     def _postprocess_result(
         self,
-        result: dict | pd.DataFrame | list[dict],
+        result: dict | pd.DataFrame,
         is_dataframe: bool,
         keys_before: set[str],
-    ) -> dict | pd.DataFrame | list[dict]:
+    ) -> dict | pd.DataFrame:
         """Validate type and output columns of a generation result."""
-        # Cell-by-cell with allow_resize: accept dict or list[dict]
-        if not is_dataframe and self.config.allow_resize:
-            if isinstance(result, dict):
-                return self._validate_output(result, keys_before, is_dataframe)
-            if isinstance(result, list):
-                if not all(isinstance(r, dict) for r in result):
-                    raise CustomColumnGenerationError(
-                        f"Custom generator for column '{self.config.name}' with allow_resize must return "
-                        "dict or list[dict]; list elements must be dicts."
-                    )
-                return [self._validate_cell_output(r, keys_before) for r in result]
-            raise CustomColumnGenerationError(
-                f"Custom generator for column '{self.config.name}' with allow_resize must return "
-                f"dict or list[dict], got {type(result).__name__}"
-            )
-
-        # Validate return type for non-resize paths
         expected_type = lazy.pd.DataFrame if is_dataframe else dict
         type_name = "DataFrame" if is_dataframe else "dict"
         if not isinstance(result, expected_type):
@@ -425,5 +405,3 @@ class CustomColumnGenerator(ColumnGenerator[CustomColumnConfig]):
             logger.info(f"{LOG_INDENT}model_aliases: {self.config.model_aliases}")
         if self.config.generator_params:
             logger.info(f"{LOG_INDENT}generator_params: {self.config.generator_params}")
-        if self.config.allow_resize:
-            logger.info(f"{LOG_INDENT}allow_resize: {self.config.allow_resize}")

data_designer_engine-0.7.0/src/data_designer/engine/column_generators/generators/expression.py

@@ -0,0 +1,130 @@
+# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+from __future__ import annotations
+
+import logging
+from collections import Counter
+from typing import TYPE_CHECKING
+
+import data_designer.lazy_heavy_imports as lazy
+from data_designer.config.column_configs import ExpressionColumnConfig
+from data_designer.engine.column_generators.generators.base import ColumnGeneratorFullColumn
+from data_designer.engine.column_generators.utils.errors import ExpressionTemplateRenderError
+from data_designer.engine.context import format_row_group_tag
+from data_designer.engine.processing.ginja.environment import WithJinja2UserTemplateRendering
+from data_designer.engine.processing.ginja.exceptions import EmptyTemplateRenderError, UserTemplateError
+from data_designer.engine.processing.utils import deserialize_json_values
+
+if TYPE_CHECKING:
+    import pandas as pd
+
+logger = logging.getLogger(__name__)
+
+EMPTY_RENDERED_EXPRESSION = "EmptyRenderedExpression"
+TEMPLATE_RENDER_ERROR = "TemplateRenderError"
+TYPE_CAST_ERROR = "TypeCastError"
+_VALID_DTYPES = {"str", "float", "int", "bool"}
+
+
+class ExpressionColumnGenerator(WithJinja2UserTemplateRendering, ColumnGeneratorFullColumn[ExpressionColumnConfig]):
+    def generate(self, data: pd.DataFrame) -> pd.DataFrame:
+        logger.info(f"🧩 {format_row_group_tag()}Generating column `{self.config.name}` from expression")
+
+        missing_columns = list(set(self.config.required_columns) - set(data.columns))
+        if len(missing_columns) > 0:
+            error_msg = (
+                f"There was an error preparing the Jinja2 expression template. "
+                f"The following columns {missing_columns} are missing!"
+            )
+            raise ExpressionTemplateRenderError(error_msg)
+
+        if self.config.dtype not in _VALID_DTYPES:
+            raise ValueError(f"Invalid dtype: {self.config.dtype}")
+
+        self.prepare_jinja2_template_renderer(self.config.expr, data.columns.to_list())
+        records: list[dict] = []
+        retained_indexes: list[object] = []
+        drop_counts: Counter[str] = Counter()
+
+        # ``DataFrame.to_dict(orient="records")`` returns ``[]`` for a 0-column
+        # DataFrame regardless of its row count, so a root expression dispatched
+        # against a fresh row group buffer (no upstream columns) would otherwise
+        # skip rendering entirely. Synthesize empty per-row dicts in that case so
+        # the loop still fires once per row and the template can render its
+        # constant value (or drop the row uniformly).
+        row_indexes = data.index.to_list()
+        row_records = data.to_dict(orient="records")
+        if not row_records and row_indexes:
+            row_records = [{} for _ in row_indexes]
+
+        for row_index, record in zip(row_indexes, row_records, strict=True):
+            prepared_record = deserialize_json_values(record)
+            try:
+                rendered_value = self.render_template(prepared_record)
+            except EmptyTemplateRenderError:
+                drop_counts[EMPTY_RENDERED_EXPRESSION] += 1
+                continue
+            except Exception:
+                logger.debug(
+                    "Expression column %r dropped row %r after template render failure.",
+                    self.config.name,
+                    row_index,
+                    exc_info=True,
+                )
+                drop_counts[TEMPLATE_RENDER_ERROR] += 1
+                continue
+
+            if self._is_empty_rendered_expression(rendered_value):
+                drop_counts[EMPTY_RENDERED_EXPRESSION] += 1
+                continue
+
+            try:
+                record[self.config.name] = self._cast_type(rendered_value)
+            except (OverflowError, TypeError, ValueError):
+                drop_counts[TYPE_CAST_ERROR] += 1
+                continue
+
+            records.append(record)
+            retained_indexes.append(row_index)
+
+        total_dropped = sum(drop_counts.values())
+        if total_dropped > 0:
+            self._log_row_drops(drop_counts, input_count=len(data), retained_count=len(records))
+            if len(records) == 0:
+                raise UserTemplateError(f"Expression column {self.config.name!r} produced no valid rows.")
+
+        return lazy.pd.DataFrame(records, index=retained_indexes)
+
+    @staticmethod
+    def _is_empty_rendered_expression(value: object) -> bool:
+        if value is None:
+            return True
+        return isinstance(value, str) and len(value.strip()) == 0
+
+    def _log_row_drops(self, drop_counts: Counter[str], *, input_count: int, retained_count: int) -> None:
+        breakdown = ", ".join(f"{name}={count}" for name, count in sorted(drop_counts.items()))
+        total_dropped = sum(drop_counts.values())
+        message = (
+            f"Expression column {self.config.name!r} dropped {total_dropped}/{input_count} rows after render: "
+            f"{breakdown}."
+        )
+        if retained_count == 0:
+            logger.error(message)
+        else:
+            logger.warning(f"{message} Continuing with {retained_count} rows.")
+
+    def _cast_type(self, value: str) -> str | float | int | bool:
+        if self.config.dtype == "str":
+            return value
+        elif self.config.dtype == "float":
+            return float(value)
+        elif self.config.dtype == "int":
+            return int(float(value))
+        elif self.config.dtype == "bool":
+            try:
+                return bool(int(float(value)))
+            except ValueError:
+                return bool(f"{value}".lower() == "true")
+        else:
+            raise ValueError(f"Invalid dtype: {self.config.dtype}")

{data_designer_engine-0.6.1rc2 → data_designer_engine-0.7.0}/src/data_designer/engine/context.py

@@ -5,8 +5,7 @@ from __future__ import annotations
 
 from contextvars import ContextVar
 
-# Set per row group by both engines: the async scheduler sets it before each
-# task executes, and the sync engine's ``_run_batch`` sets it for each batch.
+# Set per row group by the async scheduler before each task executes.
 # Value: (current_rg_index, total_rg_count) or None.
 current_row_group: ContextVar[tuple[int, int] | None] = ContextVar("current_row_group", default=None)