data-designer 0.3.8rc2__py3-none-any.whl → 0.4.0rc1__py3-none-any.whl

data_designer/engine/processing/gsonschema/validators.py

@@ -1,202 +0,0 @@
-# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
-# SPDX-License-Identifier: Apache-2.0
-
-from __future__ import annotations
-
-import logging
-import re
-from copy import deepcopy
-from decimal import ROUND_HALF_UP, Decimal
-from typing import TYPE_CHECKING, Any, overload
-
-from data_designer.engine.processing.gsonschema.exceptions import JSONSchemaValidationError
-from data_designer.engine.processing.gsonschema.schema_transformers import forbid_additional_properties
-from data_designer.engine.processing.gsonschema.types import DataObjectT, JSONSchemaT, T_primitive
-from data_designer.lazy_heavy_imports import jsonschema
-
-if TYPE_CHECKING:
-    import jsonschema
-
-DEFAULT_JSONSCHEMA_VALIDATOR = jsonschema.Draft202012Validator
-
-logger = logging.getLogger(__name__)
-
-
-def prune_additional_properties(
-    _, allow_additional_properties: bool, instance: DataObjectT, schema: JSONSchemaT
-) -> None:
-    """A JSONSchemaValidtor extension function to prune additional properties.
-
-    Operates on an individual schema in-place.
-
-    Args:
-        allow_additional_properties (bool): The value of the `additionalProperties`
-            field for this schema.
-        instance (DataObjectT): The data object being validated.
-        schema (JSONSchemaT): The schema for this object.
-
-    Returns:
-        Nothing (in place)
-    """
-    # Only act if the instance is a dict.
-    if not isinstance(instance, dict) or allow_additional_properties:
-        return
-
-    # Allowed keys are those defined in the schema's "properties".
-    allowed = schema.get("properties", {}).keys()
-
-    # Iterate over a copy of keys so we can modify the dict in place.
-    n_removed = 0
-    for key in list(instance.keys()):
-        if key not in allowed:
-            instance.pop(key)
-            n_removed += 1
-            logger.info(f"Unspecified property removed from data object: {key}.")
-
-    if n_removed > 0:
-        logger.info(f"{n_removed} unspecified properties removed from data object.")
-
-
-def extend_jsonschema_validator_with_pruning(validator):
-    """Modify behavior of a jsonschema.Validator to use pruning.
-
-    Validators extended using this function will prune extra
-    fields, rather than raising a ValidationError, when encountering
-    extra, unspecified fiends when `additionalProperties: False` is
-    set in the validating schema.
-
-    Args:
-        validator (Type[jsonschema.Validator): A validator class
-            to extend with pruning behavior.
-
-    Returns:
-        Type[jsonschema.Validator]: A validator class that will
-            prune extra fields.
-    """
-    return jsonschema.validators.extend(validator, {"additionalProperties": prune_additional_properties})
-
-
-def _get_decimal_info_from_anyof(schema: dict) -> tuple[bool, int | None]:
-    """Check if schema is a Decimal anyOf and extract decimal places.
-
-    Returns (is_decimal, decimal_places) where decimal_places is None if no constraint.
-    """
-    any_of = schema.get("anyOf")
-    if not isinstance(any_of, list):
-        return False, None
-
-    has_number = any(item.get("type") == "number" for item in any_of)
-    if not has_number:
-        return False, None
-
-    for item in any_of:
-        if item.get("type") == "string" and "pattern" in item:
-            match = re.search(r"\\d\{0,(\d+)\}", item["pattern"])
-            if match:
-                return True, int(match.group(1))
-            return True, None  # Decimal without precision constraint
-    return False, None
-
-
-def normalize_decimal_fields(obj: DataObjectT, schema: JSONSchemaT) -> DataObjectT:
-    """Normalize Decimal-like anyOf fields to floats with proper precision."""
-    if not isinstance(obj, dict):
-        return obj
-
-    defs = schema.get("$defs", {})
-    obj_schema = defs.get(schema.get("$ref", "")[len("#/$defs/") :], schema)
-    props = obj_schema.get("properties", {})
-
-    for key, value in obj.items():
-        field_schema = props.get(key, {})
-        if "$ref" in field_schema:
-            field_schema = defs.get(field_schema["$ref"][len("#/$defs/") :], {})
-
-        if isinstance(value, dict):
-            obj[key] = normalize_decimal_fields(value, schema)
-        elif isinstance(value, list):
-            obj[key] = [normalize_decimal_fields(v, schema) if isinstance(v, dict) else v for v in value]
-        elif isinstance(value, (int, float, str)) and not isinstance(value, bool):
-            is_decimal, decimal_places = _get_decimal_info_from_anyof(field_schema)
-            if is_decimal:
-                d = Decimal(str(value))
-                if decimal_places is not None:
-                    d = d.quantize(Decimal(f"0.{'0' * decimal_places}"), rounding=ROUND_HALF_UP)
-                obj[key] = float(d)
-
-    return obj
-
-
-## We don't expect the outer data type (e.g. dict, list, or const) to be
-## modified by the pruning action.
-@overload
-def validate(
-    obj: dict[str, Any],
-    schema: JSONSchemaT,
-    pruning: bool = False,
-    no_extra_properties: bool = False,
-) -> dict[str, Any]: ...
-
-
-@overload
-def validate(
-    obj: list[Any],
-    schema: JSONSchemaT,
-    pruning: bool = False,
-    no_extra_properties: bool = False,
-) -> list[Any]: ...
-
-
-@overload
-def validate(
-    obj: T_primitive,
-    schema: JSONSchemaT,
-    pruning: bool = False,
-    no_extra_properties: bool = False,
-) -> T_primitive: ...
-
-
-def validate(
-    obj: DataObjectT,
-    schema: JSONSchemaT,
-    pruning: bool = False,
-    no_extra_properties: bool = False,
-) -> DataObjectT:
-    """Validate a data object against a JSONSchema.
-
-    Args:
-        obj (DataObjectT): A data structure to validate against the
-            schema.
-        schema: (JSONSchemaT): A valid JSONSchema to use to validate
-            the provided data object.
-        pruning (bool): If set to `True`, then the default behavior for
-            `additionalProperties: False` is set to prune non-specified
-            properties instead of raising a ValidationError.
-            Default: `False`.
-        no_extra_properties (bool): If set to `True`, then
-            `additionalProperties: False` is set on all the schema
-            and all of its sub-schemas. This operation overrides any
-            existing settings of `additionalProperties` within the
-            schema. Default: `False`.
-
-    Raises:
-        JSONSchemaValidationError: This exception raised in the
-            event that the JSONSchema doesn't match the provided
-            schema.
-    """
-    final_object = deepcopy(obj)
-    validator = DEFAULT_JSONSCHEMA_VALIDATOR
-    if pruning:
-        validator = extend_jsonschema_validator_with_pruning(validator)
-
-    if no_extra_properties:
-        schema = forbid_additional_properties(schema)
-
-    try:
-        validator(schema).validate(final_object)
-    except jsonschema.ValidationError as exc:
-        raise JSONSchemaValidationError(str(exc)) from exc
-
-    final_object = normalize_decimal_fields(final_object, schema)
-
-    return final_object

data_designer/engine/processing/processors/base.py

@@ -1,13 +0,0 @@
-# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
-# SPDX-License-Identifier: Apache-2.0
-
-from __future__ import annotations
-
-from abc import ABC, abstractmethod
-
-from data_designer.engine.configurable_task import ConfigurableTask, DataT, TaskConfigT
-
-
-class Processor(ConfigurableTask[TaskConfigT], ABC):
-    @abstractmethod
-    def process(self, data: DataT, *, current_batch_number: int | None = None) -> DataT: ...

data_designer/engine/processing/processors/drop_columns.py

@@ -1,42 +0,0 @@
-# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
-# SPDX-License-Identifier: Apache-2.0
-
-from __future__ import annotations
-
-import logging
-from typing import TYPE_CHECKING
-
-from data_designer.config.processors import DropColumnsProcessorConfig
-from data_designer.engine.dataset_builders.artifact_storage import BatchStage
-from data_designer.engine.processing.processors.base import Processor
-from data_designer.lazy_heavy_imports import pd
-
-if TYPE_CHECKING:
-    import pandas as pd
-
-logger = logging.getLogger(__name__)
-
-
-class DropColumnsProcessor(Processor[DropColumnsProcessorConfig]):
-    def process(self, data: pd.DataFrame, *, current_batch_number: int | None = None) -> pd.DataFrame:
-        logger.info(f"🙈 Dropping columns: {self.config.column_names}")
-        if current_batch_number is not None:  # not in preview mode
-            self._save_dropped_columns_if_needed(data, current_batch_number)
-        for column in self.config.column_names:
-            if column in data.columns:
-                data.drop(columns=[column], inplace=True)
-            else:
-                logger.warning(f"⚠️ Cannot drop column: `{column}` not found in the dataset.")
-        return data
-
-    def _save_dropped_columns_if_needed(self, data: pd.DataFrame, current_batch_number: int) -> None:
-        logger.debug("📦 Saving dropped columns to dropped-columns directory")
-        dropped_column_parquet_file_name = self.artifact_storage.create_batch_file_path(
-            batch_number=current_batch_number,
-            batch_stage=BatchStage.DROPPED_COLUMNS,
-        ).name
-        self.artifact_storage.write_parquet_file(
-            parquet_file_name=dropped_column_parquet_file_name,
-            dataframe=data[self.config.column_names],
-            batch_stage=BatchStage.DROPPED_COLUMNS,
-        )

data_designer/engine/processing/processors/registry.py

@@ -1,25 +0,0 @@
-# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
-# SPDX-License-Identifier: Apache-2.0
-
-from __future__ import annotations
-
-from data_designer.config.base import ConfigBase
-from data_designer.config.processors import (
-    DropColumnsProcessorConfig,
-    ProcessorType,
-    SchemaTransformProcessorConfig,
-)
-from data_designer.engine.processing.processors.base import Processor
-from data_designer.engine.processing.processors.drop_columns import DropColumnsProcessor
-from data_designer.engine.processing.processors.schema_transform import SchemaTransformProcessor
-from data_designer.engine.registry.base import TaskRegistry
-
-
-class ProcessorRegistry(TaskRegistry[str, Processor, ConfigBase]): ...
-
-
-def create_default_processor_registry() -> ProcessorRegistry:
-    registry = ProcessorRegistry()
-    registry.register(ProcessorType.SCHEMA_TRANSFORM, SchemaTransformProcessor, SchemaTransformProcessorConfig, False)
-    registry.register(ProcessorType.DROP_COLUMNS, DropColumnsProcessor, DropColumnsProcessorConfig, False)
-    return registry

data_designer/engine/processing/processors/schema_transform.py

@@ -1,49 +0,0 @@
-# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
-# SPDX-License-Identifier: Apache-2.0
-
-from __future__ import annotations
-
-import json
-import logging
-from typing import TYPE_CHECKING
-
-from data_designer.config.processors import SchemaTransformProcessorConfig
-from data_designer.engine.dataset_builders.artifact_storage import BatchStage
-from data_designer.engine.processing.ginja.environment import WithJinja2UserTemplateRendering
-from data_designer.engine.processing.processors.base import Processor
-from data_designer.engine.processing.utils import deserialize_json_values
-from data_designer.lazy_heavy_imports import pd
-
-if TYPE_CHECKING:
-    import pandas as pd
-
-logger = logging.getLogger(__name__)
-
-
-class SchemaTransformProcessor(WithJinja2UserTemplateRendering, Processor[SchemaTransformProcessorConfig]):
-    @property
-    def template_as_str(self) -> str:
-        return json.dumps(self.config.template)
-
-    def process(self, data: pd.DataFrame, *, current_batch_number: int | None = None) -> pd.DataFrame:
-        self.prepare_jinja2_template_renderer(self.template_as_str, data.columns.to_list())
-        formatted_records = [
-            json.loads(self.render_template(deserialize_json_values(record)).replace("\n", "\\n"))
-            for record in data.to_dict(orient="records")
-        ]
-        formatted_data = pd.DataFrame(formatted_records)
-        if current_batch_number is not None:
-            self.artifact_storage.write_batch_to_parquet_file(
-                batch_number=current_batch_number,
-                dataframe=formatted_data,
-                batch_stage=BatchStage.PROCESSORS_OUTPUTS,
-                subfolder=self.config.name,
-            )
-        else:
-            self.artifact_storage.write_parquet_file(
-                parquet_file_name=f"{self.config.name}.parquet",
-                dataframe=formatted_data,
-                batch_stage=BatchStage.PROCESSORS_OUTPUTS,
-            )
-
-        return data

data_designer/engine/processing/utils.py

@@ -1,169 +0,0 @@
-# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
-# SPDX-License-Identifier: Apache-2.0
-
-from __future__ import annotations
-
-import ast
-import copy
-import json
-import logging
-import re
-from typing import TYPE_CHECKING, Any, TypeVar, overload
-
-from data_designer.lazy_heavy_imports import pd
-
-if TYPE_CHECKING:
-    import pandas as pd
-
-logger = logging.getLogger(__name__)
-
-T = TypeVar("T")
-K = TypeVar("K")
-V = TypeVar("V")
-
-
-def concat_datasets(datasets: list[pd.DataFrame]) -> pd.DataFrame:
-    _verify_columns_are_unique(datasets)
-    _verify_dataset_lengths_are_equal(datasets)
-    emoji = " + ".join(["💾"] * len(datasets))
-    logger.info(f"({emoji}) Concatenating {len(datasets)} datasets")
-    return pd.concat([df for df in datasets], axis=1)
-
-
-# Overloads to help static type checker better understand
-# the input/output types of the deserialize_json_values function.
-@overload
-def deserialize_json_values(data: str) -> dict[str, Any] | list[Any] | Any: ...
-
-
-@overload
-def deserialize_json_values(data: list[T]) -> list[Any]: ...
-
-
-@overload
-def deserialize_json_values(data: dict[K, V]) -> dict[K, Any]: ...
-
-
-@overload
-def deserialize_json_values(data: T) -> T: ...
-
-
-def deserialize_json_values(data):
-    """De-serialize JSON strings in various input formats.
-
-    This function creates a deep copy of the input data and does not mutate the original.
-
-    Args:
-        data: Input data in one of four formats:
-            - Single string (JSON string to deserialize)
-            - List of strings (list of JSON strings to deserialize)
-            - Dictionary (potentially with nested JSON strings to deserialize)
-            - Some other object that can't be deserialized.
-
-    Returns:
-        Deserialized data in the corresponding format:
-            - Dictionary (when input is a single string)
-            - List of dictionaries (when input is a list of strings)
-            - Dictionary (when input is a dictionary, with nested JSON strings deserialized)
-            - The original object (if there is no deserialization to perform)
-
-    """
-    # Create a deep copy to avoid mutating the original data
-    data_copy = copy.deepcopy(data)
-
-    # Case 1: Single string input
-    if isinstance(data_copy, str):
-        try:
-            return json.loads(data_copy)
-        except json.JSONDecodeError:
-            return data_copy
-
-    # Case 2: List of strings input
-    elif isinstance(data_copy, list):
-        result = []
-        for item in data_copy:
-            if isinstance(item, str):
-                try:
-                    result.append(json.loads(item))
-                except json.JSONDecodeError:
-                    result.append(item)
-            else:
-                # If list contains non-string items, recursively process them
-                result.append(deserialize_json_values(item))
-        return result
-
-    # Case 3: Dictionary input with potential nested JSON strings
-    elif isinstance(data_copy, dict):
-        result = {}
-        for key, value in data_copy.items():
-            if isinstance(value, str):
-                try:
-                    result[key] = json.loads(value)
-                except json.JSONDecodeError:
-                    result[key] = value
-            elif isinstance(value, (dict, list)):
-                # Recursively process nested dictionaries and lists
-                result[key] = deserialize_json_values(value)
-            else:
-                result[key] = value
-        return result
-
-    # Fallback for other data types
-    else:
-        return data_copy
-
-
-def parse_list_string(text: str) -> list[str]:
-    """Parse a list from a string, handling JSON arrays, Python lists, and trailing commas."""
-    text = text.strip()
-
-    # Try JSON first
-    try:
-        list_obj = json.loads(text)
-        if isinstance(list_obj, list):
-            return _clean_whitespace(list_obj)
-    except json.JSONDecodeError:
-        pass
-
-    # Remove trailing commas before closing brackets (common in JSON-like strings)
-    text_cleaned = re.sub(r",\s*]", "]", text)
-    text_cleaned = re.sub(r",\s*}", "}", text_cleaned)
-
-    # Try JSON again with cleaned text
-    try:
-        return _clean_whitespace(json.loads(text_cleaned))
-    except json.JSONDecodeError:
-        pass
-
-    # Try Python literal eval (handles single quotes)
-    try:
-        return _clean_whitespace(ast.literal_eval(text_cleaned))
-    except (ValueError, SyntaxError):
-        pass
-
-    # If all else fails, return the original text
-    return [text.strip()]
-
-
-def _clean_whitespace(texts: list[str]) -> list[str]:
-    return [text.strip() for text in texts]
-
-
-def _verify_columns_are_unique(datasets: list[pd.DataFrame]) -> None:
-    joined_columns = set()
-    for df in datasets:
-        columns = set(df.columns)
-        overlapping_columns = joined_columns & columns
-        if len(overlapping_columns) > 0:
-            raise ValueError(
-                f"🛑 Input datasets have overlapping columns: {overlapping_columns} "
-                "Please ensure that the column names are unique."
-            )
-        joined_columns.update(columns)
-
-
-def _verify_dataset_lengths_are_equal(datasets: list[pd.DataFrame]) -> None:
-    if len(set([len(df) for df in datasets])) > 1:
-        raise ValueError(
-            "🛑 Input datasets have different lengths. Please ensure that the datasets have the same number of rows."
-        )

data_designer/engine/registry/base.py

@@ -1,99 +0,0 @@
-# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
-# SPDX-License-Identifier: Apache-2.0
-
-from __future__ import annotations
-
-import threading
-from typing import Any, Generic, TypeVar
-
-from data_designer.config.base import ConfigBase
-from data_designer.config.utils.type_helpers import StrEnum
-from data_designer.engine.configurable_task import ConfigurableTask
-from data_designer.engine.registry.errors import NotFoundInRegistryError, RegistryItemNotTypeError
-
-EnumNameT = TypeVar("EnumNameT", bound=StrEnum)
-TaskT = TypeVar("TaskT", bound=ConfigurableTask)
-TaskConfigT = TypeVar("TaskConfigT", bound=ConfigBase)
-
-
-class TaskRegistry(Generic[EnumNameT, TaskT, TaskConfigT]):
-    # registered type name -> type
-    _registry: dict[EnumNameT, type[TaskT]] = {}
-    # type -> registered type name
-    _reverse_registry: dict[type[TaskT], EnumNameT] = {}
-
-    # registered type name -> config type
-    _config_registry: dict[EnumNameT, type[TaskConfigT]] = {}
-    # config type -> registered type name
-    _reverse_config_registry: dict[type[TaskConfigT], EnumNameT] = {}
-
-    # all registries are singletons
-    _instance = None
-    _lock = threading.Lock()
-
-    @classmethod
-    def register(
-        cls,
-        name: EnumNameT,
-        task: type[TaskT],
-        config: type[TaskConfigT],
-        raise_on_collision: bool = False,
-    ) -> None:
-        if cls._has_been_registered(name):
-            if not raise_on_collision:
-                return
-            raise ValueError(f"{name} has already been registered!")
-
-        cls._raise_if_not_type(task)
-        cls._raise_if_not_type(config)
-
-        with cls._lock:
-            cls._registry[name] = task
-            cls._reverse_registry[task] = name
-            cls._config_registry[name] = config
-            cls._reverse_config_registry[config] = name
-
-    @classmethod
-    def get_task_type(cls, name: EnumNameT) -> type[TaskT]:
-        cls._raise_if_not_registered(name, cls._registry)
-        return cls._registry[name]
-
-    @classmethod
-    def get_config_type(cls, name: EnumNameT) -> type[TaskConfigT]:
-        cls._raise_if_not_registered(name, cls._config_registry)
-        return cls._config_registry[name]
-
-    @classmethod
-    def get_registered_name(cls, task: type[TaskT]) -> EnumNameT:
-        cls._raise_if_not_registered(task, cls._reverse_registry)
-        return cls._reverse_registry[task]
-
-    @classmethod
-    def get_for_config_type(cls, config: type[TaskConfigT]) -> type[TaskT]:
-        cls._raise_if_not_registered(config, cls._reverse_config_registry)
-        name = cls._reverse_config_registry[config]
-        return cls.get_task_type(name)
-
-    @classmethod
-    def _has_been_registered(cls, name: EnumNameT) -> bool:
-        return name in cls._registry
-
-    @classmethod
-    def _raise_if_not_registered(cls, key: EnumNameT | type[TaskT] | type[TaskConfigT], mapping: dict) -> None:
-        if not (isinstance(key, StrEnum) or isinstance(key, str)):
-            cls._raise_if_not_type(key)
-        if key not in mapping:
-            raise NotFoundInRegistryError(f"{key} not found in registry")
-
-    @classmethod
-    def _raise_if_not_type(cls, obj: Any) -> None:
-        if not isinstance(obj, type):
-            raise RegistryItemNotTypeError(f"{obj} is not a class!")
-
-    def __new__(cls, *args, **kwargs):
-        """Registry is a singleton."""
-        if not cls._instance:
-            with cls._lock:
-                if not cls._instance:
-                    cls._instance = super().__new__(cls)
-        return cls._instance

data_designer/engine/registry/data_designer_registry.py

@@ -1,39 +0,0 @@
-# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
-# SPDX-License-Identifier: Apache-2.0
-
-from __future__ import annotations
-
-from data_designer.engine.analysis.column_profilers.registry import (
-    ColumnProfilerRegistry,
-    create_default_column_profiler_registry,
-)
-from data_designer.engine.column_generators.registry import (
-    ColumnGeneratorRegistry,
-    create_default_column_generator_registry,
-)
-from data_designer.engine.processing.processors.registry import ProcessorRegistry, create_default_processor_registry
-
-
-class DataDesignerRegistry:
-    def __init__(
-        self,
-        *,
-        column_generator_registry: ColumnGeneratorRegistry | None = None,
-        column_profiler_registry: ColumnProfilerRegistry | None = None,
-        processor_registry: ProcessorRegistry | None = None,
-    ):
-        self._column_generator_registry = column_generator_registry or create_default_column_generator_registry()
-        self._column_profiler_registry = column_profiler_registry or create_default_column_profiler_registry()
-        self._processor_registry = processor_registry or create_default_processor_registry()
-
-    @property
-    def column_generators(self) -> ColumnGeneratorRegistry:
-        return self._column_generator_registry
-
-    @property
-    def column_profilers(self) -> ColumnProfilerRegistry:
-        return self._column_profiler_registry
-
-    @property
-    def processors(self) -> ProcessorRegistry:
-        return self._processor_registry

data_designer/engine/registry/errors.py

@@ -1,12 +0,0 @@
-# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
-# SPDX-License-Identifier: Apache-2.0
-
-from __future__ import annotations
-
-from data_designer.engine.errors import DataDesignerError
-
-
-class NotFoundInRegistryError(DataDesignerError): ...
-
-
-class RegistryItemNotTypeError(DataDesignerError): ...