data-designer 0.3.8rc1__py3-none-any.whl → 0.4.0__py3-none-any.whl

data_designer/engine/validators/python.py

@@ -1,254 +0,0 @@
-# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
-# SPDX-License-Identifier: Apache-2.0
-
-from __future__ import annotations
-
-import ast
-import json
-import logging
-import subprocess
-import tempfile
-from collections import defaultdict
-from pathlib import Path
-from typing import TYPE_CHECKING
-from uuid import uuid4
-
-from pydantic import BaseModel
-from ruff.__main__ import find_ruff_bin
-
-from data_designer.config.validator_params import CodeValidatorParams
-from data_designer.engine.validators.base import BaseValidator, ValidationOutput, ValidationResult
-from data_designer.lazy_heavy_imports import pd
-
-if TYPE_CHECKING:
-    import pandas as pd
-
-logger = logging.getLogger(__name__)
-
-PYLINT_ERROR_CATEGORIES_ORDERED = [
-    "fatal",
-    "error",
-    "warning",
-    "convention",
-    "refactor",
-]
-PYLINT_VALID_LEVELS = {"none", "warning", "convention", "refactor"}
-
-TYPE_FROM_SYMBOL = {
-    "E": "refactor",
-    "F": "error",
-    "SIM": "refactor",
-    "PLC": "convention",
-    "PLE": "error",
-    "PLR": "refactor",
-    "PLW": "warning",
-    "SyntaxError": "fatal",
-}
-
-PYTHON_MESSAGES_FIELD = "python_linter_messages"
-RECORD_ID_COLUMN_NAME = "internal_code_record_id"
-
-
-class PythonValidationStat(BaseModel):
-    fatal: int = 0
-    error: int = 0
-    warning: int = 0
-    refactor: int = 0
-    convention: int = 0
-    statement: int = 0
-
-    @property
-    def score(self) -> float:
-        # https://pylint.pycqa.org/en/latest/user_guide/configuration/all-options.html#evaluation
-        if self.statement == 0:  # prevent division by zero down below
-            self.statement = max(1, self.statement)
-        return max(
-            0,
-            (
-                0
-                if self.fatal
-                else 10.0
-                - ((float(5 * self.error + self.warning + self.refactor + self.convention) / self.statement) * 10)
-            ),
-        )
-
-
-class PythonLinterMessage(BaseModel):
-    type: str
-    symbol: str
-    line: int
-    column: int
-    message: str
-
-    @property
-    def type_sort_order(self) -> int:
-        return PYLINT_ERROR_CATEGORIES_ORDERED.index(self.type)
-
-
-class PythonLinterMessages(BaseModel):
-    _messages: list[PythonLinterMessage] = []
-
-    @property
-    def messages(self) -> list[PythonLinterMessage]:
-        # Ordered by severity first then by line number
-        return sorted(self._messages, key=lambda msg: (msg.type_sort_order, msg.line))
-
-    def add(self, message: PythonLinterMessage) -> None:
-        self._messages.append(message)
-
-    def get_count_by_type(self) -> dict[str, int]:
-        count_by_type = defaultdict(int)
-        for message in self.messages:
-            count_by_type[message.type] += 1
-        return dict(count_by_type)
-
-    @property
-    def is_empty(self) -> bool:
-        return len(self.messages) == 0
-
-    @property
-    def severity(self) -> str:
-        if self.is_empty:
-            return "none"
-        return self.messages[0].type
-
-    @property
-    def is_valid(self) -> bool:
-        return self.is_empty or self.messages[0].type in PYLINT_VALID_LEVELS
-
-
-class PythonValidator(BaseValidator):
-    def __init__(self, config: CodeValidatorParams):
-        self.config = config
-
-    def run_validation(self, data: list[dict]) -> ValidationResult:
-        df = pd.DataFrame(data)
-
-        if len(df.columns) > 1:
-            raise ValueError("Python validator assumes single column input")
-        target_column = df.columns[0]
-
-        df.loc[:, RECORD_ID_COLUMN_NAME] = [uuid4() for _ in range(df.shape[0])]
-        with tempfile.TemporaryDirectory() as temp_dir:
-            _ = df.apply(
-                self._write_code_to_file,
-                args=(target_column, temp_dir),
-                axis=1,
-            )
-            results = self._validate_files_in_path(path=temp_dir)
-
-            records = df.to_dict(orient="records")
-
-            ordered_results = []
-            for record in records:
-                module_id = self._get_module_name(record[RECORD_ID_COLUMN_NAME], target_column)
-                result = results.get(module_id)
-                if result is not None:
-                    ordered_results.append(result)
-
-        return ValidationResult(data=ordered_results)
-
-    def _validate_files_in_path(self, path: str) -> dict[str, ValidationOutput]:
-        lint_results = self._run_linter(path)
-
-        scores_by_module = self._get_scores(
-            {
-                module: messages.get_count_by_type()
-                | {"statement": self._count_python_statements(f"{path}/{module}.py")}
-                for module, messages in lint_results.items()
-            }
-        )
-
-        validation_result = {}
-        for module, score in scores_by_module.items():
-            messages = lint_results.get(module, PythonLinterMessages())
-            metadata = {
-                "python_linter_score": score,
-                "python_linter_severity": messages.severity,
-                PYTHON_MESSAGES_FIELD: [m.model_dump() for m in messages.messages],
-            }
-            validation_result[module] = ValidationOutput(is_valid=messages.is_valid, **metadata)
-        return validation_result
-
-    def _write_code_to_file(self, row: pd.Series, code_column: str, path: str) -> None:
-        with open(f"{path}/{self._get_module_name(row[RECORD_ID_COLUMN_NAME], code_column)}.py", "w") as file:
-            file.write(row[code_column])
-
-    @staticmethod
-    def _get_module_name(record_id: str, column_name: str) -> str:
-        return f"{record_id}_{column_name}"
-
-    @staticmethod
-    def _run_linter(codebase_path: str) -> dict[str, PythonLinterMessages]:
-        # Create empty dict for output
-        processed = {}
-        for file in Path(codebase_path).glob("*.py"):
-            processed[file.stem] = PythonLinterMessages()
-
-        # Run ruff linter with JSON output
-        ruff_bin = find_ruff_bin()
-
-        ruff_exec = subprocess.run(
-            [
-                ruff_bin,
-                "check",
-                "--select",
-                "E,F6,F7,F8,SIM,PLC,PLE,PLR,PLW",
-                "--output-format=json",
-                codebase_path,
-            ],
-            text=True,
-            capture_output=True,
-            check=False,
-            cwd=Path.cwd(),
-        )
-        ruff_output = ruff_exec.stdout
-
-        # Parse JSON output
-        try:
-            diagnostics = json.loads(ruff_output)
-        except json.JSONDecodeError as e:
-            raise RuntimeError(f"Failed to parse ruff JSON output: {e}")
-
-        if not diagnostics:
-            return processed  # no errors or warnings
-
-        for diagnostic in diagnostics:
-            filename = diagnostic["filename"]
-            code = diagnostic["code"]
-            location = diagnostic["location"]
-            message = diagnostic["message"]
-
-            # Extract alphabetic prefix from code for type mapping
-            alpha_prefix = "".join(c for c in code if c.isalpha())
-            error_type = TYPE_FROM_SYMBOL.get(alpha_prefix, "warning")
-
-            processed[Path(filename).stem].add(
-                PythonLinterMessage(
-                    type=error_type,
-                    symbol=code,
-                    line=location["row"],
-                    column=location["column"],
-                    message=message,
-                )
-            )
-
-        return processed
-
-    @staticmethod
-    def _get_scores(stats_by_module: dict[str, dict[str, int]]) -> dict[str, float]:
-        scores = {}
-        for key, item in stats_by_module.items():
-            stat = PythonValidationStat(**item)
-            scores[key] = stat.score
-        return scores
-
-    @staticmethod
-    def _count_python_statements(file_path: str) -> int:
-        """Count the number of statements in a Python file."""
-        try:
-            with open(file_path, "r", encoding="utf-8") as f:
-                tree = ast.parse(f.read())
-            return sum(1 for node in ast.walk(tree) if isinstance(node, ast.stmt))
-        except Exception:
-            return 0

data_designer/engine/validators/remote.py

@@ -1,89 +0,0 @@
-# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
-# SPDX-License-Identifier: Apache-2.0
-
-from __future__ import annotations
-
-import logging
-from typing import TYPE_CHECKING
-
-from httpx_retries import Retry, RetryTransport
-
-from data_designer.config.validator_params import RemoteValidatorParams
-from data_designer.engine.errors import RemoteValidationSchemaError
-from data_designer.engine.processing.gsonschema.exceptions import JSONSchemaValidationError
-from data_designer.engine.processing.gsonschema.validators import validate
-from data_designer.engine.validators.base import BaseValidator, ValidationResult
-from data_designer.lazy_heavy_imports import httpx
-
-if TYPE_CHECKING:
-    import httpx
-
-logger = logging.getLogger(__name__)
-
-
-class RemoteEndpointClient:
-    """Client for making parallel HTTP requests to remote endpoints with retry, timeout, and auth support."""
-
-    def __init__(
-        self,
-        config: RemoteValidatorParams,
-    ):
-        """
-        Initialize the remote endpoint client.
-
-        Args:
-            config: Remote validator parameters
-        """
-        self.endpoint_url = config.endpoint_url
-        self.output_schema = config.output_schema
-        self.timeout = config.timeout
-        self.max_retries = config.max_retries
-        self.retry_backoff = config.retry_backoff
-
-    def post_to_remote_endpoint(self, content: dict) -> dict:
-        """
-        Make a single HTTP request with retry logic.
-
-        Args:
-            content: The content to be posted to the remote endpoint
-
-        Returns:
-            The JSON response from the remote endpoint
-
-        Raises:
-            httpx.RequestError: If all retry attempts fail
-            httpx.HTTPStatusError: If the server returns an error status
-        """
-        retry = Retry(
-            total=self.max_retries,
-            backoff_factor=self.retry_backoff,
-            status_forcelist=[429, 500, 502, 503, 504],
-        )
-        transport = RetryTransport(retry=retry)
-
-        with httpx.Client(
-            timeout=httpx.Timeout(self.timeout),
-            transport=transport,
-        ) as http_client:
-            response = http_client.post(
-                self.endpoint_url,
-                json=content,
-            )
-            response.raise_for_status()
-
-            response_json = response.json()
-            if self.output_schema:
-                try:
-                    validate(response_json, self.output_schema, no_extra_properties=True)
-                except JSONSchemaValidationError as exc:
-                    raise RemoteValidationSchemaError(str(exc)) from exc
-            return response_json
-
-
-class RemoteValidator(BaseValidator):
-    def __init__(self, config: RemoteValidatorParams):
-        self.remote_endpoint_client = RemoteEndpointClient(config=config)
-
-    def run_validation(self, data: list[dict]) -> ValidationResult:
-        result = self.remote_endpoint_client.post_to_remote_endpoint(content={"data": data})
-        return ValidationResult.model_validate(result)

data_designer/engine/validators/sql.py

@@ -1,65 +0,0 @@
-# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
-# SPDX-License-Identifier: Apache-2.0
-
-from __future__ import annotations
-
-import logging
-import re
-from typing import TYPE_CHECKING
-
-from data_designer.config.utils.code_lang import CodeLang
-from data_designer.config.validator_params import CodeValidatorParams
-from data_designer.engine.validators.base import BaseValidator, ValidationOutput, ValidationResult
-from data_designer.lazy_heavy_imports import pd, sqlfluff
-
-if TYPE_CHECKING:
-    import pandas as pd
-    import sqlfluff
-
-sqlfluff_logger = logging.getLogger("sqlfluff")
-sqlfluff_logger.setLevel(logging.WARNING)
-
-
-class SQLValidator(BaseValidator):
-    def __init__(self, config: CodeValidatorParams):
-        self.config = config
-
-    def run_validation(self, data: list[dict]) -> ValidationResult:
-        df = pd.DataFrame(data)
-
-        if len(df.columns) > 1:
-            raise ValueError("SQL validator assumes single column input")
-        target_column = df.columns[0]
-
-        records = df.to_dict(orient="records")
-
-        results = []
-        for record in records:
-            result = self._validate_query(record[target_column])
-            results.append(result)
-
-        return ValidationResult(data=results)
-
-    def _validate_query(self, content: str) -> ValidationResult:
-        try:
-            result = sqlfluff.lint(
-                content,
-                dialect=CodeLang.parse_dialect(self.config.code_lang),
-            )
-            prs_errors = [res for res in result if res["code"].startswith("PRS")]
-            error_messages = "\n".join([f"{error['code']}: {error['description']}" for error in prs_errors])
-            decimal_pattern = re.compile(r"DECIMAL\(\d+\)")
-            decimal_issues = decimal_pattern.findall(content)
-            if decimal_issues:
-                error_messages += "\nCustom Check: Found DECIMAL definitions without a scale, which may be incorrect."
-            if error_messages:
-                return ValidationOutput(
-                    is_valid=False,
-                    error_messages=error_messages,
-                )
-            return ValidationOutput(is_valid=True, error_messages="")
-        except Exception as e:
-            return ValidationOutput(
-                is_valid=False,
-                error_messages=f"Exception during SQL parsing: {e}",
-            )

data_designer/errors.py

@@ -1,7 +0,0 @@
-# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
-# SPDX-License-Identifier: Apache-2.0
-
-from __future__ import annotations
-
-
-class DataDesignerError(Exception): ...

data_designer/essentials/__init__.py

@@ -1,33 +0,0 @@
-# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
-# SPDX-License-Identifier: Apache-2.0
-
-from __future__ import annotations
-
-from data_designer.config.default_model_settings import resolve_seed_default_model_settings
-from data_designer.config.exports import *  # noqa: F403
-from data_designer.config.run_config import RunConfig
-from data_designer.config.validator_params import LocalCallableValidatorParams
-from data_designer.interface.data_designer import DataDesigner
-from data_designer.logging import LoggingConfig, configure_logging
-
-configure_logging(LoggingConfig.default())
-
-# Resolve default model settings on import to ensure they are available when the library is used.
-resolve_seed_default_model_settings()
-
-
-def get_essentials_exports() -> list[str]:
-    logging = [
-        configure_logging.__name__,
-        LoggingConfig.__name__,
-    ]
-    local = [
-        DataDesigner.__name__,
-        LocalCallableValidatorParams.__name__,
-        RunConfig.__name__,
-    ]
-
-    return logging + local + get_config_exports()  # noqa: F405
-
-
-__all__ = get_essentials_exports()

data_designer/lazy_heavy_imports.py

@@ -1,54 +0,0 @@
-# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
-# SPDX-License-Identifier: Apache-2.0
-
-"""
-Lazy imports facade for heavy third-party dependencies.
-
-This module provides a centralized facade that lazily imports heavy dependencies
-only when accessed, significantly improving import performance.
-
-Usage:
-    from data_designer.lazy_heavy_imports import pd, np, faker, litellm
-
-    df = pd.DataFrame(...)
-    arr = np.array([1, 2, 3])
-    fake = faker.Faker()
-"""
-
-from __future__ import annotations
-
-import importlib
-
-# Mapping of lazy import names to their actual module paths
-_LAZY_IMPORTS = {
-    "pd": "pandas",
-    "np": "numpy",
-    "pq": "pyarrow.parquet",
-    "pa": "pyarrow",
-    "faker": "faker",
-    "litellm": "litellm",
-    "sqlfluff": "sqlfluff",
-    "httpx": "httpx",
-    "duckdb": "duckdb",
-    "nx": "networkx",
-    "scipy": "scipy",
-    "jsonschema": "jsonschema",
-}
-
-
-def __getattr__(name: str) -> object:
-    """Lazily import heavy third-party dependencies when accessed.
-
-    This allows fast imports of data_designer while deferring loading of heavy
-    libraries until they're actually needed.
-    """
-    if name in _LAZY_IMPORTS:
-        module_name = _LAZY_IMPORTS[name]
-        return importlib.import_module(module_name)
-
-    raise AttributeError(f"module 'data_designer.lazy_heavy_imports' has no attribute {name!r}")
-
-
-def __dir__() -> list[str]:
-    """Return list of available lazy imports."""
-    return list(_LAZY_IMPORTS.keys())

data_designer/logging.py

@@ -1,163 +0,0 @@
-# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
-# SPDX-License-Identifier: Apache-2.0
-
-from __future__ import annotations
-
-import logging
-import random
-import sys
-from dataclasses import dataclass, field
-from pathlib import Path
-from typing import TextIO
-
-from pythonjsonlogger import jsonlogger
-
-
-@dataclass
-class LoggerConfig:
-    name: str
-    level: str
-
-
-@dataclass
-class OutputConfig:
-    destination: TextIO | Path
-    structured: bool
-
-
-@dataclass
-class LoggingConfig:
-    logger_configs: list[LoggerConfig]
-    output_configs: list[OutputConfig]
-    root_level: str = "INFO"
-    to_silence: list[str] = field(default_factory=lambda: _DEFAULT_NOISY_LOGGERS)
-
-    @classmethod
-    def default(cls):
-        return LoggingConfig(
-            logger_configs=[LoggerConfig(name="data_designer", level="INFO")],
-            output_configs=[OutputConfig(destination=sys.stderr, structured=False)],
-        )
-
-    @classmethod
-    def debug(cls):
-        return LoggingConfig(
-            logger_configs=[LoggerConfig(name="data_designer", level="DEBUG")],
-            output_configs=[OutputConfig(destination=sys.stderr, structured=False)],
-        )
-
-
-class RandomEmoji:
-    """A generator for various themed emoji collections."""
-
-    @staticmethod
-    def cooking() -> str:
-        """Get a random cooking or food preparation emoji."""
-        return random.choice(["👨‍🍳", "👩‍🍳", "🍳", "🥘", "🍲", "🔪", "🥄", "🍴", "⏲️", "🥗"])
-
-    @staticmethod
-    def data() -> str:
-        """Get a random data or analytics emoji."""
-        return random.choice(["📊", "📈", "📉", "💾", "💿", "📀", "🗄️", "📁", "📂", "🗃️"])
-
-    @staticmethod
-    def generating() -> str:
-        """Get a random generating or creating emoji."""
-        return random.choice(["🏭", "⚙️", "🔨", "🛠️", "🏗️", "🎨", "✍️", "📝", "🔧", "⚒️"])
-
-    @staticmethod
-    def loading() -> str:
-        """Get a random loading or waiting emoji."""
-        return random.choice(["⏳", "⌛", "🔄", "♻️", "🔃", "⏰", "⏱️", "⏲️", "📡", "🌀"])
-
-    @staticmethod
-    def magic() -> str:
-        """Get a random magical or special effect emoji."""
-        return random.choice(["✨", "⭐", "🌟", "💫", "🪄", "🔮", "🎩", "🌈", "💎", "🦄"])
-
-    @staticmethod
-    def previewing() -> str:
-        """Get a random previewing or looking ahead emoji."""
-        return random.choice(["👀", "📺", "🔁", "👁️", "🔭", "🕵️", "🧐", "📸", "🎥", "🖼️"])
-
-    @staticmethod
-    def speed() -> str:
-        """Get a random speed or fast emoji."""
-        return random.choice(["⚡", "💨", "🏃", "🏎️", "🚄", "✈️", "💥", "⏩", "🏃‍♂️", "🏃‍♀️"])
-
-    @staticmethod
-    def start() -> str:
-        """Get a random emoji representing starting or launching something."""
-        return random.choice(["🚀", "▶️", "🎬", "🌅", "🏁", "🎯", "🚦", "🔔", "📣", "🎺"])
-
-    @staticmethod
-    def success() -> str:
-        """Get a random success or celebration emoji."""
-        return random.choice(["🎉", "🎊", "👏", "🙌", "🎆", "🍾", "☀️", "🏆", "✅", "🥳"])
-
-    @staticmethod
-    def thinking() -> str:
-        """Get a random thinking or processing emoji."""
-        return random.choice(["🤔", "💭", "🧠", "💡", "🔍", "🔎", "🤨", "🧐", "📝", "🧮"])
-
-    @staticmethod
-    def working() -> str:
-        """Get a random working or in-progress emoji."""
-        return random.choice(["⚙️", "🔧", "🔨", "⚒️", "🛠️", "💼", "👷", "🏗️", "🪛", "👨‍💻"])
-
-
-def configure_logging(config: LoggingConfig) -> None:
-    root_logger = logging.getLogger()
-
-    # Remove all handlers
-    root_logger.handlers.clear()
-
-    # Create and attach handler(s)
-    handlers = [_create_handler(output_config) for output_config in config.output_configs]
-    for handler in handlers:
-        root_logger.addHandler(handler)
-
-    # Set levels
-    root_logger.setLevel(config.root_level)
-    for logger_config in config.logger_configs:
-        logger = logging.getLogger(logger_config.name)
-        logger.setLevel(logger_config.level)
-
-    # Adjust noisy loggers
-    for name in config.to_silence:
-        quiet_noisy_logger(name)
-
-
-def quiet_noisy_logger(name: str) -> None:
-    logger = logging.getLogger(name)
-    logger.handlers.clear()
-    logger.setLevel(logging.WARNING)
-
-
-def _create_handler(output_config: OutputConfig) -> logging.Handler:
-    if isinstance(output_config.destination, Path):
-        handler = logging.FileHandler(str(output_config.destination))
-    else:
-        handler = logging.StreamHandler()
-
-    if output_config.structured:
-        formatter = _make_json_formatter()
-    else:
-        formatter = _make_stream_formatter()
-
-    handler.setFormatter(formatter)
-    return handler
-
-
-def _make_json_formatter() -> logging.Formatter:
-    log_format = "%(asctime)s %(levelname)s %(name)s %(message)s"
-    return jsonlogger.JsonFormatter(log_format)
-
-
-def _make_stream_formatter() -> logging.Formatter:
-    log_format = "[%(asctime)s] [%(levelname)s] %(message)s"
-    time_format = "%H:%M:%S"
-    return logging.Formatter(log_format, time_format)
-
-
-_DEFAULT_NOISY_LOGGERS = ["httpx", "matplotlib"]