sdg-hub 0.1.4__py3-none-any.whl → 0.2.0__py3-none-any.whl

sdg_hub/core/blocks/registry.py

@@ -0,0 +1,331 @@
+# SPDX-License-Identifier: Apache-2.0
+"""Enhanced BlockRegistry with metadata and better error handling.
+
+This module provides a clean registry system for blocks with metadata,
+categorization, and improved error handling.
+"""
+
+# Standard
+from dataclasses import dataclass
+from difflib import get_close_matches
+from typing import Optional
+import inspect
+
+# Third Party
+from rich.console import Console
+from rich.table import Table
+
+# Local
+from ..utils.logger_config import setup_logger
+
+logger = setup_logger(__name__)
+console = Console()
+
+
+@dataclass
+class BlockMetadata:
+    """Metadata for registered blocks.
+
+    Parameters
+    ----------
+    name : str
+        The registered name of the block.
+    block_class : Type
+        The actual block class.
+    category : str
+        Category for organization (e.g., 'llm', 'utility', 'filtering').
+    description : str, optional
+        Human-readable description of what the block does.
+    deprecated : bool, optional
+        Whether this block is deprecated.
+    replacement : str, optional
+        Suggested replacement if deprecated.
+    """
+
+    name: str
+    block_class: type
+    category: str
+    description: str = ""
+    deprecated: bool = False
+    replacement: Optional[str] = None
+
+    def __post_init__(self) -> None:
+        """Validate metadata after initialization."""
+        if not self.name:
+            raise ValueError("Block name cannot be empty")
+        if not inspect.isclass(self.block_class):
+            raise ValueError("block_class must be a class")
+
+
+class BlockRegistry:
+    """Registry for block classes with metadata and enhanced error handling."""
+
+    _metadata: dict[str, BlockMetadata] = {}
+    _categories: dict[str, set[str]] = {}
+
+    @classmethod
+    def register(
+        cls,
+        block_name: str,
+        category: str,
+        description: str = "",
+        deprecated: bool = False,
+        replacement: Optional[str] = None,
+    ):
+        """Register a block class with metadata.
+
+        Parameters
+        ----------
+        block_name : str
+            Name under which to register the block.
+        category : str
+            Category for organization.
+        description : str, optional
+            Human-readable description of the block.
+        deprecated : bool, optional
+            Whether this block is deprecated.
+        replacement : str, optional
+            Suggested replacement if deprecated.
+
+        Returns
+        -------
+        callable
+            Decorator function.
+        """
+
+        def decorator(block_class: type) -> type:
+            # Validate the class
+            cls._validate_block_class(block_class)
+
+            # Create metadata
+            metadata = BlockMetadata(
+                name=block_name,
+                block_class=block_class,
+                category=category,
+                description=description,
+                deprecated=deprecated,
+                replacement=replacement,
+            )
+
+            # Register the metadata
+            cls._metadata[block_name] = metadata
+
+            # Update category index
+            if category not in cls._categories:
+                cls._categories[category] = set()
+            cls._categories[category].add(block_name)
+
+            logger.debug(
+                f"Registered block '{block_name}' "
+                f"({block_class.__name__}) in category '{category}'"
+            )
+
+            if deprecated:
+                warning_msg = f"Block '{block_name}' is deprecated."
+                if replacement:
+                    warning_msg += f" Use '{replacement}' instead."
+                logger.warning(warning_msg)
+
+            return block_class
+
+        return decorator
+
+    @classmethod
+    def _validate_block_class(cls, block_class: type) -> None:
+        """Validate that a class is a proper block class.
+
+        Parameters
+        ----------
+        block_class : Type
+            The class to validate.
+
+        Raises
+        ------
+        ValueError
+            If the class is not a valid block class.
+        """
+        if not inspect.isclass(block_class):
+            raise ValueError(f"Expected a class, got {type(block_class)}")
+
+        # Validate BaseBlock inheritance
+        try:
+            # Local
+            from .base import BaseBlock
+
+            if not issubclass(block_class, BaseBlock):
+                raise ValueError(
+                    f"Block class '{block_class.__name__}' must inherit from BaseBlock"
+                )
+        except ImportError as exc:
+            # BaseBlock not available, check for generate method
+            if not hasattr(block_class, "generate"):
+                raise ValueError(
+                    f"Block class '{block_class.__name__}' must implement 'generate' method"
+                ) from exc
+
+    @classmethod
+    def get(cls, block_name: str) -> type:
+        """Get a block class with enhanced error handling.
+
+        Parameters
+        ----------
+        block_name : str
+            Name of the block to retrieve.
+
+        Returns
+        -------
+        Type
+            The block class.
+
+        Raises
+        ------
+        KeyError
+            If the block is not found, with helpful suggestions.
+        """
+        if block_name not in cls._metadata:
+            available_blocks = list(cls._metadata.keys())
+            suggestions = get_close_matches(
+                block_name, available_blocks, n=3, cutoff=0.6
+            )
+
+            error_msg = f"Block '{block_name}' not found in registry."
+
+            if suggestions:
+                error_msg += f" Did you mean: {', '.join(suggestions)}?"
+
+            if available_blocks:
+                error_msg += (
+                    f"\nAvailable blocks: {', '.join(sorted(available_blocks))}"
+                )
+
+            if cls._categories:
+                error_msg += (
+                    f"\nCategories: {', '.join(sorted(cls._categories.keys()))}"
+                )
+
+            logger.error(error_msg)
+            raise KeyError(error_msg)
+
+        metadata = cls._metadata[block_name]
+
+        if metadata.deprecated:
+            warning_msg = f"Block '{block_name}' is deprecated."
+            if metadata.replacement:
+                warning_msg += f" Use '{metadata.replacement}' instead."
+            logger.warning(warning_msg)
+
+        return metadata.block_class
+
+    @classmethod
+    def info(cls, block_name: str) -> BlockMetadata:
+        """Get metadata for a specific block.
+
+        Parameters
+        ----------
+        block_name : str
+            Name of the block.
+
+        Returns
+        -------
+        BlockMetadata
+            The block's metadata.
+
+        Raises
+        ------
+        KeyError
+            If the block is not found.
+        """
+        if block_name not in cls._metadata:
+            raise KeyError(f"Block '{block_name}' not found in registry.")
+        return cls._metadata[block_name]
+
+    @classmethod
+    def categories(cls) -> list[str]:
+        """Get all available categories.
+
+        Returns
+        -------
+        List[str]
+            Sorted list of categories.
+        """
+        return sorted(cls._categories.keys())
+
+    @classmethod
+    def category(cls, category: str) -> list[str]:
+        """Get all blocks in a specific category.
+
+        Parameters
+        ----------
+        category : str
+            The category to filter by.
+
+        Returns
+        -------
+        List[str]
+            List of block names in the category.
+
+        Raises
+        ------
+        KeyError
+            If the category doesn't exist.
+        """
+        if category not in cls._categories:
+            available_categories = sorted(cls._categories.keys())
+            raise KeyError(
+                f"Category '{category}' not found. "
+                f"Available categories: {', '.join(available_categories)}"
+            )
+        return sorted(cls._categories[category])
+
+    @classmethod
+    def all(cls) -> dict[str, list[str]]:
+        """List all blocks organized by category.
+
+        Returns
+        -------
+        Dict[str, List[str]]
+            Dictionary mapping categories to lists of block names.
+        """
+        return {
+            category: sorted(blocks) for category, blocks in cls._categories.items()
+        }
+
+    @classmethod
+    def show(cls) -> None:
+        """Print a Rich-formatted table of all available blocks."""
+        if not cls._metadata:
+            console.print("[yellow]No blocks registered yet.[/yellow]")
+            return
+
+        table = Table(
+            title="Available Blocks", show_header=True, header_style="bold magenta"
+        )
+        table.add_column("Block Name", style="cyan", no_wrap=True)
+        table.add_column("Category", style="green")
+        table.add_column("Description", style="white")
+
+        # Sort blocks by category, then by name
+        sorted_blocks = sorted(
+            cls._metadata.items(), key=lambda x: (x[1].category, x[0])
+        )
+
+        for name, metadata in sorted_blocks:
+            description = metadata.description or "No description"
+
+            # Show deprecated blocks with a warning indicator in the name
+            block_name = f"⚠️ {name}" if metadata.deprecated else name
+
+            table.add_row(block_name, metadata.category, description)
+
+        console.print(table)
+
+        # Show summary
+        total_blocks = len(cls._metadata)
+        total_categories = len(cls._categories)
+        deprecated_count = sum(1 for m in cls._metadata.values() if m.deprecated)
+
+        console.print(
+            f"\n[bold]Summary:[/bold] {total_blocks} blocks across {total_categories} categories"
+        )
+        if deprecated_count > 0:
+            console.print(f"[yellow]⚠️  {deprecated_count} deprecated blocks[/yellow]")

sdg_hub/core/blocks/transform/__init__.py

@@ -0,0 +1,23 @@
+# SPDX-License-Identifier: Apache-2.0
+"""Data transformation blocks for dataset manipulation.
+
+This module provides blocks for transforming datasets including column operations,
+wide-to-long transformations, value selection, and majority value assignment.
+"""
+
+# Local
+from .duplicate_columns import DuplicateColumnsBlock
+from .index_based_mapper import IndexBasedMapperBlock
+from .melt_columns import MeltColumnsBlock
+from .rename_columns import RenameColumnsBlock
+from .text_concat import TextConcatBlock
+from .uniform_col_val_setter import UniformColumnValueSetter
+
+__all__ = [
+    "TextConcatBlock",
+    "DuplicateColumnsBlock",
+    "MeltColumnsBlock",
+    "IndexBasedMapperBlock",
+    "RenameColumnsBlock",
+    "UniformColumnValueSetter",
+]

sdg_hub/core/blocks/transform/duplicate_columns.py

@@ -0,0 +1,88 @@
+# SPDX-License-Identifier: Apache-2.0
+"""Duplicate columns block for dataset column duplication operations.
+
+This module provides a block for duplicating existing columns with new names
+according to a mapping specification.
+"""
+
+# Standard
+from typing import Any
+
+# Third Party
+from datasets import Dataset
+from pydantic import field_validator
+
+# Local
+from ...utils.logger_config import setup_logger
+from ..base import BaseBlock
+from ..registry import BlockRegistry
+
+logger = setup_logger(__name__)
+
+
+@BlockRegistry.register(
+    "DuplicateColumnsBlock",
+    "transform",
+    "Duplicates existing columns with new names according to a mapping specification",
+)
+class DuplicateColumnsBlock(BaseBlock):
+    """Block for duplicating existing columns with new names.
+
+    This block creates copies of existing columns with new names according to a mapping specification.
+    The mapping is provided through input_cols as a dictionary.
+
+    Attributes
+    ----------
+    block_name : str
+        Name of the block.
+    input_cols : Dict[str, str]
+        Dictionary mapping existing column names to new column names.
+        Keys are existing column names, values are new column names.
+    """
+
+    @field_validator("input_cols", mode="after")
+    @classmethod
+    def validate_input_cols(cls, v):
+        """Validate that input_cols is a non-empty dict."""
+        if not v:
+            raise ValueError("input_cols cannot be empty")
+        if not isinstance(v, dict):
+            raise ValueError(
+                "input_cols must be a dictionary mapping existing column names to new column names"
+            )
+        return v
+
+    def model_post_init(self, __context: Any) -> None:
+        """Initialize derived attributes after Pydantic validation."""
+        super().model_post_init(__context) if hasattr(
+            super(), "model_post_init"
+        ) else None
+
+        # Set output_cols to the new column names being created
+        if self.output_cols is None:
+            self.output_cols = list(self.input_cols.values())
+
+    def generate(self, samples: Dataset, **kwargs: Any) -> Dataset:
+        """Generate a dataset with duplicated columns.
+
+        Parameters
+        ----------
+        samples : Dataset
+            Input dataset to duplicate columns from.
+
+        Returns
+        -------
+        Dataset
+            Dataset with additional duplicated columns.
+        """
+        # Create a copy to avoid modifying the original
+        result = samples
+
+        # Duplicate each column as specified in the mapping
+        for source_col, target_col in self.input_cols.items():
+            if source_col not in result.column_names:
+                raise ValueError(f"Source column '{source_col}' not found in dataset")
+
+            result = result.add_column(target_col, result[source_col])
+
+        return result

sdg_hub/core/blocks/transform/index_based_mapper.py

@@ -0,0 +1,225 @@
+# SPDX-License-Identifier: Apache-2.0
+"""Selector block for column value selection and mapping.
+
+This module provides a block for selecting and mapping values from one column
+to another based on a choice column's value.
+"""
+
+# Standard
+from typing import Any
+
+# Third Party
+from datasets import Dataset
+from pydantic import Field, field_validator, model_validator
+
+# Local
+from ...utils.error_handling import MissingColumnError
+from ...utils.logger_config import setup_logger
+from ..base import BaseBlock
+from ..registry import BlockRegistry
+
+logger = setup_logger(__name__)
+
+
+@BlockRegistry.register(
+    "IndexBasedMapperBlock",
+    "transform",
+    "Maps values from source columns to output columns based on choice columns using shared mapping",
+)
+class IndexBasedMapperBlock(BaseBlock):
+    """Block for mapping values from source columns to output columns based on choice columns.
+
+    This block uses a shared mapping dictionary to select values from source columns and
+    store them in output columns based on corresponding choice columns' values.
+    The choice_cols and output_cols must have the same length - choice_cols[i] determines
+    the value for output_cols[i].
+
+    Attributes
+    ----------
+    block_name : str
+        Name of the block.
+    input_cols : Union[str, List[str], Dict[str, Any], None]
+        Input column specification. Should include choice columns and mapped columns.
+    output_cols : Union[str, List[str], Dict[str, Any], None]
+        Output column specification. Must have same length as choice_cols.
+    choice_map : Dict[str, str]
+        Dictionary mapping choice values to source column names.
+    choice_cols : List[str]
+        List of column names containing choice values. Must have same length as output_cols.
+    """
+
+    choice_map: dict[str, str] = Field(
+        ..., description="Dictionary mapping choice values to column names"
+    )
+    choice_cols: list[str] = Field(
+        ..., description="List of column names containing choice values"
+    )
+
+    @field_validator("choice_map")
+    @classmethod
+    def validate_choice_map(cls, v):
+        """Validate that choice_map is not empty."""
+        if not v:
+            raise ValueError("choice_map cannot be empty")
+        return v
+
+    @field_validator("choice_cols")
+    @classmethod
+    def validate_choice_cols_not_empty(cls, v):
+        """Validate that choice_cols is not empty."""
+        if not v:
+            raise ValueError("choice_cols cannot be empty")
+        return v
+
+    @model_validator(mode="after")
+    def validate_input_output_consistency(self):
+        """Validate that choice_cols and output_cols have same length and consistency."""
+        # Validate equal lengths
+        if len(self.choice_cols) != len(self.output_cols):
+            raise ValueError(
+                f"choice_cols and output_cols must have same length. "
+                f"Got choice_cols: {len(self.choice_cols)}, output_cols: {len(self.output_cols)}"
+            )
+
+        if isinstance(self.input_cols, list):
+            # Check that all choice_cols are in input_cols
+            missing_choice_cols = set(self.choice_cols) - set(self.input_cols)
+            if missing_choice_cols:
+                logger.warning(
+                    f"Choice columns {missing_choice_cols} not found in input_cols {self.input_cols}"
+                )
+
+            # Check that all mapped columns are in input_cols
+            missing_mapped_cols = set(self.choice_map.values()) - set(self.input_cols)
+            if missing_mapped_cols:
+                logger.warning(
+                    f"Mapped columns {missing_mapped_cols} not found in input_cols {self.input_cols}"
+                )
+
+        return self
+
+    def model_post_init(self, __context: Any) -> None:
+        """Initialize derived attributes after Pydantic validation."""
+        # Create mapping from choice_col to output_col for easy access
+        self.choice_to_output_map = dict(zip(self.choice_cols, self.output_cols))
+
+    def _validate_custom(self, samples: Dataset) -> None:
+        """Validate that required columns exist in the dataset.
+
+        Parameters
+        ----------
+        samples : Dataset
+            Input dataset to validate.
+
+        Raises
+        ------
+        MissingColumnError
+            If required columns are missing from the dataset.
+        ValueError
+            If choice values in data are not found in choice_map.
+        """
+        # Check that all choice_cols exist
+        missing_choice_cols = [
+            col for col in self.choice_cols if col not in samples.column_names
+        ]
+        if missing_choice_cols:
+            raise MissingColumnError(
+                block_name=self.block_name,
+                missing_columns=missing_choice_cols,
+                available_columns=samples.column_names,
+            )
+
+        # Check that all mapped columns exist
+        mapped_cols = list(self.choice_map.values())
+        missing_cols = list(set(mapped_cols) - set(samples.column_names))
+        if missing_cols:
+            raise MissingColumnError(
+                block_name=self.block_name,
+                missing_columns=missing_cols,
+                available_columns=samples.column_names,
+            )
+
+        # Check that all choice values in all choice columns have corresponding mappings
+        all_unique_choices = set()
+        for choice_col in self.choice_cols:
+            all_unique_choices.update(samples[choice_col])
+
+        mapped_choices = set(self.choice_map.keys())
+        unmapped_choices = all_unique_choices - mapped_choices
+
+        if unmapped_choices:
+            raise ValueError(
+                f"Choice values {sorted(unmapped_choices)} not found in choice_map for block '{self.block_name}'. "
+                f"Available choices in mapping: {sorted(mapped_choices)}"
+            )
+
+    def _generate(self, sample: dict[str, Any]) -> dict[str, Any]:
+        """Generate a new sample by selecting values based on choice mapping.
+
+        Parameters
+        ----------
+        sample : Dict[str, Any]
+            Input sample to process.
+
+        Returns
+        -------
+        Dict[str, Any]
+            Sample with selected values stored in corresponding output columns.
+        """
+        for choice_col, output_col in self.choice_to_output_map.items():
+            choice_value = sample[choice_col]
+            source_col = self.choice_map[
+                choice_value
+            ]  # Safe since validated in _validate_custom
+            sample[output_col] = sample[source_col]
+        return sample
+
+    def generate(self, samples: Dataset) -> Dataset:
+        """Generate a new dataset with selected values.
+
+        Parameters
+        ----------
+        samples : Dataset
+            Input dataset to process.
+
+        Returns
+        -------
+        Dataset
+            Dataset with selected values stored in output column.
+        """
+        # Log the operation
+        all_unique_choices = set()
+        for choice_col in self.choice_cols:
+            all_unique_choices.update(samples[choice_col])
+        mapped_choices = set(self.choice_map.keys())
+
+        logger.info(
+            f"Mapping values based on choice columns for block '{self.block_name}'",
+            extra={
+                "block_name": self.block_name,
+                "choice_columns": self.choice_cols,
+                "output_columns": self.output_cols,
+                "choice_mappings": len(self.choice_map),
+                "unique_choices_in_data": len(all_unique_choices),
+                "unmapped_choices": len(all_unique_choices - mapped_choices),
+            },
+        )
+
+        # Apply the mapping
+        result = samples.map(self._generate)
+
+        # Log completion
+        logger.info(
+            f"Successfully applied choice mapping for block '{self.block_name}'",
+            extra={
+                "block_name": self.block_name,
+                "rows_processed": len(result),
+                "output_columns": self.output_cols,
+                "mapping_coverage": len(mapped_choices & all_unique_choices)
+                / len(all_unique_choices)
+                if all_unique_choices
+                else 0,
+            },
+        )
+
+        return result