sdg-hub 0.1.4__py3-none-any.whl → 0.2.1__py3-none-any.whl

sdg_hub/core/blocks/transform/__init__.py

@@ -0,0 +1,23 @@
+# SPDX-License-Identifier: Apache-2.0
+"""Data transformation blocks for dataset manipulation.
+
+This module provides blocks for transforming datasets including column operations,
+wide-to-long transformations, value selection, and majority value assignment.
+"""
+
+# Local
+from .duplicate_columns import DuplicateColumnsBlock
+from .index_based_mapper import IndexBasedMapperBlock
+from .melt_columns import MeltColumnsBlock
+from .rename_columns import RenameColumnsBlock
+from .text_concat import TextConcatBlock
+from .uniform_col_val_setter import UniformColumnValueSetter
+
+__all__ = [
+    "TextConcatBlock",
+    "DuplicateColumnsBlock",
+    "MeltColumnsBlock",
+    "IndexBasedMapperBlock",
+    "RenameColumnsBlock",
+    "UniformColumnValueSetter",
+]

sdg_hub/core/blocks/transform/duplicate_columns.py

@@ -0,0 +1,88 @@
+# SPDX-License-Identifier: Apache-2.0
+"""Duplicate columns block for dataset column duplication operations.
+
+This module provides a block for duplicating existing columns with new names
+according to a mapping specification.
+"""
+
+# Standard
+from typing import Any
+
+# Third Party
+from datasets import Dataset
+from pydantic import field_validator
+
+# Local
+from ...utils.logger_config import setup_logger
+from ..base import BaseBlock
+from ..registry import BlockRegistry
+
+logger = setup_logger(__name__)
+
+
+@BlockRegistry.register(
+    "DuplicateColumnsBlock",
+    "transform",
+    "Duplicates existing columns with new names according to a mapping specification",
+)
+class DuplicateColumnsBlock(BaseBlock):
+    """Block for duplicating existing columns with new names.
+
+    This block creates copies of existing columns with new names according to a mapping specification.
+    The mapping is provided through input_cols as a dictionary.
+
+    Attributes
+    ----------
+    block_name : str
+        Name of the block.
+    input_cols : Dict[str, str]
+        Dictionary mapping existing column names to new column names.
+        Keys are existing column names, values are new column names.
+    """
+
+    @field_validator("input_cols", mode="after")
+    @classmethod
+    def validate_input_cols(cls, v):
+        """Validate that input_cols is a non-empty dict."""
+        if not v:
+            raise ValueError("input_cols cannot be empty")
+        if not isinstance(v, dict):
+            raise ValueError(
+                "input_cols must be a dictionary mapping existing column names to new column names"
+            )
+        return v
+
+    def model_post_init(self, __context: Any) -> None:
+        """Initialize derived attributes after Pydantic validation."""
+        super().model_post_init(__context) if hasattr(
+            super(), "model_post_init"
+        ) else None
+
+        # Set output_cols to the new column names being created
+        if self.output_cols is None:
+            self.output_cols = list(self.input_cols.values())
+
+    def generate(self, samples: Dataset, **kwargs: Any) -> Dataset:
+        """Generate a dataset with duplicated columns.
+
+        Parameters
+        ----------
+        samples : Dataset
+            Input dataset to duplicate columns from.
+
+        Returns
+        -------
+        Dataset
+            Dataset with additional duplicated columns.
+        """
+        # Create a copy to avoid modifying the original
+        result = samples
+
+        # Duplicate each column as specified in the mapping
+        for source_col, target_col in self.input_cols.items():
+            if source_col not in result.column_names:
+                raise ValueError(f"Source column '{source_col}' not found in dataset")
+
+            result = result.add_column(target_col, result[source_col])
+
+        return result

sdg_hub/core/blocks/transform/index_based_mapper.py

@@ -0,0 +1,225 @@
+# SPDX-License-Identifier: Apache-2.0
+"""Selector block for column value selection and mapping.
+
+This module provides a block for selecting and mapping values from one column
+to another based on a choice column's value.
+"""
+
+# Standard
+from typing import Any
+
+# Third Party
+from datasets import Dataset
+from pydantic import Field, field_validator, model_validator
+
+# Local
+from ...utils.error_handling import MissingColumnError
+from ...utils.logger_config import setup_logger
+from ..base import BaseBlock
+from ..registry import BlockRegistry
+
+logger = setup_logger(__name__)
+
+
+@BlockRegistry.register(
+    "IndexBasedMapperBlock",
+    "transform",
+    "Maps values from source columns to output columns based on choice columns using shared mapping",
+)
+class IndexBasedMapperBlock(BaseBlock):
+    """Block for mapping values from source columns to output columns based on choice columns.
+
+    This block uses a shared mapping dictionary to select values from source columns and
+    store them in output columns based on corresponding choice columns' values.
+    The choice_cols and output_cols must have the same length - choice_cols[i] determines
+    the value for output_cols[i].
+
+    Attributes
+    ----------
+    block_name : str
+        Name of the block.
+    input_cols : Union[str, List[str], Dict[str, Any], None]
+        Input column specification. Should include choice columns and mapped columns.
+    output_cols : Union[str, List[str], Dict[str, Any], None]
+        Output column specification. Must have same length as choice_cols.
+    choice_map : Dict[str, str]
+        Dictionary mapping choice values to source column names.
+    choice_cols : List[str]
+        List of column names containing choice values. Must have same length as output_cols.
+    """
+
+    choice_map: dict[str, str] = Field(
+        ..., description="Dictionary mapping choice values to column names"
+    )
+    choice_cols: list[str] = Field(
+        ..., description="List of column names containing choice values"
+    )
+
+    @field_validator("choice_map")
+    @classmethod
+    def validate_choice_map(cls, v):
+        """Validate that choice_map is not empty."""
+        if not v:
+            raise ValueError("choice_map cannot be empty")
+        return v
+
+    @field_validator("choice_cols")
+    @classmethod
+    def validate_choice_cols_not_empty(cls, v):
+        """Validate that choice_cols is not empty."""
+        if not v:
+            raise ValueError("choice_cols cannot be empty")
+        return v
+
+    @model_validator(mode="after")
+    def validate_input_output_consistency(self):
+        """Validate that choice_cols and output_cols have same length and consistency."""
+        # Validate equal lengths
+        if len(self.choice_cols) != len(self.output_cols):
+            raise ValueError(
+                f"choice_cols and output_cols must have same length. "
+                f"Got choice_cols: {len(self.choice_cols)}, output_cols: {len(self.output_cols)}"
+            )
+
+        if isinstance(self.input_cols, list):
+            # Check that all choice_cols are in input_cols
+            missing_choice_cols = set(self.choice_cols) - set(self.input_cols)
+            if missing_choice_cols:
+                logger.warning(
+                    f"Choice columns {missing_choice_cols} not found in input_cols {self.input_cols}"
+                )
+
+            # Check that all mapped columns are in input_cols
+            missing_mapped_cols = set(self.choice_map.values()) - set(self.input_cols)
+            if missing_mapped_cols:
+                logger.warning(
+                    f"Mapped columns {missing_mapped_cols} not found in input_cols {self.input_cols}"
+                )
+
+        return self
+
+    def model_post_init(self, __context: Any) -> None:
+        """Initialize derived attributes after Pydantic validation."""
+        # Create mapping from choice_col to output_col for easy access
+        self.choice_to_output_map = dict(zip(self.choice_cols, self.output_cols))
+
+    def _validate_custom(self, samples: Dataset) -> None:
+        """Validate that required columns exist in the dataset.
+
+        Parameters
+        ----------
+        samples : Dataset
+            Input dataset to validate.
+
+        Raises
+        ------
+        MissingColumnError
+            If required columns are missing from the dataset.
+        ValueError
+            If choice values in data are not found in choice_map.
+        """
+        # Check that all choice_cols exist
+        missing_choice_cols = [
+            col for col in self.choice_cols if col not in samples.column_names
+        ]
+        if missing_choice_cols:
+            raise MissingColumnError(
+                block_name=self.block_name,
+                missing_columns=missing_choice_cols,
+                available_columns=samples.column_names,
+            )
+
+        # Check that all mapped columns exist
+        mapped_cols = list(self.choice_map.values())
+        missing_cols = list(set(mapped_cols) - set(samples.column_names))
+        if missing_cols:
+            raise MissingColumnError(
+                block_name=self.block_name,
+                missing_columns=missing_cols,
+                available_columns=samples.column_names,
+            )
+
+        # Check that all choice values in all choice columns have corresponding mappings
+        all_unique_choices = set()
+        for choice_col in self.choice_cols:
+            all_unique_choices.update(samples[choice_col])
+
+        mapped_choices = set(self.choice_map.keys())
+        unmapped_choices = all_unique_choices - mapped_choices
+
+        if unmapped_choices:
+            raise ValueError(
+                f"Choice values {sorted(unmapped_choices)} not found in choice_map for block '{self.block_name}'. "
+                f"Available choices in mapping: {sorted(mapped_choices)}"
+            )
+
+    def _generate(self, sample: dict[str, Any]) -> dict[str, Any]:
+        """Generate a new sample by selecting values based on choice mapping.
+
+        Parameters
+        ----------
+        sample : Dict[str, Any]
+            Input sample to process.
+
+        Returns
+        -------
+        Dict[str, Any]
+            Sample with selected values stored in corresponding output columns.
+        """
+        for choice_col, output_col in self.choice_to_output_map.items():
+            choice_value = sample[choice_col]
+            source_col = self.choice_map[
+                choice_value
+            ]  # Safe since validated in _validate_custom
+            sample[output_col] = sample[source_col]
+        return sample
+
+    def generate(self, samples: Dataset) -> Dataset:
+        """Generate a new dataset with selected values.
+
+        Parameters
+        ----------
+        samples : Dataset
+            Input dataset to process.
+
+        Returns
+        -------
+        Dataset
+            Dataset with selected values stored in output column.
+        """
+        # Log the operation
+        all_unique_choices = set()
+        for choice_col in self.choice_cols:
+            all_unique_choices.update(samples[choice_col])
+        mapped_choices = set(self.choice_map.keys())
+
+        logger.info(
+            f"Mapping values based on choice columns for block '{self.block_name}'",
+            extra={
+                "block_name": self.block_name,
+                "choice_columns": self.choice_cols,
+                "output_columns": self.output_cols,
+                "choice_mappings": len(self.choice_map),
+                "unique_choices_in_data": len(all_unique_choices),
+                "unmapped_choices": len(all_unique_choices - mapped_choices),
+            },
+        )
+
+        # Apply the mapping
+        result = samples.map(self._generate)
+
+        # Log completion
+        logger.info(
+            f"Successfully applied choice mapping for block '{self.block_name}'",
+            extra={
+                "block_name": self.block_name,
+                "rows_processed": len(result),
+                "output_columns": self.output_cols,
+                "mapping_coverage": len(mapped_choices & all_unique_choices)
+                / len(all_unique_choices)
+                if all_unique_choices
+                else 0,
+            },
+        )
+
+        return result

sdg_hub/core/blocks/transform/melt_columns.py

@@ -0,0 +1,126 @@
+# SPDX-License-Identifier: Apache-2.0
+"""Melt columns block for wide-to-long format transformation.
+
+This module provides a block for transforming wide dataset format into long format
+by melting specified columns into rows.
+"""
+
+# Standard
+from typing import Any
+
+# Third Party
+from datasets import Dataset
+from pydantic import field_validator
+
+# Local
+from ...utils.error_handling import MissingColumnError
+from ...utils.logger_config import setup_logger
+from ..base import BaseBlock
+from ..registry import BlockRegistry
+
+logger = setup_logger(__name__)
+
+
+@BlockRegistry.register(
+    "MeltColumnsBlock",
+    "transform",
+    "Transforms wide dataset format into long format by melting columns into rows",
+)
+class MeltColumnsBlock(BaseBlock):
+    """Block for flattening multiple columns into a long format.
+
+    This block transforms a wide dataset format into a long format by melting
+    specified columns into rows, creating new variable and value columns.
+
+    The input_cols should contain the columns to be melted (variable columns).
+    The output_cols must specify exactly two columns: [value_column, variable_column].
+    Any other columns in the dataset will be treated as ID columns and preserved.
+
+    Attributes
+    ----------
+    block_name : str
+        Name of the block.
+    input_cols : Union[str, List[str], Dict[str, Any], None]
+        Columns to be melted into rows (variable columns).
+    output_cols : Union[str, List[str], Dict[str, Any], None]
+        Output column specification. Must specify exactly two columns: [value_column, variable_column].
+    """
+
+    @field_validator("input_cols", mode="after")
+    @classmethod
+    def validate_input_cols(cls, v):
+        """Validate that input_cols is not empty."""
+        if not v:
+            raise ValueError("input_cols cannot be empty")
+        return v
+
+    @field_validator("output_cols", mode="after")
+    @classmethod
+    def validate_output_cols(cls, v):
+        """Validate that exactly two output columns are specified."""
+        if len(v) != 2:
+            raise ValueError(
+                f"MeltColumnsBlock expects exactly two output columns (value, variable), got {len(v)}: {v}"
+            )
+        return v
+
+    def model_post_init(self, __context: Any) -> None:
+        """Initialize derived attributes after Pydantic validation."""
+        super().model_post_init(__context) if hasattr(
+            super(), "model_post_init"
+        ) else None
+
+        # Derive value and variable column names from output_cols
+        self.value_name = self.output_cols[0]  # First output column is value
+        self.var_name = self.output_cols[1]  # Second output column is variable
+
+        # input_cols contains the columns to be melted (what was var_cols)
+        self.var_cols = (
+            self.input_cols if isinstance(self.input_cols, list) else [self.input_cols]
+        )
+
+    def _validate_custom(self, samples: Dataset) -> None:
+        """Validate that required columns exist in the dataset.
+
+        Parameters
+        ----------
+        samples : Dataset
+            Input dataset to validate.
+
+        Raises
+        ------
+        MissingColumnError
+            If required columns are missing from the dataset.
+        """
+        # Check that all var_cols exist in the dataset
+        missing_cols = list(set(self.var_cols) - set(samples.column_names))
+        if missing_cols:
+            raise MissingColumnError(
+                block_name=self.block_name,
+                missing_columns=missing_cols,
+                available_columns=samples.column_names,
+            )
+
+    def generate(self, samples: Dataset, **kwargs: Any) -> Dataset:
+        """Generate a flattened dataset in long format.
+
+        Parameters
+        ----------
+        samples : Dataset
+            Input dataset to flatten.
+
+        Returns
+        -------
+        Dataset
+            Flattened dataset in long format with new variable and value columns.
+        """
+        # Use the original simple logic - just adapted to use derived attributes
+        df = samples.to_pandas()
+        id_cols = [col for col in samples.column_names if col not in self.var_cols]
+        flatten_df = df.melt(
+            id_vars=id_cols,
+            value_vars=self.var_cols,
+            value_name=self.value_name,
+            var_name=self.var_name,
+        )
+        return Dataset.from_pandas(flatten_df)

sdg_hub/core/blocks/transform/rename_columns.py

@@ -0,0 +1,69 @@
+# SPDX-License-Identifier: Apache-2.0
+"""Rename columns block for dataset column renaming operations.
+
+This module provides a block for renaming columns in datasets according
+to a mapping specification.
+"""
+
+# Standard
+from typing import Any
+
+# Third Party
+from datasets import Dataset
+from pydantic import field_validator
+
+# Local
+from ...utils.logger_config import setup_logger
+from ..base import BaseBlock
+from ..registry import BlockRegistry
+
+logger = setup_logger(__name__)
+
+
+@BlockRegistry.register(
+    "RenameColumnsBlock",
+    "transform",
+    "Renames columns in a dataset according to a mapping specification",
+)
+class RenameColumnsBlock(BaseBlock):
+    """Block for renaming columns in a dataset.
+
+    This block renames columns in a dataset according to a mapping specification.
+    The mapping is provided through input_cols as a dictionary.
+
+    Attributes
+    ----------
+    block_name : str
+        Name of the block.
+    input_cols : Dict[str, str]
+        Dictionary mapping existing column names to new column names.
+        Keys are existing column names, values are new column names.
+    """
+
+    @field_validator("input_cols", mode="after")
+    @classmethod
+    def validate_input_cols(cls, v):
+        """Validate that input_cols is a non-empty dict."""
+        if not v:
+            raise ValueError("input_cols cannot be empty")
+        if not isinstance(v, dict):
+            raise ValueError(
+                "input_cols must be a dictionary mapping old column names to new column names"
+            )
+        return v
+
+    def generate(self, samples: Dataset, **kwargs: Any) -> Dataset:
+        """Generate a dataset with renamed columns.
+
+        Parameters
+        ----------
+        samples : Dataset
+            Input dataset to rename columns in.
+
+        Returns
+        -------
+        Dataset
+            Dataset with renamed columns.
+        """
+        # Rename columns using HuggingFace datasets method
+        return samples.rename_columns(self.input_cols)

sdg_hub/core/blocks/transform/text_concat.py

@@ -0,0 +1,102 @@
+# SPDX-License-Identifier: Apache-2.0
+"""Text concatenation block for dataset column combination operations.
+
+This module provides a block for combining multiple columns into a single column
+using a specified separator.
+"""
+
+# Standard
+from typing import Any
+
+# Third Party
+from datasets import Dataset
+from pydantic import Field, field_validator
+
+# Local
+from ...utils.logger_config import setup_logger
+from ..base import BaseBlock
+from ..registry import BlockRegistry
+
+logger = setup_logger(__name__)
+
+
+@BlockRegistry.register(
+    "TextConcatBlock",
+    "transform",
+    "Combines multiple columns into a single column using a specified separator",
+)
+class TextConcatBlock(BaseBlock):
+    """Block for combining multiple columns into a single column.
+
+    This block concatenates values from multiple columns into a single output column,
+    using a specified separator between values.
+
+    Attributes
+    ----------
+    block_name : str
+        Name of the block.
+    input_cols : list[str]
+        List of column names to combine.
+    output_cols : list[str]
+        List containing the single output column name.
+    separator : str
+        String to use as separator between combined values.
+    """
+
+    separator: str = Field(
+        default="\n\n", description="Separator to use between combined values"
+    )
+
+    @field_validator("input_cols", mode="after")
+    @classmethod
+    def validate_input_cols(cls, v):
+        """Validate that input_cols is a non-empty list."""
+        if not v:
+            raise ValueError("input_cols cannot be empty")
+        if not isinstance(v, list):
+            raise ValueError("input_cols must be a list of column names")
+        return v
+
+    @field_validator("output_cols", mode="after")
+    @classmethod
+    def validate_output_cols(cls, v):
+        """Validate that exactly one output column is specified."""
+        if not v or len(v) != 1:
+            raise ValueError("TextConcatBlock requires exactly one output column")
+        return v
+
+    def generate(self, samples: Dataset, **kwargs: Any) -> Dataset:
+        """Generate a dataset with combined columns.
+
+        Parameters
+        ----------
+        samples : Dataset
+            Input dataset to process.
+
+        Returns
+        -------
+        Dataset
+            Dataset with combined values stored in output column.
+        """
+        if not self.output_cols:
+            raise ValueError("output_cols must be specified")
+
+        output_col = self.output_cols[0]
+
+        def _combine_columns(sample):
+            """Combine values from input columns."""
+            # Check that all input columns exist
+            for col in self.input_cols:
+                if col not in sample:
+                    raise ValueError(f"Input column '{col}' not found in sample")
+
+            # Combine values using separator
+            combined_value = self.separator.join(
+                [str(sample[col]) for col in self.input_cols]
+            )
+            sample[output_col] = combined_value
+            return sample
+
+        # Apply the combination to all samples
+        result = samples.map(_combine_columns)
+        return result