sdg-hub 0.1.4__py3-none-any.whl → 0.2.0__py3-none-any.whl

sdg_hub/core/blocks/transform/melt_columns.py

@@ -0,0 +1,126 @@
+# SPDX-License-Identifier: Apache-2.0
+"""Melt columns block for wide-to-long format transformation.
+
+This module provides a block for transforming wide dataset format into long format
+by melting specified columns into rows.
+"""
+
+# Standard
+from typing import Any
+
+# Third Party
+from datasets import Dataset
+from pydantic import field_validator
+
+# Local
+from ...utils.error_handling import MissingColumnError
+from ...utils.logger_config import setup_logger
+from ..base import BaseBlock
+from ..registry import BlockRegistry
+
+logger = setup_logger(__name__)
+
+
+@BlockRegistry.register(
+    "MeltColumnsBlock",
+    "transform",
+    "Transforms wide dataset format into long format by melting columns into rows",
+)
+class MeltColumnsBlock(BaseBlock):
+    """Block for flattening multiple columns into a long format.
+
+    This block transforms a wide dataset format into a long format by melting
+    specified columns into rows, creating new variable and value columns.
+
+    The input_cols should contain the columns to be melted (variable columns).
+    The output_cols must specify exactly two columns: [value_column, variable_column].
+    Any other columns in the dataset will be treated as ID columns and preserved.
+
+    Attributes
+    ----------
+    block_name : str
+        Name of the block.
+    input_cols : Union[str, List[str], Dict[str, Any], None]
+        Columns to be melted into rows (variable columns).
+    output_cols : Union[str, List[str], Dict[str, Any], None]
+        Output column specification. Must specify exactly two columns: [value_column, variable_column].
+    """
+
+    @field_validator("input_cols", mode="after")
+    @classmethod
+    def validate_input_cols(cls, v):
+        """Validate that input_cols is not empty."""
+        if not v:
+            raise ValueError("input_cols cannot be empty")
+        return v
+
+    @field_validator("output_cols", mode="after")
+    @classmethod
+    def validate_output_cols(cls, v):
+        """Validate that exactly two output columns are specified."""
+        if len(v) != 2:
+            raise ValueError(
+                f"MeltColumnsBlock expects exactly two output columns (value, variable), got {len(v)}: {v}"
+            )
+        return v
+
+    def model_post_init(self, __context: Any) -> None:
+        """Initialize derived attributes after Pydantic validation."""
+        super().model_post_init(__context) if hasattr(
+            super(), "model_post_init"
+        ) else None
+
+        # Derive value and variable column names from output_cols
+        self.value_name = self.output_cols[0]  # First output column is value
+        self.var_name = self.output_cols[1]  # Second output column is variable
+
+        # input_cols contains the columns to be melted (what was var_cols)
+        self.var_cols = (
+            self.input_cols if isinstance(self.input_cols, list) else [self.input_cols]
+        )
+
+    def _validate_custom(self, samples: Dataset) -> None:
+        """Validate that required columns exist in the dataset.
+
+        Parameters
+        ----------
+        samples : Dataset
+            Input dataset to validate.
+
+        Raises
+        ------
+        MissingColumnError
+            If required columns are missing from the dataset.
+        """
+        # Check that all var_cols exist in the dataset
+        missing_cols = list(set(self.var_cols) - set(samples.column_names))
+        if missing_cols:
+            raise MissingColumnError(
+                block_name=self.block_name,
+                missing_columns=missing_cols,
+                available_columns=samples.column_names,
+            )
+
+    def generate(self, samples: Dataset, **kwargs: Any) -> Dataset:
+        """Generate a flattened dataset in long format.
+
+        Parameters
+        ----------
+        samples : Dataset
+            Input dataset to flatten.
+
+        Returns
+        -------
+        Dataset
+            Flattened dataset in long format with new variable and value columns.
+        """
+        # Use the original simple logic - just adapted to use derived attributes
+        df = samples.to_pandas()
+        id_cols = [col for col in samples.column_names if col not in self.var_cols]
+        flatten_df = df.melt(
+            id_vars=id_cols,
+            value_vars=self.var_cols,
+            value_name=self.value_name,
+            var_name=self.var_name,
+        )
+        return Dataset.from_pandas(flatten_df)

sdg_hub/core/blocks/transform/rename_columns.py

@@ -0,0 +1,69 @@
+# SPDX-License-Identifier: Apache-2.0
+"""Rename columns block for dataset column renaming operations.
+
+This module provides a block for renaming columns in datasets according
+to a mapping specification.
+"""
+
+# Standard
+from typing import Any
+
+# Third Party
+from datasets import Dataset
+from pydantic import field_validator
+
+# Local
+from ...utils.logger_config import setup_logger
+from ..base import BaseBlock
+from ..registry import BlockRegistry
+
+logger = setup_logger(__name__)
+
+
+@BlockRegistry.register(
+    "RenameColumnsBlock",
+    "transform",
+    "Renames columns in a dataset according to a mapping specification",
+)
+class RenameColumnsBlock(BaseBlock):
+    """Block for renaming columns in a dataset.
+
+    This block renames columns in a dataset according to a mapping specification.
+    The mapping is provided through input_cols as a dictionary.
+
+    Attributes
+    ----------
+    block_name : str
+        Name of the block.
+    input_cols : Dict[str, str]
+        Dictionary mapping existing column names to new column names.
+        Keys are existing column names, values are new column names.
+    """
+
+    @field_validator("input_cols", mode="after")
+    @classmethod
+    def validate_input_cols(cls, v):
+        """Validate that input_cols is a non-empty dict."""
+        if not v:
+            raise ValueError("input_cols cannot be empty")
+        if not isinstance(v, dict):
+            raise ValueError(
+                "input_cols must be a dictionary mapping old column names to new column names"
+            )
+        return v
+
+    def generate(self, samples: Dataset, **kwargs: Any) -> Dataset:
+        """Generate a dataset with renamed columns.
+
+        Parameters
+        ----------
+        samples : Dataset
+            Input dataset to rename columns in.
+
+        Returns
+        -------
+        Dataset
+            Dataset with renamed columns.
+        """
+        # Rename columns using HuggingFace datasets method
+        return samples.rename_columns(self.input_cols)

sdg_hub/core/blocks/transform/text_concat.py

@@ -0,0 +1,102 @@
+# SPDX-License-Identifier: Apache-2.0
+"""Text concatenation block for dataset column combination operations.
+
+This module provides a block for combining multiple columns into a single column
+using a specified separator.
+"""
+
+# Standard
+from typing import Any
+
+# Third Party
+from datasets import Dataset
+from pydantic import Field, field_validator
+
+# Local
+from ...utils.logger_config import setup_logger
+from ..base import BaseBlock
+from ..registry import BlockRegistry
+
+logger = setup_logger(__name__)
+
+
+@BlockRegistry.register(
+    "TextConcatBlock",
+    "transform",
+    "Combines multiple columns into a single column using a specified separator",
+)
+class TextConcatBlock(BaseBlock):
+    """Block for combining multiple columns into a single column.
+
+    This block concatenates values from multiple columns into a single output column,
+    using a specified separator between values.
+
+    Attributes
+    ----------
+    block_name : str
+        Name of the block.
+    input_cols : list[str]
+        List of column names to combine.
+    output_cols : list[str]
+        List containing the single output column name.
+    separator : str
+        String to use as separator between combined values.
+    """
+
+    separator: str = Field(
+        default="\n\n", description="Separator to use between combined values"
+    )
+
+    @field_validator("input_cols", mode="after")
+    @classmethod
+    def validate_input_cols(cls, v):
+        """Validate that input_cols is a non-empty list."""
+        if not v:
+            raise ValueError("input_cols cannot be empty")
+        if not isinstance(v, list):
+            raise ValueError("input_cols must be a list of column names")
+        return v
+
+    @field_validator("output_cols", mode="after")
+    @classmethod
+    def validate_output_cols(cls, v):
+        """Validate that exactly one output column is specified."""
+        if not v or len(v) != 1:
+            raise ValueError("TextConcatBlock requires exactly one output column")
+        return v
+
+    def generate(self, samples: Dataset, **kwargs: Any) -> Dataset:
+        """Generate a dataset with combined columns.
+
+        Parameters
+        ----------
+        samples : Dataset
+            Input dataset to process.
+
+        Returns
+        -------
+        Dataset
+            Dataset with combined values stored in output column.
+        """
+        if not self.output_cols:
+            raise ValueError("output_cols must be specified")
+
+        output_col = self.output_cols[0]
+
+        def _combine_columns(sample):
+            """Combine values from input columns."""
+            # Check that all input columns exist
+            for col in self.input_cols:
+                if col not in sample:
+                    raise ValueError(f"Input column '{col}' not found in sample")
+
+            # Combine values using separator
+            combined_value = self.separator.join(
+                [str(sample[col]) for col in self.input_cols]
+            )
+            sample[output_col] = combined_value
+            return sample
+
+        # Apply the combination to all samples
+        result = samples.map(_combine_columns)
+        return result

sdg_hub/core/blocks/transform/uniform_col_val_setter.py

@@ -0,0 +1,101 @@
+# SPDX-License-Identifier: Apache-2.0
+"""Uniform column value setter block for replacing a column with a single statistic.
+
+This block sets all values in a column to a single summary statistic:
+mode, min, max, mean, or median.
+"""
+
+# Standard
+from typing import Any, Literal
+
+# Third Party
+from datasets import Dataset
+from pydantic import field_validator
+import numpy as np
+
+# Local
+from ...utils.logger_config import setup_logger
+from ..base import BaseBlock
+from ..registry import BlockRegistry
+
+logger = setup_logger(__name__)
+
+
+@BlockRegistry.register(
+    "UniformColumnValueSetter",
+    "transform",
+    "Replaces all values in a column with a single summary statistic (e.g., mode, mean, median)",
+)
+class UniformColumnValueSetter(BaseBlock):
+    """Block that replaces all values in a column with a single aggregate value.
+
+    Supported strategies include: mode, min, max, mean, median.
+
+    Attributes
+    ----------
+    block_name : str
+        Name of the block.
+    input_cols : Union[str, List[str]]
+        Must specify exactly one input column.
+    output_cols : Union[str, List[str]]
+        Output column list. Ignored — modifies in place.
+    reduction_strategy : Literal["mode", "min", "max", "mean", "median"]
+        Strategy used to compute the replacement value.
+    """
+
+    reduction_strategy: Literal["mode", "min", "max", "mean", "median"] = "mode"
+
+    @field_validator("input_cols", mode="after")
+    @classmethod
+    def validate_input_cols_single(cls, v):
+        if not v or len(v) != 1:
+            raise ValueError(
+                "UniformColumnValueSetter requires exactly one input column"
+            )
+        return v
+
+    def model_post_init(self, __context: Any) -> None:
+        if hasattr(super(), "model_post_init"):
+            super().model_post_init(__context)
+
+        if self.output_cols and len(self.output_cols) > 0:
+            logger.warning(
+                f"UniformColumnValueSetter modifies columns in-place. "
+                f"Specified output_cols {self.output_cols} will be ignored."
+            )
+        self.output_cols = []
+        self.col_name = self.input_cols[0]
+
+    def generate(self, samples: Dataset, **kwargs: Any) -> Dataset:
+        df = samples.to_pandas()
+
+        if df.empty:
+            raise ValueError("Cannot compute reduction for empty dataset")
+
+        col = df[self.col_name]
+
+        strategy = self.reduction_strategy
+        if strategy == "mode":
+            value = col.mode().iloc[0] if not col.mode().empty else None
+        elif strategy == "min":
+            value = col.min()
+        elif strategy == "max":
+            value = col.max()
+        elif strategy == "mean":
+            value = col.mean()
+        elif strategy == "median":
+            value = col.median()
+        else:
+            raise ValueError(f"Unsupported reduction strategy: {strategy}")
+
+        if value is None or (isinstance(value, float) and np.isnan(value)):
+            raise ValueError(
+                f"Could not compute {strategy} for column '{self.col_name}'"
+            )
+
+        logger.info(
+            f"Replacing all values in column '{self.col_name}' with {strategy} value: '{value}'"
+        )
+
+        df[self.col_name] = value
+        return Dataset.from_pandas(df)

sdg_hub/core/flow/__init__.py

@@ -0,0 +1,20 @@
+# SPDX-License-Identifier: Apache-2.0
+"""New flow implementation for SDG Hub.
+
+This module provides a redesigned Flow class with metadata support,
+dual initialization modes, and runtime parameter overrides.
+"""
+
+# Local
+from .base import Flow
+from .metadata import FlowMetadata, FlowParameter
+from .registry import FlowRegistry
+from .validation import FlowValidator
+
+__all__ = [
+    "Flow",
+    "FlowMetadata",
+    "FlowParameter",
+    "FlowRegistry",
+    "FlowValidator",
+]