data-engineering-exp 0.1.0__py3-none-any.whl

data_engineering_exp/cli.py

@@ -0,0 +1,63 @@
+"""Command Line Interface (CLI) entry point for dex using Click."""
+
+import os
+
+import click
+
+from data_engineering_exp.core.initializer import ProjectInitializer
+
+
+@click.group()
+def entry_point() -> None:
+    """dex - Data Engineering Experience CLI utilities.
+
+    Args:
+
+    Returns:
+        None: Main CLI entry group.
+    """
+    pass
+
+
+@entry_point.command()
+@click.option(
+    "--path",
+    type=click.Path(file_okay=False, dir_okay=True),
+    default=".",
+    help="Root directory where scaffolding will be built. Defaults to '.'",
+)
+def init(path: str) -> None:
+    """Scaffold a new data engineering project structure interactively.
+
+    Args:
+        path(str): Target directory string path for project structure.
+
+    Returns:
+        None: Side-effect creating folders and configurations.
+    """
+    click.echo("🚀 Welcome to the dex project initialization wizard!\n")
+
+    # Derives default project name dynamically from the target directory name
+    default_name = os.path.basename(os.path.abspath(path))
+    if not default_name or default_name in (".", ""):
+        default_name = "my-dex-project"
+
+    # Sequential metadata prompts with default fallbacks
+    name = click.prompt("Project name", default=default_name, type=str)
+    version = click.prompt("Version", default="0.1.0", type=str)
+    description = click.prompt(
+        "Description",
+        default="Data engineering project scaffolded by dex",
+        type=str,
+    )
+    author = click.prompt("Author", default="Anonymous", type=str)
+
+    initializer = ProjectInitializer(base_path=path)
+    initializer.init_project(
+        name=name,
+        version=version,
+        description=description,
+        author=author,
+    )
+
+    click.echo(f"\n✨ Project successfully initialized at '{path}'!")

data_engineering_exp/core/catalog.py

@@ -0,0 +1,197 @@
+"""This module implements decentralized data catalog engines for dex."""
+
+import os
+from typing import Any, Dict, List, Optional, cast
+
+import yaml
+
+
+class DataCatalog:
+    """Parses and consolidates declarative configurations from files or directories.
+
+    Scans catalog definitions recursively, allowing user architectures to be
+    split into mini-YAML files or kept within a single configuration layout.
+    """
+
+    def __init__(self, catalog_path: Optional[str] = None) -> None:
+        """Initializes the DataCatalog by scanning files or configuration paths.
+
+        If no path is provided, it automatically seeks upwards from the current
+        working directory to find the standard 'conf/catalog' directory based on
+        the pyproject.toml root anchor file.
+
+        Args:
+            catalog_path(Optional[str]): System directory path or file string
+                pointing to catalog definitions. Defaults to None.
+
+        Returns:
+            None: Initializes the class instance.
+        """
+        self.project_root: str = ""
+
+        if catalog_path is None:
+            catalog_path = self._discover_catalog_path()
+        else:
+            # Derive the root directory context if an explicit path is supplied
+            abs_path = os.path.abspath(catalog_path)
+            if os.path.isfile(abs_path):
+                self.project_root = os.path.dirname(
+                    os.path.dirname(os.path.dirname(abs_path))
+                )
+            else:
+                self.project_root = os.path.dirname(os.path.dirname(abs_path))
+
+        if not os.path.exists(catalog_path):
+            raise FileNotFoundError(f"Catalog source path not found at: {catalog_path}")
+
+        self._datasets: Dict[str, Dict[str, Any]] = {}
+        self._load_catalog_sources(catalog_path)
+
+    @property
+    def dataset_names(self) -> List[str]:
+        """Exposes the list of all dataset identifiers present in the catalog.
+
+        Args:
+
+        Returns:
+            List[str]: List of string names for all loaded datasets.
+        """
+        return list(self._datasets.keys())
+
+    def get_dataset_metadata(self, dataset_name: str) -> Dict[str, Any]:
+        """Retrieves core metadata mapping configurations for a target dataset.
+
+        Args:
+            dataset_name(str): Unique identifier string of the dataset.
+
+        Returns:
+            Dict[str, Any]: Dictionary containing properties like description,
+                format, primary_keys, and storage_path.
+
+        Raises:
+            KeyError: If the target dataset name is missing from the catalog.
+        """
+        if dataset_name not in self._datasets:
+            raise KeyError(f"Dataset '{dataset_name}' missing from catalog.")
+
+        meta = self._datasets[dataset_name].copy()
+        meta.pop("columns", None)
+        return meta
+
+    def get_column_names(self, dataset_name: str) -> List[str]:
+        """Extracts the expected list of column names for a target dataset.
+
+        Args:
+            dataset_name(str): Unique identifier string of the dataset.
+
+        Returns:
+            List[str]: Ordered list of strings representing expected columns.
+
+        Raises:
+            KeyError: If the target dataset name is missing from the catalog.
+        """
+        if dataset_name not in self._datasets:
+            raise KeyError(f"Dataset '{dataset_name}' missing from catalog.")
+
+        columns_spec = self._datasets[dataset_name].get("columns", [])
+        return [col["name"] for col in columns_spec]
+
+    def validate_schema_presence(self, df: Any, dataset_name: str) -> bool:
+        """Validates that all columns defined in the catalog exist in the dataframe.
+
+        Args:
+            df(Any): Active input Pandas or PySpark DataFrame object.
+            dataset_name(str): Catalog identifier string of the dataset target.
+
+        Returns:
+            bool: True if all expected columns are present, raises ValueError
+                otherwise.
+
+        Raises:
+            TypeError: If the input dataframe structure is not supported.
+            ValueError: If a column mismatch is discovered against the catalog.
+        """
+        expected_cols = set(self.get_column_names(dataset_name))
+
+        df_type = type(df).__name__
+        if df_type == "DataFrame" and "pandas" in type(df).__module__:
+            actual_cols = set(df.columns.tolist())
+        elif df_type == "DataFrame" and "pyspark" in type(df).__module__:
+            actual_cols = set(df.columns)
+        else:
+            raise TypeError("Unsupported DataFrame type for validation.")
+
+        missing_cols = expected_cols - actual_cols
+        if missing_cols:
+            raise ValueError(
+                f"Schema mismatch for '{dataset_name}'. "
+                f"Missing expected catalog columns: {list(missing_cols)}"
+            )
+
+        return True
+
+    def _discover_catalog_path(self) -> str:
+        """Traverses parent directories upwards to locate the pyproject.toml file.
+
+        Once the layout anchor file is found, it automatically appends the standard
+        'conf/catalog' structural workspace mapping.
+
+        Returns:
+            str: Absolute path to the discovered catalog directory.
+
+        Raises:
+            FileNotFoundError: If the pyproject.toml configuration cannot be found.
+        """
+        current_dir = os.getcwd()
+
+        while True:
+            potential_toml = os.path.join(current_dir, "pyproject.toml")
+            if os.path.exists(potential_toml):
+                self.project_root = current_dir
+                return os.path.join(current_dir, "conf", "catalog")
+
+            parent_dir = os.path.dirname(current_dir)
+            if parent_dir == current_dir:
+                break
+            current_dir = parent_dir
+
+        raise FileNotFoundError(
+            "Configuration file (pyproject.toml) could not be located. "
+            "Please run 'dex init' to establish a valid project layout."
+        )
+
+    def _load_catalog_sources(self, path: str) -> None:
+        """Internal worker to process and parse path targets recursively.
+
+        Args:
+            path(str): Targeting file path or folder configuration directory.
+
+        Returns:
+            None: Populates internal dictionary instances.
+        """
+        if os.path.isfile(path):
+            if path.endswith((".yml", ".yaml")):
+                self._parse_file(path)
+            return
+
+        for root, _, files in os.walk(path):
+            for file in files:
+                if file.endswith((".yml", ".yaml")):
+                    full_path = os.path.join(root, file)
+                    self._parse_file(full_path)
+
+    def _parse_file(self, file_path: str) -> None:
+        """Parses an individual YAML catalog file and updates target states.
+
+        Args:
+            file_path(str): Exact system string location pointing to file.
+
+        Returns:
+            None: Updates the core datasets dictionary data states.
+        """
+        with open(file_path, "r", encoding="utf-8") as stream:
+            content = yaml.safe_load(stream)
+
+        if content and isinstance(content, dict):
+            typed_content = cast(Dict[str, Dict[str, Any]], content)
+            self._datasets.update(typed_content)

data_engineering_exp/core/deduplication.py

@@ -0,0 +1,192 @@
+"""This module implements versatile data deduplication utilities for dex."""
+
+from typing import Any, List, Optional, Union
+
+# Handle optional dependencies at the top of the file
+try:
+    import pyspark.sql.functions as F
+    from pyspark.sql import Window
+
+    HAS_SPARK = True
+except ImportError:
+    Window: Any = None
+    F: Any = None
+    HAS_SPARK = False
+
+
+class Deduplicator:
+    """Utility class providing multiple strategies for data deduplication.
+
+    Supports Pandas and PySpark DataFrames dynamically, allowing extraction of
+    first, latest, multi-sorted, consolidated, or strictly unique records.
+    """
+
+    def __init__(self):
+        """Initializes the Deduplicator instance.
+
+        Args:
+
+        Returns:
+            None: Initializes the class instance.
+        """
+        pass
+
+    def latest(self, df: Any, keys: List[str], order_by_col: str) -> Any:
+        """Extracts the most recent record per business key group.
+
+        Args:
+            df(Any): Input Pandas or PySpark DataFrame.
+            keys(List[str]): Columns that identify a business entity.
+            order_by_col(str): Column used to determine chronological order.
+
+        Returns:
+            Any: Deduplicated DataFrame containing only the latest events.
+        """
+        framework = self._detect_framework(df)
+        if framework == "pandas":
+            return (
+                df.sort_values(by=order_by_col, ascending=True)
+                .groupby(keys)
+                .last()
+                .reset_index()
+            )
+
+        window_spec = Window.partitionBy(keys).orderBy(F.col(order_by_col).desc())
+        return (
+            df.withColumn("_row_num", F.row_number().over(window_spec))
+            .filter(F.col("_row_num") == 1)
+            .drop("_row_num")
+        )
+
+    def first(self, df: Any, keys: List[str], order_by_col: str) -> Any:
+        """Extracts the earliest chronological record per business key group.
+
+        Args:
+            df(Any): Input Pandas or PySpark DataFrame.
+            keys(List[str]): Columns that identify a business entity.
+            order_by_col(str): Column used to determine chronological order.
+
+        Returns:
+            Any: Deduplicated DataFrame containing only the first events.
+        """
+        framework = self._detect_framework(df)
+        if framework == "pandas":
+            return (
+                df.sort_values(by=order_by_col, ascending=True)
+                .groupby(keys)
+                .first()
+                .reset_index()
+            )
+
+        window_spec = Window.partitionBy(keys).orderBy(F.col(order_by_col).asc())
+        return (
+            df.withColumn("_row_num", F.row_number().over(window_spec))
+            .filter(F.col("_row_num") == 1)
+            .drop("_row_num")
+        )
+
+    def distinct(self, df: Any, subset: Optional[List[str]] = None) -> Any:
+        """Removes strict duplicate rows from the DataFrame.
+
+        Args:
+            df(Any): Input Pandas or PySpark DataFrame.
+            subset(Optional[List[str]]): Specific columns to consider when
+                identifying duplicates. Defaults to None (all columns).
+
+        Returns:
+            Any: Clean DataFrame with unique records only.
+        """
+        framework = self._detect_framework(df)
+        if framework == "pandas":
+            return df.drop_duplicates(subset=subset).reset_index(drop=True)
+
+        return df.dropDuplicates(subset=subset)
+
+    def by_order(
+        self,
+        df: Any,
+        keys: List[str],
+        order_by_cols: List[str],
+        ascending: Union[bool, List[bool]] = True,
+    ) -> Any:
+        """Deduplicates rows by sorting through multiple custom column criteria.
+
+        Args:
+            df(Any): Input Pandas or PySpark DataFrame.
+            keys(List[str]): Columns that identify a business entity.
+            order_by_cols(List[str]): Ordered list of columns to sort by.
+            ascending(Union[bool, List[bool]]): Type of sorting direction for
+                each column. Defaults to True.
+
+        Returns:
+            Any: Deduplicated DataFrame containing the top-ranked records.
+        """
+        framework = self._detect_framework(df)
+        if framework == "pandas":
+            return (
+                df.sort_values(by=order_by_cols, ascending=ascending)
+                .groupby(keys)
+                .first()
+                .reset_index()
+            )
+
+        if isinstance(ascending, bool):
+            asc_list = [ascending] * len(order_by_cols)
+        else:
+            asc_list = ascending
+
+        order_exprs = []
+        for col, asc_dir in zip(order_by_cols, asc_list):
+            expr = F.col(col).asc() if asc_dir else F.col(col).desc()
+            order_exprs.append(expr)
+
+        window_spec = Window.partitionBy(keys).orderBy(*order_exprs)
+        return (
+            df.withColumn("_row_num", F.row_number().over(window_spec))
+            .filter(F.col("_row_num") == 1)
+            .drop("_row_num")
+        )
+
+    def combined(self, df: Any, keys: List[str]) -> Any:
+        """Stitches rows together by compacting nulls into a golden record.
+
+        Args:
+            df(Any): Input Pandas or PySpark DataFrame.
+            keys(List[str]): Columns that identify a business entity.
+
+        Returns:
+            Any: Consolidated DataFrame with filled attributes per group.
+        """
+        framework = self._detect_framework(df)
+        if framework == "pandas":
+            return df.groupby(keys).first().reset_index()
+
+        agg_cols = [
+            F.first(c, ignorenulls=True).alias(c) for c in df.columns if c not in keys
+        ]
+        return df.groupBy(keys).agg(*agg_cols)
+
+    def _detect_framework(self, df: Any) -> str:
+        """Internal helper to identify the dataframe framework type safely.
+
+        Args:
+            df(Any): Input dataframe object.
+
+        Returns:
+            str: String identifier ("pandas" or "spark").
+        """
+        df_type = type(df).__name__
+        module_type = type(df).__module__
+
+        if "pandas" in module_type and df_type == "DataFrame":
+            return "pandas"
+
+        if "pyspark" in module_type and df_type == "DataFrame":
+            if not HAS_SPARK:
+                raise ImportError("PySpark is required but could not be imported.")
+            return "spark"
+
+        raise TypeError(
+            f"Unsupported type: {type(df)}. "
+            "Only Pandas and PySpark DataFrames are supported."
+        )

data_engineering_exp/core/initializer.py

@@ -0,0 +1,93 @@
+"""This module implements the project structural initialization engine for dex."""
+
+import os
+
+
+class ProjectInitializer:
+    """Handles scaffolding of directory structures for new data engineering setups.
+
+    Automates the creation of standard configuration folders and source directories
+    to establish a unified project layout.
+    """
+
+    def __init__(self, base_path: str = ".") -> None:
+        """Initializes the ProjectInitializer with a target base path.
+
+        Args:
+            base_path(str): Root system path where scaffolding will be built.
+                Defaults to ".".
+
+        Returns:
+            None: Initializes the class instance.
+        """
+        self.base_path = base_path
+
+    def init_project(
+        self,
+        name: str = "my-dex-project",
+        version: str = "0.1.0",
+        description: str = "Data engineering project scaffolded by dex",
+        author: str = "Anonymous",
+    ) -> None:
+        """Scaffolds the required folders and creates baseline configuration templates.
+
+        Args:
+            name(str): Operational name of the data project. Defaults to
+                "my-dex-project".
+            version(str): Initial semver string version. Defaults to "0.1.0".
+            description(str): Short purpose statement describing the repository.
+                Defaults to "Data engineering project scaffolded by dex".
+            author(str): Full name or alias identifier of the creator. Defaults
+                to "Anonymous".
+
+        Returns:
+            None: Side-effect function creating disk structures.
+        """
+        folders = [
+            os.path.join(self.base_path, "conf", "catalog"),
+            os.path.join(self.base_path, "src", "notebooks"),
+            os.path.join(self.base_path, "data"),
+        ]
+
+        for folder in folders:
+            os.makedirs(folder, exist_ok=True)
+
+        # Create a clean, standard pyproject.toml as the anchor of the project
+        toml_path = os.path.join(self.base_path, "pyproject.toml")
+        toml_content = (
+            "[project]\n"
+            f'name = "{name}"\n'
+            f'version = "{version}"\n'
+            f'description = "{description}"\n'
+            "authors = [\n"
+            f'    {{name = "{author}"}}\n'
+            "]\n"
+            'requires-python = ">=3.11,<4.0.0"\n'
+        )
+        with open(toml_path, "w", encoding="utf-8") as file:
+            file.write(toml_content)
+
+        # Drop a real physical sample CSV file into the new data folder
+        csv_path = os.path.join(self.base_path, "data", "sample_table.csv")
+        csv_content = "id,name\n1,Alice\n2,Bob\n"
+        with open(csv_path, "w", encoding="utf-8") as csv_file:
+            csv_file.write(csv_content)
+
+        # Inject a sample boilerplate YAML catalog pointing to the real CSV
+        sample_catalog_path = os.path.join(
+            self.base_path, "conf", "catalog", "sample_dataset.yaml"
+        )
+        sample_content = (
+            "sample_table:\n"
+            "  description: 'Boilerplate example dataset created by dex'\n"
+            "  format: 'csv'\n"
+            "  engine: 'pandas'\n"
+            "  storage_path: 'data/sample_table.csv'\n"
+            "  columns:\n"
+            "    - name: 'id'\n"
+            "      type: 'integer'\n"
+            "    - name: 'name'\n"
+            "      type: 'string'\n"
+        )
+        with open(sample_catalog_path, "w", encoding="utf-8") as file:
+            file.write(sample_content)

data_engineering_exp/core/io.py

@@ -0,0 +1,134 @@
+"""This module implements metadata-driven data loading utilities for dex."""
+
+import os
+from typing import Any, Optional, cast
+
+import pandas as pd
+
+from data_engineering_exp.core.catalog import DataCatalog
+
+try:
+    from pyspark.sql import SparkSession
+
+    HAS_SPARK = True
+except ImportError:
+    SparkSession = cast(Any, None)
+    HAS_SPARK = False
+
+
+class DataLoader:
+    """Handles dynamic loading of datasets using catalog metadata definitions.
+
+    Supports loading data via Pandas or PySpark based on the specified
+    execution engine and format configurations.
+    """
+
+    def __init__(self, catalog: Optional[DataCatalog] = None) -> None:
+        """Initializes the DataLoader with a DataCatalog instance.
+
+        Args:
+            catalog(Optional[DataCatalog]): An instance of DataCatalog. If
+                None, a new instance is automatically discovered.
+
+        Returns:
+            None: Initializes the class instance.
+        """
+        self.catalog = catalog if catalog is not None else DataCatalog()
+
+    def load(self, dataset_name: str, spark: Optional[Any] = None) -> Any:
+        """Loads a dataset from storage based on its catalog specification.
+
+        Args:
+            dataset_name(str): Unique identifier string of the dataset.
+            spark(Optional[Any]): Active PySpark SparkSession instance. Required
+                if engine is 'spark' and no active session is globally found.
+
+        Returns:
+            Any: Loaded Pandas or PySpark DataFrame object.
+
+        Raises:
+            KeyError: If mandatory keys like 'engine', 'format' or
+                'storage_path' are missing from metadata.
+            ValueError: If an unsupported engine or format is provided.
+            ImportError: If the PySpark engine is requested but not installed.
+        """
+        meta = self.catalog.get_dataset_metadata(dataset_name)
+
+        engine = meta.get("engine")
+        data_format = meta.get("format")
+        path = meta.get("storage_path")
+
+        if not engine or not data_format or not path:
+            raise KeyError(
+                f"Dataset '{dataset_name}' metadata must contain "
+                f"'engine', 'format', and 'storage_path'."
+            )
+
+        # Convention resolution: anchor relative paths safely to project root
+        if not os.path.isabs(path):
+            path = os.path.normpath(os.path.join(self.catalog.project_root, path))
+
+        if engine == "pandas":
+            return self._load_with_pandas(path, data_format)
+        elif engine == "spark":
+            return self._load_with_spark(path, data_format, spark)
+        else:
+            raise ValueError(f"Unsupported execution engine: '{engine}'.")
+
+    def _load_with_pandas(self, path: str, data_format: str) -> pd.DataFrame:
+        """Internal helper to route reading operations to Pandas.
+
+        Args:
+            path(str): Target system string file location path.
+            data_format(str): File format layout specification identifier.
+
+        Returns:
+            pd.DataFrame: Loaded Pandas DataFrame object.
+
+        Raises:
+            ValueError: If the file format layout is unsupported by Pandas.
+        """
+        if data_format == "parquet":
+            return pd.read_parquet(path)
+        elif data_format == "csv":
+            return pd.read_csv(path)
+        else:
+            raise ValueError(f"Unsupported Pandas format: '{data_format}'.")
+
+    def _load_with_spark(
+        self, path: str, data_format: str, spark: Optional[Any]
+    ) -> Any:
+        """Internal worker to route reading operations to PySpark.
+
+        Args:
+            path(str): Target system string file location path.
+            data_format(str): File format layout specification identifier.
+            spark(Optional[Any]): Explicit user supplied SparkSession object.
+
+        Returns:
+            Any: Loaded PySpark DataFrame object.
+
+        Raises:
+            ImportError: If the PySpark dependency library is uninstalled.
+            ValueError: If no valid operational Spark session context is found
+                or if the layout format is unsupported.
+        """
+        if not HAS_SPARK:
+            raise ImportError("PySpark is required but could not be imported.")
+
+        session = spark
+        if session is None:
+            session = SparkSession.getActiveSession()
+
+        if session is None:
+            raise ValueError(
+                "A valid SparkSession must be provided or active globally "
+                "to load data using the 'spark' engine."
+            )
+
+        if data_format == "parquet":
+            return session.read.parquet(path)
+        elif data_format == "csv":
+            return session.read.csv(path, header=True, inferSchema=True)
+        else:
+            raise ValueError(f"Unsupported Spark format: '{data_format}'.")

data_engineering_exp/pandas_core/scd2.py

@@ -0,0 +1,225 @@
+"""This module implements Slowly Changing Dimension Type 2 (SCD2) logic using Pandas."""
+
+from typing import List
+
+import pandas as pd
+
+
+class PandasSCD2Processor:
+    """Handles Slowly Changing Dimension Type 2 (SCD2) logic using Pandas.
+
+    This class encapsulates configuration parameters such as keys and metadata
+    columns, allowing clean and repeatable executions of SCD2 updates.
+    """
+
+    def __init__(
+        self,
+        business_keys: List[str],
+        compare_columns: List[str],
+        effective_date_col: str = "effective_date",
+        end_date_col: str = "end_date",
+        current_flag_col: str = "is_current",
+        input_timestamp_col: str = "input_timestamp",
+        max_date: str = "9999-12-31",
+    ):
+        """Initializes the PandasSCD2Processor with configuration parameters.
+
+        Args:
+            business_keys(List[str]): List of columns that uniquely identify
+                a record.
+            compare_columns(List[str]): List of columns used to detect
+                changes in the data.
+            effective_date_col(str): Name of the effective start date column.
+                Defaults to "effective_date".
+            end_date_col(str): Name of the effective end date column.
+                Defaults to "end_date".
+            current_flag_col(str): Name of the boolean flag column
+                indicating the active record. Defaults to "is_current".
+            input_timestamp_col(str): Name of the column containing the
+                ingestion/mutation timestamp. Defaults to "input_timestamp".
+            max_date(str): The maximum date string used to populate active
+                records. Defaults to "9999-12-31".
+
+        Returns:
+            None: This method initializes the class instance.
+        """
+        self.business_keys = business_keys
+        self.compare_columns = compare_columns
+        self.effective_date_col = effective_date_col
+        self.end_date_col = end_date_col
+        self.current_flag_col = current_flag_col
+        self.input_timestamp_col = input_timestamp_col
+        self.max_date = max_date
+
+    def process(
+        self, df_new: pd.DataFrame, df_existing: pd.DataFrame
+    ) -> pd.DataFrame:
+        """Orchestrates the SCD2 pipeline process and returns result.
+
+        Args:
+            df_new(pd.DataFrame): Incoming batch of data.
+            df_existing(pd.DataFrame): Current state of dimension.
+
+        Returns:
+            pd.DataFrame: Unified DataFrame with all processed records.
+        """
+        if not self.compare_columns:
+            raise ValueError(
+                "compare_columns cannot be empty for SCD2 processing."
+            )
+
+        # Filtrar solo los registros activos de la dimensión histórica
+        # Avoid equality comparison to True; use truthiness check instead
+        df_active = df_existing[df_existing[self.current_flag_col]]
+
+        # Indexar por llaves de negocio para aprovechar alineación de Pandas
+        df_exist_idx = df_active.set_index(self.business_keys)
+        df_new_idx = df_new.set_index(self.business_keys)
+
+        # Identificar segmentos por llaves utilizando operaciones de conjuntos
+        new_keys = df_new_idx.index.difference(df_exist_idx.index)
+        common_keys = df_new_idx.index.intersection(df_exist_idx.index)
+        preserved_keys = df_exist_idx.index.difference(df_new_idx.index)
+
+        # Separar registros comunes entre cambiados y no cambiados
+        changed_keys, unchanged_keys = self._split_common_keys(
+            df_new_idx, df_exist_idx, common_keys
+        )
+
+        # Construir cada segmento del SCD2
+        df_new_only = self._get_new_records(df_new_idx, new_keys)
+        df_changed = self._get_changed_records(df_new_idx, changed_keys)
+        df_expired = self._get_expired_records(
+            df_new_idx, df_exist_idx, changed_keys
+        )
+        df_unchanged = self._get_unchanged_records(
+            df_exist_idx, unchanged_keys
+        )
+        df_preserved = self._get_preserved_records(
+            df_exist_idx, preserved_keys
+        )
+
+        # Consolidar y ordenar columnas según el esquema original
+        target_cols = df_existing.columns.tolist()
+        return pd.concat(
+            [df_new_only, df_changed, df_expired, df_unchanged, df_preserved],
+            ignore_index=True,
+        )[target_cols]
+
+    def _split_common_keys(
+        self,
+        df_new_idx: pd.DataFrame,
+        df_exist_idx: pd.DataFrame,
+        common_keys: pd.Index,
+    ) -> tuple:
+        """Splits common keys into changed and unchanged categories.
+
+        Args:
+            df_new_idx(pd.DataFrame): New data indexed by business keys.
+            df_exist_idx(pd.DataFrame): Active existing data indexed by keys.
+            common_keys(pd.Index): Intersection of business keys.
+
+        Returns:
+            tuple: A tuple containing (changed_keys, unchanged_keys).
+        """
+        df_new_common = df_new_idx.loc[common_keys]
+        df_exist_common = df_exist_idx.loc[common_keys]
+
+        change_mask = pd.Series(False, index=common_keys)
+        for col in self.compare_columns:
+            change_mask |= df_new_common[col] != df_exist_common[col]
+
+        return common_keys[change_mask], common_keys[~change_mask]
+
+    def _get_new_records(
+        self, df_new_idx: pd.DataFrame, new_keys: pd.Index
+    ) -> pd.DataFrame:
+        """Extracts and formats brand new records.
+
+        Args:
+            df_new_idx(pd.DataFrame): New data indexed by business keys.
+            new_keys(pd.Index): Keys that only exist in the new batch.
+
+        Returns:
+            pd.DataFrame: Formatted dataframe for new insertions.
+        """
+        df_new_only = df_new_idx.loc[new_keys].reset_index()
+        df_new_only[self.effective_date_col] = df_new_only[
+            self.input_timestamp_col
+        ]
+        df_new_only[self.end_date_col] = self.max_date
+        df_new_only[self.current_flag_col] = True
+        return df_new_only
+
+    def _get_changed_records(
+        self, df_new_idx: pd.DataFrame, changed_keys: pd.Index
+    ) -> pd.DataFrame:
+        """Extracts and formats new active versions of changed records.
+
+        Args:
+            df_new_idx(pd.DataFrame): New data indexed by business keys.
+            changed_keys(pd.Index): Keys with detected attribute modifications.
+
+        Returns:
+            pd.DataFrame: Formatted dataframe for updated active rows.
+        """
+        df_changed = df_new_idx.loc[changed_keys].reset_index()
+        df_changed[self.effective_date_col] = df_changed[
+            self.input_timestamp_col
+        ]
+        df_changed[self.end_date_col] = self.max_date
+        df_changed[self.current_flag_col] = True
+        return df_changed
+
+    def _get_expired_records(
+        self,
+        df_new_idx: pd.DataFrame,
+        df_exist_idx: pd.DataFrame,
+        changed_keys: pd.Index,
+    ) -> pd.DataFrame:
+        """Closes historical records by updating end date and flag.
+
+        Args:
+            df_new_idx(pd.DataFrame): New data indexed by business keys.
+            df_exist_idx(pd.DataFrame): Active existing data indexed by keys.
+            changed_keys(pd.Index): Keys with detected attribute modifications.
+
+        Returns:
+            pd.DataFrame: Formatted dataframe for expired historical rows.
+        """
+        df_expired = df_exist_idx.loc[changed_keys].reset_index()
+        closing_timestamps = df_new_idx.loc[
+            changed_keys, self.input_timestamp_col
+        ].values
+
+        df_expired[self.end_date_col] = closing_timestamps
+        df_expired[self.current_flag_col] = False
+        return df_expired
+
+    def _get_unchanged_records(
+        self, df_exist_idx: pd.DataFrame, unchanged_keys: pd.Index
+    ) -> pd.DataFrame:
+        """Extracts active records that remain identical.
+
+        Args:
+            df_exist_idx(pd.DataFrame): Active existing data indexed by keys.
+            unchanged_keys(pd.Index): Keys with no attribute modifications.
+
+        Returns:
+            pd.DataFrame: Existing active rows without modifications.
+        """
+        return df_exist_idx.loc[unchanged_keys].reset_index()
+
+    def _get_preserved_records(
+        self, df_exist_idx: pd.DataFrame, preserved_keys: pd.Index
+    ) -> pd.DataFrame:
+        """Extracts active records completely missing from the incoming batch.
+
+        Args:
+            df_exist_idx(pd.DataFrame): Active existing data indexed by keys.
+            preserved_keys(pd.Index): Keys missing from the new batch.
+
+        Returns:
+            pd.DataFrame: Existing active rows that must be preserved.
+        """
+        return df_exist_idx.loc[preserved_keys].reset_index()

data_engineering_exp/spark_core/scd2.py

@@ -0,0 +1,283 @@
+"""
+This module implements Slowly Changing Dimension Type 2 (SCD2) logic using
+PySpark.
+"""
+
+from typing import List
+
+from pyspark.sql import Column, DataFrame
+from pyspark.sql import functions as F
+
+
+class SparkSCD2Processor:
+    """Handles Slowly Changing Dimension Type 2 (SCD2) logic using PySpark.
+
+    This class encapsulates configuration parameters such as keys and metadata
+    columns, allowing clean and repeatable executions of SCD2 updates.
+    """
+
+    def __init__(
+        self,
+        business_keys: List[str],
+        compare_columns: List[str],
+        effective_date_col: str = "effective_date",
+        end_date_col: str = "end_date",
+        current_flag_col: str = "is_current",
+        input_timestamp_col: str = "input_timestamp",
+        max_date: str = "9999-12-31",
+    ):
+        """Initializes the SparkSCD2Processor with configuration parameters.
+
+        Args:
+            business_keys(List[str]): List of columns that uniquely identify
+                a record.
+            compare_columns(List[str]): List of columns used to detect
+                changes in the data.
+            effective_date_col(str): Name of the effective start date column.
+                Defaults to "effective_date".
+            end_date_col(str): Name of the effective end date column.
+                Defaults to "end_date".
+            current_flag_col(str): Name of the boolean flag column
+                indicating the active record. Defaults to "is_current".
+            input_timestamp_col(str): Name of the column containing the
+                ingestion/mutation timestamp. Defaults to "input_timestamp".
+            max_date(str): The maximum date string used to populate active
+                records. Defaults to "9999-12-31".
+
+        Returns:
+            None: This method initializes the class instance.
+        """
+        self.business_keys = business_keys
+        self.compare_columns = compare_columns
+        self.effective_date_col = effective_date_col
+        self.end_date_col = end_date_col
+        self.current_flag_col = current_flag_col
+        self.input_timestamp_col = input_timestamp_col
+        self.max_date = max_date
+
+        self._new_alias = "new"
+        self._exist_alias = "exist"
+
+    def process(
+        self, spark_df_new: DataFrame, spark_df_existing: DataFrame
+    ) -> DataFrame:
+        """Orchestrates the SCD2 pipeline process and returns result.
+
+        Args:
+            spark_df_new(DataFrame): Incoming batch of data.
+            spark_df_existing(DataFrame): Current state of dimension.
+
+        Returns:
+            DataFrame: Unified DataFrame with all processed records.
+        """
+        target_columns = spark_df_existing.columns
+
+        df_new_prepared = self._prepare_new_df(spark_df_new)
+        df_joined = self._join_datasets(df_new_prepared, spark_df_existing)
+
+        change_cond = self._build_change_condition()
+        unchanged_cond = self._build_unchanged_condition()
+
+        df_new_only = self._get_new_records(df_joined, target_columns)
+        df_changed = self._get_changed_records(df_joined, change_cond, target_columns)
+        df_unchanged = self._get_unchanged_records(
+            df_joined, unchanged_cond, target_columns
+        )
+        df_expired = self._get_expired_records(df_joined, change_cond, target_columns)
+        df_preserved = self._get_preserved_records(
+            spark_df_new, spark_df_existing, target_columns
+        )
+
+        return (
+            df_new_only.unionByName(df_changed)
+            .unionByName(df_expired)
+            .unionByName(df_preserved)
+            .unionByName(df_unchanged)
+        )
+
+    def _prepare_new_df(self, df: DataFrame) -> DataFrame:
+        """Prepares the incoming new dataset by injecting SCD2 metadata.
+
+        Args:
+            df(DataFrame): The raw incoming PySpark DataFrame.
+
+        Returns:
+            DataFrame: PySpark DataFrame with SCD2 columns added.
+        """
+        return (
+            df.withColumn(self.effective_date_col, F.col(self.input_timestamp_col))
+            .withColumn(self.end_date_col, F.lit(self.max_date))
+            .withColumn(self.current_flag_col, F.lit(True))
+        )
+
+    def _join_datasets(self, df_new: DataFrame, df_existing: DataFrame) -> DataFrame:
+        """Performs a left join between new data and historical dimension.
+
+        Args:
+            df_new(DataFrame): Prepared incoming DataFrame.
+            df_existing(DataFrame): Historical dimension table.
+
+        Returns:
+            DataFrame: Joined PySpark DataFrame.
+        """
+        join_condition = [
+            F.col(f"{self._new_alias}.{col}") == F.col(f"{self._exist_alias}.{col}")
+            for col in self.business_keys
+        ]
+        return df_new.alias(self._new_alias).join(
+            df_existing.alias(self._exist_alias),
+            on=join_condition,
+            how="left",
+        )
+
+    def _build_change_condition(self) -> Column:
+        """Builds expression to identify mismatches across comparison columns.
+
+        Args:
+
+        Returns:
+            Column: PySpark Column representing the logical OR condition.
+        """
+        if not self.compare_columns:
+            raise ValueError(
+                "compare_columns cannot be empty for SCD2 processing."
+            )
+
+        # Inicializamos con la primera columna para asegurar un tipo Column
+        first_col = self.compare_columns[0]
+        change_condition = F.col(f"{self._new_alias}.{first_col}") != F.col(
+            f"{self._exist_alias}.{first_col}"
+        )
+
+        # Iteramos sobre el resto de las columnas usando el operador |= (OR)
+        for col in self.compare_columns[1:]:
+            condition = F.col(f"{self._new_alias}.{col}") != F.col(
+                f"{self._exist_alias}.{col}"
+            )
+            change_condition |= condition
+
+        return change_condition
+
+    def _build_unchanged_condition(self) -> Column:
+        """Builds expression to verify that columns are strictly identical.
+
+        Args:
+
+        Returns:
+            Column: PySpark Column representing the logical AND condition.
+        """
+        unchanged_condition = F.lit(True)
+        for col in self.compare_columns:
+            unchanged_condition &= F.col(f"{self._new_alias}.{col}") == F.col(
+                f"{self._exist_alias}.{col}"
+            )
+        return unchanged_condition
+
+    def _get_new_records(self, df_joined: DataFrame, columns: List[str]) -> DataFrame:
+        """Extracts records whose business keys do not exist in dimension.
+
+        Args:
+            df_joined(DataFrame): The combined/joined PySpark DataFrame.
+            columns(List[str]): List of output columns required.
+
+        Returns:
+            DataFrame: PySpark DataFrame containing brand new rows.
+        """
+        key_col = f"{self._exist_alias}.{self.business_keys[0]}"
+        select_exprs = [F.col(f"{self._new_alias}.{c}").alias(c) for c in columns]
+        return df_joined.filter(F.col(key_col).isNull()).select(select_exprs)
+
+    def _get_changed_records(
+        self, df_joined: DataFrame, change_cond: Column, columns: List[str]
+    ) -> DataFrame:
+        """Extracts newest version of records that suffered changes.
+
+        Args:
+            df_joined(DataFrame): The combined/joined PySpark DataFrame.
+            change_cond(Column): PySpark Column condition for changes.
+            columns(List[str]): List of output columns required.
+
+        Returns:
+            DataFrame: PySpark DataFrame containing active updated records.
+        """
+        is_current_cond = F.col(f"{self._exist_alias}.{self.current_flag_col}")
+        select_exprs = [F.col(f"{self._new_alias}.{c}").alias(c) for c in columns]
+        return (
+            df_joined.filter(is_current_cond).filter(change_cond).select(select_exprs)
+        )
+
+    def _get_unchanged_records(
+        self, df_joined: DataFrame, unchanged_cond: Column, columns: List[str]
+    ) -> DataFrame:
+        """Extracts existing active records that have no changes.
+
+        Args:
+            df_joined(DataFrame): The combined/joined PySpark DataFrame.
+            unchanged_cond(Column): PySpark Column condition for identity.
+            columns(List[str]): List of output columns required.
+
+        Returns:
+            DataFrame: PySpark DataFrame containing untouched active rows.
+        """
+        is_current_cond = F.col(f"{self._exist_alias}.{self.current_flag_col}")
+        select_exprs = [F.col(f"{self._exist_alias}.{c}").alias(c) for c in columns]
+        return (
+            df_joined.filter(is_current_cond)
+            .filter(unchanged_cond)
+            .select(select_exprs)
+        )
+
+    def _get_expired_records(
+        self, df_joined: DataFrame, change_cond: Column, columns: List[str]
+    ) -> DataFrame:
+        """Transforms existing active rows into historically closed versions.
+
+        Args:
+            df_joined(DataFrame): The combined/joined PySpark DataFrame.
+            change_cond(Column): PySpark Column condition for changes.
+            columns(List[str]): List of output columns required.
+
+        Returns:
+            DataFrame: PySpark DataFrame containing expired records.
+        """
+        is_current_cond = F.col(f"{self._exist_alias}.{self.current_flag_col}")
+
+        select_exprs = []
+        for c in columns:
+            if c not in [self.end_date_col, self.current_flag_col]:
+                select_exprs.append(F.col(f"{self._exist_alias}.{c}").alias(c))
+            elif c == self.end_date_col:
+                new_ts_col = f"{self._new_alias}.{self.input_timestamp_col}"
+                select_exprs.append(F.col(new_ts_col).alias(self.end_date_col))
+            else:
+                select_exprs.append(F.lit(False).alias(self.current_flag_col))
+
+        return (
+            df_joined.filter(is_current_cond).filter(change_cond).select(select_exprs)
+        )
+
+    def _get_preserved_records(
+        self, df_new: DataFrame, df_existing: DataFrame, columns: List[str]
+    ) -> DataFrame:
+        """Preserves active records missing from the new batch.
+
+        Args:
+            df_new(DataFrame): The raw incoming PySpark DataFrame.
+            df_existing(DataFrame): The existing dimension table.
+            columns(List[str]): List of output columns required.
+
+        Returns:
+            DataFrame: PySpark DataFrame containing unaffected rows.
+        """
+        is_current_cond = F.col(f"{self._exist_alias}.{self.current_flag_col}")
+        select_exprs = [F.col(f"{self._exist_alias}.{c}").alias(c) for c in columns]
+        return (
+            df_existing.alias(self._exist_alias)
+            .join(
+                df_new.alias(self._new_alias),
+                on=self.business_keys,
+                how="left_anti",
+            )
+            .filter(is_current_cond)
+            .select(select_exprs)
+        )

data_engineering_exp-0.1.0.dist-info/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 Tu Nombre Completo
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

data_engineering_exp-0.1.0.dist-info/METADATA

@@ -0,0 +1,107 @@
+Metadata-Version: 2.3
+Name: data-engineering-exp
+Version: 0.1.0
+Summary: A minimalist, agnostic Python framework to standardize data engineering pipelines.
+License: MIT
+Keywords: data-engineering,pyspark,pandas,data-catalog
+Author: idperez720
+Author-email: ivandavidperez4@gmail.com
+Requires-Python: >=3.11,<4.0.0
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Operating System :: OS Independent
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Requires-Dist: click (>=8.4.1,<9.0.0)
+Requires-Dist: pandas (>=3.0.3,<4.0.0)
+Requires-Dist: pyspark (>=4.1.2,<5.0.0)
+Requires-Dist: pyyaml (>=6.0.3,<7.0.0)
+Project-URL: Repository, https://github.com/idperez720/data-engineering-exp
+Description-Content-Type: text/markdown
+
+# Data Engineering Experience 🚀
+
+[![License: MIT](https://img.shields.io/badge/License-MIT-blue.svg)](https://opensource.org/licenses/MIT)
+[![Python Version](https://img.shields.io/badge/python-3.11%20%7C%203.12%20%7C%203.14-blue)](https://www.python.org)
+
+**dex** (*Data Engineering Experience*) is a minimalist, agnostic Python framework designed to streamline and standardize data engineering pipelines. By embracing **Convention over Configuration**, `dex` eliminates environment friction, absolute path hardcoding, and complex PySpark session management.
+
+---
+
+## ✨ Key Features
+
+* **Zero-Config File Discovery:** Automatic tree-walking directory resolution anchors your data catalog using your local `pyproject.toml` file.
+* **Decentralized Catalog:** Declare your metadata layouts inside modular, self-contained mini-YAML files.
+* **Elastic Processing Runtimes:** Switch dynamically between **Pandas** and **PySpark** execution engines using exactly the same unified interface.
+* **Interactive CLI Scaffolding:** Spin up a new production-ready data directory structure instantly with `dex init`.
+
+---
+
+## 📦 Installation
+
+*(Once published to PyPI)*
+```bash
+pip install dex
+```
+Or install it directly from the source repository using Poetry:
+```bash
+poetry add git+[https://github.com/idperez720/data-engineering-exp.git](https://github.com/idperez720/data-engineering-exp.git)
+```
+
+---
+
+## 🏁 Quick Start
+
+### 1. Initialize your workspace
+
+Navigate to an empty directory and let the interactive wizard scaffold the workspace conventions:
+
+```bash
+dex init
+
+```
+
+### 2. Declare a dataset
+
+Add a specification block inside `conf/catalog/sample_dataset.yaml`:
+
+```yaml
+customers:
+  description: "Main production customer data"
+  format: "csv"
+  engine: "pandas"
+  storage_path: "data/sample_table.csv"
+
+```
+
+### 3. Load data anywhere
+
+Create a Python script or open a Jupyter Notebook inside `src/notebooks/` and fetch your data instantly:
+
+```python
+from data_engineering_exp.core.io import DataLoader
+
+# Autodiscovers your project root boundaries and settings
+loader = DataLoader()
+
+# Loads the dataset securely as a Pandas DataFrame
+df = loader.load("customers")
+df.head()
+
+```
+
+---
+
+## 📖 Complete Documentation
+
+For comprehensive guides, testing architecture deep-dives, and complete API references, visit our documentation site:
+👉 **[http://127.0.0.1:8000/](https://www.google.com/search?q=http://127.0.0.1:8000/)** *(Replace with your deployed docs URL, e.g., GitHub Pages)*
+
+---
+
+## ⚖️ License
+
+Distributed under the **MIT License**. Any modification or distribution (including forks) must include the original copyright notice and liability waiver. See `LICENSE` for more information.
+
+```

data_engineering_exp-0.1.0.dist-info/RECORD

@@ -0,0 +1,16 @@
+data_engineering_exp/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+data_engineering_exp/cli.py,sha256=KNQ8Ei4_wU-MTEH_pca3cOhwQJ0rV0wAZ3FwHtWuht8,1783
+data_engineering_exp/core/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+data_engineering_exp/core/catalog.py,sha256=wu4QnsHus9QJi6dhQR_Eu4-elxAY-dTEor2goZpuE-U,7226
+data_engineering_exp/core/deduplication.py,sha256=aLJrJtVQG56wI39ImSKvsIaq1U36hH7_-t4CjB3fwGw,6472
+data_engineering_exp/core/initializer.py,sha256=Q0Kx7U5HxNu19pQYGvxxfy-1TVeFpHF4jMXKfji2i9c,3484
+data_engineering_exp/core/io.py,sha256=VIzAEP5iQduUJ7OWV2K82iAdelpkPT9lk7NqU1BuRXA,4825
+data_engineering_exp/pandas_core/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+data_engineering_exp/pandas_core/scd2.py,sha256=KGeRYVv2ClxEoO9WUEVfF5Bdll3l-clJmIqYlT5EbZQ,8668
+data_engineering_exp/spark_core/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+data_engineering_exp/spark_core/scd2.py,sha256=Zza0ePChxq9TV74OK5gr8odhUF7sDiX_lmoAS2qwyV4,10841
+data_engineering_exp-0.1.0.dist-info/LICENSE,sha256=AVQrXW_hCAJSW_1PiMnJmic2hJ5aT9ypRcQEqIY4sdw,1074
+data_engineering_exp-0.1.0.dist-info/METADATA,sha256=V3OeZlQeGXkdJdiuHzAgmrTQys6BJ0o7lgM7SG8W2a8,3607
+data_engineering_exp-0.1.0.dist-info/WHEEL,sha256=b4K_helf-jlQoXBBETfwnf4B04YC67LOev0jo4fX5m8,88
+data_engineering_exp-0.1.0.dist-info/entry_points.txt,sha256=xHkLyRWvtD12WAnIZ4p0XnzRvhvr3Mt0wj5x8hcJ07A,60
+data_engineering_exp-0.1.0.dist-info/RECORD,,

data_engineering_exp-0.1.0.dist-info/WHEEL

@@ -0,0 +1,4 @@
+Wheel-Version: 1.0
+Generator: poetry-core 2.1.3
+Root-Is-Purelib: true
+Tag: py3-none-any

data_engineering_exp-0.1.0.dist-info/entry_points.txt

@@ -0,0 +1,3 @@
+[console_scripts]
+dex=data_engineering_exp.cli:entry_point
+