etlpipe 2.0.0__py3-none-any.whl

etlpipe/__init__.py

@@ -0,0 +1,53 @@
+"""etlpipe — The fastest path from proprietary visual ETL to open-source Python.
+
+Accelerates migration from legacy visual ETL tools to
+open-source Python by providing 1:1 tool palette mappings (Preparation, Join,
+Transform, Parse, InOut, Developer). Uses **pandas** (local) or **PySpark** (cluster)
+as the underlying data engine. Includes automated ``.yxmd`` workflow conversion tools.
+
+Quick start::
+
+    from etlpipe import InOut, Preparation, Join, Transform, Parse, Developer
+
+    df = InOut.input_data("sales.csv")
+    high, low = Preparation.filter(df, "Revenue > 1000")
+    summary = Transform.summarize(high, group_by="Region",
+                                   aggregations={"Revenue": "sum"})
+    InOut.output_data(summary, "summary.parquet")
+"""
+
+from etlpipe._config import backend, get_backend, set_backend
+from etlpipe._contracts import SchemaViolationError, expect_schema, infer_schema
+from etlpipe._pii import scan_pii
+from etlpipe._version import __version__
+from etlpipe.convert import YxmdConverter
+from etlpipe.developer import Developer
+from etlpipe.in_out import InOut
+from etlpipe.join import Join
+from etlpipe.parse import Parse
+from etlpipe.pipeline import Pipeline
+from etlpipe.preparation import Preparation
+from etlpipe.transform import Transform
+
+__all__ = [
+    "__version__",
+    # Converter
+    "YxmdConverter",
+    # Tool palettes
+    "Developer",
+    "InOut",
+    "Join",
+    "Parse",
+    "Pipeline",
+    "Preparation",
+    "Transform",
+    "set_backend",
+    "get_backend",
+    "backend",
+    # Data contracts
+    "expect_schema",
+    "infer_schema",
+    "SchemaViolationError",
+    # PII scanning
+    "scan_pii",
+]

etlpipe/_config.py

@@ -0,0 +1,70 @@
+import threading
+from contextlib import contextmanager
+from typing import Any
+
+from etlpipe.engines.pandas_engine import PandasEngine
+
+try:
+    from etlpipe.engines.spark_engine import SparkEngine
+except ImportError:
+    SparkEngine = None
+
+# Thread-local storage for configuration so multiple pipelines can run safely
+_local_state = threading.local()
+
+
+def _init_state():
+    if not hasattr(_local_state, "backend"):
+        _local_state.backend = "pandas"
+    if not hasattr(_local_state, "engine"):
+        _local_state.engine = PandasEngine()
+    if not hasattr(_local_state, "spark_config"):
+        _local_state.spark_config = {}
+
+
+def set_backend(backend_name: str, **kwargs: Any) -> None:
+    _init_state()
+    backend_name = backend_name.lower().strip()
+    if backend_name == "pandas":
+        _local_state.backend = "pandas"
+        _local_state.engine = PandasEngine()
+    elif backend_name == "spark":
+        if SparkEngine is None:
+            raise ImportError("PySpark is required for the Spark backend. Install it with: pip install etlpipe[spark]")
+        _local_state.backend = "spark"
+        _local_state.engine = SparkEngine(**kwargs)
+        _local_state.spark_config = kwargs
+    else:
+        raise ValueError(f"Unknown backend: '{backend_name}'. Supported backends are: 'pandas', 'spark'.")
+
+
+def get_backend() -> str:
+    _init_state()
+    return _local_state.backend
+
+
+def get_engine() -> Any:
+    _init_state()
+    return _local_state.engine
+
+
+def reset_backend() -> None:
+    _local_state.backend = "pandas"
+    _local_state.engine = PandasEngine()
+    _local_state.spark_config = {}
+
+
+@contextmanager
+def backend(backend_name: str, **kwargs: Any):
+    _init_state()
+    prev_backend = _local_state.backend
+    prev_engine = _local_state.engine
+    prev_spark_config = getattr(_local_state, "spark_config", {})
+
+    try:
+        set_backend(backend_name, **kwargs)
+        yield
+    finally:
+        _local_state.backend = prev_backend
+        _local_state.engine = prev_engine
+        _local_state.spark_config = prev_spark_config

etlpipe/_contracts.py

@@ -0,0 +1,29 @@
+"""Data contract validation — backward-compatibility shim.
+
+The schema contract implementation has moved to the ``etlpipe-governance``
+sub-package.  This module re-exports everything from there so that all
+existing code continues to work unchanged:
+
+    from etlpipe._contracts import expect_schema, SchemaViolationError  # still works
+    from etlpipe import expect_schema, infer_schema                      # still works
+    from etlpipe_governance import expect_schema, profile, ContractSuite # new canonical home
+
+.. deprecated::
+    Import from ``etlpipe_governance`` directly for access to the full
+    feature set, including :func:`etlpipe_governance.contracts.profile`
+    and :class:`etlpipe_governance.contracts.ContractSuite`.
+    The shim re-exports will be removed in etlpipe 3.0.
+"""
+
+from __future__ import annotations
+
+# Re-export everything the original module exposed, sourced from the
+# governance sub-package (single source of truth).
+from etlpipe_governance.contracts import (
+    SchemaViolationError,
+    _dtype_matches,
+    expect_schema,
+    infer_schema,
+)
+
+__all__ = ["SchemaViolationError", "expect_schema", "infer_schema", "_dtype_matches"]

etlpipe/_pii.py

@@ -0,0 +1,27 @@
+"""PII scanner — backward-compatibility shim.
+
+The PII detection implementation has moved to the ``etlpipe-governance``
+sub-package.  This module re-exports everything from there so that all
+existing code continues to work unchanged:
+
+    from etlpipe._pii import scan_pii, PIIWarning        # still works
+    from etlpipe import scan_pii                          # still works
+    from etlpipe_governance import scan_pii, mask_pii    # new canonical home
+
+.. deprecated::
+    Import from ``etlpipe_governance`` directly for access to the full
+    feature set, including :func:`etlpipe_governance.pii.mask_pii`.
+    The shim re-exports will be removed in etlpipe 3.0.
+"""
+
+from __future__ import annotations
+
+# Re-export everything the original module exposed, sourced from the
+# governance sub-package (single source of truth).
+from etlpipe_governance.pii import (
+    _DEFAULT_PATTERNS,
+    PIIWarning,
+    scan_pii,
+)
+
+__all__ = ["scan_pii", "PIIWarning", "_DEFAULT_PATTERNS"]

etlpipe/_validators.py

@@ -0,0 +1,85 @@
+"""Shared input validation helpers for etlpipe tool functions.
+
+Every public tool function validates its inputs through these helpers
+to provide clear, consistent error messages across the library.
+"""
+
+from __future__ import annotations
+
+from collections.abc import Sequence
+from typing import Any
+
+import pandas as pd
+
+
+def validate_dataframe(df: Any, param_name: str = "df") -> None:
+    """Ensure the given object is a pandas DataFrame.
+
+    Args:
+        df: The object to validate.
+        param_name: Name of the parameter (for error messages).
+
+    Raises:
+        TypeError: If *df* is not a ``pandas.DataFrame``.
+    """
+    valid_types = (pd.DataFrame,)
+    try:
+        from pyspark.sql import DataFrame as SparkDF
+
+        valid_types = (pd.DataFrame, SparkDF)
+    except ImportError:
+        pass
+    if not isinstance(df, valid_types):
+        raise TypeError(f"'{param_name}' must be a pandas or PySpark DataFrame, got {type(df).__name__}.")
+
+
+def validate_columns(
+    df: Any,
+    columns: str | Sequence[str],
+    param_name: str = "columns",
+) -> list[str]:
+    """Ensure the specified columns exist in the DataFrame.
+
+    Accepts a single column name (``str``) or a sequence of names and
+    always returns a ``list[str]`` for uniform downstream handling.
+
+    Args:
+        df: The DataFrame to check against.
+        columns: Column name(s) to validate.
+        param_name: Name of the parameter (for error messages).
+
+    Returns:
+        A list of validated column names.
+
+    Raises:
+        TypeError: If *columns* is not a string or sequence of strings.
+        KeyError: If any column is missing from *df*.
+    """
+    if isinstance(columns, str):
+        columns = [columns]
+    elif not isinstance(columns, (list, tuple)):
+        raise TypeError(f"'{param_name}' must be a string or list of strings, got {type(columns).__name__}.")
+
+    missing = [c for c in columns if c not in df.columns]
+    if missing:
+        raise KeyError(f"Column(s) not found in DataFrame: {missing}. Available columns: {list(df.columns)}")
+    return list(columns)
+
+
+def validate_not_empty(df: Any, param_name: str = "df") -> None:
+    """Ensure the DataFrame is not empty.
+
+    Args:
+        df: The DataFrame to check.
+        param_name: Name of the parameter (for error messages).
+
+    Raises:
+        ValueError: If *df* has zero rows.
+    """
+    if isinstance(df, pd.DataFrame):
+        if df.empty:
+            raise ValueError(f"'{param_name}' must not be an empty DataFrame.")
+    else:
+        # For PySpark, check if at least 1 row exists without counting all
+        if len(df.head(1)) == 0:
+            raise ValueError(f"'{param_name}' must not be an empty DataFrame.")

etlpipe/_version.py

@@ -0,0 +1,3 @@
+"""Single source of truth for the etlpipe package version."""
+
+__version__ = "2.0.0"