cloe-nessy 1.0.0__py3-none-any.whl → 1.0.1__py3-none-any.whl

cloe_nessy/integration/reader/excel_reader.py

@@ -160,7 +160,7 @@ class ExcelDataFrameReader(BaseReader):
             "__metadata",
             F.create_map(
                 F.lit("timestamp"),
-                F.current_timestamp(),
+                F.current_timestamp().cast("string"),
                 F.lit("file_location"),
                 F.lit(location),
                 F.lit("sheet_name"),

cloe_nessy/integration/reader/file_reader.py

@@ -192,7 +192,8 @@ class FileReader(BaseReader):
         """Add all metadata columns to the DataFrame."""
         metadata_columns = df.select("_metadata.*").columns
 
-        entries = [(F.lit(field), F.col(f"_metadata.{field}")) for field in metadata_columns]
+        # Cast all metadata values to strings to ensure type consistency in the map
+        entries = [(F.lit(field), F.col(f"_metadata.{field}").cast("string")) for field in metadata_columns]
         flat_list = [item for tup in entries for item in tup]
 
         df = df.withColumn("__metadata", F.create_map(flat_list))

cloe_nessy/pipeline/actions/__init__.py

@@ -19,6 +19,7 @@ from .transform_group_aggregate import TransformGroupAggregate
 from .transform_hash_columns import TransformHashColumnsAction
 from .transform_join import TransformJoinAction
 from .transform_json_normalize import TransformJsonNormalize
+from .transform_regex_extract import TransformRegexExtract
 from .transform_rename_columns import TransformRenameColumnsAction
 from .transform_replace_values import TransformReplaceValuesAction
 from .transform_select_columns import TransformSelectColumnsAction
@@ -56,6 +57,7 @@ __all__ = [
     "TransformGroupAggregate",
     "TransformJoinAction",
     "TransformJsonNormalize",
+    "TransformRegexExtract",
     "TransformRenameColumnsAction",
     "TransformReplaceValuesAction",
     "TransformSelectColumnsAction",

cloe_nessy/pipeline/actions/transform_regex_extract.py

@@ -0,0 +1,169 @@
+import re
+from typing import Any
+
+import pyspark.sql.functions as F
+
+from cloe_nessy.pipeline.pipeline_action import PipelineAction
+from cloe_nessy.pipeline.pipeline_context import PipelineContext
+
+
+class TransformRegexExtract(PipelineAction):
+    r"""Extract values from a specified column in a DataFrame using regex patterns.
+
+    This action extracts values from a column based on a regex pattern and stores
+    the result in a new column. Optionally, you can replace the matched pattern in
+    the original column with a different string, remove the original column, or add
+    a boolean column indicating which rows matched the pattern.
+
+    Example:
+        ```yaml
+        Extract Action:
+            action: TRANSFORM_REGEX_EXTRACT
+            options:
+                source_column_name: Email
+                extract_column_name: org_domain
+                pattern: (?<=@)([A-Za-z0-9-]+)
+                replace_by: exampledomain.org
+        ```
+
+    This action also supports processing multiple columns simultaneously. To use this
+    functionality, structure the configuration as a dictionary mapping each source
+    column name to its extraction parameters.
+
+    Example:
+        ```yaml
+        Extract Action:
+            action: TRANSFORM_REGEX_EXTRACT
+            options:
+                extract_columns:
+                    Name:
+                        pattern: (?<=\w+) (\w+)
+                        replace_by: ''
+                        extract_column_name: last_name
+                        match_info_column_name: has_last_name
+                    Email:
+                        pattern: @\w+\.\w+
+                        extract_column_name: domain
+                        keep_original_column: False
+        ```
+
+    """
+
+    name: str = "TRANSFORM_REGEX_EXTRACT"
+
+    def run(
+        self,
+        context: PipelineContext,
+        source_column_name: str = "",
+        extract_column_name: str = "",
+        pattern: str = "",
+        keep_original_column: bool = True,
+        replace_by: str = "",
+        match_info_column_name: str = "",
+        extract_columns: dict | None = None,
+        **_: Any,
+    ) -> PipelineContext:
+        """Performs a regex extract (and replace) on a specified column in a DataFrame.
+
+        This function performs a regex extract (and optionally a replace) on one or more columns.
+
+        Args:
+            context: The context in which this action is executed.
+            source_column_name: Column name to perform the regex replace on.
+            pattern: Regex pattern to match.
+            replace_by: String that should replace the extracted pattern in the source column.
+            extract_column_name: Column name to store the extract, default: <source_column_name>_extract
+            keep_original_column: Whether to keep the original column, default: True
+            match_info_column_name: Column name to store a boolean column whether a match was found, default: None
+            extract_columns: Dictionary of column names and their corresponding 1-column-case.
+
+        Raises:
+            ValueError: If any of the required arguments are not provided.
+            ValueError: If the regex pattern is invalid.
+
+        Returns:
+            PipelineContext: Transformed context with the modified DataFrame.
+        """
+        if context.data is None:
+            raise ValueError("Data from the context is required for the operation.")
+        if not extract_columns and not source_column_name:
+            raise ValueError("Either extract_columns or source_column_name must be provided.")
+
+        df = context.data
+
+        if source_column_name:
+            self._console_logger.info(f"Extracting from column '{source_column_name}' using pattern: {pattern}")
+            df = self._process_one_column(
+                df,
+                source_column_name,
+                pattern,
+                extract_column_name,
+                replace_by,
+                keep_original_column,
+                match_info_column_name,
+            )
+
+        elif isinstance(extract_columns, dict):
+            self._console_logger.info(f"Extracting from {len(extract_columns)} columns")
+            for one_source_column_name in extract_columns:
+                parameter_dict = self._get_default_dict() | extract_columns[one_source_column_name]
+                df = self._process_one_column(df, one_source_column_name, **parameter_dict)
+
+        else:
+            raise ValueError("extract_columns must be a dictionary. See documentation for proper format.")
+
+        return context.from_existing(data=df)
+
+    def _process_one_column(
+        self,
+        df,
+        source_column_name,
+        pattern,
+        extract_column_name,
+        replace_by,
+        keep_original_column,
+        match_info_column_name,
+    ):
+        # Extract the first captured group (group 0 is the entire match)
+        matched_group_id = 0
+
+        if not extract_column_name:
+            extract_column_name = f"{source_column_name}_extracted"
+
+        if not pattern:
+            raise ValueError(f"The regex pattern (pattern) for column {source_column_name} must be provided.")
+
+        # Validate regex pattern
+        try:
+            re.compile(pattern)
+        except re.error as e:
+            raise ValueError(f"Invalid regex pattern '{pattern}' for column {source_column_name}: {e}") from e
+
+        df = df.withColumn(extract_column_name, F.regexp_extract(source_column_name, pattern, matched_group_id))
+
+        if replace_by:
+            df = df.withColumn(source_column_name, F.regexp_replace(source_column_name, pattern, replace_by))
+
+        if match_info_column_name:
+            # Check if extraction is null or empty string
+            df = df.withColumn(
+                match_info_column_name,
+                F.when((F.col(extract_column_name).isNull()) | (F.col(extract_column_name) == ""), False).otherwise(
+                    True
+                ),
+            )
+
+        if not keep_original_column:
+            df = df.drop(source_column_name)
+
+        return df
+
+    def _get_default_dict(self) -> dict[str, Any]:
+        """Return default parameters for single column extraction."""
+        return {
+            "pattern": "",
+            "extract_column_name": "",
+            "replace_by": "",
+            "keep_original_column": True,
+            "match_info_column_name": "",
+        }

{cloe_nessy-1.0.0.dist-info → cloe_nessy-1.0.1.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: cloe-nessy
-Version: 1.0.0
+Version: 1.0.1
 Summary: Your friendly datalake monster.
 Project-URL: homepage, https://initions.com/
 Author-email: initions <ICSMC_EXT_PYPIORG@accenture.com>
@@ -12,7 +12,7 @@ Classifier: License :: OSI Approved :: MIT License
 Classifier: Operating System :: OS Independent
 Classifier: Programming Language :: Python :: 3
 Classifier: Topic :: Database
-Requires-Python: <3.13,>=3.11
+Requires-Python: <3.14,>=3.11
 Requires-Dist: azure-identity<2.0.0,>=1.19.0
 Requires-Dist: cloe-logging[databricks,log-analytics]<0.4,>=0.3.8
 Requires-Dist: databricks-sdk<1.0.0,>=0.36.0

{cloe_nessy-1.0.0.dist-info → cloe_nessy-1.0.1.dist-info}/RECORD

@@ -30,9 +30,9 @@ cloe_nessy/integration/delta_loader/strategies/delta_timestamp_loader.py,sha256=
 cloe_nessy/integration/reader/__init__.py,sha256=NWQx-v6aKE8YOHhsxfeaZnMVq4KLKyRWXzUduf5aVsk,265
 cloe_nessy/integration/reader/api_reader.py,sha256=FbOyfLVG1ryL2GC-MgE1uClHICsQKBj9yZbY4TG5qrk,19637
 cloe_nessy/integration/reader/catalog_reader.py,sha256=DlnykmFjV_v8SCBh3qaCvf24QM-6TdMFVHx5Mqv7Nvs,4850
-cloe_nessy/integration/reader/excel_reader.py,sha256=JGmxQ16ux0HT-MLvAUp-9XMdKUToMb7cdObciZNsYSs,8027
+cloe_nessy/integration/reader/excel_reader.py,sha256=QXm0MaE_-tW5ix-f_3Pgn-Vx7VG5jA_uSp858rVV7lA,8042
 cloe_nessy/integration/reader/exceptions.py,sha256=_A9jFpe_RIDZCGY76qzjic9bsshxns6yXPSl141dq1c,203
-cloe_nessy/integration/reader/file_reader.py,sha256=t5zF-cmZo1X0a1rki6ry1rSiFEu5uXRP2rNGd90fwoY,8163
+cloe_nessy/integration/reader/file_reader.py,sha256=FFqqu1h003FY2Df3ru-G1JO4Bg2Ai8Rzh58fjOCN7NM,8262
 cloe_nessy/integration/reader/reader.py,sha256=YHriYkzsBduBjfI2FnP03VEo15a8UCRZ_sXtre8eaEs,1041
 cloe_nessy/integration/writer/__init__.py,sha256=3yzCAGiWZdQWtsbzlTih01sxVTJV2DDYwvl34lEAUlE,243
 cloe_nessy/integration/writer/catalog_writer.py,sha256=dQeXmtfs7J6rP6Ye3OCvxBraFScFX_3SHs7Md58hEeM,5296
@@ -74,7 +74,7 @@ cloe_nessy/pipeline/pipeline_context.py,sha256=eCOcjyE16rGRom3L85Gy_BbncfQD6i1x3
 cloe_nessy/pipeline/pipeline_parsing_service.py,sha256=eeC4RbGBILGN6zkbUyjH-qGgEMtOWV4Kv_VxrHbHMY0,9021
 cloe_nessy/pipeline/pipeline_plotting_service.py,sha256=goMQj73FzUVchKn5c2SsPcWR6fr7DtVkVrcQfJsKCq4,13111
 cloe_nessy/pipeline/pipeline_step.py,sha256=oTnlvRpB0fbOBQXbPe1URstA5fv-97igCHt_41fKCAk,2082
-cloe_nessy/pipeline/actions/__init__.py,sha256=Yf6-EoF_iTXOIOhgMN-GwhqH5DeaogUklVulh9OVj4s,2902
+cloe_nessy/pipeline/actions/__init__.py,sha256=FfAnSIl-0T6pnaWhClkDqV8nfTdvLvZZJdwycsZMLPw,2990
 cloe_nessy/pipeline/actions/read_api.py,sha256=MAc7QfmhnaRUMdE09Ywt41RSAsuW4co8zF0zXHwbM8U,16193
 cloe_nessy/pipeline/actions/read_catalog_table.py,sha256=sx3dezd33c1FawMrxORwhK5GNo1IpjCyuLATWz7esZ0,6735
 cloe_nessy/pipeline/actions/read_excel.py,sha256=IG_VmDEt1TvGVEO0SY9Fm3awHNjfisR1_7DUmhC3NEE,7968
@@ -93,6 +93,7 @@ cloe_nessy/pipeline/actions/transform_group_aggregate.py,sha256=KUHeeP-RIDi34dpb
 cloe_nessy/pipeline/actions/transform_hash_columns.py,sha256=M5_wolJwzJpPTSrZq4yWV3TH7H6BGqbjJkJCwtqPlQo,8507
 cloe_nessy/pipeline/actions/transform_join.py,sha256=ez1M1wVc9khOZj1swMArJbBKXxEpjenUHrW1wL8H330,7200
 cloe_nessy/pipeline/actions/transform_json_normalize.py,sha256=petF7pnNq1EKc8MqVdG0weFALAHNILSe_eAu4Z5XxIo,4833
+cloe_nessy/pipeline/actions/transform_regex_extract.py,sha256=vMtUW0s_oXy8DC1-4Xh-WQN3CCp8jXYsJiFYvGdYrqE,6390
 cloe_nessy/pipeline/actions/transform_rename_columns.py,sha256=4zJcPCONMU4C67qeuzsrX3AORRRHoq_selUI7FJyeg0,1952
 cloe_nessy/pipeline/actions/transform_replace_values.py,sha256=1OPHTrjcphfyGepcO7ozYfeqfwA18pjlyHpVKUS_AAU,2049
 cloe_nessy/pipeline/actions/transform_select_columns.py,sha256=-GhSEsb7iNnZIsYRm3BG9BX4_qUDJMbpj1DsKPY046w,4574
@@ -112,6 +113,6 @@ cloe_nessy/settings/settings.py,sha256=I4n129lrujriW-d8q4as2Kb4_kI932ModfZ5Ow_Up
 cloe_nessy/utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 cloe_nessy/utils/column_names.py,sha256=dCNtm61mc5aLkY2oE4rlfN3VLCrpot6fOESjAZmCmhA,361
 cloe_nessy/utils/file_and_directory_handler.py,sha256=r2EVt9xG81p6ScaJCwETC5an6pMT6WseB0jMOR-JlpU,602
-cloe_nessy-1.0.0.dist-info/METADATA,sha256=jaztxDdomowmj0qTJ4i5jtF0dE_T5TmqA8j9zywl0U0,3291
-cloe_nessy-1.0.0.dist-info/WHEEL,sha256=WLgqFyCfm_KASv4WHyYy0P3pM_m7J5L9k2skdKLirC8,87
-cloe_nessy-1.0.0.dist-info/RECORD,,
+cloe_nessy-1.0.1.dist-info/METADATA,sha256=qLn3XYfGsw2pW-pPtUUidtcHZiUtIwOESWY8LCenGEY,3291
+cloe_nessy-1.0.1.dist-info/WHEEL,sha256=WLgqFyCfm_KASv4WHyYy0P3pM_m7J5L9k2skdKLirC8,87
+cloe_nessy-1.0.1.dist-info/RECORD,,