cloe-nessy 0.3.16.7__py3-none-any.whl → 0.3.17.0__py3-none-any.whl

cloe_nessy/pipeline/actions/__init__.py

@@ -9,6 +9,7 @@ from .read_metadata_yaml import ReadMetadataYAMLAction
 from .transform_change_datatype import TransformChangeDatatypeAction
 from .transform_clean_column_names import TransformCleanColumnNamesAction
 from .transform_concat_columns import TransformConcatColumnsAction
+from .transform_convert_timestamp import TransformConvertTimestampAction
 from .transform_decode import TransformDecodeAction
 from .transform_deduplication import TransformDeduplication
 from .transform_distinct import TransformDistinctAction
@@ -46,6 +47,7 @@ __all__ = [
     "TransformChangeDatatypeAction",
     "TransformCleanColumnNamesAction",
     "TransformConcatColumnsAction",
+    "TransformConvertTimestampAction",
     "TransformDecodeAction",
     "TransformDeduplication",
     "TransformDistinctAction",

cloe_nessy/pipeline/actions/transform_convert_timestamp.py

@@ -0,0 +1,97 @@
+from typing import Any
+
+from pyspark.errors.exceptions.connect import IllegalArgumentException
+from pyspark.sql import functions as F
+from pyspark.sql.utils import AnalysisException
+
+from ..pipeline_action import PipelineAction
+from ..pipeline_context import PipelineContext
+
+
+class TransformConvertTimestampAction(PipelineAction):
+    """This action performs timestamp based conversions.
+
+    Example:
+        ```yaml
+        Convert Timestamp:
+            action: TRANSFORM_CONVERT_TIMESTAMP
+            options:
+                columns:
+                    - date
+                    - creation_timestamp
+                    - current_ts
+                source_format: unixtime_ms
+                target_format: timestamp
+        ```
+    """
+
+    name: str = "TRANSFORM_CONVERT_TIMESTAMP"
+
+    def run(
+        self,
+        context: PipelineContext,
+        *,
+        columns: list[str] | str | None = None,
+        source_format: str = "",
+        target_format: str = "",
+        **_: Any,
+    ) -> PipelineContext:
+        """Converts column(s) from a given source format to a new format.
+
+        Args:
+            context: Context in which this Action is executed.
+            columns: A column name or a list of column names that should be converted.
+            source_format: Initial format type of the column.
+            target_format: Desired format type of the column.
+                This also supports passing a format string like `yyyy-MM-dd HH:mm:ss`.
+
+        Raises:
+            ValueError: If no column, source_format or target_format are provided.
+            ValueError: If source_format or target_format are not supported.
+
+        Returns:
+            PipelineContext: Context after the execution of this Action.
+        """
+        if not columns:
+            raise ValueError("No column names provided.")
+        if not source_format:
+            raise ValueError("No source_format provided.")
+        if not target_format:
+            raise ValueError("No target_format provided.")
+        if context.data is None:
+            raise ValueError("Context DataFrame is required.")
+        df = context.data
+
+        columns = [columns] if isinstance(columns, str) else columns
+
+        match source_format:
+            # convert always to timestamp first
+            case "string" | "date" | "unixtime":
+                for column in columns:
+                    df = df.withColumn(column, F.to_timestamp(F.col(column)))
+            case "unixtime_ms":
+                for column in columns:
+                    df = df.withColumn(column, F.to_timestamp(F.col(column) / 1000))
+            case "timestamp":
+                pass
+            case _:
+                raise ValueError(f"Unknown source_format {source_format}")
+
+        match target_format:
+            # convert from timestamp to desired output type and format
+            case "timestamp":
+                pass
+            case "unixtime":
+                for column in columns:
+                    df = df.withColumn(column, F.to_unix_timestamp(F.col(column)))
+            case "date":
+                for column in columns:
+                    df = df.withColumn(column, F.to_date(F.col(column)))
+            case _:
+                try:
+                    for column in columns:
+                        df = df.withColumn(column, F.date_format(F.col(column), target_format))
+                except (IllegalArgumentException, AnalysisException) as e:
+                    raise ValueError(f"Invalid target_format {target_format}") from e
+
+        return context.from_existing(data=df)

{cloe_nessy-0.3.16.7.dist-info → cloe_nessy-0.3.17.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: cloe-nessy
-Version: 0.3.16.7
+Version: 0.3.17.0
 Summary: Your friendly datalake monster.
 Author-email: initions <ICSMC_EXT_PYPIORG@accenture.com>
 License: MIT
@@ -58,12 +58,12 @@ Extract-Transform-Load (ETL) Workflow.
 
 When you are contributing, please refer to our Contribution Guide in the *nessy*
 Docs
-[here](https://white-rock-0cabbc003.1.azurestaticapps.net/tool_docs/nessy/Developer-Guide/)!
+[here](https://yellow-mud-0b9177e03.2.azurestaticapps.net/tool_docs/nessy/Developer-Guide/)!
 
 ## Usage
 
 Please find the User Guide
-[here](https://white-rock-0cabbc003.1.azurestaticapps.net/tool_docs/nessy/User-Guide/)!
+[here](https://yellow-mud-0b9177e03.2.azurestaticapps.net/tool_docs/nessy/User-Guide/)!
 
 ## Contact
 

{cloe_nessy-0.3.16.7.dist-info → cloe_nessy-0.3.17.0.dist-info}/RECORD

@@ -71,7 +71,7 @@ cloe_nessy/pipeline/pipeline_context.py,sha256=csElDc6BsynDUtRXgQOSCH7ONc_b-ag0Y
 cloe_nessy/pipeline/pipeline_parsing_service.py,sha256=c_nAsgw81QYBM9AFiTxGgqRhNXABkDKplbeoCJPtbpE,6434
 cloe_nessy/pipeline/pipeline_plotting_service.py,sha256=goMQj73FzUVchKn5c2SsPcWR6fr7DtVkVrcQfJsKCq4,13111
 cloe_nessy/pipeline/pipeline_step.py,sha256=UlnmpS6gm_dZ7m9dD1mZvye7mvUF_DA7HjOZo0oGYDU,1977
-cloe_nessy/pipeline/actions/__init__.py,sha256=RAGwu3Xzt2JJc0AveZ_hVi3SxTIdehAG-JoQe9JPMNA,2690
+cloe_nessy/pipeline/actions/__init__.py,sha256=RZ1UVSn9v88F4GKgHy6UYDzx8zSAMQScJLCeiHO5f8A,2802
 cloe_nessy/pipeline/actions/read_api.py,sha256=RBv5XeHtjTXuCP09Fqo6JNx6iIhQQI-nuAHCuSaGs2s,7778
 cloe_nessy/pipeline/actions/read_catalog_table.py,sha256=TBlJaXJAQwLtwvh7dXsX9ebNN3rS6En6951MnT8xGG8,4101
 cloe_nessy/pipeline/actions/read_excel.py,sha256=Mhl3r_2Hqk2XN7Fl5WqqAyE4JdnwSiivbhWMglyBtkE,7961
@@ -80,6 +80,7 @@ cloe_nessy/pipeline/actions/read_metadata_yaml.py,sha256=3ZDy9qiDYtM1oDQzHPC23hL
 cloe_nessy/pipeline/actions/transform_change_datatype.py,sha256=24Tn6R3TvUkWCh8V6naLdyNbCbqvyPOOoer-hy_Ebq4,2077
 cloe_nessy/pipeline/actions/transform_clean_column_names.py,sha256=VxvWqENW63c50L96JA1V_ioe4By6gGzx_iY86njOXEM,3044
 cloe_nessy/pipeline/actions/transform_concat_columns.py,sha256=Nk8YbhxDnFZsWzW9Dj5Yl76Uq6VrcMlevQPHGms65L8,3777
+cloe_nessy/pipeline/actions/transform_convert_timestamp.py,sha256=2SL078tBcOmytDbt-cR81jZbclwqELsUB4XDLjaCnNo,3579
 cloe_nessy/pipeline/actions/transform_decode.py,sha256=JajMwHREtxa8u_1Q3RZDBVMjncoSel-WzQFVTO0MREg,4455
 cloe_nessy/pipeline/actions/transform_deduplication.py,sha256=SfTDrOL0TNSC4wITbozabC0jYvceTLnqU4urnEjYk9g,4910
 cloe_nessy/pipeline/actions/transform_distinct.py,sha256=c7aBxANyqT4aKhm0cSELDtD-bP0Se9vxlBF0K4AgQWs,1976
@@ -106,7 +107,7 @@ cloe_nessy/settings/settings.py,sha256=I4n129lrujriW-d8q4as2Kb4_kI932ModfZ5Ow_Up
 cloe_nessy/utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 cloe_nessy/utils/column_names.py,sha256=dCNtm61mc5aLkY2oE4rlfN3VLCrpot6fOESjAZmCmhA,361
 cloe_nessy/utils/file_and_directory_handler.py,sha256=r2EVt9xG81p6ScaJCwETC5an6pMT6WseB0jMOR-JlpU,602
-cloe_nessy-0.3.16.7.dist-info/METADATA,sha256=Rz35trsVqlMw7hkcDBWFr-f9EOtqII9EwZogBABy6gw,3292
-cloe_nessy-0.3.16.7.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-cloe_nessy-0.3.16.7.dist-info/top_level.txt,sha256=Z7izn8HmQpg2wBUb-0jzaKlYKMU7Ypzuc9__9vPtW_I,11
-cloe_nessy-0.3.16.7.dist-info/RECORD,,
+cloe_nessy-0.3.17.0.dist-info/METADATA,sha256=hR0GqdboYwzBrbZY_ese9kt250DIOHgMlAj3QOqLhF8,3292
+cloe_nessy-0.3.17.0.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+cloe_nessy-0.3.17.0.dist-info/top_level.txt,sha256=Z7izn8HmQpg2wBUb-0jzaKlYKMU7Ypzuc9__9vPtW_I,11
+cloe_nessy-0.3.17.0.dist-info/RECORD,,