cloe-nessy 0.3.13.4b0__py3-none-any.whl → 0.3.13.5b0__py3-none-any.whl

cloe_nessy/models/column.py

@@ -5,6 +5,7 @@ from pydantic import BaseModel, Field, field_validator, model_validator
 
 COLUMN_DATA_TYPE_LIST = {
     "string",
+    "decimal",
     "integer",
     "int",
     "smallint",
@@ -43,7 +44,7 @@ class Column(BaseModel):
         """
         val = raw.lower()
         base_data_types = re.findall(r"\b[a-z]+\b", val)
-        forbidden_characters = re.findall(r"[^a-z\<\>)]+", val)
+        forbidden_characters = re.findall(r"[^a-z0-9\(\)\<\>,\s]+", val)
 
         if forbidden_characters:
             raise ValueError(f"Forbidden characters in data type definition [ '{val}' ]: [' {forbidden_characters} ']")

cloe_nessy/pipeline/actions/transform_convert_timestamp.py

@@ -0,0 +1,87 @@
+from typing import Any
+
+from pyspark.errors.exceptions.base import IllegalArgumentException
+from pyspark.sql import functions as F
+
+from ...pipeline import PipelineAction, PipelineContext
+
+
+class TransformConvertTimestampAction(PipelineAction):
+    """This class implements a Transform action for an ETL pipeline.
+
+    This action performs timestamp based conversions.
+
+    Example:
+        ```yaml
+        Convert Timestamp:
+            action: TRANSFORM_CONVERT_TIMESTAMP
+            options:
+                column: my_timestamp_column
+                source_format: unixtime
+                target_format: yyyy-MM-dd HH:mm:ss
+        ```
+    """
+
+    name: str = "TRANSFORM_CONVERT_TIMESTAMP"
+
+    def run(
+        self,
+        context: PipelineContext,
+        *,
+        column: str = "",
+        source_format: str = "",
+        target_format: str = "",
+        **_: Any,
+    ) -> PipelineContext:
+        """Converts a column from a given source format to a new format.
+
+        Args:
+            context: Context in which this Action is executed.
+            column: The column that should be converted.
+            source_format: Initial format type of the column.
+            target_format: Desired format type of the column. This also supports
+            passing a format string like 'yyyy-MM-dd HH:mm:ss'.
+
+        Raises:
+            ValueError: If no column, source_format and target_format are provided.
+            ValueError: If source_format or target_format are not supported.
+
+        Returns:
+            PipelineContext: Context after the execution of this Action.
+        """
+        if not column:
+            raise ValueError("No column provided.")
+        if not source_format:
+            raise ValueError("No source_format provided.")
+        if not target_format:
+            raise ValueError("No target_format provided.")
+        if context.data is None:
+            raise ValueError("Context DataFrame is required.")
+        df = context.data
+
+        match source_format:
+            # convert always to timestamp first
+            case "unixtime":
+                df = df.withColumn(column, F.from_unixtime(F.col(column)))
+            case "unixtime_ms":
+                df = df.withColumn(column, F.to_timestamp(F.col(column) / 1000))
+            case "string":
+                df = df.withColumn(column, F.to_timestamp(F.col(column)))
+            case "timestamp":
+                pass
+            case _:
+                raise ValueError(f"Unknown source_format {source_format}")
+
+        match target_format:
+            # convert from timestamp to desired output format
+            case "timestamp":
+                pass
+            case "unixtime":
+                df = df.withColumn(column, F.to_unix_timestamp(F.col(column)))
+            case _:
+                try:
+                    df = df.withColumn(column, F.date_format(F.col(column), target_format))
+                except IllegalArgumentException as e:
+                    raise ValueError(f"Invalid target_format {target_format}") from e
+
+        return context.from_existing(data=df)

{cloe_nessy-0.3.13.4b0.dist-info → cloe_nessy-0.3.13.5b0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: cloe-nessy
-Version: 0.3.13.4b0
+Version: 0.3.13.5b0
 Summary: Your friendly datalake monster.
 Project-URL: homepage, https://initions.com/
 Author-email: initions <ICSMC_EXT_PYPIORG@accenture.com>
@@ -16,6 +16,7 @@ Requires-Python: <3.13,>=3.11
 Requires-Dist: azure-identity<2.0.0,>=1.19.0
 Requires-Dist: cloe-logging[databricks,log-analytics]<0.4,>=0.3.8
 Requires-Dist: databricks-sdk<1.0.0,>=0.36.0
+Requires-Dist: delta-spark>=3.3.2
 Requires-Dist: fsspec<2025.6.0,>=2025.5.1
 Requires-Dist: httpx<1.0.0,>=0.27.2
 Requires-Dist: jinja2<4.0.0,>=3.1.4

{cloe_nessy-0.3.13.4b0.dist-info → cloe_nessy-0.3.13.5b0.dist-info}/RECORD

@@ -38,7 +38,7 @@ cloe_nessy/logging/__init__.py,sha256=ySVCVbdyR3Dno_tl2ZfiER_7EVaDoQMHVkNyfdMZum
 cloe_nessy/logging/logger_mixin.py,sha256=9iy7BF6drYme-f7Rrt_imbVBRgVqQ89xjcP1X5aMtfY,7467
 cloe_nessy/models/__init__.py,sha256=-FmWEJ1Oq1njSopjc0R7GmT64mLSmALkm8PkHNzy9Y8,327
 cloe_nessy/models/catalog.py,sha256=ayC1sMp4cNLAZtu0ICVV3Us6-o4hn8U9tpzzvxC9RAs,177
-cloe_nessy/models/column.py,sha256=53fBwRnino72XKACsHZpN9QfCBqqSXyKLHZlM0huumg,1988
+cloe_nessy/models/column.py,sha256=8wR7E8PRhUc0dwM83IIlpz7kBncZim7J5FvQzd8R_Us,2012
 cloe_nessy/models/constraint.py,sha256=hsFlhn4n928z81O3dl3v5bMetewPWzMjkJK3_4kASSM,178
 cloe_nessy/models/foreign_key.py,sha256=DwRVHs9sShqqPV-NL7ow_3AmPPWX0Od26yZn_I565pU,1001
 cloe_nessy/models/schema.py,sha256=cNSrH7K4hLRrkg1E6fW6DUIBMZdR2A5B21POj5iQ4GA,3429
@@ -71,6 +71,7 @@ cloe_nessy/pipeline/actions/read_metadata_yaml.py,sha256=i8fQceV63eAqx_x0ANisCkX
 cloe_nessy/pipeline/actions/transform_change_datatype.py,sha256=24Tn6R3TvUkWCh8V6naLdyNbCbqvyPOOoer-hy_Ebq4,2077
 cloe_nessy/pipeline/actions/transform_clean_column_names.py,sha256=-CEdcXb7Fz5DQNitGlJ8EVBE_LzxfsInyCIO-D7b4iY,3042
 cloe_nessy/pipeline/actions/transform_concat_columns.py,sha256=Nk8YbhxDnFZsWzW9Dj5Yl76Uq6VrcMlevQPHGms65L8,3777
+cloe_nessy/pipeline/actions/transform_convert_timestamp.py,sha256=je6H-mtNeokU9W_-RCWaRCFvMhk4oQL9s60FVBrl8Po,3090
 cloe_nessy/pipeline/actions/transform_decode.py,sha256=JajMwHREtxa8u_1Q3RZDBVMjncoSel-WzQFVTO0MREg,4455
 cloe_nessy/pipeline/actions/transform_deduplication.py,sha256=E0ypz9qkHMSatNfnHekP-E6svQVL149M4PV02M03drg,5099
 cloe_nessy/pipeline/actions/transform_distinct.py,sha256=c7aBxANyqT4aKhm0cSELDtD-bP0Se9vxlBF0K4AgQWs,1976
@@ -94,6 +95,6 @@ cloe_nessy/settings/__init__.py,sha256=ZbkneO3WaKOxon7qHFHnou7EnBOSnBFyKMDZblIEv
 cloe_nessy/settings/settings.py,sha256=I4n129lrujriW-d8q4as2Kb4_kI932ModfZ5Ow_UpVM,3653
 cloe_nessy/utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 cloe_nessy/utils/file_and_directory_handler.py,sha256=r2EVt9xG81p6ScaJCwETC5an6pMT6WseB0jMOR-JlpU,602
-cloe_nessy-0.3.13.4b0.dist-info/METADATA,sha256=Z9Bby2T96l0nm453ZwbiAq3YDP0uau5GfPORfD9KE7E,3294
-cloe_nessy-0.3.13.4b0.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
-cloe_nessy-0.3.13.4b0.dist-info/RECORD,,
+cloe_nessy-0.3.13.5b0.dist-info/METADATA,sha256=PBFKdmm5_n8bAarqbddj81pIjctNxHIgQPw72Lru01M,3328
+cloe_nessy-0.3.13.5b0.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+cloe_nessy-0.3.13.5b0.dist-info/RECORD,,