dlt-utils-lib 0.4__tar.gz → 0.6__tar.gz

{dlt_utils_lib-0.4/dlt_utils_lib.egg-info → dlt_utils_lib-0.6}/PKG-INFO

@@ -1,5 +1,5 @@
 Metadata-Version: 2.1
 Name: dlt_utils_lib
-Version: 0.4
+Version: 0.6
 Requires-Dist: pyspark
 Requires-Dist: delta-spark

dlt_utils_lib-0.6/dlt_utils_lib.egg-info/PKG-INFO

@@ -0,0 +1,5 @@
+Metadata-Version: 2.1
+Name: dlt_utils_lib
+Version: 0.6
+Requires-Dist: pyspark
+Requires-Dist: delta-spark

{dlt_utils_lib-0.4 → dlt_utils_lib-0.6}/dlt_utils_lib.egg-info/SOURCES.txt

@@ -1,9 +1,9 @@
 setup.py
 dlt_utils/__init__.py
 dlt_utils/dlt_transformation.py
-dlt_utils/main_dlt_utils.py
 dlt_utils_lib.egg-info/PKG-INFO
 dlt_utils_lib.egg-info/SOURCES.txt
 dlt_utils_lib.egg-info/dependency_links.txt
 dlt_utils_lib.egg-info/requires.txt
+dlt_utils_lib.egg-info/tmpz2kz6zic
 dlt_utils_lib.egg-info/top_level.txt

{dlt_utils_lib-0.4 → dlt_utils_lib-0.6}/setup.py

@@ -2,7 +2,7 @@ from setuptools import setup, find_packages
 
 setup(
     name='dlt_utils_lib',
-    version='0.4',
+    version='0.6',
     packages=find_packages(),
     install_requires=[
         'pyspark',

dlt_utils_lib-0.4/dlt_utils/main_dlt_utils.py

@@ -1,59 +0,0 @@
-import dlt
-from pyspark.sql.functions import col, expr, greatest, when, datediff, abs as spark_abs
-from dlt_transformations import update_cdc_timestamp, apply_partitions
-from pyspark.sql.types import TimestampType, DateType
-from pyspark.sql import DataFrame
-from pyspark.sql import SparkSession
-
-# Assuming SparkSession is already created in your Databricks notebook
-spark = SparkSession.builder.appName("DLT Pipeline").getOrCreate()
-spark.conf.set("pipelines.tableManagedByMultiplePipelinesCheck.enabled", "false")
-
-
-# Define the transformation function dynamically
-def create_bronze_table_definition(table_name: str, files_path: str, file_format: str, partitions: dict, schema_exclude_columns: list):
-    @dlt.table(
-        name=f"bronze_{table_name}",
-        comment="This is the bronze table.",
-        temporary=False
-    )
-    def transform_cdc_to_bronze():
-        df = spark.read.format(file_format).load(files_path)
-        fields = [field for field in df.schema.fields if field.name not in schema_exclude_columns]
-        schema_string = ', '.join([f"{field.name} {field.dataType.simpleString()}" for field in fields])
-        return spark \
-            .readStream \
-            .format('cloudFiles') \
-            .option("cloudFiles.format", file_format) \
-            .option("cloudFiles.schemaHints", schema_string) \
-            .load(files_path) \
-            .withColumn('cdc_timestamp', col('cdc_timestamp').cast('timestamp')) \
-            .transform(update_cdc_timestamp) \
-            .transform(lambda df: apply_partitions(df, partitions))
-
-    return transform_cdc_to_bronze
-
-
-
-# Create silver streaming data
-def silver_streaming_process(table_name: str, keys: list, partitions: dict, exclude_columns: list):
-    dlt.create_streaming_table(
-        name=table_name,
-        table_properties={
-            "delta.autoOptimize.optimizeWrite": "true",
-            "delta.autoOptimize.autoCompact": "true"
-        },
-        comment = "This is the silver table with source in",
-        partition_cols=partitions if partitions else None
-    )
-
-    dlt.apply_changes(
-        target=table_name,
-        source=f"bronze_{table_name}",
-        keys=keys,
-        sequence_by=col("cdc_timestamp"),
-        apply_as_deletes=expr("Op = 'D'"),
-        apply_as_truncates=expr("Op = 'T'"),
-        except_column_list=["Op", "_rescued_data"] + exclude_columns,
-        stored_as_scd_type=1
-    )