datapipelab 0.2.1__py3-none-any.whl → 0.2.3__py3-none-any.whl

datapipelab/app/node/sink/hive_node.py

@@ -1,5 +1,5 @@
 from datapipelab.app.node.tnode import TNode
-
+from datapipelab.logger import logger
 
 class HiveSinkNode(TNode):
     def __init__(self, spark, tnode_config, df):
@@ -20,6 +20,7 @@ class HiveSinkNode(TNode):
                 self.df = self.df.repartition(int(self.partition_count))
             else:
                 self.df = self.df.repartition(int(self.partition_count), self.partition_by)
+        logger.info("Start writing to Hive")
         self.df.write.insertInto(f'{self.database_name}.{self.table_name}', overwrite=self.overwrite)
 
     def _process(self):

datapipelab/app/node/sink/spark_node.py

@@ -0,0 +1,27 @@
+from datapipelab.app.node.tnode import TNode
+from datapipelab.logger import logger
+
+class SparkSinkNode(TNode):
+    def __init__(self, spark, tnode_config, df):
+        from pyspark.sql import DataFrame
+        super().__init__(spark=spark)
+        self.mode = tnode_config.get('mode', None)
+        self.stream = tnode_config.get('stream', None)
+        self.database_name = tnode_config['options']['database']
+        self.table_name = tnode_config['options']['table']
+        self.partition_by = tnode_config['options'].get('partition_by', None)
+        self.partition_count = tnode_config['options'].get('partition_count', None)
+        self.overwrite = tnode_config['options']['overwrite']
+        self.df = df
+
+    def __write_dynamic_partition(self):
+        if self.partition_count:
+            if self.partition_by:
+                self.df = self.df.repartition(int(self.partition_count))
+            else:
+                self.df = self.df.repartition(int(self.partition_count), self.partition_by)
+        logger.info("Start writing to Hive")
+        self.df.write.insertInto(f'{self.database_name}.{self.table_name}', overwrite=self.overwrite)
+
+    def _process(self):
+        self.__write_dynamic_partition()

datapipelab/pipeline_handler.py

@@ -1,5 +1,6 @@
 from datapipelab.app.node.processor.custom_node import CustomNode
 from datapipelab.app.node.processor.shell_node import ShellProcessorNode
+from datapipelab.app.node.source.hive_node import HiveSourceNode
 from datapipelab.app.node.source.spark_node import SparkSourceNode
 from datapipelab.app.node.source.delta_node import DeltaSourceNode
 from datapipelab.app.node.processor.spark_node import SparkProcessorNode
@@ -33,6 +34,9 @@ class PipelineHandler:
         if input_type == 'spark':
             if input_format == 'spark':
                 source_df = SparkSourceNode(self.spark, tnode_config).run()
+        if input_type == 'hive':
+            if input_format == 'hive':
+                source_df = HiveSourceNode(self.spark, tnode_config).run()
         if input_type == 'adls_path':
             if input_format == 'delta':
                 source_df = DeltaSourceNode(self.spark, tnode_config).run()
@@ -84,6 +88,10 @@ class PipelineHandler:
             if tnode_format == 'hive':
                 from datapipelab.app.node.sink import hive_node
                 processor_df = hive_node.HiveSinkNode(self.spark, tnode_config, t_df[tnode_name_df]).run()
+        if tnode_type == 'spark':
+            if tnode_format == 'spark':
+                from datapipelab.app.node.sink import spark_node
+                processor_df = hive_node.HiveSinkNode(self.spark, tnode_config, t_df[tnode_name_df]).run()
 
 
 

{datapipelab-0.2.1.dist-info → datapipelab-0.2.3.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: datapipelab
-Version: 0.2.1
+Version: 0.2.3
 Summary: A data pipeline library with connectors, sources, processors, and sinks.
 Requires-Dist: json5
 Requires-Dist: loguru

{datapipelab-0.2.1.dist-info → datapipelab-0.2.3.dist-info}/RECORD

@@ -3,7 +3,7 @@ datapipelab/engine.py,sha256=3QRsedRYNov6xIDOZ1tukinFE-SKv39Fn3sNCnD3L6g,442
 datapipelab/logger.py,sha256=Ugv0A4TfD3JWCWXNWu0lURcnfAEyuVrK3IrvVVgcHBo,864
 datapipelab/pipeline.py,sha256=dw9D9KM_hztt9g_YzqoNgQBRyCYR92cRZwrU5duP_Pg,1464
 datapipelab/pipeline_config.py,sha256=2bFAJepViE7rT7CaRANZU07aeQpOYcZ954ISujm9pXA,3816
-datapipelab/pipeline_handler.py,sha256=mkSvEo93IqmqJKbMc9R500C8ao7JFQCyPJ813B3COOc,4303
+datapipelab/pipeline_handler.py,sha256=bzlzd5fbkhY06WxPTyLmE1wyl2ANBgjWrQtY_cDDJDA,4761
 datapipelab/app/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 datapipelab/app/connector_node/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 datapipelab/app/node/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -18,14 +18,15 @@ datapipelab/app/node/processor/spark_node.py,sha256=jzqdffIHUCgOfMFcoqjXdl8wFag-
 datapipelab/app/node/sink/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 datapipelab/app/node/sink/csv_node.py,sha256=d2hyufP5_Nmql0pfD0KeC4rFu1wXTnBxVsoGl7sWbhM,1681
 datapipelab/app/node/sink/delta_node.py,sha256=iKEdiTjJ7SHJZMrbm0jR5tms5JZ5iCFfQklZbI-Yr2o,2044
-datapipelab/app/node/sink/hive_node.py,sha256=hodxZEUDuqnJgw4hraguuNe9bJwv2UFzTbffwCsGGJg,1138
+datapipelab/app/node/sink/hive_node.py,sha256=ycknOPBBwZGH3oHram_6LjHy-ygFjhuFNvVoPaNGaCU,1220
 datapipelab/app/node/sink/pandas_csv_node.py,sha256=JsJFt2XRpwxGeJyt_PDUgqZafiQROf1Sk5TUhQPxh4c,870
+datapipelab/app/node/sink/spark_node.py,sha256=tP3tZae2jzQtAtfIm8C-166WWSLdZs54mqoIyZOSy58,1221
 datapipelab/app/node/sink/teams_notification_node.py,sha256=6ZufdbhVvRXi3QTQafLo5uKl9kLyDnkYIE_VZFT0QNw,3581
 datapipelab/app/node/source/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 datapipelab/app/node/source/delta_node.py,sha256=gg7SfuKBAAfjk6OX2jNrot9XX61HoBe3us3D8O-dscE,529
 datapipelab/app/node/source/hive_node.py,sha256=h_AMCnnmon7uLRIGsaHAPWEReD3VaWZXnz9r0TpLGNM,478
 datapipelab/app/node/source/spark_node.py,sha256=S_x2atRFPDnXmhCUtcmaLc4BDFd2H4uQq6wnEJb7Uug,480
-datapipelab-0.2.1.dist-info/METADATA,sha256=fz0lesIoc1arto7rA5e9IKFw8Bgiofc3QmCcwyb0lN8,220
-datapipelab-0.2.1.dist-info/WHEEL,sha256=zaaOINJESkSfm_4HQVc5ssNzHCPXhJm0kEUakpsEHaU,91
-datapipelab-0.2.1.dist-info/top_level.txt,sha256=HgeBjHvXorKzvNqU5BNPutoI771HtiqVit9_-0Zyrb4,12
-datapipelab-0.2.1.dist-info/RECORD,,
+datapipelab-0.2.3.dist-info/METADATA,sha256=lCuzS6-kJgmWUKMRE68gSzgCcqpMvVK0krquERsparM,220
+datapipelab-0.2.3.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+datapipelab-0.2.3.dist-info/top_level.txt,sha256=HgeBjHvXorKzvNqU5BNPutoI771HtiqVit9_-0Zyrb4,12
+datapipelab-0.2.3.dist-info/RECORD,,

{datapipelab-0.2.1.dist-info → datapipelab-0.2.3.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: setuptools (80.8.0)
+Generator: setuptools (80.9.0)
 Root-Is-Purelib: true
 Tag: py3-none-any