datapipelab 0.2.2__tar.gz → 0.2.3__tar.gz

{datapipelab-0.2.2 → datapipelab-0.2.3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: datapipelab
-Version: 0.2.2
+Version: 0.2.3
 Summary: A data pipeline library with connectors, sources, processors, and sinks.
 Requires-Dist: json5
 Requires-Dist: loguru

datapipelab-0.2.3/datapipelab/app/node/sink/spark_node.py

@@ -0,0 +1,27 @@
+from datapipelab.app.node.tnode import TNode
+from datapipelab.logger import logger
+
+class SparkSinkNode(TNode):
+    def __init__(self, spark, tnode_config, df):
+        from pyspark.sql import DataFrame
+        super().__init__(spark=spark)
+        self.mode = tnode_config.get('mode', None)
+        self.stream = tnode_config.get('stream', None)
+        self.database_name = tnode_config['options']['database']
+        self.table_name = tnode_config['options']['table']
+        self.partition_by = tnode_config['options'].get('partition_by', None)
+        self.partition_count = tnode_config['options'].get('partition_count', None)
+        self.overwrite = tnode_config['options']['overwrite']
+        self.df = df
+
+    def __write_dynamic_partition(self):
+        if self.partition_count:
+            if self.partition_by:
+                self.df = self.df.repartition(int(self.partition_count))
+            else:
+                self.df = self.df.repartition(int(self.partition_count), self.partition_by)
+        logger.info("Start writing to Hive")
+        self.df.write.insertInto(f'{self.database_name}.{self.table_name}', overwrite=self.overwrite)
+
+    def _process(self):
+        self.__write_dynamic_partition()

{datapipelab-0.2.2 → datapipelab-0.2.3}/datapipelab/pipeline_handler.py

@@ -88,6 +88,10 @@ class PipelineHandler:
             if tnode_format == 'hive':
                 from datapipelab.app.node.sink import hive_node
                 processor_df = hive_node.HiveSinkNode(self.spark, tnode_config, t_df[tnode_name_df]).run()
+        if tnode_type == 'spark':
+            if tnode_format == 'spark':
+                from datapipelab.app.node.sink import spark_node
+                processor_df = hive_node.HiveSinkNode(self.spark, tnode_config, t_df[tnode_name_df]).run()
 
 
 

{datapipelab-0.2.2 → datapipelab-0.2.3}/datapipelab.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: datapipelab
-Version: 0.2.2
+Version: 0.2.3
 Summary: A data pipeline library with connectors, sources, processors, and sinks.
 Requires-Dist: json5
 Requires-Dist: loguru

{datapipelab-0.2.2 → datapipelab-0.2.3}/datapipelab.egg-info/SOURCES.txt

@@ -28,6 +28,7 @@ datapipelab/app/node/sink/csv_node.py
 datapipelab/app/node/sink/delta_node.py
 datapipelab/app/node/sink/hive_node.py
 datapipelab/app/node/sink/pandas_csv_node.py
+datapipelab/app/node/sink/spark_node.py
 datapipelab/app/node/sink/teams_notification_node.py
 datapipelab/app/node/source/__init__.py
 datapipelab/app/node/source/delta_node.py

{datapipelab-0.2.2 → datapipelab-0.2.3}/setup.py

@@ -2,7 +2,7 @@ from setuptools import setup, find_packages
 
 setup(
     name='datapipelab',
-    version='0.2.2',
+    version='0.2.3',
     description='A data pipeline library with connectors, sources, processors, and sinks.',
     packages=find_packages(),
     include_package_data=True,