datapipelab 0.2.0__tar.gz → 0.2.2__tar.gz

{datapipelab-0.2.0 → datapipelab-0.2.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: datapipelab
-Version: 0.2.0
+Version: 0.2.2
 Summary: A data pipeline library with connectors, sources, processors, and sinks.
 Requires-Dist: json5
 Requires-Dist: loguru

{datapipelab-0.2.0 → datapipelab-0.2.2}/datapipelab/app/node/sink/hive_node.py

@@ -1,16 +1,16 @@
 from datapipelab.app.node.tnode import TNode
-
+from datapipelab.logger import logger
 
 class HiveSinkNode(TNode):
     def __init__(self, spark, tnode_config, df):
         from pyspark.sql import DataFrame
         super().__init__(spark=spark)
-        self.mode = tnode_config['mode']
-        self.stream = tnode_config['stream']
+        self.mode = tnode_config.get('mode', None)
+        self.stream = tnode_config.get('stream', None)
         self.database_name = tnode_config['options']['database']
         self.table_name = tnode_config['options']['table']
-        self.partition_by = tnode_config['options'].get('partition_by')
-        self.partition_count = tnode_config['options'].get('partition_count')
+        self.partition_by = tnode_config['options'].get('partition_by', None)
+        self.partition_count = tnode_config['options'].get('partition_count', None)
         self.overwrite = tnode_config['options']['overwrite']
         self.df = df
 
@@ -20,6 +20,7 @@ class HiveSinkNode(TNode):
                 self.df = self.df.repartition(int(self.partition_count))
             else:
                 self.df = self.df.repartition(int(self.partition_count), self.partition_by)
+        logger.info("Start writing to Hive")
         self.df.write.insertInto(f'{self.database_name}.{self.table_name}', overwrite=self.overwrite)
 
     def _process(self):

{datapipelab-0.2.0 → datapipelab-0.2.2}/datapipelab/pipeline_handler.py

@@ -1,5 +1,6 @@
 from datapipelab.app.node.processor.custom_node import CustomNode
 from datapipelab.app.node.processor.shell_node import ShellProcessorNode
+from datapipelab.app.node.source.hive_node import HiveSourceNode
 from datapipelab.app.node.source.spark_node import SparkSourceNode
 from datapipelab.app.node.source.delta_node import DeltaSourceNode
 from datapipelab.app.node.processor.spark_node import SparkProcessorNode
@@ -33,6 +34,9 @@ class PipelineHandler:
         if input_type == 'spark':
             if input_format == 'spark':
                 source_df = SparkSourceNode(self.spark, tnode_config).run()
+        if input_type == 'hive':
+            if input_format == 'hive':
+                source_df = HiveSourceNode(self.spark, tnode_config).run()
         if input_type == 'adls_path':
             if input_format == 'delta':
                 source_df = DeltaSourceNode(self.spark, tnode_config).run()
@@ -80,5 +84,10 @@ class PipelineHandler:
         if tnode_type == 'custom':
             from datapipelab.app.node import custom_node
             processor_df = custom_node.CustomNode(self.spark, tnode_config).run()
+        if tnode_type == 'spark':
+            if tnode_format == 'hive':
+                from datapipelab.app.node.sink import hive_node
+                processor_df = hive_node.HiveSinkNode(self.spark, tnode_config, t_df[tnode_name_df]).run()
+
 
 

{datapipelab-0.2.0 → datapipelab-0.2.2}/datapipelab.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: datapipelab
-Version: 0.2.0
+Version: 0.2.2
 Summary: A data pipeline library with connectors, sources, processors, and sinks.
 Requires-Dist: json5
 Requires-Dist: loguru

{datapipelab-0.2.0 → datapipelab-0.2.2}/setup.py

@@ -2,7 +2,7 @@ from setuptools import setup, find_packages
 
 setup(
     name='datapipelab',
-    version='0.2.0',
+    version='0.2.2',
     description='A data pipeline library with connectors, sources, processors, and sinks.',
     packages=find_packages(),
     include_package_data=True,