datapipelab 0.2.6__tar.gz → 0.2.8__tar.gz

{datapipelab-0.2.6 → datapipelab-0.2.8}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: datapipelab
-Version: 0.2.6
+Version: 0.2.8
 Summary: A data pipeline library with connectors, sources, processors, and sinks.
 Requires-Dist: json5
 Requires-Dist: loguru

{datapipelab-0.2.6 → datapipelab-0.2.8}/datapipelab/app/node/custom_node.py

@@ -2,7 +2,7 @@ from datapipelab.app.node.tnode import TNode
 from datapipelab.logger import logger
 
 class CustomNode(TNode):
-    def __init__(self, spark, tnode_config, t_df):
+    def __init__(self, spark, tnode_config, t_df=None):
         super().__init__(spark=spark)
         self.tnode_config = tnode_config
         self.spark = spark
@@ -10,7 +10,7 @@ class CustomNode(TNode):
         module_name = tnode_config['options']['module_name']
         module_path = tnode_config['options']['module_path']
         class_name = tnode_config['options']['class_name']
-        self.custom_processor = self.import_module(module_name, module_path, class_name)
+        self.custom_node = self.import_module(module_name, module_path, class_name)
 
     def import_module(self, module_name, module_path, class_name):
         custom_module = __import__(module_path, fromlist=[module_name])
@@ -19,4 +19,4 @@ class CustomNode(TNode):
 
     def _process(self):
         logger.info(f"Custom node type is {self.tnode_config.get('custom_node_type', 'N/A!')}")
-        return self.custom_processor.process()
+        return self.custom_node.custom_process()

{datapipelab-0.2.6 → datapipelab-0.2.8}/datapipelab/pipeline_handler.py

@@ -1,4 +1,4 @@
-from datapipelab.app.node.processor.custom_node import CustomNode
+from datapipelab.app.node.custom_node import CustomNode
 from datapipelab.app.node.processor.shell_node import ShellProcessorNode
 from datapipelab.app.node.source.hive_node import HiveSourceNode
 from datapipelab.app.node.source.spark_node import SparkSourceNode
@@ -40,6 +40,8 @@ class PipelineHandler:
         if input_type == 'adls_path':
             if input_format == 'delta':
                 source_df = DeltaSourceNode(self.spark, tnode_config).run()
+        if input_type == 'custom':
+            source_df = CustomNode(self.spark, tnode_config).run()
 
         return source_df
 
@@ -82,8 +84,7 @@ class PipelineHandler:
             if tnode_format == "csv":
                 PandasCSVSinkNode(self.spark, tnode_config, t_df).run()
         if tnode_type == 'custom':
-            from datapipelab.app.node import custom_node
-            processor_df = custom_node.CustomNode(self.spark, tnode_config).run()
+            CustomNode(self.spark, tnode_config).run()
         if tnode_type == 'spark':
             if tnode_format == 'hive':
                 from datapipelab.app.node.sink import hive_node

{datapipelab-0.2.6 → datapipelab-0.2.8}/datapipelab.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: datapipelab
-Version: 0.2.6
+Version: 0.2.8
 Summary: A data pipeline library with connectors, sources, processors, and sinks.
 Requires-Dist: json5
 Requires-Dist: loguru

{datapipelab-0.2.6 → datapipelab-0.2.8}/setup.py

@@ -2,7 +2,7 @@ from setuptools import setup, find_packages
 
 setup(
     name='datapipelab',
-    version='0.2.6',
+    version='0.2.8',
     description='A data pipeline library with connectors, sources, processors, and sinks.',
     packages=find_packages(),
     include_package_data=True,