datapipelab 0.3.0__tar.gz → 0.3.2__tar.gz

{datapipelab-0.3.0 → datapipelab-0.3.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: datapipelab
-Version: 0.3.0
+Version: 0.3.2
 Summary: A data pipeline library with connectors, sources, processors, and sinks.
 Requires-Dist: json5
 Requires-Dist: loguru

{datapipelab-0.3.0 → datapipelab-0.3.2}/datapipelab/app/node/processor/bigquery_spark_node.py

@@ -1,3 +1,5 @@
+# This node should be merged with source/spark_api_node.py
+
 from datapipelab.app.node.tnode import TNode
 from datapipelab.logger import logger
 

{datapipelab-0.3.0 → datapipelab-0.3.2}/datapipelab/app/node/processor/spark_node.py

@@ -1,3 +1,5 @@
+# This node should be renamed to spark_sql_node
+
 from datapipelab.app.node.tnode import TNode
 
 

{datapipelab-0.3.0 → datapipelab-0.3.2}/datapipelab/app/node/sink/csv_node.py

@@ -1,5 +1,5 @@
 from datapipelab.app.node.tnode import TNode
-
+from datapipelab.logger import logger
 
 class CSVSinkNode(TNode):
     def __init__(self, spark, tnode_config, t_df):

{datapipelab-0.3.0 → datapipelab-0.3.2}/datapipelab/app/node/sink/delta_node.py

@@ -1,10 +1,10 @@
-from pyspark.sql import DataFrame
 from datapipelab.app.node.tnode import TNode
-
+from datapipelab.logger import logger
 
 class DeltaSinkNode(TNode):
     def __init__(self, spark, tnode_config, t_df):
         from delta.tables import DeltaTable
+        from pyspark.sql import DataFrame
         super().__init__(spark=spark)
         self.mode = tnode_config['options']['mode']  # Can be 'append', 'overwrite', or 'upsert'
         self.partition_by = tnode_config['options'].get('partition_by')

datapipelab-0.3.2/datapipelab/app/node/sink/spark_api_node.py

@@ -0,0 +1,38 @@
+from datapipelab.app.node.tnode import TNode
+from datapipelab.logger import logger
+
+
+class SparkApiSourceNode(TNode):
+    def __init__(self, spark, tnode_config, df):
+        from pyspark.sql import DataFrame
+        super().__init__(spark=spark)
+        self.df = df
+        self.__load_options(tnode_config)
+
+    def __load_options(self, tnode_config):
+        self.spark_options = tnode_config.get('options', {})
+        self.options = {}
+        if 'format' in self.spark_options:
+            self.format = self.spark_options.get('format')
+        if 'mode' in self.spark_options:
+            self.mode = self.spark_options.get('mode')
+        if 'parent_project' in self.spark_options:
+            self.options['parentProject'] = self.spark_options.get('parent_project')
+        if 'table' in self.spark_options:
+            self.options['table'] = self.spark_options.get('table')
+        if 'write_method' in self.spark_options:
+            self.options['writeMethod'] = self.spark_options.get('write_method')
+
+    def __write_df(self):
+        writer = self.df.write
+        if self.format:
+            writer = writer.format(self.format)
+        for key, value in self.options.items():
+            if value:
+                writer = writer.option(key, value)
+        if self.mode:
+            writer = writer.mode(self.mode)
+        writer.save()
+
+    def _process(self):
+        self.__write_df()

datapipelab-0.3.2/datapipelab/app/node/source/spark_api_node.py

@@ -0,0 +1,41 @@
+from datapipelab.app.node.tnode import TNode
+from datapipelab.logger import logger
+
+
+class SparkApiSourceNode(TNode):
+    def __init__(self, spark, tnode_config):
+        super().__init__(spark=spark)
+        self.__load_options(tnode_config)
+
+
+    def __load_options(self, tnode_config):
+        self.spark_options = tnode_config.get('options', {})
+        self.options = {}
+        if 'format' in self.spark_options:
+            self.format = self.spark_options.get('format')
+        if 'query' in self.spark_options:
+            self.query = self.spark_options.get('query')
+        if 'materialization_dataset' in self.spark_options:
+            self.options['materializationDataset'] = self.spark_options.get('materialization_dataset')
+        if 'parent_project' in self.spark_options:
+            self.options['parentProject'] = self.spark_options.get('parent_project')
+        if 'table' in self.spark_options:
+            self.options['table'] = self.spark_options.get('table')
+        if 'path' in self.spark_options:
+            self.options['path'] = self.spark_options.get('path')
+
+
+
+    def __load_df(self):
+        reader = self.spark.read
+        if self.format:
+            reader = reader.format(self.format)
+        for key, value in self.options.items():
+            if value:
+                reader = reader.option(key, value)
+        self.node = reader.load()
+
+    def _process(self):
+        self.__load_df()
+        self._createOrReplaceTempView()
+        return self.node

{datapipelab-0.3.0 → datapipelab-0.3.2}/datapipelab/app/node/source/spark_node.py

@@ -1,3 +1,5 @@
+# This node should be renamed to spark_sql_node.py
+
 from datapipelab.app.node.tnode import TNode
 
 

{datapipelab-0.3.0 → datapipelab-0.3.2}/datapipelab/engine.py

@@ -3,7 +3,7 @@ from datapipelab.logger import logger
 
 
 class Engine:
-    def __init__(self, engine_config_path, spark, params=None):
+    def __init__(self, engine_config_path, spark=None, params=None):
         self.engine_config_path = engine_config_path
         self.params = params
         self.pipeline = None

{datapipelab-0.3.0 → datapipelab-0.3.2}/datapipelab/pipeline_handler.py

@@ -3,6 +3,7 @@ from datapipelab.app.node.processor.shell_node import ShellProcessorNode
 from datapipelab.app.node.source.hive_node import HiveSourceNode
 from datapipelab.app.node.source.spark_node import SparkSourceNode
 from datapipelab.app.node.source.delta_node import DeltaSourceNode
+from datapipelab.app.node.source.spark_api_node import SparkApiSourceNode
 from datapipelab.app.node.processor.spark_node import SparkProcessorNode
 from datapipelab.app.node.sink.delta_node import DeltaSinkNode
 from datapipelab.app.node.sink.csv_node import CSVSinkNode
@@ -43,6 +44,9 @@ class PipelineHandler:
                 source_df = DeltaSourceNode(self.spark, tnode_config).run()
         if input_type == 'custom':
             source_df = CustomNode(self.spark, tnode_config).run()
+        if input_type == 'spark':
+            if input_format == 'api':
+                source_df = SparkApiSourceNode(self.spark, tnode_config).run()
 
         return source_df
 
@@ -96,6 +100,10 @@ class PipelineHandler:
             if tnode_format == 'spark':
                 from datapipelab.app.node.sink import spark_node
                 processor_df = spark_node.SparkSinkNode(self.spark, tnode_config, t_df[tnode_name_df]).run()
+        if tnode_type == 'spark':
+            if tnode_format == 'api':
+                from datapipelab.app.node.sink import spark_api_node
+                processor_df = spark_api_node.SparkApiSourceNode(self.spark, tnode_config, t_df[tnode_name_df]).run()
 
 
 

{datapipelab-0.3.0 → datapipelab-0.3.2}/datapipelab.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: datapipelab
-Version: 0.3.0
+Version: 0.3.2
 Summary: A data pipeline library with connectors, sources, processors, and sinks.
 Requires-Dist: json5
 Requires-Dist: loguru

{datapipelab-0.3.0 → datapipelab-0.3.2}/datapipelab.egg-info/SOURCES.txt

@@ -29,11 +29,13 @@ datapipelab/app/node/sink/csv_node.py
 datapipelab/app/node/sink/delta_node.py
 datapipelab/app/node/sink/hive_node.py
 datapipelab/app/node/sink/pandas_csv_node.py
+datapipelab/app/node/sink/spark_api_node.py
 datapipelab/app/node/sink/spark_node.py
 datapipelab/app/node/sink/teams_notification_node.py
 datapipelab/app/node/source/__init__.py
 datapipelab/app/node/source/delta_node.py
 datapipelab/app/node/source/hive_node.py
+datapipelab/app/node/source/spark_api_node.py
 datapipelab/app/node/source/spark_node.py
 datapipelab/app/wrapper/__init__.py
 datapipelab/app/wrapper/source/__init__.py

{datapipelab-0.3.0 → datapipelab-0.3.2}/setup.py

@@ -2,7 +2,7 @@ from setuptools import setup, find_packages
 
 setup(
     name='datapipelab',
-    version='0.3.0',
+    version='0.3.2',
     description='A data pipeline library with connectors, sources, processors, and sinks.',
     packages=find_packages(),
     include_package_data=True,