datapipelab 0.3.0__py3-none-any.whl → 0.3.2__py3-none-any.whl

datapipelab/app/node/processor/bigquery_spark_node.py

@@ -1,3 +1,5 @@
+# This node should be merged with source/spark_api_node.py
+
 from datapipelab.app.node.tnode import TNode
 from datapipelab.logger import logger
 

datapipelab/app/node/processor/spark_node.py

@@ -1,3 +1,5 @@
+# This node should be renamed to spark_sql_node
+
 from datapipelab.app.node.tnode import TNode
 
 

datapipelab/app/node/sink/csv_node.py

@@ -1,5 +1,5 @@
 from datapipelab.app.node.tnode import TNode
-
+from datapipelab.logger import logger
 
 class CSVSinkNode(TNode):
     def __init__(self, spark, tnode_config, t_df):

datapipelab/app/node/sink/delta_node.py

@@ -1,10 +1,10 @@
-from pyspark.sql import DataFrame
 from datapipelab.app.node.tnode import TNode
-
+from datapipelab.logger import logger
 
 class DeltaSinkNode(TNode):
     def __init__(self, spark, tnode_config, t_df):
         from delta.tables import DeltaTable
+        from pyspark.sql import DataFrame
         super().__init__(spark=spark)
         self.mode = tnode_config['options']['mode']  # Can be 'append', 'overwrite', or 'upsert'
         self.partition_by = tnode_config['options'].get('partition_by')

datapipelab/app/node/sink/spark_api_node.py

@@ -0,0 +1,38 @@
+from datapipelab.app.node.tnode import TNode
+from datapipelab.logger import logger
+
+
+class SparkApiSourceNode(TNode):
+    def __init__(self, spark, tnode_config, df):
+        from pyspark.sql import DataFrame
+        super().__init__(spark=spark)
+        self.df = df
+        self.__load_options(tnode_config)
+
+    def __load_options(self, tnode_config):
+        self.spark_options = tnode_config.get('options', {})
+        self.options = {}
+        if 'format' in self.spark_options:
+            self.format = self.spark_options.get('format')
+        if 'mode' in self.spark_options:
+            self.mode = self.spark_options.get('mode')
+        if 'parent_project' in self.spark_options:
+            self.options['parentProject'] = self.spark_options.get('parent_project')
+        if 'table' in self.spark_options:
+            self.options['table'] = self.spark_options.get('table')
+        if 'write_method' in self.spark_options:
+            self.options['writeMethod'] = self.spark_options.get('write_method')
+
+    def __write_df(self):
+        writer = self.df.write
+        if self.format:
+            writer = writer.format(self.format)
+        for key, value in self.options.items():
+            if value:
+                writer = writer.option(key, value)
+        if self.mode:
+            writer = writer.mode(self.mode)
+        writer.save()
+
+    def _process(self):
+        self.__write_df()

datapipelab/app/node/source/spark_api_node.py

@@ -0,0 +1,41 @@
+from datapipelab.app.node.tnode import TNode
+from datapipelab.logger import logger
+
+
+class SparkApiSourceNode(TNode):
+    def __init__(self, spark, tnode_config):
+        super().__init__(spark=spark)
+        self.__load_options(tnode_config)
+
+
+    def __load_options(self, tnode_config):
+        self.spark_options = tnode_config.get('options', {})
+        self.options = {}
+        if 'format' in self.spark_options:
+            self.format = self.spark_options.get('format')
+        if 'query' in self.spark_options:
+            self.query = self.spark_options.get('query')
+        if 'materialization_dataset' in self.spark_options:
+            self.options['materializationDataset'] = self.spark_options.get('materialization_dataset')
+        if 'parent_project' in self.spark_options:
+            self.options['parentProject'] = self.spark_options.get('parent_project')
+        if 'table' in self.spark_options:
+            self.options['table'] = self.spark_options.get('table')
+        if 'path' in self.spark_options:
+            self.options['path'] = self.spark_options.get('path')
+
+
+
+    def __load_df(self):
+        reader = self.spark.read
+        if self.format:
+            reader = reader.format(self.format)
+        for key, value in self.options.items():
+            if value:
+                reader = reader.option(key, value)
+        self.node = reader.load()
+
+    def _process(self):
+        self.__load_df()
+        self._createOrReplaceTempView()
+        return self.node

datapipelab/app/node/source/spark_node.py

@@ -1,3 +1,5 @@
+# This node should be renamed to spark_sql_node.py
+
 from datapipelab.app.node.tnode import TNode
 
 

datapipelab/engine.py

@@ -3,7 +3,7 @@ from datapipelab.logger import logger
 
 
 class Engine:
-    def __init__(self, engine_config_path, spark, params=None):
+    def __init__(self, engine_config_path, spark=None, params=None):
         self.engine_config_path = engine_config_path
         self.params = params
         self.pipeline = None

datapipelab/pipeline_handler.py

@@ -3,6 +3,7 @@ from datapipelab.app.node.processor.shell_node import ShellProcessorNode
 from datapipelab.app.node.source.hive_node import HiveSourceNode
 from datapipelab.app.node.source.spark_node import SparkSourceNode
 from datapipelab.app.node.source.delta_node import DeltaSourceNode
+from datapipelab.app.node.source.spark_api_node import SparkApiSourceNode
 from datapipelab.app.node.processor.spark_node import SparkProcessorNode
 from datapipelab.app.node.sink.delta_node import DeltaSinkNode
 from datapipelab.app.node.sink.csv_node import CSVSinkNode
@@ -43,6 +44,9 @@ class PipelineHandler:
                 source_df = DeltaSourceNode(self.spark, tnode_config).run()
         if input_type == 'custom':
             source_df = CustomNode(self.spark, tnode_config).run()
+        if input_type == 'spark':
+            if input_format == 'api':
+                source_df = SparkApiSourceNode(self.spark, tnode_config).run()
 
         return source_df
 
@@ -96,6 +100,10 @@ class PipelineHandler:
             if tnode_format == 'spark':
                 from datapipelab.app.node.sink import spark_node
                 processor_df = spark_node.SparkSinkNode(self.spark, tnode_config, t_df[tnode_name_df]).run()
+        if tnode_type == 'spark':
+            if tnode_format == 'api':
+                from datapipelab.app.node.sink import spark_api_node
+                processor_df = spark_api_node.SparkApiSourceNode(self.spark, tnode_config, t_df[tnode_name_df]).run()
 
 
 

{datapipelab-0.3.0.dist-info → datapipelab-0.3.2.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: datapipelab
-Version: 0.3.0
+Version: 0.3.2
 Summary: A data pipeline library with connectors, sources, processors, and sinks.
 Requires-Dist: json5
 Requires-Dist: loguru

{datapipelab-0.3.0.dist-info → datapipelab-0.3.2.dist-info}/RECORD

@@ -1,9 +1,9 @@
 datapipelab/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-datapipelab/engine.py,sha256=3QRsedRYNov6xIDOZ1tukinFE-SKv39Fn3sNCnD3L6g,442
+datapipelab/engine.py,sha256=Dt8oM7RvIMkllPhFpUr1fynJD01ZG-hr6eqt5OSRh-Y,447
 datapipelab/logger.py,sha256=Ugv0A4TfD3JWCWXNWu0lURcnfAEyuVrK3IrvVVgcHBo,864
 datapipelab/pipeline.py,sha256=dw9D9KM_hztt9g_YzqoNgQBRyCYR92cRZwrU5duP_Pg,1464
 datapipelab/pipeline_config.py,sha256=2bFAJepViE7rT7CaRANZU07aeQpOYcZ954ISujm9pXA,3816
-datapipelab/pipeline_handler.py,sha256=Q1AzuPgOb9bElclfX-E8PiTGrdOKzFshQUjbNNXp5m0,4980
+datapipelab/pipeline_handler.py,sha256=LrVhYFAPf1FVLlDBACmQu-cJkVX-X8r4eIavwxJlAGo,5464
 datapipelab/app/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 datapipelab/app/connector_node/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 datapipelab/app/node/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -11,25 +11,27 @@ datapipelab/app/node/custom_node.py,sha256=3Se4DweMvm5VK4MTZ-pQSQ_lE_fOm6cGj-wzc
 datapipelab/app/node/tnode.py,sha256=-2hnQkIuLwEy7xVTig54TByO7L2l7UujolXMQL0CQJA,484
 datapipelab/app/node/processor/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 datapipelab/app/node/processor/bigquery_api_node.py,sha256=IclDkGxo9ltGJVkBaHKFPFCSlEEyzefgalaAOLA17bE,1752
-datapipelab/app/node/processor/bigquery_spark_node.py,sha256=S9kIYW0RE5b0RjniKFFBTzA3Tx4_plFdkFQXzhl1xTY,1039
+datapipelab/app/node/processor/bigquery_spark_node.py,sha256=pklpsqYqztidCIECkl3rpjfY6LiB0p4thvE7-PzBodE,1099
 datapipelab/app/node/processor/custom_node.py,sha256=1nqbJEhNiMP1rmN9ufpUuKO1IkuI2BEM5auW4JceGMA,933
 datapipelab/app/node/processor/gcp_bucket_node.py,sha256=bzV2c89-g5S0OH5bcKKQ-9yKOGwlmOR7h7_5uO6Gnq0,1904
 datapipelab/app/node/processor/shell_node.py,sha256=s3dKgfEqbpUIEiwORERgvp7FNDE5JkFHBo7EnJYBPnA,669
-datapipelab/app/node/processor/spark_node.py,sha256=jzqdffIHUCgOfMFcoqjXdl8wFag-3gafxfNCdssKnwc,483
+datapipelab/app/node/processor/spark_node.py,sha256=ROSp_gpqHhtS4jog7z64jEcVPaCLFaELyIhb1A2UVe8,532
 datapipelab/app/node/sink/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-datapipelab/app/node/sink/csv_node.py,sha256=d2hyufP5_Nmql0pfD0KeC4rFu1wXTnBxVsoGl7sWbhM,1681
-datapipelab/app/node/sink/delta_node.py,sha256=iKEdiTjJ7SHJZMrbm0jR5tms5JZ5iCFfQklZbI-Yr2o,2044
+datapipelab/app/node/sink/csv_node.py,sha256=ZcrMZXIwJ_ln4ZZbpCAT-iMDAZIDFI9eSHNENx4wMpA,1718
+datapipelab/app/node/sink/delta_node.py,sha256=4ajvMyz3cpXbd29_mZq0MW-gwpLJqdj6F9urmP8uHJw,2089
 datapipelab/app/node/sink/hive_node.py,sha256=ycknOPBBwZGH3oHram_6LjHy-ygFjhuFNvVoPaNGaCU,1220
 datapipelab/app/node/sink/pandas_csv_node.py,sha256=JsJFt2XRpwxGeJyt_PDUgqZafiQROf1Sk5TUhQPxh4c,870
+datapipelab/app/node/sink/spark_api_node.py,sha256=Uu25EtQEXJkqzm3eBEwvpuqSfVXpYUBbWRaybC_BoQQ,1406
 datapipelab/app/node/sink/spark_node.py,sha256=tP3tZae2jzQtAtfIm8C-166WWSLdZs54mqoIyZOSy58,1221
 datapipelab/app/node/sink/teams_notification_node.py,sha256=6ZufdbhVvRXi3QTQafLo5uKl9kLyDnkYIE_VZFT0QNw,3581
 datapipelab/app/node/source/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 datapipelab/app/node/source/delta_node.py,sha256=gg7SfuKBAAfjk6OX2jNrot9XX61HoBe3us3D8O-dscE,529
 datapipelab/app/node/source/hive_node.py,sha256=h_AMCnnmon7uLRIGsaHAPWEReD3VaWZXnz9r0TpLGNM,478
-datapipelab/app/node/source/spark_node.py,sha256=S_x2atRFPDnXmhCUtcmaLc4BDFd2H4uQq6wnEJb7Uug,480
+datapipelab/app/node/source/spark_api_node.py,sha256=HQQAkFpzIh_oA17aaQX6TVyfXtCdILGvzx4FIWXZyo8,1496
+datapipelab/app/node/source/spark_node.py,sha256=TDfezmlk8Ts2YTGkB92-God_AyGVUslTUoevXolN7W8,532
 datapipelab/app/wrapper/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 datapipelab/app/wrapper/source/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-datapipelab-0.3.0.dist-info/METADATA,sha256=uXwxvNNpO_FSDUi8rC5bxPNImGtLuZuSMUV9QDNptT0,220
-datapipelab-0.3.0.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-datapipelab-0.3.0.dist-info/top_level.txt,sha256=HgeBjHvXorKzvNqU5BNPutoI771HtiqVit9_-0Zyrb4,12
-datapipelab-0.3.0.dist-info/RECORD,,
+datapipelab-0.3.2.dist-info/METADATA,sha256=ztMAegNhlrEhJcmbHJAF7qFPMwuNvyf_CKEUlrmqsZQ,220
+datapipelab-0.3.2.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+datapipelab-0.3.2.dist-info/top_level.txt,sha256=HgeBjHvXorKzvNqU5BNPutoI771HtiqVit9_-0Zyrb4,12
+datapipelab-0.3.2.dist-info/RECORD,,