datapipelab 0.1.6__py3-none-any.whl → 0.1.7__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (5) hide show
  1. datapipelab/pipeline_handler.py +3 -2
  2. {datapipelab-0.1.6.dist-info → datapipelab-0.1.7.dist-info}/METADATA +1 -1
  3. {datapipelab-0.1.6.dist-info → datapipelab-0.1.7.dist-info}/RECORD +5 -5
  4. {datapipelab-0.1.6.dist-info → datapipelab-0.1.7.dist-info}/WHEEL +0 -0
  5. {datapipelab-0.1.6.dist-info → datapipelab-0.1.7.dist-info}/top_level.txt +0 -0
datapipelab/pipeline_handler.py CHANGED
@@ -7,6 +7,7 @@ from datapipelab.app.node.sink.csv_node import CSVSinkNode
7
7
  from datapipelab.app.node.sink.pandas_csv_node import PandasCSVSinkNode
8
8
  from datapipelab.app.node.sink.teams_notification_node import TeamsNotificationSinkNode
9
9
  from datapipelab.app.node.processor.bigquery_spark_node import BigQuerySparkProcessorNode
10
+ from datapipelab.app.node.processor.bigquery_api_node import BigQueryAPIProcessorNode
10
11
 
11
12
 
12
13
  class PipelineHandler:
@@ -46,9 +47,9 @@ class PipelineHandler:
46
47
  if tnode_format == 'query':
47
48
  processor_df = SparkProcessorNode(self.spark, tnode_config).run()
48
49
  if tnode_format == 'bigquery_api':
49
- processor_df = None
50
+ processor_df = BigQueryAPIProcessorNode(self.spark, tnode_config).run()
50
51
  if tnode_format == 'bigquery_spark':
51
- processor_df = datapipelab.app.node.processor.bigquery_spark.BigQuerySparkProcessorNode(self.spark, tnode_config).run()
52
+ processor_df = BigQuerySparkProcessorNode(self.spark, tnode_config).run()
52
53
  return processor_df
53
54
 
54
55
  def write_sink_node(self, tnode_config, t_df):
{datapipelab-0.1.6.dist-info → datapipelab-0.1.7.dist-info}/METADATA RENAMED
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: datapipelab
3
- Version: 0.1.6
3
+ Version: 0.1.7
4
4
  Summary: A data pipeline library with connectors, sources, processors, and sinks.
5
5
  Requires-Dist: json5
6
6
  Requires-Dist: loguru
{datapipelab-0.1.6.dist-info → datapipelab-0.1.7.dist-info}/RECORD RENAMED
@@ -3,7 +3,7 @@ datapipelab/engine.py,sha256=3QRsedRYNov6xIDOZ1tukinFE-SKv39Fn3sNCnD3L6g,442
3
3
  datapipelab/logger.py,sha256=Ugv0A4TfD3JWCWXNWu0lURcnfAEyuVrK3IrvVVgcHBo,864
4
4
  datapipelab/pipeline.py,sha256=dw9D9KM_hztt9g_YzqoNgQBRyCYR92cRZwrU5duP_Pg,1464
5
5
  datapipelab/pipeline_config.py,sha256=2bFAJepViE7rT7CaRANZU07aeQpOYcZ954ISujm9pXA,3816
6
- datapipelab/pipeline_handler.py,sha256=HCyvyW3Yx2QYWQ-D-ArMDZdGtMRsJ348ZHSrqbRYwd0,3779
6
+ datapipelab/pipeline_handler.py,sha256=lQv6HwwdgZDQvICgABtWiuvZQ9jG9cJjy8s_7qLZr9s,3871
7
7
  datapipelab/app/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
8
8
  datapipelab/app/connector_node/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
9
9
  datapipelab/app/node/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -25,7 +25,7 @@ datapipelab/app/node/source/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMp
25
25
  datapipelab/app/node/source/delta_node.py,sha256=gg7SfuKBAAfjk6OX2jNrot9XX61HoBe3us3D8O-dscE,529
26
26
  datapipelab/app/node/source/hive_node.py,sha256=h_AMCnnmon7uLRIGsaHAPWEReD3VaWZXnz9r0TpLGNM,478
27
27
  datapipelab/app/node/source/spark_node.py,sha256=S_x2atRFPDnXmhCUtcmaLc4BDFd2H4uQq6wnEJb7Uug,480
28
- datapipelab-0.1.6.dist-info/METADATA,sha256=yuXR8PtBINVXBFVsFncHbu9cnvJ52Jp8ZCDi-Bsmrnw,220
29
- datapipelab-0.1.6.dist-info/WHEEL,sha256=Nw36Djuh_5VDukK0H78QzOX-_FQEo6V37m3nkm96gtU,91
30
- datapipelab-0.1.6.dist-info/top_level.txt,sha256=HgeBjHvXorKzvNqU5BNPutoI771HtiqVit9_-0Zyrb4,12
31
- datapipelab-0.1.6.dist-info/RECORD,,
28
+ datapipelab-0.1.7.dist-info/METADATA,sha256=oiRy9y4GTRDU-Yiz3EQJc831TRgaTiMRXOaNimFllTQ,220
29
+ datapipelab-0.1.7.dist-info/WHEEL,sha256=Nw36Djuh_5VDukK0H78QzOX-_FQEo6V37m3nkm96gtU,91
30
+ datapipelab-0.1.7.dist-info/top_level.txt,sha256=HgeBjHvXorKzvNqU5BNPutoI771HtiqVit9_-0Zyrb4,12
31
+ datapipelab-0.1.7.dist-info/RECORD,,