datapipelab 0.1.6__tar.gz → 0.1.7__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (37) hide show
  1. {datapipelab-0.1.6 → datapipelab-0.1.7}/PKG-INFO +1 -1
  2. {datapipelab-0.1.6 → datapipelab-0.1.7}/datapipelab/pipeline_handler.py +3 -2
  3. {datapipelab-0.1.6 → datapipelab-0.1.7}/datapipelab.egg-info/PKG-INFO +1 -1
  4. {datapipelab-0.1.6 → datapipelab-0.1.7}/setup.py +1 -1
  5. {datapipelab-0.1.6 → datapipelab-0.1.7}/MANIFEST.in +0 -0
  6. {datapipelab-0.1.6 → datapipelab-0.1.7}/README.md +0 -0
  7. {datapipelab-0.1.6 → datapipelab-0.1.7}/datapipelab/__init__.py +0 -0
  8. {datapipelab-0.1.6 → datapipelab-0.1.7}/datapipelab/app/__init__.py +0 -0
  9. {datapipelab-0.1.6 → datapipelab-0.1.7}/datapipelab/app/connector_node/__init__.py +0 -0
  10. {datapipelab-0.1.6 → datapipelab-0.1.7}/datapipelab/app/node/__init__.py +0 -0
  11. {datapipelab-0.1.6 → datapipelab-0.1.7}/datapipelab/app/node/custom_node.py +0 -0
  12. {datapipelab-0.1.6 → datapipelab-0.1.7}/datapipelab/app/node/processor/__init__.py +0 -0
  13. {datapipelab-0.1.6 → datapipelab-0.1.7}/datapipelab/app/node/processor/bigquery_api_node.py +0 -0
  14. {datapipelab-0.1.6 → datapipelab-0.1.7}/datapipelab/app/node/processor/bigquery_spark_node.py +0 -0
  15. {datapipelab-0.1.6 → datapipelab-0.1.7}/datapipelab/app/node/processor/custom_node.py +0 -0
  16. {datapipelab-0.1.6 → datapipelab-0.1.7}/datapipelab/app/node/processor/shell_node.py +0 -0
  17. {datapipelab-0.1.6 → datapipelab-0.1.7}/datapipelab/app/node/processor/spark_node.py +0 -0
  18. {datapipelab-0.1.6 → datapipelab-0.1.7}/datapipelab/app/node/sink/__init__.py +0 -0
  19. {datapipelab-0.1.6 → datapipelab-0.1.7}/datapipelab/app/node/sink/csv_node.py +0 -0
  20. {datapipelab-0.1.6 → datapipelab-0.1.7}/datapipelab/app/node/sink/delta_node.py +0 -0
  21. {datapipelab-0.1.6 → datapipelab-0.1.7}/datapipelab/app/node/sink/hive_node.py +0 -0
  22. {datapipelab-0.1.6 → datapipelab-0.1.7}/datapipelab/app/node/sink/pandas_csv_node.py +0 -0
  23. {datapipelab-0.1.6 → datapipelab-0.1.7}/datapipelab/app/node/sink/teams_notification_node.py +0 -0
  24. {datapipelab-0.1.6 → datapipelab-0.1.7}/datapipelab/app/node/source/__init__.py +0 -0
  25. {datapipelab-0.1.6 → datapipelab-0.1.7}/datapipelab/app/node/source/delta_node.py +0 -0
  26. {datapipelab-0.1.6 → datapipelab-0.1.7}/datapipelab/app/node/source/hive_node.py +0 -0
  27. {datapipelab-0.1.6 → datapipelab-0.1.7}/datapipelab/app/node/source/spark_node.py +0 -0
  28. {datapipelab-0.1.6 → datapipelab-0.1.7}/datapipelab/app/node/tnode.py +0 -0
  29. {datapipelab-0.1.6 → datapipelab-0.1.7}/datapipelab/engine.py +0 -0
  30. {datapipelab-0.1.6 → datapipelab-0.1.7}/datapipelab/logger.py +0 -0
  31. {datapipelab-0.1.6 → datapipelab-0.1.7}/datapipelab/pipeline.py +0 -0
  32. {datapipelab-0.1.6 → datapipelab-0.1.7}/datapipelab/pipeline_config.py +0 -0
  33. {datapipelab-0.1.6 → datapipelab-0.1.7}/datapipelab.egg-info/SOURCES.txt +0 -0
  34. {datapipelab-0.1.6 → datapipelab-0.1.7}/datapipelab.egg-info/dependency_links.txt +0 -0
  35. {datapipelab-0.1.6 → datapipelab-0.1.7}/datapipelab.egg-info/requires.txt +0 -0
  36. {datapipelab-0.1.6 → datapipelab-0.1.7}/datapipelab.egg-info/top_level.txt +0 -0
  37. {datapipelab-0.1.6 → datapipelab-0.1.7}/setup.cfg +0 -0
{datapipelab-0.1.6 → datapipelab-0.1.7}/PKG-INFO RENAMED
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: datapipelab
3
- Version: 0.1.6
3
+ Version: 0.1.7
4
4
  Summary: A data pipeline library with connectors, sources, processors, and sinks.
5
5
  Requires-Dist: json5
6
6
  Requires-Dist: loguru
{datapipelab-0.1.6 → datapipelab-0.1.7}/datapipelab/pipeline_handler.py RENAMED
@@ -7,6 +7,7 @@ from datapipelab.app.node.sink.csv_node import CSVSinkNode
7
7
  from datapipelab.app.node.sink.pandas_csv_node import PandasCSVSinkNode
8
8
  from datapipelab.app.node.sink.teams_notification_node import TeamsNotificationSinkNode
9
9
  from datapipelab.app.node.processor.bigquery_spark_node import BigQuerySparkProcessorNode
10
+ from datapipelab.app.node.processor.bigquery_api_node import BigQueryAPIProcessorNode
10
11
 
11
12
 
12
13
  class PipelineHandler:
@@ -46,9 +47,9 @@ class PipelineHandler:
46
47
  if tnode_format == 'query':
47
48
  processor_df = SparkProcessorNode(self.spark, tnode_config).run()
48
49
  if tnode_format == 'bigquery_api':
49
- processor_df = None
50
+ processor_df = BigQueryAPIProcessorNode(self.spark, tnode_config).run()
50
51
  if tnode_format == 'bigquery_spark':
51
- processor_df = datapipelab.app.node.processor.bigquery_spark.BigQuerySparkProcessorNode(self.spark, tnode_config).run()
52
+ processor_df = BigQuerySparkProcessorNode(self.spark, tnode_config).run()
52
53
  return processor_df
53
54
 
54
55
  def write_sink_node(self, tnode_config, t_df):
{datapipelab-0.1.6 → datapipelab-0.1.7}/datapipelab.egg-info/PKG-INFO RENAMED
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: datapipelab
3
- Version: 0.1.6
3
+ Version: 0.1.7
4
4
  Summary: A data pipeline library with connectors, sources, processors, and sinks.
5
5
  Requires-Dist: json5
6
6
  Requires-Dist: loguru
{datapipelab-0.1.6 → datapipelab-0.1.7}/setup.py RENAMED
@@ -2,7 +2,7 @@ from setuptools import setup, find_packages
2
2
 
3
3
  setup(
4
4
  name='datapipelab',
5
- version='0.1.6',
5
+ version='0.1.7',
6
6
  description='A data pipeline library with connectors, sources, processors, and sinks.',
7
7
  packages=find_packages(),
8
8
  include_package_data=True,
{datapipelab-0.1.6 → datapipelab-0.1.7}/MANIFEST.in RENAMED
File without changes
{datapipelab-0.1.6 → datapipelab-0.1.7}/README.md RENAMED
File without changes
{datapipelab-0.1.6 → datapipelab-0.1.7}/setup.cfg RENAMED
File without changes