datapipelab 0.1__py3-none-any.whl → 0.1.2__py3-none-any.whl

datapipelab/engine.py

@@ -0,0 +1,14 @@
+from datapipelab.pipeline import Pipeline
+from datapipelab.logger import logger
+
+
+class Engine:
+    def __init__(self, engine_config_path, spark, params):
+        self.engine_config_path = engine_config_path
+        self.params = params
+        self.pipeline = None
+        self.spark = spark
+
+    def running_travelers(self):
+        self.pipeline = Pipeline(self.engine_config_path, self.spark, self.params)
+        self.pipeline.run()

datapipelab/logger.py

@@ -0,0 +1,35 @@
+import sys
+from loguru import logger
+from pathlib import Path
+
+# Create logs directory
+log_dir = Path(__file__).resolve().parent / "logs"
+log_dir.mkdir(exist_ok=True)
+
+# Log file path
+log_file = log_dir / "app.log"
+
+# Remove the default loguru handler
+logger.remove()
+
+# Add combined rotation: size and time
+logger.add(
+    str(log_file),
+    rotation="10 MB",              # Rotate if file exceeds 10MB
+    retention=100,                 # Keep last 100 files
+    compression="zip",             # Compress rotated logs as .zip
+    enqueue=True,
+    backtrace=True,
+    diagnose=True,
+    level="INFO",
+    serialize=True
+)
+
+# Add console handler
+logger.add(
+    sink=sys.stdout,               # Output to console
+    level="INFO",                  # Set log level
+    backtrace=True,
+    diagnose=True,
+    colorize=True                  # Enable colored output
+)

datapipelab/pipeline.py

@@ -0,0 +1,36 @@
+from datapipelab.pipeline_config import PipelineConfig
+from datapipelab.pipeline_handler import PipelineHandler
+from datapipelab.logger import Logger
+
+class Pipeline:
+    def __init__(self, pipeline_config_path, spark, config_params=None):
+        self.pipeline_config = None
+        self.pipeline_config_path = pipeline_config_path
+        self.params = config_params
+        self.__load_config()
+        self.spark = spark
+
+    def __load_config(self):
+        self.pipeline_config = PipelineConfig(self.pipeline_config_path, self.params)
+        self.pipeline_config.create_pipeline_nodes()
+
+    def __process(self):
+        logger.info('Fetch sources...')
+        print(self.pipeline_config.sources)
+        tnode = PipelineHandler(self.spark)
+        self.t_df = {}
+        for source in self.pipeline_config.sources:
+            self.t_df[source] = tnode.create_source_node(source, self.pipeline_config.sources[source])
+
+        logger.info('Running Processors...')
+        print(self.pipeline_config.processors)
+        for processor in self.pipeline_config.processors:
+            self.t_df[processor] = tnode.create_processor_node(processor, self.pipeline_config.processors[processor], self.t_df)
+
+        logger.info('Write into sinks...')
+        print(self.pipeline_config.sinks)
+        for sink in self.pipeline_config.sinks:
+            tnode.write_sink_node(self.pipeline_config.sinks[sink], self.t_df)
+
+    def run(self):
+        self.__process()

datapipelab/pipeline_config.py

@@ -0,0 +1,71 @@
+import json5
+
+
+class PipelineConfig:
+    def __init__(self, config_file, params):
+        print(config_file)
+        self.params = params
+        self.config_file = config_file
+        self.load_json_config_file()
+        self.sources = {}
+        self.processors = {}
+        self.sinks = {}
+
+    def load_json_config_file(self):
+        with open(self.config_file, 'r') as f:
+            # Read the file as a text file
+            json_config_file = f.read()
+            # Replace the placeholders with the actual values
+            for key, value in self.params.items():
+                json_config_file = json_config_file.replace(f"{{{key}}}", value)
+            # Convert to JSON file
+            self.pipeline_settings = json5.loads(json_config_file)
+
+    def create_pipeline_nodes(self):
+        for component in self.pipeline_settings:
+            if component['type'] == 'source':
+                source = {
+                    'input_format': component['format'],
+                    'name': component['name'],
+                    'input_type': component['source'],
+                    'options': component['options']
+                }
+                self.sources[component['name']] = source
+            elif component['type'] == 'processor':
+                processor = {
+                    'format': component['format'],
+                    'name': component['name'],
+                    'options': component['options']
+                }
+                parents = component['options'].get('parents', [])
+                parent_sources = []
+                for parent in parents:
+                    parent_source = self.sources.get(parent)
+                    if not parent_source:
+                        parent_processor = self.processors.get(parent)
+                        if not parent_processor:
+                            raise ValueError(f"Parent '{parent}' not found")
+                        parent_source = parent_processor['source']
+                    parent_sources.append(parent_source)
+                processor['source'] = parent_sources
+                self.processors[component['name']] = processor
+            elif component['type'] == 'sink':
+                sink = {
+                    'output_format': component['format'],
+                    'name': component['name'],
+                    'output_type': component['sink'],
+                    'options': component['options']
+                }
+                parent = component['options']['parents'][0]
+                parent_source = self.sources.get(parent)
+                if not parent_source:
+                    parent_processor = self.processors.get(parent)
+                    if not parent_processor:
+                        raise ValueError(f"Parent '{parent}' not found")
+                    parent_source = parent_processor['source']
+                sink['source'] = parent_source
+                self.sinks[component['name']] = sink
+            else:
+                raise ValueError(f"Invalid component type '{component['type']}'")
+
+

datapipelab/pipeline_handler.py

@@ -0,0 +1,74 @@
+from datapipelab.app.node.processor.custom_node import CustomNode
+from datapipelab.app.node.source.spark_node import SparkSourceNode
+from datapipelab.app.node.source.delta_node import DeltaSourceNode
+from datapipelab.app.node.processor.spark_node import SparkProcessorNode
+from datapipelab.app.node.sink.delta_node import DeltaSinkNode
+from datapipelab.app.node.sink.csv_node import CSVSinkNode
+from datapipelab.app.node.sink.pandas_csv_node import PandasCSVSinkNode
+from datapipelab.app.node.sink.teams_notification_node import TeamsNotificationSinkNode
+
+
+class PipelineHandler:
+    def __init__(self, spark=None):
+        self.spark = spark
+
+    def create_source_node(self, tnode_name, tnode_config):
+        input_type = tnode_config['input_type']
+        input_format = tnode_config['input_format']
+        print(tnode_name, input_type, input_format, tnode_config)
+
+        source_df = None
+        if input_type == 'SharedDrive':
+            if input_format == 'excel':
+                source_df = CTCSMBReaderSourceNode(tnode_config).run()
+        if input_type == "Oracle":
+            if input_format == "query":
+                source_df = OracleSourceNode(tnode_config).run()
+        if input_type == "SharePoint":
+            if input_format == "csv":
+                source_df = SharePointSourceNode(tnode_config).run()
+        if input_type == 'spark':
+            if input_format == 'spark':
+                source_df = SparkSourceNode(self.spark, tnode_config).run()
+        if input_type == 'adls_path':
+            if input_format == 'delta':
+                source_df = DeltaSourceNode(self.spark, tnode_config).run()
+
+        return source_df
+
+    def create_processor_node(self, tnode_name, tnode_config, t_df):
+        tnode_format = tnode_config['format']
+        print(tnode_name, tnode_format, tnode_config)
+        processor_df = None
+        if tnode_format == 'custom':
+            processor_df = CustomNode(self.spark, tnode_config).run()
+
+        if tnode_format == 'query':
+            processor_df = SparkProcessorNode(self.spark, tnode_config).run()
+
+        return processor_df
+
+    def write_sink_node(self, tnode_config, t_df):
+        tnode_type = tnode_config['output_type']
+        tnode_format = tnode_config['output_format']
+        tnode_name_df = tnode_config['options']['parents'][0]
+        print(tnode_type, tnode_format, tnode_name_df)
+
+        if tnode_type == 'SharePoint':
+            if tnode_format == 'csv':
+                sharepoint_sink = SharePointSinkNode(tnode_config, t_df).run()
+                print(sharepoint_sink)
+                # HiveSinkNode(self.spark, tnode_config, tnode_df[tnode_name_df]).run()
+        if tnode_type == "teams":
+            if tnode_format == "channel_notification":
+                TeamsNotificationSinkNode(self.spark, tnode_config, t_df).run() # TODO: Spark can be set to None
+        if tnode_type == "adls_path":
+            if tnode_format == "delta":
+                DeltaSinkNode(self.spark, tnode_config, t_df).run()
+            if tnode_format == "csv":
+                CSVSinkNode(self.spark, tnode_config, t_df).run()
+        if tnode_type == "local":
+            if tnode_format == "csv":
+                PandasCSVSinkNode(self.spark, tnode_config, t_df).run()
+
+

{datapipelab-0.1.dist-info → datapipelab-0.1.2.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: datapipelab
-Version: 0.1
+Version: 0.1.2
 Summary: A data pipeline library with connectors, sources, processors, and sinks.
 Requires-Dist: json5
 Requires-Dist: loguru

datapipelab-0.1.2.dist-info/RECORD

@@ -0,0 +1,27 @@
+datapipelab/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+datapipelab/engine.py,sha256=dYm39Yb0Eqt76pwdc-ilzZNxehMKdiuidE557YexHaU,437
+datapipelab/logger.py,sha256=Ugv0A4TfD3JWCWXNWu0lURcnfAEyuVrK3IrvVVgcHBo,864
+datapipelab/pipeline.py,sha256=kX_Wy-l6DZ7ega3OKk-8ZV14aQlOCNMcxX6AZ6bIHSU,1464
+datapipelab/pipeline_config.py,sha256=MyEV2jz54_ZcEAgwmMR91KP8XG8W5raGoTRWctSeIrI,3038
+datapipelab/pipeline_handler.py,sha256=1t5wwsaVUMXXmsEa-Qt-6jtMIyAZmX1hgo2I_UgbtiM,3265
+datapipelab/app/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+datapipelab/app/connector_node/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+datapipelab/app/node/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+datapipelab/app/node/tnode.py,sha256=npHG4fFZty5JZ3F_okO9xml-BRhu4DkrZuNE6oaLbvw,446
+datapipelab/app/node/processor/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+datapipelab/app/node/processor/custom_node.py,sha256=1nqbJEhNiMP1rmN9ufpUuKO1IkuI2BEM5auW4JceGMA,933
+datapipelab/app/node/processor/spark_node.py,sha256=jzqdffIHUCgOfMFcoqjXdl8wFag-3gafxfNCdssKnwc,483
+datapipelab/app/node/sink/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+datapipelab/app/node/sink/csv_node.py,sha256=YaDporq4L358dJCkheCjpGmDBsYtvai6dnGiS-uf_Mc,1673
+datapipelab/app/node/sink/delta_node.py,sha256=h3gnBsuYoXUlnhCouav-iwdQ4UVZPcvG4_5r9gxY8JM,2036
+datapipelab/app/node/sink/hive_node.py,sha256=E3pP_U7LzCFdZ0LVkqR0LDlU8HZtfbKgOo72yJsLLT0,1098
+datapipelab/app/node/sink/pandas_csv_node.py,sha256=bNF-Sb5pKMV1qAePzyDhwYLB075Rg7tjxfQ6BHcq1Wk,862
+datapipelab/app/node/sink/teams_notification_node.py,sha256=ZDE-F4nfmrK7UnXoSI6mHuhUHWlgE8rUCdPFW5ZXr7E,3564
+datapipelab/app/node/source/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+datapipelab/app/node/source/delta_node.py,sha256=gg7SfuKBAAfjk6OX2jNrot9XX61HoBe3us3D8O-dscE,529
+datapipelab/app/node/source/hive_node.py,sha256=h_AMCnnmon7uLRIGsaHAPWEReD3VaWZXnz9r0TpLGNM,478
+datapipelab/app/node/source/spark_node.py,sha256=S_x2atRFPDnXmhCUtcmaLc4BDFd2H4uQq6wnEJb7Uug,480
+datapipelab-0.1.2.dist-info/METADATA,sha256=p8_ybC58-fobA9s_WGLjfux77Ju4aroL2UNMHMx4kHA,312
+datapipelab-0.1.2.dist-info/WHEEL,sha256=CmyFI0kx5cdEMTLiONQRbGQwjIoR1aIYB7eCAQ4KPJ0,91
+datapipelab-0.1.2.dist-info/top_level.txt,sha256=HgeBjHvXorKzvNqU5BNPutoI771HtiqVit9_-0Zyrb4,12
+datapipelab-0.1.2.dist-info/RECORD,,

datapipelab-0.1.2.dist-info/top_level.txt

@@ -0,0 +1 @@
+datapipelab

datapipelab-0.1.dist-info/RECORD

@@ -1,21 +0,0 @@
-app/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-app/connector_node/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-app/node/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-app/node/tnode.py,sha256=npHG4fFZty5JZ3F_okO9xml-BRhu4DkrZuNE6oaLbvw,446
-app/node/processor/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-app/node/processor/custom_node.py,sha256=1nqbJEhNiMP1rmN9ufpUuKO1IkuI2BEM5auW4JceGMA,933
-app/node/processor/spark_node.py,sha256=jzqdffIHUCgOfMFcoqjXdl8wFag-3gafxfNCdssKnwc,483
-app/node/sink/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-app/node/sink/csv_node.py,sha256=YaDporq4L358dJCkheCjpGmDBsYtvai6dnGiS-uf_Mc,1673
-app/node/sink/delta_node.py,sha256=h3gnBsuYoXUlnhCouav-iwdQ4UVZPcvG4_5r9gxY8JM,2036
-app/node/sink/hive_node.py,sha256=E3pP_U7LzCFdZ0LVkqR0LDlU8HZtfbKgOo72yJsLLT0,1098
-app/node/sink/pandas_csv_node.py,sha256=bNF-Sb5pKMV1qAePzyDhwYLB075Rg7tjxfQ6BHcq1Wk,862
-app/node/sink/teams_notification_node.py,sha256=ZDE-F4nfmrK7UnXoSI6mHuhUHWlgE8rUCdPFW5ZXr7E,3564
-app/node/source/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-app/node/source/delta_node.py,sha256=gg7SfuKBAAfjk6OX2jNrot9XX61HoBe3us3D8O-dscE,529
-app/node/source/hive_node.py,sha256=h_AMCnnmon7uLRIGsaHAPWEReD3VaWZXnz9r0TpLGNM,478
-app/node/source/spark_node.py,sha256=S_x2atRFPDnXmhCUtcmaLc4BDFd2H4uQq6wnEJb7Uug,480
-datapipelab-0.1.dist-info/METADATA,sha256=IYnShuv2Fa5EVRfJSdJhT7kZzT4Nedr9UIEafmrP2YM,310
-datapipelab-0.1.dist-info/WHEEL,sha256=CmyFI0kx5cdEMTLiONQRbGQwjIoR1aIYB7eCAQ4KPJ0,91
-datapipelab-0.1.dist-info/top_level.txt,sha256=io9g7LCbfmTG1SFKgEOGXmCFB9uMP2H5lerm0HiHWQE,4
-datapipelab-0.1.dist-info/RECORD,,

datapipelab-0.1.dist-info/top_level.txt

@@ -1 +0,0 @@
-app