mkpipe 0.1.2__tar.gz

mkpipe-0.1.2/LICENSE

@@ -0,0 +1,201 @@
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "[]"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+   Copyright 2023-2024 Metin Karakus
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.

mkpipe-0.1.2/MANIFEST.in

@@ -0,0 +1 @@
+include mkpipe_core/jars/*

mkpipe-0.1.2/PKG-INFO

@@ -0,0 +1,11 @@
+Metadata-Version: 2.1
+Name: mkpipe
+Version: 0.1.2
+Summary: Core ETL pipeline framework for mkpipe.
+Author: Metin Karakus
+Author-email: metin_karakus@yahoo.com
+License: Apache License 2.0
+Classifier: Programming Language :: Python :: 3
+Classifier: License :: OSI Approved :: Apache Software License
+Requires-Python: >=3.8
+License-File: LICENSE

mkpipe-0.1.2/README.md

@@ -0,0 +1,59 @@
+# MkPipe
+
+**MkPipe** is a modular, open-source ETL (Extract, Transform, Load) tool that allows you to integrate various data sources and sinks easily. It is designed to be extensible with a plugin-based architecture that supports extractors, transformers, and loaders.
+
+## Features
+
+- Extract data from multiple sources (e.g., PostgreSQL, MongoDB).
+- Transform data using custom Python logic and Apache Spark.
+- Load data into various sinks (e.g., ClickHouse, PostgreSQL, Parquet).
+- Plugin-based architecture that supports future extensions.
+- Cloud-native architecture, can be deployed on Kubernetes and other environments.
+
+## Installation
+
+You can install the core package and extractors using pip:
+
+### Install the core package:
+```bash
+pip install mkpipe
+```
+
+### Install the Postgres extractor:
+```bash
+pip install mkpipe-extractor-postgres
+```
+
+### Install additional extractors or loaders as needed:
+You can find or contribute new extractors and loaders in the future.
+
+## Usage
+
+To run the ETL process, use the following command:
+
+```py
+from mkpipe_core.plugins.registry import EXTRACTORS
+
+def test_postgres_extractor():
+    postgres_extractor = EXTRACTORS.get("postgres")
+    if not postgres_extractor:
+        print("Postgres extractor not found!")
+        return
+    instance = postgres_extractor()
+    instance.extract()
+
+if __name__ == "__main__":
+    test_postgres_extractor()
+
+```
+
+Where `elt.yaml` is your configuration file that specifies the extractors, transformers, and loaders.
+
+## Documentation
+
+For more detailed documentation, please visit the [GitHub repository](https://github.com/m-karakus/mkpipe).
+
+## License
+
+This project is licensed under the Apache 2.0 License - see the [LICENSE](LICENSE) file for details.
+

mkpipe-0.1.2/mkpipe/__init__.py

@@ -0,0 +1 @@
+from .main import main as run

mkpipe-0.1.2/mkpipe/config.py

@@ -0,0 +1,63 @@
+import yaml
+from pathlib import Path
+
+ROOT_DIR = Path(__file__).parent.resolve()
+
+timezone = 'UTC'
+spark_driver_memory = '4g'
+spark_executor_memory = '3g'
+partitions_count = 2
+default_iterate_max_loop = 1_000
+default_iterate_batch_size = 500_000
+
+
+def update_globals(config):
+    """Update global variables based on the provided config dictionary."""
+    global_vars = globals()
+    for key, value in config.items():
+        if key in global_vars:  # Update only if the key exists in the globals
+            global_vars[key] = value
+
+
+def load_config(config_file):
+    config_path = Path(config_file).resolve()
+    if not config_path.exists():
+        raise FileNotFoundError(f'Configuration file not found: {config_path}')
+
+    global ROOT_DIR
+    ROOT_DIR = config_path.parent
+
+    with config_path.open('r') as f:
+        data = yaml.safe_load(f)
+        ENV = data.get(
+            'default_environment', 'prod'
+        )  # Default to 'prod' if not specified
+        env_config = data.get(ENV, {})
+
+    # Extract settings under the 'settings' key
+    settings = env_config.get('settings', {})
+    update_globals(settings)
+
+    return env_config
+
+
+def get_config_value(keys, file_name):
+    """
+    Retrieve a specific configuration value using a list of keys.
+
+    Args:
+        keys (list): List of keys to retrieve the value (e.g., ['paths', 'bucket_name']).
+        file_name (str, optional): Path to the configuration file. Defaults to None.
+
+    Returns:
+        The value corresponding to the keys or None if the path is invalid.
+    """
+    config = load_config(file_name)
+
+    value = config
+    for key in keys:
+        if isinstance(value, dict) and key in value:
+            value = value[key]
+        else:
+            return None
+    return value

mkpipe-0.1.2/mkpipe/main.py

@@ -0,0 +1,118 @@
+import os
+import time
+from .config import load_config, get_config_value, timezone
+from .run_coordinators import get_coordinator
+
+from .utils import Logger, InputTask
+
+os.environ['TZ'] = timezone
+time.tzset()
+
+
+# Function to manage priority levels and reset if necessary
+def get_priority(pipeline_name, priority, custom_priority):
+    """Adjust priority based on pipeline name or reset if necessary."""
+    if custom_priority:
+        return custom_priority
+    priority -= 1
+    return 200 if priority < 1 else priority
+
+
+def main(config_file_name: str, pipeline_name_set=None, table_name_set=None):
+    logger = Logger(config_file_name)
+    logger.log({'file_name': config_file_name})
+
+    DATA = load_config(config_file=config_file_name)
+    run_coordinator = get_config_value(
+        ['settings', 'run_coordinator'], file_name=config_file_name
+    )
+
+    # Validate that pipeline_name_set and table_name_set are sets
+    if pipeline_name_set and not isinstance(pipeline_name_set, set):
+        raise TypeError('pipeline_name_set must be a set')
+    if table_name_set and not isinstance(table_name_set, set):
+        raise TypeError('table_name_set must be a set')
+
+    jobs = DATA['jobs']
+
+    # Filter jobs based on pipeline_name_set if provided
+    if pipeline_name_set:
+        jobs = [job for job in jobs if job['name'] in pipeline_name_set]
+
+    # Initialize the priority counter starting from 200
+    priority = 200
+
+    # Create a list to hold all tasks for the chord
+    task_group = []
+
+    # Iterate over the filtered jobs
+    for job in jobs:
+        pipeline_name = job['name']
+        extract_task = job['extract_task']
+        load_task = job['load_task']
+        custom_priority = job.get('priority', None)
+
+        print(f'Running pipeline: {pipeline_name}')
+
+        # Loader Configuration
+        try:
+            loader_conf = DATA['loaders'][load_task]['config']
+            loader_variant = DATA['loaders'][load_task]['variant']
+            connection_params = DATA['connections'][loader_conf['connection_ref']]
+            loader_conf['connection_params'] = connection_params
+
+        except KeyError as e:
+            logger.log(
+                {
+                    'error': f'Loader configuration issue: {str(e)}',
+                    'loader_task': load_task,
+                }
+            )
+            continue  # Skip this job if there is an error
+
+        # Extractor Configuration
+        try:
+            extractor_conf = DATA['extractors'][extract_task]['config']
+            extractor_variant = DATA['extractors'][extract_task]['variant']
+            connection_params = DATA['connections'][extractor_conf['connection_ref']]
+            extractor_conf['connection_params'] = connection_params
+        except KeyError as e:
+            logger.log(
+                {
+                    'error': f'Extractor configuration issue: {str(e)}',
+                    'extract_task': extract_task,
+                }
+            )
+            continue  # Skip this job if there is an error
+
+        # Loop through each table in the extractor's configuration
+        for table in extractor_conf.get('tables', []):
+            # Skip this table if it's not in the provided set
+            if table_name_set and table['name'] not in table_name_set:
+                continue
+
+            # Copy the extractor config for the current table
+            current_table_conf = extractor_conf.copy()
+            current_table_conf['table'] = table  # Add the specific table
+            current_table_conf.pop('tables', None)
+
+            # Get the adjusted priority level for the current pipeline
+            priority = get_priority(pipeline_name, priority, custom_priority)
+
+            task = InputTask(
+                extractor_variant=extractor_variant,
+                current_table_conf=current_table_conf,
+                loader_variant=loader_variant,
+                loader_conf=loader_conf,
+                priority=custom_priority,
+            )
+
+            # Add the extraction task to the chord group, using kwargs
+            task_group.append(task)
+
+    if not task_group:
+        logger.log({'warning': 'No tasks were scheduled to run.'})
+        return
+
+    coordinator = get_coordinator(run_coordinator)(task_group)
+    coordinator.run()

mkpipe-0.1.2/mkpipe/plugins/__init__.py

@@ -0,0 +1,3 @@
+from .registry import EXTRACTORS, LOADERS, get_extractor, get_loader
+
+__all__ = (get_extractor, get_loader)

mkpipe-0.1.2/mkpipe/plugins/registry.py

@@ -0,0 +1,35 @@
+import importlib.metadata
+
+
+def discover_plugins(group):
+    """
+    Discover plugins registered under a specific entry point group.
+    :param group: Entry point group name (e.g., 'mkpipe.extractors')
+    :return: Dictionary of plugin names and their corresponding classes
+    """
+    try:
+        entry_points = importlib.metadata.entry_points(group=group)
+        return {ep.name: ep.load() for ep in entry_points}
+    except Exception as e:
+        print(f'Error discovering plugins: {e}')
+        return {}
+
+
+# Example usage
+EXTRACTOR_GROUP = 'mkpipe.extractors'
+LOADER_GROUP = 'mkpipe.loaders'
+
+EXTRACTORS = discover_plugins(EXTRACTOR_GROUP)
+LOADERS = discover_plugins(LOADER_GROUP)
+
+
+def get_loader(variant):
+    if variant not in LOADERS:
+        raise ValueError(f'Unsupported loader type: {variant}')
+    return LOADERS.get(variant)
+
+
+def get_extractor(variant):
+    if variant not in EXTRACTORS:
+        raise ValueError(f'Unsupported extractor type: {variant}')
+    return EXTRACTORS.get(variant)

mkpipe-0.1.2/mkpipe/run_coordinators/__init__.py

@@ -0,0 +1,3 @@
+from .registry import get_coordinator
+
+__all__ = get_coordinator

mkpipe-0.1.2/mkpipe/run_coordinators/coordinator_celery.py

@@ -0,0 +1,163 @@
+# celery --app mkpipe.run_coordinators.coordinator_celery.CoordinatorCelery.app worker --loglevel=info --concurrency=4
+from celery import Celery, chord
+from kombu import Queue
+from dotenv import load_dotenv
+import os
+import datetime
+from ..plugins import get_extractor, get_loader
+
+
+class CoordinatorCelery:
+    def __init__(self, task_group):
+        self.task_group = task_group
+        self.app = self.initialize_celery()
+        self.register_tasks()
+
+    def initialize_celery(self):
+        # Load environment variables
+        dotenv_path = os.path.join(os.path.dirname(__file__), '..', '.env')
+        load_dotenv(dotenv_path)
+
+        # Celery app configuration
+        broker_user = os.getenv('RABBITMQ_DEFAULT_USER', 'guest')
+        broker_pass = os.getenv('RABBITMQ_DEFAULT_PASS', 'guest')
+        broker_host = os.getenv('BROKER_HOST', 'rabbitmq')
+        broker_port = os.getenv('BROKER_PORT', '5672')
+
+        CELERY_BROKER_URL = (
+            f'amqp://{broker_user}:{broker_pass}@{broker_host}:{broker_port}//'
+        )
+        CELERY_BACKEND_URL = os.getenv('CELERY_BACKEND_URL', 'rpc://')
+
+        app = Celery('celery_app')
+        app.conf.update(
+            broker_url=CELERY_BROKER_URL,
+            result_backend=CELERY_BACKEND_URL,
+            task_acks_late=True,
+            worker_prefetch_multiplier=1,
+            task_queues=(
+                Queue(
+                    'mkpipe_queue',
+                    exchange='mkpipe_exchange',
+                    routing_key='mkpipe',
+                    queue_arguments={'x-max-priority': 255},
+                ),
+            ),
+            task_routes={
+                'elt.celery_app.extract_data': {'queue': 'mkpipe_queue'},
+                'elt.celery_app.load_data': {'queue': 'mkpipe_queue'},
+            },
+            task_default_queue='mkpipe_queue',
+            task_default_exchange='mkpipe_exchange',
+            task_default_routing_key='mkpipe',
+            # Retry settings
+            task_retry_limit=3,  # Maximum retries
+            task_retry_backoff=True,  # Enable exponential backoff
+            task_retry_backoff_jitter=True,  # Adds slight randomness
+            result_expires=3600,  # Result expiration time in seconds
+            result_chord_retry_interval=60,
+            broker_connection_retry_on_startup=True,
+            worker_direct=True,
+        )
+        return app
+
+    def register_tasks(self):
+        """Register tasks dynamically after app initialization."""
+
+        @self.app.task(
+            bind=True,
+            max_retries=3,
+            retry_backoff=True,
+            retry_backoff_jitter=True,
+            track_started=True,
+        )
+        def extract_data(self_task, **kwargs):
+            extractor_variant = kwargs['extractor_variant']
+            current_table_conf = kwargs['current_table_conf']
+            loader_variant = kwargs['loader_variant']
+            loader_conf = kwargs['loader_conf']
+
+            extractor = get_extractor(extractor_variant)(current_table_conf)
+            data = extractor.extract()
+
+            if data:
+                self_task.request.app.send_task(
+                    'load_data',
+                    kwargs={
+                        'loader_variant': loader_variant,
+                        'loader_conf': loader_conf,
+                        'data': data,
+                    },
+                    priority=201,
+                    queue='mkpipe_queue',
+                    exchange='mkpipe_exchange',
+                    routing_key='mkpipe',
+                )
+
+            print('Extracted data successfully!')
+            return True
+
+        @self.app.task(
+            bind=True,
+            max_retries=3,
+            retry_backoff=True,
+            retry_backoff_jitter=True,
+            track_started=True,
+        )
+        def load_data(self_task, **kwargs):
+            loader_variant = kwargs['loader_variant']
+            loader_conf = kwargs['loader_conf']
+            data = kwargs['data']
+
+            loader = get_loader(loader_variant)(loader_conf)
+            elt_start_time = datetime.datetime.now()
+            loader.load(data, elt_start_time)
+
+            print('Loaded data successfully!')
+            return True
+
+        @self.app.task
+        def on_all_tasks_completed(results):
+            print(f'All tasks completed with results: {results}')
+
+            if all(results):
+                print('Both extraction and loading tasks succeeded.')
+            else:
+                print('One or more tasks failed. DBT not triggered.')
+
+            return 'All tasks completed!' if all(results) else 'Some tasks failed!'
+
+        # Assign tasks to the class instance
+        self.extract_data = extract_data
+        self.load_data = load_data
+        self.on_all_tasks_completed = on_all_tasks_completed
+
+    def run_parallel_tasks(self, task_group):
+        chord(
+            task_group,
+            body=self.on_all_tasks_completed.s().set(
+                queue='mkpipe_queue',
+                exchange='mkpipe_exchange',
+                routing_key='mkpipe',
+            ),
+        ).apply_async()
+
+    def run(self):
+        celery_task_group = []
+        for task in self.task_group:
+            celery_task_group.append(
+                self.extract_data.s(
+                    extractor_variant=task.extractor_variant,
+                    current_table_conf=task.current_table_conf,
+                    loader_variant=task.loader_variant,
+                    loader_conf=task.loader_conf,
+                ).set(
+                    priority=task.priority,
+                    queue='mkpipe_queue',
+                    exchange='mkpipe_exchange',
+                    routing_key='mkpipe',
+                )
+            )
+
+        if celery_task_group:
+            self.run_parallel_tasks(celery_task_group)

mkpipe-0.1.2/mkpipe/run_coordinators/coordinator_single.py

@@ -0,0 +1,38 @@
+import datetime
+from ..plugins import get_extractor, get_loader
+
+
+class CoordinatorSingle:
+    def __init__(self, task_group):
+        self.task_group = task_group
+
+    def load_data(self, task):
+        # Initialize loader instance
+        loader = get_loader(task.loader_variant)(task.loader_conf)
+
+        # Record the start time of the loading process
+        elt_start_time = datetime.datetime.now()
+
+        # Load the extracted data with the start time
+        loader.load(task.data, elt_start_time)
+
+        print('Loaded data successfully!')
+        return True  # Return True to indicate success
+
+    def extract_data(self, task):
+        # Initialize extractor and loader instances
+        extractor = get_extractor(task.extractor_variant)(task.current_table_conf)
+
+        # Perform the data extraction
+        task.data = extractor.extract()
+        print('Extracted data successfully!')
+
+        if task.data:
+            # Schedule the data loading as a separate task
+            self.load_data(task)
+
+        return True  # Return True to indicate success
+
+    def run(self):
+        for task in self.task_group:
+            self.extract_data(task)

mkpipe-0.1.2/mkpipe/run_coordinators/registry.py

@@ -0,0 +1,29 @@
+from .coordinator_single import CoordinatorSingle
+from .coordinator_celery import CoordinatorCelery
+
+
+RUN_COORDINATORS = {}
+
+
+def register_coordinator(variant):
+    def decorator(fn):
+        RUN_COORDINATORS[variant] = fn
+        return fn
+
+    return decorator
+
+
+@register_coordinator('single')
+def coordinator_single(conf):
+    return CoordinatorSingle(conf)
+
+
+@register_coordinator('celery')
+def coordinator_celery(conf):
+    return CoordinatorCelery(conf)
+
+
+def get_coordinator(variant):
+    if variant not in RUN_COORDINATORS:
+        raise ValueError(f'Unsupported coordinator type: {variant}')
+    return RUN_COORDINATORS.get(variant)

mkpipe-0.1.2/mkpipe/utils/__init__.py

@@ -0,0 +1,9 @@
+from .logger import Logger
+from .logger import log_container
+from .base_class import InputTask
+
+__all__ = (
+    Logger,
+    log_container,
+    InputTask,
+)

mkpipe-0.1.2/mkpipe/utils/base_class.py

@@ -0,0 +1,11 @@
+from pydantic import BaseModel
+from typing import Optional
+
+
+class InputTask(BaseModel):
+    extractor_variant: str
+    current_table_conf: dict
+    loader_variant: str
+    loader_conf: dict
+    priority: Optional[int] = None
+    data: Optional[dict] = None

mkpipe-0.1.2/mkpipe/utils/logger.py

@@ -0,0 +1,184 @@
+import json
+import traceback
+import os
+import logging
+import time
+import logging.handlers
+from dagster import get_dagster_logger
+from ..config import ROOT_DIR
+
+path = os.path.abspath(os.path.join(ROOT_DIR, 'logs'))
+# path = ROOT_DIR / "logs"
+# path.mkdir(exist_ok=True)  # Create the directory if it doesn't exist
+
+LOG_LEVEL = os.getenv('LOG_LEVEL', 'INFO').upper()
+
+
+log_levels = {
+    'debug': logging.DEBUG,
+    'info': logging.INFO,
+    'warning': logging.WARNING,
+    'error': logging.ERROR,
+    'critical': logging.CRITICAL,
+}
+
+default_log_level = log_levels.get(LOG_LEVEL.lower())
+
+
+class Logger:
+    def __init__(self, name) -> None:
+        """
+        Custom Usage:
+        logger = Logger()
+        msg = 'This is a log message'
+        logger.log(msg, log_level="error")
+        or
+        logger.error("This is an error message")
+        """
+        self.logger = logging.getLogger(name)
+        self.dagster_logger = get_dagster_logger()
+        if not self.logger.handlers:
+            # create the handlers and call logger.addHandler(logging_handler)
+            self.logger = logging.getLogger(name)
+            # self.logger.setLevel(logging.DEBUG)
+            self.logger.setLevel(default_log_level)
+
+            # Create a formatter
+            # https://docs.python.org/3/library/logging.html#logrecord-attributes
+
+            frmt = """{"timestamp" : "%(asctime)s", "levelno" : "%(levelno)s", "level" : "%(levelname)s", "message" : %(message)s, "function" : "%(name)s" }"""
+            json_formatter = logging.Formatter(frmt)
+            json_formatter.converter = time.gmtime  # set timezone as gmtime
+
+            frmt = '%(asctime)s | %(levelno)s | %(levelname)s | %(name)s | %(message)s'
+            string_formatter = logging.Formatter(frmt)
+            string_formatter.converter = time.gmtime  # set timezone as gmtime
+
+            # Create a file handler
+            # Fetch the pod name (e.g., celery-consumer-66894599cd-85dvr)
+            pod_name = os.getenv('HOSTNAME', 'unknown_pod')
+
+            # date = datetime.today().strftime('%Y%m%d')
+            if not os.path.exists(path):
+                os.makedirs(path)
+            file_path = os.path.abspath(
+                os.path.join(
+                    path, f'{pod_name}_log.log'
+                )  # Use pod name for unique log file
+            )
+
+            # use very short interval for this example, typical 'when' would be 'midnight' and no explicit interval
+            fh = logging.handlers.TimedRotatingFileHandler(
+                file_path, when='midnight', backupCount=7
+            )
+            # fh = logging.handlers.TimedRotatingFileHandler(file_path, when="midnight", backupCount=7)
+            # fh = logging.FileHandler(file_path, mode='a', encoding='utf8')
+
+            fh.setLevel(default_log_level)
+            # fh.setFormatter(json_formatter)
+            fh.setFormatter(string_formatter)
+
+            # Create a console handler (optional)
+            ch = logging.StreamHandler()
+            # ch.setLevel(logging.ERROR)
+            ch.setLevel(default_log_level)
+            ch.setFormatter(json_formatter)
+
+            # Add handlers to the logger
+            self.logger.addHandler(fh)
+            # self.logger.addHandler(ch)
+
+    def message_formatter(self, message):
+        # msg = json.dumps(str(message))
+        msg = json.dumps(message, sort_keys=True, indent=2, separators=(',', ': '))
+        return msg
+
+    def log(self, message, log_level='info'):
+        msg = self.message_formatter(message)
+
+        logger = self.logger
+        if log_level == 'debug':
+            logger.debug(msg)
+            self.dagster_logger.debug(msg)
+        elif log_level == 'info':
+            logger.info(msg)
+            self.dagster_logger.info(msg)
+        elif log_level == 'warning':
+            logger.warning(msg)
+            self.dagster_logger.warning(msg)
+        elif log_level == 'error':
+            logger.error(msg)
+            self.dagster_logger.error(msg)
+        elif log_level == 'critical':
+            logger.critical(msg)
+            self.dagster_logger.critical(msg)
+        else:
+            logger.exception(msg)
+            self.dagster_logger.exception(msg)
+        return
+
+    def debug(self, message):
+        msg = self.message_formatter(message)
+        self.logger.debug(msg)
+        self.dagster_logger.debug(msg)
+        return
+
+    def info(self, message):
+        msg = self.message_formatter(message)
+        self.logger.info(msg)
+        self.dagster_logger.info(msg)
+        return
+
+    def warning(self, message):
+        msg = self.message_formatter(message)
+        self.logger.warning(msg)
+        self.dagster_logger.warning(msg)
+        return
+
+    def error(self, message):
+        msg = self.message_formatter(message)
+        self.logger.error(msg)
+        self.dagster_logger.error(msg)
+        return
+
+    def critical(self, message):
+        msg = self.message_formatter(message)
+        self.logger.critical(msg)
+        self.dagster_logger.critical(msg)
+        return
+
+    def shutdown(self):
+        logging.shutdown()
+
+
+def log_container(name):
+    def inner(func):
+        def wrapper(*args, **kwargs):
+            # Log some messages
+            logger = Logger(name)
+
+            try:
+                # send start message
+                # message = f'Started function: {func.__name__}'
+                # logger.log(message)
+                # start_time = time.time()
+
+                # call the function
+                result = func(*args, **kwargs)
+
+                # run_time = time.time() - start_time
+                # message = (
+                #     f'Ended function: {func.__name__}. Time Duration(sec): {run_time} '
+                # )
+                # logger.log(message)
+
+                return result
+
+            except Exception as e:
+                message = str(e) + str(traceback.format_exc()).replace('\n', ' ')
+                logger.log(message, log_level='error')
+                raise
+
+        return wrapper
+
+    return inner

mkpipe-0.1.2/mkpipe.egg-info/PKG-INFO

@@ -0,0 +1,11 @@
+Metadata-Version: 2.1
+Name: mkpipe
+Version: 0.1.2
+Summary: Core ETL pipeline framework for mkpipe.
+Author: Metin Karakus
+Author-email: metin_karakus@yahoo.com
+License: Apache License 2.0
+Classifier: Programming Language :: Python :: 3
+Classifier: License :: OSI Approved :: Apache Software License
+Requires-Python: >=3.8
+License-File: LICENSE

mkpipe-0.1.2/mkpipe.egg-info/SOURCES.txt

@@ -0,0 +1,21 @@
+LICENSE
+MANIFEST.in
+README.md
+setup.py
+mkpipe/__init__.py
+mkpipe/config.py
+mkpipe/main.py
+mkpipe.egg-info/PKG-INFO
+mkpipe.egg-info/SOURCES.txt
+mkpipe.egg-info/dependency_links.txt
+mkpipe.egg-info/top_level.txt
+mkpipe/plugins/__init__.py
+mkpipe/plugins/registry.py
+mkpipe/run_coordinators/__init__.py
+mkpipe/run_coordinators/coordinator_celery.py
+mkpipe/run_coordinators/coordinator_single.py
+mkpipe/run_coordinators/registry.py
+mkpipe/utils/__init__.py
+mkpipe/utils/base_class.py
+mkpipe/utils/logger.py
+tests/test_core.py

mkpipe-0.1.2/mkpipe.egg-info/dependency_links.txt

@@ -0,0 +1 @@
+

mkpipe-0.1.2/mkpipe.egg-info/top_level.txt

@@ -0,0 +1 @@
+mkpipe

mkpipe-0.1.2/setup.cfg

@@ -0,0 +1,4 @@
+[egg_info]
+tag_build = 
+tag_date = 0
+

mkpipe-0.1.2/setup.py

@@ -0,0 +1,23 @@
+from setuptools import setup, find_packages
+
+setup(
+    name='mkpipe',
+    version='0.1.2',
+    license='Apache License 2.0',
+    packages=find_packages(exclude=['tests', 'scripts']),
+    install_requires=[],
+    include_package_data=True,
+    entry_points={
+        'mkpipe.extractors': [],
+        'mkpipe.loaders': [],
+        'mkpipe.transformers': [],
+    },
+    description='Core ETL pipeline framework for mkpipe.',
+    author='Metin Karakus',
+    author_email='metin_karakus@yahoo.com',
+    classifiers=[
+        'Programming Language :: Python :: 3',
+        'License :: OSI Approved :: Apache Software License',
+    ],
+    python_requires='>=3.8',
+)