scalable-pypeline 2.1.31__py2.py3-none-any.whl

scalable_pypeline-2.1.31.dist-info/METADATA

@@ -0,0 +1,212 @@
+Metadata-Version: 2.1
+Name: scalable-pypeline
+Version: 2.1.31
+Summary: PypeLine - Python pipelines for the Real World
+Home-page: https://gitlab.com/bravos2/pypeline
+Author: Bravos Power Corporation
+License: Apache License 2.0
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: PyYAML (<7,>=6.0.1)
+Requires-Dist: marshmallow (<4,>=3.2.1)
+Requires-Dist: redis (<5,>=4.5.4)
+Requires-Dist: croniter (<2,>=1.0.15)
+Requires-Dist: db-medley[redis] (<2,>=1.0.2)
+Provides-Extra: build
+Requires-Dist: wheel ; extra == 'build'
+Requires-Dist: twine ; extra == 'build'
+Provides-Extra: dev
+Requires-Dist: black ; extra == 'dev'
+Provides-Extra: flask
+Requires-Dist: markupsafe (==2.0.1) ; extra == 'flask'
+Requires-Dist: flask-smorest (<1,>=0.23.0) ; extra == 'flask'
+Provides-Extra: test
+Requires-Dist: pytest-cov (<3,>=2.6.1) ; extra == 'test'
+Requires-Dist: tox (<4,>=3.14.1) ; extra == 'test'
+Requires-Dist: mock (<2,>=1) ; extra == 'test'
+Requires-Dist: responses (<0.11,>=0.10.16) ; extra == 'test'
+Requires-Dist: fakeredis (<2.31,>=2.10.3) ; extra == 'test'
+Provides-Extra: web
+Requires-Dist: gunicorn ; extra == 'web'
+Requires-Dist: gevent (<22,>=21.12.0) ; extra == 'web'
+Provides-Extra: workers
+Requires-Dist: networkx (>=2.4) ; extra == 'workers'
+Requires-Dist: dramatiq[rabbitmq] (==1.17.0) ; extra == 'workers'
+Requires-Dist: apscheduler (<4,>=3.10.4) ; extra == 'workers'
+Requires-Dist: tenacity (==8.0.1) ; extra == 'workers'
+
+```
+______ __   ________  _____  _     _____  _   _  _____ 
+| ___ \\ \ / /| ___ \|  ___|| |   |_   _|| \ | ||  ___|
+| |_/ / \ V / | |_/ /| |__  | |     | |  |  \| || |__  
+|  __/   \ /  |  __/ |  __| | |     | |  | . ` ||  __| 
+| |      | |  | |    | |___ | |_____| |_ | |\  || |___ 
+\_|      \_/  \_|    \____/ \_____/\___/ \_| \_/\____/                                 
+```
+
+## Overview
+
+PypeLine is a versatile open-source library designed to streamline the management of data workflows and APIs. With PypeLine, you can efficiently schedule cron jobs, execute complex Directed Acyclical Graph (DAG) pipelines, and set up a Flask API complete with OpenAPI documentation.
+
+#### Key Features
+- Cron Job Scheduling: Easily schedule recurring tasks with flexible cron job functionality, ensuring that your processes run reliably at specified intervals.
+- DAG Pipelines: Define and execute DAGs to manage complex data workflows with dependencies. PypeLine handles the execution order and parallelism, ensuring that each task runs in the correct sequence.
+- Flask API with OpenAPI: Quickly configure a RESTful API using Flask, with built-in support for OpenAPI documentation, allowing for clear, standardized documentation of your endpoints.
+
+## Requirements
+
+- RabbitMQ
+- Redis
+- Docker (optional for dev)
+
+## Getting Started
+
+Install PypeLines:
+
+```commandline
+pip install scalable-pypeline[flask,web,workers]>=1.2.3
+```
+
+Configure your Flask project (app.py)
+
+```python
+from flask import Flask
+from pypeline.flask import FlaskPypeline
+from pypeline_demo.api import bp
+from pypeline_demo.config import Config
+from pypeline_demo.extensions import dramatiq
+
+
+
+def create_app():
+    app = Flask(__name__)
+
+    dramatiq.init_app(app)
+
+    # Initialize your app with a configuration
+    app.config.from_object(Config)
+
+    pypeline = FlaskPypeline()
+    pypeline.init_app(app, init_api=True)
+
+    # Register API blueprints you wish 
+    app.extensions["pypeline_core_api"].register_blueprint(bp)
+    # Register application blueprints to application
+    app.register_blueprint(bp)
+
+    return app
+
+
+if __name__ == "__main__":
+    app = create_app()
+    app.run(port=5001)
+```
+
+Configure Dramatiq extension (extensions.py)
+
+```python
+from pypeline.dramatiq import Dramatiq
+
+
+dramatiq = Dramatiq()
+```
+
+Setup your yaml configuration for pypelines (pypeline.yaml)
+
+```yaml
+serviceConfig:
+    - name: pipeline-worker
+      registeredTasks:
+          - handler: pypeline_demo.pipeline.a
+          - handler: pypeline_demo.pipeline.b
+          - handler: pypeline_demo.pipeline.c
+          - handler: pypeline_demo.scheduled_tasks.cron_task
+
+pipelines:
+    demo_pipeline:
+        name: Demo Pipeline
+        description: Pipeline to show examples of DAG Adjacency
+        schemaVersion: 1
+        config:
+            dagAdjacency:
+                a:
+                    - b
+                    - c
+            metadata:
+                maxRetry: 1
+                retryBackoff: 180
+                retryBackoffMax: 300
+                retryJitter: true
+                maxTtl: 10800
+                queue: new-queue
+            taskDefinitions:
+                a:
+                    handler: pypeline_demo.pipeline.a
+                b:
+                    handler:  pypeline_demo.pipeline.b
+                c:
+                    handler:  pypeline_demo.pipeline.c
+scheduledTasks:
+    cron-task:
+        name: Example cron task
+        enabled: true
+        config:
+            task: pypeline_demo.scheduled_tasks.cron_task
+            queue: new-queue
+            schedule:
+                minute: '*'
+                hour: '*'
+                dayOfWeek: '*'
+                dayOfMonth: '*'
+                monthOfYear: '*'
+        schemaVersion: 1
+```
+
+Setup your modules to be executed by yaml (pipeline.py && scheduled_tasks.py)
+
+```python
+import time
+
+
+def a(event):
+    print("A")
+
+
+def b(event):
+    print("B")
+    time.sleep(10)
+
+
+def c(event):
+    print("C")
+```
+
+```python
+def cron_task():
+    print("HI")
+```
+
+Configure your environment variables (demo.env)
+
+```env
+SERMOS_BASE_URL=local
+PYPELINE_CLIENT_PKG_NAME=pypeline_demo
+REDIS_URL=redis://:password@localhost:6379/0
+RABBITMQ_URL=amqp://admin:password@localhost:5672
+```
+
+Start Rabbit & Redis as your message broker and backend results storage.  We use `docker compose` for this.
+
+## DEMO PROJECT COMING SOON!
+
+
+## Testing
+
+If you are developing pypeline and want to test this package,
+install the test dependencies:
+
+    $ pip install -e .[test]
+
+Now, run the tests:
+
+    $ tox

scalable_pypeline-2.1.31.dist-info/RECORD

@@ -0,0 +1,42 @@
+pypeline/__init__.py,sha256=Ot_7SOv9f1OG3mDRlKxeTuWgH4qbOFyTXh8THXd9ZXs,23
+pypeline/barrier.py,sha256=ojSgbuZnGKpKiSBYXTV4CxG9j1Z01YdzBSORli4MnzI,2376
+pypeline/constants.py,sha256=SSRWNCpJ-VjwZrwCzfE1bLXyD6_h9_bHBr4olG9Oab0,3591
+pypeline/dramatiq.py,sha256=NHwKrXG_BN7O9zjGt8y04DfynsHJmINzxPiJl4_0rZY,15862
+pypeline/executable_job_config_schema.py,sha256=P2Z8SO057Jgyt4I5oZxcbEi1iaZkLoAh7qp8PtuqcqU,1010
+pypeline/extensions.py,sha256=BzOTnXhNxap3N7uIUUh_hO6dDwx08Vc_RJDE93_K0Lo,610
+pypeline/job_runner.py,sha256=08tZLMK8KNovgaW4Jic3K7zMX0EQ5lZO60H0rQimRxs,6747
+pypeline/pipeline_config_schema.py,sha256=kRZcCMlk2FIITDzVrAfcSmHnxi1mIWmDzasTW0TnaAU,11169
+pypeline/pipeline_settings_schema.py,sha256=s_oqZ-TBiLdInSprR9k6myw1zykV2PpEbEfVaMfb5VY,21010
+pypeline/pypeline_yaml.py,sha256=hbOdwKDUg10wsZnwVaBt46FbpS3iuB3bLwVuYyXh4OY,17270
+pypeline/schedule_config_schema.py,sha256=vtZV-5wpGcAiYcXxdBPRkrjsbR6x_9E-1PC2elrKKbE,3611
+pypeline/flask/__init__.py,sha256=AdljRh0lMiS8ExgDmgzObwVs8jW7hqQuf83Ml8kn8GQ,491
+pypeline/flask/decorators.py,sha256=ki6jkjZwbDbCWuj7ET7N-ncZwrASp4Fy7257WIYiAAQ,1102
+pypeline/flask/flask_pypeline.py,sha256=Uqyu3PnSP3DoVZUJPqV9chjT4xdRgvcL3OMXxkbdTEg,5490
+pypeline/flask/api/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+pypeline/flask/api/pipelines.py,sha256=lw1ggsjp_Iha5MhyQGHtVW0akpVJnxIk0hn6NkC3c8s,9314
+pypeline/flask/api/schedules.py,sha256=8PKCMdPucaer8opchNlI5aDssK2UqT79hHpeg5BMtTA,1210
+pypeline/pipelines/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+pypeline/pipelines/factory.py,sha256=ftm6AwXIdkC8LEzy3c4CLPzXtoQonBKUo3yojbYTsUE,3309
+pypeline/pipelines/composition/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+pypeline/pipelines/composition/parallel_pipeline_composition.py,sha256=JrEsIxaB36wLWg5ztCMkANye9VizGuRIhyxyCP4DCl8,14162
+pypeline/pipelines/composition/pypeline_composition.py,sha256=lY1RO7luLqDBLfoX99YbBatx1Qov1cxGoS5lRAjB_DQ,8534
+pypeline/pipelines/middleware/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+pypeline/pipelines/middleware/get_active_worker_id_middleware.py,sha256=X4ZfRk3L8MD00DTsGHth7oOdy-W7LQV96T8vu5UC42A,755
+pypeline/pipelines/middleware/graceful_shutdown_middleware.py,sha256=k37zmFk9dOye05BoQP7KcB9MEQgvodI16kOJyYhRyAc,1764
+pypeline/pipelines/middleware/parallel_pipeline_middleware.py,sha256=kTp6niYoe2nXIiN6EGRfdpxrJyioo0GPxDkfefbGlEk,2821
+pypeline/pipelines/middleware/pypeline_middleware.py,sha256=FjREuPDdTfeYOVGVUjJgx8Szh6yu7g8OnHRc5N__448,9385
+pypeline/utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+pypeline/utils/config_utils.py,sha256=bblh8clRDDNQpQIDkBrtskZBo-csakoO1IJiaqVGyr8,2508
+pypeline/utils/dramatiq_utils.py,sha256=tbG3o5FD0zUOKtJJJECE2cM2ovDv3OLQ5CplJ9cXfM4,4001
+pypeline/utils/executable_job_util.py,sha256=Id4FYvCdzqOCzQYSoU2yeOncubKwSwITUf_BgIPZQUw,1206
+pypeline/utils/graceful_shutdown_util.py,sha256=gdyf4r_vlunZZP-prcol_9aJQEG9Hk8evivVSe_H0N0,1293
+pypeline/utils/module_utils.py,sha256=-yEJIukDCoXnmlZVXB6Dww25tH6GdPE5SoFqv6pfdVU,3682
+pypeline/utils/pipeline_utils.py,sha256=kGP1QwCJikGC5QNRtzRXCDVewyRMpWIqERTNnxGLlSY,4795
+pypeline/utils/schema_utils.py,sha256=Fgl0y9Cuo_TZeEx_S3gaSVnLjn6467LTkjb2ek7Ms98,851
+tests/fixtures/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+scalable_pypeline-2.1.31.dist-info/LICENSE,sha256=DVQuDIgE45qn836wDaWnYhSdxoLXgpRRKH4RuTjpRZQ,10174
+scalable_pypeline-2.1.31.dist-info/METADATA,sha256=nInraHJGNTj9lTVB1zOvh2mMmzYL7l7asOuBa-0e_vg,5985
+scalable_pypeline-2.1.31.dist-info/WHEEL,sha256=bb2Ot9scclHKMOLDEHY6B2sicWOgugjFKaJsT7vwMQo,110
+scalable_pypeline-2.1.31.dist-info/entry_points.txt,sha256=q5V8Qe0-5mrZ982FGbUaT2-bEURsiGjLNKEvpR6m7VU,176
+scalable_pypeline-2.1.31.dist-info/top_level.txt,sha256=C7dpkEOc_-nnsAQb28BfQknjD6XHRyS9ZrvVeoIbV7s,15
+scalable_pypeline-2.1.31.dist-info/RECORD,,

scalable_pypeline-2.1.31.dist-info/WHEEL

@@ -0,0 +1,6 @@
+Wheel-Version: 1.0
+Generator: bdist_wheel (0.38.4)
+Root-Is-Purelib: true
+Tag: py2-none-any
+Tag: py3-none-any
+

scalable_pypeline-2.1.31.dist-info/entry_points.txt

@@ -0,0 +1,6 @@
+[console_scripts]
+job-runner = pypeline.job_runner:main
+
+[flask.commands]
+cron-scheduler = pypeline.dramatiq:cron_scheduler
+pypeline-worker = pypeline.dramatiq:pypeline_worker

scalable_pypeline-2.1.31.dist-info/top_level.txt

@@ -0,0 +1,2 @@
+pypeline
+tests