scalable-pypeline 2.1.18__py2.py3-none-any.whl → 2.1.19__py2.py3-none-any.whl

pypeline/__init__.py

@@ -1 +1 @@
-__version__ = "2.1.18"
+__version__ = "2.1.19"

pypeline/dramatiq.py

@@ -45,7 +45,7 @@ from pypeline.pipelines.middleware.parallel_pipeline_middleware import ParallelP
 from pypeline.pipelines.middleware.pypeline_middleware import PypelineMiddleware
 from pypeline.utils.config_utils import (
     retrieve_latest_schedule_config,
-    get_service_config_for_worker,
+    get_service_config_for_worker, retrieve_executable_job_config,
 )
 from pypeline.utils.dramatiq_utils import (
     guess_code_directory,
@@ -107,6 +107,7 @@ def configure_default_broker(broker: Broker = None):
 def register_actors_for_workers(broker: Broker):
     service = get_service_config_for_worker(pypeline_config)
     scheduled_jobs_config = retrieve_latest_schedule_config()
+    executable_jobs_config = retrieve_executable_job_config()
 
     if not service:
         return
@@ -155,6 +156,19 @@ def register_actors_for_workers(broker: Broker):
             except Exception as e:
                 logger.exception(f"Unable to add a task to dramatiq: {e}")
 
+    for job in executable_jobs_config or []:
+        config = job["config"]
+        if config["task"] in worker_registered_tasks:
+            pipeline_meta = {"queue": config.get("queue", "default")}
+            try:
+                tmp_handler = get_callable(config["task"])
+                if pipeline_meta and pipeline_meta.get("maxRetry", 0) >= 0:
+                    pipeline_meta["store_results"] = True
+                    _ = register_lazy_actor(broker, tmp_handler, pipeline_meta, None)
+            except Exception as e:
+                logger.exception(f"Unable to add a task to dramatiq: {e}")
+
+
 
 class Dramatiq:
     """Flask extension bridging Dramatiq broker and Flask app.

pypeline/executable_job_config_schema.py

@@ -0,0 +1,35 @@
+from marshmallow import Schema, EXCLUDE, fields
+
+class ExecutableJobConfigSchema(Schema):
+    queue = fields.String(
+        required=True,
+        description="Name of queue on which to place task.",
+        example="my-default-queue",
+    )
+    task = fields.String(
+        required=True,
+        description="Path to task to invoke.",
+        example="my_app.module.method",
+    )
+
+class ExecutableJobSchema(Schema):
+    """Definition of a single schedule entry"""
+    class Meta:
+        unknown = EXCLUDE
+
+    name = fields.String(
+        required=True,
+        description="Name of schedule entry.",
+        example="My Scheduled Task",
+    )
+    schemaVersion = fields.Integer(required=True)
+    config = fields.Dict(required=True)
+    enabled = fields.Boolean(
+        required=True, description="Whether entry is enabled.", example=True
+    )
+    config = fields.Nested(
+        ExecutableJobConfigSchema,
+        required=True,
+        description="Configuration information for this job.",
+    )
+

pypeline/pypeline_yaml.py

@@ -54,6 +54,8 @@ import yaml
 from yaml.loader import SafeLoader
 from marshmallow import Schema, fields, pre_load, EXCLUDE, INCLUDE, validates_schema
 from marshmallow.exceptions import ValidationError
+
+from pypeline.executable_job_config_schema import ExecutableJobSchema
 from pypeline.utils.module_utils import PypelineModuleLoader, normalized_pkg_name
 from pypeline.constants import PYPELINE_YAML_PATH, PYPELINE_CLIENT_PKG_NAME
 from pypeline.pipeline_config_schema import BasePipelineSchema
@@ -165,6 +167,14 @@ class PypelineYamlSchema(ExcludeUnknownSchema):
         required=False,
     )
 
+    executableJobs = fields.Dict(
+        keys=fields.String(),
+        values=fields.Nested(ExecutableJobSchema),
+        description="List of executable jobs",
+        required=False,
+        allow_none=True,
+    )
+
     def validate_errors(self, schema: Schema, value: dict):
         """Run Marshmallow validate() and raise if any errors"""
         schema = schema()

pypeline/utils/config_utils.py

@@ -41,6 +41,16 @@ def retrieve_latest_schedule_config():
     return None
 
 
+def retrieve_executable_job_config():
+    pypeline_config = load_pypeline_config()
+    if "executableJobs" in pypeline_config:
+        tasks = []
+        for task_id, config in pypeline_config["executableJobs"].items():
+            tasks.append(config)
+        return tasks
+    return None
+
+
 def get_service_config_for_worker(
     pypeline_config: dict, worker_name: str = None
 ) -> Union[dict, None]:

pypeline/utils/executable_job_util.py

@@ -0,0 +1,35 @@
+from dramatiq.broker import get_broker
+
+from pypeline.utils.config_utils import retrieve_executable_job_config
+from pypeline.utils.dramatiq_utils import register_lazy_actor, LazyActor
+from pypeline.utils.module_utils import get_callable
+
+
+def execute_job(fn, *args, **kwargs):
+    executable_jobs_config = retrieve_executable_job_config()
+
+    module_path = kwargs.get("module_path", None)
+
+    job = None
+
+    for j in executable_jobs_config or []:
+        if module_path and module_path == j["config"]["task"]:
+            job = j
+            break
+        elif fn.__name__ in j["config"]["task"]:
+            if job:
+                raise ValueError(
+                    f"Multiple matches found in yaml for {fn.__name__}, "
+                    f"Consider passing module_path as a kwarg to avoid ambiguity."
+                )
+            job = j
+
+    if job is None:
+        raise ValueError(f"No match found in yaml for {fn.__name__} function.")
+
+    pipeline_meta = {"queue": job["config"].get("queue", "default")}
+    tmp_handler = get_callable(job["config"]["task"])
+
+    actor: LazyActor = register_lazy_actor(get_broker(), tmp_handler, pipeline_meta, None)
+
+    return actor.send(*args, **kwargs)

{scalable_pypeline-2.1.18.dist-info → scalable_pypeline-2.1.19.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: scalable-pypeline
-Version: 2.1.18
+Version: 2.1.19
 Summary: PypeLine - Python pipelines for the Real World
 Home-page: https://gitlab.com/bravos2/pypeline
 Author: Bravos Power Corporation

{scalable_pypeline-2.1.18.dist-info → scalable_pypeline-2.1.19.dist-info}/RECORD

@@ -1,11 +1,12 @@
-pypeline/__init__.py,sha256=Hc95NdHPIFrDSqp8_iblHu8tL5yscXH7JDPIvIVi8oI,23
+pypeline/__init__.py,sha256=ORNtksaL65IiKUhkJEaS3s0gzZq5HMSHSwIKtUEfCBA,23
 pypeline/barrier.py,sha256=oO964l9qOCOibweOHyNivmAvufdXOke9nz2tdgclouo,1172
 pypeline/constants.py,sha256=EGSuLq4KhZ4bxrbtnUgKclELRyya5ipvv0WeybCzNAs,3049
-pypeline/dramatiq.py,sha256=D-E6_oJc9he-F2rvze-DCq4eBVY3Hq7V0pSC5crHrrU,13703
+pypeline/dramatiq.py,sha256=S_rZMpGgE-VAmx9YjDBkHpSjFdhwp7mz4uuuKBUx4iQ,14405
+pypeline/executable_job_config_schema.py,sha256=P2Z8SO057Jgyt4I5oZxcbEi1iaZkLoAh7qp8PtuqcqU,1010
 pypeline/extensions.py,sha256=BzOTnXhNxap3N7uIUUh_hO6dDwx08Vc_RJDE93_K0Lo,610
 pypeline/pipeline_config_schema.py,sha256=kprvmfPfmNVP5MOqtWzDm4ON5NJWIKQC8GWbIy5EIuk,11183
 pypeline/pipeline_settings_schema.py,sha256=B_Jzh4rUKLYBRSBT2vYgeXMYdFiYRdt3_sX6t125Hkk,20899
-pypeline/pypeline_yaml.py,sha256=Og08sUKwOjq7JYPnkg-NIcGbHravYCkC5Arz22rZEtA,16981
+pypeline/pypeline_yaml.py,sha256=hbOdwKDUg10wsZnwVaBt46FbpS3iuB3bLwVuYyXh4OY,17270
 pypeline/schedule_config_schema.py,sha256=vtZV-5wpGcAiYcXxdBPRkrjsbR6x_9E-1PC2elrKKbE,3611
 pypeline/flask/__init__.py,sha256=AdljRh0lMiS8ExgDmgzObwVs8jW7hqQuf83Ml8kn8GQ,491
 pypeline/flask/decorators.py,sha256=ki6jkjZwbDbCWuj7ET7N-ncZwrASp4Fy7257WIYiAAQ,1102
@@ -24,16 +25,17 @@ pypeline/pipelines/middleware/graceful_shutdown_middleware.py,sha256=k37zmFk9dOy
 pypeline/pipelines/middleware/parallel_pipeline_middleware.py,sha256=kTp6niYoe2nXIiN6EGRfdpxrJyioo0GPxDkfefbGlEk,2821
 pypeline/pipelines/middleware/pypeline_middleware.py,sha256=AGQCRmqXcYNyCxinwtGZplHwltrmZ_sH73CVRLu_0EE,8961
 pypeline/utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-pypeline/utils/config_utils.py,sha256=rAIATyoW7kGETZ_Z2DqiXtGd7bJp5uPfcLtfNPOYsNs,2167
+pypeline/utils/config_utils.py,sha256=C-kZUS3L9zer9Ornrx9Sngf5P38c8zC-izaEpefxmBg,2459
 pypeline/utils/dramatiq_utils.py,sha256=DUdgVywm1182A4i69XzH9EIh1EJ9zAHmJLtOaVSW7pw,3844
+pypeline/utils/executable_job_util.py,sha256=Id4FYvCdzqOCzQYSoU2yeOncubKwSwITUf_BgIPZQUw,1206
 pypeline/utils/graceful_shutdown_util.py,sha256=gdyf4r_vlunZZP-prcol_9aJQEG9Hk8evivVSe_H0N0,1293
 pypeline/utils/module_utils.py,sha256=-yEJIukDCoXnmlZVXB6Dww25tH6GdPE5SoFqv6pfdVU,3682
 pypeline/utils/pipeline_utils.py,sha256=kGP1QwCJikGC5QNRtzRXCDVewyRMpWIqERTNnxGLlSY,4795
 pypeline/utils/schema_utils.py,sha256=Fgl0y9Cuo_TZeEx_S3gaSVnLjn6467LTkjb2ek7Ms98,851
 tests/fixtures/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-scalable_pypeline-2.1.18.dist-info/LICENSE,sha256=DVQuDIgE45qn836wDaWnYhSdxoLXgpRRKH4RuTjpRZQ,10174
-scalable_pypeline-2.1.18.dist-info/METADATA,sha256=GaWesZImz-Dc7QAypUT-8qi3GKte69QSQakBEOITY2c,5982
-scalable_pypeline-2.1.18.dist-info/WHEEL,sha256=bb2Ot9scclHKMOLDEHY6B2sicWOgugjFKaJsT7vwMQo,110
-scalable_pypeline-2.1.18.dist-info/entry_points.txt,sha256=uWs10ODfHSBKo2Cx_QaUjPHQTpZ3e77j9VlAdRRmMyg,119
-scalable_pypeline-2.1.18.dist-info/top_level.txt,sha256=C7dpkEOc_-nnsAQb28BfQknjD6XHRyS9ZrvVeoIbV7s,15
-scalable_pypeline-2.1.18.dist-info/RECORD,,
+scalable_pypeline-2.1.19.dist-info/LICENSE,sha256=DVQuDIgE45qn836wDaWnYhSdxoLXgpRRKH4RuTjpRZQ,10174
+scalable_pypeline-2.1.19.dist-info/METADATA,sha256=aW0WuUF0FpWnsnngqIImDLZnru5n0D0VrmqeNYoJ1I0,5982
+scalable_pypeline-2.1.19.dist-info/WHEEL,sha256=bb2Ot9scclHKMOLDEHY6B2sicWOgugjFKaJsT7vwMQo,110
+scalable_pypeline-2.1.19.dist-info/entry_points.txt,sha256=uWs10ODfHSBKo2Cx_QaUjPHQTpZ3e77j9VlAdRRmMyg,119
+scalable_pypeline-2.1.19.dist-info/top_level.txt,sha256=C7dpkEOc_-nnsAQb28BfQknjD6XHRyS9ZrvVeoIbV7s,15
+scalable_pypeline-2.1.19.dist-info/RECORD,,