climate-ref-celery 0.5.0__py3-none-any.whl → 0.5.2__py3-none-any.whl

climate_ref_celery/cli.py

@@ -54,6 +54,7 @@ def import_provider(provider_package: str) -> DiagnosticProvider:
 
 @app.command()
 def start_worker(
+    ctx: typer.Context,
     loglevel: str = typer.Option("info", help="Log level for the worker"),
     package: str | None = typer.Option(help="Package to import tasks from", default=None),
     extra_args: list[str] = typer.Argument(None, help="Additional arguments for the worker"),
@@ -75,6 +76,10 @@ def start_worker(
         # Attempt to import the provider
         provider = import_provider(package)
 
+        if hasattr(ctx.obj, "config"):
+            # Configure the provider so that it knows where the conda environments are
+            provider.configure(ctx.obj.config)
+
         # Wrap each diagnostics in the provider with a celery tasks
         register_celery_tasks(celery_app, provider)
         queue = provider.slug
@@ -84,7 +89,7 @@ def start_worker(
 
         queue = "celery"
 
-    argv = ["worker", f"--loglevel={loglevel}", f"--queues={queue}", *(extra_args or [])]
+    argv = ["worker", "-E", f"--loglevel={loglevel}", f"--queues={queue}", *(extra_args or [])]
     celery_app.worker_main(argv=argv)
 
 

climate_ref_celery/executor.py

@@ -14,9 +14,8 @@ from climate_ref.config import Config
 from climate_ref.models import Execution
 from climate_ref_celery.app import app
 from climate_ref_celery.tasks import generate_task_name
-from climate_ref_core.diagnostics import Diagnostic, ExecutionDefinition, ExecutionResult
+from climate_ref_core.diagnostics import ExecutionDefinition, ExecutionResult
 from climate_ref_core.executor import Executor
-from climate_ref_core.providers import DiagnosticProvider
 
 
 class CeleryExecutor(Executor):
@@ -43,8 +42,6 @@ class CeleryExecutor(Executor):
 
     def run(
         self,
-        provider: DiagnosticProvider,
-        diagnostic: Diagnostic,
         definition: ExecutionDefinition,
         execution: Execution | None = None,
     ) -> None:
@@ -62,10 +59,6 @@ class CeleryExecutor(Executor):
 
         Parameters
         ----------
-        provider
-            Provider for the diagnostic
-        diagnostic
-            Diagnostic to run
         definition
             A description of the information needed for this execution of the diagnostic
             This includes relative paths to the data files,
@@ -78,12 +71,14 @@ class CeleryExecutor(Executor):
         """
         from climate_ref_celery.worker_tasks import handle_result
 
-        name = generate_task_name(provider, diagnostic)
+        diagnostic = definition.diagnostic
+
+        name = generate_task_name(diagnostic.provider, diagnostic)
 
         async_result = app.send_task(
             name,
             args=[definition, self.config.log_level],
-            queue=provider.slug,
+            queue=diagnostic.provider.slug,
             link=handle_result.s(execution_id=execution.id).set(queue="celery") if execution else None,
         )
         logger.debug(f"Celery task {async_result.id} submitted")

climate_ref_celery/tasks.py

@@ -13,13 +13,10 @@ The main process is responsible for tracking what diagnostics have been register
 and to respond to new workers coming online.
 """
 
-from collections.abc import Callable
-
 from celery import Celery
-from loguru import logger
 
-from climate_ref_core.diagnostics import Diagnostic, ExecutionDefinition, ExecutionResult
-from climate_ref_core.logging import redirect_logs
+from climate_ref_core.diagnostics import Diagnostic
+from climate_ref_core.executor import execute_locally
 from climate_ref_core.providers import DiagnosticProvider
 
 
@@ -30,32 +27,6 @@ def generate_task_name(provider: DiagnosticProvider, diagnostic: Diagnostic) ->
     return f"{provider.slug}.{diagnostic.slug}"
 
 
-def _diagnostic_task_factory(
-    diagnostic: Diagnostic,
-) -> Callable[
-    [ExecutionDefinition, str],
-    ExecutionResult,
-]:
-    """
-    Create a new task for the given diagnostic
-    """
-
-    def task(definition: ExecutionDefinition, log_level: str) -> ExecutionResult:
-        """
-        Task to run the diagnostic
-        """
-        logger.info(f"Running diagnostic {diagnostic.name} with definition {definition}")
-        try:
-            with redirect_logs(definition, log_level):
-                return diagnostic.run(definition)
-        except Exception:
-            logger.exception(f"Error running diagnostic {diagnostic.slug}:{definition.key}")
-            # TODO: This exception should be caught and a unsuccessful result returned.
-            raise
-
-    return task
-
-
 def register_celery_tasks(app: Celery, provider: DiagnosticProvider) -> None:
     """
     Register all tasks for the given provider
@@ -69,10 +40,14 @@ def register_celery_tasks(app: Celery, provider: DiagnosticProvider) -> None:
     provider
         The provider to register tasks for
     """
-    for metric in provider.diagnostics():
-        print(f"Registering task for diagnostic {metric.name}")
+    for diagnostic in provider.diagnostics():
+        print(f"Registering task for diagnostic {diagnostic.name}")
+
+        # The task function is the same for all diagnostics
+        # The diagnostic is included in the definition
+        # The queue is important to ensure that the task is run in the correct worker
         app.task(  # type: ignore
-            _diagnostic_task_factory(metric),
-            name=generate_task_name(provider, metric),
+            execute_locally,
+            name=generate_task_name(provider, diagnostic),
             queue=provider.slug,
         )

{climate_ref_celery-0.5.0.dist-info → climate_ref_celery-0.5.2.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: climate-ref-celery
-Version: 0.5.0
+Version: 0.5.2
 Summary: Celery app for mananging tasks and workers
 Author-email: Jared Lewis <jared.lewis@climate-resource.com>
 License-File: LICENCE

{climate_ref_celery-0.5.0.dist-info → climate_ref_celery-0.5.2.dist-info}/RECORD

@@ -1,17 +1,17 @@
 climate_ref_celery/__init__.py,sha256=f-dS4JxMLmTCobZtftGDi6bUbtYCv_9iWqxSpC74tWw,129
 climate_ref_celery/app.py,sha256=wLzAnrpzkwZLYY6HY_riHUEtjiRQpkKDSeKZpyMTxAQ,1694
-climate_ref_celery/cli.py,sha256=-bkyEc8pJ5hV2tat7gRzGHoCGWP9qQrJIJ1KvZ3941k,2992
-climate_ref_celery/executor.py,sha256=RtpbelMoxI2i-Wu7NYrBzQIp07gJce2yPLUsM16w7kc,4944
+climate_ref_celery/cli.py,sha256=ZQ86XdrBZoBVmgV7WCFp_P6LNCT1hFfM0zXahkK0p-E,3196
+climate_ref_celery/executor.py,sha256=Qsx1bvM_-H3eMeg5VsJAV6ghHuGK3WnYyIRAd0-1DG4,4764
 climate_ref_celery/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-climate_ref_celery/tasks.py,sha256=vA7gvESuXMp3BT9Bz6ULNypDSRUUkvAHHRjumf3I3sU,2629
+climate_ref_celery/tasks.py,sha256=6Ra2H48edbcklsf6pAj1JNfzi67n6WQCUXJs-tloE80,1941
 climate_ref_celery/worker_tasks.py,sha256=slUprhM8G14D3r6RACSh2alzj-_JWXU4JKG5GR0GLpA,1194
 climate_ref_celery/celeryconf/__init__.py,sha256=xKcHhnECGZAxrjnMtfaPBU_vgqwRbRLXFw8WVQ7WxaE,314
 climate_ref_celery/celeryconf/base.py,sha256=uXaz9S_1hCdwKpvyWBDWE69YVIXHD3Ar7egNKLk9Gx8,484
 climate_ref_celery/celeryconf/dev.py,sha256=um7hvEhlTBfwey9ler_vAHAln_DrpqAP3fbbO2wdmUQ,217
 climate_ref_celery/celeryconf/prod.py,sha256=HyGTfwTZD8MjIkfkb3qyfzjjTPE9sKAoXNq3fJDFO9Q,231
-climate_ref_celery-0.5.0.dist-info/METADATA,sha256=zkB5LN8wUrSswFvN7knOH4mcUJ50bpHouaETWUshoi0,3567
-climate_ref_celery-0.5.0.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
-climate_ref_celery-0.5.0.dist-info/entry_points.txt,sha256=U9b-T6EpLV3ZXmHUpEzp8x5TZnCjQ1ynncIkFMwDuPE,58
-climate_ref_celery-0.5.0.dist-info/licenses/LICENCE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
-climate_ref_celery-0.5.0.dist-info/licenses/NOTICE,sha256=4qTlax9aX2-mswYJuVrLqJ9jK1IkN5kSBqfVvYLF3Ws,128
-climate_ref_celery-0.5.0.dist-info/RECORD,,
+climate_ref_celery-0.5.2.dist-info/METADATA,sha256=QhFLy2WjUW2PCQlBwMUoJVzXeWcn93TAr9S9Z4GQ5ps,3567
+climate_ref_celery-0.5.2.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+climate_ref_celery-0.5.2.dist-info/entry_points.txt,sha256=U9b-T6EpLV3ZXmHUpEzp8x5TZnCjQ1ynncIkFMwDuPE,58
+climate_ref_celery-0.5.2.dist-info/licenses/LICENCE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
+climate_ref_celery-0.5.2.dist-info/licenses/NOTICE,sha256=4qTlax9aX2-mswYJuVrLqJ9jK1IkN5kSBqfVvYLF3Ws,128
+climate_ref_celery-0.5.2.dist-info/RECORD,,