girder-worker 5.0.11.dev22__tar.gz → 5.0.11.dev33__tar.gz

{girder_worker-5.0.11.dev22 → girder_worker-5.0.11.dev33}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: girder-worker
-Version: 5.0.11.dev22
+Version: 5.0.11.dev33
 Summary: Batch execution engine built on celery.
 Home-page: https://github.com/girder/girder_worker
 Author: Kitware, Inc.

girder_worker-5.0.11.dev33/girder_plugin_worker/__init__.py

@@ -0,0 +1,38 @@
+import logging
+from pathlib import Path
+
+from girder_jobs.models.job import Job
+
+from girder import events
+from girder.constants import AccessType
+from girder.plugin import GirderPlugin, getPlugin, registerPluginStaticContent
+
+from . import event_handlers
+from .api.worker import Worker
+
+logger = logging.getLogger(__name__)
+
+
+class WorkerPlugin(GirderPlugin):
+    DISPLAY_NAME = 'Worker'
+
+    def load(self, info):
+        getPlugin('jobs').load(info)
+
+        info['apiRoot'].worker = Worker()
+
+        registerPluginStaticContent(
+            plugin='worker',
+            js=['/girder-plugin-worker.umd.cjs'],
+            css=['/style.css'],
+            staticDir=Path(__file__).parent / 'web_client' / 'dist',
+            tree=info['serverRoot'],
+        )
+
+        events.bind('jobs.schedule', 'worker', event_handlers.schedule)
+        events.bind('jobs.status.validate', 'worker', event_handlers.validateJobStatus)
+        events.bind('jobs.status.validTransitions', 'worker', event_handlers.validTransitions)
+        events.bind('jobs.cancel', 'worker', event_handlers.cancel)
+        events.bind('model.job.save.after', 'worker', event_handlers.attachJobInfoSpec)
+        events.bind('model.job.save', 'worker', event_handlers.attachParentJob)
+        Job().exposeFields(AccessType.SITE_ADMIN, {'celeryTaskId', 'celeryQueue'})

girder_worker-5.0.11.dev33/girder_plugin_worker/api/worker.py

@@ -0,0 +1,40 @@
+import logging
+
+import celery
+from girder_worker.app import app
+
+from girder.api import access
+from girder.api.describe import Description, autoDescribeRoute
+from girder.api.rest import Resource
+from girder.constants import TokenScope
+
+logger = logging.getLogger(__name__)
+
+
+class Worker(Resource):
+    def __init__(self):
+        super().__init__()
+        self.resourceName = 'worker'
+        self.route('GET', ('status',), self.getWorkerStatus)
+
+    @autoDescribeRoute(
+        Description('Get worker status and task information.')
+        .notes('Return -1 if the broker is inaccessible.')
+    )
+    @access.user(scope=TokenScope.DATA_READ)
+    def getWorkerStatus(self):
+        result = {}
+        conn = app.connection_for_read()
+        try:
+            conn.ensure_connection(max_retries=1)
+        except celery.exceptions.OperationalError:
+            logger.exception(f'Broker ({app.conf.broker_url}) is inaccessible.')
+            return -1
+
+        status = app.control.inspect()
+        result['report'] = status.report()
+        result['stats'] = status.stats()
+        result['ping'] = status.ping()
+        result['active'] = status.active() or {}
+        result['reserved'] = status.reserved() or {}
+        return result

girder_worker-5.0.11.dev33/girder_plugin_worker/constants.py

@@ -0,0 +1,11 @@
+# The path that will be mounted in docker containers for data IO
+DOCKER_DATA_VOLUME = '/mnt/girder_worker/data'
+
+# The path that will be mounted in docker containers for utility scripts
+DOCKER_SCRIPTS_VOLUME = '/mnt/girder_worker/scripts'
+
+
+# Settings where plugin information is stored
+class PluginSettings:
+    API_URL = 'worker.api_url'
+    DIRECT_PATH = 'worker.direct_path'

girder_worker-5.0.11.dev33/girder_plugin_worker/event_handlers.py

@@ -0,0 +1,134 @@
+import logging
+
+from celery.result import AsyncResult
+from girder_jobs.constants import JobStatus
+from girder_jobs.models.job import Job
+from girder_worker.app import app
+
+from girder.exceptions import ValidationException
+from girder.utility import setting_utilities
+
+from .constants import PluginSettings
+from .status import CustomJobStatus
+from .utils import getWorkerApiUrl, jobInfoSpec
+
+logger = logging.getLogger(__name__)
+
+
+@setting_utilities.validator({
+    PluginSettings.API_URL
+})
+def validateApiUrl(doc):
+    val = doc['value']
+    if val and not val.startswith('http://') and not val.startswith('https://'):
+        raise ValidationException('API URL must start with http:// or https://.', 'value')
+
+
+@setting_utilities.validator(PluginSettings.DIRECT_PATH)
+def _validateAutoCompute(doc):
+    if not isinstance(doc['value'], bool):
+        raise ValidationException('The direct path setting must be true or false.')
+
+
+def validateJobStatus(event):
+    """Allow our custom job status values."""
+    if CustomJobStatus.isValid(event.info):
+        event.preventDefault().addResponse(True)
+
+
+def validTransitions(event):
+    """Allow our custom job transitions."""
+    states = None
+    if event.info['job']['handler'] == 'worker_handler':
+        states = CustomJobStatus.validTransitionsWorker(event.info['status'])
+    elif event.info['job']['handler'] == 'celery_handler':
+        states = CustomJobStatus.validTransitionsCelery(event.info['status'])
+    if states is not None:
+        event.preventDefault().addResponse(states)
+
+
+def schedule(event):
+    """
+    This is bound to the "jobs.schedule" event, and will be triggered any time
+    a job is scheduled. This handler will process any job that has the
+    handler field set to "worker_handler".
+    """
+    job = event.info
+    if job['handler'] == 'worker_handler':
+        task = job.get('celeryTaskName', 'girder_worker.run')
+        queue_name = job.get('celeryQueue')
+
+        if queue_name == 'local':
+            from girder.tasks import ensure_local_worker_available
+            ensure_local_worker_available()
+        # Set the job status to queued
+        Job().updateJob(job, status=JobStatus.QUEUED)
+
+        # Send the task to celery
+        asyncResult = app.send_task(
+            task, job['args'], job['kwargs'], queue=queue_name, headers={
+                'jobInfoSpec': jobInfoSpec(job, job.get('token', None)),
+                'apiUrl': getWorkerApiUrl()
+            })
+
+        # Record the task ID from celery.
+        Job().updateJob(job, otherFields={
+            'celeryTaskId': asyncResult.task_id
+        })
+
+        # Stop event propagation since we have taken care of scheduling.
+        event.stopPropagation()
+
+
+def cancel(event):
+    """
+    This is bound to the "jobs.cancel" event, and will be triggered any time
+    a job is canceled. This handler will process any job that has the
+    handler field set to "worker_handler".
+    """
+    job = event.info
+    if job['handler'] in ['worker_handler', 'celery_handler']:
+        # Stop event propagation and prevent default, we are using a custom state
+        event.stopPropagation().preventDefault()
+
+        celeryTaskId = job.get('celeryTaskId')
+
+        if celeryTaskId is None:
+            msg = ("Unable to cancel Celery task. Job '%s' doesn't have a Celery task id."
+                   % job['_id'])
+            logger.warning(msg)
+            return
+
+        should_revoke = False
+        if job['status'] == JobStatus.INACTIVE:
+            # Move inactive jobs directly to canceled state
+            Job().updateJob(job, status=JobStatus.CANCELED)
+            should_revoke = True
+
+        elif job['status'] not in [CustomJobStatus.CANCELING, JobStatus.CANCELED,
+                                   JobStatus.SUCCESS, JobStatus.ERROR]:
+            # Give active jobs a chance to be canceled by their runner
+            Job().updateJob(job, status=CustomJobStatus.CANCELING)
+            should_revoke = True
+
+        if should_revoke:
+            # Send the revoke request.
+            asyncResult = AsyncResult(celeryTaskId, app=app)
+            asyncResult.revoke()
+
+
+def attachParentJob(event):
+    """Attach parentJob before a model is saved."""
+    job = event.info
+    if job.get('celeryParentTaskId'):
+        celeryParentTaskId = job['celeryParentTaskId']
+        parentJob = Job().findOne({'celeryTaskId': celeryParentTaskId})
+        event.info['parentId'] = parentJob['_id']
+
+
+def attachJobInfoSpec(event):
+    """Attach jobInfoSpec after a model is saved."""
+    job = event.info
+    # Local jobs have a module key
+    if not job.get('module'):
+        Job().updateJob(job, otherFields={'jobInfoSpec': jobInfoSpec(job)})

girder_worker-5.0.11.dev33/girder_plugin_worker/status.py

@@ -0,0 +1,67 @@
+from girder_jobs.constants import JobStatus
+
+
+class CustomJobStatus:
+    """The custom job status flags for the worker."""
+
+    FETCHING_INPUT = 820
+    CONVERTING_INPUT = 821
+    CONVERTING_OUTPUT = 822
+    PUSHING_OUTPUT = 823
+    CANCELING = 824
+
+    # valid transitions for worker scheduled jobs
+    valid_worker_transitions = {
+        JobStatus.QUEUED: [JobStatus.INACTIVE],
+        JobStatus.RUNNING: [JobStatus.QUEUED, FETCHING_INPUT],
+        FETCHING_INPUT: [JobStatus.RUNNING],
+        CONVERTING_INPUT: [JobStatus.RUNNING, FETCHING_INPUT],
+        CONVERTING_OUTPUT: [JobStatus.RUNNING],
+        PUSHING_OUTPUT: [JobStatus.RUNNING, CONVERTING_OUTPUT],
+        CANCELING: [JobStatus.INACTIVE, JobStatus.QUEUED, JobStatus.RUNNING],
+        JobStatus.ERROR: [FETCHING_INPUT, CONVERTING_INPUT, CONVERTING_OUTPUT,
+                          PUSHING_OUTPUT, CANCELING, JobStatus.QUEUED,
+                          JobStatus.RUNNING],
+        # The last two are allowed for revoke called from outside Girder
+        JobStatus.CANCELED: [CANCELING, JobStatus.QUEUED, JobStatus.RUNNING],
+        JobStatus.SUCCESS: [JobStatus.RUNNING, PUSHING_OUTPUT]
+    }
+
+    # valid transitions for celery scheduled jobs
+    # N.B. We have the extra worker input/output states defined here for when
+    # we are running girder_worker.run as a regular celery task
+    valid_celery_transitions = {
+        JobStatus.QUEUED: [JobStatus.INACTIVE],
+        # Note celery tasks can jump straight from INACTIVE to RUNNING
+        JobStatus.RUNNING: [JobStatus.INACTIVE, JobStatus.QUEUED,
+                            FETCHING_INPUT],
+        FETCHING_INPUT: [JobStatus.RUNNING],
+        CONVERTING_INPUT: [JobStatus.RUNNING, FETCHING_INPUT],
+        CONVERTING_OUTPUT: [JobStatus.RUNNING],
+        PUSHING_OUTPUT: [JobStatus.RUNNING, CONVERTING_OUTPUT],
+        CANCELING: [JobStatus.INACTIVE, JobStatus.QUEUED, JobStatus.RUNNING],
+        JobStatus.ERROR: [FETCHING_INPUT, CONVERTING_INPUT, CONVERTING_OUTPUT,
+                          PUSHING_OUTPUT, CANCELING, JobStatus.QUEUED,
+                          JobStatus.RUNNING],
+        JobStatus.CANCELED: [CANCELING, JobStatus.INACTIVE, JobStatus.QUEUED,
+                             JobStatus.RUNNING],
+        JobStatus.SUCCESS: [JobStatus.RUNNING, PUSHING_OUTPUT]
+    }
+
+    @classmethod
+    def isValid(cls, status):
+        return status in (
+            cls.FETCHING_INPUT,
+            cls.CONVERTING_INPUT,
+            cls.CONVERTING_OUTPUT,
+            cls.PUSHING_OUTPUT,
+            cls.CANCELING
+        )
+
+    @classmethod
+    def validTransitionsWorker(cls, status):
+        return cls.valid_worker_transitions.get(status)
+
+    @classmethod
+    def validTransitionsCelery(cls, status):
+        return cls.valid_celery_transitions.get(status)

girder_worker-5.0.11.dev33/girder_plugin_worker/utils.py

@@ -0,0 +1,155 @@
+###############################################################################
+#  Copyright Kitware Inc.
+#
+#  Licensed under the Apache License, Version 2.0 ( the "License" );
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+###############################################################################
+
+from girder_jobs.models.job import Job
+
+from girder.api.rest import getApiUrl
+from girder.exceptions import FilePathException
+from girder.models.file import File
+from girder.models.setting import Setting
+
+from .constants import PluginSettings
+
+
+def getWorkerApiUrl():
+    """
+    Return the API base URL to which the worker should callback to
+    write output information back to the server. This is controlled
+    via a system setting, and the default is to use the core server
+    root setting.
+    """
+    apiUrl = Setting().get(PluginSettings.API_URL)
+    return apiUrl or getApiUrl()
+
+
+def girderInputSpec(resource, resourceType='file', name=None, token=None,
+                    dataType='string', dataFormat='text', fetchParent=False):
+    """
+    Downstream plugins that are building Girder worker jobs that use Girder IO
+    should use this to generate the input specs more easily.
+
+    :param resource: The resource document to be downloaded at runtime.
+    :type resource: dict
+    :param resourceType: The resource type to download for the input. Should
+        be "folder", "item", or "file".
+    :type resourceType: str
+    :param name: The name of the resource to download. If not passed, uses
+        the "name" field of the resource document.
+    :type name: str or None
+    :param token: The Girder token document or raw token string to use to
+        authenticate when downloading. Pass `None` for anonymous downloads.
+    :type token: dict, str, or None
+    :param dataType: The worker `type` field.
+    :type dataType: str
+    :param dataFormat: The worker `format` field.
+    :type dataFormat: str
+    :param fetchParent: Whether to fetch the whole parent resource of the
+        specified resource as a side effect.
+    :type fetchParent: bool
+    """
+    if isinstance(token, dict):
+        token = token['_id']
+
+    result = {
+        'mode': 'girder',
+        'api_url': getWorkerApiUrl(),
+        'token': token,
+        'id': str(resource['_id']),
+        'name': name or resource['name'],
+        'resource_type': resourceType,
+        'type': dataType,
+        'format': dataFormat,
+        'fetch_parent': fetchParent
+    }
+
+    if resourceType == 'file' and not fetchParent and Setting().get(PluginSettings.DIRECT_PATH):
+        # If we are adding a file and it exists on the local filesystem include
+        # that location.  This can permit the user of the specification to
+        # access the file directly instead of downloading the file.
+        try:
+            result['direct_path'] = File().getLocalFilePath(resource)
+        except FilePathException:
+            pass
+    return result
+
+
+def girderOutputSpec(parent, token, parentType='folder', name=None,
+                     dataType='string', dataFormat='text', reference=None):
+    """
+    Downstream plugins that are building worker jobs that use Girder IO
+    should use this to generate the output specs more easily.
+
+    :param parent: The parent to upload the data into (an item or folder).
+    :type parent: dict
+    :param token: The Girder token document or raw token string to use to
+        authenticate when uploading.
+    :type token: dict or str
+    :param parentType: The type of the parent object ("item" or "folder").
+    :type parentType: str
+    :param name: Name of the resource to use when uploading. Required if
+        the output target type is "memory". If the target is "filepath", uses
+        the basename of the file being uploaded by default.
+    :type name: str or None
+    :param dataType: The worker `type` field.
+    :type dataType: str
+    :param dataFormat: The worker `format` field.
+    :type dataFormat: str
+    :param reference: Optional "reference" string to pass back to the server
+        during the upload. This can be used to attach arbitrary data to this
+        for tracking purposes, e.g., referring back to related inputs. Bind to
+        the "data.process" event to hook into the upload and inspect references.
+    :type reference: str
+    """
+    if isinstance(token, dict):
+        token = token['_id']
+
+    return {
+        'mode': 'girder',
+        'api_url': getWorkerApiUrl(),
+        'token': token,
+        'name': name,
+        'parent_id': str(parent['_id']),
+        'parent_type': parentType,
+        'type': dataType,
+        'format': dataFormat,
+        'reference': reference
+    }
+
+
+def jobInfoSpec(job, token=None, logPrint=True):
+    """
+    Build the jobInfo specification for a task to write status and log output
+    back to a Girder job.
+
+    :param job: The job document representing the worker task.
+    :type job: dict
+    :param token: The token to use. Creates a job token if not passed.
+    :type token: str or dict
+    :param logPrint: Whether standard output from the job should be
+    """
+    if token is None:
+        token = Job().createJobToken(job)
+
+    if isinstance(token, dict):
+        token = token['_id']
+
+    return {
+        'method': 'PUT',
+        'url': '/'.join((getWorkerApiUrl(), 'job', str(job['_id']))),
+        'reference': str(job['_id']),
+        'headers': {'Girder-Token': token},
+        'logPrint': logPrint
+    }

girder_worker-5.0.11.dev33/girder_plugin_worker/web_client/JobStatus.js

@@ -0,0 +1,50 @@
+const events = girder.events;
+
+// g:appload.before runs after all plugin static files have been loaded
+events.on('g:appload.before', () => {
+    const JobStatus = girder.plugins.jobs.JobStatus;
+
+    JobStatus.registerStatus({
+        WORKER_FETCHING_INPUT: {
+            value: 820,
+            text: 'Fetching input',
+            icon: 'icon-download',
+            color: '#89d2e2'
+        },
+        WORKER_CONVERTING_INPUT: {
+            value: 821,
+            text: 'Converting input',
+            icon: 'icon-shuffle',
+            color: '#92f5b5'
+        },
+        WORKER_CONVERTING_OUTPUT: {
+            value: 822,
+            text: 'Converting output',
+            icon: 'icon-shuffle',
+            color: '#92f5b5'
+        },
+        WORKER_PUSHING_OUTPUT: {
+            value: 823,
+            text: 'Pushing output',
+            icon: 'icon-upload',
+            color: '#89d2e2'
+        },
+        WORKER_CANCELING: {
+            value: 824,
+            text: 'Canceling',
+            icon: 'icon-spin3 animate-spin',
+            color: '#f89406'
+        }
+    });
+
+    const jobPluginIsCancelable = JobStatus.isCancelable;
+    JobStatus.isCancelable = function (job) {
+        const handler = job.get('handler');
+        if (handler === 'worker_handler' || handler === 'celery_handler') {
+            return [JobStatus.CANCELED, JobStatus.WORKER_CANCELING,
+                JobStatus.SUCCESS, JobStatus.ERROR].indexOf(job.get('status')) === -1;
+        }
+
+        return jobPluginIsCancelable(job);
+    };
+});

girder_worker-5.0.11.dev33/girder_plugin_worker/web_client/main.js

@@ -0,0 +1,4 @@
+import './routes';
+import './stylesheets/jobDetailsWidget.styl';
+import './stylesheets/taskStatusView.styl';
+import './JobStatus';