girder-local-checkout 5.0.23.dev7__tar.gz

girder_local_checkout-5.0.23.dev7/MANIFEST.in

@@ -0,0 +1,3 @@
+prune girder_local_checkout/web_client
+include girder_local_checkout/web_client/dist/girder-plugin-local-checkout.umd.cjs
+include girder_local_checkout/web_client/dist/style.css

girder_local_checkout-5.0.23.dev7/PKG-INFO

@@ -0,0 +1,41 @@
+Metadata-Version: 2.4
+Name: girder-local-checkout
+Version: 5.0.23.dev7
+Summary: Temporarily copy S3 assetstore data to a filesystem assetstore.
+Home-page: https://github.com/girder/girder/tree/master/plugins/local_checkout
+Author: Kitware, Inc.
+Author-email: kitware@kitware.com
+License: Apache 2.0
+Keywords: girder-plugin,local_checkout
+Classifier: Development Status :: 4 - Beta
+Classifier: Natural Language :: English
+Classifier: Programming Language :: Python :: 3
+Requires-Python: >=3.10
+Description-Content-Type: text/x-rst
+Requires-Dist: girder-jobs>=3.0.3
+Requires-Dist: girder>=3.1.23
+Dynamic: author
+Dynamic: author-email
+Dynamic: classifier
+Dynamic: description
+Dynamic: description-content-type
+Dynamic: home-page
+Dynamic: keywords
+Dynamic: license
+Dynamic: requires-dist
+Dynamic: requires-python
+Dynamic: summary
+
+local-checkout
+==============
+
+A Girder plugin that lets users temporarily "checkout" the contents of a
+folder from an S3 assetstore to a filesystem assetstore.
+
+Files on S3 assetstores are copied to a filesystem assetstore for a chosen
+number of days; while checked out, Girder serves the local copy.  When a
+checkout expires, or when the filesystem assetstore runs short of space, the
+local copies are removed and the files once again refer to the S3 data they
+came from.
+
+See the Girder documentation for details.

girder_local_checkout-5.0.23.dev7/README.rst

@@ -0,0 +1,13 @@
+local-checkout
+==============
+
+A Girder plugin that lets users temporarily "checkout" the contents of a
+folder from an S3 assetstore to a filesystem assetstore.
+
+Files on S3 assetstores are copied to a filesystem assetstore for a chosen
+number of days; while checked out, Girder serves the local copy.  When a
+checkout expires, or when the filesystem assetstore runs short of space, the
+local copies are removed and the files once again refer to the S3 data they
+came from.
+
+See the Girder documentation for details.

girder_local_checkout-5.0.23.dev7/girder_local_checkout/__init__.py

@@ -0,0 +1,120 @@
+from pathlib import Path
+
+from girder import events, plugin
+from girder.exceptions import ValidationException
+from girder.utility.filesystem_assetstore_adapter import FilesystemAssetstoreAdapter
+from girder.utility.model_importer import ModelImporter
+
+from . import rest, utils
+from .models import Checkout, CheckoutFile
+
+
+def _wrapUploadMethods():
+    """
+    Make the filesystem assetstore adapter release local copies rather than
+    reject an upload when it is short of space.
+    """
+    if getattr(FilesystemAssetstoreAdapter, '_localCheckoutWrapped', False):
+        return
+    for name in ('initUpload', 'uploadChunk'):
+        base = getattr(FilesystemAssetstoreAdapter, name)
+
+        def wrapper(self, upload, *args, _base=base, **kwargs):
+            utils.makeRoomForUpload(self, upload)
+            return _base(self, upload, *args, **kwargs)
+
+        wrapper.__name__ = name
+        wrapper.__doc__ = base.__doc__
+        setattr(FilesystemAssetstoreAdapter, name, wrapper)
+    FilesystemAssetstoreAdapter._localCheckoutWrapped = True
+
+
+def _onFileRemove(event):
+    """
+    When a checked-out file is deleted, core removes the local copy; we must
+    also remove the original data it came from.
+    """
+    record = CheckoutFile().findByFile(event.info)
+    if record is None:
+        return
+    utils.deleteOriginal(record)
+    CheckoutFile().remove(record)
+
+
+def _onFileReplaced(event):
+    """
+    When the contents of a checked-out file are replaced (including being
+    moved to another assetstore), core has already discarded the local copy,
+    so the original data is no longer needed either.
+    """
+    if 'fileId' not in event.info['upload']:
+        return
+    record = CheckoutFile().findByFile(event.info['file'])
+    if record is None:
+        return
+    utils.deleteOriginal(record)
+    CheckoutFile().remove(record)
+
+
+def _onAssetstoreRemove(event):
+    """
+    Refuse to delete an assetstore that still holds the original data of a
+    checked-out file; core only sees that those files point elsewhere.
+    """
+    if CheckoutFile().findOne(
+            {'originalAssetstoreId': event.info['_id']}, fields=['_id']) is not None:
+        raise ValidationException(
+            'You may not delete an assetstore that contains the original data of '
+            'files with local copies; release the checkouts first.')
+
+
+def _onFolderRemove(event):
+    checkout = Checkout().findByFolder(event.info)
+    if checkout is not None:
+        utils.removeCheckout(checkout)
+
+
+def _maybeCleanup(event):
+    """
+    Schedule the daily cleanup of expired local copies if it is due.  This is
+    bound to a frequent event so that it runs even on idle servers.
+    """
+    # Imported here to avoid a circular import through girder_worker's plugin
+    # discovery when girder.tasks is first loaded
+    from girder.tasks import is_local_worker_available
+
+    if is_local_worker_available() and utils.cleanupIsDue():
+        from . import tasks
+
+        tasks.cleanupTask.delay()
+
+
+class LocalCheckoutPlugin(plugin.GirderPlugin):
+    DISPLAY_NAME = 'Local Checkout'
+
+    def load(self, info):
+        plugin.getPlugin('jobs').load(info)
+
+        ModelImporter.registerModel('localCheckout', Checkout, 'local_checkout')
+        ModelImporter.registerModel('localCheckoutFile', CheckoutFile, 'local_checkout')
+
+        info['apiRoot'].local_checkout = rest.LocalCheckout()
+        info['apiRoot'].folder.route('GET', (':id', 'local_checkout'), rest.getFolderCheckout)
+        info['apiRoot'].folder.route('PUT', (':id', 'local_checkout'), rest.setFolderCheckout)
+        info['apiRoot'].folder.route('DELETE', (':id', 'local_checkout'),
+                                     rest.deleteFolderCheckout)
+
+        events.bind('model.file.remove', 'local_checkout', _onFileRemove)
+        events.bind('model.file.finalizeUpload.before', 'local_checkout', _onFileReplaced)
+        events.bind('model.folder.remove', 'local_checkout', _onFolderRemove)
+        events.bind('model.assetstore.remove', 'local_checkout', _onAssetstoreRemove)
+        events.bind('model.file.download.request', 'local_checkout', _maybeCleanup)
+        _wrapUploadMethods()
+
+        plugin.registerPluginStaticContent(
+            plugin='local_checkout',
+            css=['/style.css'],
+            js=['/girder-plugin-local-checkout.umd.cjs'],
+            staticDir=Path(__file__).parent / 'web_client' / 'dist',
+            tree=info['serverRoot'],
+        )

girder_local_checkout-5.0.23.dev7/girder_local_checkout/girder_worker_plugin.py

@@ -0,0 +1,9 @@
+from girder_worker import GirderWorkerPluginABC
+
+
+class LocalCheckoutWorkerPlugin(GirderWorkerPluginABC):
+    def __init__(self, app, *args, **kwargs):
+        self.app = app
+
+    def task_imports(self):
+        return ['girder_local_checkout.tasks']

girder_local_checkout-5.0.23.dev7/girder_local_checkout/models.py

@@ -0,0 +1,106 @@
+import datetime
+
+from bson.objectid import ObjectId
+
+from girder.exceptions import ValidationException
+from girder.models.model_base import Model
+
+
+class CheckoutStatus:
+    PENDING = 'pending'
+    RUNNING = 'running'
+    COMPLETE = 'complete'
+    # Not all files could be copied, typically because of insufficient space.
+    PARTIAL = 'partial'
+    ERROR = 'error'
+
+
+class Checkout(Model):
+    """
+    A folder-level checkout: a request that all S3 files within a folder be
+    made local until a given expiration time.
+    """
+
+    def initialize(self):
+        self.name = 'localCheckout'
+        self.ensureIndices(['folderId', 'expires'])
+
+    def validate(self, doc):
+        for key in ('folderId', 'expires', 'days'):
+            if key not in doc:
+                raise ValidationException('Field missing.', key)
+        return doc
+
+    def createCheckout(self, folder, user, days, assetstore):
+        now = datetime.datetime.now(datetime.timezone.utc)
+        return self.save({
+            'folderId': folder['_id'],
+            'assetstoreId': assetstore['_id'],
+            'userId': user['_id'] if user else None,
+            'days': days,
+            'created': now,
+            'updated': now,
+            'expires': now + datetime.timedelta(days=days),
+            'status': CheckoutStatus.PENDING,
+            'fileCount': 0,
+            'localCount': 0,
+            'bytes': 0,
+            'message': '',
+        })
+
+    def findByFolder(self, folder):
+        return self.findOne({'folderId': folder['_id']})
+
+
+class CheckoutFile(Model):
+    """
+    A file that currently has a local copy.  This records where the file's
+    data originally lived so that the file can be pointed back at it when the
+    local copy is removed.
+    """
+
+    def initialize(self):
+        self.name = 'localCheckoutFile'
+        self.ensureIndices([
+            ([('fileId', 1)], {'unique': True}),
+            'expires',
+            'checkoutIds',
+            'original.relpath',
+        ])
+
+    def validate(self, doc):
+        for key in ('fileId', 'original', 'localAssetstoreId', 'expires'):
+            if key not in doc:
+                raise ValidationException('Field missing.', key)
+        return doc
+
+    def findByFile(self, file):
+        return self.findOne({'fileId': file['_id']})
+
+    def addCheckout(self, record, checkout):
+        """
+        Associate a checkout with a file record, extending the record's
+        expiration if the checkout lasts longer.
+        """
+        update = {'$addToSet': {'checkoutIds': checkout['_id']}}
+        if checkout['expires'] > record['expires']:
+            update['$set'] = {'expires': checkout['expires']}
+            record['expires'] = checkout['expires']
+        if checkout['_id'] not in record['checkoutIds']:
+            record['checkoutIds'].append(checkout['_id'])
+        self.update({'_id': record['_id']}, update, multi=False)
+        return record
+
+    def computeExpires(self, record):
+        """
+        Return the latest expiration of any checkout that still covers the
+        record, or None if no checkout covers it any more.
+        """
+        ids = [ObjectId(i) for i in record.get('checkoutIds', [])]
+        if not ids:
+            return None
+        latest = None
+        for checkout in Checkout().find({'_id': {'$in': ids}}, fields=['expires']):
+            if latest is None or checkout['expires'] > latest:
+                latest = checkout['expires']
+        return latest

girder_local_checkout-5.0.23.dev7/girder_local_checkout/rest.py

@@ -0,0 +1,217 @@
+from girder.api import access
+from girder.api.describe import Description, autoDescribeRoute
+from girder.api.rest import Resource, boundHandler
+from girder.constants import AccessType, SortDir, TokenScope
+from girder.exceptions import GirderException, RestException
+from girder.models.assetstore import Assetstore
+from girder.models.folder import Folder
+from girder.models.setting import Setting
+from girder.models.user import User
+from girder.utility import assetstore_utilities
+from girder.utility import path as path_util
+
+from . import tasks, utils
+from .models import Checkout, CheckoutFile
+from .settings import PluginSettings
+
+
+def _checkPermission(user):
+    if Setting().get(PluginSettings.ADMIN_ONLY) and not user.get('admin'):
+        raise RestException('Only administrators may checkout folders.', code=403)
+
+
+def _ensureWorker():
+    # Imported here to avoid a circular import through girder_worker's plugin
+    # discovery when girder.tasks is first loaded
+    from girder.tasks import ensure_local_worker_available
+
+    ensure_local_worker_available()
+
+
+def _requireAssetstore(assetstoreId):
+    try:
+        return utils.resolveCheckoutAssetstore(assetstoreId)
+    except GirderException as exc:
+        raise RestException(str(exc))
+
+
+def _assetstoreInfo(assetstore):
+    capacity = assetstore_utilities.getAssetstoreAdapter(assetstore).capacityInfo()
+    return {'_id': assetstore['_id'], 'name': assetstore['name'], 'free': capacity.get('free')}
+
+
+@access.user(scope=TokenScope.DATA_READ)
+@boundHandler
+@autoDescribeRoute(
+    Description('Get the local checkout status of a folder.')
+    .modelParam('id', model=Folder, level=AccessType.READ)
+    .param('details', 'Whether to count the files in the folder that can be or '
+           'have been made local.  This can be slow for large folders.',
+           dataType='boolean', default=True, required=False)
+    .errorResponse('ID was invalid.')
+    .errorResponse('Read access was denied for the folder.', 403)
+)
+def getFolderCheckout(self, folder, details):
+    assetstore = utils.getTargetAssetstore()
+    # When there is no configured or default filesystem assetstore, the user
+    # chooses one of these
+    choices = [] if assetstore else [
+        _assetstoreInfo(choice) for choice in utils.getFilesystemAssetstores()]
+    result = {
+        'checkout': Checkout().findByFolder(folder),
+        'enabled': assetstore is not None or bool(choices),
+        'assetstoreName': assetstore['name'] if assetstore else None,
+        'assetstores': choices,
+        'maxDays': Setting().get(PluginSettings.MAX_DAYS),
+        'adminOnly': Setting().get(PluginSettings.ADMIN_ONLY),
+    }
+    if details:
+        result.update(utils.folderStatus(folder))
+    return result
+
+
+@access.user(scope=TokenScope.DATA_WRITE)
+@boundHandler
+@autoDescribeRoute(
+    Description('Checkout a folder, making local copies of its S3 files for a number of days.')
+    .notes('If the folder is already checked out, its duration is changed.  A '
+           'duration of 0 days releases the checkout.  Copying is done in a '
+           'background task.')
+    .modelParam('id', model=Folder, level=AccessType.WRITE)
+    .param('days', 'The number of days to keep local copies.', dataType='integer')
+    .param('assetstoreId', 'The filesystem assetstore to copy files to.  This is only '
+           'used when no assetstore is configured for local checkouts and the default '
+           'assetstore is not a filesystem assetstore; it may be omitted if there is only '
+           'one filesystem assetstore.', required=False)
+    .param('progress', 'Whether to record progress on the copy.', dataType='boolean',
+           default=True, required=False)
+    .errorResponse('ID was invalid.')
+    .errorResponse('Write access was denied for the folder.', 403)
+)
+def setFolderCheckout(self, folder, days, assetstoreId, progress):
+    user = self.getCurrentUser()
+    _checkPermission(user)
+    if days <= 0:
+        return _release(user, folder, progress)
+    maxDays = Setting().get(PluginSettings.MAX_DAYS)
+    if days > maxDays:
+        raise RestException('The checkout duration may not exceed %d days.' % maxDays)
+    checkout = Checkout().findByFolder(folder)
+    if not assetstoreId and checkout is not None:
+        assetstoreId = checkout.get('assetstoreId')
+    assetstore = _requireAssetstore(assetstoreId)
+    _ensureWorker()
+    if checkout is None:
+        checkout = Checkout().createCheckout(folder, user, days, assetstore)
+    else:
+        checkout = utils.updateCheckoutExpiry(checkout, days, assetstore)
+    tasks.checkoutFolderTask.delay(
+        checkoutId=str(checkout['_id']), userId=str(user['_id']), progress=progress)
+    return checkout
+
+
+@access.user(scope=TokenScope.DATA_WRITE)
+@boundHandler
+@autoDescribeRoute(
+    Description('Release the local checkout of a folder.')
+    .notes('Local copies that no other checkout covers are removed in a background task.')
+    .modelParam('id', model=Folder, level=AccessType.WRITE)
+    .param('progress', 'Whether to record progress on the release.', dataType='boolean',
+           default=True, required=False)
+    .errorResponse('ID was invalid.')
+    .errorResponse('Write access was denied for the folder.', 403)
+)
+def deleteFolderCheckout(self, folder, progress):
+    user = self.getCurrentUser()
+    _checkPermission(user)
+    return _release(user, folder, progress)
+
+
+def _release(user, folder, progress):
+    checkout = Checkout().findByFolder(folder)
+    if checkout is None:
+        return {'released': False}
+    _ensureWorker()
+    Checkout().remove(checkout)
+    tasks.detachCheckoutTask.delay(
+        checkoutId=str(checkout['_id']), userId=str(user['_id']), progress=progress,
+        folderName=folder['name'])
+    return {'released': True}
+
+
+class LocalCheckout(Resource):
+    """Administrative endpoints for local checkouts."""
+
+    def __init__(self):
+        super().__init__()
+        self.resourceName = 'local_checkout'
+        self.route('GET', (), self.listCheckouts)
+        self.route('GET', ('settings',), self.getSettings)
+        self.route('POST', ('cleanup',), self.cleanup)
+
+    @access.public
+    @autoDescribeRoute(
+        Description('Get the settings that affect whether a user may checkout folders.')
+    )
+    def getSettings(self):
+        return {
+            'adminOnly': Setting().get(PluginSettings.ADMIN_ONLY),
+            'maxDays': Setting().get(PluginSettings.MAX_DAYS),
+        }
+
+    @access.admin(scope=TokenScope.DATA_READ)
+    @autoDescribeRoute(
+        Description('List all local checkouts and summarize the space they use.')
+        .pagingParams(defaultSort='expires', defaultSortDir=SortDir.ASCENDING)
+    )
+    def listCheckouts(self, limit, offset, sort):
+        user = self.getCurrentUser()
+        checkouts = list(Checkout().find({}, limit=limit, offset=offset, sort=sort))
+        for checkout in checkouts:
+            folder = Folder().load(checkout['folderId'], force=True)
+            checkout['folderPath'] = (
+                path_util.getResourcePath('folder', folder, user=user, force=True)
+                if folder else None)
+            owner = User().load(checkout['userId'], force=True) if checkout.get('userId') else None
+            checkout['userLogin'] = owner['login'] if owner else None
+            assetstore = (Assetstore().load(checkout['assetstoreId'])
+                          if checkout.get('assetstoreId') else None)
+            checkout['assetstoreName'] = assetstore['name'] if assetstore else None
+
+        perAssetstore = {
+            entry['_id']: entry for entry in CheckoutFile().collection.aggregate([
+                {'$group': {'_id': '$localAssetstoreId', 'count': {'$sum': 1},
+                            'bytes': {'$sum': '$size'}}}
+            ])}
+        # Summarize the assetstores that hold local copies or can receive them
+        target = utils.getTargetAssetstore()
+        candidates = [target] if target else utils.getFilesystemAssetstores()
+        candidateIds = {assetstore['_id'] for assetstore in candidates}
+        candidates += [assetstore for assetstore in (
+            Assetstore().load(assetstoreId) for assetstoreId in perAssetstore
+            if assetstoreId not in candidateIds) if assetstore is not None]
+        assetstores = []
+        for assetstore in candidates:
+            totals = perAssetstore.get(assetstore['_id'], {'count': 0, 'bytes': 0})
+            assetstores.append({
+                'id': assetstore['_id'],
+                'name': assetstore['name'],
+                'capacity': assetstore_utilities.getAssetstoreAdapter(
+                    assetstore).capacityInfo(),
+                'localFileCount': totals['count'],
+                'localBytes': totals['bytes'],
+            })
+        return {
+            'checkouts': checkouts,
+            'localFileCount': sum(entry['count'] for entry in perAssetstore.values()),
+            'localBytes': sum(entry['bytes'] for entry in perAssetstore.values()),
+            'assetstores': assetstores,
+        }
+
+    @access.admin(scope=TokenScope.DATA_WRITE)
+    @autoDescribeRoute(
+        Description('Remove expired checkouts and release their local copies now.')
+        .notes('This is otherwise done daily and before each checkout.')
+    )
+    def cleanup(self):
+        return {'released': utils.cleanupExpired()}

girder_local_checkout-5.0.23.dev7/girder_local_checkout/settings.py

@@ -0,0 +1,67 @@
+from bson.objectid import ObjectId
+
+from girder.constants import AssetstoreType
+from girder.exceptions import ValidationException
+from girder.models.assetstore import Assetstore
+from girder.utility import setting_utilities
+
+
+class PluginSettings:
+    # The ID of the filesystem assetstore that local copies are placed in.  If
+    # empty, the current default assetstore is used when it is a filesystem
+    # assetstore.
+    ASSETSTORE_ID = 'local_checkout.assetstore_id'
+    # The longest checkout duration, in days, that may be requested.
+    MAX_DAYS = 'local_checkout.max_days'
+    # If true, only site administrators may checkout folders.
+    ADMIN_ONLY = 'local_checkout.admin_only'
+
+
+@setting_utilities.default(PluginSettings.ASSETSTORE_ID)
+def _defaultAssetstoreId():
+    return ''
+
+
+@setting_utilities.default(PluginSettings.MAX_DAYS)
+def _defaultMaxDays():
+    return 90
+
+
+@setting_utilities.default(PluginSettings.ADMIN_ONLY)
+def _defaultAdminOnly():
+    return False
+
+
+@setting_utilities.validator(PluginSettings.ASSETSTORE_ID)
+def _validateAssetstoreId(doc):
+    value = doc['value']
+    if value in (None, ''):
+        doc['value'] = ''
+        return
+    try:
+        assetstore = Assetstore().load(ObjectId(str(value)))
+    except Exception:
+        assetstore = None
+    if assetstore is None:
+        raise ValidationException('Assetstore ID must refer to an existing assetstore.', 'value')
+    if assetstore['type'] != AssetstoreType.FILESYSTEM:
+        raise ValidationException('The checkout assetstore must be a filesystem assetstore.',
+                                  'value')
+    doc['value'] = str(assetstore['_id'])
+
+
+@setting_utilities.validator(PluginSettings.MAX_DAYS)
+def _validateMaxDays(doc):
+    try:
+        value = int(doc['value'])
+    except (TypeError, ValueError):
+        raise ValidationException('Maximum checkout days must be an integer.', 'value')
+    if value < 1:
+        raise ValidationException('Maximum checkout days must be at least 1.', 'value')
+    doc['value'] = value
+
+
+@setting_utilities.validator(PluginSettings.ADMIN_ONLY)
+def _validateAdminOnly(doc):
+    if not isinstance(doc['value'], bool):
+        raise ValidationException('Admin-only setting must be true or false.', 'value')

girder_local_checkout-5.0.23.dev7/girder_local_checkout/tasks.py

@@ -0,0 +1,34 @@
+from girder_worker.app import app
+
+from girder.models.folder import Folder
+from girder.models.user import User
+from girder.utility.progress import ProgressContext
+
+from . import utils
+from .models import Checkout
+
+
+@app.task(queue='local')
+def checkoutFolderTask(checkoutId: str, userId: str, progress: bool):
+    user = User().load(userId, force=True)
+    checkout = Checkout().load(checkoutId)
+    if checkout is None:
+        return
+    folder = Folder().load(checkout['folderId'], force=True)
+    title = 'Making folder "%s" local' % (folder['name'] if folder else checkout['folderId'])
+    with ProgressContext(progress, user=user, title=title) as ctx:
+        utils.runCheckout(checkout, ctx)
+
+
+@app.task(queue='local')
+def detachCheckoutTask(checkoutId: str, userId: str, progress: bool, folderName: str):
+    user = User().load(userId, force=True)
+    with ProgressContext(progress, user=user,
+                         title='Releasing local copies of folder "%s"' % folderName) as ctx:
+        utils.detachCheckout(checkoutId)
+        ctx.update(message='Done', force=True)
+
+
+@app.task(queue='local')
+def cleanupTask():
+    return utils.cleanupExpired()