vortex-nwp 2.0.0b1__py3-none-any.whl

vortex/tools/storage.py

@@ -0,0 +1,1061 @@
+"""
+This package handles :class:`Storage` objects that could be in charge of
+hosting data resources both locally ("Cache") or on a remote host "Archive").
+
+* :class:`Storage` is the main abstract class that defines the user-interface for
+  every class of this module. :meth:`Storage.fullpath`, :meth:`Storage.check`,
+  :meth:`Storage.insert`, :meth:`Storage.retrieve` and :meth:`Storage.delete` are
+  frequently used from a user point of view.
+* The :class:`Cache` abstract class is a specialisation of the :class:`Storage`
+  class that handles data resources locally (i.e. data hosted on the same machine
+  that are readily and timelessly accessible). In this module, various concrete
+  implementations are provided for this class in order to support various cache
+  flavors.
+* The :class:`Archive` class (readily usable) is a specialisation of the
+  :class:`Storage` class dedicated to data resources stored remotely (e.g on a
+  mass archive system).
+
+These classes purely focus on the technical aspects (e.g. how to transfer a given
+filename, directory or file like object to its storage place). For :class:`Cache`
+based storage it determines the location of the data on the filesystem, in a
+database, ... For :class:`Archive` based storage it smoothly handles communication
+protocol between the local host and the remote archive.
+
+These classes are used by :class:`Store` objects to access data. Thus,
+:class:`Store` objects do not need to worry anymore about the technical
+aspects. Using the :mod:`footprints` package, for a given execution target, it
+allows to customise the way data are accessed leaving the :class:`Store` objects
+unchanged.
+"""
+import contextlib
+import ftplib
+import re
+import time
+from collections import defaultdict
+from datetime import datetime
+import os
+
+import footprints
+from bronx.fancies import loggers
+from bronx.stdtypes.history import History
+from bronx.syntax.decorators import nicedeco
+from vortex import sessions
+from vortex.tools.actions import actiond as ad
+from vortex.tools.delayedactions import d_action_status
+from vortex.tools.systems import istruedef
+# TODO clean instances of GenericConfigParser
+from vortex.util.config import GenericConfigParser
+from vortex import config
+
+#: No automatic export
+__all__ = []
+
+logger = loggers.getLogger(__name__)
+
+# If the source file size exceed this threshold, a hard link will be
+# used (as much as possible). Otherwise a simple copy will be used.
+HARDLINK_THRESHOLD = 1048576
+
+
+# Decorators: for internal use in the Storage class
+# -------------------------------------------------
+
+def do_recording(flag):
+    """Add a record line in the History object (if sensible)."""
+
+    @nicedeco
+    def do_flagged_recording(f):
+        def wrapped_action(self, item, *kargs, **kwargs):
+            infos = self._findout_record_infos(kwargs)
+            (rc, extrainfos) = f(self, item, *kargs, **kwargs)
+            infos.update(extrainfos)
+            self.addrecord(flag, item, status=rc, **infos)
+            return rc
+
+        return wrapped_action
+
+    return do_flagged_recording
+
+
+@nicedeco
+def enforce_readonly(f):
+    """Check that the current storage object is not readonly."""
+
+    def wrapped_action(self, item, *kargs, **kwargs):
+        if self.readonly:
+            raise OSError("This Storage place is readonly.")
+        return f(self, item, *kargs, **kwargs)
+
+    return wrapped_action
+
+
+# Main Storage abstract class
+# ---------------------------
+
+class Storage(footprints.FootprintBase):
+    """Root class for any Storage class, ex: Cache, Archive, ...
+
+    Tips for developers:
+
+    The following methods needs to be defined in the child classes:
+
+        * *_actual_fullpath*
+        * *_actual_prestageinfo*
+        * *_actual_check*
+        * *_actual_list*
+        * *_actual_insert*
+        * *_actual_retrieve*
+        * *_actual_delete*
+
+    They must return a two elements tuple consisting of a returncode and a
+    dictionary whose items will be written in the object's record.
+    """
+
+    _abstract = True,
+    _footprint = dict(
+        info = 'Default/Abstract storage place description.',
+        attr = dict(
+            kind=dict(
+                info="The storage place's kind.",
+                values=['std'],
+            ),
+            storage=dict(
+                info="The storage target.",
+            ),
+            record=dict(
+                info="Record insert, retrieve, delete actions in an History object.",
+                type=bool,
+                optional=True,
+                default=False,
+                access='rwx',
+            ),
+            readonly=dict(
+                info="Disallow insert and delete action for this storage place.",
+                type=bool,
+                optional=True,
+                default=False,
+            ),
+        )
+    )
+
+    def __init__(self, *args, **kw):
+        logger.debug('Abstract storage init %s', self.__class__)
+        super().__init__(*args, **kw)
+        self._history = History(tag=self.tag)
+
+    @property
+    def tag(self):
+        """The identifier of the storage place."""
+        raise NotImplementedError()
+
+    @property
+    def realkind(self):
+        return 'storage'
+
+    def _str_more(self):
+        return 'tag={:s}'.format(self.tag)
+
+    @property
+    def context(self):
+        """Shortcut to the active context object."""
+        return sessions.get().context
+
+    @property
+    def session(self):
+        return sessions.current()
+
+    @property
+    def sh(self):
+        """Shortcut to the active System object."""
+        return sessions.system()
+
+    @property
+    def history(self):
+        """The History object that will be used by this storage place.
+
+        :note: History objects are associated with the self.tag identifier. i.e.
+               all Storage's objects with the same tag will use the same History
+               object.
+        """
+        return self._history
+
+    def addrecord(self, action, item, **infos):
+        """Push a new record to the storage place log/history."""
+        if self.record:
+            self.history.append(action, item, infos)
+
+    def flush(self, dumpfile=None):
+        """Flush actual history to the specified ``dumpfile`` if record is on.
+
+        :note: May raise the :class:`NotImplementedError` exception.
+        """
+        raise NotImplementedError()
+
+    def _findout_record_infos(self, kwargs):
+        return dict(info=kwargs.get("info", None))
+
+    def allow_reads(self, item):  # @UnusedVariable
+        """
+        This method can be used to determine whether or not the present object
+        supports reads for **item**.
+
+        :note: This is different from **check** since, **item**'s existence is
+               not checked. It just tells if reads to **item** are supported...
+        """
+        return True
+
+    def allow_writes(self, item):  # @UnusedVariable
+        """
+        This method can be used to determine whether or not the present object
+        supports writes for **item**.
+
+        :note: This is different from **check** since, **item**'s existence is
+               not checked. It just tells if writes to **item** are supported...
+        """
+        return True
+
+    def fullpath(self, item, **kwargs):
+        """Return the path/URI to the **item**'s storage location."""
+        # Currently no recording is performed for the check action
+        (rc, _) = self._actual_fullpath(item, **kwargs)
+        return rc
+
+    def prestageinfo(self, item, **kwargs):
+        """Return the prestage infos for an **item** in the current storage place."""
+        # Currently no recording is performed for the check action
+        (rc, _) = self._actual_prestageinfo(item, **kwargs)
+        return rc
+
+    def check(self, item, **kwargs):
+        """Check/Stat an **item** from the current storage place."""
+        # Currently no recording is performed for the check action
+        (rc, _) = self._actual_check(item, **kwargs)
+        return rc
+
+    def list(self, item, **kwargs):
+        """List all data resources available in the **item** directory."""
+        # Currently no recording is performed for the check action
+        (rc, _) = self._actual_list(item, **kwargs)
+        return rc
+
+    @enforce_readonly
+    @do_recording('INSERT')
+    def insert(self, item, local, **kwargs):
+        """Insert an **item** in the current storage place.
+
+        :note: **local** may be a path to a file or any kind of file like objects.
+        """
+        return self._actual_insert(item, local, **kwargs)
+
+    @do_recording('RETRIEVE')
+    def retrieve(self, item, local, **kwargs):
+        """Retrieve an **item** from the current storage place.
+
+        :note: **local** may be a path to a file or any kind of file like objects.
+        """
+        return self._actual_retrieve(item, local, **kwargs)
+
+    def earlyretrieve(self, item, local, **kwargs):
+        """Trigger a delayed retrieve of **item** from the current storage place.
+
+        :note: **local** may be a path to a file or any kind of file like objects.
+        """
+        return self._actual_earlyretrieve(item, local, **kwargs)
+
+    def _actual_earlyretrieve(self, item, local, **kwargs):  # @UnusedVariable
+        """No earlyretrieve implemented by default."""
+        return None
+
+    def finaliseretrieve(self, retrieve_id, item, local, **kwargs):
+        """Finalise a delayed retrieve from the current storage place.
+
+        :note: **local** may be a path to a file or any kind of file like objects.
+        """
+        rc, idict = self._actual_finaliseretrieve(retrieve_id, item, local, **kwargs)
+        if rc is not None:
+            infos = self._findout_record_infos(kwargs)
+            infos.update(idict)
+            self.addrecord('RETRIEVE', item, status=rc, **infos)
+        return rc
+
+    def _actual_finaliseretrieve(self, retrieve_id, item, local, **kwargs):  # @UnusedVariable
+        """No delayedretrieve implemented by default."""
+        return None, dict()
+
+    @enforce_readonly
+    @do_recording('DELETE')
+    def delete(self, item, **kwargs):
+        """Delete an **item** from the current storage place."""
+        return self._actual_delete(item, **kwargs)
+
+
+# Defining the two main flavours of storage places
+# -----------------------------------------------
+
+class Cache(Storage):
+    """Root class for any :class:Cache subclasses."""
+
+    _abstract = True
+    _collector = ('cache',)
+    _footprint = dict(
+        info = 'Default cache description',
+        attr = dict(
+            headdir = dict(
+                info     = "The cache's subdirectory (within **rootdir**).",
+                optional = True,
+                default  = 'cache',
+            ),
+            # TODO is 'storage' used in any way?
+            storage = dict(
+                optional = True,
+                default  = 'localhost',
+            ),
+            rtouch = dict(
+                info     = "Perform the recursive touch command on the directory structure.",
+                type     = bool,
+                optional = True,
+                default  = False,
+            ),
+            rtouchskip = dict(
+                info     = "Do not 'touch' the first **rtouchskip** directories.",
+                type     = int,
+                optional = True,
+                default  = 0,
+            ),
+            rtouchdelay = dict(
+                info     = ("Do not perfom a touch if it has already been done in " +
+                            "the last X seconds."),
+                type     = float,
+                optional = True,
+                default  = 600.,  # 10 minutes
+            ),
+        )
+    )
+
+    def __init__(self, *kargs, **kwargs):
+        super().__init__(*kargs, **kwargs)
+        self._touch_tracker = dict()
+
+    @property
+    def realkind(self):
+        return 'cache'
+
+    @property
+    def tag(self):
+        """The identifier of this cache place."""
+        return '{:s}_{:s}_{:s}'.format(self.realkind, self.kind, self.headdir)
+
+    def _formatted_path(self, subpath, **kwargs):  # @UnusedVariable
+        raise NotImplementedError()
+
+    def catalog(self):
+        """List all files present in this cache.
+
+        :note: It might be quite slow...
+        """
+        raise NotImplementedError()
+
+    def _xtouch(self, path):
+        """
+        Perform a touch operation only if the last one, on te same path, was
+        less than `self.rtouchdelay` seconds ago.
+        """
+        ts = time.time()
+        ts_delay = ts - self._touch_tracker.get(path, 0)
+        if ts_delay > self.rtouchdelay:
+            logger.debug('Touching: %s (delay was %.2f)', path, ts_delay)
+            self.sh.touch(path)
+            self._touch_tracker[path] = ts
+        else:
+            logger.debug('Skipping touch: %s (delay was %.2f)', path, ts_delay)
+
+    def _recursive_touch(self, rc, item, writing=False):
+        """Make recursive touches on parent directories.
+
+        It might be useful for cleaning scripts.
+        """
+        if self.rtouch and (not self.readonly) and rc:
+            items = item.lstrip('/').split('/')
+            items = items[:-1]
+            if writing:
+                # It's useless to touch the rightmost directory
+                items = items[:-1] if len(items) > 1 else []
+            for index in range(len(items), self.rtouchskip, -1):
+                self._xtouch(self._formatted_path(self.sh.path.join(*items[:index])))
+
+    def _actual_fullpath(self, item, **kwargs):
+        """Return the path/URI to the **item**'s storage location."""
+        return self._formatted_path(item, **kwargs), dict()
+
+    def _actual_prestageinfo(self, item, **kwargs):
+        """Returns pre-staging informations."""
+        return dict(strategy=self.kind,
+                    location=self.fullpath(item, **kwargs)), dict()
+
+    def _actual_check(self, item, **kwargs):
+        """Check/Stat an **item** from the current storage place."""
+        path = self._formatted_path(item, **kwargs)
+        if path is None:
+            return None, dict()
+        try:
+            st = self.sh.stat(path)
+        except OSError:
+            st = None
+        return st, dict()
+
+    def _actual_list(self, item, **kwargs):
+        """List all data resources available in the **item** directory."""
+        path = self.fullpath(item, **kwargs)
+        if path is not None and self.sh.path.exists(path):
+            if self.sh.path.isdir(path):
+                return self.sh.listdir(path), dict()
+            else:
+                return True, dict()
+        else:
+            return None, dict()
+
+    def _actual_insert(self, item, local, **kwargs):
+        """Insert an **item** in the current storage place."""
+        # Get the relevant options
+        intent = kwargs.get("intent", "in")
+        fmt = kwargs.get("fmt", "foo")
+        # Insert the element
+        tpath = self._formatted_path(item)
+        if tpath is not None:
+            rc = self.sh.cp(local, tpath, intent=intent, fmt=fmt,
+                            smartcp_threshold=HARDLINK_THRESHOLD)
+        else:
+            logger.warning('No target location for < %s >', item)
+            rc = False
+        self._recursive_touch(rc, item, writing=True)
+        return rc, dict(intent=intent, fmt=fmt)
+
+    def _actual_retrieve(self, item, local, **kwargs):
+        """Retrieve an **item** from the current storage place."""
+        # Get the relevant options
+        intent = kwargs.get("intent", "in")
+        fmt = kwargs.get("fmt", "foo")
+        silent = kwargs.get("silent", False)
+        dirextract = kwargs.get("dirextract", False)
+        tarextract = kwargs.get("tarextract", False)
+        uniquelevel_ignore = kwargs.get("uniquelevel_ignore", True)
+        source = self._formatted_path(item)
+        if source is not None:
+            # If auto_dirextract, copy recursively each file contained in source
+            if dirextract and self.sh.path.isdir(source) and self.sh.is_tarname(local):
+                rc = True
+                destdir = self.sh.path.dirname(self.sh.path.realpath(local))
+                logger.info('Automatic directory extract to: %s', destdir)
+                for subpath in self.sh.glob(source + '/*'):
+                    rc = rc and self.sh.cp(subpath,
+                                           self.sh.path.join(destdir, self.sh.path.basename(subpath)),
+                                           intent=intent, fmt=fmt,
+                                           smartcp_threshold=HARDLINK_THRESHOLD)
+                    # For the insitu feature to work...
+                    rc = rc and self.sh.touch(local)
+            # The usual case: just copy source
+            else:
+                rc = self.sh.cp(source, local, intent=intent, fmt=fmt, silent=silent,
+                                smartcp_threshold=HARDLINK_THRESHOLD)
+                # If auto_tarextract, a potential tar file is extracted
+                if (rc and tarextract and not self.sh.path.isdir(local) and
+                        self.sh.is_tarname(local) and self.sh.is_tarfile(local)):
+                    destdir = self.sh.path.dirname(self.sh.path.realpath(local))
+                    logger.info('Automatic Tar extract to: %s', destdir)
+                    rc = rc and self.sh.smartuntar(local, destdir,
+                                                   uniquelevel_ignore=uniquelevel_ignore)
+        else:
+            getattr(logger, 'info' if silent else 'warning')('No readable source for < %s >', item)
+            rc = False
+        self._recursive_touch(rc, item)
+        return rc, dict(intent=intent, fmt=fmt)
+
+    def _actual_delete(self, item, **kwargs):
+        """Delete an **item** from the current storage place."""
+        # Get the relevant options
+        fmt = kwargs.get("fmt", "foo")
+        # Delete the element
+        tpath = self._formatted_path(item)
+        if tpath is not None:
+            rc = self.sh.remove(tpath, fmt=fmt)
+        else:
+            logger.warning('No target location for < %s >', item)
+            rc = False
+        return rc, dict(fmt=fmt)
+
+
+class AbstractArchive(Storage):
+    """The default class to handle storage to some kind if Archive."""
+
+    _abstract = True
+    _collector = ('archive',)
+    _footprint = dict(
+        info = 'Default archive description',
+        attr = dict(
+            tube = dict(
+                info     = "How to communicate with the archive ?",
+            ),
+        )
+    )
+
+    @property
+    def tag(self):
+        """The identifier of this cache place."""
+        return '{:s}_{:s}_{:s}'.format(self.realkind, self.storage, self.kind)
+
+    @property
+    def realkind(self):
+        return 'archive'
+
+    def _formatted_path(self, rawpath, **kwargs):
+        root = kwargs.get('root', None)
+        if root is not None:
+            rawpath = self.sh.path.join(root, rawpath.lstrip('/'))
+        # Deal with compression
+        compressionpipeline = kwargs.get('compressionpipeline', None)
+        if compressionpipeline is not None:
+            rawpath += compressionpipeline.suffix
+        return self.sh.anyft_remote_rewrite(rawpath, fmt=kwargs.get('fmt', 'foo'))
+
+    def _actual_proxy_method(self, pmethod):
+        """Create a proxy method based on the **pmethod** actual method."""
+
+        def actual_proxy(item, *kargs, **kwargs):
+            path = self._formatted_path(item, **kwargs)
+            if path is None:
+                raise ValueError("The archive's path is void.")
+            return pmethod(path, *kargs, **kwargs)
+
+        actual_proxy.__name__ = pmethod.__name__
+        actual_proxy.__doc__ = pmethod.__doc__
+        return actual_proxy
+
+    def __getattr__(self, attr):
+        """Provides proxy methods for _actual_* methods."""
+        methods = r'fullpath|prestageinfo|check|list|insert|retrieve|delete'
+        mattr = re.match(r'_actual_(?P<action>' + methods + r')', attr)
+        if mattr:
+            pmethod = getattr(self, '_{:s}{:s}'.format(self.tube, mattr.group('action')))
+            return self._actual_proxy_method(pmethod)
+        else:
+            raise AttributeError("The {:s} attribute was not found in this object"
+                                 .format(attr))
+
+    def _actual_earlyretrieve(self, item, local, **kwargs):
+        """Proxy to the appropriate tube dependent earlyretrieve method (if available)."""
+        pmethod = getattr(self, '_{:s}{:s}'.format(self.tube, 'earlyretrieve'), None)
+        if pmethod:
+            return self._actual_proxy_method(pmethod)(item, local, **kwargs)
+        else:
+            return None
+
+    def _actual_finaliseretrieve(self, retrieve_id, item, local, **kwargs):
+        """Proxy to the appropriate tube dependent finaliseretrieve method (if available)."""
+        pmethod = getattr(self, '_{:s}{:s}'.format(self.tube, 'finaliseretrieve'), None)
+        if pmethod:
+            return self._actual_proxy_method(pmethod)(item, local, retrieve_id, **kwargs)
+        else:
+            return None, dict()
+
+
+class Archive(AbstractArchive):
+    """The default class to handle storage to a remote location."""
+
+    _footprint = dict(
+        info = 'Default archive description',
+        attr = dict(
+            tube = dict(
+                values   = ['ftp'],
+            ),
+        )
+    )
+
+    def __init__(self, *kargs, **kwargs):
+        super().__init__(*kargs, **kwargs)
+        self.default_usejeeves = config.from_config(
+            section="storage", key="usejeeves",
+        )
+
+    @property
+    def _ftp_hostinfos(self):
+        """Return the FTP hostname end port number."""
+        s_storage = self.storage.split(':', 1)
+        hostname = s_storage[0]
+        port = None
+        if len(s_storage) > 1:
+            try:
+                port = int(s_storage[1])
+            except ValueError:
+                logger.error('Invalid port number < %s >. Ignoring it', s_storage[1])
+        return hostname, port
+
+    def _ftp_client(self, logname=None, delayed=False):
+        """Return a FTP client object."""
+        hostname, port = self._ftp_hostinfos
+        return self.sh.ftp(hostname, logname=logname, delayed=delayed, port=port)
+
+    def _ftpfullpath(self, item, **kwargs):
+        """Actual _fullpath using ftp."""
+        username = kwargs.get('username', None)
+        rc = None
+        ftp = self._ftp_client(logname=username, delayed=True)
+        if ftp:
+            try:
+                rc = ftp.netpath(item)
+            finally:
+                ftp.close()
+        return rc, dict()
+
+    def _ftpprestageinfo(self, item, **kwargs):
+        """Actual _prestageinfo using ftp."""
+        username = kwargs.get('username', None)
+        if username is None:
+            ftp = self._ftp_client(logname=username, delayed=True)
+            if ftp:
+                try:
+                    username = ftp.logname
+                finally:
+                    ftp.close()
+        baseinfo = dict(storage=self.storage,
+                        logname=username,
+                        location=item, )
+        return baseinfo, dict()
+
+    def _ftpcheck(self, item, **kwargs):
+        """Actual _check using ftp."""
+        rc = None
+        ftp = self._ftp_client(logname=kwargs.get('username', None))
+        if ftp:
+            try:
+                rc = ftp.size(item)
+            except (ValueError, TypeError):
+                pass
+            except ftplib.all_errors:
+                pass
+            finally:
+                ftp.close()
+        return rc, dict()
+
+    def _ftplist(self, item, **kwargs):
+        """Actual _list using ftp."""
+        ftp = self._ftp_client(logname=kwargs.get('username', None))
+        rc = None
+        if ftp:
+            try:
+                # Is this a directory ?
+                rc = ftp.cd(item)
+            except ftplib.all_errors:
+                # Apparently not...
+                try:
+                    # Is it a file ?
+                    if ftp.size(item) is not None:
+                        rc = True
+                except (ValueError, TypeError):
+                    pass
+                except ftplib.all_errors:
+                    pass
+            else:
+                # Content of the directory...
+                if rc:
+                    rc = ftp.nlst('.')
+            finally:
+                ftp.close()
+        return rc, dict()
+
+    def _ftpretrieve(self, item, local, **kwargs):
+        """Actual _retrieve using ftp."""
+        logger.info('ftpget on ftp://%s/%s (to: %s)', self.storage, item, local)
+        extras = dict(fmt=kwargs.get('fmt', 'foo'),
+                      cpipeline=kwargs.get('compressionpipeline', None))
+        hostname, port = self._ftp_hostinfos
+        if port is not None:
+            extras['port'] = port
+        rc = self.sh.smartftget(
+            item,
+            local,
+            # Ftp control
+            hostname=hostname,
+            logname=kwargs.get('username', None),
+            **extras
+        )
+        return rc, extras
+
+    def _ftpearlyretrieve(self, item, local, **kwargs):
+        """
+        If FtServ/ftraw is used, trigger a delayed action in order to fetch
+        several files at once.
+        """
+        cpipeline = kwargs.get('compressionpipeline', None)
+        if self.sh.rawftget_worthy(item, local, cpipeline):
+            return self.context.delayedactions_hub.register((item, kwargs.get('fmt', 'foo')),
+                                                            kind='archive',
+                                                            storage=self.storage,
+                                                            goal='get',
+                                                            tube='ftp',
+                                                            raw=True,
+                                                            logname=kwargs.get('username', None))
+        else:
+            return None
+
+    def _ftpfinaliseretrieve(self, item, local, retrieve_id, **kwargs):  # @UnusedVariable
+        """
+        Get the resource given the **retrieve_id** identifier returned by the
+        :meth:`_ftpearlyretrieve` method.
+        """
+        extras = dict(fmt=kwargs.get('fmt', 'foo'), )
+        d_action = self.context.delayedactions_hub.retrieve(retrieve_id, bareobject=True)
+        if d_action.status == d_action_status.done:
+            if self.sh.filecocoon(local):
+                rc = self.sh.mv(d_action.result, local, **extras)
+            else:
+                raise OSError('Could not cocoon: {!s}'.format(local))
+        elif d_action.status == d_action_status.failed:
+            logger.info('The earlyretrieve failed (retrieve_id=%s)', retrieve_id)
+            rc = False
+        else:
+            rc = None
+        return rc, extras
+
+    def _ftpinsert(self, item, local, **kwargs):
+        """Actual _insert using ftp."""
+        usejeeves = kwargs.get('usejeeves', None)
+        if usejeeves is None:
+            usejeeves = self.default_usejeeves
+        hostname, port = self._ftp_hostinfos
+        if not usejeeves:
+            logger.info('ftpput to ftp://%s/%s (from: %s)', self.storage, item, local)
+            extras = dict(fmt=kwargs.get('fmt', 'foo'),
+                          cpipeline=kwargs.get('compressionpipeline', None))
+            if port is not None:
+                extras['port'] = port
+            rc = self.sh.smartftput(
+                local,
+                item,
+                # Ftp control
+                hostname=hostname,
+                logname=kwargs.get('username', None),
+                sync=kwargs.get('enforcesync', False),
+                **extras
+            )
+        else:
+            logger.info('delayed ftpput to ftp://%s/%s (from: %s)', self.storage, item, local)
+            tempo = footprints.proxy.service(kind='hiddencache',
+                                             asfmt=kwargs.get('fmt'))
+            compressionpipeline = kwargs.get('compressionpipeline', '')
+            if compressionpipeline:
+                compressionpipeline = compressionpipeline.description_string
+            extras = dict(fmt=kwargs.get('fmt', 'foo'),
+                          cpipeline=compressionpipeline)
+            if port is not None:
+                extras['port'] = port
+
+            rc = ad.jeeves(
+                hostname=hostname,
+                # Explicitly resolve the logname (because jeeves FTP client is not
+                # running with the same glove (i.e. Jeeves ftuser configuration may
+                # be different).
+                logname=self.sh.fix_ftuser(hostname,
+                                           kwargs.get('username', None)),
+                todo='ftput',
+                rhandler=kwargs.get('info', None),
+                source=tempo(local),
+                destination=item,
+                original=self.sh.path.abspath(local),
+                **extras
+            )
+        return rc, extras
+
+    def _ftpdelete(self, item, **kwargs):
+        """Actual _delete using ftp."""
+        rc = None
+        ftp = self._ftp_client(logname=kwargs.get('username', None))
+        if ftp:
+            if self._ftpcheck(item, **kwargs)[0]:
+                logger.info('ftpdelete on ftp://%s/%s', self.storage, item)
+                rc = ftp.delete(item)
+                ftp.close()
+            else:
+                logger.error('Try to remove a non-existing resource <%s>', item)
+        return rc, dict()
+
+
+class AbstractLocalArchive(AbstractArchive):
+    """The default class to handle storage to the same host."""
+
+    _abstract = True
+    _footprint = dict(
+        info = 'Generic local archive description',
+        attr = dict(
+            tube = dict(
+                values   = ['inplace', ],
+            ),
+        )
+    )
+
+    def _inplacefullpath(self, item, **kwargs):
+        """Actual _fullpath."""
+        return item, dict()
+
+    def _inplacecheck(self, item, **kwargs):
+        """Actual _check."""
+        try:
+            st = self.sh.stat(item)
+        except OSError:
+            rc = None
+        else:
+            rc = st.st_size
+        return rc, dict()
+
+    def _inplacelist(self, item, **kwargs):
+        """Actual _list."""
+        if self.sh.path.exists(item):
+            if self.sh.path.isdir(item):
+                return self.sh.listdir(item), dict()
+            else:
+                return True, dict()
+        else:
+            return None, dict()
+
+    def _inplaceretrieve(self, item, local, **kwargs):
+        """Actual _retrieve using ftp."""
+        logger.info('inplaceget on file:///%s (to: %s)', item, local)
+        fmt = kwargs.get('fmt', 'foo')
+        cpipeline = kwargs.get('compressionpipeline', None)
+        if cpipeline:
+            rc = cpipeline.file2uncompress(item, local)
+        else:
+            # Do not use fmt=... on purpose (otherwise "forceunpack" may be called twice)
+            rc = self.sh.cp(item, local, intent='in')
+        rc = rc and self.sh.forceunpack(local, fmt=fmt)
+        return rc, dict(fmt=fmt, cpipeline=cpipeline)
+
+    @contextlib.contextmanager
+    def _inplaceinsert_pack(self, local, fmt):
+        local_packed = self.sh.forcepack(local, fmt=fmt)
+        if local_packed != local:
+            try:
+                yield local_packed
+            finally:
+                self.sh.rm(local_packed, fmt=fmt)
+        else:
+            yield local
+
+    def _inplaceinsert(self, item, local, **kwargs):
+        """Actual _insert using ftp."""
+        logger.info('inplaceput to file:///%s (from: %s)', item, local)
+        cpipeline = kwargs.get('compressionpipeline', None)
+        fmt = kwargs.get('fmt', 'foo')
+        with self._inplaceinsert_pack(local, fmt) as local_packed:
+            if cpipeline:
+                rc = cpipeline.compress2file(local_packed, item)
+            else:
+                # Do not use fmt=... on purpose (otherwise "forcepack" may be called twice)
+                rc = self.sh.cp(local_packed, item, intent='in')
+        return rc, dict(fmt=fmt, cpipeline=cpipeline)
+
+    def _inplacedelete(self, item, **kwargs):
+        """Actual _delete using ftp."""
+        fmt = kwargs.get('fmt', 'foo')
+        rc = None
+        if self._inplacecheck(item, **kwargs)[0]:
+            rc = self.sh.rm(item, fmt=fmt)
+        return rc, dict(fmt=fmt)
+
+
+class LocalArchive(AbstractLocalArchive):
+    """The default class to handle storage to the same host."""
+
+    _footprint = dict(
+        info = 'Default local archive description',
+        attr = dict(
+            storage = dict(
+                values   = ['localhost', ],
+            ),
+            auto_self_expand = dict(
+                info     = ('Automatically expand the current user home if ' +
+                            'a relative path is given (should always be True ' +
+                            'except during unit-testing)'),
+                type     = bool,
+                default  = True,
+                optional = True,
+            ),
+        )
+    )
+
+    def _formatted_path(self, rawpath, **kwargs):
+        rawpath = self.sh.path.expanduser(rawpath)
+        if '~' in rawpath:
+            raise OSError('User expansion failed for "{:s}"'.format(rawpath))
+        if self.auto_self_expand and not self.sh.path.isabs(rawpath):
+            rawpath = self.sh.path.expanduser(self.sh.path.join('~', rawpath))
+        return super()._formatted_path(rawpath, **kwargs)
+
+
+# Concrete cache implementations
+# ------------------------------
+
+
+class FixedEntryCache(Cache):
+    _abstract = True
+    _footprint = dict(
+        info = 'Default cache description (with a fixed entry point)',
+        attr = dict(
+            rootdir = dict(
+                info     = "The cache's location (usually on a filesystem).",
+                optional = True,
+                default  = None,
+            ),
+        )
+    )
+
+    @property
+    def entry(self):
+        """Tries to figure out what could be the actual entry point for storage space."""
+        if not self.rootdir:
+            self.rootdir = "/tmp"
+        return self.sh.path.join(self.rootdir, self.kind, self.headdir)
+
+    @property
+    def tag(self):
+        """The identifier of this cache place."""
+        return '{:s}_{:s}'.format(self.realkind, self.entry)
+
+    def _formatted_path(self, subpath, **kwargs):  # @UnusedVariable
+        return self.sh.path.join(self.entry, subpath.lstrip('/'))
+
+    def catalog(self):
+        """List all files present in this cache.
+
+        :note: It might be quite slow...
+        """
+        entry = self.sh.path.expanduser(self.entry)
+        files = self.sh.ffind(entry)
+        return [f[len(entry):] for f in files]
+
+    def flush(self, dumpfile=None):
+        """Flush actual history to the specified ``dumpfile`` if record is on."""
+        if dumpfile is None:
+            logfile = '.'.join((
+                'HISTORY',
+                datetime.now().strftime('%Y%m%d%H%M%S.%f'),
+                'P{:06d}'.format(self.sh.getpid()),
+                self.sh.getlogname()
+            ))
+            dumpfile = self.sh.path.join(self.entry, '.history', logfile)
+        if self.record:
+            self.sh.pickle_dump(self.history, dumpfile)
+
+
+class MtoolCache(FixedEntryCache):
+    """Cache items for the MTOOL jobs (or any job that acts like it)."""
+
+    _footprint = dict(
+        info = 'MTOOL like Cache',
+        attr = dict(
+            kind = dict(
+                values   = ['mtool', 'swapp'],
+                remap    = dict(swapp = 'mtool'),
+            ),
+            headdir = dict(
+                optional = True,
+                default  = "",
+            ),
+        )
+    )
+
+    @property
+    def entry(self):
+        """Tries to figure out what could be the actual entry point
+        for cache space.
+
+        """
+        if self.rootdir:
+            return os.path.join(self.rootdir, self.headdir)
+
+        if config.is_defined(section="data-tree", key="rootdir"):
+            rootdir = config.from_config(
+                section="data-tree", key="rootdir",
+            )
+        else:
+            rootdir = self.sh.path.join(os.environ["HOME"], ".vortex.d")
+
+        return self.sh.path.join(rootdir, self.headdir)
+
+
+class FtStashCache(MtoolCache):
+    """A place to store file to be sent with ftserv."""
+
+    _footprint = dict(
+        info = 'A place to store file to be sent with ftserv',
+        attr = dict(
+            kind = dict(
+                values   = ['ftstash', ],
+            ),
+            headdir = dict(
+                optional = True,
+                default  = 'ftspool',
+            ),
+        )
+    )
+
+
+class Op2ResearchCache(FixedEntryCache):
+    """Cache of the operational suite (read-only)."""
+
+    _footprint = dict(
+        info = 'MTOOL like Operations Cache (read-only)',
+        attr = dict(
+            kind = dict(
+                values   = ['op2r'],
+            ),
+            headdir = dict(
+                optional = True,
+                default  = 'vortex',
+            ),
+            readonly = dict(
+                values  = [True, ],
+                default = True,
+            )
+        )
+    )
+
+    @property
+    def entry(self):
+        cache = (
+            self.rootdir or
+            config.from_config(section="data-tree", key="op_rootdir")
+        )
+        return self.sh.path.join(cache, self.headdir)
+
+
+class HackerCache(FixedEntryCache):
+    """A dirty cache where users can hack things."""
+
+    _footprint = dict(
+        info = 'A place to hack things...',
+        attr = dict(
+            kind = dict(
+                values   = ['hack'],
+            ),
+            rootdir = dict(
+                optional = True,
+                default  = 'auto'
+            ),
+            readonly = dict(
+                default = True,
+            ),
+        )
+    )
+
+    @property
+    def entry(self):
+        """Tries to figure out what could be the actual entry point for cache space."""
+        sh = self.sh
+        if self.rootdir == 'auto':
+            gl = sessions.current().glove
+            sweethome = sh.path.join(gl.configrc, 'hack')
+            sh.mkdir(sweethome)
+            logger.debug('Using %s hack cache: %s', self.__class__, sweethome)
+        else:
+            sweethome = self.rootdir
+        return sh.path.join(sweethome, self.headdir)