vortex-nwp 2.0.0b1__py3-none-any.whl

vortex/data/stores.py

@@ -0,0 +1,1271 @@
+# pylint: disable=unused-argument
+
+"""
+This module handles store objects in charge of physically accessing resources.
+Store objects use the :mod:`footprints` mechanism.
+"""
+
+import copy
+import ftplib
+import io
+import re
+
+from bronx.fancies import loggers
+import footprints
+
+from vortex import sessions
+from vortex import config
+from vortex.data.abstractstores import Store, ArchiveStore, ConfigurableArchiveStore, CacheStore
+from vortex.data.abstractstores import MultiStore, PromiseStore
+from vortex.data.abstractstores import ARCHIVE_GET_INTENT_DEFAULT
+from vortex.layout import dataflow
+from vortex.syntax.stdattrs import hashalgo_avail_list
+from vortex.syntax.stdattrs import FreeXPid
+from vortex.syntax.stdattrs import DelayedEnvValue
+from vortex.tools.systems import ExecutionError
+
+#: Export base class
+__all__ = []
+
+logger = loggers.getLogger(__name__)
+
+
+class MagicPlace(Store):
+    """Somewhere, over the rainbow!"""
+
+    _footprint = dict(
+        info = 'Evanescent physical store',
+        attr = dict(
+            scheme = dict(
+                values   = ['magic'],
+            ),
+        ),
+        priority = dict(
+            level = footprints.priorities.top.DEFAULT  # @UndefinedVariable
+        )
+    )
+
+    @property
+    def realkind(self):
+        return 'magicstore'
+
+    def has_fast_check(self):
+        """A void check is very fast !"""
+        return True
+
+    def magiccheck(self, remote, options):
+        """Void - Always False."""
+        return False
+
+    def magiclocate(self, remote, options):
+        """Void - Empty string returned."""
+        return ''
+
+    def magicget(self, remote, local, options):
+        """Void - Always True."""
+        return True
+
+    def magicput(self, local, remote, options):
+        """Void - Always True."""
+        return True
+
+    def magicdelete(self, remote, options):
+        """Void - Always False."""
+        return False
+
+
+class FunctionStoreCallbackError(Exception):
+    pass
+
+
+class FunctionStore(Store):
+    """Calls a function that returns a File like object (get only).
+
+    This store is only able to perform the get action: it imports and calls
+    the function specified in the URI path. This function should return a
+    file like object that will be written in the local container.
+
+    The function is given an option dictionary that contains all of the
+    options provided to the store's get function, plus any additional
+    information specified in the 'query' part of the URI.
+
+    :Example:
+
+    Lets consider the following URI:
+
+      ``function:///sandbox.util.storefunctions.echofunction?msg=toto&msg=titi``
+
+    It will be seen as follows:
+
+    * scheme: ``'function'``
+    * netloc: ``''``
+    * path: ``'/sandbox.utils.storefunctions.echofunction'``
+    * query: ``dict(msg=['toto', 'titi'])``
+
+    As a result, the :func:`sandbox.utils.storefunctions.echofunction` will be
+    called with an option dictionary that contains ['toto', 'titi'] for the
+    'msg' key (plus any other options passed to the store's get method).
+    """
+
+    _footprint = dict(
+        info = 'Dummy store that calls a function',
+        attr = dict(
+            scheme = dict(
+                values   = ['function'],
+            ),
+            netloc = dict(
+                values   = [''],
+            )
+        ),
+        priority = dict(
+            level = footprints.priorities.top.DEFAULT  # @UndefinedVariable
+        )
+    )
+
+    @property
+    def realkind(self):
+        return 'functionstore'
+
+    def has_fast_check(self):
+        """A void check is very fast !"""
+        return True
+
+    def functioncheck(self, remote, options):
+        """Void - Always False."""
+        return False
+
+    def functionlocate(self, remote, options):
+        """The name of the function that will be called."""
+        cleanname = remote['path'][1:]
+        if cleanname.endswith('/'):
+            cleanname = cleanname[:-1]
+        return cleanname
+
+    def functionget(self, remote, local, options):
+        """Calls the appropriate function and writes the result."""
+        # Find the appropriate function
+        cbfunc = self.system.import_function(self.functionlocate(remote,
+                                                                 options))
+        # ... and call it
+        opts = dict()
+        opts.update(options)
+        opts.update(remote['query'])
+        try:
+            fres = cbfunc(opts)
+        except FunctionStoreCallbackError as e:
+            logger.error("An exception was raised in the Callback function")
+            logger.error("Here is the exception: %s", str(e))
+            fres = None
+        if fres is not None:
+            if 'intent' in options and options['intent'] == dataflow.intent.IN:
+                logger.info('Ignore intent <in> for function input.')
+            # Handle StringIO objects, by changing them to ByteIOs...
+            if isinstance(fres, io.StringIO):
+                s_fres = fres
+                s_fres.seek(0)
+                fres = io.BytesIO()
+                for l in s_fres:
+                    fres.write(l.encode(encoding='utf-8'))
+                fres.seek(0)
+            # NB: fres should be a file like object (BytesIO will do the trick)
+            return self.system.cp(fres, local)
+        else:
+            return False
+
+    def functionput(self, local, remote, options):
+        """This should not happen - Always False."""
+        logger.error("The function store is not able to perform PUTs.")
+        return False
+
+    def functiondelete(self, remote, options):
+        """This should not happen - Always False."""
+        logger.error("The function store is not able to perform Deletes.")
+        return False
+
+
+class Finder(Store):
+    """The most usual store: your current filesystem!"""
+
+    _footprint = dict(
+        info = 'Miscellaneous file access',
+        attr = dict(
+            scheme = dict(
+                values  = ['file', 'ftp', 'symlink', 'rcp', 'scp'],
+            ),
+            netloc = dict(
+                outcast = ['oper.inline.fr'],
+            ),
+            storehash = dict(
+                values = hashalgo_avail_list,
+            ),
+        ),
+        priority = dict(
+            level = footprints.priorities.top.DEFAULT  # @UndefinedVariable
+        )
+    )
+
+    def __init__(self, *args, **kw):
+        logger.debug('Finder store init %s', self.__class__)
+        super().__init__(*args, **kw)
+
+    @property
+    def realkind(self):
+        return 'finder'
+
+    def hostname(self):
+        """Returns the current :attr:`netloc`."""
+        return self.netloc
+
+    def fullpath(self, remote):
+        """Return actual path unless explicitly defined as relative path."""
+        if remote['query'].get('relative', False):
+            return remote['path'].lstrip('/')
+        else:
+            return remote['path']
+
+    def _localtarfix(self, local):
+        if (isinstance(local, str) and self.system.path.isfile(local) and
+                self.system.is_tarfile(local)):
+            destdir = self.system.path.dirname(self.system.path.realpath(local))
+            try:
+                self.system.smartuntar(local, destdir)
+            except ExecutionError:
+                if not self.system.is_tarname(local):
+                    logger.warning("An automatic untar was attempted but it failed. " +
+                                   "Maybe the system's is_tarfile got it wrong ?")
+                else:
+                    raise
+
+    def filecheck(self, remote, options):
+        """Returns a stat-like object if the ``remote`` exists on the ``system`` provided."""
+        try:
+            st = self.system.stat(self.fullpath(remote))
+        except OSError:
+            st = None
+        return st
+
+    def filelocate(self, remote, options):
+        """Returns the real path."""
+        return self.fullpath(remote)
+
+    def fileget(self, remote, local, options):
+        """Delegates to ``system`` the copy of ``remote`` to ``local``."""
+        rpath = self.fullpath(remote)
+        logger.info('fileget on %s (to: %s)', rpath, local)
+        if 'intent' in options and options['intent'] == dataflow.intent.IN:
+            logger.info('Ignore intent <in> for remote input %s', rpath)
+        rc = self.system.cp(rpath, local, fmt=options.get('fmt'), intent=dataflow.intent.INOUT)
+        rc = rc and self._hash_get_check(self.fileget, remote, local, options)
+        if rc:
+            self._localtarfix(local)
+        return rc
+
+    def fileput(self, local, remote, options):
+        """Delegates to ``system`` the copy of ``local`` to ``remote``."""
+        rpath = self.fullpath(remote)
+        logger.info('fileput to %s (from: %s)', rpath, local)
+        rc = self.system.cp(local, rpath, fmt=options.get('fmt'))
+        return rc and self._hash_put(self.fileput, local, remote, options)
+
+    def filedelete(self, remote, options):
+        """Delegates to ``system`` the removing of ``remote``."""
+        rc = None
+        if self.filecheck(remote, options):
+            rpath = self.fullpath(remote)
+            logger.info('filedelete on %s', rpath)
+            rc = self.system.remove(rpath, fmt=options.get('fmt'))
+        else:
+            logger.error('Try to remove a non-existing resource <%s>', self.fullpath(remote))
+        return rc
+
+    symlinkcheck = filecheck
+    symlinklocate = filelocate
+
+    def symlinkget(self, remote, local, options):
+        rpath = self.fullpath(remote)
+        if 'intent' in options and options['intent'] == dataflow.intent.INOUT:
+            logger.error('It is unsafe to have a symlink with intent=inout: %s', rpath)
+            return False
+        rc = self.system.remove(local)
+        self.system.symlink(rpath, local)
+        return rc and self.system.path.exists(local)
+
+    def symlinkput(self, local, remote, options):
+        logger.error("The Finder store with scheme:symlink is not able to perform Puts.")
+        return False
+
+    def symlinkdelete(self, remote, options):
+        logger.error("The Finder store with scheme:symlink is not able to perform Deletes.")
+        return False
+
+    def _ftpinfos(self, remote, **kwargs):
+        args = kwargs.copy()
+        args['hostname'] = self.hostname()
+        args['logname'] = remote['username']
+        port = self.hostname().netport
+        if port is not None:
+            args['port'] = port
+        return args
+
+    def ftpcheck(self, remote, options):
+        """Delegates to ``system.ftp`` a distant check."""
+        rc = None
+        ftp = self.system.ftp(** self._ftpinfos(remote))
+        if ftp:
+            try:
+                rc = ftp.size(self.fullpath(remote))
+            except (ValueError, TypeError):
+                pass
+            except ftplib.all_errors:
+                pass
+            finally:
+                ftp.close()
+        return rc
+
+    def ftplocate(self, remote, options):
+        """Delegates to ``system`` qualified name creation."""
+        ftp = self.system.ftp(** self._ftpinfos(remote, delayed=True))
+        if ftp:
+            rloc = ftp.netpath(self.fullpath(remote))
+            ftp.close()
+            return rloc
+        else:
+            return None
+
+    def ftpget(self, remote, local, options):
+        """Delegates to ``system`` the file transfer of ``remote`` to ``local``."""
+        rpath = self.fullpath(remote)
+        logger.info('ftpget on ftp://%s/%s (to: %s)', self.hostname(), rpath, local)
+        rc = self.system.smartftget(
+            rpath,
+            local,
+            fmt=options.get('fmt'),
+            # ftp control
+            ** self._ftpinfos(remote)
+        )
+        rc = rc and self._hash_get_check(self.ftpget, remote, local, options)
+        if rc:
+            self._localtarfix(local)
+        return rc
+
+    def ftpput(self, local, remote, options):
+        """Delegates to ``system`` the file transfer of ``local`` to ``remote``."""
+        rpath = self.fullpath(remote)
+        put_opts = dict()
+        put_opts['fmt'] = options.get('fmt')
+        put_opts['sync'] = options.get('enforcesync', False)
+        logger.info('ftpput to ftp://%s/%s (from: %s)', self.hostname(), rpath, local)
+        rc = self.system.smartftput(
+            local,
+            rpath,
+            # ftp control
+            ** self._ftpinfos(remote, ** put_opts)
+        )
+        return rc and self._hash_put(self.ftpput, local, remote, options)
+
+    def ftpdelete(self, remote, options):
+        """Delegates to ``system`` a distant remove."""
+        rc = None
+        actualpath = self.fullpath(remote)
+        if self.ftpcheck(remote, options):
+            logger.info('ftpdelete on ftp://%s/%s', self.hostname(), actualpath)
+            ftp = self.system.ftp(**self._ftpinfos(remote))
+            if ftp:
+                try:
+                    rc = ftp.delete(actualpath)
+                finally:
+                    ftp.close()
+        else:
+            logger.error('Try to remove a non-existing resource <%s>', actualpath)
+        return rc
+
+
+class _VortexStackedStorageMixin:
+    """Mixin class that adds utility functions to work with stacked data."""
+
+    _STACKED_RE = re.compile('stacked-')
+
+    @property
+    def stackedstore(self):
+        """Tell if the present store is looking into a stack of resources."""
+        return self._STACKED_RE.search(self.netloc)
+
+    def _stacked_remainder(self, remote, stackpath):
+        path_remainder = remote['path'].strip('/').split('/')
+        for a_spath in stackpath.split('/'):
+            if path_remainder and path_remainder[0] == a_spath:
+                del path_remainder[0]
+            else:
+                break
+        return '/'.join(path_remainder)
+
+    def _stacked_xremote(self, remote):
+        """The path to **remote** with its stack."""
+        if self.stackedstore:
+            remote = remote.copy()
+            remote['query'] = remote['query'].copy()
+            stackpath = remote['query'].pop('stackpath', (None, ))[0]
+            stackfmt = remote['query'].pop('stackfmt', (None, ))[0]
+            if stackpath is None or stackfmt is None:
+                raise ValueError('"stackpath" and "stackfmt" are not available in the query.')
+            else:
+                remote['path'] = stackpath + '/' + self._stacked_remainder(remote, stackpath)
+        return remote
+
+    def _stacked_xegglocate(self, remote):
+        """Return various informations about the stack associated with **remote**.
+
+        It returns a 3 elements tuple:
+
+        * The remote-like dictionary to the stack resource
+        * The format of the stack resource
+        * The path to **remote** within the stacked resource
+
+        """
+        remote = remote.copy()
+        remote['query'] = remote['query'].copy()
+        stackpath = remote['query'].pop('stackpath', (None, ))[0].strip('/')
+        stackfmt = remote['query'].pop('stackfmt', (None, ))[0]
+        if stackpath is None or stackfmt is None:
+            raise ValueError('"stackpath" and "stackfmt" are not available in the query.')
+        else:
+            resource_remainder = self._stacked_remainder(remote, stackpath)
+            remote['path'] = '/' + stackpath
+        return remote, stackfmt, resource_remainder
+
+
+_vortex_readonly_store = footprints.Footprint(
+    info="Abstract store' readonly=True attribute",
+    attr=dict(
+        readonly=dict(
+            values=[True, ],
+            optional=True,
+            default=True
+        )
+    )
+)
+
+
+class _VortexBaseArchiveStore(ArchiveStore, _VortexStackedStorageMixin):
+    """Some kind of archive for VORTEX experiments."""
+
+    _abstract = True
+    _footprint = dict(
+        info = 'VORTEX archive access',
+        attr = dict(
+            scheme = dict(
+                values   = ['vortex'],
+            ),
+            netloc = dict(
+            ),
+            storehead = dict(
+                optional = True,
+                default  = 'vortex',
+                outcast  = ['xp'],
+            ),
+        )
+    )
+
+    _STACKS_AUTOREFILL_CRIT = 'stacked-archive-smart'
+
+    def __init__(self, *args, **kw):
+        logger.debug('Vortex archive store init %s', self.__class__)
+        super().__init__(*args, **kw)
+
+    def remap_read(self, remote, options):
+        """Remap actual remote path to distant store path for intrusive actions."""
+        return copy.copy(remote)
+
+    def remap_list(self, remote, options):
+        """Reformulates the remote path to compatible vortex namespace."""
+        if len(remote['path'].split('/')) >= 4:
+            return self.remap_read(remote, options)
+        else:
+            logger.critical('The << %s >> path is not listable.', remote['path'])
+            return None
+
+    remap_write = remap_read
+
+    @property
+    def stacks_autorefill(self):
+        """Where to refill a stack retrieved from the archive."""
+        if self._STACKS_AUTOREFILL_CRIT in self.netloc:
+            return self.netloc.replace(self._STACKS_AUTOREFILL_CRIT, 'cache')
+        else:
+            return None
+
+    def _vortex_stacked_egg_retrieve(self, remote, result_id=None):
+        """Retrieve the stack associated with **remote**."""
+        remote, remotefmt, remainder = self._stacked_xegglocate(remote)
+        rundir = sessions.current().context.rundir
+        if not rundir:
+            rundir = self.system.pwd()
+        rundir = self.system.path.join(rundir, 'vortex_stacks_xeggs')
+        target = self.system.path.join(rundir, * remote['path'].strip('/').split('/'))
+        targetopts = dict(fmt=remotefmt, intent=dataflow.intent.IN)
+        if self.system.path.exists(target):
+            logger.info("Stack previously retrieved (in %s). Using it.", target)
+            rc = True
+        else:
+            if result_id:
+                rc = self._vortexfinaliseget(result_id, remote, target, targetopts)
+            else:
+                rc = self._vortexget(remote, target, targetopts)
+        if rc and self.stacks_autorefill:
+            rstore = footprints.proxy.store(scheme=self.scheme, netloc=self.stacks_autorefill)
+            logger.info("Refilling the stack egg to [%s]", rstore)
+            try:
+                rstore.put(target, remote.copy(), targetopts)
+            except (ExecutionError, OSError) as e:
+                logger.error("An ExecutionError happened during the refill: %s", str(e))
+                logger.error("This error is ignored... but that's ugly !")
+        return rc, target, remainder
+
+    def vortexcheck(self, remote, options):
+        """Vortex' archive check sequence."""
+        if self.stackedstore:
+            s_remote, s_remotefmt, _ = self._stacked_xegglocate(remote)
+            options = options.copy()
+            options['fmt'] = s_remotefmt
+            rc = self._vortexcheck(s_remote, options)
+            if rc:
+                rc, target, remainder = self._vortex_stacked_egg_retrieve(remote)
+                rc = rc and self.system.path.exists(self.system.path.join(target, remainder))
+            return rc
+        else:
+            return self._vortexcheck(remote, options)
+
+    def _vortexcheck(self, remote, options):
+        """Remap and ftpcheck sequence."""
+        remote = self.remap_read(remote, options)
+        return self.inarchivecheck(remote, options)
+
+    def vortexlocate(self, remote, options):
+        """Vortex' archive locate sequence."""
+        if self.stackedstore:
+            remote, s_remotefmt, _ = self._stacked_xegglocate(remote)
+            options = options.copy()
+            options['fmt'] = s_remotefmt
+        return self._vortexlocate(remote, options)
+
+    def _vortexlocate(self, remote, options):
+        """Remap and ftplocate sequence."""
+        remote = self.remap_read(remote, options)
+        return self.inarchivelocate(remote, options)
+
+    def vortexlist(self, remote, options):
+        """Vortex' archive list sequence."""
+        if self.stackedstore:
+            return None
+        else:
+            return self._vortexlist(remote, options)
+
+    def _vortexlist(self, remote, options):
+        """Remap and ftplist sequence."""
+        remote = self.remap_list(remote, options)
+        if remote:
+            return self.inarchivelist(remote, options)
+        else:
+            return None
+
+    def vortexprestageinfo(self, remote, options):
+        """Vortex' archive prestageinfo sequence."""
+        if self.stackedstore:
+            remote, s_remotefmt, _ = self._stacked_xegglocate(remote)
+            options = options.copy()
+            options['fmt'] = s_remotefmt
+        return self._vortexprestageinfo(remote, options)
+
+    def _vortexprestageinfo(self, remote, options):
+        """Remap and ftpprestageinfo sequence."""
+        remote = self.remap_read(remote, options)
+        return self.inarchiveprestageinfo(remote, options)
+
+    def vortexget(self, remote, local, options):
+        """Vortex' archive get sequence."""
+        if self.stackedstore:
+            rc, target, remainder = self._vortex_stacked_egg_retrieve(remote)
+            rc = rc and self.system.cp(self.system.path.join(target, remainder), local,
+                                       fmt=options.get('fmt'),
+                                       intent=options.get('intent',
+                                                          ARCHIVE_GET_INTENT_DEFAULT))
+            return rc
+        else:
+            return self._vortexget(remote, local, options)
+
+    def _vortexget(self, remote, local, options):
+        """Remap and ftpget sequence."""
+        remote = self.remap_read(remote, options)
+        return self.inarchiveget(remote, local, options)
+
+    def vortexearlyget(self, remote, local, options):
+        """Vortex' archive earlyget sequence."""
+        if self.stackedstore:
+            s_remote, s_remotefmt, _ = self._stacked_xegglocate(remote)
+            targetopts = dict(fmt=s_remotefmt, intent=dataflow.intent.IN)
+            return self._vortexearlyget(s_remote, 'somelocalfile', targetopts)
+        else:
+            return self._vortexearlyget(remote, local, options)
+
+    def _vortexearlyget(self, remote, local, options):
+        """Remap and ftpget sequence."""
+        remote = self.remap_read(remote, options)
+        return self.inarchiveearlyget(remote, local, options)
+
+    def vortexfinaliseget(self, result_id, remote, local, options):
+        """Vortex' archive finaliseget sequence."""
+        if self.stackedstore:
+            rc, target, remainder = self._vortex_stacked_egg_retrieve(remote, result_id=result_id)
+            rc = rc and self.system.cp(self.system.path.join(target, remainder), local,
+                                       fmt=options.get('fmt'),
+                                       intent=options.get('intent',
+                                                          ARCHIVE_GET_INTENT_DEFAULT))
+            return rc
+        else:
+            return self._vortexfinaliseget(result_id, remote, local, options)
+
+    def _vortexfinaliseget(self, result_id, remote, local, options):
+        """Remap and ftpget sequence."""
+        remote = self.remap_read(remote, options)
+        return self.inarchivefinaliseget(result_id, remote, local, options)
+
+    def vortexput(self, local, remote, options):
+        """Remap root dir and ftpput sequence."""
+        if self.stackedstore:
+            raise RuntimeError("stacked archive stores are never writable.")
+        if not self.storetrue:
+            logger.info("put deactivated for %s", str(local))
+            return True
+        remote = self.remap_write(remote, options)
+        return self.inarchiveput(local, remote, options)
+
+    def vortexdelete(self, remote, options):
+        """Remap root dir and ftpdelete sequence."""
+        if self.stackedstore:
+            raise RuntimeError("stacked archive stores are never writable.")
+        remote = self.remap_write(remote, options)
+        return self.inarchivedelete(remote, options)
+
+
+class VortexStdBaseArchiveStore(_VortexBaseArchiveStore):
+    """Archive for casual VORTEX experiments: Support for legacy/Olive XPIDs.
+
+    This 'archive-legacy' store looks into the resource 'main' location not
+    into a potential stack.
+    """
+
+    _footprint = dict(
+        info = 'VORTEX archive access for casual experiments',
+        attr = dict(
+            netloc = dict(
+                values   = ['vortex.archive-legacy.fr'],
+            ),
+        )
+    )
+
+    @property
+    def _actual_mappingroot(self):
+        """Read the get entry point form configuration."""
+        return config.from_config(
+            section="storage", key="rootdir",
+        )
+
+    def remap_read(self, remote, options):
+        """Reformulates the remote path to compatible vortex namespace."""
+        remote = copy.copy(remote)
+        xpath = remote['path'].split('/')
+        actual_mappingroot = self._actual_mappingroot
+        if not self.storeroot and actual_mappingroot:
+            remote['root'] = actual_mappingroot
+            xpath[3:4] = list(xpath[3])
+        remote['path'] = self.system.path.join(*xpath)
+        return remote
+
+
+class VortexStdStackedArchiveStore(VortexStdBaseArchiveStore):
+    """Archive for casual VORTEX experiments: Support for legacy/Olive XPIDs.
+
+    This 'stacked-archive-legacy' or 'stacked-archive-smart' store looks into
+    the stack associated to the resource. The '-smart' variant, has the ability
+    to refill the whole stack into local cache (to be faster in the future).
+    """
+
+    _footprint = [
+        _vortex_readonly_store,
+        dict(
+            attr = dict(
+                netloc = dict(
+                    values   = ['vortex.stacked-archive-legacy.fr',
+                                'vortex.stacked-archive-smart.fr'],
+                ),
+            )
+        )
+    ]
+
+
+class VortexFreeStdBaseArchiveStore(_VortexBaseArchiveStore, ConfigurableArchiveStore):
+    """Archive for casual VORTEX experiments: Support for Free XPIDs.
+
+    This 'archive-legacy' store looks into the resource 'main' location not
+    into a potential stack.
+    """
+
+    #: Path to the vortex-free Store configuration file
+    _store_global_config = '@store-vortex-free.ini'
+    _datastore_id = 'store-vortex-free-conf'
+
+    _footprint = dict(
+        info = 'VORTEX archive access for casual experiments',
+        attr = dict(
+            netloc = dict(
+                values   = ['vortex-free.archive-legacy.fr'],
+            ),
+        )
+    )
+
+    def remap_read(self, remote, options):
+        """Reformulates the remote path to compatible vortex namespace."""
+        remote = copy.copy(remote)
+        xpath = remote['path'].strip('/').split('/')
+        f_xpid = FreeXPid(xpath[2])
+        xpath[2] = f_xpid.id
+        if 'root' not in remote:
+            remote['root'] = self._actual_storeroot(f_xpid)
+        remote['path'] = self.system.path.join(*xpath)
+        return remote
+
+    remap_write = remap_read
+
+
+class VortexFreeStdStackedArchiveStore(VortexFreeStdBaseArchiveStore):
+    """Archive for casual VORTEX experiments: Support for Free XPIDs.
+
+    This 'stacked-archive-legacy' or 'stacked-archive-smart' store looks into
+    the stack associated to the resource. The '-smart' variant, has the ability
+    to refill the whole stack into local cache (to be faster in the future).
+    """
+
+    _footprint = [
+        _vortex_readonly_store,
+        dict(
+            attr = dict(
+                netloc = dict(
+                    values   = ['vortex-free.stacked-archive-legacy.fr',
+                                'vortex-free.stacked-archive-smart.fr'],
+                ),
+            )
+        )]
+
+
+class VortexOpBaseArchiveStore(_VortexBaseArchiveStore):
+    """Archive for op VORTEX experiments.
+
+    This 'archive-legacy' store looks into the resource 'main' location not
+    into a potential stack.
+    """
+
+    _footprint = dict(
+        info = 'VORTEX archive access for op experiments',
+        attr = dict(
+            netloc = dict(
+                values   = ['vsop.archive-legacy.fr'],
+            ),
+            storetrue = dict(
+                default = DelayedEnvValue('op_archive', True),
+            ),
+        )
+    )
+
+    @property
+    def _actual_storeroot(self):
+        return (
+            self.storeroot or
+            config.from_config(
+                section="storage", key="op_rootdir",
+            )
+        )
+
+    def remap_read(self, remote, options):
+        """Reformulates the remote path to compatible vortex namespace."""
+        remote = copy.copy(remote)
+        xpath = remote['path'].split('/')
+        remote['root'] = self._actual_storeroot
+        if len(xpath) >= 5 and re.match(r'^\d{8}T\d{2,4}', xpath[4]):
+            # If a date is detected
+            vxdate = list(xpath[4])
+            vxdate.insert(4, '/')
+            vxdate.insert(7, '/')
+            vxdate.insert(10, '/')
+            xpath[4] = ''.join(vxdate)
+        remote['path'] = self.system.path.join(*xpath)
+        return remote
+
+    remap_write = remap_read
+
+
+class VortexOpStackedArchiveStore(VortexOpBaseArchiveStore):
+    """Archive for op VORTEX experiments.
+
+    This 'stacked-archive-legacy' or 'stacked-archive-smart' store looks into
+    the stack associated to the resource. The '-smart' variant, has the ability
+    to refill the whole stack into local cache (to be faster in the future).
+    """
+
+    _footprint = [
+        _vortex_readonly_store,
+        dict(
+            attr = dict(
+                netloc = dict(
+                    values   = ['vsop.stacked-archive-legacy.fr',
+                                'vsop.stacked-archive-smart.fr'],
+                ),
+            )
+        )]
+
+
+class VortexArchiveStore(MultiStore):
+    """Archive store for any Vortex experiments.
+
+    Depending on the netloc, legacy/Olive XPIDs ('vortex'), free XPIDs
+    ('vortex-free') or operational experiments ('vsop') will be dealt with.
+
+    First, this multi store will look onto the resource 'main' location. In a
+    second phase, if sensible, il will also dig into the stack associated with
+    the resource.
+    """
+
+    _footprint = dict(
+        info = 'VORTEX archive access',
+        attr = dict(
+            scheme = dict(
+                values  = ['vortex'],
+            ),
+            netloc = dict(
+                values  = ['vortex.archive.fr', 'vortex-free.archive.fr', 'vsop.archive.fr'],
+            ),
+            refillstore = dict(
+                default = False,
+            ),
+            storehead = dict(
+                optional = True,
+            ),
+            storesync = dict(
+                alias    = ('archsync', 'synchro'),
+                type     = bool,
+                optional = True,
+            ),
+        )
+    )
+
+    def filtered_readable_openedstores(self, remote):
+        """Only use the stacked store if sensible."""
+        ostores = [self.openedstores[0], ]
+        ostores.extend([sto for sto in self.openedstores[1:]
+                        if not sto.stackedstore or 'stackpath' in remote['query']
+                        ])
+        return ostores
+
+    def alternates_netloc(self):
+        """Return netlocs describing both base and stacked archives."""
+        netloc_m = re.match(r'(?P<base>v.*)\.archive\.(?P<country>\w+)', self.netloc)
+        return [
+            '{base:s}.archive-legacy.{country:s}'.format(** netloc_m.groupdict()),
+            '{base:s}.stacked-archive-legacy.{country:s}'.format(** netloc_m.groupdict()),
+        ]
+
+    def alternates_fpextras(self):
+        """Deal with some ArchiveStores' specific attributes."""
+        return dict(storehead=self.storehead, storesync=self.storesync)
+
+
+class _VortexCacheBaseStore(CacheStore, _VortexStackedStorageMixin):
+    """Some kind of cache for VORTEX experiments: one still needs to choose the cache strategy."""
+
+    _abstract = True
+    _footprint = dict(
+        info = 'VORTEX cache access',
+        attr = dict(
+            scheme = dict(
+                values  = ['vortex'],
+            ),
+            headdir = dict(
+                default = "",
+                outcast = ['xp', ],
+            ),
+            rtouch = dict(
+                default = True,
+            ),
+            rtouchskip = dict(
+                default = 3,
+            ),
+        )
+    )
+
+    def __init__(self, *args, **kw):
+        logger.debug('Vortex cache store init %s', self.__class__)
+        del self.cache
+        super().__init__(*args, **kw)
+
+    def vortexcheck(self, remote, options):
+        """Proxy to :meth:`incachecheck`."""
+        return self.incachecheck(self._stacked_xremote(remote), options)
+
+    def vortexlocate(self, remote, options):
+        """Proxy to :meth:`incachelocate`."""
+        return self.incachelocate(self._stacked_xremote(remote), options)
+
+    def vortexlist(self, remote, options):
+        """Proxy to :meth:`incachelocate`."""
+        return self.incachelist(remote, options)
+
+    def vortexprestageinfo(self, remote, options):
+        """Proxy to :meth:`incacheprestageinfo`."""
+        return self.incacheprestageinfo(self._stacked_xremote(remote), options)
+
+    def vortexget(self, remote, local, options):
+        """Proxy to :meth:`incacheget`."""
+        return self.incacheget(self._stacked_xremote(remote), local, options)
+
+    def vortexput(self, local, remote, options):
+        """Proxy to :meth:`incacheput`."""
+        return self.incacheput(local, self._stacked_xremote(remote), options)
+
+    def vortexdelete(self, remote, options):
+        """Proxy to :meth:`incachedelete`."""
+        return self.incachedelete(self._stacked_xremote(remote), options)
+
+
+class VortexCacheMtStore(_VortexCacheBaseStore):
+    """Some kind of MTOOL cache for VORTEX experiments."""
+
+    _footprint = dict(
+        info = 'VORTEX MTOOL like Cache access',
+        attr = dict(
+            netloc = dict(
+                values  = ['{:s}.{:s}cache-mt.fr'.format(v, s)
+                           for v in ('vortex', 'vortex-free', 'vsop') for s in ('', 'stacked-')]
+            ),
+            strategy = dict(
+                default = 'mtool',
+            ),
+        )
+    )
+
+
+# TODO Not sure this class is needed anymore
+class VortexCacheOp2ResearchStore(_VortexCacheBaseStore):
+    """The DSI/OP VORTEX cache where researchers can get the freshest data."""
+
+    _footprint = dict(
+        info = 'VORTEX Mtool cache access',
+        attr = dict(
+            netloc = dict(
+                values  = [
+                    'vsop.{:s}cache-op2r.fr'.format(s)
+                    for s in ('', 'stacked-')
+                ],
+            ),
+            strategy = dict(
+                default = 'op2r',
+            ),
+            readonly = dict(
+                default = True,
+            )
+        )
+    )
+
+    @property
+    def underlying_cache_kind(self):
+        """The kind of cache that will be used."""
+        return self.strategy
+
+
+class _AbstractVortexCacheMultiStore(MultiStore):
+    """Any Cache based Vortex multi store."""
+
+    _abstract = True
+    _footprint = dict(
+        info = 'VORTEX cache access',
+        attr = dict(
+            scheme = dict(
+                values  = ['vortex'],
+            ),
+            refillstore = dict(
+                default = False,
+            )
+        )
+    )
+
+    def filtered_readable_openedstores(self, remote):
+        """Deals with stacked stores that are not always active."""
+        ostores = [self.openedstores[0], ]
+        # TODO is the call to cache.allow_reads still required without
+        # marketplace stores?
+        ostores.extend([sto for sto in self.openedstores[1:]
+                        if ((not sto.stackedstore or 'stackpath' in remote['query']) and
+                            sto.cache.allow_reads(remote['path']))
+                        ])
+        return ostores
+
+    def filtered_writeable_openedstores(self, remote):
+        """never writes into stack stores."""
+        ostores = [self.openedstores[0], ]
+        ostores.extend([sto for sto in self.openedstores[1:]
+                        if not sto.stackedstore and sto.cache.allow_writes(remote['path'])])
+        return ostores
+
+
+class VortexCacheStore(_AbstractVortexCacheMultiStore):
+    """The go to store for data cached by VORTEX R&D experiments."""
+
+    _footprint = dict(
+        attr = dict(
+            netloc = dict(
+                values  = ['vortex.cache.fr', 'vortex-free.cache.fr', ],
+            ),
+        )
+    )
+
+    def alternates_netloc(self):
+        """For Non-Op users, Op caches may be accessed in read-only mode."""
+        netloc_m = re.match(r'(?P<base>vortex.*)\.cache\.(?P<country>\w+)', self.netloc)
+        mt_netloc = '{base:s}.cache-mt.{country:s}'.format(** netloc_m.groupdict())
+        s_mt_netloc = '{base:s}.stacked-cache-mt.{country:s}'.format(** netloc_m.groupdict())
+        return [mt_netloc, s_mt_netloc]
+
+
+class VortexVsopCacheStore(_AbstractVortexCacheMultiStore):
+    """The go to store for data cached by VORTEX operational experiments.
+
+    It behaves differently depending on the profile of the user running the
+    code 'see the **glovekind** attribute).
+    """
+
+    _footprint = dict(
+        info = 'VORTEX vsop magic cache access',
+        attr = dict(
+            netloc = dict(
+                values  = ['vsop.cache.fr', ],
+            ),
+            glovekind = dict(
+                optional = True,
+                default = '[glove::realkind]',
+            ),
+        )
+    )
+
+    def alternates_netloc(self):
+        """For Non-Op users, Op caches may be accessed in read-only mode."""
+        todo = [
+            'vsop.cache-mt.fr',
+            'vsop.stacked-cache-mt.fr',
+        ]
+
+        # Only set up op2r cache if the associated filepath
+        # is configured
+        if (
+                (self.glovekind != 'opuser') and
+                config.is_defined(
+                    section="data-tree", key="op_rootdir",
+                )
+        ):
+            todo += [
+                'vsop.cache-op2r.fr', 'vsop.stacked-cache-op2r.fr',
+            ]
+        return todo
+
+
+class _AbstractVortexStackMultiStore(MultiStore):
+    """Any Cache based Vortex multi store."""
+
+    _abstract = True
+    _footprint = dict(
+        info = 'VORTEX stack access',
+        attr = dict(
+            scheme = dict(
+                values  = ['vortex'],
+            ),
+            refillstore = dict(
+                default = False,
+            )
+        )
+    )
+
+    # TODO is this still needed without marketplace stores?
+    def filtered_readable_openedstores(self, remote):
+        """Deals with marketplace stores that are not always active."""
+        ostores = [self.openedstores[0], ]
+        ostores.extend([sto for sto in self.openedstores[1:]
+                        if sto.cache.allow_reads(remote['path'])])
+        return ostores
+
+    def filtered_writeable_openedstores(self, remote):
+        """Deals with marketplace stores that are not always active."""
+        ostores = [self.openedstores[0], ]
+        ostores.extend([sto for sto in self.openedstores[1:]
+                        if sto.cache.allow_writes(remote['path'])])
+        return ostores
+
+
+class VortexStackStore(_AbstractVortexStackMultiStore):
+    """Store intended to read and write data into VORTEX R&D stacks."""
+
+    _footprint = dict(
+        info = 'VORTEX stack access',
+        attr = dict(
+            netloc = dict(
+                values  = ['vortex.stack.fr', 'vortex-free.stack.fr'],
+            ),
+        )
+    )
+
+    def alternates_netloc(self):
+        """Go through the various stacked stores."""
+        netloc_m = re.match(r'(?P<base>vortex.*)\.stack\.(?P<country>\w+)', self.netloc)
+        s_mt_netloc = '{base:s}.stacked-cache-mt.{country:s}'.format(** netloc_m.groupdict())
+        return [s_mt_netloc]
+
+
+class VortexVsopStackStore(_AbstractVortexStackMultiStore):
+    """Store intended to read and write data into VORTEX R&D stacks."""
+
+    _footprint = dict(
+        info = 'VORTEX stack access',
+        attr = dict(
+            netloc = dict(
+                values  = ['vsop.stack.fr'],
+            ),
+            glovekind = dict(
+                optional = True,
+                default = '[glove::realkind]',
+            ),
+        )
+    )
+
+    def alternates_netloc(self):
+        """For Non-Op users, Op caches may be accessed in read-only mode."""
+        todo = ['vsop.stacked-cache-mt.fr', ]
+        if self.glovekind != 'opuser':
+            todo.append("vsop.stacked-cache-op2r.fr")
+        return todo
+
+
+class VortexStoreLegacy(MultiStore):
+    """Combined cache and archive legacy VORTEX stores.
+
+    By '-legacy' we mean that stack resources are ignored.
+    """
+
+    _footprint = dict(
+        info='VORTEX multi access',
+        attr=dict(
+            scheme=dict(
+                values=['vortex'],
+            ),
+            netloc=dict(
+                values=['vortex.multi-legacy.fr', 'vortex-free.multi-legacy.fr', 'vsop.multi-legacy.fr'],
+            ),
+            refillstore=dict(
+                default=True,
+            )
+        )
+    )
+
+    def alternates_netloc(self):
+        """Tuple of alternates domains names, e.g. ``cache`` and ``archive``."""
+        return [self.netloc.firstname + d for d in ('.cache.fr',
+                                                    '.archive-legacy.fr')]
+
+
+class VortexStore(MultiStore):
+    """Combined cache and archive VORTEX stores.
+
+    If sensible, stack will be explored and might be refilled into cache.
+    """
+
+    _footprint = dict(
+        info = 'VORTEX multi access',
+        attr = dict(
+            scheme = dict(
+                values  = ['vortex'],
+            ),
+            netloc = dict(
+                values  = ['vortex.multi.fr', 'vortex-free.multi.fr', 'vsop.multi.fr'],
+            ),
+            refillstore = dict(
+                default = False
+            )
+        )
+    )
+
+    def filtered_readable_openedstores(self, remote):
+        """Deals with stacked stores that are not always active."""
+        ostores = [self.openedstores[0], ]
+        ostores.extend([sto for sto in self.openedstores[1:]
+                        if not sto.stackedstore or 'stackpath' in remote['query']
+                        ])
+        return ostores
+
+    def alternates_netloc(self):
+        """Tuple of alternates domains names, e.g. ``cache`` and ``archive``."""
+        return [self.netloc.firstname + d for d in ('.multi-legacy.fr',
+                                                    '.stacked-archive-smart.fr',)]
+
+
+class PromiseCacheStore(VortexCacheMtStore):
+    """Some kind of vortex cache for EXPECTED resources."""
+
+    _footprint = dict(
+        info = 'EXPECTED cache access',
+        attr = dict(
+            netloc = dict(
+                values  = ['promise.cache.fr'],
+            ),
+            headdir = dict(
+                default = 'promise',
+                outcast = ['xp', 'vortex'],
+            ),
+        )
+    )
+
+    @staticmethod
+    def _add_default_options(options):
+        options_upd = options.copy()
+        options_upd['fmt'] = 'ascii'  # Promises are always JSON files
+        options_upd['intent'] = 'in'  # Promises are always read-only
+        return options_upd
+
+    def vortexget(self, remote, local, options):
+        """Proxy to :meth:`incacheget`."""
+        return super().vortexget(remote, local, self._add_default_options(options))
+
+    def vortexput(self, local, remote, options):
+        """Proxy to :meth:`incacheput`."""
+        return super().vortexput(local, remote, self._add_default_options(options))
+
+    def vortexdelete(self, remote, options):
+        """Proxy to :meth:`incachedelete`."""
+        return super().vortexdelete(remote, self._add_default_options(options))
+
+
+class VortexPromiseStore(PromiseStore):
+    """Combine a Promise Store for expected resources and any VORTEX Store."""
+
+    _footprint = dict(
+        info = 'VORTEX promise store',
+        attr = dict(
+            scheme = dict(
+                values  = ['xvortex'],
+            ),
+            netloc=dict(
+                outcast = ['vortex-demo.cache.fr', 'vortex-demo.multi.fr',
+                           'vortex.testcache.fr', 'vortex.testmulti.fr'],
+            ),
+        )
+    )
+
+
+# Activate the footprint's fasttrack on the stores collector
+fcollect = footprints.collectors.get(tag='store')
+fcollect.fasttrack = ('netloc', 'scheme')
+del fcollect