vortex-nwp 2.0.0b1__py3-none-any.whl

vortex/toolbox.py

@@ -0,0 +1,982 @@
+"""
+Top level interface for accessing the VORTEX facilities.
+
+This module does not provides any class, constant, or any nice object.
+It defines a very basic interface to some (possibly) powerful capacities
+of the :mod:`vortex` toolbox.
+"""
+
+from contextlib import contextmanager
+import re
+import traceback
+
+from bronx.fancies import loggers
+from bronx.stdtypes.history import History
+from bronx.syntax import mktuple
+
+import footprints
+
+from vortex import sessions, data, proxy, VortexForceComplete
+from vortex.layout.dataflow import stripargs_section, intent, ixo, Section
+
+#: Automatic export of superstar interface.
+__all__ = ['rload', 'rget', 'rput']
+
+logger = loggers.getLogger(__name__)
+
+#: Shortcut to footprint env defaults
+defaults = footprints.setup.defaults
+
+sectionmap = {'input': 'get', 'output': 'put', 'executable': 'get'}
+
+
+# Toolbox defaults
+
+#: Default value for the **now** attribute of :func:`input`, :func:`executable`
+#: and :func:`output` functions
+active_now = False
+#: Default value for the **insitu** attribute of the :func:`input` and
+#: :func:`executable` functions
+active_insitu = False
+#: If *False*, drastically reduces the amount of messages printed by the
+#: toolbox module
+active_verbose = True
+#: If *False*, do not try to create/make any promise
+active_promise = True
+#: If *False*, this makes the :func:`clear_promises` function inactive
+active_clear = False
+#: If *False*, this will reset to *False* any ``metadatacheck`` attribute
+#: passed to the :func:`input` or :func:`executable` functions
+active_metadatacheck = True
+#: If *True*, archive stores will not be used at all (only cache stores will
+#: be used)
+active_incache = False
+#: Use the earlyget feature during :func:`input` calls
+active_batchinputs = True
+
+#: History recording
+history = History(tag='rload')
+
+
+# Most commonly used functions
+
+def show_toolbox_settings(ljust=24):
+    """Print the current settings of the toolbox."""
+    for key in ['active_{}'.format(act) for act in
+                ('now', 'insitu', 'verbose', 'promise', 'clear',
+                 'metadatacheck', 'incache')]:
+        kval = globals().get(key, None)
+        if kval is not None:
+            print('+', key.ljust(ljust), '=', kval)
+
+
+def quickview(args, nb=0, indent=0):
+    """Recursive call to any quick view of objects specified as arguments."""
+    if not isinstance(args, list) and not isinstance(args, tuple):
+        args = (args, )
+    for x in args:
+        if nb:
+            print()
+        nb += 1
+        quickview = getattr(x, 'quickview', None)
+        if quickview:
+            quickview(nb, indent)
+        else:
+            print('{:02d}. {:s}'.format(nb, x))
+
+
+class VortexToolboxDescError(Exception):
+    pass
+
+
+def rload(*args, **kw):
+    """
+    Resource Loader.
+
+    This function behaves as a factory for any possible pre-defined family
+    of VORTEX resources (described by an aggregation of Resource, Provider and
+    Container objects).
+
+    Arguments could be a mix of a list of dictionary-type objects and key/value
+    parameters. Other type of arguments will be discarded.
+
+    An abstract resource descriptor is built as the aggregation of these
+    arguments and then expanded according to rules defined by the
+    :func:`footprints.util.expand` function.
+
+    For each expanded descriptor, the ``rload`` method will try to pickup the
+    best candidates (if any) that could match the description (*i.e.* Resource,
+    Provider, Container). If no match is found for one of the Resource, Provider
+    or Container objects, a :class:`VortexToolboxDescError` exception is raised.
+    Otherwise,the resource's :class:`~vortex.data.handlers.Handler` built from
+    those three objects is added to the result's list.
+
+    :return: A list of :class:`vortex.data.handlers.Handler` objects.
+    """
+    rd = dict()
+    for a in args:
+        if isinstance(a, dict):
+            rd.update(a)
+        else:
+            logger.warning('Discard rload argument <%s>', a)
+    rd.update(kw)
+    if rd:
+        history.append(rd.copy())
+    rhx = []
+    for x in footprints.util.expand(rd):
+        picked_up = proxy.containers.pickup(  # @UndefinedVariable
+            * proxy.providers.pickup_and_cache(  # @UndefinedVariable
+                * proxy.resources.pickup_and_cache(x)  # @UndefinedVariable
+            )
+        )
+        logger.debug('Resource desc %s', picked_up)
+        picked_rh = data.handlers.Handler(picked_up)
+        if not picked_rh.complete:
+            raise VortexToolboxDescError("The ResourceHandler is incomplete")
+        rhx.append(picked_rh)
+
+    return rhx
+
+
+def rh(*args, **kw):
+    """
+    This function selects the first resource's handler as returned by the
+    :func:`rload` function.
+    """
+    return rload(*args, **kw)[0]
+
+
+def rget(*args, **kw):
+    """
+    This function calls the :meth:`get` method on any resource handler returned
+    by the :func:`rload` function.
+    """
+    loc_incache = kw.pop('incache', active_incache)
+    rl = rload(*args, **kw)
+    for rh in rl:
+        rh.get(incache=loc_incache)
+    return rl
+
+
+def rput(*args, **kw):
+    """
+    This function calls the :meth:`put` method on any resource handler returned
+    by the :func:`rload` function.
+    """
+    loc_incache = kw.pop('incache', active_incache)
+    rl = rload(*args, **kw)
+    for rh in rl:
+        rh.put(incache=loc_incache)
+    return rl
+
+
+def nicedump(msg, **kw):
+    """Simple dump the **kw** dict content with ``msg`` as header."""
+    print('#', msg, ':')
+    for k, v in sorted(kw.items()):
+        print('+', k.ljust(12), '=', str(v))
+    print()
+
+
+@contextmanager
+def _tb_isolate(t, loglevel):
+    """Handle the context and logger (internal use only)."""
+    # Switch off autorecording of the current context
+    ctx = t.context
+    recordswitch = ctx.record
+    if recordswitch:
+        ctx.record_off()
+    try:
+        if loglevel is not None:
+            # Possibly change the log level if necessary
+            with loggers.contextboundGlobalLevel(loglevel):
+                yield
+        else:
+            yield
+    finally:
+        if recordswitch:
+            ctx.record_on()
+
+
+def add_section(section, args, kw):
+    """
+    Add a :class:`~vortex.layout.dataflow.Section` object (of kind **section**)
+    to the current sequence.
+
+    1. The **kw** dictionary may contain keys that influence this function
+       behaviour (such attributes are popped from **kw** before going further):
+
+        * **now**: If *True*, call the appropriate action (``get()`` or ``put()``)
+          on each added :class:`~vortex.layout.dataflow.Section`. (The default
+          is given by :data:`active_now`).
+        * **loglevel**: The logging facility verbosity level that will be used
+          during the :class:`~vortex.layout.dataflow.Section` creation process.
+          If *None*, nothing is done (i.e. the current verbosity level is
+          preserved). (default: *None*).
+        * **verbose**: If *True*, print some informations on the standard output
+          (The default is given by :data:`active_verbose`).
+        * **complete**: If *True*, force the task to complete (the
+          :class:`VortexForceComplete` exception is raised) whenever an error
+          occurs. (default: False).
+        * **insitu**: It *True*, before actually getting data, we first check
+          if the data is already there (in the context of a multi-step job,
+          it might have been fetched during a previous step). (default: *False*).
+        * **incache**: It *True*, archive stores will not be used at all (only cache
+          stores will be used). (The default is given by :data:`active_incache`).
+
+    2. **kw** is then looked for items relevant to the
+       :class:`~vortex.layout.dataflow.Section` constructor (``role``, ``intent``,
+       ...). (such items are popped from kw before going further).
+
+    3. The remaining **kw** items are passed directly to the :func:`rload`
+       function in order to create the resource's
+       :class:`~vortex.data.handlers.Handler`.
+
+    :return: A list of :class:`vortex.data.handlers.Handler` objects.
+    """
+
+    t = sessions.current()
+
+    # First, retrieve arguments of the toolbox command itself
+    now = kw.pop('now', active_now)
+    loglevel = kw.pop('loglevel', None)
+    talkative = kw.pop('verbose', active_verbose)
+    complete = kw.pop('complete', False)
+    insitu = kw.get('insitu', False)
+    batch = kw.pop('batch', False)
+    lastfatal = kw.pop('lastfatal', None)
+
+    if complete:
+        kw['fatal'] = False
+
+    if batch:
+        if section not in ('input', 'excutable'):
+            logger.info("batch=True is not implemented for section=%s. overwriting to batch=Fase.",
+                        section)
+            batch = False
+
+    # Second, retrieve arguments that could be used by the now command
+    cmdopts = dict(
+        incache=kw.pop('incache', active_incache),
+        force=kw.pop('force', False)
+    )
+
+    # Third, collect arguments for triggering some hook
+    hooks = dict()
+    for ahook in [x for x in kw.keys() if x.startswith('hook_')]:
+        cbhook = mktuple(kw.pop(ahook))
+        cbfunc = cbhook[0]
+        if not callable(cbfunc):
+            cbfunc = t.sh.import_function(cbfunc)
+        hooks[ahook] = footprints.FPTuple((cbfunc, cbhook[1:]))
+
+    # Print the user inputs
+    def print_user_inputs():
+        nicedump('New {:s} section with options'.format(section), **opts)
+        nicedump('Resource handler description', **kwclean)
+        nicedump(
+            'This command options',
+            complete=complete,
+            loglevel=loglevel,
+            now=now,
+            verbose=talkative,
+        )
+        if hooks:
+            nicedump('Hooks triggered', **hooks)
+
+    with _tb_isolate(t, loglevel):
+
+        # Distinguish between section arguments, and resource loader arguments
+        opts, kwclean = stripargs_section(**kw)
+
+        # Strip the metadatacheck option depending on active_metadatacheck
+        if not active_metadatacheck and not insitu:
+            if kwclean.get('metadatacheck', False):
+                logger.info("The metadatacheck option is forced to False since " +
+                            "active_metadatacheck=False.")
+                kwclean['metadatacheck'] = False
+
+        # Show the actual set of arguments
+        if talkative and not insitu:
+            print_user_inputs()
+
+        # Let the magic of footprints resolution operate...
+        kwclean.update(hooks)
+        rl = rload(*args, **kwclean)
+        rlok = list()
+
+        # Prepare the references to the actual section method to perform
+        push = getattr(t.context.sequence, section)
+        doitmethod = sectionmap[section]
+
+        # Create a section for each resource handler
+        if rl and lastfatal is not None:
+            newsections = [push(rh=rhandler, **opts)[0] for rhandler in rl[:-1]]
+            tmpopts = opts.copy()
+            tmpopts['fatal'] = lastfatal
+            newsections.append(push(rh=rl[-1], **tmpopts)[0])
+        else:
+            newsections = [push(rh=rhandler, **opts)[0] for rhandler in rl]
+
+        # If insitu and now, try a quiet get...
+        do_quick_insitu = section in ('input', 'executable') and insitu and now
+        if do_quick_insitu:
+            quickget = [sec.rh.insitu_quickget(alternate=sec.alternate, **cmdopts) for sec in newsections]
+            if all(quickget):
+                if len(quickget) > 1:
+                    logger.info("The insitu get succeeded for all of the %d resource handlers.",
+                                len(rl))
+                else:
+                    logger.info("The insitu get succeeded for this resource handler.")
+                rlok = [sec.rh for sec in newsections]
+            else:
+                # Start again with the usual get sequence
+                print_user_inputs()
+
+        # If not insitu, not now, or if the quiet get failed
+        if not (do_quick_insitu and all(quickget)):
+            if now:
+                with t.sh.ftppool():
+                    # Create a section for each resource handler, and perform action on demand
+                    batchflags = [None, ] * len(newsections)
+                    if batch:
+                        if talkative:
+                            t.sh.subtitle('Early-{:s} for all resources.'.format(doitmethod))
+                        for ir, newsection in enumerate(newsections):
+                            rhandler = newsection.rh
+                            batchflags[ir] = getattr(newsection, 'early' + doitmethod)(**cmdopts)
+                        if talkative:
+                            if any(batchflags):
+                                for ir, newsection in enumerate(newsections):
+                                    if talkative and batchflags[ir]:
+                                        logger.info('Resource no %02d/%02d: Early-%s registered with id: %s.',
+                                                    ir + 1, len(rl), doitmethod, str(batchflags[ir]))
+                                    else:
+                                        logger.debug('Resource no %02d/%02d: Early-%s registered with id: %s.',
+                                                     ir + 1, len(rl), doitmethod, str(batchflags[ir]))
+                            else:
+                                logger.info('Early-%s was unavailable for all of the resources.',
+                                            doitmethod)
+                        # trigger finalise for all of the DelayedActions
+                        tofinalise = [r_id for r_id in batchflags if r_id and r_id is not True]
+                        if tofinalise:
+                            if talkative:
+                                t.sh.subtitle('Finalising all of the delayed actions...')
+                            t.context.delayedactions_hub.finalise(* tofinalise)
+                    secok = list()
+                    for ir, newsection in enumerate(newsections):
+                        rhandler = newsection.rh
+                        # If quick get was ok for this resource don't call get again...
+                        if talkative:
+                            t.sh.subtitle('Resource no {:02d}/{:02d}'.format(ir + 1, len(rl)))
+                            rhandler.quickview(nb=ir + 1, indent=0)
+                            if batchflags[ir] is not True or (do_quick_insitu and quickget[ir]):
+                                t.sh.highlight('Action {:s} on {:s}'.format(doitmethod.upper(),
+                                                                            rhandler.location(fatal=False)))
+                        ok = do_quick_insitu and quickget[ir]
+                        if batchflags[ir]:
+                            actual_doitmethod = 'finalise' + doitmethod
+                            ok = ok or getattr(newsection, actual_doitmethod)()
+                        else:
+                            actual_doitmethod = doitmethod
+                            ok = ok or getattr(newsection, actual_doitmethod)(**cmdopts)
+                        if talkative:
+                            t.sh.highlight('Result from {:s}: [{!s}]'.format(actual_doitmethod, ok))
+                        if talkative and not ok:
+                            logger.error('Could not %s resource %s',
+                                         doitmethod, rhandler.container.localpath())
+                        if not ok:
+                            if complete:
+                                logger.warning('Force complete for %s',
+                                               rhandler.location(fatal=False))
+                                raise VortexForceComplete('Force task complete on resource error')
+                        else:
+                            secok.append(newsection)
+                        if t.sh.trace:
+                            print()
+                    rlok.extend([newsection.rh for newsection in secok
+                                 if newsection.any_coherentgroup_opened])
+            else:
+                rlok.extend([newsection.rh for newsection in newsections])
+
+    return rlok
+
+
+# noinspection PyShadowingBuiltins
+def input(*args, **kw):  # @ReservedAssignment
+    """Add input :class:`~vortex.layout.dataflow.Section` objects to the current sequence.
+
+    Relies on the :func:`add_section` function (see its documentation), with:
+
+        * It's ``section`` attribute is automatically set to 'input';
+        * The ``kw``'s *insitu* item is set to :data:`active_insitu` by default.
+
+    :return: A list of :class:`vortex.data.handlers.Handler` objects (associated
+        with the newly created class:`~vortex.layout.dataflow.Section` objects).
+    """
+    kw.setdefault('insitu', active_insitu)
+    kw.setdefault('batch', active_batchinputs)
+    return add_section('input', args, kw)
+
+
+def inputs(ticket=None, context=None):
+    """Return effective inputs for the specified context.
+
+    It actually returns both *inputs* and *executables*.
+
+    :param ~vortex.sessions.Ticket ticket: A session's Ticket. If set to *None*,
+        the current active session will be used (default: *None*)
+    :param ~vortex.layout.contexts.Context context: A context object. If set to *None*,
+        the current active context will be used (default: *None*)
+    :return: A list of :class:`~vortex.layout.dataflow.Section` objects.
+    """
+    if context is None:
+        if ticket is None:
+            ticket = sessions.current()
+        context = ticket.context
+    return context.sequence.effective_inputs()
+
+
+def show_inputs(context=None):
+    """Dump a summary of inputs (+ executables) sections.
+
+    :param ~vortex.layout.contexts.Context context: A context object. If set to *None*,
+        the current active context will be used (default: *None*)
+    """
+    t = sessions.current()
+    for csi in inputs(ticket=t):
+        t.sh.header('Input ' + str(csi))
+        csi.show(ticket=t, context=context)
+        print()
+
+
+def output(*args, **kw):
+    """Add output :class:`~vortex.layout.dataflow.Section` objects to the current sequence.
+
+    Relies on the :func:`add_section` function (see its documentation), with:
+
+        * It's ``section`` attribute is automatically set to 'output';
+
+    :return: A list of :class:`vortex.data.handlers.Handler` objects (associated
+        with the newly created class:`~vortex.layout.dataflow.Section` objects).
+    """
+    # Strip the metadatacheck option depending on active_metadatacheck
+    if not active_promise:
+        for target in ('promised', 'expected'):
+            if target in kw and kw[target]:
+                logger.info("The %s argument is removed since active_promise=False.", target)
+                del kw[target]
+    return add_section('output', args, kw)
+
+
+def outputs(ticket=None, context=None):
+    """Return effective outputs in specified context.
+
+    :param ~vortex.sessions.Ticket ticket: A session's Ticket. If set to *None*,
+        the current active session will be used (default: *None*)
+    :param ~vortex.layout.contexts.Context context: A context object. If set to *None*,
+        the current active context will be used (default: *None*)
+    :return: A list of :class:`~vortex.layout.dataflow.Section` objects.
+    """
+    if context is None:
+        if ticket is None:
+            ticket = sessions.current()
+        context = ticket.context
+    return context.sequence.effective_outputs()
+
+
+def show_outputs(context=None):
+    """Dump a summary of outputs sections.
+
+    :param ~vortex.layout.contexts.Context context: A context object. If set to *None*,
+        the current active context will be used (default: *None*)
+    """
+    t = sessions.current()
+    for cso in outputs(ticket=t):
+        t.sh.header('Output ' + str(cso))
+        cso.show(ticket=t, context=context)
+        print()
+
+
+def promise(*args, **kw):
+    """Log promises before execution.
+
+    Relies on the :func:`add_section` function (see its documentation), with:
+
+        * It's ``section`` attribute is automatically set to 'output';
+        * The ``kw``'s *promised* item is set to *True*;
+        * The ``kw``'s *force* item is set to *True*;
+        * The ``kw``'s *now* item is set to :data:`active_promise`.
+
+    :return: A list of :class:`vortex.data.handlers.Handler` objects (associated
+        with the newly created class:`~vortex.layout.dataflow.Section` objects).
+    """
+    kw.update(
+        promised=True,
+        force=True,
+        now=active_promise,
+    )
+    if not active_promise:
+        kw.setdefault('verbose', False)
+        logger.warning('Promise flag is <%s> in that context', active_promise)
+    return add_section('output', args, kw)
+
+
+def executable(*args, **kw):
+    """Add executable :class:`~vortex.layout.dataflow.Section` objects to the current sequence.
+
+    Relies on the :func:`add_section` function (see its documentation), with:
+
+        * It's ``section`` attribute is automatically set to 'executable';
+        * The ``kw``'s *insitu* item is set to :data:`active_insitu` by default.
+
+    :return: A list of :class:`vortex.data.handlers.Handler` objects (associated
+        with the newly created class:`~vortex.layout.dataflow.Section` objects).
+    """
+    kw.setdefault('insitu', active_insitu)
+    return add_section('executable', args, kw)
+
+
+def algo(*args, **kw):
+    """Load an algo component and display its description (if **verbose**).
+
+    1. The **kw** dictionary may contain keys that influence this function
+       behaviour (such attributes are popped from **kw** before going further):
+
+        * **loglevel**: The logging facility verbosity level that will be used
+          during the :class:`~vortex.layout.dataflow.Section` creation process.
+          If *None*, nothing is done (i.e. the current verbosity level is
+          preserved). (default: *None*).
+        * **verbose**: If *True*, print some informations on the standard output
+          (The default is given by :data:`active_verbose`).
+
+    2. The remaining **kw** items are passed directly to the "algo" footprint's
+        proxy in order to create the AlgoComponent object.
+
+    :return: an object that is a subtype of :class:`vortex.algo.components.AlgoComponent`
+    """
+
+    t = sessions.current()
+
+    # First, retrieve arguments of the toolbox command itself
+    loglevel = kw.pop('loglevel', None)
+    talkative = kw.pop('verbose', active_verbose)
+
+    with _tb_isolate(t, loglevel):
+
+        if talkative:
+            nicedump('Loading algo component with description:', **kw)
+
+        ok = proxy.component(**kw)  # @UndefinedVariable
+        if ok and talkative:
+            print(t.line)
+            ok.quickview(nb=1, indent=0)
+
+    return ok
+
+
+def diff(*args, **kw):
+    """Perform a diff with a resource with the same local name.
+
+    1. The **kw** dictionary may contain keys that influence this function
+       behaviour (such attributes are popped from **kw** before going further):
+
+        * **fatal**: If *True*, a :class:`ValueError` exception will be raised
+          whenever the "diff" detects differences.
+        * **loglevel**: The logging facility verbosity level that will be used
+          during the :class:`~vortex.layout.dataflow.Section` creation process.
+          If *None*, nothing is done (i.e. the current verbosity level is
+          preserved). (default: *None*).
+        * **verbose**: If *True*, print some informations on the standard output
+          (The default is given by :data:`active_verbose`).
+
+    2. The remaining **kw** items are passed directly to the :func:`rload`
+       function in order to create the resource's
+       :class:`~vortex.data.handlers.Handler` objects for the reference files.
+
+    3. The reference files resource's :class:`~vortex.data.handlers.Handler` objects
+       are altered so that the reference files are stored in temporary Containers.
+
+    4. The reference files are fetched.
+
+    5. The diff between the containers described in the resource's description
+       and the reference files is computed.
+
+    :return: A list of *diff* results.
+    """
+
+    # First, retrieve arguments of the toolbox command itself
+    fatal = kw.pop('fatal', True)
+    loglevel = kw.pop('loglevel', None)
+    talkative = kw.pop('verbose', active_verbose)
+    batch = kw.pop('batch', active_batchinputs)
+
+    # Distinguish between section arguments, and resource loader arguments
+    opts, kwclean = stripargs_section(**kw)
+
+    # Show the actual set of arguments
+    if talkative:
+        nicedump('Discard section options', **opts)
+        nicedump('Resource handler description', **kwclean)
+
+    # Fast exit in case of undefined value
+    rlok = list()
+    none_skip = {k for k, v in kwclean.items()
+                 if v is None and k in ('experiment', 'namespace')}
+    if none_skip:
+        logger.warning('Skip diff because of undefined argument(s)')
+        return rlok
+
+    t = sessions.current()
+
+    # Swich off autorecording of the current context + deal with loggging
+    with _tb_isolate(t, loglevel):
+
+        # Do not track the reference files
+        kwclean['storetrack'] = False
+
+        rhandlers = rload(*args, **kwclean)
+        sections = list()
+        earlyget_id = list()
+        source_container = list()
+        lazzy_container = list()
+
+        if batch:
+            # Early get
+            print(t.line)
+            for ir, rhandler in enumerate(rhandlers):
+                source_container.append(rhandler.container)
+                # Create a new container to hold the reference file
+                lazzy_container.append(footprints.proxy.container(shouldfly=True,
+                                                                  actualfmt=rhandler.container.actualfmt))
+                # Swapp the original container with the lazzy one
+                rhandler.container = lazzy_container[-1]
+                # Create a new section
+                sec = Section(rh=rhandler, kind=ixo.INPUT, intent=intent.IN, fatal=False)
+                sections.append(sec)
+                # Early-get
+                if rhandler.complete:
+                    earlyget_id.append(sec.earlyget())
+                else:
+                    earlyget_id.append(None)
+            # Finalising
+            if any([r_id and r_id is not True for r_id in earlyget_id]):
+                t.sh.highlight('Finalising Early-gets')
+                t.context.delayedactions_hub.finalise(* [r_id for r_id in earlyget_id
+                                                         if r_id and r_id is not True])
+
+        for ir, rhandler in enumerate(rhandlers):
+            if talkative:
+                print(t.line)
+                rhandler.quickview(nb=ir + 1, indent=0)
+                print(t.line)
+            if not rhandler.complete:
+                logger.error('Incomplete Resource Handler for diff [%s]', rhandler)
+                if fatal:
+                    raise ValueError('Incomplete Resource Handler for diff')
+                else:
+                    rlok.append(False)
+                    continue
+
+            # Get the reference file through a Section so that intent + fatal
+            # is properly dealt with... The section is discarded afterwards.
+            if batch:
+                rc = sections[ir].finaliseget()
+            else:
+                rc = sections[ir].get()
+            if not rc:
+                try:
+                    logger.error('Cannot get the reference resource: %s',
+                                 rhandler.locate())
+                except Exception:
+                    logger.error('Cannot get the reference resource: ???')
+                if fatal:
+                    raise ValueError('Cannot get the reference resource')
+            else:
+                logger.info('The reference file is stored under: %s',
+                            rhandler.container.localpath())
+
+            # What are the differences ?
+            if rc:
+                # priority is given to the diff implemented in the DataContent
+                if rhandler.resource.clscontents.is_diffable():
+                    source_contents = rhandler.resource.contents_handler(
+                        datafmt=source_container[ir].actualfmt)
+                    source_contents.slurp(source_container[ir])
+                    ref_contents = rhandler.contents
+                    rc = source_contents.diff(ref_contents)
+                else:
+                    rc = t.sh.diff(source_container[ir].localpath(),
+                                   rhandler.container.localpath(),
+                                   fmt=rhandler.container.actualfmt)
+
+            # Delete the reference file
+            lazzy_container[ir].clear()
+
+            # Now proceed with the result
+            logger.info('Diff return %s', str(rc))
+            t.context.diff_history.append_record(rc, source_container[ir], rhandler)
+            try:
+                logger.info('Diff result %s', str(rc.result))
+            except AttributeError:
+                pass
+            if not rc:
+                try:
+                    logger.warning('Some diff occurred with %s', rhandler.locate())
+                except Exception:
+                    logger.warning('Some diff occurred with ???')
+                try:
+                    rc.result.differences()
+                except Exception:
+                    pass
+                if fatal:
+                    logger.critical('Difference in resource comparison is fatal')
+                    raise ValueError('Fatal diff')
+            if t.sh.trace:
+                print()
+            rlok.append(rc)
+
+    return rlok
+
+
+def magic(localpath, **kw):
+    """
+    Return a minimal resource handler build with an unknown resource,
+    a file container and an anonymous provider described with its URL.
+    """
+    kw.update(
+        unknown=True,
+        magic='magic://localhost/' + localpath,
+        filename=localpath,
+    )
+    rhmagic = rh(**kw)
+    rhmagic.get()
+    return rhmagic
+
+
+def archive_refill(*args, **kw):
+    """
+    Get a ressource in cache and upload it into the archive.
+
+    This will only have effect when working on a multistore.
+
+    The **kw** items are passed directly to the :func:`rload` function in
+    order to create the resource's :class:`~vortex.data.handlers.Handler`.
+    No "container" description is needed. One will be created by default.
+
+    :return: A list of :class:`vortex.data.handlers.Handler` objects.
+    """
+
+    t = sessions.current()
+
+    # First, retrieve arguments of the toolbox command itself
+    loglevel = kw.pop('loglevel', None)
+    talkative = kw.pop('verbose', active_verbose)
+
+    with _tb_isolate(t, loglevel):
+
+        # Distinguish between section arguments, and resource loader arguments
+        opts, kwclean = stripargs_section(**kw)
+        fatal = opts.get('fatal', True)
+
+        # Print the user inputs
+        if talkative:
+            nicedump('Archive Refill Ressource+Provider description', **kwclean)
+
+        # Create the resource handlers
+        kwclean['container'] = footprints.proxy.container(uuid4fly=True,
+                                                          uuid4flydir="archive_refills")
+        rl = rload(*args, **kwclean)
+
+        @contextmanager
+        def _fatal_wrap(action):
+            """Handle errors during the calls to get or put."""
+            wrap_rc = dict(rc=True)
+            try:
+                yield wrap_rc
+            except Exception as e:
+                logger.error('Something wrong (action %s): %s. %s',
+                             action, str(e), traceback.format_exc())
+                wrap_rc['rc'] = False
+                wrap_rc['exc'] = e
+            if fatal and not wrap_rc['rc']:
+                logger.critical('Fatal error with action %s.', action)
+                raise RuntimeError('Could not {:s} resource: {!s}'.format(action,
+                                                                          wrap_rc['rc']))
+
+        with t.sh.ftppool():
+            for ir, rhandler in enumerate(rl):
+                if talkative:
+                    t.sh.subtitle('Resource no {:02d}/{:02d}'.format(ir + 1, len(rl)))
+                    rhandler.quickview(nb=ir + 1, indent=0)
+                if not (rhandler.store.use_cache() and rhandler.store.use_archive()):
+                    logger.info('The requested store does not have both the cache and archive capabilities. ' +
+                                'Skipping this ressource handler.')
+                    continue
+                with _fatal_wrap('get') as get_status:
+                    get_status['rc'] = rhandler.get(incache=True, intent=intent.IN,
+                                                    fmt=rhandler.resource.nativefmt)
+                put_status = dict(rc=False)
+                if get_status['rc']:
+                    with _fatal_wrap('put') as put_status:
+                        put_status['rc'] = rhandler.put(inarchive=True,
+                                                        fmt=rhandler.resource.nativefmt)
+                    rhandler.container.clear()
+                if talkative:
+                    t.sh.highlight('Result from get: [{!s}], from put: [{!s}]'
+                                   .format(get_status['rc'], put_status['rc']))
+
+    return rl
+
+
+def stack_archive_refill(*args, **kw):
+    """Get a stack ressource in cache and upload it into the archive.
+
+    This will only have effect when working on a multistore.
+
+    The **kw** items are passed directly to the :func:`rload` function in
+    order to create the resource's :class:`~vortex.data.handlers.Handler`.
+
+    * No "container" description is needed. One will be created by default.
+    * The **block** attribute will be set automaticaly
+
+    :return: A list of :class:`vortex.data.handlers.Handler` objects.
+    """
+    kw['block'] = 'stacks'
+    return archive_refill(*args, **kw)
+
+
+def namespaces(**kw):
+    """
+    Some kind of interactive help to find out quickly which namespaces are in
+    used. By default tracks ``stores`` and ``providers`` but one could give an
+    ``only`` argument.
+    """
+    rematch = re.compile('|'.join(kw.get('match', '.').split(',')),
+                         re.IGNORECASE)
+    if 'only' in kw:
+        usedcat = kw['only'].split(',')
+    else:
+        usedcat = ('provider', 'store')
+    nameseen = dict()
+    for cat in [footprints.collectors.get(tag=x) for x in usedcat]:
+        for cls in cat():
+            fp = cls.footprint_retrieve().attr
+            netattr = fp.get('namespace', None)
+            if not netattr:
+                netattr = fp.get('netloc', None)
+            if netattr and 'values' in netattr:
+                for netname in filter(lambda x: rematch.search(x), netattr['values']):
+                    if netname not in nameseen:
+                        nameseen[netname] = list()
+                    nameseen[netname].append(cls.fullname())
+    return nameseen
+
+
+def print_namespaces(**kw):
+    """Formatted print of current namespaces."""
+    prefix = kw.pop('prefix', '+ ')
+    nd = namespaces(**kw)
+    justify = max([len(x) for x in nd.keys()])
+    linesep = ",\n" + ' ' * (justify + len(prefix) + 2)
+    for k, v in sorted(nd.items()):
+        nice_v = linesep.join(v) if len(v) > 1 else v[0]
+        print(prefix + k.ljust(justify), '[' + nice_v + ']')
+
+
+def clear_promises(clear=None, netloc='promise.cache.fr', scheme='vortex',
+                   storeoptions=None):
+    """Remove all promises that have been made in the current session.
+
+    :param netloc: Netloc of the promise's cache store to clean up
+    :param scheme: Scheme of the promise's cache store to clean up
+    :param storeoptions: Option dictionary passed to the store (may be None)
+    """
+    if clear is None:
+        clear = active_clear
+    if clear:
+        t = sessions.current()
+        myctx = t.context
+        for ctx in t.subcontexts:
+            ctx.activate()
+            ctx.clear_promises(netloc, scheme, storeoptions)
+        # Switch back to the previous context
+        myctx.activate()
+
+
+def rescue(*files, **opts):
+    """Action to be undertaken when things really went bad."""
+
+    t = sessions.current()
+    sh = t.sh
+    env = t.env
+
+    # Summarise diffs...
+    if len(t.context.diff_history):
+        sh.header('Summary of automatic toolbox diffs')
+        t.context.diff_history.show()
+
+    # Force clearing of all promises
+    clear_promises(clear=True)
+
+    sh.header('Rescuing current dir')
+    sh.dir(output=False, fatal=False)
+
+    logger.info('Rescue files %s', files)
+
+    if 'VORTEX_RESCUE' in env and env.false('VORTEX_RESCUE'):
+        logger.warning('Skip rescue <VORTEX_RESCUE=%s>', env.VORTEX_RESCUE)
+        return False
+
+    if files:
+        items = list(files)
+    else:
+        items = sh.glob('*')
+
+    rfilter = opts.get('filter', env.VORTEX_RESCUE_FILTER)
+    if rfilter is not None:
+        logger.warning('Rescue filter <%s>', rfilter)
+        select = '|'.join(re.split(r'[,;:]+', rfilter))
+        items = [x for x in items if re.search(select, x, re.IGNORECASE)]
+        logger.info('Rescue filter [%s]', select)
+
+    rdiscard = opts.get('discard', env.VORTEX_RESCUE_DISCARD)
+    if rdiscard is not None:
+        logger.warning('Rescue discard <%s>', rdiscard)
+        select = '|'.join(re.split(r'[,;:]+', rdiscard))
+        items = [x for x in items if not re.search(select, x, re.IGNORECASE)]
+        logger.info('Rescue discard [%s]', select)
+
+    if items:
+
+        bkupdir = opts.get('bkupdir', env.VORTEX_RESCUE_PATH)
+
+        if bkupdir is None:
+            logger.error('No rescue directory defined.')
+        else:
+            logger.info('Backup directory defined by user < %s >', bkupdir)
+            items.sort()
+            logger.info('Rescue items %s', str(items))
+            sh.mkdir(bkupdir)
+            mkmove = False
+            st1 = sh.stat(sh.getcwd())
+            st2 = sh.stat(bkupdir)
+            if st1 and st2 and st1.st_dev == st2.st_dev:
+                mkmove = True
+            if mkmove:
+                thisrescue = sh.mv
+            else:
+                thisrescue = sh.cp
+            for ritem in items:
+                rtarget = sh.path.join(bkupdir, ritem)
+                if sh.path.exists(ritem) and not sh.path.islink(ritem):
+                    if sh.path.isfile(ritem):
+                        sh.rm(rtarget)
+                        thisrescue(ritem, rtarget)
+                    else:
+                        thisrescue(ritem, rtarget)
+
+    else:
+        logger.warning('No item to rescue.')
+
+    return bool(items)