vortex-nwp 2.0.0b1__py3-none-any.whl

vortex/data/handlers.py

@@ -0,0 +1,1021 @@
+"""
+:class:`Handler` class is a cornerstone in any Vortex script. :class:`Handler`
+objects are in charge of manipulating data between the working directory and
+the various caches or archives".
+"""
+
+
+import functools
+import re
+import sys
+
+import bronx.fancies.dump
+from bronx.fancies import loggers
+from bronx.patterns import observer
+from bronx.stdtypes.history import History
+
+import footprints
+
+from vortex import sessions
+
+from vortex.tools import net
+from vortex.util import config
+from vortex.layout import contexts, dataflow
+from vortex.data import containers, resources, providers
+
+#: No automatic export
+__all__ = []
+
+logger = loggers.getLogger(__name__)
+
+OBSERVER_TAG = 'Resources-Handlers'
+
+
+class HandlerError(RuntimeError):
+    """Exception in case of missing resource during the wait mechanism."""
+    pass
+
+
+def observer_board(obsname=None):
+    """Proxy to :func:`footprints.observers.get`."""
+    if obsname is None:
+        obsname = OBSERVER_TAG
+    return observer.get(tag=obsname)
+
+
+class IdCardAttrDumper(bronx.fancies.dump.TxtDumper):
+    """Dump a text representation of almost any footprint object..."""
+
+    indent_size = 2
+    max_depth = 2
+
+    def __init__(self):
+        self._indent_first = 4
+
+    def _get_indent_first(self):
+        return self._indent_first
+
+    def _set_indent_first(self, val):
+        self._indent_first = val
+
+    indent_first = property(_get_indent_first, _set_indent_first)
+
+    def dump_fpattrs(self, fpobj, level=0):
+        """Dump the attributes of a footprint based object."""
+        if level + 1 > self.max_depth:
+            return "{}{{...}}{}".format(
+                self._indent(level, self.break_before_dict_begin),
+                self._indent(level, self.break_after_dict_end)
+            )
+        else:
+            items = ["{}{} = {}{},".format(self._indent(level + 1, self.break_before_dict_key),
+                                           str(k),
+                                           self._indent(level + 2, self.break_before_dict_value),
+                                           self._recursive_dump(v, level + 1))
+                     for k, v in sorted(fpobj.footprint_as_shallow_dict().items())]
+            return ' '.join(items)
+
+    def dump_default(self, obj, level=0, nextline=True):
+        """Generic dump function. Concise view for GetByTag objects."""
+        if level + 1 > self.max_depth:
+            return " <%s...>" % type(obj).__class__
+        else:
+            if hasattr(obj, 'tag'):
+                return "{:s} obj: tag={:s}".format(type(obj).__name__, obj.tag)
+            else:
+                parent_dump = super(bronx.fancies.dump.TxtDumper, self).dump_default(obj, level,
+                                                                                     nextline and
+                                                                                     self.break_default)
+                return "{:s} obj: {!s}".format(type(obj).__name__, parent_dump)
+
+
+class Handler:
+    """
+    The resource handler object gathers a provider, a resource and a container
+    for any specific resource.
+
+    Other parameters given at construct time are stored as options.
+    """
+
+    def __init__(self, rd, **kw):
+        if 'glove' in rd:
+            del rd['glove']
+        self._resource = rd.pop('resource', None)
+        self._provider = rd.pop('provider', None)
+        self._container = rd.pop('container', None)
+        self._empty = rd.pop('empty', False)
+        self._contents = None
+        self._uridata = None
+        self._options = rd.copy()
+        self._observer = observer_board(obsname=kw.pop('observer', None))
+        self._options.update(kw)
+        self._mdcheck = self._options.pop('metadatacheck', False)
+        self._mddelta = self._options.pop('metadatadelta', dict())
+        self._ghost = self._options.pop('ghost', False)
+        hook_names = [x for x in self._options.keys() if x.startswith('hook_')]
+        self._hooks = {x[5:]: self._options.pop(x) for x in hook_names}
+        self._delayhooks = self._options.pop('delayhooks', False)
+
+        self._history = History(tag='data-handler')
+        self._history.append(self.__class__.__name__, 'init', True)
+        self._stage = ['load']
+        self._observer.notify_new(self, dict(stage='load'))
+        self._localpr_cache = None  # To cache the promise dictionary
+        self._latest_earlyget_id = None
+        self._latest_earlyget_opts = None
+        logger.debug('New resource handler %s', self.__dict__)
+
+    def __str__(self):
+        return str(self.__dict__)
+
+    def _get_resource(self):
+        """Getter for ``resource`` property."""
+        return self._resource
+
+    def _set_resource(self, value):
+        """Setter for ``resource`` property."""
+        if isinstance(value, resources.Resource):
+            oldhash = self.simplified_hashkey
+            self._resource = value
+            self._notifyhash(oldhash)
+            self.reset_contents()
+        else:
+            raise ValueError('This value is not a plain Resource <{!s}>'.format(value))
+
+    resource = property(_get_resource, _set_resource)
+
+    def _get_provider(self):
+        """Getter for ``provider`` property."""
+        return self._provider
+
+    def _set_provider(self, value):
+        """Setter for ``provider`` property."""
+        if isinstance(value, providers.Provider):
+            oldhash = self.simplified_hashkey
+            self._provider = value
+            self._notifyhash(oldhash)
+            self.reset_contents()
+        else:
+            raise ValueError('This value is not a plain Provider <{!s}>'.format(value))
+
+    provider = property(_get_provider, _set_provider)
+
+    def _get_container(self):
+        """Getter for ``container`` property."""
+        return self._container
+
+    def _set_container(self, value):
+        """Setter for ``container`` property."""
+        if isinstance(value, containers.Container):
+            oldhash = self.simplified_hashkey
+            self._container = value
+            self._notifyhash(oldhash)
+        else:
+            raise ValueError('This value is not a plain Container <{!s}>'.format(value))
+
+    container = property(_get_container, _set_container)
+
+    @property
+    def history(self):
+        return self._history
+
+    @property
+    def observer(self):
+        """Footprint observer devoted to resource handlers tracking."""
+        return self._observer
+
+    def observers(self):
+        """Remote objects observing the current resource handler... and maybe others."""
+        return self._observer.observers()
+
+    def observed(self):
+        """Other objects observed by the observers of the current resource handler."""
+        return [x for x in self._observer.observed() if x is not self]
+
+    @property
+    def complete(self):
+        """Returns whether all the internal components are defined."""
+        return bool(self.resource and self.provider and self.container)
+
+    @property
+    def stage(self):
+        """Return current resource handler stage (load, get, put)."""
+        return self._stage[-1]
+
+    @property
+    def simplified_hashkey(self):
+        """Returns a tuple that can be used as a hashkey to quickly identify the handler."""
+        if self.complete:
+            rkind = getattr(self.resource, 'kind', None)
+            rfile = getattr(self.container, 'filename', None)
+            return (rkind, rfile)
+        else:
+            return ('incomplete', )
+
+    @property
+    def _cur_session(self):
+        """Return the current active session."""
+        return sessions.current()
+
+    @property
+    def _cur_context(self):
+        """Return the current active context."""
+        return contexts.current()
+
+    def external_stage_update(self, newstage):
+        """This method must not be used directly by users!
+
+        Update the stage upon request (e.g. the file has been fetched by another process).
+        """
+        self._stage.append(newstage)
+        if newstage in ('get', ):
+            self.container.updfill(True)
+
+    def _updstage(self, newstage, insitu=False):
+        """Notify the new stage to any observing system."""
+        self._stage.append(newstage)
+        self._observer.notify_upd(self, dict(stage=newstage, insitu=insitu))
+
+    def _notifyhook(self, stage, hookname):
+        """Notify that a hook function has been executed."""
+        self._observer.notify_upd(self, dict(stage=stage, hook=hookname))
+
+    def _notifyclear(self):
+        """Notify that the hashkey has changed."""
+        self._observer.notify_upd(self, dict(clear=True, ))
+
+    def _notifyhash(self, oldhash):
+        """Notify that the hashkey has changed."""
+        self._observer.notify_upd(self, dict(oldhash=oldhash, ))
+
+    def is_expected(self):
+        """Return a boolean value according to the last stage value (expected or not)."""
+        return self.stage.startswith('expect')
+
+    @property
+    def contents(self):
+        """
+        Returns an valid data layout object as long as the current handler
+        is complete and the container filled.
+        """
+        if self._empty:
+            self.container.write('')
+            self._empty = False
+        if self.complete:
+            if self.container.filled or self.stage == 'put':
+                if self._contents is None:
+                    self._contents = self.resource.contents_handler(datafmt=self.container.actualfmt)
+                    with self.container.iod_context():
+                        self._contents.slurp(self.container)
+                return self._contents
+            else:
+                logger.warning('Contents requested on an empty container [%s]', self.container)
+        else:
+            logger.warning('Contents requested for an uncomplete handler [%s]', self.container)
+            return None
+
+    def reset_contents(self):
+        """Delete actual internal reference to data contents manager."""
+        self._contents = None
+
+    @property
+    def ghost(self):
+        return self._ghost
+
+    @property
+    def hooks(self):
+        return self._hooks
+
+    @property
+    def options(self):
+        return self._options
+
+    @property
+    def delayhooks(self):
+        return self._delayhooks
+
+    def mkopts(self, *dicos, **kw):
+        """Returns options associated to that handler and a system reference."""
+        opts = dict(
+            intent=dataflow.intent.IN,
+            fmt=self.container.actualfmt,
+        )
+        opts.update(self.options)
+        for d in dicos:
+            opts.update(d)
+        opts.update(kw)
+        return opts
+
+    def location(self, fatal=True):
+        """Returns the URL as defined by the internal provider and resource."""
+        self._lasturl = None
+        if self.provider and self.resource:
+            try:
+                self._lasturl = self.provider.uri(self.resource)
+            except Exception as e:
+                if fatal:
+                    raise
+                else:
+                    return 'OOPS: {!s} (but fatal is False)'.format(e)
+            return self._lasturl
+        else:
+            logger.warning('Resource handler %s could not build location', self)
+            return None
+
+    def idcard(self, indent=2):
+        """
+        Returns a multilines documentation string with a summary
+        of the valuable information contained by this handler.
+        """
+        tab = ' ' * indent
+        card = "\n".join((
+            '{0}Handler {1!r}',
+            '{0}{0}Complete  : {2}',
+            '{0}{0}Options   : {3}',
+            '{0}{0}Location  : {4}'
+        )).format(tab,
+                  self, self.complete, self.options, self.location())
+        if self.hooks:
+            card += '\n{0}{0}Hooks     : {1}'.format(tab, ','.join(list(self.hooks.keys())))
+        d = IdCardAttrDumper(tag='idcarddumper')
+        d.reset()
+        d.indent_first = 2 * len(tab)
+        for subobj in ('resource', 'provider', 'container'):
+            obj = getattr(self, subobj, None)
+            if obj:
+                thisdoc = '{0}{0}{1:s} {2!r}'.format(tab,
+                                                     subobj.capitalize(), obj)
+                thisdoc += d.dump_fpattrs(obj)
+            else:
+                thisdoc = '{0}{0}{1:s} undefined'.format(tab, subobj.capitalize())
+            card = card + "\n" + thisdoc
+        return card
+
+    def quickview(self, nb=0, indent=0):
+        """Standard glance to objects."""
+        tab = '  ' * indent
+        print('{}{:02d}. {:s}'.format(tab, nb, repr(self)))
+        print('{}  Complete  : {!s}'.format(tab, self.complete))
+        for subobj in ('container', 'provider', 'resource'):
+            obj = getattr(self, subobj, None)
+            if obj:
+                print('{}  {:10s}: {!s}'.format(tab, subobj.capitalize(), obj))
+
+    def wide_key_lookup(self, key, exports=False, fatal=True):
+        """Return the *key* attribute if it exists in the provider or resource.
+
+        If *exports* is True, the footprint_export() or the export_dict() function
+        is called upon the return value.
+        """
+        try:
+            if key == 'safeblock':
+                # In olive experiments, the block may contain an indication of
+                # the member's number. Usually we do not want to get that...
+                a_value = getattr(self.provider, 'block')
+                a_value = re.sub(r'(member|fc)_?\d+/', '', a_value)
+            else:
+                a_value = getattr(self.provider, key)
+        except AttributeError:
+            try:
+                a_value = getattr(self.resource, key)
+            except AttributeError:
+                if fatal:
+                    raise AttributeError('The {:s} attribute could not be found in {!r}'.format(key, self))
+                else:
+                    a_value = None
+        if exports:
+            if hasattr(a_value, 'footprint_export'):
+                a_value = a_value.footprint_export()
+            elif hasattr(a_value, 'export_dict'):
+                a_value = a_value.export_dict()
+        return a_value
+
+    def as_dict(self):
+        """Produce a raw json-compatible dictionary."""
+        rhd = dict(options=dict())
+        for k, v in self.options.items():
+            try:
+                v = v.export_dict()
+            except (AttributeError, TypeError):
+                pass
+            rhd['options'][k] = v
+        for subobj in ('resource', 'provider', 'container'):
+            obj = getattr(self, subobj, None)
+            if obj is not None:
+                rhd[subobj] = obj.footprint_export()
+        return rhd
+
+    @property
+    def lasturl(self):
+        """The last actual URL value evaluated."""
+        return self._lasturl
+
+    @property
+    def uridata(self):
+        """Actual extra URI values after store definition."""
+        return self._uridata
+
+    @property
+    def store(self):
+        if self.resource and self.provider:
+            self._uridata = net.uriparse(self.location())
+            stopts = {k: v for k, v in self.options.items() if k.startswith('stor')}
+            return footprints.proxy.store(
+                scheme=self._uridata.pop('scheme'),
+                netloc=self._uridata.pop('netloc'),
+                **stopts
+            )
+        else:
+            return None
+
+    def check(self, **extras):
+        """Returns a stat-like information to the remote resource."""
+        rst = None
+        if self.resource and self.provider:
+            store = self.store
+            if store:
+                logger.debug('Check resource %s at %s from %s', self, self.lasturl, store)
+                rst = store.check(
+                    self.uridata,
+                    self.mkopts(extras)
+                )
+                if rst and self._mdcheck:
+                    logger.info('metadatacheck is on: we are forcing a real get()...')
+                    # We are using a temporary fake container
+                    mycontainer = footprints.proxy.container(shouldfly=True,
+                                                             actualfmt=self.container.actualfmt)
+                    try:
+                        tmp_options = self.mkopts(extras)
+                        tmp_options['obs_notify'] = False
+                        rst = store.get(
+                            self.uridata,
+                            mycontainer.iotarget(),
+                            tmp_options
+                        )
+                        if rst:
+                            if store.delayed:
+                                logger.warning("The resource is expected... let's say that's fine.")
+                            else:
+                                # Create the contents manually and drop it when we are done.
+                                contents = self.resource.contents_handler(datafmt=mycontainer.actualfmt)
+                                contents.slurp(mycontainer)
+                                rst = contents.metadata_check(self.resource, delta=self._mddelta)
+                    finally:
+                        # Delete the temporary container
+                        mycontainer.clear()
+                self.history.append(store.fullname(), 'check', rst)
+                if rst and self.stage == 'load':
+                    # Indicate that the resource was checked
+                    self._updstage('checked')
+                if not rst:
+                    # Always signal failures
+                    self._updstage('void')
+            else:
+                logger.error('Could not find any store to check %s', self.lasturl)
+        else:
+            logger.error('Could not check a rh without defined resource and provider %s', self)
+        return rst
+
+    def locate(self, **extras):
+        """Try to figure out what would be the physical location of the resource."""
+        rst = None
+        if self.resource and self.provider:
+            store = self.store
+            if store:
+                logger.debug('Locate resource %s at %s from %s', self, self.lasturl, store)
+                rst = store.locate(
+                    self.uridata,
+                    self.mkopts(extras)
+                )
+                self.history.append(store.fullname(), 'locate', rst)
+            else:
+                logger.error('Could not find any store to locate %s', self.lasturl)
+        else:
+            logger.error('Could not locate an incomplete rh %s', self)
+        return rst
+
+    def prestage(self, **extras):
+        """Request the pre-staging of the remote resource."""
+        rst = None
+        if self.resource and self.provider:
+            store = self.store
+            if store:
+                logger.debug('Prestage resource %s at %s from %s', self, self.lasturl, store)
+                rst = store.prestage(
+                    self.uridata,
+                    self.mkopts(extras)
+                )
+                self.history.append(store.fullname(), 'prestage', rst)
+            else:
+                logger.error('Could not find any store to prestage %s', self.lasturl)
+        else:
+            logger.error('Could not prestage an incomplete rh %s', self)
+        return rst
+
+    def _generic_apply_hooks(self, action, **extras):
+        """Apply the hooks after a get request (or verify that they were done)."""
+        if self.hooks:
+            mytracker = extras.get('mytracker', None)
+            if mytracker is None:
+                iotarget = self.container.iotarget()
+                mytracker = self._cur_context.localtracker[iotarget]
+            for hook_name in sorted(self.hooks.keys()):
+                if mytracker.redundant_hook(action, hook_name):
+                    logger.info('Hook already executed <hook_name:%s>', hook_name)
+                else:
+                    logger.info('Executing Hook <hook_name:%s>', hook_name)
+                    hook_func, hook_args = self.hooks[hook_name]
+                    hook_func(self._cur_session, self, *hook_args)
+                    self._notifyhook(action, hook_name)
+
+    def apply_get_hooks(self, **extras):
+        """Apply the hooks after a get request (or verify that they were done)."""
+        self._generic_apply_hooks(action='get', **extras)
+
+    def apply_put_hooks(self, **extras):
+        """Apply the hooks before a put request (or verify that they were done)."""
+        self._generic_apply_hooks(action='put', **extras)
+
+    def _postproc_get(self, store, rst, extras):
+        self.container.updfill(rst)
+        # Check metadata if sensible
+        if self._mdcheck and rst and not store.delayed:
+            rst = self.contents.metadata_check(self.resource,
+                                               delta=self._mddelta)
+            if not rst:
+                logger.info("We are now cleaning up the container and data contents.")
+                self.reset_contents()
+                self.clear()
+        # For the record...
+        self.history.append(store.fullname(), 'get', rst)
+        if rst:
+            # This is an expected resource
+            if store.delayed:
+                self._updstage('expected')
+                logger.info('Resource <%s> is expected', self.container.iotarget())
+            # This is a "real" resource
+            else:
+                self._updstage('get')
+                if self.hooks:
+                    if not self.delayhooks:
+                        self.apply_get_hooks(**extras)
+                    else:
+                        logger.info("(get-)Hooks were delayed")
+        else:
+            # Always signal failures
+            self._updstage('void')
+        return rst
+
+    def _actual_get(self, **extras):
+        """Internal method in charge of getting the resource.
+
+        If requested, it will check the metadata of the resource and apply the
+        hook functions.
+        """
+        rst = False
+        store = self.store
+        if store:
+            logger.debug('Get resource %s at %s from %s', self, self.lasturl, store)
+            st_options = self.mkopts(dict(rhandler=self.as_dict()), extras)
+            # Actual get
+            try:
+                rst = store.get(
+                    self.uridata,
+                    self.container.iotarget(),
+                    st_options,
+                )
+            except Exception:
+                rst = False
+                raise
+            finally:
+                rst = self._postproc_get(store, rst, extras)
+        else:
+            logger.error('Could not find any store to get %s', self.lasturl)
+
+        # Reset the promise dictionary cache
+        self._localpr_cache = None  # To cache the promise dictionary
+
+        return rst
+
+    def _actual_earlyget(self, **extras):
+        """Internal method in charge of requesting an earlyget on the resource.
+
+        :return: ``None`` if earlyget is unavailable (depending on the store's kind
+            and resource it can be perfectly fine). ``True`` if the resource was
+            actually fetched (no need to call :meth:`finaliseget`). Some kind of
+            non-null identifier that will be used to call :meth:`finaliseget`.
+        """
+        try:
+            store = self.store
+        except Exception as e:
+            logger.error("The Resource handler was unable to create a store object (%s).",
+                         str(e))
+            store = None
+        if store:
+            logger.debug('Early-Get resource %s at %s from %s', self, self.lasturl, store)
+            st_options = self.mkopts(dict(rhandler=self.as_dict()), extras)
+            # Actual earlyget
+            try:
+                return store.earlyget(
+                    self.uridata,
+                    self.container.iotarget(),
+                    st_options,
+                )
+            except Exception as e:
+                logger.error("The store's earlyget method did not return (%s): it should never append!",
+                             str(e))
+                return None
+        else:
+            logger.error('Could not find any store to get %s', self.lasturl)
+            return None
+
+    def _get_proxy(self, callback, alternate=False, **extras):
+        """
+        Process the **insitu** and **alternate** option and launch the **callback**
+        callable if sensible.
+        """
+        rst = False
+        if self.complete:
+            if self.options.get('insitu', False):  # This a second pass (or third, forth, ...)
+                cur_tracker = self._cur_context.localtracker
+                cur_seq = self._cur_context.sequence
+                iotarget = self.container.iotarget()
+                # The localpath is here and listed in the tracker
+                if self.container.exists() and cur_tracker.is_tracked_input(iotarget):
+                    # Am I consistent with the ResourceHandler recorded in the tracker ?
+                    if cur_tracker[iotarget].match_rh('get', self):
+                        rst = True
+                        # There is the tricky usecase where we are dealing with an alternate
+                        # that was already dealt with (yes, sometimes the nominal case and
+                        # the alternate is the same !)
+                        if not (alternate and iotarget in [s.rh.container.iotarget()
+                                                           for s in cur_seq.effective_inputs()]):
+                            self.container.updfill(True)
+                            self._updstage('get', insitu=True)
+                            logger.info(
+                                'The <%s> resource is already here and matches the RH description :-)',
+                                self.container.iotarget())
+                    else:
+                        # This may happen if fatal=False and the local file was fetched
+                        # by an alternate
+                        if alternate:
+                            if not self.container.is_virtual():
+                                lpath = self.container.localpath()
+                                for isec in self._cur_context.sequence.rinputs():
+                                    if (isec.stage in ('get' or 'expected') and
+                                            not isec.rh.container.is_virtual() and
+                                            isec.rh.container.localpath() == lpath):
+                                        rst = True
+                                        break
+                                if rst:
+                                    logger.info("Alternate is on and the local file exists.")
+                                else:
+                                    logger.info("Alternate is on but the local file is not yet matched.")
+                                    self._updstage('void', insitu=True)
+                            else:
+                                logger.info("Alternate is on. The local file exists. The container is virtual.")
+                                rst = True
+                        else:
+                            logger.info("The resource is already here but doesn't match the RH description :-(")
+                            cur_tracker[iotarget].match_rh('get', self, verbose=True)
+                            self._updstage('void', insitu=True)
+                # Bloody hell, the localpath doesn't exist
+                else:
+                    rst = callback(**extras)  # This might be an expected resource...
+                    if rst:
+                        logger.info("The resource was successfully fetched :-)")
+                    else:
+                        logger.info("Could not get the resource :-(")
+            else:
+                if alternate and self.container.exists():
+                    logger.info('Alternate <%s> exists', alternate)
+                    rst = True
+                else:
+                    if self.container.exists():
+                        logger.warning('The resource is already here: that should not happen at this stage !')
+                    rst = callback(**extras)
+        else:
+            logger.error('Could not get an incomplete rh %s', self)
+        return rst
+
+    def get(self, alternate=False, **extras):
+        """Method to retrieve the resource through the provider and feed the current container.
+
+        The behaviour of this method depends on the **insitu** and **alternate** options:
+
+        * When **insitu** is True, the :class:`~vortex.layout.dataflow.LocalTracker`
+          object associated with the active context is checked to determine
+          whether the resource has already been fetched or not. If not, another
+          try is made (but without using any non-cache store).
+        * When **insitu** is False, an attempt to get the resource is systematically
+          made except if **alternate** is defined and the local container already
+          exists.
+        """
+        return self._get_proxy(self._actual_get, alternate=alternate, **extras)
+
+    def earlyget(self, alternate=False, **extras):
+        """The earlyget feature is somehow a declaration of intent.
+
+        It records in the current context that, at some point in the future, we will
+        retrieve the present resource. It can be useful for some kind of stores
+        (and useless to others). For example, when using a store that targets a mass
+        archive system, this information can be used to ask for several files at
+        once, which accelerates the overall process and optimises the tape's drivers
+        usage. On the other hand, for a cache based store, it does not make much sense
+        since the data is readily available on disk.
+
+        Return values can be:
+
+        * ``None`` if earlyget is unavailable (depending on the store's kind
+          and resource it can be perfectly fine).
+        * Some kind of non-null identifier that will be used later on to actually
+          retrieve the resource. It is returned to the user as a diagnostic but is
+          also stored internally within the :class:`Handler` object.
+        * ``True`` if the resource has actually been retrieved through the provider
+          and fed into the current container.
+
+        In any case, the :meth:`finaliseget` method should be called later on
+        to actually retrieve the resource and feed the container. When ``True``
+        is returned by the :meth:`earlyget` method, the :meth:`finaliseget` call
+        can be made although it is useless.
+
+        Like with the :meth:`get` method, the behaviour of this method depends
+        on the **insitu** and **alternate** options:
+
+        * When **insitu** is True, the :class:`~vortex.layout.dataflow.LocalTracker`
+          object associated with the active context is checked to determine
+          whether the resource has already been fetched or not. If not, another
+          try is made (but without using any non-cache store).
+        * When **insitu** is False, an attempt to get the resource is systematically
+          made except if **alternate** is defined and the local container already
+          exists.
+        """
+        r_opts = extras.copy()
+        self._latest_earlyget_opts = r_opts
+        self._latest_earlyget_opts['alternate'] = alternate
+        self._latest_earlyget_id = self._get_proxy(self._actual_earlyget, alternate=alternate, **extras)
+        return self._latest_earlyget_id
+
+    def finaliseget(self):
+        """
+        When the :meth:`earlyget` method had previously been called, the
+        :meth:`finaliseget` can be called to finalise the ``get`` sequence.
+
+        When :meth:`finaliseget` returns, if the return code is non-zero, the resource
+        has been retrieved and fed into the container.
+
+        :raises HandlerError: if :meth:`earlyget` is not called prior to this
+                              method.
+        """
+        if self._latest_earlyget_id is None and self._latest_earlyget_opts is None:
+            raise HandlerError('earlyget was not called yet. Calling finaliseget is not Allowed !')
+        try:
+            if self._latest_earlyget_id is True:
+                # Nothing to be done...
+                return True
+            elif self._latest_earlyget_id is None:
+                # Delayed get not available... do the usual get !
+                e_opts = self._latest_earlyget_opts.copy()
+                e_opts['insitu'] = False
+                return self._get_proxy(self._actual_get, **e_opts)
+            else:
+                alternate = self._latest_earlyget_opts.get('alternate', False)
+                if alternate and self.container.exists():
+                    # The container may have been filled be another finaliseget
+                    logger.info('Alternate <%s> exists', alternate)
+                    rst = True
+                else:
+                    rst = False
+                    store = self.store
+                    if store:
+                        logger.debug('Finalise-Get resource %s at %s from %s', self, self.lasturl, store)
+                        st_options = self.mkopts(dict(rhandler=self.as_dict()), self._latest_earlyget_opts)
+                        # Actual get
+                        rst = store.finaliseget(
+                            self._latest_earlyget_id,
+                            self.uridata,
+                            self.container.iotarget(),
+                            st_options,
+                        )
+                        if rst is None:
+                            # Delayed get failed... attempt the usual get
+                            logger.warning('Delayed get result was unclear ! Reverting to the usual get.')
+                            e_opts = self._latest_earlyget_opts.copy()
+                            e_opts['insitu'] = False
+                            return self._get_proxy(self._actual_get, **e_opts)
+                        else:
+                            rst = self._postproc_get(store, rst, self._latest_earlyget_opts)
+                    else:
+                        logger.error('Could not find any store to get %s', self.lasturl)
+
+                    # Reset the promise dictionary cache
+                    self._localpr_cache = None  # To cache the promise dictionary
+
+                return rst
+        finally:
+            self._latest_earlyget_id = None
+            self._latest_earlyget_opts = None
+
+    def insitu_quickget(self, alternate=False, **extras):
+        """This method attempts a straightforward insitu get.
+
+        It is designed to minimise the amount of outputs when everything goes smoothly.
+        """
+        rst = False
+        if self.complete:
+            if self.options.get('insitu', False):  # This a second pass (or third, forth, ...)
+                cur_tracker = self._cur_context.localtracker
+                cur_seq = self._cur_context.sequence
+                iotarget = self.container.iotarget()
+                # The localpath is here and listed in the tracker
+                if (self.container.exists() and
+                        cur_tracker.is_tracked_input(iotarget)):
+                    if cur_tracker[iotarget].match_rh('get', self):
+                        rst = True
+                        # There is the tricky usecase where we are dealing with an alternate
+                        # that was already dealt with (yes, sometimes the nominal case and
+                        # the alternate is the same !)
+                        if not (alternate and iotarget in [s.rh.container.iotarget()
+                                                           for s in cur_seq.effective_inputs()]):
+                            self.container.updfill(True)
+                            self._updstage('get', insitu=True)
+                    elif alternate:
+                        # Alternate is on and the local file exists: check if
+                        # the file has already been fetch previously in the sequence
+                        if iotarget in [s.rh.container.iotarget()
+                                        for s in cur_seq.effective_inputs()]:
+                            rst = True
+            else:
+                logger.error('This method should not be called with insitu=False (rh %s)', self)
+        return rst
+
+    def put(self, **extras):
+        """Method to store data from the current container through the provider.
+
+        Hook functions may be applied before the put in the designated store. We
+        will ensure that a given hook function (identified by its name) is not
+        applied more than once to the local container.
+
+        Conversely, the low-level stores are made aware of the previous successful
+        put. That way, a local container is not put twice to the same destination.
+        """
+        rst = False
+        if self.complete:
+            store = self.store
+            if store:
+                iotarget = self.container.iotarget()
+                logger.debug('Put resource %s as io %s at store %s', self, iotarget, store)
+                if iotarget is not None and (self.container.exists() or self.provider.expected):
+                    mytracker = self._cur_context.localtracker[iotarget]
+                    # Execute the hooks only if the local file exists
+                    if self.container.exists():
+                        self.container.updfill(True)
+                        if self.hooks:
+                            if not self.delayhooks:
+                                self.apply_put_hooks(mytracker=mytracker, **extras)
+                            else:
+                                logger.info("(put-)Hooks were delayed")
+                    # Add a filter function to remove duplicated PUTs to the same uri
+                    extras_ext = dict(extras)
+                    extras_ext['urifilter'] = functools.partial(mytracker.redundant_uri, 'put')
+                    # Actual put
+                    logger.debug('Put resource %s at %s from %s', self, self.lasturl, store)
+                    rst = store.put(
+                        iotarget,
+                        self.uridata,
+                        self.mkopts(dict(rhandler=self.as_dict()), extras_ext)
+                    )
+                    # For the record...
+                    self.history.append(store.fullname(), 'put', rst)
+                    self._updstage('put')
+                elif self.ghost:
+                    self.history.append(store.fullname(), 'put', False)
+                    self._updstage('ghost')
+                    rst = True
+                else:
+                    logger.error('Could not find any source to put [%s]', iotarget)
+            else:
+                logger.error('Could not find any store to put [%s]', self.lasturl)
+        else:
+            logger.error('Could not put an incomplete rh [%s]', self)
+        return rst
+
+    def delete(self, **extras):
+        """Delete the remote resource from store."""
+        rst = None
+        if self.resource and self.provider:
+            store = self.store
+            if store:
+                logger.debug('Delete resource %s at %s from %s', self, self.lasturl, store)
+                rst = store.delete(
+                    self.uridata,
+                    self.mkopts(dict(rhandler=self.as_dict()), extras)
+                )
+                self.history.append(store.fullname(), 'delete', rst)
+            else:
+                logger.error('Could not find any store to delete %s', self.lasturl)
+        else:
+            logger.error('Could not delete a rh without defined resource and provider %s', self)
+        return rst
+
+    def clear(self):
+        """Clear the local container contents."""
+        rst = False
+        if self.container:
+            logger.debug('Remove resource container %s', self.container)
+            rst = self.container.clear()
+            self.history.append(self.container.actualpath(), 'clear', rst)
+            self._notifyclear()
+            stage_clear_mapping = dict(expected='checked', get='checked')
+            if self.stage in stage_clear_mapping:
+                self._updstage(stage_clear_mapping[self.stage])
+        return rst
+
+    def mkgetpr(self, pr_getter=None, tplfile=None, tplskip='@sync-skip.tpl',
+                tplfetch='@sync-fetch.tpl', py_exec=sys.executable, py_opts=''):
+        """Build a getter for the expected resource."""
+        if tplfile is None:
+            tplfile = tplfetch if self.is_expected() else tplskip
+        if pr_getter is None:
+            pr_getter = self.container.localpath() + '.getpr'
+        t = self._cur_session
+        tpl = config.load_template(t, tplfile)
+        with open(pr_getter, 'w', encoding='utf-8') as fd:
+            fd.write(tpl.substitute(
+                python=py_exec,
+                pyopts=py_opts,
+                promise=self.container.localpath(),
+            ))
+        t.sh.chmod(pr_getter, 0o555)
+        return pr_getter
+
+    @property
+    def _localpr_json(self):
+        if self.is_expected():
+            if self._localpr_cache is None:
+                self._localpr_cache = self._cur_session.sh.json_load(self.container.localpath())
+            return self._localpr_cache
+        else:
+            return None
+
+    def is_grabable(self, check_exists=False):
+        """Return if an expected resource is available or not.
+
+        Note: If it returns True, the user still needs to :meth:`get` the resource.
+        """
+        rc = True
+        if self.is_expected():
+            pr = self._localpr_json
+            itself = pr.get('itself')
+            rc = not self._cur_session.sh.path.exists(itself)
+            if rc and check_exists:
+                remote = pr.get('locate').split(';')[0]
+                rc = self._cur_session.sh.path.exists(remote)
+        return rc
+
+    def wait(self, sleep=10, timeout=300, fatal=False):
+        """Wait for an expected resource or return immediately."""
+        rc = True
+        local = self.container.localpath()
+        if self.is_expected():
+            nb = 0
+            sh = self._cur_session.sh
+            pr = self._localpr_json
+            itself = pr.get('itself')
+            nbtries = int(timeout / sleep)
+            logger.info('Waiting %d x %d s. for expected resource <%s>', nbtries, sleep, local)
+            while sh.path.exists(itself):
+                sh.sleep(sleep)
+                nb += 1
+                if nb > nbtries:
+                    logger.error('Could not wait anymore <%d>', nb)
+                    rc = False
+                    if fatal:
+                        logger.critical('Missing expected resource is fatal <%s>', local)
+                        raise HandlerError('Expected resource missing')
+                    break
+            else:
+                remote = pr.get('locate').split(';')[0]
+                if sh.path.exists(remote):
+                    logger.info('Keeping promise for remote resource <%s>', remote)
+                else:
+                    logger.warning('Empty promise for remote resource <%s>', remote)
+                    rc = False
+        else:
+            logger.info('Resource <%s> not expected', local)
+        return rc
+
+    def save(self):
+        """Rewrite data if contents have been updated."""
+        rst = False
+        if self.contents:
+            rst = self.contents.rewrite(self.container)
+            if not self.container.is_virtual():
+                self.container.close()
+        else:
+            logger.warning('Try to save undefined contents %s', self)
+        return rst
+
+    def strlast(self):
+        """String formatted log of the last action."""
+        return ' '.join([str(x) for x in self.history.last])