vortex-nwp 2.0.0__py3-none-any.whl

vortex/tools/delayedactions.py

@@ -0,0 +1,782 @@
+"""
+Advanced tools that deals with delayed actions.
+
+The entry point to the delayed action mechanism is the :class:`PrivateDelayedActionsHub`
+class. One :class:`PrivateDelayedActionsHub` object is created in each
+:class:`~vortex.dataflow.contexts.Context`
+(see :meth:`vortex.dataflow.contexts.Context.delayedactions_hub`). When working
+with delayed actions, you must always use the :class:`PrivateDelayedActionsHub`
+object associated with the current active :class:`~vortex.dataflow.contexts.Context`.
+
+Example::
+
+    # Get the DelayedActionsHub object from the current context
+
+    >>> from vortex import sessions
+
+    >>> cur_da_hub = sessions.current().context.delayedactions_hub
+
+    # Instruct Vortex to sleep (in parallel !)
+
+    >>> dactions = [cur_da_hub.register(n, kind='sleep') for n in (1, 3, 2)]
+
+    # How does it look like in the logs for the first action ?
+
+    >>> cur_da_hub.actionhistory(dactions[0])  # doctest:+ELLIPSIS
+    <....DemoSleepDelayedActionHandler object at 0x...> says:
+    [...][...] : NEW  id=sleeper_action_...: request=1
+    [...][...] : UPD  id=sleeper_action_...: result=<multiprocessing.pool.ApplyResult object at 0x...> ...
+
+    # Wait for all the sleepers to wake up... It would be possible to do that
+    # explicitly by calling the ``finalise`` method, but ``retrieve`` implicitly
+    # calls ``finalise`` so we don't bother !
+
+    >>> for daction in dactions:
+    ...     action = cur_da_hub.retrieve(daction, bareobject=True)
+    ...     if action.status == d_action_status.done:
+    ...         print('I slept for {0.request:d} seconds and It was good :-)'.format(action))
+    ...     else:
+    ...         print('I failed to get asleep for {0.request:d} seconds :-('.format(action))
+    I slept for 1 seconds and It was good :-)
+    I slept for 3 seconds and It was good :-)
+    I slept for 2 seconds and It was good :-)
+
+    # And now, what are the log saying ?
+
+    >>> cur_da_hub.actionhistory(dactions[0])  # doctest:+ELLIPSIS
+    <....DemoSleepDelayedActionHandler object at 0x...> says:
+    [...][...] : NEW  id=sleeper_action_0000000000000001: request=1
+    [...][...] : UPD  id=sleeper_action_0000000000000001: result=<multiprocessing.pool.ApplyResult object at 0x...> ...
+    [...][...] : UPD  id=sleeper_action_0000000000000001: status=done (instead of: void)
+    [...][...] : USED id=sleeper_action_0000000000000001
+
+"""
+
+from collections import namedtuple, defaultdict
+import multiprocessing
+import os
+import tempfile
+import time
+
+from bronx.fancies import loggers
+from bronx.fancies.dump import lightdump
+from bronx.stdtypes.history import PrivateHistory
+from bronx.patterns import getbytag, observer
+
+import footprints
+from footprints import proxy as fpx
+
+from vortex.tools.systems import OSExtended
+
+#: No automatic export
+__all__ = []
+
+logger = loggers.getLogger(__name__)
+
+#: Definition of a named tuple DelayedActionStatusTuple
+DelayedActionStatusTuple = namedtuple(
+    "DelayedActionStatusTuple", ["void", "failed", "done", "unclear"]
+)
+
+#: Predefined DelayedActionStatus values (void=Not ready yet,
+#                                         failed=processed but KO,
+#                                         done=processed and OK,
+#                                         unclear=processed but cannot tell whether it is KO or OK)
+d_action_status = DelayedActionStatusTuple(
+    void=0, failed=-1, done=1, unclear=-2
+)
+
+
+# Module Interface
+def get_hub(**kw):
+    """Return the actual :class:`DelayedActionsHub` object matching the *tag* (or create one)."""
+    return DelayedActionsHub(**kw)
+
+
+class DelayedAction:
+    """Simple object describing one action to be performed."""
+
+    def __init__(self, obsboard, r_id, request):
+        """
+        :param SecludedObserverBoard obsboard: The Observer board that will be used
+                                               to publish the results.
+        :param r_id: Any kind of ID that uniquely identifies the delayed action
+        :param request: Any kind of data that describes the action to be performed
+        """
+        self._obsboard = obsboard
+        self._id = r_id
+        self._request = request
+        self._status = d_action_status.void
+        self._result = None
+        self._obsboard.notify_new(self, dict())
+
+    @property
+    def id(self):
+        """The delayed action ID."""
+        return self._id
+
+    @property
+    def request(self):
+        """The data describing the action."""
+        return self._request
+
+    def _set_status(self, value):
+        oldres = self.statustext
+        self._status = value
+        self._obsboard.notify_upd(
+            self,
+            info=dict(changed="status", queryproxy="statustext", prev=oldres),
+        )
+
+    @property
+    def status(self):
+        """The delayed action status (see :data:`d_action_status` for possible values)."""
+        return self._status
+
+    def _get_result(self):
+        return self._result
+
+    def _set_result(self, result):
+        oldres = self._result
+        self._result = result
+        self._obsboard.notify_upd(
+            self, info=dict(changed="result", prev=oldres)
+        )
+
+    result = property(
+        _get_result,
+        _set_result,
+        doc="Where to find the delayed action result.",
+    )
+
+    @property
+    def statustext(self):
+        """A string that descibres the delayed action status."""
+        for k, v in d_action_status._asdict().items():
+            if self._status == v:
+                return k
+        logger.warning("What is this idiotic status (%s) ???", self.status)
+        return str(self.status)
+
+    def mark_as_failed(self):
+        """Change the status to ``failed``."""
+        logger.info(
+            "Marking early-get %s as failed (request=%s)",
+            self.id,
+            self.request,
+        )
+        self._set_status(d_action_status.failed)
+
+    def mark_as_done(self):
+        """Change the status to ``done``."""
+        logger.debug(
+            "Marking early-get %s as done (request=%s)", self.id, self.request
+        )
+        self._set_status(d_action_status.done)
+
+    def mark_as_unclear(self):
+        """Change the status to ``unclear``."""
+        logger.info(
+            "Marking early-get %s as unclear/unconclusive (request=%s)",
+            self.id,
+            self.request,
+        )
+        self._set_status(d_action_status.unclear)
+
+    def __str__(self):
+        return "id={0._id}: {0.statustext:6s} result={0.result!s}".format(self)
+
+
+class AbstractDelayedActionsHandler(
+    footprints.FootprintBase, observer.Observer
+):
+    """Abstract class that handles a bunch of similar delayed actions."""
+
+    _abstract = True
+    _collector = ("delayedactionshandler",)
+    _footprint = dict(
+        info="Abstract class that deal with delayed actions.",
+        attr=dict(
+            system=dict(info="The current system object", type=OSExtended),
+            observerboard=dict(
+                info="The observer board where delayed actions updates are published.",
+                type=observer.SecludedObserverBoard,
+            ),
+            stagedir=dict(info="The temporary directory (if need be)"),
+        ),
+    )
+
+    def __init__(self, *args, **kw):
+        super().__init__(*args, **kw)
+        self._resultsmap = dict()
+        self._history = PrivateHistory(timer=True)
+        self.observerboard.register(self)
+        self._custom_init()
+
+    def destroy(self):
+        """Cleanup everything... (useful when multiprocessing is used)"""
+        self._resultsmap = None
+        self._history = None
+
+    def _custom_init(self):
+        """This method may be specialised in actual DelayedActionsHandler classes."""
+        pass
+
+    def newobsitem(self, item, info):  # @UnusedVariable
+        """To get informed when a new :class:`DelayedAction` object is created."""
+        if item.id in self._resultsmap:
+            self._history.append(
+                "NEW ", "id={0.id!s}: request={0.request!s}".format(item)
+            )
+
+    def updobsitem(self, item, info):
+        """To get informed when a new :class:`DelayedAction` object is updates."""
+        if item.id in self._resultsmap:
+            what = info["changed"]
+            newval = getattr(item, info.get("queryproxy", what))
+            oldval = info["prev"]
+            self._history.append(
+                "UPD ",
+                "id={0.id!s}: {1:s}={2!s} (instead of: {3!s})".format(
+                    item, what, newval, oldval
+                ),
+            )
+
+    @property
+    def history(self):
+        """The :class:`PrivateHistory` object where all of this object's activity is logged."""
+        return self._history
+
+    def grephistory(self, r_id):
+        """Return the log lines matching the **r_id** delayed action ID."""
+        return self.history.grep("id={!s}".format(r_id))
+
+    def showhistory(self, r_id):
+        """Print the log lines matching the **r_id** delayed action ID."""
+        return self.history.showgrep("id={!s}".format(r_id))
+
+    def __contains__(self, r_id):
+        return r_id in self._resultsmap
+
+    def dispence_resultid(self):
+        """Return a unique ID that will identify a new :class:`DelayedAction` object."""
+        raise NotImplementedError()
+
+    def _create_delayed_action(self, r_id, request):
+        """Create a :class:`DelayedAction` object given **r_id** and **request**."""
+        return DelayedAction(self.observerboard, r_id, request)
+
+    def register(self, request):
+        """Create a new :class:`DelayedAction` object from a user's **request**."""
+        r_id = self.dispence_resultid()
+        self._resultsmap[r_id] = None  # For newobitem to work...
+        d_action = self._create_delayed_action(r_id, request)
+        self._resultsmap[r_id] = d_action
+        self._custom_register(d_action)
+        return r_id
+
+    def _custom_register(self, action):
+        """Any action to be performed each time a new delayed action is registered."""
+        pass
+
+    @property
+    def dirty(self):
+        """Is there any of the object's delayed actions that needs finalising ?"""
+        return any(
+            [
+                a.status == d_action_status.void
+                for a in self._resultsmap.values()
+            ]
+        )
+
+    def finalise(self, *r_ids):
+        """Given a **r_ids** list of delayed action IDs, wait upon actions completion."""
+        raise NotImplementedError()
+
+    def retrieve(self, r_id, bareobject=False):
+        """Given a **r_id** delayed action ID, returns the corresponding result.
+
+        If need be, :meth:`finalise` is called.
+        """
+        action = self._resultsmap[r_id]
+        try:
+            if action.status == d_action_status.void:
+                self.finalise(r_id)
+                assert action.status != d_action_status.void, (
+                    "Finalise does not seem to work."
+                )
+        finally:
+            del self._resultsmap[r_id]
+            self._history.append("USED", "id={!s}".format(action.id))
+        if bareobject:
+            return action
+        else:
+            if action.status == d_action_status.done:
+                return action.result
+            else:
+                return False
+
+    def __str__(self):
+        return self.describe(fulldump=False)
+
+    def describe(self, fulldump=False):
+        """Print the object's characteristics and content."""
+        res = "DelayedActionsHandler object of class: {:s}\n".format(
+            self.__class__
+        )
+        for k, v in self.footprint_as_shallow_dict().items():
+            res += "  * {:s}: {!s}\n".format(k, v)
+        if fulldump:
+            res += "\n  * Todo list (i.e still to be processed):\n\n"
+            res += "\n".join(
+                [
+                    "{:48s}:\n      request: {!s}".format(r_id, a.request)
+                    for r_id, a in self._resultsmap.items()
+                    if a.status == d_action_status.void
+                ]
+            )
+            res += "\n  * Done (i.e the delayed action succeeded):\n\n"
+            res += "\n".join(
+                [
+                    "{:48s}:\n      request: {!s}".format(r_id, a.request)
+                    for r_id, a in self._resultsmap.items()
+                    if a.status == d_action_status.done
+                ]
+            )
+            res += "\n  * Failed (i.e the delayed action failed):\n\n"
+            res += "\n".join(
+                [
+                    "{:48s}:\n      request: {!s}".format(r_id, a.request)
+                    for r_id, a in self._resultsmap.items()
+                    if a.status == d_action_status.failed
+                ]
+            )
+            res += (
+                "\n  * Unclear (i.e processed but the result is unclear):\n\n"
+            )
+            res += "\n".join(
+                [
+                    "{:48s}:\n      request: {!s}".format(r_id, a.request)
+                    for r_id, a in self._resultsmap.items()
+                    if a.status == d_action_status.unclear
+                ]
+            )
+        return res
+
+
+class AbstractFileBasedDelayedActionsHandler(AbstractDelayedActionsHandler):
+    """
+    A specialised version of :class:`AbstractDelayedActionsHandler` where
+    a unique file (created in the ``stagedir``) is associated with each of the
+    delayed action.
+    """
+
+    _abstract = True
+
+    @property
+    def resultid_stamp(self):
+        """Some kind of string that identifies the present object."""
+        raise NotImplementedError()
+
+    def dispence_resultid(self):
+        """Return a unique ID that will identify a new :class:`DelayedAction` object."""
+        t_temp = tempfile.mkstemp(
+            prefix="{:s}_{:d}".format(
+                self.resultid_stamp, self.system.getpid()
+            ),
+            dir=self.stagedir,
+        )
+        os.close(t_temp[0])
+        return self.system.path.basename(t_temp[1])
+
+    def _create_delayed_action(self, r_id, request):
+        """Create a :class:`DelayedAction` object given **r_id** and **request**."""
+        d_action = DelayedAction(self.observerboard, r_id, request)
+        d_action.result = self.system.path.join(self.stagedir, r_id)
+        return d_action
+
+
+def demo_sleeper_function(seconds):
+    """Sleep for a while (demo)."""
+    time.sleep(seconds)
+
+
+class DemoSleepDelayedActionHandler(AbstractDelayedActionsHandler):
+    """A Sleeper delayed action handler (Demonstration purposes)."""
+
+    _footprint = dict(
+        info="Demonstration purposes (sleep for a while).",
+        attr=dict(
+            kind=dict(
+                values=[
+                    "sleep",
+                ],
+            ),
+        ),
+    )
+
+    def dispence_resultid(self):
+        """Return a unique ID that will identify a new :class:`DelayedAction` object."""
+        self._counter += 1
+        return "sleeper_action_{:016d}".format(self._counter)
+
+    def _custom_init(self):
+        """Create the multiprocessing pool."""
+        self._ppool = multiprocessing.Pool(processes=2)
+        self._counter = 0
+
+    def destroy(self):
+        """Destry the multiprocessing pool before leaving."""
+        self._ppool.close()
+        self._ppool.terminate()
+        self._ppool = None
+        super().destroy()
+
+    def _create_delayed_action(self, r_id, request):
+        """Start the asynchronous processing."""
+        daction = DelayedAction(self.observerboard, r_id, request)
+        daction.result = self._ppool.apply_async(
+            demo_sleeper_function, (request,)
+        )
+        return daction
+
+    def finalise(self, *r_ids):
+        """Wait until completion."""
+        for r_id in r_ids:
+            action = self._resultsmap[r_id]
+            action.result.wait()
+            if action.result.successful():
+                action.mark_as_done()
+            else:
+                action.mark_as_failed()
+
+
+class AbstractFtpArchiveDelayedGetHandler(
+    AbstractFileBasedDelayedActionsHandler
+):
+    """Includes some FTP related methods"""
+
+    _abstract = True
+    _footprint = dict(
+        info="Fetch multiple files using an FTP archive.",
+        attr=dict(
+            kind=dict(
+                values=[
+                    "archive",
+                ],
+            ),
+            storage=dict(),
+            goal=dict(
+                values=[
+                    "get",
+                ]
+            ),
+            tube=dict(
+                values=[
+                    "ftp",
+                ],
+            ),
+            raw=dict(
+                type=bool,
+                optional=True,
+                default=False,
+            ),
+            logname=dict(optional=True),
+        ),
+    )
+
+    @property
+    def resultid_stamp(self):
+        bangfmt = (
+            "{0.logname:s}@{0.storage:s}" if self.logname else "{0.storage:s}"
+        )
+        return ("rawftget_" + bangfmt).format(self)
+
+    def register(self, request):
+        """Create a new :class:`DelayedAction` object from a user's **request**."""
+        assert isinstance(request, (tuple, list)) and len(request) == 2, (
+            "Request needs to be a two element tuple or list (location, format)"
+        )
+        # Check for duplicated entries...
+        target = request[0]
+        for v in self._resultsmap.values():
+            if target == v.request[0]:
+                return None
+        # Ok, let's proceed...
+        return super().register(request)
+
+    @property
+    def _ftp_hostinfos(self):
+        """Return the FTP hostname end port number."""
+        s_storage = self.storage.split(":", 1)
+        hostname = s_storage[0]
+        port = None
+        if len(s_storage) > 1:
+            try:
+                port = int(s_storage[1])
+            except ValueError:
+                logger.error(
+                    "Invalid port number < %s >. Ignoring it", s_storage[1]
+                )
+        return hostname, port
+
+
+class RawFtpDelayedGetHandler(AbstractFtpArchiveDelayedGetHandler):
+    """
+    When FtServ is used, accumulate "GET" requests for several files and fetch
+    them during a unique ``ftget`` system call.
+
+    :note: The *request* needs to be a two-elements tuple where the first element
+           is the path to the file that shoudl be fetched and the second element
+           the file format.
+    :note: The **result** returned by the :meth:`retrieve` method will be the
+           path to the temporary file where the resource has been fetched.
+    """
+
+    _footprint = dict(
+        info="Fetch multiple files using FtServ.",
+        attr=dict(
+            raw=dict(
+                optional=False,
+                values=[
+                    True,
+                ],
+            ),
+        ),
+    )
+
+    def finalise(self, *r_ids):  # @UnusedVariable
+        """Given a **r_ids** list of delayed action IDs, wait upon actions completion."""
+        todo = defaultdict(list)
+        for k, v in self._resultsmap.items():
+            if v.status == d_action_status.void:
+                a_fmt = (
+                    v.request[1]
+                    if self.system.fmtspecific_mtd(
+                        "batchrawftget", v.request[1]
+                    )
+                    else None
+                )
+                todo[a_fmt].append(k)
+        rc = True
+        if todo:
+            for a_fmt, a_todolist in todo.items():
+                sources = list()
+                destinations = list()
+                extras = dict()
+                if a_fmt is not None:
+                    extras["fmt"] = a_fmt
+                for k in a_todolist:
+                    sources.append(self._resultsmap[k].request[0])
+                    destinations.append(self._resultsmap[k].result)
+                try:
+                    logger.info(
+                        "Running the ftserv command for format=%s.", str(a_fmt)
+                    )
+                    hostname, port = self._ftp_hostinfos
+                    rc = self.system.batchrawftget(
+                        sources,
+                        destinations,
+                        hostname=hostname,
+                        logname=self.logname,
+                        port=port,
+                        **extras,
+                    )
+                except OSError:
+                    rc = [
+                        None,
+                    ] * len(sources)
+                for i, k in enumerate(a_todolist):
+                    if rc[i] is True:
+                        self._resultsmap[k].mark_as_done()
+                    elif rc[i] is False:
+                        self._resultsmap[k].mark_as_failed()
+                    else:
+                        self._resultsmap[k].mark_as_unclear()
+        return rc
+
+
+class PrivateDelayedActionsHub:
+    """
+    Manages all of the delayed actions request by forwarding them to the appropriate
+    :class:`AbstractDelayedActionsHandler` object.
+
+    If no, :class:`AbstractDelayedActionsHandler` class is able to handle
+    the delayed action, just returns ``None`` to inform the caller that the
+    requested action can't be performed
+    """
+
+    def __init__(self, sh, contextrundir):
+        """
+        :param vortex.tools.systems.OSExtended sh: The current usable System object
+        :param str contextrundir: The current context's run directory where the
+                                  staging area/directory will be created. If ``None``,
+                                  the staging directory is created in the current
+                                  working directory.
+        """
+        self._sh = sh
+        self._contextrundir = contextrundir
+        self._stagedir = None
+        self._delayedactionshandlers = set()
+        self._obsboard = observer.SecludedObserverBoard()
+        self._resultsmap = dict()
+
+    @property
+    def observerboard(self):
+        """The Observer board associated with this Hub;
+
+        :note: Anyone is free to register to it in order be kept informed when a
+               delayed action associated with this Hub is updated.
+        """
+        return self._obsboard
+
+    @property
+    def stagedir(self):
+        """This Hub staging area/directory (i.e. where results can be stored)."""
+        if self._stagedir is None:
+            self._stagedir = tempfile.mkdtemp(
+                prefix="dactions_staging_area_",
+                dir=(
+                    self._contextrundir
+                    if self._contextrundir
+                    else self._sh.pwd()
+                ),
+            )
+        return self._stagedir
+
+    def showhistory(self):
+        """
+        Print the complete logs of all of the :class:`AbstractDelayedActionsHandler`
+        objects leveraged by this Hub.
+        """
+        for handler in self._delayedactionshandlers:
+            print("{!r} says:\n".format(handler))
+            handler.history.show()
+
+    def actionhistory(self, r_id):
+        """Print the log lines associated to a given request (identified by its **r_id** ID)."""
+        for handler in self._delayedactionshandlers:
+            hst = handler.grephistory(r_id)
+            if hst:
+                print("{!r} says:".format(handler))
+                handler.showhistory(r_id)
+
+    def register(self, request, **kwargs):
+        """Take into consideration a new delayed action request.
+
+        :param request: A description of the user's request
+        :param dict kwargs: Any argument that will be used to create the
+                            :class:`AbstractDelayedActionsHandler` object
+        """
+        # Prestaging tool descriptions
+        myhandler_desc = dict(
+            system=self._sh,
+            observerboard=self._obsboard,
+            stagedir=self.stagedir,
+        )
+        myhandler_desc.update(kwargs)
+        myhandler = None
+        # Scan pre-existing prestaging tools to find a suitable one
+        for ahandler in self._delayedactionshandlers:
+            if ahandler.footprint_reusable() and ahandler.footprint_compatible(
+                myhandler_desc
+            ):
+                logger.debug(
+                    "Re-usable Actions Handler found: %s",
+                    lightdump(myhandler_desc),
+                )
+                myhandler = ahandler
+                break
+        # If necessary, create a new one
+        if myhandler is None:
+            myhandler = fpx.delayedactionshandler(
+                _emptywarning=False, **myhandler_desc
+            )
+            if myhandler is not None:
+                logger.debug(
+                    "Fresh prestaging tool created: %s",
+                    lightdump(myhandler_desc),
+                )
+                self._delayedactionshandlers.add(myhandler)
+        # Let's role
+        if myhandler is None:
+            logger.debug(
+                "Unable to find a delayed actions handler with: %s",
+                lightdump(myhandler_desc),
+            )
+            return None
+        else:
+            resultid = myhandler.register(request)
+            if resultid is not None:
+                self._resultsmap[resultid] = myhandler
+        return resultid
+
+    @property
+    def dirty(self):
+        """Is there any of the hub's delayed actions that needs finalising ?"""
+        dirtyflag = False
+        for ahandler in self._delayedactionshandlers:
+            dirtyflag = dirtyflag or ahandler.dirty
+        return dirtyflag
+
+    def finalise(self, *r_ids):
+        """Given a **r_ids** list of delayed action IDs, wait upon actions completion."""
+        todo = defaultdict(set)
+        for r_id in r_ids:
+            todo[self._resultsmap[r_id]].add(r_id)
+        for ahandler, r_ids in todo.items():
+            ahandler.finalise(*list(r_ids))
+
+    def retrieve(self, resultid, bareobject=False):
+        """Given a **resultid** delayed action ID, returns the corresponding result."""
+        try:
+            res = self._resultsmap[resultid].retrieve(
+                resultid, bareobject=bareobject
+            )
+        finally:
+            del self._resultsmap[resultid]
+        return res
+
+    def clear(self):
+        """Destroy all of the associated handlers and reset everything."""
+        for a_handler in self._delayedactionshandlers:
+            a_handler.destroy()
+        self._delayedactionshandlers = set()
+        self._obsboard = observer.SecludedObserverBoard()
+        self._resultsmap = dict()
+        self._stagedir = None
+
+    def __repr__(self):
+        return "{:s} | n_delayedactionshandlers={:d}>".format(
+            super().__repr__().rstrip(">"), len(self._delayedactionshandlers)
+        )
+
+    def __str__(self):
+        return (
+            repr(self)
+            + "\n\n"
+            + "\n\n".join(
+                [
+                    ahandler.describe(fulldump=True)
+                    for ahandler in self._delayedactionshandlers
+                ]
+            )
+        )
+
+
+class DelayedActionsHub(PrivateDelayedActionsHub, getbytag.GetByTag):
+    """
+    A subclass of :class:`PrivateDelayedActionsHub` that uses
+    :class:`footprints.util.GetByTag` to remain persistent in memory.
+
+    Therefore, a *tag* attribute needs to be specified when building/retrieving
+    an object of this class.
+    """
+
+    pass
+
+
+if __name__ == "__main__":
+    import doctest
+
+    doctest.testmod()