vortex-nwp 2.0.0b1__py3-none-any.whl → 2.1.0__py3-none-any.whl

vortex/tools/parallelism.py

@@ -34,13 +34,13 @@ class TaylorVortexWorker(taylorism.Worker):
 
     _abstract = True
     _footprint = dict(
-        attr = dict(
-            kind = dict(),
-            taskdebug = dict(
-                info        = 'Dump all stdout/stderr to a file (in real live !)',
-                type        = bool,
-                default     = False,
-                optional    = True,
+        attr=dict(
+            kind=dict(),
+            taskdebug=dict(
+                info="Dump all stdout/stderr to a file (in real live !)",
+                type=bool,
+                default=False,
+                optional=True,
             ),
         )
     )
@@ -62,7 +62,9 @@ class TaylorVortexWorker(taylorism.Worker):
     def _task(self, **kwargs):
         """Should not be overridden anymore: see :meth:`vortex_task`."""
         self._vortex_shortcuts()
-        with ParallelSilencer(self.context, self.name, debug=self.taskdebug) as psi:
+        with ParallelSilencer(
+            self.context, self.name, debug=self.taskdebug
+        ) as psi:
             rc = self.vortex_task(**kwargs)
             psi_rc = psi.export_result()
         return self._vortex_rc_wrapup(rc, psi_rc)
@@ -77,30 +79,29 @@ class VortexWorkerBlindRun(TaylorVortexWorker):
 
     _abstract = True
     _footprint = dict(
-        attr = dict(
-            progname = dict(
+        attr=dict(
+            progname=dict(),
+            progargs=dict(
+                type=footprints.FPList,
+                default=footprints.FPList(),
+                optional=True,
             ),
-            progargs = dict(
-                type = footprints.FPList,
-                default = footprints.FPList(),
-                optional = True,
+            progtaskset=dict(
+                info="Topology/Method to set up the CPU affinity of the child task.",
+                default=None,
+                optional=True,
             ),
-            progtaskset = dict(
-                info = "Topology/Method to set up the CPU affinity of the child task.",
-                default = None,
-                optional = True,
+            progtaskset_bsize=dict(
+                info="The number of threads used by one task",
+                type=int,
+                default=1,
+                optional=True,
             ),
-            progtaskset_bsize = dict(
-                info = 'The number of threads used by one task',
-                type = int,
-                default = 1,
-                optional = True
-            ),
-            progenvdelta = dict(
-                info = 'Any alteration to environment variables',
-                type = footprints.FPDict,
-                default = footprints.FPDict({}),
-                optional = True
+            progenvdelta=dict(
+                info="Any alteration to environment variables",
+                type=footprints.FPDict,
+                default=footprints.FPDict({}),
+                optional=True,
             ),
         )
     )
@@ -115,22 +116,37 @@ class VortexWorkerBlindRun(TaylorVortexWorker):
         :param stdoutfile: Path to the file where the standard/error output will
                            be saved.
         """
-        tmpio = open(stdoutfile, 'wb')
+        tmpio = open(stdoutfile, "wb")
         try:
-            self.system.softlink('/dev/null', 'core')
+            self.system.softlink("/dev/null", "core")
         except FileExistsError:
             pass
         self.local_spawn_hook()
         self.system.default_target.spawn_hook(self.system)
         logger.info("The program stdout/err will be saved to %s", stdoutfile)
-        logger.info("Starting the following command: %s (taskset=%s, id=%d)",
-                    " ".join([self.progname, ] + self.progargs),
-                    str(self.progtaskset), self.scheduler_ticket)
-        with self.system.env.delta_context(** self.progenvdelta):
-            self.system.spawn([self.progname, ] + self.progargs, output=tmpio,
-                              fatal=True, taskset=self.progtaskset,
-                              taskset_id=self.scheduler_ticket,
-                              taskset_bsize=self.progtaskset_bsize)
+        logger.info(
+            "Starting the following command: %s (taskset=%s, id=%d)",
+            " ".join(
+                [
+                    self.progname,
+                ]
+                + self.progargs
+            ),
+            str(self.progtaskset),
+            self.scheduler_ticket,
+        )
+        with self.system.env.delta_context(**self.progenvdelta):
+            self.system.spawn(
+                [
+                    self.progname,
+                ]
+                + self.progargs,
+                output=tmpio,
+                fatal=True,
+                taskset=self.progtaskset,
+                taskset_id=self.scheduler_ticket,
+                taskset_bsize=self.progtaskset_bsize,
+            )
 
     def delayed_error_local_spawn(self, stdoutfile, rcdict):
         """local_spawn wrapped in a try/except in order to trigger delayed exceptions."""
@@ -138,13 +154,16 @@ class VortexWorkerBlindRun(TaylorVortexWorker):
             self.local_spawn(stdoutfile)
         except ExecutionError as e:
             logger.error("The execution failed.")
-            rcdict['rc'] = e
+            rcdict["rc"] = e
         return rcdict
 
     def find_namelists(self, opts=None):  # @UnusedVariable
         """Find any namelists candidates in actual context inputs."""
-        namcandidates = [x.rh for x in self.context.sequence.effective_inputs(kind='namelist')]
-        self.system.subtitle('Namelist candidates')
+        namcandidates = [
+            x.rh
+            for x in self.context.sequence.effective_inputs(kind="namelist")
+        ]
+        self.system.subtitle("Namelist candidates")
         for nam in namcandidates:
             nam.quickview()
 
@@ -158,9 +177,11 @@ class TeeLikeStringIO(io.StringIO):
         super().__init__()
         self._tees = set()
 
-    def record_teefile(self, filename, mode='w', line_buffering=True):
+    def record_teefile(self, filename, mode="w", line_buffering=True):
         """Add **filename** to the set of extra logfiles."""
-        self._tees.add(open(filename, mode=mode, buffering=int(line_buffering)))
+        self._tees.add(
+            open(filename, mode=mode, buffering=int(line_buffering))
+        )
 
     def discard_tees(self):
         """Dismiss all of the extra logfiles."""
@@ -174,7 +195,7 @@ class TeeLikeStringIO(io.StringIO):
             teeio.write(t)
         super().write(t)
 
-    def filedump(self, filename, mode='w'):
+    def filedump(self, filename, mode="w"):
         """Dump all of the captured data to **filename**."""
         with open(filename, mode=mode) as fhdump:
             self.seek(0)
@@ -207,8 +228,9 @@ class ParallelSilencer:
         """
         self._ctx = context
         self._taskdebug = debug
-        self._debugfile = '{:s}_{:s}_stdeo.txt'.format(taskname,
-                                                       date.now().ymdhms)
+        self._debugfile = "{:s}_{:s}_stdeo.txt".format(
+            taskname, date.now().ymdhms
+        )
         self._ctx_r = None
         self._io_r = io.StringIO()
         # Other temporary stuff
@@ -240,7 +262,9 @@ class ParallelSilencer:
         r_logger.addHandler(self._stream_h)
         for a_handler in self._removed_h[r_logger]:
             r_logger.removeHandler(a_handler)
-        for a_logger in [logging.getLogger(x) for x in loggers.lognames | loggers.roots]:
+        for a_logger in [
+            logging.getLogger(x) for x in loggers.lognames | loggers.roots
+        ]:
             self._removed_h[a_logger] = list(a_logger.handlers)
             for a_handler in self._removed_h[a_logger]:
                 a_logger.removeHandler(a_handler)
@@ -254,8 +278,11 @@ class ParallelSilencer:
     def __exit__(self, exctype, excvalue, exctb):  # @UnusedVariable
         """The end of a context."""
         self._stop_recording()
-        if (exctype is not None and
-                not self._taskdebug and self._io_r is not None):
+        if (
+            exctype is not None
+            and not self._taskdebug
+            and self._io_r is not None
+        ):
             # Emergency dump of the outputs (even with debug=False) !
             self._io_r.filedump(self._debugfile)
 
@@ -269,7 +296,9 @@ class ParallelSilencer:
             for a_handler in self._removed_h[r_logger]:
                 r_logger.addHandler(a_handler)
             r_logger.removeHandler(self._stream_h)
-            for a_logger in [logging.getLogger(x) for x in loggers.roots | loggers.lognames]:
+            for a_logger in [
+                logging.getLogger(x) for x in loggers.roots | loggers.lognames
+            ]:
                 for a_handler in self._removed_h.get(a_logger, ()):
                     a_logger.addHandler(a_handler)
             # flush
@@ -289,8 +318,9 @@ class ParallelSilencer:
         """
         self._stop_recording()
         self._io_r.seek(0)
-        return dict(context_record=self._ctx_r,
-                    stdoe_record=self._io_r.readlines())
+        return dict(
+            context_record=self._ctx_r, stdoe_record=self._io_r.readlines()
+        )
 
 
 class ParallelResultParser:
@@ -319,18 +349,20 @@ class ParallelResultParser:
             raise res
         else:
             sys.stdout.flush()
-            logger.info('Parallel processing results for %s', res['name'])
+            logger.info("Parallel processing results for %s", res["name"])
             # Update the context
-            logger.info('... Updating the current context ...')
-            res['report']['context_record'].replay_in(self.context)
+            logger.info("... Updating the current context ...")
+            res["report"]["context_record"].replay_in(self.context)
             # Display the stdout
-            if res['report']['stdoe_record']:
-                logger.info('... Dump of the mixed standard/error output generated by the subprocess ...')
-                for l in res['report']['stdoe_record']:
+            if res["report"]["stdoe_record"]:
+                logger.info(
+                    "... Dump of the mixed standard/error output generated by the subprocess ..."
+                )
+                for l in res["report"]["stdoe_record"]:
                     sys.stdout.write(l)
-            logger.info("... That's all for all for %s ...", res['name'])
+            logger.info("... That's all for all for %s ...", res["name"])
 
-            return res['report'].get('rc', True)
+            return res["report"].get("rc", True)
 
     def __call__(self, res):
         return self.slurp(res)

vortex/tools/prestaging.py

@@ -20,7 +20,9 @@ __all__ = []
 logger = loggers.getLogger(__name__)
 
 #: Definition of a named tuple PrestagingPriorityTuple
-PrestagingPriorityTuple = namedtuple('PrestagingPriorityTuple', ['urgent', 'normal', 'low'])
+PrestagingPriorityTuple = namedtuple(
+    "PrestagingPriorityTuple", ["urgent", "normal", "low"]
+)
 
 #: Predefined PrestagingPriorities values for urgent, normal and low
 prestaging_p = PrestagingPriorityTuple(urgent=99, normal=50, low=0)
@@ -36,23 +38,20 @@ class PrestagingTool(footprints.FootprintBase, Catalog):
     """Abstract class that deal with pre-staging for a given storage target."""
 
     _abstract = True
-    _collector = ('prestagingtool',)
+    _collector = ("prestagingtool",)
     _footprint = dict(
-        info = "Abstract class that deal with pre-staging for a given storage target.",
-        attr = dict(
-            system = dict(
-                info = "The current system object",
-                type = OSExtended
+        info="Abstract class that deal with pre-staging for a given storage target.",
+        attr=dict(
+            system=dict(info="The current system object", type=OSExtended),
+            issuerkind=dict(
+                info="The kind of store issuing the prestaging request"
             ),
-            issuerkind = dict(
-                info = 'The kind of store issuing the prestaging request'
+            priority=dict(
+                info="The prestaging request priority",
+                type=int,
+                values=list(prestaging_p),
             ),
-            priority = dict(
-                info = 'The prestaging request priority',
-                type = int,
-                values = list(prestaging_p)
-            )
-        )
+        ),
     )
 
     def __init__(self, *kargs, **kwargs):
@@ -66,12 +65,14 @@ class PrestagingTool(footprints.FootprintBase, Catalog):
 
     def describe(self, fulldump=False):
         """Print the object's characteristics and content."""
-        res = 'PrestagingTool object of class: {!s}\n'.format(self.__class__)
+        res = "PrestagingTool object of class: {!s}\n".format(self.__class__)
         for k, v in self.footprint_as_shallow_dict().items():
-            res += '  * {:s}: {!s}\n'.format(k, v)
+            res += "  * {:s}: {!s}\n".format(k, v)
         if fulldump:
-            res += '\n  * Todo list:\n'
-            res += '\n'.join(['    - {:s}'.format(item) for item in sorted(self.items())])
+            res += "\n  * Todo list:\n"
+            res += "\n".join(
+                ["    - {:s}".format(item) for item in sorted(self.items())]
+            )
         return res
 
     def flush(self, email=None):
@@ -113,24 +114,35 @@ class PrivatePrestagingHub:
         # Prestaging tool descriptions
         myptool_desc = self.prestagingtools_default_opts.copy()
         myptool_desc.update(kwargs)
-        myptool_desc['priority'] = priority
-        myptool_desc['system'] = self._sh
+        myptool_desc["priority"] = priority
+        myptool_desc["system"] = self._sh
         myptool = None
         # Scan pre-existing prestaging tools to find a suitable one
         for ptool in self._prestagingtools:
-            if ptool.footprint_reusable() and ptool.footprint_compatible(myptool_desc):
-                logger.debug("Re-usable prestaging tool found: %s", lightdump(myptool_desc))
+            if ptool.footprint_reusable() and ptool.footprint_compatible(
+                myptool_desc
+            ):
+                logger.debug(
+                    "Re-usable prestaging tool found: %s",
+                    lightdump(myptool_desc),
+                )
                 myptool = ptool
                 break
         # If necessary, create a new one
         if myptool is None:
             myptool = fpx.prestagingtool(_emptywarning=False, **myptool_desc)
             if myptool is not None:
-                logger.debug("Fresh prestaging tool created: %s", lightdump(myptool_desc))
+                logger.debug(
+                    "Fresh prestaging tool created: %s",
+                    lightdump(myptool_desc),
+                )
                 self._prestagingtools.add(myptool)
         # Let's role
         if myptool is None:
-            logger.debug("Unable to perform prestaging with: %s", lightdump(myptool_desc))
+            logger.debug(
+                "Unable to perform prestaging with: %s",
+                lightdump(myptool_desc),
+            )
         else:
             logger.debug("Prestaging requested accepted for: %s", location)
             myptool.add(location)
@@ -143,13 +155,21 @@ class PrivatePrestagingHub:
         return todo
 
     def __repr__(self, *args, **kwargs):
-        return ('{:s} | n_prestagingtools={:d}>'
-                .format(super().__repr__().rstrip('>'),
-                        len(self._prestagingtools)))
+        return "{:s} | n_prestagingtools={:d}>".format(
+            super().__repr__().rstrip(">"), len(self._prestagingtools)
+        )
 
     def __str__(self):
-        return (repr(self) + "\n\n" +
-                "\n\n".join([ptool.describe(fulldump=True) for ptool in self._prestagingtools]))
+        return (
+            repr(self)
+            + "\n\n"
+            + "\n\n".join(
+                [
+                    ptool.describe(fulldump=True)
+                    for ptool in self._prestagingtools
+                ]
+            )
+        )
 
     def flush(self, priority_threshold=prestaging_p.low):
         """Actually send the pre-staging request to the appropriate location.
@@ -163,7 +183,10 @@ class PrivatePrestagingHub:
             if rc:
                 self._prestagingtools.discard(ptool)
             else:
-                logger.error("Something went wrong when flushing the %s prestaging tool", ptool)
+                logger.error(
+                    "Something went wrong when flushing the %s prestaging tool",
+                    ptool,
+                )
 
     def clear(self, priority_threshold=prestaging_p.low):
         """Erase the pre-staging requests list.
@@ -183,4 +206,5 @@ class PrestagingHub(PrivatePrestagingHub, getbytag.GetByTag):
     Therefore, a *tag* attribute needs to be specified when building/retrieving
     an object of this class.
     """
+
     pass