vortex-nwp 2.0.0b1__py3-none-any.whl → 2.1.0__py3-none-any.whl

vortex/nwp/tools/partitioning.py

@@ -16,13 +16,16 @@ from bronx.fancies import loggers
 
 logger = loggers.getLogger(__name__)
 
-__all__ = ['PartitioningError',
-           'Rectangular2DPartitioner',
-           'setup_partitioning_in_namelist']
+__all__ = [
+    "PartitioningError",
+    "Rectangular2DPartitioner",
+    "setup_partitioning_in_namelist",
+]
 
 
 class PartitioningError(ValueError):
     """Any error raised during domain partitionings."""
+
     pass
 
 
@@ -40,24 +43,33 @@ class AbstratctPartitioner:
         :param p_method_specification: The partitioning method definition
         """
         # Process the partitioning method specification string
-        p_method_parts = p_method_specification.lower().split('_')
+        p_method_parts = p_method_specification.lower().split("_")
         self.p_method_name = None
         self.p_method_args = ()
         for a_method, a_method_n_args in self._REGISTERED_METHODS:
             if p_method_parts[0] == a_method:
                 self.p_method_name = p_method_parts[0]
                 if len(p_method_parts) - 1 != a_method_n_args:
-                    raise ValueError('Erroneous number of interger args ' +
-                                     'for the {:s} p_method ({:d} required)'
-                                     .format(a_method, a_method_n_args))
-                self.p_method_args = tuple([int(s) for s in p_method_parts[1:]])
+                    raise ValueError(
+                        "Erroneous number of interger args "
+                        + "for the {:s} p_method ({:d} required)".format(
+                            a_method, a_method_n_args
+                        )
+                    )
+                self.p_method_args = tuple(
+                    [int(s) for s in p_method_parts[1:]]
+                )
         # Unknown method -> crash
         if self.p_method_name is None:
-            raise PartitioningError("Unknown partitioning method ({:s})."
-                                    .format(p_method_specification))
+            raise PartitioningError(
+                "Unknown partitioning method ({:s}).".format(
+                    p_method_specification
+                )
+            )
         # The actual class' method that will be used to compute a and b
-        self.p_method = functools.partial(getattr(self, '_' + self.p_method_name),
-                                          * self.p_method_args)
+        self.p_method = functools.partial(
+            getattr(self, "_" + self.p_method_name), *self.p_method_args
+        )
         # Implement a caching mechanism
         self._p_cache = dict()
 
@@ -124,10 +136,10 @@ class Rectangular2DPartitioner(AbstratctPartitioner):
     """
 
     _REGISTERED_METHODS = (
-        ('xcloseto', 1),
-        ('ycloseto', 1),
-        ('square', 0),
-        ('aspect', 2)
+        ("xcloseto", 1),
+        ("ycloseto", 1),
+        ("square", 0),
+        ("aspect", 2),
     )
 
     @staticmethod
@@ -143,8 +155,7 @@ class Rectangular2DPartitioner(AbstratctPartitioner):
         """Find ``x`` as the closest possible value to **close_to_what**."""
         guesses = b_iter.interleave(
             range(close_to_what, 0, -1),
-            range(close_to_what + 1, min(close_to_what * 2,
-                                         ntasks))
+            range(close_to_what + 1, min(close_to_what * 2, ntasks)),
         )
         return self._test_and_return(ntasks, guesses)
 
@@ -164,17 +175,15 @@ class Rectangular2DPartitioner(AbstratctPartitioner):
     def _aspect(self, x_spec, y_spec, ntasks):
         """Find ``x`` and ``y`` so that ``x / y =~ x_spec / y_spec``."""
         aspect_ratio = x_spec / y_spec
-        return self._xcloseto(int(math.sqrt(ntasks * aspect_ratio)),
-                              ntasks)
+        return self._xcloseto(int(math.sqrt(ntasks * aspect_ratio)), ntasks)
 
 
 _PARTITIONERS_CACHE = dict()
 
 
-def setup_partitioning_in_namelist(namcontents,
-                                   effective_tasks,
-                                   effective_threads,
-                                   namlocal=None):
+def setup_partitioning_in_namelist(
+    namcontents, effective_tasks, effective_threads, namlocal=None
+):
     """Look in a namelist Content object and replace the macros related to partitioning.
 
     :param nwp.data.namelists.NamelistContent namcontents: The namelist's Content
@@ -199,10 +208,12 @@ def setup_partitioning_in_namelist(namcontents,
       partitioning class. Any value that is accepted by the partitioning class is
       fine.
     """
-    macrovalid = re.compile('PART_' +
-                            '(?P<what>TASKS|THREADS)(?P<cls>2D)_' +
-                            '(?P<dim>[XY])_(?P<def>.*)$')
-    partitioning_classes = {'2D': Rectangular2DPartitioner}
+    macrovalid = re.compile(
+        "PART_"
+        + "(?P<what>TASKS|THREADS)(?P<cls>2D)_"
+        + "(?P<dim>[XY])_(?P<def>.*)$"
+    )
+    partitioning_classes = {"2D": Rectangular2DPartitioner}
     namw = False
     # Find the list of existing macros
     all_macros = set()
@@ -212,23 +223,36 @@ def setup_partitioning_in_namelist(namcontents,
     for macroname in all_macros:
         macroname_re = macrovalid.match(macroname)
         if macroname_re:
-            cache_key = (macroname_re.group('cls'), macroname_re.group('def'))
+            cache_key = (macroname_re.group("cls"), macroname_re.group("def"))
             if cache_key not in _PARTITIONERS_CACHE:
-                partitioning_class = partitioning_classes[macroname_re.group('cls')]
-                _PARTITIONERS_CACHE[cache_key] = partitioning_class(macroname_re.group('def'))
-            effective_n = dict(TASKS=effective_tasks,
-                               THREADS=effective_threads)[macroname_re.group('what')]
+                partitioning_class = partitioning_classes[
+                    macroname_re.group("cls")
+                ]
+                _PARTITIONERS_CACHE[cache_key] = partitioning_class(
+                    macroname_re.group("def")
+                )
+            effective_n = dict(
+                TASKS=effective_tasks, THREADS=effective_threads
+            )[macroname_re.group("what")]
             part_x, part_y = _PARTITIONERS_CACHE[cache_key](effective_n)
-            final_result = part_x if macroname_re.group('dim') == 'X' else part_y
+            final_result = (
+                part_x if macroname_re.group("dim") == "X" else part_y
+            )
             if namlocal:
-                logger.info('Setup macro %s=%s in %s', macroname, final_result, namlocal)
+                logger.info(
+                    "Setup macro %s=%s in %s",
+                    macroname,
+                    final_result,
+                    namlocal,
+                )
             else:
-                logger.info('Setup macro %s=%s', macroname, final_result)
+                logger.info("Setup macro %s=%s", macroname, final_result)
             namcontents.setmacro(macroname, final_result)
             namw = True
     return namw
 
 
-if __name__ == '__main__':
+if __name__ == "__main__":
     import doctest
+
     doctest.testmod()

vortex/nwp/tools/satrad.py

@@ -29,28 +29,43 @@ class SatRadDecoMixin(AlgoComponentDecoMixin):
 
     def _satrad_coeffdir_setup(self, rh, opts):  # @UnusedVariable
         """Look for RTTOV coefficient files and act on it."""
-        rtcoefs = self.context.sequence.effective_inputs(role='RtCoef', kind='rtcoef')
+        rtcoefs = self.context.sequence.effective_inputs(
+            role="RtCoef", kind="rtcoef"
+        )
         if rtcoefs:
             sh = self.system
-            rtpaths = {sh.path.dirname(sh.path.realpath(rtcoef.rh.container.localpath()))
-                       for rtcoef in rtcoefs}
+            rtpaths = {
+                sh.path.dirname(
+                    sh.path.realpath(rtcoef.rh.container.localpath())
+                )
+                for rtcoef in rtcoefs
+            }
             if len(rtpaths) != 1:
-                raise AlgoComponentError('The Radiative Transfer Coefficients are scattered in' +
-                                         'several directories: {!s}'.format(rtpaths))
+                raise AlgoComponentError(
+                    "The Radiative Transfer Coefficients are scattered in"
+                    + "several directories: {!s}".format(rtpaths)
+                )
             rtpath = rtpaths.pop()
-            logger.info('Setting %s = %s', 'RTTOV_COEFDIR', rtpath)
-            self.env['RTTOV_COEFDIR'] = rtpath
+            logger.info("Setting %s = %s", "RTTOV_COEFDIR", rtpath)
+            self.env["RTTOV_COEFDIR"] = rtpath
 
-    _MIXIN_PREPARE_HOOKS = (_satrad_coeffdir_setup, )
+    _MIXIN_PREPARE_HOOKS = (_satrad_coeffdir_setup,)
 
     def setchannels(self):
         """Look up for channels namelists in effective inputs."""
         namchan = [
-            x.rh for x in self.context.sequence.effective_inputs(kind='namelist')
-            if 'channel' in x.rh.options
+            x.rh
+            for x in self.context.sequence.effective_inputs(kind="namelist")
+            if "channel" in x.rh.options
         ]
         for thisnam in namchan:
-            thisloc = re.sub(r'\d+$', '', thisnam.options['channel']) + 'channels'
+            thisloc = (
+                re.sub(r"\d+$", "", thisnam.options["channel"]) + "channels"
+            )
             if thisloc != thisnam.container.localpath():
-                logger.info('Linking < %s > to < %s >', thisnam.container.localpath(), thisloc)
+                logger.info(
+                    "Linking < %s > to < %s >",
+                    thisnam.container.localpath(),
+                    thisloc,
+                )
                 self.system.softlink(thisnam.container.localpath(), thisloc)

vortex/nwp/util/async.py

@@ -18,7 +18,7 @@ def _double_ssh(sh, loginnode, transfernode):
 
     May return None when network problems occur.
     """
-    cmd = ['ssh', '-x', loginnode, 'ssh', '-x', transfernode, 'hostname', '-s']
+    cmd = ["ssh", "-x", loginnode, "ssh", "-x", transfernode, "hostname", "-s"]
     rc = sh.spawn(cmd, shell=False, output=True, fatal=False)
     if not rc:
         return None
@@ -33,54 +33,73 @@ def system_ftput(pnum, ask, config, logger, **opts):
     obtained by a double ssh.
     """
 
-    logger.info('System', todo=ask.todo, pnum=pnum, opts=opts)
-    value = dict(rpool='retry')
+    logger.info("System", todo=ask.todo, pnum=pnum, opts=opts)
+    value = dict(rpool="retry")
 
-    phasemode = opts.get('phasemode', False)
-    nbtries = opts.get('attempts', 1)
+    phasemode = opts.get("phasemode", False)
+    nbtries = opts.get("attempts", 1)
     if phasemode:
         rawftput = False
     else:
-        rawftput = opts.get('rawftput', False)
+        rawftput = opts.get("rawftput", False)
     trynum = 0
 
-    profile = config['driver'].get('profile', None)
+    profile = config["driver"].get("profile", None)
     with VortexWorker(logger=logger, profile=profile) as vwork:
         sh = vwork.session.sh
-        sh.trace = 'log'
-        sh.ftpflavour = systems.FTP_FLAVOUR.STD  # Because errors are handled directly by jeeves
+        sh.trace = "log"
+        sh.ftpflavour = (
+            systems.FTP_FLAVOUR.STD
+        )  # Because errors are handled directly by jeeves
 
         data = vwork.get_dataset(ask)
-        logger.info('ftput', source=data.source, destination=data.destination)
+        logger.info("ftput", source=data.source, destination=data.destination)
         if not sh.path.exists(data.source):
-            logger.error('The source file is missing - sorry')
-            return pnum, False, dict(rpool='error')
+            logger.error("The source file is missing - sorry")
+            return pnum, False, dict(rpool="error")
 
         if phasemode:
-            data.hostname = _double_ssh(sh, data.phase_loginnode, data.phase_transfernode)
+            data.hostname = _double_ssh(
+                sh, data.phase_loginnode, data.phase_transfernode
+            )
             if data.hostname is None:
-                return pnum, False, dict(rpool='retry')
-
-        cpipeline = (None if not hasattr(data, 'cpipeline') or not data.cpipeline
-                     else compression.CompressionPipeline(sh, data.cpipeline))
-
-        logger.info('FTPut host', hostname=data.hostname, logname=data.logname)
-        logger.info('FTPut data', source=data.source, destination=data.destination)
+                return pnum, False, dict(rpool="retry")
+
+        cpipeline = (
+            None
+            if not hasattr(data, "cpipeline") or not data.cpipeline
+            else compression.CompressionPipeline(sh, data.cpipeline)
+        )
+
+        logger.info("FTPut host", hostname=data.hostname, logname=data.logname)
+        logger.info(
+            "FTPut data", source=data.source, destination=data.destination
+        )
         while trynum < nbtries:
             trynum += 1
             if nbtries > 1:
-                logger.info('FTPut loop', attempt=trynum)
+                logger.info("FTPut loop", attempt=trynum)
             try:
                 if rawftput:
-                    putrc = sh.rawftput(data.source, data.destination, hostname=data.hostname,
-                                        logname=data.logname, cpipeline=cpipeline,
-                                        fmt=data.fmt)
+                    putrc = sh.rawftput(
+                        data.source,
+                        data.destination,
+                        hostname=data.hostname,
+                        logname=data.logname,
+                        cpipeline=cpipeline,
+                        fmt=data.fmt,
+                    )
                 else:
-                    putrc = sh.ftput(data.source, data.destination, hostname=data.hostname,
-                                     logname=data.logname, cpipeline=cpipeline,
-                                     fmt=data.fmt)
+                    putrc = sh.ftput(
+                        data.source,
+                        data.destination,
+                        hostname=data.hostname,
+                        logname=data.logname,
+                        cpipeline=cpipeline,
+                        fmt=data.fmt,
+                    )
             except Exception as e:
-                logger.warning('FTPut failed', attempt=trynum, error=e)
+                logger.warning("FTPut failed", attempt=trynum, error=e)
                 putrc = False
             if putrc:
                 value = dict(clear=sh.rm(data.source, fmt=data.fmt))
@@ -92,23 +111,23 @@ def system_ftput(pnum, ask, config, logger, **opts):
 def system_cp(pnum, ask, config, logger, **opts):
     """Local transfers (between filesystems) on a given host."""
 
-    logger.info('System', todo=ask.todo, pnum=pnum, opts=opts)
-    value = dict(rpool='retry')
+    logger.info("System", todo=ask.todo, pnum=pnum, opts=opts)
+    value = dict(rpool="retry")
 
-    profile = config['driver'].get('profile', None)
+    profile = config["driver"].get("profile", None)
     with VortexWorker(logger=logger, profile=profile) as vwork:
         sh = vwork.session.sh
-        sh.trace = 'log'
+        sh.trace = "log"
         data = vwork.get_dataset(ask)
-        logger.info('cp', source=data.source, destination=data.destination)
+        logger.info("cp", source=data.source, destination=data.destination)
         if not sh.path.exists(data.source):
-            logger.error('The source file is missing - sorry')
-            return pnum, False, dict(rpool='error')
+            logger.error("The source file is missing - sorry")
+            return pnum, False, dict(rpool="error")
 
         try:
             rc = sh.cp(data.source, data.destination, fmt=data.fmt)
         except Exception as e:
-            logger.warning('cp failed', error=e)
+            logger.warning("cp failed", error=e)
             rc = False
         if rc:
             value = dict(clear=sh.rm(data.source, fmt=data.fmt))
@@ -123,33 +142,42 @@ def system_scp(pnum, ask, config, logger, **opts):
     In phase mode, raw ftp is not allowed, and the hostname is dynamically
     obtained by a double ssh.
     """
-    logger.info('System', todo=ask.todo, pnum=pnum, opts=opts)
-    value = dict(rpool='retry')
+    logger.info("System", todo=ask.todo, pnum=pnum, opts=opts)
+    value = dict(rpool="retry")
 
-    phasemode = opts.get('phasemode', False)
+    phasemode = opts.get("phasemode", False)
 
-    profile = config['driver'].get('profile', None)
+    profile = config["driver"].get("profile", None)
     with VortexWorker(logger=logger, profile=profile) as vwork:
         sh = vwork.session.sh
-        sh.trace = 'log'
+        sh.trace = "log"
 
         data = vwork.get_dataset(ask)
-        logger.info('scp', source=data.source, destination=data.destination)
+        logger.info("scp", source=data.source, destination=data.destination)
         if not sh.path.exists(data.source):
-            logger.error('The source file is missing - sorry')
-            return pnum, False, dict(rpool='error')
+            logger.error("The source file is missing - sorry")
+            return pnum, False, dict(rpool="error")
 
         if phasemode:
-            data.hostname = _double_ssh(sh, data.phase_loginnode, data.phase_transfernode)
+            data.hostname = _double_ssh(
+                sh, data.phase_loginnode, data.phase_transfernode
+            )
             if data.hostname is None:
                 return pnum, False, value
-        logger.info('scp host', hostname=data.hostname, logname=data.logname)
-        logger.info('scp data', source=data.source, destination=data.destination)
+        logger.info("scp host", hostname=data.hostname, logname=data.logname)
+        logger.info(
+            "scp data", source=data.source, destination=data.destination
+        )
         try:
-            putrc = sh.scpput(data.source, data.destination, hostname=data.hostname,
-                              logname=data.logname, fmt=data.fmt)
+            putrc = sh.scpput(
+                data.source,
+                data.destination,
+                hostname=data.hostname,
+                logname=data.logname,
+                fmt=data.fmt,
+            )
         except Exception as e:
-            logger.warning('scp failed', error=e)
+            logger.warning("scp failed", error=e)
             putrc = False
         if putrc:
             value = dict(clear=sh.rm(data.source, fmt=data.fmt))
@@ -162,23 +190,23 @@ def system_noop(pnum, ask, config, logger, **opts):
 
     Used to desactivate jeeves when mirroring the operational suite.
     """
-    logger.info('Noop', todo=ask.todo, pnum=pnum, opts=opts)
+    logger.info("Noop", todo=ask.todo, pnum=pnum, opts=opts)
 
-    profile = config['driver'].get('profile', None)
+    profile = config["driver"].get("profile", None)
     with VortexWorker(logger=logger, profile=profile) as vwork:
         sh = vwork.session.sh
-        sh.trace = 'log'
+        sh.trace = "log"
         data = vwork.get_dataset(ask)
         value = dict(clear=sh.rm(data.source, fmt=data.fmt))
 
     return pnum, vwork.rc, value
 
 
-if __name__ == '__main__':
+if __name__ == "__main__":
     import vortex
 
     t = vortex.ticket()
     main_sh = t.sh
     main_sh.trace = True
     main_sh.verbose = True
-    print(_double_ssh(main_sh, 'beaufixoper', 'beaufixtransfert-agt'))
+    print(_double_ssh(main_sh, "beaufixoper", "beaufixtransfert-agt"))

vortex/nwp/util/beacon.py

@@ -23,18 +23,18 @@ def beaconfunction(options):
     rst = dict()
 
     # Find out if a resource handler is present and load the elements to be written
-    rhdict = options.get('rhandler', None)
+    rhdict = options.get("rhandler", None)
     if rhdict:
-        rst['vapp'] = rhdict.get('provider', {}).get('vapp', '')
-        rst['vconf'] = rhdict.get('provider', {}).get('vconf', '')
-        rst['model'] = rhdict.get('resource', {}).get('model', '')
-        rst['date'] = rhdict.get('resource', {}).get('date', '')
-        rst['cutoff'] = rhdict.get('resource', {}).get('cutoff', '')
-        member = rhdict.get('provider', {}).get('member', None)
+        rst["vapp"] = rhdict.get("provider", {}).get("vapp", "")
+        rst["vconf"] = rhdict.get("provider", {}).get("vconf", "")
+        rst["model"] = rhdict.get("resource", {}).get("model", "")
+        rst["date"] = rhdict.get("resource", {}).get("date", "")
+        rst["cutoff"] = rhdict.get("resource", {}).get("cutoff", "")
+        member = rhdict.get("provider", {}).get("member", None)
         if member is not None:
-            rst['member'] = member
+            rst["member"] = member
     else:
-        rst['error'] = 'No resource handler here'
+        rst["error"] = "No resource handler here"
     outstr = json.dumps(rst)
     # Return the string, which has to be converted to a file like object
-    return io.BytesIO(outstr.encode(encoding='utf_8'))
+    return io.BytesIO(outstr.encode(encoding="utf_8"))