vortex-nwp 2.0.0b1__py3-none-any.whl → 2.1.0__py3-none-any.whl

vortex/nwp/tools/drhook.py

@@ -5,7 +5,11 @@ Common interest classes to help setup the DrHook library environment.
 import footprints
 from bronx.fancies import loggers
 
-from vortex.algo.components import AlgoComponentDecoMixin, Parallel, algo_component_deco_mixin_autodoc
+from vortex.algo.components import (
+    AlgoComponentDecoMixin,
+    Parallel,
+    algo_component_deco_mixin_autodoc,
+)
 
 #: No automatic export
 __all__ = []
@@ -28,14 +32,15 @@ class DrHookDecoMixin(AlgoComponentDecoMixin):
         footprints.Footprint(
             attr=dict(
                 drhookprof=dict(
-                    info='Activate the DrHook profiling.',
+                    info="Activate the DrHook profiling.",
                     optional=True,
                     type=bool,
                     default=False,
                     doc_zorder=-50,
                 ),
             ),
-        )]
+        )
+    ]
 
     def _drhook_varexport(self, rh, opts):  # @UnusedVariable
         """Export proper DrHook variables"""
@@ -45,8 +50,8 @@ class DrHookDecoMixin(AlgoComponentDecoMixin):
                 ("DR_HOOK_OPT", "prof"),
                 ("DR_HOOK_IGNORE_SIGNALS", "-1"),
             ]
-            if self.drhookprof else
-            [("DR_HOOK", "0"), ("DR_HOOK_IGNORE_SIGNALS", "-1")]
+            if self.drhookprof
+            else [("DR_HOOK", "0"), ("DR_HOOK_IGNORE_SIGNALS", "-1")]
         )
         if not isinstance(self, Parallel):
             drhook_vars += [
@@ -55,8 +60,7 @@ class DrHookDecoMixin(AlgoComponentDecoMixin):
                 ("DR_HOOK_ASSERT_MPI_INITIALIZED", "0"),
             ]
         for k, v in drhook_vars:
-            logger.info('Setting DRHOOK env %s = %s', k, v)
+            logger.info("Setting DRHOOK env %s = %s", k, v)
             self.env[k] = v
 
-
-    _MIXIN_PREPARE_HOOKS = (_drhook_varexport, )
+    _MIXIN_PREPARE_HOOKS = (_drhook_varexport,)

vortex/nwp/tools/grib.py

@@ -71,7 +71,9 @@ class _GenericFilter:
             elif isinstance(a_filter, str):
                 self._filters.append(json.loads(a_filter))
             elif isinstance(a_filter, Context):
-                for a_request in a_filter.sequence.effective_inputs(kind='filtering_request'):
+                for a_request in a_filter.sequence.effective_inputs(
+                    kind="filtering_request"
+                ):
                     self._filters.append(a_request.rh.contents.data)
 
     def __len__(self):
@@ -84,7 +86,7 @@ class _GenericFilter:
         superset_ok = True
         for k, v in subset.items():
             # Ignore the comments...
-            if k.startswith('comment'):
+            if k.startswith("comment"):
                 continue
             # Check for the key inside the full dictionary
             try:
@@ -105,12 +107,14 @@ class _GenericFilter:
 
     def _filter_process(self, fid, a_filter):
         """Check if the data's fid complies with the filter."""
-        includes = a_filter.get('fields_include', [])
-        excludes = a_filter.get('fields_exclude', [])
+        includes = a_filter.get("fields_include", [])
+        excludes = a_filter.get("fields_exclude", [])
         try:
-            fid = fid[a_filter['fid_format']]
+            fid = fid[a_filter["fid_format"]]
         except KeyboardInterrupt:
-            raise ValueError("Please specify a valid fid_format in the filter description")
+            raise ValueError(
+                "Please specify a valid fid_format in the filter description"
+            )
         # First process includes
         includes_ok = True
         for include in includes:
@@ -127,13 +131,13 @@ class _GenericFilter:
 
     def __call__(self, inputfile, outfile_fmt):
         """Apply the various filters on *inputfile*. Should be implemented..."""
-        raise NotImplementedError('This method have to be overwritten.')
+        raise NotImplementedError("This method have to be overwritten.")
 
 
 class GRIBFilter(_GenericFilter):
     """Class in charge of filtering GRIB files."""
 
-    CONCATENATE_FILTER = 'concatenate'
+    CONCATENATE_FILTER = "concatenate"
 
     def __init__(self, concatenate=False):
         """
@@ -142,7 +146,7 @@ class GRIBFilter(_GenericFilter):
         """
         super().__init__()
         self.concatenate = concatenate
-        self._xgrib_support = 'grib' in self._sh.loaded_addons()
+        self._xgrib_support = "grib" in self._sh.loaded_addons()
 
     def __len__(self):
         """Returns the number of active filters (concatenate included)."""
@@ -151,16 +155,21 @@ class GRIBFilter(_GenericFilter):
     def _simple_cat(self, gribfile, outfile_fmt, intent):
         """Just concatenate a multipart GRIB."""
         if self._xgrib_support and self._sh.is_xgrib(gribfile):
-            self._sh.xgrib_pack(gribfile,
-                                outfile_fmt.format(filtername=self.CONCATENATE_FILTER),
-                                intent=intent)
+            self._sh.xgrib_pack(
+                gribfile,
+                outfile_fmt.format(filtername=self.CONCATENATE_FILTER),
+                intent=intent,
+            )
         else:
             # Just make a copy with the appropriate name...
-            self._sh.cp(gribfile,
-                        outfile_fmt.format(filtername=self.CONCATENATE_FILTER),
-                        intent=intent, fmt='grib')
-
-    def __call__(self, gribfile, outfile_fmt, intent='in'):
+            self._sh.cp(
+                gribfile,
+                outfile_fmt.format(filtername=self.CONCATENATE_FILTER),
+                intent=intent,
+                fmt="grib",
+            )
+
+    def __call__(self, gribfile, outfile_fmt, intent="in"):
         """Apply the various filters on *gribfile*.
 
         :param gribfile: The path to the input GRIB file
@@ -178,52 +187,63 @@ class GRIBFilter(_GenericFilter):
         if not self._filters:
             if self.concatenate:
                 self._simple_cat(gribfile, outfile_fmt, intent=intent)
-                return [outfile_fmt.format(filtername=self.CONCATENATE_FILTER), ]
+                return [
+                    outfile_fmt.format(filtername=self.CONCATENATE_FILTER),
+                ]
             else:
                 raise ValueError("Set concatenate=True or provide a filter.")
 
         # Open the input file using Epygram
         from ..util import usepygram
-        if not usepygram.epygram_checker.is_available(version='1.0.0'):
+
+        if not usepygram.epygram_checker.is_available(version="1.0.0"):
             raise AlgoComponentError("Epygram (v1.0.0) needs to be available")
 
         if self._xgrib_support and self._sh.is_xgrib(gribfile):
             idx = self._sh.xgrib_index_get(gribfile)
-            in_data = [footprints.proxy.dataformat(
-                filename=self._sh.path.realpath(a_gribfile),
-                openmode='r',
-                format='GRIB',
-            ) for a_gribfile in idx]
+            in_data = [
+                footprints.proxy.dataformat(
+                    filename=self._sh.path.realpath(a_gribfile),
+                    openmode="r",
+                    format="GRIB",
+                )
+                for a_gribfile in idx
+            ]
         else:
-            in_data = [footprints.proxy.dataformat(
-                filename=self._sh.path.realpath(gribfile),
-                openmode='r',
-                format='GRIB',
-            ), ]
+            in_data = [
+                footprints.proxy.dataformat(
+                    filename=self._sh.path.realpath(gribfile),
+                    openmode="r",
+                    format="GRIB",
+                ),
+            ]
 
         # Open output files
         out_data = list()
         out_filelist = list()
         for a_filter in self._filters:
-            f_name = outfile_fmt.format(filtername=a_filter['filter_name'])
+            f_name = outfile_fmt.format(filtername=a_filter["filter_name"])
             out_filelist.append(f_name)
             # It would be a lot better to use io.open but grib_api is very annoying !
-            out_data.append(open(f_name, 'wb'))
+            out_data.append(open(f_name, "wb"))
         if self.concatenate:
             f_name = outfile_fmt.format(filtername=self.CONCATENATE_FILTER)
             out_filelist.append(f_name)
             # It would be a lot better to use io.open but grib_api is very annoying !
-            out_cat = open(f_name, 'wb')
+            out_cat = open(f_name, "wb")
 
         with usepygram.epy_env_prepare(sessions.current()):
             for a_in_data in in_data:
                 msg = a_in_data.iter_messages(headers_only=False)
                 while msg is not None:
-                    for (a_out_data, a_filter) in zip(out_data, self._filters):
+                    for a_out_data, a_filter in zip(out_data, self._filters):
                         thefid = msg.genfid()
                         if self._filter_process(thefid, a_filter):
-                            logger.debug("Select succeed for filter %s: %s",
-                                         a_filter['filter_name'], thefid)
+                            logger.debug(
+                                "Select succeed for filter %s: %s",
+                                a_filter["filter_name"],
+                                thefid,
+                            )
                             msg.write_to_file(a_out_data)
                     if self.concatenate:
                         msg.write_to_file(out_cat)
@@ -248,14 +268,16 @@ def grib_inplace_cat(t, rh):
     :param t: A :class:`vortex.sessions.Ticket` object
     :param rh: A :class:`vortex.data.handlers.Handler` object
     """
-    xgrib_support = 'grib' in t.sh.loaded_addons()
+    xgrib_support = "grib" in t.sh.loaded_addons()
     if xgrib_support:
         if t.sh.is_xgrib(rh.container.localpath()):
             # Some cleanup...
             rh.reset_contents()
             rh.container.close()
             # Move the index file prior to the concatenation
-            tmpfile = rh.container.localpath() + '_concat' + t.sh.safe_filesuffix()
+            tmpfile = (
+                rh.container.localpath() + "_concat" + t.sh.safe_filesuffix()
+            )
             t.sh.move(rh.container.localpath(), tmpfile)
             # Concatenate
             t.sh.xgrib_pack(tmpfile, rh.container.localpath())
@@ -263,6 +285,10 @@ def grib_inplace_cat(t, rh):
             t.sh.grib_remove(tmpfile)
             logger.info("The multipart GRIB has been concatenated.")
         else:
-            logger.info("The localpath is not a multipart GRIB: nothing to do.")
+            logger.info(
+                "The localpath is not a multipart GRIB: nothing to do."
+            )
     else:
-        logger.info("Multipart GRIB support is not activated: nothing can be done.")
+        logger.info(
+            "Multipart GRIB support is not activated: nothing can be done."
+        )

vortex/nwp/tools/gribdiff.py

@@ -18,7 +18,7 @@ class _GRIBDIFF_Plus_St:
         self._result = result
 
     def __str__(self):
-        return '{:s} | rc={:d}>'.format(repr(self).rstrip('>'), self.rc)
+        return "{:s} | rc={:d}>".format(repr(self).rstrip(">"), self.rc)
 
     @property
     def result(self):
@@ -38,8 +38,9 @@ class _GRIBDIFF_Plus_Res:
         self._epydiff_res = epydiff_res
 
     def __str__(self):
-        return ('{0:s} | gribapi_rc={1:d} epydiff_done={2:d}>'.
-                format(repr(self).rstrip('>'), self._gapi, self._epydiff))
+        return "{0:s} | gribapi_rc={1:d} epydiff_done={2:d}>".format(
+            repr(self).rstrip(">"), self._gapi, self._epydiff
+        )
 
     def differences(self):
         """Print detailed informations about the diff."""
@@ -52,17 +53,17 @@ class GRIBDIFF_Plus(GRIBAPI_Tool):
     """
 
     _footprint = dict(
-        info = 'Default GRIBAPI system interface',
-        attr = dict(
-            maxepydiff = dict(
-                info = 'Epygram diffs are costfull, they will run only maxepydiff times',
-                type = int,
-                default = 2,
-                optional = True,
+        info="Default GRIBAPI system interface",
+        attr=dict(
+            maxepydiff=dict(
+                info="Epygram diffs are costfull, they will run only maxepydiff times",
+                type=int,
+                default=2,
+                optional=True,
             ),
         ),
-        priority = dict(
-            level = footprints.priorities.top.TOOLBOX  # @UndefinedVariable
+        priority=dict(
+            level=footprints.priorities.top.TOOLBOX  # @UndefinedVariable
         ),
     )
 
@@ -76,10 +77,12 @@ class GRIBDIFF_Plus(GRIBAPI_Tool):
         if not rc:
             if self._epyavail is None:
                 from ..util.usepygram import epygram_checker
-                self._epyavail = epygram_checker.is_available(version='1.0.0')
+
+                self._epyavail = epygram_checker.is_available(version="1.0.0")
             if self._epyavail:
                 if self._epycount < self.maxepydiff:
                     from ..util.diffpygram import EpyGribDiff
+
                     gdiff = EpyGribDiff(grib2, grib1)  # Ref file is first...
                     self._epycount += 1
                     res = _GRIBDIFF_Plus_Res(rc, True, str(gdiff))
@@ -88,12 +91,14 @@ class GRIBDIFF_Plus(GRIBAPI_Tool):
                         outfh.write(gdiff.format_diff(detailed=True))
                 else:
                     res = _GRIBDIFF_Plus_Res(
-                        rc, False,
-                        "grib_compare failed (but the Epygram diffs max number is exceeded...)"
+                        rc,
+                        False,
+                        "grib_compare failed (but the Epygram diffs max number is exceeded...)",
                     )
             else:
-                res = _GRIBDIFF_Plus_Res(rc, False,
-                                         "grib_compare failed (Epygram unavailable)")
+                res = _GRIBDIFF_Plus_Res(
+                    rc, False, "grib_compare failed (Epygram unavailable)"
+                )
         else:
             res = _GRIBDIFF_Plus_Res(rc, False, "")
         return _GRIBDIFF_Plus_St(rc, res)

vortex/nwp/tools/ifstools.py

@@ -30,7 +30,7 @@ class _IfsOutputsTimesListDesc:
             instance._tlists_store.pop(self._attr, None)
         else:
             if not isinstance(value, list):
-                raise ValueError('**value** should be a list.')
+                raise ValueError("**value** should be a list.")
             instance._tlists_store[self._attr] = [Time(t) for t in value]
 
     def __delete__(self, instance):
@@ -41,40 +41,41 @@ class IfsOutputsAbstractConfigurator(footprints.FootprintBase):
     """Abstract utility class to configure the IFS model regarding output data."""
 
     _abstract = True
-    _collector = ('ifsoutputs_configurator', )
+    _collector = ("ifsoutputs_configurator",)
     _footprint = [
         model,
         arpifs_cycle,
         dict(
             attr=dict(
-                fcterm_unit = dict(
-                    info     = 'The unit used in the *fcterm* attribute.',
-                    values   = ['h', 't']
+                fcterm_unit=dict(
+                    info="The unit used in the *fcterm* attribute.",
+                    values=["h", "t"],
                 ),
             )
-        )
+        ),
     ]
 
-    def __init__(self, * kargs, ** kwargs):
-        super().__init__(* kargs, ** kwargs)
+    def __init__(self, *kargs, **kwargs):
+        super().__init__(*kargs, **kwargs)
         self._tlists_store = dict()
 
     modelstate = _IfsOutputsTimesListDesc(
-        "modelstate",
-        "The list of terms for modelstate outputs."
+        "modelstate", "The list of terms for modelstate outputs."
     )
 
     surf_modelstate = _IfsOutputsTimesListDesc(
         "surf_modelstate",
-        "The list of terms for surface scheme modelstate outputs.")
+        "The list of terms for surface scheme modelstate outputs.",
+    )
 
     spectral_diag = _IfsOutputsTimesListDesc(
         "spectral_diag",
-        "The list of terms for spectral space diagnostics outputs.")
+        "The list of terms for spectral space diagnostics outputs.",
+    )
 
     post_processing = _IfsOutputsTimesListDesc(
         "post_processing",
-        "The list of terms for inline post-processing outputs."
+        "The list of terms for inline post-processing outputs.",
     )
 
     def _setup_nam_obj(self, namelist_object, namelist_name):
@@ -105,59 +106,98 @@ class IfsOutputsConfigurator(IfsOutputsAbstractConfigurator):
     def _set_namvar_value(namblock, var, value, namname):
         """Set a value in a **namblock** namelist and log it."""
         namblock[var] = value
-        logger.info('Setup &%s %s=%s / (file: %s)',
-                    namblock.name, var, namblock.nice(value), namname)
+        logger.info(
+            "Setup &%s %s=%s / (file: %s)",
+            namblock.name,
+            var,
+            namblock.nice(value),
+            namname,
+        )
 
     @staticmethod
     def _clean_namvar(namblock, var, namname):
         """Clean the **var** value from the **namblock** namelist."""
-        todo = {k for k in namblock.keys() if re.match(var + r'($|\(|%)', k)}
+        todo = {k for k in namblock.keys() if re.match(var + r"($|\(|%)", k)}
         if todo:
             for k in todo:
                 namblock.delvar(k)
-            logger.info('Cleaning %s variable in namelist &%s (file: %s)',
-                        var, namblock.name, namname)
+            logger.info(
+                "Cleaning %s variable in namelist &%s (file: %s)",
+                var,
+                namblock.name,
+                namname,
+            )
 
     def _generic_terms_setup(self, namct0, namct1, what, terms, namname):
         """Setup a given kind of output data (in a generic way)."""
         if terms is not None:
-            sign = -1 if self.fcterm_unit == 'h' else 1
+            sign = -1 if self.fcterm_unit == "h" else 1
             with_minutes = any([t.minute > 0 for t in terms])
-            self._clean_namvar(namct0, 'NFR{:s}'.format(what), namname)
-            self._clean_namvar(namct0, 'N{:s}TS'.format(what), namname)
-            self._clean_namvar(namct0, 'N{:s}TSMIN'.format(what), namname)
-            self._set_namvar_value(namct1, 'N1{:s}'.format(what), 1 if terms else 0, namname)
+            self._clean_namvar(namct0, "NFR{:s}".format(what), namname)
+            self._clean_namvar(namct0, "N{:s}TS".format(what), namname)
+            self._clean_namvar(namct0, "N{:s}TSMIN".format(what), namname)
+            self._set_namvar_value(
+                namct1, "N1{:s}".format(what), 1 if terms else 0, namname
+            )
             if terms:
-                self._set_namvar_value(namct0, 'N{:s}TS(0)'.format(what), sign * len(terms), namname)
+                self._set_namvar_value(
+                    namct0,
+                    "N{:s}TS(0)".format(what),
+                    sign * len(terms),
+                    namname,
+                )
                 if with_minutes:
-                    if 'cy46' <= self.cycle < 'cy47':  # Temporary fix for cy46 only
-                        self._set_namvar_value(namct0, 'N{:s}TSMIN(0)'.format(what),
-                                               len(terms), namname)
-                logger.info('Setting up N%sTS and N%sTSMIN in &%s (file: %s)',
-                            what, what, namct0.name, namname)
+                    if (
+                        "cy46" <= self.cycle < "cy47"
+                    ):  # Temporary fix for cy46 only
+                        self._set_namvar_value(
+                            namct0,
+                            "N{:s}TSMIN(0)".format(what),
+                            len(terms),
+                            namname,
+                        )
+                logger.info(
+                    "Setting up N%sTS and N%sTSMIN in &%s (file: %s)",
+                    what,
+                    what,
+                    namct0.name,
+                    namname,
+                )
                 for i, t in enumerate(terms):
-                    namct0['N{:s}TS({:d})'.format(what, i + 1)] = sign * t.hour
+                    namct0["N{:s}TS({:d})".format(what, i + 1)] = sign * t.hour
                 if with_minutes:
                     for i, t in enumerate(terms):
-                        namct0['N{:s}TSMIN({:d})'.format(what, i + 1)] = t.minute
+                        namct0["N{:s}TSMIN({:d})".format(what, i + 1)] = (
+                            t.minute
+                        )
 
     def _setup_nam_obj(self, namelist_object, namelist_name):
         """Actualy tweak the IFS namelist."""
-        namoph = self._get_namblock(namelist_object, 'NAMOPH')
-        namct0 = self._get_namblock(namelist_object, 'NAMCT0')
-        namct1 = self._get_namblock(namelist_object, 'NAMCT1')
+        namoph = self._get_namblock(namelist_object, "NAMOPH")
+        namct0 = self._get_namblock(namelist_object, "NAMCT0")
+        namct1 = self._get_namblock(namelist_object, "NAMCT1")
         # First take into account the **fcterm_unit**
-        self._set_namvar_value(namoph, 'LINC', self.fcterm_unit == 'h', namelist_name)
+        self._set_namvar_value(
+            namoph, "LINC", self.fcterm_unit == "h", namelist_name
+        )
         # Setup outputs
-        self._generic_terms_setup(namct0, namct1, 'HIS', self.modelstate, namelist_name)
-        self._generic_terms_setup(namct0, namct1, 'SFXHIS', self.surf_modelstate, namelist_name)
-        self._generic_terms_setup(namct0, namct1, 'SDI', self.spectral_diag, namelist_name)
-        self._generic_terms_setup(namct0, namct1, 'POS', self.post_processing, namelist_name)
+        self._generic_terms_setup(
+            namct0, namct1, "HIS", self.modelstate, namelist_name
+        )
+        self._generic_terms_setup(
+            namct0, namct1, "SFXHIS", self.surf_modelstate, namelist_name
+        )
+        self._generic_terms_setup(
+            namct0, namct1, "SDI", self.spectral_diag, namelist_name
+        )
+        self._generic_terms_setup(
+            namct0, namct1, "POS", self.post_processing, namelist_name
+        )
         # Extra fixup for fullpos
         if self.post_processing is not None:
             if not self.post_processing:
-                self._set_namvar_value(namct0, 'NFPOS', 0, namelist_name)
+                self._set_namvar_value(namct0, "NFPOS", 0, namelist_name)
             else:
                 # Do not overwrite a pre-existing positive value:
-                if 'NFPOS' not in namct0 or namct0['NFPOS'] == 0:
-                    self._set_namvar_value(namct0, 'NFPOS', 1, namelist_name)
+                if "NFPOS" not in namct0 or namct0["NFPOS"] == 0:
+                    self._set_namvar_value(namct0, "NFPOS", 1, namelist_name)