vortex-nwp 2.0.0b1__py3-none-any.whl → 2.1.0__py3-none-any.whl

vortex/nwp/util/ens.py

@@ -43,35 +43,44 @@ def drawingfunction(options):
 
     :rtype: A file like object
     """
-    rhdict = options.get('rhandler', None)
+    rhdict = options.get("rhandler", None)
     if rhdict:
-        date = rhdict['resource']['date']
+        date = rhdict["resource"]["date"]
         rgen = random.Random()
         rgen.seed(int(date[:-2]))
-        nbsample = rhdict['resource'].get('nbsample', 0)
+        nbsample = rhdict["resource"].get("nbsample", 0)
         if not nbsample:
-            raise FunctionStoreCallbackError('The resource must hold a non-null nbsample attribute')
-        population = rhdict['resource'].get('population', [])
+            raise FunctionStoreCallbackError(
+                "The resource must hold a non-null nbsample attribute"
+            )
+        population = rhdict["resource"].get("population", [])
         if not population:
-            raise FunctionStoreCallbackError('The resource must hold a non-empty population attribute')
+            raise FunctionStoreCallbackError(
+                "The resource must hold a non-empty population attribute"
+            )
         nbset = len(population)
 
-        tirage = (rgen.sample(population * (nbsample // nbset), (nbsample // nbset) * nbset) +
-                  rgen.sample(population, nbsample % nbset))
-        logger.info('List of random elements: %s', ', '.join([str(x) for x in tirage]))
+        tirage = rgen.sample(
+            population * (nbsample // nbset), (nbsample // nbset) * nbset
+        ) + rgen.sample(population, nbsample % nbset)
+        logger.info(
+            "List of random elements: %s", ", ".join([str(x) for x in tirage])
+        )
     else:
         raise FunctionStoreCallbackError("no resource handler here :-(")
     # NB: The result have to be a file like object !
-    outdict = dict(vapp=rhdict['provider'].get('vapp', None),
-                   vconf=rhdict['provider'].get('vconf', None),
-                   cutoff=rhdict['resource'].get('cutoff', None),
-                   date=rhdict['resource'].get('date', None),
-                   resource_kind=rhdict['resource'].get('kind', None),
-                   drawing=tirage,
-                   population=population)
-    if rhdict['provider'].get('experiment', None) is not None:
-        outdict['experiment'] = rhdict['provider']['experiment']
-    return io.BytesIO(json.dumps(outdict, indent=4).encode(encoding='utf_8'))
+    outdict = dict(
+        vapp=rhdict["provider"].get("vapp", None),
+        vconf=rhdict["provider"].get("vconf", None),
+        cutoff=rhdict["resource"].get("cutoff", None),
+        date=rhdict["resource"].get("date", None),
+        resource_kind=rhdict["resource"].get("kind", None),
+        drawing=tirage,
+        population=population,
+    )
+    if rhdict["provider"].get("experiment", None) is not None:
+        outdict["experiment"] = rhdict["provider"]["experiment"]
+    return io.BytesIO(json.dumps(outdict, indent=4).encode(encoding="utf_8"))
 
 
 def _checkingfunction_dict(options):
@@ -79,38 +88,95 @@ def _checkingfunction_dict(options):
     Internal function that returns a dictionnary that describes the available
     inputs.
     """
-    rhdict = options.get('rhandler', None)
+    rhdict = options.get("rhandler", None)
     if rhdict:
         # If no nbsample is provided, easy to achieve...
-        nbsample = rhdict['resource'].get('nbsample', None)
+        nbsample = rhdict["resource"].get("nbsample", None)
         # ...and if no explicit minimum of resources, nbsample is the minimum
-        nbmin = int(options.get('min', [(0 if nbsample is None else nbsample), ]).pop())
+        nbmin = int(
+            options.get(
+                "min",
+                [
+                    (0 if nbsample is None else nbsample),
+                ],
+            ).pop()
+        )
         if nbsample is not None and nbsample < nbmin:
-            logger.warning('%d resources needed, %d required: sin of gluttony ?', nbsample, nbmin)
+            logger.warning(
+                "%d resources needed, %d required: sin of gluttony ?",
+                nbsample,
+                nbmin,
+            )
         # What to look for ?
-        checkrole = rhdict['resource'].get('checkrole', None)
+        checkrole = rhdict["resource"].get("checkrole", None)
         if not checkrole:
-            raise FunctionStoreCallbackError('The resource must hold a non-empty checkrole attribute')
-        rolematch = re.match(r'(\w+)(?:\+(\w+))?$', checkrole)
+            raise FunctionStoreCallbackError(
+                "The resource must hold a non-empty checkrole attribute"
+            )
+        rolematch = re.match(r"(\w+)(?:\+(\w+))?$", checkrole)
         cur_t = sessions.current()
         if rolematch:
             ctx = cur_t.context
-            checklist = [sec.rh for sec in ctx.sequence.filtered_inputs(role=rolematch.group(1))]
-            mandatorylist = ([sec.rh for sec in ctx.sequence.filtered_inputs(role=rolematch.group(2))]
-                             if rolematch.group(2) else [])
+            checklist = [
+                sec.rh
+                for sec in ctx.sequence.filtered_inputs(
+                    role=rolematch.group(1)
+                )
+            ]
+            mandatorylist = (
+                [
+                    sec.rh
+                    for sec in ctx.sequence.filtered_inputs(
+                        role=rolematch.group(2)
+                    )
+                ]
+                if rolematch.group(2)
+                else []
+            )
         else:
-            raise FunctionStoreCallbackError('checkrole is not properly formatted')
+            raise FunctionStoreCallbackError(
+                "checkrole is not properly formatted"
+            )
         # Other options
-        nretries = int(options.get('nretries', [0, ]).pop())
-        retry_wait = Period(options.get('retry_wait', ['PT5M', ]).pop())
-        comp_delay = Period(options.get('comp_delay', [0, ]).pop())
-        fakecheck = options.get('fakecheck', [False, ]).pop()
+        nretries = int(
+            options.get(
+                "nretries",
+                [
+                    0,
+                ],
+            ).pop()
+        )
+        retry_wait = Period(
+            options.get(
+                "retry_wait",
+                [
+                    "PT5M",
+                ],
+            ).pop()
+        )
+        comp_delay = Period(
+            options.get(
+                "comp_delay",
+                [
+                    0,
+                ],
+            ).pop()
+        )
+        fakecheck = options.get(
+            "fakecheck",
+            [
+                False,
+            ],
+        ).pop()
 
         def _retry_cond(the_ntries, the_acceptable_time):
-            return ((the_acceptable_time is None and
-                     the_ntries <= nretries) or
-                    (the_acceptable_time and
-                     (time.time() - the_acceptable_time) < comp_delay.total_seconds()))
+            return (
+                the_acceptable_time is None and the_ntries <= nretries
+            ) or (
+                the_acceptable_time
+                and (time.time() - the_acceptable_time)
+                < comp_delay.total_seconds()
+            )
 
         # Ok let's work...
         ntries = 0
@@ -118,30 +184,41 @@ def _checkingfunction_dict(options):
         found = []
         while _retry_cond(ntries, acceptable_time):
             if ntries:
-                logger.info("Let's sleep %d sec. before the next check round...",
-                            retry_wait.total_seconds())
+                logger.info(
+                    "Let's sleep %d sec. before the next check round...",
+                    retry_wait.total_seconds(),
+                )
                 cur_t.sh.sleep(retry_wait.total_seconds())
             ntries += 1
             try:
                 logger.info("Starting an input check...")
-                found, candidates = helpers.colorfull_input_checker(nbmin,
-                                                                    checklist,
-                                                                    mandatory=mandatorylist,
-                                                                    fakecheck=fakecheck)
+                found, candidates = helpers.colorfull_input_checker(
+                    nbmin,
+                    checklist,
+                    mandatory=mandatorylist,
+                    fakecheck=fakecheck,
+                )
                 if acceptable_time is None and (found or nbmin == 0):
                     acceptable_time = time.time()
-                    if comp_delay.total_seconds() and len(found) != len(candidates):
-                        logger.info("The minimum required size was reached (nbmin=%d). " +
-                                    "That's great but we are waiting a little longer " +
-                                    "(for at most %d sec.)",
-                                    nbmin, comp_delay.total_seconds())
+                    if comp_delay.total_seconds() and len(found) != len(
+                        candidates
+                    ):
+                        logger.info(
+                            "The minimum required size was reached (nbmin=%d). "
+                            + "That's great but we are waiting a little longer "
+                            + "(for at most %d sec.)",
+                            nbmin,
+                            comp_delay.total_seconds(),
+                        )
 
                 if len(found) == len(candidates):
                     # No need to wait any longer...
                     break
             except helpers.InputCheckerError as e:
                 if not _retry_cond(ntries, acceptable_time):
-                    raise FunctionStoreCallbackError('The input checher failed ({!s})'.format(e))
+                    raise FunctionStoreCallbackError(
+                        "The input checher failed ({!s})".format(e)
+                    )
         return found
     else:
         raise FunctionStoreCallbackError("no resource handler here :-(\n")
@@ -163,17 +240,19 @@ def checkingfunction(options):
 
     :rtype: A file like object
     """
-    rhdict = options.get('rhandler', None)
+    rhdict = options.get("rhandler", None)
     avail_list = _checkingfunction_dict(options)
-    outdict = dict(vapp=rhdict['provider'].get('vapp', None),
-                   vconf=rhdict['provider'].get('vconf', None),
-                   cutoff=rhdict['resource'].get('cutoff', None),
-                   date=rhdict['resource'].get('date', None),
-                   resource_kind=rhdict['resource'].get('kind', None),
-                   population=avail_list)
-    if rhdict['provider'].get('experiment', None) is not None:
-        outdict['experiment'] = rhdict['provider']['experiment']
-    return io.BytesIO(json.dumps(outdict, indent=4).encode(encoding='utf_8'))
+    outdict = dict(
+        vapp=rhdict["provider"].get("vapp", None),
+        vconf=rhdict["provider"].get("vconf", None),
+        cutoff=rhdict["resource"].get("cutoff", None),
+        date=rhdict["resource"].get("date", None),
+        resource_kind=rhdict["resource"].get("kind", None),
+        population=avail_list,
+    )
+    if rhdict["provider"].get("experiment", None) is not None:
+        outdict["experiment"] = rhdict["provider"]["experiment"]
+    return io.BytesIO(json.dumps(outdict, indent=4).encode(encoding="utf_8"))
 
 
 def safedrawingfunction(options):
@@ -182,7 +261,7 @@ def safedrawingfunction(options):
     See the documentation of these two functions for more details.
     """
     checkedlist = _checkingfunction_dict(options)
-    options['rhandler']['resource']['population'] = checkedlist
+    options["rhandler"]["resource"]["population"] = checkedlist
     return drawingfunction(options)
 
 
@@ -192,7 +271,9 @@ def unsafedrawingfunction(options):
 
     See the documentation of these two functions for more details.
     """
-    options['fakecheck'] = [True, ]
+    options["fakecheck"] = [
+        True,
+    ]
     checkedlist = _checkingfunction_dict(options)
-    options['rhandler']['resource']['population'] = checkedlist
+    options["rhandler"]["resource"]["population"] = checkedlist
     return drawingfunction(options)

vortex/nwp/util/hooks.py

@@ -21,10 +21,15 @@ def update_namelist(t, rh, *completive_rh):
     touched = False
     for crh in completive_rh:
         if not isinstance(crh, (list, tuple)):
-            crh = [crh, ]
+            crh = [
+                crh,
+            ]
         for arh in crh:
-            logger.info('Merging: {!r} :\n{:s}'.format(arh.container,
-                                                       arh.contents.dumps()))
+            logger.info(
+                "Merging: {!r} :\n{:s}".format(
+                    arh.container, arh.contents.dumps()
+                )
+            )
             rh.contents.merge(arh.contents)
             touched = True
     if touched:
@@ -36,14 +41,16 @@ def concatenate(t, rh, *rhlist):
     blocksize = 32 * 1024 * 1024  # 32Mb
     rh.container.close()
     with rh.container.iod_context():
-        myfh = rh.container.iodesc(mode='ab')
+        myfh = rh.container.iodesc(mode="ab")
         for crh in rhlist:
             if not isinstance(crh, (list, tuple)):
-                crh = [crh, ]
+                crh = [
+                    crh,
+                ]
             for arh in crh:
-                logger.info('Appending %s to self.', str(arh.container))
+                logger.info("Appending %s to self.", str(arh.container))
                 with arh.container.iod_context():
-                    afh = arh.container.iodesc(mode='rb')
+                    afh = arh.container.iodesc(mode="rb")
                     stuff = afh.read(blocksize)
                     while stuff:
                         myfh.write(stuff)
@@ -64,14 +71,17 @@ def insert_cutoffs(t, rh, rh_cutoff_source, fuse_per_obstype=False):
             ValueError("The resource handler's list is empty.")
     # Get the CutoffDispenser
     import vortex.tools.listings
+
     assert vortex.tools.listings
-    if rh_cutoff_source.container.actualfmt == 'bdmbufr_listing':
+    if rh_cutoff_source.container.actualfmt == "bdmbufr_listing":
         c_disp_callback = functools.partial(
             rh_cutoff_source.contents.data.cutoffs_dispenser,
-            fuse_per_obstype=fuse_per_obstype
+            fuse_per_obstype=fuse_per_obstype,
         )
     else:
-        raise RuntimeError("Incompatible < {!s} > ressource handler".format(rh_cutoff_source))
+        raise RuntimeError(
+            "Incompatible < {!s} > ressource handler".format(rh_cutoff_source)
+        )
     # Fill the gaps in the original request
     rh.contents.add_cutoff_info(c_disp_callback())
     # Actually save the result to file
@@ -79,7 +89,6 @@ def insert_cutoffs(t, rh, rh_cutoff_source, fuse_per_obstype=False):
 
 
 def _new_static_cutoff_dispencer(base_date, cutoffs_def):
-
     def x_period(p):
         try:
             return Period(p)
@@ -89,8 +98,10 @@ def _new_static_cutoff_dispencer(base_date, cutoffs_def):
     if not isinstance(base_date, Date):
         base_date = Date(base_date)
     if isinstance(cutoffs_def, collections.abc.Mapping):
-        cutoffs_def = {(k if isinstance(k, Period) else x_period(k)): v
-                       for k, v in cutoffs_def.items()}
+        cutoffs_def = {
+            (k if isinstance(k, Period) else x_period(k)): v
+            for k, v in cutoffs_def.items()
+        }
         cutoffs = {base_date + k: v for k, v in cutoffs_def.items()}
         c_disp = StaticCutoffDispenser(max(cutoffs.keys()), cutoffs)
     else:
@@ -110,19 +121,19 @@ def insert_static_cutoffs(t, rh, base_date, cutoffs_def):
                         associates a cutoff with a list of `obstypes`.
     """
     # Fill the gaps in the original request
-    rh.contents.add_cutoff_info(_new_static_cutoff_dispencer(base_date, cutoffs_def))
+    rh.contents.add_cutoff_info(
+        _new_static_cutoff_dispencer(base_date, cutoffs_def)
+    )
     # Actually save the result to files
     rh.save()
 
 
 def arpifs_obs_error_correl_legacy2oops(t, rh):
     """Convert a constant file that contains observation errors correlations."""
-    if rh.resource.realkind != 'correlations':
-        raise ValueError('Incompatible resource: {!s}'.format(rh))
+    if rh.resource.realkind != "correlations":
+        raise ValueError("Incompatible resource: {!s}".format(rh))
     if rh.contents[0].startswith("SIGMAO"):
         logger.warning("Non conversion is needed...")
     else:
-        rh.contents[:0] = ["SIGMAO unused\n",
-                           "1 1.2\n",
-                           "CORRELATIONS\n"]
+        rh.contents[:0] = ["SIGMAO unused\n", "1 1.2\n", "CORRELATIONS\n"]
         rh.save()

vortex/nwp/util/taskdeco.py

@@ -34,15 +34,17 @@ def process_needs_lfi_stuff(*kargs, **kwargs):
                 pass
 
     """
-    cyclekey = kwargs.pop('cyclekey', 'cycle')
+    cyclekey = kwargs.pop("cyclekey", "cycle")
 
     def decorate_process(cls):
         """Decorator for Task: get LFI stuff before calling process."""
-        original_process = getattr(cls, 'process', None)
+        original_process = getattr(cls, "process", None)
         if original_process is not None:
+
             def process(self, *args, **kwargs):
                 _get_lfi_stuff(self, cyclekey)
                 original_process(self, *args, **kwargs)
+
             process.__doc__ = original_process.__doc__
             cls.process = process
         return cls
@@ -55,27 +57,29 @@ def process_needs_lfi_stuff(*kargs, **kwargs):
 
 def _get_lfi_stuff(self, cyclekey):
     """Get LFI stuff method (called from process)."""
-    if 'early-fetch' in self.steps or 'fetch' in self.steps:
-
+    if "early-fetch" in self.steps or "fetch" in self.steps:
         actualcycle = getattr(self.conf, cyclekey)
 
-        self.sh.title('Toolbox input tblfiscripts')
-        toolbox.input(role='LFIScripts',
-                      genv=actualcycle,
-                      kind='lfiscripts',
-                      local='usualtools/tools.lfi.tgz',
-                      )
-        self.sh.title('Toolbox input tbiopoll')
-        toolbox.input(role='IOPoll',
-                      format='unknown',
-                      genv=actualcycle,
-                      kind='iopoll',
-                      language='perl',
-                      local='usualtools/io_poll',
-                      )
-        self.sh.title('Toolbox input tblfitools')
-        toolbox.input(role='LFITOOLS',
-                      genv=actualcycle,
-                      kind='lfitools',
-                      local='usualtools/lfitools'
-                      )
+        self.sh.title("Toolbox input tblfiscripts")
+        toolbox.input(
+            role="LFIScripts",
+            genv=actualcycle,
+            kind="lfiscripts",
+            local="usualtools/tools.lfi.tgz",
+        )
+        self.sh.title("Toolbox input tbiopoll")
+        toolbox.input(
+            role="IOPoll",
+            format="unknown",
+            genv=actualcycle,
+            kind="iopoll",
+            language="perl",
+            local="usualtools/io_poll",
+        )
+        self.sh.title("Toolbox input tblfitools")
+        toolbox.input(
+            role="LFITOOLS",
+            genv=actualcycle,
+            kind="lfitools",
+            local="usualtools/lfitools",
+        )