vortex-nwp 2.2.0__py3-none-any.whl → 2.3.0__py3-none-any.whl

vortex/__init__.py

@@ -50,13 +50,14 @@ from .toolbox import promise as promise
 from .toolbox import diff as diff
 from .toolbox import defaults as defaults
 from .toolbox import algo as task
+from .toolbox import VortexForceComplete as VortexForceComplete
 
 from . import nwp as nwp  # footprints import
 
-__version__ = "2.2.0"
+__version__ = "2.3.0"
 __prompt__ = "Vortex v-" + __version__ + ":"
 
-__nextversion__ = "2.2.1"
+__nextversion__ = "2.3.1"
 __tocinfoline__ = "VORTEX core package"
 
 __all__ = [

vortex/algo/components.py

@@ -1548,12 +1548,25 @@ class Expresso(ExecutableAlgoComponent):
         attr=dict(
             interpreter=dict(
                 info="The interpreter needed to run the script.",
-                values=["current", "awk", "ksh", "bash", "perl", "python"],
+                values=[
+                    "current",
+                    "awk",
+                    "ksh",
+                    "bash",
+                    "perl",
+                    "python",
+                    "singularity",
+                ],
             ),
             interpreter_path=dict(
                 info="The interpreter command.",
                 optional=True,
             ),
+            interpreter_args=dict(
+                info="Some options to pass to the expresso",
+                optional=True,
+                default="",
+            ),
             engine=dict(values=["exec", "launch"]),
         ),
     )
@@ -1581,11 +1594,16 @@ class Expresso(ExecutableAlgoComponent):
                     )
 
     def _interpreter_args_fix(self, rh, opts):
+        cmd_opts = shlex.split(self.interpreter_args)
+
         absexec = self.absexcutable(rh.container.localpath())
         if self.interpreter == "awk":
-            return ["-f", absexec]
+            return [*cmd_opts, "-f", absexec]
+        if self.interpreter == "singularity":
+            return ["run", *cmd_opts, absexec]
         else:
             return [
+                *cmd_opts,
                 absexec,
             ]
 

vortex/algo/mpitools.py

@@ -544,10 +544,6 @@ class MpiTool(footprints.FootprintBase):
         """When group are defined, associate each MPI rank with a "real" slot."""
         if self._ranks_map_cache is None:
             self._complex_ranks_map = False
-            if not self.envelope:
-                raise RuntimeError(
-                    "Ranks mapping should always be used within an envelope."
-                )
             # First deal with bingroups
             ranks_map = dict()
             has_bin_groups = not all([b.group is None for b in self.binaries])
@@ -624,7 +620,7 @@ class MpiTool(footprints.FootprintBase):
             if self._complex_ranks_map or do_bin_distribution:
                 if not self.envelope:
                     raise RuntimeError(
-                        "Ranks mapping shoudl always be used within an envelope."
+                        "Ranks mapping should always be used within an envelope."
                     )
             if do_bin_distribution:
                 if not self._supports_manual_ranks_mapping:
@@ -1404,7 +1400,10 @@ class SRun(MpiTool):
             ),
             optmap=dict(
                 default=footprints.FPDict(
-                    nn="nodes", nnp="ntasks-per-node", np="ntasks"
+                    nn="nodes",
+                    nnp="ntasks-per-node",
+                    np="ntasks",
+                    openmp="cpus-per-task",
                 )
             ),
             slurmversion=dict(type=int, optional=True),

vortex/data/executables.py

@@ -106,6 +106,34 @@ class Script(Executable):
             return self.rawopts
 
 
+class SingularityImage(Executable):
+    """Singularity container image executable."""
+
+    _footprint = dict(
+        attr=dict(
+            kind=dict(
+                values=["image"],
+            ),
+            cmd=dict(
+                info="Command to run inside the container",
+                optional=True,
+                default="",
+            ),
+        ),
+    )
+
+    @property
+    def realkind(self):
+        return "image"
+
+    # Called by AlgoComponent.spawn_command_line
+    def command_line(self, **opts):
+        return " ".join(
+            ["--{:s} {:s}".format(opt, val) for opt, val in opts.items()]
+            + [self.cmd]
+        )
+
+
 class GnuScript(Executable):
     """Basic interpreted executable with standard command line arguments."""
 

vortex/data/geometries.ini

@@ -659,6 +659,13 @@ area       = indonesie1
 resolution = 2.50
 runit      = km
 
+[asscomindonesie2sp]
+info       = Arome asscom14 - indonesie2 spectral geometry
+kind       = projected
+area       = indonesie2
+resolution = 2.50
+runit      = km
+
 [lace20km]
 info       = Coupling grid for LACE partners
 kind       = projected
@@ -673,6 +680,13 @@ area       = lace
 resolution = 8.0
 runit      = km
 
+[lace47]
+info       = LACE canari domain
+kind       = projected
+area       = lace47
+resolution = 4.71
+runit      = km
+
 [afgt20km]
 info       = Coupling grid for afgt-Praha
 kind       = projected
@@ -1201,15 +1215,15 @@ nlon       = 641
 [paris1s100]
 info       = PARIS 0.01
 kind       = lonlat
-area       = paris1s100
+area       = PARIS1S100
 resolution = 0.01
-nlat       = 221
-nlon       = 336
+nlat       = 187
+nlon       = 279
 
 [medalp1s100]
 info       = MEDALP 0.01
 kind       = lonlat
-area       = medalp1s100
+area       = MEDALP1S100
 resolution = 0.01
 nlat       = 481
 nlon       = 501
@@ -1677,6 +1691,17 @@ nlat       = 172800
 latmax     = 83.999
 latmin     = -55.999
 
+[srtm_01s]
+info       = alternative to no_arctics7.5s used to create arome-500m PGD clim
+kind       = lonlat
+area       = srtm
+resolution = 1
+runit      = s
+nlon       = 999
+nlat       = 999
+latmax     = 999
+latmin     = 999
+
 [europe_sw3s]
 kind       = lonlat
 area       = europe_sw
@@ -2468,39 +2493,39 @@ nlon       = 896
 resolution = 0.05
 runit      = deg
 
-[hycom3dmedoc]
-info       = Shom HYCOM3D Med-occidental at 1/60
+[hycom3dmednowe]
+info       = Shom HYCOM3D Med-NorthWest at 1/60
 kind       = lonlat
-area       = medoc
-nlat       = 331
-nlon       = 661
+area       = mednowe
+nlat       = 437
+nlon       = 761
 resolution = 0.017
 runit      = deg
 
-[hycom3dmedor]
-info       = Shom HYCOM3D Med-oriental at 1/60
+[hycom3dmedsowe]
+info       = Shom HYCOM3D Med-SouthWest at 1/60
 kind       = lonlat
-area       = medor
-nlat       = 437
-nlon       = 645
+area       = medsowe
+nlat       = 481
+nlon       = 1080
 resolution = 0.017
 runit      = deg
 
-[hycom3dmedio]
-info       = Shom HYCOM3D Med-ionian at 1/60
+[hycom3dmedea]
+info       = Shom HYCOM3D Med-East at 1/60
 kind       = lonlat
-area       = medio
-nlat       = 557
-nlon       = 781
+area       = medea
+nlat       = 706
+nlon       = 855
 resolution = 0.017
 runit      = deg
 
-[hycom3dgibal]
-info       = Shom HYCOM3D Gibraltair Strait-Alboran Sea at 1/60
+[hycom3dmedinte]
+info       = Shom HYCOM3D Med-Interior at 1/60
 kind       = lonlat
-area       = gibal
-nlat       = 361
-nlon       = 571
+area       = medinte
+nlat       = 935
+nlon       = 781
 resolution = 0.017
 runit      = deg
 
@@ -2575,6 +2600,13 @@ area       = corsica
 resolution = 2.5
 runit      = km
 
+[nl2500]
+info       = HARMONIE-AROME 2.5km NL domain for DAVAI
+kind       = projected
+area       = nl
+resolution = 2.5
+runit      = km
+
 [cors1s40]
 info       = Corsica 0.025 deg
 kind       = lonlat
@@ -2669,6 +2701,39 @@ area       = chmh2325
 resolution = 2.33
 runit      = km
 
+[armcu]
+info       = MUSC Single point ARMCU
+kind       = unstructured
+area       = armcu
+
+[global31c1]
+info       = ARPEGE T31C1.0 geometry (from Mitraillette tests)
+kind       = gauss
+area       = france
+truncation = 31
+stretching = 1.0
+
+[global30c24]
+info       = ARPEGE T30C2.4 geometry (from Mitraillette tests)
+kind       = gauss
+area       = france
+truncation = 30
+stretching = 2.4
+
+[portugal21km]
+info       = Mitraillette Portugal grid
+kind       = projected
+area       = portugal
+resolution = 21.3
+runit      = km
+
+[occitanie4km]
+info       = Mitraillette Occitanie grid
+kind       = projected
+area       = occitanie
+resolution = 4.4
+runit      = km
+
 #R ===================
 #R Combined geometries
 #R ===================

vortex/data/providers.py

@@ -64,13 +64,6 @@ class Provider(footprints.FootprintBase):
         fastkeys={"namespace"},
     )
 
-    def __init__(self, *args, **kw):
-        logger.debug("Abstract provider init %s", self.__class__)
-        super().__init__(*args, **kw)
-
-        if not self.username:
-            self.username = vortex.ticket().glove.user
-
     def _str_more(self):
         """Additional information to print representation."""
         try:
@@ -92,7 +85,9 @@ class Provider(footprints.FootprintBase):
 
     def netuser_name(self, resource):  # @UnusedVariable
         """Abstract method."""
-        return self.username
+        if self.username is not None:
+            return self.username
+        return vortex.ticket().glove.user
 
     def pathname(self, resource):
         """Abstract method."""
@@ -124,10 +119,11 @@ class Provider(footprints.FootprintBase):
         The different operations of the algorithm can be redefined by subclasses.
         """
         username = self.netuser_name(resource)
+        netloc = self.netloc(resource)
         fullnetloc = (
             "{:s}@{:s}".format(username, self.netloc(resource))
-            if username
-            else self.netloc(resource)
+            if (username and netloc)
+            else netloc
         )
         logger.debug(
             "scheme %s netloc %s normpath %s urlquery %s",
@@ -279,6 +275,44 @@ def set_namespace_from_cache_settings(usecache, usearchive):
     return ".".join(("vortex", domain, "fr"))
 
 
+class Git(Provider):
+    _footprint = dict(
+        info="Git provider",
+        attr=dict(
+            ref=dict(
+                type=str,
+                optional=True,
+                default=None,
+                info="The reference's SHA-1 hash id",
+            ),
+            repo=dict(
+                type=str,
+                info="Path to the Git repository",
+            ),
+            path=dict(
+                type=str,
+                info="File path within the repository",
+            ),
+        ),
+    )
+
+    def scheme(self, resource):
+        return "git"
+
+    def urlquery(self, resource):
+        return (
+            f"repo={self.repo}&ref={self.ref}"
+            if self.ref
+            else f"repo={self.repo}"
+        )
+
+    def basename(self, resource):
+        return self.path
+
+    def pathname(sef, resource):
+        return "."
+
+
 class Vortex(Provider):
     """Main provider of the toolbox, using a fix-size path and a dedicated name factory."""
 

vortex/data/stores.py

@@ -8,8 +8,11 @@ Store objects use the :mod:`footprints` mechanism.
 import copy
 import ftplib
 import io
+from pathlib import Path
 import os
 import re
+import shutil
+import subprocess
 
 from bronx.fancies import loggers
 import footprints
@@ -33,6 +36,13 @@ __all__ = []
 
 logger = loggers.getLogger(__name__)
 
+try:
+    import pygit2
+
+    NO_PYGIT2 = False
+except ImportError:
+    NO_PYGIT2 = True
+
 
 def get_cache_location():
     try:
@@ -335,7 +345,6 @@ class Finder(Store):
     def _ftpinfos(self, remote, **kwargs):
         args = kwargs.copy()
         args["hostname"] = self.hostname()
-        args["logname"] = remote["username"]
         port = self.hostname().netport
         if port is not None:
             args["port"] = port
@@ -422,6 +431,98 @@ class Finder(Store):
         return rc
 
 
+class GitStore(Store):
+    _footprint = dict(
+        info="A store to access Git repositories",
+        attr=dict(
+            scheme=dict(
+                values=["git"],
+            ),
+        ),
+    )
+
+    def gitget(self, remote, local, options):
+        if NO_PYGIT2:
+            raise ModuleNotFoundError(
+                "pygit2 is not installed in the current environment"
+            )
+        annex_cache_path = Path(remote["query"]["repo"][0])
+
+        # If no git reference is provided, only make a copy of the
+        # worktree
+        if "ref" not in remote["query"]:
+            shutil.copy(
+                src=Path(annex_cache_path) / remote["path"].lstrip("/"),
+                dst=local,
+            )
+            return local
+
+        assert "ref" in remote["query"]
+        path = remote["path"].lstrip("/")
+        repo = pygit2.Repository(annex_cache_path)
+        oid = repo.revparse_single(remote["query"]["ref"][0])
+        obj = oid.peel(pygit2.Tree) / path
+        if obj.filemode == pygit2.enums.FileMode.LINK:
+            # Here we need to discriminate between links that point
+            # to git-annex managed files and all other symlinks.
+
+            # If the file is a git-annex link, work out the location
+            # of the data within the .git/annex dir and copy this
+            # file into the cwd with the right name
+            if ".git/annex/objects" in obj.data.decode("ASCII"):
+                gitannex_key = Path(obj.data.decode("ASCII")).name
+                subprocess.run(
+                    args=["git-annex", "get", "--key", gitannex_key],
+                    cwd=str(annex_cache_path),
+                )
+                gitannex_content_location = subprocess.run(
+                    args=["git-annex", "contentlocation", gitannex_key],
+                    cwd=str(annex_cache_path),
+                    capture_output=True,
+                    encoding="ASCII",
+                ).stdout
+                shutil.copy(
+                    src=annex_cache_path / Path(gitannex_content_location),
+                    dst=local,
+                    follow_symlinks=True,
+                )
+                return local
+            os.symlink(
+                src=obj.data.decode("ASCII"),
+                dst=local,
+            )
+            return local
+
+        if obj.filemode == pygit2.enums.FileMode.BLOB:
+            with open(local, "wb") as dst:
+                # Could also use pygit2.BlobIO to stream content
+                # without having to load the entire blob data in
+                # memory:
+                #
+                # with pygit2.BlobIO(obj) as src:
+                #     shutil.copyfileobj(fsrc=src, fdst=dst)
+                dst.write(obj.data)
+            return local
+
+        if obj.filemode == pygit2.enums.FileMode.TREE:
+            if local.endswith("/"):
+                localpath = "."
+            else:
+                localpath = local
+            os.mkdir(local)
+            for subobj in obj:
+                r = {
+                    "query": {"ref": remote["query"]["ref"]},
+                    "path": str(Path(path) / subobj.name),
+                }
+                self.gitget(
+                    remote=r,
+                    local=str(Path(localpath) / subobj.name),
+                    options=None,
+                )
+            return local
+
+
 class _VortexStackedStorageMixin:
     """Mixin class that adds utility functions to work with stacked data."""
 
@@ -985,7 +1086,10 @@ class VortexCacheMtStore(_VortexCacheBaseStore):
         except config.ConfigurationError:
             cacheloc = os.path.join(os.environ["HOME"], ".vortex.d")
 
-        if self.username != self.system.glove.user:
+        current_vortex_user = self.system.glove.user
+        cacheloc = cacheloc.replace("%usr%", current_vortex_user)
+
+        if self.username != current_vortex_user:
             return os.path.join(cacheloc, self.username)
 
         return cacheloc
@@ -1205,6 +1309,9 @@ class VortexStackStore(_AbstractVortexStackMultiStore):
         """Go through the various stacked stores."""
         return [f"{self.netloc.firstname}.stacked-cache-mt.fr"]
 
+    def alternates_fpextras(self):
+        return dict(username=self.username)
+
 
 class VortexVsopStackStore(_AbstractVortexStackMultiStore):
     """Store intended to read and write data into VORTEX R&D stacks."""
@@ -1336,7 +1443,7 @@ class PromiseCacheStore(VortexCacheMtStore):
     )
 
     @property
-    def cache_promise(self):
+    def cache_entry(self):
         return os.path.join(super().cache_entry, "promise")
 
     @staticmethod

vortex/nwp/algo/assim.py

@@ -5,7 +5,7 @@ AlgoComponents dedicated to computations related to Data Assimilation systems.
 from bronx.fancies import loggers
 from bronx.stdtypes.date import Date
 
-from vortex.algo.components import BlindRun, Parallel
+from vortex.algo.components import BlindRun
 from vortex.syntax.stdattrs import a_date
 from .ifsroot import IFSParallel
 from ..tools import odb, drhook
@@ -16,36 +16,6 @@ __all__ = []
 logger = loggers.getLogger(__name__)
 
 
-class MergeVarBC(Parallel):
-    """Merge two VarBC files.
-
-    The VarBC file resulting from the MergeVarBC contains all the items of the
-    first VarBC file plus any new item that would be present in the second file.
-    """
-
-    _footprint = dict(
-        attr=dict(
-            kind=dict(
-                values=["mergevarbc"],
-            ),
-            varbcout=dict(
-                optional=True,
-                default="VARBC.cycle_out",
-            ),
-        )
-    )
-
-    def prepare(self, rh, opts):
-        """Find any ODB candidate in input files."""
-
-        sh = self.system
-
-        sh.touch(self.varbcout)
-
-        # Let ancesters doing real stuff
-        super().prepare(rh, opts)
-
-
 class Anamix(IFSParallel):
     """Merge the surface and atmospheric analyses into a single file"""
 

vortex/nwp/algo/forecasts.py

@@ -112,6 +112,8 @@ class Forecast(IFSParallel):
             # Possibly fix post-processing clim files
             self.all_localclim_fixer(rh, thismonth)
 
+            self.grab(analysis, comment="analysis")
+
             # File linking for IAU increments
             #
             # In the case of a forecast with IAU, the IFS executable
@@ -884,3 +886,31 @@ class OfflineSurfex(Parallel, DrHookDecoMixin):
             if namsec.rh.contents.dumps_needs_update:
                 namsec.rh.save()
             logger.info("Namelist dump: \n%s", namsec.rh.container.read())
+
+
+class MUSCForecast(Forecast):
+    """Forecast for MUSC single-column model."""
+
+    _footprint = dict(
+        info="Run a forecast with a MUSC single-column model.",
+        attr=dict(
+            kind=dict(
+                values=["musc"],
+            ),
+        ),
+    )
+
+    def postfix(self, rh, opts):
+        """Post forecast information and cleaning."""
+        sh = self.system
+        # rename specific output files with hours term on 4 digits for compatibility for fmth formatting
+        fmt = re.compile(r"Out\.(?P<termh>\d{3})\.\d{4}\.lfa$")
+        for f in [
+            fmt.match(f)
+            for f in sh.listdir()
+            if f.startswith("Out.") and f.endswith(".lfa")
+        ]:
+            sh.rename(
+                f.string, "Out.{:>04}.0000.lfa".format(int(f.group("termh")))
+            )
+        super().postfix(rh, opts)

vortex/nwp/algo/ifsroot.py

@@ -91,7 +91,7 @@ class IFSParallel(
                 ),
                 fcterm=dict(
                     info="The forecast term of the Arpege/IFS model.",
-                    type=int,
+                    type=float,
                     optional=True,
                     default=0,
                 ),
@@ -346,24 +346,6 @@ class IFSParallel(
         # be done by an extra class ... and it could be generalized to mpi
         # setup by the way !
         nam_updated = False
-        # For cy41 onward, replace some namelist macros with the command line
-        # arguments
-        if rh.resource.cycle >= "cy41":
-            if "NAMARG" in namcontents:
-                opts_arg = self.spawn_command_options()
-                self._set_nam_macro(
-                    namcontents, namlocal, "CEXP", opts_arg["name"]
-                )
-                self._set_nam_macro(
-                    namcontents, namlocal, "TIMESTEP", opts_arg["timestep"]
-                )
-                fcstop = "{:s}{:d}".format(
-                    opts_arg["fcunit"], opts_arg["fcterm"]
-                )
-                self._set_nam_macro(namcontents, namlocal, "FCSTOP", fcstop)
-                nam_updated = True
-            else:
-                logger.info("No NAMARG block in %s", namlocal)
 
         if self.member is not None:
             for macro_name in ("MEMBER", "PERTURB"):
@@ -371,7 +353,39 @@ class IFSParallel(
                     namcontents, namlocal, macro_name, self.member
                 )
             nam_updated = True
-        return nam_updated
+
+        if rh.resource.cycle < "cy41":
+            return nam_updated
+
+        if "NAMARG" not in namcontents:
+            logger.info("No NAMARG block in %s", namlocal)
+            return nam_updated
+
+        # For cy41 onward, replace some namelist macros with the command line
+        # arguments
+        opts_arg = self.spawn_command_options()
+        self._set_nam_macro(namcontents, namlocal, "CEXP", opts_arg["name"])
+        self._set_nam_macro(
+            namcontents, namlocal, "TIMESTEP", opts_arg["timestep"]
+        )
+
+        if self.fcunit == "t":
+            fcstop = "t{:d}".format(int(self.fcterm))
+        elif self.fcterm.is_integer():
+            # Round number of hours
+            fcstop = "h{:d}".format(int(self.fcterm))
+        else:
+            # IFS expects the forecast term to be given as an integer,
+            # whether this integer represents hours or timesteps. This
+            # means terms that are not round hours (e.g. 01:45) can only
+            # be expressed as a number of timesteps.
+            # See http://gitlab.meteo.fr/cnrm-gmap/vortex/-/issues/9
+            nsteps = int(self.fcterm * 3600 // self.timestep)
+            fcstop = "t{:d}".format(nsteps)
+            logger.info(f"Converting {self.fcterm} hours into {nsteps}")
+
+        self._set_nam_macro(namcontents, namlocal, "FCSTOP", fcstop)
+        return True
 
     def prepare_namelists(self, rh, opts=None):
         """Update each of the namelists."""

vortex/nwp/algo/monitoring.py

@@ -4,7 +4,7 @@ AlgoComponents dedicated to computations related to observations monitoring.
 
 from bronx.fancies import loggers
 
-from vortex.algo.components import Parallel
+from vortex.algo.components import BlindRun
 from vortex.syntax.stdattrs import a_date, a_model, a_cutoff
 from ..tools import odb, drhook
 
@@ -15,7 +15,7 @@ logger = loggers.getLogger(__name__)
 
 
 class OdbMonitoring(
-    Parallel, odb.OdbComponentDecoMixin, drhook.DrHookDecoMixin
+    BlindRun, odb.OdbComponentDecoMixin, drhook.DrHookDecoMixin
 ):
     """Compute monitoring statistics."""
 

vortex/nwp/algo/odbtools.py

@@ -16,7 +16,7 @@ from taylorism import Boss
 
 from vortex.tools.systems import ExecutionError
 
-from vortex.algo.components import Parallel, ParaBlindRun
+from vortex.algo.components import Parallel, ParaBlindRun, BlindRun
 from vortex.tools.parallelism import VortexWorkerBlindRun
 
 from ..syntax.stdattrs import arpifs_cycle
@@ -1217,7 +1217,7 @@ class OdbReshuffle(
 
 
 class FlagsCompute(
-    Parallel, odb.OdbComponentDecoMixin, drhook.DrHookDecoMixin
+    BlindRun, odb.OdbComponentDecoMixin, drhook.DrHookDecoMixin
 ):
     """Compute observations flags."""
 

vortex/nwp/data/consts.py

@@ -2,12 +2,15 @@
 Various Resources for constant files used in NWP.
 """
 
+import random
+
 import footprints
+from footprints import FPList
 from ..syntax.stdattrs import gvar
 from vortex.data.contents import DataRaw, JsonDictContent, TextContent
 from vortex.data.geometries import GaussGeometry, LonlatGeometry
 from vortex.data.outflow import ModelGeoResource, ModelResource, StaticResource
-from vortex.syntax.stdattrs import month_deco
+from vortex.syntax.stdattrs import date_deco, member, month_deco
 from vortex.syntax.stddeco import (
     namebuilding_append,
     namebuilding_delete,
@@ -180,6 +183,63 @@ class RtCoef(GenvModelResource):
         return "rtcoef"
 
 
+class RtCoefMulti(GenvModelResource):
+    """
+    RtCoeff Satellite coefficients, randomly chosen depending on the member of an ensemble.
+
+    Reproducibility is ensured by the stability of the random generation for a given date (ymdh) and member.
+    With member=0 or None, or with choices=0, the choice is always 0, even if 0 is excluded.
+    """
+
+    _footprint = [
+        date_deco,
+        member,
+        dict(
+            info="Set of satellite coefficients",
+            attr=dict(
+                kind=dict(
+                    values=["rtcoef_multi", "mwave_rtcoef_multi"],
+                ),
+                choices=dict(
+                    info="Number of choices to choose from (0..choices-1)",
+                    type=int,
+                    optional=True,
+                ),
+                excluded=dict(
+                    info="List of values excluded from choice",
+                    type=footprints.stdtypes.FPList,
+                    optional=True,
+                    default=FPList([]),
+                ),
+                gvar=dict(
+                    info="Will be modified by the random choice",
+                    default="[kind]_0",
+                    access="rwx",
+                ),
+            ),
+        ),
+    ]
+
+    def __init__(self, *args, **kw):
+        super().__init__(*args, **kw)
+        if self.member is None or self.member == 0 or self.choices == 0:
+            choice = 0
+        else:
+            # a random generator entirely determined by the date
+            rgen = random.Random(int(self.date.ymdh))
+            # drawing must be reproducible for a given member:
+            # generate 'member' values, but only keep the last
+            choice = rgen.choices(
+                [n for n in range(self.choices) if n not in self.excluded],
+                k=self.member,
+            )[-1]
+        self.gvar = self.gvar[:-1] + str(choice)
+
+    @property
+    def realkind(self):
+        return "rtcoef"
+
+
 class RRTM(GenvModelResource):
     """
     Class of a tar-zip file of coefficients for radiative transfers computations.

vortex/nwp/data/diagnostics.py

@@ -145,10 +145,7 @@ _surfex_diag_decofp = footprints.DecorativeFootprint(
             ]
         ),
         nativefmt=dict(
-            values=[
-                "netcdf",
-                "grib",
-            ],
+            values=["netcdf", "grib", "fa"],
             default="netcdf",
             optional=True,
         ),
@@ -185,6 +182,7 @@ class ObjTrack(GeoFlowResource):
                 values=[
                     "json",
                     "hdf5",
+                    "tar",
                     "foo",
                 ],
                 default="foo",

vortex/nwp/tools/conftools.py

@@ -1045,11 +1045,11 @@ class ArpIfsForecastTermConfTool(ConfTool):
     The forecast term can be retrieved:
 
       >>> print(ct.fcterm('assim', 6))
-      6
+      6.0
       >>> print(ct.fcterm('production', 0))
-      102
+      102.0
       >>> print(ct.fcterm('production', 12))
-      24
+      24.0
 
     If nothing is defined it crashes:
 
@@ -1451,11 +1451,12 @@ class ArpIfsForecastTermConfTool(ConfTool):
         return self._lookup_rangex_cache[(what_desc, cutoff, hh)]
 
     def fcterm(self, cutoff, hh):
-        """The forecast term for **cutoff** and **hh**."""
+        """The forecast term for **cutoff** and **hh** as a float or int."""
         fcterm = self._cutoff_hh_lookup("fcterm", cutoff, hh)
-        if isinstance(fcterm, Time) and fcterm.minute == 0:
-            return fcterm.hour
+        if isinstance(fcterm, Time):
+            return fcterm.hour + (fcterm.minute / 60)
         else:
+            # fcterm is an int representing nb of timesteps
             return fcterm
 
     def hist_terms(self, cutoff, hh):

vortex/tools/systems.py

@@ -67,7 +67,6 @@ from vortex.tools.compression import CompressionPipeline
 from vortex.tools.env import Environment
 from vortex.tools.net import AssistedSsh, AutoRetriesFtp, DEFAULT_FTP_PORT
 from vortex.tools.net import FtpConnectionPool, LinuxNetstats, StdFtp
-import vortex.tools.storage
 from vortex import config
 
 #: No automatic export
@@ -1161,6 +1160,7 @@ class OSExtended(System):
                     logger.warning(
                         "Bad return code [%d] for %s", p.returncode, str(args)
                     )
+                    self.dump_spawn_to_script(args)
                     if isinstance(output, bool) and output:
                         sys.stderr.write(p_err.decode(plocale, "replace"))
                 if fatal:
@@ -1183,6 +1183,31 @@ class OSExtended(System):
 
         return rc
 
+    def dump_spawn_to_script(self, args):
+        """Dump spawn environment to a script that can be executed 'out-of-vortex'."""
+        script = [
+            "#!/bin/bash",
+            "",
+            'SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )',
+            "cd $SCRIPT_DIR",
+            "",
+            "ulimit -s unlimited",
+            "",
+        ]
+        # env vars
+        for k in sorted(self.env.keys()):
+            if not (k.startswith("SLURM") or k.startswith("MTOOL")):
+                script.append('export {}="{}"'.format(k, self.env[k]))
+        # command line
+        script.append("\n" + " ".join(args))
+        # replace any MTOOL dirs from spool to abort
+        for i, line in enumerate(script):
+            script[i] = line.replace("spool/spool_", "abort/dump_")
+        # write to file
+        with open("spawn_dump.sh", "w") as o:
+            for l in script:
+                o.write(l + "\n")
+
     def getlogname(self):
         """Be sure to get the actual login name."""
         return passwd.getpwuid(self._os.getuid())[0]
@@ -1860,10 +1885,15 @@ class OSExtended(System):
         """Return a cache object for the FtSpool."""
         if self._ftspool_cache is not None:
             return self._ftspool_cache
+        try:
+            cacheloc = config.from_config(section="data-tree", key="rootdir")
+        except config.ConfigurationError:
+            cacheloc = os.path.join(os.environ["HOME"], ".vortex.d")
+
         self._ftspool_cache = footprints.proxy.cache(
             entry=os.path.join(
-                vortex.data.stores.get_cache_location(), "ftspool"
-            ),
+                cacheloc.replace("%usr%", self.glove.user), "ftspool"
+            )
         )
         return self._ftspool_cache
 

{vortex_nwp-2.2.0.dist-info → vortex_nwp-2.3.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: vortex-nwp
-Version: 2.2.0
+Version: 2.3.0
 Summary: A Python library to write Numerical Weather Prediction pipelines components
 Author-email: The Vortex Team <vortex.support@meteo.fr>
 License: CECILL-C
@@ -14,6 +14,7 @@ Requires-Dist: footprints
 Requires-Dist: taylorism
 Requires-Dist: tomli
 Requires-Dist: arpifs_listings
+Requires-Dist: importlib_metadata; python_version < "3.8"
 Provides-Extra: docs
 Requires-Dist: sphinx; extra == "docs"
 Requires-Dist: sphinx-book-theme; extra == "docs"
@@ -21,6 +22,8 @@ Requires-Dist: sphinx-copybutton; extra == "docs"
 Provides-Extra: dev
 Requires-Dist: ruff==0.9.1; extra == "dev"
 Requires-Dist: pytest; extra == "dev"
+Provides-Extra: git
+Requires-Dist: pygit2; extra == "git"
 Dynamic: license-file
 
 ## vortex

{vortex_nwp-2.2.0.dist-info → vortex_nwp-2.3.0.dist-info}/RECORD

@@ -1,12 +1,12 @@
-vortex/__init__.py,sha256=MVCUcPlaXXTr7csgYIUfa02hrrJMiNL7VoawK8vKFdg,4374
+vortex/__init__.py,sha256=iF0Qyy8b-ZIznlaDuFfFtuA3PU4d3OxQZVYM1TyU7fQ,4438
 vortex/config.py,sha256=YEpcc81wR94jvn36fPONmKc1tM4ap0s8Rc7v3KfDUok,2948
 vortex/gloves.py,sha256=GKz27S8eLfRlk8fNqVL_z1gsQ8zvEj73p5uVi11ou00,8487
 vortex/proxy.py,sha256=OlPrVUJS5FoKt5pX8ApN1crFFDj8RJAqhDEilwvfrYU,127
 vortex/sessions.py,sha256=zWVhsFGKl-SovyVAZz3mpvmojjHROt5iyERjwA6kudQ,10017
 vortex/toolbox.py,sha256=fDTvfghg8Yl7ggAwhIaY3V6owO3V1IG9LBf6F97A_50,42385
 vortex/algo/__init__.py,sha256=I_9COn_QBRbwvbqhs0X3CdHeR97NZhBacIqwKjnVBTg,359
-vortex/algo/components.py,sha256=JvznzerMhxLxbKZRWtQPIX7y-JjTAMwcIzG9TAXB7Sc,89460
-vortex/algo/mpitools.py,sha256=hMVKPqu3z4ohGt9OtKCiDRNK6RlVBWgjelDoNaDH5hw,74235
+vortex/algo/components.py,sha256=BodJA593f0OM1Tj5BAh-u-p55IbL81rwQR_g_Ee-mL0,89990
+vortex/algo/mpitools.py,sha256=4J-qGZ4olkgq6kpC2F81ogXeksXp4030ARg-QbCdQ9Y,74154
 vortex/algo/serversynctools.py,sha256=fPel0txVHsrUfk6VFaeKa0D6i21fOskIAR_BbByBv9g,5601
 vortex/algo/mpitools_templates/__init__.py,sha256=Jbw903aPqVKF-AaSoB-mGMxthSvm88O_yqGoGmf_S_U,18
 vortex/algo/mpitools_templates/envelope_wrapper_default.tpl,sha256=4VhkDx_YbOYywKQ82HIxRJXGcDpLuOgqcY7Edx9Rxyw,453
@@ -16,15 +16,15 @@ vortex/data/__init__.py,sha256=XaHof5W6oCalr4X6oYAK4zW3z3F6mKuwdbKxmmmDraY,560
 vortex/data/abstractstores.py,sha256=eghm3HSlUcZM8AGdx0js4Nt_IdxEiYffOmqS5gkpVsQ,55297
 vortex/data/containers.py,sha256=Qny5rqwXEdDWxG4JlC8KcsSAsHK3pp8qukZj1PTFBOM,26004
 vortex/data/contents.py,sha256=ZrwlJfOvkTemzikFRgYBQH3ApC-TPNrbZxpZstzDdbY,19079
-vortex/data/executables.py,sha256=FeR3SA2wW97zAQXwWVecZ0v6VYT5L_3K1Czuv33Bk74,6723
+vortex/data/executables.py,sha256=aFIf0gTEw1z60SlznfDzGG58vzE4191ObDrASt0hszw,7381
 vortex/data/flow.py,sha256=P1itBnA8jaoCWnVQjqbD_Pf26rpzud1JdwSLECDnDl4,3008
-vortex/data/geometries.ini,sha256=J7hX5hYWqwBjdUAul6q8j10U6b3I-QEHrFJ98LBTQXM,52805
+vortex/data/geometries.ini,sha256=W2yDVXwG5J0amjRt3xHjOrFr_xI7h3v75WFQcFda16w,54103
 vortex/data/geometries.py,sha256=q-0xmfBFN_TrMaojljO0wEpEwY9w1cbUuoBo49cvYhc,28221
 vortex/data/handlers.py,sha256=bChvizJnN5zxQxVf0oeUqcw_Wj-jnVszSqT1ovVUXFA,50525
 vortex/data/outflow.py,sha256=IPKJkn75lRvhSqN5969TuhRAPnsZKlrWR4Cmw6VBFDs,1475
-vortex/data/providers.py,sha256=IA_7JT8TNZy-2Dtk71JTICvdA5rqrzHLsEtkdU1z2g4,15860
+vortex/data/providers.py,sha256=s1CHOuogjzyPUHfK6gPpKU3IDCtySLTjPgkmuC4xiJA,16614
 vortex/data/resources.py,sha256=UOwvQDTJxS9z66Aa7JM2nWM3IxrMWvY3L2J9D5w3sZw,6348
-vortex/data/stores.py,sha256=LOxzqZmzLkOGTgbAuPTajSOdvEbUzXhxbSy6pTOdRG0,44092
+vortex/data/stores.py,sha256=e4fhTTHcnh7FhZ21M93bH5ixl_owArSMKlvlPIxHIF4,47735
 vortex/data/sync_templates/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 vortex/layout/__init__.py,sha256=aZsDhVJrd3-648vw-UESI_2BLxyGl71sHdyrg9cL638,826
 vortex/layout/contexts.py,sha256=GkysEtOOUj3nQvGUWrFjOOjzv7SmWFPUSLTH_pMMycI,20363
@@ -32,18 +32,18 @@ vortex/layout/dataflow.py,sha256=SKOUJKbtftW4POStjyBqNrqjvfbutsNHnWQ-p_75l_8,447
 vortex/layout/monitor.py,sha256=DIUXoW242tg5B6fFEEe2Og-NOxli1T_xpcNWFvTaWXQ,34316
 vortex/nwp/__init__.py,sha256=UfB2il-lEjF87PyVO9sGxjcuy3OYhZwskZw3z52VYAE,311
 vortex/nwp/algo/__init__.py,sha256=_it4GhWxay_WAy0qr2IxLdIqsjWxXcsSSB7wx1BitB4,577
-vortex/nwp/algo/assim.py,sha256=vt3UjvAik4xBl_QC-Yj1CxY4vaiAYvGzzf27XwLoykA,15922
+vortex/nwp/algo/assim.py,sha256=HIyOAkzWc8U42ttWy7nYpHiDoBjTp9WoO7F_SQau9v8,15209
 vortex/nwp/algo/clim.py,sha256=Z9QwUfosH0X_xTg6mam68qWIvw6Ts6CZIpc7Nb0NO1s,37511
 vortex/nwp/algo/coupling.py,sha256=6KQ_YB7PCvVbt6c2VEtpCzwIyWRTjLfu54mNpKnXL9Q,29755
 vortex/nwp/algo/eda.py,sha256=Wf2mt1r6v_ZYDz4BFyt-4JP1XAsO8sZW3O1UAOSRzmU,30746
 vortex/nwp/algo/eps.py,sha256=JCBFiaFF9TQ-x7Szgw44Cd39tRD9184UXqOtoT6aRCs,26812
-vortex/nwp/algo/forecasts.py,sha256=sqs0IETanpfS8q1f1Pl3mQZtSNbfOIUOidmcduGp8EU,30063
+vortex/nwp/algo/forecasts.py,sha256=qgpSYYsD0C9agBusBX38ulDesw01uU_QvOc28gspNCI,30971
 vortex/nwp/algo/fpserver.py,sha256=98HQZsdQE22spy_mIE1G-NORQMMjfA5grEjeIMSlP9w,50163
 vortex/nwp/algo/ifsnaming.py,sha256=WfpSpRJ7ua3ihqv8Y4UkrvE4pb0CkNWKIlW9EjY-2ao,11188
-vortex/nwp/algo/ifsroot.py,sha256=Mmp0sZtexP5ob3iUFTiVbgVWog7ubAYPut0cAB0vbbg,14230
-vortex/nwp/algo/monitoring.py,sha256=JVPZPw_I5QRVtYvvSYpJpBy5RRPhqBQjcAFfIkvKuHA,8893
+vortex/nwp/algo/ifsroot.py,sha256=RIJzn1r9uZ5Cno3WHQ6swEZUAlmndvYRtn9TQ3BVIXc,14771
+vortex/nwp/algo/monitoring.py,sha256=qwWupx4ITJx4vaQVfl3XhWdkaq2T8bNwk72Ur4v-Xw8,8893
 vortex/nwp/algo/mpitools.py,sha256=QO3I9iMdY69znnA6hJy8XUAzBoejDpsVMQFRo3YzXro,24876
-vortex/nwp/algo/odbtools.py,sha256=xltXS1lpxeeJXeCGnFiqFUhGqMI6BlZzKcSLhsnd2cs,44094
+vortex/nwp/algo/odbtools.py,sha256=SjYCYseLqrxlPtEHu4APwSZoCPwjl2rLvT1XC2kvql4,44104
 vortex/nwp/algo/oopsroot.py,sha256=es3p7RPOixQT74TcglJX1q8MswxveVnLZa84kyhu3uM,33806
 vortex/nwp/algo/oopstests.py,sha256=owQAS7_fGbUGZGzMsJek32dbb3JQ6dafSVBrdmveQ-Q,6654
 vortex/nwp/algo/request.py,sha256=y5N9RUfKoPXR6EbSely37Zz6d-2HYyfV3UCvvoenqbY,22809
@@ -53,9 +53,9 @@ vortex/nwp/data/assim.py,sha256=Dv5nmRYDDzYopC6xIwAl92HFTIz3k18C1jFE_IhYzuU,1009
 vortex/nwp/data/boundaries.py,sha256=xZoIriM5L-Q66_am6rOD2ZO0s-31iAUeTE7NZIXXvHI,8114
 vortex/nwp/data/climfiles.py,sha256=hlpe51wc-qrLFV_PX3vCedfLJu8Ko4VHa_qoVmA-Wcg,12610
 vortex/nwp/data/configfiles.py,sha256=0c3VvM25dhU6-QEJs5Wo4IckIVjXFHc-tTEzh1Bho1c,3760
-vortex/nwp/data/consts.py,sha256=riUtekoj3ozBgjxW6xEzhEaWWA7QvZFhU52Y3JS02Eg,21871
+vortex/nwp/data/consts.py,sha256=LCVZEqONqgDSeWbGSVYTIcdsFnvs0tpuoA3C_fYaEJU,23894
 vortex/nwp/data/ctpini.py,sha256=wEpJG7Px2hb7Vudn4eY2vJHKi8YYO99Sd6UJAeT5kMI,3537
-vortex/nwp/data/diagnostics.py,sha256=4ZP1w0ls2FIH59441rTCYDrp0JFfRJ7tiLDWmrdvuwM,5054
+vortex/nwp/data/diagnostics.py,sha256=Ckemxx6O7-nhOLKyZK3R8dlwMdEAcbSPEC2H4lk3Ikk,5040
 vortex/nwp/data/eda.py,sha256=rVQ35SpHVdHAQ10-LhyGy1k6P9Dbd6os8Crz2I4ThvY,4112
 vortex/nwp/data/eps.py,sha256=nLKTEpEeMxV9oE_sf87xihspMqwzXA8r9uJPP6PdycE,11861
 vortex/nwp/data/executables.py,sha256=xFRHzE5rK71m7gkd-LgMO3KSs0uBeCLqW-8vmedMHqQ,25036
@@ -80,7 +80,7 @@ vortex/nwp/tools/bdap.py,sha256=GdqC4QclVHzZUNQGYQPu9EmFPNEEBadDNGsLmrE2XUs,1689
 vortex/nwp/tools/bdcp.py,sha256=cDlkIVZW9grF2YnieIoiwsDhPxQsJIcaBuOCu6rsCzo,986
 vortex/nwp/tools/bdm.py,sha256=e9o3IpR9C4GGd7NhXWt0bpogrcVkqn_hbEJBtThSV5g,360
 vortex/nwp/tools/bdmp.py,sha256=tlsaPvsPtKSXrq2T7X_xtAdWGZ3Hg2qJUJfH8Fetx3E,1506
-vortex/nwp/tools/conftools.py,sha256=bCZ1vUMc1fkuzFEKw95KDXD9Fs4-uDg546RHuJyJmtU,59494
+vortex/nwp/tools/conftools.py,sha256=Awnf9Tied8qeWhv24GXbe3VF9DWPYScnPXjN2Viez-I,59578
 vortex/nwp/tools/drhook.py,sha256=8BgHw1UGxgviLche73XUltuUtkN9rqjBPy4oBLyG4KY,1972
 vortex/nwp/tools/grib.py,sha256=Uur8z4NjGoRXV4Via-63F9Px59FKG5UxiLM9yMgNkUs,10262
 vortex/nwp/tools/gribdiff.py,sha256=VD4nH06DuFheE9HsFT1UXWnvAC5vUfVBPeqQhCK4wbo,3137
@@ -125,7 +125,7 @@ vortex/tools/schedulers.py,sha256=7JtfYfSJgViytM7T26B61V8W83dvNwkDWdb2jY_Bn-A,15
 vortex/tools/services.py,sha256=hprRFmLcQRLVGDUnFxUFHaEB7ll1tNdt38hRQz8SWF8,29795
 vortex/tools/storage.py,sha256=e2GGTN_fl2wDNdEeYoD2V7jmwpZecb8u32rYvnqPMZk,34522
 vortex/tools/surfex.py,sha256=qbCIGt-dmfIRWRRMjOQnzc9sGNnt8PIFSqQYT77trhw,1408
-vortex/tools/systems.py,sha256=Zu04yTVIZU0hCiRH_STmdGuknjVd8dNsxXWr87k_5aE,149081
+vortex/tools/systems.py,sha256=fU46ZU6feCPkyCYti2Cs_jWbbYWv4zSBZskJwfSy_8U,150269
 vortex/tools/targets.py,sha256=-n2uMoEFdz3AeoSd1IGhlr7dYYb8xantUeh9_ggk9iI,14896
 vortex/util/__init__.py,sha256=Zt0OASbKbNnnwUqFHFoq5Mk13sGYSQhxqh7bUvJH6Y8,198
 vortex/util/config.py,sha256=KJHq0GMNRlwzh2WUdpwn1FIKCuAXla7nPRe7j3IZGcI,40045
@@ -137,8 +137,8 @@ vortex/util/roles.py,sha256=9un_QAijaMn5iTS7PrdoWI5_NNw7uHxMWTnyhc5aNzg,1150
 vortex/util/storefunctions.py,sha256=uSfG-G_A88iJf3DwFBd-j0rw6eJta8opfRT39aQHsHM,3615
 vortex/util/structs.py,sha256=vapErq0MNhiKlsnjrv_a5M0Rn29KbP3WE_oiy4Hfwb8,683
 vortex/util/worker.py,sha256=zp8f2tx4SXwf1v55XMdYLAx7n3vSlg8PRGrkHgnfdmg,4721
-vortex_nwp-2.2.0.dist-info/licenses/LICENSE,sha256=ewBJPmWAcQqtBPrydH10tt6ECkcYP3b1o2RfH85pJF0,21863
-vortex_nwp-2.2.0.dist-info/METADATA,sha256=Grzw9y3IoZf2MSjJhgvC7uE6_Zd7c7IlZh1hLi__8es,2185
-vortex_nwp-2.2.0.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-vortex_nwp-2.2.0.dist-info/top_level.txt,sha256=3xfbSD7kw8xKl0jk4GNHsOPKbhubstfWHPl6bxHciRQ,7
-vortex_nwp-2.2.0.dist-info/RECORD,,
+vortex_nwp-2.3.0.dist-info/licenses/LICENSE,sha256=ewBJPmWAcQqtBPrydH10tt6ECkcYP3b1o2RfH85pJF0,21863
+vortex_nwp-2.3.0.dist-info/METADATA,sha256=IYexH93RR5LZEqUCVzTfE5IIlngnReB61R5T7bHcErs,2301
+vortex_nwp-2.3.0.dist-info/WHEEL,sha256=wUyA8OaulRlbfwMtmQsvNngGrxQHAvkKcvRmdizlJi0,92
+vortex_nwp-2.3.0.dist-info/top_level.txt,sha256=3xfbSD7kw8xKl0jk4GNHsOPKbhubstfWHPl6bxHciRQ,7
+vortex_nwp-2.3.0.dist-info/RECORD,,

{vortex_nwp-2.2.0.dist-info → vortex_nwp-2.3.0.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: setuptools (80.9.0)
+Generator: setuptools (80.10.2)
 Root-Is-Purelib: true
 Tag: py3-none-any