vortex-nwp 2.2.0__tar.gz → 2.3.0__tar.gz

{vortex_nwp-2.2.0/src/vortex_nwp.egg-info → vortex_nwp-2.3.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: vortex-nwp
-Version: 2.2.0
+Version: 2.3.0
 Summary: A Python library to write Numerical Weather Prediction pipelines components
 Author-email: The Vortex Team <vortex.support@meteo.fr>
 License: CECILL-C
@@ -14,6 +14,7 @@ Requires-Dist: footprints
 Requires-Dist: taylorism
 Requires-Dist: tomli
 Requires-Dist: arpifs_listings
+Requires-Dist: importlib_metadata; python_version < "3.8"
 Provides-Extra: docs
 Requires-Dist: sphinx; extra == "docs"
 Requires-Dist: sphinx-book-theme; extra == "docs"
@@ -21,6 +22,8 @@ Requires-Dist: sphinx-copybutton; extra == "docs"
 Provides-Extra: dev
 Requires-Dist: ruff==0.9.1; extra == "dev"
 Requires-Dist: pytest; extra == "dev"
+Provides-Extra: git
+Requires-Dist: pygit2; extra == "git"
 Dynamic: license-file
 
 ## vortex

{vortex_nwp-2.2.0 → vortex_nwp-2.3.0}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "vortex-nwp"
-version = "2.2.0"
+version = "2.3.0"
 description = "A Python library to write Numerical Weather Prediction pipelines components"
 readme = "README.md"
 requires-python = ">=3.7"
@@ -18,12 +18,18 @@ classifiers = [
     "License :: CeCILL-C Free Software License Agreement (CECILL-C)",
 ]
 dependencies = [
-    "bronx", "footprints", "taylorism", "tomli", "arpifs_listings",
+    "bronx",
+    "footprints",
+    "taylorism",
+    "tomli",
+    "arpifs_listings",
+    "importlib_metadata; python_version < '3.8'",
 ]
 
 [project.optional-dependencies]
 docs = ["sphinx", "sphinx-book-theme", "sphinx-copybutton"]
 dev = ["ruff==0.9.1", "pytest"]
+git = ["pygit2"]
 
 [tool.setuptools.packages.find]
 where = ["src"]

{vortex_nwp-2.2.0 → vortex_nwp-2.3.0}/src/vortex/__init__.py

@@ -50,13 +50,14 @@ from .toolbox import promise as promise
 from .toolbox import diff as diff
 from .toolbox import defaults as defaults
 from .toolbox import algo as task
+from .toolbox import VortexForceComplete as VortexForceComplete
 
 from . import nwp as nwp  # footprints import
 
-__version__ = "2.2.0"
+__version__ = "2.3.0"
 __prompt__ = "Vortex v-" + __version__ + ":"
 
-__nextversion__ = "2.2.1"
+__nextversion__ = "2.3.1"
 __tocinfoline__ = "VORTEX core package"
 
 __all__ = [

{vortex_nwp-2.2.0 → vortex_nwp-2.3.0}/src/vortex/algo/components.py

@@ -1548,12 +1548,25 @@ class Expresso(ExecutableAlgoComponent):
         attr=dict(
             interpreter=dict(
                 info="The interpreter needed to run the script.",
-                values=["current", "awk", "ksh", "bash", "perl", "python"],
+                values=[
+                    "current",
+                    "awk",
+                    "ksh",
+                    "bash",
+                    "perl",
+                    "python",
+                    "singularity",
+                ],
             ),
             interpreter_path=dict(
                 info="The interpreter command.",
                 optional=True,
             ),
+            interpreter_args=dict(
+                info="Some options to pass to the expresso",
+                optional=True,
+                default="",
+            ),
             engine=dict(values=["exec", "launch"]),
         ),
     )
@@ -1581,11 +1594,16 @@ class Expresso(ExecutableAlgoComponent):
                     )
 
     def _interpreter_args_fix(self, rh, opts):
+        cmd_opts = shlex.split(self.interpreter_args)
+
         absexec = self.absexcutable(rh.container.localpath())
         if self.interpreter == "awk":
-            return ["-f", absexec]
+            return [*cmd_opts, "-f", absexec]
+        if self.interpreter == "singularity":
+            return ["run", *cmd_opts, absexec]
         else:
             return [
+                *cmd_opts,
                 absexec,
             ]
 

{vortex_nwp-2.2.0 → vortex_nwp-2.3.0}/src/vortex/algo/mpitools.py

@@ -544,10 +544,6 @@ class MpiTool(footprints.FootprintBase):
         """When group are defined, associate each MPI rank with a "real" slot."""
         if self._ranks_map_cache is None:
             self._complex_ranks_map = False
-            if not self.envelope:
-                raise RuntimeError(
-                    "Ranks mapping should always be used within an envelope."
-                )
             # First deal with bingroups
             ranks_map = dict()
             has_bin_groups = not all([b.group is None for b in self.binaries])
@@ -624,7 +620,7 @@ class MpiTool(footprints.FootprintBase):
             if self._complex_ranks_map or do_bin_distribution:
                 if not self.envelope:
                     raise RuntimeError(
-                        "Ranks mapping shoudl always be used within an envelope."
+                        "Ranks mapping should always be used within an envelope."
                     )
             if do_bin_distribution:
                 if not self._supports_manual_ranks_mapping:
@@ -1404,7 +1400,10 @@ class SRun(MpiTool):
             ),
             optmap=dict(
                 default=footprints.FPDict(
-                    nn="nodes", nnp="ntasks-per-node", np="ntasks"
+                    nn="nodes",
+                    nnp="ntasks-per-node",
+                    np="ntasks",
+                    openmp="cpus-per-task",
                 )
             ),
             slurmversion=dict(type=int, optional=True),

{vortex_nwp-2.2.0 → vortex_nwp-2.3.0}/src/vortex/data/executables.py

@@ -106,6 +106,34 @@ class Script(Executable):
             return self.rawopts
 
 
+class SingularityImage(Executable):
+    """Singularity container image executable."""
+
+    _footprint = dict(
+        attr=dict(
+            kind=dict(
+                values=["image"],
+            ),
+            cmd=dict(
+                info="Command to run inside the container",
+                optional=True,
+                default="",
+            ),
+        ),
+    )
+
+    @property
+    def realkind(self):
+        return "image"
+
+    # Called by AlgoComponent.spawn_command_line
+    def command_line(self, **opts):
+        return " ".join(
+            ["--{:s} {:s}".format(opt, val) for opt, val in opts.items()]
+            + [self.cmd]
+        )
+
+
 class GnuScript(Executable):
     """Basic interpreted executable with standard command line arguments."""
 

{vortex_nwp-2.2.0 → vortex_nwp-2.3.0}/src/vortex/data/geometries.ini

@@ -659,6 +659,13 @@ area       = indonesie1
 resolution = 2.50
 runit      = km
 
+[asscomindonesie2sp]
+info       = Arome asscom14 - indonesie2 spectral geometry
+kind       = projected
+area       = indonesie2
+resolution = 2.50
+runit      = km
+
 [lace20km]
 info       = Coupling grid for LACE partners
 kind       = projected
@@ -673,6 +680,13 @@ area       = lace
 resolution = 8.0
 runit      = km
 
+[lace47]
+info       = LACE canari domain
+kind       = projected
+area       = lace47
+resolution = 4.71
+runit      = km
+
 [afgt20km]
 info       = Coupling grid for afgt-Praha
 kind       = projected
@@ -1201,15 +1215,15 @@ nlon       = 641
 [paris1s100]
 info       = PARIS 0.01
 kind       = lonlat
-area       = paris1s100
+area       = PARIS1S100
 resolution = 0.01
-nlat       = 221
-nlon       = 336
+nlat       = 187
+nlon       = 279
 
 [medalp1s100]
 info       = MEDALP 0.01
 kind       = lonlat
-area       = medalp1s100
+area       = MEDALP1S100
 resolution = 0.01
 nlat       = 481
 nlon       = 501
@@ -1677,6 +1691,17 @@ nlat       = 172800
 latmax     = 83.999
 latmin     = -55.999
 
+[srtm_01s]
+info       = alternative to no_arctics7.5s used to create arome-500m PGD clim
+kind       = lonlat
+area       = srtm
+resolution = 1
+runit      = s
+nlon       = 999
+nlat       = 999
+latmax     = 999
+latmin     = 999
+
 [europe_sw3s]
 kind       = lonlat
 area       = europe_sw
@@ -2468,39 +2493,39 @@ nlon       = 896
 resolution = 0.05
 runit      = deg
 
-[hycom3dmedoc]
-info       = Shom HYCOM3D Med-occidental at 1/60
+[hycom3dmednowe]
+info       = Shom HYCOM3D Med-NorthWest at 1/60
 kind       = lonlat
-area       = medoc
-nlat       = 331
-nlon       = 661
+area       = mednowe
+nlat       = 437
+nlon       = 761
 resolution = 0.017
 runit      = deg
 
-[hycom3dmedor]
-info       = Shom HYCOM3D Med-oriental at 1/60
+[hycom3dmedsowe]
+info       = Shom HYCOM3D Med-SouthWest at 1/60
 kind       = lonlat
-area       = medor
-nlat       = 437
-nlon       = 645
+area       = medsowe
+nlat       = 481
+nlon       = 1080
 resolution = 0.017
 runit      = deg
 
-[hycom3dmedio]
-info       = Shom HYCOM3D Med-ionian at 1/60
+[hycom3dmedea]
+info       = Shom HYCOM3D Med-East at 1/60
 kind       = lonlat
-area       = medio
-nlat       = 557
-nlon       = 781
+area       = medea
+nlat       = 706
+nlon       = 855
 resolution = 0.017
 runit      = deg
 
-[hycom3dgibal]
-info       = Shom HYCOM3D Gibraltair Strait-Alboran Sea at 1/60
+[hycom3dmedinte]
+info       = Shom HYCOM3D Med-Interior at 1/60
 kind       = lonlat
-area       = gibal
-nlat       = 361
-nlon       = 571
+area       = medinte
+nlat       = 935
+nlon       = 781
 resolution = 0.017
 runit      = deg
 
@@ -2575,6 +2600,13 @@ area       = corsica
 resolution = 2.5
 runit      = km
 
+[nl2500]
+info       = HARMONIE-AROME 2.5km NL domain for DAVAI
+kind       = projected
+area       = nl
+resolution = 2.5
+runit      = km
+
 [cors1s40]
 info       = Corsica 0.025 deg
 kind       = lonlat
@@ -2669,6 +2701,39 @@ area       = chmh2325
 resolution = 2.33
 runit      = km
 
+[armcu]
+info       = MUSC Single point ARMCU
+kind       = unstructured
+area       = armcu
+
+[global31c1]
+info       = ARPEGE T31C1.0 geometry (from Mitraillette tests)
+kind       = gauss
+area       = france
+truncation = 31
+stretching = 1.0
+
+[global30c24]
+info       = ARPEGE T30C2.4 geometry (from Mitraillette tests)
+kind       = gauss
+area       = france
+truncation = 30
+stretching = 2.4
+
+[portugal21km]
+info       = Mitraillette Portugal grid
+kind       = projected
+area       = portugal
+resolution = 21.3
+runit      = km
+
+[occitanie4km]
+info       = Mitraillette Occitanie grid
+kind       = projected
+area       = occitanie
+resolution = 4.4
+runit      = km
+
 #R ===================
 #R Combined geometries
 #R ===================

{vortex_nwp-2.2.0 → vortex_nwp-2.3.0}/src/vortex/data/providers.py

@@ -64,13 +64,6 @@ class Provider(footprints.FootprintBase):
         fastkeys={"namespace"},
     )
 
-    def __init__(self, *args, **kw):
-        logger.debug("Abstract provider init %s", self.__class__)
-        super().__init__(*args, **kw)
-
-        if not self.username:
-            self.username = vortex.ticket().glove.user
-
     def _str_more(self):
         """Additional information to print representation."""
         try:
@@ -92,7 +85,9 @@ class Provider(footprints.FootprintBase):
 
     def netuser_name(self, resource):  # @UnusedVariable
         """Abstract method."""
-        return self.username
+        if self.username is not None:
+            return self.username
+        return vortex.ticket().glove.user
 
     def pathname(self, resource):
         """Abstract method."""
@@ -124,10 +119,11 @@ class Provider(footprints.FootprintBase):
         The different operations of the algorithm can be redefined by subclasses.
         """
         username = self.netuser_name(resource)
+        netloc = self.netloc(resource)
         fullnetloc = (
             "{:s}@{:s}".format(username, self.netloc(resource))
-            if username
-            else self.netloc(resource)
+            if (username and netloc)
+            else netloc
         )
         logger.debug(
             "scheme %s netloc %s normpath %s urlquery %s",
@@ -279,6 +275,44 @@ def set_namespace_from_cache_settings(usecache, usearchive):
     return ".".join(("vortex", domain, "fr"))
 
 
+class Git(Provider):
+    _footprint = dict(
+        info="Git provider",
+        attr=dict(
+            ref=dict(
+                type=str,
+                optional=True,
+                default=None,
+                info="The reference's SHA-1 hash id",
+            ),
+            repo=dict(
+                type=str,
+                info="Path to the Git repository",
+            ),
+            path=dict(
+                type=str,
+                info="File path within the repository",
+            ),
+        ),
+    )
+
+    def scheme(self, resource):
+        return "git"
+
+    def urlquery(self, resource):
+        return (
+            f"repo={self.repo}&ref={self.ref}"
+            if self.ref
+            else f"repo={self.repo}"
+        )
+
+    def basename(self, resource):
+        return self.path
+
+    def pathname(sef, resource):
+        return "."
+
+
 class Vortex(Provider):
     """Main provider of the toolbox, using a fix-size path and a dedicated name factory."""
 

{vortex_nwp-2.2.0 → vortex_nwp-2.3.0}/src/vortex/data/stores.py

@@ -8,8 +8,11 @@ Store objects use the :mod:`footprints` mechanism.
 import copy
 import ftplib
 import io
+from pathlib import Path
 import os
 import re
+import shutil
+import subprocess
 
 from bronx.fancies import loggers
 import footprints
@@ -33,6 +36,13 @@ __all__ = []
 
 logger = loggers.getLogger(__name__)
 
+try:
+    import pygit2
+
+    NO_PYGIT2 = False
+except ImportError:
+    NO_PYGIT2 = True
+
 
 def get_cache_location():
     try:
@@ -335,7 +345,6 @@ class Finder(Store):
     def _ftpinfos(self, remote, **kwargs):
         args = kwargs.copy()
         args["hostname"] = self.hostname()
-        args["logname"] = remote["username"]
         port = self.hostname().netport
         if port is not None:
             args["port"] = port
@@ -422,6 +431,98 @@ class Finder(Store):
         return rc
 
 
+class GitStore(Store):
+    _footprint = dict(
+        info="A store to access Git repositories",
+        attr=dict(
+            scheme=dict(
+                values=["git"],
+            ),
+        ),
+    )
+
+    def gitget(self, remote, local, options):
+        if NO_PYGIT2:
+            raise ModuleNotFoundError(
+                "pygit2 is not installed in the current environment"
+            )
+        annex_cache_path = Path(remote["query"]["repo"][0])
+
+        # If no git reference is provided, only make a copy of the
+        # worktree
+        if "ref" not in remote["query"]:
+            shutil.copy(
+                src=Path(annex_cache_path) / remote["path"].lstrip("/"),
+                dst=local,
+            )
+            return local
+
+        assert "ref" in remote["query"]
+        path = remote["path"].lstrip("/")
+        repo = pygit2.Repository(annex_cache_path)
+        oid = repo.revparse_single(remote["query"]["ref"][0])
+        obj = oid.peel(pygit2.Tree) / path
+        if obj.filemode == pygit2.enums.FileMode.LINK:
+            # Here we need to discriminate between links that point
+            # to git-annex managed files and all other symlinks.
+
+            # If the file is a git-annex link, work out the location
+            # of the data within the .git/annex dir and copy this
+            # file into the cwd with the right name
+            if ".git/annex/objects" in obj.data.decode("ASCII"):
+                gitannex_key = Path(obj.data.decode("ASCII")).name
+                subprocess.run(
+                    args=["git-annex", "get", "--key", gitannex_key],
+                    cwd=str(annex_cache_path),
+                )
+                gitannex_content_location = subprocess.run(
+                    args=["git-annex", "contentlocation", gitannex_key],
+                    cwd=str(annex_cache_path),
+                    capture_output=True,
+                    encoding="ASCII",
+                ).stdout
+                shutil.copy(
+                    src=annex_cache_path / Path(gitannex_content_location),
+                    dst=local,
+                    follow_symlinks=True,
+                )
+                return local
+            os.symlink(
+                src=obj.data.decode("ASCII"),
+                dst=local,
+            )
+            return local
+
+        if obj.filemode == pygit2.enums.FileMode.BLOB:
+            with open(local, "wb") as dst:
+                # Could also use pygit2.BlobIO to stream content
+                # without having to load the entire blob data in
+                # memory:
+                #
+                # with pygit2.BlobIO(obj) as src:
+                #     shutil.copyfileobj(fsrc=src, fdst=dst)
+                dst.write(obj.data)
+            return local
+
+        if obj.filemode == pygit2.enums.FileMode.TREE:
+            if local.endswith("/"):
+                localpath = "."
+            else:
+                localpath = local
+            os.mkdir(local)
+            for subobj in obj:
+                r = {
+                    "query": {"ref": remote["query"]["ref"]},
+                    "path": str(Path(path) / subobj.name),
+                }
+                self.gitget(
+                    remote=r,
+                    local=str(Path(localpath) / subobj.name),
+                    options=None,
+                )
+            return local
+
+
 class _VortexStackedStorageMixin:
     """Mixin class that adds utility functions to work with stacked data."""
 
@@ -985,7 +1086,10 @@ class VortexCacheMtStore(_VortexCacheBaseStore):
         except config.ConfigurationError:
             cacheloc = os.path.join(os.environ["HOME"], ".vortex.d")
 
-        if self.username != self.system.glove.user:
+        current_vortex_user = self.system.glove.user
+        cacheloc = cacheloc.replace("%usr%", current_vortex_user)
+
+        if self.username != current_vortex_user:
             return os.path.join(cacheloc, self.username)
 
         return cacheloc
@@ -1205,6 +1309,9 @@ class VortexStackStore(_AbstractVortexStackMultiStore):
         """Go through the various stacked stores."""
         return [f"{self.netloc.firstname}.stacked-cache-mt.fr"]
 
+    def alternates_fpextras(self):
+        return dict(username=self.username)
+
 
 class VortexVsopStackStore(_AbstractVortexStackMultiStore):
     """Store intended to read and write data into VORTEX R&D stacks."""
@@ -1336,7 +1443,7 @@ class PromiseCacheStore(VortexCacheMtStore):
     )
 
     @property
-    def cache_promise(self):
+    def cache_entry(self):
         return os.path.join(super().cache_entry, "promise")
 
     @staticmethod

{vortex_nwp-2.2.0 → vortex_nwp-2.3.0}/src/vortex/nwp/algo/assim.py

@@ -5,7 +5,7 @@ AlgoComponents dedicated to computations related to Data Assimilation systems.
 from bronx.fancies import loggers
 from bronx.stdtypes.date import Date
 
-from vortex.algo.components import BlindRun, Parallel
+from vortex.algo.components import BlindRun
 from vortex.syntax.stdattrs import a_date
 from .ifsroot import IFSParallel
 from ..tools import odb, drhook
@@ -16,36 +16,6 @@ __all__ = []
 logger = loggers.getLogger(__name__)
 
 
-class MergeVarBC(Parallel):
-    """Merge two VarBC files.
-
-    The VarBC file resulting from the MergeVarBC contains all the items of the
-    first VarBC file plus any new item that would be present in the second file.
-    """
-
-    _footprint = dict(
-        attr=dict(
-            kind=dict(
-                values=["mergevarbc"],
-            ),
-            varbcout=dict(
-                optional=True,
-                default="VARBC.cycle_out",
-            ),
-        )
-    )
-
-    def prepare(self, rh, opts):
-        """Find any ODB candidate in input files."""
-
-        sh = self.system
-
-        sh.touch(self.varbcout)
-
-        # Let ancesters doing real stuff
-        super().prepare(rh, opts)
-
-
 class Anamix(IFSParallel):
     """Merge the surface and atmospheric analyses into a single file"""
 

{vortex_nwp-2.2.0 → vortex_nwp-2.3.0}/src/vortex/nwp/algo/forecasts.py

@@ -112,6 +112,8 @@ class Forecast(IFSParallel):
             # Possibly fix post-processing clim files
             self.all_localclim_fixer(rh, thismonth)
 
+            self.grab(analysis, comment="analysis")
+
             # File linking for IAU increments
             #
             # In the case of a forecast with IAU, the IFS executable
@@ -884,3 +886,31 @@ class OfflineSurfex(Parallel, DrHookDecoMixin):
             if namsec.rh.contents.dumps_needs_update:
                 namsec.rh.save()
             logger.info("Namelist dump: \n%s", namsec.rh.container.read())
+
+
+class MUSCForecast(Forecast):
+    """Forecast for MUSC single-column model."""
+
+    _footprint = dict(
+        info="Run a forecast with a MUSC single-column model.",
+        attr=dict(
+            kind=dict(
+                values=["musc"],
+            ),
+        ),
+    )
+
+    def postfix(self, rh, opts):
+        """Post forecast information and cleaning."""
+        sh = self.system
+        # rename specific output files with hours term on 4 digits for compatibility for fmth formatting
+        fmt = re.compile(r"Out\.(?P<termh>\d{3})\.\d{4}\.lfa$")
+        for f in [
+            fmt.match(f)
+            for f in sh.listdir()
+            if f.startswith("Out.") and f.endswith(".lfa")
+        ]:
+            sh.rename(
+                f.string, "Out.{:>04}.0000.lfa".format(int(f.group("termh")))
+            )
+        super().postfix(rh, opts)