vortex-nwp 2.1.3__py3-none-any.whl → 2.3.0__py3-none-any.whl

vortex/data/providers.py

@@ -14,6 +14,7 @@ from bronx.fancies import loggers
 import footprints
 from footprints import proxy as fpx
 
+import vortex
 from vortex import config
 from vortex.syntax.stdattrs import (
     xpid,
@@ -56,16 +57,13 @@ class Provider(footprints.FootprintBase):
                 info="The username that will be used whenever necessary.",
                 optional=True,
                 default=None,
+                access="rwx",
                 alias=("user", "logname"),
             ),
         ),
         fastkeys={"namespace"},
     )
 
-    def __init__(self, *args, **kw):
-        logger.debug("Abstract provider init %s", self.__class__)
-        super().__init__(*args, **kw)
-
     def _str_more(self):
         """Additional information to print representation."""
         try:
@@ -87,7 +85,9 @@ class Provider(footprints.FootprintBase):
 
     def netuser_name(self, resource):  # @UnusedVariable
         """Abstract method."""
-        return self.username
+        if self.username is not None:
+            return self.username
+        return vortex.ticket().glove.user
 
     def pathname(self, resource):
         """Abstract method."""
@@ -119,10 +119,11 @@ class Provider(footprints.FootprintBase):
         The different operations of the algorithm can be redefined by subclasses.
         """
         username = self.netuser_name(resource)
+        netloc = self.netloc(resource)
         fullnetloc = (
             "{:s}@{:s}".format(username, self.netloc(resource))
-            if username
-            else self.netloc(resource)
+            if (username and netloc)
+            else netloc
         )
         logger.debug(
             "scheme %s netloc %s normpath %s urlquery %s",
@@ -274,12 +275,50 @@ def set_namespace_from_cache_settings(usecache, usearchive):
     return ".".join(("vortex", domain, "fr"))
 
 
+class Git(Provider):
+    _footprint = dict(
+        info="Git provider",
+        attr=dict(
+            ref=dict(
+                type=str,
+                optional=True,
+                default=None,
+                info="The reference's SHA-1 hash id",
+            ),
+            repo=dict(
+                type=str,
+                info="Path to the Git repository",
+            ),
+            path=dict(
+                type=str,
+                info="File path within the repository",
+            ),
+        ),
+    )
+
+    def scheme(self, resource):
+        return "git"
+
+    def urlquery(self, resource):
+        return (
+            f"repo={self.repo}&ref={self.ref}"
+            if self.ref
+            else f"repo={self.repo}"
+        )
+
+    def basename(self, resource):
+        return self.path
+
+    def pathname(sef, resource):
+        return "."
+
+
 class Vortex(Provider):
     """Main provider of the toolbox, using a fix-size path and a dedicated name factory."""
 
     _DEFAULT_NAME_BUILDER = names.VortexNameBuilder()
     _CUSTOM_NAME_BUILDERS = dict()
-    _SPECIAL_EXPS = ("oper", "dble", "test", "mirr")
+    _SPECIAL_EXPS = ("OPER", "DBLE", "TEST", "MIRR")
 
     _footprint = [
         block,
@@ -366,8 +405,8 @@ class Vortex(Provider):
             self._namebuilder = self._CUSTOM_NAME_BUILDERS[self.namebuild]
         else:
             self._namebuilder = self._DEFAULT_NAME_BUILDER
-        if self.experiment in (n.upper() for n in self._SPECIAL_EXPS):
-            self.experiment = self.experiment.lower()
+        if self.experiment in (n.lower() for n in self._SPECIAL_EXPS):
+            self.experiment = self.experiment.upper()
 
         # Ensure compatibility with deprecated namespace attribute
         # Under the hood the namespace attribute is still used to

vortex/data/stores.py

@@ -8,8 +8,11 @@ Store objects use the :mod:`footprints` mechanism.
 import copy
 import ftplib
 import io
+from pathlib import Path
 import os
 import re
+import shutil
+import subprocess
 
 from bronx.fancies import loggers
 import footprints
@@ -33,6 +36,13 @@ __all__ = []
 
 logger = loggers.getLogger(__name__)
 
+try:
+    import pygit2
+
+    NO_PYGIT2 = False
+except ImportError:
+    NO_PYGIT2 = True
+
 
 def get_cache_location():
     try:
@@ -335,7 +345,6 @@ class Finder(Store):
     def _ftpinfos(self, remote, **kwargs):
         args = kwargs.copy()
         args["hostname"] = self.hostname()
-        args["logname"] = remote["username"]
         port = self.hostname().netport
         if port is not None:
             args["port"] = port
@@ -422,6 +431,98 @@ class Finder(Store):
         return rc
 
 
+class GitStore(Store):
+    _footprint = dict(
+        info="A store to access Git repositories",
+        attr=dict(
+            scheme=dict(
+                values=["git"],
+            ),
+        ),
+    )
+
+    def gitget(self, remote, local, options):
+        if NO_PYGIT2:
+            raise ModuleNotFoundError(
+                "pygit2 is not installed in the current environment"
+            )
+        annex_cache_path = Path(remote["query"]["repo"][0])
+
+        # If no git reference is provided, only make a copy of the
+        # worktree
+        if "ref" not in remote["query"]:
+            shutil.copy(
+                src=Path(annex_cache_path) / remote["path"].lstrip("/"),
+                dst=local,
+            )
+            return local
+
+        assert "ref" in remote["query"]
+        path = remote["path"].lstrip("/")
+        repo = pygit2.Repository(annex_cache_path)
+        oid = repo.revparse_single(remote["query"]["ref"][0])
+        obj = oid.peel(pygit2.Tree) / path
+        if obj.filemode == pygit2.enums.FileMode.LINK:
+            # Here we need to discriminate between links that point
+            # to git-annex managed files and all other symlinks.
+
+            # If the file is a git-annex link, work out the location
+            # of the data within the .git/annex dir and copy this
+            # file into the cwd with the right name
+            if ".git/annex/objects" in obj.data.decode("ASCII"):
+                gitannex_key = Path(obj.data.decode("ASCII")).name
+                subprocess.run(
+                    args=["git-annex", "get", "--key", gitannex_key],
+                    cwd=str(annex_cache_path),
+                )
+                gitannex_content_location = subprocess.run(
+                    args=["git-annex", "contentlocation", gitannex_key],
+                    cwd=str(annex_cache_path),
+                    capture_output=True,
+                    encoding="ASCII",
+                ).stdout
+                shutil.copy(
+                    src=annex_cache_path / Path(gitannex_content_location),
+                    dst=local,
+                    follow_symlinks=True,
+                )
+                return local
+            os.symlink(
+                src=obj.data.decode("ASCII"),
+                dst=local,
+            )
+            return local
+
+        if obj.filemode == pygit2.enums.FileMode.BLOB:
+            with open(local, "wb") as dst:
+                # Could also use pygit2.BlobIO to stream content
+                # without having to load the entire blob data in
+                # memory:
+                #
+                # with pygit2.BlobIO(obj) as src:
+                #     shutil.copyfileobj(fsrc=src, fdst=dst)
+                dst.write(obj.data)
+            return local
+
+        if obj.filemode == pygit2.enums.FileMode.TREE:
+            if local.endswith("/"):
+                localpath = "."
+            else:
+                localpath = local
+            os.mkdir(local)
+            for subobj in obj:
+                r = {
+                    "query": {"ref": remote["query"]["ref"]},
+                    "path": str(Path(path) / subobj.name),
+                }
+                self.gitget(
+                    remote=r,
+                    local=str(Path(localpath) / subobj.name),
+                    options=None,
+                )
+            return local
+
+
 class _VortexStackedStorageMixin:
     """Mixin class that adds utility functions to work with stacked data."""
 
@@ -741,19 +842,6 @@ class VortexStdBaseArchiveStore(_VortexBaseArchiveStore):
     def remap_read(self, remote, options):
         """Reformulates the remote path to compatible vortex namespace."""
         remote = copy.copy(remote)
-        try:
-            remote["root"] = config.from_config(
-                section="storage",
-                key="rootdir",
-            )
-        except config.ConfigurationError as e:
-            msg = (
-                "Trying to write to archive but location is not configured. "
-                'Make sure key "rootdir" is defined in storage section of '
-                "the configuration.\n"
-                "See https://vortex-nwp.readthedocs.io/en/latest/user-guide/configuration.html#storage"
-            )
-            raise config.ConfigurationError(msg) from e
         return remote
 
     remap_write = remap_read
@@ -801,22 +889,13 @@ class VortexOpBaseArchiveStore(_VortexBaseArchiveStore):
         ),
     )
 
+    @property
+    def archive_entry(self):
+        return config.from_config(section="storage", key="op_rootdir")
+
     def remap_read(self, remote, options):
         """Reformulates the remote path to compatible vortex namespace."""
         remote = copy.copy(remote)
-        try:
-            remote["root"] = config.from_config(
-                section="storage",
-                key="op_rootdir",
-            )
-        except config.ConfigurationError as e:
-            msg = (
-                "Trying to write to operational data archive but location"
-                ' is not configured. Make sure key "op_rootdir" is defined in '
-                "the storage section of the configuration.\n"
-                "See https://vortex-nwp.readthedocs.io/en/latest/user-guide/configuration.html#storage"
-            )
-            raise config.ConfigurationError(msg) from e
         xpath = remote["path"].split("/")
         if len(xpath) >= 5 and re.match(r"^\d{8}T\d{2,4}", xpath[4]):
             # If a date is detected
@@ -915,7 +994,11 @@ class VortexArchiveStore(MultiStore):
 
     def alternates_fpextras(self):
         """Deal with some ArchiveStores' specific attributes."""
-        return dict(storehead=self.storehead, storesync=self.storesync)
+        return dict(
+            username=self.username,
+            storehead=self.storehead,
+            storesync=self.storesync,
+        )
 
 
 class _VortexCacheBaseStore(CacheStore, _VortexStackedStorageMixin):
@@ -993,9 +1076,23 @@ class VortexCacheMtStore(_VortexCacheBaseStore):
         ),
     )
 
-    def __init__(self, *args, **kw):
-        super().__init__(*args, **kw)
-        self.location = get_cache_location()
+    @property
+    def cache_entry(self):
+        try:
+            cacheloc = config.from_config(
+                section="data-tree",
+                key="rootdir",
+            )
+        except config.ConfigurationError:
+            cacheloc = os.path.join(os.environ["HOME"], ".vortex.d")
+
+        current_vortex_user = self.system.glove.user
+        cacheloc = cacheloc.replace("%usr%", current_vortex_user)
+
+        if self.username != current_vortex_user:
+            return os.path.join(cacheloc, self.username)
+
+        return cacheloc
 
 
 class VortexCacheOp2ResearchStore(_VortexCacheBaseStore):
@@ -1016,19 +1113,17 @@ class VortexCacheOp2ResearchStore(_VortexCacheBaseStore):
         ),
     )
 
-    def __init__(self, *args, **kw):
-        super().__init__(*args, **kw)
+    @property
+    def cache_entry(self):
         if not config.is_defined(section="data-tree", key="op_rootdir"):
-            raise config.ConfigurationError(
+            msg = (
                 "Using special experiment but corresponding cache location "
                 'is not configured. Bet sure to set "op_rootdir" in configuration. '
                 "See https://vortex-nwp.readthedocs.io/en/latest/user-guide/oper-dble-data-trees"
             )
-        cachepath = config.from_config(
-            section="data-tree",
-            key="op_rootdir",
-        )
-        self.location = os.path.join(cachepath, "vortex")
+            raise config.ConfigurationError(msg)
+
+        return config.from_config(section="data-tree", key="op_rootdir")
 
 
 class _AbstractVortexCacheMultiStore(MultiStore):
@@ -1103,6 +1198,9 @@ class VortexCacheStore(_AbstractVortexCacheMultiStore):
             f"{self.netloc.firstname}.stacked-cache-mt.fr",
         ]
 
+    def alternates_fpextras(self):
+        return dict(username=self.username)
+
 
 class VortexVsopCacheStore(_AbstractVortexCacheMultiStore):
     """The go to store for data cached by VORTEX operational experiments.
@@ -1145,6 +1243,9 @@ class VortexVsopCacheStore(_AbstractVortexCacheMultiStore):
             ]
         return todo
 
+    def alternates_fpextras(self):
+        return dict(username=self.username)
+
 
 class _AbstractVortexStackMultiStore(MultiStore):
     """Any Cache based Vortex multi store."""
@@ -1208,6 +1309,9 @@ class VortexStackStore(_AbstractVortexStackMultiStore):
         """Go through the various stacked stores."""
         return [f"{self.netloc.firstname}.stacked-cache-mt.fr"]
 
+    def alternates_fpextras(self):
+        return dict(username=self.username)
+
 
 class VortexVsopStackStore(_AbstractVortexStackMultiStore):
     """Store intended to read and write data into VORTEX R&D stacks."""
@@ -1267,6 +1371,9 @@ class VortexStoreLegacy(MultiStore):
             for d in (".cache.fr", ".archive-legacy.fr")
         ]
 
+    def alternates_fpextras(self):
+        return dict(username=self.username)
+
 
 class VortexStore(MultiStore):
     """Combined cache and archive VORTEX stores.
@@ -1315,6 +1422,9 @@ class VortexStore(MultiStore):
             )
         ]
 
+    def alternates_fpextras(self):
+        return dict(username=self.username)
+
 
 class PromiseCacheStore(VortexCacheMtStore):
     """Some kind of vortex cache for EXPECTED resources."""
@@ -1332,9 +1442,9 @@ class PromiseCacheStore(VortexCacheMtStore):
         ),
     )
 
-    def __init__(self, *args, **kw):
-        super().__init__(*args, **kw)
-        self.location = os.path.join(get_cache_location(), "promise")
+    @property
+    def cache_entry(self):
+        return os.path.join(super().cache_entry, "promise")
 
     @staticmethod
     def _add_default_options(options):

vortex/nwp/algo/assim.py

@@ -5,7 +5,7 @@ AlgoComponents dedicated to computations related to Data Assimilation systems.
 from bronx.fancies import loggers
 from bronx.stdtypes.date import Date
 
-from vortex.algo.components import BlindRun, Parallel
+from vortex.algo.components import BlindRun
 from vortex.syntax.stdattrs import a_date
 from .ifsroot import IFSParallel
 from ..tools import odb, drhook
@@ -16,36 +16,6 @@ __all__ = []
 logger = loggers.getLogger(__name__)
 
 
-class MergeVarBC(Parallel):
-    """Merge two VarBC files.
-
-    The VarBC file resulting from the MergeVarBC contains all the items of the
-    first VarBC file plus any new item that would be present in the second file.
-    """
-
-    _footprint = dict(
-        attr=dict(
-            kind=dict(
-                values=["mergevarbc"],
-            ),
-            varbcout=dict(
-                optional=True,
-                default="VARBC.cycle_out",
-            ),
-        )
-    )
-
-    def prepare(self, rh, opts):
-        """Find any ODB candidate in input files."""
-
-        sh = self.system
-
-        sh.touch(self.varbcout)
-
-        # Let ancesters doing real stuff
-        super().prepare(rh, opts)
-
-
 class Anamix(IFSParallel):
     """Merge the surface and atmospheric analyses into a single file"""
 

vortex/nwp/algo/forecasts.py

@@ -112,6 +112,8 @@ class Forecast(IFSParallel):
             # Possibly fix post-processing clim files
             self.all_localclim_fixer(rh, thismonth)
 
+            self.grab(analysis, comment="analysis")
+
             # File linking for IAU increments
             #
             # In the case of a forecast with IAU, the IFS executable
@@ -884,3 +886,31 @@ class OfflineSurfex(Parallel, DrHookDecoMixin):
             if namsec.rh.contents.dumps_needs_update:
                 namsec.rh.save()
             logger.info("Namelist dump: \n%s", namsec.rh.container.read())
+
+
+class MUSCForecast(Forecast):
+    """Forecast for MUSC single-column model."""
+
+    _footprint = dict(
+        info="Run a forecast with a MUSC single-column model.",
+        attr=dict(
+            kind=dict(
+                values=["musc"],
+            ),
+        ),
+    )
+
+    def postfix(self, rh, opts):
+        """Post forecast information and cleaning."""
+        sh = self.system
+        # rename specific output files with hours term on 4 digits for compatibility for fmth formatting
+        fmt = re.compile(r"Out\.(?P<termh>\d{3})\.\d{4}\.lfa$")
+        for f in [
+            fmt.match(f)
+            for f in sh.listdir()
+            if f.startswith("Out.") and f.endswith(".lfa")
+        ]:
+            sh.rename(
+                f.string, "Out.{:>04}.0000.lfa".format(int(f.group("termh")))
+            )
+        super().postfix(rh, opts)

vortex/nwp/algo/ifsroot.py

@@ -91,7 +91,7 @@ class IFSParallel(
                 ),
                 fcterm=dict(
                     info="The forecast term of the Arpege/IFS model.",
-                    type=int,
+                    type=float,
                     optional=True,
                     default=0,
                 ),
@@ -346,24 +346,6 @@ class IFSParallel(
         # be done by an extra class ... and it could be generalized to mpi
         # setup by the way !
         nam_updated = False
-        # For cy41 onward, replace some namelist macros with the command line
-        # arguments
-        if rh.resource.cycle >= "cy41":
-            if "NAMARG" in namcontents:
-                opts_arg = self.spawn_command_options()
-                self._set_nam_macro(
-                    namcontents, namlocal, "CEXP", opts_arg["name"]
-                )
-                self._set_nam_macro(
-                    namcontents, namlocal, "TIMESTEP", opts_arg["timestep"]
-                )
-                fcstop = "{:s}{:d}".format(
-                    opts_arg["fcunit"], opts_arg["fcterm"]
-                )
-                self._set_nam_macro(namcontents, namlocal, "FCSTOP", fcstop)
-                nam_updated = True
-            else:
-                logger.info("No NAMARG block in %s", namlocal)
 
         if self.member is not None:
             for macro_name in ("MEMBER", "PERTURB"):
@@ -371,7 +353,39 @@ class IFSParallel(
                     namcontents, namlocal, macro_name, self.member
                 )
             nam_updated = True
-        return nam_updated
+
+        if rh.resource.cycle < "cy41":
+            return nam_updated
+
+        if "NAMARG" not in namcontents:
+            logger.info("No NAMARG block in %s", namlocal)
+            return nam_updated
+
+        # For cy41 onward, replace some namelist macros with the command line
+        # arguments
+        opts_arg = self.spawn_command_options()
+        self._set_nam_macro(namcontents, namlocal, "CEXP", opts_arg["name"])
+        self._set_nam_macro(
+            namcontents, namlocal, "TIMESTEP", opts_arg["timestep"]
+        )
+
+        if self.fcunit == "t":
+            fcstop = "t{:d}".format(int(self.fcterm))
+        elif self.fcterm.is_integer():
+            # Round number of hours
+            fcstop = "h{:d}".format(int(self.fcterm))
+        else:
+            # IFS expects the forecast term to be given as an integer,
+            # whether this integer represents hours or timesteps. This
+            # means terms that are not round hours (e.g. 01:45) can only
+            # be expressed as a number of timesteps.
+            # See http://gitlab.meteo.fr/cnrm-gmap/vortex/-/issues/9
+            nsteps = int(self.fcterm * 3600 // self.timestep)
+            fcstop = "t{:d}".format(nsteps)
+            logger.info(f"Converting {self.fcterm} hours into {nsteps}")
+
+        self._set_nam_macro(namcontents, namlocal, "FCSTOP", fcstop)
+        return True
 
     def prepare_namelists(self, rh, opts=None):
         """Update each of the namelists."""

vortex/nwp/algo/monitoring.py

@@ -4,7 +4,7 @@ AlgoComponents dedicated to computations related to observations monitoring.
 
 from bronx.fancies import loggers
 
-from vortex.algo.components import Parallel
+from vortex.algo.components import BlindRun
 from vortex.syntax.stdattrs import a_date, a_model, a_cutoff
 from ..tools import odb, drhook
 
@@ -15,7 +15,7 @@ logger = loggers.getLogger(__name__)
 
 
 class OdbMonitoring(
-    Parallel, odb.OdbComponentDecoMixin, drhook.DrHookDecoMixin
+    BlindRun, odb.OdbComponentDecoMixin, drhook.DrHookDecoMixin
 ):
     """Compute monitoring statistics."""
 

vortex/nwp/algo/odbtools.py

@@ -16,7 +16,7 @@ from taylorism import Boss
 
 from vortex.tools.systems import ExecutionError
 
-from vortex.algo.components import Parallel, ParaBlindRun
+from vortex.algo.components import Parallel, ParaBlindRun, BlindRun
 from vortex.tools.parallelism import VortexWorkerBlindRun
 
 from ..syntax.stdattrs import arpifs_cycle
@@ -1217,7 +1217,7 @@ class OdbReshuffle(
 
 
 class FlagsCompute(
-    Parallel, odb.OdbComponentDecoMixin, drhook.DrHookDecoMixin
+    BlindRun, odb.OdbComponentDecoMixin, drhook.DrHookDecoMixin
 ):
     """Compute observations flags."""
 

vortex/nwp/data/consts.py

@@ -2,12 +2,15 @@
 Various Resources for constant files used in NWP.
 """
 
+import random
+
 import footprints
+from footprints import FPList
 from ..syntax.stdattrs import gvar
 from vortex.data.contents import DataRaw, JsonDictContent, TextContent
 from vortex.data.geometries import GaussGeometry, LonlatGeometry
 from vortex.data.outflow import ModelGeoResource, ModelResource, StaticResource
-from vortex.syntax.stdattrs import month_deco
+from vortex.syntax.stdattrs import date_deco, member, month_deco
 from vortex.syntax.stddeco import (
     namebuilding_append,
     namebuilding_delete,
@@ -180,6 +183,63 @@ class RtCoef(GenvModelResource):
         return "rtcoef"
 
 
+class RtCoefMulti(GenvModelResource):
+    """
+    RtCoeff Satellite coefficients, randomly chosen depending on the member of an ensemble.
+
+    Reproducibility is ensured by the stability of the random generation for a given date (ymdh) and member.
+    With member=0 or None, or with choices=0, the choice is always 0, even if 0 is excluded.
+    """
+
+    _footprint = [
+        date_deco,
+        member,
+        dict(
+            info="Set of satellite coefficients",
+            attr=dict(
+                kind=dict(
+                    values=["rtcoef_multi", "mwave_rtcoef_multi"],
+                ),
+                choices=dict(
+                    info="Number of choices to choose from (0..choices-1)",
+                    type=int,
+                    optional=True,
+                ),
+                excluded=dict(
+                    info="List of values excluded from choice",
+                    type=footprints.stdtypes.FPList,
+                    optional=True,
+                    default=FPList([]),
+                ),
+                gvar=dict(
+                    info="Will be modified by the random choice",
+                    default="[kind]_0",
+                    access="rwx",
+                ),
+            ),
+        ),
+    ]
+
+    def __init__(self, *args, **kw):
+        super().__init__(*args, **kw)
+        if self.member is None or self.member == 0 or self.choices == 0:
+            choice = 0
+        else:
+            # a random generator entirely determined by the date
+            rgen = random.Random(int(self.date.ymdh))
+            # drawing must be reproducible for a given member:
+            # generate 'member' values, but only keep the last
+            choice = rgen.choices(
+                [n for n in range(self.choices) if n not in self.excluded],
+                k=self.member,
+            )[-1]
+        self.gvar = self.gvar[:-1] + str(choice)
+
+    @property
+    def realkind(self):
+        return "rtcoef"
+
+
 class RRTM(GenvModelResource):
     """
     Class of a tar-zip file of coefficients for radiative transfers computations.