vortex-nwp 2.1.1__tar.gz → 2.1.3__tar.gz

{vortex_nwp-2.1.1 → vortex_nwp-2.1.3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: vortex-nwp
-Version: 2.1.1
+Version: 2.1.3
 Summary: A Python library to write Numerical Weather Prediction pipelines components
 Author-email: The Vortex Team <vortex.support@meteo.fr>
 License: CECILL-C

{vortex_nwp-2.1.1 → vortex_nwp-2.1.3}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "vortex-nwp"
-version = "2.1.1"
+version = "2.1.3"
 description = "A Python library to write Numerical Weather Prediction pipelines components"
 readme = "README.md"
 requires-python = ">=3.7"

{vortex_nwp-2.1.1 → vortex_nwp-2.1.3}/src/vortex/__init__.py

@@ -53,10 +53,10 @@ from .toolbox import algo as task
 
 from . import nwp as nwp  # footprints import
 
-__version__ = "2.1.1"
+__version__ = "2.1.3"
 __prompt__ = "Vortex v-" + __version__ + ":"
 
-__nextversion__ = "2.1.1"
+__nextversion__ = "2.1.4"
 __tocinfoline__ = "VORTEX core package"
 
 __all__ = [

{vortex_nwp-2.1.1 → vortex_nwp-2.1.3}/src/vortex/algo/components.py

@@ -1841,7 +1841,10 @@ class Parallel(xExecutableAlgoComponent):
     def _mpitool_attributes(self, opts):
         """Return the dictionary of attributes needed to create the mpitool object."""
         # Read the appropriate configuration in the target file
-        conf_dict = config.from_config(section="mpitool")
+        if not config.is_defined(section="mpitool"):
+            conf_dict = {}
+        else:
+            conf_dict = config.from_config(section="mpitool")
         if self.mpiname:
             conf_dict["mpiname"] = self.mpiname
         # Make "mpirun" the default mpi command name

{vortex_nwp-2.1.1 → vortex_nwp-2.1.3}/src/vortex/algo/mpitools.py

@@ -84,6 +84,7 @@ from vortex.tools import env
 from vortex.tools.arm import ArmForgeTool
 from vortex.tools.systems import ExecutionError
 from vortex.util import config
+from vortex.config import is_defined, ConfigurationError
 
 #: No automatic export
 __all__ = []
@@ -1431,12 +1432,15 @@ class SRun(MpiTool):
     @property
     def _actual_slurmversion(self):
         """Return the slurm major version number."""
-        slurmversion = self.slurmversion or from_config(
-            section="mpitool", key="slurmversion"
-        )
-        if not slurmversion:
-            raise ValueError("No slurm version specified")
-        return slurmversion
+        if self.slurmversion:
+            return self.slurmversion
+
+        if not is_defined(section="mpitool", key="slurmversion"):
+            raise ConfigurationError(
+                "Using 'srun' MPI tool but slurm version is not configured. See "
+                "https://vortex-nwp.readthedocs.io/en/latest/user-guide/configuration.html#mpitool"
+            )
+        return from_config(section="mpitool", key="slurmversion")
 
     def _set_binaries_hack(self, binaries):
         """Set the list of :class:`MpiBinaryDescription` objects associated with this instance."""

{vortex_nwp-2.1.1 → vortex_nwp-2.1.3}/src/vortex/data/abstractstores.py

@@ -14,7 +14,7 @@ from bronx.system import hash as hashutils
 import footprints
 
 from vortex import sessions
-from vortex.config import from_config, ConfigurationError
+from vortex.config import from_config, ConfigurationError, is_defined
 from vortex.syntax.stdattrs import (
     hashalgo,
     hashalgo_avail_list,
@@ -870,6 +870,13 @@ class ArchiveStore(Store):
     @property
     def actual_storetube(self):
         """This archive network name (potentially read form the configuration file)."""
+        if self._actual_storetube:
+            return self._actual_storetube
+        if not is_defined(section="storage", key="protocol"):
+            raise ConfigurationError(
+                "Using remote data tree but protocol is not configured. See "
+                "https://vortex-nwp.readthedocs.io/en/latest/user-guide/configuration.html#storage"
+            )
         if self._actual_storetube is None:
             self._actual_storetube = from_config(
                 section="storage",

{vortex_nwp-2.1.1 → vortex_nwp-2.1.3}/src/vortex/data/geometries.py

@@ -764,12 +764,25 @@ def load(inifile="@geometries.ini", refresh=False, verbose=True):
 
     The class that will be instantiated depends on the "kind" keyword..
     """
+    from vortex import sessions
+
     iniconf = configparser.ConfigParser()
+
+    # Load from vortex distribution
     with importlib.resources.open_text(
         "vortex.data",
         "geometries.ini",
     ) as fh:
         iniconf.read_file(fh)
+
+    # Load from user's config directory if it exists
+    glove = sessions.current().glove
+    local = sessions.system()
+    user_geometries = glove.configrc + "/geometries.ini"
+    if local.path.exists(user_geometries):
+        with open(user_geometries, encoding="utf-8") as fh:
+            iniconf.read_file(fh)
+
     for item in iniconf.sections():
         gdesc = dict(iniconf.items(item))
         gkind = gdesc.get("kind")

{vortex_nwp-2.1.1 → vortex_nwp-2.1.3}/src/vortex/data/providers.py

@@ -279,6 +279,7 @@ class Vortex(Provider):
 
     _DEFAULT_NAME_BUILDER = names.VortexNameBuilder()
     _CUSTOM_NAME_BUILDERS = dict()
+    _SPECIAL_EXPS = ("oper", "dble", "test", "mirr")
 
     _footprint = [
         block,
@@ -365,8 +366,8 @@ class Vortex(Provider):
             self._namebuilder = self._CUSTOM_NAME_BUILDERS[self.namebuild]
         else:
             self._namebuilder = self._DEFAULT_NAME_BUILDER
-        if self.experiment in ("oper", "dble"):
-            self.experiment = self.experiment.upper()
+        if self.experiment in (n.upper() for n in self._SPECIAL_EXPS):
+            self.experiment = self.experiment.lower()
 
         # Ensure compatibility with deprecated namespace attribute
         # Under the hood the namespace attribute is still used to
@@ -426,7 +427,7 @@ class Vortex(Provider):
 
     def netloc(self, resource):
         """Returns the current ``namespace``."""
-        if self.experiment in ("OPER", "DBLE"):
+        if self.experiment in self._SPECIAL_EXPS:
             return "vsop." + self.namespace.domain
         return self.namespace.netloc
 

{vortex_nwp-2.1.1 → vortex_nwp-2.1.3}/src/vortex/data/stores.py

@@ -908,16 +908,9 @@ class VortexArchiveStore(MultiStore):
 
     def alternates_netloc(self):
         """Return netlocs describing both base and stacked archives."""
-        netloc_m = re.match(
-            r"(?P<base>v.*)\.archive\.(?P<country>\w+)", self.netloc
-        )
         return [
-            "{base:s}.archive-legacy.{country:s}".format(
-                **netloc_m.groupdict()
-            ),
-            "{base:s}.stacked-archive-legacy.{country:s}".format(
-                **netloc_m.groupdict()
-            ),
+            f"{self.netloc.firstname}.archive-legacy.fr",
+            f"{self.netloc.firstname}.stacked-archive-legacy.fr",
         ]
 
     def alternates_fpextras(self):
@@ -1025,18 +1018,16 @@ class VortexCacheOp2ResearchStore(_VortexCacheBaseStore):
 
     def __init__(self, *args, **kw):
         super().__init__(*args, **kw)
-        try:
-            cachepath = config.from_config(
-                section="data-tree",
-                key="op_rootdir",
-            )
-        except config.ConfigurationError as e:
-            logger.error(
-                "Cannot use special experiment cache without providing",
-                "cache location",
+        if not config.is_defined(section="data-tree", key="op_rootdir"):
+            raise config.ConfigurationError(
+                "Using special experiment but corresponding cache location "
+                'is not configured. Bet sure to set "op_rootdir" in configuration. '
+                "See https://vortex-nwp.readthedocs.io/en/latest/user-guide/oper-dble-data-trees"
             )
-            raise e
-
+        cachepath = config.from_config(
+            section="data-tree",
+            key="op_rootdir",
+        )
         self.location = os.path.join(cachepath, "vortex")
 
 
@@ -1341,6 +1332,10 @@ class PromiseCacheStore(VortexCacheMtStore):
         ),
     )
 
+    def __init__(self, *args, **kw):
+        super().__init__(*args, **kw)
+        self.location = os.path.join(get_cache_location(), "promise")
+
     @staticmethod
     def _add_default_options(options):
         options_upd = options.copy()

{vortex_nwp-2.1.1 → vortex_nwp-2.1.3}/src/vortex/nwp/data/stores.py

@@ -12,6 +12,7 @@ from bronx.fancies import loggers
 from bronx.stdtypes import date
 from vortex.data.abstractstores import Store
 from vortex.syntax.stdattrs import compressionpipeline
+from vortex import config
 
 #: No automatic export
 __all__ = []
@@ -109,14 +110,17 @@ class BdpeStore(Store):
         if s_archive == "True":
             extraenv["BDPE_LECTURE_ARCHIVE_AUTORISEE"] = "oui"
 
-        wsinterpreter = self.system.default_target.get(
-            "bdpe:wsclient_interpreter", None
-        )
-        wscommand = self.system.default_target.get("bdpe:wsclient_path", None)
-        if wscommand is None:
-            raise RuntimeError(
-                "bdpe:wsclient_path has to be set in the target config"
+        try:
+            wsinterpreter = config.from_config(
+                section="bdpe", key="wsclient_interpreter"
             )
+        except config.ConfigurationError:
+            wsinterpreter = "bash"
+
+        try:
+            wscommand = config.from_config(section="bdpe", key="wsclient_path")
+        except config.ConfigurationError:
+            wscommand = "/opt/softs/sopra/bin/lirepe.sh"
 
         args.insert(0, wscommand)
         if wsinterpreter is not None:

{vortex_nwp-2.1.1 → vortex_nwp-2.1.3}/src/vortex/tools/grib.py

@@ -567,7 +567,7 @@ class EcGribDecoMixin(AlgoComponentDecoMixin):
     def _eccodes_envsetup(
         self,
         eccodes_lib,
-        envvar="ECCODES_DEFINITIONS_PATH",
+        envvar="ECCODES_DEFINITION_PATH",
         tgt_path="definitions",
     ):
         """Export envirionment variables required by ECCODES
@@ -612,7 +612,7 @@ class EcGribDecoMixin(AlgoComponentDecoMixin):
         if eccodes_lib is not None:
             defvar = self._eccodes_envsetup(
                 eccodes_lib,
-                envvar="ECCODES_DEFINITIONS_PATH",
+                envvar="ECCODES_DEFINITION_PATH",
                 tgt_path="definitions",
             )
             subdir = Path("ifs_samples") / (

{vortex_nwp-2.1.1 → vortex_nwp-2.1.3}/src/vortex/tools/schedulers.py

@@ -4,12 +4,14 @@ Interface to SMS commands.
 
 import contextlib
 import functools
+import re
+import socket
 
 from bronx.fancies import loggers
 import footprints
 
 from vortex import config
-from .services import Service
+from .services import Service, get_cluster_name
 
 __all__ = []
 
@@ -363,46 +365,40 @@ class EcFlow(EcmwfLikeScheduler):
     def __init__(self, *args, **kw):
         logger.debug("EcFlow scheduler client init %s", self)
         super().__init__(*args, **kw)
-        self._actual_clientpath = self.clientpath
-
-    def path(self):
-        """Return the actual binary path to the EcFlow client."""
-        if self._actual_clientpath is None:
-            thistarget = self.sh.default_target
-            guesspath = self.env.ECF_CLIENT_PATH or thistarget.get(
-                "ecflow:clientpath"
-            )
-            ecfversion = self.env.get("ECF_VERSION", "default")
-            guesspath = guesspath.format(version=ecfversion)
-            if guesspath is None:
-                logger.warning(
-                    "ecFlow service could not guess the install location [%s]",
-                    str(guesspath),
-                )
+        if not self.clientpath:
+            if not config.is_defined(section="ecflow", key="clientpath"):
+                self.clientpath = "ecflow_client"
             else:
-                self._actual_clientpath = guesspath
-        if not self.sh.path.exists(self._actual_clientpath):
-            logger.warning(
-                "No ecFlow client found at init time [path:%s]>",
-                self._actual_clientpath,
-            )
-        return self._actual_clientpath
+                self.clientpath = config.from_config(
+                    section="ecflow",
+                    key="clientpath",
+                )
 
     @contextlib.contextmanager
     def child_session_setup(self):
         """Setup a SSH tunnel if necessary."""
         with super().child_session_setup() as setup_rc:
-            if setup_rc and not self.sh.default_target.isnetworknode:
+            name = get_cluster_name(socket.gethostname())
+            #  If the current node is a compute node, it cannot reach
+            #  the EcFlow server.  In this case, the request is made
+            #  through a SSH tunnel on taranisoper-int
+            is_compute_node = re.match(
+                rf"{name}\d+\.{name}hpc\.meteo\.fr", socket.gethostname()
+            )
+            if setup_rc and is_compute_node:
                 tunnel = None
                 # wait and retries from config
-                thistarget = self.sh.default_target
-                sshwait = float(thistarget.get("ecflow:sshproxy_wait", 6))
-                sshretries = float(
-                    thistarget.get("ecflow:sshproxy_retries", 2)
-                )
-                sshretrydelay = float(
-                    thistarget.get("ecflow:sshproxy_retrydelay", 1)
-                )
+                ssh_settings = {
+                    conf_key: default
+                    if not config.is_defined("ecflow", conf_key)
+                    else config.from_config("ecflow", conf_key)
+                    for conf_key, default in (
+                        ("sshproxy_wait", 6),
+                        ("sshproxy_retries", 2),
+                        ("sshproxy_retrydelay", 1),
+                    )
+                }
+
                 # Build up an SSH tunnel to convey the EcFlow command
                 ecconf = self.conf(dict())
                 echost = ecconf.get("{:s}HOST".format(self.env_pattern), None)
@@ -411,14 +407,15 @@ class EcFlow(EcmwfLikeScheduler):
                     setup_rc = False
                 else:
                     sshobj = self.sh.ssh(
-                        "network",
-                        virtualnode=True,
+                        hostname=f"{name}oper-int",
                         mandatory_hostcheck=False,
-                        maxtries=sshretries,
-                        triesdelay=sshretrydelay,
+                        maxtries=ssh_settings["sshproxy_retries"],
+                        triesdelay=ssh_settings["sshproxy_retrydelay"],
                     )
                     tunnel = sshobj.tunnel(
-                        echost, int(ecport), maxwait=sshwait
+                        echost,
+                        int(ecport),
+                        maxwait=ssh_settings["sshproxy_wait"],
                     )
                     if not tunnel:
                         setup_rc = False
@@ -458,9 +455,7 @@ class EcFlow(EcmwfLikeScheduler):
 
     def _actual_child(self, cmd, options, critical=True):
         """Miscellaneous ecFlow sub-command."""
-        args = [
-            self.path(),
-        ]
+        args = [self.clientpath]
         if options:
             args.append("--{:s}={!s}".format(cmd, options[0]))
             if len(options) > 1:

{vortex_nwp-2.1.1 → vortex_nwp-2.1.3}/src/vortex/tools/services.py

@@ -9,6 +9,7 @@ import configparser
 import contextlib
 import hashlib
 import pprint
+import re
 from string import Template
 
 
@@ -23,6 +24,7 @@ from vortex.util.config import (
     load_template,
     LegacyTemplatingAdapter,
 )
+from vortex import config
 
 #: No automatic export
 __all__ = []
@@ -530,8 +532,6 @@ class HideService(Service):
         """Main action: hide a cheap copy of this file under a unique name."""
 
         rootdir = self.rootdir
-        if rootdir is None:
-            rootdir = self.sh.default_target.get("hidden_rootdir", None)
         if rootdir is not None:
             rootdir = self.sh.path.expanduser(rootdir)
 
@@ -795,12 +795,11 @@ class TemplatedMailService(MailService):
         """
         tpl = self.message
         if tpl == "":
-            tplfile = self.section.get("template", self.id)
-            tplfile = self._template_name_rewrite(tplfile)
+            tplpath = self._TEMPLATES_DIR / (
+                self.section.get("template", self.id) + ".tpl"
+            )
             try:
-                tpl = load_template(
-                    self.ticket, tplfile, encoding=self.inputs_charset
-                )
+                tpl = load_template(tplpath, encoding=self.inputs_charset)
             except ValueError as exc:
                 logger.error("%s", exc.message)
                 return None
@@ -938,3 +937,18 @@ class AbstractRdTemplatedMailService(TemplatedMailService):
         # The generic host/cluster name
         sdict["host"] = self.sh.default_target.inetname
         return sdict
+
+
+def get_cluster_name(hostname):
+    if not config.is_defined(section="services", key="cluster_names"):
+        raise config.ConfigurationError(
+            'Missing configuration key "cluster_names" in section "services". '
+            "See https://vortex-nwp.readthedocs.io/en/latest/user-guide/configuration.html#services"
+        )
+    cluster_names = config.from_config(section="services", key="cluster_names")
+    m = re.match("^(" + "|".join(n for n in cluster_names) + ")", hostname)
+    if (m is None) or (m.group(0) not in cluster_names):
+        raise ValueError(
+            f"Current host should be either one of {cluster_names}"
+        )
+    return m.group(0)

{vortex_nwp-2.1.1 → vortex_nwp-2.1.3}/src/vortex_nwp.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: vortex-nwp
-Version: 2.1.1
+Version: 2.1.3
 Summary: A Python library to write Numerical Weather Prediction pipelines components
 Author-email: The Vortex Team <vortex.support@meteo.fr>
 License: CECILL-C

{vortex_nwp-2.1.1 → vortex_nwp-2.1.3}/src/vortex_nwp.egg-info/SOURCES.txt

@@ -154,10 +154,12 @@ tests/test_config.py
 tests/test_conftools.py
 tests/test_containers.py
 tests/test_doctests.py
+tests/test_ecflow.py
 tests/test_ecmwf_interface.py
 tests/test_env.py
 tests/test_epygram.py
 tests/test_gco.py
+tests/test_geometries.py
 tests/test_ifstools.py
 tests/test_import.py
 tests/test_iosponge.py

vortex_nwp-2.1.3/tests/test_ecflow.py

@@ -0,0 +1,37 @@
+from unittest.mock import MagicMock, patch
+import vortex
+from footprints import proxy as fp
+
+
+@patch("vortex.tools.systems.OSExtended.ssh")
+@patch("socket.gethostname")
+def test_child_session_setup(mocked_gethostname, mocked_ssh):
+    vortex.config.set_config(
+        "services", "cluster_names", ["belenos", "taranis"]
+    )
+    vortex.config.set_config("ecflow", "sshproxy_wait", 12)
+    vortex.config.set_config("ecflow", "sshproxy_retries", 4)
+    vortex.config.set_config("ecflow", "sshproxy_retrydelay", 2)
+    mocked_gethostname.return_value = "belenos191.belenoshpc.meteo.fr"
+    s = fp.service(kind="ecflow", clientpath="/path/to/ecflow_client")
+    s.env["ECF_HOST"] = "ecfhost"
+    s.env["ECF_PORT"] = 28943
+
+    mocked_ssh.return_value.tunnel.return_value.entranceport = 42
+
+    with s.child_session_setup() as setup:
+        assert setup
+        assert s.env["ECF_HOST"] == "localhost"
+        assert s.env["ECF_PORT"] == 42
+
+    mocked_ssh.assert_called_once_with(
+        hostname="belenosoper-int",
+        mandatory_hostcheck=False,
+        maxtries=4,
+        triesdelay=2,
+    )
+    mocked_ssh.return_value.tunnel.assert_called_once_with(
+        "ecfhost",
+        28943,
+        maxwait=12,
+    )