vortex-nwp 2.1.0__tar.gz → 2.1.2__tar.gz

{vortex_nwp-2.1.0 → vortex_nwp-2.1.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: vortex-nwp
-Version: 2.1.0
+Version: 2.1.2
 Summary: A Python library to write Numerical Weather Prediction pipelines components
 Author-email: The Vortex Team <vortex.support@meteo.fr>
 License: CECILL-C

{vortex_nwp-2.1.0 → vortex_nwp-2.1.2}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "vortex-nwp"
-version = "2.1.0"
+version = "2.1.2"
 description = "A Python library to write Numerical Weather Prediction pipelines components"
 readme = "README.md"
 requires-python = ">=3.7"

{vortex_nwp-2.1.0 → vortex_nwp-2.1.2}/src/vortex/__init__.py

@@ -21,8 +21,8 @@ strongly advised.
 """
 
 import atexit
+from pathlib import Path
 import sys
-import os
 
 # importlib.metadata included in stdlib from 3.8 onwards.
 # For older versions, import third-party importlib_metadata
@@ -53,10 +53,10 @@ from .toolbox import algo as task
 
 from . import nwp as nwp  # footprints import
 
-__version__ = "2.1.0"
+__version__ = "2.1.2"
 __prompt__ = "Vortex v-" + __version__ + ":"
 
-__nextversion__ = "2.1.1"
+__nextversion__ = "2.1.3"
 __tocinfoline__ = "VORTEX core package"
 
 __all__ = [
@@ -111,11 +111,13 @@ footprints.setup.callback = vortexfpdefaults
 ticket = sessions.get
 sh = sessions.system
 
-# If a config file can be found in current dir, load it else load .vortex.d/vortex.toml
-if os.path.isfile("vortex.toml"):
-    config.load_config("vortex.toml")
+# If a config file can be found in current dir, load it else load
+# .vortex.d/vortex.toml
+confname = Path("vortex.toml")
+if confname.exists():
+    config.load_config(confname)
 else:
-    config.load_config(os.environ["HOME"] + "/.vortex.d/vortex.toml")
+    config.load_config(Path.home() / ".vortex.d" / confname)
 
 # Load some superstars sub-packages
 

{vortex_nwp-2.1.0 → vortex_nwp-2.1.2}/src/vortex/algo/components.py

@@ -1841,7 +1841,10 @@ class Parallel(xExecutableAlgoComponent):
     def _mpitool_attributes(self, opts):
         """Return the dictionary of attributes needed to create the mpitool object."""
         # Read the appropriate configuration in the target file
-        conf_dict = config.from_config(section="mpitool")
+        if not config.is_defined(section="mpitool"):
+            conf_dict = {}
+        else:
+            conf_dict = config.from_config(section="mpitool")
         if self.mpiname:
             conf_dict["mpiname"] = self.mpiname
         # Make "mpirun" the default mpi command name

{vortex_nwp-2.1.0 → vortex_nwp-2.1.2}/src/vortex/algo/mpitools.py

@@ -84,6 +84,7 @@ from vortex.tools import env
 from vortex.tools.arm import ArmForgeTool
 from vortex.tools.systems import ExecutionError
 from vortex.util import config
+from vortex.config import is_defined, ConfigurationError
 
 #: No automatic export
 __all__ = []
@@ -1431,12 +1432,15 @@ class SRun(MpiTool):
     @property
     def _actual_slurmversion(self):
         """Return the slurm major version number."""
-        slurmversion = self.slurmversion or from_config(
-            section="mpitool", key="slurmversion"
-        )
-        if not slurmversion:
-            raise ValueError("No slurm version specified")
-        return slurmversion
+        if self.slurmversion:
+            return self.slurmversion
+
+        if not is_defined(section="mpitool", key="slurmversion"):
+            raise ConfigurationError(
+                "Using 'srun' MPI tool but slurm version is not configured. See "
+                "https://vortex-nwp.readthedocs.io/en/latest/user-guide/configuration.html#mpitool"
+            )
+        return from_config(section="mpitool", key="slurmversion")
 
     def _set_binaries_hack(self, binaries):
         """Set the list of :class:`MpiBinaryDescription` objects associated with this instance."""

{vortex_nwp-2.1.0 → vortex_nwp-2.1.2}/src/vortex/config.py

@@ -3,6 +3,7 @@ the value of configuration options, respectively.
 
 """
 
+from pathlib import Path
 import tomli
 
 from bronx.fancies import loggers
@@ -24,7 +25,7 @@ class ConfigurationError(Exception):
     """Something is wrong with the provided configuration"""
 
 
-def load_config(configpath="vortex.toml"):
+def load_config(configpath=Path("vortex.toml")):
     """Load configuration from a TOML configuration file
 
     Existing configuration values are overriden. The configuration
@@ -41,19 +42,22 @@ def load_config(configpath="vortex.toml"):
        # ...
     """
     global VORTEX_CONFIG
+    configpath = Path(configpath)
     try:
-        with open(configpath, "rb") as f:
+        with configpath.open(mode="rb") as f:
             VORTEX_CONFIG = tomli.load(f)
-        print(f"Successfully read configuration file {configpath}")
+        print(f"Successfully read configuration file {configpath.absolute()}")
     except FileNotFoundError:
-        print(f"Could not read configuration file {configpath} (not found).")
+        print(
+            f"Could not read configuration file {configpath.absolute()} (not found)."
+        )
         print("Use load_config(/path/to/config) to update the configuration")
 
 
 def print_config():
     """Print configuration (key, value) pairs"""
     if VORTEX_CONFIG:
-        for k, v in VORTEX_CONFIG:
+        for k, v in VORTEX_CONFIG.items():
             print(k.upper(), v)
 
 
@@ -65,10 +69,10 @@ def from_config(section, key=None):
     """
     try:
         subconfig = VORTEX_CONFIG[section]
-    except KeyError:
+    except KeyError as e:
         raise ConfigurationError(
             f"Missing configuration section {section}",
-        )
+        ) from e
     if not key:
         return subconfig
 

{vortex_nwp-2.1.0 → vortex_nwp-2.1.2}/src/vortex/data/abstractstores.py

@@ -14,7 +14,7 @@ from bronx.system import hash as hashutils
 import footprints
 
 from vortex import sessions
-from vortex.config import from_config, ConfigurationError
+from vortex.config import from_config, ConfigurationError, is_defined
 from vortex.syntax.stdattrs import (
     hashalgo,
     hashalgo_avail_list,
@@ -870,6 +870,13 @@ class ArchiveStore(Store):
     @property
     def actual_storetube(self):
         """This archive network name (potentially read form the configuration file)."""
+        if self._actual_storetube:
+            return self._actual_storetube
+        if not is_defined(section="storage", key="protocol"):
+            raise ConfigurationError(
+                "Using remote data tree but protocol is not configured. See "
+                "https://vortex-nwp.readthedocs.io/en/latest/user-guide/configuration.html#storage"
+            )
         if self._actual_storetube is None:
             self._actual_storetube = from_config(
                 section="storage",

{vortex_nwp-2.1.0 → vortex_nwp-2.1.2}/src/vortex/data/handlers.py

@@ -134,7 +134,26 @@ class Handler:
         return str(self.__dict__)
 
     def _get_resource(self):
-        """Getter for ``resource`` property."""
+        """Getter for ``resource`` property.
+
+        **Example**
+
+        >>> rh = vortex.input(
+        ...     vapp="arpege",
+        ...     vconf="4dvarfr",
+        ...     cutoff="production",
+        ...     date="202506160000",
+        ...     term=1,
+        ...     geometry="global1798",
+        ...     model="arpege",
+        ...     block="forecast",
+        ...     kind="modelstate",
+        ...     experiment="oper",
+        ...     local="myfile",
+        ... )
+        >>> print(rh.resource)
+        <vortex.nwp.data.modelstates.Historic object at 0x7b430874a620 | model='arpege' date='2025-06-16T00:00:00Z' cutoff='production' geometry='<vortex.data.geometries.GaussGeometry | tag='global1798' id='ARPEGE TL1798c2.2 stretched-rotated geometry' tl=1798 c=2.2>' term='01:00' subset='None'>
+        """
         return self._resource
 
     def _set_resource(self, value):
@@ -534,7 +553,29 @@ class Handler:
         return rst
 
     def locate(self, **extras):
-        """Try to figure out what would be the physical location of the resource."""
+        r"""
+        Try to figure out what would be the physical location of the resource.
+
+        :returns: A semiclon separated string listing the various locations where the resource can be found.
+
+        >>> rh = vortex.input(
+        ...     vapp="arpege",
+        ...     vconf="4dvarfr",
+        ...     cutoff="production",
+        ...     date="202506160000",
+        ...     term=1,
+        ...     geometry="global1798",
+        ...     model="arpege",
+        ...     block="forecast",
+        ...     kind="modelstate",
+        ...     experiment="oper",
+        ...     local="myfile",
+        ... )
+        >>> print("\n".join(rh.locate().split(";")))
+        /home/user/.vortex.d/arpege/4dvarfr/OPER/20250616T0000P/forecast/historic.arpege.tl1798-c22+0001:00.fa
+        user@archive:/data/archive/arpege/4dvarfr/OPER/2025/06/16/T0000P/forecast/historic.arpege.tl1798-c22+0001:00.fa
+
+        """
         rst = None
         if self.resource and self.provider:
             store = self.store
@@ -825,6 +866,32 @@ class Handler:
         * When **insitu** is False, an attempt to get the resource is systematically
           made except if **alternate** is defined and the local container already
           exists.
+
+        **Example**
+
+        .. code:: python
+
+           rhandlers = vortex.input(
+               kind='gridpoint',
+               term=1,
+               geometry='eurw1s40',
+               nativefmt='grib',
+               model='arome',
+               cutoff='production',
+               date=['2024060121', '2024060122'],
+               origin='historic',
+               vapp='arome',
+               vconf='pefrance',
+               member=[1,2,5],
+               experiment='myexp',
+               block='forecast',
+               local='gribfile_[member].grib',
+               format='grib',
+           )
+
+           for rh in rhandlers:
+               rh.get()
+
         """
         return self._get_proxy(self._actual_get, alternate=alternate, **extras)
 
@@ -1010,6 +1077,29 @@ class Handler:
 
         Conversely, the low-level stores are made aware of the previous successful
         put. That way, a local container is not put twice to the same destination.
+
+        .. code:: python
+
+           rhandlers = vortex.output(
+               kind='gridpoint',
+               term=1,
+               geometry='eurw1s40',
+               nativefmt='grib',
+               model='arome',
+               cutoff='production',
+               date=['2024060121', '2024060122'],
+               origin='historic',
+               vapp='arome',
+               vconf='pefrance',
+               member=[1,2,5],
+               experiment='myexp',
+               block='forecast',
+               local='gribfile_[member].grib',
+               format='grib',
+           )
+
+           for rh in rhandlers:
+               rh.put()
         """
         rst = False
         if self.complete:

{vortex_nwp-2.1.0 → vortex_nwp-2.1.2}/src/vortex/data/providers.py

@@ -279,6 +279,7 @@ class Vortex(Provider):
 
     _DEFAULT_NAME_BUILDER = names.VortexNameBuilder()
     _CUSTOM_NAME_BUILDERS = dict()
+    _SPECIAL_EXPS = ("oper", "dble", "test", "mirr")
 
     _footprint = [
         block,
@@ -365,8 +366,8 @@ class Vortex(Provider):
             self._namebuilder = self._CUSTOM_NAME_BUILDERS[self.namebuild]
         else:
             self._namebuilder = self._DEFAULT_NAME_BUILDER
-        if self.experiment in ("oper", "dble"):
-            self.experiment = self.experiment.upper()
+        if self.experiment in (n.upper() for n in self._SPECIAL_EXPS):
+            self.experiment = self.experiment.lower()
 
         # Ensure compatibility with deprecated namespace attribute
         # Under the hood the namespace attribute is still used to
@@ -426,7 +427,7 @@ class Vortex(Provider):
 
     def netloc(self, resource):
         """Returns the current ``namespace``."""
-        if self.experiment in ("OPER", "DBLE"):
+        if self.experiment in self._SPECIAL_EXPS:
             return "vsop." + self.namespace.domain
         return self.namespace.netloc
 

{vortex_nwp-2.1.0 → vortex_nwp-2.1.2}/src/vortex/data/stores.py

@@ -748,13 +748,12 @@ class VortexStdBaseArchiveStore(_VortexBaseArchiveStore):
             )
         except config.ConfigurationError as e:
             msg = (
-                "Trying to write to archive but location is not configured.\n"
+                "Trying to write to archive but location is not configured. "
                 'Make sure key "rootdir" is defined in storage section of '
                 "the configuration.\n"
                 "See https://vortex-nwp.readthedocs.io/en/latest/user-guide/configuration.html#storage"
             )
-            logger.error(msg)
-            raise e
+            raise config.ConfigurationError(msg) from e
         return remote
 
     remap_write = remap_read
@@ -813,12 +812,11 @@ class VortexOpBaseArchiveStore(_VortexBaseArchiveStore):
         except config.ConfigurationError as e:
             msg = (
                 "Trying to write to operational data archive but location"
-                'is not configured.\nMake sure key "rootdir" is defined in '
+                ' is not configured. Make sure key "op_rootdir" is defined in '
                 "the storage section of the configuration.\n"
                 "See https://vortex-nwp.readthedocs.io/en/latest/user-guide/configuration.html#storage"
             )
-            logger.error(msg)
-            raise e
+            raise config.ConfigurationError(msg) from e
         xpath = remote["path"].split("/")
         if len(xpath) >= 5 and re.match(r"^\d{8}T\d{2,4}", xpath[4]):
             # If a date is detected
@@ -910,16 +908,9 @@ class VortexArchiveStore(MultiStore):
 
     def alternates_netloc(self):
         """Return netlocs describing both base and stacked archives."""
-        netloc_m = re.match(
-            r"(?P<base>v.*)\.archive\.(?P<country>\w+)", self.netloc
-        )
         return [
-            "{base:s}.archive-legacy.{country:s}".format(
-                **netloc_m.groupdict()
-            ),
-            "{base:s}.stacked-archive-legacy.{country:s}".format(
-                **netloc_m.groupdict()
-            ),
+            f"{self.netloc.firstname}.archive-legacy.fr",
+            f"{self.netloc.firstname}.stacked-archive-legacy.fr",
         ]
 
     def alternates_fpextras(self):
@@ -1027,18 +1018,16 @@ class VortexCacheOp2ResearchStore(_VortexCacheBaseStore):
 
     def __init__(self, *args, **kw):
         super().__init__(*args, **kw)
-        try:
-            cachepath = config.from_config(
-                section="data-tree",
-                key="op_rootdir",
-            )
-        except config.ConfigurationError as e:
-            logger.error(
-                "Cannot use special experiment cache without providing",
-                "cache location",
+        if not config.is_defined(section="data-tree", key="op_rootdir"):
+            raise config.ConfigurationError(
+                "Using special experiment but corresponding cache location "
+                'is not configured. Bet sure to set "op_rootdir" in configuration. '
+                "See https://vortex-nwp.readthedocs.io/en/latest/user-guide/oper-dble-data-trees"
             )
-            raise e
-
+        cachepath = config.from_config(
+            section="data-tree",
+            key="op_rootdir",
+        )
         self.location = os.path.join(cachepath, "vortex")
 
 
@@ -1343,6 +1332,10 @@ class PromiseCacheStore(VortexCacheMtStore):
         ),
     )
 
+    def __init__(self, *args, **kw):
+        super().__init__(*args, **kw)
+        self.location = os.path.join(get_cache_location(), "promise")
+
     @staticmethod
     def _add_default_options(options):
         options_upd = options.copy()

{vortex_nwp-2.1.0 → vortex_nwp-2.1.2}/src/vortex/nwp/data/stores.py

@@ -12,6 +12,7 @@ from bronx.fancies import loggers
 from bronx.stdtypes import date
 from vortex.data.abstractstores import Store
 from vortex.syntax.stdattrs import compressionpipeline
+from vortex import config
 
 #: No automatic export
 __all__ = []
@@ -109,14 +110,17 @@ class BdpeStore(Store):
         if s_archive == "True":
             extraenv["BDPE_LECTURE_ARCHIVE_AUTORISEE"] = "oui"
 
-        wsinterpreter = self.system.default_target.get(
-            "bdpe:wsclient_interpreter", None
-        )
-        wscommand = self.system.default_target.get("bdpe:wsclient_path", None)
-        if wscommand is None:
-            raise RuntimeError(
-                "bdpe:wsclient_path has to be set in the target config"
+        try:
+            wsinterpreter = config.from_config(
+                section="bdpe", key="wsclient_interpreter"
             )
+        except config.ConfigurationError:
+            wsinterpreter = "bash"
+
+        try:
+            wscommand = config.from_config(section="bdpe", key="wsclient_path")
+        except config.ConfigurationError:
+            wscommand = "/opt/softs/sopra/bin/lirepe.sh"
 
         args.insert(0, wscommand)
         if wsinterpreter is not None:

{vortex_nwp-2.1.0 → vortex_nwp-2.1.2}/src/vortex/toolbox.py

@@ -485,15 +485,39 @@ def add_section(section, args, kw):
 
 # noinspection PyShadowingBuiltins
 def input(*args, **kw):  # @ReservedAssignment
-    """Add input :class:`~vortex.layout.dataflow.Section` objects to the current sequence.
+    r"""Declare one or more input resources.
+
+    This function takes an abitrary of keyword arguments forming the resource
+    description.
+
+    :return: A list of :py:class:`Handler <vortex.data.handlers.Handler>` objects.
+
+    **Example:**
+
+    The following call to ``input`` returns a list of 6
+    :py:class:`Handler <vortex.data.handlers.Handler>` objects, one
+    for each date and member:
+
+    .. code:: python
+
+       rhandlers = vortex.input(
+           kind='gridpoint',
+           term=1,
+           geometry='eurw1s40',
+           nativefmt='grib',
+           model='arome',
+           cutoff='production',
+           date=['2024060121', '2024060122'],
+           origin='historic',
+           vapp='arome',
+           vconf='pefrance',
+           member=[1,2,5],
+           experiment='myexp',
+           block='forecast',
+           local='gribfile_[member].grib',
+           format='grib',
+       )
 
-    Relies on the :func:`add_section` function (see its documentation), with:
-
-        * It's ``section`` attribute is automatically set to 'input';
-        * The ``kw``'s *insitu* item is set to :data:`active_insitu` by default.
-
-    :return: A list of :class:`vortex.data.handlers.Handler` objects (associated
-        with the newly created class:`~vortex.layout.dataflow.Section` objects).
     """
     kw.setdefault("insitu", active_insitu)
     kw.setdefault("batch", active_batchinputs)
@@ -532,14 +556,39 @@ def show_inputs(context=None):
 
 
 def output(*args, **kw):
-    """Add output :class:`~vortex.layout.dataflow.Section` objects to the current sequence.
-
-    Relies on the :func:`add_section` function (see its documentation), with:
-
-        * It's ``section`` attribute is automatically set to 'output';
+    r"""Declare one or more output resources.
+
+    This function takes an abitrary of keyword arguments forming the resource
+    description.
+
+    :return: A list of :py:class:`Handler <vortex.data.handlers.Handler>` objects.
+
+    **Example:**
+
+    The following call to ``output`` returns a list of 6
+    :py:class:`Handler <vortex.data.handlers.Handler>` objects, one
+    for each date and member:
+
+    .. code:: python
+
+       rhandlers = vortex.output(
+           kind='gridpoint',
+           term=1,
+           geometry='eurw1s40',
+           nativefmt='grib',
+           model='arome',
+           cutoff='production',
+           date=['2024060121', '2024060122'],
+           origin='historic',
+           vapp='arome',
+           vconf='pefrance',
+           member=[1,2,5],
+           experiment='myexp',
+           block='forecast',
+           local='gribfile_[member].grib',
+           format='grib',
+       )
 
-    :return: A list of :class:`vortex.data.handlers.Handler` objects (associated
-        with the newly created class:`~vortex.layout.dataflow.Section` objects).
     """
     # Strip the metadatacheck option depending on active_metadatacheck
     if not active_promise:
@@ -607,15 +656,26 @@ def promise(*args, **kw):
 
 
 def executable(*args, **kw):
-    """Add executable :class:`~vortex.layout.dataflow.Section` objects to the current sequence.
+    r"""Declare one or more executable resources.
 
-    Relies on the :func:`add_section` function (see its documentation), with:
+    This function takes an abitrary of keyword arguments forming the
+    executable resource description.
 
-        * It's ``section`` attribute is automatically set to 'executable';
-        * The ``kw``'s *insitu* item is set to :data:`active_insitu` by default.
+    :return: A list of :py:class:`Handler <vortex.data.handlers.Handler>` objects.
+
+    **Example:**
+
+    The following call to ``input`` returns a list of one
+    :py:class:`Handler <vortex.data.handlers.Handler>` object:
+
+    .. code:: python
+
+       rhandlers = vortex.executable(
+           kind="mfmodel",
+           local="ARPEGE",
+           remote="/path/to/binaries/ARPEGE.EX",
+       )
 
-    :return: A list of :class:`vortex.data.handlers.Handler` objects (associated
-        with the newly created class:`~vortex.layout.dataflow.Section` objects).
     """
     kw.setdefault("insitu", active_insitu)
     return add_section("executable", args, kw)

{vortex_nwp-2.1.0 → vortex_nwp-2.1.2}/src/vortex/tools/grib.py

@@ -567,7 +567,7 @@ class EcGribDecoMixin(AlgoComponentDecoMixin):
     def _eccodes_envsetup(
         self,
         eccodes_lib,
-        envvar="ECCODES_DEFINITIONS_PATH",
+        envvar="ECCODES_DEFINITION_PATH",
         tgt_path="definitions",
     ):
         """Export envirionment variables required by ECCODES
@@ -612,7 +612,7 @@ class EcGribDecoMixin(AlgoComponentDecoMixin):
         if eccodes_lib is not None:
             defvar = self._eccodes_envsetup(
                 eccodes_lib,
-                envvar="ECCODES_DEFINITIONS_PATH",
+                envvar="ECCODES_DEFINITION_PATH",
                 tgt_path="definitions",
             )
             subdir = Path("ifs_samples") / (