vortex-nwp 2.0.0b2__tar.gz → 2.1.1__tar.gz

{vortex_nwp-2.0.0b2 → vortex_nwp-2.1.1}/PKG-INFO

@@ -1,6 +1,6 @@
-Metadata-Version: 2.2
+Metadata-Version: 2.4
 Name: vortex-nwp
-Version: 2.0.0b2
+Version: 2.1.1
 Summary: A Python library to write Numerical Weather Prediction pipelines components
 Author-email: The Vortex Team <vortex.support@meteo.fr>
 License: CECILL-C
@@ -21,6 +21,7 @@ Requires-Dist: sphinx-copybutton; extra == "docs"
 Provides-Extra: dev
 Requires-Dist: ruff==0.9.1; extra == "dev"
 Requires-Dist: pytest; extra == "dev"
+Dynamic: license-file
 
 ## vortex
 

{vortex_nwp-2.0.0b2 → vortex_nwp-2.1.1}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "vortex-nwp"
-version = "2.0.0b2"
+version = "2.1.1"
 description = "A Python library to write Numerical Weather Prediction pipelines components"
 readme = "README.md"
 requires-python = ">=3.7"

{vortex_nwp-2.0.0b2 → vortex_nwp-2.1.1}/src/vortex/__init__.py

@@ -21,7 +21,18 @@ strongly advised.
 """
 
 import atexit
-import importlib.metadata
+from pathlib import Path
+import sys
+
+# importlib.metadata included in stdlib from 3.8 onwards.
+# For older versions, import third-party importlib_metadata
+if sys.version_info < (3, 8):
+    import importlib_metadata
+    import importlib
+
+    importlib.metadata = importlib_metadata
+else:
+    import importlib.metadata
 
 from bronx.fancies import loggers as bloggers
 import bronx.stdtypes.date
@@ -42,10 +53,10 @@ from .toolbox import algo as task
 
 from . import nwp as nwp  # footprints import
 
-__version__ = "2.0.0b2"
+__version__ = "2.1.1"
 __prompt__ = "Vortex v-" + __version__ + ":"
 
-__nextversion__ = "2.0.0b3"
+__nextversion__ = "2.1.1"
 __tocinfoline__ = "VORTEX core package"
 
 __all__ = [
@@ -100,8 +111,13 @@ footprints.setup.callback = vortexfpdefaults
 ticket = sessions.get
 sh = sessions.system
 
-# If a config file can be found in current dir, load it
-config.load_config()
+# If a config file can be found in current dir, load it else load
+# .vortex.d/vortex.toml
+confname = Path("vortex.toml")
+if confname.exists():
+    config.load_config(confname)
+else:
+    config.load_config(Path.home() / ".vortex.d" / confname)
 
 # Load some superstars sub-packages
 

{vortex_nwp-2.0.0b2 → vortex_nwp-2.1.1}/src/vortex/algo/components.py

@@ -50,7 +50,7 @@ from bronx.syntax.decorators import nicedeco
 import footprints
 from taylorism import Boss
 import vortex
-from vortex.config import from_config
+import vortex.config as config
 from vortex.algo import mpitools
 from vortex.syntax.stdattrs import DelayedEnvValue
 from vortex.tools.parallelism import ParallelResultParser
@@ -704,15 +704,15 @@ class AlgoComponent(footprints.FootprintBase, metaclass=AlgoComponentMeta):
 
     def export(self, packenv):
         """Export environment variables in given pack."""
-        for k, v in from_config(section=packenv).items():
+        for k, v in config.from_config(section=packenv).items():
             if k not in self.env:
                 logger.info("Setting %s env %s = %s", packenv.upper(), k, v)
                 self.env[k] = v
 
     def prepare(self, rh, opts):
         """Set some defaults env values."""
-        if opts.get("fortran", True):
-            self.export("fortran")
+        if config.is_defined(section="env"):
+            self.export("env")
 
     def absexcutable(self, xfile):
         """Retuns the absolute pathname of the ``xfile`` executable."""
@@ -1841,7 +1841,7 @@ class Parallel(xExecutableAlgoComponent):
     def _mpitool_attributes(self, opts):
         """Return the dictionary of attributes needed to create the mpitool object."""
         # Read the appropriate configuration in the target file
-        conf_dict = from_config(section="mpitool")
+        conf_dict = config.from_config(section="mpitool")
         if self.mpiname:
             conf_dict["mpiname"] = self.mpiname
         # Make "mpirun" the default mpi command name
@@ -1850,7 +1850,7 @@ class Parallel(xExecutableAlgoComponent):
         possible_attrs = functools.reduce(
             lambda s, t: s | t,
             [
-                set(cls._footprint.attr.keys())
+                set(cls.footprint_retrieve().attr.keys())
                 for cls in footprints.proxy.mpitools
             ],
         )

{vortex_nwp-2.0.0b2 → vortex_nwp-2.1.1}/src/vortex/algo/mpitools.py

@@ -182,8 +182,8 @@ class MpiTool(footprints.FootprintBase):
     )
 
     _envelope_bit_kind = "basicenvelopebit"
-    _envelope_wrapper_tpl = "@envelope_wrapper_default.tpl"
-    _wrapstd_wrapper_tpl = "@wrapstd_wrapper_default.tpl"
+    _envelope_wrapper_tpl = "envelope_wrapper_default.tpl"
+    _wrapstd_wrapper_tpl = "wrapstd_wrapper_default.tpl"
     _envelope_wrapper_name = "./global_envelope_wrapper.py"
     _wrapstd_wrapper_name = "./global_wrapstd_wrapper.py"
     _envelope_rank_var = "MPIRANK"
@@ -700,11 +700,11 @@ class MpiTool(footprints.FootprintBase):
         if not self.mpiwrapstd:
             return None
         # Create the launchwrapper
-        with importlib.resources.as_file(
-            importlib.resources.files("vortex.algo")
-        ) as path:
-            tplpath = path / "mpitools_templates" / self._wrapstd_wrapper_tpl
-        wtpl = config.load_template(tplpath, encoding="utf-8")
+        with importlib.resources.path(
+            "vortex.algo.mpitools_templates",
+            self._wrapstd_wrapper_tpl,
+        ) as tplpath:
+            wtpl = config.load_template(tplpath, encoding="utf-8")
         with open(self._wrapstd_wrapper_name, "w", encoding="utf-8") as fhw:
             fhw.write(
                 wtpl.substitute(
@@ -911,11 +911,11 @@ class MpiTool(footprints.FootprintBase):
             "Here are the envelope details:\n%s", "\n".join(binding_str)
         )
         # Create the launchwrapper
-        with importlib.resources.as_file(
-            importlib.resources.files("vortex.algo")
-        ) as path:
-            tplpath = path / "mpitools_templates" / self._envelope_wrapper_tpl
-        wtpl = config.load_template(tplpath, encoding="utf-8")
+        with importlib.resources.path(
+            "vortex.algo.mpitools_templates",
+            self._envelope_wrapper_tpl,
+        ) as tplpath:
+            wtpl = config.load_template(tplpath, encoding="utf-8")
         with open(self._envelope_wrapper_name, "w", encoding="utf-8") as fhw:
             fhw.write(
                 wtpl.substitute(

vortex_nwp-2.1.1/src/vortex/algo/mpitools_templates/__init__.py

@@ -0,0 +1 @@
+# Empty init file

{vortex_nwp-2.0.0b2 → vortex_nwp-2.1.1}/src/vortex/config.py

@@ -3,6 +3,7 @@ the value of configuration options, respectively.
 
 """
 
+from pathlib import Path
 import tomli
 
 from bronx.fancies import loggers
@@ -24,7 +25,7 @@ class ConfigurationError(Exception):
     """Something is wrong with the provided configuration"""
 
 
-def load_config(configpath="vortex.toml"):
+def load_config(configpath=Path("vortex.toml")):
     """Load configuration from a TOML configuration file
 
     Existing configuration values are overriden. The configuration
@@ -41,19 +42,22 @@ def load_config(configpath="vortex.toml"):
        # ...
     """
     global VORTEX_CONFIG
+    configpath = Path(configpath)
     try:
-        with open(configpath, "rb") as f:
+        with configpath.open(mode="rb") as f:
             VORTEX_CONFIG = tomli.load(f)
-        print(f"Successfully read configuration file {configpath}")
+        print(f"Successfully read configuration file {configpath.absolute()}")
     except FileNotFoundError:
-        print(f"Could not read configuration file {configpath} (not found).")
+        print(
+            f"Could not read configuration file {configpath.absolute()} (not found)."
+        )
         print("Use load_config(/path/to/config) to update the configuration")
 
 
 def print_config():
     """Print configuration (key, value) pairs"""
     if VORTEX_CONFIG:
-        for k, v in VORTEX_CONFIG:
+        for k, v in VORTEX_CONFIG.items():
             print(k.upper(), v)
 
 
@@ -65,10 +69,10 @@ def from_config(section, key=None):
     """
     try:
         subconfig = VORTEX_CONFIG[section]
-    except KeyError:
+    except KeyError as e:
         raise ConfigurationError(
             f"Missing configuration section {section}",
-        )
+        ) from e
     if not key:
         return subconfig
 

{vortex_nwp-2.0.0b2 → vortex_nwp-2.1.1}/src/vortex/data/handlers.py

@@ -5,6 +5,7 @@ the various caches or archives".
 """
 
 import functools
+import importlib
 import re
 import sys
 
@@ -133,7 +134,26 @@ class Handler:
         return str(self.__dict__)
 
     def _get_resource(self):
-        """Getter for ``resource`` property."""
+        """Getter for ``resource`` property.
+
+        **Example**
+
+        >>> rh = vortex.input(
+        ...     vapp="arpege",
+        ...     vconf="4dvarfr",
+        ...     cutoff="production",
+        ...     date="202506160000",
+        ...     term=1,
+        ...     geometry="global1798",
+        ...     model="arpege",
+        ...     block="forecast",
+        ...     kind="modelstate",
+        ...     experiment="oper",
+        ...     local="myfile",
+        ... )
+        >>> print(rh.resource)
+        <vortex.nwp.data.modelstates.Historic object at 0x7b430874a620 | model='arpege' date='2025-06-16T00:00:00Z' cutoff='production' geometry='<vortex.data.geometries.GaussGeometry | tag='global1798' id='ARPEGE TL1798c2.2 stretched-rotated geometry' tl=1798 c=2.2>' term='01:00' subset='None'>
+        """
         return self._resource
 
     def _set_resource(self, value):
@@ -533,7 +553,29 @@ class Handler:
         return rst
 
     def locate(self, **extras):
-        """Try to figure out what would be the physical location of the resource."""
+        r"""
+        Try to figure out what would be the physical location of the resource.
+
+        :returns: A semiclon separated string listing the various locations where the resource can be found.
+
+        >>> rh = vortex.input(
+        ...     vapp="arpege",
+        ...     vconf="4dvarfr",
+        ...     cutoff="production",
+        ...     date="202506160000",
+        ...     term=1,
+        ...     geometry="global1798",
+        ...     model="arpege",
+        ...     block="forecast",
+        ...     kind="modelstate",
+        ...     experiment="oper",
+        ...     local="myfile",
+        ... )
+        >>> print("\n".join(rh.locate().split(";")))
+        /home/user/.vortex.d/arpege/4dvarfr/OPER/20250616T0000P/forecast/historic.arpege.tl1798-c22+0001:00.fa
+        user@archive:/data/archive/arpege/4dvarfr/OPER/2025/06/16/T0000P/forecast/historic.arpege.tl1798-c22+0001:00.fa
+
+        """
         rst = None
         if self.resource and self.provider:
             store = self.store
@@ -824,6 +866,32 @@ class Handler:
         * When **insitu** is False, an attempt to get the resource is systematically
           made except if **alternate** is defined and the local container already
           exists.
+
+        **Example**
+
+        .. code:: python
+
+           rhandlers = vortex.input(
+               kind='gridpoint',
+               term=1,
+               geometry='eurw1s40',
+               nativefmt='grib',
+               model='arome',
+               cutoff='production',
+               date=['2024060121', '2024060122'],
+               origin='historic',
+               vapp='arome',
+               vconf='pefrance',
+               member=[1,2,5],
+               experiment='myexp',
+               block='forecast',
+               local='gribfile_[member].grib',
+               format='grib',
+           )
+
+           for rh in rhandlers:
+               rh.get()
+
         """
         return self._get_proxy(self._actual_get, alternate=alternate, **extras)
 
@@ -1009,6 +1077,29 @@ class Handler:
 
         Conversely, the low-level stores are made aware of the previous successful
         put. That way, a local container is not put twice to the same destination.
+
+        .. code:: python
+
+           rhandlers = vortex.output(
+               kind='gridpoint',
+               term=1,
+               geometry='eurw1s40',
+               nativefmt='grib',
+               model='arome',
+               cutoff='production',
+               date=['2024060121', '2024060122'],
+               origin='historic',
+               vapp='arome',
+               vconf='pefrance',
+               member=[1,2,5],
+               experiment='myexp',
+               block='forecast',
+               local='gribfile_[member].grib',
+               format='grib',
+           )
+
+           for rh in rhandlers:
+               rh.put()
         """
         rst = False
         if self.complete:
@@ -1116,18 +1207,23 @@ class Handler:
         self,
         pr_getter=None,
         tplfile=None,
-        tplskip="@sync-skip.tpl",
-        tplfetch="@sync-fetch.tpl",
         py_exec=sys.executable,
         py_opts="",
     ):
         """Build a getter for the expected resource."""
         if tplfile is None:
-            tplfile = tplfetch if self.is_expected() else tplskip
+            tplfile = (
+                "sync-" + ("fetch" if self.is_expected() else "skip") + ".tpl"
+            )
+        with importlib.resources.path(
+            "vortex.data.sync_templates",
+            tplfile,
+        ) as tplpath:
+            tpl = config.load_template(tplpath)
         if pr_getter is None:
             pr_getter = self.container.localpath() + ".getpr"
         t = self._cur_session
-        tpl = config.load_template(t, tplfile)
+
         with open(pr_getter, "w", encoding="utf-8") as fd:
             fd.write(
                 tpl.substitute(

{vortex_nwp-2.0.0b2 → vortex_nwp-2.1.1}/src/vortex/data/stores.py

@@ -527,6 +527,10 @@ class _VortexBaseArchiveStore(ArchiveStore, _VortexStackedStorageMixin):
         """Remap actual remote path to distant store path for intrusive actions."""
         raise NotImplementedError
 
+    def remap_write(self, remote, options):
+        """Remap actual remote path to distant store path for intrusive actions."""
+        raise NotImplementedError
+
     def remap_list(self, remote, options):
         """Reformulates the remote path to compatible vortex namespace."""
         if len(remote["path"].split("/")) >= 4:
@@ -537,8 +541,6 @@ class _VortexBaseArchiveStore(ArchiveStore, _VortexStackedStorageMixin):
             )
             return None
 
-    remap_write = remap_read
-
     @property
     def stacks_autorefill(self):
         """Where to refill a stack retrieved from the archive."""
@@ -746,15 +748,16 @@ class VortexStdBaseArchiveStore(_VortexBaseArchiveStore):
             )
         except config.ConfigurationError as e:
             msg = (
-                "Trying to write to archive but location is not configured.\n"
+                "Trying to write to archive but location is not configured. "
                 'Make sure key "rootdir" is defined in storage section of '
                 "the configuration.\n"
                 "See https://vortex-nwp.readthedocs.io/en/latest/user-guide/configuration.html#storage"
             )
-            logger.error(msg)
-            raise e
+            raise config.ConfigurationError(msg) from e
         return remote
 
+    remap_write = remap_read
+
 
 class VortexStdStackedArchiveStore(VortexStdBaseArchiveStore):
     """Archive for casual VORTEX experiments: Support for legacy/Olive XPIDs.
@@ -809,12 +812,11 @@ class VortexOpBaseArchiveStore(_VortexBaseArchiveStore):
         except config.ConfigurationError as e:
             msg = (
                 "Trying to write to operational data archive but location"
-                'is not configured.\nMake sure key "rootdir" is defined in '
+                ' is not configured. Make sure key "op_rootdir" is defined in '
                 "the storage section of the configuration.\n"
                 "See https://vortex-nwp.readthedocs.io/en/latest/user-guide/configuration.html#storage"
             )
-            logger.error(msg)
-            raise e
+            raise config.ConfigurationError(msg) from e
         xpath = remote["path"].split("/")
         if len(xpath) >= 5 and re.match(r"^\d{8}T\d{2,4}", xpath[4]):
             # If a date is detected
@@ -1105,16 +1107,10 @@ class VortexCacheStore(_AbstractVortexCacheMultiStore):
 
     def alternates_netloc(self):
         """For Non-Op users, Op caches may be accessed in read-only mode."""
-        netloc_m = re.match(
-            r"(?P<base>vortex.*)\.cache\.(?P<country>\w+)", self.netloc
-        )
-        mt_netloc = "{base:s}.cache-mt.{country:s}".format(
-            **netloc_m.groupdict()
-        )
-        s_mt_netloc = "{base:s}.stacked-cache-mt.{country:s}".format(
-            **netloc_m.groupdict()
-        )
-        return [mt_netloc, s_mt_netloc]
+        return [
+            f"{self.netloc.firstname}.cache-mt.fr",
+            f"{self.netloc.firstname}.stacked-cache-mt.fr",
+        ]
 
 
 class VortexVsopCacheStore(_AbstractVortexCacheMultiStore):
@@ -1219,13 +1215,7 @@ class VortexStackStore(_AbstractVortexStackMultiStore):
 
     def alternates_netloc(self):
         """Go through the various stacked stores."""
-        netloc_m = re.match(
-            r"(?P<base>vortex.*)\.stack\.(?P<country>\w+)", self.netloc
-        )
-        s_mt_netloc = "{base:s}.stacked-cache-mt.{country:s}".format(
-            **netloc_m.groupdict()
-        )
-        return [s_mt_netloc]
+        return [f"{self.netloc.firstname}.stacked-cache-mt.fr"]
 
 
 class VortexVsopStackStore(_AbstractVortexStackMultiStore):

{vortex_nwp-2.0.0b2 → vortex_nwp-2.1.1}/src/vortex/nwp/algo/forecasts.py

@@ -283,21 +283,12 @@ class LAMForecast(Forecast):
                 values=["lamfc", "lamforecast"],
                 remap=dict(lamforecast="lamfc"),
             ),
-            synctool=dict(
-                info="The name of the script called when waiting for coupling files",
-                optional=True,
-                default="atcp.alad",
-                doc_visibility=footprints.doc.visibility.ADVANCED,
-            ),
-            synctpl=dict(
-                info="The template used to generate the *synctool* script",
-                optional=True,
-                default="@sync-fetch.tpl",
-                doc_visibility=footprints.doc.visibility.ADVANCED,
-            ),
         ),
     )
 
+    synctool = "atcp.alad"
+    synctpl = "sync-fetch.tpl"
+
     def spawn_command_options(self):
         """Dictionary provided for command line factory."""
         return dict(
@@ -341,10 +332,11 @@ class LAMForecast(Forecast):
             )
             sh.softlink(thisbound, lbcnc(number=i))
             if self.mksync:
-                thistool = self.synctool + ".{:03d}".format(i)
-                bound.mkgetpr(pr_getter=thistool, tplfetch=self.synctpl)
+                bound.mkgetpr(
+                    pr_getter=self.synctool + ".{:03d}".format(i),
+                )
                 if firstsync is None:
-                    firstsync = thistool
+                    firstsync = self.synctool + ".{:03d}".format(i)
 
         # Set up the first synchronization step
         if firstsync is not None:

{vortex_nwp-2.0.0b2 → vortex_nwp-2.1.1}/src/vortex/nwp/algo/ifsroot.py

@@ -120,6 +120,11 @@ class IFSParallel(
         """Extend default tag with ``kind`` value."""
         return super().fstag() + "." + self.kind
 
+    def _mpitool_attributes(self, opts):
+        conf_dict = super()._mpitool_attributes(opts)
+        conf_dict.update({"mplbased": True})
+        return conf_dict
+
     def valid_executable(self, rh):
         """Be sure that the specifed executable is ifsmodel compatible."""
         valid = super().valid_executable(rh)

{vortex_nwp-2.0.0b2 → vortex_nwp-2.1.1}/src/vortex/nwp/algo/monitoring.py

@@ -57,6 +57,11 @@ class OdbMonitoring(
         )
     )
 
+    def _mpitool_attributes(self, opts):
+        conf_dict = super()._mpitool_attributes(opts)
+        conf_dict.update({"mplbased": True})
+        return conf_dict
+
     def _fix_nam_macro(self, rh, macro, value):
         """Set a given namelist macro and issue a log message."""
         rh.contents.setmacro(macro, value)

{vortex_nwp-2.0.0b2 → vortex_nwp-2.1.1}/src/vortex/nwp/algo/mpitools.py

@@ -64,18 +64,28 @@ class MpiAuto(mpitools.MpiTool):
             ),
             bindingmethod=dict(
                 info="How to bind the MPI processes",
-                values=["arch", "launcherspecific", "vortex"],
+                values=["vortex", "arch", "launcherspecific"],
                 optional=True,
                 doc_visibility=footprints.doc.visibility.ADVANCED,
                 doc_zorder=-90,
             ),
+            mplbased=dict(
+                info="Is the executable based on MPL?",
+                type=bool,
+                optional=True,
+                default=False,
+            ),
         )
     )
 
-    _envelope_wrapper_tpl = "@envelope_wrapper_mpiauto.tpl"
+    _envelope_wrapper_tpl = "envelope_wrapper_mpiauto.tpl"
     _envelope_rank_var = "MPIAUTORANK"
     _needs_mpilib_specific_mpienv = False
 
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        self.bindingmethod = "arch" if self.mplbased else "vortex"
+
     def _reshaped_mpiopts(self):
         """Raw list of mpi tool command line options."""
         options = super()._reshaped_mpiopts()

{vortex_nwp-2.0.0b2 → vortex_nwp-2.1.1}/src/vortex/nwp/algo/odbtools.py

@@ -850,6 +850,11 @@ class OdbAverage(Parallel, odb.OdbComponentDecoMixin, drhook.DrHookDecoMixin):
         )
     )
 
+    def _mpitool_attributes(self, opts):
+        conf_dict = super()._mpitool_attributes(opts)
+        conf_dict.update({"mplbased": True})
+        return conf_dict
+
     def prepare(self, rh, opts):
         """Find any ODB candidate in input files."""
 
@@ -962,6 +967,11 @@ class OdbCompress(Parallel, odb.OdbComponentDecoMixin, drhook.DrHookDecoMixin):
         )
     )
 
+    def _mpitool_attributes(self, opts):
+        conf_dict = super()._mpitool_attributes(opts)
+        conf_dict.update({"mplbased": True})
+        return conf_dict
+
     def prepare(self, rh, opts):
         """Find any ODB candidate in input files and fox ODB env accordingly."""
 
@@ -1027,6 +1037,11 @@ class OdbMatchup(Parallel, odb.OdbComponentDecoMixin, drhook.DrHookDecoMixin):
         )
     )
 
+    def _mpitool_attributes(self, opts):
+        conf_dict = super()._mpitool_attributes(opts)
+        conf_dict.update({"mplbased": True})
+        return conf_dict
+
     def prepare(self, rh, opts):
         """Find ODB candidates in input files."""
 
@@ -1116,6 +1131,11 @@ class OdbReshuffle(
     _OUT_DIRECTORY = "reshuffled"
     _BARE_OUT_LAYOUT = "ccma"
 
+    def _mpitool_attributes(self, opts):
+        conf_dict = super()._mpitool_attributes(opts)
+        conf_dict.update({"mplbased": True})
+        return conf_dict
+
     def prepare(self, rh, opts):
         """Find ODB candidates in input files."""
 

{vortex_nwp-2.0.0b2 → vortex_nwp-2.1.1}/src/vortex/nwp/algo/oopsroot.py

@@ -619,6 +619,11 @@ class OOPSParallel(
             logger.error("The binary < %s > has no cycle attribute", repr(rh))
             return False
 
+    def _mpitool_attributes(self, opts):
+        conf_dict = super()._mpitool_attributes(opts)
+        conf_dict.update({"mplbased": True})
+        return conf_dict
+
     def prepare(self, rh, opts):
         """Preliminary setups."""
         super().prepare(rh, opts)