vortex-nwp 2.0.0b2__py3-none-any.whl → 2.1.0__py3-none-any.whl

vortex/__init__.py

@@ -21,7 +21,18 @@ strongly advised.
 """
 
 import atexit
-import importlib.metadata
+import sys
+import os
+
+# importlib.metadata included in stdlib from 3.8 onwards.
+# For older versions, import third-party importlib_metadata
+if sys.version_info < (3, 8):
+    import importlib_metadata
+    import importlib
+
+    importlib.metadata = importlib_metadata
+else:
+    import importlib.metadata
 
 from bronx.fancies import loggers as bloggers
 import bronx.stdtypes.date
@@ -42,10 +53,10 @@ from .toolbox import algo as task
 
 from . import nwp as nwp  # footprints import
 
-__version__ = "2.0.0b2"
+__version__ = "2.1.0"
 __prompt__ = "Vortex v-" + __version__ + ":"
 
-__nextversion__ = "2.0.0b3"
+__nextversion__ = "2.1.1"
 __tocinfoline__ = "VORTEX core package"
 
 __all__ = [
@@ -100,8 +111,11 @@ footprints.setup.callback = vortexfpdefaults
 ticket = sessions.get
 sh = sessions.system
 
-# If a config file can be found in current dir, load it
-config.load_config()
+# If a config file can be found in current dir, load it else load .vortex.d/vortex.toml
+if os.path.isfile("vortex.toml"):
+    config.load_config("vortex.toml")
+else:
+    config.load_config(os.environ["HOME"] + "/.vortex.d/vortex.toml")
 
 # Load some superstars sub-packages
 

vortex/algo/components.py

@@ -50,7 +50,7 @@ from bronx.syntax.decorators import nicedeco
 import footprints
 from taylorism import Boss
 import vortex
-from vortex.config import from_config
+import vortex.config as config
 from vortex.algo import mpitools
 from vortex.syntax.stdattrs import DelayedEnvValue
 from vortex.tools.parallelism import ParallelResultParser
@@ -704,15 +704,15 @@ class AlgoComponent(footprints.FootprintBase, metaclass=AlgoComponentMeta):
 
     def export(self, packenv):
         """Export environment variables in given pack."""
-        for k, v in from_config(section=packenv).items():
+        for k, v in config.from_config(section=packenv).items():
             if k not in self.env:
                 logger.info("Setting %s env %s = %s", packenv.upper(), k, v)
                 self.env[k] = v
 
     def prepare(self, rh, opts):
         """Set some defaults env values."""
-        if opts.get("fortran", True):
-            self.export("fortran")
+        if config.is_defined(section="env"):
+            self.export("env")
 
     def absexcutable(self, xfile):
         """Retuns the absolute pathname of the ``xfile`` executable."""
@@ -1841,7 +1841,7 @@ class Parallel(xExecutableAlgoComponent):
     def _mpitool_attributes(self, opts):
         """Return the dictionary of attributes needed to create the mpitool object."""
         # Read the appropriate configuration in the target file
-        conf_dict = from_config(section="mpitool")
+        conf_dict = config.from_config(section="mpitool")
         if self.mpiname:
             conf_dict["mpiname"] = self.mpiname
         # Make "mpirun" the default mpi command name
@@ -1850,7 +1850,7 @@ class Parallel(xExecutableAlgoComponent):
         possible_attrs = functools.reduce(
             lambda s, t: s | t,
             [
-                set(cls._footprint.attr.keys())
+                set(cls.footprint_retrieve().attr.keys())
                 for cls in footprints.proxy.mpitools
             ],
         )

vortex/algo/mpitools.py

@@ -182,8 +182,8 @@ class MpiTool(footprints.FootprintBase):
     )
 
     _envelope_bit_kind = "basicenvelopebit"
-    _envelope_wrapper_tpl = "@envelope_wrapper_default.tpl"
-    _wrapstd_wrapper_tpl = "@wrapstd_wrapper_default.tpl"
+    _envelope_wrapper_tpl = "envelope_wrapper_default.tpl"
+    _wrapstd_wrapper_tpl = "wrapstd_wrapper_default.tpl"
     _envelope_wrapper_name = "./global_envelope_wrapper.py"
     _wrapstd_wrapper_name = "./global_wrapstd_wrapper.py"
     _envelope_rank_var = "MPIRANK"
@@ -700,11 +700,11 @@ class MpiTool(footprints.FootprintBase):
         if not self.mpiwrapstd:
             return None
         # Create the launchwrapper
-        with importlib.resources.as_file(
-            importlib.resources.files("vortex.algo")
-        ) as path:
-            tplpath = path / "mpitools_templates" / self._wrapstd_wrapper_tpl
-        wtpl = config.load_template(tplpath, encoding="utf-8")
+        with importlib.resources.path(
+            "vortex.algo.mpitools_templates",
+            self._wrapstd_wrapper_tpl,
+        ) as tplpath:
+            wtpl = config.load_template(tplpath, encoding="utf-8")
         with open(self._wrapstd_wrapper_name, "w", encoding="utf-8") as fhw:
             fhw.write(
                 wtpl.substitute(
@@ -911,11 +911,11 @@ class MpiTool(footprints.FootprintBase):
             "Here are the envelope details:\n%s", "\n".join(binding_str)
         )
         # Create the launchwrapper
-        with importlib.resources.as_file(
-            importlib.resources.files("vortex.algo")
-        ) as path:
-            tplpath = path / "mpitools_templates" / self._envelope_wrapper_tpl
-        wtpl = config.load_template(tplpath, encoding="utf-8")
+        with importlib.resources.path(
+            "vortex.algo.mpitools_templates",
+            self._envelope_wrapper_tpl,
+        ) as tplpath:
+            wtpl = config.load_template(tplpath, encoding="utf-8")
         with open(self._envelope_wrapper_name, "w", encoding="utf-8") as fhw:
             fhw.write(
                 wtpl.substitute(

vortex/algo/mpitools_templates/__init__.py

@@ -0,0 +1 @@
+# Empty init file

vortex/data/handlers.py

@@ -5,6 +5,7 @@ the various caches or archives".
 """
 
 import functools
+import importlib
 import re
 import sys
 
@@ -1116,18 +1117,23 @@ class Handler:
         self,
         pr_getter=None,
         tplfile=None,
-        tplskip="@sync-skip.tpl",
-        tplfetch="@sync-fetch.tpl",
         py_exec=sys.executable,
         py_opts="",
     ):
         """Build a getter for the expected resource."""
         if tplfile is None:
-            tplfile = tplfetch if self.is_expected() else tplskip
+            tplfile = (
+                "sync-" + ("fetch" if self.is_expected() else "skip") + ".tpl"
+            )
+        with importlib.resources.path(
+            "vortex.data.sync_templates",
+            tplfile,
+        ) as tplpath:
+            tpl = config.load_template(tplpath)
         if pr_getter is None:
             pr_getter = self.container.localpath() + ".getpr"
         t = self._cur_session
-        tpl = config.load_template(t, tplfile)
+
         with open(pr_getter, "w", encoding="utf-8") as fd:
             fd.write(
                 tpl.substitute(

vortex/data/stores.py

@@ -527,6 +527,10 @@ class _VortexBaseArchiveStore(ArchiveStore, _VortexStackedStorageMixin):
         """Remap actual remote path to distant store path for intrusive actions."""
         raise NotImplementedError
 
+    def remap_write(self, remote, options):
+        """Remap actual remote path to distant store path for intrusive actions."""
+        raise NotImplementedError
+
     def remap_list(self, remote, options):
         """Reformulates the remote path to compatible vortex namespace."""
         if len(remote["path"].split("/")) >= 4:
@@ -537,8 +541,6 @@ class _VortexBaseArchiveStore(ArchiveStore, _VortexStackedStorageMixin):
             )
             return None
 
-    remap_write = remap_read
-
     @property
     def stacks_autorefill(self):
         """Where to refill a stack retrieved from the archive."""
@@ -755,6 +757,8 @@ class VortexStdBaseArchiveStore(_VortexBaseArchiveStore):
             raise e
         return remote
 
+    remap_write = remap_read
+
 
 class VortexStdStackedArchiveStore(VortexStdBaseArchiveStore):
     """Archive for casual VORTEX experiments: Support for legacy/Olive XPIDs.
@@ -1105,16 +1109,10 @@ class VortexCacheStore(_AbstractVortexCacheMultiStore):
 
     def alternates_netloc(self):
         """For Non-Op users, Op caches may be accessed in read-only mode."""
-        netloc_m = re.match(
-            r"(?P<base>vortex.*)\.cache\.(?P<country>\w+)", self.netloc
-        )
-        mt_netloc = "{base:s}.cache-mt.{country:s}".format(
-            **netloc_m.groupdict()
-        )
-        s_mt_netloc = "{base:s}.stacked-cache-mt.{country:s}".format(
-            **netloc_m.groupdict()
-        )
-        return [mt_netloc, s_mt_netloc]
+        return [
+            f"{self.netloc.firstname}.cache-mt.fr",
+            f"{self.netloc.firstname}.stacked-cache-mt.fr",
+        ]
 
 
 class VortexVsopCacheStore(_AbstractVortexCacheMultiStore):
@@ -1219,13 +1217,7 @@ class VortexStackStore(_AbstractVortexStackMultiStore):
 
     def alternates_netloc(self):
         """Go through the various stacked stores."""
-        netloc_m = re.match(
-            r"(?P<base>vortex.*)\.stack\.(?P<country>\w+)", self.netloc
-        )
-        s_mt_netloc = "{base:s}.stacked-cache-mt.{country:s}".format(
-            **netloc_m.groupdict()
-        )
-        return [s_mt_netloc]
+        return [f"{self.netloc.firstname}.stacked-cache-mt.fr"]
 
 
 class VortexVsopStackStore(_AbstractVortexStackMultiStore):

vortex/nwp/algo/forecasts.py

@@ -283,21 +283,12 @@ class LAMForecast(Forecast):
                 values=["lamfc", "lamforecast"],
                 remap=dict(lamforecast="lamfc"),
             ),
-            synctool=dict(
-                info="The name of the script called when waiting for coupling files",
-                optional=True,
-                default="atcp.alad",
-                doc_visibility=footprints.doc.visibility.ADVANCED,
-            ),
-            synctpl=dict(
-                info="The template used to generate the *synctool* script",
-                optional=True,
-                default="@sync-fetch.tpl",
-                doc_visibility=footprints.doc.visibility.ADVANCED,
-            ),
         ),
     )
 
+    synctool = "atcp.alad"
+    synctpl = "sync-fetch.tpl"
+
     def spawn_command_options(self):
         """Dictionary provided for command line factory."""
         return dict(
@@ -341,10 +332,11 @@ class LAMForecast(Forecast):
             )
             sh.softlink(thisbound, lbcnc(number=i))
             if self.mksync:
-                thistool = self.synctool + ".{:03d}".format(i)
-                bound.mkgetpr(pr_getter=thistool, tplfetch=self.synctpl)
+                bound.mkgetpr(
+                    pr_getter=self.synctool + ".{:03d}".format(i),
+                )
                 if firstsync is None:
-                    firstsync = thistool
+                    firstsync = self.synctool + ".{:03d}".format(i)
 
         # Set up the first synchronization step
         if firstsync is not None:

vortex/nwp/algo/ifsroot.py

@@ -120,6 +120,11 @@ class IFSParallel(
         """Extend default tag with ``kind`` value."""
         return super().fstag() + "." + self.kind
 
+    def _mpitool_attributes(self, opts):
+        conf_dict = super()._mpitool_attributes(opts)
+        conf_dict.update({"mplbased": True})
+        return conf_dict
+
     def valid_executable(self, rh):
         """Be sure that the specifed executable is ifsmodel compatible."""
         valid = super().valid_executable(rh)

vortex/nwp/algo/monitoring.py

@@ -57,6 +57,11 @@ class OdbMonitoring(
         )
     )
 
+    def _mpitool_attributes(self, opts):
+        conf_dict = super()._mpitool_attributes(opts)
+        conf_dict.update({"mplbased": True})
+        return conf_dict
+
     def _fix_nam_macro(self, rh, macro, value):
         """Set a given namelist macro and issue a log message."""
         rh.contents.setmacro(macro, value)

vortex/nwp/algo/mpitools.py

@@ -64,18 +64,28 @@ class MpiAuto(mpitools.MpiTool):
             ),
             bindingmethod=dict(
                 info="How to bind the MPI processes",
-                values=["arch", "launcherspecific", "vortex"],
+                values=["vortex", "arch", "launcherspecific"],
                 optional=True,
                 doc_visibility=footprints.doc.visibility.ADVANCED,
                 doc_zorder=-90,
             ),
+            mplbased=dict(
+                info="Is the executable based on MPL?",
+                type=bool,
+                optional=True,
+                default=False,
+            ),
         )
     )
 
-    _envelope_wrapper_tpl = "@envelope_wrapper_mpiauto.tpl"
+    _envelope_wrapper_tpl = "envelope_wrapper_mpiauto.tpl"
     _envelope_rank_var = "MPIAUTORANK"
     _needs_mpilib_specific_mpienv = False
 
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        self.bindingmethod = "arch" if self.mplbased else "vortex"
+
     def _reshaped_mpiopts(self):
         """Raw list of mpi tool command line options."""
         options = super()._reshaped_mpiopts()

vortex/nwp/algo/odbtools.py

@@ -850,6 +850,11 @@ class OdbAverage(Parallel, odb.OdbComponentDecoMixin, drhook.DrHookDecoMixin):
         )
     )
 
+    def _mpitool_attributes(self, opts):
+        conf_dict = super()._mpitool_attributes(opts)
+        conf_dict.update({"mplbased": True})
+        return conf_dict
+
     def prepare(self, rh, opts):
         """Find any ODB candidate in input files."""
 
@@ -962,6 +967,11 @@ class OdbCompress(Parallel, odb.OdbComponentDecoMixin, drhook.DrHookDecoMixin):
         )
     )
 
+    def _mpitool_attributes(self, opts):
+        conf_dict = super()._mpitool_attributes(opts)
+        conf_dict.update({"mplbased": True})
+        return conf_dict
+
     def prepare(self, rh, opts):
         """Find any ODB candidate in input files and fox ODB env accordingly."""
 
@@ -1027,6 +1037,11 @@ class OdbMatchup(Parallel, odb.OdbComponentDecoMixin, drhook.DrHookDecoMixin):
         )
     )
 
+    def _mpitool_attributes(self, opts):
+        conf_dict = super()._mpitool_attributes(opts)
+        conf_dict.update({"mplbased": True})
+        return conf_dict
+
     def prepare(self, rh, opts):
         """Find ODB candidates in input files."""
 
@@ -1116,6 +1131,11 @@ class OdbReshuffle(
     _OUT_DIRECTORY = "reshuffled"
     _BARE_OUT_LAYOUT = "ccma"
 
+    def _mpitool_attributes(self, opts):
+        conf_dict = super()._mpitool_attributes(opts)
+        conf_dict.update({"mplbased": True})
+        return conf_dict
+
     def prepare(self, rh, opts):
         """Find ODB candidates in input files."""
 

vortex/nwp/algo/oopsroot.py

@@ -619,6 +619,11 @@ class OOPSParallel(
             logger.error("The binary < %s > has no cycle attribute", repr(rh))
             return False
 
+    def _mpitool_attributes(self, opts):
+        conf_dict = super()._mpitool_attributes(opts)
+        conf_dict.update({"mplbased": True})
+        return conf_dict
+
     def prepare(self, rh, opts):
         """Preliminary setups."""
         super().prepare(rh, opts)

vortex/tools/actions.py

@@ -10,8 +10,6 @@ to be processed: e.g. mail, routing, alarm.
 import bronx.stdtypes.catalog
 import footprints
 from bronx.fancies import loggers
-from bronx.fancies.display import dict_as_str
-from vortex import sessions
 
 #: Export nothing
 __all__ = []
@@ -134,105 +132,30 @@ class Action:
 class TunableAction(Action):
     """An Action that may be tuned
 
-    - may have it's own section in the target configuration files
-    - accepts the syntax `ad.action_tune(key=value)` (which has priority)
+    accepts the syntax `ad.action_tune(key=value)` (which has priority)
     """
 
     def __init__(self, configuration=None, **kwargs):
         super().__init__(**kwargs)
         self._tuning = dict()
-        self._conf_section = configuration
-        self._conf_dict = None
-
-    @property
-    def _shtarget(self):
-        """Warning: this may be a `vortex.syntax.stdattrs.DelayedInit` object
-        during Vortex initialization and may not have a `sections()` method
-        nor a `config` property.
-        """
-        return sessions.current().sh.default_target
-
-    @property
-    def _conf_items(self):
-        """Check and return the configuration: a section in the target-xxx.ini file.
-
-        If the configuration is None, an attempt is made to use the Action's kind.
-        Don't use before Vortex initialization is done (see `_shtarget`).
-        """
-        if self._conf_dict is None:
-            if self._conf_section is None:
-                if self.kind in self._shtarget.sections():
-                    self._conf_section = self.kind
-            else:
-                if self._conf_section not in self._shtarget.sections():
-                    raise KeyError(
-                        'No section "{}" in "{}"'.format(
-                            self._conf_section, self._shtarget.config.file
-                        )
-                    )
-            if self._conf_section is None:
-                self._conf_dict = dict()
-            else:
-                self._conf_dict = self._shtarget.items(self._conf_section)
-        return self._conf_dict
 
     def service_info(self, **kw):
-        for k, v in self._get_config_dict().items():
+        for k, v in self._tuning.items():
             kw.setdefault(k, v)
         return super().service_info(**kw)
 
-    def tune(self, section=None, **kw):
-        """Add options to override the .ini file configuration.
-
-        ``section`` is a specific section name, or ``None`` for all.
-        """
-        if section is None or section == self._conf_section:
-            self._tuning.update(kw)
-
-    def _get_config_dict(self):
-        final_dict = dict()
-        final_dict.update(self._conf_items)
-        final_dict.update(self._tuning)
-        return final_dict
-
-    def info(self):
-        """Informative string (may serve debugging purposes)."""
-        s = super().info() + " - tunable\n"
-        mix = dict()
-        mix.update(self._conf_items)
-        mix.update(self._tuning)
-        prt = dict()
-        for k, v in mix.items():
-            if k in self._tuning:
-                prt["++ " + k] = "{} (was: {})".format(
-                    v,
-                    str(self._conf_items[k])
-                    if k in self._conf_items
-                    else "<not set>",
-                )
-            else:
-                prt["   " + k] = v
-        if self._conf_section is not None:
-            s += " " * 4 + "configuration: " + self._conf_section + "\n"
-        s += dict_as_str(prt, prefix=4)
-        return s.strip()
+    def tune(self, **kw):
+        self._tuning.update(kw)
 
     def getx(self, key, *args, **kw):
         """Shortcut to access the configuration overridden by the tuning."""
         if key in self._tuning:
             return self._tuning[key]
 
-        if self._conf_section is not None:
-            return self._shtarget.getx(
-                key=self._conf_section + ":" + key, *args, **kw
-            )
-
         if "default" in kw:
             return kw["default"]
 
-        raise KeyError(
-            'The "{:s}" entry was not found in any configuration'.format(key)
-        )
+        raise KeyError('The "{:s}" entry was not found'.format(key))
 
 
 class SendMail(Action):

vortex/tools/grib.py

@@ -10,6 +10,7 @@ It also provdes an AlgoComponent's Mixin to properly setup the environment
 when using the grib_api or ecCodes libraries.
 """
 
+from pathlib import Path
 from urllib import parse as urlparse
 
 import re
@@ -563,37 +564,41 @@ class EcGribDecoMixin(AlgoComponentDecoMixin):
                 "After gribapi_setup %s = %s", a_var, self.env.getvar(a_var)
             )
 
-    def _eccodes_envsetup(self, eccodes_lib):
-        """Setup environment variables for ecCodes."""
-        dep_warn = (
-            "%s is left unconfigured because the old grib_api's variable is defined."
-            + "Please remove that !"
+    def _eccodes_envsetup(
+        self,
+        eccodes_lib,
+        envvar="ECCODES_DEFINITIONS_PATH",
+        tgt_path="definitions",
+    ):
+        """Export envirionment variables required by ECCODES
+
+        Value is
+
+        /path/to/eccodes-X.Y.Z/share/eccodes/<target_path>
+
+        eccodes_lib: Absolute path to the eccodes so file
+        envvar: Name of the environment variable to export
+        tgt_path: Name of the eccodes install subdirectory to appear
+          in the value
+        """
+        if envvar in self.env:
+            return envvar
+        if envvar.replace("ECCODES", "GRIB") in self.env:
+            logger.warning(
+                (
+                    "%s is left unconfigured because the old grib_api's"
+                    "variable is defined. ",
+                    "Please remove that!",
+                ),
+                envvar,
+            )
+            return envvar.replace("ECCODES", "GRIB")
+        eccodes_root = Path(eccodes_lib).parent.parent
+        self.env.setgenericpath(
+            envvar,
+            str(eccodes_root / "share" / "eccodes" / tgt_path),
         )
-        eccodes_root = self.system.path.dirname(eccodes_lib)
-        eccodes_root = self.system.path.split(eccodes_root)[0]
-        eccodes_share = self.system.path.join(eccodes_root, "share", "eccodes")
-        defvar = "ECCODES_DEFINITION_PATH"
-        if defvar not in self.env:
-            if "GRIB_DEFINITION_PATH" in self.env:
-                logger.warning(dep_warn, defvar)
-                defvar = "GRIB_DEFINITION_PATH"
-            else:
-                self.env.setgenericpath(
-                    defvar, self.system.path.join(eccodes_share, "definitions")
-                )
-        samplevar = "ECCODES_SAMPLES_PATH"
-        if samplevar not in self.env:
-            if "GRIB_SAMPLES_PATH" in self.env:
-                logger.warning(dep_warn, samplevar)
-                samplevar = "GRIB_SAMPLES_PATH"
-            else:
-                self.env.setgenericpath(
-                    samplevar,
-                    self.system.path.join(
-                        eccodes_share, "ifs_samples", "grib1"
-                    ),
-                )
-        return defvar, samplevar
+        return envvar
 
     def eccodes_setup(self, rh, opts, compat=False, fatal=True):
         """Setup the grib_api related stuff.
@@ -605,7 +610,19 @@ class EcGribDecoMixin(AlgoComponentDecoMixin):
         # Detect the library's path and setup appropriate variables
         eccodes_lib, gribapi_lib = self._ecgrib_libs_detext(rh)
         if eccodes_lib is not None:
-            defvar, samplevar = self._eccodes_envsetup(eccodes_lib)
+            defvar = self._eccodes_envsetup(
+                eccodes_lib,
+                envvar="ECCODES_DEFINITIONS_PATH",
+                tgt_path="definitions",
+            )
+            subdir = Path("ifs_samples") / (
+                "grib1" if rh.resource.cycle < "cy49" else "grib1_mlgrib2"
+            )
+            samplevar = self._eccodes_envsetup(
+                eccodes_lib,
+                envvar="ECCODES_SAMPLES_PATH",
+                tgt_path=subdir,
+            )
         elif compat:
             defvar, samplevar = self._gribapi_envsetup(gribapi_lib)
         else:

vortex/tools/net.py

@@ -26,6 +26,8 @@ from bronx.fancies import loggers
 from bronx.net.netrc import netrc
 from bronx.syntax.decorators import nicedeco, secure_getattr
 
+from vortex.config import get_from_config_w_default, ConfigurationError
+
 #: No automatic export
 __all__ = []
 
@@ -1192,15 +1194,35 @@ class Ssh:
         self._logname = logname
         self._remote = hostname
 
-        target = sh.default_target
-        self._sshcmd = target.get(key="services:sshcmd", default="ssh")
-        self._scpcmd = target.get(key="services:scpcmd", default="scp")
+        def _get_ssh_config(key, default):
+            config = get_from_config_w_default(
+                section="ssh", key=key, default=default
+            )
+            try:
+                val = config.pop("default")
+            except AttributeError:
+                assert isinstance(config, str)
+                return config
+            except KeyError:
+                msg = (
+                    "A default value must be specified for configuration option"
+                    f" {key}. See vortex-nwp.readthedocs.io/en/latest/user-guide/configuration.html#ssh"
+                )
+                raise ConfigurationError(msg)
+
+            for k, v in config.items():
+                if re.match(k, socket.gethostname()):
+                    val = v
+            return val
+
+        self._sshcmd = _get_ssh_config(key="sshcmd", default="ssh")
+        self._scpcmd = _get_ssh_config(key="scpcmd", default="scp")
         self._sshopts = (
-            target.get(key="services:sshopts", default="-x").split()
+            _get_ssh_config(key="sshopts", default="").split()
             + (sshopts or "").split()
         )
         self._scpopts = (
-            target.get(key="services:scpopts", default="-Bp").split()
+            _get_ssh_config(key="scpopts", default="").split()
             + (scpopts or "").split()
         )
 

vortex/tools/services.py

@@ -20,7 +20,6 @@ from bronx.stdtypes.dictionaries import UpperCaseDict
 from bronx.syntax.pretty import EncodedPrettyPrinter
 from vortex import sessions
 from vortex.util.config import (
-    GenericConfigParser,
     load_template,
     LegacyTemplatingAdapter,
 )
@@ -563,7 +562,8 @@ class Directory:
 
     def __init__(self, inifile, domain="meteo.fr", encoding=None):
         """Keep aliases in memory, as a dict of sets."""
-        config = GenericConfigParser(inifile, encoding=encoding)
+        config = configparser.ConfigParser()
+        config.read(inifile, encoding=encoding)
         try:
             self.domain = config.get("general", "default_domain")
         except configparser.NoOptionError:

vortex/tools/systems.py

@@ -68,6 +68,7 @@ from vortex.tools.env import Environment
 from vortex.tools.net import AssistedSsh, AutoRetriesFtp, DEFAULT_FTP_PORT
 from vortex.tools.net import FtpConnectionPool, LinuxNetstats, StdFtp
 import vortex.tools.storage
+from vortex import config
 
 #: No automatic export
 __all__ = []
@@ -800,7 +801,7 @@ class OSExtended(System):
 
             * **rmtreemin** - as the minimal depth needed for a :meth:`rmsafe`.
             * **cmpaftercp** - as a boolean for activating full comparison after plain cp (default: *True*).
-            * **ftraw** - allows ``smartft*`` methods to use the raw FTP commands
+            * **ftserv** - allows ``smartft*`` methods to use the raw FTP commands
               (e.g. ftget, ftput) instead of the internal Vortex's FTP client
               (default: *False*).
             * **ftputcmd** - The name of the raw FTP command for the "put" action
@@ -815,7 +816,7 @@ class OSExtended(System):
         self._rmtreemin = kw.pop("rmtreemin", 3)
         self._cmpaftercp = kw.pop("cmpaftercp", True)
         # Switches for rawft* methods
-        self._ftraw = kw.pop("ftraw", None)
+        self._ftserv = kw.pop("ftserv", None)
         self.ftputcmd = kw.pop("ftputcmd", None)
         self.ftgetcmd = kw.pop("ftgetcmd", None)
         # FTP stuff again
@@ -838,23 +839,33 @@ class OSExtended(System):
         self._signal_intercept_init()
 
     @property
-    def ftraw(self):
+    def ftserv(self):
         """Use the system's FTP service (e.g. ftserv)."""
-        if self._ftraw is None:
-            return self.default_target.ftraw_default
+        if self._ftserv is None:
+            return self._use_ftserv()
         else:
-            return self._ftraw
+            return self._ftserv
 
-    @ftraw.setter
-    def ftraw(self, value):
+    @ftserv.setter
+    def ftserv(self, value):
         """Use the system's FTP service (e.g. ftserv)."""
         self._ftraw = bool(value)
 
-    @ftraw.deleter
-    def ftraw(self):
+    @ftserv.deleter
+    def ftserv(self):
         """Use the system's FTP service (e.g. ftserv)."""
         self._ftraw = None
 
+    def _use_ftserv(self):
+        if not config.is_defined(section="ftserv"):
+            return False
+        for rgxp in config.from_config(
+            section="ftserv", key="hostname_patterns"
+        ):
+            if re.match(rgxp, self.hostname):
+                return True
+        return False
+
     def target(self, **kw):
         """
         Provide a default :class:`~vortex.tools.targets.Target` according
@@ -2064,7 +2075,7 @@ class OSExtended(System):
 
     def rawftput_worthy(self, source, destination):
         """Is it allowed to use FtServ given **source** and **destination**."""
-        return self.ftraw and self.ftserv_allowed(source, destination)
+        return self.ftserv and self.ftserv_allowed(source, destination)
 
     @fmtshcmd
     def rawftput(
@@ -2145,7 +2156,7 @@ class OSExtended(System):
 
         ``rawftput`` will be used if all of the following conditions are met:
 
-            * ``self.ftraw`` is *True*
+            * ``self.ftserv`` is *True*
             * **source** is a string (as opposed to a File like object)
             * **destination** is a string (as opposed to a File like object)
         """
@@ -2177,7 +2188,7 @@ class OSExtended(System):
     def rawftget_worthy(self, source, destination, cpipeline=None):
         """Is it allowed to use FtServ given **source** and **destination**."""
         return (
-            self.ftraw
+            self.ftserv
             and cpipeline is None
             and self.ftserv_allowed(source, destination)
         )
@@ -2259,7 +2270,7 @@ class OSExtended(System):
 
         ``rawftget`` will be used if all of the following conditions are met:
 
-            * ``self.ftraw`` is *True*
+            * ``self.ftserv`` is *True*
             * **cpipeline** is None
             * **source** is a string (as opposed to a File like object)
             * **destination** is a string (as opposed to a File like object)

vortex/util/config.py

@@ -164,7 +164,8 @@ def load_template(tplpath, encoding=None, default_templating="legacy"):
     """
     tplpath = Path(tplpath).absolute()
     if not tplpath.exists():
-        raise FileNotFoundError
+        msg = f"Template file {tplpath} not found"
+        raise FileNotFoundError(msg)
     ignored_lines = set()
     actual_encoding = None if encoding == "script" else encoding
     actual_templating = default_templating

{vortex_nwp-2.0.0b2.dist-info → vortex_nwp-2.1.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
-Metadata-Version: 2.2
+Metadata-Version: 2.4
 Name: vortex-nwp
-Version: 2.0.0b2
+Version: 2.1.0
 Summary: A Python library to write Numerical Weather Prediction pipelines components
 Author-email: The Vortex Team <vortex.support@meteo.fr>
 License: CECILL-C
@@ -21,6 +21,7 @@ Requires-Dist: sphinx-copybutton; extra == "docs"
 Provides-Extra: dev
 Requires-Dist: ruff==0.9.1; extra == "dev"
 Requires-Dist: pytest; extra == "dev"
+Dynamic: license-file
 
 ## vortex
 

{vortex_nwp-2.0.0b2.dist-info → vortex_nwp-2.1.0.dist-info}/RECORD

@@ -1,13 +1,14 @@
-vortex/__init__.py,sha256=EqLyspeGun2xW4een6ezLsh2APXX8DHphhGnu6TdMcQ,3921
+vortex/__init__.py,sha256=2D7c4cUOF2Eb4m2rCKxhttmBy5R_7Xb3iBCLcy51vkw,4352
 vortex/config.py,sha256=E7feBIG22POq5TyUN3MNXH-ZU7q08alF5nrJ1aUftG0,2815
 vortex/gloves.py,sha256=GKz27S8eLfRlk8fNqVL_z1gsQ8zvEj73p5uVi11ou00,8487
 vortex/proxy.py,sha256=OlPrVUJS5FoKt5pX8ApN1crFFDj8RJAqhDEilwvfrYU,127
 vortex/sessions.py,sha256=l_ZUUFS-l0J2Mg6KEXaUQiDhbIABrWCfcY-9ExazKEg,9988
 vortex/toolbox.py,sha256=h2QE5BoXWWEcQwK21D9YPda6t4rmCtLB5F9WeJrqT14,41379
 vortex/algo/__init__.py,sha256=I_9COn_QBRbwvbqhs0X3CdHeR97NZhBacIqwKjnVBTg,359
-vortex/algo/components.py,sha256=pD_-Lyhl8iVy3ERK7QTCkq00tHIblImofxYVv8C1LH4,89342
-vortex/algo/mpitools.py,sha256=kxq9sjvC8bt7RDxNEi7lvpByXhtPEg3Rr4xaADbeipw,74063
+vortex/algo/components.py,sha256=zs_DaKHFEHX9GQ-pn1HheAo4JvNusQi3D4u3ZhqVqe4,89362
+vortex/algo/mpitools.py,sha256=EffZ4Ok934tvEYyUjKpVA-lEUVA29FwUVBtUQ8xWnDI,73977
 vortex/algo/serversynctools.py,sha256=fPel0txVHsrUfk6VFaeKa0D6i21fOskIAR_BbByBv9g,5601
+vortex/algo/mpitools_templates/__init__.py,sha256=Jbw903aPqVKF-AaSoB-mGMxthSvm88O_yqGoGmf_S_U,18
 vortex/algo/mpitools_templates/envelope_wrapper_default.tpl,sha256=4VhkDx_YbOYywKQ82HIxRJXGcDpLuOgqcY7Edx9Rxyw,453
 vortex/algo/mpitools_templates/envelope_wrapper_mpiauto.tpl,sha256=iB4_4jz-7QDqpl6AxrlCACsotJnEMG75dtazuGfkCb4,514
 vortex/algo/mpitools_templates/wrapstd_wrapper_default.tpl,sha256=hh1Um95FIuvXAhUTMifAfixtfGgyjF1vEaF1nmbqMLw,502
@@ -19,11 +20,12 @@ vortex/data/executables.py,sha256=FeR3SA2wW97zAQXwWVecZ0v6VYT5L_3K1Czuv33Bk74,67
 vortex/data/flow.py,sha256=P1itBnA8jaoCWnVQjqbD_Pf26rpzud1JdwSLECDnDl4,3008
 vortex/data/geometries.ini,sha256=J7hX5hYWqwBjdUAul6q8j10U6b3I-QEHrFJ98LBTQXM,52805
 vortex/data/geometries.py,sha256=SjCF-74zDkbpt3qQujJU_UpeoBtgS7-xrEhbC4ywyRE,27429
-vortex/data/handlers.py,sha256=9TIu6wjuN2X64AOprqVc_uSkzRfyOvym6MHG2l1QOAQ,47426
+vortex/data/handlers.py,sha256=6tRfCw9yzQCndVhRjz3ia0dyYhV3F7_rWeXjoxBtbvs,47550
 vortex/data/outflow.py,sha256=IPKJkn75lRvhSqN5969TuhRAPnsZKlrWR4Cmw6VBFDs,1475
 vortex/data/providers.py,sha256=kLHJI3F9mkRjRcQzs0xaH_cdfvXd9H4x5h1Lm4PGSFk,15652
 vortex/data/resources.py,sha256=UOwvQDTJxS9z66Aa7JM2nWM3IxrMWvY3L2J9D5w3sZw,6348
-vortex/data/stores.py,sha256=YIRl0YgpslR7IFUasX3NWzy-Q3oswZ_EcyzGYVampyE,44777
+vortex/data/stores.py,sha256=5svCgpE8WuuAWCnOrrHOBw30EOt0cvDoxrlq9rIWKOw,44517
+vortex/data/sync_templates/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 vortex/layout/__init__.py,sha256=aZsDhVJrd3-648vw-UESI_2BLxyGl71sHdyrg9cL638,826
 vortex/layout/contexts.py,sha256=GkysEtOOUj3nQvGUWrFjOOjzv7SmWFPUSLTH_pMMycI,20363
 vortex/layout/dataflow.py,sha256=SKOUJKbtftW4POStjyBqNrqjvfbutsNHnWQ-p_75l_8,44792
@@ -35,14 +37,14 @@ vortex/nwp/algo/clim.py,sha256=Z9QwUfosH0X_xTg6mam68qWIvw6Ts6CZIpc7Nb0NO1s,37511
 vortex/nwp/algo/coupling.py,sha256=6KQ_YB7PCvVbt6c2VEtpCzwIyWRTjLfu54mNpKnXL9Q,29755
 vortex/nwp/algo/eda.py,sha256=Wf2mt1r6v_ZYDz4BFyt-4JP1XAsO8sZW3O1UAOSRzmU,30746
 vortex/nwp/algo/eps.py,sha256=JCBFiaFF9TQ-x7Szgw44Cd39tRD9184UXqOtoT6aRCs,26812
-vortex/nwp/algo/forecasts.py,sha256=vI6NOH8-RciIz47LlzEK1u-jvA2LHJHAULVlu8vcxb0,30518
+vortex/nwp/algo/forecasts.py,sha256=sqs0IETanpfS8q1f1Pl3mQZtSNbfOIUOidmcduGp8EU,30063
 vortex/nwp/algo/fpserver.py,sha256=98HQZsdQE22spy_mIE1G-NORQMMjfA5grEjeIMSlP9w,50163
 vortex/nwp/algo/ifsnaming.py,sha256=WfpSpRJ7ua3ihqv8Y4UkrvE4pb0CkNWKIlW9EjY-2ao,11188
-vortex/nwp/algo/ifsroot.py,sha256=j8uu3pMt9GF5od6DuocXvjugVFs1ovuJrwFuFgXx99E,14064
-vortex/nwp/algo/monitoring.py,sha256=QwmE9wuyNi0tpAWig3X6Z1uGg-h5k4VZ9v4BdajH344,8727
-vortex/nwp/algo/mpitools.py,sha256=TRnOwb2yzp7mGT4Sw5elSwJVpADnAISGlRSSlK_Rf9I,24539
-vortex/nwp/algo/odbtools.py,sha256=RZwxn8HNZ17qqsgvPSlWjW5cF6hnClUC_iv43XQRFXc,43430
-vortex/nwp/algo/oopsroot.py,sha256=03SCYVbaDo-ydwpJl97yHWGkIYxduZEGphLgvc9DEoU,33640
+vortex/nwp/algo/ifsroot.py,sha256=Mmp0sZtexP5ob3iUFTiVbgVWog7ubAYPut0cAB0vbbg,14230
+vortex/nwp/algo/monitoring.py,sha256=JVPZPw_I5QRVtYvvSYpJpBy5RRPhqBQjcAFfIkvKuHA,8893
+vortex/nwp/algo/mpitools.py,sha256=QO3I9iMdY69znnA6hJy8XUAzBoejDpsVMQFRo3YzXro,24876
+vortex/nwp/algo/odbtools.py,sha256=xltXS1lpxeeJXeCGnFiqFUhGqMI6BlZzKcSLhsnd2cs,44094
+vortex/nwp/algo/oopsroot.py,sha256=es3p7RPOixQT74TcglJX1q8MswxveVnLZa84kyhu3uM,33806
 vortex/nwp/algo/oopstests.py,sha256=owQAS7_fGbUGZGzMsJek32dbb3JQ6dafSVBrdmveQ-Q,6654
 vortex/nwp/algo/request.py,sha256=y5N9RUfKoPXR6EbSely37Zz6d-2HYyfV3UCvvoenqbY,22809
 vortex/nwp/algo/stdpost.py,sha256=kpZrP6s92PrrAWyLrf61bSidwkJzC3M6sxrmpete3S0,54824
@@ -101,7 +103,7 @@ vortex/syntax/__init__.py,sha256=q0eJcJ1gRbSrALTC8UaORRrHNmRxVUpbTNB1_uQbJi0,288
 vortex/syntax/stdattrs.py,sha256=OosTS6hdGGh1r5rwZNngmMb6zFj0NUrFZmYs7lDbysY,20927
 vortex/syntax/stddeco.py,sha256=Y94wAf-zEQr8RUmF_OUZkBKO1fPZnx3ju2g3DnPl9cw,6411
 vortex/tools/__init__.py,sha256=KPsj7iqvy-vlo3WEIx8PmbqJYVDXCzZZdDSbMlqDq0k,482
-vortex/tools/actions.py,sha256=BrPDuYXKiGChRVtjXVhLyEGls0-AP6C8TaeTMiQwPyo,16001
+vortex/tools/actions.py,sha256=yPzKccxHqtTI67SaYLIWJeo0s302WSXEJVpfxt6hAAU,13113
 vortex/tools/addons.py,sha256=vTfAXEz0zrFDGUbrFxwgBO67enrE2mmLbKKlakLSNpI,10439
 vortex/tools/arm.py,sha256=jl--2QOkhHkRXlxpZgo4E3Zw7Z_Gnr4z1fjPnPNyuyc,3112
 vortex/tools/compression.py,sha256=dTXX1VrzxEj1qoBDKbB_T-_8X9iGHOlPaseczfu8cuw,11221
@@ -110,23 +112,23 @@ vortex/tools/ddhpack.py,sha256=tOcQ5XQ7yTFBlisyjANDPMPQPijvHwz8yWY1eBfZMAs,206
 vortex/tools/delayedactions.py,sha256=miw6q5L5eOaxtLXhPIJlVs5Mi8vMXuIx55dY2TtU8BU,27126
 vortex/tools/env.py,sha256=ka7VaM3D5XuMRxja7oWA-eLgsUFqtWOmPvfbyHBi8Ps,18460
 vortex/tools/folder.py,sha256=R3bQtwnyn01WzkF3NdzJKb2tCkAPWYB61Bb90r0B09I,27992
-vortex/tools/grib.py,sha256=wReKDpWpTCrbiVUZKZaQdo8_YBzRLIJp0GvpEtkiHTw,25458
+vortex/tools/grib.py,sha256=aH1y2LZDRBn3TywXsVZlxTRk6kdShMVgbHJTpGcfMGU,25700
 vortex/tools/lfi.py,sha256=9mvXL-R5PQtTut7gsSMGJa0iNqpNgOFp7yuEMqYGIBA,29681
 vortex/tools/listings.py,sha256=LTfQHjT0XlK3Hxewbi06HZQjKhALmg0Gvz69UooOFog,15123
 vortex/tools/names.py,sha256=Knb2YsUsEjGpODBva1kEOBXRM2d_H0ylHwp-OoO3mnY,21644
-vortex/tools/net.py,sha256=MWamIGwm52qDpRhpxMP1BihsyZW4SwoPl6hg0GgX7WA,73734
+vortex/tools/net.py,sha256=jLablaFdAy80BybYT4Ki-LWgbpqpzMj4SG8m4CLvuN8,74535
 vortex/tools/odb.py,sha256=TZ1Qq8QmFIh3OP-bsgnwPuHPdfMKNXUAfhc-KSx9tZQ,193
 vortex/tools/parallelism.py,sha256=f4uBp37Q3BG71Qn1yy_YSTPdk-2ZZ7zoGRQsGm6pT1Y,12345
 vortex/tools/prestaging.py,sha256=lGJIo4SA8pDziF1nbBogsrK1bgwDQA6tj9Wg9VUFt6Q,7078
 vortex/tools/rawfiles.py,sha256=keUI6QtEZQrWQ7H_Sg_TY9SY32anfb-suqJkrDmdXN0,215
 vortex/tools/schedulers.py,sha256=XduOQab1dRiHwz7Mnt3mxXByGzAHMsDLCwlUOpMdXEk,15828
-vortex/tools/services.py,sha256=_cVaHQmwknJmSny6y20BL2M2K68MKN-ydOFtxWn9lPs,29228
+vortex/tools/services.py,sha256=v0uqAOXR9PsXJwv4-_yWpGzFniD7WghXW4p6WwaNtPA,29231
 vortex/tools/storage.py,sha256=YBhq00j39NipaK1EDovr0VF7EArdpp5AuFx0lqB0_p4,34411
 vortex/tools/surfex.py,sha256=qbCIGt-dmfIRWRRMjOQnzc9sGNnt8PIFSqQYT77trhw,1408
-vortex/tools/systems.py,sha256=yIs-Qy7roVdOAeWas20grfhiw-aeJVZ9JyIvTyu8mro,148751
+vortex/tools/systems.py,sha256=Zu04yTVIZU0hCiRH_STmdGuknjVd8dNsxXWr87k_5aE,149081
 vortex/tools/targets.py,sha256=-n2uMoEFdz3AeoSd1IGhlr7dYYb8xantUeh9_ggk9iI,14896
 vortex/util/__init__.py,sha256=Zt0OASbKbNnnwUqFHFoq5Mk13sGYSQhxqh7bUvJH6Y8,198
-vortex/util/config.py,sha256=-9fP9ZIJODHBfGE08UhwFB-VyVFstTscgKWibKEUmh0,39989
+vortex/util/config.py,sha256=KJHq0GMNRlwzh2WUdpwn1FIKCuAXla7nPRe7j3IZGcI,40045
 vortex/util/empty.py,sha256=EYcltpsjGxATAPPAVK2QRq_Bc5tcFLfF34dE1KO1tvk,535
 vortex/util/helpers.py,sha256=vmz2h3MCL1FElZpn-MX3MX6L_RCQfyqMXTcYWpcYyr4,7492
 vortex/util/introspection.py,sha256=VWZ6Ntr4fMJD4Xr_q-F7_lMHTSjDpHptnZ3SckBtuYY,2146
@@ -135,8 +137,8 @@ vortex/util/roles.py,sha256=9un_QAijaMn5iTS7PrdoWI5_NNw7uHxMWTnyhc5aNzg,1150
 vortex/util/storefunctions.py,sha256=uSfG-G_A88iJf3DwFBd-j0rw6eJta8opfRT39aQHsHM,3615
 vortex/util/structs.py,sha256=vapErq0MNhiKlsnjrv_a5M0Rn29KbP3WE_oiy4Hfwb8,683
 vortex/util/worker.py,sha256=zp8f2tx4SXwf1v55XMdYLAx7n3vSlg8PRGrkHgnfdmg,4721
-vortex_nwp-2.0.0b2.dist-info/LICENSE,sha256=ewBJPmWAcQqtBPrydH10tt6ECkcYP3b1o2RfH85pJF0,21863
-vortex_nwp-2.0.0b2.dist-info/METADATA,sha256=bRlOmQfR7sB8u7P5XYTrIHRVWuzCnaG6Zi9ZaoW6Cys,2165
-vortex_nwp-2.0.0b2.dist-info/WHEEL,sha256=52BFRY2Up02UkjOa29eZOS2VxUrpPORXg1pkohGGUS8,91
-vortex_nwp-2.0.0b2.dist-info/top_level.txt,sha256=3xfbSD7kw8xKl0jk4GNHsOPKbhubstfWHPl6bxHciRQ,7
-vortex_nwp-2.0.0b2.dist-info/RECORD,,
+vortex_nwp-2.1.0.dist-info/licenses/LICENSE,sha256=ewBJPmWAcQqtBPrydH10tt6ECkcYP3b1o2RfH85pJF0,21863
+vortex_nwp-2.1.0.dist-info/METADATA,sha256=58Aw65WFk1YubrWljPIRAPiGdMGfjo7JW4DYHEngbBg,2185
+vortex_nwp-2.1.0.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+vortex_nwp-2.1.0.dist-info/top_level.txt,sha256=3xfbSD7kw8xKl0jk4GNHsOPKbhubstfWHPl6bxHciRQ,7
+vortex_nwp-2.1.0.dist-info/RECORD,,

{vortex_nwp-2.0.0b2.dist-info → vortex_nwp-2.1.0.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: setuptools (76.0.0)
+Generator: setuptools (80.9.0)
 Root-Is-Purelib: true
 Tag: py3-none-any