vortex-nwp 2.0.0b2__py3-none-any.whl → 2.1.1__py3-none-any.whl

vortex/toolbox.py

@@ -485,15 +485,39 @@ def add_section(section, args, kw):
 
 # noinspection PyShadowingBuiltins
 def input(*args, **kw):  # @ReservedAssignment
-    """Add input :class:`~vortex.layout.dataflow.Section` objects to the current sequence.
+    r"""Declare one or more input resources.
+
+    This function takes an abitrary of keyword arguments forming the resource
+    description.
+
+    :return: A list of :py:class:`Handler <vortex.data.handlers.Handler>` objects.
+
+    **Example:**
+
+    The following call to ``input`` returns a list of 6
+    :py:class:`Handler <vortex.data.handlers.Handler>` objects, one
+    for each date and member:
+
+    .. code:: python
+
+       rhandlers = vortex.input(
+           kind='gridpoint',
+           term=1,
+           geometry='eurw1s40',
+           nativefmt='grib',
+           model='arome',
+           cutoff='production',
+           date=['2024060121', '2024060122'],
+           origin='historic',
+           vapp='arome',
+           vconf='pefrance',
+           member=[1,2,5],
+           experiment='myexp',
+           block='forecast',
+           local='gribfile_[member].grib',
+           format='grib',
+       )
 
-    Relies on the :func:`add_section` function (see its documentation), with:
-
-        * It's ``section`` attribute is automatically set to 'input';
-        * The ``kw``'s *insitu* item is set to :data:`active_insitu` by default.
-
-    :return: A list of :class:`vortex.data.handlers.Handler` objects (associated
-        with the newly created class:`~vortex.layout.dataflow.Section` objects).
     """
     kw.setdefault("insitu", active_insitu)
     kw.setdefault("batch", active_batchinputs)
@@ -532,14 +556,39 @@ def show_inputs(context=None):
 
 
 def output(*args, **kw):
-    """Add output :class:`~vortex.layout.dataflow.Section` objects to the current sequence.
-
-    Relies on the :func:`add_section` function (see its documentation), with:
-
-        * It's ``section`` attribute is automatically set to 'output';
+    r"""Declare one or more output resources.
+
+    This function takes an abitrary of keyword arguments forming the resource
+    description.
+
+    :return: A list of :py:class:`Handler <vortex.data.handlers.Handler>` objects.
+
+    **Example:**
+
+    The following call to ``output`` returns a list of 6
+    :py:class:`Handler <vortex.data.handlers.Handler>` objects, one
+    for each date and member:
+
+    .. code:: python
+
+       rhandlers = vortex.output(
+           kind='gridpoint',
+           term=1,
+           geometry='eurw1s40',
+           nativefmt='grib',
+           model='arome',
+           cutoff='production',
+           date=['2024060121', '2024060122'],
+           origin='historic',
+           vapp='arome',
+           vconf='pefrance',
+           member=[1,2,5],
+           experiment='myexp',
+           block='forecast',
+           local='gribfile_[member].grib',
+           format='grib',
+       )
 
-    :return: A list of :class:`vortex.data.handlers.Handler` objects (associated
-        with the newly created class:`~vortex.layout.dataflow.Section` objects).
     """
     # Strip the metadatacheck option depending on active_metadatacheck
     if not active_promise:
@@ -607,15 +656,26 @@ def promise(*args, **kw):
 
 
 def executable(*args, **kw):
-    """Add executable :class:`~vortex.layout.dataflow.Section` objects to the current sequence.
+    r"""Declare one or more executable resources.
 
-    Relies on the :func:`add_section` function (see its documentation), with:
+    This function takes an abitrary of keyword arguments forming the
+    executable resource description.
 
-        * It's ``section`` attribute is automatically set to 'executable';
-        * The ``kw``'s *insitu* item is set to :data:`active_insitu` by default.
+    :return: A list of :py:class:`Handler <vortex.data.handlers.Handler>` objects.
+
+    **Example:**
+
+    The following call to ``input`` returns a list of one
+    :py:class:`Handler <vortex.data.handlers.Handler>` object:
+
+    .. code:: python
+
+       rhandlers = vortex.executable(
+           kind="mfmodel",
+           local="ARPEGE",
+           remote="/path/to/binaries/ARPEGE.EX",
+       )
 
-    :return: A list of :class:`vortex.data.handlers.Handler` objects (associated
-        with the newly created class:`~vortex.layout.dataflow.Section` objects).
     """
     kw.setdefault("insitu", active_insitu)
     return add_section("executable", args, kw)

vortex/tools/actions.py

@@ -10,8 +10,6 @@ to be processed: e.g. mail, routing, alarm.
 import bronx.stdtypes.catalog
 import footprints
 from bronx.fancies import loggers
-from bronx.fancies.display import dict_as_str
-from vortex import sessions
 
 #: Export nothing
 __all__ = []
@@ -134,105 +132,30 @@ class Action:
 class TunableAction(Action):
     """An Action that may be tuned
 
-    - may have it's own section in the target configuration files
-    - accepts the syntax `ad.action_tune(key=value)` (which has priority)
+    accepts the syntax `ad.action_tune(key=value)` (which has priority)
     """
 
     def __init__(self, configuration=None, **kwargs):
         super().__init__(**kwargs)
         self._tuning = dict()
-        self._conf_section = configuration
-        self._conf_dict = None
-
-    @property
-    def _shtarget(self):
-        """Warning: this may be a `vortex.syntax.stdattrs.DelayedInit` object
-        during Vortex initialization and may not have a `sections()` method
-        nor a `config` property.
-        """
-        return sessions.current().sh.default_target
-
-    @property
-    def _conf_items(self):
-        """Check and return the configuration: a section in the target-xxx.ini file.
-
-        If the configuration is None, an attempt is made to use the Action's kind.
-        Don't use before Vortex initialization is done (see `_shtarget`).
-        """
-        if self._conf_dict is None:
-            if self._conf_section is None:
-                if self.kind in self._shtarget.sections():
-                    self._conf_section = self.kind
-            else:
-                if self._conf_section not in self._shtarget.sections():
-                    raise KeyError(
-                        'No section "{}" in "{}"'.format(
-                            self._conf_section, self._shtarget.config.file
-                        )
-                    )
-            if self._conf_section is None:
-                self._conf_dict = dict()
-            else:
-                self._conf_dict = self._shtarget.items(self._conf_section)
-        return self._conf_dict
 
     def service_info(self, **kw):
-        for k, v in self._get_config_dict().items():
+        for k, v in self._tuning.items():
             kw.setdefault(k, v)
         return super().service_info(**kw)
 
-    def tune(self, section=None, **kw):
-        """Add options to override the .ini file configuration.
-
-        ``section`` is a specific section name, or ``None`` for all.
-        """
-        if section is None or section == self._conf_section:
-            self._tuning.update(kw)
-
-    def _get_config_dict(self):
-        final_dict = dict()
-        final_dict.update(self._conf_items)
-        final_dict.update(self._tuning)
-        return final_dict
-
-    def info(self):
-        """Informative string (may serve debugging purposes)."""
-        s = super().info() + " - tunable\n"
-        mix = dict()
-        mix.update(self._conf_items)
-        mix.update(self._tuning)
-        prt = dict()
-        for k, v in mix.items():
-            if k in self._tuning:
-                prt["++ " + k] = "{} (was: {})".format(
-                    v,
-                    str(self._conf_items[k])
-                    if k in self._conf_items
-                    else "<not set>",
-                )
-            else:
-                prt["   " + k] = v
-        if self._conf_section is not None:
-            s += " " * 4 + "configuration: " + self._conf_section + "\n"
-        s += dict_as_str(prt, prefix=4)
-        return s.strip()
+    def tune(self, **kw):
+        self._tuning.update(kw)
 
     def getx(self, key, *args, **kw):
         """Shortcut to access the configuration overridden by the tuning."""
         if key in self._tuning:
             return self._tuning[key]
 
-        if self._conf_section is not None:
-            return self._shtarget.getx(
-                key=self._conf_section + ":" + key, *args, **kw
-            )
-
         if "default" in kw:
             return kw["default"]
 
-        raise KeyError(
-            'The "{:s}" entry was not found in any configuration'.format(key)
-        )
+        raise KeyError('The "{:s}" entry was not found'.format(key))
 
 
 class SendMail(Action):

vortex/tools/grib.py

@@ -10,6 +10,7 @@ It also provdes an AlgoComponent's Mixin to properly setup the environment
 when using the grib_api or ecCodes libraries.
 """
 
+from pathlib import Path
 from urllib import parse as urlparse
 
 import re
@@ -563,37 +564,41 @@ class EcGribDecoMixin(AlgoComponentDecoMixin):
                 "After gribapi_setup %s = %s", a_var, self.env.getvar(a_var)
             )
 
-    def _eccodes_envsetup(self, eccodes_lib):
-        """Setup environment variables for ecCodes."""
-        dep_warn = (
-            "%s is left unconfigured because the old grib_api's variable is defined."
-            + "Please remove that !"
+    def _eccodes_envsetup(
+        self,
+        eccodes_lib,
+        envvar="ECCODES_DEFINITIONS_PATH",
+        tgt_path="definitions",
+    ):
+        """Export envirionment variables required by ECCODES
+
+        Value is
+
+        /path/to/eccodes-X.Y.Z/share/eccodes/<target_path>
+
+        eccodes_lib: Absolute path to the eccodes so file
+        envvar: Name of the environment variable to export
+        tgt_path: Name of the eccodes install subdirectory to appear
+          in the value
+        """
+        if envvar in self.env:
+            return envvar
+        if envvar.replace("ECCODES", "GRIB") in self.env:
+            logger.warning(
+                (
+                    "%s is left unconfigured because the old grib_api's"
+                    "variable is defined. ",
+                    "Please remove that!",
+                ),
+                envvar,
+            )
+            return envvar.replace("ECCODES", "GRIB")
+        eccodes_root = Path(eccodes_lib).parent.parent
+        self.env.setgenericpath(
+            envvar,
+            str(eccodes_root / "share" / "eccodes" / tgt_path),
         )
-        eccodes_root = self.system.path.dirname(eccodes_lib)
-        eccodes_root = self.system.path.split(eccodes_root)[0]
-        eccodes_share = self.system.path.join(eccodes_root, "share", "eccodes")
-        defvar = "ECCODES_DEFINITION_PATH"
-        if defvar not in self.env:
-            if "GRIB_DEFINITION_PATH" in self.env:
-                logger.warning(dep_warn, defvar)
-                defvar = "GRIB_DEFINITION_PATH"
-            else:
-                self.env.setgenericpath(
-                    defvar, self.system.path.join(eccodes_share, "definitions")
-                )
-        samplevar = "ECCODES_SAMPLES_PATH"
-        if samplevar not in self.env:
-            if "GRIB_SAMPLES_PATH" in self.env:
-                logger.warning(dep_warn, samplevar)
-                samplevar = "GRIB_SAMPLES_PATH"
-            else:
-                self.env.setgenericpath(
-                    samplevar,
-                    self.system.path.join(
-                        eccodes_share, "ifs_samples", "grib1"
-                    ),
-                )
-        return defvar, samplevar
+        return envvar
 
     def eccodes_setup(self, rh, opts, compat=False, fatal=True):
         """Setup the grib_api related stuff.
@@ -605,7 +610,19 @@ class EcGribDecoMixin(AlgoComponentDecoMixin):
         # Detect the library's path and setup appropriate variables
         eccodes_lib, gribapi_lib = self._ecgrib_libs_detext(rh)
         if eccodes_lib is not None:
-            defvar, samplevar = self._eccodes_envsetup(eccodes_lib)
+            defvar = self._eccodes_envsetup(
+                eccodes_lib,
+                envvar="ECCODES_DEFINITIONS_PATH",
+                tgt_path="definitions",
+            )
+            subdir = Path("ifs_samples") / (
+                "grib1" if rh.resource.cycle < "cy49" else "grib1_mlgrib2"
+            )
+            samplevar = self._eccodes_envsetup(
+                eccodes_lib,
+                envvar="ECCODES_SAMPLES_PATH",
+                tgt_path=subdir,
+            )
         elif compat:
             defvar, samplevar = self._gribapi_envsetup(gribapi_lib)
         else:

vortex/tools/net.py

@@ -26,6 +26,8 @@ from bronx.fancies import loggers
 from bronx.net.netrc import netrc
 from bronx.syntax.decorators import nicedeco, secure_getattr
 
+from vortex.config import get_from_config_w_default, ConfigurationError
+
 #: No automatic export
 __all__ = []
 
@@ -1192,15 +1194,35 @@ class Ssh:
         self._logname = logname
         self._remote = hostname
 
-        target = sh.default_target
-        self._sshcmd = target.get(key="services:sshcmd", default="ssh")
-        self._scpcmd = target.get(key="services:scpcmd", default="scp")
+        def _get_ssh_config(key, default):
+            config = get_from_config_w_default(
+                section="ssh", key=key, default=default
+            )
+            try:
+                val = config.pop("default")
+            except AttributeError:
+                assert isinstance(config, str)
+                return config
+            except KeyError:
+                msg = (
+                    "A default value must be specified for configuration option"
+                    f" {key}. See vortex-nwp.readthedocs.io/en/latest/user-guide/configuration.html#ssh"
+                )
+                raise ConfigurationError(msg)
+
+            for k, v in config.items():
+                if re.match(k, socket.gethostname()):
+                    val = v
+            return val
+
+        self._sshcmd = _get_ssh_config(key="sshcmd", default="ssh")
+        self._scpcmd = _get_ssh_config(key="scpcmd", default="scp")
         self._sshopts = (
-            target.get(key="services:sshopts", default="-x").split()
+            _get_ssh_config(key="sshopts", default="").split()
             + (sshopts or "").split()
         )
         self._scpopts = (
-            target.get(key="services:scpopts", default="-Bp").split()
+            _get_ssh_config(key="scpopts", default="").split()
             + (scpopts or "").split()
         )
 

vortex/tools/services.py

@@ -20,7 +20,6 @@ from bronx.stdtypes.dictionaries import UpperCaseDict
 from bronx.syntax.pretty import EncodedPrettyPrinter
 from vortex import sessions
 from vortex.util.config import (
-    GenericConfigParser,
     load_template,
     LegacyTemplatingAdapter,
 )
@@ -563,7 +562,8 @@ class Directory:
 
     def __init__(self, inifile, domain="meteo.fr", encoding=None):
         """Keep aliases in memory, as a dict of sets."""
-        config = GenericConfigParser(inifile, encoding=encoding)
+        config = configparser.ConfigParser()
+        config.read(inifile, encoding=encoding)
         try:
             self.domain = config.get("general", "default_domain")
         except configparser.NoOptionError:

vortex/tools/storage.py

@@ -636,9 +636,11 @@ class Archive(AbstractArchive):
 
     def __init__(self, *kargs, **kwargs):
         super().__init__(*kargs, **kwargs)
-        self.default_usejeeves = config.from_config(
+
+        self.default_usejeeves = config.get_from_config_w_default(
             section="storage",
             key="usejeeves",
+            default=False,
         )
 
     @property

vortex/tools/systems.py

@@ -68,6 +68,7 @@ from vortex.tools.env import Environment
 from vortex.tools.net import AssistedSsh, AutoRetriesFtp, DEFAULT_FTP_PORT
 from vortex.tools.net import FtpConnectionPool, LinuxNetstats, StdFtp
 import vortex.tools.storage
+from vortex import config
 
 #: No automatic export
 __all__ = []
@@ -800,7 +801,7 @@ class OSExtended(System):
 
             * **rmtreemin** - as the minimal depth needed for a :meth:`rmsafe`.
             * **cmpaftercp** - as a boolean for activating full comparison after plain cp (default: *True*).
-            * **ftraw** - allows ``smartft*`` methods to use the raw FTP commands
+            * **ftserv** - allows ``smartft*`` methods to use the raw FTP commands
               (e.g. ftget, ftput) instead of the internal Vortex's FTP client
               (default: *False*).
             * **ftputcmd** - The name of the raw FTP command for the "put" action
@@ -815,7 +816,7 @@ class OSExtended(System):
         self._rmtreemin = kw.pop("rmtreemin", 3)
         self._cmpaftercp = kw.pop("cmpaftercp", True)
         # Switches for rawft* methods
-        self._ftraw = kw.pop("ftraw", None)
+        self._ftserv = kw.pop("ftserv", None)
         self.ftputcmd = kw.pop("ftputcmd", None)
         self.ftgetcmd = kw.pop("ftgetcmd", None)
         # FTP stuff again
@@ -838,23 +839,33 @@ class OSExtended(System):
         self._signal_intercept_init()
 
     @property
-    def ftraw(self):
+    def ftserv(self):
         """Use the system's FTP service (e.g. ftserv)."""
-        if self._ftraw is None:
-            return self.default_target.ftraw_default
+        if self._ftserv is None:
+            return self._use_ftserv()
         else:
-            return self._ftraw
+            return self._ftserv
 
-    @ftraw.setter
-    def ftraw(self, value):
+    @ftserv.setter
+    def ftserv(self, value):
         """Use the system's FTP service (e.g. ftserv)."""
         self._ftraw = bool(value)
 
-    @ftraw.deleter
-    def ftraw(self):
+    @ftserv.deleter
+    def ftserv(self):
         """Use the system's FTP service (e.g. ftserv)."""
         self._ftraw = None
 
+    def _use_ftserv(self):
+        if not config.is_defined(section="ftserv"):
+            return False
+        for rgxp in config.from_config(
+            section="ftserv", key="hostname_patterns"
+        ):
+            if re.match(rgxp, self.hostname):
+                return True
+        return False
+
     def target(self, **kw):
         """
         Provide a default :class:`~vortex.tools.targets.Target` according
@@ -2064,7 +2075,7 @@ class OSExtended(System):
 
     def rawftput_worthy(self, source, destination):
         """Is it allowed to use FtServ given **source** and **destination**."""
-        return self.ftraw and self.ftserv_allowed(source, destination)
+        return self.ftserv and self.ftserv_allowed(source, destination)
 
     @fmtshcmd
     def rawftput(
@@ -2145,7 +2156,7 @@ class OSExtended(System):
 
         ``rawftput`` will be used if all of the following conditions are met:
 
-            * ``self.ftraw`` is *True*
+            * ``self.ftserv`` is *True*
             * **source** is a string (as opposed to a File like object)
             * **destination** is a string (as opposed to a File like object)
         """
@@ -2177,7 +2188,7 @@ class OSExtended(System):
     def rawftget_worthy(self, source, destination, cpipeline=None):
         """Is it allowed to use FtServ given **source** and **destination**."""
         return (
-            self.ftraw
+            self.ftserv
             and cpipeline is None
             and self.ftserv_allowed(source, destination)
         )
@@ -2259,7 +2270,7 @@ class OSExtended(System):
 
         ``rawftget`` will be used if all of the following conditions are met:
 
-            * ``self.ftraw`` is *True*
+            * ``self.ftserv`` is *True*
             * **cpipeline** is None
             * **source** is a string (as opposed to a File like object)
             * **destination** is a string (as opposed to a File like object)

vortex/util/config.py

@@ -164,7 +164,8 @@ def load_template(tplpath, encoding=None, default_templating="legacy"):
     """
     tplpath = Path(tplpath).absolute()
     if not tplpath.exists():
-        raise FileNotFoundError
+        msg = f"Template file {tplpath} not found"
+        raise FileNotFoundError(msg)
     ignored_lines = set()
     actual_encoding = None if encoding == "script" else encoding
     actual_templating = default_templating

{vortex_nwp-2.0.0b2.dist-info → vortex_nwp-2.1.1.dist-info}/METADATA

@@ -1,6 +1,6 @@
-Metadata-Version: 2.2
+Metadata-Version: 2.4
 Name: vortex-nwp
-Version: 2.0.0b2
+Version: 2.1.1
 Summary: A Python library to write Numerical Weather Prediction pipelines components
 Author-email: The Vortex Team <vortex.support@meteo.fr>
 License: CECILL-C
@@ -21,6 +21,7 @@ Requires-Dist: sphinx-copybutton; extra == "docs"
 Provides-Extra: dev
 Requires-Dist: ruff==0.9.1; extra == "dev"
 Requires-Dist: pytest; extra == "dev"
+Dynamic: license-file
 
 ## vortex