vortex-nwp 2.0.0__py3-none-any.whl

vortex/nwp/syntax/stdattrs.py

@@ -0,0 +1,437 @@
+"""
+This module provides some pre-defined attributes descriptions or combined sets
+of attributes description that could be used in the footprint definition of any
+class which follow the :class:`footprints.Footprint` syntax.
+"""
+
+import re
+from functools import total_ordering
+
+import footprints
+from bronx.stdtypes.date import Time
+
+from vortex.syntax.stddeco import namebuilding_append
+
+#: Export some new class for attributes in footprint objects, eg : GenvKey
+__all__ = ["GenvKey", "GenvDomain"]
+
+#: Usual Footprint for a single member (in an algo Component)
+a_algo_member = dict(
+    info=(
+        "The current member's number "
+        + "(may be omitted in deterministic configurations)."
+    ),
+    optional=True,
+    type=int,
+)
+
+#: Usual Footprint of the ``outputid`` attribute.
+algo_member = footprints.Footprint(attr=dict(member=a_algo_member))
+
+#: Known OOPS testcomponent ``run``
+known_oops_testcomponent_runs = ["ootestcomponent", "testcomponent", "testvar"]
+
+#: Usual definition of the ``run`` attribute for OOPS binaries.
+a_oops_run = dict(
+    info="The OOPS run (== task).",
+    optional=False,
+)
+#: Usual Footprint of the ``run`` attribute for OOPS binaries.
+oops_run = footprints.Footprint(
+    info="OOPS kind of run", attr=dict(run=a_oops_run)
+)
+
+#: Usual definition of the ``test_type`` attribute.
+a_oops_test_type = dict(
+    info="Sub-test or family of sub-tests to be ran.",
+    optional=False,
+)
+#: Usual Footprint of the ``test_type`` attribute.
+oops_test_type = footprints.Footprint(
+    info="OOPS type of test", attr=dict(test_type=a_oops_test_type)
+)
+
+#: Usual definition of the ``expected_target`` attribute.
+an_oops_expected_target = dict(
+    info=("Expected target for the test success"),
+    type=footprints.FPDict,
+    optional=True,
+    default=None,
+)
+#: Usual Footprint of the ``expected_target`` attribute.
+oops_expected_target = footprints.Footprint(
+    attr=dict(expected_target=an_oops_expected_target)
+)
+
+#: Usual Footprint of a combined lists of members and terms
+oops_members_terms_lists = footprints.Footprint(
+    info="Abstract footprint for a members/terms list.",
+    attr=dict(
+        members=dict(
+            info="A list of members.",
+            type=footprints.FPList,
+        ),
+        terms=dict(
+            info="A list of effective terms.",
+            type=footprints.FPList,
+            optional=True,
+            default=footprints.FPList(
+                [
+                    Time(0),
+                ]
+            ),
+        ),
+    ),
+)
+
+#: Usual definition of the ``outputid`` attribute
+a_outputid = dict(
+    info="The identifier for the encoding of post-processed fields.",
+    optional=True,
+)
+
+#: Usual Footprint of the ``outputid`` attribute.
+outputid = footprints.Footprint(attr=dict(outputid=a_outputid))
+
+
+def _apply_outputid(cls):
+    """Decorator that tweak the class in order to add OUTPUTID on the namelist"""
+    orig_pnd = getattr(cls, "prepare_namelist_delta", None)
+    if orig_pnd is None:
+        raise ImportError(
+            "_apply_outputid can not be applied on {!s}".format(cls)
+        )
+
+    def prepare_namelist_delta(self, rh, namcontents, namlocal):
+        namw = orig_pnd(self, rh, namcontents, namlocal)
+        if self.outputid is not None and any(
+            ["OUTPUTID" in nam_b.macros() for nam_b in namcontents.values()]
+        ):
+            self._set_nam_macro(
+                namcontents, namlocal, "OUTPUTID", self.outputid
+            )
+            namw = True
+        return namw
+
+    cls.prepare_namelist_delta = prepare_namelist_delta
+    return cls
+
+
+#: Decorated footprint for the ``outputid`` attribute
+outputid_deco = footprints.DecorativeFootprint(
+    outputid,
+    decorator=[
+        _apply_outputid,
+    ],
+)
+
+
+def show():
+    """Returns available items and their type."""
+    dmod = globals()
+    for stda in sorted(
+        filter(
+            lambda x: x.startswith("a_")
+            or isinstance(dmod[x], footprints.Footprint),
+            dmod.keys(),
+        )
+    ):
+        print(
+            "{} ( {} ) :\n  {}\n".format(
+                stda, type(dmod[stda]).__name__, dmod[stda]
+            )
+        )
+
+
+domain_remap = dict()
+
+
+def _lowerattr(matchobj):
+    """Internal and technical function returning lower case value of the complete match item."""
+    return matchobj.group(0).lower()
+
+
+class GenvKey(str):
+    """
+    Attribute for a GEnv cycle name.
+    Implicit attributes inside brackets are translated to lower case.
+    See also :mod:`vortex_gco.tools.genv`.
+    """
+
+    def __new__(cls, value):
+        """Proxy to ``str.__new___`` with attributes inside brackets translated to lower case."""
+        return str.__new__(cls, re.sub(r"\[\w+\]", _lowerattr, value.upper()))
+
+
+a_gvar = dict(
+    info="The key that identifies the resource in the Genv database.",
+    type=GenvKey,
+    optional=True,
+    doc_visibility=footprints.doc.visibility.ADVANCED,
+)
+
+#: Usual definition of the ``genv`` attribute.
+gvar = footprints.Footprint(info="A GENV access key", attr=dict(gvar=a_gvar))
+
+
+class GenvDomain(str):
+    """
+    Remap plain area names to specific Genv short domain names.
+    See also :mod:`vortex_gco.tools.genv`.
+    """
+
+    def __new__(cls, value):
+        """Proxy to ``str.__new___`` with on the fly remapping of domain names to short values."""
+        return str.__new__(cls, domain_remap.get(value, value))
+
+
+a_gdomain = dict(
+    info="The resource's geographical domain name in the Genv database.",
+    type=GenvDomain,
+    optional=True,
+    default="[geometry::area]",
+    doc_visibility=footprints.doc.visibility.ADVANCED,
+)
+
+#: Usual definition of the ``gdomain`` attribute.
+gdomain = footprints.Footprint(
+    info="A domain name in GCO convention", attr=dict(gdomain=a_gdomain)
+)
+
+
+@total_ordering
+class ArpIfsSimplifiedCycle:
+    """
+    Type that holds a simplified representation of an ArpegeIFS cycle.
+
+    It provides basic comparison operators to determine if a given cycle is more recent or not
+    compared to another one.
+
+    It can be used in a footprint specification.
+    """
+
+    _cy_re = re.compile(
+        r"(?:u(?:env|get):)?(?:cy|al)(\d+)(?:t(\d{1,3}))?(?=_|@|\.|$)(?:.*?(?:[_-]op(\d{1,3})))?"
+    )
+    _hash_shift = 10000
+
+    def __init__(self, cyclestr):
+        cy_match = self._cy_re.match(cyclestr)
+        if cy_match:
+            self._number = int(cy_match.group(1))
+            self._toulouse = (
+                int(cy_match.group(2)) + 1
+                if cy_match.group(2) is not None
+                else 0
+            )
+            self._op = (
+                int(cy_match.group(3)) + 1
+                if cy_match.group(3) is not None
+                else 0
+            )
+        else:
+            raise ValueError("Malformed cycle: {}".format(cyclestr))
+
+    def __hash__(self):
+        return (
+            self._number * self._hash_shift + self._toulouse
+        ) * self._hash_shift + self._op
+
+    def __eq__(self, other):
+        if not isinstance(other, ArpIfsSimplifiedCycle):
+            try:
+                other = ArpIfsSimplifiedCycle(other)
+            except (ValueError, TypeError):
+                return False
+        return hash(self) == hash(other)
+
+    def __gt__(self, other):
+        if not isinstance(other, ArpIfsSimplifiedCycle):
+            other = ArpIfsSimplifiedCycle(other)
+        return hash(self) > hash(other)
+
+    def __str__(self):
+        return (
+            "cy{:d}".format(self._number)
+            + ("t{:d}".format(self._toulouse - 1) if self._toulouse else "")
+            + ("_op{:d}".format(self._op - 1) if self._op else "")
+        )
+
+    def __repr__(self):
+        return "<{} | {!s}>".format(
+            object.__repr__(self).lstrip("<").rstrip(">"), self
+        )
+
+    def export_dict(self):
+        """The pure dict/json output is the raw integer"""
+        return str(self)
+
+
+a_arpifs_cycle = dict(
+    info="An Arpege/IFS cycle name",
+    type=ArpIfsSimplifiedCycle,
+    optional=True,
+    default="cy40",  # For "old" Olive configurations to keep working
+)
+
+#: Usual definition of the ``cycle`` attribute.
+arpifs_cycle = footprints.Footprint(
+    info="An abstract arpifs_cycle in GCO convention",
+    attr=dict(cycle=a_arpifs_cycle),
+)
+
+uget_sloppy_id_regex = re.compile(
+    r"(?P<shortuget>(?P<id>\S+)@(?P<location>[-\w]+))"
+)
+uget_id_regex = (
+    r"(?P<fulluget>u(?:get|env):" + uget_sloppy_id_regex.pattern + ")"
+)
+uget_id_regex_only = re.compile("^" + uget_id_regex + "$")
+uget_id_regex = re.compile(r"\b" + uget_id_regex + r"\b")
+
+
+class GgetId(str):
+    """Basestring wrapper for Gget Ids."""
+
+    def __new__(cls, value):
+        if uget_id_regex_only.match(value):
+            raise ValueError("A GgetId cannot look like a UgetId !")
+        return str.__new__(cls, value)
+
+
+class AbstractUgetId(str):
+    """Basestring wrapper for Uget Ids."""
+
+    _ALLOWED_LOCATIONS = ()
+    _OUTCAST_LOCATIONS = ()
+
+    def __new__(cls, value):
+        vmatch = uget_id_regex_only.match(value)
+        if not vmatch:
+            raise ValueError('Invalid UgetId (got "{:s}")'.format(value))
+        me = str.__new__(cls, value)
+        me._id = vmatch.group("id")
+        me._location = vmatch.group("location")
+        if me._location in set(cls._OUTCAST_LOCATIONS):
+            raise ValueError(
+                'Invalid UgetId (got "{:s}"). Outcast Location.'.format(value)
+            )
+        if cls._ALLOWED_LOCATIONS and me._location not in set(
+            cls._ALLOWED_LOCATIONS
+        ):
+            raise ValueError(
+                'Invalid UgetId (got "{:s}"). Disallowed location'.format(
+                    value
+                )
+            )
+        return me
+
+    @property
+    def id(self):
+        return self._id
+
+    @property
+    def location(self):
+        return self._location
+
+    @property
+    def short(self):
+        return self._id + "@" + self._location
+
+    def monthlyshort(self, month):
+        return self._id + ".m{:02d}".format(month) + "@" + self._location
+
+
+class UgetId(AbstractUgetId):
+    _OUTCAST_LOCATIONS = ("demo",)
+
+
+def genv_ifs_compiler_convention(cls):
+    """Add the necessary method to handle compiler version/option in Genv."""
+    original_gget_basename = getattr(cls, "gget_basename", None)
+    if original_gget_basename is not None:
+
+        def gget_basename(self):
+            """GGET specific naming convention."""
+            b_dict = original_gget_basename(self)
+            if getattr(self, "compiler_version", None):
+                b_dict["compiler_version"] = self.compiler_version
+            if getattr(self, "compiler_option", None):
+                b_dict["compiler_option"] = self.compiler_option
+            if getattr(self, "cycle", None):
+                b_dict["cycle"] = self.cycle
+            return b_dict
+
+        cls.gget_basename = gget_basename
+    return cls
+
+
+#: Usual definition of the ``compiler_version`` and ``compiler_option`` attributes.
+gmkpack_compiler_identification = footprints.Footprint(
+    info="Add the compiler version/option in the footprint",
+    attr=dict(
+        compiler_version=dict(
+            info="The compiler version in gmkpack convention.", optional=True
+        ),
+        compiler_option=dict(
+            info="The compiler option in gmkpack convention.", optional=True
+        ),
+    ),
+)
+
+
+#: Usual definition of the ``compiler_version`` and ``compiler_option`` attributes + genv integration.
+gmkpack_compiler_identification_deco = footprints.DecorativeFootprint(
+    gmkpack_compiler_identification,
+    decorator=[
+        genv_ifs_compiler_convention,
+    ],
+)
+
+
+def genv_executable_flavour(cls):
+    """Add the necessary method to the "flavour" in Genv."""
+    original_genv_basename = getattr(cls, "genv_basename", None)
+    if original_genv_basename is not None:
+
+        def genv_basename(self):
+            """Just retrieve a potential gvar attribute."""
+            gvar = original_genv_basename(self)
+            if getattr(self, "flavour", None):
+                gvar += (
+                    "_"
+                    + {"singleprecision": "SP"}.get(
+                        self.flavour, self.flavour
+                    ).upper()
+                )
+            return gvar
+
+        cls.genv_basename = genv_basename
+    return cls
+
+
+#: Usual definition of the ``flavour`` attribute.
+executable_flavour = footprints.Footprint(
+    info="Add the executable flavour attribute to the resource",
+    attr=dict(
+        flavour=dict(
+            info="The executable flavour (This may influence the Genv's key choice).",
+            values=[
+                "singleprecision",
+            ],
+            optional=True,
+        ),
+    ),
+)
+
+
+#: Usual definition of the ``flavour``.
+executable_flavour_deco = footprints.DecorativeFootprint(
+    executable_flavour,
+    decorator=[
+        genv_executable_flavour,
+        namebuilding_append(
+            "src", lambda self: [self.flavour], none_discard=True
+        ),
+    ],
+)

vortex/nwp/tools/__init__.py

@@ -0,0 +1,10 @@
+"""
+Standalone tools for NWP
+"""
+
+# Recursive inclusion of packages with potential FootprintBase classes
+from . import conftools as conftools
+from . import ifstools as ifstools
+
+#: Automatic export of data subpackage
+__all__ = []

vortex/nwp/tools/addons.py

@@ -0,0 +1,40 @@
+"""
+Various System needed for NWP applications.
+"""
+
+from vortex.tools.addons import AddonGroup
+
+# Import the proper Addon modules for footprints
+from vortex.tools import folder as folder
+from vortex.tools import lfi as lfi
+from vortex.tools import grib as grib
+from vortex.tools import listings as listings
+from vortex.tools import surfex as surfex
+
+#: No automatic export
+__all__ = []
+
+
+class NWPAddonsGroup(AddonGroup):
+    """A set of usual NWP Addons."""
+
+    _footprint = dict(
+        info="Default NWP Addons",
+        attr=dict(
+            kind=dict(
+                values=[
+                    "nwp",
+                ],
+            ),
+        ),
+    )
+
+    _addonslist = (
+        "allfolders",  # Folder like...
+        "lfi",
+        "iopoll",  # Wonderful FA/LFI world...
+        "grib",
+        "gribapi",  # GRIB stuff...
+        "arpifs_listings",  # Obscure IFS/Arpege listings...
+        "sfx",  # Surfex...
+    )

vortex/nwp/tools/agt.py

@@ -0,0 +1,67 @@
+"""
+TODO: Module documentation.
+"""
+
+import collections
+
+
+class AgtConfigurationError(Exception):
+    """Specific Transfer Agent configuration error."""
+
+    pass
+
+
+def agt_volatile_path(sh):
+    """Prefix path to use for the Transfer Agent to consider a file as being "volatile"
+    and hard-linking to it instead of expecting it to be present "later".
+    """
+    config = sh.default_target.config
+    if not config.has_section("agt"):
+        fmt = 'Missing section "agt" in configuration file\n"{}"'
+        raise AgtConfigurationError(fmt.format(config.file))
+    return config.get("agt", "agt_volatile")
+
+
+def agt_actual_command(sh, binary_name, args, extraenv=None):
+    """Build the command able to execute a Transfer Agent binary.
+
+    The context, the execution path and the command name are
+    provided by the configuration file of the target.
+
+    The resulting command should be executed on a transfer node.
+
+    :param sh: The vortex shell that will be used
+    :param binary_name: Key in the configuration file that holds the binary name
+    :param args: Argument to the AGT binary
+    :param extraenv: Additional environment variables to export (dictionary)
+    """
+    config = sh.default_target.config
+    if not config.has_section("agt"):
+        fmt = 'Missing section "agt" in configuration file\n"{}"'
+        raise AgtConfigurationError(fmt.format(config.file))
+
+    agt_path = sh.default_target.get("agt_path", default=None)
+    agt_bin = sh.default_target.get(binary_name, default=None)
+    if not all([agt_path, agt_bin]):
+        fmt = 'Missing key "agt_path" or "{}" in configuration file\n"{}"'
+        raise AgtConfigurationError(fmt.format(binary_name, config.file))
+    cfgkeys = [
+        "HOME_SOPRA",
+        "LD_LIBRARY_PATH",
+        "base_transfert_agent",
+        "DIAP_AGENT_NUMPROG_AGENT",
+    ]
+    context = " ; ".join(
+        ["export {}={}".format(key, config.get("agt", key)) for key in cfgkeys]
+    )
+    if extraenv is not None and isinstance(extraenv, collections.abc.Mapping):
+        context = " ; ".join(
+            [
+                context,
+            ]
+            + [
+                "export {}={}".format(key.upper(), value)
+                for (key, value) in extraenv.items()
+            ]
+        )
+    return "{} ; {} {}".format(context, sh.path.join(agt_path, agt_bin), args)

vortex/nwp/tools/bdap.py

@@ -0,0 +1,59 @@
+"""
+Utility classes and function to work with the BDAP database.
+"""
+
+#: No automatic export
+__all__ = []
+
+
+class BDAPError(Exception):
+    """General BDAP error."""
+
+    pass
+
+
+class BDAPRequestConfigurationError(BDAPError):
+    """Specific Transfer Agent configuration error."""
+
+    pass
+
+
+class BDAPGetError(BDAPError):
+    """Generic BDAP get error."""
+
+    pass
+
+
+def BDAPrequest_actual_command(command, date, term, query, int_extraenv=False):
+    """Build the command able to execute a BDAP request.
+
+    The context, the execution path and the command name are
+    provided by the configuration file of the target.
+
+    The resulting command should be executed on a transfer node.
+
+    :param command: name of the BDAP request command to be used
+    :param date: the date of the file requested
+    :param term: the term of the file requested
+    :param query: the query file used for the request
+    :param int_extraenv: boolean to know if the integration BDAP is used or not
+                         (an additional environment variable has to be exported in this case).
+    """
+
+    # Environment variable to specify the date of the file
+    context = " ; ".join(
+        ["export {}={}".format("dmt_date_pivot".upper(), date.ymdhms)]
+    )
+    # Extra environment variables (integration BDAP)
+    if int_extraenv:
+        context = " ; ".join(
+            [context]
+            + [
+                "export {}={}".format(
+                    "db_file_bdap".upper(),
+                    "/usr/local/sopra/neons_db_bdap.intgr",
+                )
+            ]
+        )
+    # Return the command to be launched
+    return "{} ; {} {:d} {}".format(context, command, term.hour, query)

vortex/nwp/tools/bdcp.py

@@ -0,0 +1,41 @@
+"""
+Utility classes and function to work with the BDCP database.
+"""
+
+#: No automatic export
+__all__ = []
+
+
+class BDCPError(Exception):
+    """General BDMP error."""
+
+    pass
+
+
+class BDCPRequestConfigurationError(BDCPError):
+    """Specific Transfer Agent configuration error."""
+
+    pass
+
+
+class BDCPGetError(BDCPError):
+    """Generic BDCP get error."""
+
+    pass
+
+
+def BDCPrequest_actual_command(command, query_file, output_file):
+    """Build the command able to execute a BDCP request.
+
+    The context, the execution path and the command name are
+    provided by the configuration file of the target.
+
+    The resulting command should be executed on a transfer node.
+
+    :param command: name of the BDMP request command to be used
+    :param query_file: the query file used for the request
+    :param output_file: the file in which the result will be written
+    """
+
+    # Return the command to be launched
+    return "{} -D {} -f {}".format(command, query_file, output_file)

vortex/nwp/tools/bdm.py

@@ -0,0 +1,24 @@
+"""
+Utility classes and function to work with the BDM database.
+"""
+
+#: No automatic export
+__all__ = []
+
+
+class BDMError(Exception):
+    """General BDM error."""
+
+    pass
+
+
+class BDMRequestConfigurationError(BDMError):
+    """Specific Transfer Agent configuration error."""
+
+    pass
+
+
+class BDMGetError(BDMError):
+    """Generic BDM get error."""
+
+    pass