vortex-nwp 2.0.0b1__py3-none-any.whl

vortex/nwp/syntax/stdattrs.py

@@ -0,0 +1,375 @@
+"""
+This module provides some pre-defined attributes descriptions or combined sets
+of attributes description that could be used in the footprint definition of any
+class which follow the :class:`footprints.Footprint` syntax.
+"""
+
+from bronx.stdtypes.date import Time
+import footprints
+
+#: Export a set of attributes :data:`a_run`, etc..
+__all__ = []
+
+#: Usual Footprint for a single member (in an algo Component)
+a_algo_member = dict(
+    info=("The current member's number " +
+          "(may be omitted in deterministic configurations)."),
+    optional=True,
+    type=int
+)
+
+#: Usual Footprint of the ``outputid`` attribute.
+algo_member = footprints.Footprint(attr=dict(member=a_algo_member))
+
+#: Known OOPS testcomponent ``run``
+known_oops_testcomponent_runs = ['ootestcomponent', 'testcomponent', 'testvar']
+
+#: Usual definition of the ``run`` attribute for OOPS binaries.
+a_oops_run = dict(
+    info="The OOPS run (== task).",
+    optional=False,
+)
+#: Usual Footprint of the ``run`` attribute for OOPS binaries.
+oops_run = footprints.Footprint(info='OOPS kind of run', attr=dict(run=a_oops_run))
+
+#: Usual definition of the ``test_type`` attribute.
+a_oops_test_type = dict(
+    info='Sub-test or family of sub-tests to be ran.',
+    optional=False,
+)
+#: Usual Footprint of the ``test_type`` attribute.
+oops_test_type = footprints.Footprint(info='OOPS type of test', attr=dict(test_type=a_oops_test_type))
+
+#: Usual definition of the ``expected_target`` attribute.
+an_oops_expected_target = dict(
+    info=('Expected target for the test success'),
+    type=footprints.FPDict,
+    optional=True,
+    default=None
+)
+#: Usual Footprint of the ``expected_target`` attribute.
+oops_expected_target = footprints.Footprint(attr=dict(expected_target=an_oops_expected_target))
+
+#: Usual Footprint of a combined lists of members and terms
+oops_members_terms_lists = footprints.Footprint(
+    info="Abstract footprint for a members/terms list.",
+    attr=dict(
+        members=dict(
+            info='A list of members.',
+            type=footprints.FPList,
+        ),
+        terms=dict(
+            info='A list of effective terms.',
+            type=footprints.FPList,
+            optional=True,
+            default=footprints.FPList([Time(0), ])
+        ),
+    )
+)
+
+#: Usual definition of the ``outputid`` attribute
+a_outputid = dict(
+    info="The identifier for the encoding of post-processed fields.",
+    optional=True,
+)
+
+#: Usual Footprint of the ``outputid`` attribute.
+outputid = footprints.Footprint(attr=dict(outputid=a_outputid))
+
+
+def _apply_outputid(cls):
+    """Decorator that tweak the class in order to add OUTPUTID on the namelist"""
+    orig_pnd = getattr(cls, 'prepare_namelist_delta', None)
+    if orig_pnd is None:
+        raise ImportError('_apply_outputid can not be applied on {!s}'.format(cls))
+
+    def prepare_namelist_delta(self, rh, namcontents, namlocal):
+        namw = orig_pnd(self, rh, namcontents, namlocal)
+        if self.outputid is not None and any(['OUTPUTID' in nam_b.macros()
+                                              for nam_b in namcontents.values()]):
+            self._set_nam_macro(namcontents, namlocal, 'OUTPUTID', self.outputid)
+            namw = True
+        return namw
+
+    cls.prepare_namelist_delta = prepare_namelist_delta
+    return cls
+
+
+#: Decorated footprint for the ``outputid`` attribute
+outputid_deco = footprints.DecorativeFootprint(outputid,
+                                               decorator=[_apply_outputid, ])
+
+
+def show():
+    """Returns available items and their type."""
+    dmod = globals()
+    for stda in sorted(filter(lambda x: x.startswith('a_') or isinstance(dmod[x], footprints.Footprint),
+                              dmod.keys())):
+        print('{} ( {} ) :\n  {}\n'.format(stda, type(dmod[stda]).__name__, dmod[stda]))
+
+
+"""
+This module provides some pre-defined attributes descriptions or combined sets
+of attributes description that could be used in the footprint definition of any
+class which follow the :class:`footprints.Footprint` syntax.
+"""
+
+import re
+from functools import total_ordering
+
+import footprints
+
+from vortex.syntax.stddeco import namebuilding_append
+
+#: Export some new class for attributes in footprint objects, eg : GenvKey
+__all__ = ['GenvKey', 'GenvDomain']
+
+domain_remap = dict()
+
+
+def _lowerattr(matchobj):
+    """Internal and technical function returning lower case value of the complete match item."""
+    return matchobj.group(0).lower()
+
+
+class GenvKey(str):
+    """
+    Attribute for a GEnv cycle name.
+    Implicit attributes inside brackets are translated to lower case.
+    See also :mod:`vortex_gco.tools.genv`.
+    """
+
+    def __new__(cls, value):
+        """Proxy to ``str.__new___`` with attributes inside brackets translated to lower case."""
+        return str.__new__(cls, re.sub(r'\[\w+\]', _lowerattr, value.upper()))
+
+
+a_gvar = dict(info='The key that identifies the resource in the Genv database.',
+              type=GenvKey,
+              optional=True,
+              doc_visibility=footprints.doc.visibility.ADVANCED,
+              )
+
+#: Usual definition of the ``genv`` attribute.
+gvar = footprints.Footprint(info='A GENV access key',
+                            attr=dict(gvar=a_gvar))
+
+
+class GenvDomain(str):
+    """
+    Remap plain area names to specific Genv short domain names.
+    See also :mod:`vortex_gco.tools.genv`.
+    """
+
+    def __new__(cls, value):
+        """Proxy to ``str.__new___`` with on the fly remapping of domain names to short values."""
+        return str.__new__(cls, domain_remap.get(value, value))
+
+
+a_gdomain = dict(info="The resource's geographical domain name in the Genv database.",
+                 type=GenvDomain,
+                 optional=True,
+                 default='[geometry::area]',
+                 doc_visibility=footprints.doc.visibility.ADVANCED,)
+
+#: Usual definition of the ``gdomain`` attribute.
+gdomain = footprints.Footprint(info='A domain name in GCO convention',
+                               attr=dict(gdomain=a_gdomain))
+
+
+@total_ordering
+class ArpIfsSimplifiedCycle:
+    """
+    Type that holds a simplified representation of an ArpegeIFS cycle.
+
+    It provides basic comparison operators to determine if a given cycle is more recent or not
+    compared to another one.
+
+    It can be used in a footprint specification.
+    """
+    _cy_re = re.compile(r'(?:u(?:env|get):)?(?:cy|al)(\d+)(?:t(\d{1,3}))?(?=_|@|\.|$)(?:.*?(?:[_-]op(\d{1,3})))?')
+    _hash_shift = 10000
+
+    def __init__(self, cyclestr):
+        cy_match = self._cy_re.match(cyclestr)
+        if cy_match:
+            self._number = int(cy_match.group(1))
+            self._toulouse = (int(cy_match.group(2)) + 1
+                              if cy_match.group(2) is not None else 0)
+            self._op = (int(cy_match.group(3)) + 1
+                        if cy_match.group(3) is not None else 0)
+        else:
+            raise ValueError('Malformed cycle: {}'.format(cyclestr))
+
+    def __hash__(self):
+        return (self._number * self._hash_shift + self._toulouse) * self._hash_shift + self._op
+
+    def __eq__(self, other):
+        if not isinstance(other, ArpIfsSimplifiedCycle):
+            try:
+                other = ArpIfsSimplifiedCycle(other)
+            except (ValueError, TypeError):
+                return False
+        return hash(self) == hash(other)
+
+    def __gt__(self, other):
+        if not isinstance(other, ArpIfsSimplifiedCycle):
+            other = ArpIfsSimplifiedCycle(other)
+        return hash(self) > hash(other)
+
+    def __str__(self):
+        return ('cy{:d}'.format(self._number) +
+                ('t{:d}'.format(self._toulouse - 1) if self._toulouse else '') +
+                ('_op{:d}'.format(self._op - 1) if self._op else ''))
+
+    def __repr__(self):
+        return '<{} | {!s}>'.format(object.__repr__(self).lstrip('<').rstrip('>'), self)
+
+    def export_dict(self):
+        """The pure dict/json output is the raw integer"""
+        return str(self)
+
+
+a_arpifs_cycle = dict(info="An Arpege/IFS cycle name",
+                      type=ArpIfsSimplifiedCycle,
+                      optional=True,
+                      default='cy40',  # For "old" Olive configurations to keep working
+                      )
+
+#: Usual definition of the ``cycle`` attribute.
+arpifs_cycle = footprints.Footprint(info='An abstract arpifs_cycle in GCO convention',
+                                    attr=dict(cycle=a_arpifs_cycle))
+
+uget_sloppy_id_regex = re.compile(r'(?P<shortuget>(?P<id>\S+)@(?P<location>[-\w]+))')
+uget_id_regex = r'(?P<fulluget>u(?:get|env):' + uget_sloppy_id_regex.pattern + ')'
+uget_id_regex_only = re.compile('^' + uget_id_regex + '$')
+uget_id_regex = re.compile(r'\b' + uget_id_regex + r'\b')
+
+
+class GgetId(str):
+    """Basestring wrapper for Gget Ids."""
+    def __new__(cls, value):
+        if uget_id_regex_only.match(value):
+            raise ValueError('A GgetId cannot look like a UgetId !')
+        return str.__new__(cls, value)
+
+
+class AbstractUgetId(str):
+    """Basestring wrapper for Uget Ids."""
+
+    _ALLOWED_LOCATIONS = ()
+    _OUTCAST_LOCATIONS = ()
+
+    def __new__(cls, value):
+        vmatch = uget_id_regex_only.match(value)
+        if not vmatch:
+            raise ValueError('Invalid UgetId (got "{:s}")'.format(value))
+        me = str.__new__(cls, value)
+        me._id = vmatch.group('id')
+        me._location = vmatch.group('location')
+        if me._location in set(cls._OUTCAST_LOCATIONS):
+            raise ValueError('Invalid UgetId (got "{:s}"). Outcast Location.'.format(value))
+        if cls._ALLOWED_LOCATIONS and me._location not in set(cls._ALLOWED_LOCATIONS):
+            raise ValueError('Invalid UgetId (got "{:s}"). Disallowed location'.format(value))
+        return me
+
+    @property
+    def id(self):
+        return self._id
+
+    @property
+    def location(self):
+        return self._location
+
+    @property
+    def short(self):
+        return self._id + '@' + self._location
+
+    def monthlyshort(self, month):
+        return self._id + '.m{:02d}'.format(month) + '@' + self._location
+
+
+class UgetId(AbstractUgetId):
+
+    _OUTCAST_LOCATIONS = ('demo', )
+
+
+def genv_ifs_compiler_convention(cls):
+    """Add the necessary method to handle compiler version/option in Genv."""
+    original_gget_basename = getattr(cls, 'gget_basename', None)
+    if original_gget_basename is not None:
+
+        def gget_basename(self):
+            """GGET specific naming convention."""
+            b_dict = original_gget_basename(self)
+            if getattr(self, 'compiler_version', None):
+                b_dict['compiler_version'] = self.compiler_version
+            if getattr(self, 'compiler_option', None):
+                b_dict['compiler_option'] = self.compiler_option
+            if getattr(self, 'cycle', None):
+                b_dict['cycle'] = self.cycle
+            return b_dict
+
+        cls.gget_basename = gget_basename
+    return cls
+
+
+#: Usual definition of the ``compiler_version`` and ``compiler_option`` attributes.
+gmkpack_compiler_identification = footprints.Footprint(
+    info='Add the compiler version/option in the footprint',
+    attr=dict(
+        compiler_version=dict(
+            info="The compiler version in gmkpack convention.",
+            optional=True
+        ),
+        compiler_option=dict(
+            info="The compiler option in gmkpack convention.",
+            optional=True
+        ),
+    ),
+)
+
+
+#: Usual definition of the ``compiler_version`` and ``compiler_option`` attributes + genv integration.
+gmkpack_compiler_identification_deco = footprints.DecorativeFootprint(
+    gmkpack_compiler_identification,
+    decorator=[genv_ifs_compiler_convention, ]
+)
+
+
+def genv_executable_flavour(cls):
+    """Add the necessary method to the "flavour" in Genv."""
+    original_genv_basename = getattr(cls, 'genv_basename', None)
+    if original_genv_basename is not None:
+
+        def genv_basename(self):
+            """Just retrieve a potential gvar attribute."""
+            gvar = original_genv_basename(self)
+            if getattr(self, 'flavour', None):
+                gvar += '_' + {'singleprecision': 'SP'
+                               }.get(self.flavour, self.flavour).upper()
+            return gvar
+
+        cls.genv_basename = genv_basename
+    return cls
+
+
+#: Usual definition of the ``flavour`` attribute.
+executable_flavour = footprints.Footprint(
+    info='Add the executable flavour attribute to the resource',
+    attr=dict(
+        flavour=dict(
+            info="The executable flavour (This may influence the Genv's key choice).",
+            values=['singleprecision', ],
+            optional=True,
+        ),
+    ),
+)
+
+
+#: Usual definition of the ``flavour``.
+executable_flavour_deco = footprints.DecorativeFootprint(
+    executable_flavour,
+    decorator=[genv_executable_flavour,
+               namebuilding_append('src', lambda self: [self.flavour], none_discard=True)]
+)

vortex/nwp/tools/__init__.py

@@ -0,0 +1,10 @@
+"""
+Standalone tools for NWP
+"""
+
+# Recursive inclusion of packages with potential FootprintBase classes
+from . import conftools
+from . import ifstools
+
+#: Automatic export of data subpackage
+__all__ = []

vortex/nwp/tools/addons.py

@@ -0,0 +1,35 @@
+"""
+Various System needed for NWP applications.
+"""
+
+from vortex.tools.addons import AddonGroup
+
+# Load the proper Addon modules...
+import vortex.tools.folder  # @UnusedImport
+import vortex.tools.lfi  # @UnusedImport
+import vortex.tools.grib  # @UnusedImport
+import vortex.tools.listings  # @UnusedImport
+import vortex.tools.surfex  # @UnusedImport
+
+#: No automatic export
+__all__ = []
+
+
+class NWPAddonsGroup(AddonGroup):
+    """A set of usual NWP Addons."""
+
+    _footprint = dict(
+        info = 'Default NWP Addons',
+        attr = dict(
+            kind = dict(
+                values = ['nwp', ],
+            ),
+        )
+    )
+
+    _addonslist = ('allfolders',  # Folder like...
+                   'lfi', 'iopoll',  # Wonderful FA/LFI world...
+                   'grib', 'gribapi',  # GRIB stuff...
+                   'arpifs_listings',  # Obscure IFS/Arpege listings...
+                   'sfx',  # Surfex...
+                   )

vortex/nwp/tools/agt.py

@@ -0,0 +1,55 @@
+"""
+TODO: Module documentation.
+"""
+
+import collections
+
+
+class AgtConfigurationError(Exception):
+    """Specific Transfer Agent configuration error."""
+    pass
+
+
+def agt_volatile_path(sh):
+    """Prefix path to use for the Transfer Agent to consider a file as being "volatile"
+    and hard-linking to it instead of expecting it to be present "later".
+    """
+    config = sh.default_target.config
+    if not config.has_section('agt'):
+        fmt = 'Missing section "agt" in configuration file\n"{}"'
+        raise AgtConfigurationError(fmt.format(config.file))
+    return config.get('agt', 'agt_volatile')
+
+
+def agt_actual_command(sh, binary_name, args, extraenv=None):
+    """Build the command able to execute a Transfer Agent binary.
+
+    The context, the execution path and the command name are
+    provided by the configuration file of the target.
+
+    The resulting command should be executed on a transfer node.
+
+    :param sh: The vortex shell that will be used
+    :param binary_name: Key in the configuration file that holds the binary name
+    :param args: Argument to the AGT binary
+    :param extraenv: Additional environment variables to export (dictionary)
+    """
+    config = sh.default_target.config
+    if not config.has_section('agt'):
+        fmt = 'Missing section "agt" in configuration file\n"{}"'
+        raise AgtConfigurationError(fmt.format(config.file))
+
+    agt_path = sh.default_target.get('agt_path', default=None)
+    agt_bin = sh.default_target.get(binary_name, default=None)
+    if not all([agt_path, agt_bin]):
+        fmt = 'Missing key "agt_path" or "{}" in configuration file\n"{}"'
+        raise AgtConfigurationError(fmt.format(binary_name, config.file))
+    cfgkeys = ['HOME_SOPRA', 'LD_LIBRARY_PATH',
+               'base_transfert_agent', 'DIAP_AGENT_NUMPROG_AGENT']
+    context = ' ; '.join(["export {}={}".format(key, config.get('agt', key))
+                          for key in cfgkeys])
+    if extraenv is not None and isinstance(extraenv, collections.abc.Mapping):
+        context = ' ; '.join([context, ] +
+                             ["export {}={}".format(key.upper(), value)
+                              for (key, value) in extraenv.items()])
+    return '{} ; {} {}'.format(context, sh.path.join(agt_path, agt_bin), args)

vortex/nwp/tools/bdap.py

@@ -0,0 +1,48 @@
+"""
+Utility classes and function to work with the BDAP database.
+"""
+
+#: No automatic export
+__all__ = []
+
+
+class BDAPError(Exception):
+    """General BDAP error."""
+    pass
+
+
+class BDAPRequestConfigurationError(BDAPError):
+    """Specific Transfer Agent configuration error."""
+    pass
+
+
+class BDAPGetError(BDAPError):
+    """Generic BDAP get error."""
+    pass
+
+
+def BDAPrequest_actual_command(command, date, term, query, int_extraenv=False):
+    """Build the command able to execute a BDAP request.
+
+    The context, the execution path and the command name are
+    provided by the configuration file of the target.
+
+    The resulting command should be executed on a transfer node.
+
+    :param command: name of the BDAP request command to be used
+    :param date: the date of the file requested
+    :param term: the term of the file requested
+    :param query: the query file used for the request
+    :param int_extraenv: boolean to know if the integration BDAP is used or not
+                         (an additional environment variable has to be exported in this case).
+    """
+
+    # Environment variable to specify the date of the file
+    context = ' ; '.join(["export {}={}".format('dmt_date_pivot'.upper(), date.ymdhms)])
+    # Extra environment variables (integration BDAP)
+    if int_extraenv:
+        context = ' ; '.join([context] +
+                             ["export {}={}".format('db_file_bdap'.upper(),
+                                                    '/usr/local/sopra/neons_db_bdap.intgr')])
+    # Return the command to be launched
+    return '{} ; {} {:d} {}'.format(context, command, term.hour, query)

vortex/nwp/tools/bdcp.py

@@ -0,0 +1,38 @@
+"""
+Utility classes and function to work with the BDCP database.
+"""
+
+#: No automatic export
+__all__ = []
+
+
+class BDCPError(Exception):
+    """General BDMP error."""
+    pass
+
+
+class BDCPRequestConfigurationError(BDCPError):
+    """Specific Transfer Agent configuration error."""
+    pass
+
+
+class BDCPGetError(BDCPError):
+    """Generic BDCP get error."""
+    pass
+
+
+def BDCPrequest_actual_command(command, query_file, output_file):
+    """Build the command able to execute a BDCP request.
+
+    The context, the execution path and the command name are
+    provided by the configuration file of the target.
+
+    The resulting command should be executed on a transfer node.
+
+    :param command: name of the BDMP request command to be used
+    :param query_file: the query file used for the request
+    :param output_file: the file in which the result will be written
+    """
+
+    # Return the command to be launched
+    return '{} -D {} -f {}'.format(command, query_file, output_file)

vortex/nwp/tools/bdm.py

@@ -0,0 +1,21 @@
+"""
+Utility classes and function to work with the BDM database.
+"""
+
+#: No automatic export
+__all__ = []
+
+
+class BDMError(Exception):
+    """General BDM error."""
+    pass
+
+
+class BDMRequestConfigurationError(BDMError):
+    """Specific Transfer Agent configuration error."""
+    pass
+
+
+class BDMGetError(BDMError):
+    """Generic BDM get error."""
+    pass

vortex/nwp/tools/bdmp.py

@@ -0,0 +1,49 @@
+"""
+Utility classes and function to work with the BDMP database.
+"""
+
+#: No automatic export
+__all__ = []
+
+
+class BDMPError(Exception):
+    """General BDMP error."""
+    pass
+
+
+class BDMPRequestConfigurationError(BDMPError):
+    """Specific Transfer Agent configuration error."""
+    pass
+
+
+class BDMPGetError(BDMPError):
+    """Generic BDMP get error."""
+    pass
+
+
+def BDMPrequest_actual_command(command, query, target_bdmp):
+    """Build the command able to execute a BDMP request.
+
+    The context, the execution path and the command name are
+    provided by the configuration file of the target.
+
+    The resulting command should be executed on a transfer node.
+
+    :param command: name of the BDMP request command to be used
+    :param query: the query file used for the request
+    :param target_bdmp: string to determine the BDMP used
+    """
+    # Environment variable used for the request
+    extraenv_pwd = "export {} {}".format('pwd_file'.upper(), '/usr/local/sopra/neons_pwd')
+    if target_bdmp == 'OPER':
+        extraenv_db = '/usr/local/sopra/neons_db_bdm'
+    elif target_bdmp == 'INTE':
+        extraenv_db = '/usr/local/sopra/neons_db_bdm.archi'
+    elif target_bdmp == 'ARCH':
+        extraenv_db = '/usr/local/sopra/neons_db_bdm.intgr'
+    else:
+        raise BDMPError
+    extraenv_db = 'export {} {}'.format('db_file_bdm'.upper(), extraenv_db)
+
+    # Return the command to be launched
+    return '{} ; {} ; {} {}'.format(extraenv_pwd, extraenv_db, command, query)