vortex-nwp 2.0.0b1__py3-none-any.whl

vortex/nwp/data/oopsexec.py

@@ -0,0 +1,72 @@
+"""
+Various Resources for executables used by the OOPS software.
+"""
+
+from vortex.data.executables import NWPModel
+from ..syntax.stdattrs import gvar, arpifs_cycle, executable_flavour_deco
+from ..syntax.stdattrs import oops_run, known_oops_testcomponent_runs
+from vortex.syntax.stddeco import namebuilding_append
+
+#: No automatic export
+__all__ = []
+
+
+@namebuilding_append('src', lambda self: self.run)
+class OOPSBinary(NWPModel):
+    """Yet an other OOPS Binary."""
+
+    _footprint = [
+        arpifs_cycle,
+        gvar,
+        oops_run,
+        executable_flavour_deco,
+        dict(
+            info = 'OOPS Binary: an OOPS binary, dedicated to a task (a run in OOPS namespace).',
+            attr = dict(
+                kind = dict(
+                    values = ['oopsbinary', ],
+                ),
+                gvar = dict(
+                    default = 'master_[run]',
+                ),
+                run = dict(
+                    outcast = known_oops_testcomponent_runs,
+                ),
+            )
+        )
+    ]
+
+    @property
+    def realkind(self):
+        return 'oopsbinary'
+
+    def command_line(self, configfile):
+        """
+        Build command line for execution as a single string.
+        """
+        cmdline = '{}'.format(configfile)
+        return cmdline
+
+
+class OOPSTestComponent(OOPSBinary):
+    """Binary for OOPS Tests of components."""
+
+    _footprint = dict(
+        info = 'OOPS Component Test: can run a sub-test or a family of sub-tests',
+        attr = dict(
+            run = dict(
+                values   = known_oops_testcomponent_runs,
+                outcast  = [],
+            ),
+        ),
+    )
+
+    def command_line(self, configfile, test_type=None):
+        """
+        Build command line for execution as a single string.
+        """
+        cmdline = ''
+        if test_type is not None:
+            cmdline += '-t {} '.format(test_type)
+        cmdline += super().command_line(configfile)
+        return cmdline

vortex/nwp/data/providers.py

@@ -0,0 +1,182 @@
+"""
+Common Providers.
+
+For now, only the BDPE access is available here (Base de Donnée des Produits Élaborés).
+This provider should work both on Soprano servers and on HPC, be it experimentally for
+certain parameters combinations.
+"""
+
+from bronx.fancies import loggers
+from bronx.stdtypes.date import Time
+from vortex.data.providers import Provider
+from vortex.syntax.stdattrs import DelayedEnvValue, Namespace, namespacefp
+from vortex.util.config import GenericConfigParser
+
+#: No automatic export
+__all__ = []
+
+logger = loggers.getLogger(__name__)
+
+
+class BdpeError(Exception):
+    """Base for Bdpe errors."""
+    pass
+
+
+class BdpeConfigurationError(BdpeError):
+    """Missing BDPE product description."""
+    pass
+
+
+class BdpeMismatchError(BdpeError):
+    """BDPE product description does not match ressource description."""
+    pass
+
+
+class BdpeProvider(Provider):
+    """
+    Provider to resources stored in the BDPE database.
+
+    The BDPE only knows about product ids, base datetime, and terms.
+    A dedicated ini file describes the relationship between such ids and
+    Vortex resources. This link could be used to deduce the BDPE id from
+    the resource (a la footprints). For now, it is only checked that the
+    resource attributes are compatible with the product description.
+
+    Canvas of a complete uri:
+        bdpe://bdpe.archive.fr/EXPE/date/BDPE_num+term
+
+    The EXPE part is built from the footprint attributes that correspond
+    to env variables used by the underlying tool we use: preferred_target,
+    forbidden_target, aso., so that the BDPE store can retrieve them from
+    the uri (See :class:`BdpeStore`).
+
+    When a resource has no ``date`` attribute, the most recent data is
+    extracted from the BDPE (this feature may be used for Alert Models).
+    """
+
+    _footprint = [
+        namespacefp,
+        dict(
+            info = 'BDPE provider',
+            attr = dict(
+                namespace = dict(
+                    default  = Namespace('bdpe.archive.fr'),
+                    values   = ['bdpe.archive.fr'],
+                ),
+                bdpeid = dict(
+                ),
+                preferred_target = dict(
+                    info     = "The database we'd like to get the data from - See the BDPE documentation.",
+                    optional = True,
+                    default  = DelayedEnvValue('BDPE_CIBLE_PREFEREE', 'SEC'),
+                    values   = ['OPER', 'INT', 'SEC',
+                                'oper', 'int', 'sec', ],
+                ),
+                forbidden_target = dict(
+                    info     = "The database we don't want to access - See the BDPE documentation.",
+                    optional = True,
+                    default  = DelayedEnvValue('BDPE_CIBLE_INTERDITE', 'OPER'),
+                    values   = ['OPER', 'INT', 'SEC',
+                                'oper', 'int', 'sec', ],
+                ),
+                soprano_domain = dict(
+                    info     = 'Databases priorities profile - See the BDPE documentation.',
+                    optional = True,
+                    default  = DelayedEnvValue('DOMAINE_SOPRA', 'dev'),
+                    values   = ['oper', 'int', 'dev'],
+                ),
+                allow_archive = dict(
+                    info     = 'Allow the use of the archive version of the BDPE databases.',
+                    optional = True,
+                    type     = bool,
+                    default  = False,
+                ),
+                bdpe_timeout = dict(
+                    info     = 'Seconds before abandoning a request.',
+                    optional = True,
+                    type     = int,
+                    default  = 10,
+                ),
+                bdpe_retries = dict(
+                    info     = 'Number of retries when a request fails.',
+                    optional = True,
+                    type     = int,
+                    default  = 3,
+                ),
+                config = dict(
+                    info     = 'A ready to use configuration file object for this storage place.',
+                    type     = GenericConfigParser,
+                    optional = True,
+                    default  = None,
+                ),
+                inifile = dict(
+                    info     = ('The name of the configuration file that will be used (if ' +
+                                '**config** is not provided.'),
+                    optional = True,
+                    default  = '@bdpe-map-resources.ini',
+                ),
+            ),
+            fastkeys = {'bdpeid'},
+        )
+    ]
+
+    def __init__(self, *args, **kw):
+        logger.debug('BDPE provider init %s', self.__class__)
+        super().__init__(*args, **kw)
+        self._actual_config = self.config
+        if self._actual_config is None:
+            self._actual_config = GenericConfigParser(inifile=self.inifile)
+
+    @property
+    def realkind(self):
+        return 'bdpe'
+
+    def scheme(self, resource):
+        """A dedicated scheme."""
+        return 'bdpe'
+
+    def netloc(self, resource):
+        """The actual netloc is the ``namespace`` attribute of the current provider."""
+        return self.namespace.netloc
+
+    def basename(self, resource):
+        """Something like 'BDPE_num+term'."""
+        myterm = getattr(resource, 'term', Time(0))
+        if int(myterm) < 0:
+            myterm = Time(9000) - myterm
+        return 'BDPE_{}+{!s}'.format(self.bdpeid, myterm)
+
+    def pathname(self, resource):
+        """Something like 'PREFERRED_FORBIDDEN_DOMAIN_ARCHIVE_TIMEOUT_RETRIES/date/'."""
+        try:
+            requested_date = resource.date.vortex()
+        except AttributeError:
+            requested_date = 'most_recent'
+        return '{}_{}_{}_{}_{}_{}/{}'.format(
+            self.preferred_target, self.forbidden_target, self.soprano_domain,
+            self.allow_archive, self.bdpe_timeout, self.bdpe_retries, requested_date)
+
+    def uri(self, resource):
+        """
+        Overridden to check the resource attributes against
+        the BDPE product description from the .ini file.
+        """
+        # check that the product is described in the configuration file
+        if not self._actual_config.has_section(self.bdpeid):
+            fmt = 'Missing product n°{} in BDPE configuration file\n"{}"'
+            raise BdpeConfigurationError(fmt.format(self.bdpeid, self.config.file))
+
+        # resource description: rely on the footprint_export (also used to JSONise resources).
+        rsrcdict = {k: str(v)
+                    for k, v in resource.footprint_export().items()}
+
+        # check the BDPE pairs against the resource's
+        for (k, v) in self._actual_config.items(self.bdpeid):
+            if k not in rsrcdict:
+                raise BdpeMismatchError('Missing key "{}" in resource'.format(k))
+            if rsrcdict[k] != v:
+                fmt = 'Bad value for key "{}": rsrc="{}" bdpe="{}"'
+                raise BdpeMismatchError(fmt.format(k, rsrcdict[k], v))
+
+        return super().uri(resource)

vortex/nwp/data/query.py

@@ -0,0 +1,217 @@
+"""
+Resources for query files used for extractions in various databases.
+"""
+
+import re
+
+from bronx.fancies import loggers
+
+from vortex.data.outflow import StaticResource
+from ..syntax.stdattrs import gvar
+from vortex.data.contents import AlmostListContent, DataTemplate
+
+#: No automatic export
+__all__ = []
+
+logger = loggers.getLogger(__name__)
+
+
+class Query(StaticResource):
+    """Class to deal with queries."""
+
+    _abstract = True
+    _footprint = [
+        gvar,
+        dict(
+            info = 'Abstract class for queries.',
+            attr = dict(
+                gvar = dict(
+                    values  = ['extract_stuff'],
+                    default = 'extract_stuff'
+                ),
+                source = dict(),
+                origin = dict(),
+            ),
+        )
+    ]
+
+    def gget_urlquery(self):
+        """GGET specific query : ``extract``."""
+        return 'extract=' + self.source
+
+
+class BDAPQuery(Query):
+    """Class to deal with BDAP queries."""
+    _footprint = dict(
+        info = 'BDAP query',
+        attr = dict(
+            kind = dict(
+                values = ['bdap_query']
+            ),
+            origin = dict(
+                default = 'bdap',
+                values = ['bdap'],
+                optional = True
+            )
+        )
+    )
+
+    @property
+    def realkind(self):
+        return 'bdap_query'
+
+
+class BDMPQuery(Query):
+    """Class to deal with BDMP queries."""
+    _footprint = dict(
+        info = 'BDMP query',
+        attr = dict(
+            kind = dict(
+                values = ['bdmp_query']
+            ),
+            origin = dict(
+                default = 'bdmp',
+                values = ['bdmp'],
+                optional = True
+            )
+        )
+    )
+
+    @property
+    def realkind(self):
+        return 'bdmp_query'
+
+
+class BDCPQuery(Query):
+    """Class to deal with BDCP queries."""
+    _footprint = dict(
+        info = 'BDCP query',
+        attr = dict(
+            kind = dict(
+                values = ['bdcp_query']
+            ),
+            origin = dict(
+                default = 'bdcp',
+                values = ['bdcp'],
+                optional = True
+            ),
+        )
+    )
+
+    @property
+    def realkind(self):
+        return 'bdcp_query'
+
+
+class StaticCutoffDispenser:
+    """
+    From a dictionary of cutoff times, for a given *obstype*, find the
+    best suited cutoff time.
+
+    The __call__ method takes a unique *obstype* argument. It will return the
+    best suited cutoff time for this particular *obstype*. N.B: If no exact
+    match is found, the default cutoff time will be used.
+    """
+
+    def __init__(self, default_cutoff, obstype_cutoffs=None):
+        self._default_cutoff = default_cutoff
+        if obstype_cutoffs:
+            self._cutoffs = {k: {o.lower() for o in v}
+                             for k, v in obstype_cutoffs.items()}
+        else:
+            self._cutoffs = {}
+
+    @property
+    def max_cutoff(self):
+        return self._default_cutoff
+
+    def __call__(self, obstype):
+        """Find the best suited cutoff time for *obstype*."""
+        obstype = obstype.lower()
+        found = None
+        for k, v in self._cutoffs.items():
+            if obstype in v:
+                found = k
+        if found:
+            return found
+        else:
+            return self._default_cutoff
+
+
+class BDMQueryContent(AlmostListContent):
+    """Read the content of BDM query file."""
+
+    _RE_OBSTYPE = re.compile(r"^(\s*)(OBS\s+TYPE\s*):(\s+)(\w+)$")
+
+    def add_cutoff_info(self, cutoffs_dispenser):
+        """
+        Using a :class:`vortex.tools.listings.ListBasedCutoffDispenser` or
+        :class:`StaticCutoffDispenser` object, add the cutoff related
+        information in the BDM query.
+        """
+        if cutoffs_dispenser.max_cutoff is None:
+            logger.warning("The cutoffs_dispenser is empty. No cutoff data can be retrieved")
+        else:
+            xdata = list()
+            for line in self:
+                xdata.append(line)
+                l_match = self._RE_OBSTYPE.match(line)
+                if l_match:
+                    cutoff_fmt = '{0:s}{1:<' + str(len(l_match.group(2))) + 's}:{2:s}{3.ymdhms:s}\n'
+                    cutoff_date = cutoffs_dispenser(l_match.group(4))
+                    xdata.append(cutoff_fmt.format(l_match.group(1),
+                                                   'CUTOFF',
+                                                   l_match.group(3),
+                                                   cutoff_date))
+                    logger.info('CUTOFF=%s added for obstype < %s >.',
+                                cutoff_date.ymdhms, l_match.group(4))
+            self._data = xdata
+
+
+class BDMQuery(Query):
+    """Class to deal with BDM queries."""
+    _footprint = dict(
+        info = 'BDM query',
+        attr = dict(
+            kind = dict(
+                values = ['bdm_query']
+            ),
+            origin = dict(
+                default = 'bdm',
+                values = ['bdm'],
+                optional = True
+            ),
+            clscontents=dict(
+                default = BDMQueryContent,
+            ),
+        )
+    )
+
+    @property
+    def realkind(self):
+        return 'bdm_query'
+
+
+class MarsQuery(Query):
+    """Class to deal with Mars queries"""
+
+    _footprint = dict(
+        info = 'Mars query',
+        attr = dict(
+            kind = dict(
+                values = ['mars_query']
+            ),
+            origin = dict(
+                default = "mars",
+                values = ["mars", ],
+                optional = True
+            ),
+            clscontents=dict(
+                default = DataTemplate
+            ),
+        )
+    )
+
+    @property
+    def realkind(self):
+        return "mars_query"

vortex/nwp/data/stores.py

@@ -0,0 +1,147 @@
+# pylint: disable=unused-argument
+
+"""
+Common stores.
+
+For now, only the BDPE store is available here, dedicated to BDPE extraction.
+"""
+
+import footprints
+import vortex
+from bronx.fancies import loggers
+from bronx.stdtypes import date
+from vortex.data.abstractstores import Store
+from vortex.syntax.stdattrs import compressionpipeline
+
+#: No automatic export
+__all__ = []
+
+logger = loggers.getLogger(__name__)
+
+
+class BdpeStore(Store):
+    """Access items stored in the BDPE database (get only)."""
+
+    _footprint = [
+        compressionpipeline,
+        dict(
+            info = 'Access the BDPE database',
+            attr = dict(
+                scheme = dict(
+                    values   = ['bdpe'],
+                ),
+                netloc = dict(
+                    values   = ['bdpe.archive.fr'],
+                ),
+            ),
+            priority = dict(
+                level = footprints.priorities.top.DEFAULT
+            ),
+        ),
+    ]
+
+    @property
+    def realkind(self):
+        return 'bdpe'
+
+    def bdpelocate(self, remote, options):
+        """Reasonably close to whatever 'remote location' could mean.
+
+        e.g.: ``bdpe://bdpe.archive.fr/EXPE/date/BDPE_num+term``
+        """
+        return self.scheme + '://' + self.netloc + remote['path']
+
+    def bdpecheck(self, remote, options):
+        """Cannot check a BDPE call a priori."""
+        logger.warning("A BdpeStore is not able to perform CHECKs.")
+        return False
+
+    def bdpeput(self, local, remote, options):
+        """Cannot write to the BDPE (See :class:`BdpeService`)."""
+        logger.error("A BdpeStore is not able to perform PUTs.")
+        return False
+
+    def bdpedelete(self, remote, options):
+        """Cannot delete a BDPE product."""
+        logger.error("A BdpeStore is not able to perform DELETEs.")
+        return False
+
+    def bdpeget(self, remote, local, options):
+        """Real extraction from the BDPE database."""
+
+        # Check that local is a file (i.e not a virtual container)
+        if not isinstance(local, str):
+            raise TypeError('The BDPE provider can not deal with virtual containers')
+
+        # remote['path'] looks like '/OPER_SEC_DEV_True_10_3/20151105T0000P/BDPE_42+06:00'
+        _, targetmix, str_date, more = remote['path'].split('/')
+        p_target, f_target, domain, s_archive, timeout, retries = targetmix.split('_')
+        productid, str_term = more[5:].split('+')
+
+        # the 'oper' domain is allowed only to the operational suite
+        if domain == 'oper':
+            if not vortex.ticket().glove.profile == 'oper':
+                logger.warning("Only profile 'oper' can use 'soprano_domain=oper'. Using 'dev' instead.")
+                domain = 'dev'
+
+        if str_date == 'most_recent':
+            bdpe_date = '/'
+        else:
+            bdpe_date = date.Date(str_date).ymdhms
+        bdpe_term = date.Time(str_term).fmtraw
+        args = [
+            productid,  # id
+            bdpe_date,  # date: yyyymmddhhmmss
+            bdpe_term,  # term: HHHHmm
+            local,      # local filename
+        ]
+        extraenv = dict(
+            BDPE_CIBLE_PREFEREE=p_target,
+            BDPE_CIBLE_INTERDITE=f_target,
+            DOMAINE_SOPRA=domain,
+            BDPE_TIMEOUT=timeout,
+            BDPE_RETRYS=retries,
+        )
+        if s_archive == 'True':
+            extraenv['BDPE_LECTURE_ARCHIVE_AUTORISEE'] = 'oui'
+
+        wsinterpreter = self.system.default_target.get('bdpe:wsclient_interpreter', None)
+        wscommand = self.system.default_target.get('bdpe:wsclient_path', None)
+        if wscommand is None:
+            raise RuntimeError('bdpe:wsclient_path has to be set in the target config')
+
+        args.insert(0, wscommand)
+        if wsinterpreter is not None:
+            args.insert(0, wsinterpreter)
+
+        logger.debug('lirepe_cmd: %s', " ".join(args))
+
+        with self.system.env.delta_context(**extraenv):
+            rc = self.system.spawn(args, output=False, fatal=False)
+        rc = rc and self.system.path.exists(local)
+
+        diagfile = local + '.diag'
+        if not rc:
+            logger.warning('Something went wrong with the following command: %s',
+                           " ".join(args))
+        if not rc or bdpe_date == '/':
+            if self.system.path.exists(diagfile):
+                logger.warning('The %s file is:', diagfile)
+                self.system.cat(diagfile)
+        if rc and self._actual_cpipeline:
+            # Deal with compressed files in the BDPE using the optional attribute
+            # store_compressed of the BDPE store.
+            tempfile = local + self._actual_cpipeline.suffix
+            rc = rc and self.system.mv(local, tempfile)
+            self._actual_cpipeline.file2uncompress(tempfile, local)
+            rc = rc and self.system.path.exists(local)
+            if not rc:
+                logger.warning('Something went wrong while uncompressing the file %s.', tempfile)
+
+        # Final step : deal with format specific packing
+        rc = rc and self.system.forceunpack(local, fmt=options.get('fmt'))
+
+        if self.system.path.exists(diagfile):
+            self.system.remove(diagfile)
+
+        return rc