vortex-nwp 2.0.0__py3-none-any.whl

vortex/nwp/data/oopsexec.py

@@ -0,0 +1,74 @@
+"""
+Various Resources for executables used by the OOPS software.
+"""
+
+from vortex.data.executables import NWPModel
+from ..syntax.stdattrs import gvar, arpifs_cycle, executable_flavour_deco
+from ..syntax.stdattrs import oops_run, known_oops_testcomponent_runs
+from vortex.syntax.stddeco import namebuilding_append
+
+#: No automatic export
+__all__ = []
+
+
+@namebuilding_append("src", lambda self: self.run)
+class OOPSBinary(NWPModel):
+    """Yet an other OOPS Binary."""
+
+    _footprint = [
+        arpifs_cycle,
+        gvar,
+        oops_run,
+        executable_flavour_deco,
+        dict(
+            info="OOPS Binary: an OOPS binary, dedicated to a task (a run in OOPS namespace).",
+            attr=dict(
+                kind=dict(
+                    values=[
+                        "oopsbinary",
+                    ],
+                ),
+                gvar=dict(
+                    default="master_[run]",
+                ),
+                run=dict(
+                    outcast=known_oops_testcomponent_runs,
+                ),
+            ),
+        ),
+    ]
+
+    @property
+    def realkind(self):
+        return "oopsbinary"
+
+    def command_line(self, configfile):
+        """
+        Build command line for execution as a single string.
+        """
+        cmdline = "{}".format(configfile)
+        return cmdline
+
+
+class OOPSTestComponent(OOPSBinary):
+    """Binary for OOPS Tests of components."""
+
+    _footprint = dict(
+        info="OOPS Component Test: can run a sub-test or a family of sub-tests",
+        attr=dict(
+            run=dict(
+                values=known_oops_testcomponent_runs,
+                outcast=[],
+            ),
+        ),
+    )
+
+    def command_line(self, configfile, test_type=None):
+        """
+        Build command line for execution as a single string.
+        """
+        cmdline = ""
+        if test_type is not None:
+            cmdline += "-t {} ".format(test_type)
+        cmdline += super().command_line(configfile)
+        return cmdline

vortex/nwp/data/providers.py

@@ -0,0 +1,207 @@
+"""
+Common Providers.
+
+For now, only the BDPE access is available here (Base de Donnée des Produits Élaborés).
+This provider should work both on Soprano servers and on HPC, be it experimentally for
+certain parameters combinations.
+"""
+
+from bronx.fancies import loggers
+from bronx.stdtypes.date import Time
+from vortex.data.providers import Provider
+from vortex.syntax.stdattrs import DelayedEnvValue, Namespace, namespacefp
+from vortex.util.config import GenericConfigParser
+
+#: No automatic export
+__all__ = []
+
+logger = loggers.getLogger(__name__)
+
+
+class BdpeError(Exception):
+    """Base for Bdpe errors."""
+
+    pass
+
+
+class BdpeConfigurationError(BdpeError):
+    """Missing BDPE product description."""
+
+    pass
+
+
+class BdpeMismatchError(BdpeError):
+    """BDPE product description does not match ressource description."""
+
+    pass
+
+
+class BdpeProvider(Provider):
+    """
+    Provider to resources stored in the BDPE database.
+
+    The BDPE only knows about product ids, base datetime, and terms.
+    A dedicated ini file describes the relationship between such ids and
+    Vortex resources. This link could be used to deduce the BDPE id from
+    the resource (a la footprints). For now, it is only checked that the
+    resource attributes are compatible with the product description.
+
+    Canvas of a complete uri:
+        bdpe://bdpe.archive.fr/EXPE/date/BDPE_num+term
+
+    The EXPE part is built from the footprint attributes that correspond
+    to env variables used by the underlying tool we use: preferred_target,
+    forbidden_target, aso., so that the BDPE store can retrieve them from
+    the uri (See :class:`BdpeStore`).
+
+    When a resource has no ``date`` attribute, the most recent data is
+    extracted from the BDPE (this feature may be used for Alert Models).
+    """
+
+    _footprint = [
+        namespacefp,
+        dict(
+            info="BDPE provider",
+            attr=dict(
+                namespace=dict(
+                    default=Namespace("bdpe.archive.fr"),
+                    values=["bdpe.archive.fr"],
+                ),
+                bdpeid=dict(),
+                preferred_target=dict(
+                    info="The database we'd like to get the data from - See the BDPE documentation.",
+                    optional=True,
+                    default=DelayedEnvValue("BDPE_CIBLE_PREFEREE", "SEC"),
+                    values=[
+                        "OPER",
+                        "INT",
+                        "SEC",
+                        "oper",
+                        "int",
+                        "sec",
+                    ],
+                ),
+                forbidden_target=dict(
+                    info="The database we don't want to access - See the BDPE documentation.",
+                    optional=True,
+                    default=DelayedEnvValue("BDPE_CIBLE_INTERDITE", "OPER"),
+                    values=[
+                        "OPER",
+                        "INT",
+                        "SEC",
+                        "oper",
+                        "int",
+                        "sec",
+                    ],
+                ),
+                soprano_domain=dict(
+                    info="Databases priorities profile - See the BDPE documentation.",
+                    optional=True,
+                    default=DelayedEnvValue("DOMAINE_SOPRA", "dev"),
+                    values=["oper", "int", "dev"],
+                ),
+                allow_archive=dict(
+                    info="Allow the use of the archive version of the BDPE databases.",
+                    optional=True,
+                    type=bool,
+                    default=False,
+                ),
+                bdpe_timeout=dict(
+                    info="Seconds before abandoning a request.",
+                    optional=True,
+                    type=int,
+                    default=10,
+                ),
+                bdpe_retries=dict(
+                    info="Number of retries when a request fails.",
+                    optional=True,
+                    type=int,
+                    default=3,
+                ),
+                config=dict(
+                    info="A ready to use configuration file object for this storage place.",
+                    type=GenericConfigParser,
+                    optional=True,
+                    default=None,
+                ),
+                inifile=dict(
+                    info=(
+                        "The name of the configuration file that will be used (if "
+                        + "**config** is not provided."
+                    ),
+                    optional=True,
+                    default="@bdpe-map-resources.ini",
+                ),
+            ),
+            fastkeys={"bdpeid"},
+        ),
+    ]
+
+    def __init__(self, *args, **kw):
+        logger.debug("BDPE provider init %s", self.__class__)
+        super().__init__(*args, **kw)
+        self._actual_config = self.config
+        if self._actual_config is None:
+            self._actual_config = GenericConfigParser(inifile=self.inifile)
+
+    @property
+    def realkind(self):
+        return "bdpe"
+
+    def scheme(self, resource):
+        """A dedicated scheme."""
+        return "bdpe"
+
+    def netloc(self, resource):
+        """The actual netloc is the ``namespace`` attribute of the current provider."""
+        return self.namespace.netloc
+
+    def basename(self, resource):
+        """Something like 'BDPE_num+term'."""
+        myterm = getattr(resource, "term", Time(0))
+        if int(myterm) < 0:
+            myterm = Time(9000) - myterm
+        return "BDPE_{}+{!s}".format(self.bdpeid, myterm)
+
+    def pathname(self, resource):
+        """Something like 'PREFERRED_FORBIDDEN_DOMAIN_ARCHIVE_TIMEOUT_RETRIES/date/'."""
+        try:
+            requested_date = resource.date.vortex()
+        except AttributeError:
+            requested_date = "most_recent"
+        return "{}_{}_{}_{}_{}_{}/{}".format(
+            self.preferred_target,
+            self.forbidden_target,
+            self.soprano_domain,
+            self.allow_archive,
+            self.bdpe_timeout,
+            self.bdpe_retries,
+            requested_date,
+        )
+
+    def uri(self, resource):
+        """
+        Overridden to check the resource attributes against
+        the BDPE product description from the .ini file.
+        """
+        # check that the product is described in the configuration file
+        if not self._actual_config.has_section(self.bdpeid):
+            fmt = 'Missing product n°{} in BDPE configuration file\n"{}"'
+            raise BdpeConfigurationError(
+                fmt.format(self.bdpeid, self.config.file)
+            )
+
+        # resource description: rely on the footprint_export (also used to JSONise resources).
+        rsrcdict = {k: str(v) for k, v in resource.footprint_export().items()}
+
+        # check the BDPE pairs against the resource's
+        for k, v in self._actual_config.items(self.bdpeid):
+            if k not in rsrcdict:
+                raise BdpeMismatchError(
+                    'Missing key "{}" in resource'.format(k)
+                )
+            if rsrcdict[k] != v:
+                fmt = 'Bad value for key "{}": rsrc="{}" bdpe="{}"'
+                raise BdpeMismatchError(fmt.format(k, rsrcdict[k], v))
+
+        return super().uri(resource)

vortex/nwp/data/query.py

@@ -0,0 +1,206 @@
+"""
+Resources for query files used for extractions in various databases.
+"""
+
+import re
+
+from bronx.fancies import loggers
+
+from vortex.data.outflow import StaticResource
+from ..syntax.stdattrs import gvar
+from vortex.data.contents import AlmostListContent, DataTemplate
+
+#: No automatic export
+__all__ = []
+
+logger = loggers.getLogger(__name__)
+
+
+class Query(StaticResource):
+    """Class to deal with queries."""
+
+    _abstract = True
+    _footprint = [
+        gvar,
+        dict(
+            info="Abstract class for queries.",
+            attr=dict(
+                gvar=dict(values=["extract_stuff"], default="extract_stuff"),
+                source=dict(),
+                origin=dict(),
+            ),
+        ),
+    ]
+
+    def gget_urlquery(self):
+        """GGET specific query : ``extract``."""
+        return "extract=" + self.source
+
+
+class BDAPQuery(Query):
+    """Class to deal with BDAP queries."""
+
+    _footprint = dict(
+        info="BDAP query",
+        attr=dict(
+            kind=dict(values=["bdap_query"]),
+            origin=dict(default="bdap", values=["bdap"], optional=True),
+        ),
+    )
+
+    @property
+    def realkind(self):
+        return "bdap_query"
+
+
+class BDMPQuery(Query):
+    """Class to deal with BDMP queries."""
+
+    _footprint = dict(
+        info="BDMP query",
+        attr=dict(
+            kind=dict(values=["bdmp_query"]),
+            origin=dict(default="bdmp", values=["bdmp"], optional=True),
+        ),
+    )
+
+    @property
+    def realkind(self):
+        return "bdmp_query"
+
+
+class BDCPQuery(Query):
+    """Class to deal with BDCP queries."""
+
+    _footprint = dict(
+        info="BDCP query",
+        attr=dict(
+            kind=dict(values=["bdcp_query"]),
+            origin=dict(default="bdcp", values=["bdcp"], optional=True),
+        ),
+    )
+
+    @property
+    def realkind(self):
+        return "bdcp_query"
+
+
+class StaticCutoffDispenser:
+    """
+    From a dictionary of cutoff times, for a given *obstype*, find the
+    best suited cutoff time.
+
+    The __call__ method takes a unique *obstype* argument. It will return the
+    best suited cutoff time for this particular *obstype*. N.B: If no exact
+    match is found, the default cutoff time will be used.
+    """
+
+    def __init__(self, default_cutoff, obstype_cutoffs=None):
+        self._default_cutoff = default_cutoff
+        if obstype_cutoffs:
+            self._cutoffs = {
+                k: {o.lower() for o in v} for k, v in obstype_cutoffs.items()
+            }
+        else:
+            self._cutoffs = {}
+
+    @property
+    def max_cutoff(self):
+        return self._default_cutoff
+
+    def __call__(self, obstype):
+        """Find the best suited cutoff time for *obstype*."""
+        obstype = obstype.lower()
+        found = None
+        for k, v in self._cutoffs.items():
+            if obstype in v:
+                found = k
+        if found:
+            return found
+        else:
+            return self._default_cutoff
+
+
+class BDMQueryContent(AlmostListContent):
+    """Read the content of BDM query file."""
+
+    _RE_OBSTYPE = re.compile(r"^(\s*)(OBS\s+TYPE\s*):(\s+)(\w+)$")
+
+    def add_cutoff_info(self, cutoffs_dispenser):
+        """
+        Using a :class:`vortex.tools.listings.ListBasedCutoffDispenser` or
+        :class:`StaticCutoffDispenser` object, add the cutoff related
+        information in the BDM query.
+        """
+        if cutoffs_dispenser.max_cutoff is None:
+            logger.warning(
+                "The cutoffs_dispenser is empty. No cutoff data can be retrieved"
+            )
+        else:
+            xdata = list()
+            for line in self:
+                xdata.append(line)
+                l_match = self._RE_OBSTYPE.match(line)
+                if l_match:
+                    cutoff_fmt = (
+                        "{0:s}{1:<"
+                        + str(len(l_match.group(2)))
+                        + "s}:{2:s}{3.ymdhms:s}\n"
+                    )
+                    cutoff_date = cutoffs_dispenser(l_match.group(4))
+                    xdata.append(
+                        cutoff_fmt.format(
+                            l_match.group(1),
+                            "CUTOFF",
+                            l_match.group(3),
+                            cutoff_date,
+                        )
+                    )
+                    logger.info(
+                        "CUTOFF=%s added for obstype < %s >.",
+                        cutoff_date.ymdhms,
+                        l_match.group(4),
+                    )
+            self._data = xdata
+
+
+class BDMQuery(Query):
+    """Class to deal with BDM queries."""
+
+    _footprint = dict(
+        info="BDM query",
+        attr=dict(
+            kind=dict(values=["bdm_query"]),
+            origin=dict(default="bdm", values=["bdm"], optional=True),
+            clscontents=dict(
+                default=BDMQueryContent,
+            ),
+        ),
+    )
+
+    @property
+    def realkind(self):
+        return "bdm_query"
+
+
+class MarsQuery(Query):
+    """Class to deal with Mars queries"""
+
+    _footprint = dict(
+        info="Mars query",
+        attr=dict(
+            kind=dict(values=["mars_query"]),
+            origin=dict(
+                default="mars",
+                values=[
+                    "mars",
+                ],
+                optional=True,
+            ),
+            clscontents=dict(default=DataTemplate),
+        ),
+    )
+
+    @property
+    def realkind(self):
+        return "mars_query"

vortex/nwp/data/stores.py

@@ -0,0 +1,160 @@
+# pylint: disable=unused-argument
+
+"""
+Common stores.
+
+For now, only the BDPE store is available here, dedicated to BDPE extraction.
+"""
+
+import footprints
+import vortex
+from bronx.fancies import loggers
+from bronx.stdtypes import date
+from vortex.data.abstractstores import Store
+from vortex.syntax.stdattrs import compressionpipeline
+
+#: No automatic export
+__all__ = []
+
+logger = loggers.getLogger(__name__)
+
+
+class BdpeStore(Store):
+    """Access items stored in the BDPE database (get only)."""
+
+    _footprint = [
+        compressionpipeline,
+        dict(
+            info="Access the BDPE database",
+            attr=dict(
+                scheme=dict(
+                    values=["bdpe"],
+                ),
+                netloc=dict(
+                    values=["bdpe.archive.fr"],
+                ),
+            ),
+            priority=dict(level=footprints.priorities.top.DEFAULT),
+        ),
+    ]
+
+    @property
+    def realkind(self):
+        return "bdpe"
+
+    def bdpelocate(self, remote, options):
+        """Reasonably close to whatever 'remote location' could mean.
+
+        e.g.: ``bdpe://bdpe.archive.fr/EXPE/date/BDPE_num+term``
+        """
+        return self.scheme + "://" + self.netloc + remote["path"]
+
+    def bdpecheck(self, remote, options):
+        """Cannot check a BDPE call a priori."""
+        logger.warning("A BdpeStore is not able to perform CHECKs.")
+        return False
+
+    def bdpeput(self, local, remote, options):
+        """Cannot write to the BDPE (See :class:`BdpeService`)."""
+        logger.error("A BdpeStore is not able to perform PUTs.")
+        return False
+
+    def bdpedelete(self, remote, options):
+        """Cannot delete a BDPE product."""
+        logger.error("A BdpeStore is not able to perform DELETEs.")
+        return False
+
+    def bdpeget(self, remote, local, options):
+        """Real extraction from the BDPE database."""
+
+        # Check that local is a file (i.e not a virtual container)
+        if not isinstance(local, str):
+            raise TypeError(
+                "The BDPE provider can not deal with virtual containers"
+            )
+
+        # remote['path'] looks like '/OPER_SEC_DEV_True_10_3/20151105T0000P/BDPE_42+06:00'
+        _, targetmix, str_date, more = remote["path"].split("/")
+        p_target, f_target, domain, s_archive, timeout, retries = (
+            targetmix.split("_")
+        )
+        productid, str_term = more[5:].split("+")
+
+        # the 'oper' domain is allowed only to the operational suite
+        if domain == "oper":
+            if not vortex.ticket().glove.profile == "oper":
+                logger.warning(
+                    "Only profile 'oper' can use 'soprano_domain=oper'. Using 'dev' instead."
+                )
+                domain = "dev"
+
+        if str_date == "most_recent":
+            bdpe_date = "/"
+        else:
+            bdpe_date = date.Date(str_date).ymdhms
+        bdpe_term = date.Time(str_term).fmtraw
+        args = [
+            productid,  # id
+            bdpe_date,  # date: yyyymmddhhmmss
+            bdpe_term,  # term: HHHHmm
+            local,  # local filename
+        ]
+        extraenv = dict(
+            BDPE_CIBLE_PREFEREE=p_target,
+            BDPE_CIBLE_INTERDITE=f_target,
+            DOMAINE_SOPRA=domain,
+            BDPE_TIMEOUT=timeout,
+            BDPE_RETRYS=retries,
+        )
+        if s_archive == "True":
+            extraenv["BDPE_LECTURE_ARCHIVE_AUTORISEE"] = "oui"
+
+        wsinterpreter = self.system.default_target.get(
+            "bdpe:wsclient_interpreter", None
+        )
+        wscommand = self.system.default_target.get("bdpe:wsclient_path", None)
+        if wscommand is None:
+            raise RuntimeError(
+                "bdpe:wsclient_path has to be set in the target config"
+            )
+
+        args.insert(0, wscommand)
+        if wsinterpreter is not None:
+            args.insert(0, wsinterpreter)
+
+        logger.debug("lirepe_cmd: %s", " ".join(args))
+
+        with self.system.env.delta_context(**extraenv):
+            rc = self.system.spawn(args, output=False, fatal=False)
+        rc = rc and self.system.path.exists(local)
+
+        diagfile = local + ".diag"
+        if not rc:
+            logger.warning(
+                "Something went wrong with the following command: %s",
+                " ".join(args),
+            )
+        if not rc or bdpe_date == "/":
+            if self.system.path.exists(diagfile):
+                logger.warning("The %s file is:", diagfile)
+                self.system.cat(diagfile)
+        if rc and self._actual_cpipeline:
+            # Deal with compressed files in the BDPE using the optional attribute
+            # store_compressed of the BDPE store.
+            tempfile = local + self._actual_cpipeline.suffix
+            rc = rc and self.system.mv(local, tempfile)
+            self._actual_cpipeline.file2uncompress(tempfile, local)
+            rc = rc and self.system.path.exists(local)
+            if not rc:
+                logger.warning(
+                    "Something went wrong while uncompressing the file %s.",
+                    tempfile,
+                )
+
+        # Final step : deal with format specific packing
+        rc = rc and self.system.forceunpack(local, fmt=options.get("fmt"))
+
+        if self.system.path.exists(diagfile):
+            self.system.remove(diagfile)
+
+        return rc