vortex-nwp 2.0.0__py3-none-any.whl

vortex/util/empty.py

@@ -0,0 +1,24 @@
+"""
+An empty module to be filled with some kind of blackholes objects.
+"""
+
+from bronx.fancies import loggers
+
+#: No automatic export
+__all__ = []
+
+logger = loggers.getLogger(__name__)
+
+
+class DataConst:
+    """Constants stored as raw attributes."""
+
+    def __init__(self, **kw):
+        self.__dict__.update(kw)
+        logger.debug("DataConst init %s", self)
+
+    def __str__(self):
+        return super().__str__() + " : " + str(sorted(self.__dict__.keys()))
+
+    def __contains__(self, item):
+        return item in self.__dict__

vortex/util/helpers.py

@@ -0,0 +1,216 @@
+"""
+Some convenient functions that may simplify scripts
+"""
+
+from collections import defaultdict
+
+from bronx.compat import random
+from bronx.fancies import loggers
+from bronx.stdtypes.date import Date
+import footprints as fp
+
+from vortex.data.handlers import Handler
+from vortex.layout.dataflow import Section
+from vortex import sessions
+
+logger = loggers.getLogger(__name__)
+
+
+class InputCheckerError(Exception):
+    """Exception raised when the Input checking process fails."""
+
+    pass
+
+
+def generic_input_checker(grouping_keys, min_items, *rhandlers, **kwargs):
+    """
+    Check which input resources are present.
+
+    First, the resource handlers (*rhandlers* attribute) are split
+    into groups based on the values of their properties (only the properties
+    specified in the *grouping_keys* attribute are considered).
+
+    Then, for each group, the **check** method is called upon the resource
+    handlers. The group description is returned only if the **check** call
+    succeed for all the members of the group.
+
+    If the number of groups successfully checked is lower than *min_items*,
+    an :class:`InputCheckerError` exception is raised.
+    """
+
+    if len(rhandlers) == 0:
+        raise ValueError("At least one resource handler have to be provided")
+    # Just in case min_items is not an int...
+    min_items = int(min_items)
+
+    # Create a flat ResourceHandlers list (rhandlers may consists of lists)
+    flat_rhlist = []
+    flat_rhmandatory = []
+    for inlist, outlist in (
+        (rhandlers, flat_rhlist),
+        (kwargs.pop("mandatory", []), flat_rhmandatory),
+    ):
+        for rh in inlist:
+            if isinstance(rh, list) or isinstance(rh, tuple):
+                outlist.extend(rh)
+            else:
+                outlist.append(rh)
+
+    # Extract the group informations for each of the resource handlers
+    rhgroups = defaultdict(list)
+    for rh in flat_rhlist:
+        keylist = list()
+        for key in grouping_keys:
+            value = rh.wide_key_lookup(key, exports=True, fatal=False)
+            keylist.append(value)
+        rhgroups[tuple(keylist)].append(rh)
+
+    candidateslist = [
+        fp.stdtypes.FPDict(
+            {k: v for k, v in zip(grouping_keys, group) if v is not None}
+        )
+        for group in rhgroups.keys()
+    ]
+
+    # Activate FTP connections pooling (for enhanced performances)
+    t = sessions.current()
+    with t.sh.ftppool():
+        # Check mandatory stuff
+        mychecks = [(rh, rh.check()) for rh in flat_rhmandatory]
+        if not all([acheck[1] for acheck in mychecks]):
+            for rh in [acheck[0] for acheck in mychecks if not acheck[1]]:
+                logger.error("  Missing location: %s", str(rh.locate()))
+            raise InputCheckerError(
+                "Some of the mandatory resources are missing."
+            )
+
+        # Check call for non-mandatory stuff
+        outputlist = list()
+        # Is the check real or a delusion ?
+        fakecheck = kwargs.pop("fakecheck", False)
+        #  The keys are sorted so that results remains reproducible
+        for grouping_values in sorted(rhgroups.keys()):
+            mychecks = [
+                (rh, fakecheck or rh.check())
+                for rh in rhgroups[grouping_values]
+            ]
+            groupid = fp.stdtypes.FPDict(
+                {
+                    k: v
+                    for k, v in zip(grouping_keys, grouping_values)
+                    if v is not None
+                }
+            )
+            if all([acheck[1] for acheck in mychecks]):
+                outputlist.append(groupid)
+                logger.info(
+                    "Group (%s): All the input files are accounted for.",
+                    str(groupid),
+                )
+            else:
+                logger.warning(
+                    "Group (%s): Discarded because some of the input files are missing (see below).",
+                    str(groupid),
+                )
+                for rh in [acheck[0] for acheck in mychecks if not acheck[1]]:
+                    logger.warning("  Missing location: %s", str(rh.locate()))
+
+    # Enforce min_items
+    if len(outputlist) < min_items:
+        raise InputCheckerError(
+            "The number of input groups is too small "
+            + "({:d} < {:d})".format(len(outputlist), min_items)
+        )
+
+    return fp.stdtypes.FPList(outputlist), fp.stdtypes.FPList(candidateslist)
+
+
+def members_input_checker(min_items, *rhandlers, **kwargs):
+    """
+    This is a shortcut for the generic_input_checher: only the member number is
+    considered and the return values corresponds to a list of members.
+    """
+    mlist = [
+        desc["member"]
+        for desc in generic_input_checker(
+            ("member",), min_items, *rhandlers, **kwargs
+        )[0]
+    ]
+    return fp.stdtypes.FPList(sorted(mlist))
+
+
+def colorfull_input_checker(min_items, *rhandlers, **kwargs):
+    """
+    This is a shortcut for the generic_input_checher: it returns a list of
+    dictionaries that described the available data.
+    """
+    return generic_input_checker(
+        ("vapp", "vconf", "experiment", "cutoff", "date", "member"),
+        min_items,
+        *rhandlers,
+        **kwargs,
+    )
+
+
+def merge_contents(*kargs):
+    """Automatically merge several DataContents.
+
+    Example:
+    .. code-block:: python
+
+        mergedcontent = merge_contents(content1, content2, content3)
+        # With a list
+        mergedcontent = merge_contents([content1, content2, content3])
+        # With a list of ResourceHandlers (e.g. as returned by toolbox.input)
+        mergedcontent = merge_contents([rh1, rh2, rh3])
+        # With a list of Sections (e.g. as returned by effective_inputs)
+        mergedcontent = merge_contents([section1, section2, section3])
+
+    """
+    # Expand list or tuple elements
+    ctlist = list()
+    for elt in kargs:
+        if isinstance(elt, (list, tuple)):
+            ctlist.extend(elt)
+        else:
+            ctlist.append(elt)
+    # kargs may be a list of resource handlers (as returned by the toolbox)
+    if all([isinstance(obj, Handler) for obj in ctlist]):
+        ctlist = [obj.contents for obj in ctlist]
+    # kargs may be a list of sections
+    elif all([isinstance(obj, Section) for obj in ctlist]):
+        ctlist = [obj.rh.contents for obj in ctlist]
+    # Take the first content as a model for the new object
+    newcontent = ctlist[0].__class__()
+    newcontent.merge(*ctlist)
+    return newcontent
+
+
+def mix_list(list_elements, date=None, member=None):
+    """Mix a list using a determined seed, if member and/or date are present."""
+    dateinfo = date if date is None else Date(date)
+    memberinfo = member if member is None else int(member)
+    rgen = random.Random()
+    if (dateinfo is not None) or (memberinfo is not None):
+        if dateinfo is not None:
+            seed = dateinfo.epoch * 100
+        else:
+            seed = 9999999
+        if memberinfo:
+            seed = seed // memberinfo
+        logger.debug("The random seed is %s.", seed)
+        rgen.seed(seed)
+    else:
+        logger.info("The random seed not initialised")
+    logger.debug(
+        "The list of elements is %s.",
+        " ".join([str(x) for x in list_elements]),
+    )
+    result_list_elements = list_elements
+    result_list_elements.sort()
+    rgen.shuffle(result_list_elements)
+    logger.debug(
+        "The mixed list of elements is %s.",
+        " ".join([str(x) for x in result_list_elements]),
+    )
+    return result_list_elements

vortex/util/introspection.py

@@ -0,0 +1,69 @@
+"""
+Some convenient functions to explore the source code or its documentation.
+"""
+
+import re
+import inspect
+
+from bronx.fancies import loggers
+
+from vortex import sessions
+
+#: No automatic export
+__all__ = []
+
+logger = loggers.getLogger(__name__)
+
+
+class Sherlock:
+    """Centralized interface to introspection functions."""
+
+    def __init__(self, **kw):
+        self.verbose = False
+        self.ticket = kw.pop("ticket", sessions.current())
+        self.glove = self.ticket.glove
+        self.__dict__.update(kw)
+        logger.debug("Sherlock init %s", self)
+
+    def rstfile(self, modpath):
+        """Return the sphinx documentation associated to module reference or module path given."""
+        if not isinstance(modpath, str):
+            modpath = modpath.__file__
+        subpath = modpath
+        for installpath in self.glove.sitesrc:
+            subpath = re.sub(installpath, "", subpath)
+        subpath = re.sub(r"\.pyc?", "", subpath)
+        subpath = subpath.split("/")
+        if subpath[-1] == "__init__":
+            subpath[-1] = subpath[-2]
+        subpath[-1] += ".rst"
+
+        subpath[1:1] = [
+            "library",
+        ]
+        return self.glove.sitedoc + "/".join(subpath)
+
+    def rstshort(self, filename):
+        """Return relative path name of ``filename`` according to :meth:`siteroot`."""
+        return re.sub(self.glove.siteroot, "", filename)[1:]
+
+    def getlocalmembers(self, obj, topmodule=None):
+        """Return members of the module ``obj`` which are defined in the source file of the module."""
+        objs = dict()
+        if topmodule is None:
+            topmodule = obj
+        modfile = topmodule.__file__.rstrip("c")
+        for x, y in inspect.getmembers(obj):
+            if (
+                inspect.isclass(y)
+                or inspect.isfunction(y)
+                or inspect.ismethod(y)
+            ):
+                try:
+                    if modfile == inspect.getsourcefile(y):
+                        if self.verbose:
+                            print(x, y)
+                        objs[x] = y
+                except TypeError:
+                    pass
+        return objs

vortex/util/iosponge.py

@@ -0,0 +1,80 @@
+"""
+Provide a File-Like object that reads in the first N bytes in order to count
+them precisely.
+"""
+
+import io
+
+#: No automatic export
+__all__ = []
+
+IOSPONGE_DEFAULT_SIZECHECK = 33 * 1024 * 1024  # 33Mb
+
+
+class IoSponge(io.BufferedIOBase):
+    """Buffer the first bytes in order to compute an accurate size for the
+    underlying stream.
+
+    This class just acts as a buffer. It looks like a file object and should
+    be used as such.
+
+    If the size of the underlying stream is <= *size_check* bytes : the **size**
+    property will return an exact estimate of the file-like object size. Passed
+    that limit the maximum of *size_check* and *guessed_size* is returned.
+    """
+
+    def __init__(
+        self, rawio, size_check=IOSPONGE_DEFAULT_SIZECHECK, guessed_size=0
+    ):
+        """
+        :param file rawio: Any kind of file-like object
+        :param int size_check: The first size_check bytes will be buffered in
+            order to be properly accounted for.
+        :param int gressed_size: An estimate of the file-like object size (in
+            bytes)
+        """
+        self._rawio = rawio
+        self._size_check = size_check
+        self._guessed_size = int(guessed_size)
+        self._first_bytes = self._rawio.read(size_check)
+        self._seek = 0
+
+    @property
+    def size(self):
+        """The (exact or estimated)  size of the underlying file-like object."""
+        if len(self._first_bytes) < self._size_check:
+            return len(self._first_bytes)
+        else:
+            return max(len(self._first_bytes), self._guessed_size)
+
+    def tell(self):
+        """The amount of bytes read in this strem."""
+        return self._seek
+
+    def _generic_read(self, size, raw_read_cb):
+        ret = b""
+        if self._seek < len(self._first_bytes):
+            if size is None:
+                ret = self._first_bytes[self._seek :]
+            else:
+                ret = self._first_bytes[
+                    self._seek : min(self._size_check, self._seek + size)
+                ]
+        if size is None:
+            ret += raw_read_cb(None)
+        elif len(ret) < size:
+            ret += raw_read_cb(size - len(ret))
+        self._seek += len(ret)
+        return ret
+
+    def read(self, size=None):
+        """Read *size* bytes from the file."""
+        return self._generic_read(size, self._rawio.read)
+
+    def read1(self, size=None):
+        """Read *size* bytes from the file (at once)."""
+        return self._generic_read(size, self._rawio.read1)
+
+    def readable(self):
+        """Is this file-like object readable ?"""
+        return True

vortex/util/roles.py

@@ -0,0 +1,49 @@
+"""
+Factory for named roles.
+"""
+
+#: No automatic export
+__all__ = []
+
+_activetag = "default"
+
+
+def stdfactoryrole(role):
+    """Standard processing for role names."""
+    return "".join([s[0].upper() + s[1:] for s in role.split()])
+
+
+def switchfactory(tag="default"):
+    """Switch the current active factory to the existing one identified through its ``tag``."""
+    if tag in _rolesgateway:
+        global _activetag
+        _activetag = tag
+
+
+def setfactoryrole(factory=None, tag=None):
+    """
+    Defines the specified ``factory`` function as the current processing role translator
+    associated with ``tag``.
+    """
+    global _activetag
+    if not tag:
+        tag = _activetag
+    if factory and tag:
+        _rolesgateway[tag] = factory
+
+
+def setrole(role, tag=None):
+    """
+    Entry point for handling strings ``role``.
+    Returns the processed string according to the current active factory name
+    or using the one associated with ``tag``.
+    """
+    if not role:
+        return None
+    global _activetag
+    if not tag:
+        tag = _activetag
+    return _rolesgateway[tag](role)
+
+
+_rolesgateway = dict(default=stdfactoryrole)

vortex/util/storefunctions.py

@@ -0,0 +1,129 @@
+"""
+General purpose functions that can be used in conjunction with the
+:class:`~vortex.data.stores.FunctionStore`.
+"""
+
+import io
+
+from footprints import proxy as fpx
+
+from vortex.data.stores import FunctionStoreCallbackError
+from vortex.tools.env import vartrue
+from vortex import sessions
+from . import helpers
+
+#: No automatic export
+__all__ = []
+
+
+def mergecontents(options):
+    """
+    Merge the DataContent's of the Section objects designated by the
+    *role* option.
+
+    An additional *sort* option may be provided if the resulting merged file
+    like object needs to be sorted.
+
+    :param options: The only argument is a dictionary that contains all the options
+                    passed to the store plus anything from the query part of the URI.
+
+    :return: Content of the desired local file/container
+
+    :rtype: A file like object
+    """
+    todo = options.get("role", None)
+    sort = vartrue.match(
+        options.get(
+            "sort",
+            [
+                "false",
+            ],
+        ).pop()
+    )
+    if todo is not None:
+        ctx = sessions.current().context
+        sections = list()
+        for a_role in todo:
+            sections.extend(ctx.sequence.effective_inputs(role=a_role))
+        if len(sections) == 0:
+            raise FunctionStoreCallbackError(
+                "Nothing to store: the effective inputs sequence is void."
+            )
+        newcontent = helpers.merge_contents(sections)
+        if sort:
+            newcontent.sort()
+    else:
+        raise FunctionStoreCallbackError(
+            "At least one *role* option must be provided"
+        )
+    # Create a Virtual container and dump the new content inside it
+    virtualcont = fpx.container(incore=True)
+    newcontent.rewrite(virtualcont)
+    virtualcont.rewind()
+    # Force the new container to be in bytes mode
+    if virtualcont.actualmode and "b" not in virtualcont.actualmode:
+        virtualcont_b = fpx.container(incore=True, mode="w+b")
+        virtualcont_b.write(virtualcont.read().encode(encoding="utf-8"))
+        virtualcont = virtualcont_b
+    return virtualcont
+
+
+def dumpinputs(options):
+    """
+    Dump the content of the sequence's effective inputs into a JSON file
+
+    :note: the effective=False option can be provided. If so, all input sections
+           are dumped.
+
+    :return: a file like object
+    """
+    t = sessions.current()
+    ctx = t.context
+    if vartrue.match(
+        options.get(
+            "effective",
+            [
+                "true",
+            ],
+        ).pop()
+    ):
+        sequence = ctx.sequence.effective_inputs()
+    else:
+        sequence = list(ctx.sequence.inputs())
+    if len(sequence) == 0:
+        raise FunctionStoreCallbackError(
+            "Nothing to store: the effective inputs sequence is void."
+        )
+    fileout = io.StringIO()
+    t.sh.json_dump([s.as_dict() for s in sequence], fileout, indent=4)
+    return fileout
+
+
+def defaultinput(options):
+    """
+    Dump the content of a fake section into a JSON file
+    """
+    prefix = "d_input_"
+    content = dict()
+
+    def export_value(v):
+        if hasattr(v, "footprint_export"):
+            return v.footprint_export()
+        elif hasattr(v, "export_dict"):
+            return v.export_dict()
+        else:
+            return v
+
+    for k, v in options.items():
+        if isinstance(k, str) and k.startswith(prefix):
+            content[k[len(prefix) :]] = export_value(v)
+    t = sessions.current()
+    fileout = io.StringIO()
+    t.sh.json_dump(
+        [
+            content,
+        ],
+        fileout,
+        indent=4,
+    )
+    return fileout

vortex/util/structs.py

@@ -0,0 +1,26 @@
+"""
+This module defines common base classes for miscellaneous purposes.
+"""
+
+import json
+
+from bronx.fancies import loggers
+
+#: No automatic export
+__all__ = []
+
+logger = loggers.getLogger(__name__)
+
+
+class ShellEncoder(json.JSONEncoder):
+    """Encoder for :mod:`json` dumps method."""
+
+    def default(self, obj):
+        """Overwrite the default encoding if the current object has a ``export_dict`` method."""
+        if hasattr(obj, "export_dict"):
+            return obj.export_dict()
+        elif hasattr(obj, "footprint_export"):
+            return obj.footprint_export()
+        elif hasattr(obj, "__dict__"):
+            return vars(obj)
+        return super().default(obj)