vortex-nwp 2.0.0b1__py3-none-any.whl

vortex/util/empty.py

@@ -0,0 +1,24 @@
+"""
+An empty module to be filled with some kind of blackholes objects.
+"""
+
+from bronx.fancies import loggers
+
+#: No automatic export
+__all__ = []
+
+logger = loggers.getLogger(__name__)
+
+
+class DataConst:
+    """Constants stored as raw attributes."""
+
+    def __init__(self, **kw):
+        self.__dict__.update(kw)
+        logger.debug('DataConst init %s', self)
+
+    def __str__(self):
+        return super().__str__() + ' : ' + str(sorted(self.__dict__.keys()))
+
+    def __contains__(self, item):
+        return item in self.__dict__

vortex/util/helpers.py

@@ -0,0 +1,184 @@
+"""
+Some convenient functions that may simplify scripts
+"""
+
+from collections import defaultdict
+
+from bronx.compat import random
+from bronx.fancies import loggers
+from bronx.stdtypes.date import Date
+import footprints as fp
+
+from vortex.data.handlers import Handler
+from vortex.layout.dataflow import Section
+from vortex import sessions
+
+logger = loggers.getLogger(__name__)
+
+
+class InputCheckerError(Exception):
+    """Exception raised when the Input checking process fails."""
+    pass
+
+
+def generic_input_checker(grouping_keys, min_items, *rhandlers, **kwargs):
+    """
+    Check which input resources are present.
+
+    First, the resource handlers (*rhandlers* attribute) are split
+    into groups based on the values of their properties (only the properties
+    specified in the *grouping_keys* attribute are considered).
+
+    Then, for each group, the **check** method is called upon the resource
+    handlers. The group description is returned only if the **check** call
+    succeed for all the members of the group.
+
+    If the number of groups successfully checked is lower than *min_items*,
+    an :class:`InputCheckerError` exception is raised.
+    """
+
+    if len(rhandlers) == 0:
+        raise ValueError('At least one resource handler have to be provided')
+    # Just in case min_items is not an int...
+    min_items = int(min_items)
+
+    # Create a flat ResourceHandlers list (rhandlers may consists of lists)
+    flat_rhlist = []
+    flat_rhmandatory = []
+    for inlist, outlist in ((rhandlers, flat_rhlist),
+                            (kwargs.pop('mandatory', []), flat_rhmandatory),):
+        for rh in inlist:
+            if isinstance(rh, list) or isinstance(rh, tuple):
+                outlist.extend(rh)
+            else:
+                outlist.append(rh)
+
+    # Extract the group informations for each of the resource handlers
+    rhgroups = defaultdict(list)
+    for rh in flat_rhlist:
+        keylist = list()
+        for key in grouping_keys:
+            value = rh.wide_key_lookup(key, exports=True, fatal=False)
+            keylist.append(value)
+        rhgroups[tuple(keylist)].append(rh)
+
+    candidateslist = [
+        fp.stdtypes.FPDict({k: v
+                            for k, v in zip(grouping_keys, group)
+                            if v is not None})
+        for group in rhgroups.keys()
+    ]
+
+    # Activate FTP connections pooling (for enhanced performances)
+    t = sessions.current()
+    with t.sh.ftppool():
+
+        # Check mandatory stuff
+        mychecks = [(rh, rh.check()) for rh in flat_rhmandatory]
+        if not all([acheck[1] for acheck in mychecks]):
+            for rh in [acheck[0] for acheck in mychecks if not acheck[1]]:
+                logger.error("  Missing location: %s", str(rh.locate()))
+            raise InputCheckerError("Some of the mandatory resources are missing.")
+
+        # Check call for non-mandatory stuff
+        outputlist = list()
+        # Is the check real or a delusion ?
+        fakecheck = kwargs.pop('fakecheck', False)
+        #  The keys are sorted so that results remains reproducible
+        for grouping_values in sorted(rhgroups.keys()):
+            mychecks = [(rh, fakecheck or rh.check()) for rh in rhgroups[grouping_values]]
+            groupid = fp.stdtypes.FPDict({k: v
+                                          for k, v in zip(grouping_keys, grouping_values)
+                                          if v is not None})
+            if all([acheck[1] for acheck in mychecks]):
+                outputlist.append(groupid)
+                logger.info("Group (%s): All the input files are accounted for.", str(groupid))
+            else:
+                logger.warning("Group (%s): Discarded because some of the input files are missing (see below).",
+                               str(groupid))
+                for rh in [acheck[0] for acheck in mychecks if not acheck[1]]:
+                    logger.warning("  Missing location: %s", str(rh.locate()))
+
+    # Enforce min_items
+    if len(outputlist) < min_items:
+        raise InputCheckerError("The number of input groups is too small " +
+                                "({:d} < {:d})".format(len(outputlist), min_items))
+
+    return fp.stdtypes.FPList(outputlist), fp.stdtypes.FPList(candidateslist)
+
+
+def members_input_checker(min_items, *rhandlers, **kwargs):
+    """
+    This is a shortcut for the generic_input_checher: only the member number is
+    considered and the return values corresponds to a list of members.
+    """
+    mlist = [desc['member'] for desc in generic_input_checker(('member', ), min_items,
+                                                              *rhandlers, **kwargs)[0]]
+    return fp.stdtypes.FPList(sorted(mlist))
+
+
+def colorfull_input_checker(min_items, *rhandlers, **kwargs):
+    """
+    This is a shortcut for the generic_input_checher: it returns a list of
+    dictionaries that described the available data.
+    """
+    return generic_input_checker(('vapp', 'vconf', 'experiment', 'cutoff', 'date', 'member'),
+                                 min_items, *rhandlers, **kwargs)
+
+
+def merge_contents(*kargs):
+    """Automatically merge several DataContents.
+
+    Example:
+    .. code-block:: python
+
+        mergedcontent = merge_contents(content1, content2, content3)
+        # With a list
+        mergedcontent = merge_contents([content1, content2, content3])
+        # With a list of ResourceHandlers (e.g. as returned by toolbox.input)
+        mergedcontent = merge_contents([rh1, rh2, rh3])
+        # With a list of Sections (e.g. as returned by effective_inputs)
+        mergedcontent = merge_contents([section1, section2, section3])
+
+    """
+    # Expand list or tuple elements
+    ctlist = list()
+    for elt in kargs:
+        if isinstance(elt, (list, tuple)):
+            ctlist.extend(elt)
+        else:
+            ctlist.append(elt)
+    # kargs may be a list of resource handlers (as returned by the toolbox)
+    if all([isinstance(obj, Handler) for obj in ctlist]):
+        ctlist = [obj.contents for obj in ctlist]
+    # kargs may be a list of sections
+    elif all([isinstance(obj, Section) for obj in ctlist]):
+        ctlist = [obj.rh.contents for obj in ctlist]
+    # Take the first content as a model for the new object
+    newcontent = ctlist[0].__class__()
+    newcontent.merge(*ctlist)
+    return newcontent
+
+
+def mix_list(list_elements, date=None, member=None):
+    """Mix a list using a determined seed, if member and/or date are present."""
+    dateinfo = date if date is None else Date(date)
+    memberinfo = member if member is None else int(member)
+    rgen = random.Random()
+    if (dateinfo is not None) or (memberinfo is not None):
+        if dateinfo is not None:
+            seed = dateinfo.epoch * 100
+        else:
+            seed = 9999999
+        if memberinfo:
+            seed = seed // memberinfo
+        logger.debug("The random seed is %s.", seed)
+        rgen.seed(seed)
+    else:
+        logger.info("The random seed not initialised")
+    logger.debug("The list of elements is %s.", " ".join([str(x) for x in list_elements]))
+    result_list_elements = list_elements
+    result_list_elements.sort()
+    rgen.shuffle(result_list_elements)
+    logger.debug("The mixed list of elements is %s.", " ".join([str(x) for x in result_list_elements]))
+    return result_list_elements

vortex/util/introspection.py

@@ -0,0 +1,63 @@
+"""
+Some convenient functions to explore the source code or its documentation.
+"""
+
+import re
+import inspect
+
+from bronx.fancies import loggers
+
+from vortex import sessions
+
+#: No automatic export
+__all__ = []
+
+logger = loggers.getLogger(__name__)
+
+
+class Sherlock:
+    """Centralized interface to introspection functions."""
+
+    def __init__(self, **kw):
+        self.verbose = False
+        self.ticket = kw.pop('ticket', sessions.current())
+        self.glove = self.ticket.glove
+        self.__dict__.update(kw)
+        logger.debug('Sherlock init %s', self)
+
+    def rstfile(self, modpath):
+        """Return the sphinx documentation associated to module reference or module path given."""
+        if not isinstance(modpath, str):
+            modpath = modpath.__file__
+        subpath = modpath
+        for installpath in self.glove.sitesrc:
+            subpath = re.sub(installpath, '', subpath)
+        subpath = re.sub(r'\.pyc?', '', subpath)
+        subpath = subpath.split('/')
+        if subpath[-1] == '__init__':
+            subpath[-1] = subpath[-2]
+        subpath[-1] += '.rst'
+
+        subpath[1:1] = ['library', ]
+        return self.glove.sitedoc + '/'.join(subpath)
+
+    def rstshort(self, filename):
+        """Return relative path name of ``filename`` according to :meth:`siteroot`."""
+        return re.sub(self.glove.siteroot, '', filename)[1:]
+
+    def getlocalmembers(self, obj, topmodule=None):
+        """Return members of the module ``obj`` which are defined in the source file of the module."""
+        objs = dict()
+        if topmodule is None:
+            topmodule = obj
+        modfile = topmodule.__file__.rstrip('c')
+        for x, y in inspect.getmembers(obj):
+            if inspect.isclass(y) or inspect.isfunction(y) or inspect.ismethod(y):
+                try:
+                    if modfile == inspect.getsourcefile(y):
+                        if self.verbose:
+                            print(x, y)
+                        objs[x] = y
+                except TypeError:
+                    pass
+        return objs

vortex/util/iosponge.py

@@ -0,0 +1,76 @@
+"""
+Provide a File-Like object that reads in the first N bytes in order to count
+them precisely.
+"""
+
+import io
+
+#: No automatic export
+__all__ = []
+
+IOSPONGE_DEFAULT_SIZECHECK = 33 * 1024 * 1024  # 33Mb
+
+
+class IoSponge(io.BufferedIOBase):
+    """Buffer the first bytes in order to compute an accurate size for the
+    underlying stream.
+
+    This class just acts as a buffer. It looks like a file object and should
+    be used as such.
+
+    If the size of the underlying stream is <= *size_check* bytes : the **size**
+    property will return an exact estimate of the file-like object size. Passed
+    that limit the maximum of *size_check* and *guessed_size* is returned.
+    """
+
+    def __init__(self, rawio, size_check=IOSPONGE_DEFAULT_SIZECHECK, guessed_size=0):
+        """
+        :param file rawio: Any kind of file-like object
+        :param int size_check: The first size_check bytes will be buffered in
+            order to be properly accounted for.
+        :param int gressed_size: An estimate of the file-like object size (in
+            bytes)
+        """
+        self._rawio = rawio
+        self._size_check = size_check
+        self._guessed_size = int(guessed_size)
+        self._first_bytes = self._rawio.read(size_check)
+        self._seek = 0
+
+    @property
+    def size(self):
+        """The (exact or estimated)  size of the underlying file-like object."""
+        if len(self._first_bytes) < self._size_check:
+            return len(self._first_bytes)
+        else:
+            return max(len(self._first_bytes), self._guessed_size)
+
+    def tell(self):
+        """The amount of bytes read in this strem."""
+        return self._seek
+
+    def _generic_read(self, size, raw_read_cb):
+        ret = b''
+        if self._seek < len(self._first_bytes):
+            if size is None:
+                ret = self._first_bytes[self._seek:]
+            else:
+                ret = self._first_bytes[self._seek:min(self._size_check, self._seek + size)]
+        if size is None:
+            ret += raw_read_cb(None)
+        elif len(ret) < size:
+            ret += raw_read_cb(size - len(ret))
+        self._seek += len(ret)
+        return ret
+
+    def read(self, size=None):
+        """Read *size* bytes from the file."""
+        return self._generic_read(size, self._rawio.read)
+
+    def read1(self, size=None):
+        """Read *size* bytes from the file (at once)."""
+        return self._generic_read(size, self._rawio.read1)
+
+    def readable(self):
+        """Is this file-like object readable ?"""
+        return True

vortex/util/roles.py

@@ -0,0 +1,51 @@
+"""
+Factory for named roles.
+"""
+
+#: No automatic export
+__all__ = []
+
+_activetag = 'default'
+
+
+def stdfactoryrole(role):
+    """Standard processing for role names."""
+    return ''.join([s[0].upper() + s[1:] for s in role.split()])
+
+
+def switchfactory(tag='default'):
+    """Switch the current active factory to the existing one identified through its ``tag``."""
+    if tag in _rolesgateway:
+        global _activetag
+        _activetag = tag
+
+
+def setfactoryrole(factory=None, tag=None):
+    """
+    Defines the specified ``factory`` function as the current processing role translator
+    associated with ``tag``.
+    """
+    global _activetag
+    if not tag:
+        tag = _activetag
+    if factory and tag:
+        _rolesgateway[tag] = factory
+
+
+def setrole(role, tag=None):
+    """
+    Entry point for handling strings ``role``.
+    Returns the processed string according to the current active factory name
+    or using the one associated with ``tag``.
+    """
+    if not role:
+        return None
+    global _activetag
+    if not tag:
+        tag = _activetag
+    return _rolesgateway[tag](role)
+
+
+_rolesgateway = dict(
+    default=stdfactoryrole
+)

vortex/util/storefunctions.py

@@ -0,0 +1,103 @@
+"""
+General purpose functions that can be used in conjunction with the
+:class:`~vortex.data.stores.FunctionStore`.
+"""
+
+import io
+
+from footprints import proxy as fpx
+
+from vortex.data.stores import FunctionStoreCallbackError
+from vortex.tools.env import vartrue
+from vortex import sessions
+from . import helpers
+
+#: No automatic export
+__all__ = []
+
+
+def mergecontents(options):
+    """
+    Merge the DataContent's of the Section objects designated by the
+    *role* option.
+
+    An additional *sort* option may be provided if the resulting merged file
+    like object needs to be sorted.
+
+    :param options: The only argument is a dictionary that contains all the options
+                    passed to the store plus anything from the query part of the URI.
+
+    :return: Content of the desired local file/container
+
+    :rtype: A file like object
+    """
+    todo = options.get('role', None)
+    sort = vartrue.match(options.get('sort', ['false', ]).pop())
+    if todo is not None:
+        ctx = sessions.current().context
+        sections = list()
+        for a_role in todo:
+            sections.extend(ctx.sequence.effective_inputs(role=a_role))
+        if len(sections) == 0:
+            raise FunctionStoreCallbackError("Nothing to store: the effective inputs sequence is void.")
+        newcontent = helpers.merge_contents(sections)
+        if sort:
+            newcontent.sort()
+    else:
+        raise FunctionStoreCallbackError('At least one *role* option must be provided')
+    # Create a Virtual container and dump the new content inside it
+    virtualcont = fpx.container(incore=True)
+    newcontent.rewrite(virtualcont)
+    virtualcont.rewind()
+    # Force the new container to be in bytes mode
+    if virtualcont.actualmode and 'b' not in virtualcont.actualmode:
+        virtualcont_b = fpx.container(incore=True, mode='w+b')
+        virtualcont_b.write(virtualcont.read().encode(encoding='utf-8'))
+        virtualcont = virtualcont_b
+    return virtualcont
+
+
+def dumpinputs(options):
+    """
+    Dump the content of the sequence's effective inputs into a JSON file
+
+    :note: the effective=False option can be provided. If so, all input sections
+           are dumped.
+
+    :return: a file like object
+    """
+    t = sessions.current()
+    ctx = t.context
+    if vartrue.match(options.get('effective', ['true', ]).pop()):
+        sequence = ctx.sequence.effective_inputs()
+    else:
+        sequence = list(ctx.sequence.inputs())
+    if len(sequence) == 0:
+        raise FunctionStoreCallbackError("Nothing to store: the effective inputs sequence is void.")
+    fileout = io.StringIO()
+    t.sh.json_dump([s.as_dict() for s in sequence], fileout, indent=4)
+    return fileout
+
+
+def defaultinput(options):
+    """
+    Dump the content of a fake section into a JSON file
+    """
+    prefix = "d_input_"
+    content = dict()
+
+    def export_value(v):
+        if hasattr(v, 'footprint_export'):
+            return v.footprint_export()
+        elif hasattr(v, 'export_dict'):
+            return v.export_dict()
+        else:
+            return v
+
+    for k, v in options.items():
+        if isinstance(k, str) and k.startswith(prefix):
+            content[k[len(prefix):]] = export_value(v)
+    t = sessions.current()
+    fileout = io.StringIO()
+    t.sh.json_dump([content, ], fileout, indent=4)
+    return fileout

vortex/util/structs.py

@@ -0,0 +1,26 @@
+"""
+This module defines common base classes for miscellaneous purposes.
+"""
+
+import json
+
+from bronx.fancies import loggers
+
+#: No automatic export
+__all__ = []
+
+logger = loggers.getLogger(__name__)
+
+
+class ShellEncoder(json.JSONEncoder):
+    """Encoder for :mod:`json` dumps method."""
+
+    def default(self, obj):
+        """Overwrite the default encoding if the current object has a ``export_dict`` method."""
+        if hasattr(obj, 'export_dict'):
+            return obj.export_dict()
+        elif hasattr(obj, 'footprint_export'):
+            return obj.footprint_export()
+        elif hasattr(obj, '__dict__'):
+            return vars(obj)
+        return super().default(obj)

vortex/util/worker.py

@@ -0,0 +1,150 @@
+"""
+This package defines a class for default contexts used
+by a PoolWorker process of the Jeeves daemon.
+"""
+
+import logging
+
+from bronx.fancies import loggers
+
+logger = loggers.getLogger(__name__)
+
+
+class AttrDict(dict):
+    """Dict object that can be accessed by attributes.
+
+    >>> obj = AttrDict()
+    >>> obj.test = 'hi'
+    >>> print(obj['test'])
+    hi
+
+    >>> obj['test'] = "bye"
+    >>> print(obj.test)
+    bye
+
+    >>> print(len(obj))
+    1
+
+    >>> obj.clear()
+    >>> print(len(obj))
+    0
+
+    >>> obj.a
+    Traceback (most recent call last):
+        ...
+    AttributeError: 'AttrDict' object has no attribute 'a'
+    """
+
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        self.__dict__ = self
+
+
+class VortexWorker:
+    """Context for a vortex session handled by an asynchronous process such as Jeeves.
+
+    An _oper_ profile should be used from Jeeves: the default is to use a _research_ profile.
+    See :mod:`vortex.gloves`.
+    """
+
+    _PRIVATESESSION_TAG = 'asyncworker_view'
+    _PRIVATEGLOVE_TAG = 'asyncworker_id'
+    _PRIVATESESSION = None
+    _PRIVATEMODULES = set()
+
+    def __init__(self, modules=tuple(), verbose=False, logger=None, profile=None):
+        self._logger = logger
+        self._modules = modules
+        self._context_lock = False
+        self._context_prev_ticket = None
+        self.verbose = verbose
+        self.profile = profile
+        self.rc = True
+
+    @property
+    def logger(self):
+        return self._logger
+
+    @property
+    def modules(self):
+        return self._modules
+
+    @property
+    def session(self):
+        """The session associated with Async Worker."""
+        if self._PRIVATESESSION is None:
+            import vortex
+            t = vortex.sessions.get(
+                tag=self._PRIVATESESSION_TAG,
+                glove=vortex.sessions.getglove(
+                    tag=self._PRIVATEGLOVE_TAG,
+                    profile=self.profile
+                )
+            )
+            sh = t.system()
+            import vortex.tools.lfi  # @UnusedImport
+            import vortex.tools.grib  # @UnusedImport
+            import vortex.tools.folder  # @UnusedImport
+            import footprints as fp
+            fp.proxy.addon(kind='lfi', shell=sh)
+            fp.proxy.addon(kind='grib', shell=sh)
+            fp.proxy.addon(kind='allfolders', shell=sh, verboseload=False)
+            self._PRIVATESESSION = t
+        return self._PRIVATESESSION
+
+    def get_dataset(self, ask):
+        """Struct friendly access to data request."""
+        return AttrDict(ask.data)
+
+    def reset_loggers(self, logger):
+        if not self.verbose:
+            # footprints & bronx can be very talkative... we try to limit that !
+            global_level = logger.getEffectiveLevel()
+            f_logger = loggers.getLogger('footprints')
+            b_logger = loggers.getLogger('bronx')
+            if global_level <= logging.INFO and not self.verbose:
+                f_logger.setLevel(logging.INFO)
+                b_logger.setLevel(logging.INFO)
+            else:
+                f_logger.setLevel(logging.NOTSET)
+                b_logger.setLevel(logging.NOTSET)
+
+    def __enter__(self, *args):
+        if self._context_lock:
+            raise RuntimeError('Imbricated context manager calls are forbidden.')
+        self._context_lock = True
+        if self.logger is None:
+            self._logger = logger
+        else:
+            self.reset_loggers(self.logger)
+        # Activate our own session
+        import vortex
+        self._context_prev_ticket = vortex.sessions.current()
+        if not self.session.active:
+            self.session.activate()
+        # Import extra modules
+        for modname in self.modules:
+            if modname not in self._PRIVATEMODULES:
+                self.session.sh.import_module(modname)
+                self._PRIVATEMODULES.add(modname)
+        # Ok, let's talk...
+        self.logger.info('VORTEX enter glove_profile=%s ', self.session.glove.profile)
+        self.logger.debug('       modules=%s addons=%s', self.modules, self.session.sh.loaded_addons())
+        return self
+
+    def __exit__(self, exc_type, exc_value, exc_traceback):
+        """Well... nothing much to do..."""
+        if exc_value is not None:
+            self.logger.critical('VORTEX exits on error', exc_info=exc_value)
+            self.rc = False
+        else:
+            self.logger.debug('VORTEX exits nicely.')
+        self._context_prev_ticket.activate()
+        self._context_lock = False
+        return True
+
+
+if __name__ == '__main__':
+    import doctest
+
+    doctest.testmod(verbose=False)