vortex-nwp 2.0.0b1__py3-none-any.whl → 2.0.0b2__py3-none-any.whl

vortex/util/empty.py

@@ -15,10 +15,10 @@ class DataConst:
 
     def __init__(self, **kw):
         self.__dict__.update(kw)
-        logger.debug('DataConst init %s', self)
+        logger.debug("DataConst init %s", self)
 
     def __str__(self):
-        return super().__str__() + ' : ' + str(sorted(self.__dict__.keys()))
+        return super().__str__() + " : " + str(sorted(self.__dict__.keys()))
 
     def __contains__(self, item):
         return item in self.__dict__

vortex/util/helpers.py

@@ -18,6 +18,7 @@ logger = loggers.getLogger(__name__)
 
 class InputCheckerError(Exception):
     """Exception raised when the Input checking process fails."""
+
     pass
 
 
@@ -38,15 +39,17 @@ def generic_input_checker(grouping_keys, min_items, *rhandlers, **kwargs):
     """
 
     if len(rhandlers) == 0:
-        raise ValueError('At least one resource handler have to be provided')
+        raise ValueError("At least one resource handler have to be provided")
     # Just in case min_items is not an int...
     min_items = int(min_items)
 
     # Create a flat ResourceHandlers list (rhandlers may consists of lists)
     flat_rhlist = []
     flat_rhmandatory = []
-    for inlist, outlist in ((rhandlers, flat_rhlist),
-                            (kwargs.pop('mandatory', []), flat_rhmandatory),):
+    for inlist, outlist in (
+        (rhandlers, flat_rhlist),
+        (kwargs.pop("mandatory", []), flat_rhmandatory),
+    ):
         for rh in inlist:
             if isinstance(rh, list) or isinstance(rh, tuple):
                 outlist.extend(rh)
@@ -63,46 +66,61 @@ def generic_input_checker(grouping_keys, min_items, *rhandlers, **kwargs):
         rhgroups[tuple(keylist)].append(rh)
 
     candidateslist = [
-        fp.stdtypes.FPDict({k: v
-                            for k, v in zip(grouping_keys, group)
-                            if v is not None})
+        fp.stdtypes.FPDict(
+            {k: v for k, v in zip(grouping_keys, group) if v is not None}
+        )
         for group in rhgroups.keys()
     ]
 
     # Activate FTP connections pooling (for enhanced performances)
     t = sessions.current()
     with t.sh.ftppool():
-
         # Check mandatory stuff
         mychecks = [(rh, rh.check()) for rh in flat_rhmandatory]
         if not all([acheck[1] for acheck in mychecks]):
             for rh in [acheck[0] for acheck in mychecks if not acheck[1]]:
                 logger.error("  Missing location: %s", str(rh.locate()))
-            raise InputCheckerError("Some of the mandatory resources are missing.")
+            raise InputCheckerError(
+                "Some of the mandatory resources are missing."
+            )
 
         # Check call for non-mandatory stuff
         outputlist = list()
         # Is the check real or a delusion ?
-        fakecheck = kwargs.pop('fakecheck', False)
+        fakecheck = kwargs.pop("fakecheck", False)
         #  The keys are sorted so that results remains reproducible
         for grouping_values in sorted(rhgroups.keys()):
-            mychecks = [(rh, fakecheck or rh.check()) for rh in rhgroups[grouping_values]]
-            groupid = fp.stdtypes.FPDict({k: v
-                                          for k, v in zip(grouping_keys, grouping_values)
-                                          if v is not None})
+            mychecks = [
+                (rh, fakecheck or rh.check())
+                for rh in rhgroups[grouping_values]
+            ]
+            groupid = fp.stdtypes.FPDict(
+                {
+                    k: v
+                    for k, v in zip(grouping_keys, grouping_values)
+                    if v is not None
+                }
+            )
             if all([acheck[1] for acheck in mychecks]):
                 outputlist.append(groupid)
-                logger.info("Group (%s): All the input files are accounted for.", str(groupid))
+                logger.info(
+                    "Group (%s): All the input files are accounted for.",
+                    str(groupid),
+                )
             else:
-                logger.warning("Group (%s): Discarded because some of the input files are missing (see below).",
-                               str(groupid))
+                logger.warning(
+                    "Group (%s): Discarded because some of the input files are missing (see below).",
+                    str(groupid),
+                )
                 for rh in [acheck[0] for acheck in mychecks if not acheck[1]]:
                     logger.warning("  Missing location: %s", str(rh.locate()))
 
     # Enforce min_items
     if len(outputlist) < min_items:
-        raise InputCheckerError("The number of input groups is too small " +
-                                "({:d} < {:d})".format(len(outputlist), min_items))
+        raise InputCheckerError(
+            "The number of input groups is too small "
+            + "({:d} < {:d})".format(len(outputlist), min_items)
+        )
 
     return fp.stdtypes.FPList(outputlist), fp.stdtypes.FPList(candidateslist)
 
@@ -112,8 +130,12 @@ def members_input_checker(min_items, *rhandlers, **kwargs):
     This is a shortcut for the generic_input_checher: only the member number is
     considered and the return values corresponds to a list of members.
     """
-    mlist = [desc['member'] for desc in generic_input_checker(('member', ), min_items,
-                                                              *rhandlers, **kwargs)[0]]
+    mlist = [
+        desc["member"]
+        for desc in generic_input_checker(
+            ("member",), min_items, *rhandlers, **kwargs
+        )[0]
+    ]
     return fp.stdtypes.FPList(sorted(mlist))
 
 
@@ -122,8 +144,12 @@ def colorfull_input_checker(min_items, *rhandlers, **kwargs):
     This is a shortcut for the generic_input_checher: it returns a list of
     dictionaries that described the available data.
     """
-    return generic_input_checker(('vapp', 'vconf', 'experiment', 'cutoff', 'date', 'member'),
-                                 min_items, *rhandlers, **kwargs)
+    return generic_input_checker(
+        ("vapp", "vconf", "experiment", "cutoff", "date", "member"),
+        min_items,
+        *rhandlers,
+        **kwargs,
+    )
 
 
 def merge_contents(*kargs):
@@ -176,9 +202,15 @@ def mix_list(list_elements, date=None, member=None):
         rgen.seed(seed)
     else:
         logger.info("The random seed not initialised")
-    logger.debug("The list of elements is %s.", " ".join([str(x) for x in list_elements]))
+    logger.debug(
+        "The list of elements is %s.",
+        " ".join([str(x) for x in list_elements]),
+    )
     result_list_elements = list_elements
     result_list_elements.sort()
     rgen.shuffle(result_list_elements)
-    logger.debug("The mixed list of elements is %s.", " ".join([str(x) for x in result_list_elements]))
+    logger.debug(
+        "The mixed list of elements is %s.",
+        " ".join([str(x) for x in result_list_elements]),
+    )
     return result_list_elements

vortex/util/introspection.py

@@ -20,10 +20,10 @@ class Sherlock:
 
     def __init__(self, **kw):
         self.verbose = False
-        self.ticket = kw.pop('ticket', sessions.current())
+        self.ticket = kw.pop("ticket", sessions.current())
         self.glove = self.ticket.glove
         self.__dict__.update(kw)
-        logger.debug('Sherlock init %s', self)
+        logger.debug("Sherlock init %s", self)
 
     def rstfile(self, modpath):
         """Return the sphinx documentation associated to module reference or module path given."""
@@ -31,28 +31,34 @@ class Sherlock:
             modpath = modpath.__file__
         subpath = modpath
         for installpath in self.glove.sitesrc:
-            subpath = re.sub(installpath, '', subpath)
-        subpath = re.sub(r'\.pyc?', '', subpath)
-        subpath = subpath.split('/')
-        if subpath[-1] == '__init__':
+            subpath = re.sub(installpath, "", subpath)
+        subpath = re.sub(r"\.pyc?", "", subpath)
+        subpath = subpath.split("/")
+        if subpath[-1] == "__init__":
             subpath[-1] = subpath[-2]
-        subpath[-1] += '.rst'
+        subpath[-1] += ".rst"
 
-        subpath[1:1] = ['library', ]
-        return self.glove.sitedoc + '/'.join(subpath)
+        subpath[1:1] = [
+            "library",
+        ]
+        return self.glove.sitedoc + "/".join(subpath)
 
     def rstshort(self, filename):
         """Return relative path name of ``filename`` according to :meth:`siteroot`."""
-        return re.sub(self.glove.siteroot, '', filename)[1:]
+        return re.sub(self.glove.siteroot, "", filename)[1:]
 
     def getlocalmembers(self, obj, topmodule=None):
         """Return members of the module ``obj`` which are defined in the source file of the module."""
         objs = dict()
         if topmodule is None:
             topmodule = obj
-        modfile = topmodule.__file__.rstrip('c')
+        modfile = topmodule.__file__.rstrip("c")
         for x, y in inspect.getmembers(obj):
-            if inspect.isclass(y) or inspect.isfunction(y) or inspect.ismethod(y):
+            if (
+                inspect.isclass(y)
+                or inspect.isfunction(y)
+                or inspect.ismethod(y)
+            ):
                 try:
                     if modfile == inspect.getsourcefile(y):
                         if self.verbose:

vortex/util/iosponge.py

@@ -23,7 +23,9 @@ class IoSponge(io.BufferedIOBase):
     that limit the maximum of *size_check* and *guessed_size* is returned.
     """
 
-    def __init__(self, rawio, size_check=IOSPONGE_DEFAULT_SIZECHECK, guessed_size=0):
+    def __init__(
+        self, rawio, size_check=IOSPONGE_DEFAULT_SIZECHECK, guessed_size=0
+    ):
         """
         :param file rawio: Any kind of file-like object
         :param int size_check: The first size_check bytes will be buffered in
@@ -50,12 +52,14 @@ class IoSponge(io.BufferedIOBase):
         return self._seek
 
     def _generic_read(self, size, raw_read_cb):
-        ret = b''
+        ret = b""
         if self._seek < len(self._first_bytes):
             if size is None:
-                ret = self._first_bytes[self._seek:]
+                ret = self._first_bytes[self._seek :]
             else:
-                ret = self._first_bytes[self._seek:min(self._size_check, self._seek + size)]
+                ret = self._first_bytes[
+                    self._seek : min(self._size_check, self._seek + size)
+                ]
         if size is None:
             ret += raw_read_cb(None)
         elif len(ret) < size:

vortex/util/roles.py

@@ -5,15 +5,15 @@ Factory for named roles.
 #: No automatic export
 __all__ = []
 
-_activetag = 'default'
+_activetag = "default"
 
 
 def stdfactoryrole(role):
     """Standard processing for role names."""
-    return ''.join([s[0].upper() + s[1:] for s in role.split()])
+    return "".join([s[0].upper() + s[1:] for s in role.split()])
 
 
-def switchfactory(tag='default'):
+def switchfactory(tag="default"):
     """Switch the current active factory to the existing one identified through its ``tag``."""
     if tag in _rolesgateway:
         global _activetag
@@ -46,6 +46,4 @@ def setrole(role, tag=None):
     return _rolesgateway[tag](role)
 
 
-_rolesgateway = dict(
-    default=stdfactoryrole
-)
+_rolesgateway = dict(default=stdfactoryrole)

vortex/util/storefunctions.py

@@ -31,28 +31,39 @@ def mergecontents(options):
 
     :rtype: A file like object
     """
-    todo = options.get('role', None)
-    sort = vartrue.match(options.get('sort', ['false', ]).pop())
+    todo = options.get("role", None)
+    sort = vartrue.match(
+        options.get(
+            "sort",
+            [
+                "false",
+            ],
+        ).pop()
+    )
     if todo is not None:
         ctx = sessions.current().context
         sections = list()
         for a_role in todo:
             sections.extend(ctx.sequence.effective_inputs(role=a_role))
         if len(sections) == 0:
-            raise FunctionStoreCallbackError("Nothing to store: the effective inputs sequence is void.")
+            raise FunctionStoreCallbackError(
+                "Nothing to store: the effective inputs sequence is void."
+            )
         newcontent = helpers.merge_contents(sections)
         if sort:
             newcontent.sort()
     else:
-        raise FunctionStoreCallbackError('At least one *role* option must be provided')
+        raise FunctionStoreCallbackError(
+            "At least one *role* option must be provided"
+        )
     # Create a Virtual container and dump the new content inside it
     virtualcont = fpx.container(incore=True)
     newcontent.rewrite(virtualcont)
     virtualcont.rewind()
     # Force the new container to be in bytes mode
-    if virtualcont.actualmode and 'b' not in virtualcont.actualmode:
-        virtualcont_b = fpx.container(incore=True, mode='w+b')
-        virtualcont_b.write(virtualcont.read().encode(encoding='utf-8'))
+    if virtualcont.actualmode and "b" not in virtualcont.actualmode:
+        virtualcont_b = fpx.container(incore=True, mode="w+b")
+        virtualcont_b.write(virtualcont.read().encode(encoding="utf-8"))
         virtualcont = virtualcont_b
     return virtualcont
 
@@ -68,12 +79,21 @@ def dumpinputs(options):
     """
     t = sessions.current()
     ctx = t.context
-    if vartrue.match(options.get('effective', ['true', ]).pop()):
+    if vartrue.match(
+        options.get(
+            "effective",
+            [
+                "true",
+            ],
+        ).pop()
+    ):
         sequence = ctx.sequence.effective_inputs()
     else:
         sequence = list(ctx.sequence.inputs())
     if len(sequence) == 0:
-        raise FunctionStoreCallbackError("Nothing to store: the effective inputs sequence is void.")
+        raise FunctionStoreCallbackError(
+            "Nothing to store: the effective inputs sequence is void."
+        )
     fileout = io.StringIO()
     t.sh.json_dump([s.as_dict() for s in sequence], fileout, indent=4)
     return fileout
@@ -87,17 +107,23 @@ def defaultinput(options):
     content = dict()
 
     def export_value(v):
-        if hasattr(v, 'footprint_export'):
+        if hasattr(v, "footprint_export"):
             return v.footprint_export()
-        elif hasattr(v, 'export_dict'):
+        elif hasattr(v, "export_dict"):
             return v.export_dict()
         else:
             return v
 
     for k, v in options.items():
         if isinstance(k, str) and k.startswith(prefix):
-            content[k[len(prefix):]] = export_value(v)
+            content[k[len(prefix) :]] = export_value(v)
     t = sessions.current()
     fileout = io.StringIO()
-    t.sh.json_dump([content, ], fileout, indent=4)
+    t.sh.json_dump(
+        [
+            content,
+        ],
+        fileout,
+        indent=4,
+    )
     return fileout

vortex/util/structs.py

@@ -17,10 +17,10 @@ class ShellEncoder(json.JSONEncoder):
 
     def default(self, obj):
         """Overwrite the default encoding if the current object has a ``export_dict`` method."""
-        if hasattr(obj, 'export_dict'):
+        if hasattr(obj, "export_dict"):
             return obj.export_dict()
-        elif hasattr(obj, 'footprint_export'):
+        elif hasattr(obj, "footprint_export"):
             return obj.footprint_export()
-        elif hasattr(obj, '__dict__'):
+        elif hasattr(obj, "__dict__"):
             return vars(obj)
         return super().default(obj)

vortex/util/worker.py

@@ -47,12 +47,14 @@ class VortexWorker:
     See :mod:`vortex.gloves`.
     """
 
-    _PRIVATESESSION_TAG = 'asyncworker_view'
-    _PRIVATEGLOVE_TAG = 'asyncworker_id'
+    _PRIVATESESSION_TAG = "asyncworker_view"
+    _PRIVATEGLOVE_TAG = "asyncworker_id"
     _PRIVATESESSION = None
     _PRIVATEMODULES = set()
 
-    def __init__(self, modules=tuple(), verbose=False, logger=None, profile=None):
+    def __init__(
+        self, modules=tuple(), verbose=False, logger=None, profile=None
+    ):
         self._logger = logger
         self._modules = modules
         self._context_lock = False
@@ -74,21 +76,22 @@ class VortexWorker:
         """The session associated with Async Worker."""
         if self._PRIVATESESSION is None:
             import vortex
+
             t = vortex.sessions.get(
                 tag=self._PRIVATESESSION_TAG,
                 glove=vortex.sessions.getglove(
-                    tag=self._PRIVATEGLOVE_TAG,
-                    profile=self.profile
-                )
+                    tag=self._PRIVATEGLOVE_TAG, profile=self.profile
+                ),
             )
             sh = t.system()
             import vortex.tools.lfi  # @UnusedImport
             import vortex.tools.grib  # @UnusedImport
             import vortex.tools.folder  # @UnusedImport
             import footprints as fp
-            fp.proxy.addon(kind='lfi', shell=sh)
-            fp.proxy.addon(kind='grib', shell=sh)
-            fp.proxy.addon(kind='allfolders', shell=sh, verboseload=False)
+
+            fp.proxy.addon(kind="lfi", shell=sh)
+            fp.proxy.addon(kind="grib", shell=sh)
+            fp.proxy.addon(kind="allfolders", shell=sh, verboseload=False)
             self._PRIVATESESSION = t
         return self._PRIVATESESSION
 
@@ -100,8 +103,8 @@ class VortexWorker:
         if not self.verbose:
             # footprints & bronx can be very talkative... we try to limit that !
             global_level = logger.getEffectiveLevel()
-            f_logger = loggers.getLogger('footprints')
-            b_logger = loggers.getLogger('bronx')
+            f_logger = loggers.getLogger("footprints")
+            b_logger = loggers.getLogger("bronx")
             if global_level <= logging.INFO and not self.verbose:
                 f_logger.setLevel(logging.INFO)
                 b_logger.setLevel(logging.INFO)
@@ -111,7 +114,9 @@ class VortexWorker:
 
     def __enter__(self, *args):
         if self._context_lock:
-            raise RuntimeError('Imbricated context manager calls are forbidden.')
+            raise RuntimeError(
+                "Imbricated context manager calls are forbidden."
+            )
         self._context_lock = True
         if self.logger is None:
             self._logger = logger
@@ -119,6 +124,7 @@ class VortexWorker:
             self.reset_loggers(self.logger)
         # Activate our own session
         import vortex
+
         self._context_prev_ticket = vortex.sessions.current()
         if not self.session.active:
             self.session.activate()
@@ -128,23 +134,29 @@ class VortexWorker:
                 self.session.sh.import_module(modname)
                 self._PRIVATEMODULES.add(modname)
         # Ok, let's talk...
-        self.logger.info('VORTEX enter glove_profile=%s ', self.session.glove.profile)
-        self.logger.debug('       modules=%s addons=%s', self.modules, self.session.sh.loaded_addons())
+        self.logger.info(
+            "VORTEX enter glove_profile=%s ", self.session.glove.profile
+        )
+        self.logger.debug(
+            "       modules=%s addons=%s",
+            self.modules,
+            self.session.sh.loaded_addons(),
+        )
         return self
 
     def __exit__(self, exc_type, exc_value, exc_traceback):
         """Well... nothing much to do..."""
         if exc_value is not None:
-            self.logger.critical('VORTEX exits on error', exc_info=exc_value)
+            self.logger.critical("VORTEX exits on error", exc_info=exc_value)
             self.rc = False
         else:
-            self.logger.debug('VORTEX exits nicely.')
+            self.logger.debug("VORTEX exits nicely.")
         self._context_prev_ticket.activate()
         self._context_lock = False
         return True
 
 
-if __name__ == '__main__':
+if __name__ == "__main__":
     import doctest
 
     doctest.testmod(verbose=False)

vortex_nwp-2.0.0b2.dist-info/METADATA

@@ -0,0 +1,66 @@
+Metadata-Version: 2.2
+Name: vortex-nwp
+Version: 2.0.0b2
+Summary: A Python library to write Numerical Weather Prediction pipelines components
+Author-email: The Vortex Team <vortex.support@meteo.fr>
+License: CECILL-C
+Classifier: Development Status :: 4 - Beta
+Classifier: License :: CeCILL-C Free Software License Agreement (CECILL-C)
+Requires-Python: >=3.7
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: bronx
+Requires-Dist: footprints
+Requires-Dist: taylorism
+Requires-Dist: tomli
+Requires-Dist: arpifs_listings
+Provides-Extra: docs
+Requires-Dist: sphinx; extra == "docs"
+Requires-Dist: sphinx-book-theme; extra == "docs"
+Requires-Dist: sphinx-copybutton; extra == "docs"
+Provides-Extra: dev
+Requires-Dist: ruff==0.9.1; extra == "dev"
+Requires-Dist: pytest; extra == "dev"
+
+## vortex
+
+A Python library to write individual tasks in Numerical Weather
+Prediction pipelines.
+
+![A blue coloured vortex pointing downwards on a lighter blue background](vortex.png)
+
+Experiments in Numerical Weather Prediction (NWP) and related fields
+consist in a series of computational tasks that can depend on each
+other's output data. Each task is typically made of three successive
+steps:
+
+1. Fetch required input data.
+2. Execute a program.
+3. Make the program's output data available to subsequent tasks in the
+   pipeline.
+
+Tasks have historically been written in some variant of the UNIX
+shell, which was convenient to interact with the file system, manage
+environment variables and execute programs.  As NWP pipelines and
+tasks grow more and more complex, however, there is a need for a
+language providing more abstraction and code reuse mechanisms.
+
+On top of the popular Python language, *vortex* provides abstractions
+that encapsulate running -- potentially distributed -- programs as
+well as fetching and storing the data they consume and generate.
+
+### Documentation
+
+The documentation is available at [vortex-nwp.readthedocs.io](https://vortex-nwp.readthedocs.io).
+
+### Installation
+
+Vortex can be installed using `pip` like most Python packages:
+
+```bash
+pip install vortex-nwp
+```
+
+### Contributing
+
+See [CONTRIBUTING.md](CONTRIBUTING.md).