vortex-nwp 2.0.0b1__py3-none-any.whl

vortex/algo/mpitools_templates/envelope_wrapper_default.tpl

@@ -0,0 +1,27 @@
+#!${python}
+
+import os
+import sys
+
+sys.path.append('${sitepath}')
+rank = os.environ['${mpirankvariable}']
+
+todolist = {
+${todolist}
+}
+
+bindinglist = {
+${bindinglist}
+}
+
+if bindinglist:
+    from bronx.system.cpus import set_affinity
+    set_affinity(bindinglist[int(rank)])
+    # Also bind threads
+    os.environ['OMP_PROC_BIND'] = 'true'
+
+me = todolist[int(rank)]
+if me[2]:
+    os.environ['OMP_NUM_THREADS'] = str(me[2])
+
+os.execl(me[0], me[0], *me[1])

vortex/algo/mpitools_templates/envelope_wrapper_mpiauto.tpl

@@ -0,0 +1,29 @@
+#!${python}
+
+import os
+import sys
+
+sys.path.append('${sitepath}')
+
+actual_mpirankvariable = os.environ['${mpirankvariable}']
+rank = os.environ[actual_mpirankvariable]
+
+todolist = {
+${todolist}
+}
+
+bindinglist = {
+${bindinglist}
+}
+
+if bindinglist:
+    from bronx.system.cpus import set_affinity
+    set_affinity(bindinglist[int(rank)])
+    # Also bind threads
+    os.environ['OMP_PROC_BIND'] = 'true'
+
+me = todolist[int(rank)]
+if me[2]:
+    os.environ['OMP_NUM_THREADS'] = str(me[2])
+
+os.execl(me[0], me[0], *me[1])

vortex/algo/mpitools_templates/wrapstd_wrapper_default.tpl

@@ -0,0 +1,18 @@
+#!${python}
+
+import os
+import sys
+
+rank = os.environ['${mpirankvariable}']
+
+# Redirect stdout and stderr in a very very crude manner
+if int(rank) > 0:
+    stdout_fno = sys.stdout.fileno()
+    stderr_fno = sys.stderr.fileno()
+    red_outputs = os.open('vwrap_stdeo.{:06d}'.format(int(rank)),
+                          os.O_WRONLY | os.O_CREAT | os.O_TRUNC, 0o644)
+    os.dup2(red_outputs, stdout_fno)
+    os.dup2(red_outputs, stderr_fno)
+    os.close(red_outputs) 
+
+os.execl(sys.argv[1], *sys.argv[1:])

vortex/algo/serversynctools.py

@@ -0,0 +1,170 @@
+"""
+Utility classes to interact with long running binaries.
+"""
+
+import socket
+import sys
+
+import footprints
+from bronx.fancies import loggers
+from vortex import sessions
+from vortex.util import config
+
+# : No automatic export
+__all__ = []
+
+logger = loggers.getLogger(__name__)
+
+
+class ServerSyncTool(footprints.FootprintBase):
+    """
+    :class:`ServerSyncTool` classes are in charge of interactions between the
+    main process and the server.
+    """
+
+    _abstract = True
+    _collector = ('serversynctool',)
+    _footprint = dict(
+        info='Abstract Server Synchronisation Tool',
+        attr=dict(
+            method=dict(
+            ),
+            medium=dict(
+                optional=True,
+            ),
+            raiseonexit=dict(
+                type=bool,
+                optional=True,
+                default=True
+            ),
+            checkinterval=dict(
+                type=int,
+                optional=True,
+                default=10,
+            ),
+        )
+    )
+
+    def __init__(self, *args, **kw):
+        logger.debug('Server Synchronisation Tool init %s', self.__class__)
+        self._check_callback = lambda: True
+        super().__init__(*args, **kw)
+
+    def set_servercheck_callback(self, cb):
+        """Set a callback method that will be called to check the server state."""
+        self._check_callback = cb
+
+    def trigger_wait(self):
+        """Ask the SyncTool to wait for a request."""
+        raise NotImplementedError
+
+    def trigger_run(self):
+        """Indicate that the main process is ready for the server to run the next step.
+
+        It then wait for the server to complete this step.
+        """
+        raise NotImplementedError
+
+    def trigger_stop(self):
+        """Ask the server to stop (gently)."""
+        raise NotImplementedError
+
+
+class ServerSyncSimpleSocket(ServerSyncTool):
+    """Practical implementation of a ServerSyncTool that relies on sockets.
+
+    A script is created (its name is defined by the *medium* attribute): it
+    will be called by the server process before starting any computations. This
+    script and the main process communicate using standard UNIX sockets (through
+    the socket package).
+    """
+
+    _footprint = dict(
+        info='Server Synchronisation Tool that uses a Socket',
+        attr=dict(
+            method=dict(
+                values=['simple_socket'],
+            ),
+            medium=dict(
+                optional=False,
+            ),
+            tplname=dict(
+                optional=True,
+                default='@servsync-simplesocket.tpl',
+            ),
+        )
+    )
+
+    def __init__(self, *args, **kw):
+        super().__init__(*args, **kw)
+        # Create the socket
+        self._socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
+        try:
+            self._socket.bind((socket.getfqdn(), 0))
+        except OSError:
+            self._socket.bind(('localhost', 0))
+        self._socket.settimeout(self.checkinterval)
+        self._socket.listen(1)
+        # Current connection
+        self._socket_conn = None
+        # Create the script that will be called by the server
+        t = sessions.current()
+        tpl = config.load_template(t, self.tplname)
+        with open(self.medium, 'w') as fd:
+            fd.write(tpl.substitute(
+                python=sys.executable,
+                address=self._socket.getsockname(),
+            ))
+        t.sh.chmod(self.medium, 0o555)
+
+    def __del__(self):
+        self._socket.close()
+        if self._socket_conn is not None:
+            logger.warning("The socket is still up... that's odd.")
+        t = sessions.current()
+        if t.sh.path.exists(self.medium):
+            t.sh.remove(self.medium)
+
+    def _command(self, mess):
+        """Send a command (a string) to the server and wait for a response."""
+        if self._socket_conn is not None:
+            logger.info('Sending "%s" to the server.', mess)
+            # NB: For send/recv, the settimeout also applies...
+            self._socket_conn.send(mess.encode(encoding='utf-8'))
+            repl = self._socket_conn.recv(255).decode(encoding='utf-8')
+            logger.info('Server replied "%s" to %s.', repl, mess)
+            self._socket_conn.close()
+            self._socket_conn = None
+            if repl != 'OK':
+                raise ValueError(mess + ' failed')
+            return True
+        else:
+            # This should not happen ! If we are sitting here, it's most likely
+            # that the main process received a signal like SIGTERM...
+            return False
+
+    def trigger_wait(self):
+        logger.info('Waiting for the server to complete')
+        while self._socket_conn is None and self._check_callback():
+            try:
+                self._socket_conn, addr = self._socket.accept()  # @UnusedVariable
+            except socket.timeout:
+                logger.debug('Socket accept timed-out: checking for the server...')
+                self._socket_conn = None
+        if self._socket_conn is None:
+            if self.raiseonexit:
+                raise OSError('Apparently the server died.')
+            else:
+                logger.info('The server stopped.')
+        else:
+            self._socket_conn.settimeout(self.checkinterval)
+            logger.info('The server is now waiting')
+
+    def trigger_run(self):
+        # Tell the server that everything is ready
+        self._command('STEP')
+        # Wait for the server to complete its work
+        self.trigger_wait()
+
+    def trigger_stop(self):
+        return self._command('STOP')

vortex/config.py

@@ -0,0 +1,115 @@
+"""This module provides getter and setter functions to set and get
+the value of configuration options, respectively.
+
+"""
+import tomli
+
+from bronx.fancies import loggers
+
+__all__ = [
+    "load_config",
+    "print_config",
+    "from_config",
+    "set_config",
+    "is_defined",
+]
+
+VORTEX_CONFIG = {}
+
+logger = loggers.getLogger(__name__)
+
+
+def load_config(configpath="vortex.toml"):
+    """Load configuration from a TOML configuration file
+
+    Existing configuration values are overriden. The configuration
+    is expected to have valid TOML syntax, e.g.
+
+    .. code:: toml
+
+       [data-tree]
+       op_rootdir = "/chaine/mxpt001"
+
+       [storage]
+       address = "hendrix.meteo.fr"
+       protocol = "ftp"
+       # ...
+    """
+    global VORTEX_CONFIG
+    try:
+        with open(configpath, "rb") as f:
+             VORTEX_CONFIG = tomli.load(f)
+        print(f"Successfully read configuration file {configpath}")
+    except FileNotFoundError:
+        print(
+            f"Could not read configuration file {configpath}"
+            " (not found)."
+        )
+        print(
+            "Use load_config(/path/to/config) to update the configuration"
+        )
+
+
+def print_config():
+    """Print configuration (key, value) pairs
+    """
+    if VORTEX_CONFIG:
+        for k, v in VORTEX_CONFIG:
+            print(k.upper(), v)
+
+
+def from_config(section, key=None):
+    """Retrieve a configuration key value for a given section.
+
+    If key is ``None``, the whole section is returned as a dictionary.
+
+    """
+    try:
+        subconfig = VORTEX_CONFIG[section]
+    except KeyError as e:
+        print(f"Could not find section {section} in configuration")
+        raise(e)
+
+    if not key:
+        return subconfig
+
+    try:
+        value = subconfig[key]
+    except KeyError as e :
+        print(f"Could not find key {key} in section {section} of configuration")
+        raise(e)
+    return value
+
+
+def set_config(section, key, value):
+    """Set a configuration key to a value"""
+    global VORTEX_CONFIG
+    if section not in VORTEX_CONFIG.keys():
+        VORTEX_CONFIG[section] = {}
+    if key in VORTEX_CONFIG[section]:
+        logger.warning(
+            f"Updating existing configuration {section}:{key}"
+        )
+    VORTEX_CONFIG[section][key] = value
+
+
+def is_defined(section, key=None):
+    """Return whether or not the key is defined for the section.
+
+    If ``key`` is ``None``, return whether or not the section exists
+    in the current configuration.
+
+    """
+    if section not in VORTEX_CONFIG.keys():
+        return False
+    if key:
+        return key in VORTEX_CONFIG[section].keys()
+    return True
+
+
+def get_from_config_w_default(section, key, default):
+    logger.info(f"Reading config value {section}.{key}")
+    try:
+        return from_config(section, key)
+    except KeyError:
+        return default

vortex/data/__init__.py

@@ -0,0 +1,13 @@
+"""
+Abstract classes involved in data management within VORTEX.
+
+Actual resources and custom providers should be defined in dedicated packages.
+"""
+
+from . import handlers, resources, containers, contents, providers, \
+    executables, stores, geometries
+
+#: No automatic export
+__all__ = []
+
+__tocinfoline__ = 'Abstract classes involved in data management within VORTEX'