multivisor 6.0.2__py3-none-any.whl

multivisor/multivisor.py

@@ -0,0 +1,470 @@
+#!/usr/bin/env python
+import copy
+import logging
+import os
+import time
+import weakref
+
+from blinker import signal
+
+try:
+    from ConfigParser import SafeConfigParser as ConfigParser
+except ImportError:
+    from configparser import ConfigParser
+
+import zerorpc
+from gevent import spawn, sleep, joinall
+from supervisor.xmlrpc import Faults
+from supervisor.states import RUNNING_STATES
+
+from .util import sanitize_url, filter_patterns, parse_dict
+
+log = logging.getLogger("multivisor")
+
+
+class Supervisor(dict):
+
+    Null = {
+        "identification": None,
+        "api_version": None,
+        "version": None,
+        "supervisor_version": None,
+        "processes": {},
+        "running": False,
+        "pid": None,
+    }
+
+    def __init__(self, name, url):
+        super(Supervisor, self).__init__(self.Null)
+        self.name = self["name"] = name
+        self.url = self["url"] = url
+        self.log = log.getChild(name)
+        addr = sanitize_url(url, protocol="tcp", host=name, port=9002)
+        self.address = addr["url"]
+        self.host = self["host"] = addr["host"]
+        self.server = zerorpc.Client(self.address)
+        # fill supervisor info before events start coming in
+        self.event_loop = spawn(self.run)
+
+    def __repr__(self):
+        return "{}(name={})".format(self.__class__.__name__, self.name)
+
+    def __eq__(self, other):
+        this, other = dict(self), dict(other)
+        this_p = this.pop("processes")
+        other_p = other.pop("processes")
+        return this == other and list(this_p.keys()) == list(other_p.keys())
+
+    def run(self):
+        last_retry = time.time()
+        while True:
+            try:
+                self.log.info("(re)initializing...")
+                self.refresh()
+                for i, event in enumerate(self.server.event_stream()):
+                    # ignore first event. It serves only to trigger
+                    # connection and avoid TimeoutExpired
+                    if i != 0:
+                        self.handle_event(event)
+            except zerorpc.LostRemote:
+                self.log.info("Lost remote")
+            except zerorpc.TimeoutExpired:
+                self.log.info("Timeout expired")
+            except Exception as err:
+                self.log.warning("Unexpected error %r", err)
+            finally:
+                curr_time = time.time()
+                delta = curr_time - last_retry
+                if delta < 10:
+                    sleep(10 - delta)
+                last_retry = time.time()
+
+    def handle_event(self, event):
+        name = event["eventname"]
+        self.log.info("handling %s...", name)
+        if name.startswith("SUPERVISOR_STATE"):
+            self.refresh()
+        elif not self["running"]:
+            self.refresh()
+        elif name.startswith("PROCESS_GROUP"):
+            self.refresh()
+        elif name.startswith("PROCESS_STATE"):
+            payload = event["payload"]
+            puid = "{}:{}:{}".format(
+                self.name, payload["groupname"], payload["processname"]
+            )
+            self["processes"][puid].handle_event(event)
+
+    def create_base_info(self):
+        return dict(self.Null, name=self.name, url=self.url, host=self.host)
+
+    def read_info(self):
+        info = self.create_base_info()
+        server = self.server
+        info["pid"] = server.getPID()
+        info["running"] = True
+        info["identification"] = server.getIdentification()
+        info["api_version"] = server.getAPIVersion()
+        info["supervisor_version"] = server.getSupervisorVersion()
+        info["processes"] = processes = {}
+        procInfo = server.getAllProcessInfo()
+        for proc in procInfo:
+            process = Process(self, parse_dict(proc))
+            processes[process["uid"]] = process
+        return info
+
+    def update_info(self, info):
+        info = parse_dict(info)
+        if self == info:
+            this_p, info_p = self["processes"], info["processes"]
+            if this_p != info_p:
+                for name, process in info_p.items():
+                    if process != this_p[name]:
+                        send(process, "process_changed")
+            self.update(info)
+        else:
+            self.update(info)
+            send(self, "supervisor_changed")
+
+    def refresh(self):
+        try:
+            info = self.read_info()
+        except:
+            info = self.create_base_info()
+            raise
+        finally:
+            self.update_info(info)
+
+    def update_server(self, group_names=()):
+        server = self.server
+        try:
+            added, changed, removed = server.reloadConfig()[0]
+        except zerorpc.RemoteError as rerr:
+            error(rerr.msg)
+            return
+
+        # If any gnames are specified we need to verify that they are
+        # valid in order to print a useful error message.
+        if group_names:
+            groups = set()
+            for info in server.getAllProcessInfo():
+                groups.add(info["group"])
+            # New gnames would not currently exist in this set so
+            # add those as well.
+            groups.update(added)
+
+            for gname in group_names:
+                if gname not in groups:
+                    self.log.debug("unknown group %s", gname)
+
+        for gname in removed:
+            if group_names and gname not in group_names:
+                continue
+            results = server.stopProcessGroup(gname)
+            self.log.debug("stopped process group %s", gname)
+
+            fails = [res for res in results if res["status"] == Faults.FAILED]
+            if fails:
+                self.log.debug("%s as problems; not removing", gname)
+                continue
+            server.removeProcessGroup(gname)
+            self.log.debug("removed process group %s", gname)
+
+        for gname in changed:
+            if group_names and gname not in group_names:
+                continue
+            server.stopProcessGroup(gname)
+            self.log.debug("stopped process group %s", gname)
+
+            server.removeProcessGroup(gname)
+            server.addProcessGroup(gname)
+            self.log.debug("updated process group %s", gname)
+
+        for gname in added:
+            if group_names and gname not in group_names:
+                continue
+            server.addProcessGroup(gname)
+            self.log.debug("added process group %s", gname)
+
+        self.log.info("Updated %s", self.name)
+
+    def _reread(self):
+        return self.server.reloadConfig()
+
+    def restart(self):
+        # do a reread. If there is an error (bad config) inform the user and
+        # and refuse to restart
+        try:
+            self._reread()
+        except zerorpc.RemoteError as rerr:
+            error("Cannot restart: {}".format(rerr.msg))
+            return
+        result = self.server.restart(timeout=30)
+        if result:
+            info("Restarted {}".format(self.name))
+        else:
+            error("Error restarting {}".format(self.name))
+
+    def reread(self):
+        try:
+            added, changed, removed = self._reread()[0]
+        except zerorpc.RemoteError as rerr:
+            error(rerr.msg)
+        else:
+            info(
+                "Reread config of {} "
+                "({} added; {} changed; {} disappeared)".format(
+                    self.name, len(added), len(changed), len(removed)
+                )
+            )
+
+    def shutdown(self):
+        result = self.server.shutdown()
+        if result:
+            info("Shut down {}".format(self.name))
+        else:
+            error("Error shutting down {}".format(self.name))
+
+
+class Process(dict):
+
+    Null = {"running": False, "pid": None, "state": None, "statename": "UNKNOWN"}
+
+    def __init__(self, supervisor, *args, **kwargs):
+        super(Process, self).__init__(self.Null)
+        if args:
+            self.update(args[0])
+        self.update(kwargs)
+        supervisor_name = supervisor["name"]
+        full_name = self.get("group", "") + ":" + self.get("name", "")
+        uid = "{}:{}".format(supervisor_name, full_name)
+        self.log = log.getChild(uid)
+        self.supervisor = weakref.proxy(supervisor)
+        self["full_name"] = full_name
+        self["running"] = self["state"] in RUNNING_STATES
+        self["supervisor"] = supervisor_name
+        self["host"] = supervisor["host"]
+        self["uid"] = uid
+
+    @property
+    def server(self):
+        return self.supervisor.server
+
+    @property
+    def full_name(self):
+        return self["full_name"]
+
+    def handle_event(self, event):
+        event_name = event["eventname"]
+        if event_name.startswith("PROCESS_STATE"):
+            payload = event["payload"]
+            proc_info = payload.get("process")
+            if proc_info is not None:
+                proc_info = parse_dict(proc_info)
+                old = self.update_info(proc_info)
+                if old != self:
+                    old_state, new_state = old["statename"], self["statename"]
+                    send(self, event="process_changed")
+                    if old_state != new_state:
+                        info(
+                            "{} changed from {} to {}".format(
+                                self, old_state, new_state
+                            )
+                        )
+
+    def read_info(self):
+        proc_info = dict(self.Null)
+        try:
+            from_serv = parse_dict(self.server.getProcessInfo(self.full_name))
+            proc_info.update(from_serv)
+        except Exception as err:
+            self.log.warn("Failed to read info from %s: %s", self["uid"], err)
+        return proc_info
+
+    def update_info(self, proc_info):
+        old = dict(self)
+        proc_info["running"] = proc_info["state"] in RUNNING_STATES
+        self.update(proc_info)
+        return old
+
+    def refresh(self):
+        proc_info = self.read_info()
+        proc_info = parse_dict(proc_info)
+        self.update_info(proc_info)
+
+    def start(self):
+        try:
+            self.server.startProcess(self.full_name, False, timeout=30)
+        except:
+            message = "Error trying to start {}!".format(self)
+            error(message)
+            self.log.exception(message)
+
+    def stop(self):
+        try:
+            self.server.stopProcess(self.full_name)
+        except:
+            message = "Failed to stop {}".format(self["uid"])
+            warning(message)
+            self.log.exception(message)
+
+    def restart(self):
+        if self["running"]:
+            self.stop()
+        self.start()
+
+    def __str__(self):
+        return "{0} on {1}".format(self["name"], self["supervisor"])
+
+    def __eq__(self, proc):
+        p1, p2 = dict(self), dict(proc)
+        p1.pop("description")
+        p1.pop("now")
+        p2.pop("description")
+        p2.pop("now")
+        return p1 == p2
+
+    def __ne__(self, proc):
+        return not self == proc
+
+
+# Configuration
+
+
+def load_config(config_file):
+    parser = ConfigParser()
+    parser.read(config_file)
+    dft_global = dict(name="multivisor")
+
+    supervisors = {}
+    config = dict(dft_global, supervisors=supervisors)
+    config.update(parser.items("global"))
+    tasks = []
+    for section in parser.sections():
+        if not section.startswith("supervisor:"):
+            continue
+        name = section[len("supervisor:") :]
+        section_items = dict(parser.items(section))
+        url = section_items.get("url", "")
+        supervisors[name] = Supervisor(name, url)
+    return config
+
+
+def send(payload, event):
+    event_signal = signal(event)
+    return event_signal.send(event, payload=payload)
+
+
+def notification(message, level):
+    payload = dict(message=message, level=level, time=time.time())
+    send(payload, "notification")
+
+
+def info(message):
+    notification(message, "INFO")
+
+
+def warning(message):
+    logging.warning(message)
+    notification(message, "WARNING")
+
+
+def error(message):
+    logging.error(message)
+    notification(message, "ERROR")
+
+
+class Multivisor(object):
+    def __init__(self, options):
+        self.options = options
+        self.reload()
+
+    @property
+    def config(self):
+        if self._config is None:
+            self._config = load_config(self.options.config_file)
+        return self._config
+
+    @property
+    def safe_config(self):
+        """
+        :return: config dict without username and password
+        """
+        if not self.use_authentication:
+            return self.config
+
+        config = copy.copy(self.config)
+        config.pop("username", "")
+        config.pop("password", "")
+        return config
+
+    @property
+    def config_file_content(self):
+        with open(self.options.config_file) as config_file:
+            return config_file.read()
+
+    def reload(self):
+        self._config = None
+        return self.config
+
+    @property
+    def supervisors(self):
+        return self.config["supervisors"]
+
+    @property
+    def processes(self):
+        procs = (svisor["processes"] for svisor in self.supervisors.values())
+        return {puid: proc for sprocs in procs for puid, proc in sprocs.items()}
+
+    @property
+    def use_authentication(self):
+        """
+        :return: whether authentication should be used
+        """
+        username = self.config.get("username", False)
+        password = self.config.get("password", False)
+        return bool(username and password)
+
+    @property
+    def secret_key(self):
+        return os.environ.get("MULTIVISOR_SECRET_KEY")
+
+    def refresh(self):
+        tasks = [spawn(supervisor.refresh) for supervisor in self.supervisors.values()]
+        joinall(tasks)
+
+    def get_supervisor(self, name):
+        return self.supervisors[name]
+
+    def get_process(self, uid):
+        supervisor, _ = uid.split(":", 1)
+        return self.supervisors[supervisor]["processes"][uid]
+
+    def _do_supervisors(self, operation, *names):
+        supervisors = (self.get_supervisor(name) for name in names)
+        tasks = [spawn(operation, supervisor) for supervisor in supervisors]
+        joinall(tasks)
+
+    def _do_processes(self, operation, *patterns):
+        procs = self.processes
+        puids = filter_patterns(procs, patterns)
+        tasks = [spawn(operation, procs[puid]) for puid in puids]
+        joinall(tasks)
+
+    def update_supervisors(self, *names):
+        self._do_supervisors(Supervisor.update_server, *names)
+
+    def restart_supervisors(self, *names):
+        self._do_supervisors(Supervisor.restart, *names)
+
+    def reread_supervisors(self, *names):
+        self._do_supervisors(Supervisor.reread, *names)
+
+    def shutdown_supervisors(self, *names):
+        self._do_supervisors(Supervisor.shutdown, *names)
+
+    def restart_processes(self, *patterns):
+        self._do_processes(Process.restart, *patterns)
+
+    def stop_processes(self, *patterns):
+        self._do_processes(Process.stop, *patterns)

multivisor/rpc.py

@@ -0,0 +1,232 @@
+"""
+An extension to the standard supervisor RPC interface which subscribes
+to internal supervisor events and dispatches them to 0RPC.
+
+disadvantages: it depends on supervisor internal supervisor.events.subscribe
+               interface so its usage is quite risky.
+advantages: it avoids creating an eventlistener process just to forward events.
+
+The python environment where supervisor runs must have multivisor installed
+"""
+
+import os
+import queue
+import logging
+import functools
+import threading
+
+from gevent import spawn, hub, sleep
+from gevent.queue import Queue
+from six import text_type
+from zerorpc import stream, Server, LostRemote, Context
+
+from supervisor.http import NOT_DONE_YET
+from supervisor.rpcinterface import SupervisorNamespaceRPCInterface
+from supervisor.events import subscribe, Event, getEventNameByType
+
+# unsubscribe only appears in supervisor > 3.3.4
+try:
+    from supervisor.events import unsubscribe
+except:
+    unsubscribe = lambda x, y: None
+
+from .util import sanitize_url, parse_obj
+
+DEFAULT_BIND = "tcp://*:9002"
+
+
+def sync(klass):
+    def wrap_func(meth):
+        @functools.wraps(meth)
+        def wrapper(*args, **kwargs):
+            args[0]._log.debug("0RPC: called {}".format(meth.__name__))
+            result = meth(*args, **kwargs)
+            if callable(result):
+                r = NOT_DONE_YET
+                while r is NOT_DONE_YET:
+                    sleep(0.1)
+                    r = result()
+                result = r
+            return result
+
+        return wrapper
+
+    for name in dir(klass):
+        if name.startswith("_") or name == "event_stream":
+            continue
+        meth = getattr(klass, name)
+        if not callable(meth):
+            continue
+        setattr(klass, name, wrap_func(meth))
+    return klass
+
+
+# When supervisor is asked to restart, it closes file descriptors
+# from 5..1024. Since we are not able to restart the ZeroRPC server
+# (see https://github.com/0rpc/zerorpc-python/issues/208) this patch
+# prevents supervisor from closing the gevent pipes and 0MQ sockets
+# This is a really agressive move but seems to work until the above
+# bug is solved
+from supervisor.options import ServerOptions
+
+ServerOptions.cleanup_fds = lambda options: None
+
+
+@sync
+class MultivisorNamespaceRPCInterface(SupervisorNamespaceRPCInterface):
+    def __init__(self, supervisord, bind):
+        SupervisorNamespaceRPCInterface.__init__(self, supervisord)
+        self._bind = bind
+        self._channel = queue.Queue()
+        self._event_channels = set()
+        self._server = None
+        self._watcher = None
+        self._shutting_down = False
+        self._log = logging.getLogger("MVRPC")
+
+    def _start(self):
+        subscribe(Event, self._handle_event)
+
+    def _stop(self):
+        unsubscribe(Event, self._handle_event)
+        self._shutting_down = True
+
+    def _shutdown(self):
+        # disconnect all streams
+        for channel in self._event_channels:
+            channel.put(None)
+        if self._server is not None:
+            self._server.stop()
+            self._server.close()
+
+    def _process_event(self, event):
+        if self._shutting_down:
+            return
+        event_name = getEventNameByType(event.__class__)
+        stop_event = event_name == "SUPERVISOR_STATE_CHANGE_STOPPING"
+        if stop_event:
+            self._log.info("supervisor stopping!")
+            self._stop()
+        elif event_name.startswith("TICK"):
+            return
+        try:
+            payload_str = text_type(event.payload())
+        except AttributeError:
+            # old supervisor version
+            payload_str = text_type(event)
+        payload = dict((x.split(":") for x in payload_str.split()))
+        if event_name.startswith("PROCESS_STATE"):
+            pname = "{}:{}".format(payload["groupname"], payload["processname"])
+            payload[u"process"] = parse_obj(self.getProcessInfo(pname))
+        # broadcast the event to clients
+        server = self.supervisord.options.identifier
+        new_event = {
+            u"pool": u"multivisor",
+            u"server": text_type(server),
+            u"eventname": text_type(event_name),
+            u"payload": payload,
+        }
+        for channel in self._event_channels:
+            channel.put(new_event)
+        if stop_event:
+            self._shutdown()
+
+    # called on 0RPC server thread
+    def _dispatch_event(self):
+        while not self._channel.empty():
+            event = self._channel.get()
+            self._process_event(event)
+
+    # called on main thread
+    def _handle_event(self, event):
+        if self._server is None:
+            reply = start_rpc_server(self, self._bind)
+            if isinstance(reply, Exception):
+                self._log.critical("severe 0RPC error: %s", reply)
+                self._stop()
+                self._shutdown()
+                return
+            self._server, self._watcher = reply
+        self._channel.put(event)
+        self._watcher.send()
+
+        # handle stop synchronously
+        event_name = getEventNameByType(event.__class__)
+        if event_name == "SUPERVISOR_STATE_CHANGE_STOPPING":
+            self._server._stop_event.wait()
+            self._server = None
+            self._watcher = None
+
+    @stream
+    def event_stream(self):
+        self._log.info("client connected to stream")
+        channel = Queue()
+        self._event_channels.add(channel)
+        try:
+            yield "First event to trigger connection. Please ignore me!"
+            for event in channel:
+                if event is None:
+                    self._log.info("stop: closing client")
+                    return
+                # self._log.info(event)
+                yield event
+        except LostRemote as e:
+            self._log.info("remote end of stream disconnected")
+        finally:
+            self._event_channels.remove(channel)
+
+
+class ServerMiddleware(object):
+    def server_after_exec(self, request_event, reply_event):
+        if reply_event.args:
+            reply_event._args = parse_obj(reply_event.args)
+
+
+def start_rpc_server(multivisor, bind):
+    future_server = queue.Queue(1)
+    th = threading.Thread(
+        target=run_rpc_server, name="RPCServer", args=(multivisor, bind, future_server)
+    )
+    th.daemon = True
+    th.start()
+    return future_server.get()
+
+
+def run_rpc_server(multivisor, bind, future_server):
+    multivisor._log.info("0RPC: spawn server on {}...".format(os.getpid()))
+    watcher = hub.get_hub().loop.async_()
+    stop_event = threading.Event()
+    watcher.start(lambda: spawn(multivisor._dispatch_event))
+    server = None
+    try:
+        context = Context()
+        context.register_middleware(ServerMiddleware())
+        server = Server(multivisor, context=context)
+        server._stop_event = stop_event
+        server.bind(bind)
+        future_server.put((server, watcher))
+        multivisor._log.info("0RPC: server running!")
+        server.run()
+        multivisor._log.info("0RPC: server stopped!")
+    except Exception as err:
+        future_server.put(err)
+    finally:
+        watcher.stop()
+        del server
+        # prevent reusage of this loop because supervisor closes all ports
+        # when a restart happens. It actually doesn't help preventing a crash
+        hub.get_hub().destroy(destroy_loop=True)
+        multivisor._log.info("0RPC: server thread destroyed!")
+    stop_event.set()
+
+
+def make_rpc_interface(supervisord, bind=DEFAULT_BIND):
+    # Uncomment following lines to configure python standard logging
+    # log_level = logging.INFO
+    # log_fmt = '%(asctime)-15s %(levelname)s %(threadName)-8s %(name)s: %(message)s'
+    # logging.basicConfig(level=log_level, format=log_fmt)
+
+    url = sanitize_url(bind, protocol="tcp", host="*", port=9002)
+    multivisor = MultivisorNamespaceRPCInterface(supervisord, url["url"])
+    multivisor._start()
+    return multivisor

multivisor/server/dist/index.html

@@ -0,0 +1 @@
+<!DOCTYPE html><html><head><meta charset=utf-8><meta name=viewport content="width=device-width,initial-scale=1"><title>multivisor</title><link href="https://fonts.googleapis.com/css?family=Roboto:300,400,500,700|Material+Icons" rel=stylesheet><link href=/static/css/app.2aff3580a128440bba89b94112292cb1.css rel=stylesheet></head><body><div id=app></div><script type=text/javascript src=/static/js/manifest.2ae2e69a05c33dfc65f8.js></script><script type=text/javascript src=/static/js/vendor.1d02877727062a41e9fb.js></script><script type=text/javascript src=/static/js/app.52791f915c2f060b9cb1.js></script></body></html>