zprocess 2.22.0__tar.gz → 2.24.0__tar.gz

{zprocess-2.22.0/zprocess.egg-info → zprocess-2.24.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: zprocess
-Version: 2.22.0
+Version: 2.24.0
 Summary: A set of utilities for multiprocessing using zeromq.
 Home-page: http://github.com/chrisjbillington/zprocess
 Author: Chris Billington

{zprocess-2.22.0 → zprocess-2.24.0}/zprocess/security.py

@@ -3,6 +3,7 @@ import sys
 import base64
 import weakref
 import os
+import threading
 import ipaddress
 import zmq
 import zmq.auth.thread
@@ -24,6 +25,7 @@ if sys.version_info[0] == 2:
 else:
     from time import monotonic
 
+PYZMQ_VER_MAJOR = int(zmq.__version__.split('.')[0])
 
 _bundle_warning = """zprocess warning: pyzmq is using bundled libzmq, which on Windows
 is not built with the cryptography library libsodium. Encryption/decryption will be
@@ -165,9 +167,10 @@ class SecureSocket(zmq.Socket):
         are a server or not"""
         orig_server = self.curve_server
         if server:
+            self.curve_server = True
+            self.zap_domain = self.context.zap_domain
             self.curve_publickey = self.context.server_publickey
             self.curve_secretkey = self.context.server_secretkey
-            self.curve_server = True
         else:
             self.curve_server = False
             self.curve_publickey = self.context.client_publickey
@@ -306,6 +309,72 @@ class SecureSocket(zmq.Socket):
                 return msg
 
 
+class _ThreadAuthenticator:
+    # We roll our own thread authenticator (just implementing what we need) since a)
+    # zmq.auth.thread.ThreadAuthenticator uses asyncio, and we do not want to express an
+    # opinion on the kerfuffle that is Windows asyncio selectors (see:
+    # https://github.com/zeromq/pyzmq/issues/1423) and impose it on the rest of the
+    # interpreter, when we're not even using async stuff ourself and b)
+    # zmq.auth.thread.ThreadAuthenticator spawns a non-daemon thread which is difficult
+    # to ensure gets shut down at interpreter shutdown, and otherwise holds the
+    # interpreter open.
+    def __init__(self, ctx, zap_domain, allowed_clients):
+        if PYZMQ_VER_MAJOR >= 25:
+            self.zap_socket = ctx.socket(zmq.REP, socket_class=zmq.Socket)
+        else:
+            self.zap_socket = ctx.socket(zmq.REP)
+        self.zap_socket.linger = 1
+        self.zap_socket.bind("inproc://zeromq.zap.01")
+
+        # Note: we hold a reference to the zap socket, since if the thread crashes, we
+        # don't want the socket to be cleaned up and closed - that would have zmq
+        # (stupidly and dangerously) fall back to its default authentication method,
+        # which is to allow all.
+        self.thread = threading.Thread(
+            target=self.run,
+            args=(self.zap_socket, zap_domain, allowed_clients),
+            daemon=True,
+        )
+        self.started = threading.Event()
+        self.thread.start()
+
+    def run(self, zap_socket, zap_domain, allowed_clients):
+        VERSION = b'1.0'
+        MECHANISM = b'CURVE'
+        while True:
+            try:
+                msg = zap_socket.recv_multipart()
+            except zmq.error.ContextTerminated:
+                zap_socket.close()
+                return
+            version, request_id, domain, address, identity, mechanism = msg[:6]
+            credentials = msg[6:]
+            if version != VERSION:
+                status_code = b"400"
+                status_text = b"Invalid version"
+                user_id = b""
+            elif mechanism != MECHANISM:
+                status_code = b"400"
+                status_text = b"Security mechanism not supported"
+                user_id = b""
+            elif domain != zap_domain:
+                status_code = b"400"
+                status_text = b"Unknown domain"
+                user_id = b""
+            else:
+                key = zmq.utils.z85.encode(credentials[0])
+                if key in allowed_clients:
+                    status_code = b"200"
+                    status_text = b"OK"
+                    user_id = key
+                else:
+                    status_code = b"400"
+                    status_text = b"Unknown key"
+                    user_id = b""
+            response = [VERSION, request_id, status_code, status_text, user_id, b""]
+            zap_socket.send_multipart(response)
+
+
 class SecureContext(zmq.Context):
     """A ZeroMQ Context with SecureContext.socket() returning a
     SecureSocket(), which can authenticate and communicate securely with all
@@ -315,7 +384,10 @@ class SecureContext(zmq.Context):
 
     _socket_class = SecureSocket
     _instances = weakref.WeakValueDictionary()
+    zap_domain = b"zprocess"
+
     # Dummy class attrs to distinguish from zmq options:
+    auth = None
     secure = False
     client_publickey = None
     client_secretkey = None
@@ -331,22 +403,15 @@ class SecureContext(zmq.Context):
             self.client_publickey = zmq.curve_public(self.client_secretkey)
             self.server_publickey = zmq.curve_public(self.server_secretkey)
             
-            # There are potential reference cycles causing the authentication thread to
-            # prevent the interpreter from shutting down. In pyzmq <25, the
-            # authentication thread holds a reference to the context. So we must avoid
-            # holding a reference to it. In pyzmq 25+, the authentication thread and
-            # threadauthenticator objects both hold references to each other. So we
-            # replace one with a weakref to break the cycle.
-
-            auth = zmq.auth.thread.ThreadAuthenticator(self)
-            auth.start()
-
-            if auth.thread.authenticator is auth:
-                auth.thread.authenticator = weakref.proxy(auth)
-
-            # Allow only clients who have the client public key:
-            auth.allow_any = False
-            auth.certs['*'] = {self.client_publickey: True}
+            # Note: it is crucial we hold a reference to the authenticator, so that in
+            # the case the zap authentication thread crashes, the zap socket does not
+            # get cleaned up and closed - zmq will interpret that situation as us not
+            # requiring any authentication.
+            self.auth = _ThreadAuthenticator(
+                self,
+                zap_domain=self.zap_domain,
+                allowed_clients=[self.client_publickey],
+            )
             self.secure = True
 
     @classmethod

{zprocess-2.22.0 → zprocess-2.24.0}/zprocess/tasks.py

@@ -11,12 +11,14 @@
 #                                                                   #
 #####################################################################
 from __future__ import print_function, division, absolute_import, unicode_literals
+
 try:
     from time import monotonic
 except ImportError:
     from time import time as monotonic
 
 from bisect import insort
+import copy
 
 # This module contains a minimalistic task queue. It is used by the zlock and zlog
 # servers (zprocess.locking and zprocess.logging respectively) in order to wait for
@@ -42,23 +44,27 @@ from bisect import insort
 class Task(object):
     def __init__(self, due_in, func, *args, **kwargs):
         """Wrapper for a function call to be executed after a specified time interval.
-        due_in is how long in the future, in seconds, the function should be called,
+        `due_in` is how long in the future, in seconds, the function should be called,
         func is the function to call. All subsequent arguments and keyword arguments
-        will be passed to the function."""
+        will be passed to the function. If added to a TaskQueue with `repeat=True`, or
+        if Task.repeat is set to True, then the `due_in` argument is used as the
+        repetition interval."""
         self.due_at = monotonic() + due_in
+        self.interval = due_in
         self.func = func
         self.args = args
         self.kwargs = kwargs
-        self.called = False
+        self._called = False
+        self.repeat = False
 
     def due_in(self):
         """The time interval in seconds until the task is due"""
         return self.due_at - monotonic()
 
     def __call__(self):
-        if self.called:
+        if self._called:
             raise RuntimeError('Task has already been called')
-        self.called = True
+        self._called = True
         return self.func(*self.args, **self.kwargs)
 
     def __gt__(self, other):
@@ -71,8 +77,25 @@ class TaskQueue(list):
     """A list of pending tasks due at certain times. Tasks are stored with the soonest
     due at the end of the list, to be removed with pop()"""
 
-    def add(self, task):
-        """Insert the task into the queue, maintaining sort order"""
+    def pop(self):
+        """Returns the next due task. If Task.repeat is True, re-adds a copy of the Task
+        to the queue with a due time `Task.interval` after its previous due time."""
+        task = super().pop()
+        if task.repeat:
+            next_rep = copy.copy(task)
+            # If next due time would be in the past, add a task due now instead. We
+            # don't want to a large number of tasks if e.g. system suspension led to a
+            # discontinuous jump forward in the monotonic clock
+            next_rep.due_at = max(monotonic(), task.due_at + task.interval)
+            self.add(next_rep)
+        return task
+
+    def add(self, task, repeat=None):
+        """Insert the task into the queue, maintaining sort order. If repeat is given,
+        this sets `Task.repeat`, which controls whether or not a copy of the task will
+        be re-added to the queue after it is removed via a call to `TaskQueue.pop()`."""
+        if repeat is not None:
+            task.repeat = repeat
         insort(self, task)
 
     def next(self):
@@ -80,4 +103,25 @@ class TaskQueue(list):
         return self[-1]
 
     def cancel(self, task):
-        self.remove(task)
+        self.remove(task)
+
+
+if __name__ == '__main__':
+    # Intended use. It's up the caller to sleep() or otherwise wait until the next task
+    # is due. It is assumed that the queue is only accessed from a single thread such
+    # that it's not possible for any tasks to be added to the queue during such a sleep.
+    # More usefully, instead of calling sleep(), an application might be calling
+    # select.select() or select.poll() with the time to the next task as the timeout.
+    # This way one can implement a simple mainloop for an event-driven application that
+    # can respond to network and file events as well as executing time-based tasks. This
+    # is how the mainloops of the zlock and zlog servers are implemented.
+    import time
+
+    tasks = TaskQueue()
+    task = Task(1, print, 'hello', 'world')
+
+    tasks.add(task, repeat=True)
+    while tasks:
+        time.sleep(max(0, tasks.next().due_in()))  # don't pass sleep() a negative value
+        next_task = tasks.pop()
+        next_task()

{zprocess-2.22.0 → zprocess-2.24.0/zprocess.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: zprocess
-Version: 2.22.0
+Version: 2.24.0
 Summary: A set of utilities for multiprocessing using zeromq.
 Home-page: http://github.com/chrisjbillington/zprocess
 Author: Chris Billington