portal 3.4.3__tar.gz → 3.5.0__tar.gz

{portal-3.4.3/portal.egg-info → portal-3.5.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: portal
-Version: 3.4.3
+Version: 3.5.0
 Summary: Fast and reliable distributed systems in Python
 Home-page: http://github.com/danijar/portal
 Author: Danijar Hafner

{portal-3.4.3 → portal-3.5.0}/portal/__init__.py

@@ -1,4 +1,4 @@
-__version__ = '3.4.3'
+__version__ = '3.5.0'
 
 import multiprocessing as mp
 try:

{portal-3.4.3 → portal-3.5.0}/portal/client_socket.py

@@ -1,5 +1,6 @@
 import collections
 import dataclasses
+import os
 import queue
 import select
 import socket
@@ -30,7 +31,6 @@ class Options:
   logging: bool = True
   logging_color: str = 'yellow'
   connect_wait: float = 0.1
-  loop_sleep: float = 0.0
 
 
 class ClientSocket:
@@ -52,6 +52,7 @@ class ClientSocket:
     self.wantconn = threading.Event()
     self.sendq = collections.deque()
     self.recvq = queue.Queue()
+    self.get_signal, self.set_signal = os.pipe()
 
     self.running = True
     self.thread = thread.Thread(self._loop, name=f'{name}Loop')
@@ -76,6 +77,7 @@ class ClientSocket:
     self.require_connection(timeout)
     maxsize = self.options.max_msg_size
     self.sendq.append(buffers.SendBuffer(*data, maxsize=maxsize))
+    os.write(self.set_signal, bytes(1))
 
   def recv(self, timeout=None):
     assert self.running
@@ -98,6 +100,8 @@ class ClientSocket:
     self.running = False
     self.thread.join(timeout)
     self.thread.kill()
+    os.close(self.get_signal)
+    os.close(self.set_signal)
 
   def require_connection(self, timeout):
     if self.connected:
@@ -111,7 +115,9 @@ class ClientSocket:
     recvbuf = buffers.RecvBuffer(maxsize=self.options.max_msg_size)
     sock = None
     poll = select.poll()
+    poll.register(self.get_signal, select.POLLIN)
     isconn = False  # Local mirror of self.isconn without the lock.
+    writing = False
 
     while self.running or (self.sendq and isconn):
 
@@ -121,7 +127,7 @@ class ClientSocket:
         sock = self._connect()
         if not sock:
           break
-        poll.register(sock, select.POLLIN | select.POLLOUT)
+        poll.register(sock, select.POLLIN)
         self.isconn.set()
         isconn = True
         if not self.options.autoconn:
@@ -130,33 +136,37 @@ class ClientSocket:
 
       try:
 
-        # TODO: According to the py-spy profiler, the GIL is held during
-        # polling. Is there a way to avoid that?
-        pairs = poll.poll(0.2)
-        if not pairs:
-          continue
-        _, mask = pairs[0]
-
-        if mask & select.POLLIN:
-          try:
-            recvbuf.recv(sock)
-            if recvbuf.done():
-              if self.recvq.qsize() > self.options.max_recv_queue:
-                raise RuntimeError('Too many incoming messages enqueued')
-              msg = recvbuf.result()
-              self.recvq.put(msg)
-              [x(msg) for x in self.callbacks_recv]
-              recvbuf = buffers.RecvBuffer(maxsize=self.options.max_msg_size)
-          except BlockingIOError:
-            pass
+        if not writing:
+          fds = [fd for fd, _ in poll.poll(0.2)]
+          if self.get_signal in fds:
+            writing = True
+            os.read(self.get_signal, 1)
 
-        if self.sendq and mask & select.POLLOUT:
+        try:
+          recvbuf.recv(sock)
+          if recvbuf.done():
+            if self.recvq.qsize() > self.options.max_recv_queue:
+              raise RuntimeError('Too many incoming messages enqueued')
+            msg = recvbuf.result()
+            self.recvq.put(msg)
+            [x(msg) for x in self.callbacks_recv]
+            recvbuf = buffers.RecvBuffer(maxsize=self.options.max_msg_size)
+        except BlockingIOError:
+          pass
+
+        if self.sendq:
           try:
             self.sendq[0].send(sock)
             if self.sendq[0].done():
               self.sendq.popleft()
+              if not self.sendq:
+                writing = False
           except BlockingIOError:
             pass
+          except ConnectionResetError:
+            # The server is gone but we may have buffered messages left to
+            # read, so we keep the socket open until recv() fails.
+            pass
 
       except OSError as e:
         detail = f'{type(e).__name__}'
@@ -176,9 +186,6 @@ class ClientSocket:
         [x() for x in self.callbacks_disc]
         continue
 
-      if self.options.loop_sleep:
-        time.sleep(self.options.loop_sleep)
-
     if sock:
       sock.close()
 

{portal-3.4.3 → portal-3.5.0}/portal/server_socket.py

@@ -1,9 +1,9 @@
 import collections
 import dataclasses
+import os
 import queue
 import selectors
 import socket
-import time
 
 from . import buffers
 from . import contextlib
@@ -32,7 +32,6 @@ class Options:
   max_send_queue: int = 4096
   logging: bool = True
   logging_color: str = 'blue'
-  loop_sleep: float = 0.0
 
 
 class ServerSocket:
@@ -56,7 +55,9 @@ class ServerSocket:
     self.sock.bind(self.addr)
     self.sock.setblocking(False)
     self.sock.listen(8192)
+    self.get_signal, self.set_signal = os.pipe()
     self.sel = selectors.DefaultSelector()
+    self.sel.register(self.get_signal, selectors.EVENT_READ, data='signal')
     self.sel.register(self.sock, selectors.EVENT_READ, data=None)
     self._log(f'Listening at {self.addr[0]}:{self.addr[1]}')
     self.conns = {}
@@ -89,6 +90,7 @@ class ServerSocket:
     try:
       self.conns[addr].sendbufs.append(
           buffers.SendBuffer(*data, maxsize=maxsize))
+      os.write(self.set_signal, bytes(1))
     except KeyError:
       self._log('Dropping message to disconnected client')
 
@@ -101,36 +103,38 @@ class ServerSocket:
     [conn.sock.close() for conn in self.conns.values()]
     self.sock.close()
     self.sel.close()
+    os.close(self.get_signal)
+    os.close(self.set_signal)
 
   def _loop(self):
+    writing = False
     try:
       while self.running or self._numsending():
-        writeable = []
-        # TODO: According to the py-spy profiler, the GIL is held during
-        # polling. Is there a way to avoid that?
         for key, mask in self.sel.select(timeout=0.2):
-          if key.data is None and self.reading:
+          if key.data == 'signal':
+            writing = True
+            os.read(self.get_signal, 1)
+          elif key.data is None and self.reading:
             assert mask & selectors.EVENT_READ
             self._accept(key.fileobj)
           elif mask & selectors.EVENT_READ and self.reading:
             self._recv(key.data)
-          elif mask & selectors.EVENT_WRITE:
-            writeable.append(key.data)
-        for conn in writeable:
-          if not conn.sendbufs:
-            continue
+        if not writing:
+          continue
+        pending = [conn for conn in self.conns.values() if conn.sendbufs]
+        for conn in pending:
           try:
             conn.sendbufs[0].send(conn.sock)
             if conn.sendbufs[0].done():
               conn.sendbufs.popleft()
+              if not any(conn.sendbufs for conn in pending):
+                writing = False
           except BlockingIOError:
             pass
           except ConnectionResetError:
-            # The client is gone but we may have buffered messages left to
-            # read, so we keep the socket open until recv() fails.
+            # The client is gone but we may have buffered messages left
+            # to read, so we keep the socket open until recv() fails.
             pass
-      if self.options.loop_sleep:
-        time.sleep(self.options.loop_sleep)
     except Exception as e:
       self.error = e
 
@@ -139,8 +143,7 @@ class ServerSocket:
     self._log(f'Accepted connection from {addr[0]}:{addr[1]}')
     sock.setblocking(False)
     conn = Connection(sock, addr)
-    self.sel.register(
-        sock, selectors.EVENT_READ | selectors.EVENT_WRITE, data=conn)
+    self.sel.register(sock, selectors.EVENT_READ, data=conn)
     self.conns[addr] = conn
 
   def _recv(self, conn):

{portal-3.4.3 → portal-3.5.0/portal.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: portal
-Version: 3.4.3
+Version: 3.5.0
 Summary: Fast and reliable distributed systems in Python
 Home-page: http://github.com/danijar/portal
 Author: Danijar Hafner