portal 3.1.5__tar.gz → 3.1.7__tar.gz

{portal-3.1.5/portal.egg-info → portal-3.1.7}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: portal
-Version: 3.1.5
+Version: 3.1.7
 Summary: Fast and reliable distributed systems in Python
 Home-page: http://github.com/danijar/portal
 Author: Danijar Hafner

{portal-3.1.5 → portal-3.1.7}/portal/__init__.py

@@ -1,4 +1,4 @@
-__version__ = '3.1.5'
+__version__ = '3.1.7'
 
 import multiprocessing as mp
 try:

{portal-3.1.5 → portal-3.1.7}/portal/client.py

@@ -81,10 +81,18 @@ class Client:
     strlen = len(name).to_bytes(8, 'little', signed=False)
     sendargs = (reqnum, strlen, name, *packlib.pack(data))
     # self.socket.send(reqnum, strlen, name, *packlib.pack(data))
-    self.socket.send(*sendargs)
-    future = Future()
+    rai = [False]
+    future = Future(rai)
     future.sendargs = sendargs
     self.futures[reqnum] = future
+    # Store future before sending request because the response may come fast
+    # and the response handler runs in the socket's background thread.
+    try:
+      self.socket.send(*sendargs)
+    except client_socket.Disconnected:
+      future = self.futures.pop(reqnum)
+      future.rai[0] = True
+      raise
     return future
 
   def close(self, timeout=None):
@@ -98,9 +106,11 @@ class Client:
     reqnum = bytes(data[:8])
     status = int.from_bytes(data[8:16], 'little', signed=False)
     future = self.futures.pop(reqnum, None)
+
     assert future, (
         f'Unexpected request number: {reqnum}',
         sorted(self.futures.keys()))
+
     if status == 0:
       data = packlib.unpack(data[16:])
       future.set_result(data)
@@ -111,12 +121,25 @@ class Client:
       self.cond.notify_all()
     self.socket.recv()
 
+    # if not future:
+    #   existing = sorted(self.futures.keys())
+    #   print(f'Unexpected request number: {reqnum}', existing)
+    # elif status == 0:
+    #   data = packlib.unpack(data[16:])
+    #   future.set_result(data)
+    #   with self.cond: self.cond.notify_all()
+    # else:
+    #   message = bytes(data[16:]).decode('utf-8')
+    #   self._seterr(future, RuntimeError(message))
+    #   with self.cond: self.cond.notify_all()
+    # self.socket.recv()
+
   def _disc(self):
     if self.socket.options.autoconn:
-      for future in self.futures.values():
+      for future in list(self.futures.values()):
         future.resend = True
     else:
-      for future in self.futures.values():
+      for future in list(self.futures.values()):
         self._seterr(future, client_socket.Disconnected)
       self.futures.clear()
 
@@ -127,17 +150,17 @@ class Client:
           self.socket.send(*future.sendargs)
 
   def _seterr(self, future, e):
-    raised = [False]
-    future.raised = raised
     future.set_error(e)
+    rai = future.rai
     weakref.finalize(future, lambda: (
-        None if raised[0] else self.errors.append(e)))
+        None if rai[0] else self.errors.append(e)))
 
 
 class Future:
 
-  def __init__(self):
-    self.raised = [False]
+  def __init__(self, rai):
+    assert rai == [False]
+    self.rai = rai
     self.con = threading.Condition()
     self.don = False
     self.res = None
@@ -147,7 +170,7 @@ class Future:
     if not self.done:
       return 'Future(done=False)'
     elif self.err:
-      return f"Future(done=True, error='{self.err}', raised={self.raised[0]})"
+      return f"Future(done=True, error='{self.err}', raised={self.rai[0]})"
     else:
       return 'Future(done=True)'
 
@@ -165,8 +188,8 @@ class Future:
     assert self.don
     if self.err is None:
       return self.res
-    if not self.raised[0]:
-      self.raised[0] = True
+    if not self.rai[0]:
+      self.rai[0] = True
       raise self.err
 
   def set_result(self, result):

{portal-3.1.5 → portal-3.1.7}/portal/client_socket.py

@@ -110,6 +110,7 @@ class ClientSocket:
   def _loop(self):
     recvbuf = buffers.RecvBuffer(maxsize=self.options.max_msg_size)
     sock = None
+    poll = select.poll()
     isconn = False  # Local mirror of self.isconn without the lock.
 
     while self.running or (self.sendq and isconn):
@@ -120,6 +121,7 @@ class ClientSocket:
         sock = self._connect()
         if not sock:
           break
+        poll.register(sock, select.POLLIN | select.POLLOUT)
         self.isconn.set()
         isconn = True
         if not self.options.autoconn:
@@ -128,9 +130,15 @@ class ClientSocket:
 
       try:
 
-        readable, writable, _ = select.select([sock], [sock], [], 0.2)
+        # TODO: According to the py-spy profiler, the GIL is held during
+        # polling. Is there a way to avoid that?
+        pairs = poll.poll(0.2)
+        if not pairs:
+          continue
+        _, mask = pairs[0]
+
 
-        if readable:
+        if mask & select.POLLIN:
           try:
             recvbuf.recv(sock)
             if recvbuf.done():
@@ -143,7 +151,7 @@ class ClientSocket:
           except BlockingIOError:
             pass
 
-        if self.sendq and writable:
+        if self.sendq and mask & select.POLLOUT:
           try:
             self.sendq[0].send(sock)
             if self.sendq[0].done():
@@ -157,6 +165,7 @@ class ClientSocket:
         self._log(f'Connection to server lost ({detail})')
         self.isconn.clear()
         isconn = False
+        poll.unregister(sock)
         sock.close()
         # Clear message queue on disconnect. There is no meaningful concept of
         # sucessful delivery of a message at this level. For example, the

{portal-3.1.5 → portal-3.1.7}/portal/contextlib.py

@@ -2,6 +2,7 @@ import collections
 import multiprocessing as mp
 import os
 import pathlib
+import sys
 import threading
 import traceback
 
@@ -114,10 +115,10 @@ class Context:
     with self.printlock:
       style = utils.style(color='red')
       reset = utils.style(reset=True)
-      print(style + '\n---\n' + message + reset)
+      print(style + '\n---\n' + message + reset, file=sys.stderr)
     if self.errfile:
       self.errfile.write_text(message)
-      print(f'Wrote errorfile: {self.errfile}')
+      print(f'Wrote errorfile: {self.errfile}', file=sys.stderr)
 
   def shutdown(self, exitcode):
     # This kills the process tree forcefully to prevent hangs but results in
@@ -145,7 +146,8 @@ class Context:
   def _watcher(self):
     while True:
       if self.errfile and self.errfile.exists():
-        print(f'Shutting down due to error file: {self.errfile}')
+        message = f'Shutting down due to error file: {self.errfile}',
+        print(message, file=sys.stderr)
         self.shutdown(2)
         break
       if self.done.wait(self.interval):

{portal-3.1.5 → portal-3.1.7}/portal/server_socket.py

@@ -101,6 +101,8 @@ class ServerSocket:
     try:
       while self.running or self._numsending():
         writeable = []
+        # TODO: According to the py-spy profiler, the GIL is held during
+        # polling. Is there a way to avoid that?
         for key, mask in self.sel.select(timeout=0.2):
           if key.data is None and self.reading:
             assert mask & selectors.EVENT_READ

{portal-3.1.5 → portal-3.1.7/portal.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: portal
-Version: 3.1.5
+Version: 3.1.7
 Summary: Fast and reliable distributed systems in Python
 Home-page: http://github.com/danijar/portal
 Author: Danijar Hafner

{portal-3.1.5 → portal-3.1.7}/tests/test_client.py

@@ -48,8 +48,22 @@ class TestClient:
     client.connect()
     assert client.fn(1).result() == 1
     server.close()
-    with pytest.raises(portal.Disconnected):
-      client.fn(2).result()
+
+    assert len(client.futures) == 0
+    assert len(client.errors) == 0
+    try:
+      future = client.fn(2)
+      try:
+        future.result()
+        assert False
+      except portal.Disconnected:
+        assert True
+    except portal.Disconnected:
+      time.sleep(1)
+      future = None
+    assert len(client.futures) == 0
+    assert len(client.errors) == 0
+
     server = portal.Server(port)
     server.bind('fn', lambda x: x)
     server.start(block=False)