portal 3.4.0__tar.gz → 3.4.2__tar.gz

{portal-3.4.0/portal.egg-info → portal-3.4.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: portal
-Version: 3.4.0
+Version: 3.4.2
 Summary: Fast and reliable distributed systems in Python
 Home-page: http://github.com/danijar/portal
 Author: Danijar Hafner

{portal-3.4.0 → portal-3.4.2}/portal/__init__.py

@@ -1,4 +1,4 @@
-__version__ = '3.4.0'
+__version__ = '3.4.2'
 
 import multiprocessing as mp
 try:

{portal-3.4.0 → portal-3.4.2}/portal/client_socket.py

@@ -190,7 +190,6 @@ class ClientSocket:
       port = int(port)
       addr = (host, port, 0, 0) if self.options.ipv6 else (host, port)
       sock = self._create()
-      start = time.time()
       error = None
       try:
         sock.settimeout(10)
@@ -214,9 +213,10 @@ class ClientSocket:
   def _create(self):
     if self.options.ipv6:
       sock = socket.socket(socket.AF_INET6, socket.SOCK_STREAM)
+      sock.setsockopt(socket.IPPROTO_IPV6, socket.IPV6_V6ONLY, 1)
     else:
       sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
-    # sock.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1)
+    # sock.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1)  # TODO
 
     after = self.options.keepalive_after
     every = self.options.keepalive_every

{portal-3.4.0 → portal-3.4.2}/portal/contextlib.py

@@ -21,10 +21,13 @@ class Context:
     self.interval = 20
     self.clientkw = {}
     self.serverkw = {}
+    self.printlock = threading.Lock()
     self.done = threading.Event()
     self.watcher = None
-    self.mp = mp.get_context()
-    self.printlock = self.mp.Lock()
+
+  @property
+  def mp(self):
+    return mp.get_context('spawn')
 
   def options(self):
     return {

{portal-3.4.0 → portal-3.4.2}/portal/server.py

@@ -1,7 +1,7 @@
 import collections
 import concurrent.futures
-import threading
 import time
+import types
 
 from . import packlib
 from . import poollib
@@ -34,11 +34,11 @@ class Server:
       self.pools.append(pool)
     else:
       pool = self.pool
-    active = threading.Semaphore((workers or self.workers) + 1)
-    def workfn2(*args):
-      active.acquire()
-      return workfn(*args)
-    self.methods[name] = (workfn2, postfn, pool, active)
+    requests = collections.deque()
+    available = (workers or self.workers) + 1
+    self.methods[name] = types.SimpleNamespace(
+        workfn=workfn, postfn=postfn, pool=pool,
+        requests=requests, available=available)
 
   def start(self, block=True):
     assert not self.running
@@ -67,9 +67,10 @@ class Server:
         'numrecv': mets['recv'],
         'sendrate': mets['send'] / dur,
         'recvrate': mets['recv'] / dur,
+        'requests': sum(len(m.requests) for m in self.methods.values()),
         'jobs': len(self.jobs),
     }
-    if any(postfn for _, postfn, _, _ in self.methods.values()):
+    if any(method.postfn for method in self.methods.values()):
       stats.update({
           'post_iqueue': len(self.postfn_inp),
           'post_oqueue': len(self.postfn_out),
@@ -84,7 +85,9 @@ class Server:
     self.close()
 
   def _loop(self):
-    while self.running or self.jobs or self.postfn_inp or self.postfn_out:
+    methods = list(self.methods.values())
+    pending = 0
+    while self.running or pending:
       while True:  # Loop syntax used to break on error.
         if not self.running:  # Do not accept further requests.
           break
@@ -107,24 +110,30 @@ class Server:
           self._error(addr, reqnum, 3, f'Unknown method {name}')
           break
         self.metrics['recv'] += 1
-        workfn, postfn, pool, active = self.methods[name]
-        job = pool.submit(workfn, *data)
-        job.active = active
-        job.postfn = postfn
-        job.addr = addr
-        job.reqnum = reqnum
-        self.jobs.add(job)
-        if postfn:
-          self.postfn_inp.append(job)
+        method = self.methods[name]
+        method.requests.append((addr, reqnum, data))
+        pending += 1
         break  # We do not actually want to loop.
+
+      for method in methods:
+        if method.requests and method.available:
+          method.available -= 1
+          addr, reqnum, data = method.requests.popleft()
+          job = method.pool.submit(method.workfn, *data)
+          job.method = method
+          job.addr = addr
+          job.reqnum = reqnum
+          self.jobs.add(job)
+          if method.postfn:
+            self.postfn_inp.append(job)
+
       completed, self.jobs = concurrent.futures.wait(
           self.jobs, 0.0001, concurrent.futures.FIRST_COMPLETED)
       for job in completed:
         try:
           data = job.result()
-          if job.postfn:
-            data, info = data
-            del info
+          if job.method.postfn:
+            data, _ = data
           data = packlib.pack(data)
           status = int(0).to_bytes(8, 'little', signed=False)
           self.socket.send(job.addr, job.reqnum, status, *data)
@@ -132,19 +141,24 @@ class Server:
         except Exception as e:
           self._error(job.addr, job.reqnum, 4, f'Error in server method: {e}')
         finally:
-          if not job.postfn:
-            job.active.release()
+          if not job.method.postfn:
+            job.method.available += 1
+            pending -= 1
+
       if completed:
+        # Call postfns in the order the requests were received.
         while self.postfn_inp and self.postfn_inp[0].done():
           job = self.postfn_inp.popleft()
-          data, info = job.result()
-          postjob = self.postfn_pool.submit(job.postfn, info)
-          postjob.active = job.active
+          _, info = job.result()
+          postjob = self.postfn_pool.submit(job.method.postfn, info)
+          postjob.method = job.method
           self.postfn_out.append(postjob)
+
       while self.postfn_out and self.postfn_out[0].done():
         postjob = self.postfn_out.popleft()
-        postjob.active.release()
         postjob.result()  # Check if there was an error.
+        postjob.method.available += 1
+        pending -= 1
 
   def _error(self, addr, reqnum, status, message):
     status = status.to_bytes(8, 'little', signed=False)

{portal-3.4.0 → portal-3.4.2}/portal/server_socket.py

@@ -43,16 +43,17 @@ class ServerSocket:
     self.options = Options(**{**contextlib.context.serverkw, **kwargs})
     if self.options.ipv6:
       self.sock = socket.socket(socket.AF_INET6, socket.SOCK_STREAM)
+      self.sock.setsockopt(socket.IPPROTO_IPV6, socket.IPV6_V6ONLY, 1)
       self.addr = (self.options.host or '::', port, 0, 0)
     else:
       self.sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
       self.addr = (self.options.host or '0.0.0.0', port)
     self.sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
-    # self.sock.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1)
+    # self.sock.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1)  # TODO
     self._log(f'Binding to {self.addr[0]}:{self.addr[1]}')
     self.sock.bind(self.addr)
     self.sock.setblocking(False)
-    self.sock.listen()
+    self.sock.listen(8192)
     self.sel = selectors.DefaultSelector()
     self.sel.register(self.sock, selectors.EVENT_READ, data=None)
     self._log(f'Listening at {self.addr[0]}:{self.addr[1]}')

{portal-3.4.0 → portal-3.4.2/portal.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: portal
-Version: 3.4.0
+Version: 3.4.2
 Summary: Fast and reliable distributed systems in Python
 Home-page: http://github.com/danijar/portal
 Author: Danijar Hafner

{portal-3.4.0 → portal-3.4.2}/tests/test_server.py

@@ -165,6 +165,25 @@ class TestServer:
     assert completed != list(range(10))
     assert logged == list(range(10))
 
+  @pytest.mark.parametrize('repeat', range(10))
+  @pytest.mark.parametrize('Server', SERVERS)
+  def test_postfn_no_hang(self, repeat, Server):
+    def wrapper():
+      port = portal.free_port()
+      def workfn(x):
+        return x, x
+      def postfn(x):
+        time.sleep(0.01)
+      server = Server(port, workers=4)
+      server.bind('fn', workfn, postfn)
+      server.start(block=False)
+      client = portal.Client(port)
+      futures = [client.fn(i) for i in range(20)]
+      [future.result() for future in futures]  # Used to hang here.
+      client.close()
+      server.close()
+    assert portal.Thread(wrapper, start=True).join(timeout=10).exitcode == 0
+
   @pytest.mark.parametrize('repeat', range(5))
   @pytest.mark.parametrize('Server', SERVERS)
   @pytest.mark.parametrize('workers', (1, 4))
@@ -218,7 +237,7 @@ class TestServer:
     # The slow request is processed first, so this will wait until both
     # requests are done.
     fast_future.result()
-    assert slow_future.wait(0.01)
+    assert slow_future.wait(0.1)
     server.close()
     client.close()
 
@@ -244,7 +263,7 @@ class TestServer:
     # Both requests are processed in parallel, so the fast request returns
     # before the slow request is done.
     fast_future.result()
-    assert not slow_future.wait(0.01)
+    assert not slow_future.wait(0.1)
     server.close()
     client.close()
 
@@ -313,7 +332,7 @@ class TestServer:
     def fn(x):
       if x == 1:
         barrier.wait()
-        time.sleep(0.2)
+        time.sleep(0.5)
       return x
 
     port = portal.free_port()

{portal-3.4.0 → portal-3.4.2}/tests/test_socket.py

@@ -11,8 +11,6 @@ class TestSocket:
     port = portal.free_port()
     server = portal.ServerSocket(port, ipv6=ipv6)
     client = portal.ClientSocket(port, ipv6=ipv6)
-    client.connect()
-    assert client.connected
     client.send(b'foo')
     addr, data = server.recv()
     assert addr[0] == '::1' if ipv6 else '127.0.0.1'