portal 3.2.0__tar.gz → 3.2.3__tar.gz

{portal-3.2.0/portal.egg-info → portal-3.2.3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: portal
-Version: 3.2.0
+Version: 3.2.3
 Summary: Fast and reliable distributed systems in Python
 Home-page: http://github.com/danijar/portal
 Author: Danijar Hafner

{portal-3.2.0 → portal-3.2.3}/portal/__init__.py

@@ -1,4 +1,4 @@
-__version__ = '3.2.0'
+__version__ = '3.2.3'
 
 import multiprocessing as mp
 try:

{portal-3.2.0 → portal-3.2.3}/portal/client_socket.py

@@ -34,11 +34,11 @@ class Options:
 class ClientSocket:
 
   def __init__(self, host, port=None, name='Client', start=True, **kwargs):
-    assert (port or ':' in host) and '://' not in host, host
+    assert port or ':' in host, (host, port)
+    assert '://' not in host, (host, port)
     if port is None:
       host, port = host.rsplit(':', 1)
-      port = int(port)
-      assert host, host
+      assert host and port, (host, port)
     self.addr = (host, port)
     self.name = name
     self.options = Options(**{**contextlib.context.clientkw, **kwargs})
@@ -179,16 +179,19 @@ class ClientSocket:
       sock.close()
 
   def _connect(self):
-    host, port = self.addr
-    self._log(f'Connecting to {host}:{port}')
+    self._log(f'Connecting to {self.addr[0]}:{self.addr[1]}')
     once = True
     while self.running:
-      sock, addr = self._create()
+      # We need to resolve the address regularly.
+      host, port = self.addr
+      if contextlib.context.resolver:
+        host, port = contextlib.context.resolver((host, port))
+      assert isinstance(host, str), (host, port)
+      assert isinstance(port, int), (host, port)
+      addr = (host, port, 0, 0) if self.options.ipv6 else (host, port)
+      sock = self._create()
       error = None
       try:
-        # We need to resolve the address regularly.
-        if contextlib.context.resolver:
-          addr = contextlib.context.resolver(addr)
         sock.settimeout(10)
         sock.connect(addr)
         sock.settimeout(0)
@@ -210,10 +213,8 @@ class ClientSocket:
   def _create(self):
     if self.options.ipv6:
       sock = socket.socket(socket.AF_INET6, socket.SOCK_STREAM)
-      addr = (*self.addr, 0, 0)
     else:
       sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
-      addr = self.addr
     # sock.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1)
     after = self.options.keepalive_after
     every = self.options.keepalive_every
@@ -231,7 +232,7 @@ class ClientSocket:
       sock.setsockopt(socket.IPPROTO_TCP, socket.TCP_KEEPALIVE, every)
     if sys.platform == 'win32':
       sock.ioctl(socket.SIO_KEEPALIVE_VALS, (1, after * 1000, every * 1000))
-    return sock, addr
+    return sock
 
   def _log(self, *args):
     if not self.options.logging:

{portal-3.2.0 → portal-3.2.3}/portal/contextlib.py

@@ -4,6 +4,7 @@ import pathlib
 import sys
 import threading
 import traceback
+import warnings
 
 import cloudpickle
 import psutil
@@ -143,7 +144,12 @@ class Context:
       return
     if hasattr(worker, 'thread'):
       assert current != worker.thread
-    current.children.append(worker)
+    try:
+      current.children.append(worker)
+    except AttributeError:
+      warnings.warn(
+          'Using Portal from plain Python threads is discouraged because ' +
+          'they can cause hangs during shutdown.')
 
   def children(self, thread):
     current = thread or threading.current_thread()

{portal-3.2.0 → portal-3.2.3}/portal/process.py

@@ -1,6 +1,5 @@
 import atexit
 import errno
-import os
 import traceback
 
 import cloudpickle
@@ -37,8 +36,10 @@ class Process:
     name = name or getattr(fn, '__name__', 'process')
     fn = cloudpickle.dumps(fn)
     options = contextlib.context.options()
+    self.ready = contextlib.context.mp.Barrier(2)
     self.process = contextlib.context.mp.Process(
-        target=self._wrapper, name=name, args=(options, name, fn, args))
+        target=self._wrapper, name=name,
+        args=(options, self.ready, name, fn, args))
     self.started = False
     self.killed = False
     self.thepid = None
@@ -73,6 +74,7 @@ class Process:
     assert not self.started
     self.started = True
     self.process.start()
+    self.ready.wait()
     self.thepid = self.process.pid
     assert self.thepid is not None
     return self
@@ -83,8 +85,6 @@ class Process:
     return self
 
   def kill(self, timeout=1):
-    # Cannot early exit if process is not running, because it may just be
-    # starting up.
     assert self.started
     try:
       children = list(psutil.Process(self.pid).children(recursive=True))
@@ -108,9 +108,10 @@ class Process:
     return 'Process(' + ', '.join(attrs) + ')'
 
   @staticmethod
-  def _wrapper(options, name, fn, args):
+  def _wrapper(options, ready, name, fn, args):
     exitcode = 0
     try:
+      ready.wait()
       contextlib.setup(**options)
       fn = cloudpickle.loads(fn)
       exitcode = fn(*args)

{portal-3.2.0 → portal-3.2.3}/portal/server.py

@@ -148,4 +148,7 @@ class Server:
     data = message.encode('utf-8')
     self.socket.send(addr, reqnum, status, data)
     if self.errors:
+      # Wait until the error is delivered to the client and then raise.
+      self.socket.shutdown()
+      self.socket.close()
       raise RuntimeError(message)

{portal-3.2.0 → portal-3.2.3}/portal/thread.py

@@ -21,7 +21,6 @@ class Thread:
   """
 
   def __init__(self, fn, *args, name=None, start=False):
-    global TIDS
     self.fn = fn
     self.excode = None
     name = name or getattr(fn, '__name__', 'thread')

{portal-3.2.0 → portal-3.2.3}/portal/utils.py

@@ -8,6 +8,8 @@ import time
 
 import psutil
 
+from . import contextlib
+
 
 def run(workers, duration=None):
   [None if x.started else x.start() for x in workers]
@@ -96,9 +98,13 @@ def free_port():
   # Return a port that is currently free. This function is not thread or
   # process safe, because there is no way to guarantee that the port will still
   # be free at the time it will be used.
-  sock = socket.socket()
+  if contextlib.context.serverkw.get('ipv6', False):
+    family, addr = socket.AF_INET6, ('localhost', 0, 0, 0)
+  else:
+    family, addr = socket.AF_INET, ('localhost', 0)
+  sock = socket.socket(family, socket.SOCK_STREAM)
   sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
-  sock.bind(('localhost', 0))
+  sock.bind(addr)
   port = sock.getsockname()[1]
   sock.close()
   return port

{portal-3.2.0 → portal-3.2.3/portal.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: portal
-Version: 3.2.0
+Version: 3.2.3
 Summary: Fast and reliable distributed systems in Python
 Home-page: http://github.com/danijar/portal
 Author: Danijar Hafner

{portal-3.2.0 → portal-3.2.3}/tests/test_client.py

@@ -321,3 +321,20 @@ class TestClient:
         portal.Thread(server),
         portal.Thread(client),
     ])
+
+  def test_resolver(self):
+    portnum = portal.free_port()
+
+    def client(portnum):
+      def resolver(host, portstr):
+        assert portstr == 'name'
+        return host, portnum
+      portal.setup(resolver=resolver)
+      client = portal.Client('localhost:name')
+      assert client.fn(42).result() == 42
+
+    server = portal.Server(portnum)
+    server.bind('fn', lambda x: x)
+    server.start(block=False)
+    portal.Process(client, portnum, start=True).join()
+    server.close()

{portal-3.2.0 → portal-3.2.3}/tests/test_process.py

@@ -16,35 +16,43 @@ class TestProcess:
     assert worker.exitcode == 42
 
   def test_error(self):
+
     def fn():
       raise KeyError('foo')
+
     worker = portal.Process(fn, start=True)
     worker.join()
     assert not worker.running
     assert worker.exitcode == 1
 
   def test_error_with_children(self):
+
     def hang():
       while True:
         time.sleep(0.1)
+
     def fn():
       portal.Process(hang, start=True)
       portal.Thread(hang, start=True)
       time.sleep(0.1)
       raise KeyError('foo')
+
     worker = portal.Process(fn, start=True)
     worker.join()
     assert not worker.running
     assert worker.exitcode == 1
 
-  def test_kill(self):
+  @pytest.mark.parametrize('repeat', range(5))
+  def test_kill_basic(self, repeat):
+
     def fn():
       while True:
         time.sleep(0.1)
+
     worker = portal.Process(fn, start=True)
     worker.kill()
     assert not worker.running
-    assert worker.exitcode < 0
+    assert abs(worker.exitcode) >= 1
 
   @pytest.mark.parametrize('repeat', range(5))
   def test_kill_with_subproc(self, repeat):
@@ -52,14 +60,18 @@ class TestProcess:
     queue = portal.context.mp.Queue()
 
     def outer(ready, queue):
-      queue.put(os.getpid())
       portal.Process(inner, ready, queue, start=True)
+      queue.put(os.getpid())
+      queue.close()
+      queue.join_thread()
       ready.wait()
       while True:
         time.sleep(0.1)
 
     def inner(ready, queue):
       queue.put(os.getpid())
+      queue.close()
+      queue.join_thread()
       ready.wait()
       while True:
         time.sleep(0.1)
@@ -68,7 +80,7 @@ class TestProcess:
     ready.wait()
     worker.kill()
     assert not worker.running
-    assert worker.exitcode < 0
+    assert abs(worker.exitcode) >= 1
     assert not portal.proc_alive(queue.get())
     assert not portal.proc_alive(queue.get())
 
@@ -87,20 +99,25 @@ class TestProcess:
     ready.wait()
     worker.kill()
     assert not worker.running
-    assert worker.exitcode < 0
+    assert abs(worker.exitcode) >= 1
 
   def test_initfn(self):
+
     def init():
       portal.foo = 42
+
     portal.initfn(init)
     ready = portal.context.mp.Event()
     assert portal.foo == 42
+
     def outer(ready):
       assert portal.foo == 42
       portal.Process(inner, ready, start=True).join()
+
     def inner(ready):
       assert portal.foo == 42
       ready.set()
+
     portal.Process(outer, ready, start=True).join()
     ready.wait()
     assert ready.is_set()

{portal-3.2.0 → portal-3.2.3}/tests/test_server.py

@@ -91,6 +91,27 @@ class TestServer:
   def test_server_errors(self, Server):
     port = portal.free_port()
 
+    server = Server(port, errors=False)
+    def fn(x):
+      if x == 2:
+        raise ValueError(x)
+      return x
+    server.bind('fn', fn)
+    server.start(block=False)
+
+    client = portal.Client('localhost', port)
+    assert client.fn(1).result() == 1
+    with pytest.raises(RuntimeError):
+      client.fn(2).result()
+    assert client.fn(3).result() == 3
+
+    client.close()
+    server.close()
+
+  @pytest.mark.parametrize('Server', SERVERS)
+  def test_server_errors_raise(self, Server):
+    port = portal.free_port()
+
     def server(port):
       server = Server(port, errors=True)
       def fn(x):
@@ -108,8 +129,8 @@ class TestServer:
     client = portal.Client('localhost', port)
     assert client.fn(1).result() == 1
     assert server.running
-    with pytest.raises(RuntimeError):
-      client.fn(2).result()
+    with pytest.raises((RuntimeError, TimeoutError)):
+      client.fn(2).result(timeout=3)
 
     client.close()
     server.join()
@@ -178,7 +199,7 @@ class TestServer:
   def test_shared_pool(self, repeat, Server):
 
     def slow(x):
-      time.sleep(0.2)
+      time.sleep(0.5)
       return x
 
     def fast(x):
@@ -205,7 +226,7 @@ class TestServer:
   def test_separate_pools(self, repeat, Server):
 
     def slow(x):
-      time.sleep(0.1)
+      time.sleep(0.5)
       return x
 
     def fast(x):

{portal-3.2.0 → portal-3.2.3}/tests/test_thread.py

@@ -91,4 +91,4 @@ class TestThread:
     assert not worker.running
     assert not proc[0].running
     assert worker.exitcode == 2
-    assert proc[0].exitcode < 0
+    assert abs(proc[0].exitcode) >= 1