portal 3.1.11__tar.gz → 3.2.0__tar.gz

{portal-3.1.11/portal.egg-info → portal-3.2.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: portal
-Version: 3.1.11
+Version: 3.2.0
 Summary: Fast and reliable distributed systems in Python
 Home-page: http://github.com/danijar/portal
 Author: Danijar Hafner

{portal-3.1.11 → portal-3.2.0}/portal/__init__.py

@@ -1,4 +1,4 @@
-__version__ = '3.1.11'
+__version__ = '3.2.0'
 
 import multiprocessing as mp
 try:
@@ -29,6 +29,5 @@ from .packlib import tree_equals
 from .sharray import SharedArray
 
 from .utils import free_port
-from .utils import kill_procs
-from .utils import kill_threads
+from .utils import proc_alive
 from .utils import run

{portal-3.1.11 → portal-3.2.0}/portal/batching.py

@@ -53,8 +53,8 @@ class BatchServer:
   def close(self, timeout=None):
     assert self.started
     self.running.clear()
-    self.server.close(timeout and 0.5 * timeout)
-    self.batcher.join(timeout and 0.5 * timeout)
+    self.server.close(timeout)
+    self.batcher.join(timeout)
     self.batcher.kill()
 
   def stats(self):
@@ -160,12 +160,12 @@ def batcher(
     if errors:
       raise RuntimeError(message)
 
-  outer = server_socket.ServerSocket(outer_port, f'{name}Server', **kwargs)
-  inner = client.Client('localhost', inner_port, f'{name}Client', **kwargs)
-  batches = {}  # {method: ([addr], [reqnum], structure, [array])}
-  jobs = []
-  shutdown = False
   try:
+    outer = server_socket.ServerSocket(outer_port, f'{name}Server', **kwargs)
+    inner = client.Client('localhost', inner_port, f'{name}Client', **kwargs)
+    batches = {}  # {method: ([addr], [reqnum], structure, [array])}
+    jobs = []
+    shutdown = False
     while running.is_set() or jobs:
       if running.is_set():
         maybe_recv(outer, inner, jobs, batches)

{portal-3.1.11 → portal-3.2.0}/portal/client.py

@@ -80,7 +80,6 @@ class Client:
     name = method.encode('utf-8')
     strlen = len(name).to_bytes(8, 'little', signed=False)
     sendargs = (reqnum, strlen, name, *packlib.pack(data))
-    # self.socket.send(reqnum, strlen, name, *packlib.pack(data))
     rai = [False]
     future = Future(rai)
     future.sendargs = sendargs
@@ -117,7 +116,10 @@ class Client:
       message = bytes(data[16:]).decode('utf-8')
       self._seterr(future, RuntimeError(message))
       with self.cond: self.cond.notify_all()
-    self.socket.recv()
+    try:
+      self.socket.recv()
+    except AssertionError:
+      pass  # Socket is already closed.
 
   def _disc(self):
     if self.socket.options.autoconn:

{portal-3.1.11 → portal-3.2.0}/portal/client_socket.py

@@ -64,7 +64,6 @@ class ClientSocket:
     return self.isconn.is_set()
 
   def connect(self, timeout=None):
-    assert not self.connected
     if not self.options.autoconn:
       self.wantconn.set()
     return self.isconn.wait(timeout)
@@ -137,7 +136,6 @@ class ClientSocket:
           continue
         _, mask = pairs[0]
 
-
         if mask & select.POLLIN:
           try:
             recvbuf.recv(sock)

{portal-3.1.11 → portal-3.2.0}/portal/contextlib.py

@@ -1,4 +1,3 @@
-import collections
 import multiprocessing as mp
 import os
 import pathlib
@@ -23,7 +22,6 @@ class Context:
     self.serverkw = {}
     self.done = threading.Event()
     self.watcher = None
-    self.children = collections.defaultdict(list)
     self.mp = mp.get_context()
     self.printlock = self.mp.Lock()
 
@@ -61,6 +59,7 @@ class Context:
       assert hasattr(errfile, 'exists') and hasattr(errfile, 'write_text')
       self.errfile = errfile
       self._check_errfile()
+      self._install_excepthook()
 
     if interval:
       assert isinstance(interval, (int, float))
@@ -122,10 +121,14 @@ class Context:
       print(f'Wrote errorfile: {self.errfile}', file=sys.stderr)
 
   def shutdown(self, exitcode):
-    # This kills the process tree forcefully to prevent hangs but results in
-    # leaked semaphore warnings. However, the leaked objects are still cleaned
-    # up by the resource tracker process of Python's multiprocessing module.
-    utils.kill_procs(psutil.Process().children(recursive=True))
+    children = list(psutil.Process(os.getpid()).children(recursive=True))
+    utils.kill_proc(children, timeout=1)
+    # TODO
+    # if exitcode == 0:
+    #   for child in self.children(threading.main_thread()):
+    #     child.kill()
+    #   os._exit(0)
+    # else:
     os._exit(exitcode)
 
   def close(self):
@@ -133,21 +136,35 @@ class Context:
     if self.watcher:
       self.watcher.join()
 
-  def add_child(self, worker):
-    ident = threading.get_ident()
-    if hasattr(worker, 'ident'):
-      assert worker.ident != ident
-    self.children[ident].append(worker)
-
-  def get_children(self, ident=None):
-    if ident is None:
-      ident = threading.get_ident()
-    return self.children[ident]
+  def add_worker(self, worker):
+    assert hasattr(worker, 'kill')
+    current = threading.current_thread()
+    if current == threading.main_thread():
+      return
+    if hasattr(worker, 'thread'):
+      assert current != worker.thread
+    current.children.append(worker)
+
+  def children(self, thread):
+    current = thread or threading.current_thread()
+    if current == threading.main_thread():
+      return []
+    return current.children
 
   def _watcher(self):
     while not self.done.wait(self.interval):
       self._check_errfile()
 
+  def _install_excepthook(self):
+    existing = sys.excepthook
+    def patched(typ, val, tb):
+      if self.errfile:
+        message = ''.join(traceback.format_exception(typ, val, tb)).strip('\n')
+        self.errfile.write_text(message)
+        print(f'Wrote errorfile: {self.errfile}', file=sys.stderr)
+      return existing(typ, val, tb)
+    sys.excepthook = patched
+
   def _check_errfile(self):
     if self.errfile and self.errfile.exists():
       message = f'Shutting down due to error file: {self.errfile}'

{portal-3.1.11 → portal-3.2.0}/portal/process.py

@@ -1,4 +1,6 @@
 import atexit
+import errno
+import os
 import traceback
 
 import cloudpickle
@@ -20,24 +22,28 @@ class Process:
   2. The process terminates its nested child processes when it encounters an
   error. This prevents lingering subprocesses on error.
 
-  3. When the parent process encounters an error, the subprocess will be killed
+  3. When the parent process encounters an error, its subprocess will be killed
   via `atexit`, preventing hangs.
 
   4. It inherits the context object of its parent process, which provides
   cloudpickled initializer functions for each nested child process and error
   file watching for global shutdown on error.
+
+  5. Standard Python subprocesses do not reliably return the running() state of
+  the process. This class makes it more reliable.
   """
 
   def __init__(self, fn, *args, name=None, start=False):
     name = name or getattr(fn, '__name__', 'process')
     fn = cloudpickle.dumps(fn)
-    context = contextlib.context
-    options = context.options()
-    self.process = context.mp.Process(
+    options = contextlib.context.options()
+    self.process = contextlib.context.mp.Process(
         target=self._wrapper, name=name, args=(options, name, fn, args))
-    context.add_child(self)
     self.started = False
+    self.killed = False
+    self.thepid = None
     atexit.register(self.kill)
+    contextlib.context.add_worker(self)
     start and self.start()
 
   @property
@@ -46,44 +52,54 @@ class Process:
 
   @property
   def pid(self):
-    return self.process.pid
+    return self.thepid
 
   @property
   def running(self):
     if not self.started:
       return False
-    return self.process.is_alive()
+    if not self.process.is_alive():
+      return False
+    return utils.proc_alive(self.pid)
 
   @property
   def exitcode(self):
-    if not self.started or self.running:
-      return None
-    else:
-      return self.process.exitcode
+    exitcode = self.process.exitcode
+    if self.killed and exitcode is None:
+      return -9
+    return exitcode
 
   def start(self):
     assert not self.started
     self.started = True
     self.process.start()
-    assert self.pid is not None
+    self.thepid = self.process.pid
+    assert self.thepid is not None
     return self
 
   def join(self, timeout=None):
     assert self.started
-    if self.running:
-      self.process.join(timeout)
+    self.process.join(timeout)
     return self
 
   def kill(self, timeout=1):
+    # Cannot early exit if process is not running, because it may just be
+    # starting up.
+    assert self.started
     try:
-      proc = psutil.Process(self.pid)
-      tree = [proc] + list(proc.children(recursive=True))
+      children = list(psutil.Process(self.pid).children(recursive=True))
     except psutil.NoSuchProcess:
-      tree = []
-    self.process.terminate()
-    self.process.join(timeout / 2)
-    self.process.kill()
-    utils.kill_procs(tree, timeout / 2)
+      children = []
+    try:
+      self.process.terminate()
+      self.process.join(timeout)
+      self.process.kill()
+      utils.kill_proc(children, timeout)
+    except OSError as e:
+      if e.errno != errno.ESRCH:
+        contextlib.context.error(e, self.name)
+        contextlib.context.shutdown(exitcode=1)
+    self.killed = True
     return self
 
   def __repr__(self):

{portal-3.1.11 → portal-3.2.0}/portal/server.py

@@ -83,7 +83,7 @@ class Server:
     self.close()
 
   def _loop(self):
-    while self.running or self.jobs or self.postfn_out:
+    while self.running or self.jobs or self.postfn_inp or self.postfn_out:
       while True:  # Loop syntax used to break on error.
         if not self.running:  # Do not accept further requests.
           break

{portal-3.1.11 → portal-3.2.0}/portal/thread.py

@@ -1,5 +1,4 @@
 import threading
-import time
 import traceback
 
 from . import contextlib
@@ -22,13 +21,16 @@ class Thread:
   """
 
   def __init__(self, fn, *args, name=None, start=False):
+    global TIDS
     self.fn = fn
     self.excode = None
     name = name or getattr(fn, '__name__', 'thread')
     self.thread = threading.Thread(
         target=self._wrapper, args=args, name=name, daemon=True)
-    contextlib.context.add_child(self)
+    self.thread.children = []
     self.started = False
+    self.ready = threading.Barrier(2)
+    contextlib.context.add_worker(self)
     start and self.start()
 
   @property
@@ -36,8 +38,8 @@ class Thread:
     return self.thread.name
 
   @property
-  def ident(self):
-    return self.thread.ident
+  def tid(self):
+    return self.thetid
 
   @property
   def running(self):
@@ -53,15 +55,18 @@ class Thread:
     assert not self.started
     self.started = True
     self.thread.start()
+    self.ready.wait()
     return self
 
   def join(self, timeout=None):
-    if self.running:
-      self.thread.join(timeout)
+    self.thread.join(timeout)
     return self
 
-  def kill(self, timeout=3):
-    utils.kill_threads(self.thread, timeout)
+  def kill(self, timeout=1.0):
+    assert self.thread != threading.current_thread()
+    for child in contextlib.context.children(self.thread):
+      child.kill(timeout)
+    utils.kill_thread(self.thread, timeout)
     return self
 
   def __repr__(self):
@@ -71,6 +76,7 @@ class Thread:
 
   def _wrapper(self, *args):
     try:
+      self.ready.wait()
       exitcode = self.fn(*args)
       exitcode = exitcode if isinstance(exitcode, int) else 0
       self.excode = exitcode
@@ -78,10 +84,8 @@ class Thread:
       compact = traceback.format_tb(e.__traceback__)
       compact = '\n'.join([line.split('\n', 1)[0] for line in compact])
       print(f"Killed thread '{self.name}' at:\n{compact}")
-      [x.kill(0.1) for x in contextlib.context.get_children()]
+      [x.kill(0.1) for x in contextlib.context.children(self.thread)]
       self.excode = 2
     except Exception as e:
-      [x.kill(0.1) for x in contextlib.context.get_children()]
       contextlib.context.error(e, self.name)
-      contextlib.context.shutdown(1)
-      self.excode = 1
+      contextlib.context.shutdown(exitcode=1)

{portal-3.1.11 → portal-3.2.0}/portal/utils.py

@@ -1,10 +1,11 @@
 import ctypes
+import errno
+import os
 import socket
 import sys
 import threading
 import time
 
-import numpy as np
 import psutil
 
 
@@ -30,9 +31,8 @@ def run(workers, duration=None):
       raise RuntimeError(f"'{name}' crashed with exit code {code}")
 
 
-def kill_threads(threads, timeout=3):
+def kill_thread(threads, timeout=1):
   threads = threads if isinstance(threads, (list, tuple)) else [threads]
-  threads = [x for x in threads if x is not threading.main_thread()]
   for thread in threads:
     if thread.native_id is None:
       # Wait because thread may currently be starting.
@@ -52,7 +52,7 @@ def kill_threads(threads, timeout=3):
       print('Killed thread is still alive.')
 
 
-def kill_procs(procs, timeout=3):
+def kill_proc(procs, timeout=1):
   def eachproc(fn, procs):
     result = []
     for proc in list(procs):
@@ -75,19 +75,33 @@ def kill_procs(procs, timeout=3):
   # Should never happen but print warning if any survived.
   eachproc(lambda p: (
       print('Killed subprocess is still alive.')
-      if p.status() != psutil.STATUS_ZOMBIE else None), procs)
+      if proc_alive(p.pid) else None), procs)
 
 
-def free_port(low=10000, high=50000):
+def proc_alive(pid):
+  try:
+    if psutil.Process(pid).status() == psutil.STATUS_ZOMBIE:
+      return False
+  except psutil.NoSuchProcess:
+    return False
+  try:
+    os.kill(pid, 0)
+  except OSError as e:
+    if e.errno == errno.ESRCH:
+      return False
+  return True
+
+
+def free_port():
   # Return a port that is currently free. This function is not thread or
   # process safe, because there is no way to guarantee that the port will still
   # be free at the time it will be used.
-  rng = np.random.default_rng()
-  while True:
-    port = int(rng.integers(low, high))
-    with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s:
-      if s.connect_ex(('', port)):
-        return port
+  sock = socket.socket()
+  sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
+  sock.bind(('localhost', 0))
+  port = sock.getsockname()[1]
+  sock.close()
+  return port
 
 
 def style(color=None, background=None, bold=None, underline=None, reset=None):

{portal-3.1.11 → portal-3.2.0/portal.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: portal
-Version: 3.1.11
+Version: 3.2.0
 Summary: Fast and reliable distributed systems in Python
 Home-page: http://github.com/danijar/portal
 Author: Danijar Hafner

{portal-3.1.11 → portal-3.2.0}/tests/test_client.py

@@ -121,7 +121,7 @@ class TestClient:
       future.result(timeout=0.01)
     with pytest.raises(TimeoutError):
       future.result(timeout=0)
-    assert future.result(timeout=0.2) == 42
+    assert future.result(timeout=1) == 42
     client.close()
     server.close()
 
@@ -155,14 +155,10 @@ class TestClient:
     server = portal.Server(port)
     server.bind('fn', lambda x: x)
     server.start(block=False)
-    client = portal.Client('localhost', port)
+    client = portal.Client('localhost', port, maxinflight=1)
     client.fn(1)
     client.fn(2)
-    # Wait for the server to respond to the first two requests, so that all
-    # futures are inside the client by the time we block on the third future.
-    time.sleep(0.1)
     future3 = client.fn(3)
-    assert len(client.futures) == 1
     assert future3.result() == 3
     del future3
     assert len(client.futures) == 0

{portal-3.1.11 → portal-3.2.0}/tests/test_errfile.py

@@ -1,3 +1,4 @@
+import os
 import pathlib
 import time
 
@@ -50,37 +51,63 @@ class TestErrfile:
     assert "Error in 'fn1' (ValueError: reason):" == first_line
     assert not worker1.running
     assert not worker2.running
-    assert worker1.exitcode == 1
+    # The first worker may shut itself down or be shut down based on its own
+    # error file watcher, based on how the threads context-switch.
+    assert worker1.exitcode in (1, 2)
     assert worker2.exitcode == 2
 
   @pytest.mark.parametrize('repeat', range(3))
   def test_nested_procs(self, tmpdir, repeat):
     errfile = pathlib.Path(tmpdir) / 'error'
-    ready = portal.context.mp.Semaphore(0)
+    ready = portal.context.mp.Barrier(7)
+    queue = portal.context.mp.Queue()
 
-    def hang():
+    def outer(ready, queue, errfile):
+      portal.setup(errfile=errfile, interval=0.1)
+      portal.Process(inner, ready, queue, name='inner', start=True)
+      portal.Thread(hang_thread, ready, start=True)
+      portal.Process(hang_process, ready, queue, start=True)
+      queue.put(os.getpid())
+      queue.close()
+      queue.join_thread()
+      ready.wait()  # 1
       while True:
         time.sleep(0.1)
 
-    def outer(ready, errfile):
-      portal.setup(errfile=errfile, interval=0.1)
-      portal.Process(inner, ready, errfile, name='inner', start=True)
-      portal.Thread(hang, start=True)
-      portal.Process(hang, start=True)
-      ready.release()
-      hang()
-
-    def inner(ready, errfile):
-      portal.setup(errfile=errfile, interval=0.1)
-      portal.Thread(hang, start=True)
-      portal.Process(hang, start=True)
-      ready.release()
+    def inner(ready, queue):
+      assert portal.context.errfile
+      portal.Thread(hang_thread, ready, start=True)
+      portal.Process(hang_process, ready, queue, start=True)
+      queue.put(os.getpid())
+      queue.close()
+      queue.join_thread()
+      ready.wait()  # 2
       raise ValueError('reason')
 
-    worker = portal.Process(outer, ready, errfile, name='outer', start=True)
-    ready.acquire()
-    ready.acquire()
+    def hang_thread(ready):
+      ready.wait()  # 3, 4
+      while True:
+        time.sleep(0.1)
+
+    def hang_process(ready, queue):
+      assert portal.context.errfile
+      queue.put(os.getpid())
+      queue.close()
+      queue.join_thread()
+      ready.wait()  # 5, 6
+      while True:
+        time.sleep(0.1)
+
+    worker = portal.Process(
+        outer, ready, queue, errfile, name='outer', start=True)
+    ready.wait()  # 7
     worker.join()
     content = errfile.read_text()
     assert "Error in 'inner' (ValueError: reason):" == content.split('\n')[0]
     assert not worker.running
+    pids = [queue.get() for _ in range(4)]
+    time.sleep(2.0)  # On some systems this can take a while.
+    assert not portal.proc_alive(pids[0])
+    assert not portal.proc_alive(pids[1])
+    assert not portal.proc_alive(pids[2])
+    assert not portal.proc_alive(pids[3])

{portal-3.1.11 → portal-3.2.0}/tests/test_process.py

@@ -1,3 +1,4 @@
+import os
 import time
 
 import pytest
@@ -43,26 +44,33 @@ class TestProcess:
     worker = portal.Process(fn, start=True)
     worker.kill()
     assert not worker.running
-    assert worker.exitcode == -15
+    assert worker.exitcode < 0
 
   @pytest.mark.parametrize('repeat', range(5))
   def test_kill_with_subproc(self, repeat):
-    ready = portal.context.mp.Semaphore(0)
-    def outer(ready):
-      portal.Process(inner, ready, start=True)
-      ready.release()
+    ready = portal.context.mp.Barrier(3)
+    queue = portal.context.mp.Queue()
+
+    def outer(ready, queue):
+      queue.put(os.getpid())
+      portal.Process(inner, ready, queue, start=True)
+      ready.wait()
       while True:
         time.sleep(0.1)
-    def inner(ready):
-      ready.release()
+
+    def inner(ready, queue):
+      queue.put(os.getpid())
+      ready.wait()
       while True:
         time.sleep(0.1)
-    worker = portal.Process(outer, ready, start=True)
-    ready.acquire()
-    ready.acquire()
+
+    worker = portal.Process(outer, ready, queue, start=True)
+    ready.wait()
     worker.kill()
     assert not worker.running
-    assert worker.exitcode == -15
+    assert worker.exitcode < 0
+    assert not portal.proc_alive(queue.get())
+    assert not portal.proc_alive(queue.get())
 
   @pytest.mark.parametrize('repeat', range(5))
   def test_kill_with_subthread(self, repeat):
@@ -79,7 +87,7 @@ class TestProcess:
     ready.wait()
     worker.kill()
     assert not worker.running
-    assert worker.exitcode == -15
+    assert worker.exitcode < 0
 
   def test_initfn(self):
     def init():

{portal-3.1.11 → portal-3.2.0}/tests/test_server.py

@@ -151,16 +151,19 @@ class TestServer:
     lock = threading.Lock()
     work_calls = [0]
     done_calls = [0]
+
     def workfn(x):
       with lock:
         work_calls[0] += 1
         print(work_calls[0], done_calls[0])
         assert work_calls[0] <= done_calls[0] + workers + 1
       return x, x
+
     def postfn(x):
       with lock:
         done_calls[0] += 1
       time.sleep(0.01)
+
     server = Server(port, workers=workers)
     server.bind('fn', workfn, postfn)
     server.start(block=False)
@@ -173,11 +176,14 @@ class TestServer:
   @pytest.mark.parametrize('repeat', range(3))
   @pytest.mark.parametrize('Server', SERVERS)
   def test_shared_pool(self, repeat, Server):
+
     def slow(x):
       time.sleep(0.2)
       return x
+
     def fast(x):
       return x
+
     port = portal.free_port()
     server = Server(port, workers=1)
     server.bind('slow', slow)
@@ -197,11 +203,14 @@ class TestServer:
   @pytest.mark.parametrize('repeat', range(3))
   @pytest.mark.parametrize('Server', SERVERS)
   def test_separate_pools(self, repeat, Server):
+
     def slow(x):
       time.sleep(0.1)
       return x
+
     def fast(x):
       return x
+
     port = portal.free_port()
     server = Server(port)
     server.bind('slow', slow, workers=1)

{portal-3.1.11 → portal-3.2.0}/tests/test_thread.py

@@ -91,4 +91,4 @@ class TestThread:
     assert not worker.running
     assert not proc[0].running
     assert worker.exitcode == 2
-    assert proc[0].exitcode == -15
+    assert proc[0].exitcode < 0