portal 0.3.1__tar.gz → 3.0.0__tar.gz

portal-3.0.0/LICENSE

@@ -0,0 +1,19 @@
+Copyright 2023 Danijar Hafner
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of
+this software and associated documentation files (the "Software"), to deal in
+the Software without restriction, including without limitation the rights to
+use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies
+of the Software, and to permit persons to whom the Software is furnished to do
+so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

portal-3.0.0/MANIFEST.in

@@ -0,0 +1 @@
+include requirements.txt

portal-3.0.0/PKG-INFO

@@ -0,0 +1,79 @@
+Metadata-Version: 2.1
+Name: portal
+Version: 3.0.0
+Summary: Fast and reliable distributed systems in Python
+Home-page: http://github.com/danijar/portal
+Author: Danijar Hafner
+Author-email: mail@danijar.com
+Classifier: Intended Audience :: Science/Research
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Python :: 3
+Description-Content-Type: text/markdown
+License-File: LICENSE
+
+[![PyPI](https://img.shields.io/pypi/v/portal.svg)](https://pypi.python.org/pypi/portal/#history)
+
+# 🌀 Portal
+
+Fast and reliable distributed systems in Python.
+
+## Features
+
+- 📡 **Communication:** Portal lets you bind functions to a `Server` and call
+  them from a `Client`. Wait on results via `Future` objects. Clients can
+  automatically restore broken connections.
+- 🚀 **Performance:** Optimized for throughput and latency. Array data is
+  zero-copy serialized and deserialized for throughput near the hardware limit.
+- 🤸 **Flexibility:** Function inputs and outputs can be nested dicts and lists
+  of numbers, strings, bytes, None values, and Numpy arrays. Bytes allow
+  applications to chose their own serialization, such as `pickle`.
+- 🚨 **Error handlings:** Provides `Process` and `Thread` objects that can
+  reliably be killed by the parent. Unhandled exceptions in threads stop
+  the program. Error files can be used to stop distributed systems.
+- 📦 **Request batching:** Use `BatchServer` to collect multiple incoming
+  requests and process them at once, for example for AI inference servers.
+  Batching and dispatching happens in a separate process to free the GIL.
+- ✅ **Correctness:** Covered by over 100 unit tests for common usage and edge
+  cases and used for large scale distributed AI systems.
+
+## Installation
+
+```sh
+pip install portal
+```
+
+## Example
+
+This example runs the server and client in the same Python program using
+subprocesses, but they could also be separate Python scripts running on
+different machines.
+
+```python
+def server():
+  import portal
+  server = portal.Server(2222)
+  server.bind('add', lambda x, y: x + y)
+  server.bind('greet', lambda msg: print('Message from client:', msg))
+  server.start()
+
+def client():
+  import portal
+  client = portal.Client('localhost', 2222)
+  future = client.add(12, 42)
+  result = future.result()
+  print(result)  # 54
+  client.greet('Hello World')
+
+if __name__ == '__main__':
+  import portal
+  server_proc = portal.Process(server, start=True)
+  client_proc = portal.Process(client, start=True)
+  client_proc.join()
+  server_proc.kill()
+  print('Done')
+```
+
+## Questions
+
+Please open a separate [GitHub issue](https://github.com/danijar/portal/issues)
+for each question.

portal-3.0.0/README.md

@@ -0,0 +1,66 @@
+[![PyPI](https://img.shields.io/pypi/v/portal.svg)](https://pypi.python.org/pypi/portal/#history)
+
+# 🌀 Portal
+
+Fast and reliable distributed systems in Python.
+
+## Features
+
+- 📡 **Communication:** Portal lets you bind functions to a `Server` and call
+  them from a `Client`. Wait on results via `Future` objects. Clients can
+  automatically restore broken connections.
+- 🚀 **Performance:** Optimized for throughput and latency. Array data is
+  zero-copy serialized and deserialized for throughput near the hardware limit.
+- 🤸 **Flexibility:** Function inputs and outputs can be nested dicts and lists
+  of numbers, strings, bytes, None values, and Numpy arrays. Bytes allow
+  applications to chose their own serialization, such as `pickle`.
+- 🚨 **Error handlings:** Provides `Process` and `Thread` objects that can
+  reliably be killed by the parent. Unhandled exceptions in threads stop
+  the program. Error files can be used to stop distributed systems.
+- 📦 **Request batching:** Use `BatchServer` to collect multiple incoming
+  requests and process them at once, for example for AI inference servers.
+  Batching and dispatching happens in a separate process to free the GIL.
+- ✅ **Correctness:** Covered by over 100 unit tests for common usage and edge
+  cases and used for large scale distributed AI systems.
+
+## Installation
+
+```sh
+pip install portal
+```
+
+## Example
+
+This example runs the server and client in the same Python program using
+subprocesses, but they could also be separate Python scripts running on
+different machines.
+
+```python
+def server():
+  import portal
+  server = portal.Server(2222)
+  server.bind('add', lambda x, y: x + y)
+  server.bind('greet', lambda msg: print('Message from client:', msg))
+  server.start()
+
+def client():
+  import portal
+  client = portal.Client('localhost', 2222)
+  future = client.add(12, 42)
+  result = future.result()
+  print(result)  # 54
+  client.greet('Hello World')
+
+if __name__ == '__main__':
+  import portal
+  server_proc = portal.Process(server, start=True)
+  client_proc = portal.Process(client, start=True)
+  client_proc.join()
+  server_proc.kill()
+  print('Done')
+```
+
+## Questions
+
+Please open a separate [GitHub issue](https://github.com/danijar/portal/issues)
+for each question.

portal-3.0.0/portal/__init__.py

@@ -0,0 +1,34 @@
+__version__ = '3.0.0'
+
+import multiprocessing as mp
+try:
+  mp.set_start_method('spawn')
+except RuntimeError:
+  pass
+
+from .contextlib import context
+from .contextlib import initfn
+from .contextlib import reset
+from .contextlib import setup
+
+from .thread import Thread
+from .process import Process
+
+from .server_socket import ServerSocket
+from .client_socket import ClientSocket
+from .client_socket import Disconnected
+
+from .client import Client
+from .server import Server
+from .batching import BatchServer
+
+from .packlib import pack
+from .packlib import unpack
+from .packlib import tree_equals
+
+from .sharray import SharedArray
+
+from .utils import free_port
+from .utils import kill_procs
+from .utils import kill_threads
+from .utils import run

portal-3.0.0/portal/batching.py

@@ -0,0 +1,177 @@
+import threading
+
+import numpy as np
+import portal
+
+from . import client
+from . import packlib
+from . import process
+from . import server
+from . import server_socket
+from . import sharray
+from . import thread
+from . import utils
+
+
+class BatchServer:
+
+  def __init__(
+      self, port, name='Server', workers=1, errors=True,
+      process=True, shmem=False, **kwargs):
+    inner_port = utils.free_port()
+    self.name = name
+    self.server = server.Server(inner_port, name, workers, errors, **kwargs)
+    if process:
+      self.running = portal.context.mp.Event()
+    else:
+      self.running = threading.Event()
+    self.process = process
+    self.batsizes = {}
+    self.batargs = (
+        self.running, port, inner_port, f'{name}Batcher',
+        self.batsizes, errors, shmem, kwargs)
+    self.started = False
+
+  def bind(self, name, workfn, donefn=None, batch=0, workers=0):
+    assert not self.started
+    self.batsizes[name] = batch
+    self.server.bind(name, workfn, donefn, workers=workers)
+
+  def start(self, block=True):
+    assert not self.started
+    self.started = True
+    self.running.set()
+    if self.process:
+      self.batcher = process.Process(
+          batcher, *self.batargs, name=f'{self.name}Batcher', start=True)
+    else:
+      self.batcher = thread.Thread(
+          batcher, *self.batargs, name=f'{self.name}Batcher', start=True)
+    self.server.start(block=block)
+
+  def close(self, timeout=None):
+    assert self.started
+    self.running.clear()
+    self.server.close(timeout and 0.5 * timeout)
+    self.batcher.join(timeout and 0.5 * timeout)
+    self.batcher.kill()
+
+  def stats(self):
+    return self.server.stats()
+
+  def __enter__(self):
+    self.start(block=False)
+    return self
+
+  def __exit__(self, *e):
+    self.close()
+
+
+def batcher(
+    running, outer_port, inner_port, name, batsizes, errors, shmem,
+    kwargs):
+
+  def maybe_recv(outer, inner, jobs, batches):
+    if not running.is_set():  # Do not accept further requests.
+      return
+    try:
+      addr, data = outer.recv(timeout=0.0001)
+    except TimeoutError:
+      return
+    reqnum = bytes(data[:8])
+    data = data[8:]
+    strlen = int.from_bytes(data[:8], 'little', signed=False)
+    data = data[8:]
+    name, data = bytes(data[:strlen]).decode('utf-8'), data[strlen:]
+    if name not in batsizes:
+      send_error(addr, reqnum, 3, f'Unknown method {name}')
+      return
+    data = packlib.unpack(data)
+    batch_size = batsizes[name]
+    if not batch_size:
+      job = inner.call(name, *data)
+      job.args = (False, addr, reqnum)
+      jobs.append(job)
+      return
+    leaves, structure = packlib.tree_flatten(data)
+    leaves = [np.asarray(x) for x in leaves]
+    if any(x.dtype == object for x in leaves):
+      send_error(addr, reqnum, 5, 'Only array arguments can be batched.')
+      return
+    if name not in batches:
+      if shmem:
+        buffers = [
+            sharray.SharedArray((batch_size, *leaf.shape), leaf.dtype)
+            for leaf in leaves]
+      else:
+        buffers = [
+            np.empty((batch_size, *leaf.shape), leaf.dtype)
+            for leaf in leaves]
+      batches[name] = ([], [], structure, buffers)
+    addrs, reqnums, reference, buffers = batches[name]
+    if structure != reference:
+      send_error(addr, reqnum, 6, (
+          f'Argument structure {structure} does not match previous ' +
+          f'requests with structure {reference} for batched server ' +
+          f'method {name}.'))
+      return
+    index = len(addrs)
+    addrs.append(addr)
+    reqnums.append(reqnum)
+    for buffer, leaf in zip(buffers, leaves):
+      buffer[index] = leaf
+    if len(addrs) == batch_size:
+      del batches[name]
+      data = packlib.tree_unflatten(buffers, reference)
+      job = inner.call(name, *data)
+      job.args = (True, addrs, reqnums)
+      jobs.append(job)
+
+  def maybe_send(outer, inner, jobs):
+    done, waiting = [], []
+    [done.append(x) if x.done() else waiting.append(x) for x in jobs]
+    for job in done:
+      batched, addr, reqnum = job.args
+      try:
+        result = job.result()
+      except RuntimeError as e:
+        if batched:
+          for i, (addr, reqnum) in enumerate(zip(addr, reqnum)):
+            send_error(addr, reqnum, 6, e.args[0])
+        else:
+          send_error(addr, reqnum, 6, e.args[0])
+        continue
+      status = int(0).to_bytes(8, 'little', signed=True)
+      if batched:
+        for i, (addr, reqnum) in enumerate(zip(addr, reqnum)):
+          data = packlib.pack(packlib.tree_map(lambda x: x[i], result))
+          outer.send(addr, reqnum, status, *data)
+      else:
+        data = packlib.pack(result)
+        outer.send(addr, reqnum, status, *data)
+    return waiting
+
+  def send_error(addr, reqnum, status, message):
+    assert 1 <= status, status
+    status = status.to_bytes(8, 'little', signed=False)
+    data = message.encode('utf-8')
+    outer.send(addr, reqnum, status, data)
+    if errors:
+      raise RuntimeError(message)
+
+  outer = server_socket.ServerSocket(outer_port, f'{name}Server', **kwargs)
+  inner = client.Client('localhost', inner_port, f'{name}Client', **kwargs)
+  batches = {}  # {method: ([addr], [reqnum], structure, [array])}
+  jobs = []
+  shutdown = False
+  try:
+    while running.is_set() or jobs:
+      if running.is_set():
+        maybe_recv(outer, inner, jobs, batches)
+      elif not shutdown:
+        shutdown = True
+        outer.shutdown()
+      jobs = maybe_send(outer, inner, jobs)
+  finally:
+    outer.close()
+    inner.close()

portal-3.0.0/portal/buffers.py

@@ -0,0 +1,93 @@
+import collections
+import os
+import weakref
+
+import numpy as np
+
+
+class SendBuffer:
+
+  def __init__(self, *buffers, maxsize=None):
+    for buffer in buffers:
+      assert isinstance(buffer, (bytes, bytearray, memoryview)), type(buffer)
+      assert not isinstance(buffer, memoryview) or buffer.c_contiguous
+    buffers = tuple(
+         x.cast('c') if isinstance(x, memoryview) else x for x in buffers)
+    length = sum(len(x) for x in buffers)
+    assert all(len(x) for x in buffers)
+    assert 1 <= length, length
+    assert not maxsize or length <= length, (length, maxsize)
+    lenbuf = length.to_bytes(4, 'little', signed=False)
+    self.buffers = [lenbuf, *buffers]
+    self.remaining = collections.deque(self.buffers)
+    self.pos = 0
+
+  def __repr__(self):
+    lens = [len(x) for x in self.buffers]
+    left = [len(x) for x in self.remaining]
+    return f'SendBuffer(pos={self.pos}, lengths={lens} remaining={left})'
+
+  def reset(self):
+    self.remaining = collections.deque(self.buffers)
+    self.pos = 0
+
+  def send(self, sock):
+    first, *others = self.remaining
+    assert self.pos < len(first)
+    # The writev() call blocks but seems to be slightly faster than sendmsg().
+    size = os.writev(sock.fileno(), [memoryview(first)[self.pos:], *others])
+    # size = sock.sendmsg(
+    #     [memoryview(first)[self.pos:], *others], (), socket.MSG_DONTWAIT)
+    if size == 0:
+      raise ConnectionResetError
+    assert 0 <= size, size
+    self.pos += max(0, size)
+    while self.remaining and self.pos >= len(self.remaining[0]):
+      self.pos -= len(self.remaining.popleft())
+    return size
+
+  def done(self):
+    return not self.remaining
+
+
+class RecvBuffer:
+
+  def __init__(self, maxsize):
+    self.maxsize = maxsize
+    self.lenbuf = bytearray(4)
+    self.buffer = None
+    self.pos = 0
+
+  def __repr__(self):
+    length = self.buffer and len(self.buffer)
+    return f'RecvBuffer(pos={self.pos}, length={length})'
+
+  def recv(self, sock):
+    if self.buffer is None:
+      size = sock.recv_into(memoryview(self.lenbuf)[self.pos:])
+      self.pos += max(0, size)
+      if self.pos == 4:
+        length = int.from_bytes(self.lenbuf, 'little', signed=False)
+        assert 1 <= length <= self.maxsize, (1, length, self.maxsize)
+        # We use Numpy to allocate uninitialized memory because Python's
+        # `bytearray(length)` zero initializes which is slow. This also means
+        # the buffer cannot be pickled accidentally unless explicitly converted
+        # to a `bytes()` object, which is a nice bonus for preventing
+        # performance bugs in user code.
+        arr = np.empty(length, np.uint8)
+        self.buffer = memoryview(arr.data)
+        weakref.finalize(self.buffer, lambda arr=arr: arr)
+        self.pos = 0
+    else:
+      size = sock.recv_into(self.buffer[self.pos:])
+      self.pos += max(0, size)
+      assert 0 <= self.pos <= len(self.buffer), (0, self.pos, len(self.buffer))
+    if size == 0:
+      raise ConnectionResetError
+    return size
+
+  def done(self):
+    return self.buffer and self.pos == len(self.buffer)
+
+  def result(self):
+    return self.buffer

portal-3.0.0/portal/client.py

@@ -0,0 +1,185 @@
+import collections
+import functools
+import itertools
+import threading
+import time
+import weakref
+
+from . import client_socket
+from . import packlib
+
+
+class Client:
+
+  def __init__(
+      self, host, port, name='Client', maxinflight=16, **kwargs):
+    assert 1 <= maxinflight, maxinflight
+    self.maxinflight = maxinflight
+    self.reqnum = iter(itertools.count(0))
+    self.futures = {}
+    self.errors = collections.deque()
+    self.sendrate = [0, time.time()]
+    self.recvrate = [0, time.time()]
+    self.waitmean = [0, 0]
+    self.cond = threading.Condition()
+    self.lock = threading.Lock()
+    # Socket is created after the above attributes because the callbacks access
+    # some of the attributes.
+    self.socket = client_socket.ClientSocket(
+        host, port, name, start=False, **kwargs)
+    self.socket.callbacks_recv.append(self._recv)
+    self.socket.callbacks_disc.append(self._disc)
+    self.socket.callbacks_conn.append(self._conn)
+    self.socket.start()
+
+  @property
+  def connected(self):
+    return self.socket.connected
+
+  def __getattr__(self, name):
+    if name.startswith('_'):
+      raise AttributeError(name)
+    try:
+      return functools.partial(self.call, name)
+    except AttributeError:
+      raise ValueError(name)
+
+  def stats(self):
+    now = time.time()
+    stats = {
+        'inflight': self._numinflight(),
+        'numsend': self.sendrate[0],
+        'numrecv': self.recvrate[0],
+        'sendrate': self.sendrate[0] / (now - self.sendrate[1]),
+        'recvrate': self.recvrate[0] / (now - self.recvrate[1]),
+        'waitmean': self.waitmean[0] and (self.waitmean[1] / self.waitmean[0]),
+    }
+    self.sendrate = [0, now]
+    self.recvrate = [0, now]
+    self.waitmean = [0, 0]
+    return stats
+
+  def connect(self, timeout=None):
+    return self.socket.connect(timeout)
+
+  def call(self, method, *data):
+    reqnum = next(self.reqnum).to_bytes(8, 'little', signed=False)
+    start = time.time()
+    while self._numinflight() >= self.maxinflight:
+      with self.cond: self.cond.wait(timeout=0.2)
+      try:
+        self.socket.require_connection(timeout=0)
+      except TimeoutError:
+        pass
+    with self.lock:
+      self.waitmean[1] += time.time() - start
+      self.waitmean[0] += 1
+      self.sendrate[0] += 1
+    if self.errors:  # Raise errors of dropped futures.
+      raise self.errors.popleft()
+    name = method.encode('utf-8')
+    strlen = len(name).to_bytes(8, 'little', signed=False)
+    sendargs = (reqnum, strlen, name, *packlib.pack(data))
+    # self.socket.send(reqnum, strlen, name, *packlib.pack(data))
+    self.socket.send(*sendargs)
+    future = Future()
+    future.sendargs = sendargs
+    self.futures[reqnum] = future
+    return future
+
+  def close(self, timeout=None):
+    for future in self.futures.values():
+      self._seterr(future, client_socket.Disconnected)
+    self.futures.clear()
+    self.socket.close(timeout)
+
+  def _numinflight(self):
+    return len([x for x in self.futures.values() if not x.don])
+
+  def _recv(self, data):
+    assert len(data) >= 16, 'Unexpectedly short response'
+    reqnum = bytes(data[:8])
+    status = int.from_bytes(data[8:16], 'little', signed=False)
+    future = self.futures.pop(reqnum, None)
+    assert future, (
+        f'Unexpected request number: {reqnum}',
+        sorted(self.futures.keys()))
+    if status == 0:
+      data = packlib.unpack(data[16:])
+      future.set_result(data)
+    else:
+      message = bytes(data[16:]).decode('utf-8')
+      self._seterr(future, RuntimeError(message))
+    with self.cond:
+      self.cond.notify_all()
+    self.socket.recv()
+
+  def _disc(self):
+    if self.socket.options.autoconn:
+      for future in self.futures.values():
+        future.resend = True
+    else:
+      for future in self.futures.values():
+        self._seterr(future, client_socket.Disconnected)
+      self.futures.clear()
+
+  def _conn(self):
+    if self.socket.options.autoconn:
+      for future in list(self.futures.values()):
+        if getattr(future, 'resend', False):
+          self.socket.send(*future.sendargs)
+
+  def _seterr(self, future, e):
+    raised = [False]
+    future.raised = raised
+    future.set_error(e)
+    weakref.finalize(future, lambda: (
+        None if raised[0] else self.errors.append(e)))
+
+
+class Future:
+
+  def __init__(self):
+    self.raised = [False]
+    self.con = threading.Condition()
+    self.don = False
+    self.res = None
+    self.err = None
+
+  def __repr__(self):
+    if not self.done:
+      return 'Future(done=False)'
+    elif self.err:
+      return f"Future(done=True, error='{self.err}', raised={self.raised[0]})"
+    else:
+      return 'Future(done=True)'
+
+  def wait(self, timeout=None):
+    if self.don:
+      return self.don
+    with self.con: return self.con.wait(timeout)
+
+  def done(self):
+    return self.don
+
+  def result(self, timeout=None):
+    if not self.wait(timeout):
+      raise TimeoutError
+    assert self.don
+    if self.err is None:
+      return self.res
+    if not self.raised[0]:
+      self.raised[0] = True
+      raise self.err
+
+  def set_result(self, result):
+    assert not self.don
+    self.don = True
+    self.res = result
+    with self.con: self.con.notify_all()
+
+  def set_error(self, e):
+    assert not self.don
+    self.don = True
+    self.err = e
+    with self.con: self.con.notify_all()