opentrons 8.6.0a1__py3-none-any.whl

opentrons/util/async_helpers.py

@@ -0,0 +1,166 @@
+"""
+util.async_helpers - various utilities for asyncio functions and tasks.
+"""
+
+from functools import wraps
+from threading import Thread
+from typing import (
+    Any,
+    AsyncContextManager,
+    Awaitable,
+    Callable,
+    Generator,
+    Tuple,
+    TypeVar,
+    cast,
+)
+
+import asyncio
+import contextlib
+import queue
+
+
+async def asyncio_yield() -> None:
+    """
+    Ensure that the current task yields to the event loop.
+
+    The python async framework only switches between concurrently-running tasks when
+    the currently-running task hits a leaf call that requires waiting for an event on
+    the loop. That means that you can have a very long call of nice coroutines with
+    async def and await call() that still effectively "block" other concurrent tasks
+    from actually executing.
+
+    There's also not really a nice way to yield to the event loop explicitly. The best
+    way to do it is to drop in await asyncio.sleep(0), which will yield to the loop and
+    let something else run.
+
+    If you have an async call chain that is used in a tight loop and at no point contains
+    an await() call that will yield the loop - this will be anything that
+    - Involves touching a file descriptor that is registered with an asyncio protocol
+    - Involves waiting for a Future
+    - Involves waiting for another task
+    - Involves specifically invoking an asyncio scheduling operation like gather()
+    - Uses asyncio.sleep()
+
+    You should drop an await asyncio_yield() at the most frequent leaf call to make sure
+    that other tasks on the loop get a chance to run.
+    """
+    await asyncio.sleep(0)
+
+
+_Wrapped = TypeVar("_Wrapped", bound=Callable[..., Awaitable[Any]])
+
+
+def ensure_yield(async_def_func: _Wrapped) -> _Wrapped:
+    """
+    A decorator that makes sure that asyncio_yield() is called after the decorated async
+    function finishes executing.
+    """
+
+    # _wrapper can be typed using ParamSpec https://docs.python.org/3/library/typing.html#typing.ParamSpec
+    # when we
+    # - bump to a mypy version that supports it (>0.930)
+    # - either go to python 3.10 or bump typing_extensions to 4.2.0
+    # Until then, this is a mess of opaque type vars and disabling internal checking because of their
+    # opacity. The cast on line 58 should keep the external interface of the annotated function correct.
+    @wraps(async_def_func)
+    async def _wrapper(*args: Any, **kwargs: Any) -> Any:
+        ret = await async_def_func(*args, **kwargs)
+        await asyncio_yield()
+        return ret
+
+    return cast(_Wrapped, _wrapper)
+
+
+_ContextManagerResult = TypeVar("_ContextManagerResult")
+
+
+@contextlib.contextmanager
+def async_context_manager_in_thread(
+    async_context_manager: AsyncContextManager[_ContextManagerResult],
+) -> Generator[Tuple[_ContextManagerResult, asyncio.AbstractEventLoop], None, None]:
+    """Enter an async context manager in a worker thread.
+
+    When you enter this context manager, it:
+
+    1. Spawns a worker thread.
+    2. In that thread, starts an asyncio event loop.
+    3. In that event loop, enters the context manager that you passed in.
+    4. Returns: the result of entering that context manager, and the running event loop.
+       Use functions like `asyncio.run_coroutine_threadsafe()` to safely interact
+       with the returned object from your thread.
+
+    When you exit this context manager, it:
+
+    1. In the worker thread's event loop, exits the context manager that you passed in.
+    2. Stops and cleans up the worker thread's event loop.
+    3. Joins the worker thread.
+    """
+    with _run_loop_in_thread() as loop_in_thread:
+        async_object = asyncio.run_coroutine_threadsafe(
+            async_context_manager.__aenter__(),
+            loop=loop_in_thread,
+        ).result()
+
+        try:
+            yield async_object, loop_in_thread
+
+        finally:
+            exit = asyncio.run_coroutine_threadsafe(
+                async_context_manager.__aexit__(None, None, None),
+                loop=loop_in_thread,
+            )
+            exit.result()
+
+
+@contextlib.contextmanager
+def _run_loop_in_thread() -> Generator[asyncio.AbstractEventLoop, None, None]:
+    """Run an event loop in a worker thread.
+
+    Entering this context manager spawns a thread, starts an asyncio event loop in it,
+    and returns that loop.
+
+    Exiting this context manager stops and cleans up the event loop, and then joins the thread.
+    """
+    loop_mailbox: "queue.SimpleQueue[asyncio.AbstractEventLoop]" = queue.SimpleQueue()
+
+    def _in_thread() -> None:
+        loop = asyncio.new_event_loop()
+
+        # We assume that the lines above this will never fail,
+        # so we will always reach this point to unblock the parent thread.
+        loop_mailbox.put(loop)
+
+        loop.run_forever()
+
+        # If we've reached here, the loop has been stopped from outside this thread. Clean it up.
+        #
+        # This cleanup is naive because asyncio makes it difficult and confusing to get it right.
+        # Compare this with asyncio.run()'s cleanup, which:
+        #
+        # * Cancels and awaits any remaining tasks
+        #   (according to the source code--this seems undocumented)
+        # * Shuts down asynchronous generators
+        #   (see asyncio.shutdown_asyncgens())
+        # * Shuts down the default thread pool executor
+        #   (see https://bugs.python.org/issue34037 and asyncio.shutdown_default_executor())
+        #
+        # In Python >=3.11, we should rewrite this to use asyncio.Runner,
+        # which can take care of these nuances for us.
+        loop.close()
+
+    thread = Thread(
+        target=_in_thread,
+        name=f"{__name__} event loop thread",
+        # This is a load-bearing daemon=True. It avoids @atexit-related deadlocks when this is used
+        # by opentrons.execute and cleaned up by opentrons.execute's @atexit handler.
+        # https://github.com/Opentrons/opentrons/pull/12970#issuecomment-1648243785
+        daemon=True,
+    )
+    thread.start()
+    loop_in_thread = loop_mailbox.get()
+    try:
+        yield loop_in_thread
+    finally:
+        loop_in_thread.call_soon_threadsafe(loop_in_thread.stop)
+        thread.join()

opentrons/util/broker.py

@@ -0,0 +1,84 @@
+"""A simple pub/sub message broker."""
+
+
+from abc import ABC, abstractmethod
+from contextlib import contextmanager
+from typing import Callable, ContextManager, Generator, Generic, Set, TypeVar
+
+
+_MessageT = TypeVar("_MessageT")
+_CallbackT = Callable[[_MessageT], None]
+
+
+class ReadOnlyBroker(ABC, Generic[_MessageT]):
+    """The read-only subset of `Broker`.
+
+    Useful for typing if you want people to be able to subscribe to your `Broker`,
+    but don't want them to be able to publish their own messages to it.
+    """
+
+    @abstractmethod
+    def subscribed(self, callback: _CallbackT[_MessageT]) -> ContextManager[None]:
+        """See `Broker.subscribed()`."""  # noqa: D402
+        pass
+
+    @abstractmethod
+    def subscribe(self, callback: _CallbackT[_MessageT]) -> Callable[[], None]:
+        """See `Broker.subscribe()`."""  # noqa: D402
+        pass
+
+
+class Broker(Generic[_MessageT], ReadOnlyBroker[_MessageT]):
+    """A simple pub/sub message broker.
+
+    Subscribers can listen to events. Publishers can push events to all subscribers.
+    """
+
+    def __init__(self) -> None:
+        self._callbacks: Set[_CallbackT[_MessageT]] = set()
+
+    @contextmanager
+    def subscribed(
+        self, callback: _CallbackT[_MessageT]
+    ) -> Generator[None, None, None]:
+        """Register a callback to be called on each message.
+
+        The callback is subscribed when this context manager is entered,
+        and unsubscribed when it's exited.
+
+        You must not subscribe the same callback again unless you first usubscribe it.
+        """
+        unsubscribe = self.subscribe(callback)
+        try:
+            yield
+        finally:
+            unsubscribe()
+
+    def subscribe(self, callback: _CallbackT[_MessageT]) -> Callable[[], None]:
+        """Register a callback to be called on each message.
+
+        You must not subscribe the same callback again unless you first unsubscribe it.
+
+        Returns:
+            A function that you can call to unsubscribe ``callback``.
+            You must not call it more than once.
+        """
+
+        def unsubscribe() -> None:
+            self._callbacks.remove(callback)
+
+        self._callbacks.add(callback)
+        return unsubscribe
+
+    def publish(self, message: _MessageT) -> None:
+        """Call every subscribed callback, with ``message`` as the argument.
+
+        The order in which the callbacks are called is undefined.
+
+        If any callback raises an exception, it's propagated,
+        and any remaining callbacks will be left uncalled.
+        """
+        # Callback order is undefined because
+        # Python sets don't preserve insertion order.
+        for callback in self._callbacks:
+            callback(message)

opentrons/util/change_notifier.py

@@ -0,0 +1,47 @@
+"""Simple state change notification interface."""
+import asyncio
+
+
+class ChangeNotifier:
+    """An interface to emit or subscribe to state change notifications."""
+
+    def __init__(self) -> None:
+        """Initialize the ChangeNotifier with an internal Event."""
+        self._event = asyncio.Event()
+
+    def notify(self) -> None:
+        """Notify all `wait`'ers that the state has changed."""
+        self._event.set()
+
+    async def wait(self) -> None:
+        """Wait until the next state change notification."""
+        await self._event.wait()
+        self._event.clear()
+
+
+class ChangeNotifier_ts(ChangeNotifier):
+    """ChangeNotifier initialized with Event_ts."""
+
+    def __init__(self) -> None:
+        """Initialize the ChangeNotifier_Ts with an internal Event_ts."""
+        super().__init__()
+        self._event = Event_ts()
+
+
+class Event_ts(asyncio.Event):
+    """asyncio.Event with threadsafe methods."""
+
+    def __init__(self) -> None:
+        """Initialize Event_ts with the active event_loop or event_loop_policy if not active."""
+        super().__init__()
+        if self._loop is None:
+            self._loop: asyncio.AbstractEventLoop = asyncio.get_event_loop()
+
+    def set(self) -> None:
+        """Primarily intended for calling from a thread not responsible for the event loop.
+
+        Calling set() from the event loop thread will actually delay the execution of the set() until the
+        calling method either yields, awaits, or exits altogether. This is usually fine but might occasionally cause
+        unexpected behavior.
+        """
+        self._loop.call_soon_threadsafe(super().set)

opentrons/util/entrypoint_util.py

@@ -0,0 +1,278 @@
+""" opentrons.util.entrypoint_util: functions common to entrypoints
+"""
+
+import asyncio
+import contextlib
+from dataclasses import dataclass
+import json
+import logging
+import pathlib
+import subprocess
+import sys
+import tempfile
+from typing import (
+    Dict,
+    Generator,
+    List,
+    Optional,
+    Sequence,
+    Union,
+    TYPE_CHECKING,
+)
+
+from opentrons.calibration_storage.deck_configuration import (
+    deserialize_deck_configuration,
+)
+from opentrons.config import (
+    ARCHITECTURE,
+    IS_ROBOT,
+    JUPYTER_NOTEBOOK_LABWARE_DIR,
+    SystemArchitecture,
+)
+from opentrons.protocols import labware
+from opentrons.calibration_storage import helpers
+from opentrons.protocol_engine.errors.error_occurrence import (
+    ErrorOccurrence as ProtocolEngineErrorOccurrence,
+)
+from opentrons.protocol_engine.types import DeckConfigurationType
+from opentrons.protocol_reader import ProtocolReader, ProtocolSource
+from opentrons.protocols.types import JsonProtocol, Protocol, PythonProtocol
+
+if TYPE_CHECKING:
+    from opentrons_shared_data.labware.types import LabwareDefinition
+
+
+log = logging.getLogger(__name__)
+
+
+@dataclass
+class FoundLabware:
+    """An individual labware found by `labware_from_paths()`."""
+
+    path: pathlib.Path
+    definition: "LabwareDefinition"
+
+
+def labware_from_paths(
+    paths: Sequence[Union[str, pathlib.Path]]
+) -> Dict[str, FoundLabware]:
+    """Search paths for labware definitions.
+
+    Returns:
+        A dict, keyed by labware URI, where each value has the file path and the parsed def.
+    """
+    labware_defs: Dict[str, FoundLabware] = {}
+
+    for strpath in paths:
+        log.info(f"local labware: checking path {strpath}")
+        purepath = pathlib.PurePath(strpath)
+        if purepath.is_absolute():
+            path = pathlib.Path(purepath)
+        else:
+            path = pathlib.Path.cwd() / purepath
+        if not path.is_dir():
+            raise RuntimeError(f"{path} is not a directory")
+        for child in path.iterdir():
+            if child.is_file() and child.suffix.endswith("json"):
+                try:
+                    defn = labware.verify_definition(child.read_bytes())
+                except labware.NotALabwareError:
+                    log.info(f"{child}: invalid labware, ignoring")
+                    log.debug(
+                        f"{child}: labware invalid because of this exception.",
+                        exc_info=True,
+                    )
+                else:
+                    uri = helpers.uri_from_definition(defn)
+                    labware_defs[uri] = FoundLabware(path=child, definition=defn)
+                    log.info(f"loaded labware {uri} from {child}")
+            else:
+                log.info(f"ignoring {child} in labware path")
+    return labware_defs
+
+
+def find_jupyter_labware() -> Optional[Dict[str, FoundLabware]]:
+    """Return labware files in this robot's Jupyter Notebook directory.
+
+    Returns:
+        If we're running on an Opentrons robot:
+        A dict, keyed by labware URI, where each value has the file path and the parsed def.
+
+        Otherwise: None.
+    """
+    if IS_ROBOT:
+        # JUPYTER_NOTEBOOK_LABWARE_DIR should never be None when IS_ROBOT == True.
+        assert JUPYTER_NOTEBOOK_LABWARE_DIR is not None
+        if JUPYTER_NOTEBOOK_LABWARE_DIR.is_dir():
+            return labware_from_paths([JUPYTER_NOTEBOOK_LABWARE_DIR])
+
+    return None
+
+
+def datafiles_from_paths(paths: Sequence[Union[str, pathlib.Path]]) -> Dict[str, bytes]:
+    datafiles: Dict[str, bytes] = {}
+    for strpath in paths:
+        log.info(f"data files: checking path {strpath}")
+        purepath = pathlib.PurePath(strpath)
+        if purepath.is_absolute():
+            path = pathlib.Path(purepath)
+        else:
+            path = pathlib.Path.cwd() / purepath
+        if path.is_file():
+            datafiles[path.name] = path.read_bytes()
+            log.info(f"read {path} into custom data as {path.name}")
+        elif path.is_dir():
+            for child in path.iterdir():
+                if child.is_file():
+                    datafiles[child.name] = child.read_bytes()
+                    log.info(f"read {child} into data path as {child.name}")
+                else:
+                    log.info(f"ignoring {child} in data path")
+    return datafiles
+
+
+def get_deck_configuration() -> DeckConfigurationType:
+    """Return the host robot's current deck configuration.
+
+    This is a hacky implementation because the deck configuration is owned by
+    robot-server. See `robot_server.deck_configuration.cli`.
+    """
+    match ARCHITECTURE:
+        # These hard-coded paths need to be kept in sync with robot-server's
+        # systemd configuration:
+        case SystemArchitecture.BUILDROOT:
+            robot_server_persistence_dir = "/data/opentrons_robot_server"
+        case SystemArchitecture.YOCTO:
+            robot_server_persistence_dir = "/var/lib/opentrons-robot-server"
+        case SystemArchitecture.HOST:
+            # todo(mm, 2024-08-13):
+            # It's unclear what should happen if you run an `opentrons.execute` entry
+            # point on a non-robot device that doesn't have robot-server. I can't
+            # imagine we'd let actual users do this, but it might happen in internal
+            # testing. This empty list return value is totally arbitrary and will
+            # probably not do the right thing.
+            return []
+
+    proc = subprocess.run(
+        [
+            sys.executable,
+            "-m",
+            "robot_server.deck_configuration.cli",
+            "--persistence-directory",
+            robot_server_persistence_dir,
+        ],
+        check=True,
+        capture_output=True,
+    )
+    deserialized_deck_configuration = deserialize_deck_configuration(proc.stdout)
+    if deserialized_deck_configuration is None:
+        raise RuntimeError("Error getting the host robot's deck configuration.")
+
+    cutout_fixture_placements, _ = deserialized_deck_configuration
+
+    return [
+        (
+            cutout_fixture_placement.cutout_id,
+            cutout_fixture_placement.cutout_fixture_id,
+            cutout_fixture_placement.opentrons_module_serial_number,
+        )
+        for cutout_fixture_placement in cutout_fixture_placements
+    ]
+
+
+@contextlib.contextmanager
+def adapt_protocol_source(protocol: Protocol) -> Generator[ProtocolSource, None, None]:
+    """Convert a `Protocol` to a `ProtocolSource`.
+
+    `Protocol` and `ProtocolSource` do basically the same thing. `Protocol` is the traditional
+    interface. `ProtocolSource` is the newer, but not necessarily better, interface that's required
+    to run stuff through Protocol Engine. Ideally, the two would be unified. Until then, we have
+    this shim.
+
+    This is a context manager because it needs to keep some temp files around.
+    """
+    # ProtocolReader needs to know the filename of the main protocol file so it can infer from its
+    # extension whether it's a JSON or Python protocol. But that filename doesn't necessarily exist,
+    # like when a user passes a text stream to opentrons.simulate.simulate(). As a hack, work
+    # backwards and synthesize a dummy filename with the correct extension.
+    if protocol.filename is not None:
+        # We were given a filename, so no need to guess.
+        #
+        # It's not well-defined in our customer-facing interfaces whether the supplied protocol_name
+        # should be just the filename part, or a path with separators. In case it contains stuff
+        # like "../", sanitize it to just the filename part so we don't save files somewhere bad.
+        main_file_name = pathlib.Path(protocol.filename).name
+    elif isinstance(protocol, JsonProtocol):
+        main_file_name = "protocol.json"
+    else:
+        main_file_name = "protocol.py"
+
+    with tempfile.TemporaryDirectory() as temporary_directory:
+        # FIXME(mm, 2023-06-26): Copying these files is pure overhead, and it introduces encoding
+        # hazards. Remove this when we can parse JSONv6+ and PAPIv2.14+ protocols without going
+        # through the filesystem. https://opentrons.atlassian.net/browse/RSS-281
+
+        main_file = pathlib.Path(temporary_directory) / main_file_name
+        if isinstance(protocol.text, str):
+            main_file.write_text(protocol.text, encoding="utf-8")
+        else:
+            main_file.write_bytes(protocol.text)
+
+        labware_files: List[pathlib.Path] = []
+        if isinstance(protocol, PythonProtocol) and protocol.extra_labware is not None:
+            for labware_index, labware_definition in enumerate(
+                protocol.extra_labware.values()
+            ):
+                new_labware_file = (
+                    pathlib.Path(temporary_directory) / f"{labware_index}.json"
+                )
+                new_labware_file.write_text(
+                    json.dumps(labware_definition), encoding="utf-8"
+                )
+                labware_files.append(new_labware_file)
+
+        protocol_source = asyncio.run(
+            ProtocolReader().read_saved(
+                files=[main_file] + labware_files,
+                directory=None,
+                files_are_prevalidated=False,
+            )
+        )
+
+        yield protocol_source
+
+
+class ProtocolEngineExecuteError(Exception):
+    def __init__(self, errors: List[ProtocolEngineErrorOccurrence]) -> None:
+        """Raised when there was any fatal error running a protocol through Protocol Engine.
+
+        Protocol Engine reports errors as data, not as exceptions.
+        But the only way for `opentrons.execute.execute()` and `opentrons.simulate.simulate()`
+        to signal problems to their callers is to raise something.
+        So we need this class to wrap them.
+
+        Params:
+            errors: The errors that Protocol Engine reported.
+        """
+        # Show the full error details if this is part of a traceback. Don't try to summarize.
+        super().__init__(errors)
+
+        self._error_occurrences = errors
+
+    def to_stderr_string(self) -> str:
+        """Return a string suitable as the stderr output of the `opentrons_execute` CLI.
+
+        This summarizes from the full error details.
+        """
+        # It's unclear what exactly we should extract here.
+        #
+        # First, do we print the first element, or the last, or all of them?
+        #
+        # Second, do we print the .detail? .errorCode? .errorInfo? .wrappedErrors?
+        # By contract, .detail seems like it would be insufficient, but experimentally,
+        # it includes a lot, like:
+        #
+        #     ProtocolEngineError [line 3]: Error 4000 GENERAL_ERROR (ProtocolEngineError):
+        #     UnexpectedProtocolError: Labware "fixture_12_trough" not found with version 1
+        #     in namespace "fixture".
+        return self._error_occurrences[0].detail

opentrons/util/get_union_elements.py

@@ -0,0 +1,26 @@
+# noqa: D100
+
+import typing
+
+
+def get_union_elements(type_alias: typing.Any) -> tuple[typing.Any, ...]:
+    """Return the elements comprising a `typing.Union`.
+
+    e.g. `get_union_elements(Union[int, str, float])` returns `(int, str, float)`.
+
+    If the union itself is wrapped with `typing.Annotated`, it's discarded.
+    If an individual element is wrapped with `typing.Annotated`, it's returned as-is.
+
+    Ideally, we would use `typing.get_type_hints(..., include_extras=False)` for this,
+    but that only works for getting the annotations of a function, method, module, or
+    class, and sometimes we want to unpack an individual union type alias.
+    """
+    origin, args = typing.get_origin(type_alias), typing.get_args(type_alias)
+    if origin is typing.Annotated:
+        # Unwrap Annotated[Union[A, B, C]] to Union[A, B, C].
+        union_annotation = args[0]
+        return get_union_elements(union_annotation)
+    elif origin is typing.Union:
+        return args
+    else:
+        raise TypeError(f"get_union_elements() got unrecognized type {type_alias}")

opentrons/util/helpers.py

@@ -0,0 +1,6 @@
+from datetime import datetime, timezone
+
+
+def utc_now() -> datetime:
+    """Return the current time in the UTC timezone."""
+    return datetime.now(tz=timezone.utc)