appose 0.4.0__py3-none-any.whl → 0.6.0__py3-none-any.whl

appose/python_worker.py

@@ -43,7 +43,7 @@ import sys
 import traceback
 from threading import Thread
 from time import sleep
-from typing import Any, Dict, Optional
+from typing import Any
 
 # NB: Avoid relative imports so that this script can be run standalone.
 from appose.service import RequestType, ResponseType
@@ -51,19 +51,23 @@ from appose.types import Args, _set_worker, decode, encode
 
 
 class Task:
-    def __init__(self, uuid: str) -> None:
-        self.uuid = uuid
+    def __init__(self, uuid: str, script: str, inputs: Args | None = None) -> None:
+        self._uuid = uuid
+        self._script = script
+        self._inputs = inputs
+        self._finished = False
+        self._thread = None
+
+        # Public-facing fields for use within the task script.
         self.outputs = {}
-        self.finished = False
         self.cancel_requested = False
-        self.thread = None  # Initialize thread attribute
 
     def update(
         self,
-        message: Optional[str] = None,
-        current: Optional[int] = None,
-        maximum: Optional[int] = None,
-        info: Optional[Dict[str, Any]] = None,
+        message: str | None = None,
+        current: int | None = None,
+        maximum: int | None = None,
+        info: dict[str, Any] | None = None,
     ) -> None:
         args = {}
         if message is not None:
@@ -84,16 +88,16 @@ class Task:
     def cancel(self) -> None:
         self._respond(ResponseType.CANCELATION, None)
 
-    def fail(self, error: Optional[str] = None) -> None:
+    def fail(self, error: str | None = None) -> None:
         args = None if error is None else {"error": error}
         self._respond(ResponseType.FAILURE, args)
 
-    def _start(self, script: str, inputs: Optional[Args]) -> None:
-        def execute_script():
+    def _run(self) -> None:
+        try:
             # Populate script bindings.
             binding = {"task": self}
-            if inputs is not None:
-                binding.update(inputs)
+            if self._inputs is not None:
+                binding.update(self._inputs)
 
             # Inform the calling process that the script is launching.
             self._report_launch()
@@ -101,37 +105,32 @@ class Task:
             # Execute the script.
             # result = exec(script, locals=binding)
             result = None
-            try:
-                # NB: Execute the block, except for the last statement,
-                # which we evaluate instead to get its return value.
-                # Credit: https://stackoverflow.com/a/39381428/1207769
-
-                block = ast.parse(script, mode="exec")
-                last = None
-                if (
-                    len(block.body) > 0
-                    and hasattr(block.body[-1], "value")
-                    and not isinstance(block.body[-1], ast.Assign)
-                ):
-                    # Last statement of the script looks like an expression. Evaluate!
-                    last = ast.Expression(block.body.pop().value)
-
-                # NB: When `exec` gets two separate objects as *globals* and
-                # *locals*, the code will be executed as if it were embedded in
-                # a class definition. This means functions and classes defined
-                # in the executed code will not be able to access variables
-                # assigned at the top level, because the "top level" variables
-                # are treated as class variables in a class definition.
-                # See: https://docs.python.org/3/library/functions.html#exec
-                _globals = binding
-                exec(compile(block, "<string>", mode="exec"), _globals, binding)
-                if last is not None:
-                    result = eval(
-                        compile(last, "<string>", mode="eval"), _globals, binding
-                    )
-            except Exception:
-                self.fail(traceback.format_exc())
-                return
+
+            # NB: Execute the block, except for the last statement,
+            # which we evaluate instead to get its return value.
+            # Credit: https://stackoverflow.com/a/39381428/1207769
+
+            block = ast.parse(self._script, mode="exec")
+            last = None
+            if (
+                len(block.body) > 0
+                and hasattr(block.body[-1], "value")
+                and not isinstance(block.body[-1], ast.Assign)
+            ):
+                # Last statement of the script looks like an expression. Evaluate!
+                last = ast.Expression(block.body.pop().value)
+
+            # NB: When `exec` gets two separate objects as *globals* and
+            # *locals*, the code will be executed as if it were embedded in
+            # a class definition. This means functions and classes defined
+            # in the executed code will not be able to access variables
+            # assigned at the top level, because the "top level" variables
+            # are treated as class variables in a class definition.
+            # See: https://docs.python.org/3/library/functions.html#exec
+            _globals = binding
+            exec(compile(block, "<string>", mode="exec"), _globals, binding)
+            if last is not None:
+                result = eval(compile(last, "<string>", mode="eval"), _globals, binding)
 
             # Report the results to the Appose calling process.
             if isinstance(result, dict):
@@ -140,27 +139,9 @@ class Task:
             elif result is not None:
                 # Script produced a non-dict; add it alone to the outputs.
                 self.outputs["result"] = result
-
             self._report_completion()
-
-        # HACK: Pre-load toplevel import statements before running the script
-        # as a whole on its own Thread. Why? Because on Windows, some imports
-        # (e.g. numpy) may lead to hangs if loaded from a separate thread.
-        # See https://github.com/apposed/appose/issues/13.
-        block = ast.parse(script, mode="exec")
-        import_nodes = [
-            node
-            for node in block.body
-            if isinstance(node, (ast.Import, ast.ImportFrom))
-        ]
-        import_block = ast.Module(body=import_nodes, type_ignores=[])
-        compiled_imports = compile(import_block, filename="<imports>", mode="exec")
-        exec(compiled_imports, globals())
-
-        # Create a thread and save a reference to it, in case its script
-        # ends up killing the thread. This happens e.g. if it calls sys.exit.
-        self.thread = Thread(target=execute_script, name=f"Appose-{self.uuid}")
-        self.thread.start()
+        except Exception:
+            self.fail(traceback.format_exc())
 
     def _report_launch(self) -> None:
         self._respond(ResponseType.LAUNCH, None)
@@ -169,20 +150,20 @@ class Task:
         args = None if self.outputs is None else {"outputs": self.outputs}
         self._respond(ResponseType.COMPLETION, args)
 
-    def _respond(self, response_type: ResponseType, args: Optional[Args]) -> None:
+    def _respond(self, response_type: ResponseType, args: Args | None) -> None:
         already_terminated = False
         if response_type.is_terminal():
-            if self.finished:
+            if self._finished:
                 # This is not the first terminal response. Let's
                 # remember, in case an exception is generated below,
                 # so that we can avoid infinite recursion loops.
                 already_terminated = True
-            self.finished = True
+            self._finished = True
 
         response = {}
         if args is not None:
             response.update(args)
-        response.update({"task": self.uuid, "responseType": response_type.value})
+        response.update({"task": self._uuid, "responseType": response_type.value})
         # NB: Flush is necessary to ensure service receives the data!
         try:
             print(encode(response), flush=True)
@@ -198,57 +179,86 @@ class Task:
             self.fail(traceback.format_exc())
 
 
-def main() -> None:
-    _set_worker(True)
+class Worker:
 
-    tasks = {}
-    running = True
+    def __init__(self):
+        self.tasks = {}
+        self.queue: list[Task] = []
+        self.running = True
 
-    def cleanup_threads():
-        while running:
+        # Flag this process as a worker, not a service.
+        _set_worker(True)
+
+        Thread(target=self._process_input, name="Appose-Receiver").start()
+        Thread(target=self._cleanup_threads, name="Appose-Janitor").start()
+
+    def run(self) -> None:
+        """
+        Processes tasks from the task queue.
+        """
+        while self.running:
+            if len(self.queue) == 0:
+                # Nothing queued, so wait a bit.
+                sleep(0.05)
+                continue
+            task = self.queue.pop()
+            task._run()
+
+    def _process_input(self) -> None:
+        while True:
+            try:
+                line = input().strip()
+            except EOFError:
+                line = None
+            if not line:
+                self.running = False
+                return
+
+            request = decode(line)
+            uuid = request.get("task")
+            request_type = request.get("requestType")
+
+            match RequestType(request_type):
+                case RequestType.EXECUTE:
+                    script = request.get("script")
+                    inputs = request.get("inputs")
+                    queue = request.get("queue")
+                    task = Task(uuid, script, inputs)
+                    self.tasks[uuid] = task
+                    if queue == "main":
+                        # Add the task to the main thread queue.
+                        self.queue.append(task)
+                    else:
+                        # Create a thread and save a reference to it, in case its script
+                        # kills the thread. This happens e.g. if it calls sys.exit.
+                        task._thread = Thread(target=task._run, name=f"Appose-{uuid}")
+                        task._thread.start()
+
+                case RequestType.CANCEL:
+                    task = self.tasks.get(uuid)
+                    if task is None:
+                        print(f"No such task: {uuid}", file=sys.stderr)
+                        continue
+                    task.cancel_requested = True
+
+    def _cleanup_threads(self) -> None:
+        while self.running:
             sleep(0.05)
             dead = {
                 uuid: task
-                for uuid, task in tasks.items()
-                if task.thread is not None and not task.thread.is_alive()
+                for uuid, task in self.tasks.items()
+                if task._thread is not None and not task._thread.is_alive()
             }
             for uuid, task in dead.items():
-                tasks.pop(uuid)
-                if not task.finished:
+                self.tasks.pop(uuid)
+                if not task._finished:
                     # The task died before reporting a terminal status.
                     # We report this situation as failure by thread death.
                     task.fail("thread death")
 
-    Thread(target=cleanup_threads, name="Appose-Janitor").start()
 
-    while True:
-        try:
-            line = input().strip()
-        except EOFError:
-            break
-        if not line:
-            break
-
-        request = decode(line)
-        uuid = request.get("task")
-        request_type = request.get("requestType")
-
-        match RequestType(request_type):
-            case RequestType.EXECUTE:
-                script = request.get("script")
-                inputs = request.get("inputs")
-                task = Task(uuid)
-                tasks[uuid] = task
-                task._start(script, inputs)
-
-            case RequestType.CANCEL:
-                task = tasks.get(uuid)
-                if task is None:
-                    print(f"No such task: {uuid}", file=sys.stderr)
-                    continue
-                task.cancel_requested = True
-
-    running = False
+def main() -> None:
+    Worker().run()
 
 
 if __name__ == "__main__":

appose/service.py

@@ -36,7 +36,7 @@ import threading
 from enum import Enum
 from pathlib import Path
 from traceback import format_exc
-from typing import Any, Callable, Dict, List, Optional, Sequence, Union
+from typing import Any, Callable
 from uuid import uuid4
 
 from .types import Args, decode, encode
@@ -52,17 +52,17 @@ class Service:
 
     _service_count = 0
 
-    def __init__(self, cwd: Union[str, Path], args: Sequence[str]) -> None:
+    def __init__(self, cwd: str | Path, args: list[str]) -> None:
         self._cwd = cwd
         self._args = args[:]
-        self._tasks: Dict[str, "Task"] = {}
+        self._tasks: dict[str, "Task"] = {}
         self._service_id = Service._service_count
         Service._service_count += 1
-        self._process: Optional[subprocess.Popen] = None
-        self._stdout_thread: Optional[threading.Thread] = None
-        self._stderr_thread: Optional[threading.Thread] = None
-        self._monitor_thread: Optional[threading.Thread] = None
-        self._debug_callback: Optional[Callable[[Any], Any]] = None
+        self._process: subprocess.Popen | None = None
+        self._stdout_thread: threading.Thread | None = None
+        self._stderr_thread: threading.Thread | None = None
+        self._monitor_thread: threading.Thread | None = None
+        self._debug_callback: Callable[[Any], Any] | None = None
 
     def debug(self, debug_callback: Callable[[Any], Any]) -> None:
         """
@@ -109,16 +109,20 @@ class Service:
         self._stderr_thread.start()
         self._monitor_thread.start()
 
-    def task(self, script: str, inputs: Optional[Args] = None) -> "Task":
+    def task(
+        self, script: str, inputs: Args | None = None, queue: str | None = None
+    ) -> "Task":
         """
         Create a new task, passing the given script to the worker for execution.
         :param script:
             The script for the worker to execute in its environment.
         :param inputs:
             Optional list of key/value pairs to feed into the script as inputs.
+        :param queue:
+            Optional queue target. Pass "main" to queue to worker's main thread.
         """
         self.start()
-        return Task(self, script, inputs)
+        return Task(self, script, inputs, queue)
 
     def close(self) -> None:
         """
@@ -274,17 +278,17 @@ class TaskEvent:
         self,
         task: "Task",
         response_type: ResponseType,
-        message: Optional[str] = None,
-        current: Optional[int] = None,
-        maximum: Optional[int] = None,
-        info: Optional[Dict[str, Any]] = None,
+        message: str | None = None,
+        current: int | None = None,
+        maximum: int | None = None,
+        info: dict[str, Any] | None = None,
     ) -> None:
         self.task: "Task" = task
         self.response_type: ResponseType = response_type
-        self.message: Optional[str] = message
-        self.current: Optional[int] = current
-        self.maximum: Optional[int] = maximum
-        self.info: Optional[Dict[str, Any]] = info
+        self.message: str | None = message
+        self.current: int | None = current
+        self.maximum: int | None = maximum
+        self.info: dict[str, Any] | None = info
 
     def __str__(self):
         return f"[{self.response_type}] {self.task}"
@@ -298,21 +302,26 @@ class Task:
     """
 
     def __init__(
-        self, service: Service, script: str, inputs: Optional[Args] = None
+        self,
+        service: Service,
+        script: str,
+        inputs: Args | None = None,
+        queue: str | None = None,
     ) -> None:
         self.uuid = uuid4().hex
         self.service = service
         self.script = script
         self.inputs: Args = {}
+        self.queue: str | None = queue
         if inputs is not None:
             self.inputs.update(inputs)
         self.outputs: Args = {}
         self.status: TaskStatus = TaskStatus.INITIAL
-        self.message: Optional[str] = None
+        self.message: str | None = None
         self.current: int = 0
         self.maximum: int = 1
-        self.error: Optional[str] = None
-        self.listeners: List[Callable[["TaskEvent"], None]] = []
+        self.error: str | None = None
+        self.listeners: list[Callable[["TaskEvent"], None]] = []
         self.cv = threading.Condition()
         self.service._tasks[self.uuid] = self
 
@@ -323,7 +332,7 @@ class Task:
 
             self.status = TaskStatus.QUEUED
 
-        args = {"script": self.script, "inputs": self.inputs}
+        args = {"script": self.script, "inputs": self.inputs, "queue": self.queue}
         self._request(RequestType.EXECUTE, args)
 
         return self

{appose-0.4.0.dist-info → appose-0.6.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: appose
-Version: 0.4.0
+Version: 0.6.0
 Summary: Appose: multi-language interprocess cooperation with shared memory.
 Author: Appose developers
 License-Expression: BSD-2-Clause

appose-0.6.0.dist-info/RECORD

@@ -0,0 +1,11 @@
+appose/__init__.py,sha256=SRXmNEqFIAajVlwVCSAdhHQytRKZJiWI5Y5-xwioW0w,7544
+appose/environment.py,sha256=Kg-MfAcmVPDltwqvmrcvus-pP01SNUdn4bE2kdkIVuk,7597
+appose/paths.py,sha256=WgBHwnZdS3Ot8_hssFqUiTGnunolYRrHLf9rAfEU5r4,2182
+appose/python_worker.py,sha256=rkssHh3YTBueGTQ9qAuRVqq7bXncYulscG3kbM0D2TQ,10061
+appose/service.py,sha256=jdtGOOoWjut9eZj441fL8t_P_ln2mfgilBvJdH0bDus,14750
+appose/types.py,sha256=uidbt2JwWJ2D3Mbtc4Vl2sRP6WNgKjmfcHjJ9QLWkcs,10557
+appose-0.6.0.dist-info/licenses/LICENSE.txt,sha256=ZedidA6NyZclNlsx-vl9IVBWRAQSqXdNsCWgtFcwzIE,1313
+appose-0.6.0.dist-info/METADATA,sha256=F900U1qP-pPiPgklzrnjg7og3LuY5h8pXYDdadVtfdY,5598
+appose-0.6.0.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+appose-0.6.0.dist-info/top_level.txt,sha256=oqHhw2QGlaFfH3jMR9H5Cd6XYHdEv5DvK1am0iEFPW0,7
+appose-0.6.0.dist-info/RECORD,,

appose-0.4.0.dist-info/RECORD

@@ -1,11 +0,0 @@
-appose/__init__.py,sha256=SRXmNEqFIAajVlwVCSAdhHQytRKZJiWI5Y5-xwioW0w,7544
-appose/environment.py,sha256=Kg-MfAcmVPDltwqvmrcvus-pP01SNUdn4bE2kdkIVuk,7597
-appose/paths.py,sha256=WgBHwnZdS3Ot8_hssFqUiTGnunolYRrHLf9rAfEU5r4,2182
-appose/python_worker.py,sha256=iQoz2t0glbXlR1lgynN-yXPIgtlAbCFtMYYpev-LZyU,9844
-appose/service.py,sha256=UQn1aoL3uQ8gPKS6nf5oYg0Ea1ZRsaQ3TfLCKeCZLPQ,14581
-appose/types.py,sha256=uidbt2JwWJ2D3Mbtc4Vl2sRP6WNgKjmfcHjJ9QLWkcs,10557
-appose-0.4.0.dist-info/licenses/LICENSE.txt,sha256=ZedidA6NyZclNlsx-vl9IVBWRAQSqXdNsCWgtFcwzIE,1313
-appose-0.4.0.dist-info/METADATA,sha256=cI_E4xetwcHlusk2UzSPOk6SWrNr_vBrGjOC_0H4CFo,5598
-appose-0.4.0.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-appose-0.4.0.dist-info/top_level.txt,sha256=oqHhw2QGlaFfH3jMR9H5Cd6XYHdEv5DvK1am0iEFPW0,7
-appose-0.4.0.dist-info/RECORD,,