appose 0.1.0__py3-none-any.whl → 0.2.0__py3-none-any.whl

appose/__init__.py

@@ -2,7 +2,7 @@
 # #%L
 # Appose: multi-language interprocess cooperation with shared memory.
 # %%
-# Copyright (C) 2023 Appose developers.
+# Copyright (C) 2023 - 2024 Appose developers.
 # %%
 # Redistribution and use in source and binary forms, with or without
 # modification, are permitted provided that the following conditions are met:
@@ -52,7 +52,7 @@ Here is a very simple example written in Python:
     Task task = groovy.task("""
         5 + 6
     """)
-    task.waitFor()
+    task.wait_for()
     result = task.outputs.get("result")
     assert 11 == result
 
@@ -103,7 +103,7 @@ And here is an example using a few more of Appose's features:
         # Task is taking too long; request a cancelation.
         task.cancel()
 
-    task.waitFor()
+    task.wait_for()
 
 Of course, the above examples could have been done all in Python. But
 hopefully they hint at the possibilities of easy cross-language integration.
@@ -134,6 +134,7 @@ JSON, one line per request/response.
 from pathlib import Path
 
 from .environment import Builder, Environment
+from .types import NDArray, SharedMemory  # noqa: F401
 
 
 def base(directory: Path) -> Builder:

appose/environment.py

@@ -2,7 +2,7 @@
 # #%L
 # Appose: multi-language interprocess cooperation with shared memory.
 # %%
-# Copyright (C) 2023 Appose developers.
+# Copyright (C) 2023 - 2024 Appose developers.
 # %%
 # Redistribution and use in source and binary forms, with or without
 # modification, are permitted provided that the following conditions are met:
@@ -58,6 +58,7 @@ class Environment:
         """
         python_exes = [
             "python",
+            "python3",
             "python.exe",
             "bin/python",
             "bin/python.exe",
@@ -109,8 +110,9 @@ class Environment:
         # TODO: Ensure that the classpath includes Appose and its dependencies.
 
         # Append any explicitly requested classpath elements.
-        for element in class_path:
-            cp[element] = None
+        if class_path is not None:
+            for element in class_path:
+                cp[element] = None
 
         # Build up the service arguments.
         args = [

appose/paths.py

@@ -2,7 +2,7 @@
 # #%L
 # Appose: multi-language interprocess cooperation with shared memory.
 # %%
-# Copyright (C) 2023 Appose developers.
+# Copyright (C) 2023 - 2024 Appose developers.
 # %%
 # Redistribution and use in source and binary forms, with or without
 # modification, are permitted provided that the following conditions are met:
@@ -51,6 +51,6 @@ def find_exe(dirs: Sequence[str], exes: Sequence[str]) -> Optional[Path]:
             # Candidate is a relative path; check beneath each given directory.
             for d in dirs:
                 f = Path(d) / exe
-                if can_execute(f):
+                if can_execute(f) and not f.is_dir():
                     return f
     return None

appose/python_worker.py

@@ -2,7 +2,7 @@
 # #%L
 # Appose: multi-language interprocess cooperation with shared memory.
 # %%
-# Copyright (C) 2023 Appose developers.
+# Copyright (C) 2023 - 2024 Appose developers.
 # %%
 # Redistribution and use in source and binary forms, with or without
 # modification, are permitted provided that the following conditions are met:
@@ -28,7 +28,14 @@
 ###
 
 """
-TODO
+The Appose worker for running Python scripts.
+
+Like all Appose workers, this program conforms to the Appose worker process
+contract, meaning it accepts requests on stdin and produces responses on
+stdout, both formatted according to Appose's assumptions.
+
+For details, see the Appose README:
+https://github.com/apposed/appose/blob/-/README.md#workers
 """
 
 import ast
@@ -39,7 +46,7 @@ from typing import Optional
 
 # NB: Avoid relative imports so that this script can be run standalone.
 from appose.service import RequestType, ResponseType
-from appose.types import Args, decode, encode
+from appose.types import Args, _set_worker, decode, encode
 
 
 class Task:
@@ -56,11 +63,17 @@ class Task:
     ) -> None:
         args = {}
         if message is not None:
-            args["message"] = message
+            args["message"] = str(message)
         if current is not None:
-            args["current"] = current
+            try:
+                args["current"] = int(current)
+            except ValueError:
+                pass
         if maximum is not None:
-            args["maximum"] = maximum
+            try:
+                args["maximum"] = int(maximum)
+            except ValueError:
+                pass
         self._respond(ResponseType.UPDATE, args)
 
     def cancel(self) -> None:
@@ -74,7 +87,6 @@ class Task:
         def execute_script():
             # Populate script bindings.
             binding = {"task": self}
-            # TODO: Magically convert shared memory image inputs.
             if inputs is not None:
                 binding.update(inputs)
 
@@ -136,10 +148,22 @@ class Task:
         if args is not None:
             response.update(args)
         # NB: Flush is necessary to ensure service receives the data!
-        print(encode(response), flush=True)
+        try:
+            print(encode(response), flush=True)
+        except Exception:
+            # Encoding can fail due to unsupported types, when the response
+            # or its elements are not supported by JSON encoding.
+            # No matter what goes wrong, we want to tell the caller.
+            if response_type is ResponseType.FAILURE:
+                # TODO: How to address this hypothetical case
+                # of a failure message triggering another failure?
+                raise
+            self.fail(traceback.format_exc())
 
 
 def main() -> None:
+    _set_worker(True)
+
     tasks = {}
 
     while True:
@@ -165,8 +189,6 @@ def main() -> None:
             case RequestType.CANCEL:
                 task = tasks.get(uuid)
                 if task is None:
-                    # TODO: proper logging
-                    # Maybe should stdout the error back to Appose calling process.
                     print(f"No such task: {uuid}", file=sys.stderr)
                     continue
                 task.cancel_requested = True

appose/service.py

@@ -2,7 +2,7 @@
 # #%L
 # Appose: multi-language interprocess cooperation with shared memory.
 # %%
-# Copyright (C) 2023 Appose developers.
+# Copyright (C) 2023 - 2024 Appose developers.
 # %%
 # Redistribution and use in source and binary forms, with or without
 # modification, are permitted provided that the following conditions are met:
@@ -61,6 +61,7 @@ class Service:
         self._process: Optional[subprocess.Popen] = None
         self._stdout_thread: Optional[threading.Thread] = None
         self._stderr_thread: Optional[threading.Thread] = None
+        self._monitor_thread: Optional[threading.Thread] = None
         self._debug_callback: Optional[Callable[[Any], Any]] = None
 
     def debug(self, debug_callback: Callable[[Any], Any]) -> None:
@@ -101,8 +102,12 @@ class Service:
         self._stderr_thread = threading.Thread(
             target=self._stderr_loop, name=f"{prefix}-Stderr"
         )
+        self._monitor_thread = threading.Thread(
+            target=self._monitor_loop, name=f"{prefix}-Monitor"
+        )
         self._stdout_thread.start()
         self._stderr_thread.start()
+        self._monitor_thread.start()
 
     def task(self, script: str, inputs: Optional[Args] = None) -> "Task":
         """
@@ -131,15 +136,24 @@ class Service:
         """
         Input loop processing lines from the worker's stdout stream.
         """
-        # noinspection PyBroadException
-        try:
-            while True:
-                line = self._process.stdout.readline()
-                self._debug_service("<worker stdout closed>" if line is None else line)
+        while True:
+            stdout = self._process.stdout
+            # noinspection PyBroadException
+            try:
+                line = None if stdout is None else stdout.readline()
+            except Exception:
+                # Something went wrong reading the line. Panic!
+                self._debug_service(format_exc())
+                break
+
+            if not line:  # readline returns empty string upon EOF
+                self._debug_service("<worker stdout closed>")
+                return
 
-                if line is None:
-                    return  # pipe closed
+            # noinspection PyBroadException
+            try:
                 response = decode(line)
+                self._debug_service(line)  # Echo the line to the debug listener.
                 uuid = response.get("task")
                 if uuid is None:
                     self._debug_service("Invalid service message: {line}")
@@ -150,8 +164,10 @@ class Service:
                     continue
                 # noinspection PyProtectedMember
                 task._handle(response)
-        except Exception:
-            self._debug_service(format_exc())
+            except Exception:
+                # Something went wrong decoding the line of JSON.
+                # Skip it and keep going, but log it first.
+                self._debug_service(f"<INVALID> {line}")
 
     def _stderr_loop(self) -> None:
         """
@@ -160,14 +176,37 @@ class Service:
         # noinspection PyBroadException
         try:
             while True:
-                line = self._process.stderr.readline()
-                if line is None:
+                stderr = self._process.stderr
+                line = None if stderr is None else stderr.readline()
+                if not line:  # readline returns empty string upon EOF
                     self._debug_service("<worker stderr closed>")
                     return
                 self._debug_worker(line)
         except Exception:
             self._debug_service(format_exc())
 
+    def _monitor_loop(self) -> None:
+        # Wait until the worker process terminates.
+        self._process.wait()
+
+        # Do some sanity checks.
+        exit_code = self._process.returncode
+        if exit_code != 0:
+            self._debug_service(
+                f"<worker process terminated with exit code {exit_code}>"
+            )
+        task_count = len(self._tasks)
+        if task_count > 0:
+            self._debug_service(
+                f"<worker process terminated with {task_count} pending tasks>"
+            )
+
+        # Notify any remaining tasks about the process crash.
+        for task in self._tasks.values():
+            task._crash()
+
+        self._tasks.clear()
+
     def _debug_service(self, message: str) -> None:
         self._debug("SERVICE", message)
 
@@ -190,15 +229,17 @@ class TaskStatus(Enum):
     COMPLETE = "COMPLETE"
     CANCELED = "CANCELED"
     FAILED = "FAILED"
+    CRASHED = "CRASHED"
 
     def is_finished(self):
         """
-        True iff status is COMPLETE, CANCELED, or FAILED.
+        True iff status is COMPLETE, CANCELED, FAILED, or CRASHED.
         """
         return self in (
             TaskStatus.COMPLETE,
             TaskStatus.CANCELED,
             TaskStatus.FAILED,
+            TaskStatus.CRASHED,
         )
 
 
@@ -213,6 +254,7 @@ class ResponseType(Enum):
     COMPLETION = "COMPLETION"
     CANCELATION = "CANCELATION"
     FAILURE = "FAILURE"
+    CRASH = "CRASH"
 
 
 class TaskEvent:
@@ -220,6 +262,9 @@ class TaskEvent:
         self.task: "Task" = task
         self.response_type: ResponseType = response_type
 
+    def __str__(self):
+        return f"[{self.response_type}] {self.task}"
+
 
 # noinspection PyProtectedMember
 class Task:
@@ -342,3 +387,17 @@ class Task:
         if self.status.is_finished():
             with self.cv:
                 self.cv.notify_all()
+
+    def _crash(self):
+        event = TaskEvent(self, ResponseType.CRASH)
+        self.status = TaskStatus.CRASHED
+        for listener in self.listeners:
+            listener(event)
+        with self.cv:
+            self.cv.notify_all()
+
+    def __str__(self):
+        return (
+            f"{self.uuid=}, {self.status=}, {self.message=}, "
+            f"{self.current=}, {self.maximum=}, {self.error=}"
+        )

appose/types.py

@@ -2,7 +2,7 @@
 # #%L
 # Appose: multi-language interprocess cooperation with shared memory.
 # %%
-# Copyright (C) 2023 Appose developers.
+# Copyright (C) 2023 - 2024 Appose developers.
 # %%
 # Redistribution and use in source and binary forms, with or without
 # modification, are permitted provided that the following conditions are met:
@@ -28,14 +28,170 @@
 ###
 
 import json
-from typing import Any, Dict
+import re
+from math import ceil, prod
+from multiprocessing import resource_tracker, shared_memory
+from typing import Any, Dict, Sequence, Union
 
 Args = Dict[str, Any]
 
 
+class SharedMemory(shared_memory.SharedMemory):
+    """
+    An enhanced version of Python's multiprocessing.shared_memory.SharedMemory
+    class which can be used with a `with` statement. When the program flow
+    exits the `with` block, this class's `dispose()` method will be invoked,
+    which might call `close()` or `unlink()` depending on the value of its
+    `unlink_on_dispose` flag.
+    """
+
+    def __init__(self, name: str = None, create: bool = False, size: int = 0):
+        super().__init__(name=name, create=create, size=size)
+        self._unlink_on_dispose = create
+        if _is_worker:
+            # HACK: Remove this shared memory block from the resource_tracker,
+            # which wants to clean up shared memory blocks after all known
+            # references are done using them.
+            #
+            # There is one resource_tracker per Python process, and they will
+            # each try to delete shared memory blocks known to them when they
+            # are shutting down, even when other processes still need them.
+            #
+            # As such, the rule Appose follows is: let the service process
+            # always handle cleanup of shared memory blocks, regardless of
+            # which process initially allocated it.
+            resource_tracker.unregister(self._name, "shared_memory")
+
+    def unlink_on_dispose(self, value: bool) -> None:
+        """
+        Set whether the `unlink()` method should be invoked to destroy
+        the shared memory block when the `dispose()` method is called.
+
+        Note: dispose() is the method called when exiting a `with` block.
+
+        By default, shared memory objects constructed with `create=True`
+        will behave this way, whereas shared memory objects constructed
+        with `create=False` will not. But this method allows to override
+        the behavior.
+        """
+        self._unlink_on_dispose = value
+
+    def dispose(self) -> None:
+        if self._unlink_on_dispose:
+            self.unlink()
+        else:
+            self.close()
+
+    def __enter__(self) -> "SharedMemory":
+        return self
+
+    def __exit__(self, exc_type, exc_value, exc_tb) -> None:
+        self.dispose()
+
+
 def encode(data: Args) -> str:
-    return json.dumps(data)
+    return json.dumps(data, cls=_ApposeJSONEncoder, separators=(",", ":"))
 
 
 def decode(the_json: str) -> Args:
-    return json.loads(the_json)
+    return json.loads(the_json, object_hook=_appose_object_hook)
+
+
+class NDArray:
+    """
+    Data structure for a multi-dimensional array.
+    The array contains elements of a data type, arranged in
+    a particular shape, and flattened into SharedMemory.
+    """
+
+    def __init__(self, dtype: str, shape: Sequence[int], shm: SharedMemory = None):
+        """
+        Create an NDArray.
+        :param dtype: The type of the data elements; e.g. int8, uint8, float32, float64.
+        :param shape: The dimensional extents; e.g. a stack of 7 image planes
+                      with resolution 512x512 would have shape [7, 512, 512].
+        :param shm: The SharedMemory containing the array data, or None to create it.
+        """
+        self.dtype = dtype
+        self.shape = shape
+        self.shm = (
+            SharedMemory(
+                create=True, size=ceil(prod(shape) * _bytes_per_element(dtype))
+            )
+            if shm is None
+            else shm
+        )
+
+    def __str__(self):
+        return (
+            f"NDArray("
+            f"dtype='{self.dtype}', "
+            f"shape={self.shape}, "
+            f"shm='{self.shm.name}' ({self.shm.size}))"
+        )
+
+    def ndarray(self):
+        """
+        Create a NumPy ndarray object for working with the array data.
+        No array data is copied; the NumPy array wraps the same SharedMemory.
+        Requires the numpy package to be installed.
+        """
+        try:
+            import numpy
+
+            return numpy.ndarray(
+                prod(self.shape), dtype=self.dtype, buffer=self.shm.buf
+            ).reshape(self.shape)
+        except ModuleNotFoundError:
+            raise ImportError("NumPy is not available.")
+
+    def __enter__(self) -> "NDArray":
+        return self
+
+    def __exit__(self, exc_type, exc_value, exc_tb) -> None:
+        self.shm.dispose()
+
+
+class _ApposeJSONEncoder(json.JSONEncoder):
+    def default(self, obj):
+        if isinstance(obj, SharedMemory):
+            return {
+                "appose_type": "shm",
+                "name": obj.name,
+                "size": obj.size,
+            }
+        if isinstance(obj, NDArray):
+            return {
+                "appose_type": "ndarray",
+                "dtype": obj.dtype,
+                "shape": obj.shape,
+                "shm": obj.shm,
+            }
+        return super().default(obj)
+
+
+def _appose_object_hook(obj: Dict):
+    atype = obj.get("appose_type")
+    if atype == "shm":
+        # Attach to existing shared memory block.
+        return SharedMemory(name=(obj["name"]), size=(obj["size"]))
+    elif atype == "ndarray":
+        return NDArray(obj["dtype"], obj["shape"], obj["shm"])
+    else:
+        return obj
+
+
+def _bytes_per_element(dtype: str) -> Union[int, float]:
+    try:
+        bits = int(re.sub("[^0-9]", "", dtype))
+    except ValueError:
+        raise ValueError(f"Invalid dtype: {dtype}")
+    return bits / 8
+
+
+_is_worker = False
+
+
+def _set_worker(value: bool) -> None:
+    global _is_worker
+    _is_worker = value

{appose-0.1.0.dist-info → appose-0.2.0.dist-info}/LICENSE.txt

@@ -1,4 +1,4 @@
-Copyright (c) 2023, Appose developers.
+Copyright (c) 2023 - 2024, Appose developers.
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without modification,

{appose-0.1.0.dist-info → appose-0.2.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: appose
-Version: 0.1.0
+Version: 0.2.0
 Summary: Appose: multi-language interprocess cooperation with shared memory.
 Author: Appose developers
 License: Simplified BSD License
@@ -8,7 +8,7 @@ Project-URL: homepage, https://github.com/apposed/appose-python
 Project-URL: documentation, https://github.com/apposed/appose-python/blob/main/README.md
 Project-URL: source, https://github.com/apposed/appose-python
 Project-URL: download, https://pypi.org/project/appose-python
-Project-URL: tracker, https://github.com/apposed/appose-python/issues
+Project-URL: tracker, https://github.com/apposed/appose/issues
 Keywords: java,javascript,python,cross-language,interprocess
 Classifier: Development Status :: 2 - Pre-Alpha
 Classifier: Intended Audience :: Developers
@@ -71,16 +71,6 @@ This is the **Python implementation of Appose**.
 
 The name of the package is `appose`.
 
-### Conda/Mamba
-
-To use [the conda-forge package](https://anaconda.org/conda-forge/appose),
-add `appose` to your `environment.yml`'s `dependencies` section:
-
-```yaml
-dependencies:
-  - appose
-```
-
 ### PyPI/Pip
 
 To use [the PyPI package](https://pypi.org/project/appose),
@@ -98,6 +88,16 @@ dependencies = [
 ]
 ```
 
+### Conda/Mamba
+
+To use [the conda-forge package](https://anaconda.org/conda-forge/appose),
+add `appose` to your `environment.yml`'s `dependencies` section:
+
+```yaml
+dependencies:
+  - appose
+```
+
 ## Examples
 
 Here is a minimal example for calling into Java from Python:
@@ -107,12 +107,12 @@ import appose
 env = appose.java(vendor="zulu", version="17").build()
 with env.groovy() as groovy:
     task = groovy.task("5 + 6")
-    task.waitFor()
-    result = task.outputs.get("result")
+    task.wait_for()
+    result = task.outputs["result"]
     assert 11 == result
 ```
 
-*Note: The `Appose.java` builder is planned, but not yet implemented.*
+*Note: The `appose.java` builder is planned, but not yet implemented.*
 
 Here is an example using a few more of Appose's features:
 
@@ -169,3 +169,9 @@ with env.groovy() as groovy:
 
 Of course, the above examples could have been done all in one language. But
 hopefully they hint at the possibilities of easy cross-language integration.
+
+## Issue tracker
+
+All implementations of Appose use the same issue tracker:
+
+https://github.com/apposed/appose/issues

appose-0.2.0.dist-info/RECORD

@@ -0,0 +1,11 @@
+appose/__init__.py,sha256=sGkVQv1eZQp0BOoySsVOGcVMr47rE3PSyqxsk8_EOn4,5404
+appose/environment.py,sha256=8BBvRIzr6ZIjBeZMQYOHG6Va1gSxMkYc3ieJ9HafeXA,7597
+appose/paths.py,sha256=dcXD54rOUiCWfM58vqttRY0Eu6YUFMD65ZO_2V4hhEk,2182
+appose/python_worker.py,sha256=jhe_iFnCS7vJ_N1EycDTt6oq09JhYIOnvvhyqnLG40U,7335
+appose/service.py,sha256=v5u_HAliJlsoghwC6QIn3CZV2YpnPU8V3GdBcjgA62w,14070
+appose/types.py,sha256=m5mX5BhdR7REnkSCuP5fGnnQWc_o7qQDyOFHWND8Vvs,6975
+appose-0.2.0.dist-info/LICENSE.txt,sha256=LcTntIsPpu7JOZfEBAVslv-PVV4vnD9tfvXrS4cROe8,1313
+appose-0.2.0.dist-info/METADATA,sha256=lqL66Xs9gTSJLr_ISb7ge8lK80xSQh04_Amfsf1jQWg,5586
+appose-0.2.0.dist-info/WHEEL,sha256=R0nc6qTxuoLk7ShA2_Y-UWkN8ZdfDBG2B6Eqpz2WXbs,91
+appose-0.2.0.dist-info/top_level.txt,sha256=oqHhw2QGlaFfH3jMR9H5Cd6XYHdEv5DvK1am0iEFPW0,7
+appose-0.2.0.dist-info/RECORD,,

{appose-0.1.0.dist-info → appose-0.2.0.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: bdist_wheel (0.40.0)
+Generator: setuptools (72.1.0)
 Root-Is-Purelib: true
 Tag: py3-none-any
 

appose-0.1.0.dist-info/RECORD

@@ -1,11 +0,0 @@
-appose/__init__.py,sha256=IwFrDvIa6QOgV_RVgbIUfwyzswWB4RckLrzA9cts6IE,5340
-appose/environment.py,sha256=m1a7166dJq69XPWX53d63KoITqmH4fKMNPKbEGSxeC0,7524
-appose/paths.py,sha256=qBEWWf0LB0Cpk-l7u5uoPx9DTley6HltMTg0vJ2iFEQ,2156
-appose/python_worker.py,sha256=ZR6AtWXXk_Xcp0nSoWehbl7LYc6z18ZTTvo4QjO_MwU,6474
-appose/service.py,sha256=LqchMIxQXV6YF_l8sAGEyJapc2lSL6KBttcbLzdABFw,11966
-appose/types.py,sha256=uuVNLt21eT3JEPtRQCMQ_IcFnp2yLgrJTSxzAnxkKFY,1618
-appose-0.1.0.dist-info/LICENSE.txt,sha256=BOQExkI6YKvnUVMUqDbPuSxsY7XXL1XRUTTbnfEWII0,1306
-appose-0.1.0.dist-info/METADATA,sha256=6ohYqhKiVnCvof0UX4HAnMmarKKIyi7hRScNtEO1T9E,5477
-appose-0.1.0.dist-info/WHEEL,sha256=pkctZYzUS4AYVn6dJ-7367OJZivF2e8RA9b_ZBjif18,92
-appose-0.1.0.dist-info/top_level.txt,sha256=oqHhw2QGlaFfH3jMR9H5Cd6XYHdEv5DvK1am0iEFPW0,7
-appose-0.1.0.dist-info/RECORD,,