appose 0.2.0__py3-none-any.whl → 0.6.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- appose/__init__.py +90 -6
- appose/environment.py +1 -1
- appose/paths.py +1 -1
- appose/python_worker.py +146 -79
- appose/service.py +58 -29
- appose/types.py +56 -10
- {appose-0.2.0.dist-info → appose-0.6.0.dist-info}/METADATA +16 -15
- appose-0.6.0.dist-info/RECORD +11 -0
- {appose-0.2.0.dist-info → appose-0.6.0.dist-info}/WHEEL +1 -1
- {appose-0.2.0.dist-info → appose-0.6.0.dist-info/licenses}/LICENSE.txt +1 -1
- appose-0.2.0.dist-info/RECORD +0 -11
- {appose-0.2.0.dist-info → appose-0.6.0.dist-info}/top_level.txt +0 -0
appose/__init__.py
CHANGED
@@ -2,7 +2,7 @@
|
|
2
2
|
# #%L
|
3
3
|
# Appose: multi-language interprocess cooperation with shared memory.
|
4
4
|
# %%
|
5
|
-
# Copyright (C) 2023 -
|
5
|
+
# Copyright (C) 2023 - 2025 Appose developers.
|
6
6
|
# %%
|
7
7
|
# Redistribution and use in source and binary forms, with or without
|
8
8
|
# modification, are permitted provided that the following conditions are met:
|
@@ -42,8 +42,6 @@ The steps for using Appose are:
|
|
42
42
|
|
43
43
|
## Examples
|
44
44
|
|
45
|
-
* TODO - move the below code somewhere linkable, for succinctness here.
|
46
|
-
|
47
45
|
Here is a very simple example written in Python:
|
48
46
|
|
49
47
|
import appose
|
@@ -84,7 +82,7 @@ And here is an example using a few more of Appose's features:
|
|
84
82
|
def task_listener(event):
|
85
83
|
match event.responseType:
|
86
84
|
case UPDATE:
|
87
|
-
print(f"Progress {
|
85
|
+
print(f"Progress {event.current}/{event.maximum}")
|
88
86
|
case COMPLETION:
|
89
87
|
numer = task.outputs["numer"]
|
90
88
|
denom = task.outputs["denom"]
|
@@ -127,8 +125,94 @@ But Appose is compatible with any program that abides by the
|
|
127
125
|
2. The worker must issue responses in Appose's response format on its
|
128
126
|
standard output (stdout) stream.
|
129
127
|
|
130
|
-
|
131
|
-
|
128
|
+
### Requests to worker from service
|
129
|
+
|
130
|
+
A *request* is a single line of JSON sent to the worker process via
|
131
|
+
its standard input stream. It has a `task` key taking the form of a
|
132
|
+
UUID, and a `requestType` key with one of the following values:
|
133
|
+
|
134
|
+
#### EXECUTE
|
135
|
+
|
136
|
+
Asynchronously execute a script within the worker process. E.g.:
|
137
|
+
|
138
|
+
{
|
139
|
+
"task" : "87427f91-d193-4b25-8d35-e1292a34b5c4",
|
140
|
+
"requestType" : "EXECUTE",
|
141
|
+
"script" : "task.outputs[\"result\"] = computeResult(gamma)\n",
|
142
|
+
"inputs" : {"gamma": 2.2}
|
143
|
+
}
|
144
|
+
|
145
|
+
#### CANCEL
|
146
|
+
|
147
|
+
Cancel a running script. E.g.:
|
148
|
+
|
149
|
+
{
|
150
|
+
"task" : "87427f91-d193-4b25-8d35-e1292a34b5c4",
|
151
|
+
"requestType" : "CANCEL"
|
152
|
+
}
|
153
|
+
|
154
|
+
### Responses from worker to service
|
155
|
+
|
156
|
+
A *response* is a single line of JSON with a `task` key taking the form
|
157
|
+
of a UUID, and a `responseType` key with one of the following values:
|
158
|
+
|
159
|
+
#### LAUNCH
|
160
|
+
|
161
|
+
A LAUNCH response is issued to confirm the success of an EXECUTE
|
162
|
+
request.
|
163
|
+
|
164
|
+
{
|
165
|
+
"task" : "87427f91-d193-4b25-8d35-e1292a34b5c4",
|
166
|
+
"responseType" : "LAUNCH"
|
167
|
+
}
|
168
|
+
|
169
|
+
#### UPDATE
|
170
|
+
|
171
|
+
An UPDATE response is issued to convey that a task has somehow made
|
172
|
+
progress. The UPDATE response typically comes bundled with a
|
173
|
+
`message` string indicating what has changed, `current` and/or
|
174
|
+
`maximum` progress indicators conveying the step the task has
|
175
|
+
reached, or both.
|
176
|
+
|
177
|
+
{
|
178
|
+
"task" : "87427f91-d193-4b25-8d35-e1292a34b5c4",
|
179
|
+
"responseType" : "UPDATE",
|
180
|
+
"message" : "Processing step 0 of 91",
|
181
|
+
"current" : 0,
|
182
|
+
"maximum" : 91
|
183
|
+
}
|
184
|
+
|
185
|
+
#### COMPLETION
|
186
|
+
|
187
|
+
A COMPLETION response is issued to convey that a task has successfully
|
188
|
+
completed execution, as well as report the values of any task outputs.
|
189
|
+
|
190
|
+
{
|
191
|
+
"task" : "87427f91-d193-4b25-8d35-e1292a34b5c4",
|
192
|
+
"responseType" : "COMPLETION",
|
193
|
+
"outputs" : {"result" : 91}
|
194
|
+
}
|
195
|
+
|
196
|
+
#### CANCELATION
|
197
|
+
|
198
|
+
A CANCELATION response is issued to confirm the success of a CANCEL
|
199
|
+
request.
|
200
|
+
|
201
|
+
{
|
202
|
+
"task" : "87427f91-d193-4b25-8d35-e1292a34b5c4",
|
203
|
+
"responseType" : "CANCELATION"
|
204
|
+
}
|
205
|
+
|
206
|
+
#### FAILURE
|
207
|
+
|
208
|
+
A FAILURE response is issued to convey that a task did not completely
|
209
|
+
and successfully execute, such as an exception being raised.
|
210
|
+
|
211
|
+
{
|
212
|
+
"task" : "87427f91-d193-4b25-8d35-e1292a34b5c4",
|
213
|
+
"responseType" : "FAILURE",
|
214
|
+
"error", "Invalid gamma value"
|
215
|
+
}
|
132
216
|
'''
|
133
217
|
|
134
218
|
from pathlib import Path
|
appose/environment.py
CHANGED
@@ -2,7 +2,7 @@
|
|
2
2
|
# #%L
|
3
3
|
# Appose: multi-language interprocess cooperation with shared memory.
|
4
4
|
# %%
|
5
|
-
# Copyright (C) 2023 -
|
5
|
+
# Copyright (C) 2023 - 2025 Appose developers.
|
6
6
|
# %%
|
7
7
|
# Redistribution and use in source and binary forms, with or without
|
8
8
|
# modification, are permitted provided that the following conditions are met:
|
appose/paths.py
CHANGED
@@ -2,7 +2,7 @@
|
|
2
2
|
# #%L
|
3
3
|
# Appose: multi-language interprocess cooperation with shared memory.
|
4
4
|
# %%
|
5
|
-
# Copyright (C) 2023 -
|
5
|
+
# Copyright (C) 2023 - 2025 Appose developers.
|
6
6
|
# %%
|
7
7
|
# Redistribution and use in source and binary forms, with or without
|
8
8
|
# modification, are permitted provided that the following conditions are met:
|
appose/python_worker.py
CHANGED
@@ -2,7 +2,7 @@
|
|
2
2
|
# #%L
|
3
3
|
# Appose: multi-language interprocess cooperation with shared memory.
|
4
4
|
# %%
|
5
|
-
# Copyright (C) 2023 -
|
5
|
+
# Copyright (C) 2023 - 2025 Appose developers.
|
6
6
|
# %%
|
7
7
|
# Redistribution and use in source and binary forms, with or without
|
8
8
|
# modification, are permitted provided that the following conditions are met:
|
@@ -42,7 +42,8 @@ import ast
|
|
42
42
|
import sys
|
43
43
|
import traceback
|
44
44
|
from threading import Thread
|
45
|
-
from
|
45
|
+
from time import sleep
|
46
|
+
from typing import Any
|
46
47
|
|
47
48
|
# NB: Avoid relative imports so that this script can be run standalone.
|
48
49
|
from appose.service import RequestType, ResponseType
|
@@ -50,16 +51,23 @@ from appose.types import Args, _set_worker, decode, encode
|
|
50
51
|
|
51
52
|
|
52
53
|
class Task:
|
53
|
-
def __init__(self, uuid: str) -> None:
|
54
|
-
self.
|
54
|
+
def __init__(self, uuid: str, script: str, inputs: Args | None = None) -> None:
|
55
|
+
self._uuid = uuid
|
56
|
+
self._script = script
|
57
|
+
self._inputs = inputs
|
58
|
+
self._finished = False
|
59
|
+
self._thread = None
|
60
|
+
|
61
|
+
# Public-facing fields for use within the task script.
|
55
62
|
self.outputs = {}
|
56
63
|
self.cancel_requested = False
|
57
64
|
|
58
65
|
def update(
|
59
66
|
self,
|
60
|
-
message:
|
61
|
-
current:
|
62
|
-
maximum:
|
67
|
+
message: str | None = None,
|
68
|
+
current: int | None = None,
|
69
|
+
maximum: int | None = None,
|
70
|
+
info: dict[str, Any] | None = None,
|
63
71
|
) -> None:
|
64
72
|
args = {}
|
65
73
|
if message is not None:
|
@@ -74,21 +82,22 @@ class Task:
|
|
74
82
|
args["maximum"] = int(maximum)
|
75
83
|
except ValueError:
|
76
84
|
pass
|
85
|
+
args["info"] = info
|
77
86
|
self._respond(ResponseType.UPDATE, args)
|
78
87
|
|
79
88
|
def cancel(self) -> None:
|
80
89
|
self._respond(ResponseType.CANCELATION, None)
|
81
90
|
|
82
|
-
def fail(self, error:
|
91
|
+
def fail(self, error: str | None = None) -> None:
|
83
92
|
args = None if error is None else {"error": error}
|
84
93
|
self._respond(ResponseType.FAILURE, args)
|
85
94
|
|
86
|
-
def
|
87
|
-
|
95
|
+
def _run(self) -> None:
|
96
|
+
try:
|
88
97
|
# Populate script bindings.
|
89
98
|
binding = {"task": self}
|
90
|
-
if
|
91
|
-
binding.update(
|
99
|
+
if self._inputs is not None:
|
100
|
+
binding.update(self._inputs)
|
92
101
|
|
93
102
|
# Inform the calling process that the script is launching.
|
94
103
|
self._report_launch()
|
@@ -96,30 +105,32 @@ class Task:
|
|
96
105
|
# Execute the script.
|
97
106
|
# result = exec(script, locals=binding)
|
98
107
|
result = None
|
99
|
-
|
100
|
-
|
101
|
-
|
102
|
-
|
103
|
-
|
104
|
-
|
105
|
-
|
106
|
-
|
107
|
-
|
108
|
-
|
109
|
-
|
110
|
-
|
111
|
-
|
112
|
-
|
113
|
-
|
114
|
-
|
115
|
-
|
116
|
-
|
117
|
-
|
118
|
-
|
119
|
-
|
120
|
-
|
121
|
-
|
122
|
-
|
108
|
+
|
109
|
+
# NB: Execute the block, except for the last statement,
|
110
|
+
# which we evaluate instead to get its return value.
|
111
|
+
# Credit: https://stackoverflow.com/a/39381428/1207769
|
112
|
+
|
113
|
+
block = ast.parse(self._script, mode="exec")
|
114
|
+
last = None
|
115
|
+
if (
|
116
|
+
len(block.body) > 0
|
117
|
+
and hasattr(block.body[-1], "value")
|
118
|
+
and not isinstance(block.body[-1], ast.Assign)
|
119
|
+
):
|
120
|
+
# Last statement of the script looks like an expression. Evaluate!
|
121
|
+
last = ast.Expression(block.body.pop().value)
|
122
|
+
|
123
|
+
# NB: When `exec` gets two separate objects as *globals* and
|
124
|
+
# *locals*, the code will be executed as if it were embedded in
|
125
|
+
# a class definition. This means functions and classes defined
|
126
|
+
# in the executed code will not be able to access variables
|
127
|
+
# assigned at the top level, because the "top level" variables
|
128
|
+
# are treated as class variables in a class definition.
|
129
|
+
# See: https://docs.python.org/3/library/functions.html#exec
|
130
|
+
_globals = binding
|
131
|
+
exec(compile(block, "<string>", mode="exec"), _globals, binding)
|
132
|
+
if last is not None:
|
133
|
+
result = eval(compile(last, "<string>", mode="eval"), _globals, binding)
|
123
134
|
|
124
135
|
# Report the results to the Appose calling process.
|
125
136
|
if isinstance(result, dict):
|
@@ -128,13 +139,9 @@ class Task:
|
|
128
139
|
elif result is not None:
|
129
140
|
# Script produced a non-dict; add it alone to the outputs.
|
130
141
|
self.outputs["result"] = result
|
131
|
-
|
132
142
|
self._report_completion()
|
133
|
-
|
134
|
-
|
135
|
-
# expose a "force" option for cancelation that kills it forcibly; see:
|
136
|
-
# https://www.geeksforgeeks.org/python-different-ways-to-kill-a-thread/
|
137
|
-
Thread(target=execute_script, name=f"Appose-{self.uuid}").start()
|
143
|
+
except Exception:
|
144
|
+
self.fail(traceback.format_exc())
|
138
145
|
|
139
146
|
def _report_launch(self) -> None:
|
140
147
|
self._respond(ResponseType.LAUNCH, None)
|
@@ -143,55 +150,115 @@ class Task:
|
|
143
150
|
args = None if self.outputs is None else {"outputs": self.outputs}
|
144
151
|
self._respond(ResponseType.COMPLETION, args)
|
145
152
|
|
146
|
-
def _respond(self, response_type: ResponseType, args:
|
147
|
-
|
153
|
+
def _respond(self, response_type: ResponseType, args: Args | None) -> None:
|
154
|
+
already_terminated = False
|
155
|
+
if response_type.is_terminal():
|
156
|
+
if self._finished:
|
157
|
+
# This is not the first terminal response. Let's
|
158
|
+
# remember, in case an exception is generated below,
|
159
|
+
# so that we can avoid infinite recursion loops.
|
160
|
+
already_terminated = True
|
161
|
+
self._finished = True
|
162
|
+
|
163
|
+
response = {}
|
148
164
|
if args is not None:
|
149
165
|
response.update(args)
|
166
|
+
response.update({"task": self._uuid, "responseType": response_type.value})
|
150
167
|
# NB: Flush is necessary to ensure service receives the data!
|
151
168
|
try:
|
152
169
|
print(encode(response), flush=True)
|
153
170
|
except Exception:
|
154
|
-
|
155
|
-
|
171
|
+
if already_terminated:
|
172
|
+
# An exception triggered a failure response which
|
173
|
+
# then triggered another exception. Let's stop here
|
174
|
+
# to avoid the risk of infinite recursion loops.
|
175
|
+
return
|
176
|
+
# Encoding can fail due to unsupported types, when the
|
177
|
+
# response or its elements are not supported by JSON encoding.
|
156
178
|
# No matter what goes wrong, we want to tell the caller.
|
157
|
-
if response_type is ResponseType.FAILURE:
|
158
|
-
# TODO: How to address this hypothetical case
|
159
|
-
# of a failure message triggering another failure?
|
160
|
-
raise
|
161
179
|
self.fail(traceback.format_exc())
|
162
180
|
|
163
181
|
|
164
|
-
|
165
|
-
_set_worker(True)
|
182
|
+
class Worker:
|
166
183
|
|
167
|
-
|
184
|
+
def __init__(self):
|
185
|
+
self.tasks = {}
|
186
|
+
self.queue: list[Task] = []
|
187
|
+
self.running = True
|
168
188
|
|
169
|
-
|
170
|
-
|
171
|
-
|
172
|
-
|
173
|
-
|
174
|
-
|
175
|
-
|
176
|
-
|
177
|
-
|
178
|
-
|
179
|
-
|
180
|
-
|
181
|
-
|
182
|
-
|
183
|
-
|
184
|
-
|
185
|
-
|
186
|
-
|
187
|
-
|
188
|
-
|
189
|
-
|
190
|
-
|
191
|
-
|
192
|
-
|
193
|
-
|
194
|
-
|
189
|
+
# Flag this process as a worker, not a service.
|
190
|
+
_set_worker(True)
|
191
|
+
|
192
|
+
Thread(target=self._process_input, name="Appose-Receiver").start()
|
193
|
+
Thread(target=self._cleanup_threads, name="Appose-Janitor").start()
|
194
|
+
|
195
|
+
def run(self) -> None:
|
196
|
+
"""
|
197
|
+
Processes tasks from the task queue.
|
198
|
+
"""
|
199
|
+
while self.running:
|
200
|
+
if len(self.queue) == 0:
|
201
|
+
# Nothing queued, so wait a bit.
|
202
|
+
sleep(0.05)
|
203
|
+
continue
|
204
|
+
task = self.queue.pop()
|
205
|
+
task._run()
|
206
|
+
|
207
|
+
def _process_input(self) -> None:
|
208
|
+
while True:
|
209
|
+
try:
|
210
|
+
line = input().strip()
|
211
|
+
except EOFError:
|
212
|
+
line = None
|
213
|
+
if not line:
|
214
|
+
self.running = False
|
215
|
+
return
|
216
|
+
|
217
|
+
request = decode(line)
|
218
|
+
uuid = request.get("task")
|
219
|
+
request_type = request.get("requestType")
|
220
|
+
|
221
|
+
match RequestType(request_type):
|
222
|
+
case RequestType.EXECUTE:
|
223
|
+
script = request.get("script")
|
224
|
+
inputs = request.get("inputs")
|
225
|
+
queue = request.get("queue")
|
226
|
+
task = Task(uuid, script, inputs)
|
227
|
+
self.tasks[uuid] = task
|
228
|
+
if queue == "main":
|
229
|
+
# Add the task to the main thread queue.
|
230
|
+
self.queue.append(task)
|
231
|
+
else:
|
232
|
+
# Create a thread and save a reference to it, in case its script
|
233
|
+
# kills the thread. This happens e.g. if it calls sys.exit.
|
234
|
+
task._thread = Thread(target=task._run, name=f"Appose-{uuid}")
|
235
|
+
task._thread.start()
|
236
|
+
|
237
|
+
case RequestType.CANCEL:
|
238
|
+
task = self.tasks.get(uuid)
|
239
|
+
if task is None:
|
240
|
+
print(f"No such task: {uuid}", file=sys.stderr)
|
241
|
+
continue
|
242
|
+
task.cancel_requested = True
|
243
|
+
|
244
|
+
def _cleanup_threads(self) -> None:
|
245
|
+
while self.running:
|
246
|
+
sleep(0.05)
|
247
|
+
dead = {
|
248
|
+
uuid: task
|
249
|
+
for uuid, task in self.tasks.items()
|
250
|
+
if task._thread is not None and not task._thread.is_alive()
|
251
|
+
}
|
252
|
+
for uuid, task in dead.items():
|
253
|
+
self.tasks.pop(uuid)
|
254
|
+
if not task._finished:
|
255
|
+
# The task died before reporting a terminal status.
|
256
|
+
# We report this situation as failure by thread death.
|
257
|
+
task.fail("thread death")
|
258
|
+
|
259
|
+
|
260
|
+
def main() -> None:
|
261
|
+
Worker().run()
|
195
262
|
|
196
263
|
|
197
264
|
if __name__ == "__main__":
|
appose/service.py
CHANGED
@@ -2,7 +2,7 @@
|
|
2
2
|
# #%L
|
3
3
|
# Appose: multi-language interprocess cooperation with shared memory.
|
4
4
|
# %%
|
5
|
-
# Copyright (C) 2023 -
|
5
|
+
# Copyright (C) 2023 - 2025 Appose developers.
|
6
6
|
# %%
|
7
7
|
# Redistribution and use in source and binary forms, with or without
|
8
8
|
# modification, are permitted provided that the following conditions are met:
|
@@ -36,7 +36,7 @@ import threading
|
|
36
36
|
from enum import Enum
|
37
37
|
from pathlib import Path
|
38
38
|
from traceback import format_exc
|
39
|
-
from typing import Any, Callable
|
39
|
+
from typing import Any, Callable
|
40
40
|
from uuid import uuid4
|
41
41
|
|
42
42
|
from .types import Args, decode, encode
|
@@ -52,17 +52,17 @@ class Service:
|
|
52
52
|
|
53
53
|
_service_count = 0
|
54
54
|
|
55
|
-
def __init__(self, cwd:
|
55
|
+
def __init__(self, cwd: str | Path, args: list[str]) -> None:
|
56
56
|
self._cwd = cwd
|
57
57
|
self._args = args[:]
|
58
|
-
self._tasks:
|
58
|
+
self._tasks: dict[str, "Task"] = {}
|
59
59
|
self._service_id = Service._service_count
|
60
60
|
Service._service_count += 1
|
61
|
-
self._process:
|
62
|
-
self._stdout_thread:
|
63
|
-
self._stderr_thread:
|
64
|
-
self._monitor_thread:
|
65
|
-
self._debug_callback:
|
61
|
+
self._process: subprocess.Popen | None = None
|
62
|
+
self._stdout_thread: threading.Thread | None = None
|
63
|
+
self._stderr_thread: threading.Thread | None = None
|
64
|
+
self._monitor_thread: threading.Thread | None = None
|
65
|
+
self._debug_callback: Callable[[Any], Any] | None = None
|
66
66
|
|
67
67
|
def debug(self, debug_callback: Callable[[Any], Any]) -> None:
|
68
68
|
"""
|
@@ -109,16 +109,20 @@ class Service:
|
|
109
109
|
self._stderr_thread.start()
|
110
110
|
self._monitor_thread.start()
|
111
111
|
|
112
|
-
def task(
|
112
|
+
def task(
|
113
|
+
self, script: str, inputs: Args | None = None, queue: str | None = None
|
114
|
+
) -> "Task":
|
113
115
|
"""
|
114
116
|
Create a new task, passing the given script to the worker for execution.
|
115
117
|
:param script:
|
116
118
|
The script for the worker to execute in its environment.
|
117
119
|
:param inputs:
|
118
120
|
Optional list of key/value pairs to feed into the script as inputs.
|
121
|
+
:param queue:
|
122
|
+
Optional queue target. Pass "main" to queue to worker's main thread.
|
119
123
|
"""
|
120
124
|
self.start()
|
121
|
-
return Task(self, script, inputs)
|
125
|
+
return Task(self, script, inputs, queue)
|
122
126
|
|
123
127
|
def close(self) -> None:
|
124
128
|
"""
|
@@ -256,11 +260,35 @@ class ResponseType(Enum):
|
|
256
260
|
FAILURE = "FAILURE"
|
257
261
|
CRASH = "CRASH"
|
258
262
|
|
263
|
+
"""
|
264
|
+
True iff response type is COMPLETE, CANCELED, FAILED, or CRASHED.
|
265
|
+
"""
|
266
|
+
|
267
|
+
def is_terminal(self):
|
268
|
+
return self in (
|
269
|
+
ResponseType.COMPLETION,
|
270
|
+
ResponseType.CANCELATION,
|
271
|
+
ResponseType.FAILURE,
|
272
|
+
ResponseType.CRASH,
|
273
|
+
)
|
274
|
+
|
259
275
|
|
260
276
|
class TaskEvent:
|
261
|
-
def __init__(
|
277
|
+
def __init__(
|
278
|
+
self,
|
279
|
+
task: "Task",
|
280
|
+
response_type: ResponseType,
|
281
|
+
message: str | None = None,
|
282
|
+
current: int | None = None,
|
283
|
+
maximum: int | None = None,
|
284
|
+
info: dict[str, Any] | None = None,
|
285
|
+
) -> None:
|
262
286
|
self.task: "Task" = task
|
263
287
|
self.response_type: ResponseType = response_type
|
288
|
+
self.message: str | None = message
|
289
|
+
self.current: int | None = current
|
290
|
+
self.maximum: int | None = maximum
|
291
|
+
self.info: dict[str, Any] | None = info
|
264
292
|
|
265
293
|
def __str__(self):
|
266
294
|
return f"[{self.response_type}] {self.task}"
|
@@ -274,21 +302,26 @@ class Task:
|
|
274
302
|
"""
|
275
303
|
|
276
304
|
def __init__(
|
277
|
-
self,
|
305
|
+
self,
|
306
|
+
service: Service,
|
307
|
+
script: str,
|
308
|
+
inputs: Args | None = None,
|
309
|
+
queue: str | None = None,
|
278
310
|
) -> None:
|
279
311
|
self.uuid = uuid4().hex
|
280
312
|
self.service = service
|
281
313
|
self.script = script
|
282
314
|
self.inputs: Args = {}
|
315
|
+
self.queue: str | None = queue
|
283
316
|
if inputs is not None:
|
284
317
|
self.inputs.update(inputs)
|
285
318
|
self.outputs: Args = {}
|
286
319
|
self.status: TaskStatus = TaskStatus.INITIAL
|
287
|
-
self.message:
|
320
|
+
self.message: str | None = None
|
288
321
|
self.current: int = 0
|
289
322
|
self.maximum: int = 1
|
290
|
-
self.error:
|
291
|
-
self.listeners:
|
323
|
+
self.error: str | None = None
|
324
|
+
self.listeners: list[Callable[["TaskEvent"], None]] = []
|
292
325
|
self.cv = threading.Condition()
|
293
326
|
self.service._tasks[self.uuid] = self
|
294
327
|
|
@@ -299,7 +332,7 @@ class Task:
|
|
299
332
|
|
300
333
|
self.status = TaskStatus.QUEUED
|
301
334
|
|
302
|
-
args = {"script": self.script, "inputs": self.inputs}
|
335
|
+
args = {"script": self.script, "inputs": self.inputs, "queue": self.queue}
|
303
336
|
self._request(RequestType.EXECUTE, args)
|
304
337
|
|
305
338
|
return self
|
@@ -354,13 +387,8 @@ class Task:
|
|
354
387
|
case ResponseType.LAUNCH:
|
355
388
|
self.status = TaskStatus.RUNNING
|
356
389
|
case ResponseType.UPDATE:
|
357
|
-
|
358
|
-
|
359
|
-
maximum = response.get("maximum")
|
360
|
-
if current is not None:
|
361
|
-
self.current = int(current)
|
362
|
-
if maximum is not None:
|
363
|
-
self.maximum = int(maximum)
|
390
|
+
# No extra action needed.
|
391
|
+
pass
|
364
392
|
case ResponseType.COMPLETION:
|
365
393
|
self.service._tasks.pop(self.uuid, None)
|
366
394
|
self.status = TaskStatus.COMPLETE
|
@@ -380,7 +408,11 @@ class Task:
|
|
380
408
|
)
|
381
409
|
return
|
382
410
|
|
383
|
-
|
411
|
+
message = response.get("message")
|
412
|
+
current = response.get("current")
|
413
|
+
maximum = response.get("maximum")
|
414
|
+
info = response.get("info")
|
415
|
+
event = TaskEvent(self, response_type, message, current, maximum, info)
|
384
416
|
for listener in self.listeners:
|
385
417
|
listener(event)
|
386
418
|
|
@@ -397,7 +429,4 @@ class Task:
|
|
397
429
|
self.cv.notify_all()
|
398
430
|
|
399
431
|
def __str__(self):
|
400
|
-
return
|
401
|
-
f"{self.uuid=}, {self.status=}, {self.message=}, "
|
402
|
-
f"{self.current=}, {self.maximum=}, {self.error=}"
|
403
|
-
)
|
432
|
+
return f"{self.uuid=}, {self.status=}, {self.error=}"
|
appose/types.py
CHANGED
@@ -2,7 +2,7 @@
|
|
2
2
|
# #%L
|
3
3
|
# Appose: multi-language interprocess cooperation with shared memory.
|
4
4
|
# %%
|
5
|
-
# Copyright (C) 2023 -
|
5
|
+
# Copyright (C) 2023 - 2025 Appose developers.
|
6
6
|
# %%
|
7
7
|
# Redistribution and use in source and binary forms, with or without
|
8
8
|
# modification, are permitted provided that the following conditions are met:
|
@@ -45,13 +45,29 @@ class SharedMemory(shared_memory.SharedMemory):
|
|
45
45
|
`unlink_on_dispose` flag.
|
46
46
|
"""
|
47
47
|
|
48
|
-
def __init__(self, name: str = None, create: bool = False,
|
49
|
-
|
48
|
+
def __init__(self, name: str = None, create: bool = False, rsize: int = 0):
|
49
|
+
"""
|
50
|
+
Create a new shared memory block, or attach to an existing one.
|
51
|
+
|
52
|
+
:param name:
|
53
|
+
The unique name for the requested shared memory, specified as a
|
54
|
+
string. If create is True (i.e. a new shared memory block) and
|
55
|
+
no name is given, a novel name will be generated.
|
56
|
+
:param create:
|
57
|
+
Whether a new shared memory block is created (True)
|
58
|
+
or an existing one is attached to (False).
|
59
|
+
:param rsize:
|
60
|
+
Requested size in bytes. The true allocated size will be at least
|
61
|
+
this much, but may be rounded up to the next block size multiple,
|
62
|
+
depending on the running platform.
|
63
|
+
"""
|
64
|
+
super().__init__(name=name, create=create, size=rsize)
|
65
|
+
self.rsize = rsize
|
50
66
|
self._unlink_on_dispose = create
|
51
67
|
if _is_worker:
|
52
68
|
# HACK: Remove this shared memory block from the resource_tracker,
|
53
|
-
# which
|
54
|
-
# references are done using them.
|
69
|
+
# which would otherwise want to clean up shared memory blocks
|
70
|
+
# after all known references are done using them.
|
55
71
|
#
|
56
72
|
# There is one resource_tracker per Python process, and they will
|
57
73
|
# each try to delete shared memory blocks known to them when they
|
@@ -60,7 +76,37 @@ class SharedMemory(shared_memory.SharedMemory):
|
|
60
76
|
# As such, the rule Appose follows is: let the service process
|
61
77
|
# always handle cleanup of shared memory blocks, regardless of
|
62
78
|
# which process initially allocated it.
|
63
|
-
|
79
|
+
try:
|
80
|
+
resource_tracker.unregister(self._name, "shared_memory")
|
81
|
+
except ModuleNotFoundError:
|
82
|
+
# Unfortunately, on (some?) Windows systems, we see the error:
|
83
|
+
#
|
84
|
+
# Traceback (most recent call last): # noqa: E501
|
85
|
+
# File "...\site-packages\appose\types.py", line 97, in decode # noqa: E501
|
86
|
+
# return json.loads(the_json, object_hook=_appose_object_hook) # noqa: E501
|
87
|
+
# File "...\lib\json\__init__.py", line 359, in loads # noqa: E501
|
88
|
+
# return cls(**kw).decode(s) # noqa: E501
|
89
|
+
# File "...\lib\json\decoder.py", line 337, in decode # noqa: E501
|
90
|
+
# obj, end = self.raw_decode(s, idx=_w(s, 0).end()) # noqa: E501
|
91
|
+
# File "...\lib\json\decoder.py", line 353, in raw_decode # noqa: E501
|
92
|
+
# obj, end = self.scan_once(s, idx) # noqa: E501
|
93
|
+
# File "...\site-packages\appose\types.py", line 177, in _appose_object_hook # noqa: E501
|
94
|
+
# return SharedMemory(name=(obj["name"]), size=(obj["size"])) # noqa: E501
|
95
|
+
# File "...\site-packages\appose\types.py", line 63, in __init__ # noqa: E501
|
96
|
+
# resource_tracker.unregister(self._name, "shared_memory") # noqa: E501
|
97
|
+
# File "...\lib\multiprocessing\resource_tracker.py", line 159, in unregister # noqa: E501
|
98
|
+
# self._send('UNREGISTER', name, rtype) # noqa: E501
|
99
|
+
# File "...\lib\multiprocessing\resource_tracker.py", line 162, in _send # noqa: E501
|
100
|
+
# self.ensure_running() # noqa: E501
|
101
|
+
# File "...\lib\multiprocessing\resource_tracker.py", line 129, in ensure_running # noqa: E501
|
102
|
+
# pid = util.spawnv_passfds(exe, args, fds_to_pass) # noqa: E501
|
103
|
+
# File "...\lib\multiprocessing\util.py", line 448, in spawnv_passfds # noqa: E501
|
104
|
+
# import _posixsubprocess # noqa: E501
|
105
|
+
# ModuleNotFoundError: No module named '_posixsubprocess' # noqa: E501
|
106
|
+
#
|
107
|
+
# A bug in Python? Regardless: we guard against it here.
|
108
|
+
# See also: https://github.com/imglib/imglib2-appose/issues/1
|
109
|
+
pass
|
64
110
|
|
65
111
|
def unlink_on_dispose(self, value: bool) -> None:
|
66
112
|
"""
|
@@ -116,7 +162,7 @@ class NDArray:
|
|
116
162
|
self.shape = shape
|
117
163
|
self.shm = (
|
118
164
|
SharedMemory(
|
119
|
-
create=True,
|
165
|
+
create=True, rsize=ceil(prod(shape) * _bytes_per_element(dtype))
|
120
166
|
)
|
121
167
|
if shm is None
|
122
168
|
else shm
|
@@ -127,7 +173,7 @@ class NDArray:
|
|
127
173
|
f"NDArray("
|
128
174
|
f"dtype='{self.dtype}', "
|
129
175
|
f"shape={self.shape}, "
|
130
|
-
f"shm='{self.shm.name}' ({self.shm.
|
176
|
+
f"shm='{self.shm.name}' ({self.shm.rsize}))"
|
131
177
|
)
|
132
178
|
|
133
179
|
def ndarray(self):
|
@@ -158,7 +204,7 @@ class _ApposeJSONEncoder(json.JSONEncoder):
|
|
158
204
|
return {
|
159
205
|
"appose_type": "shm",
|
160
206
|
"name": obj.name,
|
161
|
-
"
|
207
|
+
"rsize": obj.rsize,
|
162
208
|
}
|
163
209
|
if isinstance(obj, NDArray):
|
164
210
|
return {
|
@@ -174,7 +220,7 @@ def _appose_object_hook(obj: Dict):
|
|
174
220
|
atype = obj.get("appose_type")
|
175
221
|
if atype == "shm":
|
176
222
|
# Attach to existing shared memory block.
|
177
|
-
return SharedMemory(name=(obj["name"]),
|
223
|
+
return SharedMemory(name=(obj["name"]), rsize=(obj["rsize"]))
|
178
224
|
elif atype == "ndarray":
|
179
225
|
return NDArray(obj["dtype"], obj["shape"], obj["shm"])
|
180
226
|
else:
|
@@ -1,9 +1,9 @@
|
|
1
|
-
Metadata-Version: 2.
|
1
|
+
Metadata-Version: 2.4
|
2
2
|
Name: appose
|
3
|
-
Version: 0.
|
3
|
+
Version: 0.6.0
|
4
4
|
Summary: Appose: multi-language interprocess cooperation with shared memory.
|
5
5
|
Author: Appose developers
|
6
|
-
License:
|
6
|
+
License-Expression: BSD-2-Clause
|
7
7
|
Project-URL: homepage, https://github.com/apposed/appose-python
|
8
8
|
Project-URL: documentation, https://github.com/apposed/appose-python/blob/main/README.md
|
9
9
|
Project-URL: source, https://github.com/apposed/appose-python
|
@@ -17,7 +17,7 @@ Classifier: Intended Audience :: Science/Research
|
|
17
17
|
Classifier: Programming Language :: Python :: 3 :: Only
|
18
18
|
Classifier: Programming Language :: Python :: 3.10
|
19
19
|
Classifier: Programming Language :: Python :: 3.11
|
20
|
-
Classifier:
|
20
|
+
Classifier: Programming Language :: Python :: 3.12
|
21
21
|
Classifier: Operating System :: Microsoft :: Windows
|
22
22
|
Classifier: Operating System :: Unix
|
23
23
|
Classifier: Operating System :: MacOS
|
@@ -29,17 +29,18 @@ Requires-Python: >=3.10
|
|
29
29
|
Description-Content-Type: text/markdown
|
30
30
|
License-File: LICENSE.txt
|
31
31
|
Provides-Extra: dev
|
32
|
-
Requires-Dist: autopep8
|
33
|
-
Requires-Dist: black
|
34
|
-
Requires-Dist: build
|
35
|
-
Requires-Dist: flake8
|
36
|
-
Requires-Dist: flake8-pyproject
|
37
|
-
Requires-Dist: flake8-typing-imports
|
38
|
-
Requires-Dist: isort
|
39
|
-
Requires-Dist: pytest
|
40
|
-
Requires-Dist: numpy
|
41
|
-
Requires-Dist: toml
|
42
|
-
Requires-Dist: validate-pyproject[all]
|
32
|
+
Requires-Dist: autopep8; extra == "dev"
|
33
|
+
Requires-Dist: black; extra == "dev"
|
34
|
+
Requires-Dist: build; extra == "dev"
|
35
|
+
Requires-Dist: flake8; extra == "dev"
|
36
|
+
Requires-Dist: flake8-pyproject; extra == "dev"
|
37
|
+
Requires-Dist: flake8-typing-imports; extra == "dev"
|
38
|
+
Requires-Dist: isort; extra == "dev"
|
39
|
+
Requires-Dist: pytest; extra == "dev"
|
40
|
+
Requires-Dist: numpy; extra == "dev"
|
41
|
+
Requires-Dist: toml; extra == "dev"
|
42
|
+
Requires-Dist: validate-pyproject[all]; extra == "dev"
|
43
|
+
Dynamic: license-file
|
43
44
|
|
44
45
|
# Appose Python
|
45
46
|
|
@@ -0,0 +1,11 @@
|
|
1
|
+
appose/__init__.py,sha256=SRXmNEqFIAajVlwVCSAdhHQytRKZJiWI5Y5-xwioW0w,7544
|
2
|
+
appose/environment.py,sha256=Kg-MfAcmVPDltwqvmrcvus-pP01SNUdn4bE2kdkIVuk,7597
|
3
|
+
appose/paths.py,sha256=WgBHwnZdS3Ot8_hssFqUiTGnunolYRrHLf9rAfEU5r4,2182
|
4
|
+
appose/python_worker.py,sha256=rkssHh3YTBueGTQ9qAuRVqq7bXncYulscG3kbM0D2TQ,10061
|
5
|
+
appose/service.py,sha256=jdtGOOoWjut9eZj441fL8t_P_ln2mfgilBvJdH0bDus,14750
|
6
|
+
appose/types.py,sha256=uidbt2JwWJ2D3Mbtc4Vl2sRP6WNgKjmfcHjJ9QLWkcs,10557
|
7
|
+
appose-0.6.0.dist-info/licenses/LICENSE.txt,sha256=ZedidA6NyZclNlsx-vl9IVBWRAQSqXdNsCWgtFcwzIE,1313
|
8
|
+
appose-0.6.0.dist-info/METADATA,sha256=F900U1qP-pPiPgklzrnjg7og3LuY5h8pXYDdadVtfdY,5598
|
9
|
+
appose-0.6.0.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
|
10
|
+
appose-0.6.0.dist-info/top_level.txt,sha256=oqHhw2QGlaFfH3jMR9H5Cd6XYHdEv5DvK1am0iEFPW0,7
|
11
|
+
appose-0.6.0.dist-info/RECORD,,
|
appose-0.2.0.dist-info/RECORD
DELETED
@@ -1,11 +0,0 @@
|
|
1
|
-
appose/__init__.py,sha256=sGkVQv1eZQp0BOoySsVOGcVMr47rE3PSyqxsk8_EOn4,5404
|
2
|
-
appose/environment.py,sha256=8BBvRIzr6ZIjBeZMQYOHG6Va1gSxMkYc3ieJ9HafeXA,7597
|
3
|
-
appose/paths.py,sha256=dcXD54rOUiCWfM58vqttRY0Eu6YUFMD65ZO_2V4hhEk,2182
|
4
|
-
appose/python_worker.py,sha256=jhe_iFnCS7vJ_N1EycDTt6oq09JhYIOnvvhyqnLG40U,7335
|
5
|
-
appose/service.py,sha256=v5u_HAliJlsoghwC6QIn3CZV2YpnPU8V3GdBcjgA62w,14070
|
6
|
-
appose/types.py,sha256=m5mX5BhdR7REnkSCuP5fGnnQWc_o7qQDyOFHWND8Vvs,6975
|
7
|
-
appose-0.2.0.dist-info/LICENSE.txt,sha256=LcTntIsPpu7JOZfEBAVslv-PVV4vnD9tfvXrS4cROe8,1313
|
8
|
-
appose-0.2.0.dist-info/METADATA,sha256=lqL66Xs9gTSJLr_ISb7ge8lK80xSQh04_Amfsf1jQWg,5586
|
9
|
-
appose-0.2.0.dist-info/WHEEL,sha256=R0nc6qTxuoLk7ShA2_Y-UWkN8ZdfDBG2B6Eqpz2WXbs,91
|
10
|
-
appose-0.2.0.dist-info/top_level.txt,sha256=oqHhw2QGlaFfH3jMR9H5Cd6XYHdEv5DvK1am0iEFPW0,7
|
11
|
-
appose-0.2.0.dist-info/RECORD,,
|
File without changes
|