process-bigraph 0.0.44__tar.gz → 0.0.46__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {process_bigraph-0.0.44/process_bigraph.egg-info → process_bigraph-0.0.46}/PKG-INFO +1 -1
- process_bigraph-0.0.46/process_bigraph/package/__init__.py +0 -0
- process_bigraph-0.0.46/process_bigraph/package/discover.py +99 -0
- process_bigraph-0.0.46/process_bigraph/protocols/__init__.py +17 -0
- process_bigraph-0.0.46/process_bigraph/protocols/docker.py +262 -0
- process_bigraph-0.0.46/process_bigraph/protocols/local.py +43 -0
- process_bigraph-0.0.46/process_bigraph/protocols/parallel.py +241 -0
- process_bigraph-0.0.46/process_bigraph/protocols/protocol.py +3 -0
- process_bigraph-0.0.46/process_bigraph/protocols/rest.py +171 -0
- process_bigraph-0.0.46/process_bigraph/protocols/socket.py +2 -0
- {process_bigraph-0.0.44 → process_bigraph-0.0.46/process_bigraph.egg-info}/PKG-INFO +1 -1
- {process_bigraph-0.0.44 → process_bigraph-0.0.46}/process_bigraph.egg-info/SOURCES.txt +10 -1
- {process_bigraph-0.0.44 → process_bigraph-0.0.46}/pyproject.toml +5 -2
- {process_bigraph-0.0.44 → process_bigraph-0.0.46}/.github/workflows/notebook_to_html.yml +0 -0
- {process_bigraph-0.0.44 → process_bigraph-0.0.46}/.github/workflows/pytest.yml +0 -0
- {process_bigraph-0.0.44 → process_bigraph-0.0.46}/.gitignore +0 -0
- {process_bigraph-0.0.44 → process_bigraph-0.0.46}/AUTHORS.md +0 -0
- {process_bigraph-0.0.44 → process_bigraph-0.0.46}/CLA.md +0 -0
- {process_bigraph-0.0.44 → process_bigraph-0.0.46}/CODE_OF_CONDUCT.md +0 -0
- {process_bigraph-0.0.44 → process_bigraph-0.0.46}/CONTRIBUTING.md +0 -0
- {process_bigraph-0.0.44 → process_bigraph-0.0.46}/LICENSE +0 -0
- {process_bigraph-0.0.44 → process_bigraph-0.0.46}/README.md +0 -0
- {process_bigraph-0.0.44 → process_bigraph-0.0.46}/doc/_static/process-bigraph.png +0 -0
- {process_bigraph-0.0.44 → process_bigraph-0.0.46}/notebooks/process-bigraphs.ipynb +0 -0
- {process_bigraph-0.0.44 → process_bigraph-0.0.46}/notebooks/visualize_processes.ipynb +0 -0
- {process_bigraph-0.0.44 → process_bigraph-0.0.46}/process_bigraph/__init__.py +0 -0
- {process_bigraph-0.0.44 → process_bigraph-0.0.46}/process_bigraph/composite.py +0 -0
- {process_bigraph-0.0.44 → process_bigraph-0.0.46}/process_bigraph/emitter.py +0 -0
- {process_bigraph-0.0.44 → process_bigraph-0.0.46}/process_bigraph/experiments/__init__.py +0 -0
- {process_bigraph-0.0.44 → process_bigraph-0.0.46}/process_bigraph/experiments/minimal_gillespie.py +0 -0
- {process_bigraph-0.0.44 → process_bigraph-0.0.46}/process_bigraph/process_types.py +0 -0
- {process_bigraph-0.0.44 → process_bigraph-0.0.46}/process_bigraph/processes/__init__.py +0 -0
- {process_bigraph-0.0.44 → process_bigraph-0.0.46}/process_bigraph/processes/growth_division.py +0 -0
- {process_bigraph-0.0.44 → process_bigraph-0.0.46}/process_bigraph/processes/parameter_scan.py +0 -0
- {process_bigraph-0.0.44 → process_bigraph-0.0.46}/process_bigraph/tests.py +0 -0
- {process_bigraph-0.0.44 → process_bigraph-0.0.46}/process_bigraph/units.py +0 -0
- {process_bigraph-0.0.44 → process_bigraph-0.0.46}/process_bigraph.egg-info/dependency_links.txt +0 -0
- {process_bigraph-0.0.44 → process_bigraph-0.0.46}/process_bigraph.egg-info/requires.txt +0 -0
- {process_bigraph-0.0.44 → process_bigraph-0.0.46}/process_bigraph.egg-info/top_level.txt +0 -0
- {process_bigraph-0.0.44 → process_bigraph-0.0.46}/pytest.ini +0 -0
- {process_bigraph-0.0.44 → process_bigraph-0.0.46}/release.sh +0 -0
- {process_bigraph-0.0.44 → process_bigraph-0.0.46}/setup.cfg +0 -0
- {process_bigraph-0.0.44 → process_bigraph-0.0.46}/setup.py +0 -0
|
File without changes
|
|
@@ -0,0 +1,99 @@
|
|
|
1
|
+
import importlib.metadata
|
|
2
|
+
import pkgutil
|
|
3
|
+
import inspect
|
|
4
|
+
|
|
5
|
+
from process_bigraph import Process, Step, ProcessTypes
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
def recursive_dynamic_import(core, package_name: str, verbose: bool = False) -> list[tuple[str, Process | Step ]]:
|
|
9
|
+
classes_to_import = []
|
|
10
|
+
adjusted_package_name: str = package_name.replace("-", "_")
|
|
11
|
+
# TODO: fix module name discovery based on package name
|
|
12
|
+
if adjusted_package_name == "vivarium_interface":
|
|
13
|
+
adjusted_package_name = "vivarium"
|
|
14
|
+
try:
|
|
15
|
+
module = importlib.import_module(adjusted_package_name)
|
|
16
|
+
except ModuleNotFoundError:
|
|
17
|
+
# TODO: Add code to try and find correct module name via accessing `top_level.txt`,
|
|
18
|
+
# and getting the correct module name
|
|
19
|
+
# find top-level.txt
|
|
20
|
+
# find correct module name
|
|
21
|
+
# return recursive_dynamic_import(correct_module_name)
|
|
22
|
+
raise ModuleNotFoundError(f"Error: module `{adjusted_package_name}` not found when trying to dynamically import!")
|
|
23
|
+
|
|
24
|
+
if hasattr(module, 'register_types'):
|
|
25
|
+
core = module.register_types(core)
|
|
26
|
+
|
|
27
|
+
class_members = inspect.getmembers(module, inspect.isclass)
|
|
28
|
+
if verbose:
|
|
29
|
+
print(f"Processing {len(class_members)} members...")
|
|
30
|
+
for class_name, clazz in class_members:
|
|
31
|
+
if clazz == Process:
|
|
32
|
+
if verbose:
|
|
33
|
+
print(f'Process `{class_name}` skipped, because it is `Process` itself.')
|
|
34
|
+
continue
|
|
35
|
+
if clazz == Step:
|
|
36
|
+
if verbose:
|
|
37
|
+
print(f'Process `{class_name}` skipped, because it is `Step` itself.')
|
|
38
|
+
continue
|
|
39
|
+
if issubclass(clazz, Process):
|
|
40
|
+
if verbose:
|
|
41
|
+
print(f'Process `{class_name}` added to queue!')
|
|
42
|
+
classes_to_import.append((f"{package_name}.{class_name}", clazz))
|
|
43
|
+
if issubclass(clazz, Step):
|
|
44
|
+
if verbose:
|
|
45
|
+
print(f'Step `{class_name}` added to queue!')
|
|
46
|
+
classes_to_import.append((f"{package_name}.{class_name}", clazz))
|
|
47
|
+
if verbose:
|
|
48
|
+
print(f'Invalid `{class_name}` skipped; not a Process nor Step!')
|
|
49
|
+
|
|
50
|
+
path = module.__path__ if hasattr(module, '__path__') else "<No `__path__` attr>"
|
|
51
|
+
modules_to_check = pkgutil.iter_modules(module.__path__) if hasattr(module, '__path__') else []
|
|
52
|
+
if verbose:
|
|
53
|
+
print(f"Checking for modules in `{module.__name__}` [path: {path}]...")
|
|
54
|
+
for _module_loader, subname, isPkg in modules_to_check:
|
|
55
|
+
# if not isPkg: continue
|
|
56
|
+
if verbose:
|
|
57
|
+
print(f"Found: {adjusted_package_name}.{subname}")
|
|
58
|
+
classes_to_import += recursive_dynamic_import(core, f"{adjusted_package_name}.{subname}", verbose)
|
|
59
|
+
if verbose:
|
|
60
|
+
print(f'Found {len(classes_to_import)} classes in `{package_name}` to import'"")
|
|
61
|
+
return classes_to_import
|
|
62
|
+
|
|
63
|
+
|
|
64
|
+
def load_local_modules(core, verbose: bool = False) -> list[tuple[str, Process | Step ]]:
|
|
65
|
+
if verbose:
|
|
66
|
+
print("Loading local registry...")
|
|
67
|
+
packages = importlib.metadata.distributions()
|
|
68
|
+
classes_to_load = []
|
|
69
|
+
for package in packages:
|
|
70
|
+
if not does_package_require_process_bigraph(package):
|
|
71
|
+
continue
|
|
72
|
+
# If a package requires BSail, it probably has abstractions for us; worth importing.
|
|
73
|
+
if verbose:
|
|
74
|
+
print(f'Relevant Processes found in `{package.name}`...')
|
|
75
|
+
classes_to_load += recursive_dynamic_import(core, package.name, verbose)
|
|
76
|
+
|
|
77
|
+
return classes_to_load
|
|
78
|
+
|
|
79
|
+
def does_package_require_process_bigraph(package: importlib.metadata.Distribution) -> bool:
|
|
80
|
+
for entry in ([] if package.requires is None else package.requires):
|
|
81
|
+
if "process-bigraph" in entry:
|
|
82
|
+
return True
|
|
83
|
+
return False
|
|
84
|
+
|
|
85
|
+
|
|
86
|
+
def discover_packages(core, verbose: bool) -> ProcessTypes:
|
|
87
|
+
# core = ProcessTypes()
|
|
88
|
+
counter = 0
|
|
89
|
+
for name, clazz in load_local_modules(core, verbose):
|
|
90
|
+
if name not in core.process_registry.registry:
|
|
91
|
+
core.register_process(name, clazz)
|
|
92
|
+
if verbose:
|
|
93
|
+
counter += 1
|
|
94
|
+
print(f'Registered `{name}` to `{clazz.__name__}`')
|
|
95
|
+
if verbose:
|
|
96
|
+
print(f"Registered {counter} processes...")
|
|
97
|
+
|
|
98
|
+
return core
|
|
99
|
+
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
"""
|
|
2
|
+
===============================================
|
|
3
|
+
Protocols for retrieving processes from address
|
|
4
|
+
===============================================
|
|
5
|
+
"""
|
|
6
|
+
|
|
7
|
+
from process_bigraph.protocols.local import local_lookup, LocalProtocol
|
|
8
|
+
from process_bigraph.protocols.parallel import ParallelProtocol
|
|
9
|
+
from process_bigraph.protocols.docker import DockerProtocol
|
|
10
|
+
from process_bigraph.protocols.rest import RestProtocol
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
BASE_PROTOCOLS = {
|
|
14
|
+
'local': LocalProtocol,
|
|
15
|
+
'parallel': ParallelProtocol,
|
|
16
|
+
'docker': DockerProtocol,
|
|
17
|
+
'rest': RestProtocol}
|
|
@@ -0,0 +1,262 @@
|
|
|
1
|
+
"""
|
|
2
|
+
===============================================
|
|
3
|
+
Protocol for running processes in parallel using
|
|
4
|
+
python multiprocessing
|
|
5
|
+
===============================================
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
import sys
|
|
9
|
+
import json
|
|
10
|
+
import time
|
|
11
|
+
import uuid
|
|
12
|
+
import pstats
|
|
13
|
+
import socket
|
|
14
|
+
from pathlib import Path
|
|
15
|
+
from typing import Any, Dict, Optional, Union, List, Tuple
|
|
16
|
+
|
|
17
|
+
import docker
|
|
18
|
+
|
|
19
|
+
from docker import DockerClient
|
|
20
|
+
|
|
21
|
+
from process_bigraph.composite import Process, SyncUpdate
|
|
22
|
+
from process_bigraph.protocols.protocol import Protocol
|
|
23
|
+
from process_bigraph.protocols.local import LocalProtocol
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
def setup_working_directory(path):
|
|
27
|
+
if not path.exists():
|
|
28
|
+
path.mkdir(
|
|
29
|
+
parents=True,
|
|
30
|
+
exist_ok=True)
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
DOCKER_WORKING_PATH = Path('containers')
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
def receive(sock, buffer_size=4096):
|
|
37
|
+
data = bytearray()
|
|
38
|
+
while True:
|
|
39
|
+
packet = sock.recv(buffer_size)
|
|
40
|
+
data.extend(packet)
|
|
41
|
+
if packet[-1] == 10:
|
|
42
|
+
break
|
|
43
|
+
return data
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
class DockerProcess(Process):
|
|
47
|
+
def __init__(self, client, data, config, core) -> None:
|
|
48
|
+
"""
|
|
49
|
+
Sets up a docker container to be a process and communicates with
|
|
50
|
+
it over a socket.
|
|
51
|
+
"""
|
|
52
|
+
|
|
53
|
+
self._ended = False
|
|
54
|
+
self._pending_command: Optional[
|
|
55
|
+
Tuple[str, Optional[tuple], Optional[dict]]] = None
|
|
56
|
+
|
|
57
|
+
self.client = client
|
|
58
|
+
self.data = data
|
|
59
|
+
self.image = self.data['image']
|
|
60
|
+
self.host = self.data.get('host', '0.0.0.0')
|
|
61
|
+
self.port = self.data.get('port', 11111)
|
|
62
|
+
|
|
63
|
+
self.uuid = str(uuid.uuid4())
|
|
64
|
+
self.work_path = DOCKER_WORKING_PATH / self.uuid
|
|
65
|
+
self.config_path = self.work_path / 'config'
|
|
66
|
+
|
|
67
|
+
setup_working_directory(self.config_path)
|
|
68
|
+
|
|
69
|
+
with open(self.config_path / 'config.json', 'w') as config_file:
|
|
70
|
+
json.dump(config, config_file)
|
|
71
|
+
|
|
72
|
+
self.container = self.client.containers.run(
|
|
73
|
+
self.image,
|
|
74
|
+
ports={f'{self.port}/tcp': str(self.port)},
|
|
75
|
+
volumes={
|
|
76
|
+
str(self.config_path.absolute()): {
|
|
77
|
+
'bind': '/config', 'mode': 'ro'}},
|
|
78
|
+
detach=True)
|
|
79
|
+
|
|
80
|
+
# wait to start the socket until the container is running (!)
|
|
81
|
+
while self.container.status != "running":
|
|
82
|
+
print(f"'{self.image}' status: {self.container.status} - waiting...")
|
|
83
|
+
time.sleep(1)
|
|
84
|
+
self.container.reload()
|
|
85
|
+
|
|
86
|
+
print(f'{self.image} now running')
|
|
87
|
+
|
|
88
|
+
self.socket = socket.socket(
|
|
89
|
+
socket.AF_INET,
|
|
90
|
+
socket.SOCK_STREAM)
|
|
91
|
+
|
|
92
|
+
self.socket.connect((
|
|
93
|
+
self.host,
|
|
94
|
+
self.port))
|
|
95
|
+
|
|
96
|
+
super().__init__(config, core=core)
|
|
97
|
+
|
|
98
|
+
def send_command(
|
|
99
|
+
self, command: str, args: Optional[tuple] = None,
|
|
100
|
+
kwargs: Optional[dict] = None,
|
|
101
|
+
run_pre_check: bool = True) -> None:
|
|
102
|
+
'''Send a command to the parallel process.
|
|
103
|
+
|
|
104
|
+
See :py:func:``_handle_parallel_process`` for details on how the
|
|
105
|
+
command will be handled.
|
|
106
|
+
'''
|
|
107
|
+
if run_pre_check:
|
|
108
|
+
self.pre_send_command(command, args, kwargs)
|
|
109
|
+
|
|
110
|
+
send = {
|
|
111
|
+
'command': command,
|
|
112
|
+
'arguments': {}}
|
|
113
|
+
|
|
114
|
+
if command == 'update':
|
|
115
|
+
state, interval = args
|
|
116
|
+
send['arguments'] = {
|
|
117
|
+
'state': state,
|
|
118
|
+
'interval': interval}
|
|
119
|
+
|
|
120
|
+
send_json = f'{json.dumps(send)}\n'.encode('utf-8')
|
|
121
|
+
|
|
122
|
+
print(f'sending {send_json}')
|
|
123
|
+
|
|
124
|
+
self.socket.sendall(
|
|
125
|
+
send_json)
|
|
126
|
+
|
|
127
|
+
def get_command_result(self):
|
|
128
|
+
"""Get the result of a command sent to the parallel process.
|
|
129
|
+
|
|
130
|
+
Commands and their results work like a queue, so unlike
|
|
131
|
+
:py:class:`Process`, you can technically call this method
|
|
132
|
+
multiple times and get different return values each time.
|
|
133
|
+
This behavior is subject to change, so you should not rely on
|
|
134
|
+
it.
|
|
135
|
+
|
|
136
|
+
Returns:
|
|
137
|
+
The command result.
|
|
138
|
+
"""
|
|
139
|
+
|
|
140
|
+
if not self._pending_command:
|
|
141
|
+
raise RuntimeError(
|
|
142
|
+
'Trying to retrieve command result, but no command is '
|
|
143
|
+
'pending.')
|
|
144
|
+
self._pending_command = None
|
|
145
|
+
|
|
146
|
+
result_json = receive(self.socket)
|
|
147
|
+
result = json.loads(result_json)
|
|
148
|
+
|
|
149
|
+
return result
|
|
150
|
+
|
|
151
|
+
def get(self):
|
|
152
|
+
return self.get_command_result()
|
|
153
|
+
|
|
154
|
+
@staticmethod
|
|
155
|
+
def generate_state(config=None):
|
|
156
|
+
"""Generate static initial state for user configuration or inspection."""
|
|
157
|
+
return {}
|
|
158
|
+
|
|
159
|
+
@property
|
|
160
|
+
def config(self) -> Dict[str, Any]:
|
|
161
|
+
return self._config
|
|
162
|
+
|
|
163
|
+
@config.setter
|
|
164
|
+
def config(self, config):
|
|
165
|
+
self._config = config
|
|
166
|
+
|
|
167
|
+
@property
|
|
168
|
+
def composition(self) -> Dict[str, Any]:
|
|
169
|
+
return {
|
|
170
|
+
'_type': 'process',
|
|
171
|
+
'_inputs': self.inputs(),
|
|
172
|
+
'_outputs': self.outputs()}
|
|
173
|
+
|
|
174
|
+
@composition.setter
|
|
175
|
+
def composition(self, composition):
|
|
176
|
+
pass
|
|
177
|
+
|
|
178
|
+
@property
|
|
179
|
+
def state(self) -> Dict[str, Any]:
|
|
180
|
+
return self
|
|
181
|
+
|
|
182
|
+
@state.setter
|
|
183
|
+
def state(self, state):
|
|
184
|
+
pass
|
|
185
|
+
|
|
186
|
+
def initial_state(self):
|
|
187
|
+
"""Return initial state values, if applicable."""
|
|
188
|
+
return {}
|
|
189
|
+
|
|
190
|
+
def inputs(self):
|
|
191
|
+
"""
|
|
192
|
+
Return a dictionary mapping input port names to bigraph types.
|
|
193
|
+
|
|
194
|
+
Example:
|
|
195
|
+
{'glucose': 'float', 'biomass': 'map[float]'}
|
|
196
|
+
"""
|
|
197
|
+
return self.run_command('inputs', ())
|
|
198
|
+
|
|
199
|
+
def outputs(self):
|
|
200
|
+
"""
|
|
201
|
+
Return a dictionary mapping output port names to bigraph types.
|
|
202
|
+
|
|
203
|
+
Example:
|
|
204
|
+
{'growth_rate': 'float'}
|
|
205
|
+
"""
|
|
206
|
+
return self.run_command('outputs', ())
|
|
207
|
+
|
|
208
|
+
def invoke(self, state, interval):
|
|
209
|
+
result = self.run_command('update', (state, interval))
|
|
210
|
+
return SyncUpdate(result)
|
|
211
|
+
|
|
212
|
+
def update(self, state, interval):
|
|
213
|
+
return self.run_command('update', (state, interval))
|
|
214
|
+
|
|
215
|
+
def end(self) -> None:
|
|
216
|
+
"""
|
|
217
|
+
remove the container
|
|
218
|
+
"""
|
|
219
|
+
# Only end once.
|
|
220
|
+
if self._ended:
|
|
221
|
+
return
|
|
222
|
+
|
|
223
|
+
self.socket.close()
|
|
224
|
+
|
|
225
|
+
self.container.stop()
|
|
226
|
+
self.container.remove()
|
|
227
|
+
|
|
228
|
+
self._ended = True
|
|
229
|
+
|
|
230
|
+
def __del__(self) -> None:
|
|
231
|
+
self.end()
|
|
232
|
+
|
|
233
|
+
def initialize_docker():
|
|
234
|
+
client: DockerClient = None
|
|
235
|
+
try:
|
|
236
|
+
client = docker.from_env()
|
|
237
|
+
except Exception:
|
|
238
|
+
pass # We handle this by checking if client is None elsewhere.
|
|
239
|
+
return client
|
|
240
|
+
|
|
241
|
+
class DockerProtocol(Protocol):
|
|
242
|
+
client = initialize_docker()
|
|
243
|
+
|
|
244
|
+
@classmethod
|
|
245
|
+
def interface(cls, core, data):
|
|
246
|
+
if cls.client is None:
|
|
247
|
+
raise NotImplementedError("Docker was unable to be initialized; check your installation and try again.")
|
|
248
|
+
image = cls.client.images.get(
|
|
249
|
+
data['image'])
|
|
250
|
+
|
|
251
|
+
config_schema = json.loads(
|
|
252
|
+
image.labels['config_schema'])
|
|
253
|
+
|
|
254
|
+
def instantiate(config, core=None):
|
|
255
|
+
return DockerProcess(
|
|
256
|
+
cls.client,
|
|
257
|
+
data,
|
|
258
|
+
config,
|
|
259
|
+
core)
|
|
260
|
+
|
|
261
|
+
instantiate.config_schema = config_schema
|
|
262
|
+
return instantiate
|
|
@@ -0,0 +1,43 @@
|
|
|
1
|
+
"""
|
|
2
|
+
===============================================
|
|
3
|
+
Protocol for processes running locally
|
|
4
|
+
===============================================
|
|
5
|
+
"""
|
|
6
|
+
|
|
7
|
+
from bigraph_schema.protocols import local_lookup_module
|
|
8
|
+
from process_bigraph.protocols.protocol import Protocol
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
def local_lookup_registry(core, address):
|
|
12
|
+
"""Process Registry Protocol
|
|
13
|
+
|
|
14
|
+
Retrieves from the process registry
|
|
15
|
+
"""
|
|
16
|
+
return core.process_registry.access(address)
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
def local_lookup(core, address):
|
|
20
|
+
"""Local Lookup Protocol
|
|
21
|
+
|
|
22
|
+
Retrieves local processes, from the process registry or from a local module
|
|
23
|
+
"""
|
|
24
|
+
if address[0] == '!':
|
|
25
|
+
instantiate = local_lookup_module(address[1:])
|
|
26
|
+
else:
|
|
27
|
+
instantiate = local_lookup_registry(core, address)
|
|
28
|
+
return instantiate
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
class LocalProtocol(Protocol):
|
|
32
|
+
@staticmethod
|
|
33
|
+
def interface(core, address):
|
|
34
|
+
if isinstance(address, str):
|
|
35
|
+
return local_lookup(core, address)
|
|
36
|
+
elif isinstance(address, dict):
|
|
37
|
+
if 'address' not in address:
|
|
38
|
+
raise Exception(f'must include address in local protocol: {address}')
|
|
39
|
+
else:
|
|
40
|
+
return local_lookup(core, address['address'])
|
|
41
|
+
else:
|
|
42
|
+
raise Exception(f'address must be str or dict, not {address}')
|
|
43
|
+
|
|
@@ -0,0 +1,241 @@
|
|
|
1
|
+
"""
|
|
2
|
+
===============================================
|
|
3
|
+
Protocol for running processes in parallel using
|
|
4
|
+
python multiprocessing
|
|
5
|
+
===============================================
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
import sys
|
|
9
|
+
import pstats
|
|
10
|
+
from typing import Any, Dict, Optional, Union, List, Tuple
|
|
11
|
+
|
|
12
|
+
import multiprocessing
|
|
13
|
+
from multiprocessing.connection import Connection
|
|
14
|
+
|
|
15
|
+
from process_bigraph.composite import Process
|
|
16
|
+
from process_bigraph.protocols.protocol import Protocol
|
|
17
|
+
from process_bigraph.protocols.local import LocalProtocol
|
|
18
|
+
|
|
19
|
+
def _handle_parallel_process(
|
|
20
|
+
connection: Connection, process: Process,
|
|
21
|
+
profile: bool) -> None:
|
|
22
|
+
'''Handle a parallel Vivarium :term:`process`.
|
|
23
|
+
|
|
24
|
+
This function is designed to be passed as ``target`` to
|
|
25
|
+
``Multiprocess()``. In a loop, it receives :term:`process commands`
|
|
26
|
+
from a pipe, passes those commands to the parallel process, and
|
|
27
|
+
passes the result back along the pipe.
|
|
28
|
+
|
|
29
|
+
The special command ``end`` is handled directly by this function.
|
|
30
|
+
This command causes the function to exit and therefore shut down the
|
|
31
|
+
OS process created by multiprocessing.
|
|
32
|
+
|
|
33
|
+
Args:
|
|
34
|
+
connection: The child end of a multiprocessing pipe. All
|
|
35
|
+
communications received from the pipe should be a 3-tuple of
|
|
36
|
+
the form ``(command, args, kwargs)``, and the tuple contents
|
|
37
|
+
will be passed to :py:meth:`Process.run_command`. The
|
|
38
|
+
result, which may be of any type, will be sent back through
|
|
39
|
+
the pipe.
|
|
40
|
+
process: The process running in parallel.
|
|
41
|
+
profile: Whether to profile the process.
|
|
42
|
+
'''
|
|
43
|
+
if profile:
|
|
44
|
+
profiler = cProfile.Profile()
|
|
45
|
+
profiler.enable()
|
|
46
|
+
running = True
|
|
47
|
+
|
|
48
|
+
while running:
|
|
49
|
+
command, args, kwargs = connection.recv()
|
|
50
|
+
|
|
51
|
+
if command == 'end':
|
|
52
|
+
running = False
|
|
53
|
+
else:
|
|
54
|
+
result = process.run_command(command, args, kwargs)
|
|
55
|
+
connection.send(result)
|
|
56
|
+
|
|
57
|
+
if profile:
|
|
58
|
+
profiler.disable()
|
|
59
|
+
stats = pstats.Stats(profiler)
|
|
60
|
+
connection.send(stats.stats) # type: ignore
|
|
61
|
+
|
|
62
|
+
connection.close()
|
|
63
|
+
|
|
64
|
+
|
|
65
|
+
class ParallelProcess(Process):
|
|
66
|
+
def __init__(
|
|
67
|
+
self, process: Process, profile: bool = False,
|
|
68
|
+
stats_objs: Optional[List[pstats.Stats]] = None) -> None:
|
|
69
|
+
"""Wraps a :py:class:`Process` for multiprocessing.
|
|
70
|
+
|
|
71
|
+
To run a simulation distributed across multiple processors, we
|
|
72
|
+
use Python's multiprocessing tools. This object runs in the main
|
|
73
|
+
process and manages communication between the main (parent)
|
|
74
|
+
process and the child process with the :py:class:`Process` that
|
|
75
|
+
this object manages.
|
|
76
|
+
|
|
77
|
+
Most methods pass their name and arguments to
|
|
78
|
+
:py:class:`Process.run_command`.
|
|
79
|
+
|
|
80
|
+
Args:
|
|
81
|
+
process: The Process to manage.
|
|
82
|
+
profile: Whether to use cProfile to profile the subprocess.
|
|
83
|
+
stats_objs: List to add cProfile stats objs to when process
|
|
84
|
+
is deleted. Only used if ``profile`` is true.
|
|
85
|
+
"""
|
|
86
|
+
|
|
87
|
+
self._ended = False
|
|
88
|
+
self._pending_command: Optional[
|
|
89
|
+
Tuple[str, Optional[tuple], Optional[dict]]] = None
|
|
90
|
+
self.process = process
|
|
91
|
+
|
|
92
|
+
super().__init__(process.config, core=process.core)
|
|
93
|
+
|
|
94
|
+
self.profile = profile
|
|
95
|
+
self._stats_objs = stats_objs
|
|
96
|
+
assert not self.profile or self._stats_objs is not None
|
|
97
|
+
# Linux's default ``fork`` start method causes a lot of random
|
|
98
|
+
# issues, including python/cpython#110770 (prompted this change)
|
|
99
|
+
# and python/cpython#84559 (general discussion). This default
|
|
100
|
+
# will be changed to ``forkserver`` in Python 3.14. MacOS and
|
|
101
|
+
# Windows use the much safer but slightly slower ``spawn`` method
|
|
102
|
+
if sys.platform not in ("darwin", "win32"):
|
|
103
|
+
start_method = "forkserver"
|
|
104
|
+
else:
|
|
105
|
+
start_method = "spawn"
|
|
106
|
+
mp_ctx = multiprocessing.get_context(start_method)
|
|
107
|
+
self.parent, child = mp_ctx.Pipe()
|
|
108
|
+
self.multiprocess = mp_ctx.Process( # type: ignore[attr-defined]
|
|
109
|
+
target=_handle_parallel_process,
|
|
110
|
+
args=(child, process, self.profile))
|
|
111
|
+
self.multiprocess.start()
|
|
112
|
+
|
|
113
|
+
def send_command(
|
|
114
|
+
self, command: str, args: Optional[tuple] = None,
|
|
115
|
+
kwargs: Optional[dict] = None,
|
|
116
|
+
run_pre_check: bool = True) -> None:
|
|
117
|
+
'''Send a command to the parallel process.
|
|
118
|
+
|
|
119
|
+
See :py:func:``_handle_parallel_process`` for details on how the
|
|
120
|
+
command will be handled.
|
|
121
|
+
'''
|
|
122
|
+
if run_pre_check:
|
|
123
|
+
self.pre_send_command(command, args, kwargs)
|
|
124
|
+
self.parent.send((command, args, kwargs))
|
|
125
|
+
|
|
126
|
+
def get_command_result(self):
|
|
127
|
+
"""Get the result of a command sent to the parallel process.
|
|
128
|
+
|
|
129
|
+
Commands and their results work like a queue, so unlike
|
|
130
|
+
:py:class:`Process`, you can technically call this method
|
|
131
|
+
multiple times and get different return values each time.
|
|
132
|
+
This behavior is subject to change, so you should not rely on
|
|
133
|
+
it.
|
|
134
|
+
|
|
135
|
+
Returns:
|
|
136
|
+
The command result.
|
|
137
|
+
"""
|
|
138
|
+
if not self._pending_command:
|
|
139
|
+
raise RuntimeError(
|
|
140
|
+
'Trying to retrieve command result, but no command is '
|
|
141
|
+
'pending.')
|
|
142
|
+
self._pending_command = None
|
|
143
|
+
return self.parent.recv()
|
|
144
|
+
|
|
145
|
+
def get(self):
|
|
146
|
+
return self.get_command_result()
|
|
147
|
+
|
|
148
|
+
@staticmethod
|
|
149
|
+
def generate_state(config=None):
|
|
150
|
+
"""Generate static initial state for user configuration or inspection."""
|
|
151
|
+
return type(self.process).generate_state(config)
|
|
152
|
+
|
|
153
|
+
@property
|
|
154
|
+
def config(self) -> Dict[str, Any]:
|
|
155
|
+
return self.run_command('config')
|
|
156
|
+
|
|
157
|
+
@config.setter
|
|
158
|
+
def config(self, config):
|
|
159
|
+
self.run_command('set_config', (config,))
|
|
160
|
+
|
|
161
|
+
@property
|
|
162
|
+
def composition(self) -> Dict[str, Any]:
|
|
163
|
+
return self.run_command('composition')
|
|
164
|
+
|
|
165
|
+
@composition.setter
|
|
166
|
+
def composition(self, composition):
|
|
167
|
+
self.run_command('set_composition', (composition,))
|
|
168
|
+
|
|
169
|
+
@property
|
|
170
|
+
def state(self) -> Dict[str, Any]:
|
|
171
|
+
return self.run_command('state')
|
|
172
|
+
|
|
173
|
+
@state.setter
|
|
174
|
+
def state(self, state):
|
|
175
|
+
self.run_command('set_state', (state,))
|
|
176
|
+
|
|
177
|
+
def initial_state(self):
|
|
178
|
+
"""Return initial state values, if applicable."""
|
|
179
|
+
return self.run_command('initial_state', ())
|
|
180
|
+
|
|
181
|
+
def inputs(self):
|
|
182
|
+
"""
|
|
183
|
+
Return a dictionary mapping input port names to bigraph types.
|
|
184
|
+
|
|
185
|
+
Example:
|
|
186
|
+
{'glucose': 'float', 'biomass': 'map[float]'}
|
|
187
|
+
"""
|
|
188
|
+
# return self.run_command('inputs', ())
|
|
189
|
+
return self.process.inputs()
|
|
190
|
+
|
|
191
|
+
def outputs(self):
|
|
192
|
+
"""
|
|
193
|
+
Return a dictionary mapping output port names to bigraph types.
|
|
194
|
+
|
|
195
|
+
Example:
|
|
196
|
+
{'growth_rate': 'float'}
|
|
197
|
+
"""
|
|
198
|
+
return self.process.outputs()
|
|
199
|
+
# return self.run_command('outputs', ())
|
|
200
|
+
|
|
201
|
+
def invoke(self, state, interval):
|
|
202
|
+
self.send_command('update', (state, interval))
|
|
203
|
+
return self
|
|
204
|
+
|
|
205
|
+
def update(self, state, interval):
|
|
206
|
+
return self.run_command('update', (state, interval))
|
|
207
|
+
|
|
208
|
+
def end(self) -> None:
|
|
209
|
+
"""End the child process.
|
|
210
|
+
|
|
211
|
+
If profiling was enabled, then when the child process ends, it
|
|
212
|
+
will compile its profiling stats and send those to the parent.
|
|
213
|
+
The parent then saves those stats in ``self.stats``.
|
|
214
|
+
"""
|
|
215
|
+
# Only end once.
|
|
216
|
+
if self._ended:
|
|
217
|
+
return
|
|
218
|
+
self.send_command('end')
|
|
219
|
+
if self.profile:
|
|
220
|
+
stats = pstats.Stats()
|
|
221
|
+
stats.stats = self.get_command_result() # type: ignore
|
|
222
|
+
assert self._stats_objs is not None
|
|
223
|
+
self._stats_objs.append(stats)
|
|
224
|
+
self.multiprocess.join()
|
|
225
|
+
self.multiprocess.close()
|
|
226
|
+
self._ended = True
|
|
227
|
+
|
|
228
|
+
def __del__(self) -> None:
|
|
229
|
+
self.end()
|
|
230
|
+
|
|
231
|
+
|
|
232
|
+
class ParallelProtocol(Protocol):
|
|
233
|
+
@staticmethod
|
|
234
|
+
def interface(core, address):
|
|
235
|
+
local_instantiate = LocalProtocol.interface(core, address)
|
|
236
|
+
def instantiate(config, core=None):
|
|
237
|
+
instance = local_instantiate(config, core=core)
|
|
238
|
+
return ParallelProcess(instance)
|
|
239
|
+
|
|
240
|
+
instantiate.config_schema = local_instantiate.config_schema
|
|
241
|
+
return instantiate
|
|
@@ -0,0 +1,171 @@
|
|
|
1
|
+
"""
|
|
2
|
+
===============================================
|
|
3
|
+
Protocol for running processes in parallel using
|
|
4
|
+
python multiprocessing
|
|
5
|
+
===============================================
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
import sys
|
|
9
|
+
import json
|
|
10
|
+
import time
|
|
11
|
+
import uuid
|
|
12
|
+
import pstats
|
|
13
|
+
import socket
|
|
14
|
+
from pathlib import Path
|
|
15
|
+
from typing import Any, Dict, Optional, Union, List, Tuple
|
|
16
|
+
|
|
17
|
+
from urllib.parse import urlparse, urlunparse
|
|
18
|
+
import requests
|
|
19
|
+
|
|
20
|
+
from process_bigraph.composite import Process, SyncUpdate
|
|
21
|
+
from process_bigraph.protocols.protocol import Protocol
|
|
22
|
+
from process_bigraph.protocols.local import LocalProtocol
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
def rest_get(url, parameters=None):
|
|
26
|
+
return requests.get(
|
|
27
|
+
urlunparse(url),
|
|
28
|
+
json=parameters).json()
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
def rest_post(url, parameters=None):
|
|
32
|
+
return requests.post(
|
|
33
|
+
urlunparse(url),
|
|
34
|
+
json=parameters).json()
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
class RestProcess(Process):
|
|
38
|
+
def __init__(self, data, config, core) -> None:
|
|
39
|
+
self._ended = False
|
|
40
|
+
|
|
41
|
+
self.process_name = data['process']
|
|
42
|
+
|
|
43
|
+
self.base_url = data['base_url'] or urlparse('http://localhost:22222')
|
|
44
|
+
|
|
45
|
+
self.initialize_url = self.base_url._replace(
|
|
46
|
+
path=f'/process/{self.process_name}/initialize')
|
|
47
|
+
self.process_id = rest_post(
|
|
48
|
+
self.initialize_url,
|
|
49
|
+
config)
|
|
50
|
+
|
|
51
|
+
self.end_url = self.base_url._replace(
|
|
52
|
+
path=f'/process/{self.process_name}/end/{self.process_id}')
|
|
53
|
+
self.inputs_url = self.base_url._replace(
|
|
54
|
+
path=f'/process/{self.process_name}/inputs/{self.process_id}')
|
|
55
|
+
self.outputs_url = self.base_url._replace(
|
|
56
|
+
path=f'/process/{self.process_name}/outputs/{self.process_id}')
|
|
57
|
+
self.update_url = self.base_url._replace(
|
|
58
|
+
path=f'/process/{self.process_name}/update/{self.process_id}')
|
|
59
|
+
|
|
60
|
+
super().__init__(config, core=core)
|
|
61
|
+
|
|
62
|
+
def get(self):
|
|
63
|
+
return self.get_command_result()
|
|
64
|
+
|
|
65
|
+
@staticmethod
|
|
66
|
+
def generate_state(config=None):
|
|
67
|
+
"""Generate static initial state for user configuration or inspection."""
|
|
68
|
+
return {}
|
|
69
|
+
|
|
70
|
+
@property
|
|
71
|
+
def config(self) -> Dict[str, Any]:
|
|
72
|
+
return self._config
|
|
73
|
+
|
|
74
|
+
@config.setter
|
|
75
|
+
def config(self, config):
|
|
76
|
+
self._config = config
|
|
77
|
+
|
|
78
|
+
@property
|
|
79
|
+
def composition(self) -> Dict[str, Any]:
|
|
80
|
+
return {
|
|
81
|
+
'_type': 'process',
|
|
82
|
+
'_inputs': self.inputs(),
|
|
83
|
+
'_outputs': self.outputs()}
|
|
84
|
+
|
|
85
|
+
@composition.setter
|
|
86
|
+
def composition(self, composition):
|
|
87
|
+
pass
|
|
88
|
+
|
|
89
|
+
@property
|
|
90
|
+
def state(self) -> Dict[str, Any]:
|
|
91
|
+
return self
|
|
92
|
+
|
|
93
|
+
@state.setter
|
|
94
|
+
def state(self, state):
|
|
95
|
+
pass
|
|
96
|
+
|
|
97
|
+
def initial_state(self):
|
|
98
|
+
"""Return initial state values, if applicable."""
|
|
99
|
+
return {}
|
|
100
|
+
|
|
101
|
+
def inputs(self):
|
|
102
|
+
"""
|
|
103
|
+
Return a dictionary mapping input port names to bigraph types.
|
|
104
|
+
|
|
105
|
+
Example:
|
|
106
|
+
{'glucose': 'float', 'biomass': 'map[float]'}
|
|
107
|
+
"""
|
|
108
|
+
response = rest_get(self.inputs_url)
|
|
109
|
+
|
|
110
|
+
return response
|
|
111
|
+
|
|
112
|
+
def outputs(self):
|
|
113
|
+
"""
|
|
114
|
+
Return a dictionary mapping output port names to bigraph types.
|
|
115
|
+
|
|
116
|
+
Example:
|
|
117
|
+
{'growth_rate': 'float'}
|
|
118
|
+
"""
|
|
119
|
+
response = rest_get(self.outputs_url)
|
|
120
|
+
|
|
121
|
+
return response
|
|
122
|
+
|
|
123
|
+
def update(self, state, interval):
|
|
124
|
+
response = rest_post(self.update_url, {
|
|
125
|
+
'state': state,
|
|
126
|
+
'interval': interval})
|
|
127
|
+
return response
|
|
128
|
+
|
|
129
|
+
def end(self) -> None:
|
|
130
|
+
"""
|
|
131
|
+
remove the container
|
|
132
|
+
"""
|
|
133
|
+
# Only end once.
|
|
134
|
+
if self._ended:
|
|
135
|
+
return
|
|
136
|
+
|
|
137
|
+
rest_post(self.end_url)
|
|
138
|
+
|
|
139
|
+
self._ended = True
|
|
140
|
+
|
|
141
|
+
def __del__(self) -> None:
|
|
142
|
+
self.end()
|
|
143
|
+
|
|
144
|
+
|
|
145
|
+
class RestProtocol(Protocol):
|
|
146
|
+
@classmethod
|
|
147
|
+
def interface(cls, core, data):
|
|
148
|
+
ssh = ''
|
|
149
|
+
process_name = data['process']
|
|
150
|
+
host = data['host']
|
|
151
|
+
port = data['port']
|
|
152
|
+
base_raw = f'http{ssh}://{host}:{port}'
|
|
153
|
+
base_url = urlparse(base_raw)
|
|
154
|
+
|
|
155
|
+
config_schema_url = base_url._replace(
|
|
156
|
+
path=f'/process/{process_name}/config-schema')
|
|
157
|
+
config_schema = rest_get(
|
|
158
|
+
config_schema_url)
|
|
159
|
+
|
|
160
|
+
instance = {
|
|
161
|
+
'base_url': base_url,
|
|
162
|
+
'process': process_name}
|
|
163
|
+
|
|
164
|
+
def instantiate(config, core=None):
|
|
165
|
+
return RestProcess(
|
|
166
|
+
instance,
|
|
167
|
+
config,
|
|
168
|
+
core)
|
|
169
|
+
|
|
170
|
+
instantiate.config_schema = config_schema
|
|
171
|
+
return instantiate
|
|
@@ -27,6 +27,15 @@ process_bigraph.egg-info/requires.txt
|
|
|
27
27
|
process_bigraph.egg-info/top_level.txt
|
|
28
28
|
process_bigraph/experiments/__init__.py
|
|
29
29
|
process_bigraph/experiments/minimal_gillespie.py
|
|
30
|
+
process_bigraph/package/__init__.py
|
|
31
|
+
process_bigraph/package/discover.py
|
|
30
32
|
process_bigraph/processes/__init__.py
|
|
31
33
|
process_bigraph/processes/growth_division.py
|
|
32
|
-
process_bigraph/processes/parameter_scan.py
|
|
34
|
+
process_bigraph/processes/parameter_scan.py
|
|
35
|
+
process_bigraph/protocols/__init__.py
|
|
36
|
+
process_bigraph/protocols/docker.py
|
|
37
|
+
process_bigraph/protocols/local.py
|
|
38
|
+
process_bigraph/protocols/parallel.py
|
|
39
|
+
process_bigraph/protocols/protocol.py
|
|
40
|
+
process_bigraph/protocols/rest.py
|
|
41
|
+
process_bigraph/protocols/socket.py
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
[project]
|
|
2
2
|
name = "process-bigraph"
|
|
3
|
-
version = "0.0.
|
|
3
|
+
version = "0.0.46"
|
|
4
4
|
description = "protocol and execution for compositional systems biology"
|
|
5
5
|
readme = "README.md"
|
|
6
6
|
requires-python = ">=3.11"
|
|
@@ -16,7 +16,10 @@ dependencies = [
|
|
|
16
16
|
[tool.setuptools]
|
|
17
17
|
packages = [
|
|
18
18
|
"process_bigraph",
|
|
19
|
-
"process_bigraph.processes"
|
|
19
|
+
"process_bigraph.processes",
|
|
20
|
+
"process_bigraph.protocols",
|
|
21
|
+
"process_bigraph.experiments",
|
|
22
|
+
"process_bigraph.package"
|
|
20
23
|
]
|
|
21
24
|
|
|
22
25
|
[tool.uv.sources]
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{process_bigraph-0.0.44 → process_bigraph-0.0.46}/process_bigraph/experiments/minimal_gillespie.py
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
{process_bigraph-0.0.44 → process_bigraph-0.0.46}/process_bigraph/processes/growth_division.py
RENAMED
|
File without changes
|
{process_bigraph-0.0.44 → process_bigraph-0.0.46}/process_bigraph/processes/parameter_scan.py
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
{process_bigraph-0.0.44 → process_bigraph-0.0.46}/process_bigraph.egg-info/dependency_links.txt
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|