manta-node 0.5b0.dev9__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- manta_node/__init__.py +5 -0
- manta_node/__main__.py +57 -0
- manta_node/cli/__init__.py +6 -0
- manta_node/cli/commands/__init__.py +21 -0
- manta_node/cli/commands/cluster.py +419 -0
- manta_node/cli/commands/config.py +490 -0
- manta_node/cli/commands/logs.py +168 -0
- manta_node/cli/commands/start.py +459 -0
- manta_node/cli/commands/status.py +204 -0
- manta_node/cli/commands/stop.py +253 -0
- manta_node/cli/config_manager.py +139 -0
- manta_node/cli/main.py +133 -0
- manta_node/cli/version.py +106 -0
- manta_node/domain/__init__.py +8 -0
- manta_node/domain/task_lifecycle.py +83 -0
- manta_node/infrastructure/__init__.py +13 -0
- manta_node/infrastructure/config/__init__.py +31 -0
- manta_node/infrastructure/config/node_config_manager.py +918 -0
- manta_node/infrastructure/container/__init__.py +9 -0
- manta_node/infrastructure/container/docker_adapter.py +253 -0
- manta_node/infrastructure/container/manager.py +536 -0
- manta_node/infrastructure/filesystem/__init__.py +7 -0
- manta_node/infrastructure/filesystem/dataset_manager.py +469 -0
- manta_node/infrastructure/grpc/__init__.py +11 -0
- manta_node/infrastructure/grpc/client.py +373 -0
- manta_node/infrastructure/grpc/local_servicer.py +151 -0
- manta_node/infrastructure/grpc/world_servicer.py +284 -0
- manta_node/infrastructure/metrics/__init__.py +10 -0
- manta_node/infrastructure/metrics/collector.py +94 -0
- manta_node/infrastructure/metrics/metrics_collector.py +351 -0
- manta_node/infrastructure/mqtt/__init__.py +7 -0
- manta_node/infrastructure/mqtt/command_handler.py +315 -0
- manta_node/node_orchestrator.py +462 -0
- manta_node/task_manager.py +519 -0
- manta_node/tasks.py +272 -0
- manta_node/utils.py +52 -0
- manta_node-0.5b0.dev9.dist-info/METADATA +799 -0
- manta_node-0.5b0.dev9.dist-info/RECORD +42 -0
- manta_node-0.5b0.dev9.dist-info/WHEEL +5 -0
- manta_node-0.5b0.dev9.dist-info/entry_points.txt +2 -0
- manta_node-0.5b0.dev9.dist-info/licenses/LICENSE +683 -0
- manta_node-0.5b0.dev9.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,253 @@
|
|
|
1
|
+
from datetime import datetime
|
|
2
|
+
from typing import Optional
|
|
3
|
+
|
|
4
|
+
from aiodocker import Docker
|
|
5
|
+
from aiodocker.docker import DockerContainer
|
|
6
|
+
from docker import DockerClient
|
|
7
|
+
|
|
8
|
+
from manta_common.errors import MantaDockerError
|
|
9
|
+
from manta_common.traces import Tracer
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
class DockerRaiser:
|
|
13
|
+
async_client: Docker
|
|
14
|
+
tracer: Tracer
|
|
15
|
+
docker_client: DockerClient
|
|
16
|
+
|
|
17
|
+
async def run_or_raise(self, config: dict) -> DockerContainer:
|
|
18
|
+
"""
|
|
19
|
+
Start, pull or run a container
|
|
20
|
+
|
|
21
|
+
Parameters
|
|
22
|
+
----------
|
|
23
|
+
config : dict
|
|
24
|
+
Configuration
|
|
25
|
+
|
|
26
|
+
Returns
|
|
27
|
+
-------
|
|
28
|
+
DockerContainer
|
|
29
|
+
Container
|
|
30
|
+
|
|
31
|
+
Raises
|
|
32
|
+
------
|
|
33
|
+
MantaDockerError
|
|
34
|
+
Cannot run container
|
|
35
|
+
"""
|
|
36
|
+
container = None
|
|
37
|
+
try:
|
|
38
|
+
container = await self.async_client.containers.run(config)
|
|
39
|
+
except Exception as exc:
|
|
40
|
+
self.tracer.error("Task cannot be started")
|
|
41
|
+
raise MantaDockerError(
|
|
42
|
+
"Task cannot be started", metadata=self.tracer.metadata
|
|
43
|
+
) from exc
|
|
44
|
+
return container
|
|
45
|
+
|
|
46
|
+
async def pull_image_or_raise(self, image: str):
|
|
47
|
+
"""
|
|
48
|
+
Pull an image
|
|
49
|
+
|
|
50
|
+
Parameters
|
|
51
|
+
----------
|
|
52
|
+
image : str
|
|
53
|
+
Image to pull
|
|
54
|
+
|
|
55
|
+
Raises
|
|
56
|
+
------
|
|
57
|
+
MantaDockerError
|
|
58
|
+
Cannot pull image
|
|
59
|
+
"""
|
|
60
|
+
try:
|
|
61
|
+
await self.async_client.images.pull(image)
|
|
62
|
+
except Exception as exc:
|
|
63
|
+
self.tracer.error(f"Image {image} cannot be pulled")
|
|
64
|
+
raise MantaDockerError(
|
|
65
|
+
f"Image {image} cannot be pulled", metadata=self.tracer.metadata
|
|
66
|
+
) from exc
|
|
67
|
+
|
|
68
|
+
async def list_containers_or_raise(self, all_containers: bool = True) -> list:
|
|
69
|
+
"""
|
|
70
|
+
List all containers in the docker engine
|
|
71
|
+
|
|
72
|
+
Parameters
|
|
73
|
+
----------
|
|
74
|
+
all_containers : bool, optional
|
|
75
|
+
Show all containers; only running containers are shown by default
|
|
76
|
+
|
|
77
|
+
Returns
|
|
78
|
+
-------
|
|
79
|
+
list
|
|
80
|
+
List of containers
|
|
81
|
+
|
|
82
|
+
Raises
|
|
83
|
+
------
|
|
84
|
+
MantaDockerError
|
|
85
|
+
Cannot list containers
|
|
86
|
+
"""
|
|
87
|
+
containers = []
|
|
88
|
+
try:
|
|
89
|
+
containers = await self.async_client.containers.list(all=all_containers)
|
|
90
|
+
except Exception as exc:
|
|
91
|
+
self.tracer.error("Unable to list containers")
|
|
92
|
+
raise MantaDockerError(
|
|
93
|
+
"Unable to list containers", metadata=self.tracer.metadata
|
|
94
|
+
) from exc
|
|
95
|
+
return containers
|
|
96
|
+
|
|
97
|
+
def list_images_or_raise(self) -> list:
|
|
98
|
+
"""
|
|
99
|
+
List all images in the docker engine
|
|
100
|
+
|
|
101
|
+
Returns
|
|
102
|
+
-------
|
|
103
|
+
list
|
|
104
|
+
List of images
|
|
105
|
+
|
|
106
|
+
Raises
|
|
107
|
+
------
|
|
108
|
+
MantaDockerError
|
|
109
|
+
Cannot list images
|
|
110
|
+
"""
|
|
111
|
+
images = []
|
|
112
|
+
try:
|
|
113
|
+
images = [
|
|
114
|
+
tag for image in self.docker_client.images.list() for tag in image.tags
|
|
115
|
+
]
|
|
116
|
+
except Exception as exc:
|
|
117
|
+
self.tracer.error("Unable to list images")
|
|
118
|
+
raise MantaDockerError(
|
|
119
|
+
"Unable to list images", metadata=self.tracer.metadata
|
|
120
|
+
) from exc
|
|
121
|
+
return images
|
|
122
|
+
|
|
123
|
+
async def restart_or_raise(self, container: DockerContainer):
|
|
124
|
+
"""
|
|
125
|
+
Restart container
|
|
126
|
+
|
|
127
|
+
Parameters
|
|
128
|
+
----------
|
|
129
|
+
container : DockerContainer
|
|
130
|
+
Container
|
|
131
|
+
|
|
132
|
+
Raises
|
|
133
|
+
------
|
|
134
|
+
MantaDockerError
|
|
135
|
+
Cannot restart container
|
|
136
|
+
"""
|
|
137
|
+
try:
|
|
138
|
+
await container.restart()
|
|
139
|
+
except Exception as exc:
|
|
140
|
+
self.tracer.error("Container cannot be restarted")
|
|
141
|
+
raise MantaDockerError(
|
|
142
|
+
"Container cannot be restart", metadata=self.tracer.metadata
|
|
143
|
+
) from exc
|
|
144
|
+
|
|
145
|
+
async def stop_or_raise(self, container: DockerContainer):
|
|
146
|
+
"""
|
|
147
|
+
Stop container
|
|
148
|
+
|
|
149
|
+
Parameters
|
|
150
|
+
----------
|
|
151
|
+
container : DockerContainer
|
|
152
|
+
Container
|
|
153
|
+
|
|
154
|
+
Raises
|
|
155
|
+
------
|
|
156
|
+
MantaDockerError
|
|
157
|
+
Cannot stop container
|
|
158
|
+
"""
|
|
159
|
+
try:
|
|
160
|
+
await container.stop()
|
|
161
|
+
except Exception as exc:
|
|
162
|
+
self.tracer.error("Container cannot be stopped")
|
|
163
|
+
raise MantaDockerError(
|
|
164
|
+
"Container cannot be stopped", metadata=self.tracer.metadata
|
|
165
|
+
) from exc
|
|
166
|
+
|
|
167
|
+
async def remove_or_raise(self, container: DockerContainer):
|
|
168
|
+
"""
|
|
169
|
+
Remove container
|
|
170
|
+
|
|
171
|
+
Parameters
|
|
172
|
+
----------
|
|
173
|
+
container : DockerContainer
|
|
174
|
+
Container
|
|
175
|
+
|
|
176
|
+
Raises
|
|
177
|
+
------
|
|
178
|
+
MantaDockerError
|
|
179
|
+
Cannot remove container
|
|
180
|
+
"""
|
|
181
|
+
try:
|
|
182
|
+
await container.delete()
|
|
183
|
+
except Exception as exc:
|
|
184
|
+
self.tracer.error("Container cannot be removed")
|
|
185
|
+
raise MantaDockerError(
|
|
186
|
+
"Container cannot be removed", metadata=self.tracer.metadata
|
|
187
|
+
) from exc
|
|
188
|
+
|
|
189
|
+
async def get_status_or_raise(self, container: DockerContainer):
|
|
190
|
+
"""
|
|
191
|
+
Reload container
|
|
192
|
+
|
|
193
|
+
Parameters
|
|
194
|
+
----------
|
|
195
|
+
container : DockerContainer
|
|
196
|
+
Container
|
|
197
|
+
|
|
198
|
+
Raises
|
|
199
|
+
------
|
|
200
|
+
MantaDockerError
|
|
201
|
+
Cannot reload container
|
|
202
|
+
"""
|
|
203
|
+
status = "unknown"
|
|
204
|
+
try:
|
|
205
|
+
status = await container.show()
|
|
206
|
+
status = status["State"]["Status"]
|
|
207
|
+
except Exception as exc:
|
|
208
|
+
self.tracer.error("Cannot get status of container")
|
|
209
|
+
raise MantaDockerError(
|
|
210
|
+
"Cannot get status of container", metadata=self.tracer.metadata
|
|
211
|
+
) from exc
|
|
212
|
+
return status
|
|
213
|
+
|
|
214
|
+
async def get_container_logs_or_raise(
|
|
215
|
+
self, container: DockerContainer, since: Optional[datetime] = None
|
|
216
|
+
) -> bytes:
|
|
217
|
+
"""
|
|
218
|
+
Get container logs (async)
|
|
219
|
+
|
|
220
|
+
Parameters
|
|
221
|
+
----------
|
|
222
|
+
container : DockerContainer
|
|
223
|
+
Container
|
|
224
|
+
since: Optional[datetime]
|
|
225
|
+
Start time
|
|
226
|
+
|
|
227
|
+
Returns
|
|
228
|
+
-------
|
|
229
|
+
bytes
|
|
230
|
+
Logs
|
|
231
|
+
|
|
232
|
+
Raises
|
|
233
|
+
------
|
|
234
|
+
MantaDockerError
|
|
235
|
+
Cannot get logs
|
|
236
|
+
"""
|
|
237
|
+
logs = b""
|
|
238
|
+
try:
|
|
239
|
+
# Use async Docker client to avoid blocking the event loop
|
|
240
|
+
async_container = await self.async_client.containers.get(container.id)
|
|
241
|
+
# aiodocker >= 0.25: log() returns a coroutine yielding a list of strings
|
|
242
|
+
# since must be a unix timestamp (int), not a datetime object
|
|
243
|
+
since_ts = int(since.timestamp()) if since else None
|
|
244
|
+
log_lines = await async_container.log(
|
|
245
|
+
stdout=True, stderr=True, since=since_ts
|
|
246
|
+
)
|
|
247
|
+
logs = "".join(log_lines).encode()
|
|
248
|
+
except Exception as exc:
|
|
249
|
+
self.tracer.error(f"Cannot get logs of container: {exc}")
|
|
250
|
+
raise MantaDockerError(
|
|
251
|
+
"Cannot get logs of container", metadata=self.tracer.metadata
|
|
252
|
+
) from exc
|
|
253
|
+
return logs
|