analogcloud-common 0.1.4__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- anabrid/cloud/deploy/__init__.py +0 -0
- anabrid/cloud/deploy/generate.py +503 -0
- anabrid/cloud/deploy/templates/docker-compose.workers-compile-remote.yaml.j2 +30 -0
- anabrid/cloud/deploy/templates/docker-compose.workers.yaml.j2 +66 -0
- anabrid/cloud/deploy/templates/docker-compose.yaml.j2 +111 -0
- anabrid/cloud/devices.py +570 -0
- anabrid/cloud/models/__init__.py +0 -0
- anabrid/cloud/models/api/__init__.py +0 -0
- anabrid/cloud/models/api/v1.py +199 -0
- anabrid/cloud/models/auth.py +35 -0
- anabrid/cloud/models/celery.py +14 -0
- anabrid/cloud/models/config.py +191 -0
- anabrid/cloud/models/db.py +555 -0
- anabrid/cloud/models/enums.py +21 -0
- anabrid/cloud/storage/__init__.py +27 -0
- anabrid/cloud/storage/backend.py +303 -0
- anabrid/cloud/storage/blob_handle.py +55 -0
- anabrid/cloud/storage/log_handle.py +73 -0
- anabrid/cloud/storage/session.py +43 -0
- anabrid/cloud/storage/types.py +65 -0
- anabrid/cloud/util/__init__.py +0 -0
- anabrid/cloud/util/config.py +72 -0
- analogcloud_common-0.1.4.dist-info/METADATA +71 -0
- analogcloud_common-0.1.4.dist-info/RECORD +26 -0
- analogcloud_common-0.1.4.dist-info/WHEEL +4 -0
- analogcloud_common-0.1.4.dist-info/entry_points.txt +5 -0
|
File without changes
|
|
@@ -0,0 +1,503 @@
|
|
|
1
|
+
"""Deploy configuration generator for Analog Cloud services.
|
|
2
|
+
|
|
3
|
+
Renders Docker Compose YAML and supervisord.conf from a ``AnalogCloudConfig``
|
|
4
|
+
instance. Two compose targets are supported:
|
|
5
|
+
|
|
6
|
+
* The main ``docker-compose.yaml`` (postgres + rabbitmq + server). The only
|
|
7
|
+
template parameter is the server image reference; all other services and
|
|
8
|
+
environment-variable interpolations are fixed by the bundled template.
|
|
9
|
+
|
|
10
|
+
* The ``docker-compose.workers.yaml`` worker fragment — one service per
|
|
11
|
+
compile instance plus one per device-worker slot.
|
|
12
|
+
|
|
13
|
+
Both compose YAMLs are rendered from Jinja2 templates packaged under
|
|
14
|
+
``anabrid.cloud.deploy.templates``. The supervisord sidecar is emitted
|
|
15
|
+
alongside the worker compose only.
|
|
16
|
+
|
|
17
|
+
The database DSN is never baked into the generated files. The compose
|
|
18
|
+
fragment emits a compose-level ``${POSTGRES_PASSWORD}`` interpolation so
|
|
19
|
+
docker resolves the secret from ``.env`` at container start; the
|
|
20
|
+
supervisord fragment emits ``%(ENV_POSTGRES_PASSWORD)s`` so supervisord
|
|
21
|
+
resolves the secret from its own environment at program-start. No
|
|
22
|
+
plaintext password appears in either artefact.
|
|
23
|
+
"""
|
|
24
|
+
|
|
25
|
+
from __future__ import annotations
|
|
26
|
+
|
|
27
|
+
import argparse
|
|
28
|
+
import sys
|
|
29
|
+
from pathlib import Path
|
|
30
|
+
|
|
31
|
+
import yaml
|
|
32
|
+
from jinja2 import Environment, PackageLoader
|
|
33
|
+
|
|
34
|
+
from anabrid.cloud.models.config import AnalogCloudConfig
|
|
35
|
+
from anabrid.cloud.models.enums import DeviceTypeEnum
|
|
36
|
+
|
|
37
|
+
DEFAULT_WORKER_IMAGE = "analogcloud-worker:latest"
|
|
38
|
+
|
|
39
|
+
# Supervisord runs in the dev-host-API topology only; the API and workers
|
|
40
|
+
# reach postgres via the published port on the loopback. Supervisord's
|
|
41
|
+
# %(ENV_X)s syntax has no default-value form, so non-secret fields are
|
|
42
|
+
# inlined here and only the password remains a placeholder resolved from
|
|
43
|
+
# the shell (which sources .env before launching supervisord).
|
|
44
|
+
_SUPERVISORD_DATABASE_URL = (
|
|
45
|
+
"postgresql+asyncpg://analogcloud:%(ENV_POSTGRES_PASSWORD)s"
|
|
46
|
+
"@localhost:5432/analogcloud"
|
|
47
|
+
)
|
|
48
|
+
_SUPERVISORD_BROKER_URL = "amqp://localhost:5672//"
|
|
49
|
+
_SUPERVISORD_SOCK = "/tmp/supervisor.sock"
|
|
50
|
+
|
|
51
|
+
_API_COMMAND = (
|
|
52
|
+
"uv run uvicorn anabrid.cloud.server.main:app"
|
|
53
|
+
" --reload"
|
|
54
|
+
" --reload-dir %(ENV_ANALOGCLOUD_BASE_DIR)s/components/server/src"
|
|
55
|
+
" --reload-dir %(ENV_ANALOGCLOUD_BASE_DIR)s/components/common/src"
|
|
56
|
+
" --port 8000"
|
|
57
|
+
)
|
|
58
|
+
_WORKER_COMMAND = (
|
|
59
|
+
"uv run watchfiles --filter python"
|
|
60
|
+
' "python -m anabrid.cloud.worker.main"'
|
|
61
|
+
" %(ENV_ANALOGCLOUD_BASE_DIR)s/components/worker/src"
|
|
62
|
+
" %(ENV_ANALOGCLOUD_BASE_DIR)s/components/common/src"
|
|
63
|
+
)
|
|
64
|
+
|
|
65
|
+
_TEMPLATE_ENV = Environment(
|
|
66
|
+
loader=PackageLoader("anabrid.cloud.deploy"),
|
|
67
|
+
autoescape=False,
|
|
68
|
+
keep_trailing_newline=True,
|
|
69
|
+
)
|
|
70
|
+
|
|
71
|
+
|
|
72
|
+
def _check_duplicate_simulator_ports(config: AnalogCloudConfig) -> None:
|
|
73
|
+
"""Raise ``ValueError`` if two SIMULATOR devices share the same (host, port).
|
|
74
|
+
|
|
75
|
+
This guard fires before any output is produced, so a misconfigured file
|
|
76
|
+
fails fast with a clear message naming both devices and the conflicting
|
|
77
|
+
address.
|
|
78
|
+
"""
|
|
79
|
+
port_map: dict[tuple[str, int], list[str]] = {}
|
|
80
|
+
for entry in config.devices.available:
|
|
81
|
+
if entry.type == DeviceTypeEnum.SIMULATOR:
|
|
82
|
+
key = (entry.host, entry.port)
|
|
83
|
+
port_map.setdefault(key, []).append(entry.name)
|
|
84
|
+
|
|
85
|
+
for (host, port), names in port_map.items():
|
|
86
|
+
if len(names) > 1:
|
|
87
|
+
first, second = names[0], names[1]
|
|
88
|
+
suffix = f" and {len(names) - 2} others" if len(names) > 2 else ""
|
|
89
|
+
raise ValueError(
|
|
90
|
+
f"SIMULATOR devices {first!r} and {second!r}{suffix} share"
|
|
91
|
+
f" host:port {host}:{port}"
|
|
92
|
+
)
|
|
93
|
+
|
|
94
|
+
|
|
95
|
+
def render_server_compose(
|
|
96
|
+
server_image: str,
|
|
97
|
+
compile_remote: bool = False,
|
|
98
|
+
) -> str:
|
|
99
|
+
"""Render the main ``docker-compose.yaml`` with the given server image.
|
|
100
|
+
|
|
101
|
+
All other services (postgres, rabbitmq) and environment-variable
|
|
102
|
+
interpolations are fixed by the bundled template; only the server
|
|
103
|
+
image reference is substituted. When ``compile_remote`` is true the
|
|
104
|
+
template additionally emits the SeaweedFS S3 sidecar. The returned
|
|
105
|
+
string is valid Compose YAML.
|
|
106
|
+
"""
|
|
107
|
+
template = _TEMPLATE_ENV.get_template("docker-compose.yaml.j2")
|
|
108
|
+
return template.render(
|
|
109
|
+
server_image=server_image,
|
|
110
|
+
compile_remote=compile_remote,
|
|
111
|
+
)
|
|
112
|
+
|
|
113
|
+
|
|
114
|
+
def render_worker_compose(
|
|
115
|
+
config: AnalogCloudConfig,
|
|
116
|
+
worker_image: str = DEFAULT_WORKER_IMAGE,
|
|
117
|
+
compile_remote: bool = False,
|
|
118
|
+
) -> str:
|
|
119
|
+
"""Render ``docker-compose.workers.yaml`` for all worker services.
|
|
120
|
+
|
|
121
|
+
Emits one service per compile instance (unless ``compile_remote`` is
|
|
122
|
+
``True``, in which case no compile-worker services are emitted) and
|
|
123
|
+
one per device-worker slot. Raises ``ValueError`` if two SIMULATOR
|
|
124
|
+
devices share the same ``(host, port)`` pair.
|
|
125
|
+
|
|
126
|
+
``worker_image`` controls the image tag on every generated service.
|
|
127
|
+
Defaults to the local ``analogcloud-worker:latest`` tag built by the dev
|
|
128
|
+
init script; production bundles pass a registry-qualified reference
|
|
129
|
+
so the prod host can pull directly from the registry.
|
|
130
|
+
|
|
131
|
+
``compile_remote`` is a deploy-time topology signal passed in by the
|
|
132
|
+
caller (``ci/build.sh``, derived from ``environments.yaml``). When
|
|
133
|
+
``True``, compile-worker services are omitted from this bundle because
|
|
134
|
+
they are shipped to a separate host via the ``workers/compile`` bundle.
|
|
135
|
+
|
|
136
|
+
Security:
|
|
137
|
+
The generated file carries the DB DSN as a compose-level
|
|
138
|
+
interpolation literal (``${POSTGRES_PASSWORD}`` etc). Docker
|
|
139
|
+
resolves the secret from the ``.env`` file at container start,
|
|
140
|
+
so no plaintext password is ever written to disk by this
|
|
141
|
+
generator.
|
|
142
|
+
"""
|
|
143
|
+
_check_duplicate_simulator_ports(config)
|
|
144
|
+
|
|
145
|
+
devices = [
|
|
146
|
+
{
|
|
147
|
+
"name": entry.name,
|
|
148
|
+
"workers": entry.workers,
|
|
149
|
+
"is_simulator": entry.type == DeviceTypeEnum.SIMULATOR,
|
|
150
|
+
}
|
|
151
|
+
for entry in config.devices.available
|
|
152
|
+
]
|
|
153
|
+
template = _TEMPLATE_ENV.get_template("docker-compose.workers.yaml.j2")
|
|
154
|
+
return template.render(
|
|
155
|
+
worker_image=worker_image,
|
|
156
|
+
compile_instances=config.workers.compile.num_instances,
|
|
157
|
+
compile_remote=compile_remote,
|
|
158
|
+
compile_timeout=config.workers.compile.instance_timeout,
|
|
159
|
+
compile_cpu_cores=config.workers.compile.instance_cpu_cores,
|
|
160
|
+
compile_memory_gb=config.workers.compile.instance_memory_gb,
|
|
161
|
+
run_op_time_limit=config.workers.run_op_time_limit,
|
|
162
|
+
devices=devices,
|
|
163
|
+
)
|
|
164
|
+
|
|
165
|
+
|
|
166
|
+
def render_compile_remote_workers_compose(
|
|
167
|
+
config: AnalogCloudConfig,
|
|
168
|
+
env_host: str,
|
|
169
|
+
worker_image: str = DEFAULT_WORKER_IMAGE,
|
|
170
|
+
) -> str:
|
|
171
|
+
"""Render the compile-worker docker-compose YAML for a remote host.
|
|
172
|
+
|
|
173
|
+
Emits one ``worker-compile-{i}`` service per
|
|
174
|
+
``config.workers.compile.num_instances``. All broker, database, and
|
|
175
|
+
blob URLs target ``env_host`` — the main host's address as seen by
|
|
176
|
+
the remote compile-worker machine. In practice this must be the
|
|
177
|
+
main host's VPN interface IP (e.g. WireGuard), not the public
|
|
178
|
+
FQDN, since Postgres (5432), RabbitMQ (5672), and SeaweedFS (8333)
|
|
179
|
+
are bound to the VPN address and never exposed publicly. The
|
|
180
|
+
generated services attach to the default bridge network on the
|
|
181
|
+
remote host; no ``analogcloud_internal`` bridge exists there.
|
|
182
|
+
|
|
183
|
+
``worker_image`` controls the image tag on every generated service.
|
|
184
|
+
Defaults to the local ``analogcloud-worker:latest`` tag built by the dev
|
|
185
|
+
init script; production bundles pass a registry-qualified reference
|
|
186
|
+
so the remote host can pull directly from the registry.
|
|
187
|
+
|
|
188
|
+
Raises ``ValueError`` if ``config.workers.compile.num_instances`` is
|
|
189
|
+
less than 1; the remote bundle has no meaning at zero and must fail
|
|
190
|
+
fast rather than emit an empty compose file.
|
|
191
|
+
"""
|
|
192
|
+
if config.workers.compile.num_instances < 1:
|
|
193
|
+
raise ValueError(
|
|
194
|
+
"workers/compile bundle requires compile.num_instances >= 1, got"
|
|
195
|
+
f" {config.workers.compile.num_instances}"
|
|
196
|
+
)
|
|
197
|
+
template = _TEMPLATE_ENV.get_template(
|
|
198
|
+
"docker-compose.workers-compile-remote.yaml.j2"
|
|
199
|
+
)
|
|
200
|
+
return template.render(
|
|
201
|
+
worker_image=worker_image,
|
|
202
|
+
env_host=env_host,
|
|
203
|
+
compile_instances=config.workers.compile.num_instances,
|
|
204
|
+
compile_timeout=config.workers.compile.instance_timeout,
|
|
205
|
+
compile_cpu_cores=config.workers.compile.instance_cpu_cores,
|
|
206
|
+
compile_memory_gb=config.workers.compile.instance_memory_gb,
|
|
207
|
+
)
|
|
208
|
+
|
|
209
|
+
|
|
210
|
+
def generate_supervisord(config: AnalogCloudConfig) -> str:
|
|
211
|
+
"""Generate a supervisord.conf string for the dev-host-API topology.
|
|
212
|
+
|
|
213
|
+
Emits four harness stanzas (supervisord, unix_http_server,
|
|
214
|
+
rpcinterface:supervisor, supervisorctl), one ``[program:api]`` stanza
|
|
215
|
+
running uvicorn with --reload, one ``[program:worker-compile-N]`` per
|
|
216
|
+
compile instance running the worker entrypoint wrapped by watchfiles,
|
|
217
|
+
and one ``[program:worker-run-<device>-N]`` per device-worker slot.
|
|
218
|
+
|
|
219
|
+
All program paths reference ``%(ENV_ANALOGCLOUD_BASE_DIR)s`` so the
|
|
220
|
+
generated file is independent of where the project is checked out.
|
|
221
|
+
Log files are written to
|
|
222
|
+
``%(ENV_ANALOGCLOUD_BASE_DIR)s/logs/<program-name>.{out,err}``.
|
|
223
|
+
|
|
224
|
+
Raises ``ValueError`` if two SIMULATOR devices share the same
|
|
225
|
+
``(host, port)`` pair.
|
|
226
|
+
"""
|
|
227
|
+
_check_duplicate_simulator_ports(config)
|
|
228
|
+
|
|
229
|
+
common_env_pairs = [
|
|
230
|
+
f'BROKER_URL="{_SUPERVISORD_BROKER_URL}"',
|
|
231
|
+
f'ANALOGCLOUD_DATABASE_URL="{_SUPERVISORD_DATABASE_URL}"',
|
|
232
|
+
'ANALOGCLOUD_BASE_DIR="%(ENV_ANALOGCLOUD_BASE_DIR)s"',
|
|
233
|
+
]
|
|
234
|
+
|
|
235
|
+
def _log_base(name: str) -> str:
|
|
236
|
+
return "%(ENV_ANALOGCLOUD_BASE_DIR)s/logs/" + name
|
|
237
|
+
|
|
238
|
+
def _program_block(
|
|
239
|
+
name: str,
|
|
240
|
+
command: str,
|
|
241
|
+
directory: str,
|
|
242
|
+
extra_env_pairs: list[str] | None = None,
|
|
243
|
+
) -> str:
|
|
244
|
+
env_pairs = list(extra_env_pairs or []) + common_env_pairs
|
|
245
|
+
env_line = ",".join(env_pairs)
|
|
246
|
+
lines = [
|
|
247
|
+
f"[program:{name}]",
|
|
248
|
+
f"command={command}",
|
|
249
|
+
f"directory={directory}",
|
|
250
|
+
"autostart=true",
|
|
251
|
+
"autorestart=true",
|
|
252
|
+
f"stdout_logfile={_log_base(name)}.out",
|
|
253
|
+
f"stderr_logfile={_log_base(name)}.err",
|
|
254
|
+
f"environment={env_line}",
|
|
255
|
+
"",
|
|
256
|
+
]
|
|
257
|
+
return "\n".join(lines)
|
|
258
|
+
|
|
259
|
+
base_dir = "%(ENV_ANALOGCLOUD_BASE_DIR)s"
|
|
260
|
+
server_dir = f"{base_dir}/components/server"
|
|
261
|
+
worker_dir = f"{base_dir}/components/worker"
|
|
262
|
+
|
|
263
|
+
sections: list[str] = []
|
|
264
|
+
|
|
265
|
+
sections.append(
|
|
266
|
+
"[supervisord]\n"
|
|
267
|
+
"nodaemon=true\n"
|
|
268
|
+
"\n"
|
|
269
|
+
)
|
|
270
|
+
sections.append(
|
|
271
|
+
"[unix_http_server]\n"
|
|
272
|
+
f"file={_SUPERVISORD_SOCK}\n"
|
|
273
|
+
"\n"
|
|
274
|
+
)
|
|
275
|
+
sections.append(
|
|
276
|
+
"[rpcinterface:supervisor]\n"
|
|
277
|
+
"supervisor.rpcinterface_factory=supervisor.rpcinterface:make_main_rpcinterface\n"
|
|
278
|
+
"\n"
|
|
279
|
+
)
|
|
280
|
+
sections.append(
|
|
281
|
+
"[supervisorctl]\n"
|
|
282
|
+
f"serverurl=unix://{_SUPERVISORD_SOCK}\n"
|
|
283
|
+
"\n"
|
|
284
|
+
)
|
|
285
|
+
|
|
286
|
+
sections.append(
|
|
287
|
+
_program_block(
|
|
288
|
+
name="api",
|
|
289
|
+
command=_API_COMMAND,
|
|
290
|
+
directory=server_dir,
|
|
291
|
+
)
|
|
292
|
+
)
|
|
293
|
+
|
|
294
|
+
for i in range(config.workers.compile.num_instances):
|
|
295
|
+
name = f"worker-compile-{i}"
|
|
296
|
+
compile_env_pairs = [
|
|
297
|
+
'WORKER_TYPE="compile"',
|
|
298
|
+
'REDACC_EXECUTABLE="%(ENV_ANALOGCLOUD_BASE_DIR)s/bin/redacc"',
|
|
299
|
+
f'ANALOGCLOUD_COMPILE_TIMEOUT="{config.workers.compile.instance_timeout}"',
|
|
300
|
+
]
|
|
301
|
+
if config.workers.compile.instance_cpu_cores is not None:
|
|
302
|
+
compile_env_pairs.append(
|
|
303
|
+
f'ANALOGCLOUD_COMPILE_CPU_CORES="{config.workers.compile.instance_cpu_cores}"'
|
|
304
|
+
)
|
|
305
|
+
if config.workers.compile.instance_memory_gb is not None:
|
|
306
|
+
compile_env_pairs.append(
|
|
307
|
+
f'ANALOGCLOUD_COMPILE_MEMORY_GB="{config.workers.compile.instance_memory_gb}"'
|
|
308
|
+
)
|
|
309
|
+
sections.append(
|
|
310
|
+
_program_block(
|
|
311
|
+
name=name,
|
|
312
|
+
command=_WORKER_COMMAND,
|
|
313
|
+
directory=worker_dir,
|
|
314
|
+
extra_env_pairs=compile_env_pairs,
|
|
315
|
+
)
|
|
316
|
+
)
|
|
317
|
+
|
|
318
|
+
for entry in config.devices.available:
|
|
319
|
+
for i in range(entry.workers):
|
|
320
|
+
name = f"worker-run-{entry.name}-{i}"
|
|
321
|
+
env_pairs = [
|
|
322
|
+
'WORKER_TYPE="run"',
|
|
323
|
+
f'DEVICE_NAME="{entry.name}"',
|
|
324
|
+
f'ANALOGCLOUD_RUN_OP_TIME_LIMIT="{config.workers.run_op_time_limit}"',
|
|
325
|
+
]
|
|
326
|
+
sections.append(
|
|
327
|
+
_program_block(
|
|
328
|
+
name=name,
|
|
329
|
+
command=_WORKER_COMMAND,
|
|
330
|
+
directory=worker_dir,
|
|
331
|
+
extra_env_pairs=env_pairs,
|
|
332
|
+
)
|
|
333
|
+
)
|
|
334
|
+
|
|
335
|
+
return "\n".join(sections)
|
|
336
|
+
|
|
337
|
+
|
|
338
|
+
def main() -> None:
|
|
339
|
+
"""Entry point for the ``analogcloud-deploy`` CLI tool."""
|
|
340
|
+
parser = argparse.ArgumentParser(
|
|
341
|
+
description=(
|
|
342
|
+
"Generate Docker Compose configuration from an Analog Cloud config file."
|
|
343
|
+
)
|
|
344
|
+
)
|
|
345
|
+
subparsers = parser.add_subparsers(dest="command", required=True)
|
|
346
|
+
|
|
347
|
+
gen = subparsers.add_parser(
|
|
348
|
+
"generate",
|
|
349
|
+
help=(
|
|
350
|
+
"Generate a compose file. Pass --bundle-target "
|
|
351
|
+
"{main, workers, workers/compile} to select the bundle "
|
|
352
|
+
"(default: main)."
|
|
353
|
+
),
|
|
354
|
+
)
|
|
355
|
+
gen.add_argument("-c", "--config", required=True, help="Path to analogcloud.yaml.")
|
|
356
|
+
gen.add_argument(
|
|
357
|
+
"-o", "--output", required=True, help="Path for the output compose file."
|
|
358
|
+
)
|
|
359
|
+
target_group = gen.add_mutually_exclusive_group()
|
|
360
|
+
target_group.add_argument(
|
|
361
|
+
"--worker",
|
|
362
|
+
action="store_true",
|
|
363
|
+
help=(
|
|
364
|
+
"Render docker-compose.workers.yaml (plus the supervisord.conf "
|
|
365
|
+
"sidecar) instead of the main docker-compose.yaml. Synonym for "
|
|
366
|
+
"--bundle-target workers."
|
|
367
|
+
),
|
|
368
|
+
)
|
|
369
|
+
target_group.add_argument(
|
|
370
|
+
"--bundle-target",
|
|
371
|
+
choices=("main", "workers", "workers/compile"),
|
|
372
|
+
default=None,
|
|
373
|
+
help=(
|
|
374
|
+
"Which bundle to render. 'main' (default) emits the main "
|
|
375
|
+
"docker-compose.yaml; 'workers' emits the local worker "
|
|
376
|
+
"fragment plus supervisord.conf; 'workers/compile' emits the "
|
|
377
|
+
"remote compile-worker compose targeted at --env-host."
|
|
378
|
+
),
|
|
379
|
+
)
|
|
380
|
+
gen.add_argument(
|
|
381
|
+
"--worker-image",
|
|
382
|
+
default=None,
|
|
383
|
+
help=(
|
|
384
|
+
"Worker image reference to emit on every worker service. "
|
|
385
|
+
f"Defaults to {DEFAULT_WORKER_IMAGE!r}. Only meaningful with "
|
|
386
|
+
"--bundle-target workers or workers/compile."
|
|
387
|
+
),
|
|
388
|
+
)
|
|
389
|
+
gen.add_argument(
|
|
390
|
+
"--server-image",
|
|
391
|
+
default=None,
|
|
392
|
+
help=(
|
|
393
|
+
"Server image reference to emit on the server service. "
|
|
394
|
+
"Required when generating the main compose; rejected for "
|
|
395
|
+
"worker bundle targets."
|
|
396
|
+
),
|
|
397
|
+
)
|
|
398
|
+
gen.add_argument(
|
|
399
|
+
"--env-host",
|
|
400
|
+
default=None,
|
|
401
|
+
help=(
|
|
402
|
+
"Address of the env's main host as reachable from the "
|
|
403
|
+
"remote compile-worker machine. In practice this is the "
|
|
404
|
+
"main host's VPN interface IP (e.g. WireGuard), since "
|
|
405
|
+
"Postgres/RabbitMQ/SeaweedFS are bound to the VPN address "
|
|
406
|
+
"and not exposed on the public FQDN. Required with "
|
|
407
|
+
"--bundle-target workers/compile; rejected otherwise."
|
|
408
|
+
),
|
|
409
|
+
)
|
|
410
|
+
gen.add_argument(
|
|
411
|
+
"--compile-remote",
|
|
412
|
+
action="store_true",
|
|
413
|
+
default=False,
|
|
414
|
+
help=(
|
|
415
|
+
"Signal that this env runs compile workers on a separate host "
|
|
416
|
+
"(derived by ci/build.sh from environments.yaml: the env has at "
|
|
417
|
+
"least one additional_hosts[type=compile] entry). Affects the "
|
|
418
|
+
"main and workers bundles: the main compose gains the SeaweedFS "
|
|
419
|
+
"sidecar and the workers compose omits compile-worker services. "
|
|
420
|
+
"Valid only with --bundle-target main or workers; rejected for "
|
|
421
|
+
"--bundle-target workers/compile (that target is intrinsically "
|
|
422
|
+
"the remote-compile path)."
|
|
423
|
+
),
|
|
424
|
+
)
|
|
425
|
+
|
|
426
|
+
args = parser.parse_args()
|
|
427
|
+
|
|
428
|
+
# Resolve the effective bundle target. --worker is a synonym for
|
|
429
|
+
# --bundle-target workers (the two are already mutually exclusive
|
|
430
|
+
# at the argparse layer).
|
|
431
|
+
if args.worker:
|
|
432
|
+
bundle_target = "workers"
|
|
433
|
+
elif args.bundle_target is not None:
|
|
434
|
+
bundle_target = args.bundle_target
|
|
435
|
+
else:
|
|
436
|
+
bundle_target = "main"
|
|
437
|
+
|
|
438
|
+
if bundle_target == "main":
|
|
439
|
+
if args.server_image is None:
|
|
440
|
+
parser.error(
|
|
441
|
+
"--server-image is required when generating the main compose"
|
|
442
|
+
)
|
|
443
|
+
if args.env_host is not None:
|
|
444
|
+
parser.error("--env-host is only valid with --bundle-target workers/compile")
|
|
445
|
+
if args.worker_image is not None:
|
|
446
|
+
parser.error(
|
|
447
|
+
"--worker-image is not valid with --bundle-target main"
|
|
448
|
+
)
|
|
449
|
+
elif bundle_target == "workers":
|
|
450
|
+
if args.server_image is not None:
|
|
451
|
+
parser.error("--server-image is not valid for the workers bundle")
|
|
452
|
+
if args.env_host is not None:
|
|
453
|
+
parser.error("--env-host is only valid with --bundle-target workers/compile")
|
|
454
|
+
elif bundle_target == "workers/compile":
|
|
455
|
+
if args.server_image is not None:
|
|
456
|
+
parser.error(
|
|
457
|
+
"--server-image is not valid for the workers/compile bundle"
|
|
458
|
+
)
|
|
459
|
+
if args.env_host is None:
|
|
460
|
+
parser.error(
|
|
461
|
+
"--env-host is required with --bundle-target workers/compile"
|
|
462
|
+
)
|
|
463
|
+
if args.compile_remote:
|
|
464
|
+
parser.error(
|
|
465
|
+
"--compile-remote is not valid with --bundle-target workers/compile "
|
|
466
|
+
"(that target is intrinsically the remote-compile path)"
|
|
467
|
+
)
|
|
468
|
+
|
|
469
|
+
config_path = Path(args.config)
|
|
470
|
+
config = AnalogCloudConfig.model_validate(
|
|
471
|
+
yaml.safe_load(config_path.read_text(encoding="utf-8"))
|
|
472
|
+
)
|
|
473
|
+
|
|
474
|
+
worker_image = args.worker_image or DEFAULT_WORKER_IMAGE
|
|
475
|
+
|
|
476
|
+
output_path = Path(args.output)
|
|
477
|
+
try:
|
|
478
|
+
if bundle_target == "workers":
|
|
479
|
+
yaml_text = render_worker_compose(
|
|
480
|
+
config,
|
|
481
|
+
worker_image=worker_image,
|
|
482
|
+
compile_remote=args.compile_remote,
|
|
483
|
+
)
|
|
484
|
+
supervisord_text = generate_supervisord(config)
|
|
485
|
+
output_path.write_text(yaml_text, encoding="utf-8")
|
|
486
|
+
supervisord_path = output_path.parent / "supervisord.conf"
|
|
487
|
+
supervisord_path.write_text(supervisord_text, encoding="utf-8")
|
|
488
|
+
elif bundle_target == "workers/compile":
|
|
489
|
+
yaml_text = render_compile_remote_workers_compose(
|
|
490
|
+
config,
|
|
491
|
+
env_host=args.env_host,
|
|
492
|
+
worker_image=worker_image,
|
|
493
|
+
)
|
|
494
|
+
output_path.write_text(yaml_text, encoding="utf-8")
|
|
495
|
+
else:
|
|
496
|
+
yaml_text = render_server_compose(
|
|
497
|
+
server_image=args.server_image,
|
|
498
|
+
compile_remote=args.compile_remote,
|
|
499
|
+
)
|
|
500
|
+
output_path.write_text(yaml_text, encoding="utf-8")
|
|
501
|
+
except ValueError as exc:
|
|
502
|
+
print(f"error: {exc}", file=sys.stderr)
|
|
503
|
+
sys.exit(2)
|
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
services:
|
|
2
|
+
{% for i in range(compile_instances) %}
|
|
3
|
+
worker-compile-{{ i }}:
|
|
4
|
+
image: {{ worker_image }}
|
|
5
|
+
restart: unless-stopped
|
|
6
|
+
user: "${ANALOGCLOUD_UID:?ANALOGCLOUD_UID is not set in .env}:${ANALOGCLOUD_GID:?ANALOGCLOUD_GID is not set in .env}"
|
|
7
|
+
volumes:
|
|
8
|
+
- ./config:/config:ro
|
|
9
|
+
- ./data:/data
|
|
10
|
+
- ./logs:/logs
|
|
11
|
+
environment:
|
|
12
|
+
WORKER_TYPE: compile
|
|
13
|
+
ANALOGCLOUD_BASE_DIR: "/"
|
|
14
|
+
ANALOGCLOUD_LOG_NAME: worker-compile-{{ i }}
|
|
15
|
+
REDACC_EXECUTABLE: /usr/local/bin/redacc
|
|
16
|
+
ANALOGCLOUD_COMPILE_TIMEOUT: "{{ compile_timeout }}"
|
|
17
|
+
{%- if compile_cpu_cores is not none %}
|
|
18
|
+
ANALOGCLOUD_COMPILE_CPU_CORES: "{{ compile_cpu_cores }}"
|
|
19
|
+
{%- endif %}
|
|
20
|
+
{%- if compile_memory_gb is not none %}
|
|
21
|
+
ANALOGCLOUD_COMPILE_MEMORY_GB: "{{ compile_memory_gb }}"
|
|
22
|
+
{%- endif %}
|
|
23
|
+
BROKER_URL: "amqp://${RABBITMQ_DEFAULT_USER:-analogcloud}:${RABBITMQ_DEFAULT_PASS:?RABBITMQ_DEFAULT_PASS is not set in .env}@{{ env_host }}:5672//"
|
|
24
|
+
ANALOGCLOUD_DATABASE_URL: "postgresql+asyncpg://${POSTGRES_USER:-analogcloud}:${POSTGRES_PASSWORD:?POSTGRES_PASSWORD is not set in .env}@{{ env_host }}:5432/${POSTGRES_DB:-analogcloud}"
|
|
25
|
+
ANALOGCLOUD_BLOB_BACKEND: s3
|
|
26
|
+
S3_BLOB_ENDPOINT: "http://{{ env_host }}:8333"
|
|
27
|
+
S3_BLOB_BUCKET: ${S3_BLOB_BUCKET:-analogcloud}
|
|
28
|
+
S3_BLOB_ACCESS_KEY: ${S3_BLOB_ACCESS_KEY:?S3_BLOB_ACCESS_KEY is not set in .env}
|
|
29
|
+
S3_BLOB_SECRET_KEY: ${S3_BLOB_SECRET_KEY:?S3_BLOB_SECRET_KEY is not set in .env}
|
|
30
|
+
{% endfor %}
|
|
@@ -0,0 +1,66 @@
|
|
|
1
|
+
services:
|
|
2
|
+
{% if not compile_remote %}
|
|
3
|
+
{% for i in range(compile_instances) %}
|
|
4
|
+
worker-compile-{{ i }}:
|
|
5
|
+
image: {{ worker_image }}
|
|
6
|
+
restart: unless-stopped
|
|
7
|
+
user: "${ANALOGCLOUD_UID:?ANALOGCLOUD_UID is not set in .env}:${ANALOGCLOUD_GID:?ANALOGCLOUD_GID is not set in .env}"
|
|
8
|
+
networks:
|
|
9
|
+
- internal
|
|
10
|
+
volumes:
|
|
11
|
+
- "${ANALOGCLOUD_BASE_DIR:?ANALOGCLOUD_BASE_DIR is not set in .env}/config:/config:ro"
|
|
12
|
+
- "${ANALOGCLOUD_BASE_DIR:?ANALOGCLOUD_BASE_DIR is not set in .env}/data:/data"
|
|
13
|
+
- "${ANALOGCLOUD_BASE_DIR:?ANALOGCLOUD_BASE_DIR is not set in .env}/logs:/logs"
|
|
14
|
+
environment:
|
|
15
|
+
WORKER_TYPE: compile
|
|
16
|
+
BROKER_URL: "amqp://${RABBITMQ_DEFAULT_USER:-analogcloud}:${RABBITMQ_DEFAULT_PASS:?RABBITMQ_DEFAULT_PASS is not set in .env}@rabbitmq:5672//"
|
|
17
|
+
ANALOGCLOUD_DATABASE_URL: "postgresql+asyncpg://${POSTGRES_USER:-analogcloud}:${POSTGRES_PASSWORD}@postgres:${POSTGRES_PORT:-5432}/${POSTGRES_DB:-analogcloud}"
|
|
18
|
+
ANALOGCLOUD_BASE_DIR: "/"
|
|
19
|
+
ANALOGCLOUD_LOG_NAME: worker-compile-{{ i }}
|
|
20
|
+
REDACC_EXECUTABLE: /usr/local/bin/redacc
|
|
21
|
+
ANALOGCLOUD_COMPILE_TIMEOUT: "{{ compile_timeout }}"
|
|
22
|
+
{%- if compile_cpu_cores is not none %}
|
|
23
|
+
ANALOGCLOUD_COMPILE_CPU_CORES: "{{ compile_cpu_cores }}"
|
|
24
|
+
{%- endif %}
|
|
25
|
+
{%- if compile_memory_gb is not none %}
|
|
26
|
+
ANALOGCLOUD_COMPILE_MEMORY_GB: "{{ compile_memory_gb }}"
|
|
27
|
+
{%- endif %}
|
|
28
|
+
{% endfor %}
|
|
29
|
+
{% endif %}
|
|
30
|
+
{%- for entry in devices %}
|
|
31
|
+
{%- for i in range(entry.workers) %}
|
|
32
|
+
worker-run-{{ entry.name }}-{{ i }}:
|
|
33
|
+
image: {{ worker_image }}
|
|
34
|
+
restart: unless-stopped
|
|
35
|
+
user: "${ANALOGCLOUD_UID:?ANALOGCLOUD_UID is not set in .env}:${ANALOGCLOUD_GID:?ANALOGCLOUD_GID is not set in .env}"
|
|
36
|
+
networks:
|
|
37
|
+
- internal
|
|
38
|
+
volumes:
|
|
39
|
+
- "${ANALOGCLOUD_BASE_DIR:?ANALOGCLOUD_BASE_DIR is not set in .env}/config:/config:ro"
|
|
40
|
+
- "${ANALOGCLOUD_BASE_DIR:?ANALOGCLOUD_BASE_DIR is not set in .env}/data:/data"
|
|
41
|
+
- "${ANALOGCLOUD_BASE_DIR:?ANALOGCLOUD_BASE_DIR is not set in .env}/logs:/logs"
|
|
42
|
+
environment:
|
|
43
|
+
WORKER_TYPE: run
|
|
44
|
+
BROKER_URL: "amqp://${RABBITMQ_DEFAULT_USER:-analogcloud}:${RABBITMQ_DEFAULT_PASS:?RABBITMQ_DEFAULT_PASS is not set in .env}@rabbitmq:5672//"
|
|
45
|
+
ANALOGCLOUD_DATABASE_URL: "postgresql+asyncpg://${POSTGRES_USER:-analogcloud}:${POSTGRES_PASSWORD}@postgres:${POSTGRES_PORT:-5432}/${POSTGRES_DB:-analogcloud}"
|
|
46
|
+
ANALOGCLOUD_BASE_DIR: "/"
|
|
47
|
+
ANALOGCLOUD_LOG_NAME: worker-run-{{ entry.name }}-{{ i }}
|
|
48
|
+
DEVICE_NAME: "{{ entry.name }}"
|
|
49
|
+
ANALOGCLOUD_RUN_OP_TIME_LIMIT: "{{ run_op_time_limit }}"
|
|
50
|
+
{%- if entry.is_simulator %}
|
|
51
|
+
ANALOGCLOUD_SIMULATOR_BINARY: /usr/local/bin/compiler_sim
|
|
52
|
+
{%- endif %}
|
|
53
|
+
{%- if compile_remote %}
|
|
54
|
+
ANALOGCLOUD_BLOB_BACKEND: s3
|
|
55
|
+
S3_BLOB_ENDPOINT: "http://seaweedfs:8333"
|
|
56
|
+
S3_BLOB_BUCKET: ${S3_BLOB_BUCKET:-analogcloud}
|
|
57
|
+
S3_BLOB_ACCESS_KEY: ${S3_BLOB_ACCESS_KEY:?S3_BLOB_ACCESS_KEY is not set in .env}
|
|
58
|
+
S3_BLOB_SECRET_KEY: ${S3_BLOB_SECRET_KEY:?S3_BLOB_SECRET_KEY is not set in .env}
|
|
59
|
+
{%- endif %}
|
|
60
|
+
|
|
61
|
+
{% endfor %}
|
|
62
|
+
{%- endfor %}
|
|
63
|
+
networks:
|
|
64
|
+
internal:
|
|
65
|
+
external: true
|
|
66
|
+
name: analogcloud_internal
|