composable-data-stack 0.4.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- cli/__init__.py +1 -0
- cli/diagnostics.py +13 -0
- cli/graph.py +58 -0
- cli/image_updates.py +326 -0
- cli/image_verification.py +484 -0
- cli/loader.py +180 -0
- cli/main.py +1656 -0
- cli/overlay.py +239 -0
- cli/planner.py +618 -0
- cli/preflight.py +418 -0
- cli/renderer.py +791 -0
- cli/resolver.py +28 -0
- cli/resources/__init__.py +1 -0
- cli/resources/rule-schema.json +274 -0
- cli/resources/rule-set.json +919 -0
- cli/secrets.py +169 -0
- cli/security.py +768 -0
- cli/security_common.py +41 -0
- cli/state.py +112 -0
- cli/up_runner.py +257 -0
- cli/validator.py +570 -0
- composable_data_stack-0.4.0.dist-info/METADATA +872 -0
- composable_data_stack-0.4.0.dist-info/RECORD +27 -0
- composable_data_stack-0.4.0.dist-info/WHEEL +5 -0
- composable_data_stack-0.4.0.dist-info/entry_points.txt +2 -0
- composable_data_stack-0.4.0.dist-info/licenses/LICENSE +201 -0
- composable_data_stack-0.4.0.dist-info/top_level.txt +1 -0
cli/renderer.py
ADDED
|
@@ -0,0 +1,791 @@
|
|
|
1
|
+
# pyright: reportMissingModuleSource=false
|
|
2
|
+
# cli/renderer.py
|
|
3
|
+
"""
|
|
4
|
+
Render docker-compose YAML from a composition plan.
|
|
5
|
+
"""
|
|
6
|
+
from __future__ import annotations
|
|
7
|
+
|
|
8
|
+
import re
|
|
9
|
+
import os
|
|
10
|
+
import tempfile
|
|
11
|
+
import yaml
|
|
12
|
+
from copy import deepcopy
|
|
13
|
+
from pathlib import Path
|
|
14
|
+
from typing import Any
|
|
15
|
+
|
|
16
|
+
from .diagnostics import Diagnostic
|
|
17
|
+
from .loader import resolve_module_dir
|
|
18
|
+
|
|
19
|
+
# Guards recursive interpolation of user-controlled service/volume templates
|
|
20
|
+
# against maliciously or accidentally deeply nested documents that would
|
|
21
|
+
# otherwise raise an unhandled RecursionError / stack overflow.
|
|
22
|
+
MAX_NESTING_DEPTH = 100
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
class MaxNestingDepthExceeded(Exception):
|
|
26
|
+
"""Raised when a recursive structure exceeds MAX_NESTING_DEPTH."""
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
def _atomic_write_compose(path: Path, content: str) -> None:
|
|
30
|
+
"""Write the rendered compose file atomically via a temp file + os.replace,
|
|
31
|
+
so a crash/kill mid-write can't leave a truncated docker-compose.yml behind.
|
|
32
|
+
"""
|
|
33
|
+
fd, tmp_name = tempfile.mkstemp(dir=str(path.parent), prefix=f".{path.name}.", suffix=".tmp")
|
|
34
|
+
try:
|
|
35
|
+
with os.fdopen(fd, "w", encoding="utf-8") as tmp_file:
|
|
36
|
+
tmp_file.write(content)
|
|
37
|
+
os.replace(tmp_name, path)
|
|
38
|
+
except OSError:
|
|
39
|
+
try:
|
|
40
|
+
os.unlink(tmp_name)
|
|
41
|
+
except OSError:
|
|
42
|
+
pass
|
|
43
|
+
raise
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
def render_compose(
|
|
47
|
+
plan: dict[str, Any],
|
|
48
|
+
output_path: str | None = None,
|
|
49
|
+
env_file: str | None = None,
|
|
50
|
+
) -> tuple[str, list[Diagnostic]]:
|
|
51
|
+
"""
|
|
52
|
+
Render docker-compose from plan.
|
|
53
|
+
|
|
54
|
+
Args:
|
|
55
|
+
plan: Composition plan.
|
|
56
|
+
output_path: Optional output file path.
|
|
57
|
+
env_file: Reserved for compatibility; not used.
|
|
58
|
+
|
|
59
|
+
Returns:
|
|
60
|
+
Tuple of (output_yaml, diagnostics).
|
|
61
|
+
"""
|
|
62
|
+
_ = env_file
|
|
63
|
+
diagnostics: list[Diagnostic] = []
|
|
64
|
+
secrets = plan.get("secrets", {})
|
|
65
|
+
compose_dir = Path(output_path).resolve().parent if output_path else Path.cwd().resolve()
|
|
66
|
+
profile_dir = _resolve_profile_dir(plan)
|
|
67
|
+
project_root = _resolve_project_root(profile_dir)
|
|
68
|
+
|
|
69
|
+
# Extract networks from runtime config
|
|
70
|
+
runtime = plan.get("runtime", {})
|
|
71
|
+
networks_config = runtime.get("networks", [])
|
|
72
|
+
|
|
73
|
+
# Build networks section
|
|
74
|
+
networks: dict[str, Any] = {}
|
|
75
|
+
for net in networks_config:
|
|
76
|
+
net_name = net.get("name", "default")
|
|
77
|
+
networks[net_name] = {}
|
|
78
|
+
driver = net.get("driver")
|
|
79
|
+
if driver:
|
|
80
|
+
networks[net_name]["driver"] = driver
|
|
81
|
+
|
|
82
|
+
# Build the default network name from namespace or profile name
|
|
83
|
+
default_network_name = runtime.get("namespace") or plan.get("metadata", {}).get("name", "cds")
|
|
84
|
+
|
|
85
|
+
compose: dict[str, Any] = {
|
|
86
|
+
"name": plan.get("metadata", {}).get("name", "cds"),
|
|
87
|
+
"services": {},
|
|
88
|
+
"volumes": {},
|
|
89
|
+
}
|
|
90
|
+
module_service_names: dict[str, list[str]] = {}
|
|
91
|
+
|
|
92
|
+
# Add networks if defined
|
|
93
|
+
if networks or default_network_name:
|
|
94
|
+
if not networks:
|
|
95
|
+
networks[default_network_name] = {}
|
|
96
|
+
compose["networks"] = networks
|
|
97
|
+
|
|
98
|
+
for module in plan.get("modules", []):
|
|
99
|
+
implementation = module.get("implementation", {})
|
|
100
|
+
|
|
101
|
+
if implementation.get("kind") != "docker-compose":
|
|
102
|
+
diagnostics.append(Diagnostic(
|
|
103
|
+
level="error",
|
|
104
|
+
code="E070",
|
|
105
|
+
message=(
|
|
106
|
+
f'Module "{module.get("id")}" has unsupported implementation '
|
|
107
|
+
f'kind "{implementation.get("kind")}".'
|
|
108
|
+
),
|
|
109
|
+
path=f'module:{module.get("id")}.implementation.kind',
|
|
110
|
+
))
|
|
111
|
+
continue
|
|
112
|
+
|
|
113
|
+
compose_impl = implementation.get("compose")
|
|
114
|
+
if not compose_impl:
|
|
115
|
+
diagnostics.append(Diagnostic(
|
|
116
|
+
level="warning",
|
|
117
|
+
code="W071",
|
|
118
|
+
message=(
|
|
119
|
+
f'Module "{module.get("id")}" has kind "docker-compose" '
|
|
120
|
+
f'but no "compose" definition.'
|
|
121
|
+
),
|
|
122
|
+
path=f'module:{module.get("id")}.implementation.compose',
|
|
123
|
+
))
|
|
124
|
+
continue
|
|
125
|
+
|
|
126
|
+
services = compose_impl.get("services", {})
|
|
127
|
+
volumes = compose_impl.get("volumes", {})
|
|
128
|
+
|
|
129
|
+
try:
|
|
130
|
+
rendered_services = _render_services(
|
|
131
|
+
module,
|
|
132
|
+
services,
|
|
133
|
+
secrets,
|
|
134
|
+
profile_dir=profile_dir,
|
|
135
|
+
project_root=project_root,
|
|
136
|
+
compose_dir=compose_dir,
|
|
137
|
+
network_name=default_network_name,
|
|
138
|
+
)
|
|
139
|
+
rendered_volumes = _render_volumes(module, volumes, secrets)
|
|
140
|
+
except MaxNestingDepthExceeded:
|
|
141
|
+
diagnostics.append(Diagnostic(
|
|
142
|
+
level="error",
|
|
143
|
+
code="E094",
|
|
144
|
+
message=(
|
|
145
|
+
f'Module "{module.get("id")}" service/volume templates exceed the '
|
|
146
|
+
f"maximum supported nesting depth ({MAX_NESTING_DEPTH})."
|
|
147
|
+
),
|
|
148
|
+
path=f'module:{module.get("id")}.implementation.compose',
|
|
149
|
+
))
|
|
150
|
+
continue
|
|
151
|
+
|
|
152
|
+
# Handle initDbEnv for postgres service (merge additional env vars)
|
|
153
|
+
_merge_init_db_env(rendered_services, module, secrets)
|
|
154
|
+
|
|
155
|
+
for service_name, service_def in rendered_services.items():
|
|
156
|
+
compose_service_name = _compose_service_name(module["id"], service_name)
|
|
157
|
+
compose["services"][compose_service_name] = service_def
|
|
158
|
+
module_service_names.setdefault(module["id"], []).append(compose_service_name)
|
|
159
|
+
|
|
160
|
+
for volume_name, volume_def in rendered_volumes.items():
|
|
161
|
+
compose["volumes"][f'{module["id"]}-{volume_name}'] = volume_def
|
|
162
|
+
|
|
163
|
+
if not compose["volumes"]:
|
|
164
|
+
compose.pop("volumes")
|
|
165
|
+
|
|
166
|
+
_add_cross_module_dependencies(compose, plan, module_service_names)
|
|
167
|
+
|
|
168
|
+
output = yaml.safe_dump(compose, sort_keys=False)
|
|
169
|
+
diagnostics.extend(_check_unresolved_expressions(output))
|
|
170
|
+
|
|
171
|
+
if output_path:
|
|
172
|
+
path = Path(output_path)
|
|
173
|
+
path.parent.mkdir(parents=True, exist_ok=True)
|
|
174
|
+
_atomic_write_compose(path, output)
|
|
175
|
+
|
|
176
|
+
return output, diagnostics
|
|
177
|
+
|
|
178
|
+
|
|
179
|
+
_UNRESOLVED_EXPRESSION_PATTERN = re.compile(
|
|
180
|
+
r"\$\{((?:config|bindings|service)\.[^}]*)\}"
|
|
181
|
+
)
|
|
182
|
+
|
|
183
|
+
|
|
184
|
+
def _check_unresolved_expressions(rendered_yaml: str) -> list[Diagnostic]:
|
|
185
|
+
"""
|
|
186
|
+
Detect leftover ${config.*}/${bindings.*}/${service.*} template expressions
|
|
187
|
+
that survived rendering unresolved (e.g. an optional consumed contract that
|
|
188
|
+
was never bound, but is unconditionally referenced by the module's
|
|
189
|
+
template). These are always a rendering bug -- unlike ${CDS_*}/${VAR}
|
|
190
|
+
placeholders, CDS's own template vocabulary is meant to be fully resolved
|
|
191
|
+
by render time, so leaving one in place would silently ship a broken
|
|
192
|
+
Compose file instead of failing loudly.
|
|
193
|
+
"""
|
|
194
|
+
unresolved = sorted(set(_UNRESOLVED_EXPRESSION_PATTERN.findall(rendered_yaml)))
|
|
195
|
+
return [
|
|
196
|
+
Diagnostic(
|
|
197
|
+
level="error",
|
|
198
|
+
code="E071",
|
|
199
|
+
message=(
|
|
200
|
+
f'Unresolved template expression "${{{expr}}}" remains in the rendered '
|
|
201
|
+
"output. This usually means an optional contract binding referenced by "
|
|
202
|
+
"a module's template was never satisfied by the profile."
|
|
203
|
+
),
|
|
204
|
+
path=f"rendered.{expr}",
|
|
205
|
+
)
|
|
206
|
+
for expr in unresolved
|
|
207
|
+
]
|
|
208
|
+
|
|
209
|
+
|
|
210
|
+
# ---------------------------------------------------------------------------
|
|
211
|
+
# Internal rendering helpers
|
|
212
|
+
# ---------------------------------------------------------------------------
|
|
213
|
+
|
|
214
|
+
def _render_services(
|
|
215
|
+
module: dict[str, Any],
|
|
216
|
+
services: dict[str, Any],
|
|
217
|
+
secrets: dict[str, str],
|
|
218
|
+
profile_dir: Path | None,
|
|
219
|
+
project_root: Path | None,
|
|
220
|
+
compose_dir: Path,
|
|
221
|
+
network_name: str | None = None,
|
|
222
|
+
) -> dict[str, Any]:
|
|
223
|
+
rendered: dict[str, Any] = {}
|
|
224
|
+
context = _build_context(module, secrets)
|
|
225
|
+
|
|
226
|
+
for service_name, service_def in services.items():
|
|
227
|
+
if not isinstance(service_def, dict):
|
|
228
|
+
continue
|
|
229
|
+
|
|
230
|
+
# Top-level enabledFrom guard
|
|
231
|
+
enabled_from = service_def.get("enabledFrom")
|
|
232
|
+
if enabled_from and _resolve_expr(enabled_from, context) is False:
|
|
233
|
+
continue
|
|
234
|
+
|
|
235
|
+
service_copy = deepcopy(service_def)
|
|
236
|
+
service_copy.pop("enabledFrom", None)
|
|
237
|
+
|
|
238
|
+
# Conditional healthcheck
|
|
239
|
+
healthcheck = service_copy.get("healthcheck")
|
|
240
|
+
if isinstance(healthcheck, dict):
|
|
241
|
+
cond = healthcheck.get("conditionallyEnabledFrom")
|
|
242
|
+
if cond:
|
|
243
|
+
hc_copy = deepcopy(healthcheck)
|
|
244
|
+
hc_copy.pop("conditionallyEnabledFrom", None)
|
|
245
|
+
if _resolve_expr(cond, context) is False:
|
|
246
|
+
service_copy.pop("healthcheck", None)
|
|
247
|
+
else:
|
|
248
|
+
service_copy["healthcheck"] = _substitute_values(hc_copy, context)
|
|
249
|
+
|
|
250
|
+
service_copy = _substitute_values(service_copy, context)
|
|
251
|
+
service_copy = _rewrite_service_volumes(
|
|
252
|
+
service_copy,
|
|
253
|
+
module,
|
|
254
|
+
profile_dir=profile_dir,
|
|
255
|
+
project_root=project_root,
|
|
256
|
+
compose_dir=compose_dir,
|
|
257
|
+
)
|
|
258
|
+
service_copy = _rewrite_depends_on(service_copy, module)
|
|
259
|
+
service_copy = _rewrite_build_context(
|
|
260
|
+
service_copy,
|
|
261
|
+
module,
|
|
262
|
+
profile_dir=profile_dir,
|
|
263
|
+
project_root=project_root,
|
|
264
|
+
compose_dir=compose_dir,
|
|
265
|
+
)
|
|
266
|
+
|
|
267
|
+
# Attach to the network if network_name is provided
|
|
268
|
+
if network_name:
|
|
269
|
+
service_copy["networks"] = [network_name]
|
|
270
|
+
|
|
271
|
+
rendered[service_name] = service_copy
|
|
272
|
+
|
|
273
|
+
return rendered
|
|
274
|
+
|
|
275
|
+
|
|
276
|
+
def _render_volumes(
|
|
277
|
+
module: dict[str, Any],
|
|
278
|
+
volumes: dict[str, Any],
|
|
279
|
+
secrets: dict[str, str],
|
|
280
|
+
) -> dict[str, Any]:
|
|
281
|
+
rendered: dict[str, Any] = {}
|
|
282
|
+
context = _build_context(module, secrets)
|
|
283
|
+
|
|
284
|
+
for volume_name, volume_def in volumes.items():
|
|
285
|
+
if isinstance(volume_def, dict):
|
|
286
|
+
enabled_from = volume_def.get("enabledFrom")
|
|
287
|
+
if enabled_from and _resolve_expr(enabled_from, context) is False:
|
|
288
|
+
continue
|
|
289
|
+
volume_copy = deepcopy(volume_def)
|
|
290
|
+
volume_copy.pop("enabledFrom", None)
|
|
291
|
+
rendered[volume_name] = _substitute_values(volume_copy, context)
|
|
292
|
+
else:
|
|
293
|
+
rendered[volume_name] = volume_def
|
|
294
|
+
|
|
295
|
+
return rendered
|
|
296
|
+
|
|
297
|
+
|
|
298
|
+
def _merge_init_db_env(
|
|
299
|
+
services: dict[str, Any],
|
|
300
|
+
module: dict[str, Any],
|
|
301
|
+
secrets: dict[str, str],
|
|
302
|
+
) -> None:
|
|
303
|
+
"""
|
|
304
|
+
Merge initDbEnv into postgres service environment variables.
|
|
305
|
+
|
|
306
|
+
If a module has config.initDbEnv, merge those environment variables
|
|
307
|
+
into any service named "postgres" in the rendered services.
|
|
308
|
+
"""
|
|
309
|
+
init_db_env = module.get("config", {}).get("initDbEnv")
|
|
310
|
+
if not init_db_env or not isinstance(init_db_env, dict):
|
|
311
|
+
return
|
|
312
|
+
|
|
313
|
+
# Find postgres service and merge env vars
|
|
314
|
+
postgres_service = services.get("postgres")
|
|
315
|
+
if postgres_service and isinstance(postgres_service, dict):
|
|
316
|
+
env = postgres_service.setdefault("environment", {})
|
|
317
|
+
if not isinstance(env, dict):
|
|
318
|
+
return
|
|
319
|
+
|
|
320
|
+
# Substitute values in init_db_env and merge
|
|
321
|
+
context = _build_context(module, secrets)
|
|
322
|
+
for key, value in init_db_env.items():
|
|
323
|
+
if isinstance(value, str):
|
|
324
|
+
# Resolve references like "secrets.xyz"
|
|
325
|
+
env[key] = _resolve_expr(value, context)
|
|
326
|
+
else:
|
|
327
|
+
env[key] = value
|
|
328
|
+
|
|
329
|
+
|
|
330
|
+
def _build_context(
|
|
331
|
+
module: dict[str, Any],
|
|
332
|
+
secrets: dict[str, str] | None = None,
|
|
333
|
+
) -> dict[str, Any]:
|
|
334
|
+
secrets = secrets or {}
|
|
335
|
+
bindings: dict[str, Any] = {}
|
|
336
|
+
|
|
337
|
+
for consume_name, consume_value in module.get("consumes", {}).items():
|
|
338
|
+
contract = consume_value.get("contract", {})
|
|
339
|
+
if isinstance(contract, dict):
|
|
340
|
+
spec = contract.get("spec", {})
|
|
341
|
+
bindings[consume_name] = spec if isinstance(spec, dict) else {}
|
|
342
|
+
|
|
343
|
+
return {
|
|
344
|
+
"config": module.get("config", {}),
|
|
345
|
+
"bindings": bindings,
|
|
346
|
+
"service": {"host": module.get("id")},
|
|
347
|
+
"secrets": secrets,
|
|
348
|
+
}
|
|
349
|
+
|
|
350
|
+
|
|
351
|
+
def _substitute_values(obj: Any, context: dict[str, Any], _depth: int = 0) -> Any:
|
|
352
|
+
"""Recursively substitute interpolation expressions in obj."""
|
|
353
|
+
if _depth > MAX_NESTING_DEPTH:
|
|
354
|
+
raise MaxNestingDepthExceeded(
|
|
355
|
+
f"Service/volume template nesting exceeds the maximum supported depth ({MAX_NESTING_DEPTH})."
|
|
356
|
+
)
|
|
357
|
+
if isinstance(obj, dict):
|
|
358
|
+
return {k: _substitute_values(v, context, _depth + 1) for k, v in obj.items()}
|
|
359
|
+
if isinstance(obj, list):
|
|
360
|
+
return [_substitute_values(v, context, _depth + 1) for v in obj]
|
|
361
|
+
if isinstance(obj, str):
|
|
362
|
+
return _substitute_string(obj, context)
|
|
363
|
+
return obj
|
|
364
|
+
|
|
365
|
+
|
|
366
|
+
def _substitute_string(value: str, context: dict[str, Any]) -> Any:
|
|
367
|
+
"""
|
|
368
|
+
Substitute interpolations in a string.
|
|
369
|
+
|
|
370
|
+
Supports:
|
|
371
|
+
- Pure: "${config.name}" → value of config.name (any type)
|
|
372
|
+
- Mixed: "db://${bindings.db.host}:5432" → "db://postgres:5432"
|
|
373
|
+
- Secret: "${secrets.alias_or_env_name}" → "${CDS_ENV_NAME}"
|
|
374
|
+
"""
|
|
375
|
+
_PATTERN = re.compile(r"\$\{([^}]+)\}")
|
|
376
|
+
matches = _PATTERN.findall(value)
|
|
377
|
+
|
|
378
|
+
if not matches:
|
|
379
|
+
return value
|
|
380
|
+
|
|
381
|
+
# Pure substitution: entire string is a single expression
|
|
382
|
+
if len(matches) == 1 and value == f"${{{matches[0]}}}":
|
|
383
|
+
result = _resolve_expr(matches[0], context)
|
|
384
|
+
return result if result is not None else value
|
|
385
|
+
|
|
386
|
+
# Mixed substitution: string-concat all expressions
|
|
387
|
+
def _replace(match: re.Match) -> str:
|
|
388
|
+
resolved = _resolve_expr(match.group(1), context)
|
|
389
|
+
return str(resolved) if resolved is not None else match.group(0)
|
|
390
|
+
|
|
391
|
+
return _PATTERN.sub(_replace, value)
|
|
392
|
+
|
|
393
|
+
|
|
394
|
+
def _resolve_expr(expr: str, context: dict[str, Any]) -> Any:
|
|
395
|
+
"""
|
|
396
|
+
Resolve a dot-notation expression against context.
|
|
397
|
+
|
|
398
|
+
Secrets are emitted as Docker Compose runtime placeholders (${VAR}).
|
|
399
|
+
Raw secret values are never returned.
|
|
400
|
+
|
|
401
|
+
Returns None if the path is not found.
|
|
402
|
+
"""
|
|
403
|
+
if expr.startswith("secrets."):
|
|
404
|
+
alias = expr.split(".", 1)[1]
|
|
405
|
+
secret_map = context.get("secrets", {})
|
|
406
|
+
env_name = secret_map.get(alias, alias)
|
|
407
|
+
return f"${{{env_name}}}"
|
|
408
|
+
|
|
409
|
+
current: Any = context
|
|
410
|
+
for part in expr.split("."):
|
|
411
|
+
if not isinstance(current, dict) or part not in current:
|
|
412
|
+
return None
|
|
413
|
+
current = current[part]
|
|
414
|
+
|
|
415
|
+
if isinstance(current, str) and current.startswith("secrets."):
|
|
416
|
+
alias = current.split(".", 1)[1]
|
|
417
|
+
secret_map = context.get("secrets", {})
|
|
418
|
+
env_name = secret_map.get(alias, alias)
|
|
419
|
+
return f"${{{env_name}}}"
|
|
420
|
+
|
|
421
|
+
return current
|
|
422
|
+
|
|
423
|
+
|
|
424
|
+
def _rewrite_service_volumes(
|
|
425
|
+
service_def: dict[str, Any],
|
|
426
|
+
module: dict[str, Any],
|
|
427
|
+
profile_dir: Path | None,
|
|
428
|
+
project_root: Path | None,
|
|
429
|
+
compose_dir: Path,
|
|
430
|
+
) -> dict[str, Any]:
|
|
431
|
+
volumes = service_def.get("volumes")
|
|
432
|
+
if not isinstance(volumes, list):
|
|
433
|
+
return service_def
|
|
434
|
+
|
|
435
|
+
rewritten: list[Any] = []
|
|
436
|
+
for item in volumes:
|
|
437
|
+
if isinstance(item, str):
|
|
438
|
+
parts = item.split(":", 1)
|
|
439
|
+
if len(parts) == 2 and _is_named_volume(parts[0]):
|
|
440
|
+
item = f"{module['id']}-{parts[0]}:{parts[1]}"
|
|
441
|
+
elif len(parts) >= 2:
|
|
442
|
+
source = parts[0]
|
|
443
|
+
rewritten_source = _rewrite_local_path(
|
|
444
|
+
source,
|
|
445
|
+
module=module,
|
|
446
|
+
profile_dir=profile_dir,
|
|
447
|
+
project_root=project_root,
|
|
448
|
+
compose_dir=compose_dir,
|
|
449
|
+
)
|
|
450
|
+
if rewritten_source != source:
|
|
451
|
+
item = f"{rewritten_source}:{parts[1]}"
|
|
452
|
+
elif isinstance(item, dict):
|
|
453
|
+
item_copy = deepcopy(item)
|
|
454
|
+
if item_copy.get("type") == "bind" and isinstance(item_copy.get("source"), str):
|
|
455
|
+
item_copy["source"] = _rewrite_local_path(
|
|
456
|
+
item_copy["source"],
|
|
457
|
+
module=module,
|
|
458
|
+
profile_dir=profile_dir,
|
|
459
|
+
project_root=project_root,
|
|
460
|
+
compose_dir=compose_dir,
|
|
461
|
+
)
|
|
462
|
+
elif item_copy.get("type") == "volume" and isinstance(item_copy.get("source"), str):
|
|
463
|
+
source = item_copy["source"]
|
|
464
|
+
if _is_named_volume(source):
|
|
465
|
+
item_copy["source"] = f"{module['id']}-{source}"
|
|
466
|
+
item = item_copy
|
|
467
|
+
rewritten.append(item)
|
|
468
|
+
|
|
469
|
+
return {**service_def, "volumes": rewritten}
|
|
470
|
+
|
|
471
|
+
|
|
472
|
+
def _rewrite_local_path(
|
|
473
|
+
path_value: str,
|
|
474
|
+
module: dict[str, Any],
|
|
475
|
+
profile_dir: Path | None,
|
|
476
|
+
project_root: Path | None,
|
|
477
|
+
compose_dir: Path,
|
|
478
|
+
) -> str:
|
|
479
|
+
if Path(path_value).is_absolute() or _looks_remote_context(path_value) or "${" in path_value:
|
|
480
|
+
return path_value
|
|
481
|
+
|
|
482
|
+
candidates: list[Path] = []
|
|
483
|
+
for base in _local_path_bases(module, profile_dir, project_root, compose_dir):
|
|
484
|
+
candidate = (base / path_value).resolve()
|
|
485
|
+
if candidate not in candidates:
|
|
486
|
+
candidates.append(candidate)
|
|
487
|
+
|
|
488
|
+
if not candidates:
|
|
489
|
+
return path_value
|
|
490
|
+
|
|
491
|
+
chosen = _choose_best_local_path_candidate(candidates)
|
|
492
|
+
if project_root is not None and not _path_is_within_root(compose_dir, project_root):
|
|
493
|
+
try:
|
|
494
|
+
return Path(chosen).relative_to(project_root).as_posix()
|
|
495
|
+
except ValueError:
|
|
496
|
+
pass
|
|
497
|
+
try:
|
|
498
|
+
rel = Path(chosen).relative_to(compose_dir)
|
|
499
|
+
except ValueError:
|
|
500
|
+
# relative_to() only works for descendants; relpath preserves ../ segments.
|
|
501
|
+
try:
|
|
502
|
+
rel = Path(os.path.relpath(chosen, compose_dir))
|
|
503
|
+
except ValueError:
|
|
504
|
+
# On Windows, relpath raises when chosen and compose_dir are on
|
|
505
|
+
# different drives (e.g. C:\ vs D:\), no relative path can
|
|
506
|
+
# express that. Fall back to the absolute path, same as the
|
|
507
|
+
# is_absolute() short-circuit above.
|
|
508
|
+
return Path(chosen).as_posix()
|
|
509
|
+
return rel.as_posix()
|
|
510
|
+
|
|
511
|
+
|
|
512
|
+
def _local_path_bases(
|
|
513
|
+
module: dict[str, Any],
|
|
514
|
+
profile_dir: Path | None,
|
|
515
|
+
project_root: Path | None,
|
|
516
|
+
compose_dir: Path,
|
|
517
|
+
) -> list[Path]:
|
|
518
|
+
bases: list[Path] = []
|
|
519
|
+
|
|
520
|
+
if project_root is not None:
|
|
521
|
+
bases.append(project_root)
|
|
522
|
+
|
|
523
|
+
if profile_dir is not None:
|
|
524
|
+
bases.append(profile_dir)
|
|
525
|
+
|
|
526
|
+
module_dir = _resolve_module_dir(module, profile_dir)
|
|
527
|
+
if module_dir is not None:
|
|
528
|
+
bases.append(module_dir)
|
|
529
|
+
|
|
530
|
+
bases.append(compose_dir)
|
|
531
|
+
|
|
532
|
+
if project_root is not None:
|
|
533
|
+
bases.append((project_root / "build").resolve())
|
|
534
|
+
|
|
535
|
+
return bases
|
|
536
|
+
|
|
537
|
+
|
|
538
|
+
def _choose_best_local_path_candidate(candidates: list[Path]) -> Path:
|
|
539
|
+
for candidate in candidates:
|
|
540
|
+
if candidate.exists():
|
|
541
|
+
return candidate
|
|
542
|
+
|
|
543
|
+
return candidates[0]
|
|
544
|
+
|
|
545
|
+
|
|
546
|
+
def _rewrite_depends_on(
|
|
547
|
+
service_def: dict[str, Any],
|
|
548
|
+
module: dict[str, Any],
|
|
549
|
+
) -> dict[str, Any]:
|
|
550
|
+
depends_on = service_def.get("depends_on")
|
|
551
|
+
if depends_on is None:
|
|
552
|
+
return service_def
|
|
553
|
+
|
|
554
|
+
if isinstance(depends_on, list):
|
|
555
|
+
rewritten = {
|
|
556
|
+
_compose_service_name(module["id"], dep): {"condition": "service_started"}
|
|
557
|
+
for dep in depends_on
|
|
558
|
+
}
|
|
559
|
+
elif isinstance(depends_on, dict):
|
|
560
|
+
rewritten = {
|
|
561
|
+
_compose_service_name(module["id"], dep): val
|
|
562
|
+
for dep, val in depends_on.items()
|
|
563
|
+
}
|
|
564
|
+
else:
|
|
565
|
+
return service_def
|
|
566
|
+
|
|
567
|
+
return {**service_def, "depends_on": rewritten}
|
|
568
|
+
|
|
569
|
+
|
|
570
|
+
def _rewrite_build_context(
|
|
571
|
+
service_def: dict[str, Any],
|
|
572
|
+
module: dict[str, Any],
|
|
573
|
+
profile_dir: Path | None,
|
|
574
|
+
project_root: Path | None,
|
|
575
|
+
compose_dir: Path,
|
|
576
|
+
) -> dict[str, Any]:
|
|
577
|
+
build = service_def.get("build")
|
|
578
|
+
if build is None:
|
|
579
|
+
return service_def
|
|
580
|
+
|
|
581
|
+
if isinstance(build, str):
|
|
582
|
+
rewritten = _resolve_context_path(
|
|
583
|
+
context=build,
|
|
584
|
+
dockerfile=None,
|
|
585
|
+
module=module,
|
|
586
|
+
profile_dir=profile_dir,
|
|
587
|
+
project_root=project_root,
|
|
588
|
+
compose_dir=compose_dir,
|
|
589
|
+
)
|
|
590
|
+
return {**service_def, "build": rewritten}
|
|
591
|
+
|
|
592
|
+
if isinstance(build, dict):
|
|
593
|
+
context = build.get("context")
|
|
594
|
+
if not isinstance(context, str):
|
|
595
|
+
return service_def
|
|
596
|
+
|
|
597
|
+
dockerfile = build.get("dockerfile")
|
|
598
|
+
rewritten = _resolve_context_path(
|
|
599
|
+
context=context,
|
|
600
|
+
dockerfile=dockerfile if isinstance(dockerfile, str) else None,
|
|
601
|
+
module=module,
|
|
602
|
+
profile_dir=profile_dir,
|
|
603
|
+
project_root=project_root,
|
|
604
|
+
compose_dir=compose_dir,
|
|
605
|
+
)
|
|
606
|
+
return {**service_def, "build": {**build, "context": rewritten}}
|
|
607
|
+
|
|
608
|
+
return service_def
|
|
609
|
+
|
|
610
|
+
|
|
611
|
+
def _resolve_context_path(
|
|
612
|
+
context: str,
|
|
613
|
+
dockerfile: str | None,
|
|
614
|
+
module: dict[str, Any],
|
|
615
|
+
profile_dir: Path | None,
|
|
616
|
+
project_root: Path | None,
|
|
617
|
+
compose_dir: Path,
|
|
618
|
+
) -> str:
|
|
619
|
+
# Keep absolute paths and remote contexts unchanged.
|
|
620
|
+
if Path(context).is_absolute() or _looks_remote_context(context) or "${" in context:
|
|
621
|
+
return context
|
|
622
|
+
|
|
623
|
+
candidates: list[Path] = []
|
|
624
|
+
for base in _context_bases(module, profile_dir, project_root, compose_dir):
|
|
625
|
+
candidate = (base / context).resolve()
|
|
626
|
+
if candidate not in candidates:
|
|
627
|
+
candidates.append(candidate)
|
|
628
|
+
|
|
629
|
+
if not candidates:
|
|
630
|
+
return context
|
|
631
|
+
|
|
632
|
+
chosen = _choose_best_context_candidate(candidates, dockerfile)
|
|
633
|
+
if project_root is not None and not _path_is_within_root(compose_dir, project_root):
|
|
634
|
+
try:
|
|
635
|
+
return Path(chosen).relative_to(project_root).as_posix()
|
|
636
|
+
except ValueError:
|
|
637
|
+
pass
|
|
638
|
+
try:
|
|
639
|
+
rel = Path(chosen).relative_to(compose_dir)
|
|
640
|
+
except ValueError:
|
|
641
|
+
# relative_to() only works for descendants; relpath preserves ../ segments.
|
|
642
|
+
try:
|
|
643
|
+
rel = Path(os.path.relpath(chosen, compose_dir))
|
|
644
|
+
except ValueError:
|
|
645
|
+
# On Windows, relpath raises when chosen and compose_dir are on
|
|
646
|
+
# different drives (e.g. C:\ vs D:\), no relative path can
|
|
647
|
+
# express that. Fall back to the absolute path, same as the
|
|
648
|
+
# is_absolute() short-circuit above.
|
|
649
|
+
return Path(chosen).as_posix()
|
|
650
|
+
return Path(rel).as_posix()
|
|
651
|
+
|
|
652
|
+
|
|
653
|
+
def _context_bases(
|
|
654
|
+
module: dict[str, Any],
|
|
655
|
+
profile_dir: Path | None,
|
|
656
|
+
project_root: Path | None,
|
|
657
|
+
compose_dir: Path,
|
|
658
|
+
) -> list[Path]:
|
|
659
|
+
bases: list[Path] = []
|
|
660
|
+
|
|
661
|
+
if project_root is not None:
|
|
662
|
+
bases.append(project_root)
|
|
663
|
+
|
|
664
|
+
bases.append(compose_dir)
|
|
665
|
+
|
|
666
|
+
module_dir = _resolve_module_dir(module, profile_dir)
|
|
667
|
+
if module_dir is not None:
|
|
668
|
+
bases.append(module_dir)
|
|
669
|
+
|
|
670
|
+
if project_root is not None:
|
|
671
|
+
# Legacy compose path in this repo used to be project_root/build.
|
|
672
|
+
bases.append((project_root / "build").resolve())
|
|
673
|
+
|
|
674
|
+
return bases
|
|
675
|
+
|
|
676
|
+
|
|
677
|
+
def _choose_best_context_candidate(candidates: list[Path], dockerfile: str | None) -> Path:
|
|
678
|
+
if dockerfile:
|
|
679
|
+
for candidate in candidates:
|
|
680
|
+
if candidate.exists() and (candidate / dockerfile).exists():
|
|
681
|
+
return candidate
|
|
682
|
+
|
|
683
|
+
for candidate in candidates:
|
|
684
|
+
if candidate.exists():
|
|
685
|
+
return candidate
|
|
686
|
+
|
|
687
|
+
return candidates[0]
|
|
688
|
+
|
|
689
|
+
|
|
690
|
+
def _looks_remote_context(value: str) -> bool:
|
|
691
|
+
return "://" in value or value.startswith("git@")
|
|
692
|
+
|
|
693
|
+
|
|
694
|
+
def _path_is_within_root(path_value: Path, root: Path) -> bool:
|
|
695
|
+
try:
|
|
696
|
+
path_value.resolve().relative_to(root.resolve())
|
|
697
|
+
except ValueError:
|
|
698
|
+
return False
|
|
699
|
+
return True
|
|
700
|
+
|
|
701
|
+
|
|
702
|
+
def _resolve_profile_dir(plan: dict[str, Any]) -> Path | None:
|
|
703
|
+
source_profile = plan.get("sourceProfile")
|
|
704
|
+
if not isinstance(source_profile, str):
|
|
705
|
+
return None
|
|
706
|
+
return Path(source_profile).resolve().parent
|
|
707
|
+
|
|
708
|
+
|
|
709
|
+
def _resolve_project_root(profile_dir: Path | None) -> Path | None:
|
|
710
|
+
if profile_dir is None:
|
|
711
|
+
return None
|
|
712
|
+
|
|
713
|
+
for directory in [profile_dir, *profile_dir.parents]:
|
|
714
|
+
if (directory / "pyproject.toml").exists() or (directory / ".git").exists():
|
|
715
|
+
return directory
|
|
716
|
+
|
|
717
|
+
return None
|
|
718
|
+
|
|
719
|
+
|
|
720
|
+
def _resolve_module_dir(module: dict[str, Any], profile_dir: Path | None) -> Path | None:
|
|
721
|
+
source = module.get("source")
|
|
722
|
+
if not isinstance(source, str):
|
|
723
|
+
return None
|
|
724
|
+
|
|
725
|
+
module_root = os.getenv("CDS_MODULE_PATH")
|
|
726
|
+
module_root_path = Path(module_root) if module_root else None
|
|
727
|
+
|
|
728
|
+
return resolve_module_dir(source, profile_dir, module_root=module_root_path)
|
|
729
|
+
|
|
730
|
+
|
|
731
|
+
def _is_named_volume(value: str) -> bool:
|
|
732
|
+
if value.startswith((".", "/", "~")):
|
|
733
|
+
return False
|
|
734
|
+
return "/" not in value and "\\" not in value
|
|
735
|
+
|
|
736
|
+
|
|
737
|
+
def _compose_service_name(module_id: str, service_name: str) -> str:
|
|
738
|
+
"""Normalize compose service names so module prefixes are not duplicated."""
|
|
739
|
+
if service_name == module_id or service_name.startswith(f"{module_id}-"):
|
|
740
|
+
return service_name
|
|
741
|
+
return f"{module_id}-{service_name}"
|
|
742
|
+
|
|
743
|
+
|
|
744
|
+
def _add_cross_module_dependencies(
|
|
745
|
+
compose: dict[str, Any],
|
|
746
|
+
plan: dict[str, Any],
|
|
747
|
+
module_service_names: dict[str, list[str]],
|
|
748
|
+
) -> None:
|
|
749
|
+
"""
|
|
750
|
+
Add explicit cross-module dependencies to docker-compose services.
|
|
751
|
+
|
|
752
|
+
For each module that has dependsOn declarations, add depends_on entries
|
|
753
|
+
to all its services, referencing all services from the dependent modules.
|
|
754
|
+
"""
|
|
755
|
+
modules = plan.get("modules", [])
|
|
756
|
+
services = compose.get("services", {})
|
|
757
|
+
|
|
758
|
+
# For each module, add depends_on for its dependencies
|
|
759
|
+
for module in modules:
|
|
760
|
+
module_id = module.get("id")
|
|
761
|
+
depends_on = module.get("dependsOn", [])
|
|
762
|
+
|
|
763
|
+
if not depends_on or not module_id:
|
|
764
|
+
continue
|
|
765
|
+
|
|
766
|
+
# Find all services belonging to this module
|
|
767
|
+
current_module_service_names = module_service_names.get(module_id, [])
|
|
768
|
+
|
|
769
|
+
# For each service in this module, add depends_on entries
|
|
770
|
+
for service_name in current_module_service_names:
|
|
771
|
+
service_def = services.get(service_name)
|
|
772
|
+
if not service_def:
|
|
773
|
+
continue
|
|
774
|
+
|
|
775
|
+
# Collect all services from dependent modules
|
|
776
|
+
for dep_module_id in depends_on:
|
|
777
|
+
dep_services = module_service_names.get(dep_module_id, [])
|
|
778
|
+
|
|
779
|
+
for dep_service_name in dep_services:
|
|
780
|
+
# Initialize depends_on if not present
|
|
781
|
+
if "depends_on" not in service_def:
|
|
782
|
+
service_def["depends_on"] = {}
|
|
783
|
+
|
|
784
|
+
# Add the dependency with a started condition
|
|
785
|
+
if isinstance(service_def["depends_on"], dict):
|
|
786
|
+
service_def["depends_on"][dep_service_name] = {
|
|
787
|
+
"condition": "service_healthy"
|
|
788
|
+
}
|
|
789
|
+
elif isinstance(service_def["depends_on"], list):
|
|
790
|
+
if dep_service_name not in service_def["depends_on"]:
|
|
791
|
+
service_def["depends_on"].append(dep_service_name)
|