composable-data-stack 0.14.0__py3-none-any.whl → 0.15.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- cli/getter.py +624 -13
- cli/image_updates.py +12 -6
- cli/k8s_renderer.py +639 -2
- cli/k8s_runner.py +1 -1
- cli/k8s_security.py +96 -3
- cli/main.py +144 -30
- cli/maturity.py +271 -0
- cli/planner.py +220 -15
- cli/renderer.py +223 -10
- cli/resources/module.schema.json +147 -3
- cli/resources/profile.schema.json +11 -1
- cli/resources/rule-schema.json +5 -0
- cli/resources/rule-set.json +10 -7
- cli/secrets.py +4 -1
- cli/security.py +122 -29
- cli/security_common.py +22 -0
- cli/up_runner.py +38 -4
- cli/utils.py +50 -0
- cli/validator.py +513 -11
- {composable_data_stack-0.14.0.dist-info → composable_data_stack-0.15.0.dist-info}/METADATA +22 -3
- composable_data_stack-0.15.0.dist-info/RECORD +46 -0
- composable_data_stack-0.14.0.dist-info/RECORD +0 -45
- {composable_data_stack-0.14.0.dist-info → composable_data_stack-0.15.0.dist-info}/WHEEL +0 -0
- {composable_data_stack-0.14.0.dist-info → composable_data_stack-0.15.0.dist-info}/entry_points.txt +0 -0
- {composable_data_stack-0.14.0.dist-info → composable_data_stack-0.15.0.dist-info}/licenses/LICENSE +0 -0
- {composable_data_stack-0.14.0.dist-info → composable_data_stack-0.15.0.dist-info}/top_level.txt +0 -0
cli/getter.py
CHANGED
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
from __future__ import annotations
|
|
2
2
|
|
|
3
|
+
import hashlib
|
|
3
4
|
import json
|
|
4
5
|
import os
|
|
5
6
|
import re
|
|
@@ -12,7 +13,7 @@ from collections.abc import Iterator, Sequence
|
|
|
12
13
|
from contextlib import contextmanager
|
|
13
14
|
from dataclasses import dataclass
|
|
14
15
|
from datetime import UTC, datetime
|
|
15
|
-
from pathlib import Path
|
|
16
|
+
from pathlib import Path, PurePosixPath
|
|
16
17
|
from typing import Any
|
|
17
18
|
from urllib.error import HTTPError, URLError
|
|
18
19
|
from urllib.parse import quote
|
|
@@ -21,6 +22,7 @@ from urllib.request import Request, urlopen
|
|
|
21
22
|
from .loader import load_yaml_file, resolve_module_dir
|
|
22
23
|
from .overlay import _derive_profiles_root, _resolve_extends_ref, resolve_extends
|
|
23
24
|
from .planner import MaxNestingDepthExceeded, apply_defaults, substitute_string
|
|
25
|
+
from .utils import _atomic_write
|
|
24
26
|
|
|
25
27
|
# The upstream repository `cds get` downloads from when no `--remote` is
|
|
26
28
|
# given. Keep in sync with the `Repository` URL in pyproject.toml.
|
|
@@ -103,6 +105,15 @@ class CopyAction:
|
|
|
103
105
|
repo_relative_path: str
|
|
104
106
|
|
|
105
107
|
|
|
108
|
+
@dataclass
|
|
109
|
+
class UpdateResult:
|
|
110
|
+
manifest_path: Path
|
|
111
|
+
updated: list[str]
|
|
112
|
+
skipped: list[str]
|
|
113
|
+
conflicts: list[str]
|
|
114
|
+
errors: list[str]
|
|
115
|
+
|
|
116
|
+
|
|
106
117
|
def fetch_profile(
|
|
107
118
|
profile: str,
|
|
108
119
|
*,
|
|
@@ -159,6 +170,359 @@ def fetch_profile(
|
|
|
159
170
|
return actions, target_root / _TRACKING_FILE, conflicts
|
|
160
171
|
|
|
161
172
|
|
|
173
|
+
def update_tracked_assets(
|
|
174
|
+
*,
|
|
175
|
+
profile: str | None = None,
|
|
176
|
+
destination_root: Path | None = None,
|
|
177
|
+
categories: Sequence[str] | None = None,
|
|
178
|
+
dry_run: bool = False,
|
|
179
|
+
allowed_sources: Sequence[str] | None = None,
|
|
180
|
+
) -> UpdateResult:
|
|
181
|
+
"""Refresh previously fetched assets without overwriting local edits."""
|
|
182
|
+
target_root = (destination_root or Path.cwd()).expanduser().resolve()
|
|
183
|
+
manifest_path = target_root / _TRACKING_FILE
|
|
184
|
+
if manifest_path.parent.is_symlink() or manifest_path.is_symlink():
|
|
185
|
+
raise GetError(f"Refusing to update through symlinked tracking metadata: {manifest_path}")
|
|
186
|
+
if not manifest_path.is_file():
|
|
187
|
+
raise GetError(f"No fetched-asset tracking manifest found at {manifest_path}; run `cds get` first.")
|
|
188
|
+
|
|
189
|
+
manifest = _read_tracking_manifest(manifest_path)
|
|
190
|
+
profiles = manifest.get("profiles")
|
|
191
|
+
if not isinstance(profiles, dict) or not profiles:
|
|
192
|
+
raise GetError(f"No tracked profiles found in {manifest_path}; run `cds get` first.")
|
|
193
|
+
|
|
194
|
+
selected_categories = set(categories) if categories else None
|
|
195
|
+
supported_categories = {"profiles", "modules", "contracts"}
|
|
196
|
+
if selected_categories and not selected_categories.issubset(supported_categories):
|
|
197
|
+
raise GetError(
|
|
198
|
+
"Unsupported update category: "
|
|
199
|
+
+ ", ".join(sorted(selected_categories - supported_categories))
|
|
200
|
+
)
|
|
201
|
+
|
|
202
|
+
selected_profiles = _select_tracked_profiles(profiles, profile)
|
|
203
|
+
if not selected_profiles:
|
|
204
|
+
raise GetError(
|
|
205
|
+
f'No fetched profile matching "{profile}" was found in {manifest_path}.'
|
|
206
|
+
if profile
|
|
207
|
+
else f"No tracked profiles found in {manifest_path}."
|
|
208
|
+
)
|
|
209
|
+
|
|
210
|
+
result = UpdateResult(manifest_path, [], [], [], [])
|
|
211
|
+
for profile_name, entry in selected_profiles:
|
|
212
|
+
if not isinstance(entry, dict):
|
|
213
|
+
result.errors.append(f"{profile_name}: tracking entry is not an object")
|
|
214
|
+
continue
|
|
215
|
+
try:
|
|
216
|
+
_update_tracked_profile(
|
|
217
|
+
profile_name=profile_name,
|
|
218
|
+
entry=entry,
|
|
219
|
+
manifest=manifest,
|
|
220
|
+
manifest_path=manifest_path,
|
|
221
|
+
destination_root=target_root,
|
|
222
|
+
categories=selected_categories,
|
|
223
|
+
dry_run=dry_run,
|
|
224
|
+
allowed_sources=allowed_sources,
|
|
225
|
+
result=result,
|
|
226
|
+
)
|
|
227
|
+
except GetError as exc:
|
|
228
|
+
result.errors.append(f"{profile_name}: {exc}")
|
|
229
|
+
return result
|
|
230
|
+
|
|
231
|
+
|
|
232
|
+
def _select_tracked_profiles(
|
|
233
|
+
profiles: dict[str, Any], requested: str | None
|
|
234
|
+
) -> list[tuple[str, Any]]:
|
|
235
|
+
entries = list(profiles.items())
|
|
236
|
+
if requested is None:
|
|
237
|
+
return [(name, entry) for name, entry in entries]
|
|
238
|
+
matches = [
|
|
239
|
+
(name, entry)
|
|
240
|
+
for name, entry in entries
|
|
241
|
+
if requested == name
|
|
242
|
+
or (
|
|
243
|
+
isinstance(entry, dict)
|
|
244
|
+
and requested in (entry.get("requestedProfile"), entry.get("sourceProfile"))
|
|
245
|
+
)
|
|
246
|
+
]
|
|
247
|
+
return matches
|
|
248
|
+
|
|
249
|
+
|
|
250
|
+
def _update_tracked_profile(
|
|
251
|
+
*,
|
|
252
|
+
profile_name: str,
|
|
253
|
+
entry: dict[str, Any],
|
|
254
|
+
manifest: dict[str, Any],
|
|
255
|
+
manifest_path: Path,
|
|
256
|
+
destination_root: Path,
|
|
257
|
+
categories: set[str] | None,
|
|
258
|
+
dry_run: bool,
|
|
259
|
+
allowed_sources: Sequence[str] | None,
|
|
260
|
+
result: UpdateResult,
|
|
261
|
+
) -> None:
|
|
262
|
+
requested_profile = entry.get("requestedProfile", profile_name)
|
|
263
|
+
if not isinstance(requested_profile, str) or not requested_profile:
|
|
264
|
+
raise GetError("tracking entry is missing its requested profile")
|
|
265
|
+
remote = entry.get("remote")
|
|
266
|
+
if not isinstance(remote, str) or not remote:
|
|
267
|
+
raise GetError("tracking entry is missing its source repository")
|
|
268
|
+
stored_ref = entry.get("ref")
|
|
269
|
+
if stored_ref is not None and not isinstance(stored_ref, str):
|
|
270
|
+
raise GetError("tracking entry has an invalid source revision")
|
|
271
|
+
if stored_ref == "":
|
|
272
|
+
raise GetError("tracking entry has an empty source revision")
|
|
273
|
+
hashes = entry.get("fileHashes")
|
|
274
|
+
if not isinstance(hashes, dict) or not isinstance(entry.get("assets"), list):
|
|
275
|
+
raise GetError(
|
|
276
|
+
"tracking metadata lacks file hashes or asset provenance; run `cds get` "
|
|
277
|
+
"with a version that records tracked assets before updating"
|
|
278
|
+
)
|
|
279
|
+
|
|
280
|
+
local_source = remote if stored_ref is None else None
|
|
281
|
+
source_remote = None if stored_ref is None else remote
|
|
282
|
+
source_ref = DEFAULT_REF if stored_ref is None else stored_ref
|
|
283
|
+
pinned_commit = entry.get("pinnedCommit")
|
|
284
|
+
if pinned_commit is not None and not isinstance(pinned_commit, str):
|
|
285
|
+
raise GetError("tracking entry has an invalid pinned commit")
|
|
286
|
+
if local_source is not None and not Path(local_source).is_absolute():
|
|
287
|
+
raise GetError(
|
|
288
|
+
"tracking entry contains a relative local source path; run `cds get` again "
|
|
289
|
+
"to record an absolute source path before updating"
|
|
290
|
+
)
|
|
291
|
+
|
|
292
|
+
with _prepare_source(
|
|
293
|
+
source_remote,
|
|
294
|
+
source_ref,
|
|
295
|
+
local_source,
|
|
296
|
+
allowed_sources=allowed_sources,
|
|
297
|
+
commit=pinned_commit,
|
|
298
|
+
) as (source_repo, resolved_commit):
|
|
299
|
+
source_profile = entry.get("sourceProfile")
|
|
300
|
+
if not isinstance(source_profile, str) or not source_profile:
|
|
301
|
+
source_profile = requested_profile
|
|
302
|
+
profile_path = _resolve_source_profile_path(source_repo, source_profile)
|
|
303
|
+
asset_roots = _collect_asset_roots(source_repo, profile_path)
|
|
304
|
+
assets = _build_tracking_assets(
|
|
305
|
+
source_repo=source_repo,
|
|
306
|
+
requested_profile=requested_profile,
|
|
307
|
+
profile_path=profile_path,
|
|
308
|
+
asset_roots=asset_roots,
|
|
309
|
+
)
|
|
310
|
+
roots = _select_update_roots(
|
|
311
|
+
source_repo=source_repo,
|
|
312
|
+
asset_roots=asset_roots,
|
|
313
|
+
assets=assets,
|
|
314
|
+
categories=categories,
|
|
315
|
+
)
|
|
316
|
+
if not roots:
|
|
317
|
+
result.skipped.append(
|
|
318
|
+
f"{profile_name}: no tracked assets match the selected categories"
|
|
319
|
+
)
|
|
320
|
+
return
|
|
321
|
+
actions = _build_copy_plan(source_repo, roots, destination_root)
|
|
322
|
+
action_paths = {action.repo_relative_path for action in actions}
|
|
323
|
+
|
|
324
|
+
updates: list[CopyAction] = []
|
|
325
|
+
latest_hashes: dict[str, str] = {}
|
|
326
|
+
for action in actions:
|
|
327
|
+
rel_path = action.repo_relative_path
|
|
328
|
+
try:
|
|
329
|
+
source_hash = _sha256_file(action.source)
|
|
330
|
+
except OSError as exc:
|
|
331
|
+
raise GetError(f"Could not read source asset {action.source}: {exc}") from exc
|
|
332
|
+
destination = action.destination
|
|
333
|
+
if _has_symlinked_parent(destination, destination_root):
|
|
334
|
+
result.conflicts.append(f"{profile_name}: {rel_path} (symlinked parent directory)")
|
|
335
|
+
continue
|
|
336
|
+
if destination.is_symlink():
|
|
337
|
+
result.conflicts.append(f"{profile_name}: {rel_path} (destination is a symlink)")
|
|
338
|
+
continue
|
|
339
|
+
if destination.is_dir():
|
|
340
|
+
result.conflicts.append(f"{profile_name}: {rel_path} (destination is a directory)")
|
|
341
|
+
continue
|
|
342
|
+
if not destination.exists():
|
|
343
|
+
if rel_path in hashes:
|
|
344
|
+
result.conflicts.append(f"{profile_name}: {rel_path} (tracked file was deleted locally)")
|
|
345
|
+
else:
|
|
346
|
+
updates.append(action)
|
|
347
|
+
latest_hashes[rel_path] = source_hash
|
|
348
|
+
continue
|
|
349
|
+
|
|
350
|
+
try:
|
|
351
|
+
local_hash = _sha256_file(destination)
|
|
352
|
+
except OSError as exc:
|
|
353
|
+
raise GetError(f"Could not read local asset {destination}: {exc}") from exc
|
|
354
|
+
if local_hash == source_hash:
|
|
355
|
+
result.skipped.append(f"{profile_name}: {rel_path} (already current)")
|
|
356
|
+
latest_hashes[rel_path] = source_hash
|
|
357
|
+
elif rel_path in hashes and isinstance(hashes[rel_path], str) and local_hash == hashes[rel_path]:
|
|
358
|
+
updates.append(action)
|
|
359
|
+
latest_hashes[rel_path] = source_hash
|
|
360
|
+
else:
|
|
361
|
+
result.conflicts.append(f"{profile_name}: {rel_path} (modified locally)")
|
|
362
|
+
|
|
363
|
+
_report_removed_source_files(
|
|
364
|
+
profile_name=profile_name,
|
|
365
|
+
entry=entry,
|
|
366
|
+
destination_root=destination_root,
|
|
367
|
+
categories=categories,
|
|
368
|
+
current_paths=action_paths,
|
|
369
|
+
result=result,
|
|
370
|
+
)
|
|
371
|
+
|
|
372
|
+
if not dry_run and updates:
|
|
373
|
+
_write_actions(updates)
|
|
374
|
+
for action in updates:
|
|
375
|
+
label = f"{profile_name}: {action.repo_relative_path}"
|
|
376
|
+
result.updated.append(f"would update {label}" if dry_run else label)
|
|
377
|
+
|
|
378
|
+
if dry_run:
|
|
379
|
+
return
|
|
380
|
+
|
|
381
|
+
for rel_path, file_hash in latest_hashes.items():
|
|
382
|
+
hashes[rel_path] = file_hash
|
|
383
|
+
tracked_files = entry.get("files")
|
|
384
|
+
if not isinstance(tracked_files, list):
|
|
385
|
+
tracked_files = []
|
|
386
|
+
entry["files"] = sorted(set(
|
|
387
|
+
[path for path in tracked_files if isinstance(path, str)] + list(latest_hashes)
|
|
388
|
+
))
|
|
389
|
+
entry["fileHashes"] = hashes
|
|
390
|
+
entry["sourceProfile"] = profile_path.relative_to(source_repo).as_posix()
|
|
391
|
+
entry["assetRoots"] = [
|
|
392
|
+
_asset_root_relative_path(root, source_repo) for root in asset_roots
|
|
393
|
+
]
|
|
394
|
+
entry["assets"] = assets
|
|
395
|
+
entry["commit"] = resolved_commit
|
|
396
|
+
entry["fetchedAt"] = datetime.now(UTC).isoformat()
|
|
397
|
+
_write_updated_manifest(manifest_path, manifest)
|
|
398
|
+
|
|
399
|
+
|
|
400
|
+
def _select_update_roots(
|
|
401
|
+
*,
|
|
402
|
+
source_repo: Path,
|
|
403
|
+
asset_roots: list[Path],
|
|
404
|
+
assets: list[dict[str, Any]],
|
|
405
|
+
categories: set[str] | None,
|
|
406
|
+
) -> list[Path]:
|
|
407
|
+
if categories is None:
|
|
408
|
+
return asset_roots
|
|
409
|
+
|
|
410
|
+
selected: set[Path] = set()
|
|
411
|
+
for source_path in _selected_asset_source_paths(assets, categories):
|
|
412
|
+
relative = PurePosixPath(source_path)
|
|
413
|
+
source = (source_repo / Path(*relative.parts)).resolve()
|
|
414
|
+
_require_within_repo(source, source_repo, f'tracked asset "{source_path}"')
|
|
415
|
+
selected.add(source)
|
|
416
|
+
return sorted(selected)
|
|
417
|
+
|
|
418
|
+
|
|
419
|
+
def _selected_asset_source_paths(
|
|
420
|
+
assets: Any, categories: set[str] | None
|
|
421
|
+
) -> set[str]:
|
|
422
|
+
if not isinstance(assets, list):
|
|
423
|
+
return set()
|
|
424
|
+
selected: set[str] = set()
|
|
425
|
+
for asset in assets:
|
|
426
|
+
if not isinstance(asset, dict):
|
|
427
|
+
continue
|
|
428
|
+
asset_type = asset.get("type")
|
|
429
|
+
source_path = asset.get("sourcePath")
|
|
430
|
+
if not isinstance(source_path, str):
|
|
431
|
+
continue
|
|
432
|
+
include = categories is None or (
|
|
433
|
+
(asset_type == "profile" and "profiles" in categories)
|
|
434
|
+
or (asset_type in {"module", "runtime"} and "modules" in categories)
|
|
435
|
+
or (
|
|
436
|
+
asset_type == "contract"
|
|
437
|
+
and "contracts" in categories
|
|
438
|
+
and (source_path == "shared/contracts" or source_path.startswith("shared/contracts/"))
|
|
439
|
+
)
|
|
440
|
+
)
|
|
441
|
+
if not include:
|
|
442
|
+
continue
|
|
443
|
+
relative = PurePosixPath(source_path)
|
|
444
|
+
if relative.is_absolute() or ".." in relative.parts:
|
|
445
|
+
raise GetError(f"Tracked asset path is unsafe: {source_path}")
|
|
446
|
+
selected.add(relative.as_posix())
|
|
447
|
+
return selected
|
|
448
|
+
|
|
449
|
+
|
|
450
|
+
def _report_removed_source_files(
|
|
451
|
+
*,
|
|
452
|
+
profile_name: str,
|
|
453
|
+
entry: dict[str, Any],
|
|
454
|
+
destination_root: Path,
|
|
455
|
+
categories: set[str] | None,
|
|
456
|
+
current_paths: set[str],
|
|
457
|
+
result: UpdateResult,
|
|
458
|
+
) -> None:
|
|
459
|
+
roots = _selected_asset_source_paths(entry.get("assets"), categories)
|
|
460
|
+
if not roots:
|
|
461
|
+
return
|
|
462
|
+
hashes = entry.get("fileHashes")
|
|
463
|
+
if not isinstance(hashes, dict):
|
|
464
|
+
return
|
|
465
|
+
tracked_files = entry.get("files")
|
|
466
|
+
paths = {path for path in hashes if isinstance(path, str)}
|
|
467
|
+
if isinstance(tracked_files, list):
|
|
468
|
+
paths.update(path for path in tracked_files if isinstance(path, str))
|
|
469
|
+
for rel_path in sorted(paths - current_paths):
|
|
470
|
+
relative = PurePosixPath(rel_path)
|
|
471
|
+
if relative.is_absolute() or ".." in relative.parts:
|
|
472
|
+
raise GetError(f"Tracked file path is unsafe: {rel_path}")
|
|
473
|
+
# Category-scoped updates only report removals inside the selected
|
|
474
|
+
# categories' asset roots. A default update has no category filter,
|
|
475
|
+
# and the assets list is rewritten from the *current* source on
|
|
476
|
+
# every update -- so a file removed upstream is no longer an asset
|
|
477
|
+
# on the next pass and must stay reportable through its tracked
|
|
478
|
+
# hashes/files alone.
|
|
479
|
+
if categories is not None and not any(
|
|
480
|
+
relative == PurePosixPath(root) or PurePosixPath(root) in relative.parents
|
|
481
|
+
for root in roots
|
|
482
|
+
):
|
|
483
|
+
continue
|
|
484
|
+
destination = destination_root / Path(*relative.parts)
|
|
485
|
+
label = f"{profile_name}: {rel_path}"
|
|
486
|
+
if _has_symlinked_parent(destination, destination_root) or destination.is_symlink():
|
|
487
|
+
result.conflicts.append(f"{label} (removed upstream; symlinked destination)")
|
|
488
|
+
continue
|
|
489
|
+
if not destination.exists() and not destination.is_symlink():
|
|
490
|
+
result.skipped.append(f"{label} (already absent locally)")
|
|
491
|
+
continue
|
|
492
|
+
if destination.is_dir():
|
|
493
|
+
result.conflicts.append(f"{label} (removed upstream; destination is a directory)")
|
|
494
|
+
continue
|
|
495
|
+
try:
|
|
496
|
+
local_hash = _sha256_file(destination)
|
|
497
|
+
except OSError as exc:
|
|
498
|
+
raise GetError(f"Could not read local asset {destination}: {exc}") from exc
|
|
499
|
+
expected_hash = hashes.get(rel_path)
|
|
500
|
+
if isinstance(expected_hash, str) and local_hash == expected_hash:
|
|
501
|
+
result.skipped.append(f"{label} (removed upstream; left in place)")
|
|
502
|
+
else:
|
|
503
|
+
result.conflicts.append(f"{label} (removed upstream and modified locally)")
|
|
504
|
+
|
|
505
|
+
|
|
506
|
+
def _has_symlinked_parent(path: Path, root: Path) -> bool:
|
|
507
|
+
try:
|
|
508
|
+
relative = path.relative_to(root)
|
|
509
|
+
except ValueError:
|
|
510
|
+
return True
|
|
511
|
+
parent = root
|
|
512
|
+
for part in relative.parts[:-1]:
|
|
513
|
+
parent = parent / part
|
|
514
|
+
if parent.is_symlink():
|
|
515
|
+
return True
|
|
516
|
+
return False
|
|
517
|
+
|
|
518
|
+
|
|
519
|
+
def _write_updated_manifest(path: Path, manifest: dict[str, Any]) -> None:
|
|
520
|
+
try:
|
|
521
|
+
_atomic_write(path, json.dumps(manifest, indent=2) + "\n")
|
|
522
|
+
except OSError as exc:
|
|
523
|
+
raise GetError(f"Could not update tracking manifest {path}: {exc}") from exc
|
|
524
|
+
|
|
525
|
+
|
|
162
526
|
def format_get_plan(
|
|
163
527
|
actions: list[CopyAction],
|
|
164
528
|
*,
|
|
@@ -486,7 +850,12 @@ def _collect_asset_roots(source_repo: Path, profile_path: Path) -> list[Path]:
|
|
|
486
850
|
_require_within_repo(module_dir, source_repo, f'module source "{source}"')
|
|
487
851
|
asset_roots.add(module_dir.resolve())
|
|
488
852
|
asset_roots.update(
|
|
489
|
-
_collect_module_runtime_assets(
|
|
853
|
+
_collect_module_runtime_assets(
|
|
854
|
+
source_repo,
|
|
855
|
+
module_dir.resolve(),
|
|
856
|
+
module,
|
|
857
|
+
environment=_profile_environment(profile_doc),
|
|
858
|
+
)
|
|
490
859
|
)
|
|
491
860
|
|
|
492
861
|
return sorted(asset_roots)
|
|
@@ -496,6 +865,8 @@ def _collect_module_runtime_assets(
|
|
|
496
865
|
source_repo: Path,
|
|
497
866
|
module_dir: Path,
|
|
498
867
|
module_instance: dict[str, Any],
|
|
868
|
+
*,
|
|
869
|
+
environment: str | None = None,
|
|
499
870
|
) -> set[Path]:
|
|
500
871
|
module_doc, module_diags = load_yaml_file(module_dir / "module.yaml")
|
|
501
872
|
if module_diags or module_doc is None:
|
|
@@ -514,19 +885,34 @@ def _collect_module_runtime_assets(
|
|
|
514
885
|
if not isinstance(services, dict):
|
|
515
886
|
return assets
|
|
516
887
|
|
|
517
|
-
|
|
888
|
+
contexts: list[dict[str, Any] | None] = [None]
|
|
889
|
+
for selected_environment in (None, "production", environment):
|
|
890
|
+
context = _module_interpolation_context(
|
|
891
|
+
module_doc, module_instance, environment=selected_environment
|
|
892
|
+
)
|
|
893
|
+
if context is not None and context not in contexts:
|
|
894
|
+
contexts.append(context)
|
|
518
895
|
|
|
519
896
|
for service in services.values():
|
|
520
897
|
if not isinstance(service, dict):
|
|
521
898
|
continue
|
|
522
|
-
|
|
523
|
-
|
|
524
|
-
|
|
525
|
-
|
|
526
|
-
|
|
527
|
-
|
|
528
|
-
)
|
|
529
|
-
|
|
899
|
+
build_specs = [service.get("build")]
|
|
900
|
+
# A production mapping can carry its own build block (the hardened
|
|
901
|
+
# local image); collect it too so fetched profiles can still build
|
|
902
|
+
# the production variant.
|
|
903
|
+
marker = service.get("production")
|
|
904
|
+
if isinstance(marker, dict):
|
|
905
|
+
build_specs.append(marker.get("build"))
|
|
906
|
+
for context in contexts:
|
|
907
|
+
for build in build_specs:
|
|
908
|
+
assets.update(
|
|
909
|
+
_collect_build_assets(
|
|
910
|
+
source_repo,
|
|
911
|
+
module_dir,
|
|
912
|
+
build,
|
|
913
|
+
interpolation_context=context,
|
|
914
|
+
)
|
|
915
|
+
)
|
|
530
916
|
|
|
531
917
|
return assets
|
|
532
918
|
|
|
@@ -725,6 +1111,8 @@ def _strip_dockerfile_comment(line: str) -> str:
|
|
|
725
1111
|
def _module_interpolation_context(
|
|
726
1112
|
module_doc: dict[str, Any],
|
|
727
1113
|
module_instance: dict[str, Any],
|
|
1114
|
+
*,
|
|
1115
|
+
environment: str | None = None,
|
|
728
1116
|
) -> dict[str, Any] | None:
|
|
729
1117
|
config_schema = module_doc.get("spec", {}).get("configSchema")
|
|
730
1118
|
if not isinstance(config_schema, dict):
|
|
@@ -737,7 +1125,9 @@ def _module_interpolation_context(
|
|
|
737
1125
|
return None
|
|
738
1126
|
|
|
739
1127
|
try:
|
|
740
|
-
normalized_config = apply_defaults(
|
|
1128
|
+
normalized_config = apply_defaults(
|
|
1129
|
+
raw_config, config_schema, environment=environment
|
|
1130
|
+
)
|
|
741
1131
|
except MaxNestingDepthExceeded:
|
|
742
1132
|
return None
|
|
743
1133
|
|
|
@@ -749,6 +1139,12 @@ def _module_interpolation_context(
|
|
|
749
1139
|
}
|
|
750
1140
|
|
|
751
1141
|
|
|
1142
|
+
def _profile_environment(profile: dict[str, Any]) -> str | None:
|
|
1143
|
+
metadata = profile.get("metadata")
|
|
1144
|
+
environment = metadata.get("environment") if isinstance(metadata, dict) else None
|
|
1145
|
+
return environment if isinstance(environment, str) else None
|
|
1146
|
+
|
|
1147
|
+
|
|
752
1148
|
def _resolve_template_value(
|
|
753
1149
|
value: str,
|
|
754
1150
|
interpolation_context: dict[str, Any] | None,
|
|
@@ -917,10 +1313,14 @@ def _write_tracking_manifest(
|
|
|
917
1313
|
manifest_path = target_root / _TRACKING_FILE
|
|
918
1314
|
manifest = _read_tracking_manifest(manifest_path)
|
|
919
1315
|
|
|
1316
|
+
relative_files = {
|
|
1317
|
+
action.repo_relative_path: _sha256_file(action.source)
|
|
1318
|
+
for action in actions
|
|
1319
|
+
}
|
|
920
1320
|
entry = {
|
|
921
1321
|
"requestedProfile": requested_profile,
|
|
922
1322
|
"sourceProfile": profile_path.relative_to(source_repo).as_posix(),
|
|
923
|
-
"remote": local
|
|
1323
|
+
"remote": str(source_repo) if local is not None else remote or DEFAULT_REMOTE,
|
|
924
1324
|
"ref": None if local else ref,
|
|
925
1325
|
"commit": resolved_commit,
|
|
926
1326
|
"pinnedCommit": pinned_commit,
|
|
@@ -929,8 +1329,16 @@ def _write_tracking_manifest(
|
|
|
929
1329
|
_asset_root_relative_path(asset_root, source_repo) for asset_root in asset_roots
|
|
930
1330
|
],
|
|
931
1331
|
"files": [action.repo_relative_path for action in actions],
|
|
1332
|
+
"fileHashes": relative_files,
|
|
1333
|
+
"assets": _build_tracking_assets(
|
|
1334
|
+
source_repo=source_repo,
|
|
1335
|
+
requested_profile=requested_profile,
|
|
1336
|
+
profile_path=profile_path,
|
|
1337
|
+
asset_roots=asset_roots,
|
|
1338
|
+
),
|
|
932
1339
|
}
|
|
933
1340
|
|
|
1341
|
+
manifest["version"] = 2
|
|
934
1342
|
profiles = manifest.setdefault("profiles", {})
|
|
935
1343
|
if not isinstance(profiles, dict):
|
|
936
1344
|
profiles = {}
|
|
@@ -945,6 +1353,209 @@ def _write_tracking_manifest(
|
|
|
945
1353
|
manifest_path.write_text(json.dumps(manifest, indent=2) + "\n", encoding="utf-8")
|
|
946
1354
|
|
|
947
1355
|
|
|
1356
|
+
def _build_tracking_assets(
|
|
1357
|
+
*,
|
|
1358
|
+
source_repo: Path,
|
|
1359
|
+
requested_profile: str,
|
|
1360
|
+
profile_path: Path,
|
|
1361
|
+
asset_roots: list[Path],
|
|
1362
|
+
) -> list[dict[str, Any]]:
|
|
1363
|
+
if not asset_roots:
|
|
1364
|
+
return []
|
|
1365
|
+
|
|
1366
|
+
profile_doc, _provenance, diagnostics = resolve_extends(str(profile_path))
|
|
1367
|
+
if profile_doc is None or any(d.level == "error" for d in diagnostics):
|
|
1368
|
+
raise GetError(f"Could not load source profile {profile_path}")
|
|
1369
|
+
|
|
1370
|
+
spec = profile_doc.get("spec")
|
|
1371
|
+
modules = spec.get("modules", []) if isinstance(spec, dict) else []
|
|
1372
|
+
if not isinstance(modules, list):
|
|
1373
|
+
raise GetError(f"Profile modules must be a list in {profile_path}")
|
|
1374
|
+
|
|
1375
|
+
assets: list[dict[str, Any]] = []
|
|
1376
|
+
profile_roots = [
|
|
1377
|
+
root
|
|
1378
|
+
for root in asset_roots
|
|
1379
|
+
if (
|
|
1380
|
+
root == profile_path
|
|
1381
|
+
or root == profile_path.parent
|
|
1382
|
+
or (
|
|
1383
|
+
root.is_relative_to(source_repo / "profiles")
|
|
1384
|
+
and (root / "profile.yaml").is_file()
|
|
1385
|
+
)
|
|
1386
|
+
)
|
|
1387
|
+
]
|
|
1388
|
+
profile_dependencies: set[tuple[str, str]] = set()
|
|
1389
|
+
for parent_dir in _collect_extends_profile_dirs(source_repo, profile_path):
|
|
1390
|
+
parent_relative = parent_dir.relative_to(source_repo).as_posix()
|
|
1391
|
+
profile_dependencies.add(("profile", parent_relative))
|
|
1392
|
+
|
|
1393
|
+
module_dirs: dict[str, Path] = {}
|
|
1394
|
+
for index, module in enumerate(modules):
|
|
1395
|
+
if not isinstance(module, dict):
|
|
1396
|
+
raise GetError(f"Profile module at index {index} must be a mapping")
|
|
1397
|
+
module_id = module.get("id")
|
|
1398
|
+
source = module.get("source")
|
|
1399
|
+
if not isinstance(module_id, str) or not module_id:
|
|
1400
|
+
raise GetError(f"Profile module at index {index} is missing an id")
|
|
1401
|
+
if not isinstance(source, str) or not source:
|
|
1402
|
+
raise GetError(f'Profile module "{module_id}" is missing a source')
|
|
1403
|
+
|
|
1404
|
+
module_dir = resolve_module_dir(source, profile_path.parent, module_root=None)
|
|
1405
|
+
if module_dir is None or not module_dir.is_dir():
|
|
1406
|
+
raise GetError(f'Could not resolve module source "{source}" from {profile_path.parent}')
|
|
1407
|
+
_require_within_repo(module_dir, source_repo, f'module source "{source}"')
|
|
1408
|
+
module_dir = module_dir.resolve()
|
|
1409
|
+
module_dirs[module_id] = module_dir
|
|
1410
|
+
module_path = module_dir.relative_to(source_repo).as_posix()
|
|
1411
|
+
dependencies: set[tuple[str, str]] = set()
|
|
1412
|
+
depends_on = module.get("dependsOn", [])
|
|
1413
|
+
if isinstance(depends_on, list):
|
|
1414
|
+
dependencies.update(
|
|
1415
|
+
("module", dependency)
|
|
1416
|
+
for dependency in depends_on
|
|
1417
|
+
if isinstance(dependency, str) and dependency
|
|
1418
|
+
)
|
|
1419
|
+
# A contractRef "<module>.<name>" targets another module's *provided*
|
|
1420
|
+
# contract, so the dependency edge carries that contract's full asset
|
|
1421
|
+
# id ("<module>.<name>.provided") to stay referentially intact.
|
|
1422
|
+
dependencies.update(
|
|
1423
|
+
("contract", f"{contract_ref}.provided")
|
|
1424
|
+
for contract_ref in _find_contract_refs(module.get("config", {}))
|
|
1425
|
+
)
|
|
1426
|
+
assets.append(
|
|
1427
|
+
{
|
|
1428
|
+
"type": "module",
|
|
1429
|
+
"id": module_id,
|
|
1430
|
+
"sourcePath": module_path,
|
|
1431
|
+
"destinationPath": module_path,
|
|
1432
|
+
"dependencies": _serialize_dependencies(dependencies),
|
|
1433
|
+
}
|
|
1434
|
+
)
|
|
1435
|
+
profile_dependencies.add(("module", module_id))
|
|
1436
|
+
|
|
1437
|
+
if profile_path.name == "profile.yaml":
|
|
1438
|
+
primary_profile_root = profile_path.parent
|
|
1439
|
+
else:
|
|
1440
|
+
primary_profile_root = profile_path
|
|
1441
|
+
for profile_root in profile_roots:
|
|
1442
|
+
relative = profile_root.relative_to(source_repo).as_posix()
|
|
1443
|
+
assets.append(
|
|
1444
|
+
{
|
|
1445
|
+
"type": "profile",
|
|
1446
|
+
"id": requested_profile if profile_root == primary_profile_root else relative,
|
|
1447
|
+
"sourcePath": relative,
|
|
1448
|
+
"destinationPath": relative,
|
|
1449
|
+
"dependencies": (
|
|
1450
|
+
_serialize_dependencies(profile_dependencies)
|
|
1451
|
+
if profile_root == primary_profile_root
|
|
1452
|
+
else []
|
|
1453
|
+
),
|
|
1454
|
+
}
|
|
1455
|
+
)
|
|
1456
|
+
|
|
1457
|
+
for module_id, module_dir in module_dirs.items():
|
|
1458
|
+
module_yaml_path = module_dir / "module.yaml"
|
|
1459
|
+
module_doc, module_diagnostics = load_yaml_file(module_yaml_path)
|
|
1460
|
+
if module_doc is None or any(d.level == "error" for d in module_diagnostics):
|
|
1461
|
+
raise GetError(f"Could not load module {module_yaml_path}")
|
|
1462
|
+
module_spec = module_doc.get("spec")
|
|
1463
|
+
if not isinstance(module_spec, dict):
|
|
1464
|
+
continue
|
|
1465
|
+
module_path = module_yaml_path.relative_to(source_repo).as_posix()
|
|
1466
|
+
for field, direction in (("provides", "provided"), ("consumes", "consumed")):
|
|
1467
|
+
declarations = module_spec.get(field, [])
|
|
1468
|
+
if not isinstance(declarations, list):
|
|
1469
|
+
continue
|
|
1470
|
+
for index, declaration in enumerate(declarations):
|
|
1471
|
+
if not isinstance(declaration, dict):
|
|
1472
|
+
continue
|
|
1473
|
+
name = declaration.get("name")
|
|
1474
|
+
if not isinstance(name, str) or not name:
|
|
1475
|
+
continue
|
|
1476
|
+
contract = declaration.get("contract")
|
|
1477
|
+
kind = contract.get("kind") if isinstance(contract, dict) else None
|
|
1478
|
+
assets.append(
|
|
1479
|
+
{
|
|
1480
|
+
"type": "contract",
|
|
1481
|
+
# The direction is part of the id: a module may
|
|
1482
|
+
# provide and consume a contract with the same
|
|
1483
|
+
# name, and both records must stay addressable.
|
|
1484
|
+
"id": f"{module_id}.{name}.{direction}",
|
|
1485
|
+
"direction": direction,
|
|
1486
|
+
"kind": kind if isinstance(kind, str) else None,
|
|
1487
|
+
"sourcePath": module_path,
|
|
1488
|
+
"sourcePointer": f"/spec/{field}/{index}",
|
|
1489
|
+
"destinationPath": module_path,
|
|
1490
|
+
"destinationPointer": f"/spec/{field}/{index}",
|
|
1491
|
+
"dependencies": [{"type": "module", "id": module_id}],
|
|
1492
|
+
}
|
|
1493
|
+
)
|
|
1494
|
+
|
|
1495
|
+
profile_root_paths = {root.resolve() for root in profile_roots}
|
|
1496
|
+
module_root_paths = {path.resolve() for path in module_dirs.values()}
|
|
1497
|
+
for root in asset_roots:
|
|
1498
|
+
resolved_root = root.resolve()
|
|
1499
|
+
if any(
|
|
1500
|
+
resolved_root == profile_root or profile_root in resolved_root.parents
|
|
1501
|
+
for profile_root in profile_root_paths
|
|
1502
|
+
):
|
|
1503
|
+
continue
|
|
1504
|
+
if any(
|
|
1505
|
+
resolved_root == module_root or module_root in resolved_root.parents
|
|
1506
|
+
for module_root in module_root_paths
|
|
1507
|
+
):
|
|
1508
|
+
continue
|
|
1509
|
+
relative = _asset_root_relative_path(root, source_repo)
|
|
1510
|
+
assets.append(
|
|
1511
|
+
{
|
|
1512
|
+
# The contracts root itself (referenced as a whole by a
|
|
1513
|
+
# COPY line) must classify as a contract too, not just
|
|
1514
|
+
# paths nested inside it.
|
|
1515
|
+
"type": (
|
|
1516
|
+
"contract"
|
|
1517
|
+
if relative == "shared/contracts" or relative.startswith("shared/contracts/")
|
|
1518
|
+
else "runtime"
|
|
1519
|
+
),
|
|
1520
|
+
"id": relative,
|
|
1521
|
+
"sourcePath": relative,
|
|
1522
|
+
"destinationPath": relative,
|
|
1523
|
+
"dependencies": [],
|
|
1524
|
+
}
|
|
1525
|
+
)
|
|
1526
|
+
|
|
1527
|
+
return sorted(assets, key=lambda asset: (asset["type"], asset["id"]))
|
|
1528
|
+
|
|
1529
|
+
|
|
1530
|
+
def _sha256_file(path: Path) -> str:
|
|
1531
|
+
digest = hashlib.sha256()
|
|
1532
|
+
with path.open("rb") as file:
|
|
1533
|
+
for chunk in iter(lambda: file.read(1024 * 1024), b""):
|
|
1534
|
+
digest.update(chunk)
|
|
1535
|
+
return digest.hexdigest()
|
|
1536
|
+
|
|
1537
|
+
|
|
1538
|
+
def _find_contract_refs(value: Any) -> set[str]:
|
|
1539
|
+
references: set[str] = set()
|
|
1540
|
+
if isinstance(value, dict):
|
|
1541
|
+
contract_ref = value.get("contractRef")
|
|
1542
|
+
if isinstance(contract_ref, str) and contract_ref:
|
|
1543
|
+
references.add(contract_ref)
|
|
1544
|
+
for nested in value.values():
|
|
1545
|
+
references.update(_find_contract_refs(nested))
|
|
1546
|
+
elif isinstance(value, list):
|
|
1547
|
+
for nested in value:
|
|
1548
|
+
references.update(_find_contract_refs(nested))
|
|
1549
|
+
return references
|
|
1550
|
+
|
|
1551
|
+
|
|
1552
|
+
def _serialize_dependencies(dependencies: set[tuple[str, str]]) -> list[dict[str, str]]:
|
|
1553
|
+
return [
|
|
1554
|
+
{"type": dependency_type, "id": identifier}
|
|
1555
|
+
for dependency_type, identifier in sorted(dependencies)
|
|
1556
|
+
]
|
|
1557
|
+
|
|
1558
|
+
|
|
948
1559
|
def _read_tracking_manifest(path: Path) -> dict[str, Any]:
|
|
949
1560
|
if not path.exists():
|
|
950
1561
|
return {"version": 1, "profiles": {}}
|