boatyard 0.6.0.dev170__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- boatyard/__init__.py +0 -0
- boatyard/build.py +267 -0
- boatyard/cli.py +23 -0
- boatyard/compose.py +24 -0
- boatyard/config/__init__.py +0 -0
- boatyard/config/config.py +787 -0
- boatyard/config/lock.py +152 -0
- boatyard/config/values.py +188 -0
- boatyard/crane.py +23 -0
- boatyard/dockerfiles.py +201 -0
- boatyard/lock.py +233 -0
- boatyard/markers.py +201 -0
- boatyard/paths.py +5 -0
- boatyard/publish.py +72 -0
- boatyard/py.typed +0 -0
- boatyard/score.py +267 -0
- boatyard/score_compose.py +274 -0
- boatyard/score_k8s.py +152 -0
- boatyard/score_patch_templates/gpu-cuda.tpl +24 -0
- boatyard/score_patch_templates/gpu-pod.tpl +43 -0
- boatyard/score_patch_templates/gpu-rocm.tpl +30 -0
- boatyard/score_patch_templates/probe-adapter-healthchecks.tpl +55 -0
- boatyard/score_patch_templates/restart.tpl +21 -0
- boatyard/score_patch_templates/scale-to-zero.tpl +30 -0
- boatyard/temp.py +5 -0
- boatyard/verbs/__init__.py +0 -0
- boatyard/verbs/build.py +54 -0
- boatyard/verbs/down.py +39 -0
- boatyard/verbs/lock.py +34 -0
- boatyard/verbs/matrix.py +57 -0
- boatyard/verbs/mirror.py +50 -0
- boatyard/verbs/publish.py +138 -0
- boatyard/verbs/score.py +161 -0
- boatyard/verbs/tree_tags.py +35 -0
- boatyard/verbs/up.py +265 -0
- boatyard/yaml.py +34 -0
- boatyard-0.6.0.dev170.dist-info/METADATA +12 -0
- boatyard-0.6.0.dev170.dist-info/RECORD +42 -0
- boatyard-0.6.0.dev170.dist-info/WHEEL +4 -0
- boatyard-0.6.0.dev170.dist-info/entry_points.txt +2 -0
- boatyard-0.6.0.dev170.dist-info/licenses/LICENSE +201 -0
- boatyard-0.6.0.dev170.dist-info/licenses/NOTICE +1 -0
|
@@ -0,0 +1,787 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import importlib.metadata
|
|
4
|
+
import re
|
|
5
|
+
import tomllib
|
|
6
|
+
from collections.abc import Mapping
|
|
7
|
+
from functools import partial
|
|
8
|
+
from pathlib import Path
|
|
9
|
+
from typing import Annotated, Literal, NamedTuple, Self, TypeIs
|
|
10
|
+
|
|
11
|
+
from packaging.specifiers import SpecifierSet
|
|
12
|
+
from pydantic import (
|
|
13
|
+
AfterValidator,
|
|
14
|
+
BaseModel,
|
|
15
|
+
ConfigDict,
|
|
16
|
+
Discriminator,
|
|
17
|
+
Field,
|
|
18
|
+
Tag,
|
|
19
|
+
ValidationInfo,
|
|
20
|
+
field_validator,
|
|
21
|
+
model_validator,
|
|
22
|
+
)
|
|
23
|
+
import yaml
|
|
24
|
+
|
|
25
|
+
from boatyard.dockerfiles import (
|
|
26
|
+
DOCKERFILE_NAME,
|
|
27
|
+
ServiceBuild,
|
|
28
|
+
derive_services,
|
|
29
|
+
service_sha_env,
|
|
30
|
+
)
|
|
31
|
+
from boatyard.temp import PLATFORM_PATTERN, SHA256_DIGEST_REGEX
|
|
32
|
+
from boatyard.yaml import load_yaml_model
|
|
33
|
+
|
|
34
|
+
DEV_SENTINEL = "0.0.0.dev0"
|
|
35
|
+
|
|
36
|
+
DEFAULT_MIRROR_PREFIX = "ghcr.io/outernet-foundation/mirror"
|
|
37
|
+
|
|
38
|
+
PYPROJECT_PATH = Path("pyproject.toml")
|
|
39
|
+
PYTHON_VERSION_PATH = Path(".python-version")
|
|
40
|
+
|
|
41
|
+
ENV_NAME_PATTERN = re.compile(r"^[A-Z][A-Z0-9_]*$")
|
|
42
|
+
|
|
43
|
+
# Match ${ALL_CAPS} placeholder slots in workload files.
|
|
44
|
+
PLACEHOLDER = re.compile(r"\$\{([A-Z][A-Z0-9_]*)\}")
|
|
45
|
+
|
|
46
|
+
# Match a compose brace slot with its preceding dollar run.
|
|
47
|
+
COMPOSE_SLOT = re.compile(r"(\$+)\{([A-Z][A-Z0-9_]*)(:[^{}]*)?\}")
|
|
48
|
+
|
|
49
|
+
# Match compose env slots: ${NAME}, ${NAME:?err}, ${NAME:-default}.
|
|
50
|
+
ENV_SLOT_PATTERN = re.compile(r"^\$\{[A-Z][A-Z0-9_]*(?::[^{}]*)?\}$")
|
|
51
|
+
PORT_PATTERN = re.compile(r"^[0-9]+$")
|
|
52
|
+
|
|
53
|
+
type Yaml = str | int | float | bool | list[Yaml] | dict[str, Yaml] | None
|
|
54
|
+
|
|
55
|
+
|
|
56
|
+
class BundleScan(NamedTuple):
|
|
57
|
+
service_names: set[str]
|
|
58
|
+
requires: list[str]
|
|
59
|
+
|
|
60
|
+
|
|
61
|
+
def _require_port_or_slot(value: str) -> str:
|
|
62
|
+
if PORT_PATTERN.fullmatch(value) is None and ENV_SLOT_PATTERN.fullmatch(value) is None:
|
|
63
|
+
raise ValueError(f"{value!r} must be a literal port number or a ${{…}} env slot")
|
|
64
|
+
return value
|
|
65
|
+
|
|
66
|
+
|
|
67
|
+
PortOrSlot = Annotated[str, AfterValidator(_require_port_or_slot)]
|
|
68
|
+
|
|
69
|
+
|
|
70
|
+
def _require_all_caps_name(value: str) -> str:
|
|
71
|
+
if ENV_NAME_PATTERN.fullmatch(value) is None:
|
|
72
|
+
raise ValueError(f"{value!r} must be an ALL_CAPS env name")
|
|
73
|
+
return value
|
|
74
|
+
|
|
75
|
+
|
|
76
|
+
AllCapsName = Annotated[str, AfterValidator(_require_all_caps_name)]
|
|
77
|
+
|
|
78
|
+
# Match path:tag or path@digest.
|
|
79
|
+
DECLARED_REFERENCE_PATTERN = re.compile(
|
|
80
|
+
rf"^(?P<path>[^:@\s$]+)(?::(?P<tag>[^@\s$]+))?(?:@(?P<digest>{SHA256_DIGEST_REGEX}))?$"
|
|
81
|
+
)
|
|
82
|
+
|
|
83
|
+
TEMPLATE_KEY_PATTERN = re.compile(r"\{([a-z][a-z0-9_]*)\}")
|
|
84
|
+
|
|
85
|
+
# Match the range operators a ref tag must never carry; exact tags only.
|
|
86
|
+
REF_TAG_OPERATOR_PATTERN = re.compile(r"[<>=!~^*]")
|
|
87
|
+
|
|
88
|
+
OCI_REFERENCE_PREFIX = "oci://"
|
|
89
|
+
|
|
90
|
+
AUTH_GROUP_NAME = "auth"
|
|
91
|
+
EDGE_SERVICE_NAME = "edge"
|
|
92
|
+
CADDY_IMAGE = "CADDY_IMAGE"
|
|
93
|
+
|
|
94
|
+
COMPOSE_OUTPUT_DIR = Path("stack/generated/compose")
|
|
95
|
+
|
|
96
|
+
Target = Literal["both", "compose", "k8s"]
|
|
97
|
+
|
|
98
|
+
|
|
99
|
+
class UvToolTable(BaseModel):
|
|
100
|
+
model_config = ConfigDict(extra="ignore")
|
|
101
|
+
|
|
102
|
+
required_version: str | None = Field(default=None, alias="required-version")
|
|
103
|
+
|
|
104
|
+
|
|
105
|
+
class ToolTable(BaseModel):
|
|
106
|
+
model_config = ConfigDict(extra="ignore")
|
|
107
|
+
|
|
108
|
+
uv: UvToolTable | None = None
|
|
109
|
+
|
|
110
|
+
|
|
111
|
+
class PyprojectToml(BaseModel):
|
|
112
|
+
model_config = ConfigDict(extra="ignore")
|
|
113
|
+
|
|
114
|
+
tool: ToolTable | None = None
|
|
115
|
+
|
|
116
|
+
|
|
117
|
+
class ConfigFile(BaseModel):
|
|
118
|
+
model_config = ConfigDict(extra="forbid")
|
|
119
|
+
|
|
120
|
+
requires: str
|
|
121
|
+
gateway: PortOrSlot | None = None
|
|
122
|
+
repository: str | None = None
|
|
123
|
+
base_images: dict[AllCapsName, str] = Field(default_factory=dict)
|
|
124
|
+
variants: dict[AllCapsName, list[str]] = Field(default_factory=dict)
|
|
125
|
+
platforms: list[str] = Field(default_factory=list)
|
|
126
|
+
project_name: str
|
|
127
|
+
services: dict[str, ServiceMember] = Field(default_factory=dict)
|
|
128
|
+
k8s: K8sConfig | None = None
|
|
129
|
+
mirror_prefix: str = DEFAULT_MIRROR_PREFIX
|
|
130
|
+
|
|
131
|
+
score_dir: Path = Path("stack/score")
|
|
132
|
+
k8s_output: Path = Path("stack/generated/k8s/manifests.yaml")
|
|
133
|
+
k8s_local_output: Path = Path("stack/generated/k8s/manifests.local.yaml")
|
|
134
|
+
score_compose_version: str = "0.42.0"
|
|
135
|
+
score_k8s_version: str = "0.15.0"
|
|
136
|
+
helm_version: str = "3.22.1"
|
|
137
|
+
|
|
138
|
+
service_builds: dict[str, ServiceBuild] = Field(default_factory=dict, exclude=True)
|
|
139
|
+
root: Path = Field(default_factory=Path.cwd, exclude=True)
|
|
140
|
+
|
|
141
|
+
@classmethod
|
|
142
|
+
def load(cls, path: Path) -> ConfigFile:
|
|
143
|
+
config = load_yaml_model(cls, path)
|
|
144
|
+
config.resolve(path.parent)
|
|
145
|
+
return config
|
|
146
|
+
|
|
147
|
+
def resolve(self, root: Path) -> Self:
|
|
148
|
+
# Self-check the installed version.
|
|
149
|
+
installed = importlib.metadata.version("boatyard")
|
|
150
|
+
if installed != DEV_SENTINEL and not SpecifierSet(self.requires).contains(installed, prereleases=True):
|
|
151
|
+
raise ValueError(f"boatyard {installed} does not satisfy requires={self.requires!r}")
|
|
152
|
+
self.root = root
|
|
153
|
+
|
|
154
|
+
# Resolve {uv} and {python} templates in base_images against the toolchain pins.
|
|
155
|
+
pins: dict[str, str] = {}
|
|
156
|
+
used = sorted({
|
|
157
|
+
key for reference in self.base_images.values() for key in TEMPLATE_KEY_PATTERN.findall(reference)
|
|
158
|
+
})
|
|
159
|
+
for key in used:
|
|
160
|
+
if key == "uv":
|
|
161
|
+
pyproject = root / PYPROJECT_PATH
|
|
162
|
+
pyproject_document = PyprojectToml.model_validate(
|
|
163
|
+
tomllib.loads(pyproject.read_text(encoding="utf-8")) if pyproject.is_file() else {}
|
|
164
|
+
)
|
|
165
|
+
declared = (
|
|
166
|
+
pyproject_document.tool.uv.required_version
|
|
167
|
+
if pyproject_document.tool is not None and pyproject_document.tool.uv is not None
|
|
168
|
+
else None
|
|
169
|
+
)
|
|
170
|
+
if not declared:
|
|
171
|
+
raise ValueError(
|
|
172
|
+
f"{PYPROJECT_PATH.as_posix()}: tool.uv.required-version is not declared; "
|
|
173
|
+
"the {uv} template requires it"
|
|
174
|
+
)
|
|
175
|
+
pin_match = re.fullmatch(r"==(?P<version>[^,*\s]+)", declared.strip())
|
|
176
|
+
if pin_match is None:
|
|
177
|
+
raise ValueError(
|
|
178
|
+
f"{PYPROJECT_PATH.as_posix()}: tool.uv.required-version {declared!r} "
|
|
179
|
+
"must be an exact == pin like '==0.12.15'"
|
|
180
|
+
)
|
|
181
|
+
pins[key] = pin_match.group("version")
|
|
182
|
+
elif key == "python":
|
|
183
|
+
python_version = root / PYTHON_VERSION_PATH
|
|
184
|
+
if not python_version.is_file():
|
|
185
|
+
raise ValueError(
|
|
186
|
+
f"{PYTHON_VERSION_PATH.as_posix()}: the file is absent; the {{{key}}} template requires it"
|
|
187
|
+
)
|
|
188
|
+
# Take the first non-empty line.
|
|
189
|
+
version = next(
|
|
190
|
+
(line.strip() for line in python_version.read_text(encoding="utf-8").splitlines() if line.strip()),
|
|
191
|
+
"",
|
|
192
|
+
)
|
|
193
|
+
if not version:
|
|
194
|
+
raise ValueError(
|
|
195
|
+
f"{PYTHON_VERSION_PATH.as_posix()}: the file is empty; the {{{key}}} template requires it"
|
|
196
|
+
)
|
|
197
|
+
pins[key] = version
|
|
198
|
+
else:
|
|
199
|
+
raise ValueError(
|
|
200
|
+
f"base_images template key {{{key}}} is unknown; the toolchain pins are {{uv}}, {{python}}"
|
|
201
|
+
)
|
|
202
|
+
self.base_images = {
|
|
203
|
+
name: TEMPLATE_KEY_PATTERN.sub(partial(_resolve_template_key, pins), reference)
|
|
204
|
+
for name, reference in self.base_images.items()
|
|
205
|
+
}
|
|
206
|
+
|
|
207
|
+
# Derive the service build set from the Dockerfile tree.
|
|
208
|
+
service_builds = derive_services(root, self.variants)
|
|
209
|
+
|
|
210
|
+
computable = {service_sha_env(name) for name in service_builds}
|
|
211
|
+
for service in service_builds.values():
|
|
212
|
+
unknown = [argument for argument in service.sha_args if argument not in computable]
|
|
213
|
+
if unknown:
|
|
214
|
+
raise ValueError(
|
|
215
|
+
f"workloads/{service.name}/{DOCKERFILE_NAME}: ARG {', '.join(unknown)} names no service directory"
|
|
216
|
+
)
|
|
217
|
+
if service_builds and self.repository is None:
|
|
218
|
+
listed = ", ".join(sorted(service_builds))
|
|
219
|
+
raise ValueError(f"Dockerfile service directories exist ({listed}); repository is required")
|
|
220
|
+
self.service_builds = service_builds
|
|
221
|
+
|
|
222
|
+
# Enforce the lock pairing laws.
|
|
223
|
+
declared_charts = self.declared_charts
|
|
224
|
+
declares_routes = any(
|
|
225
|
+
entry.routes
|
|
226
|
+
or (isinstance(entry, ServiceGroup) and any(option.routes for option in entry.options.values()))
|
|
227
|
+
for entry in self.services.values()
|
|
228
|
+
)
|
|
229
|
+
if declares_routes and self.gateway is None:
|
|
230
|
+
raise ValueError("routes are declared; gateway is required")
|
|
231
|
+
if self.gateway is not None and CADDY_IMAGE not in self.base_images:
|
|
232
|
+
raise ValueError("gateway is declared; base_images must declare CADDY_IMAGE")
|
|
233
|
+
for chart in declared_charts:
|
|
234
|
+
values = root / self.score_dir / "k8s" / "values" / f"{chart}.yaml"
|
|
235
|
+
if not values.is_file():
|
|
236
|
+
raise ValueError(f"{values.as_posix()}: the chart's values file is required")
|
|
237
|
+
|
|
238
|
+
return self
|
|
239
|
+
|
|
240
|
+
@property
|
|
241
|
+
def declared_charts(self) -> dict[str, str]:
|
|
242
|
+
return self.k8s.charts if self.k8s is not None else {}
|
|
243
|
+
|
|
244
|
+
@property
|
|
245
|
+
def selectable_options(self) -> dict[str, tuple[str, str]]:
|
|
246
|
+
return {
|
|
247
|
+
row.name: (row.root, row.option)
|
|
248
|
+
for row in _flatten_service_members(self.services)
|
|
249
|
+
if row.option is not None
|
|
250
|
+
}
|
|
251
|
+
|
|
252
|
+
@property
|
|
253
|
+
def bare_entries(self) -> list[str]:
|
|
254
|
+
return [row.root for row in _flatten_service_members(self.services) if row.option is None]
|
|
255
|
+
|
|
256
|
+
def local_service_entries(self) -> list[tuple[str, ServiceEntry]]:
|
|
257
|
+
return [(row.name, row.entry) for row in _flatten_service_members(self.services) if not row.entry.remote]
|
|
258
|
+
|
|
259
|
+
def remote_declared_entries(self) -> list[tuple[str, str, ServiceEntry]]:
|
|
260
|
+
return [
|
|
261
|
+
(row.name, row.entry.ref.removeprefix(OCI_REFERENCE_PREFIX), row.entry)
|
|
262
|
+
for row in _flatten_service_members(self.services)
|
|
263
|
+
if row.entry.ref is not None
|
|
264
|
+
]
|
|
265
|
+
|
|
266
|
+
def resolve_services(self, choices: Mapping[str, str]) -> list[ResolvedService]:
|
|
267
|
+
resolved: list[ResolvedService] = []
|
|
268
|
+
for name, member in self.services.items():
|
|
269
|
+
if isinstance(member, ServiceGroup):
|
|
270
|
+
option_name = choices.get(name)
|
|
271
|
+
if option_name is None:
|
|
272
|
+
continue
|
|
273
|
+
option = member.options[option_name]
|
|
274
|
+
resolved.append(
|
|
275
|
+
ResolvedService(
|
|
276
|
+
name=qualified_option_name(name, option_name),
|
|
277
|
+
entry=option,
|
|
278
|
+
routes=option.routes + member.routes,
|
|
279
|
+
ports=option.ports + member.ports,
|
|
280
|
+
auth=name == AUTH_GROUP_NAME,
|
|
281
|
+
)
|
|
282
|
+
)
|
|
283
|
+
else:
|
|
284
|
+
resolved.append(ResolvedService(name=name, entry=member, routes=member.routes, ports=member.ports))
|
|
285
|
+
return resolved
|
|
286
|
+
|
|
287
|
+
@field_validator("requires")
|
|
288
|
+
@classmethod
|
|
289
|
+
def _validate_requires(cls, value: str) -> str:
|
|
290
|
+
if not value:
|
|
291
|
+
raise ValueError("requires must be a non-empty PEP 440 specifier")
|
|
292
|
+
SpecifierSet(value)
|
|
293
|
+
return value
|
|
294
|
+
|
|
295
|
+
@field_validator("repository")
|
|
296
|
+
@classmethod
|
|
297
|
+
def _validate_repository(cls, value: str | None) -> str | None:
|
|
298
|
+
if value is not None:
|
|
299
|
+
host, separator, path = value.partition("/")
|
|
300
|
+
if not separator or not host or not path or value.endswith("/"):
|
|
301
|
+
raise ValueError(
|
|
302
|
+
"repository must be registry-host/repository-path with no trailing slash (ghcr.io/org/repo)"
|
|
303
|
+
)
|
|
304
|
+
return value
|
|
305
|
+
|
|
306
|
+
@field_validator("base_images")
|
|
307
|
+
@classmethod
|
|
308
|
+
def _validate_base_images(cls, value: dict[AllCapsName, str]) -> dict[AllCapsName, str]:
|
|
309
|
+
for name, reference in value.items():
|
|
310
|
+
match = DECLARED_REFERENCE_PATTERN.fullmatch(reference)
|
|
311
|
+
if match is None or not (match.group("tag") or match.group("digest")):
|
|
312
|
+
raise ValueError(f"base_images.{name}: {reference!r} must be 'path:tag' or 'path@digest'")
|
|
313
|
+
return value
|
|
314
|
+
|
|
315
|
+
@field_validator("variants")
|
|
316
|
+
@classmethod
|
|
317
|
+
def _validate_variants(cls, value: dict[AllCapsName, list[str]]) -> dict[AllCapsName, list[str]]:
|
|
318
|
+
for axis, values in value.items():
|
|
319
|
+
if not values:
|
|
320
|
+
raise ValueError(f"variants.{axis}: must declare at least one value")
|
|
321
|
+
seen: set[str] = set()
|
|
322
|
+
for variant in values:
|
|
323
|
+
if re.fullmatch(r"[a-z0-9][a-z0-9_-]*", variant) is None:
|
|
324
|
+
raise ValueError(f"variants.{axis}: value {variant!r} must be a bare lowercase spelling like cuda")
|
|
325
|
+
if variant in seen:
|
|
326
|
+
raise ValueError(f"variants.{axis}: value {variant!r} is declared twice")
|
|
327
|
+
seen.add(variant)
|
|
328
|
+
return value
|
|
329
|
+
|
|
330
|
+
@field_validator("platforms")
|
|
331
|
+
@classmethod
|
|
332
|
+
def _validate_platforms(cls, value: list[str]) -> list[str]:
|
|
333
|
+
for platform in value:
|
|
334
|
+
if PLATFORM_PATTERN.fullmatch(platform) is None:
|
|
335
|
+
raise ValueError(f"platforms entry {platform!r} must be an os/arch pair like linux/arm64")
|
|
336
|
+
return value
|
|
337
|
+
|
|
338
|
+
@field_validator("mirror_prefix")
|
|
339
|
+
@classmethod
|
|
340
|
+
def _validate_mirror_prefix(cls, value: str) -> str:
|
|
341
|
+
if not value or value.endswith("/"):
|
|
342
|
+
raise ValueError("mirror_prefix must be non-empty with no trailing slash")
|
|
343
|
+
return value
|
|
344
|
+
|
|
345
|
+
@field_validator("services")
|
|
346
|
+
@classmethod
|
|
347
|
+
def _validate_services(
|
|
348
|
+
cls, services: dict[str, ServiceGroup | ServiceEntry]
|
|
349
|
+
) -> dict[str, ServiceGroup | ServiceEntry]:
|
|
350
|
+
# The qualified vocabulary (options as group-dash-option, bare entries as their root
|
|
351
|
+
# name) must be collision-free: two declaration sites never produce one spelling.
|
|
352
|
+
vocabulary: dict[str, str] = {}
|
|
353
|
+
for row in _flatten_service_members(services):
|
|
354
|
+
if row.name in vocabulary:
|
|
355
|
+
raise ValueError(f"{row.site}: {row.name} is already claimed at {vocabulary[row.name]}")
|
|
356
|
+
vocabulary[row.name] = row.site
|
|
357
|
+
|
|
358
|
+
# Collect every route with its declaration site; paths are edge-wide unique.
|
|
359
|
+
route_sites = [
|
|
360
|
+
(where, route)
|
|
361
|
+
for name, entry in services.items()
|
|
362
|
+
for where, entry_routes in (
|
|
363
|
+
[(f"services.{name}", entry.routes)]
|
|
364
|
+
if not isinstance(entry, ServiceGroup)
|
|
365
|
+
else [
|
|
366
|
+
(f"services.{name}.options.{option_name}", option.routes)
|
|
367
|
+
for option_name, option in entry.options.items()
|
|
368
|
+
]
|
|
369
|
+
+ [(f"services.{name}", entry.routes)]
|
|
370
|
+
)
|
|
371
|
+
for route in entry_routes
|
|
372
|
+
]
|
|
373
|
+
owner: dict[str, str] = {}
|
|
374
|
+
for where, route in route_sites:
|
|
375
|
+
if route.path in owner:
|
|
376
|
+
raise ValueError(
|
|
377
|
+
f"{where}: route path {route.path} is already declared at {owner[route.path]}; "
|
|
378
|
+
"paths are edge-wide unique"
|
|
379
|
+
)
|
|
380
|
+
owner[route.path] = where
|
|
381
|
+
|
|
382
|
+
if AUTH_GROUP_NAME in services:
|
|
383
|
+
auth = services[AUTH_GROUP_NAME]
|
|
384
|
+
if not isinstance(auth, ServiceGroup):
|
|
385
|
+
raise ValueError(f"services.{AUTH_GROUP_NAME}: auth is a reserved group name; it must be a group")
|
|
386
|
+
for option_name, option in auth.options.items():
|
|
387
|
+
if len(option.routes) != 1:
|
|
388
|
+
raise ValueError(
|
|
389
|
+
f"services.{AUTH_GROUP_NAME}.options.{option_name}: an auth option declares exactly one route"
|
|
390
|
+
)
|
|
391
|
+
return services
|
|
392
|
+
|
|
393
|
+
@model_validator(mode="after")
|
|
394
|
+
def _validate(self) -> Self:
|
|
395
|
+
if self.services:
|
|
396
|
+
any_local = any(not row.entry.remote for row in _flatten_service_members(self.services))
|
|
397
|
+
if any_local and not self.repository:
|
|
398
|
+
raise ValueError("local entries (workloads:) are declared; repository is required")
|
|
399
|
+
|
|
400
|
+
if self.base_images and not self.platforms:
|
|
401
|
+
raise ValueError(
|
|
402
|
+
"base_images are declared; platforms is required (an amd64-only repo declares [linux/amd64])"
|
|
403
|
+
)
|
|
404
|
+
return self
|
|
405
|
+
|
|
406
|
+
|
|
407
|
+
class ServiceGroup(BaseModel):
|
|
408
|
+
model_config = ConfigDict(extra="forbid")
|
|
409
|
+
|
|
410
|
+
options: dict[str, ServiceEntry]
|
|
411
|
+
routes: list[RouteConfig] = Field(default_factory=list)
|
|
412
|
+
ports: list[PortDeclaration] = Field(default_factory=list)
|
|
413
|
+
|
|
414
|
+
|
|
415
|
+
class ServiceEntry(BaseModel):
|
|
416
|
+
model_config = ConfigDict(extra="forbid")
|
|
417
|
+
|
|
418
|
+
ref: str | None = None
|
|
419
|
+
workloads: list[str] = Field(default_factory=list)
|
|
420
|
+
vars: dict[str, str] = Field(default_factory=dict)
|
|
421
|
+
routes: list[RouteConfig] = Field(default_factory=list)
|
|
422
|
+
ports: list[PortDeclaration] = Field(default_factory=list)
|
|
423
|
+
|
|
424
|
+
@property
|
|
425
|
+
def remote(self) -> bool:
|
|
426
|
+
return self.ref is not None
|
|
427
|
+
|
|
428
|
+
@model_validator(mode="after")
|
|
429
|
+
def _validate(self) -> Self:
|
|
430
|
+
if self.ref is not None:
|
|
431
|
+
require_bundle_reference(self.ref)
|
|
432
|
+
if self.ref is not None and self.workloads:
|
|
433
|
+
raise ValueError("an entry declares ref: or workloads:, not both")
|
|
434
|
+
if self.ref is None and not self.workloads:
|
|
435
|
+
raise ValueError("ref: or workloads: is required")
|
|
436
|
+
if self.ref is not None and self.vars:
|
|
437
|
+
raise ValueError("vars is not allowed with ref:")
|
|
438
|
+
for name in self.workloads:
|
|
439
|
+
require_workload_reference(name)
|
|
440
|
+
return self
|
|
441
|
+
|
|
442
|
+
|
|
443
|
+
class FlattenedServiceEntry(NamedTuple):
|
|
444
|
+
root: str
|
|
445
|
+
option: str | None
|
|
446
|
+
name: str
|
|
447
|
+
site: str
|
|
448
|
+
entry: ServiceEntry
|
|
449
|
+
|
|
450
|
+
|
|
451
|
+
def _flatten_service_members(services: dict[str, ServiceGroup | ServiceEntry]) -> list[FlattenedServiceEntry]:
|
|
452
|
+
# Options ride under their qualified name; bare entries are their own root.
|
|
453
|
+
flattened: list[FlattenedServiceEntry] = []
|
|
454
|
+
for root_name, member in services.items():
|
|
455
|
+
if isinstance(member, ServiceGroup):
|
|
456
|
+
flattened.extend(
|
|
457
|
+
FlattenedServiceEntry(
|
|
458
|
+
root=root_name,
|
|
459
|
+
option=option_name,
|
|
460
|
+
name=qualified_option_name(root_name, option_name),
|
|
461
|
+
site=f"services.{root_name}.options.{option_name}",
|
|
462
|
+
entry=option,
|
|
463
|
+
)
|
|
464
|
+
for option_name, option in member.options.items()
|
|
465
|
+
)
|
|
466
|
+
else:
|
|
467
|
+
flattened.append(
|
|
468
|
+
FlattenedServiceEntry(
|
|
469
|
+
root=root_name, option=None, name=root_name, site=f"services.{root_name}", entry=member
|
|
470
|
+
)
|
|
471
|
+
)
|
|
472
|
+
return flattened
|
|
473
|
+
|
|
474
|
+
|
|
475
|
+
def _service_member_kind(value: Yaml) -> str:
|
|
476
|
+
return "group" if is_mapping(value) and "options" in value else "entry"
|
|
477
|
+
|
|
478
|
+
|
|
479
|
+
ServiceMember = Annotated[
|
|
480
|
+
Annotated[ServiceGroup, Tag("group")] | Annotated[ServiceEntry, Tag("entry")],
|
|
481
|
+
Discriminator(_service_member_kind),
|
|
482
|
+
]
|
|
483
|
+
|
|
484
|
+
|
|
485
|
+
class RouteConfig(BaseModel):
|
|
486
|
+
model_config = ConfigDict(extra="forbid")
|
|
487
|
+
|
|
488
|
+
path: str
|
|
489
|
+
upstream: str
|
|
490
|
+
strip_prefix: bool = False
|
|
491
|
+
raw: str | None = None
|
|
492
|
+
|
|
493
|
+
@model_validator(mode="after")
|
|
494
|
+
def _validate(self) -> Self:
|
|
495
|
+
if re.fullmatch(r"(?:/\*|/(?:[^/\s*]+(?:/[^/\s*]+)*)?(?:/\*)?)", self.path) is None:
|
|
496
|
+
raise ValueError(f"route path {self.path!r} must be a /-rooted segment chain, optionally wildcarded as …/*")
|
|
497
|
+
if re.fullmatch(r"[a-zA-Z0-9][a-zA-Z0-9_.-]*:[0-9]+", self.upstream) is None:
|
|
498
|
+
raise ValueError(
|
|
499
|
+
f"route upstream {self.upstream!r} must be a service name and a literal port (grafana:3000)"
|
|
500
|
+
)
|
|
501
|
+
if self.strip_prefix and not self.path.endswith("/*"):
|
|
502
|
+
raise ValueError(f"route {self.path}: the path must end in /* for strip_prefix")
|
|
503
|
+
return self
|
|
504
|
+
|
|
505
|
+
|
|
506
|
+
class PortDeclaration(BaseModel):
|
|
507
|
+
model_config = ConfigDict(extra="forbid")
|
|
508
|
+
|
|
509
|
+
service: str
|
|
510
|
+
published: PortOrSlot
|
|
511
|
+
target: PortOrSlot
|
|
512
|
+
protocol: Literal["tcp", "udp"] = "tcp"
|
|
513
|
+
host_ip: str | None = None
|
|
514
|
+
|
|
515
|
+
|
|
516
|
+
class ResolvedService(NamedTuple):
|
|
517
|
+
name: str
|
|
518
|
+
entry: ServiceEntry
|
|
519
|
+
routes: list[RouteConfig]
|
|
520
|
+
ports: list[PortDeclaration]
|
|
521
|
+
auth: bool = False
|
|
522
|
+
|
|
523
|
+
|
|
524
|
+
class K8sConfig(BaseModel):
|
|
525
|
+
model_config = ConfigDict(extra="forbid")
|
|
526
|
+
|
|
527
|
+
workloads: list[str] = Field(default_factory=list)
|
|
528
|
+
vars: dict[str, str] = Field(default_factory=dict)
|
|
529
|
+
charts: dict[str, str] = Field(default_factory=dict)
|
|
530
|
+
|
|
531
|
+
@model_validator(mode="after")
|
|
532
|
+
def _validate(self) -> Self:
|
|
533
|
+
for name in self.workloads:
|
|
534
|
+
require_workload_reference(name)
|
|
535
|
+
for name, reference in self.charts.items():
|
|
536
|
+
if not name:
|
|
537
|
+
raise ValueError("charts: names must be non-empty")
|
|
538
|
+
if not reference.startswith(OCI_REFERENCE_PREFIX):
|
|
539
|
+
raise ValueError(
|
|
540
|
+
f"charts.{name}: {reference!r} must be an upstream-native OCI reference "
|
|
541
|
+
f"({OCI_REFERENCE_PREFIX}host/path:tag or {OCI_REFERENCE_PREFIX}host/path@digest)"
|
|
542
|
+
)
|
|
543
|
+
remainder = reference.removeprefix(OCI_REFERENCE_PREFIX)
|
|
544
|
+
match = DECLARED_REFERENCE_PATTERN.fullmatch(remainder)
|
|
545
|
+
if match is None or not (match.group("tag") or match.group("digest")):
|
|
546
|
+
raise ValueError(f"charts.{name}: {reference!r} must carry a tag or a digest")
|
|
547
|
+
if not (self.workloads or self.charts):
|
|
548
|
+
raise ValueError("the k8s section must declare workloads or charts")
|
|
549
|
+
return self
|
|
550
|
+
|
|
551
|
+
|
|
552
|
+
PROBE_ADAPTER_ANNOTATION = "outernet.internal/probe-adapter"
|
|
553
|
+
PROBE_ADAPTERS = ("python-urllib", "bash-http")
|
|
554
|
+
RESTART_ANNOTATION = "outernet.internal/restart"
|
|
555
|
+
COMPOSE_RESTART_POLICIES = ("no", "always", "on-failure", "unless-stopped")
|
|
556
|
+
CONSUMER_INTERPOLATED_ANNOTATION = "outernet.internal/consumer-interpolated"
|
|
557
|
+
|
|
558
|
+
|
|
559
|
+
class WorkloadMetadata(BaseModel):
|
|
560
|
+
model_config = ConfigDict(extra="allow")
|
|
561
|
+
|
|
562
|
+
name: str | None = None
|
|
563
|
+
annotations: dict[str, str] = Field(default_factory=dict)
|
|
564
|
+
|
|
565
|
+
|
|
566
|
+
class WorkloadProbe(BaseModel):
|
|
567
|
+
model_config = ConfigDict(extra="allow")
|
|
568
|
+
|
|
569
|
+
http_get: Yaml | None = Field(default=None, alias="httpGet")
|
|
570
|
+
|
|
571
|
+
|
|
572
|
+
class WorkloadContainer(BaseModel):
|
|
573
|
+
model_config = ConfigDict(extra="allow")
|
|
574
|
+
|
|
575
|
+
image: str | None = None
|
|
576
|
+
files: dict[str, Yaml] = Field(default_factory=dict)
|
|
577
|
+
readiness_probe: WorkloadProbe | None = Field(default=None, alias="readinessProbe")
|
|
578
|
+
liveness_probe: WorkloadProbe | None = Field(default=None, alias="livenessProbe")
|
|
579
|
+
|
|
580
|
+
@field_validator("image")
|
|
581
|
+
@classmethod
|
|
582
|
+
def _require_image_slot(cls, value: str | None) -> str | None:
|
|
583
|
+
if value is not None and "${" not in value and "{{" not in value:
|
|
584
|
+
raise ValueError(f"image {value!r} must bind to the lock or a service sha via a ${{NAME}} slot")
|
|
585
|
+
return value
|
|
586
|
+
|
|
587
|
+
|
|
588
|
+
class WorkloadResource(BaseModel):
|
|
589
|
+
model_config = ConfigDict(extra="allow")
|
|
590
|
+
|
|
591
|
+
type: str
|
|
592
|
+
|
|
593
|
+
|
|
594
|
+
class WorkloadDocument(BaseModel):
|
|
595
|
+
model_config = ConfigDict(extra="allow", populate_by_name=True)
|
|
596
|
+
|
|
597
|
+
metadata: WorkloadMetadata = Field(default_factory=WorkloadMetadata)
|
|
598
|
+
containers: dict[str, WorkloadContainer] = Field(default_factory=dict)
|
|
599
|
+
resources: dict[str, WorkloadResource] = Field(default_factory=dict)
|
|
600
|
+
|
|
601
|
+
@classmethod
|
|
602
|
+
def parse(cls, raw: Yaml, file_name: str) -> Self:
|
|
603
|
+
return cls.model_validate(raw, context={"source": file_name})
|
|
604
|
+
|
|
605
|
+
@model_validator(mode="after")
|
|
606
|
+
def _require_name_matching_file_stem(self, info: ValidationInfo) -> Self:
|
|
607
|
+
source = info.context.get("source") if info.context is not None else None
|
|
608
|
+
if source is not None and self.metadata.name != Path(source).stem:
|
|
609
|
+
raise ValueError(f"metadata.name must be {Path(source).stem!r} to match the workload file name")
|
|
610
|
+
return self
|
|
611
|
+
|
|
612
|
+
@model_validator(mode="after")
|
|
613
|
+
def _validate_deployment_surface_annotations(self) -> Self:
|
|
614
|
+
annotations = self.metadata.annotations
|
|
615
|
+
|
|
616
|
+
adapter = annotations.get(PROBE_ADAPTER_ANNOTATION)
|
|
617
|
+
if adapter is not None and adapter not in PROBE_ADAPTERS:
|
|
618
|
+
raise ValueError(
|
|
619
|
+
f"{PROBE_ADAPTER_ANNOTATION} is {adapter!r}; the supported adapters are {list(PROBE_ADAPTERS)}"
|
|
620
|
+
)
|
|
621
|
+
|
|
622
|
+
restart = annotations.get(RESTART_ANNOTATION)
|
|
623
|
+
if restart is not None and restart not in COMPOSE_RESTART_POLICIES:
|
|
624
|
+
raise ValueError(
|
|
625
|
+
f"{RESTART_ANNOTATION} is {restart!r}; the compose restart policies are "
|
|
626
|
+
f"{list(COMPOSE_RESTART_POLICIES)}"
|
|
627
|
+
)
|
|
628
|
+
|
|
629
|
+
consumer_interpolated = annotations.get(CONSUMER_INTERPOLATED_ANNOTATION)
|
|
630
|
+
if consumer_interpolated is not None:
|
|
631
|
+
if not consumer_interpolated.strip():
|
|
632
|
+
raise ValueError(
|
|
633
|
+
f"{CONSUMER_INTERPOLATED_ANNOTATION} must be a non-empty comma-separated "
|
|
634
|
+
"list of the workload's files: paths"
|
|
635
|
+
)
|
|
636
|
+
declared_files = {target for container in self.containers.values() for target in container.files}
|
|
637
|
+
for entry in consumer_interpolated.split(","):
|
|
638
|
+
if entry != entry.strip() or not entry:
|
|
639
|
+
raise ValueError(
|
|
640
|
+
f"{CONSUMER_INTERPOLATED_ANNOTATION} entry {entry!r} must be non-empty and unpadded"
|
|
641
|
+
)
|
|
642
|
+
if entry not in declared_files:
|
|
643
|
+
raise ValueError(
|
|
644
|
+
f"{CONSUMER_INTERPOLATED_ANNOTATION} entry {entry!r} names no declared files: path"
|
|
645
|
+
)
|
|
646
|
+
|
|
647
|
+
return self
|
|
648
|
+
|
|
649
|
+
@property
|
|
650
|
+
def consumer_interpolated_targets(self) -> list[str]:
|
|
651
|
+
annotation = self.metadata.annotations.get(CONSUMER_INTERPOLATED_ANNOTATION)
|
|
652
|
+
return annotation.split(",") if annotation is not None else []
|
|
653
|
+
|
|
654
|
+
|
|
655
|
+
class ComposeWorkloadDocument(WorkloadDocument):
|
|
656
|
+
@model_validator(mode="after")
|
|
657
|
+
def _require_probe_adapter_for_http_get(self) -> Self:
|
|
658
|
+
if self.metadata.annotations.get(PROBE_ADAPTER_ANNOTATION) is not None:
|
|
659
|
+
return self
|
|
660
|
+
for container in self.containers.values():
|
|
661
|
+
for probe, block in (
|
|
662
|
+
("readinessProbe", container.readiness_probe),
|
|
663
|
+
("livenessProbe", container.liveness_probe),
|
|
664
|
+
):
|
|
665
|
+
if block is not None and block.http_get is not None:
|
|
666
|
+
raise ValueError(f"a {probe} httpGet requires {PROBE_ADAPTER_ANNOTATION}")
|
|
667
|
+
return self
|
|
668
|
+
|
|
669
|
+
|
|
670
|
+
def require_workload_reference(name: str) -> None:
|
|
671
|
+
if re.fullmatch(r"[A-Za-z0-9._-]+\.yaml", name) is None:
|
|
672
|
+
raise ValueError(f"workload reference {name!r} must be a flat .yaml file name")
|
|
673
|
+
|
|
674
|
+
|
|
675
|
+
def require_bundle_reference(value: str) -> None:
|
|
676
|
+
if not value.startswith(OCI_REFERENCE_PREFIX):
|
|
677
|
+
raise ValueError(f"ref {value!r} must be an OCI include ({OCI_REFERENCE_PREFIX}host/path:tag)")
|
|
678
|
+
remainder = value.removeprefix(OCI_REFERENCE_PREFIX)
|
|
679
|
+
if "@" in remainder:
|
|
680
|
+
raise ValueError(f"ref {value!r} pins a digest; a ref carries a published tag and the lock records the digest")
|
|
681
|
+
reference, separator, tag = remainder.rpartition(":")
|
|
682
|
+
if not separator or not reference or not tag or re.search(r"\s", remainder):
|
|
683
|
+
raise ValueError(f"ref {value!r} must carry a tag ({OCI_REFERENCE_PREFIX}host/path:tag)")
|
|
684
|
+
if tag == "latest":
|
|
685
|
+
raise ValueError(f"ref {value!r}: the tag 'latest' is mutable and never a pin; pin a stable, dev, or sha tag")
|
|
686
|
+
if REF_TAG_OPERATOR_PATTERN.search(tag) or any(part in ("x", "X") for part in tag.split(".")):
|
|
687
|
+
raise ValueError(f"ref {value!r}: the tag {tag!r} is a range; exact tags only")
|
|
688
|
+
|
|
689
|
+
|
|
690
|
+
def compose_bundle_output_path(root: Path, name: str) -> Path:
|
|
691
|
+
return root / COMPOSE_OUTPUT_DIR / f"compose.{name}.yaml"
|
|
692
|
+
|
|
693
|
+
|
|
694
|
+
def qualified_option_name(group: str, option_name: str) -> str:
|
|
695
|
+
return f"{group}-{option_name}"
|
|
696
|
+
|
|
697
|
+
|
|
698
|
+
def lock_key_prefix(name: str) -> str:
|
|
699
|
+
return name.upper().replace("-", "_") + "_"
|
|
700
|
+
|
|
701
|
+
|
|
702
|
+
LOCK_SLOT_MARKER = "__LOCK__"
|
|
703
|
+
|
|
704
|
+
|
|
705
|
+
def env_lock_slot(entry_name: str, lock_key: str) -> str:
|
|
706
|
+
# The marker reserves the fan spelling against values-file keys; the values parse refuses it
|
|
707
|
+
return entry_name.upper().replace("-", "_") + LOCK_SLOT_MARKER + lock_key
|
|
708
|
+
|
|
709
|
+
|
|
710
|
+
def chart_lock_key(name: str) -> str:
|
|
711
|
+
return lock_key_prefix(name) + "CHART"
|
|
712
|
+
|
|
713
|
+
|
|
714
|
+
def bundle_lock_key(name: str) -> str:
|
|
715
|
+
return lock_key_prefix(name) + "BUNDLE"
|
|
716
|
+
|
|
717
|
+
|
|
718
|
+
def scan_bundle_texts(
|
|
719
|
+
entry_name: str,
|
|
720
|
+
routes: list[RouteConfig],
|
|
721
|
+
ports: list[PortDeclaration],
|
|
722
|
+
texts: list[str],
|
|
723
|
+
) -> BundleScan:
|
|
724
|
+
# Refuse declared routes and ports that name no service in the bundle and answer its requirement slots.
|
|
725
|
+
service_names = refuse_unresolved_references(entry_name, routes, ports, texts, prefix=f"{entry_name}-")
|
|
726
|
+
return BundleScan(service_names=service_names, requires=sorted(required_compose_slots(texts)))
|
|
727
|
+
|
|
728
|
+
|
|
729
|
+
def required_compose_slots(texts: list[str]) -> set[str]:
|
|
730
|
+
# Odd $-parity slots are interpolation the consumer must fill; a :- default spells optional
|
|
731
|
+
# and $$-escaped literals are not slots at all.
|
|
732
|
+
slots: set[str] = set()
|
|
733
|
+
for text in texts:
|
|
734
|
+
slots.update(
|
|
735
|
+
match.group(2)
|
|
736
|
+
for match in COMPOSE_SLOT.finditer(text)
|
|
737
|
+
if len(match.group(1)) % 2 == 1 and not (match.group(3) or "").startswith(":-")
|
|
738
|
+
)
|
|
739
|
+
return slots
|
|
740
|
+
|
|
741
|
+
|
|
742
|
+
def refuse_unresolved_references(
|
|
743
|
+
entry_name: str,
|
|
744
|
+
routes: list[RouteConfig],
|
|
745
|
+
ports: list[PortDeclaration],
|
|
746
|
+
texts: list[str],
|
|
747
|
+
prefix: str = "",
|
|
748
|
+
) -> set[str]:
|
|
749
|
+
# Union the services keys across the compose texts; a document without a services mapping contributes none.
|
|
750
|
+
service_names: set[str] = set()
|
|
751
|
+
for text in texts:
|
|
752
|
+
document = yaml.safe_load(text)
|
|
753
|
+
services = document.get("services") if is_mapping(document) else None
|
|
754
|
+
if is_mapping(services):
|
|
755
|
+
service_names.update(services)
|
|
756
|
+
references = {f"{prefix}{route.upstream.partition(':')[0]}": route.path for route in routes}
|
|
757
|
+
references.update({f"{prefix}{port.service}": str(port.published) for port in ports})
|
|
758
|
+
unknown = sorted(name for name in references if name not in service_names)
|
|
759
|
+
if unknown:
|
|
760
|
+
name = unknown[0]
|
|
761
|
+
raise ValueError(
|
|
762
|
+
f"services entry {entry_name!r}: {references[name]!r} references {name!r}, "
|
|
763
|
+
f"which resolves to no service in the bundle (services: {sorted(service_names)})"
|
|
764
|
+
)
|
|
765
|
+
return service_names
|
|
766
|
+
|
|
767
|
+
|
|
768
|
+
def mirrored_ref(reference: str, prefix: str) -> str:
|
|
769
|
+
if reference.startswith(f"{prefix}/"):
|
|
770
|
+
raise ValueError(f"declarations are upstream-native; {reference} already carries the mirror prefix {prefix}")
|
|
771
|
+
return f"{prefix}/{reference}"
|
|
772
|
+
|
|
773
|
+
|
|
774
|
+
def _resolve_template_key(pins: Mapping[str, str], match: re.Match[str]) -> str:
|
|
775
|
+
return pins[match.group(1)]
|
|
776
|
+
|
|
777
|
+
|
|
778
|
+
def is_mapping(value: object) -> TypeIs[dict[str, Yaml]]:
|
|
779
|
+
return isinstance(value, dict)
|
|
780
|
+
|
|
781
|
+
|
|
782
|
+
def is_list(value: object) -> TypeIs[list[Yaml]]:
|
|
783
|
+
return isinstance(value, list)
|
|
784
|
+
|
|
785
|
+
|
|
786
|
+
def double_dollars(text: str) -> str:
|
|
787
|
+
return text.replace("$", "$$")
|