dploydb 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- dploydb/__init__.py +1 -0
- dploydb/__main__.py +6 -0
- dploydb/backup.py +455 -0
- dploydb/candidate.py +868 -0
- dploydb/cli.py +1111 -0
- dploydb/config.py +784 -0
- dploydb/cutover.py +326 -0
- dploydb/deploy.py +1163 -0
- dploydb/deployment_dependencies.py +366 -0
- dploydb/deployment_evidence.py +136 -0
- dploydb/diagnostics.py +1020 -0
- dploydb/errors.py +140 -0
- dploydb/health.py +625 -0
- dploydb/locking.py +609 -0
- dploydb/manual_restore.py +825 -0
- dploydb/migration.py +575 -0
- dploydb/models.py +927 -0
- dploydb/recovery.py +1210 -0
- dploydb/redaction.py +229 -0
- dploydb/releases.py +611 -0
- dploydb/restore.py +461 -0
- dploydb/retention.py +165 -0
- dploydb/runners/__init__.py +33 -0
- dploydb/runners/base.py +389 -0
- dploydb/runners/docker_compose.py +590 -0
- dploydb/runners/docker_compose_production.py +966 -0
- dploydb/sqlite_checks.py +136 -0
- dploydb/state.py +604 -0
- dploydb/storage/__init__.py +13 -0
- dploydb/storage/base.py +52 -0
- dploydb/storage/local.py +301 -0
- dploydb/storage/s3.py +737 -0
- dploydb/subprocesses.py +641 -0
- dploydb/traffic.py +165 -0
- dploydb-0.1.0.dist-info/METADATA +583 -0
- dploydb-0.1.0.dist-info/RECORD +40 -0
- dploydb-0.1.0.dist-info/WHEEL +4 -0
- dploydb-0.1.0.dist-info/entry_points.txt +2 -0
- dploydb-0.1.0.dist-info/licenses/LICENSE +201 -0
- dploydb-0.1.0.dist-info/licenses/NOTICE +4 -0
dploydb/config.py
ADDED
|
@@ -0,0 +1,784 @@
|
|
|
1
|
+
"""Strict, side-effect-free configuration parsing for DployDB.
|
|
2
|
+
|
|
3
|
+
Configuration handling is deliberately split into phases. Structural parsing does
|
|
4
|
+
not inspect the host. Environment interpolation and secret registration happen in
|
|
5
|
+
a separate in-memory phase. Filesystem, database, socket, Docker, and executable
|
|
6
|
+
checks belong to ``doctor`` in Milestone 1G.
|
|
7
|
+
"""
|
|
8
|
+
|
|
9
|
+
from __future__ import annotations
|
|
10
|
+
|
|
11
|
+
import hashlib
|
|
12
|
+
import json
|
|
13
|
+
import os
|
|
14
|
+
import re
|
|
15
|
+
from collections.abc import Mapping
|
|
16
|
+
from dataclasses import dataclass
|
|
17
|
+
from pathlib import Path, PurePosixPath
|
|
18
|
+
from typing import Annotated, Any, Final, Literal, Self
|
|
19
|
+
from urllib.parse import urlsplit
|
|
20
|
+
|
|
21
|
+
import yaml
|
|
22
|
+
from pydantic import (
|
|
23
|
+
AfterValidator,
|
|
24
|
+
BaseModel,
|
|
25
|
+
BeforeValidator,
|
|
26
|
+
ConfigDict,
|
|
27
|
+
Field,
|
|
28
|
+
ValidationError,
|
|
29
|
+
ValidationInfo,
|
|
30
|
+
field_validator,
|
|
31
|
+
model_validator,
|
|
32
|
+
)
|
|
33
|
+
|
|
34
|
+
from dploydb.errors import ConfigurationError
|
|
35
|
+
from dploydb.redaction import SecretRegistry, is_sensitive_key
|
|
36
|
+
|
|
37
|
+
DEFAULT_CONFIG_PATH = Path("dploydb.yaml")
|
|
38
|
+
CONFIG_FILE_MODE = 0o600
|
|
39
|
+
|
|
40
|
+
_ENVIRONMENT_NAME = re.compile(r"[A-Za-z_][A-Za-z0-9_]*\Z")
|
|
41
|
+
_INTERPOLATION = re.compile(r"\$\{([A-Za-z_][A-Za-z0-9_]*)\}")
|
|
42
|
+
_PROJECT_NAME = re.compile(r"[A-Za-z0-9][A-Za-z0-9._-]{0,63}\Z")
|
|
43
|
+
_COMPOSE_SERVICE_NAME = re.compile(r"[A-Za-z0-9][A-Za-z0-9._-]{0,127}\Z")
|
|
44
|
+
_S3_BUCKET_NAME = re.compile(r"[a-z0-9][a-z0-9.-]{1,61}[a-z0-9]\Z")
|
|
45
|
+
_LOCAL_HEALTH_HOSTS: Final[frozenset[str]] = frozenset({"127.0.0.1", "localhost", "::1"})
|
|
46
|
+
_RESERVED_CANDIDATE_ENVIRONMENT: Final[frozenset[str]] = frozenset({"DPLOYDB_VERSION"})
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
def _configuration_error(what_failed: str) -> ConfigurationError:
|
|
50
|
+
return ConfigurationError(
|
|
51
|
+
what_failed,
|
|
52
|
+
production_changed=False,
|
|
53
|
+
previous_application_running=None,
|
|
54
|
+
next_safe_action="Correct the configuration and run the command again.",
|
|
55
|
+
)
|
|
56
|
+
|
|
57
|
+
|
|
58
|
+
def _allow_unresolved(info: ValidationInfo) -> bool:
|
|
59
|
+
context = info.context
|
|
60
|
+
return bool(context and context.get("allow_unresolved_environment"))
|
|
61
|
+
|
|
62
|
+
|
|
63
|
+
def _contains_interpolation(value: str) -> bool:
|
|
64
|
+
return _INTERPOLATION.search(value) is not None
|
|
65
|
+
|
|
66
|
+
|
|
67
|
+
def _non_empty_text(value: str) -> str:
|
|
68
|
+
if not value.strip():
|
|
69
|
+
raise ValueError("must not be empty")
|
|
70
|
+
if "\x00" in value:
|
|
71
|
+
raise ValueError("must not contain a NUL byte")
|
|
72
|
+
return value
|
|
73
|
+
|
|
74
|
+
|
|
75
|
+
def _absolute_path(value: object, info: ValidationInfo) -> Path:
|
|
76
|
+
if not isinstance(value, str):
|
|
77
|
+
raise ValueError("must be an absolute path string")
|
|
78
|
+
_non_empty_text(value)
|
|
79
|
+
if _allow_unresolved(info) and _contains_interpolation(value):
|
|
80
|
+
return Path(value)
|
|
81
|
+
|
|
82
|
+
path = Path(value)
|
|
83
|
+
if not path.is_absolute():
|
|
84
|
+
raise ValueError("must be an absolute path")
|
|
85
|
+
if path == Path("/"):
|
|
86
|
+
raise ValueError("must not be the filesystem root")
|
|
87
|
+
return path
|
|
88
|
+
|
|
89
|
+
|
|
90
|
+
def _argument_array(value: object) -> tuple[str, ...]:
|
|
91
|
+
if not isinstance(value, list):
|
|
92
|
+
raise ValueError("must be a YAML argument array")
|
|
93
|
+
if not value:
|
|
94
|
+
raise ValueError("must contain at least one argument")
|
|
95
|
+
arguments: list[str] = []
|
|
96
|
+
for argument in value:
|
|
97
|
+
if not isinstance(argument, str):
|
|
98
|
+
raise ValueError("arguments must be strings")
|
|
99
|
+
arguments.append(_non_empty_text(argument))
|
|
100
|
+
return tuple(arguments)
|
|
101
|
+
|
|
102
|
+
|
|
103
|
+
NonEmptyText = Annotated[str, Field(strict=True), AfterValidator(_non_empty_text)]
|
|
104
|
+
AbsolutePath = Annotated[Path, BeforeValidator(_absolute_path)]
|
|
105
|
+
ArgumentArray = Annotated[tuple[str, ...], BeforeValidator(_argument_array)]
|
|
106
|
+
PositiveInt = Annotated[int, Field(strict=True, gt=0)]
|
|
107
|
+
PositiveNumber = Annotated[float, Field(strict=True, gt=0)]
|
|
108
|
+
Port = Annotated[int, Field(strict=True, ge=1, le=65535)]
|
|
109
|
+
|
|
110
|
+
|
|
111
|
+
class StrictConfigModel(BaseModel):
|
|
112
|
+
"""Base model that rejects typos and implicit type coercion."""
|
|
113
|
+
|
|
114
|
+
model_config = ConfigDict(extra="forbid", frozen=True, strict=True)
|
|
115
|
+
|
|
116
|
+
def __repr__(self) -> str:
|
|
117
|
+
"""Avoid exposing interpolated values through incidental diagnostics."""
|
|
118
|
+
return f"{type(self).__name__}(<configuration values hidden>)"
|
|
119
|
+
|
|
120
|
+
def __str__(self) -> str:
|
|
121
|
+
"""Avoid exposing interpolated values through incidental diagnostics."""
|
|
122
|
+
return repr(self)
|
|
123
|
+
|
|
124
|
+
|
|
125
|
+
class DatabaseConfig(StrictConfigModel):
|
|
126
|
+
"""SQLite database location and migration environment contract."""
|
|
127
|
+
|
|
128
|
+
path: AbsolutePath
|
|
129
|
+
path_env: NonEmptyText
|
|
130
|
+
minimum_free_space_multiplier: PositiveNumber = 3.0
|
|
131
|
+
|
|
132
|
+
@field_validator("path_env")
|
|
133
|
+
@classmethod
|
|
134
|
+
def validate_path_environment_name(cls, value: str) -> str:
|
|
135
|
+
if _ENVIRONMENT_NAME.fullmatch(value) is None:
|
|
136
|
+
raise ValueError("must be a valid environment-variable name")
|
|
137
|
+
return value
|
|
138
|
+
|
|
139
|
+
|
|
140
|
+
class MigrationConfig(StrictConfigModel):
|
|
141
|
+
"""Developer-supplied migration command and its mandatory timeout."""
|
|
142
|
+
|
|
143
|
+
command: ArgumentArray
|
|
144
|
+
timeout_seconds: PositiveInt
|
|
145
|
+
|
|
146
|
+
|
|
147
|
+
def _validate_local_health_url(
|
|
148
|
+
value: str,
|
|
149
|
+
*,
|
|
150
|
+
expected_port: int,
|
|
151
|
+
field_name: str,
|
|
152
|
+
port_name: str,
|
|
153
|
+
info: ValidationInfo,
|
|
154
|
+
) -> None:
|
|
155
|
+
if _allow_unresolved(info) and _contains_interpolation(value):
|
|
156
|
+
return
|
|
157
|
+
try:
|
|
158
|
+
parsed = urlsplit(value)
|
|
159
|
+
parsed_port = parsed.port
|
|
160
|
+
except ValueError as exc:
|
|
161
|
+
raise ValueError(f"{field_name} must be a valid local HTTP URL") from exc
|
|
162
|
+
|
|
163
|
+
if parsed.scheme != "http":
|
|
164
|
+
raise ValueError(f"{field_name} must use http")
|
|
165
|
+
if parsed.hostname is None or parsed.hostname.lower() not in _LOCAL_HEALTH_HOSTS:
|
|
166
|
+
raise ValueError(f"{field_name} must use a loopback host")
|
|
167
|
+
if parsed.username is not None or parsed.password is not None:
|
|
168
|
+
raise ValueError(f"{field_name} must not contain credentials")
|
|
169
|
+
if parsed_port is None:
|
|
170
|
+
raise ValueError(f"{field_name} must include the {port_name}")
|
|
171
|
+
if parsed_port != expected_port:
|
|
172
|
+
raise ValueError(f"{field_name} port must match {port_name}")
|
|
173
|
+
if not parsed.path.startswith("/"):
|
|
174
|
+
raise ValueError(f"{field_name} must contain an absolute URL path")
|
|
175
|
+
if parsed.query or parsed.fragment:
|
|
176
|
+
raise ValueError(f"{field_name} must not contain a query or fragment")
|
|
177
|
+
|
|
178
|
+
|
|
179
|
+
class ApplicationConfig(StrictConfigModel):
|
|
180
|
+
"""Single Docker Compose application and isolated candidate settings."""
|
|
181
|
+
|
|
182
|
+
runner: Literal["docker_compose"]
|
|
183
|
+
compose_file: AbsolutePath
|
|
184
|
+
service: NonEmptyText
|
|
185
|
+
production_project: NonEmptyText | None = None
|
|
186
|
+
production_port: Port | None = None
|
|
187
|
+
production_health_url: NonEmptyText | None = None
|
|
188
|
+
candidate_port: Port
|
|
189
|
+
candidate_container_port: Port = 8080
|
|
190
|
+
database_volume_target: NonEmptyText = "/data"
|
|
191
|
+
candidate_health_url: NonEmptyText
|
|
192
|
+
startup_timeout_seconds: PositiveInt
|
|
193
|
+
smoke_command: ArgumentArray | None = None
|
|
194
|
+
test_mode_env: dict[str, str] = Field(default_factory=dict)
|
|
195
|
+
|
|
196
|
+
@field_validator("test_mode_env", mode="before")
|
|
197
|
+
@classmethod
|
|
198
|
+
def validate_test_environment_shape(cls, value: object) -> object:
|
|
199
|
+
if not isinstance(value, dict):
|
|
200
|
+
raise ValueError("must be a mapping of environment names to string values")
|
|
201
|
+
return value
|
|
202
|
+
|
|
203
|
+
@field_validator("test_mode_env")
|
|
204
|
+
@classmethod
|
|
205
|
+
def validate_test_environment(cls, value: dict[str, str]) -> dict[str, str]:
|
|
206
|
+
for name, environment_value in value.items():
|
|
207
|
+
if _ENVIRONMENT_NAME.fullmatch(name) is None:
|
|
208
|
+
raise ValueError("contains an invalid environment-variable name")
|
|
209
|
+
if name in _RESERVED_CANDIDATE_ENVIRONMENT:
|
|
210
|
+
raise ValueError(f"must not override reserved environment variable {name}")
|
|
211
|
+
_non_empty_text(environment_value)
|
|
212
|
+
return value
|
|
213
|
+
|
|
214
|
+
@field_validator("service", "production_project")
|
|
215
|
+
@classmethod
|
|
216
|
+
def validate_compose_name(cls, value: str | None, info: ValidationInfo) -> str | None:
|
|
217
|
+
if value is None:
|
|
218
|
+
return None
|
|
219
|
+
if _allow_unresolved(info) and _contains_interpolation(value):
|
|
220
|
+
return value
|
|
221
|
+
if _COMPOSE_SERVICE_NAME.fullmatch(value) is None:
|
|
222
|
+
raise ValueError("must be a safe Docker Compose service name")
|
|
223
|
+
return value
|
|
224
|
+
|
|
225
|
+
@field_validator("database_volume_target")
|
|
226
|
+
@classmethod
|
|
227
|
+
def validate_database_volume_target(cls, value: str, info: ValidationInfo) -> str:
|
|
228
|
+
if _allow_unresolved(info) and _contains_interpolation(value):
|
|
229
|
+
return value
|
|
230
|
+
path = PurePosixPath(value)
|
|
231
|
+
if not path.is_absolute() or path == PurePosixPath("/"):
|
|
232
|
+
raise ValueError("must be an absolute non-root container directory")
|
|
233
|
+
if str(path) != value or value.startswith("//"):
|
|
234
|
+
raise ValueError("must be a normalized container path")
|
|
235
|
+
if any(part in {".", ".."} for part in path.parts):
|
|
236
|
+
raise ValueError("must not contain traversal segments")
|
|
237
|
+
if ":" in value:
|
|
238
|
+
raise ValueError("must not contain a colon")
|
|
239
|
+
return value
|
|
240
|
+
|
|
241
|
+
@model_validator(mode="after")
|
|
242
|
+
def validate_health_topology(self, info: ValidationInfo) -> Self:
|
|
243
|
+
_validate_local_health_url(
|
|
244
|
+
self.candidate_health_url,
|
|
245
|
+
expected_port=self.candidate_port,
|
|
246
|
+
field_name="candidate_health_url",
|
|
247
|
+
port_name="candidate_port",
|
|
248
|
+
info=info,
|
|
249
|
+
)
|
|
250
|
+
|
|
251
|
+
production_values = (
|
|
252
|
+
self.production_project,
|
|
253
|
+
self.production_port,
|
|
254
|
+
self.production_health_url,
|
|
255
|
+
)
|
|
256
|
+
if all(value is None for value in production_values):
|
|
257
|
+
return self
|
|
258
|
+
if any(value is None for value in production_values):
|
|
259
|
+
raise ValueError(
|
|
260
|
+
"production_project, production_port, and production_health_url "
|
|
261
|
+
"must be configured together"
|
|
262
|
+
)
|
|
263
|
+
assert self.production_port is not None
|
|
264
|
+
assert self.production_health_url is not None
|
|
265
|
+
if self.production_port == self.candidate_port:
|
|
266
|
+
raise ValueError("production_port must differ from candidate_port")
|
|
267
|
+
_validate_local_health_url(
|
|
268
|
+
self.production_health_url,
|
|
269
|
+
expected_port=self.production_port,
|
|
270
|
+
field_name="production_health_url",
|
|
271
|
+
port_name="production_port",
|
|
272
|
+
info=info,
|
|
273
|
+
)
|
|
274
|
+
return self
|
|
275
|
+
|
|
276
|
+
|
|
277
|
+
class TrafficConfig(StrictConfigModel):
|
|
278
|
+
"""Bounded command hooks used by the later cutover milestone."""
|
|
279
|
+
|
|
280
|
+
maintenance_on_command: ArgumentArray
|
|
281
|
+
maintenance_off_command: ArgumentArray
|
|
282
|
+
activate_new_command: ArgumentArray
|
|
283
|
+
activate_old_command: ArgumentArray
|
|
284
|
+
timeout_seconds: PositiveInt = 30
|
|
285
|
+
|
|
286
|
+
|
|
287
|
+
class RemoteBackupConfig(StrictConfigModel):
|
|
288
|
+
"""Optional S3-compatible backup configuration."""
|
|
289
|
+
|
|
290
|
+
enabled: bool = False
|
|
291
|
+
required: bool = False
|
|
292
|
+
provider: Literal["s3"] = "s3"
|
|
293
|
+
bucket: NonEmptyText | None = None
|
|
294
|
+
prefix: str = ""
|
|
295
|
+
region_name: NonEmptyText = "auto"
|
|
296
|
+
storage_class: Literal["STANDARD", "STANDARD_IA"] = "STANDARD"
|
|
297
|
+
endpoint_url: NonEmptyText | None = None
|
|
298
|
+
endpoint_url_env: NonEmptyText | None = None
|
|
299
|
+
access_key_env: NonEmptyText | None = None
|
|
300
|
+
secret_key_env: NonEmptyText | None = None
|
|
301
|
+
session_token_env: NonEmptyText | None = None
|
|
302
|
+
timeout_seconds: PositiveInt = 30
|
|
303
|
+
max_attempts: PositiveInt = 3
|
|
304
|
+
|
|
305
|
+
@field_validator(
|
|
306
|
+
"endpoint_url_env",
|
|
307
|
+
"access_key_env",
|
|
308
|
+
"secret_key_env",
|
|
309
|
+
"session_token_env",
|
|
310
|
+
)
|
|
311
|
+
@classmethod
|
|
312
|
+
def validate_environment_name(cls, value: str | None) -> str | None:
|
|
313
|
+
if value is not None and _ENVIRONMENT_NAME.fullmatch(value) is None:
|
|
314
|
+
raise ValueError("must be a valid environment-variable name")
|
|
315
|
+
return value
|
|
316
|
+
|
|
317
|
+
@field_validator("bucket")
|
|
318
|
+
@classmethod
|
|
319
|
+
def validate_bucket(cls, value: str | None, info: ValidationInfo) -> str | None:
|
|
320
|
+
if value is None or (_allow_unresolved(info) and _contains_interpolation(value)):
|
|
321
|
+
return value
|
|
322
|
+
if _S3_BUCKET_NAME.fullmatch(value) is None or ".." in value:
|
|
323
|
+
raise ValueError("must be a valid lowercase S3-compatible bucket name")
|
|
324
|
+
return value
|
|
325
|
+
|
|
326
|
+
@field_validator("endpoint_url")
|
|
327
|
+
@classmethod
|
|
328
|
+
def validate_endpoint_url(cls, value: str | None, info: ValidationInfo) -> str | None:
|
|
329
|
+
if value is None or (_allow_unresolved(info) and _contains_interpolation(value)):
|
|
330
|
+
return value
|
|
331
|
+
try:
|
|
332
|
+
parsed = urlsplit(value)
|
|
333
|
+
port = parsed.port
|
|
334
|
+
except ValueError as exc:
|
|
335
|
+
raise ValueError("must be a valid S3-compatible endpoint URL") from exc
|
|
336
|
+
if parsed.scheme not in {"http", "https"} or parsed.hostname is None:
|
|
337
|
+
raise ValueError("must be an HTTP or HTTPS endpoint URL")
|
|
338
|
+
if parsed.scheme == "http" and parsed.hostname.lower() not in _LOCAL_HEALTH_HOSTS:
|
|
339
|
+
raise ValueError("must use HTTPS unless the endpoint host is loopback")
|
|
340
|
+
if parsed.username is not None or parsed.password is not None:
|
|
341
|
+
raise ValueError("must not contain credentials")
|
|
342
|
+
if parsed.query or parsed.fragment:
|
|
343
|
+
raise ValueError("must not contain a query or fragment")
|
|
344
|
+
if parsed.path not in {"", "/"}:
|
|
345
|
+
raise ValueError("must not contain an object path")
|
|
346
|
+
if port is not None and not 1 <= port <= 65535:
|
|
347
|
+
raise ValueError("contains an invalid port")
|
|
348
|
+
return value.rstrip("/")
|
|
349
|
+
|
|
350
|
+
@field_validator("prefix")
|
|
351
|
+
@classmethod
|
|
352
|
+
def validate_prefix(cls, value: str, info: ValidationInfo) -> str:
|
|
353
|
+
if _allow_unresolved(info) and _contains_interpolation(value):
|
|
354
|
+
return value
|
|
355
|
+
if value.startswith("/"):
|
|
356
|
+
raise ValueError("must be a relative object prefix")
|
|
357
|
+
if value and any(part in {"", ".", ".."} for part in value.split("/")):
|
|
358
|
+
raise ValueError("must be a normalized relative object prefix")
|
|
359
|
+
if "\x00" in value:
|
|
360
|
+
raise ValueError("must not contain a NUL byte")
|
|
361
|
+
if len(value.encode("utf-8")) > 768:
|
|
362
|
+
raise ValueError("must not exceed 768 UTF-8 bytes")
|
|
363
|
+
return value
|
|
364
|
+
|
|
365
|
+
@model_validator(mode="after")
|
|
366
|
+
def validate_enabled_remote(self) -> Self:
|
|
367
|
+
if self.required and not self.enabled:
|
|
368
|
+
raise ValueError("required remote backup must also be enabled")
|
|
369
|
+
if self.endpoint_url is not None and self.endpoint_url_env is not None:
|
|
370
|
+
raise ValueError("endpoint_url and endpoint_url_env are mutually exclusive")
|
|
371
|
+
if not self.enabled:
|
|
372
|
+
return self
|
|
373
|
+
missing = [
|
|
374
|
+
name
|
|
375
|
+
for name, value in (
|
|
376
|
+
("bucket", self.bucket),
|
|
377
|
+
("access_key_env", self.access_key_env),
|
|
378
|
+
("secret_key_env", self.secret_key_env),
|
|
379
|
+
)
|
|
380
|
+
if value is None
|
|
381
|
+
]
|
|
382
|
+
if missing:
|
|
383
|
+
raise ValueError("enabled remote backup requires " + ", ".join(missing))
|
|
384
|
+
return self
|
|
385
|
+
|
|
386
|
+
|
|
387
|
+
class BackupConfig(StrictConfigModel):
|
|
388
|
+
"""Verified local backup retention and optional remote target."""
|
|
389
|
+
|
|
390
|
+
local_directory: AbsolutePath
|
|
391
|
+
keep_last: PositiveInt
|
|
392
|
+
remote: RemoteBackupConfig | None = None
|
|
393
|
+
|
|
394
|
+
|
|
395
|
+
class DployDBConfig(StrictConfigModel):
|
|
396
|
+
"""Complete supported hackathon configuration contract."""
|
|
397
|
+
|
|
398
|
+
project: NonEmptyText
|
|
399
|
+
state_directory: AbsolutePath
|
|
400
|
+
database: DatabaseConfig
|
|
401
|
+
migration: MigrationConfig
|
|
402
|
+
application: ApplicationConfig
|
|
403
|
+
traffic: TrafficConfig
|
|
404
|
+
backup: BackupConfig
|
|
405
|
+
|
|
406
|
+
@field_validator("project")
|
|
407
|
+
@classmethod
|
|
408
|
+
def validate_project(cls, value: str, info: ValidationInfo) -> str:
|
|
409
|
+
if _allow_unresolved(info) and _contains_interpolation(value):
|
|
410
|
+
return value
|
|
411
|
+
if _PROJECT_NAME.fullmatch(value) is None:
|
|
412
|
+
raise ValueError(
|
|
413
|
+
"must be 1-64 letters, digits, dots, underscores, or hyphens and "
|
|
414
|
+
"start with a letter or digit"
|
|
415
|
+
)
|
|
416
|
+
return value
|
|
417
|
+
|
|
418
|
+
@model_validator(mode="after")
|
|
419
|
+
def validate_reserved_candidate_environment(self) -> Self:
|
|
420
|
+
if self.database.path_env in _RESERVED_CANDIDATE_ENVIRONMENT:
|
|
421
|
+
raise ValueError(
|
|
422
|
+
"database.path_env must not use reserved environment variable DPLOYDB_VERSION"
|
|
423
|
+
)
|
|
424
|
+
return self
|
|
425
|
+
|
|
426
|
+
|
|
427
|
+
@dataclass(frozen=True, slots=True)
|
|
428
|
+
class LoadedConfiguration:
|
|
429
|
+
"""Resolved configuration paired with its non-serializable secret registry."""
|
|
430
|
+
|
|
431
|
+
config: DployDBConfig
|
|
432
|
+
secrets: SecretRegistry
|
|
433
|
+
|
|
434
|
+
|
|
435
|
+
@dataclass(frozen=True, slots=True)
|
|
436
|
+
class ProductionTopology:
|
|
437
|
+
"""Deploy-only production application topology proven by configuration."""
|
|
438
|
+
|
|
439
|
+
compose_project: str
|
|
440
|
+
host_port: int
|
|
441
|
+
health_url: str
|
|
442
|
+
|
|
443
|
+
|
|
444
|
+
def require_deploy_topology(config: DployDBConfig) -> ProductionTopology:
|
|
445
|
+
"""Return complete deploy topology or fail before any operational access."""
|
|
446
|
+
application = config.application
|
|
447
|
+
if (
|
|
448
|
+
application.production_project is None
|
|
449
|
+
or application.production_port is None
|
|
450
|
+
or application.production_health_url is None
|
|
451
|
+
):
|
|
452
|
+
raise _configuration_error(
|
|
453
|
+
"deployment requires application.production_project, production_port, "
|
|
454
|
+
"and production_health_url"
|
|
455
|
+
)
|
|
456
|
+
return ProductionTopology(
|
|
457
|
+
compose_project=application.production_project,
|
|
458
|
+
host_port=application.production_port,
|
|
459
|
+
health_url=application.production_health_url,
|
|
460
|
+
)
|
|
461
|
+
|
|
462
|
+
|
|
463
|
+
def configuration_fingerprint(config: DployDBConfig, *, secrets: SecretRegistry) -> str:
|
|
464
|
+
"""Hash a canonical redacted configuration without persisting resolved secrets."""
|
|
465
|
+
safe = secrets.redact(config.model_dump(mode="json"))
|
|
466
|
+
payload = json.dumps(
|
|
467
|
+
safe,
|
|
468
|
+
sort_keys=True,
|
|
469
|
+
separators=(",", ":"),
|
|
470
|
+
ensure_ascii=False,
|
|
471
|
+
allow_nan=False,
|
|
472
|
+
).encode("utf-8")
|
|
473
|
+
return hashlib.sha256(payload).hexdigest()
|
|
474
|
+
|
|
475
|
+
|
|
476
|
+
class _DuplicateKeyError(yaml.YAMLError):
|
|
477
|
+
def __init__(self, mark: yaml.Mark) -> None:
|
|
478
|
+
self.mark = mark
|
|
479
|
+
super().__init__("duplicate YAML key")
|
|
480
|
+
|
|
481
|
+
|
|
482
|
+
class _UniqueKeyLoader(yaml.SafeLoader):
|
|
483
|
+
"""Safe YAML loader that refuses duplicate keys at every mapping level."""
|
|
484
|
+
|
|
485
|
+
|
|
486
|
+
def _construct_unique_mapping(
|
|
487
|
+
loader: _UniqueKeyLoader, node: yaml.MappingNode, deep: bool = False
|
|
488
|
+
) -> dict[object, object]:
|
|
489
|
+
mapping: dict[object, object] = {}
|
|
490
|
+
for key_node, value_node in node.value:
|
|
491
|
+
key = loader.construct_object(key_node, deep=deep)
|
|
492
|
+
try:
|
|
493
|
+
duplicate = key in mapping
|
|
494
|
+
except TypeError as exc:
|
|
495
|
+
raise yaml.constructor.ConstructorError(
|
|
496
|
+
"while constructing a mapping",
|
|
497
|
+
node.start_mark,
|
|
498
|
+
"found an unhashable mapping key",
|
|
499
|
+
key_node.start_mark,
|
|
500
|
+
) from exc
|
|
501
|
+
if duplicate:
|
|
502
|
+
raise _DuplicateKeyError(key_node.start_mark)
|
|
503
|
+
mapping[key] = loader.construct_object(value_node, deep=deep)
|
|
504
|
+
return mapping
|
|
505
|
+
|
|
506
|
+
|
|
507
|
+
_UniqueKeyLoader.add_constructor(
|
|
508
|
+
yaml.resolver.BaseResolver.DEFAULT_MAPPING_TAG,
|
|
509
|
+
_construct_unique_mapping,
|
|
510
|
+
)
|
|
511
|
+
|
|
512
|
+
|
|
513
|
+
def _location_text(location: tuple[int | str, ...]) -> str:
|
|
514
|
+
if not location:
|
|
515
|
+
return "configuration"
|
|
516
|
+
parts: list[str] = []
|
|
517
|
+
for item in location:
|
|
518
|
+
text = str(item)
|
|
519
|
+
parts.append("[sensitive-key]" if is_sensitive_key(text) else text)
|
|
520
|
+
return ".".join(parts)
|
|
521
|
+
|
|
522
|
+
|
|
523
|
+
def _validation_summary(error: ValidationError) -> str:
|
|
524
|
+
details: list[str] = []
|
|
525
|
+
for item in error.errors(include_url=False, include_input=False):
|
|
526
|
+
location = _location_text(item["loc"])
|
|
527
|
+
details.append(f"{location}: {item['msg']}")
|
|
528
|
+
return "; ".join(details)
|
|
529
|
+
|
|
530
|
+
|
|
531
|
+
def _validate_interpolation_syntax(value: object, location: tuple[int | str, ...] = ()) -> None:
|
|
532
|
+
if isinstance(value, str):
|
|
533
|
+
remainder = _INTERPOLATION.sub("", value)
|
|
534
|
+
if "${" in remainder:
|
|
535
|
+
raise _configuration_error(
|
|
536
|
+
f"{_location_text(location)} contains invalid environment interpolation"
|
|
537
|
+
)
|
|
538
|
+
return
|
|
539
|
+
if isinstance(value, list):
|
|
540
|
+
for index, item in enumerate(value):
|
|
541
|
+
_validate_interpolation_syntax(item, (*location, index))
|
|
542
|
+
return
|
|
543
|
+
if isinstance(value, dict):
|
|
544
|
+
for key, item in value.items():
|
|
545
|
+
_validate_interpolation_syntax(item, (*location, str(key)))
|
|
546
|
+
|
|
547
|
+
|
|
548
|
+
def parse_configuration(text: str) -> DployDBConfig:
|
|
549
|
+
"""Parse and structurally validate YAML without consulting the host."""
|
|
550
|
+
if not isinstance(text, str):
|
|
551
|
+
raise TypeError("configuration text must be a string")
|
|
552
|
+
try:
|
|
553
|
+
raw = yaml.load(text, Loader=_UniqueKeyLoader)
|
|
554
|
+
except _DuplicateKeyError as exc:
|
|
555
|
+
raise _configuration_error(
|
|
556
|
+
f"configuration contains a duplicate YAML key at line "
|
|
557
|
+
f"{exc.mark.line + 1}, column {exc.mark.column + 1}"
|
|
558
|
+
) from None
|
|
559
|
+
except yaml.YAMLError as exc:
|
|
560
|
+
mark = getattr(exc, "problem_mark", None)
|
|
561
|
+
location = f" at line {mark.line + 1}, column {mark.column + 1}" if mark is not None else ""
|
|
562
|
+
raise _configuration_error(f"configuration is not valid YAML{location}") from None
|
|
563
|
+
|
|
564
|
+
if not isinstance(raw, dict):
|
|
565
|
+
raise _configuration_error("configuration root must be a YAML mapping")
|
|
566
|
+
|
|
567
|
+
_validate_interpolation_syntax(raw)
|
|
568
|
+
try:
|
|
569
|
+
return DployDBConfig.model_validate(
|
|
570
|
+
raw,
|
|
571
|
+
context={"allow_unresolved_environment": True},
|
|
572
|
+
)
|
|
573
|
+
except ValidationError as exc:
|
|
574
|
+
raise _configuration_error(
|
|
575
|
+
f"configuration validation failed: {_validation_summary(exc)}"
|
|
576
|
+
) from None
|
|
577
|
+
|
|
578
|
+
|
|
579
|
+
def _missing_environment_names(value: object, environment: Mapping[str, str]) -> set[str]:
|
|
580
|
+
missing: set[str] = set()
|
|
581
|
+
if isinstance(value, str):
|
|
582
|
+
missing.update(name for name in _INTERPOLATION.findall(value) if name not in environment)
|
|
583
|
+
elif isinstance(value, list):
|
|
584
|
+
for item in value:
|
|
585
|
+
missing.update(_missing_environment_names(item, environment))
|
|
586
|
+
elif isinstance(value, dict):
|
|
587
|
+
for item in value.values():
|
|
588
|
+
missing.update(_missing_environment_names(item, environment))
|
|
589
|
+
return missing
|
|
590
|
+
|
|
591
|
+
|
|
592
|
+
def _resolve_value(
|
|
593
|
+
value: Any,
|
|
594
|
+
*,
|
|
595
|
+
environment: Mapping[str, str],
|
|
596
|
+
secrets: SecretRegistry,
|
|
597
|
+
field_name: str | None = None,
|
|
598
|
+
) -> Any:
|
|
599
|
+
if isinstance(value, str):
|
|
600
|
+
names = _INTERPOLATION.findall(value)
|
|
601
|
+
resolved = _INTERPOLATION.sub(lambda match: environment[match.group(1)], value)
|
|
602
|
+
if (field_name is not None and is_sensitive_key(field_name)) or any(
|
|
603
|
+
is_sensitive_key(name) for name in names
|
|
604
|
+
):
|
|
605
|
+
secrets.register_many(environment[name] for name in names)
|
|
606
|
+
if field_name is not None and is_sensitive_key(field_name):
|
|
607
|
+
secrets.register(resolved)
|
|
608
|
+
return resolved
|
|
609
|
+
if isinstance(value, list):
|
|
610
|
+
return [_resolve_value(item, environment=environment, secrets=secrets) for item in value]
|
|
611
|
+
if isinstance(value, dict):
|
|
612
|
+
return {
|
|
613
|
+
key: _resolve_value(
|
|
614
|
+
item,
|
|
615
|
+
environment=environment,
|
|
616
|
+
secrets=secrets,
|
|
617
|
+
field_name=key,
|
|
618
|
+
)
|
|
619
|
+
for key, item in value.items()
|
|
620
|
+
}
|
|
621
|
+
return value
|
|
622
|
+
|
|
623
|
+
|
|
624
|
+
def resolve_configuration(
|
|
625
|
+
config: DployDBConfig,
|
|
626
|
+
*,
|
|
627
|
+
environment: Mapping[str, str],
|
|
628
|
+
secrets: SecretRegistry,
|
|
629
|
+
) -> DployDBConfig:
|
|
630
|
+
"""Resolve ``${NAME}`` values and register secrets without host checks."""
|
|
631
|
+
raw = config.model_dump(mode="json")
|
|
632
|
+
missing = sorted(_missing_environment_names(raw, environment))
|
|
633
|
+
if missing:
|
|
634
|
+
raise _configuration_error(
|
|
635
|
+
"configuration references missing environment variables: " + ", ".join(missing)
|
|
636
|
+
)
|
|
637
|
+
|
|
638
|
+
resolved = _resolve_value(raw, environment=environment, secrets=secrets)
|
|
639
|
+
try:
|
|
640
|
+
return DployDBConfig.model_validate(
|
|
641
|
+
resolved,
|
|
642
|
+
context={"allow_unresolved_environment": False},
|
|
643
|
+
)
|
|
644
|
+
except ValidationError as exc:
|
|
645
|
+
raise _configuration_error(
|
|
646
|
+
f"resolved configuration validation failed: {_validation_summary(exc)}"
|
|
647
|
+
) from None
|
|
648
|
+
|
|
649
|
+
|
|
650
|
+
def load_configuration(
|
|
651
|
+
path: Path = DEFAULT_CONFIG_PATH,
|
|
652
|
+
*,
|
|
653
|
+
environment: Mapping[str, str] | None = None,
|
|
654
|
+
secrets: SecretRegistry | None = None,
|
|
655
|
+
) -> LoadedConfiguration:
|
|
656
|
+
"""Read, structurally validate, and resolve one configuration file."""
|
|
657
|
+
registry = secrets if secrets is not None else SecretRegistry()
|
|
658
|
+
try:
|
|
659
|
+
text = path.read_text(encoding="utf-8")
|
|
660
|
+
except OSError:
|
|
661
|
+
raise _configuration_error(f"configuration file could not be read: {path}") from None
|
|
662
|
+
structural = parse_configuration(text)
|
|
663
|
+
resolved = resolve_configuration(
|
|
664
|
+
structural,
|
|
665
|
+
environment=os.environ if environment is None else environment,
|
|
666
|
+
secrets=registry,
|
|
667
|
+
)
|
|
668
|
+
return LoadedConfiguration(config=resolved, secrets=registry)
|
|
669
|
+
|
|
670
|
+
|
|
671
|
+
STARTER_CONFIGURATION = """\
|
|
672
|
+
# DployDB configuration for one SQLite application on one Linux server.
|
|
673
|
+
# Adjust every /srv/example path before running doctor or a deployment.
|
|
674
|
+
project: example-app
|
|
675
|
+
|
|
676
|
+
state_directory: /srv/example/.dploydb
|
|
677
|
+
|
|
678
|
+
database:
|
|
679
|
+
path: /srv/example/data/app.db
|
|
680
|
+
path_env: DATABASE_PATH
|
|
681
|
+
minimum_free_space_multiplier: 3
|
|
682
|
+
|
|
683
|
+
migration:
|
|
684
|
+
# Commands are argument arrays. DployDB never enables a shell implicitly.
|
|
685
|
+
command: [python, scripts/migrate.py]
|
|
686
|
+
timeout_seconds: 120
|
|
687
|
+
|
|
688
|
+
application:
|
|
689
|
+
runner: docker_compose
|
|
690
|
+
compose_file: /srv/example/compose.yaml
|
|
691
|
+
service: app
|
|
692
|
+
# The existing production service is preserved exactly for rollback.
|
|
693
|
+
production_project: example-app
|
|
694
|
+
production_port: 4510
|
|
695
|
+
production_health_url: http://127.0.0.1:4510/health
|
|
696
|
+
candidate_port: 4511
|
|
697
|
+
# Container-side defaults are explicit so candidate isolation can be inspected.
|
|
698
|
+
candidate_container_port: 8080
|
|
699
|
+
database_volume_target: /data
|
|
700
|
+
candidate_health_url: http://127.0.0.1:4511/health
|
|
701
|
+
startup_timeout_seconds: 45
|
|
702
|
+
# Omit smoke_command when the HTTP health check is sufficient.
|
|
703
|
+
smoke_command: [python, scripts/smoke_test.py]
|
|
704
|
+
test_mode_env:
|
|
705
|
+
DPLOYDB_TEST_MODE: "1"
|
|
706
|
+
|
|
707
|
+
traffic:
|
|
708
|
+
maintenance_on_command: [/srv/example/ops/maintenance, "on"]
|
|
709
|
+
maintenance_off_command: [/srv/example/ops/maintenance, "off"]
|
|
710
|
+
activate_new_command: [/srv/example/ops/activate, candidate]
|
|
711
|
+
activate_old_command: [/srv/example/ops/activate, current]
|
|
712
|
+
timeout_seconds: 30
|
|
713
|
+
|
|
714
|
+
backup:
|
|
715
|
+
local_directory: /srv/dploydb/backups/example-app
|
|
716
|
+
keep_last: 10
|
|
717
|
+
# Remote backup is optional. Credentials are read only from named variables.
|
|
718
|
+
remote:
|
|
719
|
+
enabled: false
|
|
720
|
+
required: false
|
|
721
|
+
provider: s3
|
|
722
|
+
bucket: example-backups
|
|
723
|
+
prefix: dploydb/example-app
|
|
724
|
+
region_name: auto
|
|
725
|
+
storage_class: STANDARD
|
|
726
|
+
# Set endpoint_url for S3-compatible services such as Cloudflare R2, or
|
|
727
|
+
# name an environment variable containing it with endpoint_url_env.
|
|
728
|
+
endpoint_url_env: S3_ENDPOINT_URL
|
|
729
|
+
access_key_env: S3_ACCESS_KEY_ID
|
|
730
|
+
secret_key_env: S3_SECRET_ACCESS_KEY
|
|
731
|
+
timeout_seconds: 30
|
|
732
|
+
max_attempts: 3
|
|
733
|
+
"""
|
|
734
|
+
|
|
735
|
+
|
|
736
|
+
def initialize_configuration(path: Path = DEFAULT_CONFIG_PATH) -> Path:
|
|
737
|
+
"""Create a valid mode-0600 starter configuration without overwriting."""
|
|
738
|
+
# Keep the shipped template subject to the same parser as user configuration.
|
|
739
|
+
structural = parse_configuration(STARTER_CONFIGURATION)
|
|
740
|
+
resolve_configuration(structural, environment={}, secrets=SecretRegistry())
|
|
741
|
+
|
|
742
|
+
flags = os.O_WRONLY | os.O_CREAT | os.O_EXCL
|
|
743
|
+
flags |= getattr(os, "O_CLOEXEC", 0)
|
|
744
|
+
flags |= getattr(os, "O_NOFOLLOW", 0)
|
|
745
|
+
try:
|
|
746
|
+
descriptor = os.open(path, flags, CONFIG_FILE_MODE)
|
|
747
|
+
except FileExistsError:
|
|
748
|
+
raise _configuration_error(
|
|
749
|
+
f"configuration path already exists and was preserved: {path}"
|
|
750
|
+
) from None
|
|
751
|
+
except OSError:
|
|
752
|
+
raise _configuration_error(f"configuration file could not be created: {path}") from None
|
|
753
|
+
|
|
754
|
+
created = True
|
|
755
|
+
try:
|
|
756
|
+
os.fchmod(descriptor, CONFIG_FILE_MODE)
|
|
757
|
+
data = STARTER_CONFIGURATION.encode("utf-8")
|
|
758
|
+
written = 0
|
|
759
|
+
while written < len(data):
|
|
760
|
+
write_count = os.write(descriptor, data[written:])
|
|
761
|
+
if write_count <= 0:
|
|
762
|
+
raise OSError("configuration write made no progress")
|
|
763
|
+
written += write_count
|
|
764
|
+
os.fsync(descriptor)
|
|
765
|
+
os.close(descriptor)
|
|
766
|
+
descriptor = -1
|
|
767
|
+
except OSError:
|
|
768
|
+
if descriptor >= 0:
|
|
769
|
+
try:
|
|
770
|
+
os.close(descriptor)
|
|
771
|
+
except OSError:
|
|
772
|
+
pass
|
|
773
|
+
try:
|
|
774
|
+
path.unlink()
|
|
775
|
+
created = False
|
|
776
|
+
except OSError:
|
|
777
|
+
pass
|
|
778
|
+
detail = (
|
|
779
|
+
f"configuration creation failed and the incomplete file may remain: {path}"
|
|
780
|
+
if created
|
|
781
|
+
else f"configuration file could not be written: {path}"
|
|
782
|
+
)
|
|
783
|
+
raise _configuration_error(detail) from None
|
|
784
|
+
return path
|