offline-docker-cli 2.2.2__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- offline_docker/__init__.py +3 -0
- offline_docker/__main__.py +4 -0
- offline_docker/actions.py +173 -0
- offline_docker/build.py +245 -0
- offline_docker/cli.py +486 -0
- offline_docker/config.py +692 -0
- offline_docker/configure.py +282 -0
- offline_docker/context.py +111 -0
- offline_docker/defaults.py +48 -0
- offline_docker/discover.py +139 -0
- offline_docker/docker_cmd.py +136 -0
- offline_docker/download_debs.py +142 -0
- offline_docker/download_images.py +26 -0
- offline_docker/download_npm.py +106 -0
- offline_docker/download_trivy.py +90 -0
- offline_docker/download_wheels.py +363 -0
- offline_docker/download_yarn.py +208 -0
- offline_docker/ensure_base.py +61 -0
- offline_docker/hooks.py +84 -0
- offline_docker/init_cmd.py +54 -0
- offline_docker/load_images.py +34 -0
- offline_docker/oc_discovery.py +206 -0
- offline_docker/resolve.py +306 -0
- offline_docker/scan.py +496 -0
- offline_docker/sources/__init__.py +18 -0
- offline_docker/sources/base.py +18 -0
- offline_docker/sources/filesystem.py +17 -0
- offline_docker/sources/nexus.py +61 -0
- offline_docker/sources/nexus_client.py +67 -0
- offline_docker/sources/proget.py +49 -0
- offline_docker/sources/proget_client.py +48 -0
- offline_docker/sources/upstream.py +36 -0
- offline_docker/status.py +247 -0
- offline_docker/sync_cache.py +125 -0
- offline_docker/trivy_scan.py +86 -0
- offline_docker/tui/__init__.py +19 -0
- offline_docker/tui/app.py +56 -0
- offline_docker/tui/runner.py +87 -0
- offline_docker/tui/screens.py +333 -0
- offline_docker/user_config.py +89 -0
- offline_docker/utils.py +73 -0
- offline_docker/validate.py +53 -0
- offline_docker/wheel_prune.py +141 -0
- offline_docker_cli-2.2.2.dist-info/METADATA +417 -0
- offline_docker_cli-2.2.2.dist-info/RECORD +49 -0
- offline_docker_cli-2.2.2.dist-info/WHEEL +5 -0
- offline_docker_cli-2.2.2.dist-info/entry_points.txt +2 -0
- offline_docker_cli-2.2.2.dist-info/licenses/LICENSE +21 -0
- offline_docker_cli-2.2.2.dist-info/top_level.txt +1 -0
offline_docker/config.py
ADDED
|
@@ -0,0 +1,692 @@
|
|
|
1
|
+
"""Project configuration: load YAML/JSON, merge with auto-discovery."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import json
|
|
6
|
+
import os
|
|
7
|
+
from dataclasses import dataclass, field
|
|
8
|
+
from pathlib import Path
|
|
9
|
+
from typing import Any
|
|
10
|
+
|
|
11
|
+
from offline_docker.defaults import (
|
|
12
|
+
DEFAULT_CRITICAL_WHEELS,
|
|
13
|
+
DEFAULT_DEB_PACKAGES,
|
|
14
|
+
DEFAULT_DOCKER_IMAGES,
|
|
15
|
+
WheelPattern,
|
|
16
|
+
)
|
|
17
|
+
from offline_docker.resolve import CacheSettings, PackageResolver, default_local_only
|
|
18
|
+
from offline_docker.utils import read_torch_version
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
@dataclass
|
|
22
|
+
class DebExtraRepo:
|
|
23
|
+
name: str
|
|
24
|
+
gpg_url: str
|
|
25
|
+
list_url: str
|
|
26
|
+
gpg_dest: str
|
|
27
|
+
list_dest: str
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
@dataclass
|
|
31
|
+
class DockerfilesConfig:
|
|
32
|
+
base: str = "Dockerfile.base"
|
|
33
|
+
base_offline: str = "Dockerfile.base.offline"
|
|
34
|
+
backend: str = "Dockerfile"
|
|
35
|
+
backend_offline: str = "Dockerfile.offline"
|
|
36
|
+
frontend: str = "frontend/Dockerfile"
|
|
37
|
+
frontend_offline: str = "frontend/Dockerfile.offline"
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
@dataclass
|
|
41
|
+
class ImagesConfig:
|
|
42
|
+
base: str = "neuroai-base:cuda12"
|
|
43
|
+
backend: str = "app-backend:latest"
|
|
44
|
+
frontend: str = "app-frontend:latest"
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
@dataclass
|
|
48
|
+
class TrivyConfig:
|
|
49
|
+
scanner_image: str = "aquasec/trivy:latest"
|
|
50
|
+
cache_subdir: str = "trivy"
|
|
51
|
+
db_max_age_days: int = 7
|
|
52
|
+
severity: str = "HIGH,CRITICAL"
|
|
53
|
+
scanners: str = "vuln"
|
|
54
|
+
|
|
55
|
+
|
|
56
|
+
@dataclass
|
|
57
|
+
class CacheHook:
|
|
58
|
+
"""Project hook run after standard offline-cache downloads."""
|
|
59
|
+
|
|
60
|
+
name: str
|
|
61
|
+
win: list[str] = field(default_factory=list)
|
|
62
|
+
sh: list[str] = field(default_factory=list)
|
|
63
|
+
script: str | None = None
|
|
64
|
+
|
|
65
|
+
|
|
66
|
+
@dataclass
|
|
67
|
+
class VendorPathCheck:
|
|
68
|
+
"""Required path under offline-cache (glob allowed), e.g. vendor/VERSIONS."""
|
|
69
|
+
|
|
70
|
+
path: str
|
|
71
|
+
label: str
|
|
72
|
+
|
|
73
|
+
|
|
74
|
+
@dataclass
|
|
75
|
+
class ProjectConfig:
|
|
76
|
+
root: Path
|
|
77
|
+
offline_cache: Path
|
|
78
|
+
log_dir: Path
|
|
79
|
+
compose_files: list[str] = field(default_factory=list)
|
|
80
|
+
compose_build: list[str] = field(default_factory=list)
|
|
81
|
+
compose_up_offline: list[str] = field(default_factory=list)
|
|
82
|
+
compose_up_offline_lab: list[str] = field(default_factory=list)
|
|
83
|
+
compose_offline: str = "docker-compose.offline.yml"
|
|
84
|
+
compose_online: str = "docker-compose.yml"
|
|
85
|
+
compose_env_file: str | None = None
|
|
86
|
+
compose_env_files: list[str] = field(default_factory=list)
|
|
87
|
+
frontend_dir: str = "frontend"
|
|
88
|
+
frontend_lock: str = "package-lock.json"
|
|
89
|
+
python_lock_files: list[str] = field(
|
|
90
|
+
default_factory=lambda: ["pyproject.toml", "uv.lock", "requirements-base.txt"]
|
|
91
|
+
)
|
|
92
|
+
torch_version_file: str = "requirements-base.txt"
|
|
93
|
+
pytorch_index: str = "https://download.pytorch.org/whl/cu128"
|
|
94
|
+
pypi_index: str = "https://pypi.org/simple"
|
|
95
|
+
deb_ubuntu_image: str = "ubuntu:24.04"
|
|
96
|
+
deb_packages: list[str] = field(default_factory=list)
|
|
97
|
+
deb_extra_repos: list[DebExtraRepo] = field(default_factory=list)
|
|
98
|
+
docker_images: list[str] = field(default_factory=list)
|
|
99
|
+
dockerfiles: DockerfilesConfig = field(default_factory=DockerfilesConfig)
|
|
100
|
+
images: ImagesConfig = field(default_factory=ImagesConfig)
|
|
101
|
+
critical_wheels: list[WheelPattern] = field(default_factory=list)
|
|
102
|
+
critical_vendor: list[VendorPathCheck] = field(default_factory=list)
|
|
103
|
+
after_cache_hooks: list[CacheHook] = field(default_factory=list)
|
|
104
|
+
trivy: TrivyConfig = field(default_factory=TrivyConfig)
|
|
105
|
+
build_services: list[str] = field(
|
|
106
|
+
default_factory=lambda: ["base", "backend", "frontend"]
|
|
107
|
+
)
|
|
108
|
+
app_name: str = "app"
|
|
109
|
+
node_image: str = "node:22-bookworm-slim"
|
|
110
|
+
use_uv_export: bool = True
|
|
111
|
+
explicit_use_uv_export: bool = field(default=False, repr=False)
|
|
112
|
+
explicit_python_project_dir: bool = field(default=False, repr=False)
|
|
113
|
+
python_project_dir: str | None = None
|
|
114
|
+
uv_export_prune: list[str] = field(
|
|
115
|
+
default_factory=lambda: ["torch", "torchaudio"]
|
|
116
|
+
)
|
|
117
|
+
extra_wheel_specs: list[str] = field(default_factory=list)
|
|
118
|
+
config_source: str | None = None
|
|
119
|
+
# Raw project "cache" section if present (wins over user-config).
|
|
120
|
+
project_cache_data: dict[str, Any] | None = field(default=None, repr=False)
|
|
121
|
+
cache_settings: CacheSettings = field(default_factory=default_local_only)
|
|
122
|
+
cache_config_origin: str = "default"
|
|
123
|
+
|
|
124
|
+
def path(self, rel: str) -> Path:
|
|
125
|
+
return self.root / rel
|
|
126
|
+
|
|
127
|
+
def package_resolver(self) -> PackageResolver:
|
|
128
|
+
return PackageResolver(self.cache_settings)
|
|
129
|
+
|
|
130
|
+
def python_lock_paths(self) -> list[Path]:
|
|
131
|
+
return [self.path(f) for f in self.python_lock_files if self.path(f).is_file()]
|
|
132
|
+
|
|
133
|
+
def frontend_path(self) -> Path | None:
|
|
134
|
+
d = self.path(self.frontend_dir)
|
|
135
|
+
return d if d.is_dir() else None
|
|
136
|
+
|
|
137
|
+
def frontend_lock_path(self) -> Path | None:
|
|
138
|
+
fe = self.frontend_path()
|
|
139
|
+
if fe is None:
|
|
140
|
+
return None
|
|
141
|
+
lock = fe / self.frontend_lock
|
|
142
|
+
return lock if lock.is_file() else None
|
|
143
|
+
|
|
144
|
+
def compose_build_file_args(self) -> list[str]:
|
|
145
|
+
"""`-f` args: online manifest + offline build override."""
|
|
146
|
+
files = self.compose_build or [self.compose_online, self.compose_offline]
|
|
147
|
+
args: list[str] = []
|
|
148
|
+
for rel in files:
|
|
149
|
+
args.extend(["-f", rel])
|
|
150
|
+
return args
|
|
151
|
+
|
|
152
|
+
def compose_env_file_args(self) -> list[str]:
|
|
153
|
+
"""`--env-file` args for compose ${VAR} interpolation.
|
|
154
|
+
|
|
155
|
+
Uses ``compose_env_files`` (list) when set; otherwise legacy
|
|
156
|
+
``compose_env_file``. Later files override earlier ones (Compose
|
|
157
|
+
behaviour), so put secrets last.
|
|
158
|
+
"""
|
|
159
|
+
rels = list(self.compose_env_files)
|
|
160
|
+
if not rels and self.compose_env_file:
|
|
161
|
+
rels = [self.compose_env_file]
|
|
162
|
+
args: list[str] = []
|
|
163
|
+
for rel in rels:
|
|
164
|
+
path = self.path(rel)
|
|
165
|
+
if not path.is_file():
|
|
166
|
+
continue
|
|
167
|
+
args.extend(["--env-file", rel])
|
|
168
|
+
return args
|
|
169
|
+
|
|
170
|
+
def compose_up_file_args(self, *, lab: bool = False) -> list[str]:
|
|
171
|
+
"""`-f` args для offline runtime (up-offline)."""
|
|
172
|
+
if lab:
|
|
173
|
+
files = self.compose_up_offline_lab or [
|
|
174
|
+
*(
|
|
175
|
+
self.compose_up_offline
|
|
176
|
+
or self.compose_build
|
|
177
|
+
or [self.compose_online, self.compose_offline]
|
|
178
|
+
),
|
|
179
|
+
"docker-compose.lab.yml",
|
|
180
|
+
]
|
|
181
|
+
else:
|
|
182
|
+
files = self.compose_up_offline or self.compose_build or [
|
|
183
|
+
self.compose_online,
|
|
184
|
+
self.compose_offline,
|
|
185
|
+
]
|
|
186
|
+
args: list[str] = []
|
|
187
|
+
for rel in files:
|
|
188
|
+
args.extend(["-f", rel])
|
|
189
|
+
return args
|
|
190
|
+
|
|
191
|
+
def torch_version(self) -> str | None:
|
|
192
|
+
tv_file = self.path(self.torch_version_file)
|
|
193
|
+
if not tv_file.is_file():
|
|
194
|
+
return None
|
|
195
|
+
try:
|
|
196
|
+
return read_torch_version(tv_file)
|
|
197
|
+
except RuntimeError:
|
|
198
|
+
return None
|
|
199
|
+
|
|
200
|
+
@staticmethod
|
|
201
|
+
def _normalize_project_dir(rel: str) -> str:
|
|
202
|
+
d = rel.strip().strip("/")
|
|
203
|
+
return "." if not d else d
|
|
204
|
+
|
|
205
|
+
def infer_python_project_dir(self) -> str:
|
|
206
|
+
pyproject_dirs: list[str] = []
|
|
207
|
+
uv_lock_dirs: list[str] = []
|
|
208
|
+
for rel in self.python_lock_files:
|
|
209
|
+
p = Path(rel)
|
|
210
|
+
if not self.path(rel).is_file():
|
|
211
|
+
continue
|
|
212
|
+
parent = self._normalize_project_dir(
|
|
213
|
+
"." if p.parent == Path(".") else p.parent.as_posix()
|
|
214
|
+
)
|
|
215
|
+
if p.name == "pyproject.toml":
|
|
216
|
+
pyproject_dirs.append(parent)
|
|
217
|
+
elif p.name == "uv.lock":
|
|
218
|
+
uv_lock_dirs.append(parent)
|
|
219
|
+
|
|
220
|
+
if pyproject_dirs and uv_lock_dirs:
|
|
221
|
+
common = set(pyproject_dirs) & set(uv_lock_dirs)
|
|
222
|
+
if common:
|
|
223
|
+
return min(common, key=len)
|
|
224
|
+
return pyproject_dirs[0]
|
|
225
|
+
if pyproject_dirs:
|
|
226
|
+
return pyproject_dirs[0]
|
|
227
|
+
|
|
228
|
+
for dockerfile in (self.dockerfiles.backend_offline, self.dockerfiles.backend):
|
|
229
|
+
parent = Path(dockerfile).parent
|
|
230
|
+
if parent != Path(".") and self.path(str(parent / "pyproject.toml")).is_file():
|
|
231
|
+
return self._normalize_project_dir(parent.as_posix())
|
|
232
|
+
|
|
233
|
+
if self.path("pyproject.toml").is_file():
|
|
234
|
+
return "."
|
|
235
|
+
return "."
|
|
236
|
+
|
|
237
|
+
def effective_python_project_dir(self) -> str:
|
|
238
|
+
if self.python_project_dir is not None:
|
|
239
|
+
return self._normalize_project_dir(self.python_project_dir)
|
|
240
|
+
return self.infer_python_project_dir()
|
|
241
|
+
|
|
242
|
+
def python_requires_min(self) -> str | None:
|
|
243
|
+
"""`requires-python` из pyproject.toml как 'X.Y' (нижняя граница)."""
|
|
244
|
+
import re
|
|
245
|
+
|
|
246
|
+
proj = self.effective_python_project_dir()
|
|
247
|
+
base = self.root if proj == "." else self.root / proj
|
|
248
|
+
pyproject = base / "pyproject.toml"
|
|
249
|
+
if not pyproject.is_file():
|
|
250
|
+
return None
|
|
251
|
+
text = pyproject.read_text(encoding="utf-8", errors="replace")
|
|
252
|
+
requires: str | None = None
|
|
253
|
+
try:
|
|
254
|
+
import tomllib
|
|
255
|
+
|
|
256
|
+
requires = tomllib.loads(text).get("project", {}).get("requires-python")
|
|
257
|
+
except Exception:
|
|
258
|
+
match = re.search(r'requires-python\s*=\s*"([^"]+)"', text)
|
|
259
|
+
requires = match.group(1) if match else None
|
|
260
|
+
if not requires:
|
|
261
|
+
return None
|
|
262
|
+
match = re.search(r"(\d+)\.(\d+)", requires)
|
|
263
|
+
return f"{match.group(1)}.{match.group(2)}" if match else None
|
|
264
|
+
|
|
265
|
+
def infer_use_uv_export(self) -> bool:
|
|
266
|
+
proj = self.effective_python_project_dir()
|
|
267
|
+
base = self.root if proj == "." else self.root / proj
|
|
268
|
+
return (base / "uv.lock").is_file() and (base / "pyproject.toml").is_file()
|
|
269
|
+
|
|
270
|
+
|
|
271
|
+
def _load_yaml_or_json(path: Path) -> dict[str, Any]:
|
|
272
|
+
text = path.read_text(encoding="utf-8")
|
|
273
|
+
if path.suffix.lower() == ".json":
|
|
274
|
+
data = json.loads(text)
|
|
275
|
+
elif path.suffix.lower() in (".yaml", ".yml"):
|
|
276
|
+
try:
|
|
277
|
+
import yaml # type: ignore[import-untyped]
|
|
278
|
+
except ImportError as exc:
|
|
279
|
+
json_alt = path.with_suffix(".json")
|
|
280
|
+
if json_alt.is_file():
|
|
281
|
+
return _load_yaml_or_json(json_alt)
|
|
282
|
+
raise SystemExit(
|
|
283
|
+
f"ERROR: для {path.name} нужен PyYAML или JSON-версия ({json_alt.name})"
|
|
284
|
+
) from exc
|
|
285
|
+
data = yaml.safe_load(text)
|
|
286
|
+
else:
|
|
287
|
+
raise SystemExit(f"ERROR: неподдерживаемый формат конфига: {path}")
|
|
288
|
+
if not isinstance(data, dict):
|
|
289
|
+
raise SystemExit(f"ERROR: конфиг {path} должен быть объектом JSON/YAML")
|
|
290
|
+
return data
|
|
291
|
+
|
|
292
|
+
|
|
293
|
+
def _parse_deb_extra_repos(raw: list[Any]) -> list[DebExtraRepo]:
|
|
294
|
+
repos: list[DebExtraRepo] = []
|
|
295
|
+
for item in raw:
|
|
296
|
+
if not isinstance(item, dict):
|
|
297
|
+
continue
|
|
298
|
+
repos.append(
|
|
299
|
+
DebExtraRepo(
|
|
300
|
+
name=str(item.get("name", "extra")),
|
|
301
|
+
gpg_url=str(item["gpg_url"]),
|
|
302
|
+
list_url=str(item["list_url"]),
|
|
303
|
+
gpg_dest=str(item.get("gpg_dest", f"apt-meta/{item.get('name', 'extra')}.gpg")),
|
|
304
|
+
list_dest=str(
|
|
305
|
+
item.get("list_dest", f"apt-meta/{item.get('name', 'extra')}.list")
|
|
306
|
+
),
|
|
307
|
+
)
|
|
308
|
+
)
|
|
309
|
+
return repos
|
|
310
|
+
|
|
311
|
+
|
|
312
|
+
def _parse_critical_wheels(raw: list[Any]) -> list[WheelPattern]:
|
|
313
|
+
wheels: list[WheelPattern] = []
|
|
314
|
+
for item in raw:
|
|
315
|
+
if isinstance(item, dict):
|
|
316
|
+
wheels.append(
|
|
317
|
+
WheelPattern(str(item["pattern"]), str(item.get("label", item["pattern"])))
|
|
318
|
+
)
|
|
319
|
+
elif isinstance(item, (list, tuple)) and len(item) >= 2:
|
|
320
|
+
wheels.append(WheelPattern(str(item[0]), str(item[1])))
|
|
321
|
+
return wheels
|
|
322
|
+
|
|
323
|
+
|
|
324
|
+
def _parse_critical_vendor(raw: list[Any]) -> list[VendorPathCheck]:
|
|
325
|
+
items: list[VendorPathCheck] = []
|
|
326
|
+
for item in raw:
|
|
327
|
+
if isinstance(item, dict):
|
|
328
|
+
path = str(item.get("path") or item.get("pattern") or "")
|
|
329
|
+
if not path:
|
|
330
|
+
continue
|
|
331
|
+
items.append(VendorPathCheck(path, str(item.get("label", path))))
|
|
332
|
+
elif isinstance(item, str):
|
|
333
|
+
items.append(VendorPathCheck(item, item))
|
|
334
|
+
elif isinstance(item, (list, tuple)) and len(item) >= 1:
|
|
335
|
+
path = str(item[0])
|
|
336
|
+
label = str(item[1]) if len(item) >= 2 else path
|
|
337
|
+
items.append(VendorPathCheck(path, label))
|
|
338
|
+
return items
|
|
339
|
+
|
|
340
|
+
|
|
341
|
+
def _parse_after_cache_hooks(raw: list[Any]) -> list[CacheHook]:
|
|
342
|
+
hooks: list[CacheHook] = []
|
|
343
|
+
for item in raw:
|
|
344
|
+
if not isinstance(item, dict):
|
|
345
|
+
continue
|
|
346
|
+
name = str(item.get("name") or item.get("script") or "hook")
|
|
347
|
+
win = [str(x) for x in item.get("win", [])] if isinstance(item.get("win"), list) else []
|
|
348
|
+
sh = [str(x) for x in item.get("sh", [])] if isinstance(item.get("sh"), list) else []
|
|
349
|
+
script = str(item["script"]) if item.get("script") else None
|
|
350
|
+
# Convenience: "run" as platform-neutral argv
|
|
351
|
+
run = item.get("run")
|
|
352
|
+
if isinstance(run, list) and run:
|
|
353
|
+
argv = [str(x) for x in run]
|
|
354
|
+
if not win:
|
|
355
|
+
win = list(argv)
|
|
356
|
+
if not sh:
|
|
357
|
+
sh = list(argv)
|
|
358
|
+
hooks.append(CacheHook(name=name, win=win, sh=sh, script=script))
|
|
359
|
+
return hooks
|
|
360
|
+
|
|
361
|
+
|
|
362
|
+
def dockerfile_uses_vendor_mount(cfg: ProjectConfig) -> bool:
|
|
363
|
+
"""True if any offline Dockerfile bind-mounts source=vendor from oc."""
|
|
364
|
+
candidates = (
|
|
365
|
+
cfg.dockerfiles.backend_offline,
|
|
366
|
+
cfg.dockerfiles.base_offline,
|
|
367
|
+
cfg.dockerfiles.frontend_offline,
|
|
368
|
+
)
|
|
369
|
+
for rel in candidates:
|
|
370
|
+
path = cfg.path(rel)
|
|
371
|
+
if not path.is_file():
|
|
372
|
+
continue
|
|
373
|
+
text = path.read_text(encoding="utf-8", errors="ignore")
|
|
374
|
+
if "source=vendor" in text:
|
|
375
|
+
return True
|
|
376
|
+
return False
|
|
377
|
+
|
|
378
|
+
|
|
379
|
+
def _merge_config_dict(base: ProjectConfig, data: dict[str, Any]) -> ProjectConfig:
|
|
380
|
+
if "offline_cache" in data:
|
|
381
|
+
base.offline_cache = base.path(str(data["offline_cache"]))
|
|
382
|
+
if "log_dir" in data:
|
|
383
|
+
base.log_dir = base.path(str(data["log_dir"]))
|
|
384
|
+
if "compose_files" in data:
|
|
385
|
+
base.compose_files = [str(x) for x in data["compose_files"]]
|
|
386
|
+
if "compose_offline" in data:
|
|
387
|
+
base.compose_offline = str(data["compose_offline"])
|
|
388
|
+
if "compose_online" in data:
|
|
389
|
+
base.compose_online = str(data["compose_online"])
|
|
390
|
+
if "compose_build" in data:
|
|
391
|
+
base.compose_build = [str(x) for x in data["compose_build"]]
|
|
392
|
+
if "compose_up_offline" in data:
|
|
393
|
+
base.compose_up_offline = [str(x) for x in data["compose_up_offline"]]
|
|
394
|
+
if "compose_up_offline_lab" in data:
|
|
395
|
+
base.compose_up_offline_lab = [str(x) for x in data["compose_up_offline_lab"]]
|
|
396
|
+
if "compose_env_file" in data:
|
|
397
|
+
base.compose_env_file = str(data["compose_env_file"])
|
|
398
|
+
if "compose_env_files" in data:
|
|
399
|
+
base.compose_env_files = [str(x) for x in data["compose_env_files"]]
|
|
400
|
+
if "app_name" in data:
|
|
401
|
+
base.app_name = str(data["app_name"])
|
|
402
|
+
if "node_image" in data:
|
|
403
|
+
base.node_image = str(data["node_image"])
|
|
404
|
+
if "use_uv_export" in data:
|
|
405
|
+
base.use_uv_export = bool(data["use_uv_export"])
|
|
406
|
+
base.explicit_use_uv_export = True
|
|
407
|
+
|
|
408
|
+
py = data.get("python", {})
|
|
409
|
+
if isinstance(py, dict):
|
|
410
|
+
if "project_dir" in py:
|
|
411
|
+
base.python_project_dir = str(py["project_dir"])
|
|
412
|
+
base.explicit_python_project_dir = True
|
|
413
|
+
if "lock_files" in py:
|
|
414
|
+
base.python_lock_files = [str(x) for x in py["lock_files"]]
|
|
415
|
+
if "torch_version_file" in py:
|
|
416
|
+
base.torch_version_file = str(py["torch_version_file"])
|
|
417
|
+
if "pytorch_index" in py:
|
|
418
|
+
base.pytorch_index = str(py["pytorch_index"])
|
|
419
|
+
if "pypi_index" in py:
|
|
420
|
+
base.pypi_index = str(py["pypi_index"])
|
|
421
|
+
if "uv_export_prune" in py:
|
|
422
|
+
base.uv_export_prune = [str(x) for x in py["uv_export_prune"]]
|
|
423
|
+
if "extra_wheel_specs" in py:
|
|
424
|
+
base.extra_wheel_specs = [str(x) for x in py["extra_wheel_specs"]]
|
|
425
|
+
|
|
426
|
+
fe = data.get("frontend", {})
|
|
427
|
+
if isinstance(fe, dict):
|
|
428
|
+
if "dir" in fe:
|
|
429
|
+
base.frontend_dir = str(fe["dir"])
|
|
430
|
+
if "lock_file" in fe:
|
|
431
|
+
base.frontend_lock = str(fe["lock_file"])
|
|
432
|
+
|
|
433
|
+
debs = data.get("debs", {})
|
|
434
|
+
if isinstance(debs, dict):
|
|
435
|
+
if "ubuntu_image" in debs:
|
|
436
|
+
base.deb_ubuntu_image = str(debs["ubuntu_image"])
|
|
437
|
+
if "packages" in debs:
|
|
438
|
+
base.deb_packages = [str(x) for x in debs["packages"]]
|
|
439
|
+
if "extra_repos" in debs:
|
|
440
|
+
base.deb_extra_repos = _parse_deb_extra_repos(debs["extra_repos"])
|
|
441
|
+
|
|
442
|
+
if "docker_images" in data:
|
|
443
|
+
base.docker_images = [str(x) for x in data["docker_images"]]
|
|
444
|
+
|
|
445
|
+
df = data.get("dockerfiles", {})
|
|
446
|
+
if isinstance(df, dict):
|
|
447
|
+
for key in (
|
|
448
|
+
"base",
|
|
449
|
+
"base_offline",
|
|
450
|
+
"backend",
|
|
451
|
+
"backend_offline",
|
|
452
|
+
"frontend",
|
|
453
|
+
"frontend_offline",
|
|
454
|
+
):
|
|
455
|
+
if key in df:
|
|
456
|
+
setattr(base.dockerfiles, key, str(df[key]))
|
|
457
|
+
|
|
458
|
+
imgs = data.get("images", {})
|
|
459
|
+
if isinstance(imgs, dict):
|
|
460
|
+
if "base" in imgs:
|
|
461
|
+
base.images.base = str(imgs["base"])
|
|
462
|
+
elif "base_online" in imgs:
|
|
463
|
+
base.images.base = str(imgs["base_online"])
|
|
464
|
+
elif "base_offline" in imgs:
|
|
465
|
+
base.images.base = str(imgs["base_offline"]).replace(":offline", ":cuda12")
|
|
466
|
+
|
|
467
|
+
if "backend" in imgs:
|
|
468
|
+
base.images.backend = str(imgs["backend"])
|
|
469
|
+
elif "backend_offline" in imgs:
|
|
470
|
+
tag = str(imgs["backend_offline"])
|
|
471
|
+
base.images.backend = tag.replace(":offline", ":latest")
|
|
472
|
+
|
|
473
|
+
if "frontend" in imgs:
|
|
474
|
+
base.images.frontend = str(imgs["frontend"])
|
|
475
|
+
elif "frontend_offline" in imgs:
|
|
476
|
+
tag = str(imgs["frontend_offline"])
|
|
477
|
+
base.images.frontend = tag.replace(":offline", ":latest")
|
|
478
|
+
|
|
479
|
+
val = data.get("validate", {})
|
|
480
|
+
if isinstance(val, dict):
|
|
481
|
+
if "critical_wheels" in val:
|
|
482
|
+
base.critical_wheels = _parse_critical_wheels(val["critical_wheels"])
|
|
483
|
+
if "critical_vendor" in val:
|
|
484
|
+
base.critical_vendor = _parse_critical_vendor(val["critical_vendor"])
|
|
485
|
+
|
|
486
|
+
hooks = data.get("hooks", {})
|
|
487
|
+
if isinstance(hooks, dict) and "after_cache" in hooks:
|
|
488
|
+
raw_hooks = hooks["after_cache"]
|
|
489
|
+
if isinstance(raw_hooks, list):
|
|
490
|
+
base.after_cache_hooks = _parse_after_cache_hooks(raw_hooks)
|
|
491
|
+
|
|
492
|
+
trivy = data.get("trivy", {})
|
|
493
|
+
if isinstance(trivy, dict):
|
|
494
|
+
for key in ("scanner_image", "cache_subdir", "severity", "scanners"):
|
|
495
|
+
if key in trivy:
|
|
496
|
+
setattr(base.trivy, key, str(trivy[key]))
|
|
497
|
+
if "db_max_age_days" in trivy:
|
|
498
|
+
base.trivy.db_max_age_days = int(trivy["db_max_age_days"])
|
|
499
|
+
|
|
500
|
+
if "build_services" in data:
|
|
501
|
+
base.build_services = [str(x) for x in data["build_services"]]
|
|
502
|
+
|
|
503
|
+
if "cache" in data and isinstance(data["cache"], dict):
|
|
504
|
+
base.project_cache_data = dict(data["cache"])
|
|
505
|
+
|
|
506
|
+
return base
|
|
507
|
+
|
|
508
|
+
|
|
509
|
+
def project_config_to_dict(cfg: ProjectConfig) -> dict[str, Any]:
|
|
510
|
+
"""Serialize ProjectConfig to .offline-docker.json shape (example schema)."""
|
|
511
|
+
offline_cache = cfg.offline_cache
|
|
512
|
+
if offline_cache.is_absolute():
|
|
513
|
+
try:
|
|
514
|
+
offline_cache_rel = offline_cache.relative_to(cfg.root).as_posix()
|
|
515
|
+
except ValueError:
|
|
516
|
+
offline_cache_rel = str(offline_cache)
|
|
517
|
+
else:
|
|
518
|
+
offline_cache_rel = Path(offline_cache).as_posix()
|
|
519
|
+
|
|
520
|
+
log_dir = cfg.log_dir
|
|
521
|
+
if log_dir.is_absolute():
|
|
522
|
+
try:
|
|
523
|
+
log_dir_rel = log_dir.relative_to(cfg.root).as_posix()
|
|
524
|
+
except ValueError:
|
|
525
|
+
log_dir_rel = str(log_dir)
|
|
526
|
+
else:
|
|
527
|
+
log_dir_rel = Path(log_dir).as_posix()
|
|
528
|
+
|
|
529
|
+
data: dict[str, Any] = {
|
|
530
|
+
"offline_cache": offline_cache_rel,
|
|
531
|
+
"log_dir": log_dir_rel,
|
|
532
|
+
"app_name": cfg.app_name,
|
|
533
|
+
"compose_online": cfg.compose_online,
|
|
534
|
+
"compose_offline": cfg.compose_offline,
|
|
535
|
+
"compose_build": cfg.compose_build
|
|
536
|
+
or [cfg.compose_online, cfg.compose_offline],
|
|
537
|
+
"compose_up_offline": cfg.compose_up_offline
|
|
538
|
+
or [cfg.compose_online, cfg.compose_offline],
|
|
539
|
+
"python": {
|
|
540
|
+
"lock_files": list(cfg.python_lock_files),
|
|
541
|
+
"torch_version_file": cfg.torch_version_file,
|
|
542
|
+
"pytorch_index": cfg.pytorch_index,
|
|
543
|
+
},
|
|
544
|
+
"frontend": {
|
|
545
|
+
"dir": cfg.frontend_dir,
|
|
546
|
+
"lock_file": cfg.frontend_lock,
|
|
547
|
+
},
|
|
548
|
+
"dockerfiles": {
|
|
549
|
+
"base": cfg.dockerfiles.base,
|
|
550
|
+
"base_offline": cfg.dockerfiles.base_offline,
|
|
551
|
+
"backend": cfg.dockerfiles.backend,
|
|
552
|
+
"backend_offline": cfg.dockerfiles.backend_offline,
|
|
553
|
+
"frontend": cfg.dockerfiles.frontend,
|
|
554
|
+
"frontend_offline": cfg.dockerfiles.frontend_offline,
|
|
555
|
+
},
|
|
556
|
+
"images": {
|
|
557
|
+
"base": cfg.images.base,
|
|
558
|
+
"backend": cfg.images.backend,
|
|
559
|
+
"frontend": cfg.images.frontend,
|
|
560
|
+
},
|
|
561
|
+
"debs": {
|
|
562
|
+
"ubuntu_image": cfg.deb_ubuntu_image,
|
|
563
|
+
"packages": list(cfg.deb_packages),
|
|
564
|
+
"extra_repos": [
|
|
565
|
+
{
|
|
566
|
+
"name": r.name,
|
|
567
|
+
"gpg_url": r.gpg_url,
|
|
568
|
+
"list_url": r.list_url,
|
|
569
|
+
"gpg_dest": r.gpg_dest,
|
|
570
|
+
"list_dest": r.list_dest,
|
|
571
|
+
}
|
|
572
|
+
for r in cfg.deb_extra_repos
|
|
573
|
+
],
|
|
574
|
+
},
|
|
575
|
+
"docker_images": list(cfg.docker_images),
|
|
576
|
+
"validate": {
|
|
577
|
+
"critical_wheels": [
|
|
578
|
+
{"pattern": w.pattern, "label": w.label} for w in cfg.critical_wheels
|
|
579
|
+
],
|
|
580
|
+
"critical_vendor": [
|
|
581
|
+
{"path": v.path, "label": v.label} for v in cfg.critical_vendor
|
|
582
|
+
],
|
|
583
|
+
},
|
|
584
|
+
"trivy": {
|
|
585
|
+
"scanner_image": cfg.trivy.scanner_image,
|
|
586
|
+
"db_max_age_days": cfg.trivy.db_max_age_days,
|
|
587
|
+
"severity": cfg.trivy.severity,
|
|
588
|
+
"scanners": cfg.trivy.scanners,
|
|
589
|
+
},
|
|
590
|
+
"build_services": list(cfg.build_services),
|
|
591
|
+
}
|
|
592
|
+
if cfg.after_cache_hooks:
|
|
593
|
+
data["hooks"] = {
|
|
594
|
+
"after_cache": [
|
|
595
|
+
{
|
|
596
|
+
"name": h.name,
|
|
597
|
+
**({"script": h.script} if h.script else {}),
|
|
598
|
+
**({"win": list(h.win)} if h.win else {}),
|
|
599
|
+
**({"sh": list(h.sh)} if h.sh else {}),
|
|
600
|
+
}
|
|
601
|
+
for h in cfg.after_cache_hooks
|
|
602
|
+
]
|
|
603
|
+
}
|
|
604
|
+
if cfg.compose_up_offline_lab:
|
|
605
|
+
data["compose_up_offline_lab"] = list(cfg.compose_up_offline_lab)
|
|
606
|
+
if cfg.python_project_dir and cfg.python_project_dir != ".":
|
|
607
|
+
data["python"]["project_dir"] = cfg.python_project_dir
|
|
608
|
+
if cfg.compose_env_files:
|
|
609
|
+
data["compose_env_files"] = list(cfg.compose_env_files)
|
|
610
|
+
elif cfg.compose_env_file:
|
|
611
|
+
data["compose_env_file"] = cfg.compose_env_file
|
|
612
|
+
return data
|
|
613
|
+
|
|
614
|
+
|
|
615
|
+
def find_project_config_file(root: Path) -> Path | None:
|
|
616
|
+
for name in (".offline-docker.yaml", ".offline-docker.yml", ".offline-docker.json"):
|
|
617
|
+
path = root / name
|
|
618
|
+
if path.is_file():
|
|
619
|
+
return path
|
|
620
|
+
return None
|
|
621
|
+
|
|
622
|
+
|
|
623
|
+
def resolve_offline_cache(root: Path, app_name: str, configured: Path | None) -> Path:
|
|
624
|
+
for key in ("OFFLINE_CACHE", f"{app_name.upper()}_OFFLINE_CACHE"):
|
|
625
|
+
val = os.environ.get(key)
|
|
626
|
+
if val:
|
|
627
|
+
p = Path(val)
|
|
628
|
+
return p if p.is_absolute() else root / p
|
|
629
|
+
if configured is not None:
|
|
630
|
+
return configured if configured.is_absolute() else root / configured
|
|
631
|
+
return root / "offline-cache"
|
|
632
|
+
|
|
633
|
+
|
|
634
|
+
def load_config(
|
|
635
|
+
root: Path,
|
|
636
|
+
*,
|
|
637
|
+
config_file: Path | None = None,
|
|
638
|
+
) -> ProjectConfig:
|
|
639
|
+
from offline_docker.discover import discover_project
|
|
640
|
+
|
|
641
|
+
root = root.resolve()
|
|
642
|
+
cfg: ProjectConfig | None = None
|
|
643
|
+
source: str | None = None
|
|
644
|
+
|
|
645
|
+
if config_file and config_file.is_file():
|
|
646
|
+
cfg = _merge_config_dict(discover_project(root), _load_yaml_or_json(config_file))
|
|
647
|
+
source = str(config_file.resolve())
|
|
648
|
+
else:
|
|
649
|
+
project_config = find_project_config_file(root)
|
|
650
|
+
if project_config is not None:
|
|
651
|
+
cfg = _merge_config_dict(
|
|
652
|
+
discover_project(root), _load_yaml_or_json(project_config)
|
|
653
|
+
)
|
|
654
|
+
source = str(project_config.resolve())
|
|
655
|
+
|
|
656
|
+
if cfg is None:
|
|
657
|
+
cfg = discover_project(root)
|
|
658
|
+
source = "(auto-discovery)"
|
|
659
|
+
|
|
660
|
+
cfg.config_source = source
|
|
661
|
+
cfg.root = root
|
|
662
|
+
cfg.offline_cache = resolve_offline_cache(root, cfg.app_name, cfg.offline_cache)
|
|
663
|
+
if not cfg.log_dir.is_absolute():
|
|
664
|
+
cfg.log_dir = root / cfg.log_dir
|
|
665
|
+
|
|
666
|
+
if not cfg.deb_packages:
|
|
667
|
+
cfg.deb_packages = list(DEFAULT_DEB_PACKAGES)
|
|
668
|
+
if not cfg.docker_images:
|
|
669
|
+
cfg.docker_images = list(DEFAULT_DOCKER_IMAGES)
|
|
670
|
+
if not cfg.critical_wheels:
|
|
671
|
+
cfg.critical_wheels = list(DEFAULT_CRITICAL_WHEELS)
|
|
672
|
+
|
|
673
|
+
if not cfg.explicit_python_project_dir:
|
|
674
|
+
cfg.python_project_dir = cfg.infer_python_project_dir()
|
|
675
|
+
elif cfg.python_project_dir is None:
|
|
676
|
+
cfg.python_project_dir = cfg.infer_python_project_dir()
|
|
677
|
+
if not cfg.explicit_use_uv_export:
|
|
678
|
+
cfg.use_uv_export = cfg.infer_use_uv_export()
|
|
679
|
+
|
|
680
|
+
from offline_docker.resolve import merge_cache_settings
|
|
681
|
+
from offline_docker.user_config import load_user_sources
|
|
682
|
+
|
|
683
|
+
user_data = load_user_sources()
|
|
684
|
+
cfg.cache_settings = merge_cache_settings(
|
|
685
|
+
user_data=user_data,
|
|
686
|
+
project_data=cfg.project_cache_data,
|
|
687
|
+
pypi_index=cfg.pypi_index,
|
|
688
|
+
pytorch_index=cfg.pytorch_index,
|
|
689
|
+
)
|
|
690
|
+
cfg.cache_config_origin = cfg.cache_settings.cache_config_origin
|
|
691
|
+
|
|
692
|
+
return cfg
|