offline-docker-cli 2.2.2__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- offline_docker/__init__.py +3 -0
- offline_docker/__main__.py +4 -0
- offline_docker/actions.py +173 -0
- offline_docker/build.py +245 -0
- offline_docker/cli.py +486 -0
- offline_docker/config.py +692 -0
- offline_docker/configure.py +282 -0
- offline_docker/context.py +111 -0
- offline_docker/defaults.py +48 -0
- offline_docker/discover.py +139 -0
- offline_docker/docker_cmd.py +136 -0
- offline_docker/download_debs.py +142 -0
- offline_docker/download_images.py +26 -0
- offline_docker/download_npm.py +106 -0
- offline_docker/download_trivy.py +90 -0
- offline_docker/download_wheels.py +363 -0
- offline_docker/download_yarn.py +208 -0
- offline_docker/ensure_base.py +61 -0
- offline_docker/hooks.py +84 -0
- offline_docker/init_cmd.py +54 -0
- offline_docker/load_images.py +34 -0
- offline_docker/oc_discovery.py +206 -0
- offline_docker/resolve.py +306 -0
- offline_docker/scan.py +496 -0
- offline_docker/sources/__init__.py +18 -0
- offline_docker/sources/base.py +18 -0
- offline_docker/sources/filesystem.py +17 -0
- offline_docker/sources/nexus.py +61 -0
- offline_docker/sources/nexus_client.py +67 -0
- offline_docker/sources/proget.py +49 -0
- offline_docker/sources/proget_client.py +48 -0
- offline_docker/sources/upstream.py +36 -0
- offline_docker/status.py +247 -0
- offline_docker/sync_cache.py +125 -0
- offline_docker/trivy_scan.py +86 -0
- offline_docker/tui/__init__.py +19 -0
- offline_docker/tui/app.py +56 -0
- offline_docker/tui/runner.py +87 -0
- offline_docker/tui/screens.py +333 -0
- offline_docker/user_config.py +89 -0
- offline_docker/utils.py +73 -0
- offline_docker/validate.py +53 -0
- offline_docker/wheel_prune.py +141 -0
- offline_docker_cli-2.2.2.dist-info/METADATA +417 -0
- offline_docker_cli-2.2.2.dist-info/RECORD +49 -0
- offline_docker_cli-2.2.2.dist-info/WHEEL +5 -0
- offline_docker_cli-2.2.2.dist-info/entry_points.txt +2 -0
- offline_docker_cli-2.2.2.dist-info/licenses/LICENSE +21 -0
- offline_docker_cli-2.2.2.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,363 @@
|
|
|
1
|
+
"""Download Linux wheels into offline-cache/wheels."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import os
|
|
6
|
+
from pathlib import Path
|
|
7
|
+
|
|
8
|
+
from offline_docker import docker_cmd
|
|
9
|
+
from offline_docker.context import ProjectContext
|
|
10
|
+
from offline_docker.utils import docker_mount_path
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
WHEELS_GUEST_UV = r"""
|
|
14
|
+
set -euo pipefail
|
|
15
|
+
export DEBIAN_FRONTEND=noninteractive
|
|
16
|
+
|
|
17
|
+
apt-get update
|
|
18
|
+
apt-get install -y --no-install-recommends \
|
|
19
|
+
ca-certificates curl build-essential pkg-config cython3 \
|
|
20
|
+
ffmpeg libavformat-dev libavcodec-dev libavdevice-dev \
|
|
21
|
+
libavutil-dev libavfilter-dev libswscale-dev libswresample-dev
|
|
22
|
+
rm -rf /var/lib/apt/lists/*
|
|
23
|
+
|
|
24
|
+
# Python берём из базового образа (python:3.12-slim и т.п.): его версия задаёт
|
|
25
|
+
# ABI скачанных wheel-ов. Пакеты python3.12* из apt ставить нельзя — в Debian 13
|
|
26
|
+
# (trixie) их нет, там python3 = 3.13. apt-пакеты python3* нужны только когда
|
|
27
|
+
# интерпретатора/venv в образе нет (ubuntu:24.04, debian:bookworm-slim).
|
|
28
|
+
if ! command -v python3 >/dev/null 2>&1 || ! python3 -c 'import venv, ensurepip' >/dev/null 2>&1; then
|
|
29
|
+
echo ">>> python3/venv в образе нет — ставим из apt"
|
|
30
|
+
apt-get update
|
|
31
|
+
apt-get install -y --no-install-recommends python3 python3-venv python3-dev python3-pip
|
|
32
|
+
rm -rf /var/lib/apt/lists/*
|
|
33
|
+
fi
|
|
34
|
+
PYTHON=python3
|
|
35
|
+
echo ">>> python: $($PYTHON -V) ($(command -v $PYTHON))"
|
|
36
|
+
if [[ -n "${PYTHON_MIN:-}" ]] && ! $PYTHON -c \
|
|
37
|
+
"import sys; sys.exit(0 if sys.version_info[:2] >= tuple(int(x) for x in '${PYTHON_MIN}'.split('.')) else 1)"; then
|
|
38
|
+
echo "ERROR: python в образе $($PYTHON -V) старее requires-python проекта (>=${PYTHON_MIN});" \
|
|
39
|
+
"wheel-ы получились бы для чужого ABI. Возьмите образ с python ${PYTHON_MIN}+." >&2
|
|
40
|
+
exit 1
|
|
41
|
+
fi
|
|
42
|
+
|
|
43
|
+
$PYTHON -m venv /opt/venv
|
|
44
|
+
export PATH="/opt/venv/bin:$PATH"
|
|
45
|
+
export UV_PROJECT_ENVIRONMENT=/opt/venv
|
|
46
|
+
export UV_LINK_MODE=copy
|
|
47
|
+
pip install --no-cache-dir -U pip uv
|
|
48
|
+
|
|
49
|
+
WHEELS=/wheels
|
|
50
|
+
PYTORCH_INDEX="${PYTORCH_INDEX}"
|
|
51
|
+
PYPI_INDEX="${PYPI_INDEX}"
|
|
52
|
+
TORCH_VERSION="${TORCH_VERSION}"
|
|
53
|
+
USE_UV="${USE_UV_EXPORT}"
|
|
54
|
+
UV_PROJECT_DIR="${UV_PROJECT_DIR:-.}"
|
|
55
|
+
# Кэш uv — только внутри контейнера (bind-mount Windows ломает atomic rename в .uv-cache)
|
|
56
|
+
UV_CACHE_DIR="/tmp/uv-cache"
|
|
57
|
+
export UV_CACHE_DIR
|
|
58
|
+
mkdir -p "$UV_CACHE_DIR"
|
|
59
|
+
|
|
60
|
+
echo "mount-check-$(date +%s)" > "${WHEELS}/.mount-check"
|
|
61
|
+
if [[ ! -f "${WHEELS}/.mount-check" ]]; then
|
|
62
|
+
echo "ERROR: /wheels недоступен - проверьте Docker mount path" >&2
|
|
63
|
+
exit 1
|
|
64
|
+
fi
|
|
65
|
+
|
|
66
|
+
harvest_uv_wheels() {
|
|
67
|
+
# Остаточный сбор из /tmp (если где-то вызывался uv pip)
|
|
68
|
+
find "$UV_CACHE_DIR" -type f -name "*.whl" -exec cp -un {} "$WHEELS"/ \; 2>/dev/null || true
|
|
69
|
+
}
|
|
70
|
+
|
|
71
|
+
# PIP_INDEX_ARGS: space-separated --index-url/--extra-index-url from resolver
|
|
72
|
+
# shellcheck disable=SC2206
|
|
73
|
+
pip_index_args=( ${PIP_INDEX_ARGS} )
|
|
74
|
+
|
|
75
|
+
# Allow HTTP corporate indexes (Nexus without TLS)
|
|
76
|
+
for ((i=0; i<${#pip_index_args[@]}; i++)); do
|
|
77
|
+
arg="${pip_index_args[$i]}"
|
|
78
|
+
if [[ "$arg" == --index-url || "$arg" == --extra-index-url ]]; then
|
|
79
|
+
url="${pip_index_args[$((i+1))]:-}"
|
|
80
|
+
case "$url" in
|
|
81
|
+
http://*|https://*)
|
|
82
|
+
host="$(echo "$url" | sed -E 's#^https?://([^/:]+).*#\1#')"
|
|
83
|
+
pip_index_args+=(--trusted-host "$host")
|
|
84
|
+
;;
|
|
85
|
+
esac
|
|
86
|
+
fi
|
|
87
|
+
done
|
|
88
|
+
|
|
89
|
+
pip_common=(
|
|
90
|
+
--dest "$WHEELS"
|
|
91
|
+
--prefer-binary
|
|
92
|
+
--ignore-requires-python
|
|
93
|
+
--retries 5
|
|
94
|
+
--timeout 60
|
|
95
|
+
--find-links "$WHEELS"
|
|
96
|
+
"${pip_index_args[@]}"
|
|
97
|
+
)
|
|
98
|
+
|
|
99
|
+
# Index-only (без find-links) — recovery, когда в /wheels лежит чужая версия пакета
|
|
100
|
+
pip_index_only=(
|
|
101
|
+
--dest "$WHEELS"
|
|
102
|
+
--prefer-binary
|
|
103
|
+
--ignore-requires-python
|
|
104
|
+
--retries 5
|
|
105
|
+
--timeout 60
|
|
106
|
+
"${pip_index_args[@]}"
|
|
107
|
+
)
|
|
108
|
+
|
|
109
|
+
download_req_file() {
|
|
110
|
+
# $1=requirements path; $2=optional "no-deps"
|
|
111
|
+
local req="$1"
|
|
112
|
+
local no_deps="${2:-}"
|
|
113
|
+
local extra=( )
|
|
114
|
+
local deps_flag=( )
|
|
115
|
+
if [[ "$no_deps" == "no-deps" ]]; then
|
|
116
|
+
deps_flag=( --no-deps )
|
|
117
|
+
fi
|
|
118
|
+
if [[ -n "$TORCH_VERSION" && -f /tmp/constraints.txt ]]; then
|
|
119
|
+
extra=( -c /tmp/constraints.txt )
|
|
120
|
+
fi
|
|
121
|
+
if [[ -f /opt/prune_stale_wheels.py ]]; then
|
|
122
|
+
echo ">>> prune stale wheels vs ${req}"
|
|
123
|
+
python3 /opt/prune_stale_wheels.py "$req" "$WHEELS" || true
|
|
124
|
+
fi
|
|
125
|
+
if pip download "${pip_common[@]}" "${deps_flag[@]}" "${extra[@]}" -r "$req"; then
|
|
126
|
+
return 0
|
|
127
|
+
fi
|
|
128
|
+
echo "WARN: pip download -r failed; retry via index (без find-links)" >&2
|
|
129
|
+
if [[ -f /opt/prune_stale_wheels.py ]]; then
|
|
130
|
+
python3 /opt/prune_stale_wheels.py "$req" "$WHEELS" || true
|
|
131
|
+
fi
|
|
132
|
+
# Повтор без find-links — убирает «from versions: только локальная»
|
|
133
|
+
pip download "${pip_index_only[@]}" "${deps_flag[@]}" "${extra[@]}" -r "$req"
|
|
134
|
+
}
|
|
135
|
+
|
|
136
|
+
echo ""
|
|
137
|
+
echo ">>> [1/4] pip, uv, setuptools, wheel"
|
|
138
|
+
pip download "${pip_common[@]}" pip uv setuptools wheel
|
|
139
|
+
|
|
140
|
+
if [[ -n "$TORCH_VERSION" ]]; then
|
|
141
|
+
echo ""
|
|
142
|
+
echo ">>> [2/4] torch + nvidia-* (base image)"
|
|
143
|
+
if compgen -G "${WHEELS}/torch-${TORCH_VERSION}+cu128*.whl" > /dev/null \
|
|
144
|
+
&& compgen -G "${WHEELS}/nvidia_cublas_cu12*.whl" > /dev/null; then
|
|
145
|
+
echo " torch cu128 и nvidia-* уже в wheels/, пропуск"
|
|
146
|
+
else
|
|
147
|
+
pip download "${pip_common[@]}" "torch==${TORCH_VERSION}"
|
|
148
|
+
fi
|
|
149
|
+
|
|
150
|
+
echo ""
|
|
151
|
+
echo ">>> [2b] torchaudio==${TORCH_VERSION}"
|
|
152
|
+
pip download "${pip_common[@]}" "torchaudio==${TORCH_VERSION}"
|
|
153
|
+
fi
|
|
154
|
+
|
|
155
|
+
IFS="|" read -ra _extra_specs <<< "${EXTRA_WHEEL_SPECS}"
|
|
156
|
+
for spec in "${_extra_specs[@]}"; do
|
|
157
|
+
[[ -z "$spec" ]] && continue
|
|
158
|
+
echo ""
|
|
159
|
+
echo ">>> [2c] extra wheel: ${spec}"
|
|
160
|
+
pip download "${pip_common[@]}" "$spec"
|
|
161
|
+
done
|
|
162
|
+
|
|
163
|
+
if [[ "$USE_UV" == "1" ]]; then
|
|
164
|
+
echo ""
|
|
165
|
+
echo ">>> [3/4] uv export deps (prune: ${UV_EXPORT_PRUNE})"
|
|
166
|
+
cd "/app/${UV_PROJECT_DIR}"
|
|
167
|
+
prune_args=()
|
|
168
|
+
for pkg in ${UV_EXPORT_PRUNE}; do
|
|
169
|
+
prune_args+=(--prune "$pkg")
|
|
170
|
+
done
|
|
171
|
+
uv export --frozen --no-dev --no-emit-project \
|
|
172
|
+
"${prune_args[@]}" \
|
|
173
|
+
--no-hashes \
|
|
174
|
+
-o /tmp/app-requirements.txt
|
|
175
|
+
# --no-deps: uv export уже полный + учитывает override-dependencies
|
|
176
|
+
# (напр. ctranslate2>=4.5 при whisperx, который пинит <4.5). pip иначе
|
|
177
|
+
# падает с ResolutionImpossible.
|
|
178
|
+
if [[ -n "$TORCH_VERSION" ]]; then
|
|
179
|
+
printf 'torch==%s\ntorchaudio==%s\n' "$TORCH_VERSION" "$TORCH_VERSION" > /tmp/constraints.txt
|
|
180
|
+
fi
|
|
181
|
+
download_req_file /tmp/app-requirements.txt no-deps
|
|
182
|
+
rm -f /wheels/torch-2.1[0-9].0-*.whl /wheels/torch-2.13*.whl \
|
|
183
|
+
/wheels/torchaudio-2.1[0-9]* /wheels/torchaudio-2.2* 2>/dev/null || true
|
|
184
|
+
elif [[ -f /app/requirements.txt ]]; then
|
|
185
|
+
echo ""
|
|
186
|
+
echo ">>> [3/4] requirements.txt (без uv.lock)"
|
|
187
|
+
cp /app/requirements.txt /tmp/app-requirements.txt
|
|
188
|
+
download_req_file /tmp/app-requirements.txt
|
|
189
|
+
fi
|
|
190
|
+
|
|
191
|
+
echo ""
|
|
192
|
+
echo ">>> [3b] cython/setuptools + av fallback"
|
|
193
|
+
pip download "${pip_common[@]}" "cython>=3.1.0a1,<4" "setuptools>=77" || true
|
|
194
|
+
if ! compgen -G "/wheels/av-14.4.0-*.whl" > /dev/null; then
|
|
195
|
+
pip download "${pip_common[@]}" --only-binary=:all: "av==14.2.0" || true
|
|
196
|
+
pip wheel --no-deps --wheel-dir /wheels "av==14.4.0" \
|
|
197
|
+
|| echo " WARN: av 14.4.0 build failed"
|
|
198
|
+
fi
|
|
199
|
+
rm -f /wheels/av-*.tar.gz 2>/dev/null || true
|
|
200
|
+
|
|
201
|
+
echo ""
|
|
202
|
+
echo ">>> [3c] wheel из sdist (docopt, julius, antlr4)"
|
|
203
|
+
pip install --no-cache-dir wheel
|
|
204
|
+
for pkg in docopt julius antlr4-python3-runtime; do
|
|
205
|
+
if compgen -G "/wheels/${pkg}-*.whl" > /dev/null; then
|
|
206
|
+
continue
|
|
207
|
+
fi
|
|
208
|
+
if compgen -G "/wheels/${pkg}-*.tar.gz" > /dev/null; then
|
|
209
|
+
pip wheel --no-deps --wheel-dir /wheels /wheels/${pkg}-*.tar.gz \
|
|
210
|
+
&& rm -f /wheels/${pkg}-*.tar.gz
|
|
211
|
+
fi
|
|
212
|
+
done
|
|
213
|
+
|
|
214
|
+
echo ""
|
|
215
|
+
echo ">>> [4/4] ctranslate2>=4.5.0"
|
|
216
|
+
pip download "${pip_common[@]}" "ctranslate2>=4.5.0" || true
|
|
217
|
+
|
|
218
|
+
harvest_uv_wheels
|
|
219
|
+
|
|
220
|
+
count="$(find "$WHEELS" -maxdepth 1 -name "*.whl" | wc -l)"
|
|
221
|
+
nvidia_count="$(find "$WHEELS" -maxdepth 1 -name "nvidia_*.whl" | wc -l)"
|
|
222
|
+
echo ""
|
|
223
|
+
echo "=== Готово: ${count} wheel-файлов (${nvidia_count} nvidia_*) в ${WHEELS} ==="
|
|
224
|
+
du -sh "$WHEELS"
|
|
225
|
+
"""
|
|
226
|
+
|
|
227
|
+
|
|
228
|
+
def download_wheels(ctx: ProjectContext) -> None:
|
|
229
|
+
if not ctx.config.python_lock_paths() and not (ctx.root / "requirements.txt").is_file():
|
|
230
|
+
print("=== download-wheels: SKIP (нет pyproject.toml/uv.lock/requirements.txt) ===")
|
|
231
|
+
return
|
|
232
|
+
|
|
233
|
+
ctx.ensure_offline_dirs()
|
|
234
|
+
try:
|
|
235
|
+
current = ctx.python_deps_hash()
|
|
236
|
+
except RuntimeError:
|
|
237
|
+
current = "no-lock-files"
|
|
238
|
+
|
|
239
|
+
whl_count = len(list(ctx.wheels_dir.glob("*.whl")))
|
|
240
|
+
mount_check = ctx.wheels_dir / ".mount-check"
|
|
241
|
+
critical_ok = all(
|
|
242
|
+
any(ctx.wheels_dir.glob(wp.pattern)) for wp in ctx.config.critical_wheels
|
|
243
|
+
)
|
|
244
|
+
if (
|
|
245
|
+
ctx.wheels_hash_file.is_file()
|
|
246
|
+
and ctx.wheels_hash_file.read_text(encoding="utf-8").strip() == current
|
|
247
|
+
and mount_check.is_file()
|
|
248
|
+
and whl_count >= 10
|
|
249
|
+
and critical_ok
|
|
250
|
+
):
|
|
251
|
+
print(
|
|
252
|
+
f"=== download-wheels: lockfile не менялся, cache актуален ({whl_count} .whl) ==="
|
|
253
|
+
)
|
|
254
|
+
print(docker_cmd.dir_size_human(ctx.wheels_dir))
|
|
255
|
+
return
|
|
256
|
+
|
|
257
|
+
tv = ctx.config.torch_version() or ""
|
|
258
|
+
pytorch_index = os.environ.get("PYTORCH_INDEX", ctx.config.pytorch_index)
|
|
259
|
+
pypi_index = os.environ.get("PYPI_INDEX", ctx.config.pypi_index)
|
|
260
|
+
resolver = ctx.config.package_resolver()
|
|
261
|
+
from offline_docker.resolve import pip_index_chain_for_download
|
|
262
|
+
|
|
263
|
+
chain, used_fallback = pip_index_chain_for_download(
|
|
264
|
+
resolver,
|
|
265
|
+
pypi_index=pypi_index,
|
|
266
|
+
pytorch_index=pytorch_index,
|
|
267
|
+
)
|
|
268
|
+
pip_index_args: list[str] = []
|
|
269
|
+
if chain:
|
|
270
|
+
pip_index_args.extend(["--index-url", chain[0]])
|
|
271
|
+
for url in chain[1:]:
|
|
272
|
+
pip_index_args.extend(["--extra-index-url", url])
|
|
273
|
+
pip_index_args_str = " ".join(pip_index_args)
|
|
274
|
+
|
|
275
|
+
root_docker = docker_mount_path(ctx.root)
|
|
276
|
+
wheels_docker = docker_mount_path(ctx.wheels_dir)
|
|
277
|
+
import offline_docker.wheel_prune as wheel_prune
|
|
278
|
+
|
|
279
|
+
prune_host = Path(wheel_prune.__file__).resolve()
|
|
280
|
+
prune_docker = docker_mount_path(prune_host)
|
|
281
|
+
|
|
282
|
+
print("=== download-wheels ===")
|
|
283
|
+
print(f"Корень: {ctx.root}")
|
|
284
|
+
print(f"Offline cache: {ctx.offline_cache}")
|
|
285
|
+
print(f"Каталог wheels: {ctx.wheels_dir}")
|
|
286
|
+
print(f"torch: {tv or '(нет)'}")
|
|
287
|
+
print(f"use_uv_export: {ctx.config.use_uv_export}")
|
|
288
|
+
print(f"python project: {ctx.config.effective_python_project_dir()}")
|
|
289
|
+
print(
|
|
290
|
+
"Python (образ): "
|
|
291
|
+
f"{ctx.config.deb_ubuntu_image} (нужно >= {ctx.config.python_requires_min() or '—'})"
|
|
292
|
+
)
|
|
293
|
+
print(f"Resolve: {' -> '.join(resolver.settings.resolve.order)}")
|
|
294
|
+
print(f"PyPI chain: {' -> '.join(chain) if chain else '(find-links only)'}")
|
|
295
|
+
if used_fallback and resolver.is_local_only():
|
|
296
|
+
print()
|
|
297
|
+
print(
|
|
298
|
+
"WARN: sources.json = local-only; для download временно добавлен "
|
|
299
|
+
"PyPI/PyTorch index."
|
|
300
|
+
)
|
|
301
|
+
print(
|
|
302
|
+
" Для явного режима заполнения кэша: "
|
|
303
|
+
"offdock configure --with-upstream"
|
|
304
|
+
)
|
|
305
|
+
print()
|
|
306
|
+
|
|
307
|
+
from offline_docker.sync_cache import maybe_materialize
|
|
308
|
+
|
|
309
|
+
maybe_materialize(ctx)
|
|
310
|
+
|
|
311
|
+
prune = " ".join(ctx.config.uv_export_prune)
|
|
312
|
+
extra = "|".join(ctx.config.extra_wheel_specs)
|
|
313
|
+
project_dir = ctx.config.effective_python_project_dir()
|
|
314
|
+
|
|
315
|
+
env = [
|
|
316
|
+
f"TORCH_VERSION={tv}",
|
|
317
|
+
f"PYTORCH_INDEX={pytorch_index}",
|
|
318
|
+
f"PYPI_INDEX={chain[0] if chain else pypi_index}",
|
|
319
|
+
f"PIP_INDEX_ARGS={pip_index_args_str}",
|
|
320
|
+
f"USE_UV_EXPORT={'1' if ctx.config.use_uv_export else '0'}",
|
|
321
|
+
f"UV_PROJECT_DIR={project_dir}",
|
|
322
|
+
f"UV_EXPORT_PRUNE={prune}",
|
|
323
|
+
f"EXTRA_WHEEL_SPECS={extra}",
|
|
324
|
+
f"PYTHON_MIN={ctx.config.python_requires_min() or ''}",
|
|
325
|
+
]
|
|
326
|
+
|
|
327
|
+
docker_cmd.run(
|
|
328
|
+
[
|
|
329
|
+
"docker",
|
|
330
|
+
"run",
|
|
331
|
+
"--rm",
|
|
332
|
+
*[item for pair in env for item in ("-e", pair)],
|
|
333
|
+
"-v",
|
|
334
|
+
f"{root_docker}:/app:ro",
|
|
335
|
+
"-v",
|
|
336
|
+
f"{wheels_docker}:/wheels",
|
|
337
|
+
"-v",
|
|
338
|
+
f"{prune_docker}:/opt/prune_stale_wheels.py:ro",
|
|
339
|
+
ctx.config.deb_ubuntu_image,
|
|
340
|
+
"bash",
|
|
341
|
+
"-ec",
|
|
342
|
+
WHEELS_GUEST_UV,
|
|
343
|
+
],
|
|
344
|
+
env={"DOCKER_BUILDKIT": "0"},
|
|
345
|
+
)
|
|
346
|
+
|
|
347
|
+
if not mount_check.is_file():
|
|
348
|
+
raise SystemExit(
|
|
349
|
+
"ERROR: wheels не попали на хост - проверьте Docker mount path (D:/..., не /d/...)"
|
|
350
|
+
)
|
|
351
|
+
|
|
352
|
+
ctx.wheels_hash_file.write_text(current + "\n", encoding="utf-8")
|
|
353
|
+
print()
|
|
354
|
+
nvidia = sorted(ctx.wheels_dir.glob("nvidia_*.whl"))
|
|
355
|
+
if nvidia:
|
|
356
|
+
print("NVIDIA-пакеты на хосте:")
|
|
357
|
+
for p in nvidia:
|
|
358
|
+
print(p.name)
|
|
359
|
+
|
|
360
|
+
if resolver.settings.resolve.fill_local:
|
|
361
|
+
from offline_docker.sync_cache import publish_to_remotes
|
|
362
|
+
|
|
363
|
+
publish_to_remotes(ctx)
|
|
@@ -0,0 +1,208 @@
|
|
|
1
|
+
"""Download yarn cache and dependent oc:// archives (discovered from Dockerfiles)."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import shutil
|
|
6
|
+
import subprocess
|
|
7
|
+
import tempfile
|
|
8
|
+
from pathlib import Path
|
|
9
|
+
|
|
10
|
+
from offline_docker import docker_cmd
|
|
11
|
+
from offline_docker.context import ProjectContext
|
|
12
|
+
from offline_docker.oc_discovery import (
|
|
13
|
+
YarnCacheSpec,
|
|
14
|
+
archive_tar_path,
|
|
15
|
+
discover_yarn_cache_specs,
|
|
16
|
+
resolve_build_arg,
|
|
17
|
+
)
|
|
18
|
+
from offline_docker.utils import docker_mount_path, file_sha256
|
|
19
|
+
|
|
20
|
+
YARN_GUEST = r"""
|
|
21
|
+
set -euo pipefail
|
|
22
|
+
export DEBIAN_FRONTEND=noninteractive
|
|
23
|
+
apt-get update
|
|
24
|
+
apt-get install -y --no-install-recommends ca-certificates git
|
|
25
|
+
rm -rf /var/lib/apt/lists/*
|
|
26
|
+
yarn config set cache-folder /yarn-cache
|
|
27
|
+
yarn install --network-timeout 100000
|
|
28
|
+
echo "mount-check-$(date +%s)" > /yarn-cache/.mount-check
|
|
29
|
+
"""
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
def _archive_tag_file(archive_dir: Path, tag: str) -> Path:
|
|
33
|
+
safe = tag.replace("/", "_")
|
|
34
|
+
return archive_dir / f".archive-{safe}.tag"
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
def sanitize_yarn_cache_for_buildkit(cache_dir: Path) -> int:
|
|
38
|
+
"""Remove package ``.bin`` dirs from yarn cache.
|
|
39
|
+
|
|
40
|
+
Yarn classic stores Linux symlinks under ``*/.bin/*`` when the cache is
|
|
41
|
+
filled via a Linux container bind-mounted to a Windows host. Docker
|
|
42
|
+
BuildKit then fails with ``invalid file request .../.bin/parser`` while
|
|
43
|
+
transferring the ``oc`` additional context. Offline install uses
|
|
44
|
+
``.yarn-tarball.tgz`` and does not need those ``.bin`` links.
|
|
45
|
+
"""
|
|
46
|
+
if not cache_dir.is_dir():
|
|
47
|
+
return 0
|
|
48
|
+
removed = 0
|
|
49
|
+
for bin_dir in sorted(cache_dir.rglob(".bin"), reverse=True):
|
|
50
|
+
if not bin_dir.is_dir():
|
|
51
|
+
continue
|
|
52
|
+
try:
|
|
53
|
+
shutil.rmtree(bin_dir)
|
|
54
|
+
removed += 1
|
|
55
|
+
continue
|
|
56
|
+
except OSError:
|
|
57
|
+
pass
|
|
58
|
+
for child in list(bin_dir.iterdir()):
|
|
59
|
+
try:
|
|
60
|
+
child.unlink(missing_ok=True)
|
|
61
|
+
except OSError:
|
|
62
|
+
try:
|
|
63
|
+
child.unlink()
|
|
64
|
+
except OSError:
|
|
65
|
+
pass
|
|
66
|
+
try:
|
|
67
|
+
bin_dir.rmdir()
|
|
68
|
+
removed += 1
|
|
69
|
+
except OSError:
|
|
70
|
+
pass
|
|
71
|
+
return removed
|
|
72
|
+
|
|
73
|
+
|
|
74
|
+
def download_oc_archive(ctx: ProjectContext, spec: YarnCacheSpec) -> Path:
|
|
75
|
+
if shutil.which("git") is None:
|
|
76
|
+
raise SystemExit("ERROR: git не найден в PATH — нужен для подготовки oc-архива")
|
|
77
|
+
|
|
78
|
+
ctx.ensure_offline_dirs()
|
|
79
|
+
version = resolve_build_arg(ctx.config, spec.version_arg, ctx.config.path(spec.source_dockerfile))
|
|
80
|
+
tar = archive_tar_path(ctx.config, spec)
|
|
81
|
+
tag = f"{spec.git_tag_prefix}{version}"
|
|
82
|
+
tag_file = _archive_tag_file(tar.parent, tag)
|
|
83
|
+
|
|
84
|
+
if tar.is_file() and tag_file.is_file() and tag_file.read_text(encoding="utf-8").strip() == tag:
|
|
85
|
+
print(f"=== download-oc-archive [{spec.archive_subdir}]: {tag} актуален ===")
|
|
86
|
+
print(docker_cmd.dir_size_human(tar))
|
|
87
|
+
return tar
|
|
88
|
+
|
|
89
|
+
print("=== download-oc-archive ===")
|
|
90
|
+
print(f"Subdir: {spec.archive_subdir}")
|
|
91
|
+
print(f"Version: {version} ({spec.version_arg})")
|
|
92
|
+
print(f"Repo: {spec.git_url}")
|
|
93
|
+
print(f"Output: {tar}")
|
|
94
|
+
print()
|
|
95
|
+
|
|
96
|
+
with tempfile.TemporaryDirectory(prefix="od-oc-archive-") as tmp_name:
|
|
97
|
+
tmp = Path(tmp_name)
|
|
98
|
+
clone_dir = tmp / "upstream"
|
|
99
|
+
print(f">>> git clone --depth 1 --branch {tag}")
|
|
100
|
+
subprocess.run(
|
|
101
|
+
[
|
|
102
|
+
"git",
|
|
103
|
+
"clone",
|
|
104
|
+
"--depth",
|
|
105
|
+
"1",
|
|
106
|
+
"--branch",
|
|
107
|
+
tag,
|
|
108
|
+
spec.git_url,
|
|
109
|
+
str(clone_dir),
|
|
110
|
+
],
|
|
111
|
+
check=True,
|
|
112
|
+
)
|
|
113
|
+
tar.parent.mkdir(parents=True, exist_ok=True)
|
|
114
|
+
subprocess.run(
|
|
115
|
+
["tar", "-czf", str(tar), "-C", str(tmp), "upstream"],
|
|
116
|
+
check=True,
|
|
117
|
+
)
|
|
118
|
+
|
|
119
|
+
tag_file.write_text(tag + "\n", encoding="utf-8")
|
|
120
|
+
print()
|
|
121
|
+
print(f"=== Готово: {tar} ===")
|
|
122
|
+
print(docker_cmd.dir_size_human(tar))
|
|
123
|
+
return tar
|
|
124
|
+
|
|
125
|
+
|
|
126
|
+
def download_yarn_cache(ctx: ProjectContext) -> None:
|
|
127
|
+
specs = discover_yarn_cache_specs(ctx.config)
|
|
128
|
+
if not specs:
|
|
129
|
+
print("=== download-yarn-cache: SKIP (нет source=yarn в *Dockerfile.offline) ===")
|
|
130
|
+
return
|
|
131
|
+
|
|
132
|
+
for spec in specs:
|
|
133
|
+
_download_yarn_cache_one(ctx, spec)
|
|
134
|
+
|
|
135
|
+
|
|
136
|
+
def _download_yarn_cache_one(ctx: ProjectContext, spec: YarnCacheSpec) -> None:
|
|
137
|
+
ctx.ensure_offline_dirs()
|
|
138
|
+
cache_dir = ctx.offline_cache / spec.cache_subdir
|
|
139
|
+
cache_dir.mkdir(parents=True, exist_ok=True)
|
|
140
|
+
|
|
141
|
+
tar = archive_tar_path(ctx.config, spec)
|
|
142
|
+
if not tar.is_file():
|
|
143
|
+
print(f">>> oc/{spec.archive_subdir} не найден, загружаем архив...")
|
|
144
|
+
tar = download_oc_archive(ctx, spec)
|
|
145
|
+
|
|
146
|
+
hash_file = cache_dir / f".{spec.archive_subdir}.sha256"
|
|
147
|
+
current = file_sha256(tar)
|
|
148
|
+
marker = cache_dir / ".mount-check"
|
|
149
|
+
|
|
150
|
+
if (
|
|
151
|
+
hash_file.is_file()
|
|
152
|
+
and hash_file.read_text(encoding="utf-8").strip() == current
|
|
153
|
+
and marker.is_file()
|
|
154
|
+
):
|
|
155
|
+
n = sanitize_yarn_cache_for_buildkit(cache_dir)
|
|
156
|
+
print(f"=== download-yarn-cache [{spec.cache_subdir}]: cache актуален ===")
|
|
157
|
+
if n:
|
|
158
|
+
print(f"sanitize: removed {n} .bin dir(s) for BuildKit/Windows")
|
|
159
|
+
print(docker_cmd.dir_size_human(cache_dir))
|
|
160
|
+
return
|
|
161
|
+
|
|
162
|
+
yarn_docker = docker_mount_path(cache_dir)
|
|
163
|
+
|
|
164
|
+
print("=== download-yarn-cache ===")
|
|
165
|
+
print(f"Cache subdir: {spec.cache_subdir}")
|
|
166
|
+
print(f"Archive: {tar}")
|
|
167
|
+
print(f"Node image: {spec.node_image}")
|
|
168
|
+
print(f"Frontend dir: {spec.frontend_dir}")
|
|
169
|
+
print()
|
|
170
|
+
|
|
171
|
+
with tempfile.TemporaryDirectory(prefix="od-yarn-") as tmp_name:
|
|
172
|
+
tmp = Path(tmp_name)
|
|
173
|
+
subprocess.run(
|
|
174
|
+
["tar", "-xzf", str(tar), "-C", str(tmp), "--strip-components=1"],
|
|
175
|
+
check=True,
|
|
176
|
+
)
|
|
177
|
+
frontend = tmp / spec.frontend_dir
|
|
178
|
+
if not frontend.is_dir():
|
|
179
|
+
raise SystemExit(f"ERROR: в архиве нет {spec.frontend_dir}/: {tar}")
|
|
180
|
+
|
|
181
|
+
frontend_docker = docker_mount_path(frontend)
|
|
182
|
+
docker_cmd.run(
|
|
183
|
+
[
|
|
184
|
+
"docker",
|
|
185
|
+
"run",
|
|
186
|
+
"--rm",
|
|
187
|
+
"--entrypoint",
|
|
188
|
+
"/bin/bash",
|
|
189
|
+
"-v",
|
|
190
|
+
f"{yarn_docker}:/yarn-cache",
|
|
191
|
+
"-v",
|
|
192
|
+
f"{frontend_docker}:/work/frontend",
|
|
193
|
+
"-w",
|
|
194
|
+
"/work/frontend",
|
|
195
|
+
spec.node_image,
|
|
196
|
+
"-ec",
|
|
197
|
+
YARN_GUEST,
|
|
198
|
+
],
|
|
199
|
+
env={"DOCKER_BUILDKIT": "0", "MSYS_NO_PATHCONV": "1"},
|
|
200
|
+
)
|
|
201
|
+
|
|
202
|
+
hash_file.write_text(current + "\n", encoding="utf-8")
|
|
203
|
+
n = sanitize_yarn_cache_for_buildkit(cache_dir)
|
|
204
|
+
print()
|
|
205
|
+
print(f"=== Готово: yarn cache в {cache_dir} ===")
|
|
206
|
+
if n:
|
|
207
|
+
print(f"sanitize: removed {n} .bin dir(s) for BuildKit/Windows")
|
|
208
|
+
print(docker_cmd.dir_size_human(cache_dir))
|
|
@@ -0,0 +1,61 @@
|
|
|
1
|
+
"""Ensure base Docker images (reuse / build)."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from offline_docker import docker_cmd
|
|
6
|
+
from offline_docker.context import ProjectContext
|
|
7
|
+
from offline_docker.utils import docker_mount_path
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
def ensure_base_online(
|
|
11
|
+
ctx: ProjectContext, image: str | None = None, *, no_cache: bool = False
|
|
12
|
+
) -> None:
|
|
13
|
+
image = image or ctx.config.images.base
|
|
14
|
+
if not no_cache and docker_cmd.image_exists(image):
|
|
15
|
+
print(f">>> reuse existing base: {image}")
|
|
16
|
+
return
|
|
17
|
+
df = ctx.config.dockerfiles.base
|
|
18
|
+
print(f">>> docker build base -> {image}")
|
|
19
|
+
cmd = ["docker", "build"]
|
|
20
|
+
if no_cache:
|
|
21
|
+
cmd.append("--no-cache")
|
|
22
|
+
cmd.extend(["-f", df, "-t", image, "."])
|
|
23
|
+
docker_cmd.run(cmd, cwd=ctx.root, env={"DOCKER_BUILDKIT": "1"})
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
def require_base_offline(ctx: ProjectContext, image: str | None = None) -> None:
|
|
27
|
+
image = image or ctx.config.images.base
|
|
28
|
+
if not docker_cmd.image_exists(image):
|
|
29
|
+
raise SystemExit(
|
|
30
|
+
f"Ошибка: образ {image} не найден. "
|
|
31
|
+
"Сначала: offdock build base"
|
|
32
|
+
)
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
def build_base_offline(
|
|
36
|
+
ctx: ProjectContext, image: str | None = None, *, no_cache: bool = False
|
|
37
|
+
) -> None:
|
|
38
|
+
image = image or ctx.config.images.base
|
|
39
|
+
if not no_cache and docker_cmd.image_exists(image):
|
|
40
|
+
print(f">>> reuse existing base: {image}")
|
|
41
|
+
return
|
|
42
|
+
|
|
43
|
+
log_file = ctx.offline_build_log("base")
|
|
44
|
+
df = ctx.config.dockerfiles.base_offline
|
|
45
|
+
oc_ctx = docker_mount_path(ctx.offline_cache)
|
|
46
|
+
print(f">>> docker build base -> {log_file}")
|
|
47
|
+
cmd = ["docker", "build"]
|
|
48
|
+
if no_cache:
|
|
49
|
+
cmd.append("--no-cache")
|
|
50
|
+
cmd.extend(
|
|
51
|
+
[
|
|
52
|
+
"-f",
|
|
53
|
+
df,
|
|
54
|
+
"--build-context",
|
|
55
|
+
f"oc={oc_ctx}",
|
|
56
|
+
"-t",
|
|
57
|
+
image,
|
|
58
|
+
".",
|
|
59
|
+
]
|
|
60
|
+
)
|
|
61
|
+
docker_cmd.run_tee(cmd, log_file, cwd=ctx.root, env={"DOCKER_BUILDKIT": "1"})
|