offline-docker-cli 2.2.2__py3-none-any.whl

offline_docker/docker_cmd.py

@@ -0,0 +1,136 @@
+"""Thin wrappers around docker / docker compose CLI."""
+
+from __future__ import annotations
+
+import os
+import shutil
+import subprocess
+import sys
+from pathlib import Path
+
+
+_SUBPROCESS_ENCODING = "utf-8"
+
+
+def _write_stdout(text: str) -> None:
+    try:
+        sys.stdout.write(text)
+    except UnicodeEncodeError:
+        sys.stdout.buffer.write(
+            text.encode(sys.stdout.encoding or _SUBPROCESS_ENCODING, errors="replace")
+        )
+    sys.stdout.flush()
+
+
+class CommandError(RuntimeError):
+    def __init__(self, cmd: list[str], returncode: int):
+        self.cmd = cmd
+        self.returncode = returncode
+        super().__init__(f"Command failed ({returncode}): {' '.join(cmd)}")
+
+
+def require_docker() -> None:
+    if shutil.which("docker") is None:
+        raise SystemExit("ERROR: docker not found in PATH")
+
+
+def run(
+    cmd: list[str],
+    *,
+    cwd: Path | None = None,
+    env: dict[str, str] | None = None,
+    check: bool = True,
+) -> int:
+    print(f"$ {' '.join(cmd)}", flush=True)
+    merged = os.environ.copy()
+    if env:
+        merged.update(env)
+    result = subprocess.run(cmd, cwd=cwd, env=merged)
+    if check and result.returncode != 0:
+        raise CommandError(cmd, result.returncode)
+    return result.returncode
+
+
+def run_capture(
+    cmd: list[str],
+    *,
+    cwd: Path | None = None,
+    env: dict[str, str] | None = None,
+) -> subprocess.CompletedProcess[str]:
+    merged = os.environ.copy()
+    if env:
+        merged.update(env)
+    return subprocess.run(
+        cmd,
+        cwd=cwd,
+        env=merged,
+        capture_output=True,
+        text=True,
+        encoding=_SUBPROCESS_ENCODING,
+        errors="replace",
+    )
+
+
+def image_exists(image: str) -> bool:
+    r = run_capture(["docker", "image", "inspect", image])
+    return r.returncode == 0
+
+
+def run_tee(
+    cmd: list[str],
+    log_file: Path,
+    *,
+    cwd: Path | None = None,
+    env: dict[str, str] | None = None,
+) -> None:
+    """Run command, stream stdout/stderr to console and log file."""
+    log_file.parent.mkdir(parents=True, exist_ok=True)
+    print(f"$ {' '.join(cmd)}  -> {log_file}", flush=True)
+    merged = os.environ.copy()
+    if env:
+        merged.update(env)
+    with log_file.open("w", encoding="utf-8") as log:
+        proc = subprocess.Popen(
+            cmd,
+            cwd=cwd,
+            env=merged,
+            stdout=subprocess.PIPE,
+            stderr=subprocess.STDOUT,
+            text=True,
+            encoding=_SUBPROCESS_ENCODING,
+            errors="replace",
+            bufsize=1,
+        )
+        assert proc.stdout is not None
+        for line in proc.stdout:
+            _write_stdout(line)
+            log.write(line)
+        rc = proc.wait()
+    if rc != 0:
+        raise CommandError(cmd, rc)
+
+
+def dir_size_human(path: Path) -> str:
+    """Human-readable size. Skips broken symlinks (common on Windows yarn caches)."""
+    total = 0
+    try:
+        if path.is_file():
+            total = path.stat().st_size
+        elif path.is_dir():
+            for f in path.rglob("*"):
+                try:
+                    # is_file() follows symlinks and can raise WinError 1920
+                    if f.is_file():
+                        total += f.stat().st_size
+                except OSError:
+                    pass
+    except OSError:
+        pass
+    size = float(total)
+    for unit in ("B", "K", "M", "G"):
+        if size < 1024.0 or unit == "G":
+            if unit == "B":
+                return f"{int(size)}{unit}"
+            return f"{size:.1f}{unit}"
+        size /= 1024.0
+    return f"{size:.1f}G"

offline_docker/download_debs.py

@@ -0,0 +1,142 @@
+"""Download .deb packages into offline-cache/debs."""
+
+from __future__ import annotations
+
+from pathlib import Path
+
+from offline_docker import docker_cmd
+from offline_docker.context import ProjectContext
+from offline_docker.utils import docker_mount_path
+
+
+def _build_debs_guest(ctx: ProjectContext) -> str:
+    pkgs_bash = " ".join(f'"{p}"' for p in ctx.config.deb_packages)
+    extra_repo_script = ""
+    for repo in ctx.config.deb_extra_repos:
+        gpg_name = Path(repo.gpg_dest).name
+        list_name = Path(repo.list_dest).name
+        extra_repo_script += f"""
+echo ">>> Extra repo: {repo.name}"
+curl -fsSL {repo.gpg_url} | gpg --dearmor -o /apt-meta/{gpg_name}
+curl -fsSL {repo.list_url} > /apt-meta/{list_name}
+cp /apt-meta/{gpg_name} /usr/share/keyrings/{gpg_name}
+cp /apt-meta/{list_name} /etc/apt/sources.list.d/{list_name}
+apt-get update
+"""
+    # Optional apt mirrors from package resolver (ProGet deb / Nexus apt)
+    apt_mirror_script = ""
+    for i, mirror in enumerate(ctx.config.package_resolver().apt_mirrors()):
+        # Best-effort: prepend as extra signed-by-less list (corp mirrors vary)
+        apt_mirror_script += f"""
+echo ">>> Apt mirror from sources[{i}]: {mirror}"
+echo "deb [trusted=yes] {mirror} ./" > /etc/apt/sources.list.d/od-mirror-{i}.list
+"""
+    return f"""
+set -euo pipefail
+export DEBIAN_FRONTEND=noninteractive
+
+# Cache debs is bound to the guest distro: Ubuntu/Debian .deb несовместимы, и
+# dpkg -i чужих пакетов ломает базовый образ. Тег образа может переехать
+# (python:3.12-slim bookworm -> trixie) — сбрасываем кэш при смене базы.
+. /etc/os-release
+BASE_OS="$ID-$VERSION_ID"
+if [ -f /debs/.base-os ] && [ "$(cat /debs/.base-os)" != "$BASE_OS" ]; then
+  echo ">>> База изменилась: $(cat /debs/.base-os) -> $BASE_OS - очищаю старые .deb"
+  find /debs -maxdepth 1 -name '*.deb' -delete
+fi
+printf '%s\\n' "$BASE_OS" > /debs/.base-os
+
+apt-get update
+apt-get install -y --no-install-recommends ca-certificates curl gnupg
+{apt_mirror_script}
+{extra_repo_script}
+echo "ok-$(date +%s)" > /apt-meta/.ready
+
+PKGS=({pkgs_bash})
+
+echo ""
+echo ">>> Сбор транзитивных зависимостей"
+ALL=""
+for pkg in "${{PKGS[@]}}"; do
+  deps="$(apt-cache depends --recurse --no-recommends \\
+    --no-suggests --no-conflicts --no-breaks --no-replaces --no-enhances \\
+    "$pkg" 2>/dev/null | grep -E "^[[:alnum:]][[:alnum:]._+-]*$" || true)"
+  ALL="$ALL $deps"
+done
+UNIQUE="$(echo "$ALL" | tr " " "\\n" | sort -u | grep -v "^$")"
+
+cd /debs
+total="$(echo "$UNIQUE" | wc -l)"
+n=0
+skipped=0
+for pkg in $UNIQUE; do
+  n=$((n + 1))
+  if compgen -G "${{pkg}}_*.deb" > /dev/null \\
+     || compgen -G "${{pkg}}-*.deb" > /dev/null; then
+    skipped=$((skipped + 1))
+    continue
+  fi
+  if (( n % 25 == 1 )); then
+    echo "    [$n/$total] $pkg ..."
+  fi
+  apt-get download "$pkg" 2>/dev/null || echo "    WARN: не скачан $pkg" >&2
+done
+
+count="$(find /debs -maxdepth 1 -name "*.deb" | wc -l)"
+echo ""
+echo "=== Готово: ${{count}} .deb в /debs (пропущено уже скачанных: ${{skipped}}) ==="
+du -sh /debs
+rm -rf /var/lib/apt/lists/*
+"""
+
+
+def download_debs(ctx: ProjectContext) -> None:
+    ctx.ensure_offline_dirs()
+    from offline_docker.sync_cache import maybe_materialize
+
+    maybe_materialize(ctx)
+
+    debs_docker = docker_mount_path(ctx.debs_dir)
+    apt_meta_docker = docker_mount_path(ctx.apt_meta_dir)
+    resolver = ctx.config.package_resolver()
+
+    print("=== download-debs ===")
+    print(f"Корень:           {ctx.root}")
+    print(f"Offline cache:    {ctx.offline_cache}")
+    print(f"Ubuntu image:     {ctx.config.deb_ubuntu_image}")
+    print(f"Пакетов (seed):   {len(ctx.config.deb_packages)}")
+    mirrors = resolver.apt_mirrors()
+    if mirrors:
+        print(f"Apt mirrors:      {', '.join(mirrors)}")
+    print()
+
+    docker_cmd.run(
+        [
+            "docker",
+            "run",
+            "--rm",
+            "-v",
+            f"{debs_docker}:/debs",
+            "-v",
+            f"{apt_meta_docker}:/apt-meta",
+            ctx.config.deb_ubuntu_image,
+            "bash",
+            "-ec",
+            _build_debs_guest(ctx),
+        ],
+        env={"DOCKER_BUILDKIT": "0"},
+    )
+
+    if resolver.settings.resolve.fill_local:
+        from offline_docker.sync_cache import publish_to_remotes
+
+        publish_to_remotes(ctx)
+
+    if not (ctx.apt_meta_dir / ".ready").is_file():
+        raise SystemExit(
+            "ERROR: apt-meta не попал на хост - проверьте Docker mount path (D:/..., не /d/...)"
+        )
+
+    deb_count = len(list(ctx.debs_dir.glob("*.deb")))
+    print()
+    print(f"Всего .deb: {deb_count}")

offline_docker/download_images.py

@@ -0,0 +1,26 @@
+"""Pull and save Docker images into offline-cache/docker."""
+
+from __future__ import annotations
+
+from offline_docker import docker_cmd
+from offline_docker.context import ProjectContext
+
+
+def download_docker_images(ctx: ProjectContext) -> None:
+    ctx.ensure_offline_dirs()
+    print("=== download-docker-images ===")
+    print(f"Каталог: {ctx.docker_cache_dir}")
+    print()
+
+    for image in ctx.config.docker_images:
+        safe_name = image.replace("/", "_").replace(":", "-")
+        tar_path = ctx.docker_cache_dir / f"{safe_name}.tar"
+        print(f">>> pull {image}")
+        docker_cmd.run(["docker", "pull", image])
+        print(f">>> save -> {tar_path}")
+        docker_cmd.run(["docker", "save", "-o", str(tar_path), image])
+        print(docker_cmd.dir_size_human(tar_path))
+        print()
+
+    tar_count = len(list(ctx.docker_cache_dir.glob("*.tar")))
+    print(f"=== Готово: {tar_count} tar-файлов ===")

offline_docker/download_npm.py

@@ -0,0 +1,106 @@
+"""Download npm cache for frontend."""
+
+from __future__ import annotations
+
+from offline_docker import docker_cmd
+from offline_docker.context import ProjectContext
+from offline_docker.utils import docker_mount_path, file_sha256
+
+# POSIX /bin/sh (node:*-alpine has no bash; node ENTRYPOINT would otherwise exec "node bash").
+# Не использовать pipefail: dash (Debian /bin/sh) его не поддерживает.
+NPM_GUEST = r"""
+set -eu
+cp /app-src/package.json /app-src/package-lock.json . 2>/dev/null || cp /app-src/package.json .
+if [ ! -f package-lock.json ]; then
+  echo "ERROR: package-lock.json не найден - нужен lockfile для offline npm ci" >&2
+  exit 1
+fi
+npm config set cache /npm-cache
+npm config set registry "${NPM_REGISTRY}"
+# HTTP corporate registries (e.g. Nexus without TLS)
+case "${NPM_REGISTRY}" in
+  http://*) npm config set strict-ssl false ;;
+esac
+npm ci --no-audit --no-fund
+echo "mount-check-$(date +%s)" > /npm-cache/.mount-check
+"""
+
+
+def download_npm_cache(ctx: ProjectContext) -> None:
+    fe = ctx.config.frontend_path()
+    lock = ctx.config.frontend_lock_path()
+    if fe is None or lock is None:
+        print("=== download-npm-cache: SKIP (frontend/package-lock.json не найден) ===")
+        return
+
+    ctx.ensure_offline_dirs()
+    hash_file = ctx.npm_cache_dir / ".package-lock.sha256"
+    current = file_sha256(lock)
+    cacache = ctx.npm_cache_dir / "_cacache"
+
+    if (
+        hash_file.is_file()
+        and hash_file.read_text(encoding="utf-8").strip() == current
+        and cacache.is_dir()
+    ):
+        print("=== download-npm-cache: package-lock.json не менялся, cache актуален ===")
+        print(docker_cmd.dir_size_human(ctx.npm_cache_dir))
+        return
+
+    from offline_docker.sync_cache import maybe_materialize
+
+    maybe_materialize(ctx)
+
+    resolver = ctx.config.package_resolver()
+    registry = resolver.primary_npm_registry() or "https://registry.npmjs.org/"
+    used_default = resolver.primary_npm_registry() is None
+
+    frontend_docker = docker_mount_path(fe)
+    npm_docker = docker_mount_path(ctx.npm_cache_dir)
+
+    print("=== download-npm-cache ===")
+    print(f"Frontend:       {fe}")
+    print(f"npm cache dir:  {ctx.npm_cache_dir}")
+    print(f"Node image:     {ctx.config.node_image}")
+    print(f"Resolve:        {' -> '.join(resolver.settings.resolve.order)}")
+    print(f"npm registry:   {registry}")
+    if used_default and resolver.is_local_only():
+        print(
+            "WARN: sources.json = local-only; для download используется "
+            "https://registry.npmjs.org/"
+        )
+        print(
+            "      Явный режим: offdock configure --with-upstream"
+        )
+
+    docker_cmd.run(
+        [
+            "docker",
+            "run",
+            "--rm",
+            "--entrypoint",
+            "/bin/sh",
+            "-e",
+            f"NPM_REGISTRY={registry}",
+            "-v",
+            f"{frontend_docker}:/app-src:ro",
+            "-v",
+            f"{npm_docker}:/npm-cache",
+            "-w",
+            "/work",
+            ctx.config.node_image,
+            "-ec",
+            NPM_GUEST,
+        ],
+        env={"DOCKER_BUILDKIT": "0", "MSYS_NO_PATHCONV": "1"},
+    )
+
+    hash_file.write_text(current + "\n", encoding="utf-8")
+    print()
+    print(f"=== Готово: npm cache в {ctx.npm_cache_dir} ===")
+    print(docker_cmd.dir_size_human(ctx.npm_cache_dir))
+
+    if resolver.settings.resolve.fill_local:
+        from offline_docker.sync_cache import publish_to_remotes
+
+        publish_to_remotes(ctx)

offline_docker/download_trivy.py

@@ -0,0 +1,90 @@
+"""Download Trivy vulnerability DB into offline-cache/trivy."""
+
+from __future__ import annotations
+
+import time
+
+from offline_docker import docker_cmd
+from offline_docker.context import ProjectContext
+from offline_docker.utils import docker_mount_path
+
+
+def trivy_db_stale(ctx: ProjectContext) -> bool:
+    """True if DB is missing or older than configured max age."""
+    cache = ctx.trivy_cache_dir
+    ts_file = cache / ".trivy-db.timestamp"
+    max_age = ctx.config.trivy.db_max_age_days * 86400
+
+    if not cache.is_dir():
+        return True
+
+    others = [
+        p
+        for p in cache.rglob("*")
+        if p.is_file() and p.name != ".trivy-db.timestamp"
+    ]
+    if not others:
+        return True
+
+    if not ts_file.is_file():
+        return True
+
+    try:
+        stamp = int(ts_file.read_text(encoding="utf-8").strip())
+    except ValueError:
+        return True
+
+    return (time.time() - stamp) > max_age
+
+
+def ensure_trivy_image(ctx: ProjectContext) -> None:
+    image = ctx.config.trivy.scanner_image
+    if docker_cmd.image_exists(image):
+        return
+    from offline_docker.load_images import load_docker_images
+
+    print(f"Образ {image} не найден — load-docker-images")
+    load_docker_images(ctx)
+    if not docker_cmd.image_exists(image):
+        raise SystemExit(
+            f"ERROR: образ {image} недоступен. Выполните offdock cache images"
+        )
+
+
+def download_trivy_db(ctx: ProjectContext) -> None:
+    ctx.ensure_offline_dirs()
+    if not trivy_db_stale(ctx):
+        print(
+            f"=== download-trivy-db: БД актуальна (моложе {ctx.config.trivy.db_max_age_days} дн.) ==="
+        )
+        print(docker_cmd.dir_size_human(ctx.trivy_cache_dir))
+        return
+
+    print("=== download-trivy-db ===")
+    print(f"Каталог:  {ctx.trivy_cache_dir}")
+    print(f"Образ:    {ctx.config.trivy.scanner_image}")
+    print()
+
+    ensure_trivy_image(ctx)
+    cache_docker = docker_mount_path(ctx.trivy_cache_dir)
+
+    docker_cmd.run(
+        [
+            "docker",
+            "run",
+            "--rm",
+            "-v",
+            f"{cache_docker}:/root/.cache/trivy",
+            ctx.config.trivy.scanner_image,
+            "image",
+            "--download-db-only",
+        ],
+        env={"MSYS_NO_PATHCONV": "1"},
+    )
+
+    (ctx.trivy_cache_dir / ".trivy-db.timestamp").write_text(
+        str(int(time.time())) + "\n", encoding="utf-8"
+    )
+    print()
+    print(f"=== Готово: vulnerability DB в {ctx.trivy_cache_dir} ===")
+    print(docker_cmd.dir_size_human(ctx.trivy_cache_dir))