offline-docker-cli 2.2.2__py3-none-any.whl

offline_docker/sources/proget.py

@@ -0,0 +1,49 @@
+"""Inedo ProGet feed source."""
+
+from __future__ import annotations
+
+from offline_docker.sources.base import Ecosystem
+
+
+class ProGetSource:
+    source_type = "proget"
+
+    def __init__(
+        self,
+        name: str = "proget",
+        *,
+        url: str,
+        feeds: dict[str, str] | None = None,
+        auth_env: str = "PROGET_API_KEY",
+        enabled: bool = True,
+        publish: bool = False,
+    ) -> None:
+        self.name = name
+        self.url = url.rstrip("/")
+        self.feeds = dict(feeds or {})
+        self.auth_env = auth_env
+        self.enabled = enabled and bool(url.strip())
+        self.publish = publish
+
+    def index_urls(self, ecosystem: Ecosystem) -> list[str]:
+        if not self.enabled:
+            return []
+        if ecosystem == "pypi":
+            feed = self.feeds.get("pypi")
+            if not feed:
+                return []
+            return [f"{self.url}/pypi/{feed}/simple"]
+        if ecosystem == "npm":
+            feed = self.feeds.get("npm")
+            if not feed:
+                return []
+            return [f"{self.url}/npm/{feed}/"]
+        if ecosystem == "apt":
+            feed = self.feeds.get("deb") or self.feeds.get("apt")
+            if not feed:
+                return []
+            return [f"{self.url}/debian/{feed}/"]
+        return []
+
+    def universal_feed(self) -> str | None:
+        return self.feeds.get("universal") or self.feeds.get("asset")

offline_docker/sources/proget_client.py

@@ -0,0 +1,48 @@
+"""ProGet universal/asset feed helpers (optional mirror of offline-cache tree)."""
+
+from __future__ import annotations
+
+import os
+import urllib.error
+import urllib.request
+from pathlib import Path
+from urllib.parse import quote
+
+
+def _api_key_header(auth_env: str) -> dict[str, str]:
+    key = os.environ.get(auth_env, "").strip()
+    if not key:
+        return {}
+    if key.lower().startswith("x-api-key:"):
+        return {"X-ApiKey": key.split(":", 1)[1].strip()}
+    return {"X-ApiKey": key}
+
+
+def download_asset(base_url: str, feed: str, path: str, dest: Path, *, auth_env: str) -> None:
+    url = f"{base_url.rstrip('/')}/endpoints/{quote(feed)}/content/{path.lstrip('/')}"
+    req = urllib.request.Request(url, headers=_api_key_header(auth_env))
+    try:
+        with urllib.request.urlopen(req, timeout=120) as resp:
+            dest.parent.mkdir(parents=True, exist_ok=True)
+            dest.write_bytes(resp.read())
+    except urllib.error.HTTPError as exc:
+        if exc.code == 404:
+            raise FileNotFoundError(path) from exc
+        raise RuntimeError(f"ProGet HTTP {exc.code} {url}") from exc
+    except urllib.error.URLError as exc:
+        raise RuntimeError(f"ProGet unreachable {url}: {exc}") from exc
+
+
+def upload_asset(base_url: str, feed: str, path: str, src: Path, *, auth_env: str) -> None:
+    url = f"{base_url.rstrip('/')}/endpoints/{quote(feed)}/content/{path.lstrip('/')}"
+    data = src.read_bytes()
+    headers = dict(_api_key_header(auth_env))
+    headers["Content-Type"] = "application/octet-stream"
+    req = urllib.request.Request(url, data=data, headers=headers, method="PUT")
+    try:
+        with urllib.request.urlopen(req, timeout=120) as resp:
+            resp.read()
+    except urllib.error.HTTPError as exc:
+        raise RuntimeError(f"ProGet HTTP {exc.code} {url}") from exc
+    except urllib.error.URLError as exc:
+        raise RuntimeError(f"ProGet unreachable {url}: {exc}") from exc

offline_docker/sources/upstream.py

@@ -0,0 +1,36 @@
+"""Public upstream registries (PyPI / npmjs)."""
+
+from __future__ import annotations
+
+from offline_docker.sources.base import Ecosystem
+
+
+class UpstreamSource:
+    source_type = "upstream"
+
+    def __init__(
+        self,
+        name: str = "upstream",
+        *,
+        enabled: bool = False,
+        pypi_index: str = "https://pypi.org/simple",
+        npm_registry: str = "https://registry.npmjs.org/",
+        pytorch_index: str | None = None,
+    ) -> None:
+        self.name = name
+        self.enabled = enabled
+        self.pypi_index = pypi_index.rstrip("/")
+        self.npm_registry = npm_registry.rstrip("/") + "/"
+        self.pytorch_index = pytorch_index.rstrip("/") if pytorch_index else None
+
+    def index_urls(self, ecosystem: Ecosystem) -> list[str]:
+        if not self.enabled:
+            return []
+        if ecosystem == "pypi":
+            urls = [self.pypi_index]
+            if self.pytorch_index:
+                urls.append(self.pytorch_index)
+            return urls
+        if ecosystem == "npm":
+            return [self.npm_registry]
+        return []

offline_docker/status.py

@@ -0,0 +1,247 @@
+"""Structured offline-cache status for CLI validate and TUI."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import Literal
+
+from offline_docker.config import dockerfile_uses_vendor_mount
+from offline_docker.context import ProjectContext
+from offline_docker.oc_discovery import archive_tar_path, discover_yarn_cache_specs
+from offline_docker.utils import file_sha256
+
+StatusLevel = Literal["ok", "warn", "error", "skip"]
+
+
+@dataclass(frozen=True)
+class CacheStatusItem:
+    level: StatusLevel
+    label: str
+    detail: str = ""
+
+
+@dataclass
+class CacheStatusReport:
+    project: Path
+    offline_cache: Path
+    resolve_order: list[str]
+    on_miss: str
+    soft: bool
+    items: list[CacheStatusItem] = field(default_factory=list)
+
+    @property
+    def error_count(self) -> int:
+        return sum(1 for i in self.items if i.level == "error")
+
+    @property
+    def warn_count(self) -> int:
+        return sum(1 for i in self.items if i.level == "warn")
+
+    @property
+    def ok_count(self) -> int:
+        return sum(1 for i in self.items if i.level == "ok")
+
+    @property
+    def ok(self) -> bool:
+        return self.error_count == 0
+
+
+def collect_cache_status(ctx: ProjectContext) -> CacheStatusReport:
+    cfg = ctx.config
+    resolver = cfg.package_resolver()
+    soft = resolver.settings.resolve.on_miss == "warn" and not resolver.is_local_only()
+    report = CacheStatusReport(
+        project=ctx.root,
+        offline_cache=ctx.offline_cache,
+        resolve_order=list(resolver.settings.resolve.order),
+        on_miss=resolver.settings.resolve.on_miss,
+        soft=soft,
+    )
+
+    def add(level: StatusLevel, label: str, detail: str = "") -> None:
+        report.items.append(CacheStatusItem(level=level, label=label, detail=detail))
+
+    def fail(msg: str, *, label: str | None = None) -> None:
+        if soft:
+            add("warn", label or "cache", msg)
+        else:
+            add("error", label or "cache", msg)
+
+    def check_dir(d: Path, name: str) -> bool:
+        if not d.is_dir():
+            fail(f"нет каталога {name} ({d})", label=name)
+            return False
+        add("ok", name, str(d))
+        return True
+
+    def check_min_files(d: Path, pattern: str, minimum: int, label: str) -> None:
+        count = len(list(d.glob(pattern)))
+        if count < minimum:
+            fail(f"найдено {count} (минимум {minimum}) в {d}", label=label)
+        else:
+            add("ok", label, f"{count} файлов")
+
+    has_python = bool(cfg.python_lock_paths()) or (ctx.root / "requirements.txt").is_file()
+    has_frontend = cfg.frontend_lock_path() is not None
+
+    check_dir(ctx.wheels_dir, "wheels")
+    check_dir(ctx.debs_dir, "debs")
+    check_dir(ctx.npm_cache_dir, "npm")
+    check_dir(ctx.docker_cache_dir, "docker")
+
+    if ctx.trivy_cache_dir.is_dir():
+        add("ok", "trivy", str(ctx.trivy_cache_dir))
+    else:
+        add("warn", "trivy", f"нет каталога {ctx.trivy_cache_dir}")
+
+    if has_python:
+        if (ctx.wheels_dir / ".mount-check").is_file():
+            add("ok", "wheels/.mount-check")
+        else:
+            fail("wheels/.mount-check отсутствует - выполните offdock cache wheels", label="wheels")
+
+        if ctx.wheels_hash_file.is_file():
+            try:
+                current = ctx.python_deps_hash()
+                stored = ctx.wheels_hash_file.read_text(encoding="utf-8").strip()
+                if current == stored:
+                    lock_names = ", ".join(cfg.python_lock_files)
+                    add("ok", "wheels cache", f"соответствует {lock_names}")
+                else:
+                    fail("Python lockfile изменился - выполните offdock cache wheels", label="wheels")
+            except RuntimeError:
+                pass
+        else:
+            fail("нет wheels/.deps.sha256 - выполните offdock cache wheels", label="wheels")
+
+        check_min_files(ctx.wheels_dir, "*.whl", 1, "Python wheels")
+        for wp in cfg.critical_wheels:
+            if list(ctx.wheels_dir.glob(wp.pattern)):
+                add("ok", f"wheel {wp.label}")
+            else:
+                fail(f"нет {wp.pattern} - выполните offdock cache wheels", label=f"wheel {wp.label}")
+    else:
+        add("skip", "Python wheels", "нет lockfile")
+
+    check_min_files(ctx.debs_dir, "*.deb", 5, "deb-пакеты")
+
+    for repo in cfg.deb_extra_repos:
+        gpg = ctx.apt_meta_dir / Path(repo.gpg_dest).name
+        lst = ctx.apt_meta_dir / Path(repo.list_dest).name
+        if gpg.is_file() and lst.is_file():
+            add("ok", f"apt-meta/{repo.name}")
+        else:
+            fail(f"нет apt-meta для {repo.name} - выполните offdock cache debs", label=f"apt-meta/{repo.name}")
+
+    if has_frontend:
+        if (ctx.npm_cache_dir / "_cacache").is_dir():
+            add("ok", "npm/_cacache")
+        else:
+            fail("нет npm/_cacache - выполните offdock cache npm", label="npm")
+
+        npm_hash = ctx.npm_cache_dir / ".package-lock.sha256"
+        lock = cfg.frontend_lock_path()
+        assert lock is not None
+        if npm_hash.is_file():
+            current = file_sha256(lock)
+            stored = npm_hash.read_text(encoding="utf-8").strip()
+            if current == stored:
+                add("ok", "npm cache", f"соответствует {cfg.frontend_dir}/{cfg.frontend_lock}")
+            else:
+                fail("package-lock.json изменился - выполните offdock cache npm", label="npm")
+        else:
+            fail("нет npm/.package-lock.sha256 - выполните offdock cache npm", label="npm")
+    else:
+        add("skip", "npm cache", "нет frontend lockfile")
+
+    yarn_specs = discover_yarn_cache_specs(cfg)
+    if yarn_specs:
+        for spec in yarn_specs:
+            cache_dir = ctx.offline_cache / spec.cache_subdir
+            archive_tar = archive_tar_path(cfg, spec)
+            check_dir(cache_dir, spec.cache_subdir)
+            check_dir(archive_tar.parent, spec.archive_subdir)
+            if archive_tar.is_file():
+                add("ok", f"{spec.archive_subdir} archive", archive_tar.name)
+            else:
+                fail(
+                    f"нет {archive_tar.name} - выполните offdock cache yarn",
+                    label=spec.archive_subdir,
+                )
+
+            yarn_hash = cache_dir / f".{spec.archive_subdir}.sha256"
+            if (cache_dir / ".mount-check").is_file():
+                add("ok", f"{spec.cache_subdir}/.mount-check")
+            else:
+                fail(
+                    f"{spec.cache_subdir}/.mount-check отсутствует - выполните offdock cache yarn",
+                    label=spec.cache_subdir,
+                )
+            if yarn_hash.is_file() and archive_tar.is_file():
+                current = file_sha256(archive_tar)
+                stored = yarn_hash.read_text(encoding="utf-8").strip()
+                if current == stored:
+                    add(
+                        "ok",
+                        f"{spec.cache_subdir} hash",
+                        f"соответствует {archive_tar.name}",
+                    )
+                else:
+                    fail("archive изменился - выполните offdock cache yarn", label=spec.cache_subdir)
+            else:
+                fail(
+                    f"нет {spec.cache_subdir}/.{spec.archive_subdir}.sha256 - выполните offdock cache yarn",
+                    label=spec.cache_subdir,
+                )
+            if cache_dir.is_dir():
+                cache_entries = [
+                    p
+                    for p in cache_dir.iterdir()
+                    if p.name not in {".mount-check"} and not p.name.startswith(".")
+                ]
+                v6_dir = cache_dir / "v6"
+                if v6_dir.is_dir():
+                    cache_entries.append(v6_dir)
+                if not cache_entries:
+                    fail(
+                        f"{spec.cache_subdir}/ пуст (нет пакетов) - выполните offdock cache yarn",
+                        label=spec.cache_subdir,
+                    )
+                else:
+                    add("ok", f"{spec.cache_subdir} entries", str(len(cache_entries)))
+    else:
+        add("skip", "yarn cache", "нет source=yarn в *Dockerfile.offline")
+
+    tar_count = len(list(ctx.docker_cache_dir.glob("*.tar"))) if ctx.docker_cache_dir.is_dir() else 0
+    if tar_count >= 1:
+        add("ok", "docker tars", f"{tar_count} файлов")
+    else:
+        add("warn", "docker tars", "нет docker/*.tar - выполните offdock cache images")
+
+    # vendor: explicit critical_vendor and/or source=vendor in Dockerfile.offline
+    vendor_hint = "выполните offdock cache (hooks.after_cache) или cache-vendor"
+    if cfg.critical_vendor:
+        for vp in cfg.critical_vendor:
+            pattern = vp.path.replace("\\", "/")
+            if any(ch in pattern for ch in "*?["):
+                matches = list(ctx.offline_cache.glob(pattern))
+            else:
+                candidate = ctx.offline_cache / pattern
+                matches = [candidate] if candidate.is_file() or candidate.is_dir() else []
+            # files only for globs; dirs ok for exact paths like vendor/
+            ok_matches = [m for m in matches if m.is_file() or (m.is_dir() and any(m.iterdir()))]
+            if ok_matches:
+                add("ok", f"vendor {vp.label}", ok_matches[0].name if ok_matches[0].is_file() else str(ok_matches[0]))
+            else:
+                fail(f"нет {pattern} - {vendor_hint}", label=f"vendor {vp.label}")
+    elif dockerfile_uses_vendor_mount(cfg):
+        vendor_dir = ctx.offline_cache / "vendor"
+        if vendor_dir.is_dir() and any(vendor_dir.iterdir()):
+            add("ok", "vendor", str(vendor_dir))
+        else:
+            fail(f"нет offline-cache/vendor - {vendor_hint}", label="vendor")
+    else:
+        add("skip", "vendor", "нет source=vendor / critical_vendor")
+
+    return report

offline_docker/sync_cache.py

@@ -0,0 +1,125 @@
+"""Materialize / publish offline-cache trees from remote raw/universal feeds."""
+
+from __future__ import annotations
+
+from pathlib import Path
+
+from offline_docker.context import ProjectContext
+from offline_docker.resolve import PackageResolver
+from offline_docker.sources.nexus import NexusSource
+from offline_docker.sources.nexus_client import download_file, join_url, upload_file
+from offline_docker.sources.proget import ProGetSource
+from offline_docker.sources.proget_client import download_asset, upload_asset
+
+# Marker files / dirs commonly present after download-*
+_SYNC_REL_PATHS = (
+    "wheels/.mount-check",
+    "wheels/.deps.sha256",
+    "npm/.package-lock.sha256",
+    "apt-meta/.ready",
+)
+
+
+def materialize_from_remotes(ctx: ProjectContext) -> None:
+    """Best-effort pull of known marker/meta files from Nexus raw / ProGet universal."""
+    resolver = ctx.config.package_resolver()
+    if resolver.is_local_only():
+        return
+
+    offline = ctx.offline_cache
+    offline.mkdir(parents=True, exist_ok=True)
+
+    for src in resolver.nexus_sources():
+        base = src.raw_base_url()
+        if not base:
+            continue
+        print(f">>> materialize from Nexus raw [{src.name}]: {base}")
+        for rel in _SYNC_REL_PATHS:
+            url = join_url(base, rel)
+            dest = offline / rel
+            try:
+                download_file(url, dest, auth_env=src.auth_env)
+                print(f"    OK {rel}")
+            except Exception as exc:  # noqa: BLE001 — best-effort sync
+                print(f"    skip {rel}: {exc}")
+
+    for src in resolver.proget_sources():
+        feed = src.universal_feed()
+        if not feed:
+            continue
+        print(f">>> materialize from ProGet universal [{src.name}/{feed}]")
+        for rel in _SYNC_REL_PATHS:
+            dest = offline / rel
+            try:
+                download_asset(src.url, feed, rel, dest, auth_env=src.auth_env)
+                print(f"    OK {rel}")
+            except FileNotFoundError:
+                print(f"    skip {rel}: not found")
+            except Exception as exc:  # noqa: BLE001
+                print(f"    skip {rel}: {exc}")
+
+
+def publish_to_remotes(ctx: ProjectContext) -> None:
+    """Upload local offline-cache files to remotes with publish=true."""
+    resolver = ctx.config.package_resolver()
+    offline = ctx.offline_cache
+    if not offline.is_dir():
+        return
+
+    for src in resolver.nexus_sources():
+        if not getattr(src, "publish", False):
+            continue
+        base = src.raw_base_url()
+        if not base:
+            print(f"WARN: nexus [{src.name}] publish=true but raw repo not set")
+            continue
+        print(f">>> publish to Nexus raw [{src.name}]")
+        _upload_tree_nexus(offline, base, src.auth_env)
+
+    for src in resolver.proget_sources():
+        if not getattr(src, "publish", False):
+            continue
+        feed = src.universal_feed()
+        if not feed:
+            print(f"WARN: proget [{src.name}] publish=true but universal feed not set")
+            continue
+        print(f">>> publish to ProGet [{src.name}/{feed}]")
+        _upload_tree_proget(offline, src.url, feed, src.auth_env)
+
+
+def _upload_tree_nexus(root: Path, base: str, auth_env: str) -> None:
+    for path in root.rglob("*"):
+        if not path.is_file():
+            continue
+        rel = path.relative_to(root).as_posix()
+        url = join_url(base, rel)
+        try:
+            upload_file(url, path, auth_env=auth_env)
+            print(f"    uploaded {rel}")
+        except Exception as exc:  # noqa: BLE001
+            print(f"    FAIL {rel}: {exc}")
+
+
+def _upload_tree_proget(root: Path, base_url: str, feed: str, auth_env: str) -> None:
+    for path in root.rglob("*"):
+        if not path.is_file():
+            continue
+        rel = path.relative_to(root).as_posix()
+        try:
+            upload_asset(base_url, feed, rel, path, auth_env=auth_env)
+            print(f"    uploaded {rel}")
+        except Exception as exc:  # noqa: BLE001
+            print(f"    FAIL {rel}: {exc}")
+
+
+def maybe_materialize(ctx: ProjectContext) -> None:
+    resolver: PackageResolver = ctx.config.package_resolver()
+    if resolver.has_remote_indexes() or any(
+        isinstance(s, NexusSource) and s.raw_base_url() for s in resolver.nexus_sources()
+    ) or any(
+        isinstance(s, ProGetSource) and s.universal_feed() for s in resolver.proget_sources()
+    ):
+        try:
+            materialize_from_remotes(ctx)
+        except Exception as exc:  # noqa: BLE001
+            print(f"WARN: materialize skipped: {exc}")

offline_docker/trivy_scan.py

@@ -0,0 +1,86 @@
+"""Offline Trivy image scanning."""
+
+from __future__ import annotations
+
+import shutil
+
+from offline_docker import docker_cmd
+from offline_docker.context import ProjectContext
+from offline_docker.download_trivy import download_trivy_db, ensure_trivy_image, trivy_db_stale
+from offline_docker.utils import docker_mount_path
+
+
+def scan_trivy_offline(
+    ctx: ProjectContext,
+    target: str,
+    image: str | None = None,
+) -> None:
+    if target not in ("backend", "frontend"):
+        raise SystemExit(f"ERROR: target должен быть backend или frontend, получено: {target}")
+
+    if image is None:
+        image = (
+            ctx.config.images.backend
+            if target == "backend"
+            else ctx.config.images.frontend
+        )
+
+    if not docker_cmd.image_exists(image):
+        raise SystemExit(f"ERROR: образ {image} не найден локально")
+
+    if trivy_db_stale(ctx):
+        print("БД Trivy устарела или отсутствует — offdock cache trivy")
+        try:
+            download_trivy_db(ctx)
+        except SystemExit:
+            raise
+        except Exception as exc:
+            raise SystemExit(
+                "ERROR: не удалось обновить БД Trivy. "
+                "Выполните offdock cache trivy на машине с интернетом."
+            ) from exc
+
+    ensure_trivy_image(ctx)
+    log_file = ctx.trivy_scan_log(target)
+    cache_docker = docker_mount_path(ctx.trivy_cache_dir)
+    trivy = ctx.config.trivy
+
+    fanal = ctx.trivy_cache_dir / "fanal"
+    if fanal.exists():
+        shutil.rmtree(fanal, ignore_errors=True)
+
+    print("=== scan-trivy-offline ===")
+    print(f"Target:   {target}")
+    print(f"Image:    {image}")
+    print(f"Severity: {trivy.severity}")
+    print(f"Scanners: {trivy.scanners}")
+    print(f"Лог:      {log_file}")
+    print()
+
+    docker_cmd.run_tee(
+        [
+            "docker",
+            "run",
+            "--rm",
+            "-v",
+            "/var/run/docker.sock:/var/run/docker.sock",
+            "-v",
+            f"{cache_docker}:/root/.cache/trivy",
+            trivy.scanner_image,
+            "image",
+            "--skip-db-update",
+            "--scanners",
+            trivy.scanners,
+            "--severity",
+            trivy.severity,
+            "--exit-code",
+            "1",
+            "--ignore-unfixed",
+            image,
+        ],
+        log_file,
+        env={"MSYS_NO_PATHCONV": "1"},
+    )
+
+    print()
+    print(f"=== scan-trivy-offline: OK (лог: {log_file}) ===")

offline_docker/tui/__init__.py

@@ -0,0 +1,19 @@
+"""Optional Textual TUI for offline-docker."""
+
+from __future__ import annotations
+
+from pathlib import Path
+
+
+def run_tui(
+    project: Path | None = None,
+    *,
+    config_file: Path | None = None,
+) -> int:
+    """Launch Textual TUI. Returns process exit code."""
+    from offline_docker.tui.app import OfflineDockerApp
+
+    root = (project or Path.cwd()).resolve()
+    app = OfflineDockerApp(project=root, config_file=config_file)
+    app.run()
+    return 0

offline_docker/tui/app.py

@@ -0,0 +1,56 @@
+"""Textual application entry."""
+
+from __future__ import annotations
+
+from pathlib import Path
+
+from textual.app import App
+
+from offline_docker.context import ProjectContext
+from offline_docker.tui.screens import HomeScreen
+
+
+class OfflineDockerApp(App[None]):
+    """Interactive TUI for offline-docker actions."""
+
+    TITLE = "offdock"
+    CSS = """
+    Screen {
+        background: $surface;
+    }
+    .title {
+        text-style: bold;
+        margin: 1 1 0 1;
+    }
+    #discover-modal {
+        width: 90%;
+        height: 90%;
+        border: thick $primary;
+        background: $surface;
+        padding: 1;
+    }
+    #discover-log {
+        height: 1fr;
+        margin-bottom: 1;
+    }
+    """
+
+    def __init__(
+        self,
+        project: Path,
+        *,
+        config_file: Path | None = None,
+    ) -> None:
+        super().__init__()
+        self.project = project.resolve()
+        self.config_file = config_file
+        self.ctx = ProjectContext.load(self.project, config_file=config_file)
+
+    def set_project(self, project: Path, *, config_file: Path | None = None) -> None:
+        self.project = project.resolve()
+        if config_file is not None:
+            self.config_file = config_file
+        self.ctx = ProjectContext.load(self.project, config_file=self.config_file)
+
+    def on_mount(self) -> None:
+        self.push_screen(HomeScreen())