fc-data 0.2.0__py3-none-any.whl

datasmith/docker/images.py

@@ -0,0 +1,158 @@
+"""Docker image management via python-on-whales."""
+
+from __future__ import annotations
+
+import os
+from pathlib import Path
+
+from python_on_whales import DockerClient
+
+from datasmith.utils import get_logger
+from datasmith.utils.core import Settings
+
+logger = get_logger("docker.images")
+
+_TEMPLATES_DIR = Path(__file__).parent / "templates"
+
+# The three-tier hierarchy (base -> repo -> PR) requires each image to be
+# available locally for the next FROM.  The built-in "default" builder uses
+# the docker driver, which builds directly in the daemon's image store.
+# Container-based builders (docker-container) run in isolation and cannot
+# resolve locally-built images, so we pin to the docker driver here.
+_BUILDER = "default"
+
+
+def _default_context() -> str:
+    """Return the path to the built-in templates directory."""
+    return str(_TEMPLATES_DIR)
+
+
+def _docker_namespace() -> str:
+    """Return the Docker namespace from settings (DOCKERHUB_USERNAME env var)."""
+    return Settings().dockerhub_username
+
+
+def get_base_image_name() -> str:
+    """Return the canonical tag for the base image."""
+    return f"{_docker_namespace()}/base:latest"
+
+
+def get_repo_image_name(owner: str, repo: str) -> str:
+    """Return the canonical tag for a repository image."""
+    owner = owner.lower()
+    repo = repo.lower()
+    return f"{_docker_namespace()}/{owner}-{repo}:latest".lower()
+
+
+def get_pr_image_name(owner: str, repo: str, issue_number: int) -> str:
+    """Return the canonical tag for a PR image."""
+    return f"{_docker_namespace()}/{owner}-{repo}:{issue_number}".lower()
+
+
+class ImageManager:
+    def __init__(self, timeout: int = 3600) -> None:
+        self._docker = DockerClient()
+        self._timeout = timeout
+
+    @staticmethod
+    def _default_context() -> str:
+        """Return the path to the built-in templates directory."""
+        return _default_context()
+
+    def build_base_image(
+        self,
+        context: str | None = None,
+        *,
+        py_version: str = "",
+    ) -> str:
+        ctx = context or _default_context()
+        tag = get_base_image_name()
+        logger.info("Building base image: %s", tag)
+        kwargs: dict[str, object] = {
+            "tags": [tag],
+            "file": os.path.join(ctx, "Dockerfile.base"),
+            "builder": _BUILDER,
+        }
+        if py_version:
+            kwargs["build_args"] = {"PY_VERSION": py_version}
+        self._docker.build(ctx, **kwargs)  # type: ignore[arg-type]
+        return tag
+
+    def build_repo_image(
+        self,
+        owner: str,
+        repo: str,
+        context: str | None = None,
+        *,
+        repo_url: str | None = None,
+        py_version: str = "",
+    ) -> str:
+        ctx = context or _default_context()
+        url = repo_url or f"https://github.com/{owner}/{repo}.git"
+        tag = get_repo_image_name(owner, repo)
+        logger.info("Building repo image: %s", tag)
+        build_args: dict[str, str] = {
+            "BASE_IMAGE": get_base_image_name(),
+            "REPO_URL": url,
+        }
+        if py_version:
+            build_args["PY_VERSION"] = py_version
+        self._docker.build(
+            ctx,
+            tags=[tag],
+            file=os.path.join(ctx, "Dockerfile.repo"),
+            build_args=build_args,
+            builder=_BUILDER,
+        )
+        return tag
+
+    def build_pr_image(
+        self,
+        owner: str,
+        repo: str,
+        issue_number: int,
+        context: str | None = None,
+        build_script: str = "",
+        *,
+        commit_sha: str = "HEAD",
+        env_payload: str = "[]",
+        py_version: str = "",
+    ) -> str:
+        ctx = context or _default_context()
+        tag = get_pr_image_name(owner, repo, issue_number)
+        repo_image = get_repo_image_name(owner, repo)
+        logger.info("Building PR image: %s", tag)
+        build_args: dict[str, str] = {
+            "REPO_IMAGE": repo_image,
+            "COMMIT_SHA": commit_sha,
+            "ENV_PAYLOAD": env_payload,
+        }
+        if build_script:
+            build_args["BUILD_SCRIPT"] = build_script
+        if py_version:
+            build_args["PY_VERSION"] = py_version
+        self._docker.build(
+            ctx,
+            tags=[tag],
+            file=os.path.join(ctx, "Dockerfile.pr"),
+            build_args=build_args,
+            builder=_BUILDER,
+        )
+        return tag
+
+    def image_exists(self, tag: str) -> bool:
+        try:
+            self._docker.image.inspect(tag)
+        except Exception:
+            return False
+        else:
+            return True
+
+    def remove_image(self, tag: str) -> None:
+        try:
+            self._docker.image.remove(tag, force=True)
+        except Exception:
+            logger.warning("Failed to remove image: %s", tag)
+
+    def prune_dangling(self) -> None:
+        self._docker.image.prune(all=False)

datasmith/docker/publish.py

@@ -0,0 +1,56 @@
+"""DockerHub publisher."""
+
+from __future__ import annotations
+
+import os
+from datetime import datetime, timezone
+
+import httpx
+from python_on_whales import DockerClient
+
+from datasmith.utils import get_logger, with_backoff
+
+logger = get_logger("docker.publish")
+
+
+class DockerHubPublisher:
+    def __init__(self, namespace: str = "formulacode") -> None:
+        self._docker = DockerClient()
+        self._namespace = namespace
+        self._logged_in = False
+
+    def _login(self) -> None:
+        if self._logged_in:
+            return
+        username = os.environ.get("DOCKERHUB_USERNAME", "")
+        token = os.environ.get("DOCKERHUB_TOKEN", "")
+        if username and token:
+            self._docker.login(username=username, password=token)
+            self._logged_in = True
+
+    @with_backoff(max_retries=3, base_delay=2.0)
+    def push(self, image_tag: str) -> None:
+        self._login()
+        logger.info("Pushing image: %s", image_tag)
+        self._docker.push(image_tag)
+
+    def tag_with_version(self, image_tag: str) -> str:
+        version = datetime.now(tz=timezone.utc).strftime("@%Y-%m")
+        new_tag = f"{image_tag}{version}"
+        self._docker.tag(image_tag, new_tag)
+        return new_tag
+
+    def list_remote_tags(self, repo: str) -> list[str]:
+        """List tags for a DockerHub repository."""
+        url = f"https://hub.docker.com/v2/repositories/{self._namespace}/{repo}/tags/"
+        try:
+            resp = httpx.get(url, timeout=10.0)
+            if resp.status_code == 200:
+                return [t["name"] for t in resp.json().get("results", [])]
+        except Exception:
+            logger.warning("Failed to list remote tags for %s", repo)
+        return []
+
+    def filter_unpublished(self, local_tags: list[str], remote_tags: list[str]) -> list[str]:
+        remote_set = set(remote_tags)
+        return [t for t in local_tags if t.split(":")[-1] not in remote_set]

datasmith/docker/templates/Dockerfile.base

@@ -0,0 +1,26 @@
+# syntax=docker/dockerfile:1.7
+
+ARG PY_VERSION=""
+FROM buildpack-deps:jammy AS base
+ARG PY_VERSION=""
+
+ENV DEBIAN_FRONTEND=noninteractive \
+    MAMBA_ROOT_PREFIX=/opt/conda \
+    PATH=/opt/conda/bin:/root/.cargo/bin:/root/.local/bin:$PATH \
+    MAMBA_DOCKERFILE_ACTIVATE=1 \
+    OPENBLAS_NUM_THREADS=1 \
+    MKL_NUM_THREADS=1 \
+    OMP_NUM_THREADS=1
+
+RUN mkdir -p /workspace /output
+WORKDIR /workspace
+
+COPY docker_build_base.sh /workspace/docker_build_base.sh
+RUN chmod +x /workspace/docker_build_base.sh && \
+    if [ -n "${PY_VERSION}" ]; then \
+        PY_VERSION="${PY_VERSION}" /workspace/docker_build_base.sh --py-version "${PY_VERSION}"; \
+    else \
+        /workspace/docker_build_base.sh; \
+    fi
+
+RUN micromamba clean --all --yes

datasmith/docker/templates/Dockerfile.pr

@@ -0,0 +1,42 @@
+# syntax=docker/dockerfile:1.7
+
+ARG REPO_IMAGE
+
+FROM ${REPO_IMAGE} AS env
+ARG COMMIT_SHA
+ARG ENV_PAYLOAD="[]"
+ARG PY_VERSION=""
+RUN git checkout "$COMMIT_SHA"
+LABEL vcs.ref="$COMMIT_SHA"
+
+COPY docker_build_env.sh /workspace/repo/docker_build_env.sh
+RUN chmod +x /workspace/repo/docker_build_env.sh && \
+    echo "${ENV_PAYLOAD}" > /tmp/env_payload.json && \
+    /workspace/repo/docker_build_env.sh --env-payload /tmp/env_payload.json
+
+FROM env AS pkg
+
+COPY docker_build_pkg.sh /workspace/repo/docker_build_pkg.sh
+RUN chmod +x /workspace/repo/docker_build_pkg.sh && \
+    /workspace/repo/docker_build_pkg.sh
+
+COPY profile.sh /profile.sh
+COPY run-tests.sh /run-tests.sh
+RUN chmod +x /profile.sh /run-tests.sh
+
+RUN micromamba clean --all --yes
+
+FROM pkg AS run
+
+COPY docker_build_run.sh /docker_build_run.sh
+RUN chmod +x /docker_build_run.sh \
+    && /docker_build_run.sh
+
+FROM run AS final
+ARG BENCHMARKS=""
+ARG BUILD_SCRIPT=""
+
+COPY docker_build_final.sh /docker_build_final.sh
+RUN chmod +x /docker_build_final.sh \
+    && printf "%s\n" "${BENCHMARKS}" > /tmp/asv_benchmarks_fallback.txt; \
+    /docker_build_final.sh /tmp/asv_benchmarks_fallback.txt;

datasmith/docker/templates/Dockerfile.repo

@@ -0,0 +1,11 @@
+# syntax=docker/dockerfile:1.7
+
+ARG BASE_IMAGE=formulacode/base:latest
+
+FROM ${BASE_IMAGE} AS repo
+ARG REPO_URL
+ARG COMMIT_SHA=""
+RUN git clone "$REPO_URL" /workspace/repo && \
+    if [ -n "$COMMIT_SHA" ]; then cd /workspace/repo && git checkout "$COMMIT_SHA"; fi
+WORKDIR /workspace/repo
+ENTRYPOINT ["/bin/bash"]