coffea-casa 2023.12.12.post1__tar.gz → 2024.8.7.post0__tar.gz

{coffea_casa-2023.12.12.post1 → coffea_casa-2024.8.7.post0}/.github/workflows/dockercd-dev.yml

@@ -20,7 +20,8 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        image: [cc-base-ubuntu, cc-analysis-ubuntu]
+        #image: [cc-base-ubuntu, cc-analysis-ubuntu, cc-dask-alma8, cc-analysis-alma8, cc-dask-alma9, cc-analysis-alma9]
+        image: [cc-dask-alma9, cc-analysis-alma9]
         # Keep this line in sync with gh actions @ coffea-dask repo
         # python: [3.8, 3.9, '3.10']
         #exclude:

{coffea_casa-2023.12.12.post1 → coffea_casa-2024.8.7.post0}/.github/workflows/dockerci-dev.yml

@@ -18,7 +18,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        image: [cc-base-ubuntu, cc-analysis-ubuntu]
+        image: [cc-dask-alma9, cc-analysis-alma9]
         # Keep this line in sync with gh actions @ coffea-dask repo
         # python: [3.8, 3.9, '3.10']
         #exclude:

{coffea_casa-2023.12.12.post1 → coffea_casa-2024.8.7.post0}/PKG-INFO

@@ -1,6 +1,6 @@
-Metadata-Version: 2.1
+Metadata-Version: 2.3
 Name: coffea-casa
-Version: 2023.12.12.post1
+Version: 2024.8.7.post0
 Summary: Wrappers for Dask clusters to be used from coffea-casa AF
 Project-URL: Homepage, https://github.com/CoffeaTeam/coffea-casa
 Project-URL: Documentation, https://coffea-casa.readthedocs.io/en/latest/

{coffea_casa-2023.12.12.post1 → coffea_casa-2024.8.7.post0}/coffea_casa/_version.py

@@ -12,5 +12,5 @@ __version__: str
 __version_tuple__: VERSION_TUPLE
 version_tuple: VERSION_TUPLE
 
-__version__ = version = '2023.12.12.post1'
-__version_tuple__ = version_tuple = (2023, 12, 12)
+__version__ = version = '2024.8.7.post0'
+__version_tuple__ = version_tuple = (2024, 8, 7)

{coffea_casa-2023.12.12.post1 → coffea_casa-2024.8.7.post0}/coffea_casa/coffea_casa.py

@@ -205,7 +205,9 @@ class CoffeaCasaCluster(HTCondorCluster):
             {"should_transfer_files": "YES"},
             {"Stream_Output": "False"},
             {"Stream_Error": "False"},
+            {"+CoffeaCasaWorkerType": "dask"},
             {"+DaskSchedulerAddress": external_ip_string},
+            {"+AccountingGroup": "cms.other.coffea.$ENV(HOSTNAME)"},
             job_kwargs.get(
                 "job_extra_directives", dask.config.get(f"jobqueue.{cls.config_name}.job_extra_directives")
             ),

{coffea_casa-2023.12.12.post1 → coffea_casa-2024.8.7.post0}/coffea_casa/jobqueue-coffea-casa.yaml

@@ -3,11 +3,12 @@ jobqueue:
     name: dask-worker
     # Dask worker options, taken from https://github.com/dask/dask-jobqueue/tree/master/dask_jobqueue
     cores: 1                 # Total number of cores per job
-    memory: "2GiB"                # Total amount of memory per job
+    memory: "3GiB"                # Total amount of memory per job
     processes: 1                # Number of Python processes per jobs
     worker-image: "hub.opensciencegrid.org/coffea-casa/cc-analysis-ubuntu:development"
 
     # Comunication settings
+    python: null                # Python executable
     interface: null             # Network interface to use like eth0 or ib0
     death-timeout: 60           # Number of seconds to wait if a worker can not find a scheduler
     local-directory: null       # Location of fast local storage like /scratch or $TMPDIR

coffea_casa-2024.8.7.post0/docker/Dockerfile.cc-analysis-alma8

@@ -0,0 +1,221 @@
+#FROM coffeateam/coffea-base-almalinux8:0.7.22-py3.10
+FROM coffeateam/coffea-dask-almalinux8:latest-py3.10
+
+USER root
+LABEL maintainer="Oksana Shadura <ksu.shadura@gmail.com>"
+# Jupyterhub
+ARG TAG="development"
+ARG NB_USER="cms-jovyan"
+ARG NB_UID="6440"
+ARG NB_GID="11265"
+ARG CERT_DIR="/etc/cmsaf-secrets"
+# Hack for GH Actions
+ARG GITHUB_ACTIONS="false"
+# FIX ME AFTER TEST:
+#ARG BEARER_TOKEN_FILE="/tmp/.xcache/access_token"
+ARG BEARER_TOKEN_FILE="/tmp/.xcache/access_token"
+ARG SEC_TOKEN_SYSTEM_DIRECTORY="/tmp/.condor"
+ARG XCACHE_HOST="xcache"
+#ARG XCACHE_HOST="red-xcache1.unl.edu"
+
+
+# Configure environment
+ENV CONDA_DIR /usr/local
+ENV CERT_DIR $CERT_DIR
+ENV XCACHE_HOST $XCACHE_HOST
+ENV BEARER_TOKEN_FILE $BEARER_TOKEN_FILE
+ENV SEC_TOKEN_SYSTEM_DIRECTORY $SEC_TOKEN_SYSTEM_DIRECTORY
+ENV SHELL /bin/bash
+ENV NB_USER $NB_USER
+ENV USER $NB_USER
+ENV NB_UID $NB_UID
+ENV NB_GID $NB_GID
+ENV HOME /home/$NB_USER
+ENV PATH "${CONDA_DIR}/bin/:$PATH"
+ENV LC_ALL en_US.UTF-8
+ENV LANG en_US.UTF-8
+ENV LANGUAGE en_US.UTF-8
+
+# Install all OS dependencies for notebook server that starts but lacks all
+# features (e.g., download as all possible file formats)
+RUN yum -y update \
+ && yum -y group install "Development Tools" \
+ && yum -y install \
+    wget \
+    epel-release \
+    gettext \
+    bzip2 \
+    ca-certificates \
+    sudo \
+    langpacks-en \
+    glibc-langpack-en \
+    glibc-all-langpacks \
+    liberation-fonts \
+    wget \
+    nss_wrapper \
+    openssl \
+    emacs \
+    git \
+    unzip \
+    nano \
+    vim  \
+    curl \
+    net-tools \
+    libXext \
+    libSM \
+    libXrender \
+    nc \
+    openssh-clients \
+    tzdata \
+    unzip \
+    gdb \
+    && yum clean all && rm -rf /var/cache/yum
+
+ENV TINI_VERSION v0.19.0
+ADD https://github.com/krallin/tini/releases/download/${TINI_VERSION}/tini-static-amd64 /usr/bin/tini
+RUN chmod +x /usr/bin/tini
+
+# Copy a script that we will use to correct permissions after running certain commands
+COPY jupyterhub/fix-permissions /usr/local/bin/fix-permissions
+RUN chmod a+rx /usr/local/bin/fix-permissions
+
+RUN rm -rf /etc/grid-security && \
+    ln -s /usr/local/etc/grid-security /etc/grid-security && \
+    chmod 755 /etc/grid-security/certificates && \
+    chmod g-w /etc/grid-security/certificates
+
+# Enable prompt color in the skeleton .bashrc before creating the default NB_USER
+# hadolint ignore=SC2016
+RUN sed -i 's/^#force_color_prompt=yes/force_color_prompt=yes/' /etc/skel/.bashrc && \
+   # Add call to conda init script see https://stackoverflow.com/a/58081608/4413446
+   echo 'eval "$(command conda shell.bash hook 2> /dev/null)"' >> /etc/skel/.bashrc
+
+# Create NB_USER with name jovyan user with given UID and in the 'users' group
+# and make sure these dirs are writable by the `users` group.
+RUN echo "auth requisite pam_deny.so" >> /etc/pam.d/su && \
+    sed -i.bak -e 's/^%admin/#%admin/' /etc/sudoers && \
+    sed -i.bak -e 's/^%sudo/#%sudo/' /etc/sudoers && \
+    groupadd -r "${NB_USER}" --gid "$NB_GID" && \
+    useradd -l -m -s /bin/bash -N -u "${NB_UID}" "${NB_USER}" && \
+    mkdir -p "${CONDA_DIR}" && \
+    mkdir -p "${CERT_DIR}" && \
+    mkdir -p /var/lib/condor && \
+    chown -R "${NB_USER}:${NB_GID}" "${CONDA_DIR}" && \
+    chown -R "${NB_USER}:${NB_GID}" "${HOME}" && \
+    chmod g+w /etc/passwd && \
+    fix-permissions "${HOME}" && \
+    fix-permissions "${CONDA_DIR}"
+
+USER $NB_USER
+WORKDIR $HOME
+# Setup work directory for backward-compatibility
+RUN mkdir $HOME/work && \
+    mkdir -p $HOME/.local && \
+    mkdir -p $HOME/.condor/tokens.d && \
+    fix-permissions $HOME
+
+RUN mamba remove htcondor
+
+#Dask dependencies and HTCondor
+RUN mamba install --yes \
+    -c conda-forge \
+    htcondor==10.8.0 \
+    xgboost \
+    pyhf \
+    xrootd \
+    scipy>=1.8.1 \
+    cabinetry \
+    vector \
+    hist \
+    mplhep \
+    iminuit \
+    cmake \
+    ndcctools \
+    && mamba clean \
+        --all \
+        --force-pkgs-dirs \
+        --yes
+
+RUN pip install --no-cache-dir \
+    aiostream \
+    supervisor \
+    correctionlib \
+    funcx \
+    pyyaml \
+    # ML packages
+    dask-ml \
+    dask-gateway \
+    prometheus_client \
+    comm>=0.1.2 \
+    mlflow
+
+#RUN if [ "${TAG:-}" == "development" ]; then pip install --no-cache-dir git+https://github.com/CoffeaTeam/coffea-casa.git#egg=coffea_casa ; else pip install --no-cache-dir coffea_casa -U ; fi
+RUN pip install --no-cache-dir git+https://github.com/CoffeaTeam/coffea-casa.git#egg=coffea_casa
+
+# ------- xrootd-xcache-plugin -------------------------------
+RUN cd /tmp && \
+    git clone -b xcache https://github.com/jthiltges/xrdcl-authz-plugin.git && \
+    cd xrdcl-authz-plugin && \
+    mkdir build && \
+    cd  build && \
+    cmake /tmp/xrdcl-authz-plugin -DCMAKE_INSTALL_PREFIX=${CONDA_DIR} && \
+    make && \
+    make install
+
+ENV XRD_PLUGINCONFDIR="${CONDA_DIR}/etc/xrootd/client.plugins.d/"
+ENV XRD_PLUGIN="${CONDA_DIR}/lib/libXrdClXcachePlugin-5.so"
+
+# Include additional CA certificates beyond ca-policy-lcg
+COPY certs/* /etc/grid-security/certificates/
+RUN openssl rehash /etc/grid-security/certificates/
+
+# TODO: RETEST IF WE STILL NEED THIS 
+ENV LD_LIBRARY_PATH="${CONDA_DIR}/lib/:$LD_LIBRARY_PATH"
+ENV PATH="${CONDA_DIR}/bin/:$PATH"
+
+USER root
+# Setup supervisord files
+COPY k8s-worker/supervisord.conf  /etc/supervisor/
+# Setup HTCondor user/group and change group for user $NB_USER
+# Fix error (submitting jobs as user/group 0 (root) is not allowed for security reasons) and
+# it configured from kubernetes side and updated in docker container to match it
+RUN groupadd -r condor && \
+    useradd -r -g condor -d /var/lib/condor -s /sbin/nologin condor
+
+# FIXME: merge PRs open in distributed.git (oshadura)
+# Distributed: we need to install patched version of distributed version
+COPY dask/distributed ${CONDA_DIR}/lib/python3.10/site-packages/distributed
+RUN cd ${CONDA_DIR}/lib/python3.10/site-packages/distributed && \
+    patch -p2 < 0001-Patch-from-bbockelman-adaptive-scaling.patch && \
+    patch -p2 < 0002-Allow-scheduler-to-preserve-worker-hostnames.patch
+    # && patch -p2 < 0003-Activate-patch.patch
+    # && patch -p2 < 0004-Add-possibility-to-setup-external_adress-for-schedul.patch
+    # && patch -p2 < 0005-Add-nanny-patch.patch
+
+# FIXME: we have a wrong path, let's make a link.
+# cms-jovyan@jupyter-oksana-2eshadura-40cern-2ech:~$ echo $PATH
+# ${CONDA_DIR}/condabin:/usr/local/bin:/usr/bin:/bin:/usr/local/games:/usr/games
+RUN rm -rf ${CONDA_DIR}/condabin && ln -s ${CONDA_DIR}/bin ${CONDA_DIR}/condabin
+
+# Cleanup
+RUN rm -rf /tmp/* \
+    && rm -rf $HOME/.cache/.pip/* \
+    && mamba clean --all -f -y \
+    && find ${CONDA_DIR} -type f -name '*.a' -delete \
+    && find ${CONDA_DIR} -type f -name '*.pyc' -delete \
+    && find ${CONDA_DIR} -type f -name '*.js.map' -delete \
+    && (find ${CONDA_DIR}/lib/python3.10/site-packages/bokeh/server/static -type f,l -name '*.js' -not -name '*.min.js' -delete || echo "no bokeh static files to cleanup") \
+    && rm -rf ${CONDA_DIR}/pkgs
+
+# Mkdir CVMFS mount directory.  Mount done upstream in configs. Harmless
+# if not utilized, crucial if CVMFS is desired
+RUN mkdir /cvmfs
+
+# Prepare HTCondor spec. environment and execute dask-worker command
+# If we using this container as a sidecar, we don't setup any HTCondor spec. environment
+# nor and execute dask-worker command
+ADD prepare-env/prepare-env-cc-analysis.sh /usr/local/bin/prepare-env.sh
+RUN chmod ugo+x /usr/local/bin/prepare-env.sh
+
+USER $NB_USER
+ENTRYPOINT ["tini", "-g", "--", "/usr/local/bin/prepare-env.sh"]

coffea_casa-2024.8.7.post0/docker/Dockerfile.cc-analysis-alma9

@@ -0,0 +1,217 @@
+#FROM coffeateam/coffea-base-almalinux8:0.7.22-py3.10
+FROM coffeateam/coffea-dask-almalinux9:latest-py3.10
+
+
+USER root
+LABEL maintainer="Oksana Shadura <ksu.shadura@gmail.com>"
+# Jupyterhub
+ARG TAG="development"
+ARG NB_USER="cms-jovyan"
+ARG NB_UID="6440"
+ARG NB_GID="11265"
+ARG CERT_DIR="/etc/cmsaf-secrets"
+# Hack for GH Actions
+ARG GITHUB_ACTIONS="false"
+# FIX ME AFTER TEST:
+#ARG BEARER_TOKEN_FILE="/tmp/.xcache/access_token"
+ARG BEARER_TOKEN_FILE="/tmp/.xcache/access_token"
+ARG SEC_TOKEN_SYSTEM_DIRECTORY="/tmp/.condor"
+ARG XCACHE_HOST="xcache.cmsaf-dev.flatiron.hollandhpc.org"
+#ARG XCACHE_HOST="red-xcache1.unl.edu"
+
+# Configure environment
+ENV CONDA_DIR /usr/local
+ENV CERT_DIR $CERT_DIR
+ENV XCACHE_HOST $XCACHE_HOST
+ENV BEARER_TOKEN_FILE $BEARER_TOKEN_FILE
+ENV SEC_TOKEN_SYSTEM_DIRECTORY $SEC_TOKEN_SYSTEM_DIRECTORY
+ENV SHELL /bin/bash
+ENV NB_USER $NB_USER
+ENV USER $NB_USER
+ENV NB_UID $NB_UID
+ENV NB_GID $NB_GID
+ENV HOME /home/$NB_USER
+ENV PATH "${CONDA_DIR}/bin/:$PATH"
+ENV LC_ALL en_US.UTF-8
+ENV LANG en_US.UTF-8
+ENV LANGUAGE en_US.UTF-8
+
+# Install all OS dependencies for notebook server that starts but lacks all
+# features (e.g., download as all possible file formats)
+RUN yum -y update \
+ && yum -y group install "Development Tools" \
+ && yum -y install \
+    wget \
+    epel-release \
+    gettext \
+    bzip2 \
+    ca-certificates \
+    sudo \
+    langpacks-en \
+    glibc-langpack-en \
+    glibc-all-langpacks \
+    liberation-fonts \
+    wget \
+    nss_wrapper \
+    openssl \
+    emacs \
+    git \
+    unzip \
+    nano \
+    vim  \
+    net-tools \
+    libXext \
+    libSM \
+    libXrender \
+    nc \
+    openssh-clients \
+    tzdata \
+    unzip \
+    gdb \
+    && yum clean all && rm -rf /var/cache/yum
+
+ENV TINI_VERSION v0.19.0
+ADD https://github.com/krallin/tini/releases/download/${TINI_VERSION}/tini-static-amd64 /usr/bin/tini
+RUN chmod +x /usr/bin/tini
+
+# Copy a script that we will use to correct permissions after running certain commands
+COPY jupyterhub/fix-permissions /usr/local/bin/fix-permissions
+RUN chmod a+rx /usr/local/bin/fix-permissions
+
+RUN rm -rf /etc/grid-security && \
+    ln -s /usr/local/etc/grid-security /etc/grid-security && \
+    chmod 755 /etc/grid-security/certificates && \
+    chmod g-w /etc/grid-security/certificates
+
+# Enable prompt color in the skeleton .bashrc before creating the default NB_USER
+# hadolint ignore=SC2016
+RUN sed -i 's/^#force_color_prompt=yes/force_color_prompt=yes/' /etc/skel/.bashrc && \
+   # Add call to conda init script see https://stackoverflow.com/a/58081608/4413446
+   echo 'eval "$(command conda shell.bash hook 2> /dev/null)"' >> /etc/skel/.bashrc
+
+# Create NB_USER with name jovyan user with given UID and in the 'users' group
+# and make sure these dirs are writable by the `users` group.
+RUN echo "auth requisite pam_deny.so" >> /etc/pam.d/su && \
+    sed -i.bak -e 's/^%admin/#%admin/' /etc/sudoers && \
+    sed -i.bak -e 's/^%sudo/#%sudo/' /etc/sudoers && \
+    groupadd -r "${NB_USER}" --gid "$NB_GID" && \
+    useradd -l -m -s /bin/bash -N -u "${NB_UID}" "${NB_USER}" && \
+    mkdir -p "${CONDA_DIR}" && \
+    mkdir -p "${CERT_DIR}" && \
+    mkdir -p /var/lib/condor && \
+    chown -R "${NB_USER}:${NB_GID}" "${CONDA_DIR}" && \
+    chown -R "${NB_USER}:${NB_GID}" "${HOME}" && \
+    chmod g+w /etc/passwd && \
+    fix-permissions "${HOME}" && \
+    fix-permissions "${CONDA_DIR}"
+
+USER $NB_USER
+WORKDIR $HOME
+# Setup work directory for backward-compatibility
+RUN mkdir $HOME/work && \
+    mkdir -p $HOME/.local && \
+    mkdir -p $HOME/.condor/tokens.d && \
+    fix-permissions $HOME
+
+#Dask dependencies and HTCondor
+RUN mamba install --yes \
+    -c conda-forge \
+    xgboost \
+    pyhf \
+    xrootd \
+    scipy>=1.8.1 \
+    cabinetry \
+    vector \
+    hist \
+    mplhep \
+    iminuit \
+    cmake \
+    ndcctools \
+    && mamba clean \
+        --all \
+        --force-pkgs-dirs \
+        --yes
+
+RUN pip install --no-cache-dir \
+    aiostream \
+    supervisor \
+    correctionlib \
+    funcx \
+    pyyaml \
+    # ML packages
+    dask-ml \
+    dask-gateway \
+    prometheus_client \
+    comm>=0.1.2 \
+    mlflow
+
+#RUN if [ "${TAG:-}" == "development" ]; then pip install --no-cache-dir git+https://github.com/CoffeaTeam/coffea-casa.git#egg=coffea_casa ; else pip install --no-cache-dir coffea_casa -U ; fi
+RUN pip install --no-cache-dir git+https://github.com/CoffeaTeam/coffea-casa.git#egg=coffea_casa
+
+# ------- xrootd-xcache-plugin -------------------------------
+RUN cd /tmp && \
+    git clone -b xcache https://github.com/jthiltges/xrdcl-authz-plugin.git && \
+    cd xrdcl-authz-plugin && \
+    mkdir build && \
+    cd  build && \
+    cmake /tmp/xrdcl-authz-plugin -DCMAKE_INSTALL_PREFIX=${CONDA_DIR} && \
+    make && \
+    make install
+
+ENV XRD_PLUGINCONFDIR="${CONDA_DIR}/etc/xrootd/client.plugins.d/"
+ENV XRD_PLUGIN="${CONDA_DIR}/lib/libXrdClXcachePlugin-5.so"
+
+# Include additional CA certificates beyond ca-policy-lcg
+COPY certs/* /etc/grid-security/certificates/
+RUN openssl rehash /etc/grid-security/certificates/
+
+# TODO: RETEST IF WE STILL NEED THIS 
+ENV LD_LIBRARY_PATH="${CONDA_DIR}/lib/:$LD_LIBRARY_PATH"
+ENV PATH="${CONDA_DIR}/bin/:$PATH"
+
+USER root
+# Setup supervisord files
+COPY k8s-worker/supervisord.conf  /etc/supervisor/
+# Setup HTCondor user/group and change group for user $NB_USER
+# Fix error (submitting jobs as user/group 0 (root) is not allowed for security reasons) and
+# it configured from kubernetes side and updated in docker container to match it
+RUN groupadd -r condor && \
+    useradd -r -g condor -d /var/lib/condor -s /sbin/nologin condor
+
+# FIXME: merge PRs open in distributed.git (oshadura)
+# Distributed: we need to install patched version of distributed version
+COPY dask/distributed ${CONDA_DIR}/lib/python3.10/site-packages/distributed
+RUN cd ${CONDA_DIR}/lib/python3.10/site-packages/distributed && \
+    patch -p2 < 0001-Patch-from-bbockelman-adaptive-scaling.patch && \
+    patch -p2 < 0002-Allow-scheduler-to-preserve-worker-hostnames.patch
+    # && patch -p2 < 0003-Activate-patch.patch
+    # && patch -p2 < 0004-Add-possibility-to-setup-external_adress-for-schedul.patch
+    # && patch -p2 < 0005-Add-nanny-patch.patch
+
+# FIXME: we have a wrong path, let's make a link.
+# cms-jovyan@jupyter-oksana-2eshadura-40cern-2ech:~$ echo $PATH
+# ${CONDA_DIR}/condabin:/usr/local/bin:/usr/bin:/bin:/usr/local/games:/usr/games
+RUN rm -rf ${CONDA_DIR}/condabin && ln -s ${CONDA_DIR}/bin ${CONDA_DIR}/condabin
+
+# Cleanup
+RUN rm -rf /tmp/* \
+    && rm -rf $HOME/.cache/.pip/* \
+    && mamba clean --all -f -y \
+    && find ${CONDA_DIR} -type f -name '*.a' -delete \
+    && find ${CONDA_DIR} -type f -name '*.pyc' -delete \
+    && find ${CONDA_DIR} -type f -name '*.js.map' -delete \
+    && (find ${CONDA_DIR}/lib/python3.10/site-packages/bokeh/server/static -type f,l -name '*.js' -not -name '*.min.js' -delete || echo "no bokeh static files to cleanup") \
+    && rm -rf ${CONDA_DIR}/pkgs
+
+# Mkdir CVMFS mount directory.  Mount done upstream in configs. Harmless
+# if not utilized, crucial if CVMFS is desired
+RUN mkdir /cvmfs
+
+# Prepare HTCondor spec. environment and execute dask-worker command
+# If we using this container as a sidecar, we don't setup any HTCondor spec. environment
+# nor and execute dask-worker command
+ADD prepare-env/prepare-env-cc-analysis.sh /usr/local/bin/prepare-env.sh
+RUN chmod ugo+x /usr/local/bin/prepare-env.sh
+
+USER $NB_USER
+ENTRYPOINT ["tini", "-g", "--", "/usr/local/bin/prepare-env.sh"]