alibuild 1.17.19__py3-none-any.whl

alibuild_helpers/build_template.sh

@@ -0,0 +1,314 @@
+#!/bin/bash
+
+# Automatically generated build script
+unset DYLD_LIBRARY_PATH
+
+# Cleanup variables which should not be exposed to user code
+unset AWS_ACCESS_KEY_ID
+unset AWS_SECRET_ACCESS_KEY
+
+set -e
+set +h
+function hash() { true; }
+export WORK_DIR="${WORK_DIR_OVERRIDE:-%(workDir)s}"
+export ALIBUILD_CONFIG_DIR="${ALIBUILD_CONFIG_DIR_OVERRIDE:-%(configDir)s}"
+
+# Insert our own wrapper scripts into $PATH, patched to use the system OpenSSL,
+# instead of the one we build ourselves.
+export PATH=$WORK_DIR/wrapper-scripts:$PATH
+
+# The following environment variables are setup by
+# the aliBuild script itself
+#
+# - ARCHITECTURE
+# - BUILD_REQUIRES
+# - CACHED_TARBALL
+# - CAN_DELETE
+# - COMMIT_HASH
+# - DEPS_HASH
+# - DEVEL_HASH
+# - DEVEL_PREFIX
+# - INCREMENTAL_BUILD_HASH
+# - JOBS
+# - PKGHASH
+# - PKGNAME
+# - PKGREVISION
+# - PKGVERSION
+# - REQUIRES
+# - RUNTIME_REQUIRES
+
+export PKG_NAME="$PKGNAME"
+export PKG_VERSION="$PKGVERSION"
+export PKG_BUILDNUM="$PKGREVISION"
+
+export PKGPATH=${ARCHITECTURE}/${PKGNAME}/${PKGVERSION}-${PKGREVISION}
+mkdir -p "$WORK_DIR/BUILD" "$WORK_DIR/SOURCES" "$WORK_DIR/TARS" \
+         "$WORK_DIR/SPECS" "$WORK_DIR/INSTALLROOT"
+# If we are in development mode, then install directly in $WORK_DIR/$PKGPATH,
+# so that we can do "make install" directly into BUILD/$PKGPATH and have
+# changes being propagated.
+# Moreover, devel packages should always go in the official WORK_DIR
+if [ -n "$DEVEL_HASH" ]; then
+  export ALIBUILD_BUILD_WORK_DIR="${WORK_DIR}"
+  export INSTALLROOT="$WORK_DIR/$PKGPATH"
+else
+  export INSTALLROOT="$WORK_DIR/INSTALLROOT/$PKGHASH/$PKGPATH"
+  export ALIBUILD_BUILD_WORK_DIR="${ALIBUILD_BUILD_WORK_DIR:-$WORK_DIR}"
+fi
+
+export BUILDROOT="$ALIBUILD_BUILD_WORK_DIR/BUILD/$PKGHASH"
+export SOURCEDIR="$WORK_DIR/SOURCES/$PKGNAME/$PKGVERSION/$COMMIT_HASH"
+export BUILDDIR="$BUILDROOT/$PKGNAME"
+
+rm -fr "$WORK_DIR/INSTALLROOT/$PKGHASH"
+# We remove the build directory only if we are not in incremental mode.
+if [[ "$INCREMENTAL_BUILD_HASH" == 0 ]] && ! rm -rf "$BUILDROOT"; then
+  # Golang installs stuff without write permissions for ourselves sometimes.
+  # This makes the `rm -rf` above fail, so give ourselves write permission.
+  chmod -R o+w "$BUILDROOT" || :
+  rm -rf "$BUILDROOT"
+fi
+mkdir -p "$INSTALLROOT" "$BUILDROOT" "$BUILDDIR" "$WORK_DIR/INSTALLROOT/$PKGHASH/$PKGPATH"
+
+cd "$WORK_DIR/INSTALLROOT/$PKGHASH"
+cat > "$INSTALLROOT/.meta.json" <<\EOF
+%(provenance)s
+EOF
+
+# Add "source" command for dependencies to init.sh.
+# Install init.sh now, so that it is available for debugging in case the build fails.
+mkdir -p "$INSTALLROOT/etc/profile.d"
+rm -f "$INSTALLROOT/etc/profile.d/init.sh"
+cat <<\EOF > "$INSTALLROOT/etc/profile.d/init.sh"
+%(initdotsh_deps)s
+EOF
+
+# Apply dependency initialisation now, but skip setting the variables below until after the build.
+. "$INSTALLROOT/etc/profile.d/init.sh"
+
+# Add support for direnv https://github.com/direnv/direnv/
+#
+# This is beneficial for all the cases where the build step requires some
+# environment to be properly setup in order to work. e.g. to support ninja or
+# protoc.
+cat << EOF > "$BUILDDIR/.envrc"
+# Source the build environment which was used for this package
+WORK_DIR=\${WORK_DIR:-$WORK_DIR} source "\${WORK_DIR:-$WORK_DIR}/${INSTALLROOT#$WORK_DIR/}/etc/profile.d/init.sh"
+source_up
+# On mac we build with the proper installation relative RPATH,
+# so this is not actually used and it's actually harmful since
+# startup time is reduced a lot by the extra overhead from the
+# dynamic loader
+unset DYLD_LIBRARY_PATH
+EOF
+
+cd "$BUILDROOT"
+ln -snf $PKGHASH "${BUILDROOT}-latest"
+if [[ $DEVEL_PREFIX ]]; then
+  ln -snf $PKGHASH "${BUILDROOT}-latest-$DEVEL_PREFIX"
+fi
+
+cd "$BUILDDIR"
+
+# Actual build script, as defined in the recipe
+
+# This actually does the build, taking in to account shortcuts like
+# having a pre build tarball or having an incremental recipe (in the
+# case of development mode).
+#
+# - If the build was never done and we do not have a cached tarball,
+#   build everything as usual.
+# - If the build was started, we do not have a tarball, and we
+#   have a non trivial incremental recipe, use it to continue the build.
+# - If the build was started, but we do not have a incremental build recipe,
+#   simply rebuild as usual.
+# - In case we have a cached tarball, we skip the build and expand it, change
+#   the relocation script so that it takes into account the new location.
+if [[ "$CACHED_TARBALL" == "" && ! -f $BUILDROOT/log ]]; then
+  set -o pipefail
+  (set -x; unset DYLD_LIBRARY_PATH; source "$WORK_DIR/SPECS/$ARCHITECTURE/$PKGNAME/$PKGVERSION-$PKGREVISION/$PKGNAME.sh" 2>&1) | tee "$BUILDROOT/log"
+elif [[ "$CACHED_TARBALL" == "" && $INCREMENTAL_BUILD_HASH != "0" && -f "$BUILDDIR/.build_succeeded" ]]; then
+  set -o pipefail
+  (%(incremental_recipe)s) 2>&1 | tee "$BUILDROOT/log"
+elif [[ "$CACHED_TARBALL" == "" ]]; then
+  set -o pipefail
+  (set -x; unset DYLD_LIBRARY_PATH; source "$WORK_DIR/SPECS/$ARCHITECTURE/$PKGNAME/$PKGVERSION-$PKGREVISION/$PKGNAME.sh" 2>&1) | tee "$BUILDROOT/log"
+else
+  # Unpack the cached tarball in the $INSTALLROOT and remove the unrelocated
+  # files.
+  rm -rf "$BUILDROOT/log"
+  mkdir -p $WORK_DIR/TMP/$PKGHASH
+  tar -xzf "$CACHED_TARBALL" -C "$WORK_DIR/TMP/$PKGHASH"
+  mkdir -p $(dirname $INSTALLROOT)
+  rm -rf $INSTALLROOT
+  mv $WORK_DIR/TMP/$PKGHASH/$ARCHITECTURE/$PKGNAME/$PKGVERSION-* $INSTALLROOT
+  pushd $WORK_DIR/INSTALLROOT/$PKGHASH
+    if [ -w "$INSTALLROOT" ]; then
+      WORK_DIR=$WORK_DIR/INSTALLROOT/$PKGHASH bash -ex $INSTALLROOT/relocate-me.sh
+    fi
+  popd
+  find $INSTALLROOT -name "*.unrelocated" -delete
+  rm -rf $WORK_DIR/TMP/$PKGHASH
+fi
+
+# Regenerate init.sh, in case the package build clobbered it. This
+# particularly happens in the AliEn-Runtime package, since it copies other
+# packages into its installroot wholesale.
+# Notice how we only do it if $INSTALLROOT is writeable. If it is
+# not, we assume it points to a CVMFS store which should be left untouched.
+if [ -w $INSTALLROOT ]; then
+mkdir -p "$INSTALLROOT/etc/profile.d"
+rm -f "$INSTALLROOT/etc/profile.d/init.sh"
+cat <<\EOF > "$INSTALLROOT/etc/profile.d/init.sh"
+%(initdotsh_full)s
+EOF
+
+cd "$WORK_DIR/INSTALLROOT/$PKGHASH"
+# Replace the .envrc to point to the final installation directory.
+cat << EOF > "$BUILDDIR/.envrc"
+# Source the build environment which was used for this package
+WORK_DIR=\${WORK_DIR:-$WORK_DIR} source ../../../$PKGPATH/etc/profile.d/init.sh
+source_up
+# On mac we build with the proper installation relative RPATH,
+# so this is not actually used and it's actually harmful since
+# startup time is reduced a lot by the extra overhead from the
+# dynamic loader
+unset DYLD_LIBRARY_PATH
+EOF
+
+cat > "$INSTALLROOT/.meta.json" <<\EOF
+%(provenance)s
+EOF
+
+cd "$WORK_DIR/INSTALLROOT/$PKGHASH/$PKGPATH"
+# Find which files need relocation.
+{ grep -I -H -l -R "\(INSTALLROOT/$PKGHASH\|[@][@]PKGREVISION[@]$PKGHASH[@][@]\)" . || true; } | sed -e 's|^\./||' > "$INSTALLROOT/.original-unrelocated"
+
+# Relocate script for <arch>/<pkgname>/<pkgver> structure
+cat > "$INSTALLROOT/relocate-me.sh" <<EoF
+#!/bin/bash -e
+if [[ "\$WORK_DIR" == '' ]]; then
+  echo 'Please, define \$WORK_DIR'
+  exit 1
+fi
+OP=${PKGPATH}
+PP=\${PKGPATH:-${PKGPATH}}
+PH=${PKGHASH}
+EoF
+
+while read -r unrelocated; do
+  echo "sed -e \"s|/[^ ;:]*INSTALLROOT/\$PH/\$OP|\$WORK_DIR/\$PP|g; s|[@][@]PKGREVISION[@]\$PH[@][@]|$PKGREVISION|g\"" \
+       "\$PP/$unrelocated.unrelocated > \$PP/$unrelocated"
+done < "$INSTALLROOT/.original-unrelocated" >> "$INSTALLROOT/relocate-me.sh"
+
+# Always relocate the modulefile (if present) so that it works also in devel mode.
+if [[ ! -s "$INSTALLROOT/.original-unrelocated" && -f "$INSTALLROOT/etc/modulefiles/$PKGNAME" ]]; then
+  echo "mv -f \$PP/etc/modulefiles/$PKGNAME \$PP/etc/modulefiles/${PKGNAME}.forced-relocation && sed -e \"s|[@][@]PKGREVISION[@]\$PH[@][@]|$PKGREVISION|g\" \$PP/etc/modulefiles/${PKGNAME}.forced-relocation > \$PP/etc/modulefiles/$PKGNAME" >> "$INSTALLROOT/relocate-me.sh"
+fi
+
+# Find libraries and executables needing relocation on macOS
+if [[ ${ARCHITECTURE:0:3} == "osx" ]]; then
+  otool_arch=$(echo "${ARCHITECTURE#osx_}" | tr - _)  # otool knows x86_64, not x86-64
+
+  /usr/bin/find ${RELOCATE_PATHS:-bin lib lib64} -type d \( -name '*.dist-info' -o -path '*/pytz/zoneinfo' \) -prune -false -o -type f \
+                -not -name '*.py' -not -name '*.pyc' -not -name '*.pyi' -not -name '*.pxd' -not -name '*.inc' -not -name '*.js' -not -name '*.json' \
+                -not -name '*.xml' -not -name '*.xsl' -not -name '*.txt' -not -name '*.dat' -not -name '*.mat' -not -name '*.sav' -not -name '*.csv' \
+                -not -name '*.wav' -not -name '*.png' -not -name '*.svg' -not -name '*.css' -not -name '*.html' -not -name '*.woff' -not -name '*.woff2' -not -name '*.ttf' \
+                -not -name LICENSE -not -name COPYING -not -name '*.c' -not -name '*.cc' -not -name '*.cxx' -not -name '*.cpp' -not -name '*.h' -not -name '*.hpp' |
+    while read -r BIN; do
+      MACHOTYPE=$(set +o pipefail; otool -arch "$otool_arch" -h "$PWD/$BIN" 2> /dev/null | grep filetype -A1 | awk 'END{print $5}')
+
+      # See mach-o/loader.h from XNU sources: 2 == executable, 6 == dylib, 8 == bundle
+      if [[ $MACHOTYPE == 6 || $MACHOTYPE == 8 ]]; then
+        # Only dylibs: relocate LC_ID_DYLIB
+        if otool -arch "$otool_arch" -D "$PWD/$BIN" 2> /dev/null | tail -n1 | grep -q "$PKGHASH"; then
+          cat <<EOF >> "$INSTALLROOT/relocate-me.sh"
+install_name_tool -id "\$(otool -arch $otool_arch -D "\$PP/$BIN" | tail -n1 | sed -e "s|/[^ ]*INSTALLROOT/\$PH/\$OP|\$WORK_DIR/\$PP|g")" "\$PP/$BIN"
+EOF
+        elif otool -arch "$otool_arch" -D "$PWD/$BIN" 2> /dev/null | tail -n1 | grep -vq /; then
+          cat <<EOF >> "$INSTALLROOT/relocate-me.sh"
+install_name_tool -id "\$WORK_DIR/\$PP/$BIN" "\$PP/$BIN"
+EOF
+        fi
+      fi
+
+      if [[ $MACHOTYPE == 2 || $MACHOTYPE == 6 || $MACHOTYPE == 8 ]]; then
+        # Both libs and binaries: relocate LC_RPATH
+        if otool -arch "$otool_arch" -l "$PWD/$BIN" 2> /dev/null | grep -A2 LC_RPATH | grep path | grep -q "$PKGHASH"; then
+          cat <<EOF >> "$INSTALLROOT/relocate-me.sh"
+OLD_RPATHS=\$(otool -arch $otool_arch -l "\$PP/$BIN" | grep -A2 LC_RPATH | grep path | grep "\$PH" | sed -e 's|^.*path ||' -e 's| .*$||' | sort -u)
+for OLD_RPATH in \$OLD_RPATHS; do
+  NEW_RPATH=\${OLD_RPATH/#*INSTALLROOT\/\$PH\/\$OP/\$WORK_DIR/\$PP}
+  install_name_tool -rpath "\$OLD_RPATH" "\$NEW_RPATH" "\$PP/$BIN"
+done
+EOF
+        fi
+
+        # Both libs and binaries: relocate LC_LOAD_DYLIB
+        if otool -arch "$otool_arch" -l "$PWD/$BIN" 2> /dev/null | grep -A2 LC_LOAD_DYLIB | grep name | grep -q $PKGHASH; then
+          cat <<EOF >> "$INSTALLROOT/relocate-me.sh"
+OLD_LOAD_DYLIBS=\$(otool -arch $otool_arch -l "\$PP/$BIN" | grep -A2 LC_LOAD_DYLIB | grep name | grep "\$PH" | sed -e 's|^.*name ||' -e 's| .*$||' | sort -u)
+for OLD_LOAD_DYLIB in \$OLD_LOAD_DYLIBS; do
+  NEW_LOAD_DYLIB=\${OLD_LOAD_DYLIB/#*INSTALLROOT\/\$PH\/\$OP/\$WORK_DIR/\$PP}
+  install_name_tool -change "\$OLD_LOAD_DYLIB" "\$NEW_LOAD_DYLIB" "\$PP/$BIN"
+done
+EOF
+        fi
+      fi
+    done || true
+fi
+
+cat "$INSTALLROOT/relocate-me.sh"
+cat "$INSTALLROOT/.original-unrelocated" | xargs -n1 -I{} cp '{}' '{}'.unrelocated
+fi
+cd "$WORK_DIR/INSTALLROOT/$PKGHASH"
+
+# Archive creation
+HASHPREFIX=`echo $PKGHASH | cut -b1,2`
+HASH_PATH=$ARCHITECTURE/store/$HASHPREFIX/$PKGHASH
+mkdir -p "${WORK_DIR}/TARS/$HASH_PATH" \
+         "${WORK_DIR}/TARS/$ARCHITECTURE/$PKGNAME"
+
+PACKAGE_WITH_REV=$PKGNAME-$PKGVERSION-$PKGREVISION.$ARCHITECTURE.tar.gz
+# Copy and tar/compress (if applicable) in parallel.
+# Use -H to match tar's behaviour of preserving hardlinks.
+rsync -aH "$WORK_DIR/INSTALLROOT/$PKGHASH/" "$WORK_DIR" & rsync_pid=$!
+if [ "$CAN_DELETE" = 1 ]; then
+  # We're deleting the tarball anyway, so no point in creating a new one.
+  # There might be an old existing tarball, and we should delete it.
+  rm -f "$WORK_DIR/TARS/$HASH_PATH/$PACKAGE_WITH_REV"
+elif [ -z "$CACHED_TARBALL" ]; then
+  # Use pigz to compress, if we can, because it's multicore.
+  gzip=$(command -v pigz) || gzip=$(command -v gzip)
+  # We don't have an existing tarball, and we want to keep the one we create now.
+  tar -cC "$WORK_DIR/INSTALLROOT/$PKGHASH" . |
+    # Avoid having broken left overs if the tar fails.
+    $gzip -c > "$WORK_DIR/TARS/$HASH_PATH/$PACKAGE_WITH_REV.processing"
+  mv "$WORK_DIR/TARS/$HASH_PATH/$PACKAGE_WITH_REV.processing" \
+     "$WORK_DIR/TARS/$HASH_PATH/$PACKAGE_WITH_REV"
+  ln -nfs "../../$HASH_PATH/$PACKAGE_WITH_REV" \
+     "$WORK_DIR/TARS/$ARCHITECTURE/$PKGNAME/$PACKAGE_WITH_REV"
+fi
+wait "$rsync_pid"
+
+# We've copied files into their final place; now relocate.
+cd "$WORK_DIR"
+if [ -w "$WORK_DIR/$ARCHITECTURE/$PKGNAME/$PKGVERSION-$PKGREVISION" ]; then
+  bash -ex "$ARCHITECTURE/$PKGNAME/$PKGVERSION-$PKGREVISION/relocate-me.sh"
+fi
+# Last package built gets a "latest" mark.
+ln -snf $PKGVERSION-$PKGREVISION $ARCHITECTURE/$PKGNAME/latest
+
+# Latest package built for a given devel prefix gets latest-$BUILD_FAMILY
+if [[ $BUILD_FAMILY ]]; then
+  ln -snf $PKGVERSION-$PKGREVISION $ARCHITECTURE/$PKGNAME/latest-$BUILD_FAMILY
+fi
+
+# When the package is definitely fully installed, install the file that marks
+# the package as successful.
+if [ -w "$WORK_DIR/$PKGPATH" ]; then
+  echo "$PKGHASH" > "$WORK_DIR/$PKGPATH/.build-hash"
+fi
+# Mark the build as successful with a placeholder. Allows running incremental
+# recipe in case the package is in development mode.
+echo "${DEVEL_HASH}${DEPS_HASH}" > "$BUILDDIR/.build_succeeded"

alibuild_helpers/clean.py

@@ -0,0 +1,83 @@
+# Import as function if they do not have any side effects
+from os.path import dirname, basename
+
+# Import as modules if I need to mock them later
+import os.path as path
+import os
+import glob
+import sys
+import shutil
+from alibuild_helpers import log
+
+
+def decideClean(workDir, architecture, aggressiveCleanup):
+  """Decide what to delete, without actually doing it.
+
+  To clean up obsolete build directories:
+  - Find all the symlinks in "BUILD"
+  - Find all the directories in "BUILD"
+  - Schedule a directory for deletion if it does not have a symlink
+
+  Installed packages are deleted from the final installation directory
+  according to the above scheme as well.
+
+  The temporary directory and temporary install roots are always cleaned up.
+
+  In aggressive mode, the following are also cleaned up:
+
+  - Tarballs (but not their symlinks), since these are expected to either be
+    unpacked in the installation directory, available from the remote store
+    for download, or not needed any more if their installation directory is
+    gone.
+  - Git checkouts for specific tags, since we expect to be able to rebuild
+    those easily from the mirror directory.
+
+  In the case of installed packages and tarballs, only those for the given
+  architecture are considered for deletion.
+  """
+  symlinksBuild = [os.readlink(x) for x in glob.glob("%s/BUILD/*-latest*" % workDir)]
+  # $WORK_DIR/TMP should always be cleaned up. This does not happen only
+  # in the case we run out of space while unpacking. INSTALLROOT is similar,
+  # though it is not cleaned up automatically in case of build errors.
+  # $WORK_DIR/<architecture>/store can be cleaned up as well, because
+  # we do not need the actual tarballs after they have been built.
+  toDelete = ["%s/TMP" % workDir, "%s/INSTALLROOT" % workDir]
+  if aggressiveCleanup:
+    toDelete += ["%s/TARS/%s/store" % (workDir, architecture),
+                 "%s/SOURCES" % (workDir)]
+  allBuildStuff = glob.glob("%s/BUILD/*" % workDir)
+  toDelete += [x for x in allBuildStuff
+               if not path.islink(x) and basename(x) not in symlinksBuild]
+  installGlob ="%s/%s/*/" % (workDir, architecture)
+  installedPackages = set([dirname(x) for x in glob.glob(installGlob)])
+  symlinksInstall = []
+  for x in installedPackages:
+    symlinksInstall += [path.realpath(y) for y in glob.glob(x + "/latest*")]
+  toDelete += [x for x in glob.glob(installGlob+ "*")
+               if not path.islink(x) and path.realpath(x) not in symlinksInstall]
+  toDelete = [x for x in toDelete if path.exists(x)]
+  return toDelete
+
+
+def doClean(workDir, architecture, aggressiveCleanup, dryRun):
+  """ CLI API to cleanup build area """
+  toDelete = decideClean(workDir, architecture, aggressiveCleanup)
+  if not toDelete:
+    log.info("Nothing to delete.")
+    sys.exit(0)
+
+  log.banner("This %s delete the following directories:\n%s",
+             "would" if dryRun else "will", "\n".join(toDelete))
+  if dryRun:
+    log.info("--dry-run / -n specified. Doing nothing.")
+    sys.exit(0)
+
+  have_error = False
+  for directory in toDelete:
+    try:
+      shutil.rmtree(directory)
+    except OSError as exc:
+      have_error = True
+      log.error("Unable to delete %s:", directory, exc_info=exc)
+
+  sys.exit(1 if have_error else 0)

alibuild_helpers/cmd.py

@@ -0,0 +1,154 @@
+import os
+import os.path
+import time
+from subprocess import Popen, PIPE, STDOUT
+from textwrap import dedent
+from subprocess import TimeoutExpired
+from shlex import quote
+
+from alibuild_helpers.log import debug, warning, dieOnError
+
+def decode_with_fallback(data):
+  """Try to decode DATA as utf-8; if that doesn't work, fall back to latin-1.
+
+  This combination should cover every possible byte string, as latin-1 covers
+  every possible single byte.
+  """
+  if isinstance(data, bytes):
+    try:
+      return data.decode("utf-8")
+    except UnicodeDecodeError:
+      return data.decode("latin-1")
+  else:
+    return str(data)
+
+
+def getoutput(command, timeout=None):
+  """Run command, check it succeeded, and return its stdout as a string."""
+  proc = Popen(command, shell=isinstance(command, str), stdout=PIPE, stderr=PIPE)
+  try:
+    stdout, stderr = proc.communicate(timeout=timeout)
+  except TimeoutExpired:
+    warning("Process %r timed out; terminated", command)
+    proc.terminate()
+    stdout, stderr = proc.communicate()
+  dieOnError(proc.returncode, "Command %s failed with code %d: %s" %
+             (command, proc.returncode, decode_with_fallback(stderr)))
+  return decode_with_fallback(stdout)
+
+
+def getstatusoutput(command, timeout=None, cwd=None):
+  """Run command and return its return code and output (stdout and stderr)."""
+  proc = Popen(command, shell=isinstance(command, str), stdout=PIPE, stderr=STDOUT, cwd=cwd)
+  try:
+    merged_output, _ = proc.communicate(timeout=timeout)
+  except TimeoutExpired:
+    warning("Process %r timed out; terminated", command)
+    proc.terminate()
+    merged_output, _ = proc.communicate()
+  merged_output = decode_with_fallback(merged_output)
+  # Strip a single trailing newline, if one exists, to match the behaviour of
+  # subprocess.getstatusoutput.
+  if merged_output.endswith("\n"):
+    merged_output = merged_output[:-1]
+  return proc.returncode, merged_output
+
+
+def execute(command, printer=debug, timeout=None):
+  popen = Popen(command, shell=isinstance(command, str), stdout=PIPE, stderr=STDOUT)
+  start_time = time.time()
+  for line in iter(popen.stdout.readline, b""):
+    printer("%s", decode_with_fallback(line).strip("\n"))
+    if timeout is not None and time.time() > start_time + timeout:
+      popen.terminate()
+      break
+  out = decode_with_fallback(popen.communicate()[0]).strip("\n")
+  if out:
+    printer("%s", out)
+  return popen.returncode
+
+
+BASH = "bash" if getstatusoutput("/bin/bash --version")[0] else "/bin/bash"
+
+
+class DockerRunner:
+  """A context manager for running commands inside a Docker container.
+
+  If the Docker image given is None or empty, the commands are run on the host
+  instead.
+  """
+
+  def __init__(self, docker_image, docker_run_args=(), extra_env={}, extra_volumes=[]) -> None:
+    self._docker_image = docker_image
+    self._docker_run_args = docker_run_args
+    self._container = None
+    self._extra_env = extra_env
+    self._extra_volumes = extra_volumes
+
+  def __enter__(self):
+    if self._docker_image:
+      # "sleep inf" pauses forever, until we kill it.
+      envOpts = []
+      volumes = []
+      for env in self._extra_env.items():
+        envOpts.append("-e")
+        envOpts.append(f"{env[0]}={env[1]}")
+      for v in self._extra_volumes:
+        volumes.append("-v")
+        volumes.append(v)
+      cmd = ["docker", "run", "--detach"] + envOpts + volumes + ["--rm", "--entrypoint="]
+      cmd += self._docker_run_args
+      cmd += [self._docker_image, "sleep", "inf"]
+      self._container = getoutput(cmd).strip()
+
+    def getstatusoutput_docker(cmd, cwd=None):
+      if self._container is None:
+        command_prefix=""
+        if self._extra_env:
+          command_prefix="env " + " ".join("{}={}".format(k, v) for (k,v) in self._extra_env.items()) + " "
+        return getstatusoutput("{}{} -c {}".format(command_prefix, BASH, quote(cmd))
+                             , cwd=cwd)
+      envOpts = []
+      for env in self._extra_env.items():
+        envOpts.append("-e")
+        envOpts.append("{}={}".format(env[0], env[1]))
+      exec_cmd = ["docker", "container", "exec"] + envOpts + [self._container, "bash", "-c", cmd]
+      return getstatusoutput(exec_cmd, cwd=cwd)
+
+    return getstatusoutput_docker
+
+  def __exit__(self, exc_type, exc_value, traceback):
+    if self._container is not None:
+      # 'docker container stop' sends SIGTERM, which doesn't work on 'sleep'
+      # for some reason. Kill it directly instead, so we don't have to wait.
+      getstatusoutput("docker container kill " + quote(self._container))
+    self._container = None
+    return False   # propagate any exception that may have occurred
+
+
+def install_wrapper_script(name, work_dir):
+  script_dir = os.path.join(work_dir, "wrapper-scripts")
+  try:
+    os.makedirs(script_dir)
+  except OSError as exc:
+    # Errno 17 means the directory already exists.
+    if exc.errno != 17:
+      raise
+  # Create a wrapper script that cleans up the environment, so we don't see the
+  # OpenSSL built by aliBuild.
+  with open(os.path.join(script_dir, name), "w") as scriptf:
+    # Compute the "real" executable path each time, as the wrapper script might
+    # be called on the host or in a container.
+    scriptf.write(dedent("""\
+    #!/bin/sh
+    exec env -u LD_LIBRARY_PATH -u DYLD_LIBRARY_PATH \\
+         "$(which -a "$(basename "$0")" | grep -Fxv "$0" | head -1)" "$@"
+    """))
+    os.fchmod(scriptf.fileno(), 0o755)  # make the wrapper script executable
+  # If $PATH is empty, this is bad, because we need to fall back to the "real"
+  # executable that our script is wrapping.
+  dieOnError(not os.environ.get("PATH"),
+             "$PATH is unset or empty. Cannot find any executables. Try "
+             "rerunning this command inside a login shell (e.g. `bash -l`). "
+             "If that doesn't work, run `export PATH` manually.")
+  os.environ["PATH"] = script_dir + ":" + os.environ["PATH"]

alibuild_helpers/deps.py

@@ -0,0 +1,116 @@
+#!/usr/bin/env python3
+
+from alibuild_helpers.log import debug, error, info, dieOnError
+from alibuild_helpers.utilities import parseDefaults, readDefaults, getPackageList, validateDefaults
+from alibuild_helpers.cmd import DockerRunner, execute
+from tempfile import NamedTemporaryFile
+from os import remove, path
+
+def doDeps(args, parser):
+
+  # Check if we have an output file
+  if not args.outgraph:
+    parser.error("Specify a PDF output file with --outgraph")
+
+  # Resolve all the package parsing boilerplate
+  specs = {}
+  defaultsReader = lambda: readDefaults(args.configDir, args.defaults, parser.error, args.architecture)
+  (err, overrides, taps) = parseDefaults(args.disable, defaultsReader, debug)
+  with DockerRunner(args.dockerImage, args.docker_extra_args, extra_env={"ALIBUILD_CONFIG_DIR": "/alidist" if args.docker else path.abspath(args.configDir)}, extra_volumes=[f"{path.abspath(args.configDir)}:/alidist:ro"] if args.docker else []) as getstatusoutput_docker:
+    def performCheck(pkg, cmd):
+      return getstatusoutput_docker(cmd)
+    
+    systemPackages, ownPackages, failed, validDefaults = \
+      getPackageList(packages                = [args.package],
+                     specs                   = specs,
+                     configDir               = args.configDir,
+                     preferSystem            = args.preferSystem,
+                     noSystem                = args.noSystem,
+                     architecture            = args.architecture,
+                     disable                 = args.disable,
+                     defaults                = args.defaults,
+                     performPreferCheck      = performCheck,
+                     performRequirementCheck = performCheck,
+                     performValidateDefaults = lambda spec: validateDefaults(spec, args.defaults),
+                     overrides               = overrides,
+                     taps                    = taps,
+                     log                     = debug)
+
+  dieOnError(validDefaults and args.defaults not in validDefaults,
+             "Specified default `%s' is not compatible with the packages you want to build.\n" % args.defaults +
+             "Valid defaults:\n\n- " +
+             "\n- ".join(sorted(validDefaults)))
+
+  for s in specs.values():
+    # Remove disabled packages
+    s["requires"] = [r for r in s["requires"] if r not in args.disable and r != "defaults-release"]
+    s["build_requires"] = [r for r in s["build_requires"] if r not in args.disable and r != "defaults-release"]
+    s["runtime_requires"] = [r for r in s["runtime_requires"] if r not in args.disable and r != "defaults-release"]
+
+  # Determine which packages are only build/runtime dependencies
+  all_build   = set()
+  all_runtime = set()
+  for k,spec in specs.items():
+    all_build.update(spec["build_requires"])
+    all_runtime.update(spec["runtime_requires"])
+  all_both = all_build.intersection(all_runtime)
+
+  dot = "digraph {\n"
+  dot += "ratio=\"0.52\"\n"
+  dot += 'graph [nodesep=0.25, ranksep=0.2];\n'
+  dot += 'node [width=1.5, height=1, fonsize=46, margin=0.1];\n'
+  dot += 'edge [penwidth=2];\n'
+
+  for k,spec in specs.items():
+    if k == "defaults-release":
+      continue
+
+    # Determine node color based on its dependency status
+    color = None
+    if k in all_both:
+      color = "tomato1"
+    elif k in all_runtime:
+      color = "greenyellow"
+    elif k in all_build:
+      color = "plum"
+    elif k == args.package:
+      color = "gold"
+    else:
+      assert color, "This should not happen (happened for %s)" % k
+
+    # Node definition
+    dot += '"%s" [shape=box, style="rounded,filled", fontname="helvetica", fillcolor=%s]\n' % (k,color)
+
+    # Connections (different whether it's a build dependency or a runtime one)
+    for dep in spec["build_requires"]:
+     dot += '"%s" -> "%s" [color=grey70]\n' % (k, dep)
+    for dep in spec["runtime_requires"]:
+     dot += '"%s" -> "%s" [color=dodgerblue3]\n' % (k, dep)
+
+  dot += "}\n"
+
+  if args.outdot:
+    fp = open(args.outdot, "wt")
+  else:
+    fp = NamedTemporaryFile(delete=False, mode="wt")
+  fp.write(dot)
+  fp.close()
+
+  # Check if we have dot in PATH
+  try:
+    execute(["dot", "-V"])
+  except Exception:
+    dieOnError(True, "Could not find dot in PATH. Please install graphviz and add it to PATH.")
+  try:
+    if args.neat:
+      execute("tred {dotFile} > {dotFile}.0 && mv {dotFile}.0 {dotFile}".format(dotFile=fp.name))
+    execute(["dot", fp.name, "-Tpdf", "-o", args.outgraph])
+  except Exception as e:
+    error("Error generating dependencies with dot: %s: %s", type(e).__name__, e)
+  else:
+    info("Dependencies graph generated: %s" % args.outgraph)
+  if fp.name != args.outdot:
+    remove(fp.name)
+  else:
+    info("Intermediate dot file for Graphviz saved: %s" % args.outdot)
+  return True