alibuild 1.17.19__py3-none-any.whl

alibuild_helpers/workarea.py

@@ -0,0 +1,179 @@
+import codecs
+import errno
+import os
+import os.path
+import shutil
+import tempfile
+from collections import OrderedDict
+
+from alibuild_helpers.log import dieOnError, debug, error
+from alibuild_helpers.utilities import call_ignoring_oserrors, symlink, short_commit_hash
+
+FETCH_LOG_NAME = "fetch-log.txt"
+
+
+def cleanup_git_log(referenceSources):
+  """Remove a stale fetch-log.txt.
+
+  You must call this function before running updateReferenceRepoSpec or
+  updateReferenceRepo any number of times. This is not done automatically, so
+  that running those functions in parallel works properly.
+  """
+  try:
+    os.unlink(os.path.join(referenceSources, FETCH_LOG_NAME))
+  except OSError as exc:
+    # Ignore errors when deleting a nonexistent file.
+    dieOnError(exc.errno != errno.ENOENT,
+               "Could not delete stale git log: %s" % exc)
+
+
+def logged_scm(scm, package, referenceSources,
+               command, directory, prompt, logOutput=True):
+  """Run an SCM command, but produce an output file if it fails.
+
+  This is useful in CI, so that we can pick up SCM failures and show them in
+  the final produced log. For this reason, the file we write in this function
+  must not contain any secrets. We only output the SCM command we ran, its exit
+  code, and the package name, so this should be safe.
+  """
+  debug("%s %s for repository for %s...", scm.name, command[0], package)
+  err, output = scm.exec(command, directory=directory, check=False, prompt=prompt)
+  if logOutput:
+    debug(output)
+  if err:
+    try:
+      with codecs.open(os.path.join(referenceSources, FETCH_LOG_NAME),
+                       "a", encoding="utf-8", errors="replace") as logf:
+        logf.write("%s command for package %r failed.\n"
+                   "Command: %s %s\nIn directory: %s\nExit code: %d\n" %
+                   (scm.name, package, scm.name.lower(), " ".join(command), directory, err))
+    except OSError as exc:
+      error("Could not write error log from SCM command:", exc_info=exc)
+  dieOnError(err, "Error during %s %s for reference repo for %s." %
+             (scm.name.lower(), command[0], package))
+  debug("Done %s %s for repository for %s", scm.name.lower(), command[0], package)
+  return output
+
+
+def updateReferenceRepoSpec(referenceSources, p, spec,
+                            fetch=True, usePartialClone=True, allowGitPrompt=True):
+  """
+  Update source reference area whenever possible, and set the spec's "reference"
+  if available for reading.
+
+  @referenceSources : a string containing the path to the sources to be updated
+  @p                : the name of the package to be updated
+  @spec             : the spec of the package to be updated (an OrderedDict)
+  @fetch            : whether to fetch updates: if False, only clone if not found
+  """
+  spec["reference"] = updateReferenceRepo(referenceSources, p, spec, fetch,
+                                          usePartialClone, allowGitPrompt)
+  if not spec["reference"]:
+    del spec["reference"]
+
+
+def updateReferenceRepo(referenceSources, p, spec,
+                        fetch=True, usePartialClone=True, allowGitPrompt=True):
+  """
+  Update source reference area, if possible.
+  If the area is already there and cannot be written, assume it maintained
+  by someone else.
+
+  If the area can be created, clone a bare repository with the sources.
+
+  Returns the reference repository's local path if available, otherwise None.
+  Throws a fatal error in case repository cannot be updated even if it appears
+  to be writeable.
+
+  @referenceSources : a string containing the path to the sources to be updated
+  @p                : the name of the package to be updated
+  @spec             : the spec of the package to be updated (an OrderedDict)
+  @fetch            : whether to fetch updates: if False, only clone if not found
+  """
+  assert isinstance(spec, OrderedDict)
+  if spec["is_devel_pkg"] or "source" not in spec:
+    return None
+
+  scm = spec["scm"]
+
+  debug("Updating references.")
+  referenceRepo = os.path.join(os.path.abspath(referenceSources), p.lower())
+
+  call_ignoring_oserrors(os.makedirs, os.path.abspath(referenceSources), exist_ok=True)
+
+  if not is_writeable(referenceSources):
+    if os.path.exists(referenceRepo):
+      debug("Using %s as reference for %s", referenceRepo, p)
+      return referenceRepo  # reference is read-only
+    else:
+      debug("Cannot create reference for %s in %s", p, referenceSources)
+      return None  # no reference can be found and created (not fatal)
+
+  if not os.path.exists(referenceRepo):
+    cmd = scm.cloneReferenceCmd(spec["source"], referenceRepo, usePartialClone)
+    logged_scm(scm, p, referenceSources, cmd, ".", allowGitPrompt)
+  elif fetch:
+    cmd = scm.fetchCmd(spec["source"], "+refs/tags/*:refs/tags/*", "+refs/heads/*:refs/heads/*")
+    logged_scm(scm, p, referenceSources, cmd, referenceRepo, allowGitPrompt)
+
+  return referenceRepo  # reference is read-write
+
+
+def is_writeable(dirpath):
+  try:
+    with tempfile.NamedTemporaryFile(dir=dirpath):
+      return True
+  except:
+    return False
+
+
+def checkout_sources(spec, work_dir, reference_sources, containerised_build):
+  """Check out sources to be compiled, potentially from a given reference."""
+  scm = spec["scm"]
+
+  def scm_exec(command, directory=".", check=True):
+    """Run the given SCM command, simulating a shell exit code."""
+    try:
+      logged_scm(scm, spec["package"], reference_sources, command, directory, prompt=False)
+    except SystemExit as exc:
+      if check:
+        raise
+      return exc.code
+    return 0
+
+  source_parent_dir = os.path.join(work_dir, "SOURCES", spec["package"], spec["version"])
+  # The build script expects SOURCEDIR to be named after the shortened commit
+  # hash, not the full one.
+  source_dir = os.path.join(source_parent_dir, short_commit_hash(spec))
+  os.makedirs(source_parent_dir, exist_ok=True)
+
+  if spec["commit_hash"] != spec["tag"]:
+    symlink(spec["commit_hash"], os.path.join(source_parent_dir, spec["tag"].replace("/", "_")))
+
+  if "source" not in spec:
+    # There are no sources, so just create an empty SOURCEDIR.
+    os.makedirs(source_dir, exist_ok=True)
+  elif spec["is_devel_pkg"]:
+    shutil.rmtree(source_dir, ignore_errors=True)
+    # In a container, we mount development packages' source dirs in /.
+    # Outside a container, we have access to the source dir directly.
+    symlink("/" + os.path.basename(spec["source"])
+            if containerised_build else spec["source"],
+            source_dir)
+  elif os.path.isdir(source_dir):
+    # Sources are a relative path or URL and the local repo already exists, so
+    # checkout the right commit there.
+    err = scm_exec(scm.checkoutCmd(spec["tag"]), source_dir, check=False)
+    if err:
+      # If we can't find the tag, it might be new. Fetch tags and try again.
+      tag_ref = "refs/tags/{0}:refs/tags/{0}".format(spec["tag"])
+      scm_exec(scm.fetchCmd(spec["source"], tag_ref), source_dir)
+      scm_exec(scm.checkoutCmd(spec["tag"]), source_dir)
+  else:
+    # Sources are a relative path or URL and don't exist locally yet, so clone
+    # and checkout the git repo from there.
+    shutil.rmtree(source_dir, ignore_errors=True)
+    scm_exec(scm.cloneSourceCmd(spec["source"], source_dir, spec.get("reference"),
+                                usePartialClone=True))
+    scm_exec(scm.setWriteUrlCmd(spec.get("write_repo", spec["source"])), source_dir)
+    scm_exec(scm.checkoutCmd(spec["tag"]), source_dir)

debian/changelog

@@ -0,0 +1,11 @@
+python3-alibuild (1.8.4~focal) focal; urgency=medium
+
+  * Bump to v1.8.4
+
+ -- Giulio Eulisse <giulio.eulisse@cern.ch>  Thu, 27 May 2021 19:18:00 +0200
+
+python3-alibuild (1.8.2) bionic; urgency=medium
+
+  * Initial release.
+
+ -- Giulio Eulisse <giulio.eulisse@cern.ch>  Tue, 04 May 2021 00:08:40 +0200

debian/compat

@@ -0,0 +1 @@
+10

debian/control

@@ -0,0 +1,14 @@
+Source: python3-alibuild
+Maintainer: Giulio Eulisse <giulio.eulisse@cern.ch>
+Build-Depends: debhelper,dh-python,python3-all,python3-setuptools,python3-setuptools-scm
+Section: devel
+Priority: optional
+Standards-Version: 3.9.6
+X-Python3-Version: >= 3.6
+
+Package: python3-alibuild
+Architecture: all
+Description: ALICE build tool.
+	Please refer to https://alisw.github.io/alibuild for full documentation.
+Depends: ${python3:Depends},python3-requests,python3-yaml
+Recommends: pigz

debian/copyright

@@ -0,0 +1,10 @@
+Copyright CERN and copyright holders of ALICE O2. This software is
+distributed under the terms of the GNU General Public License v3 (GPL
+Version 3), copied verbatim in the file "/usr/share/common-licenses".
+
+See http://alice-o2.web.cern.ch/license for full licensing information.
+
+In applying this license CERN does not waive the privileges and immunities
+granted to it by virtue of its status as an Intergovernmental Organization
+or submit itself to any jurisdiction.
+

debian/files

@@ -0,0 +1 @@
+python3-alibuild_1.8.2_source.buildinfo devel optional

debian/rules

@@ -0,0 +1,7 @@
+#! /usr/bin/make -f
+
+#export DH_VERBOSE = 1
+export PYBUILD_NAME = python3-alibuild
+
+%:
+	dh $@ --with python3 --buildsystem=pybuild

docs/README.md

@@ -0,0 +1 @@
+Read the documentation here: <http://alisw.github.io/alibuild/>

docs/SUPPORT

@@ -0,0 +1,3 @@
+Before submitting an issue, please refer to the [troubleshooting
+section](http://alisw.github.io/alibuild/troubleshooting.html) of our
+documentation.

docs/docs/index.md

@@ -0,0 +1,75 @@
+---
+subtile: About this tool
+layout: main
+---
+
+<div style="text-align:center;width:100%">
+  <a href="https://badge.fury.io/py/alibuild"><img src="https://badge.fury.io/py/alibuild.svg" alt="PyPI version" height="22"></a>
+  <a href="https://github.com/alisw/alibuild/actions/workflows/pr-check.yml"><img src="https://github.com/alisw/alibuild/actions/workflows/pr-check.yml/badge.svg?branch=master&event=push" alt="Build status" height="22"></a>
+</div>
+
+A simple build tool for ALICE experiment software and its externals. Recipes
+for the externals and ALICE software are stored in
+[alidist](https://github.com/alisw/alidist).
+
+You can install aliBuild on [Ubuntu][ubuntu], [MacOS][mac], [CentOS 7][centos7], [Alma 8][alma8], [Alma 9][alma9] and [Fedora][fedora].
+
+[centos7]: https://alice-doc.github.io/alice-analysis-tutorial/building/prereq-centos7.html
+[alma8]: https://alice-doc.github.io/alice-analysis-tutorial/building/prereq-centos8.html
+[alma9]: https://alice-doc.github.io/alice-analysis-tutorial/building/prereq-alma9.html
+[ubuntu]: https://alice-doc.github.io/alice-analysis-tutorial/building/prereq-ubuntu.html
+[mac]: https://alice-doc.github.io/alice-analysis-tutorial/building/prereq-macos.html
+[fedora]: https://alice-doc.github.io/alice-analysis-tutorial/building/prereq-fedora.html
+
+Then, build ALICE's software with:
+
+    git clone https://github.com/alisw/alidist.git
+    aliBuild build O2Physics
+
+For a more verbose documentation of what is happening have a look at
+the [quickstart guide](quick.md). See the [user guide](user.md)
+for more command line options or have a look at the [troubleshooting
+pages](troubleshooting.md) for hints on how to debug build errors.
+Have a look at the [reference guide](reference.md) if you want to
+package your own software.
+
+<div style="display:grid;
+  grid-template-columns: repeat(3,1fr);  /* 3 columns */
+  grid-template-rows: repeat(2,1fr); /* 2 rows  */
+  grid-gap:50px 30px;
+
+">
+    <div><h2>Simple build recipes</h2>
+      Build recipes are simple bash scripts with a YAML header. Whenever
+      a dependency or a package changes only what is affected by the
+      change is rebuilt.
+      <br/><a href="reference.html">Read more</a>
+    </div>
+    <div><h2>Reuses system tools</h2>
+      If desired, aliBuild will do its best to reuse what is available
+      on the system, if compatible to what is found in the recipe.
+      <br/><a href="user.html#controlling-which-system-packages-are-picked-up">Read more</a>
+    </div>
+    <div><h2>Docker support</h2>
+      aliBuild allows builds to happen inside a docker container, so
+      that you can develop on Mac and build on your production Linux
+      platform.
+      <br/><a href="user.html#running-in-docker">Read more</a>
+    </div>
+    <div><h2>Binary packages</h2>
+      aliBuild provides the ability to reuse binary packages which were
+      previously centrally built, when they match the one that would be
+      built locally.
+      <br/><a href="user.html#using-precompiled-packages">Read more</a>
+    </div>
+    <div><h2>Developer mode</h2>
+      Besides building and packaging your dependencies, aliBuild
+      provides you the ability to develop those via a simple git clone.
+      <br/><a href="user.html#developing-packages-locally">Read more</a>
+    </div>
+    <div><h2>Integrates with modules</h2>
+      Easily setup your work environment using `alienv`, which is based on
+      standard <a href="http://modules.sourceforge.net">modulefiles</a>.
+      <br/><a href="quick.html#loading-the-package-environment">Read more</a>
+    </div>
+</div>

docs/docs/quick.md

@@ -0,0 +1,89 @@
+---
+subtitle: Quick Start
+layout: main
+---
+
+aliBuild is a tool to simplify building and installing ALICE / ALFA
+software. This is a quickstart Guide which will show you how to build
+and use a package, for extended documentation please have a look at the
+[user guide](user.md).
+
+## Setting up
+
+The tool itself is available as a standard PyPi package. You
+can install it via:
+
+    pip install alibuild
+
+Alternatively, if you cannot use pip, you can checkout the Github repository and
+use it from there:
+
+    git clone https://github.com/alisw/alibuild.git
+
+This will provide you the tool itself. 
+
+In order to work you will need a set of recipes from a repository called
+[alidist](https://github.com/alisw/alidist.git). On the first invocation of
+`alibuild` the recipes will be downloaded and put in a `alidist` folder. 
+In case you need to use a special branch / repository you can always `git clone` 
+the repository yourself. By default alibuild will pickup the recipes found
+in `$PWD/alidist`.
+
+## Building a package
+
+Once you have obtained both repository, you can trigger a build via:
+
+    aliBuild [-d] -j <jobs> build <package>
+
+(or alibuild/aliBuild if you are working from sources) where:
+
+- `<package>`: is the name of the package you want to build, e.g.: 
+  - `AliRoot`
+  - `AliPhysics`
+  - `O2`
+  - `ROOT`
+- `-d` can be used to have verbose debug output.
+- `<jobs>` is the maximum number of parallel processes to be used for
+  building where possible (defaults to the number of CPUs available if
+  omitted).
+
+If you need to modify the compile options, you can do so by looking at the
+recipes in your local `alidist` folder and amend them.
+
+## Results of a build
+
+By default (can be changed using the `-c` option) the installation of your builds
+can be found in:
+
+    sw/<architecture>/<package-name>/<package-version>-<revision>/
+
+where:
+
+- `<architecture>` is the same as the one passed via the `-a` option.
+- `<package-name>`: is the same as the one passed as an argument.
+- `<package-version>`: is the same as the one found in the related recipe in alidist.
+- `<package-revision>`: is the number of times you rebuilt the same version of
+  a package, using a different recipe. In general this will be 1.
+
+For example, on Centos7:
+
+    sw/slc7_x86-64/AliRoot/v6-16-01-1
+
+## Using the built package
+
+Environment for packages built using aliBuild is managed by [Environment
+Modules](http://modules.sourceforge.net) and a wrapper script called alienv.
+Notice you will need the package `environment-modules` on Linux or `modules` on
+macOS for the following to work.
+
+Assuming you are in the toplevel directory containing `alibuild`, `alidist` and
+`sw` you can do:
+
+    alienv q
+
+to list the available packages, and:
+
+    alienv enter VO_ALICE@PackageA::VersionA[,VO_ALICE@PackageB::VersionB...]
+
+to enter a shell with the appropriate environment set. To learn more about alienv you
+can also look at the [user guide](user.md#using-the-packages-you-have-built).