alibuild 1.17.7__tar.gz → 1.17.10__tar.gz

{alibuild-1.17.7 → alibuild-1.17.10}/.github/workflows/documentation.yml

@@ -25,6 +25,6 @@ jobs:
           restore-keys: |
             mkdocs-material-
       - run: |
-          pip install mkdocs-material 
+          pip install mkdocs-material mkdocs-redirects
           cd docs
           mkdocs gh-deploy --force # Always keep in sync with master branch

{alibuild-1.17.7 → alibuild-1.17.10}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: alibuild
-Version: 1.17.7
+Version: 1.17.10
 Summary: ALICE Build Tool
 Home-page: https://alisw.github.io/alibuild
 Author: Giulio Eulisse

{alibuild-1.17.7 → alibuild-1.17.10}/alibuild.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: alibuild
-Version: 1.17.7
+Version: 1.17.10
 Summary: ALICE Build Tool
 Home-page: https://alisw.github.io/alibuild
 Author: Giulio Eulisse

{alibuild-1.17.7 → alibuild-1.17.10}/alibuild.egg-info/SOURCES.txt

@@ -55,14 +55,14 @@ debian/rules
 docs/README.md
 docs/SUPPORT
 docs/mkdocs.yml
+docs/docs/alice_logo.png
 docs/docs/deps.png
 docs/docs/index.md
-docs/docs/o2-tutorial.md
 docs/docs/quick.md
 docs/docs/reference.md
 docs/docs/troubleshooting.md
-docs/docs/tutorial.md
 docs/docs/user.md
+docs/docs/stylesheets/extra.css
 tests/test_analytics.py
 tests/test_args.py
 tests/test_build.py

{alibuild-1.17.7 → alibuild-1.17.10}/alibuild_helpers/_version.py

@@ -12,5 +12,5 @@ __version__: str
 __version_tuple__: VERSION_TUPLE
 version_tuple: VERSION_TUPLE
 
-__version__ = version = '1.17.7'
-__version_tuple__ = version_tuple = (1, 17, 7)
+__version__ = version = '1.17.10'
+__version_tuple__ = version_tuple = (1, 17, 10)

{alibuild-1.17.7 → alibuild-1.17.10}/alibuild_helpers/args.py

@@ -380,6 +380,9 @@ On Linux, x86-64:
    Fedora 33 compatible: fedora33_x86-64
    Fedora 34 compatible: fedora34_x86-64
 
+On Linux, ARM:
+   RHEL9 / ALMA9 compatible: slc9_aarch64
+
 On Linux, POWER8 / PPC64 (little endian):
    RHEL7 / CC7 compatible: slc7_ppc64
 
@@ -389,7 +392,7 @@ On Mac, 1-2 latest supported OSX versions:
 """
 
 # When updating this variable, also update docs/user.markdown!
-S3_SUPPORTED_ARCHS = "slc7_x86-64", "slc8_x86-64", "ubuntu2004_x86-64", "ubuntu2204_x86-64", "slc9_x86-64"
+S3_SUPPORTED_ARCHS = "slc7_x86-64", "slc8_x86-64", "ubuntu2004_x86-64", "ubuntu2204_x86-64", "slc9_x86-64", "slc9_aarch64"
 
 def finaliseArgs(args, parser):
 

{alibuild-1.17.7 → alibuild-1.17.10}/alibuild_helpers/build.py

@@ -899,12 +899,18 @@ def doBuild(args, parser):
 
     # Now that we have all the information about the package we want to build, let's
     # check if it wasn't built / unpacked already.
-    hashFile = "%s/%s/%s/%s-%s/.build-hash" % (workDir,
-                                               args.architecture,
-                                               spec["package"],
-                                               spec["version"],
-                                               spec["revision"])
-    fileHash = readHashFile(hashFile)
+    hashPath= "%s/%s/%s/%s-%s" % (workDir,
+                                  args.architecture,
+                                  spec["package"],
+                                  spec["version"],
+                                  spec["revision"])
+    hashFile = hashPath + "/.build-hash"
+    # If the folder is a symlink, we consider it to be to CVMFS and
+    # take the hash for good.
+    if os.path.islink(hashPath):
+      fileHash = spec["hash"]
+    else:
+      fileHash = readHashFile(hashFile)
     # Development packages have their own rebuild-detection logic above.
     # spec["hash"] is only useful here for regular packages.
     if fileHash == spec["hash"] and not spec["is_devel_pkg"]:

{alibuild-1.17.7 → alibuild-1.17.10}/alibuild_helpers/build_template.sh

@@ -139,7 +139,9 @@ else
   rm -rf $INSTALLROOT
   mv $WORK_DIR/TMP/$PKGHASH/$ARCHITECTURE/$PKGNAME/$PKGVERSION-* $INSTALLROOT
   pushd $WORK_DIR/INSTALLROOT/$PKGHASH
-    WORK_DIR=$WORK_DIR/INSTALLROOT/$PKGHASH bash -ex $INSTALLROOT/relocate-me.sh
+    if [ -w "$INSTALLROOT" ]; then
+      WORK_DIR=$WORK_DIR/INSTALLROOT/$PKGHASH bash -ex $INSTALLROOT/relocate-me.sh
+    fi
   popd
   find $INSTALLROOT -name "*.unrelocated" -delete
   rm -rf $WORK_DIR/TMP/$PKGHASH
@@ -148,6 +150,9 @@ fi
 # Regenerate init.sh, in case the package build clobbered it. This
 # particularly happens in the AliEn-Runtime package, since it copies other
 # packages into its installroot wholesale.
+# Notice how we only do it if $INSTALLROOT is writeable. If it is
+# not, we assume it points to a CVMFS store which should be left untouched.
+if [ -w $INSTALLROOT ]; then
 mkdir -p "$INSTALLROOT/etc/profile.d"
 rm -f "$INSTALLROOT/etc/profile.d/init.sh"
 cat <<\EOF > "$INSTALLROOT/etc/profile.d/init.sh"
@@ -251,6 +256,7 @@ fi
 
 cat "$INSTALLROOT/relocate-me.sh"
 cat "$INSTALLROOT/.original-unrelocated" | xargs -n1 -I{} cp '{}' '{}'.unrelocated
+fi
 cd "$WORK_DIR/INSTALLROOT/$PKGHASH"
 
 # Archive creation
@@ -283,7 +289,9 @@ wait "$rsync_pid"
 
 # We've copied files into their final place; now relocate.
 cd "$WORK_DIR"
-bash -ex "$ARCHITECTURE/$PKGNAME/$PKGVERSION-$PKGREVISION/relocate-me.sh"
+if [ -w "$WORK_DIR/$ARCHITECTURE/$PKGNAME/$PKGVERSION-$PKGREVISION" ]; then
+  bash -ex "$ARCHITECTURE/$PKGNAME/$PKGVERSION-$PKGREVISION/relocate-me.sh"
+fi
 # Last package built gets a "latest" mark.
 ln -snf $PKGVERSION-$PKGREVISION $ARCHITECTURE/$PKGNAME/latest
 
@@ -294,7 +302,9 @@ fi
 
 # When the package is definitely fully installed, install the file that marks
 # the package as successful.
-echo "$PKGHASH" > "$WORK_DIR/$PKGPATH/.build-hash"
+if [ -w "$WORK_DIR/$PKGPATH" ]; then
+  echo "$PKGHASH" > "$WORK_DIR/$PKGPATH/.build-hash"
+fi
 # Mark the build as successful with a placeholder. Allows running incremental
 # recipe in case the package is in development mode.
 echo "${DEVEL_HASH}${DEPS_HASH}" > "$BUILDDIR/.build_succeeded"

{alibuild-1.17.7 → alibuild-1.17.10}/alibuild_helpers/sync.py

@@ -22,6 +22,8 @@ def remote_from_url(read_url, write_url, architecture, work_dir, insecure=False)
     return S3RemoteSync(read_url, write_url, architecture, work_dir)
   if read_url.startswith("b3://"):
     return Boto3RemoteSync(read_url, write_url, architecture, work_dir)
+  if read_url.startswith("cvmfs://"):
+    return CVMFSRemoteSync(read_url, None, architecture, work_dir)
   if read_url:
     return RsyncRemoteSync(read_url, write_url, architecture, work_dir)
   return NoRemoteSync()
@@ -304,6 +306,74 @@ class RsyncRemoteSync:
       revision=spec["revision"],
     )), "Unable to upload tarball.")
 
+class CVMFSRemoteSync:
+  """ Sync packages build directory from CVMFS or similar
+      FS based deployment. The tarball will be created on the fly with a single
+      symlink to the remote store in it, so that unpacking really
+      means unpacking the symlink to the wanted package.
+  """
+
+  def __init__(self, remoteStore, writeStore, architecture, workdir):
+    self.remoteStore = re.sub("^cvmfs://", "", remoteStore)
+    # We do not support uploading directly to CVMFS, for obvious
+    # reasons.
+    assert(writeStore == None)
+    self.writeStore = None
+    self.architecture = architecture
+    self.workdir = workdir
+
+  def fetch_tarball(self, spec):
+    info("Downloading tarball for %s@%s-%s, if available", spec["package"], spec["version"], spec["revision"])
+    # If we already have a tarball with any equivalent hash, don't check S3.
+    for pkg_hash in spec["remote_hashes"] + spec["local_hashes"]:
+      store_path = resolve_store_path(self.architecture, pkg_hash)
+      pattern = os.path.join(self.workdir, store_path, "%s-*.tar.gz" % spec["package"])
+      if glob.glob(pattern):
+        info("Reusing existing tarball for %s@%s", spec["package"], pkg_hash)
+        return
+    info("Could not find prebuilt tarball for %s@%s-%s, will be rebuilt",
+         spec["package"], spec["version"], spec["revision"])
+
+  def fetch_symlinks(self, spec):
+    # When using CVMFS, we create the symlinks grass by reading the .
+    info("Fetching available build hashes for %s, from %s", spec["package"], self.remoteStore)
+    links_path = resolve_links_path(self.architecture, spec["package"])
+    os.makedirs(os.path.join(self.workdir, links_path), exist_ok=True)
+
+    cvmfs_architecture = re.sub(r"slc(\d+)_x86-64", r"el\1-x86_64", self.architecture)
+    err = execute("""\
+    set -x
+    # Exit without error in case we do not have any package published
+    test -d "{remote_store}/{cvmfs_architecture}/Packages/{package}" || exit 0
+    mkdir -p "{workDir}/{links_path}"
+    for install_path in $(find "{remote_store}/{cvmfs_architecture}/Packages/{package}" -type d -mindepth 1 -maxdepth 1); do
+      full_version="${{install_path##*/}}"
+      tarball={package}-$full_version.{architecture}.tar.gz
+      pkg_hash=$(cat "${{install_path}}/.build-hash" || jq -r '.package.hash' <${{install_path}}/.meta.json)
+      if [ "X$pkg_hash" = X ]; then
+        continue
+      fi
+      ln -sf ../../{architecture}/store/${{pkg_hash:0:2}}/$pkg_hash/$tarball "{workDir}/{links_path}/$tarball"
+      # Create the dummy tarball, if it does not exists
+      test -f "{workDir}/{architecture}/store/${{pkg_hash:0:2}}/$pkg_hash/$tarball" && continue
+      mkdir -p "{workDir}/INSTALLROOT/$pkg_hash/{architecture}/{package}"
+      ln -sf "{remote_store}/{cvmfs_architecture}/Packages/{package}/$full_version" "{workDir}/INSTALLROOT/$pkg_hash/{architecture}/{package}"
+      mkdir -p "{workDir}/TARS/{architecture}/store/${{pkg_hash:0:2}}/$pkg_hash"
+      tar -C "{workDir}/INSTALLROOT/$pkg_hash" -czf "{workDir}/TARS/{architecture}/store/${{pkg_hash:0:2}}/$pkg_hash/$tarball" .
+      rm -rf "{workDir}/INSTALLROOT/$pkg_hash"
+    done
+    """.format(
+      workDir=self.workdir,
+      b=self.remoteStore,
+      architecture=self.architecture,
+      cvmfs_architecture=cvmfs_architecture,
+      package=spec["package"],
+      remote_store=self.remoteStore,
+      links_path=links_path,
+    ))
+
+  def upload_symlinks_and_tarball(self, spec):
+    dieOnError(self.writeStore, "CVMFS backend does not support uploading directly")
 
 class S3RemoteSync:
   """Sync package build directory from and to S3 using s3cmd.

alibuild-1.17.10/docs/docs/stylesheets/extra.css

@@ -0,0 +1,7 @@
+.md-header__button.md-logo img, .md-header__button.md-logo svg {
+  height: 1.5rem;
+}
+
+.md-header {
+  background-color: #111;
+}

{alibuild-1.17.7 → alibuild-1.17.10}/docs/docs/troubleshooting.md

@@ -330,3 +330,17 @@ git config --global credential.helper 'cache --timeout 3600'
 ```
 
 You can adjust the timeout (3600 seconds, above) to your liking, if you would prefer git to remember your passwords for longer.
+
+#### I get an HTTP/2 related error
+
+Some network provider do not support HTTP/2 apparently. If you get:
+
+```bash
+error: RPC failed; curl 92 HTTP/2 stream 5 was not closed cleanly: CANCEL (err 8)
+```
+
+or similar message, try to disable HTTP/2 with something like:
+
+```
+git config --global http.version HTTP/1.1
+```

{alibuild-1.17.7 → alibuild-1.17.10}/docs/mkdocs.yml

@@ -4,6 +4,9 @@ repo_url: https://github.com/alisw/alibuild
 
 theme:
   name: material
+  logo: alice_logo.png
+
+extra_css: [stylesheets/extra.css]
 
 use_directory_urls: false
 
@@ -12,6 +15,7 @@ nav:
   - Quickstart: quick.md
   - User manual: user.md
   - Reference: reference.md
+  - Troubleshooting: troubleshooting.md
   - ALICE/O2 tutorial: "https://alice-doc.github.io/alice-analysis-tutorial/building/"
 
 markdown_extensions:
@@ -24,3 +28,9 @@ markdown_extensions:
   - pymdownx.superfences
   - toc:
       permalink: "#"
+
+plugins:
+  - redirects:
+      redirect_maps:
+        'o2-tutorial.html': 'https://alice-doc.github.io/alice-analysis-tutorial/building/'
+        'tutorial.html': 'https://alice-doc.github.io/alice-analysis-tutorial/building/'

{alibuild-1.17.7 → alibuild-1.17.10}/setup.py

@@ -81,7 +81,7 @@ setup(
         # The 7.* series removed support for Python 3.6.
         'setuptools_scm<7.0.0' if sys.version_info < (3, 7) else
         'setuptools_scm'
-    ],
+    ] + (['packaging<=23'] if sys.version_info <(3, 7) else []),
 
     # List run-time dependencies here.  These will be installed by pip when
     # your project is installed. For an analysis of "install_requires" vs pip's

alibuild-1.17.7/docs/docs/o2-tutorial.md

@@ -1,5 +0,0 @@
----
-subtitle: O2 build tutorial
-dest_url: https://alice-doc.github.io/alice-analysis-tutorial/building/
-layout: redirect
----

alibuild-1.17.7/docs/docs/tutorial.md

@@ -1,5 +0,0 @@
----
-subtitle: Basic tutorial
-dest_url: https://alice-doc.github.io/alice-analysis-tutorial/building/
-layout: redirect
----