alibuild 2.0.0__tar.gz → 2.0.1__tar.gz

{alibuild-2.0.0 → alibuild-2.0.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: alibuild
-Version: 2.0.0
+Version: 2.0.1
 Summary: ALICE Build Tool
 Home-page: https://alisw.github.io/alibuild
 Author: Giulio Eulisse

{alibuild-2.0.0 → alibuild-2.0.1}/alibuild.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: alibuild
-Version: 2.0.0
+Version: 2.0.1
 Summary: ALICE Build Tool
 Home-page: https://alisw.github.io/alibuild
 Author: Giulio Eulisse

{alibuild-2.0.0 → alibuild-2.0.1}/alibuild_helpers/_version.py

@@ -28,7 +28,7 @@ version_tuple: VERSION_TUPLE
 commit_id: COMMIT_ID
 __commit_id__: COMMIT_ID
 
-__version__ = version = '2.0.0'
-__version_tuple__ = version_tuple = (2, 0, 0)
+__version__ = version = '2.0.1'
+__version_tuple__ = version_tuple = (2, 0, 1)
 
-__commit_id__ = commit_id = 'g13a7d02c4'
+__commit_id__ = commit_id = 'g0096f2c8c'

{alibuild-2.0.0 → alibuild-2.0.1}/alibuild_helpers/build.py

@@ -1382,7 +1382,7 @@ def doBuild(args, parser):
     published = True
     if fileHash == spec["hash"] and not spec["is_devel_pkg"] and \
        isinstance(syncHelper, REAPIRemoteSync) and getattr(syncHelper, "writeStore", ""):
-      published = syncHelper.is_published(spec["remote_revision_hash"])
+      published = syncHelper.is_published(spec["remote_revision_hash"], keep_alive=True)
       if not published:
         bound_unpublished_rebuild(p, rebuilt_unpublished, syncHelper.acWriteStore)
         info("%s is built locally but not published to %s; rebuilding it so it can "

{alibuild-2.0.0 → alibuild-2.0.1}/alibuild_helpers/log.py

@@ -10,6 +10,22 @@ def dieOnError(err, msg) -> None:
     error("%s", msg)
     sys.exit(1)
 
+# Credentials reach the log through command lines: the docker invocation carries
+# every -e NAME=VALUE, and CI passes JALIEN_TOKEN_KEY (a PEM private key) and
+# CCDB auth tokens that way. The value may be shell-quoted or, where the command
+# is " ".join()ed, bare and spanning several lines. A key whose END marker is
+# not in the same message is redacted to the end of the message.
+_PRIVATE_KEY_RE = re.compile(
+  r"-----BEGIN ([A-Z ]*)PRIVATE KEY-----.*?(?:-----END \1PRIVATE KEY-----|\Z)", re.DOTALL)
+_SECRET_ASSIGNMENT_RE = re.compile(
+  r"\b([A-Za-z0-9_]*(?:TOKEN|SECRET|PASSWORD|PASSWD)[A-Za-z0-9_]*=)"
+  r"('[^']*'|\"[^\"]*\"|[^\s'\"]+)")
+
+def redact_secrets(msg):
+  """Hide private keys and the values of *TOKEN*/*SECRET*/*PASSWORD* assignments."""
+  msg = _PRIVATE_KEY_RE.sub(r"-----BEGIN \1PRIVATE KEY----- <redacted>", msg)
+  return _SECRET_ASSIGNMENT_RE.sub(r"\1<redacted>", msg)
+
 class LogFormatter(logging.Formatter):
   def __init__(self, fmtstr) -> None:
     self.fmtstr = fmtstr
@@ -25,7 +41,7 @@ class LogFormatter(logging.Formatter):
     # args to an already-expanded string -- "not all arguments converted".
     # Harmless with only our handler attached, which is why it went unnoticed;
     # under pytest, whose logging plugin adds its own handler, it aborts.
-    msg = record.getMessage()
+    msg = redact_secrets(record.getMessage())
     if record.levelno == logging.BANNER and sys.stdout.isatty():
       lines = msg.split("\n")
       return "\n\033[1;34m==>\033[m \033[1m%s\033[m" % lines[0] + \

{alibuild-2.0.0 → alibuild-2.0.1}/alibuild_helpers/sync.py

@@ -1,6 +1,7 @@
 """Sync backends for alibuild."""
 
 import glob
+import hashlib
 import os
 import os.path
 import re
@@ -93,6 +94,7 @@ class HttpRemoteSync:
           # Destination specified -- file (dest) or buffer (returnResult).
           # Use requests in stream mode
           resp = get(url, stream=True, verify=not self.insecure, timeout=self.httpTimeoutSec)
+          cas_digest = None
           if redirect_base:
             # S3 only turns x-amz-website-redirect-location into a 301 when the
             # bucket is served as a website; on the REST endpoint the object is
@@ -115,6 +117,16 @@ class HttpRemoteSync:
               debug("Following store redirect to %s", url)
               resp = get(url, stream=True, verify=not self.insecure,
                          timeout=self.httpTimeoutSec)
+              # A content-addressed target names its own digest: check against it.
+              cas_digest = re.search(r"(?:^|/)cas/sha256/[0-9a-f]{2}/([0-9a-f]{64})$", target)
+          if resp.status_code == 404:
+            # Retrying will not bring it back. After a store redirect this is a
+            # stub whose CAS blob is gone (an expired ephemeral artifact): not
+            # available, so the package is rebuilt, as the reapi backend does.
+            (warning if redirect_base else debug)("GET %s: not found", url)
+            return None
+          resp.raise_for_status()
+          sha256 = hashlib.sha256()
           size = int(resp.headers.get("content-length", "-1"))
           downloaded = 0
           reportTime = time.time()
@@ -125,6 +137,8 @@ class HttpRemoteSync:
             for chunk in filter(bool, resp.iter_content(chunk_size=32768)):
               if destFp:
                 destFp.write(chunk)
+              if cas_digest:
+                sha256.update(chunk)
               if returnResult:
                 result.append(chunk)
               downloaded += len(chunk)
@@ -141,6 +155,11 @@ class HttpRemoteSync:
 
           if size not in (downloaded, -1):
             raise PartialDownloadError(downloaded, size)
+          if cas_digest and sha256.hexdigest() != cas_digest.group(1):
+            warning("GET %s: content does not match its sha256, ignoring it", url)
+            if dest:
+              os.unlink(dest+".tmp")
+            return None
           if dest:
             os.rename(dest+".tmp", dest)  # we should not have errors here
           return b''.join(result) if returnResult else True
@@ -240,7 +259,7 @@ class HttpRemoteSync:
       # Fetch manifest file with initial symlinks. This file is updated
       # regularly; we use it to avoid many small network requests.
       manifest = self.getRetry(f"{self.remoteStore}/{links_path}.manifest",
-                               returnResult=True, session=session)
+                               returnResult=True, session=session) or b""
       symlinks = {
         linkname.decode("utf-8"): target.decode("utf-8")
         for linkname, sep, target in (line.partition(b"\t")

{alibuild-2.0.0 → alibuild-2.0.1}/alibuild_helpers/sync_reapi.py

@@ -520,7 +520,7 @@ class REAPIRemoteSync(Boto3RemoteSync):
       return None
     return json.loads(obj["Body"].read())
 
-  def is_published(self, action_hash):
+  def is_published(self, action_hash, keep_alive=False):
     """Whether an action is fully published: its Action Cache entry exists *and* the
     CAS blob that entry names is present.
 
@@ -531,7 +531,12 @@ class REAPIRemoteSync(Boto3RemoteSync):
     entry and bytes) is not something a dependent can be published against either.
 
     Two HEAD-class requests, no download -- the blob is checked by presence, since it
-    is content-addressed and its key is its own digest."""
+    is content-addressed and its key is its own digest.
+
+    keep_alive also promotes or LRU-refreshes the blob (see _maybe_promote). The
+    build asks this for every package it reuses, rebuilt, fetched or installed
+    already: a fetched one never reaches the upload, which is the only other
+    place a blob is kept alive."""
     entry = self.read_ac_entry(action_hash)
     if entry is None:
       return False
@@ -539,7 +544,12 @@ class REAPIRemoteSync(Boto3RemoteSync):
     if not content_hash:
       # validate-system nodes produce no tarball: the entry alone is the artifact.
       return (entry.get("action") or {}).get("kind") == "validate-system"
-    return self._exists(self.remoteStore, resolve_cas_path(content_hash, algo))
+    cas_path = resolve_cas_path(content_hash, algo)
+    if not self._exists(self.remoteStore, cas_path):
+      return False
+    if keep_alive:
+      self._maybe_promote(cas_path)
+    return True
 
   def download_blob(self, content_hash, dest, algo="sha256"):
     """Download a ledger (input) blob -- e.g. a source bundle -- to dest."""
@@ -601,8 +611,10 @@ class REAPIRemoteSync(Boto3RemoteSync):
 
   def _maybe_promote(self, cas_path):
     """When uploading as 'permanent' and the blob already exists tagged
-    ephemeral, promote it to permanent so it is no longer LRU-expired."""
+    ephemeral, promote it to permanent so it is no longer LRU-expired.
+    Otherwise LRU-refresh it: reusing a blob must keep it alive."""
     if self.storage != "permanent":
+      self._touch_if_expiring(cas_path)
       return
     if self._retention_of(self.writeStore, cas_path) == "ephemeral":
       self.s3.put_object_tagging(
@@ -628,21 +640,25 @@ class REAPIRemoteSync(Boto3RemoteSync):
   def _touch_if_expiring(self, cas_path):
     """LRU-refresh: if the blob is ephemeral and within REFRESH_WITHIN_DAYS of
     its EPHEMERAL_TTL_DAYS expiry, copy it onto itself (preserving the tag) to
-    reset last-modified. Best-effort and only when we can write the same bucket."""
+    reset last-modified. Best-effort, in the bucket we write: an upload that
+    dedups needs its blob there to stay, wherever it reads from."""
     from botocore.exceptions import ClientError
-    if not self.writeStore or self.writeStore != self.remoteStore:
+    if not self.writeStore:
       return
     try:
-      if self._retention_of(self.remoteStore, cas_path) != "ephemeral":
-        return
-      head = self.s3.head_object(Bucket=self.remoteStore, Key=cas_path)
+      head = self.s3.head_object(Bucket=self.writeStore, Key=cas_path)
       age_days = (datetime.now(timezone.utc) - head["LastModified"]).days
-      if age_days < self.EPHEMERAL_TTL_DAYS - self.REFRESH_WITHIN_DAYS:
+      if age_days < self.EPHEMERAL_TTL_DAYS - self.REFRESH_WITHIN_DAYS or \
+         self._retention_of(self.writeStore, cas_path) != "ephemeral":
         return
       debug("Refreshing ephemeral %s (%d days old)", cas_path, age_days)
+      # REPLACE, with the same metadata: S3 and Ceph RGW refuse to copy an
+      # object onto itself with MetadataDirective=COPY (InvalidRequest).
       self.s3.copy_object(Bucket=self.writeStore, Key=cas_path,
-                          CopySource={"Bucket": self.remoteStore, "Key": cas_path},
-                          MetadataDirective="COPY", TaggingDirective="COPY")
+                          CopySource={"Bucket": self.writeStore, "Key": cas_path},
+                          MetadataDirective="REPLACE", Metadata=head.get("Metadata", {}),
+                          ContentType=head.get("ContentType", "binary/octet-stream"),
+                          TaggingDirective="COPY")
     except ClientError as exc:
       debug("Could not refresh %s: %s", cas_path, exc)
 

{alibuild-2.0.0 → alibuild-2.0.1}/alienv

@@ -122,10 +122,21 @@ function detectShell() {
 
 function collectModules() {
   if [[ -z "$NO_REFRESH" ]]; then
-    # Collect all modulefiles in one place
-    rm -rf $WORK_DIR/MODULES/$ARCHITECTURE
-    mkdir -p $WORK_DIR/MODULES/$ARCHITECTURE/BASE
-    cat > $WORK_DIR/MODULES/$ARCHITECTURE/BASE/1.0 <<EOF
+    # Collect all modulefiles in one place. Other alienv calls may read the
+    # directory meanwhile, so it is never emptied: each modulefile is written
+    # next to it and renamed into place, and only then are the modulefiles of
+    # packages which are gone removed.
+    local DEST=$WORK_DIR/MODULES/$ARCHITECTURE STAGE PKG PKGVER PKGNAME MODFILE MODNAME
+    mkdir -p "$DEST"
+    STAGE=$(mktemp -d "$WORK_DIR/MODULES/.refresh-$ARCHITECTURE.XXXXXX") || return 1
+    function publish() {
+      # Install $1 (a file, or - for stdin) as the modulefile $2, relative to $DEST.
+      mkdir -p "$DEST/${2%/*}"
+      if [[ $1 == - ]]; then cat > "$STAGE/file"; else cp "$1" "$STAGE/file"; fi
+      mv -f "$STAGE/file" "$DEST/$2"
+      echo "$2" >> "$STAGE/published"
+    }
+    publish - BASE/1.0 <<EOF
 #%Module1.0
 set base_path $WORK_DIR/$ARCHITECTURE
 setenv BASEDIR \$base_path
@@ -144,10 +155,15 @@ EOF
       for MODFILE in "$PKG"/etc/modulefiles/*; do
         [[ -f "$MODFILE" ]] || continue
         MODNAME=${MODFILE##*/}
-        mkdir -p "$WORK_DIR/MODULES/$ARCHITECTURE/$MODNAME"
-        cp "$MODFILE" "$WORK_DIR/MODULES/$ARCHITECTURE/$MODNAME/$PKGVER"
+        publish "$MODFILE" "$MODNAME/$PKGVER"
       done
     done < <(find $WORK_DIR/$ARCHITECTURE -maxdepth 2 -mindepth 2 2> /dev/null)
+    # BASE/1.0 is always published, so the list is never empty here.
+    (cd "$DEST" && find . -type f ! -name '.*' | sed -e 's|^\./||') | grep -vxF -f "$STAGE/published" | \
+      while read MODFILE; do rm -f "$DEST/$MODFILE"; done
+    find "$DEST" -mindepth 1 -type d -empty -delete 2> /dev/null
+    rm -rf "$STAGE"
+    unset -f publish
   else
     printf "${EY}WARNING: not updating modulefiles${EZ}\n" >&2
   fi

alibuild-2.0.1/tests/test_log.py

@@ -0,0 +1,90 @@
+import unittest
+from unittest.mock import MagicMock, patch
+
+import logging
+
+from alibuild_helpers.log import dieOnError, ProgressPrint, LogFormatter, redact_secrets
+
+
+FAKE_KEY = ("-----BEGIN RSA PRIVATE KEY-----\nMIIEfakefakefake\nAAAAsecretbody\n"
+            "-----END RSA PRIVATE KEY-----")
+
+
+class LogTestCase(unittest.TestCase):
+    def test_redact_docker_command(self) -> None:
+        """The docker command line, joined unquoted, must not show the key."""
+        cmd = " ".join(["docker", "run", "-e", "JALIEN_TOKEN_KEY=" + FAKE_KEY,
+                        "-e", "ALICEO2_CCDB_AUTH_TOKEN=abc123", "-e", "ARCHITECTURE=slc9_x86-64",
+                        "-e", "ALIBUILD_PREFER_SYSTEM_KEY=gcc", "image"])
+        out = redact_secrets(cmd)
+        self.assertNotIn("secretbody", out)
+        self.assertNotIn("MIIEfake", out)
+        self.assertNotIn("abc123", out)
+        self.assertIn("ARCHITECTURE=slc9_x86-64", out)
+        self.assertIn("ALIBUILD_PREFER_SYSTEM_KEY=gcc", out)
+        self.assertIn("image", out)
+
+    def test_redact_quoted(self) -> None:
+        """Shell-quoted values are redacted whole, including the quotes."""
+        out = redact_secrets("env GITHUB_TOKEN='tok en' JALIEN_TOKEN_KEY=\"%s\" bash" % FAKE_KEY)
+        self.assertEqual(out.count("<redacted>"), 2)
+        self.assertNotIn("tok en", out)
+        self.assertNotIn("secretbody", out)
+        self.assertTrue(out.endswith(" bash"))
+
+    def test_redact_unterminated_key(self) -> None:
+        """A key cut off before its END marker is redacted to the end."""
+        out = redact_secrets("before " + FAKE_KEY.split("-----END")[0])
+        self.assertEqual(out, "before -----BEGIN RSA PRIVATE KEY----- <redacted>")
+
+    def test_formatter_redacts(self) -> None:
+        """Everything going through the formatter is redacted, args included."""
+        record = logging.LogRecord("alibuild", logging.DEBUG, __file__, 1,
+                                   "Build command: %s", ("docker run -e JALIEN_TOKEN_KEY=" + FAKE_KEY,), None)
+        out = LogFormatter("%(levelname)s: %(message)s").format(record)
+        self.assertNotIn("secretbody", out)
+        self.assertIn("Build command: docker run -e JALIEN_TOKEN_KEY=<redacted>", out)
+
+    @patch("alibuild_helpers.log.error")
+    @patch("alibuild_helpers.log.sys")
+    def test_dieOnError(self, mock_sys, mock_error):
+        """Check dieOnError dies on error."""
+        dieOnError(True, "Message")
+        mock_sys.exit.assert_called_once_with(1)
+        mock_error.assert_called_once_with("%s", "Message")
+        mock_sys.reset_mock()
+        mock_error.reset_mock()
+        dieOnError(False, "Message")
+        mock_error.assert_not_called()
+        mock_sys.exit.assert_not_called()
+
+    @patch("sys.stdout.isatty", new=MagicMock(return_value=True))
+    @patch("sys.stderr", new=MagicMock(return_value=True))
+    def test_ProgressPrint(self) -> None:
+        """Make sure ProgressPrint updates correctly."""
+        # ProgressPrint only parses messages every 0.5s. Trick it into thinking
+        # the last message was <interval> seconds ago.
+        interval = 0.6
+        p = ProgressPrint("begin")
+        p("%s", "First message")
+        self.assertEqual(p.percent, -1)
+        p("%s", "[100/200] Update too fast")
+        self.assertEqual(p.percent, -1)  # unchanged
+        p.lasttime -= interval
+        p("%s", "Has percentage: 80%")
+        self.assertEqual(p.percent, 80)
+        p.lasttime -= interval
+        p("%s", "No percentage")
+        self.assertEqual(p.percent, 80)  # unchanged
+        p.lasttime -= interval
+        p("%s", "[100/200] Building CXX object this/is/a/ninja/test")
+        self.assertEqual(p.percent, 50)
+        p.lasttime -= interval
+        p("%s", "[100/0] Wrong-styled Ninja progress")
+        self.assertEqual(p.percent, 50)  # unchanged
+        p.lasttime -= interval
+        p("%s", "Last message")
+        p.end()
+
+if __name__ == '__main__':
+  unittest.main()

{alibuild-2.0.0 → alibuild-2.0.1}/tests/test_sync.py

@@ -1,6 +1,8 @@
+import hashlib
 import os
 import os.path
 import sys
+import tempfile
 import unittest
 from io import BytesIO
 
@@ -219,6 +221,51 @@ class SyncTestCase(unittest.TestCase):
         self.assertFalse([u for u in requested if u.startswith(store + "/https")],
                          "the absolute target was joined onto the store: %s" % requested)
 
+    def fetch_through_stub(self, cas_response):
+        """Fetch GOOD_SPEC from a store whose object redirects to a CAS blob, which
+        answers with cas_response. Returns (requested URLs, files left behind)."""
+        store = "https://localhost/legacy"
+        blob = "https://localhost/cas/cas/sha256/%s/%s" % (self.digest[:2], self.digest)
+        requested = []
+
+        def get(url, *args, **kw):
+            requested.append(url)
+            if url.endswith(tarball_name(GOOD_SPEC)) and "/store/" in url:
+                return MockRequest([{"name": tarball_name(GOOD_SPEC)}], redirect=blob)
+            if url == blob:
+                return cas_response
+            return self.mock_get(url, *args, **kw)
+
+        with tempfile.TemporaryDirectory() as workdir, \
+             patch("requests.Session.get", side_effect=get):
+            syncer = sync.HttpRemoteSync(remoteStore=store, architecture=ARCHITECTURE,
+                                         workdir=workdir, insecure=False)
+            syncer.httpBackoff = 0
+            syncer.fetch_tarball(GOOD_SPEC)
+            left = [f for _, _, files in os.walk(workdir) for f in files]
+        return [u for u in requested if u == blob], left
+
+    @patch("alibuild_helpers.sync.warning", new=MagicMock())
+    def test_http_missing_cas_blob_is_not_a_tarball(self):
+        """A stub whose CAS blob is gone (expired) gives no tarball, no retries and
+        no exception, so the package is rebuilt. Writing the 404 body as the
+        tarball is what made a build fail with "gzip: not in gzip format"."""
+        self.digest = "fa" * 32
+        gone = MockRequest(None)
+        gone.iter_content = lambda chunk_size=10: iter([b"NoSuchKey"])
+        gone.headers["content-length"] = "9"
+        requested, left = self.fetch_through_stub(gone)
+        self.assertEqual(len(requested), 1)
+        self.assertEqual(left, [])
+
+    @patch("alibuild_helpers.sync.warning", new=MagicMock())
+    def test_http_cas_blob_checked_against_its_digest(self):
+        """Bytes behind a cas/sha256/<digest> redirect must hash to <digest>."""
+        good = hashlib.sha256(b"x" * 123456).hexdigest()
+        for self.digest, kept in ((good, [tarball_name(GOOD_SPEC)]), ("ab" * 32, [])):
+            _, left = self.fetch_through_stub(MockRequest([{"name": "blob"}]))
+            self.assertEqual(left, kept)
+
     def test_s3cmd_follows_store_redirect(self):
         """s3cmd cannot see the redirect header, so the stub is spotted by content."""
         commands = []

{alibuild-2.0.0 → alibuild-2.0.1}/tests/test_sync_reapi.py

@@ -78,18 +78,23 @@ class REAPIRemoteSyncTestCase(unittest.TestCase):
         reapi.s3 = client
         return reapi
 
-    def make_client(self, existing=()):
+    def make_client(self, existing=(), age_days=0, retention=None):
         """Mock S3 client: head_object 404s unless the key is in `existing`,
-        and all directory listings are empty (so uploads see no conflicts)."""
+        and all directory listings are empty (so uploads see no conflicts).
+        Existing objects are `age_days` old and tagged with `retention`."""
         from botocore.exceptions import ClientError
+        from datetime import datetime, timezone, timedelta
 
         def head_object(Bucket, Key):
             if Key in existing:
-                return {"ContentLength": 4096}
+                return {"ContentLength": 4096, "ContentType": "application/gzip",
+                        "LastModified": datetime.now(timezone.utc) - timedelta(days=age_days)}
             raise ClientError({"Error": {"Code": "404"}}, "head_object")
 
         return MagicMock(
             head_object=MagicMock(side_effect=head_object),
+            get_object_tagging=MagicMock(return_value={"TagSet": [
+                {"Key": "retention", "Value": retention}] if retention else []}),
             get_paginator=lambda method: MagicMock(
                 paginate=lambda **kw: [{"Contents": []}]),
             put_object=MagicMock(return_value=None),
@@ -231,6 +236,62 @@ class REAPIRemoteSyncTestCase(unittest.TestCase):
         client.upload_file.assert_not_called()
         self.assertIn(resolve_ac_path(ARCHITECTURE, REAPI_HASH), self.put_keys(client))
 
+    @patch("alibuild_helpers.sync_reapi.file_digest",
+           new=MagicMock(return_value=REAPI_CONTENT_HASH))
+    @patch("os.path.getsize", new=MagicMock(return_value=4096))
+    @patch("os.listdir",
+           new=lambda path: [tarball_name(REAPI_SPEC)] if path.endswith("-1") else [])
+    @patch("os.readlink", new=MagicMock(return_value="../../store/de/dead/x.tar.gz"))
+    @patch("os.path.islink", new=MagicMock(return_value=True))
+    def test_upload_dedup_refreshes_expiring_ephemeral(self) -> None:
+        """Identical rebuilds always dedup, so the dedup must keep the blob alive:
+        otherwise a package that never changes expires 90 days after its first
+        build, while its AC entry and store stub still point at it."""
+        cas = resolve_cas_path(REAPI_CONTENT_HASH)
+        for age_days, refreshed in ((75, True), (10, False)):
+            client = self.make_client(existing={cas}, age_days=age_days,
+                                      retention="ephemeral")
+            self.make_sync(client).upload_symlinks_and_tarball(REAPI_SPEC)
+            client.upload_file.assert_not_called()
+            if not refreshed:
+                client.copy_object.assert_not_called()
+                continue
+            # Copied onto itself in the bucket we write, tags kept. COPY would be
+            # refused by S3 and Ceph RGW for a self-copy, so it must be REPLACE.
+            client.copy_object.assert_called_once()
+            kw = client.copy_object.call_args.kwargs
+            self.assertEqual((kw["Bucket"], kw["Key"]), ("bucket", cas))
+            self.assertEqual(kw["CopySource"], {"Bucket": "bucket", "Key": cas})
+            self.assertEqual(kw["MetadataDirective"], "REPLACE")
+            self.assertEqual(kw["ContentType"], "application/gzip")
+            self.assertEqual(kw["TaggingDirective"], "COPY")
+
+    def test_is_published_keep_alive(self) -> None:
+        """A package the build takes from the store never reaches the upload, so
+        is_published(keep_alive=True), asked for every reused package, is what
+        promotes its blob on a permanent build (or LRU-refreshes it)."""
+        cas = resolve_cas_path(REAPI_CONTENT_HASH)
+        entry = {"result": {"outputDigest": "sha256:" + REAPI_CONTENT_HASH}}
+        for keep_alive in (False, True):
+            client = self.make_client(existing={cas}, age_days=5, retention="ephemeral")
+            reapi = self.make_sync(client)
+            reapi.storage = "permanent"
+            reapi.read_ac_entry = MagicMock(return_value=entry)
+            self.assertTrue(reapi.is_published(REAPI_HASH, keep_alive=keep_alive))
+            if keep_alive:
+                client.put_object_tagging.assert_called_once_with(
+                    Bucket="bucket", Key=cas,
+                    Tagging={"TagSet": [{"Key": "retention", "Value": "permanent"}]})
+            else:
+                client.put_object_tagging.assert_not_called()
+        # An ephemeral build refreshes it only when it is close to expiry.
+        client = self.make_client(existing={cas}, age_days=70, retention="ephemeral")
+        reapi = self.make_sync(client)
+        reapi.read_ac_entry = MagicMock(return_value=entry)
+        self.assertTrue(reapi.is_published(REAPI_HASH, keep_alive=True))
+        client.copy_object.assert_called_once()
+        client.put_object_tagging.assert_not_called()
+
     @patch("alibuild_helpers.sync_reapi.file_digest",
            new=MagicMock(return_value=REAPI_CONTENT_HASH))
     @patch("os.path.getsize", new=MagicMock(return_value=4096))

alibuild-2.0.0/tests/test_log.py

@@ -1,50 +0,0 @@
-import unittest
-from unittest.mock import MagicMock, patch
-
-from alibuild_helpers.log import dieOnError, ProgressPrint
-
-
-class LogTestCase(unittest.TestCase):
-    @patch("alibuild_helpers.log.error")
-    @patch("alibuild_helpers.log.sys")
-    def test_dieOnError(self, mock_sys, mock_error):
-        """Check dieOnError dies on error."""
-        dieOnError(True, "Message")
-        mock_sys.exit.assert_called_once_with(1)
-        mock_error.assert_called_once_with("%s", "Message")
-        mock_sys.reset_mock()
-        mock_error.reset_mock()
-        dieOnError(False, "Message")
-        mock_error.assert_not_called()
-        mock_sys.exit.assert_not_called()
-
-    @patch("sys.stdout.isatty", new=MagicMock(return_value=True))
-    @patch("sys.stderr", new=MagicMock(return_value=True))
-    def test_ProgressPrint(self) -> None:
-        """Make sure ProgressPrint updates correctly."""
-        # ProgressPrint only parses messages every 0.5s. Trick it into thinking
-        # the last message was <interval> seconds ago.
-        interval = 0.6
-        p = ProgressPrint("begin")
-        p("%s", "First message")
-        self.assertEqual(p.percent, -1)
-        p("%s", "[100/200] Update too fast")
-        self.assertEqual(p.percent, -1)  # unchanged
-        p.lasttime -= interval
-        p("%s", "Has percentage: 80%")
-        self.assertEqual(p.percent, 80)
-        p.lasttime -= interval
-        p("%s", "No percentage")
-        self.assertEqual(p.percent, 80)  # unchanged
-        p.lasttime -= interval
-        p("%s", "[100/200] Building CXX object this/is/a/ninja/test")
-        self.assertEqual(p.percent, 50)
-        p.lasttime -= interval
-        p("%s", "[100/0] Wrong-styled Ninja progress")
-        self.assertEqual(p.percent, 50)  # unchanged
-        p.lasttime -= interval
-        p("%s", "Last message")
-        p.end()
-
-if __name__ == '__main__':
-  unittest.main()