scriptworker 63.3.0__tar.gz → 63.4.1__tar.gz

{scriptworker-63.3.0 → scriptworker-63.4.1}/HISTORY.rst

@@ -4,6 +4,21 @@ Change Log
 All notable changes to this project will be documented in this file.
 This project adheres to `Semantic Versioning <http://semver.org/>`__.
 
+63.4.1 - 2026-08-11
+-------------------
+
+Fixed
+~~~~~
+
+- Allow enterprise tasks to have comm source repositories
+
+63.4.0 - 2026-08-11
+-------------------
+
+Added
+~~~~~
+- set Content-MD5 header to avoid silent data corruption uploading artifacts (#815)
+
 63.3.0 - 2026-08-05
 -------------------
 

{scriptworker-63.3.0 → scriptworker-63.4.1}/PKG-INFO

@@ -1,6 +1,6 @@
-Metadata-Version: 2.4
+Metadata-Version: 2.5
 Name: scriptworker
-Version: 63.3.0
+Version: 63.4.1
 Summary: Taskcluster scriptworker
 Author-email: Mozilla Release Engineering <releng@mozilla.com>
 License-File: LICENSE

{scriptworker-63.3.0 → scriptworker-63.4.1}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "scriptworker"
-version = "63.3.0"
+version = "63.4.1"
 description = "Taskcluster scriptworker"
 authors = [
     { name = "Mozilla Release Engineering", email = "releng@mozilla.com" },

{scriptworker-63.3.0 → scriptworker-63.4.1}/src/scriptworker/artifacts.py

@@ -6,6 +6,7 @@ in S3.
 """
 
 import asyncio
+import base64
 import fnmatch
 import gzip
 import logging
@@ -21,7 +22,15 @@ from taskcluster.exceptions import TaskclusterFailure
 from scriptworker.client import validate_artifact_url
 from scriptworker.exceptions import DownloadError, ScriptWorkerRetryException, ScriptWorkerTaskException
 from scriptworker.task import get_decision_task_id, get_run_id, get_task_id
-from scriptworker.utils import add_enumerable_item_to_dict, download_file, get_loggable_url, raise_future_exceptions, retry_async, semaphore_wrapper
+from scriptworker.utils import (
+    add_enumerable_item_to_dict,
+    download_file,
+    get_hash,
+    get_loggable_url,
+    raise_future_exceptions,
+    retry_async,
+    semaphore_wrapper,
+)
 
 log = logging.getLogger(__name__)
 
@@ -163,6 +172,8 @@ async def create_artifact(context, path, target_path, content_type, content_enco
     payload = {"storageType": storage_type, "expires": expires or get_expiration_arrow(context).isoformat(), "contentType": content_type}
     args = [get_task_id(context.claim_task), get_run_id(context.claim_task), target_path, payload]
 
+    content_md5 = await asyncio.to_thread(get_content_md5, path)
+
     tc_response = await context.temp_queue.createArtifact(*args)
     skip_auto_headers = [aiohttp.hdrs.CONTENT_TYPE]
     loggable_url = get_loggable_url(tc_response["putUrl"])
@@ -172,7 +183,7 @@ async def create_artifact(context, path, target_path, content_type, content_enco
             async with context.session.put(
                 tc_response["putUrl"],
                 data=fh,
-                headers=_craft_artifact_put_headers(content_type, content_encoding),
+                headers=_craft_artifact_put_headers(content_type, content_md5, content_encoding),
                 skip_auto_headers=skip_auto_headers,
                 compress=False,
             ) as resp:
@@ -180,6 +191,12 @@ async def create_artifact(context, path, target_path, content_type, content_enco
                 response_text = await resp.text()
                 log.info(response_text)
                 if resp.status not in (200, 204):
+                    if content_md5 is not None and "BadDigest" in (response_text or ""):
+                        log.error(
+                            "{} was corrupted in transit: the storage backend rejected the body as not matching Content-MD5 {}. Retrying.".format(
+                                target_path, content_md5
+                            )
+                        )
                     raise ScriptWorkerRetryException("Bad status {}".format(resp.status))
 
 
@@ -206,9 +223,22 @@ async def create_link_artifact(context, target_path, link_to, content_type, expi
     await context.temp_queue.createArtifact(*args)
 
 
-def _craft_artifact_put_headers(content_type, encoding=None):
+def get_content_md5(path):
+    """Get the base64-encoded md5 digest of a file, formatted for ``Content-MD5``.
+
+    Args:
+        path (str): the path to the file to digest.
+
+    Returns:
+        str: the base64-encoded md5 digest.
+
+    """
+    return base64.b64encode(bytes.fromhex(get_hash(path, hash_alg="md5"))).decode("ascii")
+
+
+def _craft_artifact_put_headers(content_type, content_md5, encoding=None):
     log.debug("{} {}".format(content_type, encoding))
-    headers = {aiohttp.hdrs.CONTENT_TYPE: content_type}
+    headers = {aiohttp.hdrs.CONTENT_TYPE: content_type, aiohttp.hdrs.CONTENT_MD5: content_md5}
 
     if encoding is not None:
         headers[aiohttp.hdrs.CONTENT_ENCODING] = encoding

{scriptworker-63.3.0 → scriptworker-63.4.1}/src/scriptworker/cot/verify.py

@@ -1952,12 +1952,26 @@ def get_source_url(obj):
     log.debug("Getting source url for {} {}...".format(obj.name, obj.task_id))
     repo = get_repo(obj.task, source_env_prefix=source_env_prefix)
     source = task["metadata"]["source"]
-    if repo and not verify_repo_matches_url(repo, source):
-        raise CoTError(
-            "{name} {task_id}: {source_env_prefix} {repo} doesn't match source {source}!".format(
-                name=obj.name, task_id=obj.task_id, source_env_prefix=source_env_prefix, repo=repo, source=source
+    cot_product = obj.context.config["cot_product"]
+    match_repo = repo and verify_repo_matches_url(repo, source)
+    if cot_product != "enterprise":
+        if repo and not match_repo:
+            raise CoTError(
+                "{name} {task_id}: {source_env_prefix} {repo} doesn't match source {source}!".format(
+                    name=obj.name, task_id=obj.task_id, source_env_prefix=source_env_prefix, repo=repo, source=source
+                )
             )
-        )
+    else:
+        comm_repo = get_repo(obj.task, source_env_prefix="COMM", repo_type="head")
+        match_comm_repo = comm_repo and verify_repo_matches_url(comm_repo, source)
+
+        if not match_repo and not match_comm_repo:
+            raise CoTError(
+                "{name} {task_id}: {source_env_prefix} {repo} or {comm_repo} doesn't match source {source}!".format(
+                    name=obj.name, task_id=obj.task_id, source_env_prefix=source_env_prefix, repo=repo, comm_repo=comm_repo, source=source
+                )
+            )
+
     log.info("{} {}: found {}".format(obj.name, obj.task_id, source))
     return source