scriptworker 63.2.0__tar.gz → 63.4.1__tar.gz

{scriptworker-63.2.0 → scriptworker-63.4.1}/HISTORY.rst

@@ -4,6 +4,43 @@ Change Log
 All notable changes to this project will be documented in this file.
 This project adheres to `Semantic Versioning <http://semver.org/>`__.
 
+63.4.1 - 2026-08-11
+-------------------
+
+Fixed
+~~~~~
+
+- Allow enterprise tasks to have comm source repositories
+
+63.4.0 - 2026-08-11
+-------------------
+
+Added
+~~~~~
+- set Content-MD5 header to avoid silent data corruption uploading artifacts (#815)
+
+63.3.0 - 2026-08-05
+-------------------
+
+Added
+~~~~~
+
+- `cot_product` is now optional. When unset, `by-cot-product` mappings resolve to
+  `None`, and `check_config` requires `verify_chain_of_trust` to be off (#810)
+- `download_file` accepts an optional `expected_content_type` kwarg that rejects
+  `text/html` responses when another type was requested (#791)
+
+Fixed
+~~~~~
+
+- Event loop creation in a fresh process, which raised a `RuntimeError` on
+  Python 3.14 since `asyncio.get_event_loop()` no longer creates a loop (#811)
+- JSON/YAML load failures now include the file path and, for downloaded
+  artifacts, the source URL in the error message (#791)
+- `load_json_or_yaml_from_url` retries the download when the cached file fails
+  to parse, turning transient artifact-server glitches into self-healing
+  retries instead of hard failures (#791)
+
 63.2.0 - 2026-06-18
 -------------------
 

{scriptworker-63.2.0 → scriptworker-63.4.1}/PKG-INFO

@@ -1,6 +1,6 @@
-Metadata-Version: 2.4
+Metadata-Version: 2.5
 Name: scriptworker
-Version: 63.2.0
+Version: 63.4.1
 Summary: Taskcluster scriptworker
 Author-email: Mozilla Release Engineering <releng@mozilla.com>
 License-File: LICENSE

{scriptworker-63.2.0 → scriptworker-63.4.1}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "scriptworker"
-version = "63.2.0"
+version = "63.4.1"
 description = "Taskcluster scriptworker"
 authors = [
     { name = "Mozilla Release Engineering", email = "releng@mozilla.com" },

{scriptworker-63.2.0 → scriptworker-63.4.1}/src/scriptworker/artifacts.py

@@ -6,6 +6,7 @@ in S3.
 """
 
 import asyncio
+import base64
 import fnmatch
 import gzip
 import logging
@@ -21,7 +22,15 @@ from taskcluster.exceptions import TaskclusterFailure
 from scriptworker.client import validate_artifact_url
 from scriptworker.exceptions import DownloadError, ScriptWorkerRetryException, ScriptWorkerTaskException
 from scriptworker.task import get_decision_task_id, get_run_id, get_task_id
-from scriptworker.utils import add_enumerable_item_to_dict, download_file, get_loggable_url, raise_future_exceptions, retry_async, semaphore_wrapper
+from scriptworker.utils import (
+    add_enumerable_item_to_dict,
+    download_file,
+    get_hash,
+    get_loggable_url,
+    raise_future_exceptions,
+    retry_async,
+    semaphore_wrapper,
+)
 
 log = logging.getLogger(__name__)
 
@@ -163,6 +172,8 @@ async def create_artifact(context, path, target_path, content_type, content_enco
     payload = {"storageType": storage_type, "expires": expires or get_expiration_arrow(context).isoformat(), "contentType": content_type}
     args = [get_task_id(context.claim_task), get_run_id(context.claim_task), target_path, payload]
 
+    content_md5 = await asyncio.to_thread(get_content_md5, path)
+
     tc_response = await context.temp_queue.createArtifact(*args)
     skip_auto_headers = [aiohttp.hdrs.CONTENT_TYPE]
     loggable_url = get_loggable_url(tc_response["putUrl"])
@@ -172,7 +183,7 @@ async def create_artifact(context, path, target_path, content_type, content_enco
             async with context.session.put(
                 tc_response["putUrl"],
                 data=fh,
-                headers=_craft_artifact_put_headers(content_type, content_encoding),
+                headers=_craft_artifact_put_headers(content_type, content_md5, content_encoding),
                 skip_auto_headers=skip_auto_headers,
                 compress=False,
             ) as resp:
@@ -180,6 +191,12 @@ async def create_artifact(context, path, target_path, content_type, content_enco
                 response_text = await resp.text()
                 log.info(response_text)
                 if resp.status not in (200, 204):
+                    if content_md5 is not None and "BadDigest" in (response_text or ""):
+                        log.error(
+                            "{} was corrupted in transit: the storage backend rejected the body as not matching Content-MD5 {}. Retrying.".format(
+                                target_path, content_md5
+                            )
+                        )
                     raise ScriptWorkerRetryException("Bad status {}".format(resp.status))
 
 
@@ -206,9 +223,22 @@ async def create_link_artifact(context, target_path, link_to, content_type, expi
     await context.temp_queue.createArtifact(*args)
 
 
-def _craft_artifact_put_headers(content_type, encoding=None):
+def get_content_md5(path):
+    """Get the base64-encoded md5 digest of a file, formatted for ``Content-MD5``.
+
+    Args:
+        path (str): the path to the file to digest.
+
+    Returns:
+        str: the base64-encoded md5 digest.
+
+    """
+    return base64.b64encode(bytes.fromhex(get_hash(path, hash_alg="md5"))).decode("ascii")
+
+
+def _craft_artifact_put_headers(content_type, content_md5, encoding=None):
     log.debug("{} {}".format(content_type, encoding))
-    headers = {aiohttp.hdrs.CONTENT_TYPE: content_type}
+    headers = {aiohttp.hdrs.CONTENT_TYPE: content_type, aiohttp.hdrs.CONTENT_MD5: content_md5}
 
     if encoding is not None:
         headers[aiohttp.hdrs.CONTENT_ENCODING] = encoding

{scriptworker-63.2.0 → scriptworker-63.4.1}/src/scriptworker/config.py

@@ -125,11 +125,17 @@ def check_config(config, path):
         if key not in DEFAULT_CONFIG:
             messages.append("Unknown key {} in {}!".format(key, path))
             continue
+        is_by_cot_product = isinstance(DEFAULT_CONFIG[key], Mapping) and "by-cot-product" in DEFAULT_CONFIG[key]
         if value is None:
-            messages.append(_VALUE_UNDEFINED_MESSAGE.format(path=path, key=key))
+            if key != "cot_product" and not is_by_cot_product:
+                messages.append(_VALUE_UNDEFINED_MESSAGE.format(path=path, key=key))
         else:
             value_type = type(value)
-            if isinstance(DEFAULT_CONFIG[key], Mapping) and "by-cot-product" in DEFAULT_CONFIG[key]:
+            if key == "cot_product":
+                # `cot_product`'s own default is `None`, so its type can't be
+                # inferred from `DEFAULT_CONFIG` the way other keys' can.
+                default_type = str
+            elif is_by_cot_product:
                 default_type = type(DEFAULT_CONFIG[key]["by-cot-product"][config["cot_product"]])
             else:
                 default_type = type(DEFAULT_CONFIG[key])
@@ -139,6 +145,8 @@ def check_config(config, path):
             messages.append(_VALUE_UNDEFINED_MESSAGE.format(path=path, key=key))
         if key in ("provisioner_id", "worker_group", "worker_type", "worker_id") and not _is_id_valid(value):
             messages.append('{} doesn\'t match "{}" (required by Taskcluster)'.format(key, _GENERIC_ID_REGEX.pattern))
+    if config.get("cot_product") is None and config.get("verify_chain_of_trust"):
+        messages.append("{} verify_chain_of_trust can't be True when cot_product is not set!".format(path))
     return messages
 
 
@@ -161,6 +169,9 @@ def apply_product_config(config):
 
     for key in config:
         if isinstance(config[key], Mapping) and "by-cot-product" in config[key]:
+            if cot_product is None:
+                config[key] = None
+                continue
             try:
                 config[key] = config[key]["by-cot-product"][cot_product]
             except KeyError: