scriptworker 62.7.0__tar.gz → 63.0.1__tar.gz

{scriptworker-62.7.0 → scriptworker-63.0.1}/HISTORY.rst

@@ -4,6 +4,31 @@ Change Log
 All notable changes to this project will be documented in this file.
 This project adheres to `Semantic Versioning <http://semver.org/>`__.
 
+63.0.1 - 2026-04-27
+
+Fixed
+~~~~~
+
+- File corruption caused by parallel downloads (#792)
+
+63.0.0 - 2026-04-09
+-------------------
+
+Changed
+~~~~~~~
+
+- Parallelized several steps of Chain of Trust verification, significantly
+  improving end-to-end times. As a result, `scriptworker.cot.verify.build_link`
+  has been renamed to `scriptworker.cot.verify.add_link` and no longer recurses
+  into dependencies . One should call `build_task_dependencies` after adding
+  the link to walk the graph.
+
+Fixed
+~~~~~
+
+- The root task isn't verified twice anymore when using `check_task=True` in
+  `verify_chain_of_trust` on non decision tasks.
+
 62.7.0 - 2026-03-18
 -------------------
 

{scriptworker-62.7.0 → scriptworker-63.0.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: scriptworker
-Version: 62.7.0
+Version: 63.0.1
 Summary: Taskcluster scriptworker
 Author-email: Mozilla Release Engineering <releng@mozilla.com>
 License-File: LICENSE

{scriptworker-62.7.0 → scriptworker-63.0.1}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "scriptworker"
-version = "62.7.0"
+version = "63.0.1"
 description = "Taskcluster scriptworker"
 authors = [
     { name = "Mozilla Release Engineering", email = "releng@mozilla.com" },

{scriptworker-62.7.0 → scriptworker-63.0.1}/src/scriptworker/cot/verify.py

@@ -222,7 +222,7 @@ class ChainOfTrust(object):
             list: of all ``LinkOfTrust``s to verify.
 
         """
-        if self.is_decision() and self.get_link(self.task_id):
+        if self.task_id in self.dependent_task_ids():
             return self.links
         return [self] + self.links
 
@@ -619,8 +619,8 @@ def _sort_dependencies_by_name_then_task_id(dependencies):
 
 
 # build_task_dependencies {{{1
-async def build_link(chain, task_name, task_id):
-    """Build a LinkOfTrust and add it to the chain.
+async def add_link(chain, task_name, task_id):
+    """Fetch a task definition and add it as a LinkOfTrust to the chain.
 
     Args:
         chain (ChainOfTrust): the chain of trust to add to.
@@ -633,17 +633,14 @@ async def build_link(chain, task_name, task_id):
     """
     link = LinkOfTrust(chain.context, task_name, task_id)
     json_path = link.get_artifact_full_path("task.json")
-    task_defn = await retry_get_task_definition(chain.context.queue, task_id, exception=CoTError)
-    link.task = task_defn
+    link.task = await retry_get_task_definition(chain.context.queue, task_id, exception=CoTError)
     chain.links.append(link)
-    # write task json to disk
     makedirs(os.path.dirname(json_path))
     with open(json_path, "w") as fh:
-        fh.write(format_json(task_defn))
-    await build_task_dependencies(chain, task_defn, task_name, task_id)
+        fh.write(format_json(link.task))
 
 
-async def build_task_dependencies(chain, task, name, my_task_id):
+async def build_task_dependencies(chain, task, name, my_task_id, seen=None):
     """Recursively build the task dependencies of a task.
 
     Args:
@@ -651,6 +648,7 @@ async def build_task_dependencies(chain, task, name, my_task_id):
         task (dict): the task definition to operate on.
         name (str): the name of the task to operate on.
         my_task_id (str): the taskId of the task to operate on.
+        seen (set): shared set of already-seen task IDs to avoid duplicates
 
     Raises:
         CoTError: on failure.
@@ -661,9 +659,20 @@ async def build_task_dependencies(chain, task, name, my_task_id):
         raise CoTError("Too deep recursion!\n{}".format(name))
     sorted_dependencies = find_sorted_task_dependencies(task, name, my_task_id)
 
+    if seen is None:
+        seen = set(chain.dependent_task_ids())
+    new_deps = []
     for task_name, task_id in sorted_dependencies:
-        if task_id not in chain.dependent_task_ids():
-            await build_link(chain, task_name, task_id)
+        if task_id not in seen:
+            seen.add(task_id)
+            new_deps.append((task_name, task_id))
+
+    if not new_deps:
+        return
+
+    await asyncio.gather(*[add_link(chain, task_name, task_id) for task_name, task_id in new_deps])
+
+    await asyncio.gather(*[build_task_dependencies(chain, chain.get_link(task_id).task, task_name, task_id, seen) for task_name, task_id in new_deps])
 
 
 # download_cot {{{1
@@ -1565,8 +1574,10 @@ async def get_jsone_context_and_template(chain, parent_link, decision_link, task
     if tasks_for in ("action", "pr-action"):
         jsone_context, tmpl = await get_action_context_and_template(chain, parent_link, decision_link, tasks_for)
     else:
-        tmpl = await get_in_tree_template(decision_link)
-        jsone_context = await populate_jsone_context(chain, parent_link, decision_link, tasks_for)
+        tmpl, jsone_context = await asyncio.gather(
+            get_in_tree_template(decision_link),
+            populate_jsone_context(chain, parent_link, decision_link, tasks_for),
+        )
     return jsone_context, tmpl
 
 
@@ -1803,14 +1814,14 @@ async def verify_task_types(chain):
     """
     valid_task_types = get_valid_task_types()
     task_count = {}
+    tasks = []
     for obj in chain.get_all_links_in_chain():
         task_type = obj.task_type
         log.info("Verifying {} {} as a {} task...".format(obj.name, obj.task_id, task_type))
         task_count.setdefault(task_type, 0)
         task_count[task_type] += 1
-        # Run tests synchronously for now.  We can parallelize if efficiency
-        # is more important than a single simple logfile.
-        await valid_task_types[task_type](chain, obj)
+        tasks.append(valid_task_types[task_type](chain, obj))
+    await asyncio.gather(*tasks)
     return task_count
 
 
@@ -1880,12 +1891,12 @@ async def verify_worker_impls(chain):
 
     """
     valid_worker_impls = get_valid_worker_impls()
+    tasks = []
     for obj in chain.get_all_links_in_chain():
         worker_impl = obj.worker_impl
         log.info("Verifying {} {} as a {} task...".format(obj.name, obj.task_id, worker_impl))
-        # Run tests synchronously for now.  We can parallelize if efficiency
-        # is more important than a single simple logfile.
-        await valid_worker_impls[worker_impl](chain, obj)
+        tasks.append(valid_worker_impls[worker_impl](chain, obj))
+    await asyncio.gather(*tasks)
 
 
 # get_source_url {{{1
@@ -2042,9 +2053,8 @@ async def verify_chain_of_trust(chain, *, check_task=False):
         try:
             # build LinkOfTrust objects
             if check_task:
-                await build_link(chain, chain.name, chain.task_id)
-            else:
-                await build_task_dependencies(chain, chain.task, chain.name, chain.task_id)
+                await add_link(chain, chain.name, chain.task_id)
+            await build_task_dependencies(chain, chain.task, chain.name, chain.task_id)
             # download the signed chain of trust artifacts
             await download_cot(chain)
             # verify the signatures and populate the ``link.cot``s

{scriptworker-62.7.0 → scriptworker-63.0.1}/src/scriptworker/utils.py

@@ -16,6 +16,7 @@ import random
 import re
 import shutil
 import time
+import uuid
 from copy import deepcopy
 from typing import IO, TYPE_CHECKING, Any, Awaitable, Callable, Dict, Match, Optional, Sequence, Tuple, Type, Union, cast, overload  # noqa
 from urllib.parse import unquote, urlparse
@@ -680,12 +681,21 @@ async def download_file(context, url, abs_filename, session=None, chunk_size=128
             await _log_download_error(resp, "Failed to download %(url)s: %(status)s; body=%(body)s")
             raise DownloadError("{} status {} is not 200!".format(loggable_url, resp.status))
         makedirs(parent_dir)
-        with open(abs_filename, "wb") as fd:
-            while True:
-                chunk = await resp.content.read(chunk_size)
-                if not chunk:
-                    break
-                fd.write(chunk)
+        tmp_filename = "{}.{}.part".format(abs_filename, uuid.uuid4().hex)
+        try:
+            with open(tmp_filename, "wb") as fd:
+                while True:
+                    chunk = await resp.content.read(chunk_size)
+                    if not chunk:
+                        break
+                    fd.write(chunk)
+            os.replace(tmp_filename, abs_filename)
+        except Exception:
+            try:
+                os.unlink(tmp_filename)
+            except OSError:
+                pass
+            raise
     log.info("Done")
 
 

{scriptworker-62.7.0 → scriptworker-63.0.1}/tox.ini

@@ -66,6 +66,7 @@ commands =
 
 [flake8]
 max-line-length = 160
+extend-ignore = E203
 # test_github.py ignored because of https://gitlab.com/pycqa/flake8/issues/375
 exclude = .ropeproject,.tox,sandbox,docs,.eggs,*.egg,*.egg-info,setup.py,build/,tests/test_github.py,.venv
 per-file-ignores =