scriptworker 63.0.0__tar.gz → 63.1.0__tar.gz

{scriptworker-63.0.0 → scriptworker-63.1.0}/HISTORY.rst

@@ -4,6 +4,23 @@ Change Log
 All notable changes to this project will be documented in this file.
 This project adheres to `Semantic Versioning <http://semver.org/>`__.
 
+63.1.0 - 2026-05-26
+-------------------
+
+Added
+~~~~~
+- The TASKCLUSTER_CREDENTIALS_FD environment variable is passed to scripts, and
+  the referenced file descriptor (a memfd) contains temporary taskcluster
+  credentials using the task's scopes, updated after each reclaimTask
+
+63.0.1 - 2026-04-27
+-------------------
+
+Fixed
+~~~~~
+
+- File corruption caused by parallel downloads (#792)
+
 63.0.0 - 2026-04-09
 -------------------
 

{scriptworker-63.0.0 → scriptworker-63.1.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: scriptworker
-Version: 63.0.0
+Version: 63.1.0
 Summary: Taskcluster scriptworker
 Author-email: Mozilla Release Engineering <releng@mozilla.com>
 License-File: LICENSE

{scriptworker-63.0.0 → scriptworker-63.1.0}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "scriptworker"
-version = "63.0.0"
+version = "63.1.0"
 description = "Taskcluster scriptworker"
 authors = [
     { name = "Mozilla Release Engineering", email = "releng@mozilla.com" },
@@ -43,11 +43,8 @@ dev = [
     "tox-uv",
     "aiohttp>=3",
     "asyncio_extras",
-    "black",
     "coveralls",
-    "flake8",
-    "flake8_docstrings",
-    "isort>=5",
+    "ruff",
     "mock",
     "pytest",
     "pytest-asyncio",
@@ -85,34 +82,24 @@ verify_cot = "scriptworker.cot.verify:verify_cot_cmdln"
 create_test_workdir = "scriptworker.cot.verify:create_test_workdir"
 verify_ed25519_signature = "scriptworker.ed25519:verify_ed25519_signature_cmdln"
 
-[tool.black]
+[tool.ruff]
 line-length = 160
-target-version = ["py311", "py312", "py313"]
-include = '\.(wsgi|pyi?)$'
-exclude = '''
-/(
-    \.eggs
-  | \.git
-  | \.hg
-  | \.mypy_cache
-  | \.tox
-  | \.venv
-  | \.cache
-  | \.cache_py3
-  | _build
-  | buck-out
-  | build
-  | dist
-  | ui
-)/
-'''
+target-version = "py311"
+extend-exclude = [
+    "docs"
+]
+
+[tool.ruff.lint]
+# F = pyflakes, E/W = pycodestyle, I = isort, D = pydocstyle
+select = ["F", "E", "W", "I", "D"]
+
+[tool.ruff.lint.per-file-ignores]
+"tests/**" = ["D"]
+"tests/test_github.py" = ["D", "E501"]  # ignore long lines in test_github.py
+
 
-[tool.isort]
-multi_line_output = 3
-include_trailing_comma = true
-force_grid_wrap = 0
-use_parentheses = true
-line_length = 160
+[tool.ruff.lint.pydocstyle]
+convention = "pep257"
 
 [tool.pytest.ini_options]
 asyncio_mode = "auto"

{scriptworker-63.0.0 → scriptworker-63.1.0}/src/scriptworker/artifacts.py

@@ -225,6 +225,7 @@ def get_artifact_url(context, task_id, path):
 
 # list_latest_artifacts {{{1
 async def list_latest_artifacts(queue, task_id):
+    """List all artifacts for the latest run of a task, following pagination."""
     artifacts = []
 
     def pagination_handler(response):
@@ -235,6 +236,7 @@ async def list_latest_artifacts(queue, task_id):
 
 
 async def retry_list_latest_artifacts(queue, task_id, exception=TaskclusterFailure, **kwargs):
+    """Retry ``list_latest_artifacts`` on transient Taskcluster failures."""
     kwargs.setdefault("retry_exceptions", tuple(set([TaskclusterFailure, exception])))
     return await retry_async(list_latest_artifacts, args=(queue, task_id), **kwargs)
 
@@ -442,6 +444,7 @@ def assert_is_parent(path, parent_dir):
 
 
 def get_artifacts_matching_glob(context, task_id, pattern):
+    """Return paths of downloaded artifacts matching ``pattern`` under the task's cot/<id> dir."""
     parent_dir = os.path.abspath(os.path.join(context.config["work_dir"], "cot", task_id))
     matching = []
     for root, _, files in os.walk(parent_dir):

{scriptworker-63.0.0 → scriptworker-63.1.0}/src/scriptworker/context.py

@@ -59,6 +59,7 @@ class Context(object):
 
     config: Optional[Dict[str, Any]] = None
     credentials_timestamp: Optional[int] = None
+    credentials_fd: int = -1
     proc: Optional[task_process.TaskProcess] = None
     queue: Optional[Queue] = None
     session: Optional[aiohttp.ClientSession] = None
@@ -98,6 +99,8 @@ class Context(object):
         if claim_task:
             self.task = claim_task["task"]
             self.verify_task()
+            # flags=0 to let the child inherit this fd
+            self.credentials_fd = os.memfd_create("scriptworker_temp_creds", flags=0)
             self.temp_credentials = claim_task["credentials"]
             path = os.path.join(self.config["work_dir"], "task.json")
             assert self.task
@@ -105,6 +108,8 @@ class Context(object):
         else:
             self.temp_credentials = None
             self.task = None
+            os.close(self.credentials_fd)
+            self.credentials_fd = -1
 
     def verify_task(self) -> None:
         """Run some task sanity checks on ``self.task``."""
@@ -193,6 +198,10 @@ class Context(object):
     def temp_credentials(self, credentials: Optional[Dict[str, Any]]) -> None:
         self._temp_credentials = credentials
         self.temp_queue = self.create_queue(self.temp_credentials)
+        if credentials:
+            data = json.dumps(credentials, indent=2, sort_keys=True).encode("ascii")
+            # use pwrite so we don't confuse the child by changing the file offset
+            assert os.pwrite(self.credentials_fd, data, 0) == len(data)
 
     def write_json(self, path: str, contents: Dict[str, Any], message: str) -> None:
         """Write json to disk.
@@ -261,6 +270,7 @@ class Context(object):
 
     @property
     def download_semaphore(self) -> asyncio.BoundedSemaphore:
+        """Return a lazily-built semaphore limiting concurrent downloads to ``max_concurrent_downloads``."""
         assert self.config
         if self._download_semaphore is None:
             try:

{scriptworker-63.0.0 → scriptworker-63.1.0}/src/scriptworker/cot/verify.py

@@ -180,7 +180,8 @@ class ChainOfTrust(object):
 
     def is_scope_in_restricted_scopes(self, scope, restricted_scopes):
         """Determine if a scope matches in a list of restricted_scopes.
-           if one of the restricted scopes ends with '*', find a partial match.
+
+        If one of the restricted scopes ends with '*', find a partial match.
 
         Returns:
             String: string of matching restricted_scope, if no match ""
@@ -728,8 +729,10 @@ async def download_cot_artifact(chain, task_id, path):
     link = chain.get_link(task_id)
     log.debug("Verifying {} is in {} cot artifacts...".format(path, task_id))
     if not link.cot:
-        log.warning('Chain of Trust for "{}" in {} does not exist. See above log for more details. \
-Skipping download of this artifact'.format(path, task_id))
+        log.warning(
+            'Chain of Trust for "{}" in {} does not exist. See above log for more details. \
+Skipping download of this artifact'.format(path, task_id)
+        )
         return
 
     if path not in link.cot["artifacts"]:
@@ -1094,8 +1097,9 @@ async def _get_additional_hg_push_jsone_context(parent_link, decision_link):
         allowed_comments.append(push_comment[try_idx:].split("\n")[0])
     if decision_comment not in allowed_comments:
         raise CoTError(
-            "Decision task {} comment doesn't match the push comment!\n"
-            "Decision comment: \n{}\nPush comment: \n{}".format(decision_link.name, decision_comment, push_comment)
+            "Decision task {} comment doesn't match the push comment!\nDecision comment: \n{}\nPush comment: \n{}".format(
+                decision_link.name, decision_comment, push_comment
+            )
         )
     return {
         "push": {
@@ -2113,7 +2117,8 @@ def verify_cot_cmdln(args=None, event_loop=None):
 
     """
     args = args or sys.argv[1:]
-    parser = argparse.ArgumentParser(description="""Verify a given task's chain of trust.
+    parser = argparse.ArgumentParser(
+        description="""Verify a given task's chain of trust.
 
 Given a task's `task_id`, get its task definition, then trace its chain of
 trust back to the tree.  This doesn't verify chain of trust artifact signatures,
@@ -2125,7 +2130,8 @@ To use, first either set your taskcluster creds in your env http://bit.ly/2eDMa6
 or in the CREDS_FILES http://bit.ly/2fVMu0A
 
 If you are verifying against a private github repo, please also set in environment
-SCRIPTWORKER_GITHUB_OAUTH_TOKEN to an OAUTH token with read permissions to the repo""")
+SCRIPTWORKER_GITHUB_OAUTH_TOKEN to an OAUTH token with read permissions to the repo"""
+    )
     parser.add_argument("task_id", help="the task id to test")
     parser.add_argument("--task-type", help="the task type to test", choices=sorted(get_valid_task_types().keys()), required=True)
     parser.add_argument("--cleanup", help="clean up the temp dir afterwards", dest="cleanup", action="store_true", default=False)
@@ -2192,13 +2198,15 @@ def create_test_workdir(args=None, event_loop=None):
 
     """
     args = args or sys.argv[1:]
-    parser = argparse.ArgumentParser(description="""Populate a test `work_dir`.
+    parser = argparse.ArgumentParser(
+        description="""Populate a test `work_dir`.
 
 Given a scriptworker task's `task_id`, get its task definition, write it to
 `./work/task.json`, then download its `upstreamArtifacts` and put them in
 `./work/cot/TASK_ID/PATH`.
 
-This is helpful in manually testing a *script run.""")
+This is helpful in manually testing a *script run."""
+    )
     parser.add_argument("--path", help="relative path to the work_dir", default="work")
     parser.add_argument("--overwrite", help="overwrite an existing work_dir", action="store_true")
     parser.add_argument("task_id", help="the task id to test")

{scriptworker-63.0.0 → scriptworker-63.1.0}/src/scriptworker/ed25519.py

@@ -141,11 +141,13 @@ def verify_ed25519_signature_cmdln(args=None, exception=SystemExit):
 
     """
     args = args or sys.argv[1:]
-    parser = argparse.ArgumentParser(description="""Verify an ed25519 signature from the command line.
+    parser = argparse.ArgumentParser(
+        description="""Verify an ed25519 signature from the command line.
 
 Given a file and its detached signature, verify that it has been signed with
 a valid key. This key can be specified on the command line; otherwise we'll
-default to ``config['ed25519_public_keys']``.""")
+default to ``config['ed25519_public_keys']``."""
+    )
     parser.add_argument("--pubkey", help="path to a base64-encoded ed25519 pubkey, optional")
     parser.add_argument("file_path")
     parser.add_argument("sig_path")

{scriptworker-63.0.0 → scriptworker-63.1.0}/src/scriptworker/github.py

@@ -121,7 +121,7 @@ class GitHubRepository:
         if any(vcs_rule.get("require_secret") for vcs_rule in context.config["trusted_vcs_rules"]):
             # This check uses unofficial API on github, which we can't easily
             # check for private repos, assume its true in the private case.
-            log.info("has_commit_landed_on_repository() not implemented for private" "repositories, assume True")
+            log.info("has_commit_landed_on_repository() not implemented for private repositories, assume True")
             return True
 
         # Revision may be a tag name. `branch_commits` doesn't work on tags
@@ -293,7 +293,7 @@ def is_github_repo_owner_the_official_one(context, repo_owner):
     official_repo_owner = context.config["official_github_repos_owner"]
     if not official_repo_owner:
         raise ConfigError(
-            "This worker does not have a defined owner for official GitHub repositories. " 'Given "official_github_repos_owner": {}'.format(official_repo_owner)
+            'This worker does not have a defined owner for official GitHub repositories. Given "official_github_repos_owner": {}'.format(official_repo_owner)
         )
 
     return official_repo_owner == repo_owner

{scriptworker-63.0.0 → scriptworker-63.1.0}/src/scriptworker/task.py

@@ -117,7 +117,7 @@ def get_run_id(claim_task):
 
 # get_task_maxruntime {{{1
 def get_task_maxruntime(task_def, max_timeout):
-    """Given a task definition, return the lower of max_timeout and maxRunTime if set"""
+    """Given a task definition, return the lower of max_timeout and maxRunTime if set."""
     max_run_time = task_def["payload"].get("maxRunTime")
     if max_run_time is None:
         return max_timeout
@@ -672,7 +672,16 @@ async def run_task(context, to_cancellable_process):
     env["TASK_ID"] = context.task_id or "None"
     env["RUN_ID"] = str(get_run_id(context.claim_task))
     env["TASKCLUSTER_ROOT_URL"] = context.config["taskcluster_root_url"]
-    kwargs = {"stdout": PIPE, "stderr": PIPE, "stdin": None, "close_fds": True, "preexec_fn": lambda: os.setsid(), "env": env}  # pragma: no branch
+    env["TASKCLUSTER_CREDENTIALS_FD"] = str(context.credentials_fd)
+    kwargs = {
+        "stdout": PIPE,
+        "stderr": PIPE,
+        "stdin": None,
+        "close_fds": True,
+        "preexec_fn": lambda: os.setsid(),
+        "env": env,
+        "pass_fds": (context.credentials_fd,),
+    }  # pragma: no branch
 
     timeout = get_task_maxruntime(context.task, context.config["task_max_timeout"])
     subprocess = await asyncio.create_subprocess_exec(*context.config["task_script"], **kwargs)

{scriptworker-63.0.0 → scriptworker-63.1.0}/src/scriptworker/task_process.py

@@ -19,7 +19,7 @@ class TaskProcess:
     """Wraps worker task's process."""
 
     def __init__(self, process: Process):
-        """Constructor.
+        """Initialize the wrapper around the worker task's subprocess.
 
         Args:
             process (Process): task process

{scriptworker-63.0.0 → scriptworker-63.1.0}/src/scriptworker/utils.py

@@ -16,6 +16,7 @@ import random
 import re
 import shutil
 import time
+import uuid
 from copy import deepcopy
 from typing import IO, TYPE_CHECKING, Any, Awaitable, Callable, Dict, Match, Optional, Sequence, Tuple, Type, Union, cast, overload  # noqa
 from urllib.parse import unquote, urlparse
@@ -41,6 +42,7 @@ log = logging.getLogger(__name__)
 
 # scriptworker_session {{{1
 def scriptworker_session(*args, **kwargs):
+    """Return an ``aiohttp.ClientSession`` with the scriptworker User-Agent header set."""
     kwargs.setdefault("headers", {}).setdefault("User-Agent", f"scriptworker {scriptworker.__version__}")
     return aiohttp.ClientSession(*args, **kwargs)
 
@@ -680,12 +682,21 @@ async def download_file(context, url, abs_filename, session=None, chunk_size=128
             await _log_download_error(resp, "Failed to download %(url)s: %(status)s; body=%(body)s")
             raise DownloadError("{} status {} is not 200!".format(loggable_url, resp.status))
         makedirs(parent_dir)
-        with open(abs_filename, "wb") as fd:
-            while True:
-                chunk = await resp.content.read(chunk_size)
-                if not chunk:
-                    break
-                fd.write(chunk)
+        tmp_filename = "{}.{}.part".format(abs_filename, uuid.uuid4().hex)
+        try:
+            with open(tmp_filename, "wb") as fd:
+                while True:
+                    chunk = await resp.content.read(chunk_size)
+                    if not chunk:
+                        break
+                    fd.write(chunk)
+            os.replace(tmp_filename, abs_filename)
+        except Exception:
+            try:
+                os.unlink(tmp_filename)
+            except OSError:
+                pass
+            raise
     log.info("Done")
 
 
@@ -865,8 +876,9 @@ def remove_empty_keys(values, remove=({}, None, [], "null")):
 
 # add_projectid {{{1
 def add_projectid(task_def):
-    """Add a projectId property to a task, if none already exists, using
-    the Taskcluster default value of 'none'.
+    """Add a projectId property to a task, if none already exists.
+
+    Uses the Taskcluster default value of 'none'.
 
     Args:
         task_def (dict): the task definition
@@ -879,8 +891,9 @@ def add_projectid(task_def):
 
 # add_taskqueueid {{{1
 def add_taskqueueid(task_def):
-    """Add a taskQueueId property to a task, if none already exists, based
-    on the provisionerId and workerType.  Then remove those two properties.
+    """Add a taskQueueId property to a task, if none already exists.
+
+    Builds the value from the provisionerId and workerType, then removes those two properties.
 
     Args:
         task_def (dict): the task definition

{scriptworker-63.0.0 → scriptworker-63.1.0}/src/scriptworker/worker.py

@@ -104,7 +104,7 @@ class RunTasks:
     """Manages processing of Taskcluster tasks."""
 
     def __init__(self):
-        """Constructor."""
+        """Initialize the task-processing state."""
         self.future = None
         self.task_process = None
         self.is_cancelled = False

{scriptworker-63.0.0 → scriptworker-63.1.0}/tox.ini

@@ -54,25 +54,16 @@ commands=
 
 [testenv:check]
 skip_install = true
+deps = ruff
 commands =
-    black --check {toxinidir}
-    isort --check --df {toxinidir}
-    flake8 {toxinidir}
+    ruff check {toxinidir}
+    ruff format --check {toxinidir}
 
 [testenv:mypy]
 commands =
     mypy --config {toxinidir}/mypi.ini {toxinidir}/src
 
 
-[flake8]
-max-line-length = 160
-# test_github.py ignored because of https://gitlab.com/pycqa/flake8/issues/375
-exclude = .ropeproject,.tox,sandbox,docs,.eggs,*.egg,*.egg-info,setup.py,build/,tests/test_github.py,.venv
-per-file-ignores =
-    # Ignore documentation issues in tests
-    tests:*:D
-show-source = True
-
 [pytest]
 norecursedirs = .tox .git .hg sandbox .eggs build
 python_files = test_*.py