scriptworker 60.0.0__tar.gz → 60.2.0__tar.gz

{scriptworker-60.0.0 → scriptworker-60.2.0}/HISTORY.rst

@@ -4,6 +4,30 @@ Change Log
 All notable changes to this project will be documented in this file.
 This project adheres to `Semantic Versioning <http://semver.org/>`__.
 
+60.2.0 - 2024-07-12
+-------------------
+
+Added
+~~~~~
+- Set User-Agent in http requests (#661)
+- Support github-pull-request-untrusted in CoT verification (https://bugzilla.mozilla.org/show_bug.cgi?id=1906748)
+
+Fixed
+~~~~~
+- Pull project from the default branch in projects.yml (#665)
+
+60.1.0 - 2024-06-24
+-------------------
+
+Added
+~~~~~
+- Values for Firefox Translations training repository scriptworker constants
+- Support for deferring upstream artifact selection until runtime
+
+Fixed
+~~~~~
+- Ensure GitHub pull request head sha is set correctly during chain of trust verification
+
 60.0.0 - 2024-05-27
 -------------------
 

scriptworker-60.2.0/PKG-INFO

@@ -0,0 +1,86 @@
+Metadata-Version: 2.1
+Name: scriptworker
+Version: 60.2.0
+Summary: TaskCluster Script Worker
+Home-page: https://github.com/mozilla-releng/scriptworker
+Author: Mozilla Release Engineering
+Author-email: release+python@mozilla.com
+License: MPL 2.0
+Classifier: Intended Audience :: Developers
+Classifier: Natural Language :: English
+Classifier: Programming Language :: Python :: 3.8
+Classifier: Programming Language :: Python :: 3.9
+Classifier: Programming Language :: Python :: 3.10
+Requires-Python: >=3.7
+License-File: LICENSE
+
+===================
+Scriptworker Readme
+===================
+
+.. image:: https://travis-ci.org/mozilla-releng/scriptworker.svg?branch=master
+    :target: https://travis-ci.org/mozilla-releng/scriptworker
+
+.. image:: https://coveralls.io/repos/github/mozilla-releng/scriptworker/badge.svg?branch=master
+    :target: https://coveralls.io/github/mozilla-releng/scriptworker?branch=master
+
+.. image:: https://readthedocs.org/projects/scriptworker/badge/?version=latest
+    :target: http://scriptworker.readthedocs.io/en/latest/?badge=latest
+    :alt: Documentation Status
+
+Scriptworker implements the `TaskCluster worker model`_, then launches a pre-defined script.
+
+.. _TaskCluster worker model: https://firefox-ci-tc.services.mozilla.com/docs/reference/platform/queue/worker-interaction
+
+This worker was designed for `Releng processes`_ that need specific, limited, and pre-defined capabilities.
+
+.. _Releng processes: https://bugzilla.mozilla.org/show_bug.cgi?id=1245837
+
+Free software: MPL2 License
+
+-----
+Usage
+-----
+* Create a config file.  By default scriptworker will look in ``./scriptworker.yaml``, but this config path can be specified as the first and only commandline argument.  There is an `example config file`_, and all config items are specified in `scriptworker.constants.DEFAULT_CONFIG`_.
+
+.. _example config file: https://github.com/mozilla-releng/scriptworker/blob/master/scriptworker.yaml.tmpl
+.. _scriptworker.constants.DEFAULT_CONFIG: https://github.com/mozilla-releng/scriptworker/blob/master/src/scriptworker/constants.py
+
+Credentials can live in ``./scriptworker.yaml``, ``./secrets.json``, ``~/.scriptworker``.
+
+* Launch: ``scriptworker [config_path]``
+
+-------
+Testing
+-------
+
+Without integration tests install tox, then
+
+``NO_CREDENTIALS_TESTS=1 tox -e py36``
+
+Without any tests connecting to the net, then ``NO_TESTS_OVER_WIRE=1 tox -e py36``
+
+With integration tests, first create a client in the Taskcluster UI with the scopes::
+
+    queue:cancel-task:test-dummy-scheduler/*
+    queue:claim-work:test-dummy-provisioner/dummy-worker-*
+    queue:create-task:lowest:test-dummy-provisioner/dummy-worker-*
+    queue:define-task:test-dummy-provisioner/dummy-worker-*
+    queue:get-artifact:SampleArtifacts/_/X.txt
+    queue:scheduler-id:test-dummy-scheduler
+    queue:schedule-task:test-dummy-scheduler/*
+    queue:task-group-id:test-dummy-scheduler/*
+    queue:worker-id:test-dummy-workers/dummy-worker-*
+
+Then generate a no priviledge personal access token in Github for the scriptworker_github_token (to avoid rate limiting) and create a ``./secrets.json`` or ``~/.scriptworker`` that looks like::
+
+    {
+        "integration_credentials": {
+            "clientId": "...",
+            "accessToken": "...",
+        }
+        "scriptworker_github_token": "..."
+    }
+
+
+then to run all tests: ``tox``

{scriptworker-60.0.0 → scriptworker-60.2.0}/setup.py

@@ -7,6 +7,8 @@ import sys
 from setuptools import setup
 from setuptools.command.test import test as TestCommand
 
+project_dir = os.path.abspath(os.path.dirname(__file__))
+
 if {"register", "upload"}.intersection(set(sys.argv)):
     print(
         "                        ***** WARNING *****\n"
@@ -39,6 +41,9 @@ with open(PATH) as filehandle:
 with open(os.path.join(os.path.abspath(os.path.dirname(__file__)), "requirements.txt")) as f:
     install_requires = f.readlines()
 
+with open(os.path.join(project_dir, "README.rst")) as fh:
+    long_description = fh.read()
+
 
 class Tox(TestCommand):
     """http://bit.ly/1T0dwvG"""
@@ -71,6 +76,7 @@ setup(
     name="scriptworker",
     version=VERSION,
     description="TaskCluster Script Worker",
+    long_description=long_description,
     author="Mozilla Release Engineering",
     author_email="release+python@mozilla.com",
     url="https://github.com/mozilla-releng/scriptworker",

{scriptworker-60.0.0 → scriptworker-60.2.0}/src/scriptworker/artifacts.py

@@ -6,6 +6,7 @@ in S3.
 """
 
 import asyncio
+import fnmatch
 import gzip
 import logging
 import mimetypes
@@ -15,6 +16,7 @@ from pathlib import Path
 import aiohttp
 import arrow
 import async_timeout
+from taskcluster.exceptions import TaskclusterFailure
 
 from scriptworker.client import validate_artifact_url
 from scriptworker.exceptions import DownloadError, ScriptWorkerRetryException, ScriptWorkerTaskException
@@ -221,6 +223,16 @@ def get_artifact_url(context, task_id, path):
     return url
 
 
+# list_latest_artifacts {{{1
+async def list_latest_artifacts(queue, task_id, exception=TaskclusterFailure):
+    return await queue.listLatestArtifacts(task_id)
+
+
+async def retry_list_latest_artifacts(queue, task_id, exception=TaskclusterFailure, **kwargs):
+    kwargs.setdefault("retry_exceptions", tuple(set([TaskclusterFailure, exception])))
+    return await retry_async(list_latest_artifacts, args=(queue, task_id), kwargs={"exception": exception}, **kwargs)
+
+
 # get_expiration_arrow {{{1
 def get_expiration_arrow(context):
     """Return an arrow matching `context.task['expires']`.
@@ -321,8 +333,12 @@ def get_upstream_artifacts_full_paths_per_task_id(context):
     for task_id, paths in task_ids_and_relative_paths:
         for path in paths:
             try:
-                path_to_add = get_and_check_single_upstream_artifact_full_path(context, task_id, path)
-                add_enumerable_item_to_dict(dict_=upstream_artifacts_full_paths_per_task_id, key=task_id, item=path_to_add)
+                if "*" in path:
+                    for path_to_add in get_artifacts_matching_glob(context, task_id, path):
+                        add_enumerable_item_to_dict(dict_=upstream_artifacts_full_paths_per_task_id, key=task_id, item=path_to_add)
+                else:
+                    path_to_add = get_and_check_single_upstream_artifact_full_path(context, task_id, path)
+                    add_enumerable_item_to_dict(dict_=upstream_artifacts_full_paths_per_task_id, key=task_id, item=path_to_add)
             except ScriptWorkerTaskException:
                 if path in optional_artifacts_per_task_id.get(task_id, []):
                     log.warning('Optional artifact "{}" of task "{}" not found'.format(path, task_id))
@@ -417,3 +433,13 @@ def assert_is_parent(path, parent_dir):
     p2 = Path(os.path.realpath(parent_dir))
     if p1 != p2 and p2 not in p1.parents:
         raise ScriptWorkerTaskException("{} is not under {}!".format(p1, p2))
+
+
+def get_artifacts_matching_glob(context, task_id, pattern):
+    parent_dir = os.path.abspath(os.path.join(context.config["work_dir"], "cot", task_id))
+    matching = []
+    for root, _, files in os.walk(parent_dir):
+        for f in files:
+            if fnmatch.fnmatch(f, pattern):
+                matching.append(os.path.join(root, f))
+    return matching

{scriptworker-60.0.0 → scriptworker-60.2.0}/src/scriptworker/client.py

@@ -17,13 +17,12 @@ from asyncio import AbstractEventLoop
 from typing import Any, Awaitable, Callable, Dict, List, Match, NoReturn, Optional, Tuple
 from urllib.parse import unquote
 
-import aiohttp
 import jsonschema
 
 from scriptworker.constants import STATUSES
 from scriptworker.context import Context
 from scriptworker.exceptions import ScriptWorkerException, ScriptWorkerTaskException, TaskVerificationError
-from scriptworker.utils import load_json_or_yaml, match_url_regex
+from scriptworker.utils import load_json_or_yaml, match_url_regex, scriptworker_session
 
 log = logging.getLogger(__name__)
 
@@ -199,7 +198,7 @@ def _init_logging(context: Any) -> None:
 
 
 async def _handle_asyncio_loop(async_main: Callable[[Any], Awaitable[None]], context: Any) -> None:
-    async with aiohttp.ClientSession() as session:
+    async with scriptworker_session() as session:
         context.session = session
         try:
             await async_main(context)

{scriptworker-60.0.0 → scriptworker-60.2.0}/src/scriptworker/constants.py

@@ -140,6 +140,7 @@ DEFAULT_CONFIG: immutabledict[str, Any] = immutabledict(
                         "xpi": "github",
                         "adhoc": "github",
                         "scriptworker": "github",
+                        "translations": "github",
                     }
                 )
             }
@@ -172,6 +173,7 @@ DEFAULT_CONFIG: immutabledict[str, Any] = immutabledict(
                         "xpi": ("xpi-1/decision", "xpi-3/decision", "xpi-1/decision-gcp", "xpi-3/decision-gcp"),
                         "adhoc": ("adhoc-1/decision", "adhoc-3/decision", "adhoc-1/decision-gcp", "adhoc-3/decision-gcp"),
                         "scriptworker": ("scriptworker-1/decision", "scriptworker-3/decision", "scriptworker-1/decision-gcp", "scriptworker-3/decision-gcp"),
+                        "translations": ("translations-1/decision-gcp",),
                     }
                 )
             }
@@ -190,6 +192,7 @@ DEFAULT_CONFIG: immutabledict[str, Any] = immutabledict(
                         "xpi": ("xpi-1/images", "xpi-3/images", "xpi-1/images-gcp", "xpi-3/images-gcp"),
                         "adhoc": ("adhoc-1/images", "adhoc-3/images", "adhoc-1/images-gcp", "adhoc-3/images-gcp"),
                         "scriptworker": ("scriptworker-1/images", "scriptworker-3/images", "scriptworker-1/images-gcp", "scriptworker-3/images-gcp"),
+                        "translations": ("translations-1/images-gcp",),
                     }
                 )
             }
@@ -277,6 +280,15 @@ DEFAULT_CONFIG: immutabledict[str, Any] = immutabledict(
                             }
                         ),
                     ),
+                    "translations": (
+                        immutabledict(
+                            {
+                                "schemes": ("https", "ssh"),
+                                "netlocs": ("github.com",),
+                                "path_regexes": tuple([r"^(?P<path>/mozilla/firefox-translations-training)(/|.git|$)"]),
+                            }
+                        ),
+                    ),
                 }
             )
         },
@@ -288,10 +300,8 @@ DEFAULT_CONFIG: immutabledict[str, Any] = immutabledict(
                     "mobile": (
                         "action",
                         "cron",
-                        # On staging releases, level 1 docker images may be built in the pull-request graph
                         "github-pull-request",
-                        # Similarly, docker images can be built on regular push. This is usually the case
-                        # for level 3 images
+                        "github-pull-request-untrusted",
                         "github-push",
                         "github-release",
                     ),
@@ -299,26 +309,21 @@ DEFAULT_CONFIG: immutabledict[str, Any] = immutabledict(
                     "app-services": (
                         "action",
                         "cron",
-                        # On staging releases, level 1 docker images may be built in the pull-request graph
                         "github-pull-request",
-                        # Similarly, docker images can be built on regular push. This is usually the case
-                        # for level 3 images
                         "github-push",
                         "github-release",
                     ),
                     "glean": (
                         "action",
                         "cron",
-                        # On staging releases, level 1 docker images may be built in the pull-request graph
                         "github-pull-request",
-                        # Similarly, docker images can be built on regular push. This is usually the case
-                        # for level 3 images
                         "github-push",
                         "github-release",
                     ),
                     "xpi": ("action", "cron", "github-pull-request", "github-push", "github-release"),
                     "adhoc": ("action", "github-pull-request", "github-push"),
                     "scriptworker": ("action", "cron", "github-pull-request", "github-push", "github-release"),
+                    "translations": ("action", "github-pull-request", "github-push"),
                 }
             )
         },
@@ -334,6 +339,7 @@ DEFAULT_CONFIG: immutabledict[str, Any] = immutabledict(
                     "xpi": "mozilla-extensions",
                     "adhoc": "mozilla-releng",
                     "scriptworker": "mozilla-releng",
+                    "translations": "mozilla",
                 }
             )
         },
@@ -422,6 +428,11 @@ DEFAULT_CONFIG: immutabledict[str, Any] = immutabledict(
                             "project:scriptworker:pypi:production": "all-production-repos",
                         }
                     ),
+                    "translations": immutabledict(
+                        {
+                            "project:translations:releng:beetmover:bucket:release": "translations-repo",
+                        }
+                    ),
                 }
             )
         },
@@ -517,6 +528,11 @@ DEFAULT_CONFIG: immutabledict[str, Any] = immutabledict(
                             "all-production-repos": ("/mozilla-releng/scriptworker", "/mozilla-releng/scriptworker-scripts"),
                         }
                     ),
+                    "translations": immutabledict(
+                        {
+                            "translations-repo": ("/mozilla/firefox-translations-training",),
+                        }
+                    ),
                 }
             )
         },
@@ -533,6 +549,7 @@ DEFAULT_CONFIG: immutabledict[str, Any] = immutabledict(
                     "xpi": "any",  # all allowed
                     "adhoc": "any",  # all allowed
                     "scriptworker": ("decision", "action", "docker-image"),
+                    "translations": "any",  # all allowed
                 }
             )
         },
@@ -548,6 +565,7 @@ DEFAULT_CONFIG: immutabledict[str, Any] = immutabledict(
                     "xpi": "XPI",
                     "adhoc": "ADHOC",
                     "scriptworker": "SCRIPTWORKER",
+                    "translations": "FIREFOX_TRANSLATIONS_TRAINING",
                 }
             )
         },

{scriptworker-60.0.0 → scriptworker-60.2.0}/src/scriptworker/context.py

@@ -24,7 +24,7 @@ from taskcluster.aio import Queue
 
 from scriptworker import task_process
 from scriptworker.exceptions import CoTError
-from scriptworker.utils import load_json_or_yaml_from_url, makedirs
+from scriptworker.utils import load_json_or_yaml_from_url, makedirs, scriptworker_session
 
 log = logging.getLogger(__name__)
 
@@ -108,7 +108,9 @@ class Context(object):
             task_id: str = upstream_artifact["taskId"]
             for path in upstream_artifact["paths"]:
                 if os.path.isabs(path) or ".." in path:
-                    raise CoTError("upstreamArtifacts taskId {} has illegal path {}!".format(task_id, path))
+                    raise CoTError(f"upstreamArtifacts taskId {task_id} has illegal path {path}!")
+                if "*" in path and not upstream_artifact.get("optional", False):
+                    raise CoTError(f"upstreamArtifacts taskId {task_id} has globbed path {path} as a non-optional artifact!")
 
     @property
     def credentials(self) -> Optional[Dict[str, Any]]:
@@ -146,7 +148,7 @@ class Context(object):
         """
         assert self.config
         if credentials:
-            session = self.session or aiohttp.ClientSession(loop=self.event_loop)
+            session = self.session or scriptworker_session(loop=self.event_loop)
             return Queue(options={"credentials": credentials, "rootUrl": self.config["taskcluster_root_url"]}, session=session)
         return None
 

{scriptworker-60.0.0 → scriptworker-60.2.0}/src/scriptworker/cot/verify.py

@@ -9,6 +9,7 @@ Attributes:
 
 import argparse
 import asyncio
+import fnmatch
 import logging
 import os
 import pprint
@@ -23,7 +24,13 @@ import jsone
 from immutabledict import immutabledict
 from taskcluster.aio import Queue
 
-from scriptworker.artifacts import download_artifacts, get_artifact_url, get_optional_artifacts_per_task_id, get_single_upstream_artifact_full_path
+from scriptworker.artifacts import (
+    download_artifacts,
+    get_artifact_url,
+    get_optional_artifacts_per_task_id,
+    get_single_upstream_artifact_full_path,
+    retry_list_latest_artifacts,
+)
 from scriptworker.config import apply_product_config, read_worker_creds
 from scriptworker.constants import DEFAULT_CONFIG
 from scriptworker.context import Context
@@ -39,6 +46,7 @@ from scriptworker.task import (
     get_branch,
     get_commit_message,
     get_decision_task_id,
+    get_head_revision,
     get_parent_task_id,
     get_project,
     get_provisioner_id,
@@ -70,6 +78,7 @@ from scriptworker.utils import (
     read_from_file,
     remove_empty_keys,
     rm,
+    scriptworker_session,
     write_to_file,
 )
 from scriptworker.version import __version_string__
@@ -761,14 +770,28 @@ async def download_cot_artifacts(chain):
 
     mandatory_artifact_tasks = []
     optional_artifact_tasks = []
+    latest_artifacts = {}
     for task_id, paths in all_artifacts_per_task_id.items():
         for path in paths:
-            coroutine = asyncio.ensure_future(download_cot_artifact(chain, task_id, path))
+            if "*" in path:
+                # Paths with wildcards in them indicate that the concrete
+                # artifact names aren't known when the task definition is
+                # created. For these cases, we need to fetch the list of
+                # artifacts from the completed tasks and then determine
+                # which are needed based on the pattern given.
+                if not latest_artifacts.get(task_id):
+                    latest_artifacts[task_id] = (await retry_list_latest_artifacts(chain.context.queue, task_id))["artifacts"]
+                coroutines = []
+                for artifact in latest_artifacts[task_id]:
+                    if fnmatch.fnmatch(artifact["name"], path):
+                        coroutines.append(asyncio.ensure_future(download_cot_artifact(chain, task_id, artifact["name"])))
+            else:
+                coroutines = [asyncio.ensure_future(download_cot_artifact(chain, task_id, path))]
 
             if is_artifact_optional(chain, task_id, path):
-                optional_artifact_tasks.append(coroutine)
+                optional_artifact_tasks.extend(coroutines)
             else:
-                mandatory_artifact_tasks.append(coroutine)
+                mandatory_artifact_tasks.extend(coroutines)
 
     mandatory_artifacts_paths = await raise_future_exceptions(mandatory_artifact_tasks)
     succeeded_optional_artifacts_paths, failed_optional_artifacts = await get_results_and_future_exceptions(optional_artifact_tasks)
@@ -1020,12 +1043,17 @@ async def get_scm_level(context, project):
     """
     await context.populate_projects()
     config = context.projects[project]
-    if "access" in config:
+    if config["repo_type"] == "hg":
         return config["access"].replace("scm_level_", "")
-    elif "level" in config:
-        return str(config["level"])
-    else:
-        raise ValueError("Can't find level for project {}".format(project))
+    elif config["repo_type"] == "git":
+        # TODO: we should be using the branch that the task is actually
+        # being run on
+        default_branch = config.get("default_branch", "main")
+        for branch in config["branches"]:
+            if branch["name"] == default_branch:
+                return str(branch["level"])
+
+    raise ValueError("Can't find level for project {}".format(project))
 
 
 async def _get_additional_hg_action_jsone_context(parent_link, decision_link):
@@ -1193,6 +1221,10 @@ async def _get_additional_github_pull_request_jsone_context(decision_link):
     pull_request_data["head"]["updated_at"] = get_push_date_time(task, source_env_prefix)
     # Similarly, the base branch may get new commits by the time a PR job runs
     pull_request_data["base"]["sha"] = get_base_revision(task, source_env_prefix)
+    # The head sha may also be different. This commonly happens in pull requests that
+    # have tasks that have cached tasks from an earlier revision of the PR, or even
+    # another PR altogether, upstream of one from the latest revision of the PR.
+    pull_request_data["head"]["sha"] = get_head_revision(task, source_env_prefix)
 
     return {
         "event": {
@@ -1319,7 +1351,7 @@ async def populate_jsone_context(chain, parent_link, decision_link, tasks_for):
             jsone_context.update(await _get_additional_git_cron_jsone_context(decision_link))
         elif tasks_for == "action":
             jsone_context.update(await _get_additional_git_action_jsone_context(decision_link, parent_link))
-        elif tasks_for == "github-pull-request":
+        elif tasks_for in ("github-pull-request", "github-pull-request-untrusted"):
             jsone_context.update(await _get_additional_github_pull_request_jsone_context(decision_link))
         elif tasks_for == "github-push":
             jsone_context.update(await _get_additional_github_push_jsone_context(decision_link))
@@ -2044,7 +2076,7 @@ async def verify_chain_of_trust(chain, *, check_task=False):
 
 # verify_cot_cmdln {{{1
 async def _async_verify_cot_cmdln(opts, tmp):
-    async with aiohttp.ClientSession() as session:
+    async with scriptworker_session() as session:
         context = Context()
         context.session = session
         context.config = dict(deepcopy(DEFAULT_CONFIG))
@@ -2120,7 +2152,7 @@ SCRIPTWORKER_GITHUB_OAUTH_TOKEN to an OAUTH token with read permissions to the r
 
 # create_test_workdir {{{1
 async def _async_create_test_workdir(task_id, path, queue=None):
-    async with aiohttp.ClientSession() as session:
+    async with scriptworker_session() as session:
         context = Context()
         context.session = session
         context.config = dict(deepcopy(DEFAULT_CONFIG))

{scriptworker-60.0.0 → scriptworker-60.2.0}/src/scriptworker/task.py

@@ -230,6 +230,22 @@ def get_base_revision(task, source_env_prefix):
     return _extract_from_env_in_payload(task, source_env_prefix + "_BASE_REV")
 
 
+def get_head_revision(task, source_env_prefix):
+    """Get the base revision for a task.
+
+    Args:
+        obj (ChainOfTrust or LinkOfTrust): the trust object to inspect
+        source_env_prefix (str): The environment variable prefix that is used
+            to get repository information.
+
+    Returns:
+        str: the revision.
+        None: if not defined for this task.
+
+    """
+    return _extract_from_env_in_payload(task, source_env_prefix + "_HEAD_REV")
+
+
 def get_branch(task, source_env_prefix):
     """Get the branch on top of which the graph was made.
 
@@ -487,7 +503,7 @@ async def is_pull_request(context, task):
 
     This checks for the following things::
 
-        * ``task.extra.env.tasks_for`` == "github-pull-request"
+        * ``task.extra.env.tasks_for`` is "github-pull-request" or "github-pull-request-untrusted"
         * ``task.payload.env.MOBILE_HEAD_REPOSITORY`` doesn't come from an official repo
         * ``task.metadata.source`` doesn't come from an official repo, either
         * The last 2 items are landed on the official repo
@@ -509,7 +525,7 @@ async def is_pull_request(context, task):
     metadata_source_url = task["metadata"].get("source", "")
     repo_from_source_url, revision_from_source_url = extract_github_repo_and_revision_from_source_url(metadata_source_url)
 
-    conditions = [tasks_for == "github-pull-request"]
+    conditions = [tasks_for in ("github-pull-request", "github-pull-request-untrusted")]
     urls_revisions_and_can_skip = ((repo_url_from_payload, revision_from_payload, True), (repo_from_source_url, revision_from_source_url, False))
     for repo_url, revision, can_skip in urls_revisions_and_can_skip:
         # XXX In the case of scriptworker tasks, neither the repo nor the revision is defined

{scriptworker-60.0.0 → scriptworker-60.2.0}/src/scriptworker/utils.py

@@ -25,6 +25,7 @@ import async_timeout
 import yaml
 from taskcluster.client import createTemporaryCredentials
 
+import scriptworker.version
 from scriptworker.exceptions import Download404, DownloadError, ScriptWorkerException, ScriptWorkerRetryException, ScriptWorkerTaskException
 
 if TYPE_CHECKING:
@@ -37,6 +38,12 @@ else:
 log = logging.getLogger(__name__)
 
 
+# scriptworker_session {{{1
+def scriptworker_session(*args, **kwargs):
+    kwargs.setdefault("headers", {}).setdefault("User-Agent", f"scriptworker {scriptworker.version.__version_string__}")
+    return aiohttp.ClientSession(*args, **kwargs)
+
+
 # request {{{1
 async def request(context, url, timeout=60, method="get", good=(200,), retry=tuple(range(500, 512)), return_type="text", **kwargs):
     """Async aiohttp request wrapper.

{scriptworker-60.0.0 → scriptworker-60.2.0}/src/scriptworker/version.py

@@ -54,7 +54,7 @@ def get_version_string(version: Union[ShortVerType, LongVerType]) -> str:
 
 # 1}}}
 # Semantic versioning 2.0.0  http://semver.org/
-__version__ = (60, 0, 0)
+__version__ = (60, 2, 0)
 __version_string__ = get_version_string(__version__)
 
 

{scriptworker-60.0.0 → scriptworker-60.2.0}/src/scriptworker/worker.py

@@ -25,7 +25,7 @@ from scriptworker.cot.verify import ChainOfTrust, verify_chain_of_trust
 from scriptworker.exceptions import ScriptWorkerException, WorkerShutdownDuringTask
 from scriptworker.task import claim_work, complete_task, prepare_to_run_task, reclaim_task, run_task, worst_level
 from scriptworker.task_process import TaskProcess
-from scriptworker.utils import cleanup, filepaths_in_dir
+from scriptworker.utils import cleanup, filepaths_in_dir, scriptworker_session
 
 log = logging.getLogger(__name__)
 
@@ -210,7 +210,7 @@ async def async_main(context, credentials):
     Args:
         context (scriptworker.context.Context): the scriptworker context.
     """
-    async with aiohttp.ClientSession() as session:
+    async with scriptworker_session() as session:
         context.session = session
         context.credentials = credentials
         await run_tasks(context)

scriptworker-60.2.0/src/scriptworker.egg-info/PKG-INFO

@@ -0,0 +1,86 @@
+Metadata-Version: 2.1
+Name: scriptworker
+Version: 60.2.0
+Summary: TaskCluster Script Worker
+Home-page: https://github.com/mozilla-releng/scriptworker
+Author: Mozilla Release Engineering
+Author-email: release+python@mozilla.com
+License: MPL 2.0
+Classifier: Intended Audience :: Developers
+Classifier: Natural Language :: English
+Classifier: Programming Language :: Python :: 3.8
+Classifier: Programming Language :: Python :: 3.9
+Classifier: Programming Language :: Python :: 3.10
+Requires-Python: >=3.7
+License-File: LICENSE
+
+===================
+Scriptworker Readme
+===================
+
+.. image:: https://travis-ci.org/mozilla-releng/scriptworker.svg?branch=master
+    :target: https://travis-ci.org/mozilla-releng/scriptworker
+
+.. image:: https://coveralls.io/repos/github/mozilla-releng/scriptworker/badge.svg?branch=master
+    :target: https://coveralls.io/github/mozilla-releng/scriptworker?branch=master
+
+.. image:: https://readthedocs.org/projects/scriptworker/badge/?version=latest
+    :target: http://scriptworker.readthedocs.io/en/latest/?badge=latest
+    :alt: Documentation Status
+
+Scriptworker implements the `TaskCluster worker model`_, then launches a pre-defined script.
+
+.. _TaskCluster worker model: https://firefox-ci-tc.services.mozilla.com/docs/reference/platform/queue/worker-interaction
+
+This worker was designed for `Releng processes`_ that need specific, limited, and pre-defined capabilities.
+
+.. _Releng processes: https://bugzilla.mozilla.org/show_bug.cgi?id=1245837
+
+Free software: MPL2 License
+
+-----
+Usage
+-----
+* Create a config file.  By default scriptworker will look in ``./scriptworker.yaml``, but this config path can be specified as the first and only commandline argument.  There is an `example config file`_, and all config items are specified in `scriptworker.constants.DEFAULT_CONFIG`_.
+
+.. _example config file: https://github.com/mozilla-releng/scriptworker/blob/master/scriptworker.yaml.tmpl
+.. _scriptworker.constants.DEFAULT_CONFIG: https://github.com/mozilla-releng/scriptworker/blob/master/src/scriptworker/constants.py
+
+Credentials can live in ``./scriptworker.yaml``, ``./secrets.json``, ``~/.scriptworker``.
+
+* Launch: ``scriptworker [config_path]``
+
+-------
+Testing
+-------
+
+Without integration tests install tox, then
+
+``NO_CREDENTIALS_TESTS=1 tox -e py36``
+
+Without any tests connecting to the net, then ``NO_TESTS_OVER_WIRE=1 tox -e py36``
+
+With integration tests, first create a client in the Taskcluster UI with the scopes::
+
+    queue:cancel-task:test-dummy-scheduler/*
+    queue:claim-work:test-dummy-provisioner/dummy-worker-*
+    queue:create-task:lowest:test-dummy-provisioner/dummy-worker-*
+    queue:define-task:test-dummy-provisioner/dummy-worker-*
+    queue:get-artifact:SampleArtifacts/_/X.txt
+    queue:scheduler-id:test-dummy-scheduler
+    queue:schedule-task:test-dummy-scheduler/*
+    queue:task-group-id:test-dummy-scheduler/*
+    queue:worker-id:test-dummy-workers/dummy-worker-*
+
+Then generate a no priviledge personal access token in Github for the scriptworker_github_token (to avoid rate limiting) and create a ``./secrets.json`` or ``~/.scriptworker`` that looks like::
+
+    {
+        "integration_credentials": {
+            "clientId": "...",
+            "accessToken": "...",
+        }
+        "scriptworker_github_token": "..."
+    }
+
+
+then to run all tests: ``tox``

{scriptworker-60.0.0 → scriptworker-60.2.0}/src/scriptworker.egg-info/entry_points.txt

@@ -3,4 +3,3 @@ create_test_workdir = scriptworker.cot.verify:create_test_workdir
 scriptworker = scriptworker.worker:main
 verify_cot = scriptworker.cot.verify:verify_cot_cmdln
 verify_ed25519_signature = scriptworker.ed25519:verify_ed25519_signature_cmdln
-

{scriptworker-60.0.0 → scriptworker-60.2.0}/version.json

@@ -1,8 +1,8 @@
 {
     "version":[
         60,
-        0,
+        2,
         0
     ],
-    "version_string":"60.0.0"
+    "version_string":"60.2.0"
 }

scriptworker-60.0.0/PKG-INFO

@@ -1,19 +0,0 @@
-Metadata-Version: 2.1
-Name: scriptworker
-Version: 60.0.0
-Summary: TaskCluster Script Worker
-Home-page: https://github.com/mozilla-releng/scriptworker
-Author: Mozilla Release Engineering
-Author-email: release+python@mozilla.com
-License: MPL 2.0
-Platform: UNKNOWN
-Classifier: Intended Audience :: Developers
-Classifier: Natural Language :: English
-Classifier: Programming Language :: Python :: 3.8
-Classifier: Programming Language :: Python :: 3.9
-Classifier: Programming Language :: Python :: 3.10
-Requires-Python: >=3.7
-License-File: LICENSE
-
-UNKNOWN
-

scriptworker-60.0.0/src/scriptworker.egg-info/PKG-INFO

@@ -1,19 +0,0 @@
-Metadata-Version: 2.1
-Name: scriptworker
-Version: 60.0.0
-Summary: TaskCluster Script Worker
-Home-page: https://github.com/mozilla-releng/scriptworker
-Author: Mozilla Release Engineering
-Author-email: release+python@mozilla.com
-License: MPL 2.0
-Platform: UNKNOWN
-Classifier: Intended Audience :: Developers
-Classifier: Natural Language :: English
-Classifier: Programming Language :: Python :: 3.8
-Classifier: Programming Language :: Python :: 3.9
-Classifier: Programming Language :: Python :: 3.10
-Requires-Python: >=3.7
-License-File: LICENSE
-
-UNKNOWN
-