scriptworker 63.0.1__tar.gz → 63.2.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (30) hide show
  1. {scriptworker-63.0.1 → scriptworker-63.2.0}/HISTORY.rst +24 -0
  2. {scriptworker-63.0.1 → scriptworker-63.2.0}/PKG-INFO +1 -1
  3. {scriptworker-63.0.1 → scriptworker-63.2.0}/pyproject.toml +18 -31
  4. {scriptworker-63.0.1 → scriptworker-63.2.0}/src/scriptworker/artifacts.py +26 -0
  5. {scriptworker-63.0.1 → scriptworker-63.2.0}/src/scriptworker/constants.py +12 -1
  6. {scriptworker-63.0.1 → scriptworker-63.2.0}/src/scriptworker/context.py +10 -0
  7. {scriptworker-63.0.1 → scriptworker-63.2.0}/src/scriptworker/cot/verify.py +16 -11
  8. {scriptworker-63.0.1 → scriptworker-63.2.0}/src/scriptworker/ed25519.py +4 -2
  9. {scriptworker-63.0.1 → scriptworker-63.2.0}/src/scriptworker/github.py +2 -2
  10. {scriptworker-63.0.1 → scriptworker-63.2.0}/src/scriptworker/log.py +13 -0
  11. {scriptworker-63.0.1 → scriptworker-63.2.0}/src/scriptworker/task.py +12 -2
  12. {scriptworker-63.0.1 → scriptworker-63.2.0}/src/scriptworker/task_process.py +1 -1
  13. {scriptworker-63.0.1 → scriptworker-63.2.0}/src/scriptworker/utils.py +7 -4
  14. {scriptworker-63.0.1 → scriptworker-63.2.0}/src/scriptworker/worker.py +19 -3
  15. {scriptworker-63.0.1 → scriptworker-63.2.0}/tox.ini +3 -13
  16. {scriptworker-63.0.1 → scriptworker-63.2.0}/.gitignore +0 -0
  17. {scriptworker-63.0.1 → scriptworker-63.2.0}/CONTRIBUTING.rst +0 -0
  18. {scriptworker-63.0.1 → scriptworker-63.2.0}/LICENSE +0 -0
  19. {scriptworker-63.0.1 → scriptworker-63.2.0}/README.rst +0 -0
  20. {scriptworker-63.0.1 → scriptworker-63.2.0}/renovate.json +0 -0
  21. {scriptworker-63.0.1 → scriptworker-63.2.0}/scripts/gen_ed25519_key.py +0 -0
  22. {scriptworker-63.0.1 → scriptworker-63.2.0}/scriptworker.yaml.tmpl +0 -0
  23. {scriptworker-63.0.1 → scriptworker-63.2.0}/src/scriptworker/__init__.py +0 -0
  24. {scriptworker-63.0.1 → scriptworker-63.2.0}/src/scriptworker/client.py +0 -0
  25. {scriptworker-63.0.1 → scriptworker-63.2.0}/src/scriptworker/config.py +0 -0
  26. {scriptworker-63.0.1 → scriptworker-63.2.0}/src/scriptworker/cot/__init__.py +0 -0
  27. {scriptworker-63.0.1 → scriptworker-63.2.0}/src/scriptworker/cot/generate.py +0 -0
  28. {scriptworker-63.0.1 → scriptworker-63.2.0}/src/scriptworker/data/cot_v1_schema.json +0 -0
  29. {scriptworker-63.0.1 → scriptworker-63.2.0}/src/scriptworker/data/scriptworker_task_schema.json +0 -0
  30. {scriptworker-63.0.1 → scriptworker-63.2.0}/src/scriptworker/exceptions.py +0 -0
@@ -4,7 +4,31 @@ Change Log
4
4
  All notable changes to this project will be documented in this file.
5
5
  This project adheres to `Semantic Versioning <http://semver.org/>`__.
6
6
 
7
+ 63.2.0 - 2026-06-18
8
+ -------------------
9
+
10
+ Added
11
+ ~~~~~
12
+ - When Chain of Trust verification fails, the log is also exposed as public/logs/live_backing.log
13
+ (https://bugzilla.mozilla.org/show_bug.cgi?id=2039779)
14
+ - Values for ESR 153 to scriptworker constants (https://bugzilla.mozilla.org/show_bug.cgi?id=2046216)
15
+
16
+ Fixed
17
+ ~~~~~
18
+ - The TASKCLUSTER_CREDENTIALS_FD feature uses a regular temporary file rather than a memfd, to
19
+ make things work again on macOS
20
+
21
+ 63.1.0 - 2026-05-26
22
+ -------------------
23
+
24
+ Added
25
+ ~~~~~
26
+ - The TASKCLUSTER_CREDENTIALS_FD environment variable is passed to scripts, and
27
+ the referenced file descriptor (a memfd) contains temporary taskcluster
28
+ credentials using the task's scopes, updated after each reclaimTask
29
+
7
30
  63.0.1 - 2026-04-27
31
+ -------------------
8
32
 
9
33
  Fixed
10
34
  ~~~~~
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: scriptworker
3
- Version: 63.0.1
3
+ Version: 63.2.0
4
4
  Summary: Taskcluster scriptworker
5
5
  Author-email: Mozilla Release Engineering <releng@mozilla.com>
6
6
  License-File: LICENSE
@@ -1,6 +1,6 @@
1
1
  [project]
2
2
  name = "scriptworker"
3
- version = "63.0.1"
3
+ version = "63.2.0"
4
4
  description = "Taskcluster scriptworker"
5
5
  authors = [
6
6
  { name = "Mozilla Release Engineering", email = "releng@mozilla.com" },
@@ -43,11 +43,8 @@ dev = [
43
43
  "tox-uv",
44
44
  "aiohttp>=3",
45
45
  "asyncio_extras",
46
- "black",
47
46
  "coveralls",
48
- "flake8",
49
- "flake8_docstrings",
50
- "isort>=5",
47
+ "ruff",
51
48
  "mock",
52
49
  "pytest",
53
50
  "pytest-asyncio",
@@ -85,34 +82,24 @@ verify_cot = "scriptworker.cot.verify:verify_cot_cmdln"
85
82
  create_test_workdir = "scriptworker.cot.verify:create_test_workdir"
86
83
  verify_ed25519_signature = "scriptworker.ed25519:verify_ed25519_signature_cmdln"
87
84
 
88
- [tool.black]
85
+ [tool.ruff]
89
86
  line-length = 160
90
- target-version = ["py311", "py312", "py313"]
91
- include = '\.(wsgi|pyi?)$'
92
- exclude = '''
93
- /(
94
- \.eggs
95
- | \.git
96
- | \.hg
97
- | \.mypy_cache
98
- | \.tox
99
- | \.venv
100
- | \.cache
101
- | \.cache_py3
102
- | _build
103
- | buck-out
104
- | build
105
- | dist
106
- | ui
107
- )/
108
- '''
87
+ target-version = "py311"
88
+ extend-exclude = [
89
+ "docs"
90
+ ]
91
+
92
+ [tool.ruff.lint]
93
+ # F = pyflakes, E/W = pycodestyle, I = isort, D = pydocstyle
94
+ select = ["F", "E", "W", "I", "D"]
95
+
96
+ [tool.ruff.lint.per-file-ignores]
97
+ "tests/**" = ["D"]
98
+ "tests/test_github.py" = ["D", "E501"] # ignore long lines in test_github.py
99
+
109
100
 
110
- [tool.isort]
111
- multi_line_output = 3
112
- include_trailing_comma = true
113
- force_grid_wrap = 0
114
- use_parentheses = true
115
- line_length = 160
101
+ [tool.ruff.lint.pydocstyle]
102
+ convention = "pep257"
116
103
 
117
104
  [tool.pytest.ini_options]
118
105
  asyncio_mode = "auto"
@@ -183,6 +183,29 @@ async def create_artifact(context, path, target_path, content_type, content_enco
183
183
  raise ScriptWorkerRetryException("Bad status {}".format(resp.status))
184
184
 
185
185
 
186
+ async def create_link_artifact(context, target_path, link_to, content_type, expires=None):
187
+ """Create a Taskcluster link artifact that redirects to another artifact in the same task.
188
+
189
+ Args:
190
+ context (scriptworker.context.Context): the scriptworker context.
191
+ target_path (str): the artifact name to create (e.g., ``public/logs/live_backing.log``).
192
+ link_to (str): the artifact name to link to (e.g., ``public/logs/chain_of_trust.log``).
193
+ content_type (str): Content type (MIME type) of the linked artifact.
194
+ expires (str, optional): ISO datestring of when the artifact expires.
195
+ Defaults to ``context.task["expires"]``.
196
+
197
+ """
198
+ payload = {
199
+ "storageType": "link",
200
+ "expires": expires or get_expiration_arrow(context).isoformat(),
201
+ "contentType": content_type,
202
+ "artifact": link_to,
203
+ }
204
+ args = [get_task_id(context.claim_task), get_run_id(context.claim_task), target_path, payload]
205
+ log.info("Creating link artifact {} -> {}".format(target_path, link_to))
206
+ await context.temp_queue.createArtifact(*args)
207
+
208
+
186
209
  def _craft_artifact_put_headers(content_type, encoding=None):
187
210
  log.debug("{} {}".format(content_type, encoding))
188
211
  headers = {aiohttp.hdrs.CONTENT_TYPE: content_type}
@@ -225,6 +248,7 @@ def get_artifact_url(context, task_id, path):
225
248
 
226
249
  # list_latest_artifacts {{{1
227
250
  async def list_latest_artifacts(queue, task_id):
251
+ """List all artifacts for the latest run of a task, following pagination."""
228
252
  artifacts = []
229
253
 
230
254
  def pagination_handler(response):
@@ -235,6 +259,7 @@ async def list_latest_artifacts(queue, task_id):
235
259
 
236
260
 
237
261
  async def retry_list_latest_artifacts(queue, task_id, exception=TaskclusterFailure, **kwargs):
262
+ """Retry ``list_latest_artifacts`` on transient Taskcluster failures."""
238
263
  kwargs.setdefault("retry_exceptions", tuple(set([TaskclusterFailure, exception])))
239
264
  return await retry_async(list_latest_artifacts, args=(queue, task_id), **kwargs)
240
265
 
@@ -442,6 +467,7 @@ def assert_is_parent(path, parent_dir):
442
467
 
443
468
 
444
469
  def get_artifacts_matching_glob(context, task_id, pattern):
470
+ """Return paths of downloaded artifacts matching ``pattern`` under the task's cot/<id> dir."""
445
471
  parent_dir = os.path.abspath(os.path.join(context.config["work_dir"], "cot", task_id))
446
472
  matching = []
447
473
  for root, _, files in os.walk(parent_dir):
@@ -559,6 +559,7 @@ DEFAULT_CONFIG: immutabledict[str, Any] = immutabledict(
559
559
  "project:releng:lando:repo:release": "release",
560
560
  "project:releng:lando:repo:esr115": "esr115",
561
561
  "project:releng:lando:repo:esr140": "esr140",
562
+ "project:releng:lando:repo:esr153": "esr153",
562
563
  }
563
564
  ),
564
565
  "glean": immutabledict({"project:mozilla:glean:releng:beetmover:bucket:maven-production": "glean-repo"}),
@@ -645,6 +646,7 @@ DEFAULT_CONFIG: immutabledict[str, Any] = immutabledict(
645
646
  "/releases/mozilla-release",
646
647
  "/releases/mozilla-esr115",
647
648
  "/releases/mozilla-esr140",
649
+ "/releases/mozilla-esr153",
648
650
  "/projects/birch",
649
651
  # Bug 1821839 hneiva using to test hardened signing
650
652
  "/projects/maple",
@@ -654,6 +656,7 @@ DEFAULT_CONFIG: immutabledict[str, Any] = immutabledict(
654
656
  "/releases/mozilla-release",
655
657
  "/releases/mozilla-esr115",
656
658
  "/releases/mozilla-esr140",
659
+ "/releases/mozilla-esr153",
657
660
  "/projects/birch",
658
661
  "/projects/maple",
659
662
  "/integration/autoland",
@@ -667,9 +670,11 @@ DEFAULT_CONFIG: immutabledict[str, Any] = immutabledict(
667
670
  ),
668
671
  "esr115": ("/releases/mozilla-esr115",),
669
672
  "esr140": ("/releases/mozilla-esr140",),
673
+ "esr153": ("/releases/mozilla-esr153",),
670
674
  "esr": (
671
675
  "/releases/mozilla-esr115",
672
676
  "/releases/mozilla-esr140",
677
+ "/releases/mozilla-esr153",
673
678
  ),
674
679
  "nightly": ("/mozilla-central",),
675
680
  "autoland": ("/integration/autoland",),
@@ -681,6 +686,7 @@ DEFAULT_CONFIG: immutabledict[str, Any] = immutabledict(
681
686
  "/releases/mozilla-release",
682
687
  "/releases/mozilla-esr115",
683
688
  "/releases/mozilla-esr140",
689
+ "/releases/mozilla-esr153",
684
690
  # bug 1845368: a permanent project branch for testing updates
685
691
  "/projects/pine",
686
692
  "/projects/larch",
@@ -693,6 +699,7 @@ DEFAULT_CONFIG: immutabledict[str, Any] = immutabledict(
693
699
  "/releases/mozilla-release",
694
700
  "/releases/mozilla-esr115",
695
701
  "/releases/mozilla-esr140",
702
+ "/releases/mozilla-esr153",
696
703
  ),
697
704
  "all-production-branches-and-autoland": (
698
705
  "/mozilla-central",
@@ -700,6 +707,7 @@ DEFAULT_CONFIG: immutabledict[str, Any] = immutabledict(
700
707
  "/releases/mozilla-release",
701
708
  "/releases/mozilla-esr115",
702
709
  "/releases/mozilla-esr140",
710
+ "/releases/mozilla-esr153",
703
711
  "/integration/autoland",
704
712
  ),
705
713
  }
@@ -728,6 +736,7 @@ DEFAULT_CONFIG: immutabledict[str, Any] = immutabledict(
728
736
  "/releases/comm-release",
729
737
  "/releases/comm-esr115",
730
738
  "/releases/comm-esr140",
739
+ "/releases/comm-esr153",
731
740
  ),
732
741
  "beta": ("/releases/comm-beta",),
733
742
  "release": ("/releases/comm-release",),
@@ -735,14 +744,16 @@ DEFAULT_CONFIG: immutabledict[str, Any] = immutabledict(
735
744
  "/releases/comm-release",
736
745
  "/releases/comm-esr115",
737
746
  "/releases/comm-esr140",
747
+ "/releases/comm-esr153",
738
748
  ),
739
- "esr": ("/releases/comm-esr115", "/releases/comm-esr140"),
749
+ "esr": ("/releases/comm-esr115", "/releases/comm-esr140", "/releases/comm-esr153"),
740
750
  "all-nightly-branches": (
741
751
  "/comm-central",
742
752
  "/releases/comm-beta",
743
753
  "/releases/comm-release",
744
754
  "/releases/comm-esr115",
745
755
  "/releases/comm-esr140",
756
+ "/releases/comm-esr153",
746
757
  ),
747
758
  "nightly": ("/comm-central",),
748
759
  }
@@ -59,6 +59,7 @@ class Context(object):
59
59
 
60
60
  config: Optional[Dict[str, Any]] = None
61
61
  credentials_timestamp: Optional[int] = None
62
+ credentials_file = None
62
63
  proc: Optional[task_process.TaskProcess] = None
63
64
  queue: Optional[Queue] = None
64
65
  session: Optional[aiohttp.ClientSession] = None
@@ -98,6 +99,7 @@ class Context(object):
98
99
  if claim_task:
99
100
  self.task = claim_task["task"]
100
101
  self.verify_task()
102
+ self.credentials_file = tempfile.TemporaryFile()
101
103
  self.temp_credentials = claim_task["credentials"]
102
104
  path = os.path.join(self.config["work_dir"], "task.json")
103
105
  assert self.task
@@ -105,6 +107,9 @@ class Context(object):
105
107
  else:
106
108
  self.temp_credentials = None
107
109
  self.task = None
110
+ if self.credentials_file is not None:
111
+ self.credentials_file.close()
112
+ self.credentials_file = None
108
113
 
109
114
  def verify_task(self) -> None:
110
115
  """Run some task sanity checks on ``self.task``."""
@@ -193,6 +198,10 @@ class Context(object):
193
198
  def temp_credentials(self, credentials: Optional[Dict[str, Any]]) -> None:
194
199
  self._temp_credentials = credentials
195
200
  self.temp_queue = self.create_queue(self.temp_credentials)
201
+ if credentials:
202
+ data = json.dumps(credentials, indent=2, sort_keys=True).encode("ascii")
203
+ # use pwrite so we don't confuse the child by changing the file offset
204
+ assert os.pwrite(self.credentials_file.fileno(), data, 0) == len(data)
196
205
 
197
206
  def write_json(self, path: str, contents: Dict[str, Any], message: str) -> None:
198
207
  """Write json to disk.
@@ -261,6 +270,7 @@ class Context(object):
261
270
 
262
271
  @property
263
272
  def download_semaphore(self) -> asyncio.BoundedSemaphore:
273
+ """Return a lazily-built semaphore limiting concurrent downloads to ``max_concurrent_downloads``."""
264
274
  assert self.config
265
275
  if self._download_semaphore is None:
266
276
  try:
@@ -41,7 +41,7 @@ from scriptworker.context import Context
41
41
  from scriptworker.ed25519 import ed25519_public_key_from_string, verify_ed25519_signature
42
42
  from scriptworker.exceptions import BaseDownloadError, CoTError, ScriptWorkerEd25519Error
43
43
  from scriptworker.github import GitHubRepository, extract_github_repo_full_name, extract_github_repo_owner_and_name, extract_github_repo_ssh_url
44
- from scriptworker.log import contextual_log_handler
44
+ from scriptworker.log import contextual_log_handler, get_chain_of_trust_log_filename
45
45
  from scriptworker.task import (
46
46
  get_action_callback_name,
47
47
  get_and_check_tasks_for,
@@ -180,7 +180,8 @@ class ChainOfTrust(object):
180
180
 
181
181
  def is_scope_in_restricted_scopes(self, scope, restricted_scopes):
182
182
  """Determine if a scope matches in a list of restricted_scopes.
183
- if one of the restricted scopes ends with '*', find a partial match.
183
+
184
+ If one of the restricted scopes ends with '*', find a partial match.
184
185
 
185
186
  Returns:
186
187
  String: string of matching restricted_scope, if no match ""
@@ -728,8 +729,7 @@ async def download_cot_artifact(chain, task_id, path):
728
729
  link = chain.get_link(task_id)
729
730
  log.debug("Verifying {} is in {} cot artifacts...".format(path, task_id))
730
731
  if not link.cot:
731
- log.warning('Chain of Trust for "{}" in {} does not exist. See above log for more details. \
732
- Skipping download of this artifact'.format(path, task_id))
732
+ log.warning(f'Chain of Trust for "{path}" in {task_id} does not exist. See above log for more details. Skipping download of this artifact.')
733
733
  return
734
734
 
735
735
  if path not in link.cot["artifacts"]:
@@ -1094,8 +1094,9 @@ async def _get_additional_hg_push_jsone_context(parent_link, decision_link):
1094
1094
  allowed_comments.append(push_comment[try_idx:].split("\n")[0])
1095
1095
  if decision_comment not in allowed_comments:
1096
1096
  raise CoTError(
1097
- "Decision task {} comment doesn't match the push comment!\n"
1098
- "Decision comment: \n{}\nPush comment: \n{}".format(decision_link.name, decision_comment, push_comment)
1097
+ "Decision task {} comment doesn't match the push comment!\nDecision comment: \n{}\nPush comment: \n{}".format(
1098
+ decision_link.name, decision_comment, push_comment
1099
+ )
1099
1100
  )
1100
1101
  return {
1101
1102
  "push": {
@@ -2041,7 +2042,7 @@ async def verify_chain_of_trust(chain, *, check_task=False):
2041
2042
  CoTError: on failure
2042
2043
 
2043
2044
  """
2044
- log_path = os.path.join(chain.context.config["task_log_dir"], "chain_of_trust.log")
2045
+ log_path = get_chain_of_trust_log_filename(chain.context)
2045
2046
  scriptworker_log = logging.getLogger("scriptworker")
2046
2047
  with contextual_log_handler(
2047
2048
  chain.context,
@@ -2113,7 +2114,8 @@ def verify_cot_cmdln(args=None, event_loop=None):
2113
2114
 
2114
2115
  """
2115
2116
  args = args or sys.argv[1:]
2116
- parser = argparse.ArgumentParser(description="""Verify a given task's chain of trust.
2117
+ parser = argparse.ArgumentParser(
2118
+ description="""Verify a given task's chain of trust.
2117
2119
 
2118
2120
  Given a task's `task_id`, get its task definition, then trace its chain of
2119
2121
  trust back to the tree. This doesn't verify chain of trust artifact signatures,
@@ -2125,7 +2127,8 @@ To use, first either set your taskcluster creds in your env http://bit.ly/2eDMa6
2125
2127
  or in the CREDS_FILES http://bit.ly/2fVMu0A
2126
2128
 
2127
2129
  If you are verifying against a private github repo, please also set in environment
2128
- SCRIPTWORKER_GITHUB_OAUTH_TOKEN to an OAUTH token with read permissions to the repo""")
2130
+ SCRIPTWORKER_GITHUB_OAUTH_TOKEN to an OAUTH token with read permissions to the repo"""
2131
+ )
2129
2132
  parser.add_argument("task_id", help="the task id to test")
2130
2133
  parser.add_argument("--task-type", help="the task type to test", choices=sorted(get_valid_task_types().keys()), required=True)
2131
2134
  parser.add_argument("--cleanup", help="clean up the temp dir afterwards", dest="cleanup", action="store_true", default=False)
@@ -2192,13 +2195,15 @@ def create_test_workdir(args=None, event_loop=None):
2192
2195
 
2193
2196
  """
2194
2197
  args = args or sys.argv[1:]
2195
- parser = argparse.ArgumentParser(description="""Populate a test `work_dir`.
2198
+ parser = argparse.ArgumentParser(
2199
+ description="""Populate a test `work_dir`.
2196
2200
 
2197
2201
  Given a scriptworker task's `task_id`, get its task definition, write it to
2198
2202
  `./work/task.json`, then download its `upstreamArtifacts` and put them in
2199
2203
  `./work/cot/TASK_ID/PATH`.
2200
2204
 
2201
- This is helpful in manually testing a *script run.""")
2205
+ This is helpful in manually testing a *script run."""
2206
+ )
2202
2207
  parser.add_argument("--path", help="relative path to the work_dir", default="work")
2203
2208
  parser.add_argument("--overwrite", help="overwrite an existing work_dir", action="store_true")
2204
2209
  parser.add_argument("task_id", help="the task id to test")
@@ -141,11 +141,13 @@ def verify_ed25519_signature_cmdln(args=None, exception=SystemExit):
141
141
 
142
142
  """
143
143
  args = args or sys.argv[1:]
144
- parser = argparse.ArgumentParser(description="""Verify an ed25519 signature from the command line.
144
+ parser = argparse.ArgumentParser(
145
+ description="""Verify an ed25519 signature from the command line.
145
146
 
146
147
  Given a file and its detached signature, verify that it has been signed with
147
148
  a valid key. This key can be specified on the command line; otherwise we'll
148
- default to ``config['ed25519_public_keys']``.""")
149
+ default to ``config['ed25519_public_keys']``."""
150
+ )
149
151
  parser.add_argument("--pubkey", help="path to a base64-encoded ed25519 pubkey, optional")
150
152
  parser.add_argument("file_path")
151
153
  parser.add_argument("sig_path")
@@ -121,7 +121,7 @@ class GitHubRepository:
121
121
  if any(vcs_rule.get("require_secret") for vcs_rule in context.config["trusted_vcs_rules"]):
122
122
  # This check uses unofficial API on github, which we can't easily
123
123
  # check for private repos, assume its true in the private case.
124
- log.info("has_commit_landed_on_repository() not implemented for private" "repositories, assume True")
124
+ log.info("has_commit_landed_on_repository() not implemented for private repositories, assume True")
125
125
  return True
126
126
 
127
127
  # Revision may be a tag name. `branch_commits` doesn't work on tags
@@ -293,7 +293,7 @@ def is_github_repo_owner_the_official_one(context, repo_owner):
293
293
  official_repo_owner = context.config["official_github_repos_owner"]
294
294
  if not official_repo_owner:
295
295
  raise ConfigError(
296
- "This worker does not have a defined owner for official GitHub repositories. " 'Given "official_github_repos_owner": {}'.format(official_repo_owner)
296
+ 'This worker does not have a defined owner for official GitHub repositories. Given "official_github_repos_owner": {}'.format(official_repo_owner)
297
297
  )
298
298
 
299
299
  return official_repo_owner == repo_owner
@@ -115,6 +115,19 @@ def get_log_filename(context: Any) -> str:
115
115
  return os.path.join(context.config["task_log_dir"], "live_backing.log")
116
116
 
117
117
 
118
+ def get_chain_of_trust_log_filename(context: Any) -> str:
119
+ """Get the chain of trust verification log file path.
120
+
121
+ Args:
122
+ context (scriptworker.context.Context): the scriptworker context.
123
+
124
+ Returns:
125
+ string: chain of trust log file path
126
+
127
+ """
128
+ return os.path.join(context.config["task_log_dir"], "chain_of_trust.log")
129
+
130
+
118
131
  @contextmanager
119
132
  def get_log_filehandle(context: Any) -> Iterator[IO[str]]:
120
133
  """Open the log and error filehandles.
@@ -117,7 +117,7 @@ def get_run_id(claim_task):
117
117
 
118
118
  # get_task_maxruntime {{{1
119
119
  def get_task_maxruntime(task_def, max_timeout):
120
- """Given a task definition, return the lower of max_timeout and maxRunTime if set"""
120
+ """Given a task definition, return the lower of max_timeout and maxRunTime if set."""
121
121
  max_run_time = task_def["payload"].get("maxRunTime")
122
122
  if max_run_time is None:
123
123
  return max_timeout
@@ -672,7 +672,17 @@ async def run_task(context, to_cancellable_process):
672
672
  env["TASK_ID"] = context.task_id or "None"
673
673
  env["RUN_ID"] = str(get_run_id(context.claim_task))
674
674
  env["TASKCLUSTER_ROOT_URL"] = context.config["taskcluster_root_url"]
675
- kwargs = {"stdout": PIPE, "stderr": PIPE, "stdin": None, "close_fds": True, "preexec_fn": lambda: os.setsid(), "env": env} # pragma: no branch
675
+ credentials_fd = context.credentials_file.fileno()
676
+ env["TASKCLUSTER_CREDENTIALS_FD"] = str(credentials_fd)
677
+ kwargs = {
678
+ "stdout": PIPE,
679
+ "stderr": PIPE,
680
+ "stdin": None,
681
+ "close_fds": True,
682
+ "preexec_fn": lambda: os.setsid(),
683
+ "env": env,
684
+ "pass_fds": (credentials_fd,),
685
+ } # pragma: no branch
676
686
 
677
687
  timeout = get_task_maxruntime(context.task, context.config["task_max_timeout"])
678
688
  subprocess = await asyncio.create_subprocess_exec(*context.config["task_script"], **kwargs)
@@ -19,7 +19,7 @@ class TaskProcess:
19
19
  """Wraps worker task's process."""
20
20
 
21
21
  def __init__(self, process: Process):
22
- """Constructor.
22
+ """Initialize the wrapper around the worker task's subprocess.
23
23
 
24
24
  Args:
25
25
  process (Process): task process
@@ -42,6 +42,7 @@ log = logging.getLogger(__name__)
42
42
 
43
43
  # scriptworker_session {{{1
44
44
  def scriptworker_session(*args, **kwargs):
45
+ """Return an ``aiohttp.ClientSession`` with the scriptworker User-Agent header set."""
45
46
  kwargs.setdefault("headers", {}).setdefault("User-Agent", f"scriptworker {scriptworker.__version__}")
46
47
  return aiohttp.ClientSession(*args, **kwargs)
47
48
 
@@ -875,8 +876,9 @@ def remove_empty_keys(values, remove=({}, None, [], "null")):
875
876
 
876
877
  # add_projectid {{{1
877
878
  def add_projectid(task_def):
878
- """Add a projectId property to a task, if none already exists, using
879
- the Taskcluster default value of 'none'.
879
+ """Add a projectId property to a task, if none already exists.
880
+
881
+ Uses the Taskcluster default value of 'none'.
880
882
 
881
883
  Args:
882
884
  task_def (dict): the task definition
@@ -889,8 +891,9 @@ def add_projectid(task_def):
889
891
 
890
892
  # add_taskqueueid {{{1
891
893
  def add_taskqueueid(task_def):
892
- """Add a taskQueueId property to a task, if none already exists, based
893
- on the provisionerId and workerType. Then remove those two properties.
894
+ """Add a taskQueueId property to a task, if none already exists.
895
+
896
+ Builds the value from the provisionerId and workerType, then removes those two properties.
894
897
 
895
898
  Args:
896
899
  task_def (dict): the task definition
@@ -8,6 +8,7 @@ Attributes:
8
8
 
9
9
  import asyncio
10
10
  import logging
11
+ import os
11
12
  import signal
12
13
  import socket
13
14
  import sys
@@ -17,12 +18,13 @@ from typing import Any
17
18
  import aiohttp
18
19
  import arrow
19
20
 
20
- from scriptworker.artifacts import upload_artifacts
21
+ from scriptworker.artifacts import create_link_artifact, upload_artifacts
21
22
  from scriptworker.config import get_context_from_cmdln
22
23
  from scriptworker.constants import STATUSES
23
24
  from scriptworker.cot.generate import generate_cot
24
25
  from scriptworker.cot.verify import ChainOfTrust, verify_chain_of_trust
25
26
  from scriptworker.exceptions import ScriptWorkerException, WorkerShutdownDuringTask
27
+ from scriptworker.log import get_chain_of_trust_log_filename
26
28
  from scriptworker.task import claim_work, complete_task, prepare_to_run_task, reclaim_task, run_task, worst_level
27
29
  from scriptworker.task_process import TaskProcess
28
30
  from scriptworker.utils import cleanup, filepaths_in_dir, scriptworker_session
@@ -53,7 +55,21 @@ async def do_run_task(context, run_cancellable, to_cancellable_process):
53
55
  try:
54
56
  if context.config["verify_chain_of_trust"]:
55
57
  chain = ChainOfTrust(context, context.config["cot_job_type"])
56
- await run_cancellable(verify_chain_of_trust(chain))
58
+ try:
59
+ await run_cancellable(verify_chain_of_trust(chain))
60
+ except Exception:
61
+ # Point live_backing.log at chain_of_trust.log so Treeherder parses the CoT failure
62
+ if os.path.exists(get_chain_of_trust_log_filename(context)):
63
+ try:
64
+ await create_link_artifact(
65
+ context,
66
+ target_path="public/logs/live_backing.log",
67
+ link_to="public/logs/chain_of_trust.log",
68
+ content_type="text/plain",
69
+ )
70
+ except Exception as link_exc:
71
+ log.exception("Failed to create live_backing.log link artifact: {}".format(link_exc))
72
+ raise
57
73
  status = await run_task(context, to_cancellable_process)
58
74
  generate_cot(context)
59
75
  except asyncio.CancelledError:
@@ -104,7 +120,7 @@ class RunTasks:
104
120
  """Manages processing of Taskcluster tasks."""
105
121
 
106
122
  def __init__(self):
107
- """Constructor."""
123
+ """Initialize the task-processing state."""
108
124
  self.future = None
109
125
  self.task_process = None
110
126
  self.is_cancelled = False
@@ -54,26 +54,16 @@ commands=
54
54
 
55
55
  [testenv:check]
56
56
  skip_install = true
57
+ deps = ruff
57
58
  commands =
58
- black --check {toxinidir}
59
- isort --check --df {toxinidir}
60
- flake8 {toxinidir}
59
+ ruff check {toxinidir}
60
+ ruff format --check {toxinidir}
61
61
 
62
62
  [testenv:mypy]
63
63
  commands =
64
64
  mypy --config {toxinidir}/mypi.ini {toxinidir}/src
65
65
 
66
66
 
67
- [flake8]
68
- max-line-length = 160
69
- extend-ignore = E203
70
- # test_github.py ignored because of https://gitlab.com/pycqa/flake8/issues/375
71
- exclude = .ropeproject,.tox,sandbox,docs,.eggs,*.egg,*.egg-info,setup.py,build/,tests/test_github.py,.venv
72
- per-file-ignores =
73
- # Ignore documentation issues in tests
74
- tests:*:D
75
- show-source = True
76
-
77
67
  [pytest]
78
68
  norecursedirs = .tox .git .hg sandbox .eggs build
79
69
  python_files = test_*.py
File without changes
File without changes
File without changes