scriptworker 63.0.1__tar.gz → 63.2.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {scriptworker-63.0.1 → scriptworker-63.2.0}/HISTORY.rst +24 -0
- {scriptworker-63.0.1 → scriptworker-63.2.0}/PKG-INFO +1 -1
- {scriptworker-63.0.1 → scriptworker-63.2.0}/pyproject.toml +18 -31
- {scriptworker-63.0.1 → scriptworker-63.2.0}/src/scriptworker/artifacts.py +26 -0
- {scriptworker-63.0.1 → scriptworker-63.2.0}/src/scriptworker/constants.py +12 -1
- {scriptworker-63.0.1 → scriptworker-63.2.0}/src/scriptworker/context.py +10 -0
- {scriptworker-63.0.1 → scriptworker-63.2.0}/src/scriptworker/cot/verify.py +16 -11
- {scriptworker-63.0.1 → scriptworker-63.2.0}/src/scriptworker/ed25519.py +4 -2
- {scriptworker-63.0.1 → scriptworker-63.2.0}/src/scriptworker/github.py +2 -2
- {scriptworker-63.0.1 → scriptworker-63.2.0}/src/scriptworker/log.py +13 -0
- {scriptworker-63.0.1 → scriptworker-63.2.0}/src/scriptworker/task.py +12 -2
- {scriptworker-63.0.1 → scriptworker-63.2.0}/src/scriptworker/task_process.py +1 -1
- {scriptworker-63.0.1 → scriptworker-63.2.0}/src/scriptworker/utils.py +7 -4
- {scriptworker-63.0.1 → scriptworker-63.2.0}/src/scriptworker/worker.py +19 -3
- {scriptworker-63.0.1 → scriptworker-63.2.0}/tox.ini +3 -13
- {scriptworker-63.0.1 → scriptworker-63.2.0}/.gitignore +0 -0
- {scriptworker-63.0.1 → scriptworker-63.2.0}/CONTRIBUTING.rst +0 -0
- {scriptworker-63.0.1 → scriptworker-63.2.0}/LICENSE +0 -0
- {scriptworker-63.0.1 → scriptworker-63.2.0}/README.rst +0 -0
- {scriptworker-63.0.1 → scriptworker-63.2.0}/renovate.json +0 -0
- {scriptworker-63.0.1 → scriptworker-63.2.0}/scripts/gen_ed25519_key.py +0 -0
- {scriptworker-63.0.1 → scriptworker-63.2.0}/scriptworker.yaml.tmpl +0 -0
- {scriptworker-63.0.1 → scriptworker-63.2.0}/src/scriptworker/__init__.py +0 -0
- {scriptworker-63.0.1 → scriptworker-63.2.0}/src/scriptworker/client.py +0 -0
- {scriptworker-63.0.1 → scriptworker-63.2.0}/src/scriptworker/config.py +0 -0
- {scriptworker-63.0.1 → scriptworker-63.2.0}/src/scriptworker/cot/__init__.py +0 -0
- {scriptworker-63.0.1 → scriptworker-63.2.0}/src/scriptworker/cot/generate.py +0 -0
- {scriptworker-63.0.1 → scriptworker-63.2.0}/src/scriptworker/data/cot_v1_schema.json +0 -0
- {scriptworker-63.0.1 → scriptworker-63.2.0}/src/scriptworker/data/scriptworker_task_schema.json +0 -0
- {scriptworker-63.0.1 → scriptworker-63.2.0}/src/scriptworker/exceptions.py +0 -0
|
@@ -4,7 +4,31 @@ Change Log
|
|
|
4
4
|
All notable changes to this project will be documented in this file.
|
|
5
5
|
This project adheres to `Semantic Versioning <http://semver.org/>`__.
|
|
6
6
|
|
|
7
|
+
63.2.0 - 2026-06-18
|
|
8
|
+
-------------------
|
|
9
|
+
|
|
10
|
+
Added
|
|
11
|
+
~~~~~
|
|
12
|
+
- When Chain of Trust verification fails, the log is also exposed as public/logs/live_backing.log
|
|
13
|
+
(https://bugzilla.mozilla.org/show_bug.cgi?id=2039779)
|
|
14
|
+
- Values for ESR 153 to scriptworker constants (https://bugzilla.mozilla.org/show_bug.cgi?id=2046216)
|
|
15
|
+
|
|
16
|
+
Fixed
|
|
17
|
+
~~~~~
|
|
18
|
+
- The TASKCLUSTER_CREDENTIALS_FD feature uses a regular temporary file rather than a memfd, to
|
|
19
|
+
make things work again on macOS
|
|
20
|
+
|
|
21
|
+
63.1.0 - 2026-05-26
|
|
22
|
+
-------------------
|
|
23
|
+
|
|
24
|
+
Added
|
|
25
|
+
~~~~~
|
|
26
|
+
- The TASKCLUSTER_CREDENTIALS_FD environment variable is passed to scripts, and
|
|
27
|
+
the referenced file descriptor (a memfd) contains temporary taskcluster
|
|
28
|
+
credentials using the task's scopes, updated after each reclaimTask
|
|
29
|
+
|
|
7
30
|
63.0.1 - 2026-04-27
|
|
31
|
+
-------------------
|
|
8
32
|
|
|
9
33
|
Fixed
|
|
10
34
|
~~~~~
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
[project]
|
|
2
2
|
name = "scriptworker"
|
|
3
|
-
version = "63.0
|
|
3
|
+
version = "63.2.0"
|
|
4
4
|
description = "Taskcluster scriptworker"
|
|
5
5
|
authors = [
|
|
6
6
|
{ name = "Mozilla Release Engineering", email = "releng@mozilla.com" },
|
|
@@ -43,11 +43,8 @@ dev = [
|
|
|
43
43
|
"tox-uv",
|
|
44
44
|
"aiohttp>=3",
|
|
45
45
|
"asyncio_extras",
|
|
46
|
-
"black",
|
|
47
46
|
"coveralls",
|
|
48
|
-
"
|
|
49
|
-
"flake8_docstrings",
|
|
50
|
-
"isort>=5",
|
|
47
|
+
"ruff",
|
|
51
48
|
"mock",
|
|
52
49
|
"pytest",
|
|
53
50
|
"pytest-asyncio",
|
|
@@ -85,34 +82,24 @@ verify_cot = "scriptworker.cot.verify:verify_cot_cmdln"
|
|
|
85
82
|
create_test_workdir = "scriptworker.cot.verify:create_test_workdir"
|
|
86
83
|
verify_ed25519_signature = "scriptworker.ed25519:verify_ed25519_signature_cmdln"
|
|
87
84
|
|
|
88
|
-
[tool.
|
|
85
|
+
[tool.ruff]
|
|
89
86
|
line-length = 160
|
|
90
|
-
target-version =
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
| buck-out
|
|
104
|
-
| build
|
|
105
|
-
| dist
|
|
106
|
-
| ui
|
|
107
|
-
)/
|
|
108
|
-
'''
|
|
87
|
+
target-version = "py311"
|
|
88
|
+
extend-exclude = [
|
|
89
|
+
"docs"
|
|
90
|
+
]
|
|
91
|
+
|
|
92
|
+
[tool.ruff.lint]
|
|
93
|
+
# F = pyflakes, E/W = pycodestyle, I = isort, D = pydocstyle
|
|
94
|
+
select = ["F", "E", "W", "I", "D"]
|
|
95
|
+
|
|
96
|
+
[tool.ruff.lint.per-file-ignores]
|
|
97
|
+
"tests/**" = ["D"]
|
|
98
|
+
"tests/test_github.py" = ["D", "E501"] # ignore long lines in test_github.py
|
|
99
|
+
|
|
109
100
|
|
|
110
|
-
[tool.
|
|
111
|
-
|
|
112
|
-
include_trailing_comma = true
|
|
113
|
-
force_grid_wrap = 0
|
|
114
|
-
use_parentheses = true
|
|
115
|
-
line_length = 160
|
|
101
|
+
[tool.ruff.lint.pydocstyle]
|
|
102
|
+
convention = "pep257"
|
|
116
103
|
|
|
117
104
|
[tool.pytest.ini_options]
|
|
118
105
|
asyncio_mode = "auto"
|
|
@@ -183,6 +183,29 @@ async def create_artifact(context, path, target_path, content_type, content_enco
|
|
|
183
183
|
raise ScriptWorkerRetryException("Bad status {}".format(resp.status))
|
|
184
184
|
|
|
185
185
|
|
|
186
|
+
async def create_link_artifact(context, target_path, link_to, content_type, expires=None):
|
|
187
|
+
"""Create a Taskcluster link artifact that redirects to another artifact in the same task.
|
|
188
|
+
|
|
189
|
+
Args:
|
|
190
|
+
context (scriptworker.context.Context): the scriptworker context.
|
|
191
|
+
target_path (str): the artifact name to create (e.g., ``public/logs/live_backing.log``).
|
|
192
|
+
link_to (str): the artifact name to link to (e.g., ``public/logs/chain_of_trust.log``).
|
|
193
|
+
content_type (str): Content type (MIME type) of the linked artifact.
|
|
194
|
+
expires (str, optional): ISO datestring of when the artifact expires.
|
|
195
|
+
Defaults to ``context.task["expires"]``.
|
|
196
|
+
|
|
197
|
+
"""
|
|
198
|
+
payload = {
|
|
199
|
+
"storageType": "link",
|
|
200
|
+
"expires": expires or get_expiration_arrow(context).isoformat(),
|
|
201
|
+
"contentType": content_type,
|
|
202
|
+
"artifact": link_to,
|
|
203
|
+
}
|
|
204
|
+
args = [get_task_id(context.claim_task), get_run_id(context.claim_task), target_path, payload]
|
|
205
|
+
log.info("Creating link artifact {} -> {}".format(target_path, link_to))
|
|
206
|
+
await context.temp_queue.createArtifact(*args)
|
|
207
|
+
|
|
208
|
+
|
|
186
209
|
def _craft_artifact_put_headers(content_type, encoding=None):
|
|
187
210
|
log.debug("{} {}".format(content_type, encoding))
|
|
188
211
|
headers = {aiohttp.hdrs.CONTENT_TYPE: content_type}
|
|
@@ -225,6 +248,7 @@ def get_artifact_url(context, task_id, path):
|
|
|
225
248
|
|
|
226
249
|
# list_latest_artifacts {{{1
|
|
227
250
|
async def list_latest_artifacts(queue, task_id):
|
|
251
|
+
"""List all artifacts for the latest run of a task, following pagination."""
|
|
228
252
|
artifacts = []
|
|
229
253
|
|
|
230
254
|
def pagination_handler(response):
|
|
@@ -235,6 +259,7 @@ async def list_latest_artifacts(queue, task_id):
|
|
|
235
259
|
|
|
236
260
|
|
|
237
261
|
async def retry_list_latest_artifacts(queue, task_id, exception=TaskclusterFailure, **kwargs):
|
|
262
|
+
"""Retry ``list_latest_artifacts`` on transient Taskcluster failures."""
|
|
238
263
|
kwargs.setdefault("retry_exceptions", tuple(set([TaskclusterFailure, exception])))
|
|
239
264
|
return await retry_async(list_latest_artifacts, args=(queue, task_id), **kwargs)
|
|
240
265
|
|
|
@@ -442,6 +467,7 @@ def assert_is_parent(path, parent_dir):
|
|
|
442
467
|
|
|
443
468
|
|
|
444
469
|
def get_artifacts_matching_glob(context, task_id, pattern):
|
|
470
|
+
"""Return paths of downloaded artifacts matching ``pattern`` under the task's cot/<id> dir."""
|
|
445
471
|
parent_dir = os.path.abspath(os.path.join(context.config["work_dir"], "cot", task_id))
|
|
446
472
|
matching = []
|
|
447
473
|
for root, _, files in os.walk(parent_dir):
|
|
@@ -559,6 +559,7 @@ DEFAULT_CONFIG: immutabledict[str, Any] = immutabledict(
|
|
|
559
559
|
"project:releng:lando:repo:release": "release",
|
|
560
560
|
"project:releng:lando:repo:esr115": "esr115",
|
|
561
561
|
"project:releng:lando:repo:esr140": "esr140",
|
|
562
|
+
"project:releng:lando:repo:esr153": "esr153",
|
|
562
563
|
}
|
|
563
564
|
),
|
|
564
565
|
"glean": immutabledict({"project:mozilla:glean:releng:beetmover:bucket:maven-production": "glean-repo"}),
|
|
@@ -645,6 +646,7 @@ DEFAULT_CONFIG: immutabledict[str, Any] = immutabledict(
|
|
|
645
646
|
"/releases/mozilla-release",
|
|
646
647
|
"/releases/mozilla-esr115",
|
|
647
648
|
"/releases/mozilla-esr140",
|
|
649
|
+
"/releases/mozilla-esr153",
|
|
648
650
|
"/projects/birch",
|
|
649
651
|
# Bug 1821839 hneiva using to test hardened signing
|
|
650
652
|
"/projects/maple",
|
|
@@ -654,6 +656,7 @@ DEFAULT_CONFIG: immutabledict[str, Any] = immutabledict(
|
|
|
654
656
|
"/releases/mozilla-release",
|
|
655
657
|
"/releases/mozilla-esr115",
|
|
656
658
|
"/releases/mozilla-esr140",
|
|
659
|
+
"/releases/mozilla-esr153",
|
|
657
660
|
"/projects/birch",
|
|
658
661
|
"/projects/maple",
|
|
659
662
|
"/integration/autoland",
|
|
@@ -667,9 +670,11 @@ DEFAULT_CONFIG: immutabledict[str, Any] = immutabledict(
|
|
|
667
670
|
),
|
|
668
671
|
"esr115": ("/releases/mozilla-esr115",),
|
|
669
672
|
"esr140": ("/releases/mozilla-esr140",),
|
|
673
|
+
"esr153": ("/releases/mozilla-esr153",),
|
|
670
674
|
"esr": (
|
|
671
675
|
"/releases/mozilla-esr115",
|
|
672
676
|
"/releases/mozilla-esr140",
|
|
677
|
+
"/releases/mozilla-esr153",
|
|
673
678
|
),
|
|
674
679
|
"nightly": ("/mozilla-central",),
|
|
675
680
|
"autoland": ("/integration/autoland",),
|
|
@@ -681,6 +686,7 @@ DEFAULT_CONFIG: immutabledict[str, Any] = immutabledict(
|
|
|
681
686
|
"/releases/mozilla-release",
|
|
682
687
|
"/releases/mozilla-esr115",
|
|
683
688
|
"/releases/mozilla-esr140",
|
|
689
|
+
"/releases/mozilla-esr153",
|
|
684
690
|
# bug 1845368: a permanent project branch for testing updates
|
|
685
691
|
"/projects/pine",
|
|
686
692
|
"/projects/larch",
|
|
@@ -693,6 +699,7 @@ DEFAULT_CONFIG: immutabledict[str, Any] = immutabledict(
|
|
|
693
699
|
"/releases/mozilla-release",
|
|
694
700
|
"/releases/mozilla-esr115",
|
|
695
701
|
"/releases/mozilla-esr140",
|
|
702
|
+
"/releases/mozilla-esr153",
|
|
696
703
|
),
|
|
697
704
|
"all-production-branches-and-autoland": (
|
|
698
705
|
"/mozilla-central",
|
|
@@ -700,6 +707,7 @@ DEFAULT_CONFIG: immutabledict[str, Any] = immutabledict(
|
|
|
700
707
|
"/releases/mozilla-release",
|
|
701
708
|
"/releases/mozilla-esr115",
|
|
702
709
|
"/releases/mozilla-esr140",
|
|
710
|
+
"/releases/mozilla-esr153",
|
|
703
711
|
"/integration/autoland",
|
|
704
712
|
),
|
|
705
713
|
}
|
|
@@ -728,6 +736,7 @@ DEFAULT_CONFIG: immutabledict[str, Any] = immutabledict(
|
|
|
728
736
|
"/releases/comm-release",
|
|
729
737
|
"/releases/comm-esr115",
|
|
730
738
|
"/releases/comm-esr140",
|
|
739
|
+
"/releases/comm-esr153",
|
|
731
740
|
),
|
|
732
741
|
"beta": ("/releases/comm-beta",),
|
|
733
742
|
"release": ("/releases/comm-release",),
|
|
@@ -735,14 +744,16 @@ DEFAULT_CONFIG: immutabledict[str, Any] = immutabledict(
|
|
|
735
744
|
"/releases/comm-release",
|
|
736
745
|
"/releases/comm-esr115",
|
|
737
746
|
"/releases/comm-esr140",
|
|
747
|
+
"/releases/comm-esr153",
|
|
738
748
|
),
|
|
739
|
-
"esr": ("/releases/comm-esr115", "/releases/comm-esr140"),
|
|
749
|
+
"esr": ("/releases/comm-esr115", "/releases/comm-esr140", "/releases/comm-esr153"),
|
|
740
750
|
"all-nightly-branches": (
|
|
741
751
|
"/comm-central",
|
|
742
752
|
"/releases/comm-beta",
|
|
743
753
|
"/releases/comm-release",
|
|
744
754
|
"/releases/comm-esr115",
|
|
745
755
|
"/releases/comm-esr140",
|
|
756
|
+
"/releases/comm-esr153",
|
|
746
757
|
),
|
|
747
758
|
"nightly": ("/comm-central",),
|
|
748
759
|
}
|
|
@@ -59,6 +59,7 @@ class Context(object):
|
|
|
59
59
|
|
|
60
60
|
config: Optional[Dict[str, Any]] = None
|
|
61
61
|
credentials_timestamp: Optional[int] = None
|
|
62
|
+
credentials_file = None
|
|
62
63
|
proc: Optional[task_process.TaskProcess] = None
|
|
63
64
|
queue: Optional[Queue] = None
|
|
64
65
|
session: Optional[aiohttp.ClientSession] = None
|
|
@@ -98,6 +99,7 @@ class Context(object):
|
|
|
98
99
|
if claim_task:
|
|
99
100
|
self.task = claim_task["task"]
|
|
100
101
|
self.verify_task()
|
|
102
|
+
self.credentials_file = tempfile.TemporaryFile()
|
|
101
103
|
self.temp_credentials = claim_task["credentials"]
|
|
102
104
|
path = os.path.join(self.config["work_dir"], "task.json")
|
|
103
105
|
assert self.task
|
|
@@ -105,6 +107,9 @@ class Context(object):
|
|
|
105
107
|
else:
|
|
106
108
|
self.temp_credentials = None
|
|
107
109
|
self.task = None
|
|
110
|
+
if self.credentials_file is not None:
|
|
111
|
+
self.credentials_file.close()
|
|
112
|
+
self.credentials_file = None
|
|
108
113
|
|
|
109
114
|
def verify_task(self) -> None:
|
|
110
115
|
"""Run some task sanity checks on ``self.task``."""
|
|
@@ -193,6 +198,10 @@ class Context(object):
|
|
|
193
198
|
def temp_credentials(self, credentials: Optional[Dict[str, Any]]) -> None:
|
|
194
199
|
self._temp_credentials = credentials
|
|
195
200
|
self.temp_queue = self.create_queue(self.temp_credentials)
|
|
201
|
+
if credentials:
|
|
202
|
+
data = json.dumps(credentials, indent=2, sort_keys=True).encode("ascii")
|
|
203
|
+
# use pwrite so we don't confuse the child by changing the file offset
|
|
204
|
+
assert os.pwrite(self.credentials_file.fileno(), data, 0) == len(data)
|
|
196
205
|
|
|
197
206
|
def write_json(self, path: str, contents: Dict[str, Any], message: str) -> None:
|
|
198
207
|
"""Write json to disk.
|
|
@@ -261,6 +270,7 @@ class Context(object):
|
|
|
261
270
|
|
|
262
271
|
@property
|
|
263
272
|
def download_semaphore(self) -> asyncio.BoundedSemaphore:
|
|
273
|
+
"""Return a lazily-built semaphore limiting concurrent downloads to ``max_concurrent_downloads``."""
|
|
264
274
|
assert self.config
|
|
265
275
|
if self._download_semaphore is None:
|
|
266
276
|
try:
|
|
@@ -41,7 +41,7 @@ from scriptworker.context import Context
|
|
|
41
41
|
from scriptworker.ed25519 import ed25519_public_key_from_string, verify_ed25519_signature
|
|
42
42
|
from scriptworker.exceptions import BaseDownloadError, CoTError, ScriptWorkerEd25519Error
|
|
43
43
|
from scriptworker.github import GitHubRepository, extract_github_repo_full_name, extract_github_repo_owner_and_name, extract_github_repo_ssh_url
|
|
44
|
-
from scriptworker.log import contextual_log_handler
|
|
44
|
+
from scriptworker.log import contextual_log_handler, get_chain_of_trust_log_filename
|
|
45
45
|
from scriptworker.task import (
|
|
46
46
|
get_action_callback_name,
|
|
47
47
|
get_and_check_tasks_for,
|
|
@@ -180,7 +180,8 @@ class ChainOfTrust(object):
|
|
|
180
180
|
|
|
181
181
|
def is_scope_in_restricted_scopes(self, scope, restricted_scopes):
|
|
182
182
|
"""Determine if a scope matches in a list of restricted_scopes.
|
|
183
|
-
|
|
183
|
+
|
|
184
|
+
If one of the restricted scopes ends with '*', find a partial match.
|
|
184
185
|
|
|
185
186
|
Returns:
|
|
186
187
|
String: string of matching restricted_scope, if no match ""
|
|
@@ -728,8 +729,7 @@ async def download_cot_artifact(chain, task_id, path):
|
|
|
728
729
|
link = chain.get_link(task_id)
|
|
729
730
|
log.debug("Verifying {} is in {} cot artifacts...".format(path, task_id))
|
|
730
731
|
if not link.cot:
|
|
731
|
-
log.warning('Chain of Trust for "{}" in {} does not exist. See above log for more details.
|
|
732
|
-
Skipping download of this artifact'.format(path, task_id))
|
|
732
|
+
log.warning(f'Chain of Trust for "{path}" in {task_id} does not exist. See above log for more details. Skipping download of this artifact.')
|
|
733
733
|
return
|
|
734
734
|
|
|
735
735
|
if path not in link.cot["artifacts"]:
|
|
@@ -1094,8 +1094,9 @@ async def _get_additional_hg_push_jsone_context(parent_link, decision_link):
|
|
|
1094
1094
|
allowed_comments.append(push_comment[try_idx:].split("\n")[0])
|
|
1095
1095
|
if decision_comment not in allowed_comments:
|
|
1096
1096
|
raise CoTError(
|
|
1097
|
-
"Decision task {} comment doesn't match the push comment!\n"
|
|
1098
|
-
|
|
1097
|
+
"Decision task {} comment doesn't match the push comment!\nDecision comment: \n{}\nPush comment: \n{}".format(
|
|
1098
|
+
decision_link.name, decision_comment, push_comment
|
|
1099
|
+
)
|
|
1099
1100
|
)
|
|
1100
1101
|
return {
|
|
1101
1102
|
"push": {
|
|
@@ -2041,7 +2042,7 @@ async def verify_chain_of_trust(chain, *, check_task=False):
|
|
|
2041
2042
|
CoTError: on failure
|
|
2042
2043
|
|
|
2043
2044
|
"""
|
|
2044
|
-
log_path =
|
|
2045
|
+
log_path = get_chain_of_trust_log_filename(chain.context)
|
|
2045
2046
|
scriptworker_log = logging.getLogger("scriptworker")
|
|
2046
2047
|
with contextual_log_handler(
|
|
2047
2048
|
chain.context,
|
|
@@ -2113,7 +2114,8 @@ def verify_cot_cmdln(args=None, event_loop=None):
|
|
|
2113
2114
|
|
|
2114
2115
|
"""
|
|
2115
2116
|
args = args or sys.argv[1:]
|
|
2116
|
-
parser = argparse.ArgumentParser(
|
|
2117
|
+
parser = argparse.ArgumentParser(
|
|
2118
|
+
description="""Verify a given task's chain of trust.
|
|
2117
2119
|
|
|
2118
2120
|
Given a task's `task_id`, get its task definition, then trace its chain of
|
|
2119
2121
|
trust back to the tree. This doesn't verify chain of trust artifact signatures,
|
|
@@ -2125,7 +2127,8 @@ To use, first either set your taskcluster creds in your env http://bit.ly/2eDMa6
|
|
|
2125
2127
|
or in the CREDS_FILES http://bit.ly/2fVMu0A
|
|
2126
2128
|
|
|
2127
2129
|
If you are verifying against a private github repo, please also set in environment
|
|
2128
|
-
SCRIPTWORKER_GITHUB_OAUTH_TOKEN to an OAUTH token with read permissions to the repo"""
|
|
2130
|
+
SCRIPTWORKER_GITHUB_OAUTH_TOKEN to an OAUTH token with read permissions to the repo"""
|
|
2131
|
+
)
|
|
2129
2132
|
parser.add_argument("task_id", help="the task id to test")
|
|
2130
2133
|
parser.add_argument("--task-type", help="the task type to test", choices=sorted(get_valid_task_types().keys()), required=True)
|
|
2131
2134
|
parser.add_argument("--cleanup", help="clean up the temp dir afterwards", dest="cleanup", action="store_true", default=False)
|
|
@@ -2192,13 +2195,15 @@ def create_test_workdir(args=None, event_loop=None):
|
|
|
2192
2195
|
|
|
2193
2196
|
"""
|
|
2194
2197
|
args = args or sys.argv[1:]
|
|
2195
|
-
parser = argparse.ArgumentParser(
|
|
2198
|
+
parser = argparse.ArgumentParser(
|
|
2199
|
+
description="""Populate a test `work_dir`.
|
|
2196
2200
|
|
|
2197
2201
|
Given a scriptworker task's `task_id`, get its task definition, write it to
|
|
2198
2202
|
`./work/task.json`, then download its `upstreamArtifacts` and put them in
|
|
2199
2203
|
`./work/cot/TASK_ID/PATH`.
|
|
2200
2204
|
|
|
2201
|
-
This is helpful in manually testing a *script run."""
|
|
2205
|
+
This is helpful in manually testing a *script run."""
|
|
2206
|
+
)
|
|
2202
2207
|
parser.add_argument("--path", help="relative path to the work_dir", default="work")
|
|
2203
2208
|
parser.add_argument("--overwrite", help="overwrite an existing work_dir", action="store_true")
|
|
2204
2209
|
parser.add_argument("task_id", help="the task id to test")
|
|
@@ -141,11 +141,13 @@ def verify_ed25519_signature_cmdln(args=None, exception=SystemExit):
|
|
|
141
141
|
|
|
142
142
|
"""
|
|
143
143
|
args = args or sys.argv[1:]
|
|
144
|
-
parser = argparse.ArgumentParser(
|
|
144
|
+
parser = argparse.ArgumentParser(
|
|
145
|
+
description="""Verify an ed25519 signature from the command line.
|
|
145
146
|
|
|
146
147
|
Given a file and its detached signature, verify that it has been signed with
|
|
147
148
|
a valid key. This key can be specified on the command line; otherwise we'll
|
|
148
|
-
default to ``config['ed25519_public_keys']``."""
|
|
149
|
+
default to ``config['ed25519_public_keys']``."""
|
|
150
|
+
)
|
|
149
151
|
parser.add_argument("--pubkey", help="path to a base64-encoded ed25519 pubkey, optional")
|
|
150
152
|
parser.add_argument("file_path")
|
|
151
153
|
parser.add_argument("sig_path")
|
|
@@ -121,7 +121,7 @@ class GitHubRepository:
|
|
|
121
121
|
if any(vcs_rule.get("require_secret") for vcs_rule in context.config["trusted_vcs_rules"]):
|
|
122
122
|
# This check uses unofficial API on github, which we can't easily
|
|
123
123
|
# check for private repos, assume its true in the private case.
|
|
124
|
-
log.info("has_commit_landed_on_repository() not implemented for private
|
|
124
|
+
log.info("has_commit_landed_on_repository() not implemented for private repositories, assume True")
|
|
125
125
|
return True
|
|
126
126
|
|
|
127
127
|
# Revision may be a tag name. `branch_commits` doesn't work on tags
|
|
@@ -293,7 +293,7 @@ def is_github_repo_owner_the_official_one(context, repo_owner):
|
|
|
293
293
|
official_repo_owner = context.config["official_github_repos_owner"]
|
|
294
294
|
if not official_repo_owner:
|
|
295
295
|
raise ConfigError(
|
|
296
|
-
|
|
296
|
+
'This worker does not have a defined owner for official GitHub repositories. Given "official_github_repos_owner": {}'.format(official_repo_owner)
|
|
297
297
|
)
|
|
298
298
|
|
|
299
299
|
return official_repo_owner == repo_owner
|
|
@@ -115,6 +115,19 @@ def get_log_filename(context: Any) -> str:
|
|
|
115
115
|
return os.path.join(context.config["task_log_dir"], "live_backing.log")
|
|
116
116
|
|
|
117
117
|
|
|
118
|
+
def get_chain_of_trust_log_filename(context: Any) -> str:
|
|
119
|
+
"""Get the chain of trust verification log file path.
|
|
120
|
+
|
|
121
|
+
Args:
|
|
122
|
+
context (scriptworker.context.Context): the scriptworker context.
|
|
123
|
+
|
|
124
|
+
Returns:
|
|
125
|
+
string: chain of trust log file path
|
|
126
|
+
|
|
127
|
+
"""
|
|
128
|
+
return os.path.join(context.config["task_log_dir"], "chain_of_trust.log")
|
|
129
|
+
|
|
130
|
+
|
|
118
131
|
@contextmanager
|
|
119
132
|
def get_log_filehandle(context: Any) -> Iterator[IO[str]]:
|
|
120
133
|
"""Open the log and error filehandles.
|
|
@@ -117,7 +117,7 @@ def get_run_id(claim_task):
|
|
|
117
117
|
|
|
118
118
|
# get_task_maxruntime {{{1
|
|
119
119
|
def get_task_maxruntime(task_def, max_timeout):
|
|
120
|
-
"""Given a task definition, return the lower of max_timeout and maxRunTime if set"""
|
|
120
|
+
"""Given a task definition, return the lower of max_timeout and maxRunTime if set."""
|
|
121
121
|
max_run_time = task_def["payload"].get("maxRunTime")
|
|
122
122
|
if max_run_time is None:
|
|
123
123
|
return max_timeout
|
|
@@ -672,7 +672,17 @@ async def run_task(context, to_cancellable_process):
|
|
|
672
672
|
env["TASK_ID"] = context.task_id or "None"
|
|
673
673
|
env["RUN_ID"] = str(get_run_id(context.claim_task))
|
|
674
674
|
env["TASKCLUSTER_ROOT_URL"] = context.config["taskcluster_root_url"]
|
|
675
|
-
|
|
675
|
+
credentials_fd = context.credentials_file.fileno()
|
|
676
|
+
env["TASKCLUSTER_CREDENTIALS_FD"] = str(credentials_fd)
|
|
677
|
+
kwargs = {
|
|
678
|
+
"stdout": PIPE,
|
|
679
|
+
"stderr": PIPE,
|
|
680
|
+
"stdin": None,
|
|
681
|
+
"close_fds": True,
|
|
682
|
+
"preexec_fn": lambda: os.setsid(),
|
|
683
|
+
"env": env,
|
|
684
|
+
"pass_fds": (credentials_fd,),
|
|
685
|
+
} # pragma: no branch
|
|
676
686
|
|
|
677
687
|
timeout = get_task_maxruntime(context.task, context.config["task_max_timeout"])
|
|
678
688
|
subprocess = await asyncio.create_subprocess_exec(*context.config["task_script"], **kwargs)
|
|
@@ -42,6 +42,7 @@ log = logging.getLogger(__name__)
|
|
|
42
42
|
|
|
43
43
|
# scriptworker_session {{{1
|
|
44
44
|
def scriptworker_session(*args, **kwargs):
|
|
45
|
+
"""Return an ``aiohttp.ClientSession`` with the scriptworker User-Agent header set."""
|
|
45
46
|
kwargs.setdefault("headers", {}).setdefault("User-Agent", f"scriptworker {scriptworker.__version__}")
|
|
46
47
|
return aiohttp.ClientSession(*args, **kwargs)
|
|
47
48
|
|
|
@@ -875,8 +876,9 @@ def remove_empty_keys(values, remove=({}, None, [], "null")):
|
|
|
875
876
|
|
|
876
877
|
# add_projectid {{{1
|
|
877
878
|
def add_projectid(task_def):
|
|
878
|
-
"""Add a projectId property to a task, if none already exists
|
|
879
|
-
|
|
879
|
+
"""Add a projectId property to a task, if none already exists.
|
|
880
|
+
|
|
881
|
+
Uses the Taskcluster default value of 'none'.
|
|
880
882
|
|
|
881
883
|
Args:
|
|
882
884
|
task_def (dict): the task definition
|
|
@@ -889,8 +891,9 @@ def add_projectid(task_def):
|
|
|
889
891
|
|
|
890
892
|
# add_taskqueueid {{{1
|
|
891
893
|
def add_taskqueueid(task_def):
|
|
892
|
-
"""Add a taskQueueId property to a task, if none already exists
|
|
893
|
-
|
|
894
|
+
"""Add a taskQueueId property to a task, if none already exists.
|
|
895
|
+
|
|
896
|
+
Builds the value from the provisionerId and workerType, then removes those two properties.
|
|
894
897
|
|
|
895
898
|
Args:
|
|
896
899
|
task_def (dict): the task definition
|
|
@@ -8,6 +8,7 @@ Attributes:
|
|
|
8
8
|
|
|
9
9
|
import asyncio
|
|
10
10
|
import logging
|
|
11
|
+
import os
|
|
11
12
|
import signal
|
|
12
13
|
import socket
|
|
13
14
|
import sys
|
|
@@ -17,12 +18,13 @@ from typing import Any
|
|
|
17
18
|
import aiohttp
|
|
18
19
|
import arrow
|
|
19
20
|
|
|
20
|
-
from scriptworker.artifacts import upload_artifacts
|
|
21
|
+
from scriptworker.artifacts import create_link_artifact, upload_artifacts
|
|
21
22
|
from scriptworker.config import get_context_from_cmdln
|
|
22
23
|
from scriptworker.constants import STATUSES
|
|
23
24
|
from scriptworker.cot.generate import generate_cot
|
|
24
25
|
from scriptworker.cot.verify import ChainOfTrust, verify_chain_of_trust
|
|
25
26
|
from scriptworker.exceptions import ScriptWorkerException, WorkerShutdownDuringTask
|
|
27
|
+
from scriptworker.log import get_chain_of_trust_log_filename
|
|
26
28
|
from scriptworker.task import claim_work, complete_task, prepare_to_run_task, reclaim_task, run_task, worst_level
|
|
27
29
|
from scriptworker.task_process import TaskProcess
|
|
28
30
|
from scriptworker.utils import cleanup, filepaths_in_dir, scriptworker_session
|
|
@@ -53,7 +55,21 @@ async def do_run_task(context, run_cancellable, to_cancellable_process):
|
|
|
53
55
|
try:
|
|
54
56
|
if context.config["verify_chain_of_trust"]:
|
|
55
57
|
chain = ChainOfTrust(context, context.config["cot_job_type"])
|
|
56
|
-
|
|
58
|
+
try:
|
|
59
|
+
await run_cancellable(verify_chain_of_trust(chain))
|
|
60
|
+
except Exception:
|
|
61
|
+
# Point live_backing.log at chain_of_trust.log so Treeherder parses the CoT failure
|
|
62
|
+
if os.path.exists(get_chain_of_trust_log_filename(context)):
|
|
63
|
+
try:
|
|
64
|
+
await create_link_artifact(
|
|
65
|
+
context,
|
|
66
|
+
target_path="public/logs/live_backing.log",
|
|
67
|
+
link_to="public/logs/chain_of_trust.log",
|
|
68
|
+
content_type="text/plain",
|
|
69
|
+
)
|
|
70
|
+
except Exception as link_exc:
|
|
71
|
+
log.exception("Failed to create live_backing.log link artifact: {}".format(link_exc))
|
|
72
|
+
raise
|
|
57
73
|
status = await run_task(context, to_cancellable_process)
|
|
58
74
|
generate_cot(context)
|
|
59
75
|
except asyncio.CancelledError:
|
|
@@ -104,7 +120,7 @@ class RunTasks:
|
|
|
104
120
|
"""Manages processing of Taskcluster tasks."""
|
|
105
121
|
|
|
106
122
|
def __init__(self):
|
|
107
|
-
"""
|
|
123
|
+
"""Initialize the task-processing state."""
|
|
108
124
|
self.future = None
|
|
109
125
|
self.task_process = None
|
|
110
126
|
self.is_cancelled = False
|
|
@@ -54,26 +54,16 @@ commands=
|
|
|
54
54
|
|
|
55
55
|
[testenv:check]
|
|
56
56
|
skip_install = true
|
|
57
|
+
deps = ruff
|
|
57
58
|
commands =
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
flake8 {toxinidir}
|
|
59
|
+
ruff check {toxinidir}
|
|
60
|
+
ruff format --check {toxinidir}
|
|
61
61
|
|
|
62
62
|
[testenv:mypy]
|
|
63
63
|
commands =
|
|
64
64
|
mypy --config {toxinidir}/mypi.ini {toxinidir}/src
|
|
65
65
|
|
|
66
66
|
|
|
67
|
-
[flake8]
|
|
68
|
-
max-line-length = 160
|
|
69
|
-
extend-ignore = E203
|
|
70
|
-
# test_github.py ignored because of https://gitlab.com/pycqa/flake8/issues/375
|
|
71
|
-
exclude = .ropeproject,.tox,sandbox,docs,.eggs,*.egg,*.egg-info,setup.py,build/,tests/test_github.py,.venv
|
|
72
|
-
per-file-ignores =
|
|
73
|
-
# Ignore documentation issues in tests
|
|
74
|
-
tests:*:D
|
|
75
|
-
show-source = True
|
|
76
|
-
|
|
77
67
|
[pytest]
|
|
78
68
|
norecursedirs = .tox .git .hg sandbox .eggs build
|
|
79
69
|
python_files = test_*.py
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{scriptworker-63.0.1 → scriptworker-63.2.0}/src/scriptworker/data/scriptworker_task_schema.json
RENAMED
|
File without changes
|
|
File without changes
|