pubrules 0.48.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- pubrules/__init__.py +14 -0
- pubrules/audit.py +116 -0
- pubrules/audit_draft.py +295 -0
- pubrules/audit_prepare.py +1128 -0
- pubrules/cache.py +319 -0
- pubrules/cbor.py +45 -0
- pubrules/ci.py +910 -0
- pubrules/cli.py +1069 -0
- pubrules/conform.py +295 -0
- pubrules/doc_coverage.py +235 -0
- pubrules/freshness.py +75 -0
- pubrules/gherkin.py +348 -0
- pubrules/hooks.py +59 -0
- pubrules/hosting.py +372 -0
- pubrules/ignores.py +86 -0
- pubrules/new.py +242 -0
- pubrules/nightly.py +1002 -0
- pubrules/paths.py +32 -0
- pubrules/protect.py +596 -0
- pubrules/providers.py +129 -0
- pubrules/refs.py +213 -0
- pubrules/release-commit +1 -0
- pubrules/releases.py +107 -0
- pubrules/repos.py +67 -0
- pubrules/ruleset.json +43 -0
- pubrules/secrets_config.py +165 -0
- pubrules/security.py +134 -0
- pubrules/service_needs.py +467 -0
- pubrules/standard.json +635 -0
- pubrules/standard.py +266 -0
- pubrules/supply_chain.py +188 -0
- pubrules/tag-ruleset.json +14 -0
- pubrules/templates/common/AGENTS.md +32 -0
- pubrules/templates/common/pubrules +26 -0
- pubrules/templates/common/tests/features/repository.feature +11 -0
- pubrules/templates/common/tests/install-lock +65 -0
- pubrules/templates/common/tests/pytest.ini +4 -0
- pubrules/templates/common/tests/requirements-pubrules.txt +17 -0
- pubrules/templates/common/tests/requirements.txt +3 -0
- pubrules/templates/common/tests/steps/test_repository.py +14 -0
- pubrules/templates/deploy/pubdeploy +28 -0
- pubrules/templates/new-rust/CHANGELOG.md +3 -0
- pubrules/templates/new-rust/Cargo.toml +46 -0
- pubrules/templates/new-rust/LICENSE +202 -0
- pubrules/templates/new-rust/Makefile +18 -0
- pubrules/templates/new-rust/README.md +34 -0
- pubrules/templates/new-rust/SECURITY.md +26 -0
- pubrules/templates/new-rust/corpus/corpus.json +11 -0
- pubrules/templates/new-rust/deny.toml +32 -0
- pubrules/templates/new-rust/fuzz/Cargo.toml +23 -0
- pubrules/templates/new-rust/fuzz/fuzz_targets/greet.rs +8 -0
- pubrules/templates/new-rust/fuzz/gitignore +4 -0
- pubrules/templates/new-rust/fuzz/regressions/README.md +4 -0
- pubrules/templates/new-rust/gitignore +10 -0
- pubrules/templates/new-rust/src/lib.rs +91 -0
- pubrules/templates/new-rust/src/main.rs +19 -0
- pubrules/templates/new-rust/supply-chain/audits.toml +4 -0
- pubrules/templates/new-rust/supply-chain/config.toml +23 -0
- pubrules/templates/new-rust/supply-chain/imports.lock +2 -0
- pubrules/templates/new-rust/tests/conftest.py +9 -0
- pubrules/templates/new-rust/tests/features/cli.feature +14 -0
- pubrules/templates/new-rust/tests/steps/test_cli.py +15 -0
- pubrules/templates/new-rust/tests/vectors.rs +30 -0
- pubrules/templates/new-rust/tools/vectors/generate.py +24 -0
- pubrules/templates/python/Makefile +5 -0
- pubrules/templates/python/pubrules-python.mk +47 -0
- pubrules/templates/rust/Makefile +9 -0
- pubrules/templates/rust/pubrules.mk +138 -0
- pubrules/templates/rust/rust-toolchain.toml +8 -0
- pubrules/templates/service/pubsec +27 -0
- pubrules/templates/service/with-secrets +20 -0
- pubrules/templates/service/with-secrets.json +8 -0
- pubrules/variables.py +45 -0
- pubrules-0.48.0.dist-info/METADATA +1241 -0
- pubrules-0.48.0.dist-info/RECORD +79 -0
- pubrules-0.48.0.dist-info/WHEEL +5 -0
- pubrules-0.48.0.dist-info/entry_points.txt +2 -0
- pubrules-0.48.0.dist-info/licenses/LICENSE +202 -0
- pubrules-0.48.0.dist-info/top_level.txt +1 -0
pubrules/ci.py
ADDED
|
@@ -0,0 +1,910 @@
|
|
|
1
|
+
"""The CI `pubrules sync` writes from the `ci` block of pubrules.json.
|
|
2
|
+
|
|
3
|
+
A reusable workflow that serves every repository has to skip the jobs a
|
|
4
|
+
repository does not use, and GitHub shows each as skipped. Here pubrules
|
|
5
|
+
writes the repository's workflows instead, one per event, each calling
|
|
6
|
+
only the jobs that event runs: one reusable workflow per job
|
|
7
|
+
(job-*.yml), pinned to the commit of this pubrules' release tag, with the
|
|
8
|
+
version in a comment (refs.py).
|
|
9
|
+
|
|
10
|
+
"ci": {
|
|
11
|
+
"language": "node",
|
|
12
|
+
"events": ["pr", "queue", "main", "nightly"],
|
|
13
|
+
"cron": "17 3 * * *",
|
|
14
|
+
"with": {"browser-test": "npm run test:browser", "coverage-report": "coverage/"},
|
|
15
|
+
"jobs": {"review": false, "sonar": {"events": ["pr", "main"]}},
|
|
16
|
+
"release": {"pubrel": "0.5.4", "with": {}},
|
|
17
|
+
"tools": [{"crate": "some-cli", "git": "https://github.com/example-org/some-cli",
|
|
18
|
+
"tag": "v1.2.3"}]
|
|
19
|
+
}
|
|
20
|
+
|
|
21
|
+
`language` is a utility's, node or python; without it, a package.json
|
|
22
|
+
with no pyproject.toml beside it makes it node. A rust, service or content
|
|
23
|
+
repository's CI is its kind's. `events` are the workflows written: pr.yml
|
|
24
|
+
(pull requests), queue.yml (the merge queue's runs of them, `merge_group`,
|
|
25
|
+
so the checks a ruleset requires report on the queue's merge too),
|
|
26
|
+
main.yml (pushes to main) and nightly.yml (the schedule `cron`, and by
|
|
27
|
+
hand); pr, queue and main by default, and all four for Rust. The queue
|
|
28
|
+
runs every job a pull request does, and the heavy ones (Rust's msrv, fuzz
|
|
29
|
+
and cross) besides, so a pull request gets its answer sooner and nothing
|
|
30
|
+
merges untested. Where the queue would run nothing a pull request does
|
|
31
|
+
not, it has no workflow of its own: pr.yml runs on `merge_group` too. A
|
|
32
|
+
repository without a merge queue (a user's, or a private one) leaves
|
|
33
|
+
`queue` out, and its pull requests run the heavy jobs. `with` gives the
|
|
34
|
+
jobs' inputs: each goes to every job that takes it, and an input no job
|
|
35
|
+
takes stops sync. A job runs on the events JOBS gives it unless `jobs`
|
|
36
|
+
narrows them, and `false` drops it, as a private repository drops review.
|
|
37
|
+
The browser job runs only when `with` gives a browser-test, and a
|
|
38
|
+
service's terraform and tests jobs only when it gives a terraform-dir and
|
|
39
|
+
a test. Rust's jobs are the ones the Rust crate standard asks for here,
|
|
40
|
+
from the rules and pubrules.json's exceptions, and sync writes each job's
|
|
41
|
+
required make targets into its workflow, so a change to either shows as a
|
|
42
|
+
change there. `release` adds pubrel's release workflow to pr.yml,
|
|
43
|
+
queue.yml and main.yml, at the pubrel it names (where it names none, the one
|
|
44
|
+
the workflows already pin, or else pubrel's latest release; the nightly
|
|
45
|
+
moves it), passing it the
|
|
46
|
+
release app's RELEASE_APP_ID and RELEASE_APP_KEY from pubrel 0.6.1, the
|
|
47
|
+
first to take them, so `pubrel tag` tags with the app the tag ruleset lets
|
|
48
|
+
through; `false` leaves it
|
|
49
|
+
out, and a service, which deploys, has none, nor has a content repository,
|
|
50
|
+
which pubrules neither releases nor deploys. A content repository's CI is
|
|
51
|
+
its checks (the managed files in step, and shellcheck on its **/*.sh
|
|
52
|
+
unless `with` gives shellcheck) and, when `with` gives a test, the tests
|
|
53
|
+
job, which runs the checks its own tooling makes; it has no scan. `{"uses": "local"}` calls the
|
|
54
|
+
repository's own .github/workflows/release.yml instead, as pubrel does, so
|
|
55
|
+
a change to its release workflow is tested before it ships.
|
|
56
|
+
|
|
57
|
+
`tools` lists what a Rust repository's functional suite drives: each crate
|
|
58
|
+
is installed from its git repository at its tag, by `cargo install --locked`,
|
|
59
|
+
and the nightly moves each tag to the latest release of that repository.
|
|
60
|
+
|
|
61
|
+
`runs-on` names the runners: a label, such as a self-hosted runner scale
|
|
62
|
+
set's, for every job, or an object of labels by job name with "default" for
|
|
63
|
+
the rest, such as {"default": "my-runners", "review": "ubuntu-latest"}. It
|
|
64
|
+
goes to each job pubrules calls as its `runs-on` input, and into the jobs
|
|
65
|
+
pubrules writes itself (tested, passed, site). Rust's test job runs on its
|
|
66
|
+
os-matrix instead. pubrel's release workflow takes it from pubrel 0.7.0,
|
|
67
|
+
which installs Rust on a runner without it; an older pubrel's runs on
|
|
68
|
+
GitHub's runners. Without it nothing is written, and every job runs on
|
|
69
|
+
ubuntu-latest.
|
|
70
|
+
|
|
71
|
+
A repository that deploys a static site with a deploy-static action (a
|
|
72
|
+
workflow of its own uses it, or pubrules.json has a `site` block) gets a `site` job in pr.yml (and queue.yml): it builds the site
|
|
73
|
+
(`site.pr_build`, else `site.build`, else nothing) and runs the
|
|
74
|
+
deploy-static its deploy workflow pins, with check-only, on
|
|
75
|
+
`site.check_dir` (else the `dir` that workflow gives it). A site that would
|
|
76
|
+
fail to deploy fails the pull request.
|
|
77
|
+
|
|
78
|
+
"site": {"check_dir": "public", "pr_build": "npm run build:sample",
|
|
79
|
+
"url": "https://example.eu/"}
|
|
80
|
+
|
|
81
|
+
Each workflow ends in `passed` (the check a ruleset requires, `ci /
|
|
82
|
+
passed`), which passes only when every job it needs succeeded.
|
|
83
|
+
|
|
84
|
+
The merge queue runs queue.yml (or pr.yml) on the pull request merged
|
|
85
|
+
into main, and merging moves main to that very commit, so main.yml would
|
|
86
|
+
test it again. It starts instead with `tested` (job-tested.yml), which
|
|
87
|
+
looks for a successful merge_group run of that workflow on the commit:
|
|
88
|
+
found, main skips the jobs the queue ran, and runs the scan (with the queue run's coverage,
|
|
89
|
+
since SonarQube Cloud measures main from a push to it) and the release;
|
|
90
|
+
not found, as without a queue, every job runs as before. The merge queue
|
|
91
|
+
never scans: SonarQube Cloud analyses pull requests and main and refuses
|
|
92
|
+
the queue's gh-readonly-queue branch, the pull request's run scanned the
|
|
93
|
+
change, and main's push scans the merge. A Rust queue.yml has no sonar
|
|
94
|
+
job, and in pr.yml, which the queue runs too, it does not run on
|
|
95
|
+
merge_group, which `passed` takes. Every job
|
|
96
|
+
is named `ci`, so its checks read `ci / test`, `ci / sonar` and so on.
|
|
97
|
+
"""
|
|
98
|
+
|
|
99
|
+
from __future__ import annotations
|
|
100
|
+
|
|
101
|
+
import json
|
|
102
|
+
import re
|
|
103
|
+
import sys
|
|
104
|
+
from dataclasses import dataclass, field
|
|
105
|
+
from pathlib import Path
|
|
106
|
+
from typing import Any
|
|
107
|
+
|
|
108
|
+
from . import __version__
|
|
109
|
+
from .refs import owner, ref, release_commit
|
|
110
|
+
|
|
111
|
+
EVENTS = ("pr", "queue", "main", "nightly")
|
|
112
|
+
DEFAULT_EVENTS = ("pr", "queue", "main")
|
|
113
|
+
# The jobs too slow to hold up every push to a pull request: they run in
|
|
114
|
+
# the merge queue, before anything merges, and on main and the nightly.
|
|
115
|
+
HEAVY = ("queue", "main", "nightly")
|
|
116
|
+
CRON = "17 3 * * *"
|
|
117
|
+
TRIGGERS = {
|
|
118
|
+
"pr": " pull_request:\n",
|
|
119
|
+
"queue": " merge_group:\n",
|
|
120
|
+
"main": " push: { branches: [main] }\n",
|
|
121
|
+
"nightly": " schedule: [{ cron: 'CRON' }]\n workflow_dispatch:\n",
|
|
122
|
+
}
|
|
123
|
+
# The events the release workflow runs on: a pull request records and
|
|
124
|
+
# checks its release; a push to main tags and releases it. The merge
|
|
125
|
+
# queue's run of a pull request (merge_group) only reports the check:
|
|
126
|
+
# pubrel tags from a push to main alone.
|
|
127
|
+
RELEASE_EVENTS = ("pr", "queue", "main")
|
|
128
|
+
# The events the site is checked on, before anything deploys it.
|
|
129
|
+
SITE_EVENTS = ("pr", "queue")
|
|
130
|
+
# Jobs the merge queue never runs: SonarQube Cloud analyses pull requests
|
|
131
|
+
# and main, and refuses the queue's gh-readonly-queue branch. The pull
|
|
132
|
+
# request's run scanned the change, and main's push scans the merge.
|
|
133
|
+
NOT_QUEUED = ("sonar",)
|
|
134
|
+
SETTINGS = ("language", "events", "cron", "with", "jobs", "release", "tools", "runs-on")
|
|
135
|
+
HEADER = ("# Written by `pubrules sync` from the ci block of pubrules.json. Do not edit:\n"
|
|
136
|
+
"# change pubrules.json and run `pubrules sync`.\n")
|
|
137
|
+
|
|
138
|
+
|
|
139
|
+
@dataclass(frozen=True)
|
|
140
|
+
class Job:
|
|
141
|
+
"""One job: the reusable workflow it calls, the inputs it takes, and
|
|
142
|
+
the jobs whose results it waits for, where they run."""
|
|
143
|
+
workflow: str
|
|
144
|
+
inputs: tuple[str, ...] = ()
|
|
145
|
+
needs: tuple[str, ...] = ()
|
|
146
|
+
events: tuple[str, ...] = EVENTS
|
|
147
|
+
when: str | None = None # the input it runs only when given
|
|
148
|
+
secrets: bool = False
|
|
149
|
+
fixed: dict[str, str] = field(default_factory=dict) # inputs pubrules gives
|
|
150
|
+
# Inputs that are pubrules' to give, whether or not this language gives
|
|
151
|
+
# them, never the repository's.
|
|
152
|
+
internal: tuple[str, ...] = ()
|
|
153
|
+
# Inputs this kind gives unless the ci block's `with` does: the
|
|
154
|
+
# workflow's own default suits another kind.
|
|
155
|
+
defaults: dict[str, str] = field(default_factory=dict)
|
|
156
|
+
# Rust: the job of the Rust crate standard this one runs. Its required
|
|
157
|
+
# targets, if it has any, go in the input `required`; with none, or a
|
|
158
|
+
# workflow job the standard does not turn on, the job is left out.
|
|
159
|
+
standard: str | None = None
|
|
160
|
+
# Its runners come from an input of its own (os-matrix), so `runs-on`
|
|
161
|
+
# does not reach it.
|
|
162
|
+
matrix: bool = False
|
|
163
|
+
|
|
164
|
+
|
|
165
|
+
NODE_TEST = ("node-version", "python-version", "apt-packages", "setup", "test", "sync-check",
|
|
166
|
+
"coverage-report", "fuzz-runs", "fuzz-runs-scheduled")
|
|
167
|
+
PYTHON_TEST = ("python-version", "apt-packages", "setup", "test", "sync-check",
|
|
168
|
+
"coverage-report", "fuzz-runs", "fuzz-runs-scheduled")
|
|
169
|
+
SONAR_INTERNAL = ("reports", "layout", "coverage-run")
|
|
170
|
+
RUST_REQUIRED = ("required",)
|
|
171
|
+
JOBS: dict[str, dict[str, Job]] = {
|
|
172
|
+
"node": {
|
|
173
|
+
"test": Job("job-node-test.yml", NODE_TEST),
|
|
174
|
+
"browser": Job("job-node-browser.yml",
|
|
175
|
+
("node-version", "apt-packages", "setup", "browser-setup", "browser-test",
|
|
176
|
+
"browser-coverage-report"), when="browser-test"),
|
|
177
|
+
"audit": Job("job-node-audit.yml", ("node-version",)),
|
|
178
|
+
"osv": Job("job-osv.yml"),
|
|
179
|
+
"review": Job("job-review.yml", events=("pr",)),
|
|
180
|
+
"sonar": Job("job-sonar.yml", ("sonar-args",), needs=("test", "browser"), secrets=True,
|
|
181
|
+
fixed={"reports": "sonar.javascript.lcov.reportPaths=*"},
|
|
182
|
+
internal=SONAR_INTERNAL),
|
|
183
|
+
},
|
|
184
|
+
"python": {
|
|
185
|
+
"functional": Job("job-python-functional.yml", PYTHON_TEST),
|
|
186
|
+
"browser": Job("job-python-browser.yml",
|
|
187
|
+
("python-version", "apt-packages", "setup", "browser-setup", "browser-test",
|
|
188
|
+
"browser-coverage-report"), when="browser-test"),
|
|
189
|
+
"audit": Job("job-python-audit.yml", ("python-version", "audit-requirements")),
|
|
190
|
+
"osv": Job("job-osv.yml"),
|
|
191
|
+
"review": Job("job-review.yml", events=("pr",)),
|
|
192
|
+
"sonar": Job("job-sonar.yml", ("sonar-args",), needs=("functional", "browser"), secrets=True,
|
|
193
|
+
fixed={"reports": "sonar.python.coverage.reportPaths=*.xml"},
|
|
194
|
+
internal=SONAR_INTERNAL),
|
|
195
|
+
},
|
|
196
|
+
"rust": {
|
|
197
|
+
"gates": Job("job-rust-gates.yml", ("python-version", "gates"), internal=RUST_REQUIRED,
|
|
198
|
+
standard="gates"),
|
|
199
|
+
"test": Job("job-rust-test.yml", ("os-matrix", "test-args"), needs=("gates",),
|
|
200
|
+
standard="test", matrix=True),
|
|
201
|
+
"functional": Job("job-rust-functional.yml",
|
|
202
|
+
("python-version", "apt-packages", "functional",
|
|
203
|
+
"cargo-llvm-cov-version"), needs=("gates",), secrets=True,
|
|
204
|
+
internal=(*RUST_REQUIRED, "tools"), standard="functional"),
|
|
205
|
+
"msrv": Job("job-rust-msrv.yml", needs=("gates",), events=HEAVY, standard="msrv"),
|
|
206
|
+
"fuzz": Job("job-rust-fuzz.yml",
|
|
207
|
+
("python-version", "fuzz-seconds", "fuzz-seconds-scheduled", "cargo-fuzz-version"),
|
|
208
|
+
needs=("gates",), events=HEAVY, standard="fuzz"),
|
|
209
|
+
"cross": Job("job-rust-cross.yml", ("python-version", "cross-targets", "cross-test-args"),
|
|
210
|
+
needs=("gates",), events=HEAVY, standard="cross"),
|
|
211
|
+
"audit": Job("job-rust-audit.yml", ("cargo-deny-version", "cargo-vet-version"),
|
|
212
|
+
internal=RUST_REQUIRED, standard="audit"),
|
|
213
|
+
"osv": Job("job-osv.yml"),
|
|
214
|
+
"review": Job("job-review.yml", events=("pr",)),
|
|
215
|
+
"toolchains": Job("job-rust-toolchains.yml", ("scheduled-toolchains", "test-args"),
|
|
216
|
+
needs=("gates",), events=("nightly",)),
|
|
217
|
+
"sonar": Job("job-sonar.yml", ("sonar-args",), needs=("functional",), secrets=True,
|
|
218
|
+
fixed={"reports": "sonar.rust.lcov.reportPaths=lcov.info "
|
|
219
|
+
"sonar.python.coverage.reportPaths=*.xml"},
|
|
220
|
+
internal=SONAR_INTERNAL),
|
|
221
|
+
},
|
|
222
|
+
# A service deploys rather than releases: no release. Public code is
|
|
223
|
+
# scanned, a service's too; a private service, which SonarQube Cloud
|
|
224
|
+
# does not analyse for free, drops the scan with "jobs": {"sonar": false}.
|
|
225
|
+
"service": {
|
|
226
|
+
"checks": Job("job-service-checks.yml", ("python-version", "shellcheck", "sync-check")),
|
|
227
|
+
"terraform": Job("job-service-terraform.yml", ("terraform-dir",), when="terraform-dir"),
|
|
228
|
+
"tests": Job("job-service-tests.yml", ("python-version", "apt-packages", "setup", "test", "coverage-report"),
|
|
229
|
+
when="test"),
|
|
230
|
+
"sonar": Job("job-sonar.yml", ("sonar-args",), needs=("tests",), secrets=True,
|
|
231
|
+
fixed={"reports": "sonar.python.coverage.reportPaths=*.xml "
|
|
232
|
+
"sonar.javascript.lcov.reportPaths=lcov.info"},
|
|
233
|
+
internal=SONAR_INTERNAL),
|
|
234
|
+
},
|
|
235
|
+
# Authored content, private, that pubrules neither releases nor deploys:
|
|
236
|
+
# the managed files in step and its shell scripts checked, and its own
|
|
237
|
+
# tooling's checks as the tests job, when it names a test. No scan: there
|
|
238
|
+
# is no code for SonarQube Cloud to analyse, and it would not analyse a
|
|
239
|
+
# private repository for free.
|
|
240
|
+
"content": {
|
|
241
|
+
"checks": Job("job-service-checks.yml", ("python-version", "shellcheck", "sync-check"),
|
|
242
|
+
defaults={"shellcheck": "**/*.sh"}),
|
|
243
|
+
"tests": Job("job-service-tests.yml", ("python-version", "apt-packages", "setup", "test", "coverage-report"),
|
|
244
|
+
when="test"),
|
|
245
|
+
},
|
|
246
|
+
}
|
|
247
|
+
# Kinds that never release: a service deploys, and content is versioned, if
|
|
248
|
+
# at all, by its own tooling.
|
|
249
|
+
UNRELEASED = {"service": "a service, which deploys rather than releases",
|
|
250
|
+
"content": "a content repository, which pubrules neither releases nor deploys"}
|
|
251
|
+
# The languages a utility can be; a rust, service or content repository is its kind.
|
|
252
|
+
UTILITY = ("node", "python")
|
|
253
|
+
# Where SonarQube Cloud finds the code and its tests, by language: the
|
|
254
|
+
# directories that hold code, in the order tried, and those that hold
|
|
255
|
+
# tests. Without them it reads the whole repository as code, and warns that
|
|
256
|
+
# some of it looks like tests. Each is given only when the repository has
|
|
257
|
+
# it, since the scan fails on a directory that is not there.
|
|
258
|
+
LAYOUT = {"python": (("src",), ("tests",)),
|
|
259
|
+
"rust": (("src", "crates"), ("tests", "fuzz", "benches"))}
|
|
260
|
+
|
|
261
|
+
# What a tool in the ci block's `tools` is: what cargo installs, and from where.
|
|
262
|
+
TOOL = ("crate", "git", "tag")
|
|
263
|
+
CRATE = re.compile(r"[A-Za-z0-9][A-Za-z0-9_-]*")
|
|
264
|
+
TOOL_GIT = re.compile(r"https://[A-Za-z0-9.-]+(?::[0-9]+)?(?:/[A-Za-z0-9_.-]+)+/?")
|
|
265
|
+
TOOL_TAG = re.compile(r"[A-Za-z0-9][A-Za-z0-9_.+-]*")
|
|
266
|
+
# The git repositories whose latest release the nightly can read.
|
|
267
|
+
GITHUB_REPO = re.compile(r"https://github\.com/(?P<repo>[A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+?)"
|
|
268
|
+
r"(?:\.git)?/?")
|
|
269
|
+
|
|
270
|
+
# The checkout the site job runs, pinned as pubrules' own workflows pin it.
|
|
271
|
+
CHECKOUT = "actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5.1.0"
|
|
272
|
+
SITE = ("check_dir", "pr_build", "build", "url")
|
|
273
|
+
# A use of a deploy-static action, from whichever repository ships it,
|
|
274
|
+
# with what follows `@` up to the end of the line: the site job runs that
|
|
275
|
+
# same pin, so pubrules holds no pin of its own for it.
|
|
276
|
+
DEPLOY_STATIC = re.compile(r"(?P<uses>[A-Za-z0-9][A-Za-z0-9-]*/[A-Za-z0-9_.-]+/actions/deploy-static@[^\s#]+"
|
|
277
|
+
r"(?:[ \t]+#[ \t]*v[0-9][^\s]*)?)")
|
|
278
|
+
# deploy-static's `dir`, in the lines of the step that uses it.
|
|
279
|
+
DEPLOY_DIR = re.compile(r"""^\s*(?:with:\s*\{[^}\n]*?\b)?dir:\s*['"]?([^'"\s,}]+)""", re.M)
|
|
280
|
+
|
|
281
|
+
# The first pubrel whose release workflow takes the release app's secrets.
|
|
282
|
+
# Passing a secret a reusable workflow does not declare fails the caller.
|
|
283
|
+
RELEASE_APP_FROM = (0, 6, 1)
|
|
284
|
+
RELEASE_SECRETS = (" secrets: { RELEASE_APP_ID: '${{ secrets.RELEASE_APP_ID }}', "
|
|
285
|
+
"RELEASE_APP_KEY: '${{ secrets.RELEASE_APP_KEY }}' }\n")
|
|
286
|
+
|
|
287
|
+
|
|
288
|
+
def pubrel_from(pubrel: str, first: tuple[int, int, int]) -> bool:
|
|
289
|
+
"""Whether pubrel `pubrel` is release `first` or later."""
|
|
290
|
+
parts = []
|
|
291
|
+
for part in pubrel.split("-", 1)[0].split(".")[:3]:
|
|
292
|
+
if not part.isdigit():
|
|
293
|
+
return False
|
|
294
|
+
parts.append(int(part))
|
|
295
|
+
return tuple(parts) >= first
|
|
296
|
+
|
|
297
|
+
|
|
298
|
+
def takes_app(pubrel: str) -> bool:
|
|
299
|
+
"""Whether pubrel `pubrel`'s release workflow takes the app's secrets."""
|
|
300
|
+
return pubrel_from(pubrel, RELEASE_APP_FROM)
|
|
301
|
+
|
|
302
|
+
|
|
303
|
+
# The first pubrel whose release workflow takes runs-on.
|
|
304
|
+
RELEASE_RUNS_ON_FROM = (0, 7, 0)
|
|
305
|
+
|
|
306
|
+
|
|
307
|
+
# Inputs the release workflow takes besides pubrel-version, by language.
|
|
308
|
+
RELEASE_WITH = {"rust": {"semver-checks-version": "0.51.0"}}
|
|
309
|
+
# The events a language runs on unless the ci block names them: Rust adds
|
|
310
|
+
# the nightly, for longer fuzzing and the latest stable and beta Rust.
|
|
311
|
+
LANGUAGE_EVENTS = {"rust": EVENTS}
|
|
312
|
+
|
|
313
|
+
|
|
314
|
+
def stop(text: str) -> None:
|
|
315
|
+
sys.exit(f"pubrules: pubrules.json's ci {text}")
|
|
316
|
+
|
|
317
|
+
|
|
318
|
+
def is_node(root: Path) -> bool:
|
|
319
|
+
"""A Node package: a package.json, and no pyproject.toml beside it."""
|
|
320
|
+
return (root / "package.json").exists() and not (root / "pyproject.toml").exists()
|
|
321
|
+
|
|
322
|
+
|
|
323
|
+
def plan(config: dict[str, Any], root: Path) -> dict:
|
|
324
|
+
"""What the Rust crate standard requires here: the make targets of each
|
|
325
|
+
job and the workflow jobs it turns on, as `pubrules required` gives them."""
|
|
326
|
+
from . import conform, standard # noqa: PLC0415
|
|
327
|
+
rules = standard.load()
|
|
328
|
+
waived, _, _ = conform.exceptions(config, {r["id"] for r in rules["rules"]})
|
|
329
|
+
# Before there is a crate, every rule a library would meet.
|
|
330
|
+
library = not (root / "Cargo.toml").exists() or standard.has_library(root)
|
|
331
|
+
return standard.required(rules, library, frozenset(waived))
|
|
332
|
+
|
|
333
|
+
|
|
334
|
+
def planned(job: Job, rules: dict | None) -> bool:
|
|
335
|
+
"""Whether the standard asks for this job: some required targets, or a
|
|
336
|
+
workflow job it turns on."""
|
|
337
|
+
if job.standard is None or rules is None:
|
|
338
|
+
return True
|
|
339
|
+
if job.standard in rules["targets"]:
|
|
340
|
+
return bool(rules["targets"][job.standard])
|
|
341
|
+
return job.standard in rules["jobs"]
|
|
342
|
+
|
|
343
|
+
|
|
344
|
+
def settings(config: dict[str, Any], root: Path) -> dict[str, Any] | None:
|
|
345
|
+
"""The ci block, checked and filled in, or None when pubrules.json has
|
|
346
|
+
none and the repository writes its own workflows."""
|
|
347
|
+
given = config.get("ci")
|
|
348
|
+
if given is None:
|
|
349
|
+
return None
|
|
350
|
+
kind = config.get("kind")
|
|
351
|
+
if not isinstance(given, dict):
|
|
352
|
+
stop('block must be an object, such as {"with": {"test": "npm test"}}')
|
|
353
|
+
unknown = sorted(set(given) - set(SETTINGS))
|
|
354
|
+
if unknown:
|
|
355
|
+
stop(f"block has no setting {', '.join(unknown)}; it takes {', '.join(SETTINGS)}")
|
|
356
|
+
if kind == "utility":
|
|
357
|
+
language = given.get("language", "node" if is_node(root) else "python")
|
|
358
|
+
if language not in UTILITY:
|
|
359
|
+
stop(f"language must be one of {', '.join(UTILITY)}, not {language!r}")
|
|
360
|
+
else:
|
|
361
|
+
language = kind
|
|
362
|
+
if "language" in given and given["language"] != kind:
|
|
363
|
+
stop(f"language is for a utility; a {kind} repository's CI is {kind}'s")
|
|
364
|
+
table = JOBS[language]
|
|
365
|
+
events = given.get("events", list(LANGUAGE_EVENTS.get(language, DEFAULT_EVENTS)))
|
|
366
|
+
if not isinstance(events, list) or not events or not set(events) <= set(EVENTS):
|
|
367
|
+
stop(f"events must list some of {', '.join(EVENTS)}")
|
|
368
|
+
if "queue" in events and "pr" not in events:
|
|
369
|
+
stop("events lists queue without pr: the merge queue runs pull requests")
|
|
370
|
+
cron = given.get("cron", CRON)
|
|
371
|
+
if not isinstance(cron, str) or len(cron.split()) != 5:
|
|
372
|
+
stop(f"cron must be a schedule of five fields, such as {CRON!r}")
|
|
373
|
+
inputs = given.get("with", {})
|
|
374
|
+
if not isinstance(inputs, dict) or not all(
|
|
375
|
+
isinstance(v, (str, bool, int, float)) for v in inputs.values()):
|
|
376
|
+
stop("with must be an object of the jobs' inputs: strings, numbers or true/false")
|
|
377
|
+
taken = {i for job in table.values() for i in job.inputs}
|
|
378
|
+
unknown = sorted(set(inputs) - taken)
|
|
379
|
+
if unknown:
|
|
380
|
+
stop(f"with gives {', '.join(unknown)}, which no {language} job takes; "
|
|
381
|
+
f"they take {', '.join(sorted(taken))}")
|
|
382
|
+
overrides = given.get("jobs", {})
|
|
383
|
+
if not isinstance(overrides, dict):
|
|
384
|
+
stop('jobs must be an object, such as {"review": false}')
|
|
385
|
+
listed = tools(given)
|
|
386
|
+
if listed and language != "rust":
|
|
387
|
+
stop(f"tools is for a Rust repository's functional job; {language} has none")
|
|
388
|
+
runners = runs_on(given, table)
|
|
389
|
+
rules = plan(config, root) if language == "rust" else None
|
|
390
|
+
jobs: dict[str, tuple[str, ...]] = {}
|
|
391
|
+
for name, job in table.items():
|
|
392
|
+
override = overrides.get(name, {})
|
|
393
|
+
if override is False:
|
|
394
|
+
continue
|
|
395
|
+
if not isinstance(override, dict) or set(override) - {"events"}:
|
|
396
|
+
stop(f'jobs.{name} must be false or {{"events": [...]}}')
|
|
397
|
+
on = override.get("events", list(job.events))
|
|
398
|
+
if not isinstance(on, list) or not set(on) <= set(EVENTS):
|
|
399
|
+
stop(f"jobs.{name}.events must list some of {', '.join(EVENTS)}")
|
|
400
|
+
if job.when and inputs.get(job.when, "") == "":
|
|
401
|
+
continue
|
|
402
|
+
if not planned(job, rules) or (name == "cross" and inputs.get("cross-targets") == "[]"):
|
|
403
|
+
continue
|
|
404
|
+
jobs[name] = tuple(e for e in EVENTS if e in on and (e in events or e == "queue")
|
|
405
|
+
and not (name in NOT_QUEUED and e == "queue"))
|
|
406
|
+
unknown = sorted(set(overrides) - set(table))
|
|
407
|
+
if unknown:
|
|
408
|
+
stop(f"jobs names {', '.join(unknown)}, which {language} has not; it has {', '.join(table)}")
|
|
409
|
+
# The queue has a workflow of its own only when it runs a job pull
|
|
410
|
+
# requests do not; else pr.yml runs in the queue too, with the queue's
|
|
411
|
+
# jobs. Without a queue, its jobs run on pull requests instead.
|
|
412
|
+
separate = "queue" in events and any("queue" in on and "pr" not in on for on in jobs.values())
|
|
413
|
+
if not separate:
|
|
414
|
+
events = [e for e in events if e != "queue"]
|
|
415
|
+
jobs = {name: tuple(e for e in EVENTS if e in events and (
|
|
416
|
+
e in on or (e == "pr" and "queue" in on))) for name, on in jobs.items()}
|
|
417
|
+
release = given.get("release", language not in UNRELEASED)
|
|
418
|
+
if language in UNRELEASED and release is not False:
|
|
419
|
+
stop(f"release is not for {UNRELEASED[language]}")
|
|
420
|
+
if release is True:
|
|
421
|
+
release = {}
|
|
422
|
+
if release is not False:
|
|
423
|
+
if not isinstance(release, dict) or set(release) - {"pubrel", "with", "uses"}:
|
|
424
|
+
stop('release must be true, false or {"pubrel": "0.5.4", "with": {...}}')
|
|
425
|
+
extra = release.get("with", {})
|
|
426
|
+
if not isinstance(extra, dict):
|
|
427
|
+
stop("release.with must be an object of the release workflow's inputs")
|
|
428
|
+
if "uses" in release:
|
|
429
|
+
# pubrel itself: its own release workflow, from the checkout.
|
|
430
|
+
if release["uses"] != "local" or "pubrel" in release:
|
|
431
|
+
stop('release.uses can only be "local", the repository\'s own '
|
|
432
|
+
'.github/workflows/release.yml, and then names no pubrel')
|
|
433
|
+
release = {"local": True, "with": extra}
|
|
434
|
+
else:
|
|
435
|
+
pubrel = release["pubrel"] if "pubrel" in release else pinned_pubrel(config, root)
|
|
436
|
+
if not isinstance(pubrel, str) or not pubrel[:1].isdigit():
|
|
437
|
+
stop('release.pubrel must be a version, such as "0.5.4"')
|
|
438
|
+
release = {"pubrel": pubrel, "with": {**RELEASE_WITH.get(language, {}), **extra}}
|
|
439
|
+
return {"language": language, "events": [e for e in EVENTS if e in events], "cron": cron,
|
|
440
|
+
"with": inputs, "jobs": jobs, "release": release, "layout": layout(language, root),
|
|
441
|
+
"site": site(config, root), "tools": listed, "runs-on": runners,
|
|
442
|
+
"targets": rules["targets"] if rules else {}}
|
|
443
|
+
|
|
444
|
+
|
|
445
|
+
def pinned_pubrel(config: dict[str, Any], root: Path) -> str:
|
|
446
|
+
"""The pubrel a release job calls where ci.release names none: the one the
|
|
447
|
+
workflows sync last wrote pin, else its latest release (releases.py),
|
|
448
|
+
which the nightly then moves."""
|
|
449
|
+
from .refs import RELEASE_PIN # noqa: PLC0415
|
|
450
|
+
for event in EVENTS:
|
|
451
|
+
written = root / path(event)
|
|
452
|
+
if written.is_file() and is_generated(text := written.read_text(encoding="utf-8")):
|
|
453
|
+
if m := RELEASE_PIN.search(text):
|
|
454
|
+
return m["version"]
|
|
455
|
+
from .releases import version # noqa: PLC0415
|
|
456
|
+
return version("pubrel", owner(config))
|
|
457
|
+
|
|
458
|
+
|
|
459
|
+
# A runner label: GitHub's, such as ubuntu-latest, or a self-hosted runner
|
|
460
|
+
# scale set's name.
|
|
461
|
+
LABEL = re.compile(r"[A-Za-z0-9][A-Za-z0-9_.-]*")
|
|
462
|
+
# The jobs pubrules writes into the workflow itself, besides the table's.
|
|
463
|
+
OWN_JOBS = ("tested", "passed", "site", "release")
|
|
464
|
+
|
|
465
|
+
|
|
466
|
+
def runs_on(given: dict[str, Any], table: dict[str, Job]) -> dict[str, str]:
|
|
467
|
+
"""The ci block's `runs-on`, checked: the runner label of each job that
|
|
468
|
+
has one, by job name, under "default" for every other. Empty without
|
|
469
|
+
it, so nothing is written and every job runs on ubuntu-latest."""
|
|
470
|
+
value = given.get("runs-on")
|
|
471
|
+
if value is None:
|
|
472
|
+
return {}
|
|
473
|
+
if isinstance(value, str):
|
|
474
|
+
value = {"default": value}
|
|
475
|
+
if not isinstance(value, dict) or not value:
|
|
476
|
+
stop('runs-on must be a runner label, such as "my-runners", or an object of them '
|
|
477
|
+
'by job, such as {"default": "my-runners", "review": "ubuntu-latest"}')
|
|
478
|
+
named = [n for n, job in table.items() if not job.matrix]
|
|
479
|
+
allowed = ("default", *named, *OWN_JOBS)
|
|
480
|
+
unknown = sorted(set(value) - set(allowed))
|
|
481
|
+
if unknown:
|
|
482
|
+
matrix = sorted(n for n in unknown if n in table)
|
|
483
|
+
if matrix:
|
|
484
|
+
stop(f"runs-on names {', '.join(matrix)}, whose runners its os-matrix input gives")
|
|
485
|
+
stop(f"runs-on names {', '.join(unknown)}; it takes {', '.join(allowed)}")
|
|
486
|
+
for name, label in value.items():
|
|
487
|
+
if not isinstance(label, str) or not LABEL.fullmatch(label):
|
|
488
|
+
stop(f"runs-on.{name} must be a runner label, such as \"ubuntu-latest\", "
|
|
489
|
+
f"not {json.dumps(label)}")
|
|
490
|
+
return value
|
|
491
|
+
|
|
492
|
+
|
|
493
|
+
def runner(ci: dict[str, Any], name: str) -> str | None:
|
|
494
|
+
"""The label job `name` runs on where the ci block gives one."""
|
|
495
|
+
return ci["runs-on"].get(name, ci["runs-on"].get("default"))
|
|
496
|
+
|
|
497
|
+
|
|
498
|
+
def tools(given: dict[str, Any]) -> list[dict[str, str]]:
|
|
499
|
+
"""The ci block's `tools`, checked; each is a crate, a git and a tag."""
|
|
500
|
+
listed = given.get("tools", [])
|
|
501
|
+
if not isinstance(listed, list):
|
|
502
|
+
stop('tools must be a list of objects, each with a crate, a git and a tag')
|
|
503
|
+
for tool in listed:
|
|
504
|
+
if not isinstance(tool, dict):
|
|
505
|
+
stop(f"tools must list objects, not {json.dumps(tool)}")
|
|
506
|
+
unknown = sorted(set(tool) - set(TOOL))
|
|
507
|
+
if unknown:
|
|
508
|
+
stop(f"tools has no setting {', '.join(unknown)}; each tool takes {', '.join(TOOL)}")
|
|
509
|
+
for key, pattern, what in (("crate", CRATE, "a crate's name"),
|
|
510
|
+
("git", TOOL_GIT, "the git repository's https:// address"),
|
|
511
|
+
("tag", TOOL_TAG, "a tag of that repository")):
|
|
512
|
+
if not isinstance(tool.get(key), str) or not pattern.fullmatch(tool[key]):
|
|
513
|
+
stop(f"tools: {key} must be {what}, not {json.dumps(tool.get(key))}")
|
|
514
|
+
crates = [tool["crate"] for tool in listed]
|
|
515
|
+
if len(set(crates)) != len(crates):
|
|
516
|
+
stop("tools lists a crate twice")
|
|
517
|
+
return listed
|
|
518
|
+
|
|
519
|
+
|
|
520
|
+
def layout(language: str, root: Path) -> str:
|
|
521
|
+
"""The sonar job's `layout`: sonar.sources and sonar.tests, from the
|
|
522
|
+
directories the repository has; empty without tests to name. With no
|
|
523
|
+
code directory the code is the repository, less its tests, so no file
|
|
524
|
+
is read as both."""
|
|
525
|
+
if language not in LAYOUT:
|
|
526
|
+
return ""
|
|
527
|
+
code, tests = ([d for d in dirs if (root / d).is_dir()] for dirs in LAYOUT[language])
|
|
528
|
+
if not tests:
|
|
529
|
+
return ""
|
|
530
|
+
out = [f"-Dsonar.sources={','.join(code or ['.'])}", f"-Dsonar.tests={','.join(tests)}"]
|
|
531
|
+
if not code:
|
|
532
|
+
out.append(f"-Dsonar.exclusions={','.join(f'{d}/**' for d in tests)}")
|
|
533
|
+
return " ".join(out)
|
|
534
|
+
|
|
535
|
+
|
|
536
|
+
def site(config: dict[str, Any], root: Path) -> dict[str, str] | None:
|
|
537
|
+
"""The site job's settings: pubrules.json's site block, checked, filled
|
|
538
|
+
in from the workflow that runs deploy-static, and the deploy-static
|
|
539
|
+
that workflow pins (`uses`); None when the repository deploys no static
|
|
540
|
+
site with it."""
|
|
541
|
+
given = config.get("site")
|
|
542
|
+
deploys = [p for p in sorted((root / ".github" / "workflows").glob("*.y*ml"))
|
|
543
|
+
if not is_generated(text := p.read_text(encoding="utf-8"))
|
|
544
|
+
and DEPLOY_STATIC.search(text)]
|
|
545
|
+
if given is None and not deploys:
|
|
546
|
+
return None
|
|
547
|
+
|
|
548
|
+
def refuse(text: str) -> None:
|
|
549
|
+
sys.exit(f"pubrules: pubrules.json's site {text}")
|
|
550
|
+
|
|
551
|
+
given = {} if given is None else given
|
|
552
|
+
if not isinstance(given, dict):
|
|
553
|
+
refuse('block must be an object, such as {"check_dir": "public"}')
|
|
554
|
+
unknown = sorted(set(given) - set(SITE))
|
|
555
|
+
if unknown:
|
|
556
|
+
refuse(f"block has no setting {', '.join(unknown)}; it takes {', '.join(SITE)}")
|
|
557
|
+
for key, value in given.items():
|
|
558
|
+
if not isinstance(value, str) or not value.strip() or "\n" in value:
|
|
559
|
+
refuse(f"{key} must be one line of text")
|
|
560
|
+
if "url" in given and not re.fullmatch(r"https://[^\s/]+/\S*", given["url"]):
|
|
561
|
+
refuse("url must be the site's https:// address, such as https://example.eu/")
|
|
562
|
+
out = dict(given)
|
|
563
|
+
if "check_dir" not in out:
|
|
564
|
+
found = []
|
|
565
|
+
for path in deploys:
|
|
566
|
+
text = path.read_text(encoding="utf-8")
|
|
567
|
+
for m in DEPLOY_STATIC.finditer(text):
|
|
568
|
+
if d := DEPLOY_DIR.search("\n".join(text[m.end():].splitlines()[:12])):
|
|
569
|
+
found.append(d[1])
|
|
570
|
+
if len(set(found)) != 1:
|
|
571
|
+
refuse("block needs check_dir, the directory deploy-static deploys: "
|
|
572
|
+
+ (f"the workflows give {', '.join(sorted(set(found)))}" if found
|
|
573
|
+
else "no workflow's deploy-static step gives one pubrules can read"))
|
|
574
|
+
out["check_dir"] = found[0]
|
|
575
|
+
rel = Path(out["check_dir"])
|
|
576
|
+
if rel.is_absolute() or ".." in rel.parts:
|
|
577
|
+
refuse("check_dir must be a directory inside the repository")
|
|
578
|
+
pins = sorted({m["uses"] for path in deploys
|
|
579
|
+
for m in DEPLOY_STATIC.finditer(path.read_text(encoding="utf-8"))})
|
|
580
|
+
if len(pins) != 1:
|
|
581
|
+
refuse("block needs the deploy-static its deploy workflow pins, for the pull request's "
|
|
582
|
+
"check to run the same one: "
|
|
583
|
+
+ (f"the workflows pin {', '.join(pins)}; pin one" if pins
|
|
584
|
+
else "no workflow of the repository's own uses deploy-static"))
|
|
585
|
+
out["uses"] = pins[0]
|
|
586
|
+
return out
|
|
587
|
+
|
|
588
|
+
|
|
589
|
+
def flow(values: dict[str, Any]) -> str:
|
|
590
|
+
"""A `with:` mapping on one line: JSON, which YAML reads as it is."""
|
|
591
|
+
return json.dumps(values, ensure_ascii=False)
|
|
592
|
+
|
|
593
|
+
|
|
594
|
+
# The condition each testing job runs under, on main and in the merge
|
|
595
|
+
# queue: no earlier run already passed on the commit's tree.
|
|
596
|
+
UNTESTED = "needs.tested.outputs.tested != 'true'"
|
|
597
|
+
NO_QUEUE = "github.event_name != 'merge_group'"
|
|
598
|
+
|
|
599
|
+
|
|
600
|
+
def queued(ci: dict[str, Any]) -> str:
|
|
601
|
+
"""The event whose workflow the merge queue runs."""
|
|
602
|
+
return "queue" if "queue" in ci["events"] else "pr"
|
|
603
|
+
|
|
604
|
+
|
|
605
|
+
def tested_job(ci: dict[str, Any]) -> str:
|
|
606
|
+
return (" # Whether an earlier run already passed on this commit's tree: then the\n"
|
|
607
|
+
" # jobs it ran are skipped, and nothing here tests less than it did.\n"
|
|
608
|
+
" tested:\n name: ci\n"
|
|
609
|
+
f" uses: {ci['owner']}/pubrules/.github/workflows/job-tested.yml"
|
|
610
|
+
f"@{ref(__version__, ci['commit'])}\n"
|
|
611
|
+
f" with: {flow({'workflow': path(queued(ci)), **runs_with(ci, 'tested')})}\n"
|
|
612
|
+
" permissions:\n actions: read\n contents: read\n")
|
|
613
|
+
|
|
614
|
+
|
|
615
|
+
def runs_with(ci: dict[str, Any], name: str) -> dict[str, str]:
|
|
616
|
+
"""The `runs-on` input of a job pubrules calls, where the ci block gives
|
|
617
|
+
it a runner; nothing else, so the job keeps its default."""
|
|
618
|
+
label = runner(ci, name)
|
|
619
|
+
return {"runs-on": label} if label else {}
|
|
620
|
+
|
|
621
|
+
|
|
622
|
+
def trigger(ci: dict[str, Any], event: str) -> str:
|
|
623
|
+
"""What starts one event's workflow: pr.yml's runs in the merge queue
|
|
624
|
+
too when the queue has none of its own."""
|
|
625
|
+
out = TRIGGERS[event].replace("CRON", ci["cron"])
|
|
626
|
+
if event == "pr" and queued(ci) == "pr":
|
|
627
|
+
out += TRIGGERS["queue"]
|
|
628
|
+
return out
|
|
629
|
+
|
|
630
|
+
|
|
631
|
+
def workflow(ci: dict[str, Any], event: str) -> str:
|
|
632
|
+
"""The text of one event's workflow."""
|
|
633
|
+
table = JOBS[ci["language"]]
|
|
634
|
+
names = [name for name, on in ci["jobs"].items() if event in on]
|
|
635
|
+
# Main takes an earlier run of the queue's workflow on the same tree, and
|
|
636
|
+
# so does the queue, where it runs pr.yml itself: a pull request up to
|
|
637
|
+
# date with main merges as the very files its own run tested.
|
|
638
|
+
shortcut = (event == "main" and "pr" in ci["events"]) or (event == "pr" and queued(ci) == "pr")
|
|
639
|
+
out = [HEADER, f"name: {event}\n\non:\n", trigger(ci, event),
|
|
640
|
+
"\npermissions:\n contents: read\n\njobs:\n"]
|
|
641
|
+
if shortcut:
|
|
642
|
+
out.append(tested_job(ci))
|
|
643
|
+
for name in names:
|
|
644
|
+
job = table[name]
|
|
645
|
+
given = {**job.defaults, **{i: ci["with"][i] for i in job.inputs if i in ci["with"]},
|
|
646
|
+
**job.fixed}
|
|
647
|
+
if "required" in job.internal:
|
|
648
|
+
given["required"] = " ".join(ci["targets"][job.standard])
|
|
649
|
+
if "tools" in job.internal and ci["tools"]:
|
|
650
|
+
given["tools"] = json.dumps(ci["tools"])
|
|
651
|
+
if "layout" in job.internal and ci["layout"]:
|
|
652
|
+
given["layout"] = ci["layout"]
|
|
653
|
+
if not job.matrix:
|
|
654
|
+
given.update(runs_with(ci, name))
|
|
655
|
+
needs = [n for n in job.needs if n in names]
|
|
656
|
+
# On main, a job the queue ran waits to hear whether it passed.
|
|
657
|
+
# In the queue, a tree its pull request already passed is not scanned
|
|
658
|
+
# again either: main's push scans it, with that run's coverage.
|
|
659
|
+
skips = shortcut and (queued(ci) in ci["jobs"][name] or (name == "sonar" and event != "main"))
|
|
660
|
+
if skips or (shortcut and name == "sonar"):
|
|
661
|
+
needs = ["tested", *needs]
|
|
662
|
+
first = name == names[0] and not shortcut
|
|
663
|
+
out.append(f"{'' if first else chr(10)} {name}:\n name: ci\n")
|
|
664
|
+
if needs:
|
|
665
|
+
out.append(f" needs: [{', '.join(needs)}]\n")
|
|
666
|
+
scan = shortcut and name == "sonar" and event == "main"
|
|
667
|
+
if scan:
|
|
668
|
+
# The scan runs either way, after the tests or in their place.
|
|
669
|
+
out.append(" if: ${{ !failure() && !cancelled() }}\n")
|
|
670
|
+
given["coverage-run"] = "${{ needs.tested.outputs.run-id }}"
|
|
671
|
+
elif skips and name in NOT_QUEUED and event == "pr":
|
|
672
|
+
# pr.yml runs in the merge queue too: there, never.
|
|
673
|
+
out.append(f" if: ${{{{ {NO_QUEUE} && {UNTESTED} }}}}\n")
|
|
674
|
+
elif skips:
|
|
675
|
+
out.append(f" if: ${{{{ {UNTESTED} }}}}\n")
|
|
676
|
+
out.append(f" uses: {ci['owner']}/pubrules/.github/workflows/{job.workflow}"
|
|
677
|
+
f"@{ref(__version__, ci['commit'])}\n")
|
|
678
|
+
if scan:
|
|
679
|
+
out.append(" permissions:\n actions: read\n contents: read\n")
|
|
680
|
+
if given:
|
|
681
|
+
out.append(f" with: {flow(given)}\n")
|
|
682
|
+
# By name, not `secrets: inherit`: GitHub hands inherited secrets
|
|
683
|
+
# only to a reusable workflow of the caller's own owner, and drops
|
|
684
|
+
# them without a word for pubrules', once the two owners differ.
|
|
685
|
+
if job.secrets:
|
|
686
|
+
out.append(" secrets: { SONAR_TOKEN: '${{ secrets.SONAR_TOKEN }}' }\n")
|
|
687
|
+
if ci["site"] and event in SITE_EVENTS:
|
|
688
|
+
names = [*names, "site"]
|
|
689
|
+
out.append(site_job(ci))
|
|
690
|
+
if shortcut and event == "pr" and any(n in NOT_QUEUED for n in names):
|
|
691
|
+
out.append(
|
|
692
|
+
"\n # The one check a ruleset requires: it passes only when every job\n"
|
|
693
|
+
" # above succeeded or, where an earlier run already passed on the\n"
|
|
694
|
+
" # commit's tree, was skipped for it, or is the scan, which the merge\n"
|
|
695
|
+
" # queue never runs. None is skipped for not being asked for.\n"
|
|
696
|
+
" passed:\n name: ci / passed\n if: always()\n"
|
|
697
|
+
f" needs: [{', '.join(['tested', *names])}]\n runs-on: {runner(ci, 'passed') or 'ubuntu-latest'}\n"
|
|
698
|
+
" steps:\n"
|
|
699
|
+
" - run: |\n echo \"$RESULTS\"\n"
|
|
700
|
+
" echo \"$RESULTS\" | jq -e --arg tested \"$TESTED\" --arg event \"$EVENT\" '.tested.result == \"success\"\n"
|
|
701
|
+
" and all(to_entries[]; .value.result == \"success\" or (.value.result == \"skipped\"\n"
|
|
702
|
+
f" and ($tested == \"true\" or (.key == \"{NOT_QUEUED[0]}\" and $event == \"merge_group\"))))'"
|
|
703
|
+
" > /dev/null\n"
|
|
704
|
+
" env: { RESULTS: '${{ toJSON(needs) }}', TESTED: '${{ needs.tested.outputs.tested }}', "
|
|
705
|
+
"EVENT: '${{ github.event_name }}' }\n")
|
|
706
|
+
elif shortcut:
|
|
707
|
+
out.append(
|
|
708
|
+
"\n # The one check a ruleset requires: it passes only when every job\n"
|
|
709
|
+
" # above succeeded or, where an earlier run already passed on the\n"
|
|
710
|
+
" # commit's tree, was skipped for it. None is skipped for not being asked for.\n"
|
|
711
|
+
" passed:\n name: ci / passed\n if: always()\n"
|
|
712
|
+
f" needs: [{', '.join(['tested', *names])}]\n runs-on: {runner(ci, 'passed') or 'ubuntu-latest'}\n"
|
|
713
|
+
" steps:\n"
|
|
714
|
+
" - run: |\n echo \"$RESULTS\"\n"
|
|
715
|
+
" echo \"$RESULTS\" | jq -e --arg tested \"$TESTED\" '.tested.result == \"success\"\n"
|
|
716
|
+
" and all(.[]; .result == \"success\" or (.result == \"skipped\" and $tested == \"true\"))'"
|
|
717
|
+
" > /dev/null\n"
|
|
718
|
+
" env: { RESULTS: '${{ toJSON(needs) }}', TESTED: '${{ needs.tested.outputs.tested }}' }\n")
|
|
719
|
+
else:
|
|
720
|
+
out.append(
|
|
721
|
+
"\n # The one check a ruleset requires: it passes only when every job\n"
|
|
722
|
+
" # above succeeded. None of them is ever skipped for not being asked for.\n"
|
|
723
|
+
" passed:\n name: ci / passed\n if: always()\n"
|
|
724
|
+
f" needs: [{', '.join(names)}]\n runs-on: {runner(ci, 'passed') or 'ubuntu-latest'}\n steps:\n"
|
|
725
|
+
" - run: |\n echo \"$RESULTS\"\n"
|
|
726
|
+
" echo \"$RESULTS\" | jq -e 'all(.[]; .result == \"success\")' > /dev/null\n"
|
|
727
|
+
" env: { RESULTS: '${{ toJSON(needs) }}' }\n")
|
|
728
|
+
# A job skipped upstream skips the release too, unless it says otherwise.
|
|
729
|
+
after = (" if: ${{ !cancelled() && needs.passed.result == 'success' }}\n"
|
|
730
|
+
if shortcut else "")
|
|
731
|
+
release = ci["release"]
|
|
732
|
+
if release and event in RELEASE_EVENTS and release.get("local"):
|
|
733
|
+
out.append(
|
|
734
|
+
"\n # The repository's own release workflow, built from this checkout,\n"
|
|
735
|
+
" # so a change to it is tested here before a release carries it to\n"
|
|
736
|
+
" # anyone else.\n"
|
|
737
|
+
" release:\n needs: passed\n" + after + " uses: ./.github/workflows/release.yml\n"
|
|
738
|
+
+ (f" with: {flow(release['with'])}\n" if release["with"] else "")
|
|
739
|
+
+ RELEASE_SECRETS
|
|
740
|
+
+ " permissions:\n contents: write\n")
|
|
741
|
+
elif release and event in RELEASE_EVENTS:
|
|
742
|
+
pubrel = release["pubrel"]
|
|
743
|
+
out.append(
|
|
744
|
+
"\n # pubrel's release workflow: on a pull request, a code change records\n"
|
|
745
|
+
" # what it changes and a release is the bump its changes require; on\n"
|
|
746
|
+
" # main, a published version is tagged and released.\n"
|
|
747
|
+
" release:\n needs: passed\n" + after +
|
|
748
|
+
f" uses: {ci['owner']}/pubrel/.github/workflows/release.yml"
|
|
749
|
+
f"@{ref(pubrel, release['commit'])}\n"
|
|
750
|
+
f" with: {flow({'pubrel-version': f'v{pubrel}', **release['with'], **release_runner(ci, pubrel)})}\n"
|
|
751
|
+
+ (RELEASE_SECRETS if takes_app(pubrel) else "")
|
|
752
|
+
+ " permissions:\n contents: write\n")
|
|
753
|
+
return "".join(out)
|
|
754
|
+
|
|
755
|
+
|
|
756
|
+
def release_runner(ci: dict[str, Any], pubrel: str) -> dict[str, str]:
|
|
757
|
+
"""The release job's runs-on input: the ci block's runner for it, where
|
|
758
|
+
that pubrel's release workflow takes one (0.7.0 on); before that it
|
|
759
|
+
always runs on ubuntu-latest."""
|
|
760
|
+
return runs_with(ci, "release") if pubrel_from(pubrel, RELEASE_RUNS_ON_FROM) else {}
|
|
761
|
+
|
|
762
|
+
|
|
763
|
+
def site_job(ci: dict[str, Any]) -> str:
|
|
764
|
+
"""The pull request's site job: the site built as it is for a deploy,
|
|
765
|
+
and checked by the deploy-static its deploy workflow pins, which deploys nothing."""
|
|
766
|
+
s = ci["site"]
|
|
767
|
+
build = s.get("pr_build") or s.get("build")
|
|
768
|
+
given = {"dir": s["check_dir"], "check-only": "true", **({"site-url": s["url"]} if "url" in s else {})}
|
|
769
|
+
return ("\n # The site as deploy-static checks it before a deploy, so a site that\n"
|
|
770
|
+
" # would fail to deploy fails the pull request. It deploys nothing.\n"
|
|
771
|
+
f" site:\n name: ci / site\n runs-on: {runner(ci, 'site') or 'ubuntu-latest'}\n steps:\n"
|
|
772
|
+
f" - uses: {CHECKOUT}\n"
|
|
773
|
+
+ (f" - run: {json.dumps(build)}\n" if build else "")
|
|
774
|
+
+ f" - uses: {s['uses']}\n"
|
|
775
|
+
f" with: {flow(given)}\n")
|
|
776
|
+
|
|
777
|
+
|
|
778
|
+
def path(event: str) -> str:
|
|
779
|
+
return f".github/workflows/{event}.yml"
|
|
780
|
+
|
|
781
|
+
|
|
782
|
+
INTEGRATIONS = "integrations.json"
|
|
783
|
+
WITH_SECRETS = "with-secrets"
|
|
784
|
+
OIDC = "github.actions-oidc"
|
|
785
|
+
SERVICE_NEEDS = ".github/workflows/service-needs.yml" # service_needs.CALLER
|
|
786
|
+
|
|
787
|
+
|
|
788
|
+
def declared(root: Path, wanted: list[tuple[str, str]], gone: frozenset[str] = frozenset()) -> tuple[str, str] | None:
|
|
789
|
+
"""integrations.json as it must read for the integrations scan to pass: every
|
|
790
|
+
workflow pubrules writes that takes SONAR_TOKEN listed under a sonarcloud use, adding
|
|
791
|
+
them to "sonarcloud.scan" where none lists them, and the service-needs caller, whose OIDC
|
|
792
|
+
token is GitHub's identity, under "github.actions-oidc". None where the repository keeps no
|
|
793
|
+
integrations.json, or one that isn't JSON, which the scan reports itself. A path
|
|
794
|
+
already listed, by name or by a pattern, leaves the file as it is, byte for byte;
|
|
795
|
+
nothing else is taken out, since something else may need it, but for the paths this
|
|
796
|
+
sync removes (`gone`, such as the go-live caller service-needs replaced), which come
|
|
797
|
+
out of the OIDC use, so one sync leaves the file as the scan wants it.
|
|
798
|
+
|
|
799
|
+
The one path moved: the with-secrets pubrules writes runs `./pubsec run` and
|
|
800
|
+
no longer calls bws, so the scan calls a Bitwarden use declared on it stale. Where
|
|
801
|
+
pubrules writes it, such a use names with-secrets.json, which the scan reads, instead."""
|
|
802
|
+
path = root / INTEGRATIONS
|
|
803
|
+
if not path.is_file():
|
|
804
|
+
return None
|
|
805
|
+
text = path.read_text(encoding="utf-8")
|
|
806
|
+
try:
|
|
807
|
+
data = json.loads(text)
|
|
808
|
+
uses = data["uses"]
|
|
809
|
+
except (ValueError, KeyError, TypeError):
|
|
810
|
+
return None
|
|
811
|
+
if not isinstance(uses, list) or not all(isinstance(u, dict) for u in uses):
|
|
812
|
+
return None
|
|
813
|
+
from fnmatch import fnmatch # noqa: PLC0415
|
|
814
|
+
listed = [p for u in uses if str(u.get("plugin", "")).startswith("sonarcloud.")
|
|
815
|
+
for p in u.get("paths") or [] if isinstance(p, str)]
|
|
816
|
+
missing = [name for name, body in wanted
|
|
817
|
+
if name.startswith(".github/workflows/") and "secrets.SONAR_TOKEN" in body
|
|
818
|
+
and not any(name == p or fnmatch(name, p) for p in listed)]
|
|
819
|
+
moved = False
|
|
820
|
+
if any(name == WITH_SECRETS for name, _ in wanted):
|
|
821
|
+
for use in uses:
|
|
822
|
+
paths = use.get("paths")
|
|
823
|
+
if (str(use.get("plugin", "")).startswith("bitwarden.") and isinstance(paths, list)
|
|
824
|
+
and WITH_SECRETS in paths):
|
|
825
|
+
use["paths"] = sorted({*(p for p in paths if p != WITH_SECRETS), f"{WITH_SECRETS}.json"})
|
|
826
|
+
moved = True
|
|
827
|
+
# The service-needs caller's OIDC token (id-token: write), under GitHub's OIDC use.
|
|
828
|
+
oidc = [p for u in uses if u.get("plugin") == OIDC for p in u.get("paths") or [] if isinstance(p, str)]
|
|
829
|
+
identity = [name for name, body in wanted if name == SERVICE_NEEDS and "id-token: write" in body
|
|
830
|
+
and not any(name == p or fnmatch(name, p) for p in oidc)]
|
|
831
|
+
for use in uses:
|
|
832
|
+
paths = use.get("paths")
|
|
833
|
+
if use.get("plugin") == OIDC and isinstance(paths, list) and set(paths) & gone:
|
|
834
|
+
use["paths"] = [p for p in paths if p not in gone]
|
|
835
|
+
moved = True
|
|
836
|
+
if not missing and not moved and not identity:
|
|
837
|
+
return INTEGRATIONS, text
|
|
838
|
+
for plugin, paths in (("sonarcloud.scan", missing), (OIDC, identity)):
|
|
839
|
+
if not paths:
|
|
840
|
+
continue
|
|
841
|
+
use = next((u for u in uses if u.get("plugin") == plugin), None)
|
|
842
|
+
if use is None:
|
|
843
|
+
use = {"plugin": plugin, "paths": [], "mode": "required"}
|
|
844
|
+
uses.append(use)
|
|
845
|
+
uses.sort(key=lambda u: str(u.get("plugin", "")))
|
|
846
|
+
use["paths"] = sorted({*use.get("paths", []), *paths})
|
|
847
|
+
return INTEGRATIONS, json.dumps(data, indent=2) + "\n"
|
|
848
|
+
|
|
849
|
+
|
|
850
|
+
REVIEW_LICENSES = ".github/dependency-review-licenses.txt"
|
|
851
|
+
CALLED = re.compile(r"""^[ \t-]*uses:[ \t]*['"]?(?P<owner>[A-Za-z0-9-]+)/(?P<path>[^@\s'"]+)@""", re.M)
|
|
852
|
+
|
|
853
|
+
|
|
854
|
+
def review_licenses(config: dict[str, Any], root: Path, wanted: list[tuple[str, str]],
|
|
855
|
+
gone: frozenset[str] = frozenset()) -> tuple[str, str] | None:
|
|
856
|
+
"""The review job's license exceptions, where a workflow pubrules writes runs
|
|
857
|
+
job-review.yml: every reusable workflow and action of the owner's that this
|
|
858
|
+
repository's workflows call, as the purl GitHub's dependency graph gives it,
|
|
859
|
+
without its version. GitHub reads no license for them, and the action takes
|
|
860
|
+
no wildcard, so each is named; the job keeps only the owner's. A workflow this
|
|
861
|
+
sync removes (`gone`) calls nothing any more."""
|
|
862
|
+
bodies = {p: b for p, b in wanted if p.startswith(".github/workflows/")}
|
|
863
|
+
if not any("/job-review.yml@" in b for b in bodies.values()):
|
|
864
|
+
return None
|
|
865
|
+
for found in sorted((root / ".github" / "workflows").glob("*.y*ml")):
|
|
866
|
+
if f".github/workflows/{found.name}" not in gone:
|
|
867
|
+
bodies.setdefault(f".github/workflows/{found.name}", found.read_text(encoding="utf-8"))
|
|
868
|
+
own = owner(config)
|
|
869
|
+
purls = sorted({f"pkg:githubactions/{m['owner']}/{m['path']}" for body in bodies.values()
|
|
870
|
+
for m in CALLED.finditer(body) if m["owner"] == own})
|
|
871
|
+
return REVIEW_LICENSES, ("# Written by `pubrules sync` from this repository's workflows. Do not edit:\n"
|
|
872
|
+
"# the review job leaves these out of its license check, and no other.\n"
|
|
873
|
+
+ "".join(f"{p}\n" for p in purls))
|
|
874
|
+
|
|
875
|
+
|
|
876
|
+
def generated(config: dict[str, Any], root: Path) -> list[tuple[str, str]]:
|
|
877
|
+
"""Each workflow pubrules writes, by path: none without a ci block."""
|
|
878
|
+
ci = settings(config, root)
|
|
879
|
+
if ci is None:
|
|
880
|
+
return []
|
|
881
|
+
ci["owner"] = owner(config)
|
|
882
|
+
ci["commit"] = release_commit()
|
|
883
|
+
if ci["release"] and not ci["release"].get("local"):
|
|
884
|
+
from .refs import pubrel_commit # noqa: PLC0415
|
|
885
|
+
written = "".join((root / path(e)).read_text(encoding="utf-8") for e in EVENTS
|
|
886
|
+
if (root / path(e)).is_file())
|
|
887
|
+
ci["release"]["commit"] = pubrel_commit(ci["release"]["pubrel"], written, ci["owner"])
|
|
888
|
+
return [(path(event), workflow(ci, event)) for event in ci["events"]]
|
|
889
|
+
|
|
890
|
+
|
|
891
|
+
def is_generated(text: str) -> bool:
|
|
892
|
+
return text.startswith(HEADER)
|
|
893
|
+
|
|
894
|
+
|
|
895
|
+
def leftovers(config: dict[str, Any], root: Path) -> list[str]:
|
|
896
|
+
"""Workflows the generated ones replace: a hand-written ci.yml, and a
|
|
897
|
+
generated one for an event the ci block no longer runs."""
|
|
898
|
+
ci = settings(config, root)
|
|
899
|
+
if ci is None:
|
|
900
|
+
return []
|
|
901
|
+
out = []
|
|
902
|
+
old = root / ".github" / "workflows" / "ci.yml"
|
|
903
|
+
if old.exists():
|
|
904
|
+
out.append(".github/workflows/ci.yml")
|
|
905
|
+
for event in EVENTS:
|
|
906
|
+
target = root / path(event)
|
|
907
|
+
if event not in ci["events"] and target.exists() and is_generated(
|
|
908
|
+
target.read_text(encoding="utf-8")):
|
|
909
|
+
out.append(path(event))
|
|
910
|
+
return out
|