pubrules 0.48.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- pubrules/__init__.py +14 -0
- pubrules/audit.py +116 -0
- pubrules/audit_draft.py +295 -0
- pubrules/audit_prepare.py +1128 -0
- pubrules/cache.py +319 -0
- pubrules/cbor.py +45 -0
- pubrules/ci.py +910 -0
- pubrules/cli.py +1069 -0
- pubrules/conform.py +295 -0
- pubrules/doc_coverage.py +235 -0
- pubrules/freshness.py +75 -0
- pubrules/gherkin.py +348 -0
- pubrules/hooks.py +59 -0
- pubrules/hosting.py +372 -0
- pubrules/ignores.py +86 -0
- pubrules/new.py +242 -0
- pubrules/nightly.py +1002 -0
- pubrules/paths.py +32 -0
- pubrules/protect.py +596 -0
- pubrules/providers.py +129 -0
- pubrules/refs.py +213 -0
- pubrules/release-commit +1 -0
- pubrules/releases.py +107 -0
- pubrules/repos.py +67 -0
- pubrules/ruleset.json +43 -0
- pubrules/secrets_config.py +165 -0
- pubrules/security.py +134 -0
- pubrules/service_needs.py +467 -0
- pubrules/standard.json +635 -0
- pubrules/standard.py +266 -0
- pubrules/supply_chain.py +188 -0
- pubrules/tag-ruleset.json +14 -0
- pubrules/templates/common/AGENTS.md +32 -0
- pubrules/templates/common/pubrules +26 -0
- pubrules/templates/common/tests/features/repository.feature +11 -0
- pubrules/templates/common/tests/install-lock +65 -0
- pubrules/templates/common/tests/pytest.ini +4 -0
- pubrules/templates/common/tests/requirements-pubrules.txt +17 -0
- pubrules/templates/common/tests/requirements.txt +3 -0
- pubrules/templates/common/tests/steps/test_repository.py +14 -0
- pubrules/templates/deploy/pubdeploy +28 -0
- pubrules/templates/new-rust/CHANGELOG.md +3 -0
- pubrules/templates/new-rust/Cargo.toml +46 -0
- pubrules/templates/new-rust/LICENSE +202 -0
- pubrules/templates/new-rust/Makefile +18 -0
- pubrules/templates/new-rust/README.md +34 -0
- pubrules/templates/new-rust/SECURITY.md +26 -0
- pubrules/templates/new-rust/corpus/corpus.json +11 -0
- pubrules/templates/new-rust/deny.toml +32 -0
- pubrules/templates/new-rust/fuzz/Cargo.toml +23 -0
- pubrules/templates/new-rust/fuzz/fuzz_targets/greet.rs +8 -0
- pubrules/templates/new-rust/fuzz/gitignore +4 -0
- pubrules/templates/new-rust/fuzz/regressions/README.md +4 -0
- pubrules/templates/new-rust/gitignore +10 -0
- pubrules/templates/new-rust/src/lib.rs +91 -0
- pubrules/templates/new-rust/src/main.rs +19 -0
- pubrules/templates/new-rust/supply-chain/audits.toml +4 -0
- pubrules/templates/new-rust/supply-chain/config.toml +23 -0
- pubrules/templates/new-rust/supply-chain/imports.lock +2 -0
- pubrules/templates/new-rust/tests/conftest.py +9 -0
- pubrules/templates/new-rust/tests/features/cli.feature +14 -0
- pubrules/templates/new-rust/tests/steps/test_cli.py +15 -0
- pubrules/templates/new-rust/tests/vectors.rs +30 -0
- pubrules/templates/new-rust/tools/vectors/generate.py +24 -0
- pubrules/templates/python/Makefile +5 -0
- pubrules/templates/python/pubrules-python.mk +47 -0
- pubrules/templates/rust/Makefile +9 -0
- pubrules/templates/rust/pubrules.mk +138 -0
- pubrules/templates/rust/rust-toolchain.toml +8 -0
- pubrules/templates/service/pubsec +27 -0
- pubrules/templates/service/with-secrets +20 -0
- pubrules/templates/service/with-secrets.json +8 -0
- pubrules/variables.py +45 -0
- pubrules-0.48.0.dist-info/METADATA +1241 -0
- pubrules-0.48.0.dist-info/RECORD +79 -0
- pubrules-0.48.0.dist-info/WHEEL +5 -0
- pubrules-0.48.0.dist-info/entry_points.txt +2 -0
- pubrules-0.48.0.dist-info/licenses/LICENSE +202 -0
- pubrules-0.48.0.dist-info/top_level.txt +1 -0
pubrules/nightly.py
ADDED
|
@@ -0,0 +1,1002 @@
|
|
|
1
|
+
"""The nightly update of a pub* repository, as two commits' worth of change.
|
|
2
|
+
|
|
3
|
+
`pubrules pins --pubrel V` moves every outdated pub* pin to its latest
|
|
4
|
+
release: this pubrules (managed files, lock, `rust-ci.yml`), pubrel's release
|
|
5
|
+
workflow and `pubrel-version` (ci.release.pubrel, written there where the
|
|
6
|
+
repository's release job named none), the pubsec or pubdeploy a launcher
|
|
7
|
+
fetches where deploy.providers lists none, with its sha256, each
|
|
8
|
+
tool `ci.tools` pins by tag, to the latest release of its git repository,
|
|
9
|
+
every other repository's reusable workflow a workflow pins at a release
|
|
10
|
+
(a control plane's reusable workflow, say), to that repository's
|
|
11
|
+
latest release, every Python package a requirements file or pyproject.toml
|
|
12
|
+
pins by another repository's release wheel (`name @ https://github.com/OWNER/
|
|
13
|
+
REPO/releases/download/vX.Y.Z/name-X.Y.Z-....whl`, pubtest's in
|
|
14
|
+
tests/requirements.txt among them) to that repository's latest release,
|
|
15
|
+
forward only, with each `uv pip compile` lock made from a moved file
|
|
16
|
+
compiled again, and the crates `nightly.pins.crates` names in
|
|
17
|
+
`Cargo.lock`, within their requirements.
|
|
18
|
+
|
|
19
|
+
`pubrules crates --report FILE` then tries each upgrade `cargo update` would
|
|
20
|
+
make to the other crates, one at a time on top of those it kept, and keeps
|
|
21
|
+
one only when `cargo vet` covers it with an audit, imported or by a trusted
|
|
22
|
+
publisher, without adding an exemption, and `cargo deny` passes with it.
|
|
23
|
+
Each update it drops is listed in the report with the audits cargo vet
|
|
24
|
+
recommends for it, as `cargo vet suggest` prints them.
|
|
25
|
+
|
|
26
|
+
`pubrules actions` moves each third-party GitHub Action the hand-written
|
|
27
|
+
workflows and composite actions pin by commit to its latest release, by
|
|
28
|
+
the commit of its tag, in place of Dependabot's version updates, whose pull
|
|
29
|
+
requests can add no pubrel row and get no SonarQube Cloud token.
|
|
30
|
+
|
|
31
|
+
`pubrules nightly` then updates the npm packages within their ranges and
|
|
32
|
+
re-locks the Python requirements, when the repository has them. A
|
|
33
|
+
repository turns each language's updates off with `update`, and the long
|
|
34
|
+
nightly fuzz run with `security.fuzz`, in its pubrules.json (see
|
|
35
|
+
`settings`); the pub* pins always come, and the advisory checks every CI
|
|
36
|
+
run makes have no switch.
|
|
37
|
+
|
|
38
|
+
Each prints what it changed, one line per change, for the commit message
|
|
39
|
+
and the pull request.
|
|
40
|
+
"""
|
|
41
|
+
|
|
42
|
+
from __future__ import annotations
|
|
43
|
+
|
|
44
|
+
import contextlib
|
|
45
|
+
import functools
|
|
46
|
+
import io
|
|
47
|
+
import json
|
|
48
|
+
import os
|
|
49
|
+
import re
|
|
50
|
+
import subprocess
|
|
51
|
+
import sys
|
|
52
|
+
from pathlib import Path
|
|
53
|
+
|
|
54
|
+
from . import __version__
|
|
55
|
+
from .paths import confined
|
|
56
|
+
from .refs import (ANY_OWNER, AT, PUBRULES_PIN, RELEASE_PIN, TF_SOURCE, VERSION, owner_of, pubrel_commit, ref,
|
|
57
|
+
release_commit, repin, shipped, tagged)
|
|
58
|
+
|
|
59
|
+
# A row's text, which must not start with '-' (pubrel has no `--`, so it
|
|
60
|
+
# would read that as an option) or a space.
|
|
61
|
+
ROW = re.compile(r"[^-\s].*", re.S)
|
|
62
|
+
|
|
63
|
+
# The categories `pubrel add` takes.
|
|
64
|
+
CATEGORIES = ("changed", "removed", "added", "fixed", "security", "internal")
|
|
65
|
+
|
|
66
|
+
# RELEASE_PIN (refs.py) is pubrel's release workflow, by commit or by tag.
|
|
67
|
+
PUBREL_VERSION = re.compile(r"(pubrel-version:\s*v)(?P<version>[0-9][^\s,}'\"]*)")
|
|
68
|
+
# Every pubrules reusable workflow a repository can call, not only rust-ci.
|
|
69
|
+
CI_PIN = re.compile(ANY_OWNER + r"(?P<path>pubrules/\.github/workflows/"
|
|
70
|
+
r"(?:rust|python|node|service)-ci\.yml@)" + AT + VERSION)
|
|
71
|
+
# Any other repository's reusable workflow (.github/workflows/<name>) or
|
|
72
|
+
# action (actions/<name>) at a release, the repository read from the line
|
|
73
|
+
# itself, so pubrules names none of them. pubrules' and pubrel's move with
|
|
74
|
+
# PUBRULES_PIN and RELEASE_PIN; a deploy.providers package's with its entry,
|
|
75
|
+
# which its sustenaut.bump hook moves.
|
|
76
|
+
OTHER_PIN = re.compile(ANY_OWNER + r"(?P<repo>[A-Za-z0-9_.-]+)"
|
|
77
|
+
r"(?P<path>/(?:\.github/workflows/[\w.-]+\.ya?ml|actions/[\w.-]+)@)"
|
|
78
|
+
+ AT + VERSION)
|
|
79
|
+
RELEASE_TAG = re.compile(r"v[0-9]+\.[0-9]+\.[0-9]+")
|
|
80
|
+
# A Python package pinned by another repository's release wheel, in a requirements
|
|
81
|
+
# file or pyproject.toml: the version is in the tag and in the file's name.
|
|
82
|
+
WHEEL_PIN = re.compile(r"(?P<lead>(?P<name>[A-Za-z0-9][A-Za-z0-9_.-]*)(?:\[[^\]]*\])?\s*@\s*)https://github\.com/"
|
|
83
|
+
r"(?P<owner>[A-Za-z0-9][A-Za-z0-9-]*)/(?P<repo>[A-Za-z0-9_.-]+)/releases/download/"
|
|
84
|
+
r"v(?P<version>[0-9]+\.[0-9]+\.[0-9]+)/(?P<file>[A-Za-z0-9_.+-]+\.whl)")
|
|
85
|
+
# The command a `uv pip compile` lock was made with, in its header.
|
|
86
|
+
UV_COMPILE = re.compile(r"^#\s+(uv pip compile .*)$", re.M)
|
|
87
|
+
# A third-party action in a `uses:` line, pinned as Dependabot pins one: by
|
|
88
|
+
# the commit of a release's tag, with the version in a comment. OWNER/REPO,
|
|
89
|
+
# then any path inside it (actions/cache/restore), then `@COMMIT # vX.Y.Z`.
|
|
90
|
+
# A pin by tag, or to a branch (`# main`), or of a pre-release, is left.
|
|
91
|
+
ACTION_PIN = re.compile(r"(?P<lead>uses:[ \t]*['\"]?)" + ANY_OWNER + r"(?P<repo>[A-Za-z0-9_.-]+)"
|
|
92
|
+
r"(?P<path>(?:/[\w.-]+)*)@(?P<commit>[0-9a-f]{40})(?P<gap>[ \t]+#[ \t]*)"
|
|
93
|
+
r"v(?P<version>[0-9]+(?:\.[0-9]+)*)(?=[\s'\"]|$)", re.M)
|
|
94
|
+
# Directories a repository's own composite actions are never in.
|
|
95
|
+
NOT_OWN = ("node_modules", "target")
|
|
96
|
+
|
|
97
|
+
# The languages pubrules' settings name, each by the ecosystem pubrules works
|
|
98
|
+
# with: crates, npm packages, and the hash-locked Python requirements.
|
|
99
|
+
LANGUAGES = ("rust", "node", "python")
|
|
100
|
+
|
|
101
|
+
# What `update` and `security.fuzz` default to for each language: all on.
|
|
102
|
+
# `update` also takes `actions`, the third-party GitHub Actions the
|
|
103
|
+
# repository's workflows and composite actions use (see `actions`).
|
|
104
|
+
# Fuzzing is cargo-fuzz for Rust, and the property tests (Hypothesis,
|
|
105
|
+
# fast-check) for Python and Node; on, a scheduled run fuzzes for longer.
|
|
106
|
+
UPDATES = dict.fromkeys((*LANGUAGES, "actions"), True)
|
|
107
|
+
FUZZ = dict.fromkeys(LANGUAGES, True)
|
|
108
|
+
|
|
109
|
+
# The keys `security` takes (security.py reads dependabot and sonar). The
|
|
110
|
+
# advisory checks have none: they always run.
|
|
111
|
+
SECURITY = ("fuzz", "dependabot", "sonar")
|
|
112
|
+
|
|
113
|
+
|
|
114
|
+
def by_language(name: str, given: object, defaults: dict[str, bool]) -> dict[str, bool]:
|
|
115
|
+
"""A per-language switch: an object naming some languages, or one
|
|
116
|
+
boolean for all of them, over the defaults."""
|
|
117
|
+
if isinstance(given, bool):
|
|
118
|
+
return dict.fromkeys(defaults, given)
|
|
119
|
+
if not isinstance(given, dict):
|
|
120
|
+
sys.exit(f"pubrules: pubrules.json's {name} must be an object, such as "
|
|
121
|
+
'{"rust": true, "python": false}, or true or false for every language')
|
|
122
|
+
unknown = sorted(set(given) - set(defaults))
|
|
123
|
+
if unknown:
|
|
124
|
+
sys.exit(f"pubrules: pubrules.json's {name} has no language {', '.join(unknown)}; "
|
|
125
|
+
f"it takes {', '.join(defaults)}")
|
|
126
|
+
for key, value in given.items():
|
|
127
|
+
if not isinstance(value, bool):
|
|
128
|
+
sys.exit(f"pubrules: pubrules.json's {name}.{key} must be true or false, not {value!r}")
|
|
129
|
+
return {**defaults, **given}
|
|
130
|
+
|
|
131
|
+
|
|
132
|
+
def named_crates(config: dict) -> tuple[str, ...] | None:
|
|
133
|
+
"""The crates pubrules.json's `nightly.pins.crates` names, or None when it
|
|
134
|
+
names none. Anything else in the nightly block stops the command."""
|
|
135
|
+
from .ci import CRATE # noqa: PLC0415
|
|
136
|
+
block = config.get("nightly", {})
|
|
137
|
+
if not isinstance(block, dict):
|
|
138
|
+
sys.exit('pubrules: pubrules.json\'s nightly must be an object, such as '
|
|
139
|
+
'{"pins": {"crates": ["some-crate"]}}')
|
|
140
|
+
unknown = sorted(set(block) - {"pins"})
|
|
141
|
+
if unknown:
|
|
142
|
+
sys.exit(f"pubrules: pubrules.json's nightly has no setting {', '.join(unknown)}; "
|
|
143
|
+
"it takes pins")
|
|
144
|
+
pinned = block.get("pins", {})
|
|
145
|
+
if not isinstance(pinned, dict):
|
|
146
|
+
sys.exit('pubrules: pubrules.json\'s nightly.pins must be an object, such as '
|
|
147
|
+
'{"crates": ["some-crate"]}')
|
|
148
|
+
unknown = sorted(set(pinned) - {"crates"})
|
|
149
|
+
if unknown:
|
|
150
|
+
sys.exit(f"pubrules: pubrules.json's nightly.pins has no setting {', '.join(unknown)}; "
|
|
151
|
+
"it takes crates")
|
|
152
|
+
if "crates" not in pinned:
|
|
153
|
+
return None
|
|
154
|
+
names = pinned["crates"]
|
|
155
|
+
if not isinstance(names, list) or not all(isinstance(n, str) and CRATE.fullmatch(n)
|
|
156
|
+
for n in names):
|
|
157
|
+
sys.exit("pubrules: pubrules.json's nightly.pins.crates must list crate names, "
|
|
158
|
+
f"not {names!r}")
|
|
159
|
+
return tuple(names)
|
|
160
|
+
|
|
161
|
+
|
|
162
|
+
def pin_crates(root: Path) -> tuple[str, ...]:
|
|
163
|
+
"""The crates `pins` moves here rather than `crates`."""
|
|
164
|
+
path = root / "pubrules.json"
|
|
165
|
+
config = json.loads(path.read_text(encoding="utf-8")) if path.is_file() else {}
|
|
166
|
+
named = named_crates(config)
|
|
167
|
+
return named or ()
|
|
168
|
+
|
|
169
|
+
|
|
170
|
+
def settings(root: Path) -> dict[str, dict[str, bool]]:
|
|
171
|
+
"""The repository's `update` and `security.fuzz`, per language, from
|
|
172
|
+
pubrules.json over the defaults. Anything else stops the nightly."""
|
|
173
|
+
path = root / "pubrules.json"
|
|
174
|
+
config = json.loads(path.read_text(encoding="utf-8")) if path.is_file() else {}
|
|
175
|
+
security = config.get("security", {})
|
|
176
|
+
if not isinstance(security, dict):
|
|
177
|
+
sys.exit("pubrules: pubrules.json's security must be an object, such as "
|
|
178
|
+
'{"fuzz": {"rust": true}}')
|
|
179
|
+
unknown = sorted(set(security) - set(SECURITY))
|
|
180
|
+
if unknown:
|
|
181
|
+
sys.exit(f"pubrules: pubrules.json's security has no setting {', '.join(unknown)}; "
|
|
182
|
+
f"it takes {', '.join(SECURITY)} (the advisory checks always run)")
|
|
183
|
+
fuzz = by_language("security.fuzz", security.get("fuzz", {}), FUZZ)
|
|
184
|
+
from .security import dependabot_settings, sonar_settings # noqa: PLC0415
|
|
185
|
+
dependabot_settings(config)
|
|
186
|
+
sonar_settings(config)
|
|
187
|
+
named_crates(config)
|
|
188
|
+
return {"update": by_language("update", config.get("update", {}), UPDATES), "fuzz": fuzz}
|
|
189
|
+
|
|
190
|
+
|
|
191
|
+
def locked(root: Path) -> dict[str, str]:
|
|
192
|
+
"""Crate name -> version, from Cargo.lock (the highest when several)."""
|
|
193
|
+
lock = root / "Cargo.lock"
|
|
194
|
+
if not lock.is_file():
|
|
195
|
+
return {}
|
|
196
|
+
out: dict[str, str] = {}
|
|
197
|
+
for m in re.finditer(r'\[\[package\]\]\nname = "([^"]+)"\nversion = "([^"]+)"',
|
|
198
|
+
lock.read_text(encoding="utf-8")):
|
|
199
|
+
out[m.group(1)] = m.group(2)
|
|
200
|
+
return out
|
|
201
|
+
|
|
202
|
+
|
|
203
|
+
def cargo(root: Path, *args: str) -> subprocess.CompletedProcess:
|
|
204
|
+
return subprocess.run(["cargo", *args], cwd=root, capture_output=True, text=True, check=False)
|
|
205
|
+
|
|
206
|
+
|
|
207
|
+
def refresh_reports(root: Path) -> None:
|
|
208
|
+
"""Regenerate each published crate's SUPPLY-CHAIN.md, which names the
|
|
209
|
+
versions in Cargo.lock: an update otherwise leaves it stale."""
|
|
210
|
+
from .conform import packages # noqa: PLC0415
|
|
211
|
+
from .supply_chain import run as supply_chain # noqa: PLC0415
|
|
212
|
+
|
|
213
|
+
config = json.loads((root / "pubrules.json").read_text(encoding="utf-8"))
|
|
214
|
+
names = [p["name"] for p in packages(root, config)
|
|
215
|
+
if (Path(p["manifest_path"]).parent / "SUPPLY-CHAIN.md").is_file()]
|
|
216
|
+
if names:
|
|
217
|
+
supply_chain(root, names, check=False)
|
|
218
|
+
|
|
219
|
+
|
|
220
|
+
def record(root: Path, text: str, category: str = "internal") -> list[str]:
|
|
221
|
+
"""Record `text` for every package `pubrel check` says needs a row for the
|
|
222
|
+
committed change, and return the packages."""
|
|
223
|
+
if category not in CATEGORIES:
|
|
224
|
+
sys.exit(f"pubrules: pubrel has no category {category!r}; use one of "
|
|
225
|
+
f"{', '.join(CATEGORIES)}")
|
|
226
|
+
row = ROW.fullmatch(text)
|
|
227
|
+
if not row:
|
|
228
|
+
sys.exit(f"pubrules: the row's text {text!r} is empty or starts with a space or "
|
|
229
|
+
"'-', which pubrel would take for an option")
|
|
230
|
+
text = row.group(0)
|
|
231
|
+
out = subprocess.run(["pubrel", "check", "origin/main"], cwd=root, capture_output=True,
|
|
232
|
+
text=True, check=False)
|
|
233
|
+
hints = re.findall(r"run `pubrel add( --package ([A-Za-z0-9_][A-Za-z0-9_-]*))? CATEGORY",
|
|
234
|
+
out.stdout + out.stderr)
|
|
235
|
+
added = []
|
|
236
|
+
for flag, package in hints:
|
|
237
|
+
argv = ["pubrel", "add", *(["--package", package] if flag else []), category, text]
|
|
238
|
+
subprocess.run(argv, cwd=root, check=True, capture_output=True)
|
|
239
|
+
added.append(package or "(the package)")
|
|
240
|
+
return added
|
|
241
|
+
|
|
242
|
+
|
|
243
|
+
def released_by_config(root: Path, pubrel: str, changes: list[str]) -> bool:
|
|
244
|
+
"""Move the pubrel the repository's release job calls to `pubrel`; True
|
|
245
|
+
when it moved. It is ci.release.pubrel, or, where that names none, the
|
|
246
|
+
one the workflows sync wrote pin, which is then written there."""
|
|
247
|
+
path = root / "pubrules.json"
|
|
248
|
+
if not path.exists():
|
|
249
|
+
return False
|
|
250
|
+
config = json.loads(path.read_text(encoding="utf-8"))
|
|
251
|
+
ci = config.get("ci")
|
|
252
|
+
release = ci.get("release") if isinstance(ci, dict) else None
|
|
253
|
+
if isinstance(release, dict) and "pubrel" in release:
|
|
254
|
+
current = release["pubrel"]
|
|
255
|
+
elif release is False or (isinstance(release, dict) and "uses" in release) or not isinstance(ci, dict):
|
|
256
|
+
return False
|
|
257
|
+
else:
|
|
258
|
+
current = written_pubrel(root)
|
|
259
|
+
if current in (None, pubrel):
|
|
260
|
+
return False
|
|
261
|
+
changes.append(f"pubrules.json: ci.release.pubrel {current} -> {pubrel}")
|
|
262
|
+
ci["release"] = {**(release if isinstance(release, dict) else {}), "pubrel": pubrel}
|
|
263
|
+
path.write_text(json.dumps(config, indent=2) + "\n", encoding="utf-8")
|
|
264
|
+
return True
|
|
265
|
+
|
|
266
|
+
|
|
267
|
+
def written_pubrel(root: Path) -> str | None:
|
|
268
|
+
"""The pubrel the workflows sync wrote call, or None."""
|
|
269
|
+
from .ci import is_generated # noqa: PLC0415
|
|
270
|
+
for path in sorted((root / ".github" / "workflows").glob("*.y*ml")):
|
|
271
|
+
text = path.read_text(encoding="utf-8")
|
|
272
|
+
if is_generated(text) and (m := RELEASE_PIN.search(text)):
|
|
273
|
+
return m["version"]
|
|
274
|
+
return None
|
|
275
|
+
|
|
276
|
+
|
|
277
|
+
def latest_release(repo: str) -> str | None:
|
|
278
|
+
"""The tag of `repo`'s latest release, or None when GitHub does not say."""
|
|
279
|
+
try:
|
|
280
|
+
out = subprocess.run(["gh", "release", "view", "--repo", repo, "--json", "tagName",
|
|
281
|
+
"--jq", ".tagName"], capture_output=True, text=True, check=False)
|
|
282
|
+
except OSError:
|
|
283
|
+
return None
|
|
284
|
+
return out.stdout.strip() if out.returncode == 0 else None
|
|
285
|
+
|
|
286
|
+
|
|
287
|
+
def tools_by_config(root: Path, changes: list[str]) -> bool:
|
|
288
|
+
"""Move each tool pubrules.json's ci.tools pins by tag to the latest release
|
|
289
|
+
of its git repository; True when one moved. One whose latest release
|
|
290
|
+
cannot be read stays as it is, with a warning."""
|
|
291
|
+
from .ci import GITHUB_REPO, TOOL_TAG, tools # noqa: PLC0415
|
|
292
|
+
|
|
293
|
+
path = root / "pubrules.json"
|
|
294
|
+
if not path.is_file():
|
|
295
|
+
return False
|
|
296
|
+
config = json.loads(path.read_text(encoding="utf-8"))
|
|
297
|
+
listed = tools(config["ci"]) if isinstance(config.get("ci"), dict) else []
|
|
298
|
+
moved = False
|
|
299
|
+
for tool in listed:
|
|
300
|
+
m = GITHUB_REPO.fullmatch(tool["git"])
|
|
301
|
+
latest = latest_release(m["repo"]) if m else None
|
|
302
|
+
if not latest or not TOOL_TAG.fullmatch(latest):
|
|
303
|
+
print(f"pubrules: warning: no latest release of {tool['git']} to read, "
|
|
304
|
+
f"so {tool['crate']} stays at {tool['tag']}", file=sys.stderr)
|
|
305
|
+
elif latest != tool["tag"]:
|
|
306
|
+
changes.append(f"pubrules.json: ci.tools {tool['crate']} {tool['tag']} -> {latest}")
|
|
307
|
+
tool["tag"] = latest
|
|
308
|
+
moved = True
|
|
309
|
+
if moved:
|
|
310
|
+
config["ci"]["tools"] = listed
|
|
311
|
+
path.write_text(json.dumps(config, indent=2) + "\n", encoding="utf-8")
|
|
312
|
+
return moved
|
|
313
|
+
|
|
314
|
+
|
|
315
|
+
def release_assets(repo: str, tag: str) -> list[str] | None:
|
|
316
|
+
"""The file names release `tag` of `repo` carries, or None when GitHub does not say."""
|
|
317
|
+
try:
|
|
318
|
+
out = subprocess.run(["gh", "release", "view", tag, "--repo", repo, "--json", "assets",
|
|
319
|
+
"--jq", ".assets[].name"], capture_output=True, text=True, check=False)
|
|
320
|
+
except OSError:
|
|
321
|
+
return None
|
|
322
|
+
return out.stdout.split() if out.returncode == 0 else None
|
|
323
|
+
|
|
324
|
+
|
|
325
|
+
@functools.cache
|
|
326
|
+
def release_wheel(slug: str, version: str) -> str | None:
|
|
327
|
+
"""The file name of the pure-Python wheel release vVERSION of `slug`
|
|
328
|
+
carries (`<distribution>-VERSION-py3-none-any.whl`), or None when it
|
|
329
|
+
carries none or GitHub does not say. A repository's distribution may be
|
|
330
|
+
named otherwise than the repository (pubtest's is pubtestrules), so the
|
|
331
|
+
release itself says what its wheel is called."""
|
|
332
|
+
wheels = [name for name in release_assets(slug, f"v{version}") or []
|
|
333
|
+
if name.endswith(f"-{version}-py3-none-any.whl")]
|
|
334
|
+
return wheels[0] if len(wheels) == 1 else None
|
|
335
|
+
|
|
336
|
+
|
|
337
|
+
def distribution(file: str) -> str:
|
|
338
|
+
"""The distribution a wheel's file name names, as the file spells it."""
|
|
339
|
+
return file.split("-", 1)[0]
|
|
340
|
+
|
|
341
|
+
|
|
342
|
+
def same_name(a: str, b: str) -> bool:
|
|
343
|
+
"""Whether two distribution names are one (PEP 503: case, `-`, `_` and `.` alike)."""
|
|
344
|
+
return re.sub(r"[-_.]+", "-", a).lower() == re.sub(r"[-_.]+", "-", b).lower()
|
|
345
|
+
|
|
346
|
+
|
|
347
|
+
@functools.cache
|
|
348
|
+
def latest_tag(slug: str) -> str | None:
|
|
349
|
+
"""`slug`'s latest release tag, when it is vX.Y.Z; None, with a warning, when not."""
|
|
350
|
+
tag = latest_release(slug)
|
|
351
|
+
if tag and RELEASE_TAG.fullmatch(tag):
|
|
352
|
+
return tag
|
|
353
|
+
print(f"pubrules: warning: no latest release of {slug} to read, so its workflows and actions "
|
|
354
|
+
"stay where they are pinned", file=sys.stderr)
|
|
355
|
+
return None
|
|
356
|
+
|
|
357
|
+
|
|
358
|
+
def provided(root: Path) -> set[str]:
|
|
359
|
+
"""The repositories deploy.providers takes wheels from, by name in lower case."""
|
|
360
|
+
from .providers import WHEEL # noqa: PLC0415
|
|
361
|
+
path = root / "pubrules.json"
|
|
362
|
+
config = json.loads(path.read_text(encoding="utf-8")) if path.is_file() else {}
|
|
363
|
+
deploy = config.get("deploy") if isinstance(config, dict) else None
|
|
364
|
+
given = deploy.get("providers") if isinstance(deploy, dict) else None
|
|
365
|
+
return {m["repo"].split("/", 1)[1].lower() for p in given or [] if isinstance(p, dict)
|
|
366
|
+
and isinstance(p.get("wheel"), str) and (m := WHEEL.fullmatch(p["wheel"]))}
|
|
367
|
+
|
|
368
|
+
|
|
369
|
+
def others_by_release(text: str, name: str, skip: set[str], changes: list[str]) -> str:
|
|
370
|
+
"""`text` with each other repository's workflow pin (OTHER_PIN) moved to that
|
|
371
|
+
repository's latest release, by the commit of its tag; the moves said in
|
|
372
|
+
`changes`. One whose latest release cannot be read stays."""
|
|
373
|
+
def move(m: re.Match[str]) -> str:
|
|
374
|
+
if m["repo"].lower() in skip:
|
|
375
|
+
return m.group(0)
|
|
376
|
+
slug = f"{m['owner']}/{m['repo']}"
|
|
377
|
+
latest = latest_tag(slug)
|
|
378
|
+
if latest is None:
|
|
379
|
+
return m.group(0)
|
|
380
|
+
version = latest[1:]
|
|
381
|
+
commit = tagged(f"{os.environ.get('PUBRULES_GIT_BASE', 'https://github.com')}/{slug}", latest)
|
|
382
|
+
if version == m["version"] and (commit is None or commit == m["commit"]):
|
|
383
|
+
return m.group(0)
|
|
384
|
+
changes.append(f"{name}: {slug} {m['version']} -> {version}" if version != m["version"]
|
|
385
|
+
else f"{name}: {slug} {version} pinned to {commit}")
|
|
386
|
+
return f"{slug}{m['path']}{ref(version, commit)}"
|
|
387
|
+
return OTHER_PIN.sub(move, text)
|
|
388
|
+
|
|
389
|
+
|
|
390
|
+
def wheel_files(root: Path) -> list[Path]:
|
|
391
|
+
"""The files that can pin a release wheel: requirements files (*.txt, *.in
|
|
392
|
+
with "requirements" in the name) and pyproject.toml, outside dot-directories,
|
|
393
|
+
node_modules and target; never a lock, which follows its input, nor the
|
|
394
|
+
requirements pubrules sync writes."""
|
|
395
|
+
out = []
|
|
396
|
+
for path in sorted(root.rglob("*")):
|
|
397
|
+
rel = path.relative_to(root)
|
|
398
|
+
if any(part.startswith(".") or part in NOT_OWN for part in rel.parts[:-1]) or not path.is_file():
|
|
399
|
+
continue
|
|
400
|
+
base = path.name
|
|
401
|
+
if base == "pyproject.toml" or ("requirements" in base and base.endswith((".txt", ".in"))
|
|
402
|
+
and base != "requirements-pubrules.txt"):
|
|
403
|
+
out.append(path)
|
|
404
|
+
return out
|
|
405
|
+
|
|
406
|
+
|
|
407
|
+
def wheels_by_release(root: Path, skip: set[str], changes: list[str]) -> list[Path]:
|
|
408
|
+
"""Each release wheel pin (WHEEL_PIN) of another repository moved to that
|
|
409
|
+
repository's latest release, when that release is newer: the tag and the
|
|
410
|
+
file's version both. The moves said in `changes`; the files moved returned.
|
|
411
|
+
One whose latest release cannot be read, or is not newer, stays."""
|
|
412
|
+
moved_files = []
|
|
413
|
+
|
|
414
|
+
def move(m: re.Match[str], rel: str) -> str:
|
|
415
|
+
if m["repo"].lower() in skip or m["version"] not in m["file"]:
|
|
416
|
+
return m.group(0)
|
|
417
|
+
slug = f"{m['owner']}/{m['repo']}"
|
|
418
|
+
latest = latest_tag(slug)
|
|
419
|
+
if latest is None or release_key(latest[1:]) <= release_key(m["version"]):
|
|
420
|
+
return m.group(0)
|
|
421
|
+
version = latest[1:]
|
|
422
|
+
line = f"{rel}: {slug} {m['version']} -> {version}"
|
|
423
|
+
if line not in changes:
|
|
424
|
+
changes.append(line)
|
|
425
|
+
# The new release's wheel, by the name it carries: a distribution
|
|
426
|
+
# renamed since (pubtest's, as pubtestrules) takes the requirement's
|
|
427
|
+
# name with it. A release that says nothing keeps the old file's name.
|
|
428
|
+
file = release_wheel(slug, version) or m["file"].replace(m["version"], version, 1)
|
|
429
|
+
lead = m["lead"]
|
|
430
|
+
old, new = distribution(m["file"]), distribution(file)
|
|
431
|
+
if not same_name(old, new) and same_name(m["name"], old):
|
|
432
|
+
lead = new + lead[len(m["name"]):]
|
|
433
|
+
return f"{lead}https://github.com/{slug}/releases/download/v{version}/{file}"
|
|
434
|
+
|
|
435
|
+
for path in wheel_files(root):
|
|
436
|
+
text = path.read_text(encoding="utf-8")
|
|
437
|
+
rel = path.relative_to(root).as_posix()
|
|
438
|
+
new = WHEEL_PIN.sub(lambda m, rel=rel: move(m, rel), text)
|
|
439
|
+
if new != text:
|
|
440
|
+
path.write_text(new, encoding="utf-8")
|
|
441
|
+
moved_files.append(path)
|
|
442
|
+
return moved_files
|
|
443
|
+
|
|
444
|
+
|
|
445
|
+
def launchers_by_release(root: Path, changes: list[str]) -> bool:
|
|
446
|
+
"""The pubsec or pubdeploy a managed launcher fetches where deploy.providers
|
|
447
|
+
lists none (a service without a deploy block runs ./pubsec), moved to its
|
|
448
|
+
latest release, the wheel's sha256 with it; sync then writes the launcher
|
|
449
|
+
from that pin. True when one moved. One whose latest release cannot be
|
|
450
|
+
read, or is not newer, stays."""
|
|
451
|
+
from .providers import PUBDEPLOY, PUBSEC, WHEEL, in_launcher, listed # noqa: PLC0415
|
|
452
|
+
path = root / "pubrules.json"
|
|
453
|
+
config = json.loads(path.read_text(encoding="utf-8")) if path.is_file() else {}
|
|
454
|
+
moved = False
|
|
455
|
+
for name in (PUBSEC, PUBDEPLOY):
|
|
456
|
+
pin = in_launcher(root, name)
|
|
457
|
+
m = WHEEL.fullmatch(pin["wheel"]) if pin and listed(config, name) is None else None
|
|
458
|
+
if m is None:
|
|
459
|
+
continue
|
|
460
|
+
latest = latest_tag(m["repo"])
|
|
461
|
+
if latest is None or release_key(latest[1:]) <= release_key(m["version"]):
|
|
462
|
+
continue
|
|
463
|
+
from .cli import sha256_of # noqa: PLC0415
|
|
464
|
+
version = latest[1:]
|
|
465
|
+
wheel = pin["wheel"].replace(f"/v{m['version']}/", f"/{latest}/").replace(
|
|
466
|
+
f"-{m['version']}-", f"-{version}-")
|
|
467
|
+
launcher = root / name
|
|
468
|
+
launcher.write_text(launcher.read_text(encoding="utf-8").replace(
|
|
469
|
+
f"fetch {pin['wheel']} {pin['sha256']} ", f"fetch {wheel} {sha256_of(wheel)} "), encoding="utf-8")
|
|
470
|
+
changes.append(f"{name}: {m['repo']} {m['version']} -> {version}")
|
|
471
|
+
moved = True
|
|
472
|
+
return moved
|
|
473
|
+
|
|
474
|
+
|
|
475
|
+
def recompile(root: Path, moved: list[Path], changes: list[str]) -> None:
|
|
476
|
+
"""Each `uv pip compile` lock (its header names the command) whose input
|
|
477
|
+
moved, compiled again with that command, so its hashes are the new wheels'.
|
|
478
|
+
One that fails is said, and left for the pull request's CI to show."""
|
|
479
|
+
import shlex # noqa: PLC0415
|
|
480
|
+
names = {p.relative_to(root).as_posix() for p in moved}
|
|
481
|
+
for lock in sorted(root.rglob("*.lock")):
|
|
482
|
+
if any(part.startswith(".") or part in NOT_OWN for part in lock.relative_to(root).parts[:-1]):
|
|
483
|
+
continue
|
|
484
|
+
try:
|
|
485
|
+
found = UV_COMPILE.search(lock.read_text(encoding="utf-8")[:2000])
|
|
486
|
+
except (OSError, UnicodeDecodeError):
|
|
487
|
+
continue
|
|
488
|
+
if not found:
|
|
489
|
+
continue
|
|
490
|
+
argv = shlex.split(found.group(1))
|
|
491
|
+
here = lock.parent.relative_to(root)
|
|
492
|
+
if not any((here / a).as_posix() in names for a in argv if a.endswith((".in", ".txt", ".toml"))):
|
|
493
|
+
continue
|
|
494
|
+
done = subprocess.run(argv, cwd=lock.parent, capture_output=True, text=True, check=False)
|
|
495
|
+
rel = lock.relative_to(root).as_posix()
|
|
496
|
+
if done.returncode != 0:
|
|
497
|
+
print(f"pubrules: warning: {rel} could not be compiled again: "
|
|
498
|
+
f"{(done.stderr.strip().splitlines() or ['uv failed'])[-1]}", file=sys.stderr)
|
|
499
|
+
continue
|
|
500
|
+
changes.append(f"{rel}: compiled again")
|
|
501
|
+
|
|
502
|
+
|
|
503
|
+
def modules_by_release(root: Path, skip: set[str], changes: list[str]) -> None:
|
|
504
|
+
"""Each other repository's Terraform module source (TF_SOURCE) in the
|
|
505
|
+
repository's .tf files, moved to that repository's latest release by the
|
|
506
|
+
commit of its tag, with the version in a comment after the closing
|
|
507
|
+
quote; the moves said in `changes`. A source pinned by commit with no
|
|
508
|
+
version comment, or whose latest release cannot be read, stays."""
|
|
509
|
+
from .refs import tf_files # noqa: PLC0415
|
|
510
|
+
|
|
511
|
+
def move(m: re.Match[str], name: str) -> str:
|
|
512
|
+
current = m["tag"] or m["version"]
|
|
513
|
+
if m["repo"].lower() in skip or current is None:
|
|
514
|
+
return m.group(0)
|
|
515
|
+
slug = f"{m['owner']}/{m['repo']}"
|
|
516
|
+
latest = latest_tag(slug)
|
|
517
|
+
if latest is None:
|
|
518
|
+
return m.group(0)
|
|
519
|
+
version = latest[1:]
|
|
520
|
+
commit = tagged(f"{os.environ.get('PUBRULES_GIT_BASE', 'https://github.com')}/{slug}", latest)
|
|
521
|
+
if version == current and (commit is None or commit == m["commit"]):
|
|
522
|
+
return m.group(0)
|
|
523
|
+
changes.append(f"{name}: {slug} {current} -> {version}" if version != current
|
|
524
|
+
else f"{name}: {slug} {version} pinned to {commit}")
|
|
525
|
+
at = f'{commit}" # v{version}' if commit else f'v{version}"'
|
|
526
|
+
return f'{m["lead"]}"github.com/{slug}//{m["path"]}?ref={at}'
|
|
527
|
+
|
|
528
|
+
for path in tf_files(root):
|
|
529
|
+
text = path.read_text(encoding="utf-8")
|
|
530
|
+
rel = path.relative_to(root).as_posix()
|
|
531
|
+
new = TF_SOURCE.sub(lambda m, rel=rel: move(m, rel), text)
|
|
532
|
+
if new != text:
|
|
533
|
+
path.write_text(new, encoding="utf-8")
|
|
534
|
+
|
|
535
|
+
|
|
536
|
+
def action_files(root: Path) -> list[Path]:
|
|
537
|
+
"""The repository's hand-written workflows and its own composite actions
|
|
538
|
+
(action.yml anywhere outside dot-directories other than .github, and
|
|
539
|
+
outside node_modules and target)."""
|
|
540
|
+
workflows = sorted((root / ".github" / "workflows").glob("*.y*ml"))
|
|
541
|
+
own = sorted(p for p in root.rglob("action.y*ml")
|
|
542
|
+
if not any((part.startswith(".") and part != ".github") or part in NOT_OWN
|
|
543
|
+
for part in p.relative_to(root).parts[:-1]))
|
|
544
|
+
return workflows + [p for p in own if p not in workflows]
|
|
545
|
+
|
|
546
|
+
|
|
547
|
+
def release_key(version: str) -> tuple[int, ...]:
|
|
548
|
+
return tuple(int(n) for n in version.split("."))
|
|
549
|
+
|
|
550
|
+
|
|
551
|
+
@functools.cache
|
|
552
|
+
def tag_commit(slug: str, tag: str) -> str | None:
|
|
553
|
+
"""The commit `slug`'s tag names, or None. `tagged` would warn that it
|
|
554
|
+
pins by tag; `actions` never does, and says so itself."""
|
|
555
|
+
with contextlib.redirect_stderr(io.StringIO()):
|
|
556
|
+
return tagged(f"{os.environ.get('PUBRULES_GIT_BASE', 'https://github.com')}/{slug}", tag)
|
|
557
|
+
|
|
558
|
+
|
|
559
|
+
def actions(root: Path) -> list[str]:
|
|
560
|
+
"""Each third-party action (ACTION_PIN) in the repository's hand-written
|
|
561
|
+
workflows and composite actions moved to its repository's latest
|
|
562
|
+
release, by the commit of its tag, with the version in a comment: what
|
|
563
|
+
Dependabot's version updates did, but on the nightly's branch, with a
|
|
564
|
+
pubrel row and the nightly's token. One line per file and repository
|
|
565
|
+
moved. pubrules' and pubrel's pins, and a deploy.providers package's,
|
|
566
|
+
move with `pins`, as does any OWNER/REPO/actions/NAME or reusable
|
|
567
|
+
workflow (OTHER_PIN). An action only moves forward, and one whose
|
|
568
|
+
latest release or tag's commit cannot be read stays, with a warning.
|
|
569
|
+
Workflows `pubrules sync` writes move with pubrules."""
|
|
570
|
+
from .ci import is_generated # noqa: PLC0415
|
|
571
|
+
|
|
572
|
+
skip = {"pubrules", "pubrel"} | provided(root)
|
|
573
|
+
changes: list[str] = []
|
|
574
|
+
for path in action_files(root):
|
|
575
|
+
text = path.read_text(encoding="utf-8")
|
|
576
|
+
if is_generated(text):
|
|
577
|
+
continue
|
|
578
|
+
rel = path.relative_to(root).as_posix()
|
|
579
|
+
|
|
580
|
+
def move(m: re.Match[str], rel: str = rel) -> str:
|
|
581
|
+
if m["repo"].lower() in skip or m["path"].startswith(("/actions/", "/.github/workflows/")):
|
|
582
|
+
return m.group(0)
|
|
583
|
+
slug = f"{m['owner']}/{m['repo']}"
|
|
584
|
+
latest = latest_tag(slug)
|
|
585
|
+
if latest is None or release_key(latest[1:]) <= release_key(m["version"]):
|
|
586
|
+
return m.group(0)
|
|
587
|
+
commit = tag_commit(slug, latest)
|
|
588
|
+
if commit is None:
|
|
589
|
+
print(f"pubrules: warning: could not read the commit of {slug}'s {latest}, so "
|
|
590
|
+
f"{rel} stays at {m['version']}", file=sys.stderr)
|
|
591
|
+
return m.group(0)
|
|
592
|
+
line = f"{rel}: {slug} {m['version']} -> {latest[1:]}"
|
|
593
|
+
if line not in changes:
|
|
594
|
+
changes.append(line)
|
|
595
|
+
return f"{m['lead']}{slug}{m['path']}@{commit}{m['gap']}{latest}"
|
|
596
|
+
|
|
597
|
+
new = ACTION_PIN.sub(move, text)
|
|
598
|
+
if new != text:
|
|
599
|
+
path.write_text(new, encoding="utf-8")
|
|
600
|
+
return changes
|
|
601
|
+
|
|
602
|
+
|
|
603
|
+
def pins(root: Path, pubrel: str, lock: bool = True) -> list[str]:
|
|
604
|
+
from .freshness import pinned, pinned_package # noqa: PLC0415
|
|
605
|
+
|
|
606
|
+
changes = []
|
|
607
|
+
# What installed packages move, as `pubrules bump` has them do: pubdeploy's
|
|
608
|
+
# moves deploy.providers (pubsec and pubdeploy among them) and the uses of
|
|
609
|
+
# pubdeploy's workflows, before sync writes the launchers and the tests'
|
|
610
|
+
# requirements from them. A failing hook is said, and moves nothing.
|
|
611
|
+
from .hooks import bump as bump_hooks # noqa: PLC0415
|
|
612
|
+
moved = []
|
|
613
|
+
for line in bump_hooks(root):
|
|
614
|
+
if line.startswith("updated"):
|
|
615
|
+
moved.append(line.removeprefix("updated").strip())
|
|
616
|
+
else:
|
|
617
|
+
print(line, file=sys.stderr)
|
|
618
|
+
changes += moved
|
|
619
|
+
package = (pinned_package(root) or ("pubrules", None))[0]
|
|
620
|
+
before = pinned(root)
|
|
621
|
+
if before and RELEASE_TAG.fullmatch(f"v{before}") and release_key(before) > release_key(__version__):
|
|
622
|
+
# Never backwards: sync would write this older pubrules' managed files.
|
|
623
|
+
print(f"pubrules: warning: this repository runs pubrules {before}, newer than this {__version__}, "
|
|
624
|
+
"so pubrules' managed files and lock stay", file=sys.stderr)
|
|
625
|
+
before, moved = __version__, []
|
|
626
|
+
if (before and before != __version__) or moved:
|
|
627
|
+
from .cli import lock as lock_requirements, sync # noqa: PLC0415
|
|
628
|
+
sync(root, check=False)
|
|
629
|
+
if lock:
|
|
630
|
+
lock_requirements(root)
|
|
631
|
+
if before and before != __version__:
|
|
632
|
+
# A rule this pubrules brings that the repository fails is waived,
|
|
633
|
+
# for a while, by the move that brings it.
|
|
634
|
+
from .gherkin import grandfather # noqa: PLC0415
|
|
635
|
+
changes += grandfather(root, package, before)
|
|
636
|
+
if before and before != __version__:
|
|
637
|
+
changes.append(f"pubrules {before} -> {__version__}")
|
|
638
|
+
released = bool(pubrel) and released_by_config(root, pubrel, changes)
|
|
639
|
+
launched = launchers_by_release(root, changes)
|
|
640
|
+
if tools_by_config(root, changes) or released or launched:
|
|
641
|
+
from .cli import sync # noqa: PLC0415
|
|
642
|
+
sync(root, check=False)
|
|
643
|
+
from .ci import is_generated # noqa: PLC0415
|
|
644
|
+
calls = owner_of(root)
|
|
645
|
+
skip = {"pubrules", "pubrel"} | provided(root)
|
|
646
|
+
for path in sorted((root / ".github" / "workflows").glob("*.y*ml")):
|
|
647
|
+
text = path.read_text(encoding="utf-8")
|
|
648
|
+
if is_generated(text): # pubrules sync writes these, from pubrules.json
|
|
649
|
+
continue
|
|
650
|
+
new = text
|
|
651
|
+
for pattern, version, what in ((PUBRULES_PIN, __version__, "pubrules"),
|
|
652
|
+
(RELEASE_PIN, pubrel, "pubrel release workflow"),
|
|
653
|
+
(PUBREL_VERSION, pubrel, "pubrel-version")):
|
|
654
|
+
if not version or not pattern.search(new):
|
|
655
|
+
continue
|
|
656
|
+
if pattern is PUBREL_VERSION:
|
|
657
|
+
commit = None
|
|
658
|
+
moved = pattern.sub(rf"\g<1>{version}", new)
|
|
659
|
+
else:
|
|
660
|
+
# A workflow's pin: the commit moves with the version. A pin
|
|
661
|
+
# of a pubrules action or workflow this pubrules does not ship
|
|
662
|
+
# stays on the commit that has it; sync --check says so.
|
|
663
|
+
commit = (pubrel_commit(version, new, calls) if pattern is RELEASE_PIN
|
|
664
|
+
else release_commit())
|
|
665
|
+
moved = repin(pattern, new, calls, version, commit,
|
|
666
|
+
keep=(lambda m: not shipped(m["name"])) if pattern is PUBRULES_PIN else None)
|
|
667
|
+
for m in pattern.finditer(new):
|
|
668
|
+
if pattern is PUBRULES_PIN and not shipped(m["name"]):
|
|
669
|
+
changes.append(f"{path.name}: {m['name']} stays at pubrules {m['version']}: "
|
|
670
|
+
f"pubrules {version} has no {m['name']}")
|
|
671
|
+
continue
|
|
672
|
+
# "pubrules A -> B" already says that pubrules' own pins moved.
|
|
673
|
+
if m["version"] != version and pattern is not PUBRULES_PIN:
|
|
674
|
+
changes.append(f"{path.name}: {what} {m['version']} -> {version}")
|
|
675
|
+
elif m["version"] == version and commit and m["commit"] != commit:
|
|
676
|
+
changes.append(f"{path.name}: {what} {version} pinned to {commit}")
|
|
677
|
+
new = moved
|
|
678
|
+
new = others_by_release(new, path.name, skip, changes)
|
|
679
|
+
if new != text:
|
|
680
|
+
path.write_text(new, encoding="utf-8")
|
|
681
|
+
modules_by_release(root, skip, changes)
|
|
682
|
+
# deploy.providers' wheels too: a hand-written requirement of one keeps up with
|
|
683
|
+
# the providers entry its package's hook moved. pubrules' are sync's to write.
|
|
684
|
+
wheels = wheels_by_release(root, {"pubrules"}, changes)
|
|
685
|
+
if wheels and lock:
|
|
686
|
+
from .cli import REQUIREMENTS, lock as lock_requirements # noqa: PLC0415
|
|
687
|
+
if (root / REQUIREMENTS).is_file():
|
|
688
|
+
with contextlib.redirect_stdout(io.StringIO()):
|
|
689
|
+
lock_requirements(root)
|
|
690
|
+
recompile(root, wheels, changes)
|
|
691
|
+
old = locked(root)
|
|
692
|
+
moving = pin_crates(root)
|
|
693
|
+
for crate in moving:
|
|
694
|
+
if crate in old:
|
|
695
|
+
cargo(root, "update", "--quiet", "-p", crate)
|
|
696
|
+
new_lock = locked(root)
|
|
697
|
+
changes += [f"{c} {old[c]} -> {new_lock[c]}" for c in moving
|
|
698
|
+
if c in old and new_lock.get(c) != old[c]]
|
|
699
|
+
if new_lock != old:
|
|
700
|
+
refresh_reports(root)
|
|
701
|
+
return changes
|
|
702
|
+
|
|
703
|
+
|
|
704
|
+
UPDATE = re.compile(r"^\s*Updating (\S+) v(\S+) -> v(\S+)", re.M)
|
|
705
|
+
|
|
706
|
+
|
|
707
|
+
def npm_locked(root: Path) -> dict[str, str]:
|
|
708
|
+
"""Direct dependency -> version, from package-lock.json."""
|
|
709
|
+
lock = root / "package-lock.json"
|
|
710
|
+
if not lock.is_file():
|
|
711
|
+
return {}
|
|
712
|
+
packages = json.loads(lock.read_text(encoding="utf-8")).get("packages", {})
|
|
713
|
+
top = packages.get("", {})
|
|
714
|
+
names = {**top.get("dependencies", {}), **top.get("devDependencies", {})}
|
|
715
|
+
return {n: packages[f"node_modules/{n}"]["version"] for n in sorted(names)
|
|
716
|
+
if "version" in packages.get(f"node_modules/{n}", {})}
|
|
717
|
+
|
|
718
|
+
|
|
719
|
+
def changed(old: dict[str, str], new: dict[str, str]) -> list[str]:
|
|
720
|
+
"""One line per package whose version moved, in name order."""
|
|
721
|
+
return [f"{n} {old[n]} -> {new[n]}" for n in sorted(old) if n in new and new[n] != old[n]]
|
|
722
|
+
|
|
723
|
+
|
|
724
|
+
def npm(root: Path) -> list[str]:
|
|
725
|
+
"""Every npm package updated to the newest version its range in
|
|
726
|
+
package.json allows, in package-lock.json only and without running an
|
|
727
|
+
install script. The direct dependencies that moved, one line each."""
|
|
728
|
+
if not (root / "package-lock.json").is_file():
|
|
729
|
+
return []
|
|
730
|
+
old = npm_locked(root)
|
|
731
|
+
subprocess.run(["npm", "update", "--package-lock-only", "--ignore-scripts",
|
|
732
|
+
"--no-audit", "--no-fund"], cwd=root, check=True, capture_output=True)
|
|
733
|
+
return changed(old, npm_locked(root)) or (["indirect dependencies"]
|
|
734
|
+
if git(root, "status", "--porcelain") else [])
|
|
735
|
+
|
|
736
|
+
|
|
737
|
+
def python_locked(root: Path) -> dict[str, str]:
|
|
738
|
+
"""Package -> version, from tests/requirements.lock."""
|
|
739
|
+
from .cli import LOCK # noqa: PLC0415
|
|
740
|
+
|
|
741
|
+
lock = root / LOCK
|
|
742
|
+
if not lock.is_file():
|
|
743
|
+
return {}
|
|
744
|
+
return dict(re.findall(r"^([A-Za-z0-9_.-]+)==(\S+)", lock.read_text(encoding="utf-8"), re.M))
|
|
745
|
+
|
|
746
|
+
|
|
747
|
+
def python(root: Path) -> list[str]:
|
|
748
|
+
"""tests/requirements.lock re-resolved, so each package moves to the
|
|
749
|
+
newest version requirements.txt allows. The packages that moved."""
|
|
750
|
+
from .cli import LOCK, REQUIREMENTS, lock # noqa: PLC0415
|
|
751
|
+
|
|
752
|
+
if not (root / REQUIREMENTS).is_file() or not (root / LOCK).is_file():
|
|
753
|
+
return []
|
|
754
|
+
old = python_locked(root)
|
|
755
|
+
with contextlib.redirect_stdout(io.StringIO()):
|
|
756
|
+
lock(root)
|
|
757
|
+
return changed(old, python_locked(root))
|
|
758
|
+
|
|
759
|
+
|
|
760
|
+
def candidates(root: Path) -> list[tuple[str, str, str]]:
|
|
761
|
+
"""(crate, from, to) for every upgrade `cargo update` would make, the
|
|
762
|
+
crates `pins` moves aside. Downgrades (which the MSRV-aware resolver
|
|
763
|
+
proposes when a locked version needs a newer Rust) are not taken."""
|
|
764
|
+
return upgrades(cargo(root, "update", "--dry-run").stderr, pin_crates(root))
|
|
765
|
+
|
|
766
|
+
|
|
767
|
+
def upgrades(dry_run: str, pinned: tuple[str, ...] = ()) -> list[tuple[str, str, str]]:
|
|
768
|
+
"""The upgrades in `cargo update --dry-run`'s output, `pinned` aside."""
|
|
769
|
+
return [(m.group(1), m.group(2), m.group(3)) for m in UPDATE.finditer(dry_run)
|
|
770
|
+
if m.group(1) not in pinned]
|
|
771
|
+
|
|
772
|
+
|
|
773
|
+
def recommended(vet_output: str) -> str:
|
|
774
|
+
"""What cargo vet recommends auditing, from a failing run's output."""
|
|
775
|
+
lines = [line.rstrip() for line in vet_output.splitlines() if line.strip()]
|
|
776
|
+
at = next((i for i, line in enumerate(lines) if "recommended audits" in line), None)
|
|
777
|
+
return "\n".join(lines[at:] if at is not None else lines[-8:])
|
|
778
|
+
|
|
779
|
+
|
|
780
|
+
def failing(vet_output: str) -> list[str]:
|
|
781
|
+
"""The crates `cargo vet` reports missing audits for."""
|
|
782
|
+
return sorted({m.group(1) for m in re.finditer(r"^\s*([A-Za-z0-9_-]+):\S+ missing",
|
|
783
|
+
vet_output, re.M)})
|
|
784
|
+
|
|
785
|
+
|
|
786
|
+
def has(root: Path, crate: str, version: str) -> bool:
|
|
787
|
+
lock = root / "Cargo.lock"
|
|
788
|
+
return lock.is_file() and f'name = "{crate}"\nversion = "{version}"' in lock.read_text(
|
|
789
|
+
encoding="utf-8")
|
|
790
|
+
|
|
791
|
+
|
|
792
|
+
def snapshot(root: Path) -> dict[Path, bytes]:
|
|
793
|
+
"""Cargo.lock and supply-chain/, to restore if the updates do not pass."""
|
|
794
|
+
paths = [root / "Cargo.lock", *sorted((root / "supply-chain").glob("*"))]
|
|
795
|
+
return {p: p.read_bytes() for p in paths if p.is_file()}
|
|
796
|
+
|
|
797
|
+
|
|
798
|
+
def restore(saved: dict[Path, bytes]) -> None:
|
|
799
|
+
for path, data in saved.items():
|
|
800
|
+
path.write_bytes(data)
|
|
801
|
+
|
|
802
|
+
|
|
803
|
+
def own_packages(root: Path) -> set[str]:
|
|
804
|
+
meta = json.loads(cargo(root, "metadata", "--no-deps", "--format-version", "1").stdout
|
|
805
|
+
or '{"packages": []}')
|
|
806
|
+
return {p["name"] for p in meta["packages"]}
|
|
807
|
+
|
|
808
|
+
|
|
809
|
+
def first_party_policy(root: Path, vet_output: str) -> list[str]:
|
|
810
|
+
"""When cargo vet stops because the repository's own crates now also
|
|
811
|
+
exist on crates.io, record that they are not audited as crates.io
|
|
812
|
+
crates: `[policy.NAME] audit-as-crates-io = false`. Returns the names."""
|
|
813
|
+
if "audit-as-crates-io" not in vet_output:
|
|
814
|
+
return []
|
|
815
|
+
named = set(re.findall(r"^[ \t]*(?:│[ \t]*)?([A-Za-z0-9_-]+):\S+[ \t]*$", vet_output, re.M))
|
|
816
|
+
own = sorted(named & own_packages(root))
|
|
817
|
+
config = root / "supply-chain" / "config.toml"
|
|
818
|
+
if not own or not config.is_file():
|
|
819
|
+
return []
|
|
820
|
+
text = config.read_text(encoding="utf-8").rstrip("\n")
|
|
821
|
+
for name in own:
|
|
822
|
+
text += (f"\n\n[policy.{name}]\naudit-as-crates-io = false\n"
|
|
823
|
+
if f"[policy.{name}]" not in text else "")
|
|
824
|
+
config.write_text(text.rstrip("\n") + "\n", encoding="utf-8")
|
|
825
|
+
return own
|
|
826
|
+
|
|
827
|
+
|
|
828
|
+
def tail(proc: subprocess.CompletedProcess, n: int = 8) -> str:
|
|
829
|
+
return "\n".join((proc.stdout + proc.stderr).strip().splitlines()[-n:])
|
|
830
|
+
|
|
831
|
+
|
|
832
|
+
def crates(root: Path, report: Path | None) -> int:
|
|
833
|
+
"""Try each upgrade on its own, on top of those already kept, and keep
|
|
834
|
+
it only when `cargo vet` passes without a new exemption and `cargo deny`
|
|
835
|
+
passes. A repository with no Cargo.lock has no crates to update."""
|
|
836
|
+
target = confined(report, "the --report file") if report else None
|
|
837
|
+
lines, skipped = upgrade(root) if (root / "Cargo.lock").is_file() else ([], [])
|
|
838
|
+
for line in lines:
|
|
839
|
+
print(line)
|
|
840
|
+
if target:
|
|
841
|
+
target.write_text(json.dumps({"updated": lines, "skipped": skipped}, indent=2) + "\n",
|
|
842
|
+
encoding="utf-8")
|
|
843
|
+
return 0
|
|
844
|
+
|
|
845
|
+
|
|
846
|
+
def upgrade(root: Path) -> tuple[list[str], list[dict]]:
|
|
847
|
+
"""The crate updates `crates` keeps, and those it skips with why."""
|
|
848
|
+
saved = snapshot(root)
|
|
849
|
+
notes: list[str] = []
|
|
850
|
+
updated: list[tuple[str, str, str]] = []
|
|
851
|
+
skipped: list[dict] = []
|
|
852
|
+
todo = candidates(root)
|
|
853
|
+
|
|
854
|
+
vet = cargo(root, "vet")
|
|
855
|
+
own = first_party_policy(root, vet.stdout + vet.stderr) if vet.returncode else []
|
|
856
|
+
if own:
|
|
857
|
+
notes.append("supply-chain/config.toml: " + ", ".join(own) + " (this repository's own) "
|
|
858
|
+
"not audited as crates.io crates, now that they are published there")
|
|
859
|
+
vet = cargo(root, "vet")
|
|
860
|
+
deny = cargo(root, "deny", "--locked", "check", "--hide-inclusion-graph")
|
|
861
|
+
if vet.returncode or deny.returncode:
|
|
862
|
+
restore(saved)
|
|
863
|
+
why = "main itself does not pass " + ("cargo vet" if vet.returncode else "cargo deny") + \
|
|
864
|
+
":\n" + tail(vet if vet.returncode else deny)
|
|
865
|
+
skipped = [{"crate": c, "from": a, "to": b, "why": why} for c, a, b in todo]
|
|
866
|
+
todo, notes = [], []
|
|
867
|
+
|
|
868
|
+
for crate, old, new in todo:
|
|
869
|
+
if has(root, crate, new):
|
|
870
|
+
# Another kept update brought it along, and vet and deny passed.
|
|
871
|
+
updated.append((crate, old, new))
|
|
872
|
+
continue
|
|
873
|
+
if not has(root, crate, old):
|
|
874
|
+
continue
|
|
875
|
+
before = snapshot(root)
|
|
876
|
+
step = cargo(root, "update", "--quiet", "-p", f"{crate}@{old}", "--precise", new)
|
|
877
|
+
why = ""
|
|
878
|
+
if step.returncode:
|
|
879
|
+
why = "cargo updates it only together with other crates:\n" + tail(step)
|
|
880
|
+
else:
|
|
881
|
+
vet = cargo(root, "vet")
|
|
882
|
+
if vet.returncode:
|
|
883
|
+
why = "not covered by an audit; cargo vet recommends:\n" + recommended(
|
|
884
|
+
vet.stdout + vet.stderr)
|
|
885
|
+
else:
|
|
886
|
+
# The advisory database was fetched above: --frozen is
|
|
887
|
+
# --locked, offline.
|
|
888
|
+
deny = cargo(root, "deny", "--frozen", "check", "--hide-inclusion-graph")
|
|
889
|
+
if deny.returncode:
|
|
890
|
+
why = "cargo deny fails with it:\n" + tail(deny)
|
|
891
|
+
if why:
|
|
892
|
+
restore(before)
|
|
893
|
+
skipped.append({"crate": crate, "from": old, "to": new, "why": why})
|
|
894
|
+
else:
|
|
895
|
+
updated.append((crate, old, new))
|
|
896
|
+
|
|
897
|
+
if updated and (cargo(root, "vet", "--locked").returncode
|
|
898
|
+
or cargo(root, "deny", "--locked", "check").returncode):
|
|
899
|
+
# Each passed on its own but not all together: take none.
|
|
900
|
+
restore(saved)
|
|
901
|
+
skipped += [{"crate": c, "from": a, "to": b,
|
|
902
|
+
"why": "the updates together did not pass cargo vet and cargo deny"}
|
|
903
|
+
for c, a, b in updated]
|
|
904
|
+
updated, notes = [], []
|
|
905
|
+
if not updated and not notes:
|
|
906
|
+
restore(saved)
|
|
907
|
+
if updated:
|
|
908
|
+
refresh_reports(root)
|
|
909
|
+
return notes + [f"{c} {a} -> {b}" for c, a, b in updated], skipped
|
|
910
|
+
|
|
911
|
+
|
|
912
|
+
def git(root: Path, *args: str) -> str:
|
|
913
|
+
return subprocess.run(["git", *args], cwd=root, capture_output=True, text=True,
|
|
914
|
+
check=True).stdout
|
|
915
|
+
|
|
916
|
+
|
|
917
|
+
def commit(root: Path, title: str, lines: list[str]) -> bool:
|
|
918
|
+
"""Commit everything changed as `title`, with `lines` as its body and an
|
|
919
|
+
internal pubrel row for each package that needs one. False when nothing
|
|
920
|
+
changed."""
|
|
921
|
+
if not git(root, "status", "--porcelain"):
|
|
922
|
+
return False
|
|
923
|
+
who = ["-c", "user.name=pubrules nightly", "-c", "user.email=noreply@github.com"]
|
|
924
|
+
git(root, "add", "-A")
|
|
925
|
+
git(root, *who, "commit", "-q", "-m", title, "-m", "\n".join(lines))
|
|
926
|
+
if record(root, title):
|
|
927
|
+
git(root, "add", "-A")
|
|
928
|
+
git(root, *who, "commit", "-q", "--amend", "--no-edit")
|
|
929
|
+
return True
|
|
930
|
+
|
|
931
|
+
|
|
932
|
+
def body(pinned: list[str], updated: list[str], skipped: list[dict],
|
|
933
|
+
others: dict[str, list[str]] | None = None, update: bool = True) -> str:
|
|
934
|
+
"""The pull request's body. `others` is each further section's lines,
|
|
935
|
+
by heading; with `update` off, the body says the repository takes no
|
|
936
|
+
third-party updates instead of listing them."""
|
|
937
|
+
out = ["Opened by pubrules' nightly workflow, which updates this pull request each night "
|
|
938
|
+
"rather than opening another. Nothing merges it: merge it when CI is green.", ""]
|
|
939
|
+
out += ["## 1. pub* pins", ""] + ([f"- {line}" for line in pinned] or ["Nothing outdated."])
|
|
940
|
+
if not update:
|
|
941
|
+
out += ["", "## Other dependencies", "",
|
|
942
|
+
"None: this repository takes no third-party updates; its pubrules.json sets `\"update\": false`."]
|
|
943
|
+
return "\n".join(out) + "\n"
|
|
944
|
+
number = 2
|
|
945
|
+
if updated or skipped:
|
|
946
|
+
out += ["", f"## {number}. Other crates", "",
|
|
947
|
+
"Updates within their requirements that `cargo vet` covers with an audit "
|
|
948
|
+
"(no new exemption) and `cargo deny` passes.", ""]
|
|
949
|
+
out += [f"- {line}" for line in updated] or ["None."]
|
|
950
|
+
number += 1
|
|
951
|
+
for heading, lines in (others or {}).items():
|
|
952
|
+
if lines:
|
|
953
|
+
out += ["", f"## {number}. {heading}", ""] + [f"- {line}" for line in lines]
|
|
954
|
+
number += 1
|
|
955
|
+
if skipped:
|
|
956
|
+
out += ["", "### Skipped", "",
|
|
957
|
+
"Each needs an audit before the nightly takes it. cargo vet recommends, as `cargo vet suggest` would:", ""]
|
|
958
|
+
for s in skipped:
|
|
959
|
+
out += [f"<details><summary>{s['crate']} {s['from']} -> {s['to']}</summary>", "",
|
|
960
|
+
"```", s["why"], "```", "", "</details>", ""]
|
|
961
|
+
return "\n".join(out).rstrip("\n") + "\n"
|
|
962
|
+
|
|
963
|
+
|
|
964
|
+
def nightly(root: Path, pubrel: str, body_path: Path) -> int:
|
|
965
|
+
"""The nightly's commits on the current branch, and the pull request's
|
|
966
|
+
body. Prints the commits it made, in order: `pins` first when it made
|
|
967
|
+
that one, then any of `actions`, `crates`, `npm` and `python`, or nothing."""
|
|
968
|
+
import tempfile # noqa: PLC0415
|
|
969
|
+
|
|
970
|
+
body_path = confined(body_path, "the --body file")
|
|
971
|
+
update = settings(root)["update"]
|
|
972
|
+
pinned = pins(root, pubrel)
|
|
973
|
+
made = ["pins"] if commit(root, "Move to the latest pub* releases", pinned) else []
|
|
974
|
+
result: dict = {"updated": [], "skipped": []}
|
|
975
|
+
others: dict[str, list[str]] = {}
|
|
976
|
+
if update["actions"]:
|
|
977
|
+
heading = "GitHub Actions, to their latest releases"
|
|
978
|
+
others[heading] = actions(root)
|
|
979
|
+
if commit(root, "Move GitHub Actions to their latest releases", others[heading]):
|
|
980
|
+
made.append("actions")
|
|
981
|
+
if update["rust"]:
|
|
982
|
+
with tempfile.TemporaryDirectory() as tmp:
|
|
983
|
+
report = Path(tmp) / "crates.json"
|
|
984
|
+
with contextlib.redirect_stdout(io.StringIO()):
|
|
985
|
+
crates(root, report)
|
|
986
|
+
result = json.loads(report.read_text(encoding="utf-8"))
|
|
987
|
+
if commit(root, "Update other crates the supply-chain audits cover", result["updated"]):
|
|
988
|
+
made.append("crates")
|
|
989
|
+
for language, heading, update_with, title, name in (
|
|
990
|
+
("node", "npm packages, within their ranges in package.json", npm,
|
|
991
|
+
"Update npm packages within their ranges", "npm"),
|
|
992
|
+
("python", "Python packages, re-locked from requirements.txt", python,
|
|
993
|
+
"Update the locked Python packages", "python")):
|
|
994
|
+
if update[language]:
|
|
995
|
+
others[heading] = update_with(root)
|
|
996
|
+
if commit(root, title, others[heading]):
|
|
997
|
+
made.append(name)
|
|
998
|
+
body_path.write_text(body(pinned, result["updated"], result["skipped"], others,
|
|
999
|
+
any(update.values())),
|
|
1000
|
+
encoding="utf-8")
|
|
1001
|
+
print(" ".join(made))
|
|
1002
|
+
return 0
|