pubrules 0.48.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (79) hide show
  1. pubrules/__init__.py +14 -0
  2. pubrules/audit.py +116 -0
  3. pubrules/audit_draft.py +295 -0
  4. pubrules/audit_prepare.py +1128 -0
  5. pubrules/cache.py +319 -0
  6. pubrules/cbor.py +45 -0
  7. pubrules/ci.py +910 -0
  8. pubrules/cli.py +1069 -0
  9. pubrules/conform.py +295 -0
  10. pubrules/doc_coverage.py +235 -0
  11. pubrules/freshness.py +75 -0
  12. pubrules/gherkin.py +348 -0
  13. pubrules/hooks.py +59 -0
  14. pubrules/hosting.py +372 -0
  15. pubrules/ignores.py +86 -0
  16. pubrules/new.py +242 -0
  17. pubrules/nightly.py +1002 -0
  18. pubrules/paths.py +32 -0
  19. pubrules/protect.py +596 -0
  20. pubrules/providers.py +129 -0
  21. pubrules/refs.py +213 -0
  22. pubrules/release-commit +1 -0
  23. pubrules/releases.py +107 -0
  24. pubrules/repos.py +67 -0
  25. pubrules/ruleset.json +43 -0
  26. pubrules/secrets_config.py +165 -0
  27. pubrules/security.py +134 -0
  28. pubrules/service_needs.py +467 -0
  29. pubrules/standard.json +635 -0
  30. pubrules/standard.py +266 -0
  31. pubrules/supply_chain.py +188 -0
  32. pubrules/tag-ruleset.json +14 -0
  33. pubrules/templates/common/AGENTS.md +32 -0
  34. pubrules/templates/common/pubrules +26 -0
  35. pubrules/templates/common/tests/features/repository.feature +11 -0
  36. pubrules/templates/common/tests/install-lock +65 -0
  37. pubrules/templates/common/tests/pytest.ini +4 -0
  38. pubrules/templates/common/tests/requirements-pubrules.txt +17 -0
  39. pubrules/templates/common/tests/requirements.txt +3 -0
  40. pubrules/templates/common/tests/steps/test_repository.py +14 -0
  41. pubrules/templates/deploy/pubdeploy +28 -0
  42. pubrules/templates/new-rust/CHANGELOG.md +3 -0
  43. pubrules/templates/new-rust/Cargo.toml +46 -0
  44. pubrules/templates/new-rust/LICENSE +202 -0
  45. pubrules/templates/new-rust/Makefile +18 -0
  46. pubrules/templates/new-rust/README.md +34 -0
  47. pubrules/templates/new-rust/SECURITY.md +26 -0
  48. pubrules/templates/new-rust/corpus/corpus.json +11 -0
  49. pubrules/templates/new-rust/deny.toml +32 -0
  50. pubrules/templates/new-rust/fuzz/Cargo.toml +23 -0
  51. pubrules/templates/new-rust/fuzz/fuzz_targets/greet.rs +8 -0
  52. pubrules/templates/new-rust/fuzz/gitignore +4 -0
  53. pubrules/templates/new-rust/fuzz/regressions/README.md +4 -0
  54. pubrules/templates/new-rust/gitignore +10 -0
  55. pubrules/templates/new-rust/src/lib.rs +91 -0
  56. pubrules/templates/new-rust/src/main.rs +19 -0
  57. pubrules/templates/new-rust/supply-chain/audits.toml +4 -0
  58. pubrules/templates/new-rust/supply-chain/config.toml +23 -0
  59. pubrules/templates/new-rust/supply-chain/imports.lock +2 -0
  60. pubrules/templates/new-rust/tests/conftest.py +9 -0
  61. pubrules/templates/new-rust/tests/features/cli.feature +14 -0
  62. pubrules/templates/new-rust/tests/steps/test_cli.py +15 -0
  63. pubrules/templates/new-rust/tests/vectors.rs +30 -0
  64. pubrules/templates/new-rust/tools/vectors/generate.py +24 -0
  65. pubrules/templates/python/Makefile +5 -0
  66. pubrules/templates/python/pubrules-python.mk +47 -0
  67. pubrules/templates/rust/Makefile +9 -0
  68. pubrules/templates/rust/pubrules.mk +138 -0
  69. pubrules/templates/rust/rust-toolchain.toml +8 -0
  70. pubrules/templates/service/pubsec +27 -0
  71. pubrules/templates/service/with-secrets +20 -0
  72. pubrules/templates/service/with-secrets.json +8 -0
  73. pubrules/variables.py +45 -0
  74. pubrules-0.48.0.dist-info/METADATA +1241 -0
  75. pubrules-0.48.0.dist-info/RECORD +79 -0
  76. pubrules-0.48.0.dist-info/WHEEL +5 -0
  77. pubrules-0.48.0.dist-info/entry_points.txt +2 -0
  78. pubrules-0.48.0.dist-info/licenses/LICENSE +202 -0
  79. pubrules-0.48.0.dist-info/top_level.txt +1 -0
pubrules/nightly.py ADDED
@@ -0,0 +1,1002 @@
1
+ """The nightly update of a pub* repository, as two commits' worth of change.
2
+
3
+ `pubrules pins --pubrel V` moves every outdated pub* pin to its latest
4
+ release: this pubrules (managed files, lock, `rust-ci.yml`), pubrel's release
5
+ workflow and `pubrel-version` (ci.release.pubrel, written there where the
6
+ repository's release job named none), the pubsec or pubdeploy a launcher
7
+ fetches where deploy.providers lists none, with its sha256, each
8
+ tool `ci.tools` pins by tag, to the latest release of its git repository,
9
+ every other repository's reusable workflow a workflow pins at a release
10
+ (a control plane's reusable workflow, say), to that repository's
11
+ latest release, every Python package a requirements file or pyproject.toml
12
+ pins by another repository's release wheel (`name @ https://github.com/OWNER/
13
+ REPO/releases/download/vX.Y.Z/name-X.Y.Z-....whl`, pubtest's in
14
+ tests/requirements.txt among them) to that repository's latest release,
15
+ forward only, with each `uv pip compile` lock made from a moved file
16
+ compiled again, and the crates `nightly.pins.crates` names in
17
+ `Cargo.lock`, within their requirements.
18
+
19
+ `pubrules crates --report FILE` then tries each upgrade `cargo update` would
20
+ make to the other crates, one at a time on top of those it kept, and keeps
21
+ one only when `cargo vet` covers it with an audit, imported or by a trusted
22
+ publisher, without adding an exemption, and `cargo deny` passes with it.
23
+ Each update it drops is listed in the report with the audits cargo vet
24
+ recommends for it, as `cargo vet suggest` prints them.
25
+
26
+ `pubrules actions` moves each third-party GitHub Action the hand-written
27
+ workflows and composite actions pin by commit to its latest release, by
28
+ the commit of its tag, in place of Dependabot's version updates, whose pull
29
+ requests can add no pubrel row and get no SonarQube Cloud token.
30
+
31
+ `pubrules nightly` then updates the npm packages within their ranges and
32
+ re-locks the Python requirements, when the repository has them. A
33
+ repository turns each language's updates off with `update`, and the long
34
+ nightly fuzz run with `security.fuzz`, in its pubrules.json (see
35
+ `settings`); the pub* pins always come, and the advisory checks every CI
36
+ run makes have no switch.
37
+
38
+ Each prints what it changed, one line per change, for the commit message
39
+ and the pull request.
40
+ """
41
+
42
+ from __future__ import annotations
43
+
44
+ import contextlib
45
+ import functools
46
+ import io
47
+ import json
48
+ import os
49
+ import re
50
+ import subprocess
51
+ import sys
52
+ from pathlib import Path
53
+
54
+ from . import __version__
55
+ from .paths import confined
56
+ from .refs import (ANY_OWNER, AT, PUBRULES_PIN, RELEASE_PIN, TF_SOURCE, VERSION, owner_of, pubrel_commit, ref,
57
+ release_commit, repin, shipped, tagged)
58
+
59
+ # A row's text, which must not start with '-' (pubrel has no `--`, so it
60
+ # would read that as an option) or a space.
61
+ ROW = re.compile(r"[^-\s].*", re.S)
62
+
63
+ # The categories `pubrel add` takes.
64
+ CATEGORIES = ("changed", "removed", "added", "fixed", "security", "internal")
65
+
66
+ # RELEASE_PIN (refs.py) is pubrel's release workflow, by commit or by tag.
67
+ PUBREL_VERSION = re.compile(r"(pubrel-version:\s*v)(?P<version>[0-9][^\s,}'\"]*)")
68
+ # Every pubrules reusable workflow a repository can call, not only rust-ci.
69
+ CI_PIN = re.compile(ANY_OWNER + r"(?P<path>pubrules/\.github/workflows/"
70
+ r"(?:rust|python|node|service)-ci\.yml@)" + AT + VERSION)
71
+ # Any other repository's reusable workflow (.github/workflows/<name>) or
72
+ # action (actions/<name>) at a release, the repository read from the line
73
+ # itself, so pubrules names none of them. pubrules' and pubrel's move with
74
+ # PUBRULES_PIN and RELEASE_PIN; a deploy.providers package's with its entry,
75
+ # which its sustenaut.bump hook moves.
76
+ OTHER_PIN = re.compile(ANY_OWNER + r"(?P<repo>[A-Za-z0-9_.-]+)"
77
+ r"(?P<path>/(?:\.github/workflows/[\w.-]+\.ya?ml|actions/[\w.-]+)@)"
78
+ + AT + VERSION)
79
+ RELEASE_TAG = re.compile(r"v[0-9]+\.[0-9]+\.[0-9]+")
80
+ # A Python package pinned by another repository's release wheel, in a requirements
81
+ # file or pyproject.toml: the version is in the tag and in the file's name.
82
+ WHEEL_PIN = re.compile(r"(?P<lead>(?P<name>[A-Za-z0-9][A-Za-z0-9_.-]*)(?:\[[^\]]*\])?\s*@\s*)https://github\.com/"
83
+ r"(?P<owner>[A-Za-z0-9][A-Za-z0-9-]*)/(?P<repo>[A-Za-z0-9_.-]+)/releases/download/"
84
+ r"v(?P<version>[0-9]+\.[0-9]+\.[0-9]+)/(?P<file>[A-Za-z0-9_.+-]+\.whl)")
85
+ # The command a `uv pip compile` lock was made with, in its header.
86
+ UV_COMPILE = re.compile(r"^#\s+(uv pip compile .*)$", re.M)
87
+ # A third-party action in a `uses:` line, pinned as Dependabot pins one: by
88
+ # the commit of a release's tag, with the version in a comment. OWNER/REPO,
89
+ # then any path inside it (actions/cache/restore), then `@COMMIT # vX.Y.Z`.
90
+ # A pin by tag, or to a branch (`# main`), or of a pre-release, is left.
91
+ ACTION_PIN = re.compile(r"(?P<lead>uses:[ \t]*['\"]?)" + ANY_OWNER + r"(?P<repo>[A-Za-z0-9_.-]+)"
92
+ r"(?P<path>(?:/[\w.-]+)*)@(?P<commit>[0-9a-f]{40})(?P<gap>[ \t]+#[ \t]*)"
93
+ r"v(?P<version>[0-9]+(?:\.[0-9]+)*)(?=[\s'\"]|$)", re.M)
94
+ # Directories a repository's own composite actions are never in.
95
+ NOT_OWN = ("node_modules", "target")
96
+
97
+ # The languages pubrules' settings name, each by the ecosystem pubrules works
98
+ # with: crates, npm packages, and the hash-locked Python requirements.
99
+ LANGUAGES = ("rust", "node", "python")
100
+
101
+ # What `update` and `security.fuzz` default to for each language: all on.
102
+ # `update` also takes `actions`, the third-party GitHub Actions the
103
+ # repository's workflows and composite actions use (see `actions`).
104
+ # Fuzzing is cargo-fuzz for Rust, and the property tests (Hypothesis,
105
+ # fast-check) for Python and Node; on, a scheduled run fuzzes for longer.
106
+ UPDATES = dict.fromkeys((*LANGUAGES, "actions"), True)
107
+ FUZZ = dict.fromkeys(LANGUAGES, True)
108
+
109
+ # The keys `security` takes (security.py reads dependabot and sonar). The
110
+ # advisory checks have none: they always run.
111
+ SECURITY = ("fuzz", "dependabot", "sonar")
112
+
113
+
114
+ def by_language(name: str, given: object, defaults: dict[str, bool]) -> dict[str, bool]:
115
+ """A per-language switch: an object naming some languages, or one
116
+ boolean for all of them, over the defaults."""
117
+ if isinstance(given, bool):
118
+ return dict.fromkeys(defaults, given)
119
+ if not isinstance(given, dict):
120
+ sys.exit(f"pubrules: pubrules.json's {name} must be an object, such as "
121
+ '{"rust": true, "python": false}, or true or false for every language')
122
+ unknown = sorted(set(given) - set(defaults))
123
+ if unknown:
124
+ sys.exit(f"pubrules: pubrules.json's {name} has no language {', '.join(unknown)}; "
125
+ f"it takes {', '.join(defaults)}")
126
+ for key, value in given.items():
127
+ if not isinstance(value, bool):
128
+ sys.exit(f"pubrules: pubrules.json's {name}.{key} must be true or false, not {value!r}")
129
+ return {**defaults, **given}
130
+
131
+
132
+ def named_crates(config: dict) -> tuple[str, ...] | None:
133
+ """The crates pubrules.json's `nightly.pins.crates` names, or None when it
134
+ names none. Anything else in the nightly block stops the command."""
135
+ from .ci import CRATE # noqa: PLC0415
136
+ block = config.get("nightly", {})
137
+ if not isinstance(block, dict):
138
+ sys.exit('pubrules: pubrules.json\'s nightly must be an object, such as '
139
+ '{"pins": {"crates": ["some-crate"]}}')
140
+ unknown = sorted(set(block) - {"pins"})
141
+ if unknown:
142
+ sys.exit(f"pubrules: pubrules.json's nightly has no setting {', '.join(unknown)}; "
143
+ "it takes pins")
144
+ pinned = block.get("pins", {})
145
+ if not isinstance(pinned, dict):
146
+ sys.exit('pubrules: pubrules.json\'s nightly.pins must be an object, such as '
147
+ '{"crates": ["some-crate"]}')
148
+ unknown = sorted(set(pinned) - {"crates"})
149
+ if unknown:
150
+ sys.exit(f"pubrules: pubrules.json's nightly.pins has no setting {', '.join(unknown)}; "
151
+ "it takes crates")
152
+ if "crates" not in pinned:
153
+ return None
154
+ names = pinned["crates"]
155
+ if not isinstance(names, list) or not all(isinstance(n, str) and CRATE.fullmatch(n)
156
+ for n in names):
157
+ sys.exit("pubrules: pubrules.json's nightly.pins.crates must list crate names, "
158
+ f"not {names!r}")
159
+ return tuple(names)
160
+
161
+
162
+ def pin_crates(root: Path) -> tuple[str, ...]:
163
+ """The crates `pins` moves here rather than `crates`."""
164
+ path = root / "pubrules.json"
165
+ config = json.loads(path.read_text(encoding="utf-8")) if path.is_file() else {}
166
+ named = named_crates(config)
167
+ return named or ()
168
+
169
+
170
+ def settings(root: Path) -> dict[str, dict[str, bool]]:
171
+ """The repository's `update` and `security.fuzz`, per language, from
172
+ pubrules.json over the defaults. Anything else stops the nightly."""
173
+ path = root / "pubrules.json"
174
+ config = json.loads(path.read_text(encoding="utf-8")) if path.is_file() else {}
175
+ security = config.get("security", {})
176
+ if not isinstance(security, dict):
177
+ sys.exit("pubrules: pubrules.json's security must be an object, such as "
178
+ '{"fuzz": {"rust": true}}')
179
+ unknown = sorted(set(security) - set(SECURITY))
180
+ if unknown:
181
+ sys.exit(f"pubrules: pubrules.json's security has no setting {', '.join(unknown)}; "
182
+ f"it takes {', '.join(SECURITY)} (the advisory checks always run)")
183
+ fuzz = by_language("security.fuzz", security.get("fuzz", {}), FUZZ)
184
+ from .security import dependabot_settings, sonar_settings # noqa: PLC0415
185
+ dependabot_settings(config)
186
+ sonar_settings(config)
187
+ named_crates(config)
188
+ return {"update": by_language("update", config.get("update", {}), UPDATES), "fuzz": fuzz}
189
+
190
+
191
+ def locked(root: Path) -> dict[str, str]:
192
+ """Crate name -> version, from Cargo.lock (the highest when several)."""
193
+ lock = root / "Cargo.lock"
194
+ if not lock.is_file():
195
+ return {}
196
+ out: dict[str, str] = {}
197
+ for m in re.finditer(r'\[\[package\]\]\nname = "([^"]+)"\nversion = "([^"]+)"',
198
+ lock.read_text(encoding="utf-8")):
199
+ out[m.group(1)] = m.group(2)
200
+ return out
201
+
202
+
203
+ def cargo(root: Path, *args: str) -> subprocess.CompletedProcess:
204
+ return subprocess.run(["cargo", *args], cwd=root, capture_output=True, text=True, check=False)
205
+
206
+
207
+ def refresh_reports(root: Path) -> None:
208
+ """Regenerate each published crate's SUPPLY-CHAIN.md, which names the
209
+ versions in Cargo.lock: an update otherwise leaves it stale."""
210
+ from .conform import packages # noqa: PLC0415
211
+ from .supply_chain import run as supply_chain # noqa: PLC0415
212
+
213
+ config = json.loads((root / "pubrules.json").read_text(encoding="utf-8"))
214
+ names = [p["name"] for p in packages(root, config)
215
+ if (Path(p["manifest_path"]).parent / "SUPPLY-CHAIN.md").is_file()]
216
+ if names:
217
+ supply_chain(root, names, check=False)
218
+
219
+
220
+ def record(root: Path, text: str, category: str = "internal") -> list[str]:
221
+ """Record `text` for every package `pubrel check` says needs a row for the
222
+ committed change, and return the packages."""
223
+ if category not in CATEGORIES:
224
+ sys.exit(f"pubrules: pubrel has no category {category!r}; use one of "
225
+ f"{', '.join(CATEGORIES)}")
226
+ row = ROW.fullmatch(text)
227
+ if not row:
228
+ sys.exit(f"pubrules: the row's text {text!r} is empty or starts with a space or "
229
+ "'-', which pubrel would take for an option")
230
+ text = row.group(0)
231
+ out = subprocess.run(["pubrel", "check", "origin/main"], cwd=root, capture_output=True,
232
+ text=True, check=False)
233
+ hints = re.findall(r"run `pubrel add( --package ([A-Za-z0-9_][A-Za-z0-9_-]*))? CATEGORY",
234
+ out.stdout + out.stderr)
235
+ added = []
236
+ for flag, package in hints:
237
+ argv = ["pubrel", "add", *(["--package", package] if flag else []), category, text]
238
+ subprocess.run(argv, cwd=root, check=True, capture_output=True)
239
+ added.append(package or "(the package)")
240
+ return added
241
+
242
+
243
+ def released_by_config(root: Path, pubrel: str, changes: list[str]) -> bool:
244
+ """Move the pubrel the repository's release job calls to `pubrel`; True
245
+ when it moved. It is ci.release.pubrel, or, where that names none, the
246
+ one the workflows sync wrote pin, which is then written there."""
247
+ path = root / "pubrules.json"
248
+ if not path.exists():
249
+ return False
250
+ config = json.loads(path.read_text(encoding="utf-8"))
251
+ ci = config.get("ci")
252
+ release = ci.get("release") if isinstance(ci, dict) else None
253
+ if isinstance(release, dict) and "pubrel" in release:
254
+ current = release["pubrel"]
255
+ elif release is False or (isinstance(release, dict) and "uses" in release) or not isinstance(ci, dict):
256
+ return False
257
+ else:
258
+ current = written_pubrel(root)
259
+ if current in (None, pubrel):
260
+ return False
261
+ changes.append(f"pubrules.json: ci.release.pubrel {current} -> {pubrel}")
262
+ ci["release"] = {**(release if isinstance(release, dict) else {}), "pubrel": pubrel}
263
+ path.write_text(json.dumps(config, indent=2) + "\n", encoding="utf-8")
264
+ return True
265
+
266
+
267
+ def written_pubrel(root: Path) -> str | None:
268
+ """The pubrel the workflows sync wrote call, or None."""
269
+ from .ci import is_generated # noqa: PLC0415
270
+ for path in sorted((root / ".github" / "workflows").glob("*.y*ml")):
271
+ text = path.read_text(encoding="utf-8")
272
+ if is_generated(text) and (m := RELEASE_PIN.search(text)):
273
+ return m["version"]
274
+ return None
275
+
276
+
277
+ def latest_release(repo: str) -> str | None:
278
+ """The tag of `repo`'s latest release, or None when GitHub does not say."""
279
+ try:
280
+ out = subprocess.run(["gh", "release", "view", "--repo", repo, "--json", "tagName",
281
+ "--jq", ".tagName"], capture_output=True, text=True, check=False)
282
+ except OSError:
283
+ return None
284
+ return out.stdout.strip() if out.returncode == 0 else None
285
+
286
+
287
+ def tools_by_config(root: Path, changes: list[str]) -> bool:
288
+ """Move each tool pubrules.json's ci.tools pins by tag to the latest release
289
+ of its git repository; True when one moved. One whose latest release
290
+ cannot be read stays as it is, with a warning."""
291
+ from .ci import GITHUB_REPO, TOOL_TAG, tools # noqa: PLC0415
292
+
293
+ path = root / "pubrules.json"
294
+ if not path.is_file():
295
+ return False
296
+ config = json.loads(path.read_text(encoding="utf-8"))
297
+ listed = tools(config["ci"]) if isinstance(config.get("ci"), dict) else []
298
+ moved = False
299
+ for tool in listed:
300
+ m = GITHUB_REPO.fullmatch(tool["git"])
301
+ latest = latest_release(m["repo"]) if m else None
302
+ if not latest or not TOOL_TAG.fullmatch(latest):
303
+ print(f"pubrules: warning: no latest release of {tool['git']} to read, "
304
+ f"so {tool['crate']} stays at {tool['tag']}", file=sys.stderr)
305
+ elif latest != tool["tag"]:
306
+ changes.append(f"pubrules.json: ci.tools {tool['crate']} {tool['tag']} -> {latest}")
307
+ tool["tag"] = latest
308
+ moved = True
309
+ if moved:
310
+ config["ci"]["tools"] = listed
311
+ path.write_text(json.dumps(config, indent=2) + "\n", encoding="utf-8")
312
+ return moved
313
+
314
+
315
+ def release_assets(repo: str, tag: str) -> list[str] | None:
316
+ """The file names release `tag` of `repo` carries, or None when GitHub does not say."""
317
+ try:
318
+ out = subprocess.run(["gh", "release", "view", tag, "--repo", repo, "--json", "assets",
319
+ "--jq", ".assets[].name"], capture_output=True, text=True, check=False)
320
+ except OSError:
321
+ return None
322
+ return out.stdout.split() if out.returncode == 0 else None
323
+
324
+
325
+ @functools.cache
326
+ def release_wheel(slug: str, version: str) -> str | None:
327
+ """The file name of the pure-Python wheel release vVERSION of `slug`
328
+ carries (`<distribution>-VERSION-py3-none-any.whl`), or None when it
329
+ carries none or GitHub does not say. A repository's distribution may be
330
+ named otherwise than the repository (pubtest's is pubtestrules), so the
331
+ release itself says what its wheel is called."""
332
+ wheels = [name for name in release_assets(slug, f"v{version}") or []
333
+ if name.endswith(f"-{version}-py3-none-any.whl")]
334
+ return wheels[0] if len(wheels) == 1 else None
335
+
336
+
337
+ def distribution(file: str) -> str:
338
+ """The distribution a wheel's file name names, as the file spells it."""
339
+ return file.split("-", 1)[0]
340
+
341
+
342
+ def same_name(a: str, b: str) -> bool:
343
+ """Whether two distribution names are one (PEP 503: case, `-`, `_` and `.` alike)."""
344
+ return re.sub(r"[-_.]+", "-", a).lower() == re.sub(r"[-_.]+", "-", b).lower()
345
+
346
+
347
+ @functools.cache
348
+ def latest_tag(slug: str) -> str | None:
349
+ """`slug`'s latest release tag, when it is vX.Y.Z; None, with a warning, when not."""
350
+ tag = latest_release(slug)
351
+ if tag and RELEASE_TAG.fullmatch(tag):
352
+ return tag
353
+ print(f"pubrules: warning: no latest release of {slug} to read, so its workflows and actions "
354
+ "stay where they are pinned", file=sys.stderr)
355
+ return None
356
+
357
+
358
+ def provided(root: Path) -> set[str]:
359
+ """The repositories deploy.providers takes wheels from, by name in lower case."""
360
+ from .providers import WHEEL # noqa: PLC0415
361
+ path = root / "pubrules.json"
362
+ config = json.loads(path.read_text(encoding="utf-8")) if path.is_file() else {}
363
+ deploy = config.get("deploy") if isinstance(config, dict) else None
364
+ given = deploy.get("providers") if isinstance(deploy, dict) else None
365
+ return {m["repo"].split("/", 1)[1].lower() for p in given or [] if isinstance(p, dict)
366
+ and isinstance(p.get("wheel"), str) and (m := WHEEL.fullmatch(p["wheel"]))}
367
+
368
+
369
+ def others_by_release(text: str, name: str, skip: set[str], changes: list[str]) -> str:
370
+ """`text` with each other repository's workflow pin (OTHER_PIN) moved to that
371
+ repository's latest release, by the commit of its tag; the moves said in
372
+ `changes`. One whose latest release cannot be read stays."""
373
+ def move(m: re.Match[str]) -> str:
374
+ if m["repo"].lower() in skip:
375
+ return m.group(0)
376
+ slug = f"{m['owner']}/{m['repo']}"
377
+ latest = latest_tag(slug)
378
+ if latest is None:
379
+ return m.group(0)
380
+ version = latest[1:]
381
+ commit = tagged(f"{os.environ.get('PUBRULES_GIT_BASE', 'https://github.com')}/{slug}", latest)
382
+ if version == m["version"] and (commit is None or commit == m["commit"]):
383
+ return m.group(0)
384
+ changes.append(f"{name}: {slug} {m['version']} -> {version}" if version != m["version"]
385
+ else f"{name}: {slug} {version} pinned to {commit}")
386
+ return f"{slug}{m['path']}{ref(version, commit)}"
387
+ return OTHER_PIN.sub(move, text)
388
+
389
+
390
+ def wheel_files(root: Path) -> list[Path]:
391
+ """The files that can pin a release wheel: requirements files (*.txt, *.in
392
+ with "requirements" in the name) and pyproject.toml, outside dot-directories,
393
+ node_modules and target; never a lock, which follows its input, nor the
394
+ requirements pubrules sync writes."""
395
+ out = []
396
+ for path in sorted(root.rglob("*")):
397
+ rel = path.relative_to(root)
398
+ if any(part.startswith(".") or part in NOT_OWN for part in rel.parts[:-1]) or not path.is_file():
399
+ continue
400
+ base = path.name
401
+ if base == "pyproject.toml" or ("requirements" in base and base.endswith((".txt", ".in"))
402
+ and base != "requirements-pubrules.txt"):
403
+ out.append(path)
404
+ return out
405
+
406
+
407
+ def wheels_by_release(root: Path, skip: set[str], changes: list[str]) -> list[Path]:
408
+ """Each release wheel pin (WHEEL_PIN) of another repository moved to that
409
+ repository's latest release, when that release is newer: the tag and the
410
+ file's version both. The moves said in `changes`; the files moved returned.
411
+ One whose latest release cannot be read, or is not newer, stays."""
412
+ moved_files = []
413
+
414
+ def move(m: re.Match[str], rel: str) -> str:
415
+ if m["repo"].lower() in skip or m["version"] not in m["file"]:
416
+ return m.group(0)
417
+ slug = f"{m['owner']}/{m['repo']}"
418
+ latest = latest_tag(slug)
419
+ if latest is None or release_key(latest[1:]) <= release_key(m["version"]):
420
+ return m.group(0)
421
+ version = latest[1:]
422
+ line = f"{rel}: {slug} {m['version']} -> {version}"
423
+ if line not in changes:
424
+ changes.append(line)
425
+ # The new release's wheel, by the name it carries: a distribution
426
+ # renamed since (pubtest's, as pubtestrules) takes the requirement's
427
+ # name with it. A release that says nothing keeps the old file's name.
428
+ file = release_wheel(slug, version) or m["file"].replace(m["version"], version, 1)
429
+ lead = m["lead"]
430
+ old, new = distribution(m["file"]), distribution(file)
431
+ if not same_name(old, new) and same_name(m["name"], old):
432
+ lead = new + lead[len(m["name"]):]
433
+ return f"{lead}https://github.com/{slug}/releases/download/v{version}/{file}"
434
+
435
+ for path in wheel_files(root):
436
+ text = path.read_text(encoding="utf-8")
437
+ rel = path.relative_to(root).as_posix()
438
+ new = WHEEL_PIN.sub(lambda m, rel=rel: move(m, rel), text)
439
+ if new != text:
440
+ path.write_text(new, encoding="utf-8")
441
+ moved_files.append(path)
442
+ return moved_files
443
+
444
+
445
+ def launchers_by_release(root: Path, changes: list[str]) -> bool:
446
+ """The pubsec or pubdeploy a managed launcher fetches where deploy.providers
447
+ lists none (a service without a deploy block runs ./pubsec), moved to its
448
+ latest release, the wheel's sha256 with it; sync then writes the launcher
449
+ from that pin. True when one moved. One whose latest release cannot be
450
+ read, or is not newer, stays."""
451
+ from .providers import PUBDEPLOY, PUBSEC, WHEEL, in_launcher, listed # noqa: PLC0415
452
+ path = root / "pubrules.json"
453
+ config = json.loads(path.read_text(encoding="utf-8")) if path.is_file() else {}
454
+ moved = False
455
+ for name in (PUBSEC, PUBDEPLOY):
456
+ pin = in_launcher(root, name)
457
+ m = WHEEL.fullmatch(pin["wheel"]) if pin and listed(config, name) is None else None
458
+ if m is None:
459
+ continue
460
+ latest = latest_tag(m["repo"])
461
+ if latest is None or release_key(latest[1:]) <= release_key(m["version"]):
462
+ continue
463
+ from .cli import sha256_of # noqa: PLC0415
464
+ version = latest[1:]
465
+ wheel = pin["wheel"].replace(f"/v{m['version']}/", f"/{latest}/").replace(
466
+ f"-{m['version']}-", f"-{version}-")
467
+ launcher = root / name
468
+ launcher.write_text(launcher.read_text(encoding="utf-8").replace(
469
+ f"fetch {pin['wheel']} {pin['sha256']} ", f"fetch {wheel} {sha256_of(wheel)} "), encoding="utf-8")
470
+ changes.append(f"{name}: {m['repo']} {m['version']} -> {version}")
471
+ moved = True
472
+ return moved
473
+
474
+
475
+ def recompile(root: Path, moved: list[Path], changes: list[str]) -> None:
476
+ """Each `uv pip compile` lock (its header names the command) whose input
477
+ moved, compiled again with that command, so its hashes are the new wheels'.
478
+ One that fails is said, and left for the pull request's CI to show."""
479
+ import shlex # noqa: PLC0415
480
+ names = {p.relative_to(root).as_posix() for p in moved}
481
+ for lock in sorted(root.rglob("*.lock")):
482
+ if any(part.startswith(".") or part in NOT_OWN for part in lock.relative_to(root).parts[:-1]):
483
+ continue
484
+ try:
485
+ found = UV_COMPILE.search(lock.read_text(encoding="utf-8")[:2000])
486
+ except (OSError, UnicodeDecodeError):
487
+ continue
488
+ if not found:
489
+ continue
490
+ argv = shlex.split(found.group(1))
491
+ here = lock.parent.relative_to(root)
492
+ if not any((here / a).as_posix() in names for a in argv if a.endswith((".in", ".txt", ".toml"))):
493
+ continue
494
+ done = subprocess.run(argv, cwd=lock.parent, capture_output=True, text=True, check=False)
495
+ rel = lock.relative_to(root).as_posix()
496
+ if done.returncode != 0:
497
+ print(f"pubrules: warning: {rel} could not be compiled again: "
498
+ f"{(done.stderr.strip().splitlines() or ['uv failed'])[-1]}", file=sys.stderr)
499
+ continue
500
+ changes.append(f"{rel}: compiled again")
501
+
502
+
503
+ def modules_by_release(root: Path, skip: set[str], changes: list[str]) -> None:
504
+ """Each other repository's Terraform module source (TF_SOURCE) in the
505
+ repository's .tf files, moved to that repository's latest release by the
506
+ commit of its tag, with the version in a comment after the closing
507
+ quote; the moves said in `changes`. A source pinned by commit with no
508
+ version comment, or whose latest release cannot be read, stays."""
509
+ from .refs import tf_files # noqa: PLC0415
510
+
511
+ def move(m: re.Match[str], name: str) -> str:
512
+ current = m["tag"] or m["version"]
513
+ if m["repo"].lower() in skip or current is None:
514
+ return m.group(0)
515
+ slug = f"{m['owner']}/{m['repo']}"
516
+ latest = latest_tag(slug)
517
+ if latest is None:
518
+ return m.group(0)
519
+ version = latest[1:]
520
+ commit = tagged(f"{os.environ.get('PUBRULES_GIT_BASE', 'https://github.com')}/{slug}", latest)
521
+ if version == current and (commit is None or commit == m["commit"]):
522
+ return m.group(0)
523
+ changes.append(f"{name}: {slug} {current} -> {version}" if version != current
524
+ else f"{name}: {slug} {version} pinned to {commit}")
525
+ at = f'{commit}" # v{version}' if commit else f'v{version}"'
526
+ return f'{m["lead"]}"github.com/{slug}//{m["path"]}?ref={at}'
527
+
528
+ for path in tf_files(root):
529
+ text = path.read_text(encoding="utf-8")
530
+ rel = path.relative_to(root).as_posix()
531
+ new = TF_SOURCE.sub(lambda m, rel=rel: move(m, rel), text)
532
+ if new != text:
533
+ path.write_text(new, encoding="utf-8")
534
+
535
+
536
+ def action_files(root: Path) -> list[Path]:
537
+ """The repository's hand-written workflows and its own composite actions
538
+ (action.yml anywhere outside dot-directories other than .github, and
539
+ outside node_modules and target)."""
540
+ workflows = sorted((root / ".github" / "workflows").glob("*.y*ml"))
541
+ own = sorted(p for p in root.rglob("action.y*ml")
542
+ if not any((part.startswith(".") and part != ".github") or part in NOT_OWN
543
+ for part in p.relative_to(root).parts[:-1]))
544
+ return workflows + [p for p in own if p not in workflows]
545
+
546
+
547
+ def release_key(version: str) -> tuple[int, ...]:
548
+ return tuple(int(n) for n in version.split("."))
549
+
550
+
551
+ @functools.cache
552
+ def tag_commit(slug: str, tag: str) -> str | None:
553
+ """The commit `slug`'s tag names, or None. `tagged` would warn that it
554
+ pins by tag; `actions` never does, and says so itself."""
555
+ with contextlib.redirect_stderr(io.StringIO()):
556
+ return tagged(f"{os.environ.get('PUBRULES_GIT_BASE', 'https://github.com')}/{slug}", tag)
557
+
558
+
559
+ def actions(root: Path) -> list[str]:
560
+ """Each third-party action (ACTION_PIN) in the repository's hand-written
561
+ workflows and composite actions moved to its repository's latest
562
+ release, by the commit of its tag, with the version in a comment: what
563
+ Dependabot's version updates did, but on the nightly's branch, with a
564
+ pubrel row and the nightly's token. One line per file and repository
565
+ moved. pubrules' and pubrel's pins, and a deploy.providers package's,
566
+ move with `pins`, as does any OWNER/REPO/actions/NAME or reusable
567
+ workflow (OTHER_PIN). An action only moves forward, and one whose
568
+ latest release or tag's commit cannot be read stays, with a warning.
569
+ Workflows `pubrules sync` writes move with pubrules."""
570
+ from .ci import is_generated # noqa: PLC0415
571
+
572
+ skip = {"pubrules", "pubrel"} | provided(root)
573
+ changes: list[str] = []
574
+ for path in action_files(root):
575
+ text = path.read_text(encoding="utf-8")
576
+ if is_generated(text):
577
+ continue
578
+ rel = path.relative_to(root).as_posix()
579
+
580
+ def move(m: re.Match[str], rel: str = rel) -> str:
581
+ if m["repo"].lower() in skip or m["path"].startswith(("/actions/", "/.github/workflows/")):
582
+ return m.group(0)
583
+ slug = f"{m['owner']}/{m['repo']}"
584
+ latest = latest_tag(slug)
585
+ if latest is None or release_key(latest[1:]) <= release_key(m["version"]):
586
+ return m.group(0)
587
+ commit = tag_commit(slug, latest)
588
+ if commit is None:
589
+ print(f"pubrules: warning: could not read the commit of {slug}'s {latest}, so "
590
+ f"{rel} stays at {m['version']}", file=sys.stderr)
591
+ return m.group(0)
592
+ line = f"{rel}: {slug} {m['version']} -> {latest[1:]}"
593
+ if line not in changes:
594
+ changes.append(line)
595
+ return f"{m['lead']}{slug}{m['path']}@{commit}{m['gap']}{latest}"
596
+
597
+ new = ACTION_PIN.sub(move, text)
598
+ if new != text:
599
+ path.write_text(new, encoding="utf-8")
600
+ return changes
601
+
602
+
603
+ def pins(root: Path, pubrel: str, lock: bool = True) -> list[str]:
604
+ from .freshness import pinned, pinned_package # noqa: PLC0415
605
+
606
+ changes = []
607
+ # What installed packages move, as `pubrules bump` has them do: pubdeploy's
608
+ # moves deploy.providers (pubsec and pubdeploy among them) and the uses of
609
+ # pubdeploy's workflows, before sync writes the launchers and the tests'
610
+ # requirements from them. A failing hook is said, and moves nothing.
611
+ from .hooks import bump as bump_hooks # noqa: PLC0415
612
+ moved = []
613
+ for line in bump_hooks(root):
614
+ if line.startswith("updated"):
615
+ moved.append(line.removeprefix("updated").strip())
616
+ else:
617
+ print(line, file=sys.stderr)
618
+ changes += moved
619
+ package = (pinned_package(root) or ("pubrules", None))[0]
620
+ before = pinned(root)
621
+ if before and RELEASE_TAG.fullmatch(f"v{before}") and release_key(before) > release_key(__version__):
622
+ # Never backwards: sync would write this older pubrules' managed files.
623
+ print(f"pubrules: warning: this repository runs pubrules {before}, newer than this {__version__}, "
624
+ "so pubrules' managed files and lock stay", file=sys.stderr)
625
+ before, moved = __version__, []
626
+ if (before and before != __version__) or moved:
627
+ from .cli import lock as lock_requirements, sync # noqa: PLC0415
628
+ sync(root, check=False)
629
+ if lock:
630
+ lock_requirements(root)
631
+ if before and before != __version__:
632
+ # A rule this pubrules brings that the repository fails is waived,
633
+ # for a while, by the move that brings it.
634
+ from .gherkin import grandfather # noqa: PLC0415
635
+ changes += grandfather(root, package, before)
636
+ if before and before != __version__:
637
+ changes.append(f"pubrules {before} -> {__version__}")
638
+ released = bool(pubrel) and released_by_config(root, pubrel, changes)
639
+ launched = launchers_by_release(root, changes)
640
+ if tools_by_config(root, changes) or released or launched:
641
+ from .cli import sync # noqa: PLC0415
642
+ sync(root, check=False)
643
+ from .ci import is_generated # noqa: PLC0415
644
+ calls = owner_of(root)
645
+ skip = {"pubrules", "pubrel"} | provided(root)
646
+ for path in sorted((root / ".github" / "workflows").glob("*.y*ml")):
647
+ text = path.read_text(encoding="utf-8")
648
+ if is_generated(text): # pubrules sync writes these, from pubrules.json
649
+ continue
650
+ new = text
651
+ for pattern, version, what in ((PUBRULES_PIN, __version__, "pubrules"),
652
+ (RELEASE_PIN, pubrel, "pubrel release workflow"),
653
+ (PUBREL_VERSION, pubrel, "pubrel-version")):
654
+ if not version or not pattern.search(new):
655
+ continue
656
+ if pattern is PUBREL_VERSION:
657
+ commit = None
658
+ moved = pattern.sub(rf"\g<1>{version}", new)
659
+ else:
660
+ # A workflow's pin: the commit moves with the version. A pin
661
+ # of a pubrules action or workflow this pubrules does not ship
662
+ # stays on the commit that has it; sync --check says so.
663
+ commit = (pubrel_commit(version, new, calls) if pattern is RELEASE_PIN
664
+ else release_commit())
665
+ moved = repin(pattern, new, calls, version, commit,
666
+ keep=(lambda m: not shipped(m["name"])) if pattern is PUBRULES_PIN else None)
667
+ for m in pattern.finditer(new):
668
+ if pattern is PUBRULES_PIN and not shipped(m["name"]):
669
+ changes.append(f"{path.name}: {m['name']} stays at pubrules {m['version']}: "
670
+ f"pubrules {version} has no {m['name']}")
671
+ continue
672
+ # "pubrules A -> B" already says that pubrules' own pins moved.
673
+ if m["version"] != version and pattern is not PUBRULES_PIN:
674
+ changes.append(f"{path.name}: {what} {m['version']} -> {version}")
675
+ elif m["version"] == version and commit and m["commit"] != commit:
676
+ changes.append(f"{path.name}: {what} {version} pinned to {commit}")
677
+ new = moved
678
+ new = others_by_release(new, path.name, skip, changes)
679
+ if new != text:
680
+ path.write_text(new, encoding="utf-8")
681
+ modules_by_release(root, skip, changes)
682
+ # deploy.providers' wheels too: a hand-written requirement of one keeps up with
683
+ # the providers entry its package's hook moved. pubrules' are sync's to write.
684
+ wheels = wheels_by_release(root, {"pubrules"}, changes)
685
+ if wheels and lock:
686
+ from .cli import REQUIREMENTS, lock as lock_requirements # noqa: PLC0415
687
+ if (root / REQUIREMENTS).is_file():
688
+ with contextlib.redirect_stdout(io.StringIO()):
689
+ lock_requirements(root)
690
+ recompile(root, wheels, changes)
691
+ old = locked(root)
692
+ moving = pin_crates(root)
693
+ for crate in moving:
694
+ if crate in old:
695
+ cargo(root, "update", "--quiet", "-p", crate)
696
+ new_lock = locked(root)
697
+ changes += [f"{c} {old[c]} -> {new_lock[c]}" for c in moving
698
+ if c in old and new_lock.get(c) != old[c]]
699
+ if new_lock != old:
700
+ refresh_reports(root)
701
+ return changes
702
+
703
+
704
+ UPDATE = re.compile(r"^\s*Updating (\S+) v(\S+) -> v(\S+)", re.M)
705
+
706
+
707
+ def npm_locked(root: Path) -> dict[str, str]:
708
+ """Direct dependency -> version, from package-lock.json."""
709
+ lock = root / "package-lock.json"
710
+ if not lock.is_file():
711
+ return {}
712
+ packages = json.loads(lock.read_text(encoding="utf-8")).get("packages", {})
713
+ top = packages.get("", {})
714
+ names = {**top.get("dependencies", {}), **top.get("devDependencies", {})}
715
+ return {n: packages[f"node_modules/{n}"]["version"] for n in sorted(names)
716
+ if "version" in packages.get(f"node_modules/{n}", {})}
717
+
718
+
719
+ def changed(old: dict[str, str], new: dict[str, str]) -> list[str]:
720
+ """One line per package whose version moved, in name order."""
721
+ return [f"{n} {old[n]} -> {new[n]}" for n in sorted(old) if n in new and new[n] != old[n]]
722
+
723
+
724
+ def npm(root: Path) -> list[str]:
725
+ """Every npm package updated to the newest version its range in
726
+ package.json allows, in package-lock.json only and without running an
727
+ install script. The direct dependencies that moved, one line each."""
728
+ if not (root / "package-lock.json").is_file():
729
+ return []
730
+ old = npm_locked(root)
731
+ subprocess.run(["npm", "update", "--package-lock-only", "--ignore-scripts",
732
+ "--no-audit", "--no-fund"], cwd=root, check=True, capture_output=True)
733
+ return changed(old, npm_locked(root)) or (["indirect dependencies"]
734
+ if git(root, "status", "--porcelain") else [])
735
+
736
+
737
+ def python_locked(root: Path) -> dict[str, str]:
738
+ """Package -> version, from tests/requirements.lock."""
739
+ from .cli import LOCK # noqa: PLC0415
740
+
741
+ lock = root / LOCK
742
+ if not lock.is_file():
743
+ return {}
744
+ return dict(re.findall(r"^([A-Za-z0-9_.-]+)==(\S+)", lock.read_text(encoding="utf-8"), re.M))
745
+
746
+
747
+ def python(root: Path) -> list[str]:
748
+ """tests/requirements.lock re-resolved, so each package moves to the
749
+ newest version requirements.txt allows. The packages that moved."""
750
+ from .cli import LOCK, REQUIREMENTS, lock # noqa: PLC0415
751
+
752
+ if not (root / REQUIREMENTS).is_file() or not (root / LOCK).is_file():
753
+ return []
754
+ old = python_locked(root)
755
+ with contextlib.redirect_stdout(io.StringIO()):
756
+ lock(root)
757
+ return changed(old, python_locked(root))
758
+
759
+
760
+ def candidates(root: Path) -> list[tuple[str, str, str]]:
761
+ """(crate, from, to) for every upgrade `cargo update` would make, the
762
+ crates `pins` moves aside. Downgrades (which the MSRV-aware resolver
763
+ proposes when a locked version needs a newer Rust) are not taken."""
764
+ return upgrades(cargo(root, "update", "--dry-run").stderr, pin_crates(root))
765
+
766
+
767
+ def upgrades(dry_run: str, pinned: tuple[str, ...] = ()) -> list[tuple[str, str, str]]:
768
+ """The upgrades in `cargo update --dry-run`'s output, `pinned` aside."""
769
+ return [(m.group(1), m.group(2), m.group(3)) for m in UPDATE.finditer(dry_run)
770
+ if m.group(1) not in pinned]
771
+
772
+
773
+ def recommended(vet_output: str) -> str:
774
+ """What cargo vet recommends auditing, from a failing run's output."""
775
+ lines = [line.rstrip() for line in vet_output.splitlines() if line.strip()]
776
+ at = next((i for i, line in enumerate(lines) if "recommended audits" in line), None)
777
+ return "\n".join(lines[at:] if at is not None else lines[-8:])
778
+
779
+
780
+ def failing(vet_output: str) -> list[str]:
781
+ """The crates `cargo vet` reports missing audits for."""
782
+ return sorted({m.group(1) for m in re.finditer(r"^\s*([A-Za-z0-9_-]+):\S+ missing",
783
+ vet_output, re.M)})
784
+
785
+
786
+ def has(root: Path, crate: str, version: str) -> bool:
787
+ lock = root / "Cargo.lock"
788
+ return lock.is_file() and f'name = "{crate}"\nversion = "{version}"' in lock.read_text(
789
+ encoding="utf-8")
790
+
791
+
792
+ def snapshot(root: Path) -> dict[Path, bytes]:
793
+ """Cargo.lock and supply-chain/, to restore if the updates do not pass."""
794
+ paths = [root / "Cargo.lock", *sorted((root / "supply-chain").glob("*"))]
795
+ return {p: p.read_bytes() for p in paths if p.is_file()}
796
+
797
+
798
+ def restore(saved: dict[Path, bytes]) -> None:
799
+ for path, data in saved.items():
800
+ path.write_bytes(data)
801
+
802
+
803
+ def own_packages(root: Path) -> set[str]:
804
+ meta = json.loads(cargo(root, "metadata", "--no-deps", "--format-version", "1").stdout
805
+ or '{"packages": []}')
806
+ return {p["name"] for p in meta["packages"]}
807
+
808
+
809
+ def first_party_policy(root: Path, vet_output: str) -> list[str]:
810
+ """When cargo vet stops because the repository's own crates now also
811
+ exist on crates.io, record that they are not audited as crates.io
812
+ crates: `[policy.NAME] audit-as-crates-io = false`. Returns the names."""
813
+ if "audit-as-crates-io" not in vet_output:
814
+ return []
815
+ named = set(re.findall(r"^[ \t]*(?:│[ \t]*)?([A-Za-z0-9_-]+):\S+[ \t]*$", vet_output, re.M))
816
+ own = sorted(named & own_packages(root))
817
+ config = root / "supply-chain" / "config.toml"
818
+ if not own or not config.is_file():
819
+ return []
820
+ text = config.read_text(encoding="utf-8").rstrip("\n")
821
+ for name in own:
822
+ text += (f"\n\n[policy.{name}]\naudit-as-crates-io = false\n"
823
+ if f"[policy.{name}]" not in text else "")
824
+ config.write_text(text.rstrip("\n") + "\n", encoding="utf-8")
825
+ return own
826
+
827
+
828
+ def tail(proc: subprocess.CompletedProcess, n: int = 8) -> str:
829
+ return "\n".join((proc.stdout + proc.stderr).strip().splitlines()[-n:])
830
+
831
+
832
+ def crates(root: Path, report: Path | None) -> int:
833
+ """Try each upgrade on its own, on top of those already kept, and keep
834
+ it only when `cargo vet` passes without a new exemption and `cargo deny`
835
+ passes. A repository with no Cargo.lock has no crates to update."""
836
+ target = confined(report, "the --report file") if report else None
837
+ lines, skipped = upgrade(root) if (root / "Cargo.lock").is_file() else ([], [])
838
+ for line in lines:
839
+ print(line)
840
+ if target:
841
+ target.write_text(json.dumps({"updated": lines, "skipped": skipped}, indent=2) + "\n",
842
+ encoding="utf-8")
843
+ return 0
844
+
845
+
846
+ def upgrade(root: Path) -> tuple[list[str], list[dict]]:
847
+ """The crate updates `crates` keeps, and those it skips with why."""
848
+ saved = snapshot(root)
849
+ notes: list[str] = []
850
+ updated: list[tuple[str, str, str]] = []
851
+ skipped: list[dict] = []
852
+ todo = candidates(root)
853
+
854
+ vet = cargo(root, "vet")
855
+ own = first_party_policy(root, vet.stdout + vet.stderr) if vet.returncode else []
856
+ if own:
857
+ notes.append("supply-chain/config.toml: " + ", ".join(own) + " (this repository's own) "
858
+ "not audited as crates.io crates, now that they are published there")
859
+ vet = cargo(root, "vet")
860
+ deny = cargo(root, "deny", "--locked", "check", "--hide-inclusion-graph")
861
+ if vet.returncode or deny.returncode:
862
+ restore(saved)
863
+ why = "main itself does not pass " + ("cargo vet" if vet.returncode else "cargo deny") + \
864
+ ":\n" + tail(vet if vet.returncode else deny)
865
+ skipped = [{"crate": c, "from": a, "to": b, "why": why} for c, a, b in todo]
866
+ todo, notes = [], []
867
+
868
+ for crate, old, new in todo:
869
+ if has(root, crate, new):
870
+ # Another kept update brought it along, and vet and deny passed.
871
+ updated.append((crate, old, new))
872
+ continue
873
+ if not has(root, crate, old):
874
+ continue
875
+ before = snapshot(root)
876
+ step = cargo(root, "update", "--quiet", "-p", f"{crate}@{old}", "--precise", new)
877
+ why = ""
878
+ if step.returncode:
879
+ why = "cargo updates it only together with other crates:\n" + tail(step)
880
+ else:
881
+ vet = cargo(root, "vet")
882
+ if vet.returncode:
883
+ why = "not covered by an audit; cargo vet recommends:\n" + recommended(
884
+ vet.stdout + vet.stderr)
885
+ else:
886
+ # The advisory database was fetched above: --frozen is
887
+ # --locked, offline.
888
+ deny = cargo(root, "deny", "--frozen", "check", "--hide-inclusion-graph")
889
+ if deny.returncode:
890
+ why = "cargo deny fails with it:\n" + tail(deny)
891
+ if why:
892
+ restore(before)
893
+ skipped.append({"crate": crate, "from": old, "to": new, "why": why})
894
+ else:
895
+ updated.append((crate, old, new))
896
+
897
+ if updated and (cargo(root, "vet", "--locked").returncode
898
+ or cargo(root, "deny", "--locked", "check").returncode):
899
+ # Each passed on its own but not all together: take none.
900
+ restore(saved)
901
+ skipped += [{"crate": c, "from": a, "to": b,
902
+ "why": "the updates together did not pass cargo vet and cargo deny"}
903
+ for c, a, b in updated]
904
+ updated, notes = [], []
905
+ if not updated and not notes:
906
+ restore(saved)
907
+ if updated:
908
+ refresh_reports(root)
909
+ return notes + [f"{c} {a} -> {b}" for c, a, b in updated], skipped
910
+
911
+
912
+ def git(root: Path, *args: str) -> str:
913
+ return subprocess.run(["git", *args], cwd=root, capture_output=True, text=True,
914
+ check=True).stdout
915
+
916
+
917
+ def commit(root: Path, title: str, lines: list[str]) -> bool:
918
+ """Commit everything changed as `title`, with `lines` as its body and an
919
+ internal pubrel row for each package that needs one. False when nothing
920
+ changed."""
921
+ if not git(root, "status", "--porcelain"):
922
+ return False
923
+ who = ["-c", "user.name=pubrules nightly", "-c", "user.email=noreply@github.com"]
924
+ git(root, "add", "-A")
925
+ git(root, *who, "commit", "-q", "-m", title, "-m", "\n".join(lines))
926
+ if record(root, title):
927
+ git(root, "add", "-A")
928
+ git(root, *who, "commit", "-q", "--amend", "--no-edit")
929
+ return True
930
+
931
+
932
+ def body(pinned: list[str], updated: list[str], skipped: list[dict],
933
+ others: dict[str, list[str]] | None = None, update: bool = True) -> str:
934
+ """The pull request's body. `others` is each further section's lines,
935
+ by heading; with `update` off, the body says the repository takes no
936
+ third-party updates instead of listing them."""
937
+ out = ["Opened by pubrules' nightly workflow, which updates this pull request each night "
938
+ "rather than opening another. Nothing merges it: merge it when CI is green.", ""]
939
+ out += ["## 1. pub* pins", ""] + ([f"- {line}" for line in pinned] or ["Nothing outdated."])
940
+ if not update:
941
+ out += ["", "## Other dependencies", "",
942
+ "None: this repository takes no third-party updates; its pubrules.json sets `\"update\": false`."]
943
+ return "\n".join(out) + "\n"
944
+ number = 2
945
+ if updated or skipped:
946
+ out += ["", f"## {number}. Other crates", "",
947
+ "Updates within their requirements that `cargo vet` covers with an audit "
948
+ "(no new exemption) and `cargo deny` passes.", ""]
949
+ out += [f"- {line}" for line in updated] or ["None."]
950
+ number += 1
951
+ for heading, lines in (others or {}).items():
952
+ if lines:
953
+ out += ["", f"## {number}. {heading}", ""] + [f"- {line}" for line in lines]
954
+ number += 1
955
+ if skipped:
956
+ out += ["", "### Skipped", "",
957
+ "Each needs an audit before the nightly takes it. cargo vet recommends, as `cargo vet suggest` would:", ""]
958
+ for s in skipped:
959
+ out += [f"<details><summary>{s['crate']} {s['from']} -> {s['to']}</summary>", "",
960
+ "```", s["why"], "```", "", "</details>", ""]
961
+ return "\n".join(out).rstrip("\n") + "\n"
962
+
963
+
964
+ def nightly(root: Path, pubrel: str, body_path: Path) -> int:
965
+ """The nightly's commits on the current branch, and the pull request's
966
+ body. Prints the commits it made, in order: `pins` first when it made
967
+ that one, then any of `actions`, `crates`, `npm` and `python`, or nothing."""
968
+ import tempfile # noqa: PLC0415
969
+
970
+ body_path = confined(body_path, "the --body file")
971
+ update = settings(root)["update"]
972
+ pinned = pins(root, pubrel)
973
+ made = ["pins"] if commit(root, "Move to the latest pub* releases", pinned) else []
974
+ result: dict = {"updated": [], "skipped": []}
975
+ others: dict[str, list[str]] = {}
976
+ if update["actions"]:
977
+ heading = "GitHub Actions, to their latest releases"
978
+ others[heading] = actions(root)
979
+ if commit(root, "Move GitHub Actions to their latest releases", others[heading]):
980
+ made.append("actions")
981
+ if update["rust"]:
982
+ with tempfile.TemporaryDirectory() as tmp:
983
+ report = Path(tmp) / "crates.json"
984
+ with contextlib.redirect_stdout(io.StringIO()):
985
+ crates(root, report)
986
+ result = json.loads(report.read_text(encoding="utf-8"))
987
+ if commit(root, "Update other crates the supply-chain audits cover", result["updated"]):
988
+ made.append("crates")
989
+ for language, heading, update_with, title, name in (
990
+ ("node", "npm packages, within their ranges in package.json", npm,
991
+ "Update npm packages within their ranges", "npm"),
992
+ ("python", "Python packages, re-locked from requirements.txt", python,
993
+ "Update the locked Python packages", "python")):
994
+ if update[language]:
995
+ others[heading] = update_with(root)
996
+ if commit(root, title, others[heading]):
997
+ made.append(name)
998
+ body_path.write_text(body(pinned, result["updated"], result["skipped"], others,
999
+ any(update.values())),
1000
+ encoding="utf-8")
1001
+ print(" ".join(made))
1002
+ return 0