pubrules 0.48.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- pubrules/__init__.py +14 -0
- pubrules/audit.py +116 -0
- pubrules/audit_draft.py +295 -0
- pubrules/audit_prepare.py +1128 -0
- pubrules/cache.py +319 -0
- pubrules/cbor.py +45 -0
- pubrules/ci.py +910 -0
- pubrules/cli.py +1069 -0
- pubrules/conform.py +295 -0
- pubrules/doc_coverage.py +235 -0
- pubrules/freshness.py +75 -0
- pubrules/gherkin.py +348 -0
- pubrules/hooks.py +59 -0
- pubrules/hosting.py +372 -0
- pubrules/ignores.py +86 -0
- pubrules/new.py +242 -0
- pubrules/nightly.py +1002 -0
- pubrules/paths.py +32 -0
- pubrules/protect.py +596 -0
- pubrules/providers.py +129 -0
- pubrules/refs.py +213 -0
- pubrules/release-commit +1 -0
- pubrules/releases.py +107 -0
- pubrules/repos.py +67 -0
- pubrules/ruleset.json +43 -0
- pubrules/secrets_config.py +165 -0
- pubrules/security.py +134 -0
- pubrules/service_needs.py +467 -0
- pubrules/standard.json +635 -0
- pubrules/standard.py +266 -0
- pubrules/supply_chain.py +188 -0
- pubrules/tag-ruleset.json +14 -0
- pubrules/templates/common/AGENTS.md +32 -0
- pubrules/templates/common/pubrules +26 -0
- pubrules/templates/common/tests/features/repository.feature +11 -0
- pubrules/templates/common/tests/install-lock +65 -0
- pubrules/templates/common/tests/pytest.ini +4 -0
- pubrules/templates/common/tests/requirements-pubrules.txt +17 -0
- pubrules/templates/common/tests/requirements.txt +3 -0
- pubrules/templates/common/tests/steps/test_repository.py +14 -0
- pubrules/templates/deploy/pubdeploy +28 -0
- pubrules/templates/new-rust/CHANGELOG.md +3 -0
- pubrules/templates/new-rust/Cargo.toml +46 -0
- pubrules/templates/new-rust/LICENSE +202 -0
- pubrules/templates/new-rust/Makefile +18 -0
- pubrules/templates/new-rust/README.md +34 -0
- pubrules/templates/new-rust/SECURITY.md +26 -0
- pubrules/templates/new-rust/corpus/corpus.json +11 -0
- pubrules/templates/new-rust/deny.toml +32 -0
- pubrules/templates/new-rust/fuzz/Cargo.toml +23 -0
- pubrules/templates/new-rust/fuzz/fuzz_targets/greet.rs +8 -0
- pubrules/templates/new-rust/fuzz/gitignore +4 -0
- pubrules/templates/new-rust/fuzz/regressions/README.md +4 -0
- pubrules/templates/new-rust/gitignore +10 -0
- pubrules/templates/new-rust/src/lib.rs +91 -0
- pubrules/templates/new-rust/src/main.rs +19 -0
- pubrules/templates/new-rust/supply-chain/audits.toml +4 -0
- pubrules/templates/new-rust/supply-chain/config.toml +23 -0
- pubrules/templates/new-rust/supply-chain/imports.lock +2 -0
- pubrules/templates/new-rust/tests/conftest.py +9 -0
- pubrules/templates/new-rust/tests/features/cli.feature +14 -0
- pubrules/templates/new-rust/tests/steps/test_cli.py +15 -0
- pubrules/templates/new-rust/tests/vectors.rs +30 -0
- pubrules/templates/new-rust/tools/vectors/generate.py +24 -0
- pubrules/templates/python/Makefile +5 -0
- pubrules/templates/python/pubrules-python.mk +47 -0
- pubrules/templates/rust/Makefile +9 -0
- pubrules/templates/rust/pubrules.mk +138 -0
- pubrules/templates/rust/rust-toolchain.toml +8 -0
- pubrules/templates/service/pubsec +27 -0
- pubrules/templates/service/with-secrets +20 -0
- pubrules/templates/service/with-secrets.json +8 -0
- pubrules/variables.py +45 -0
- pubrules-0.48.0.dist-info/METADATA +1241 -0
- pubrules-0.48.0.dist-info/RECORD +79 -0
- pubrules-0.48.0.dist-info/WHEEL +5 -0
- pubrules-0.48.0.dist-info/entry_points.txt +2 -0
- pubrules-0.48.0.dist-info/licenses/LICENSE +202 -0
- pubrules-0.48.0.dist-info/top_level.txt +1 -0
pubrules/providers.py
ADDED
|
@@ -0,0 +1,129 @@
|
|
|
1
|
+
"""deploy.providers: the packages a deploy block's walk runs with, among them
|
|
2
|
+
pubsec and pubdeploy (`ensure`, in `pubrules sync`). pubrules holds no deploy
|
|
3
|
+
code and no version of either: pubdeploy walks, and its sustenaut.bump hook
|
|
4
|
+
moves each provider to its latest release.
|
|
5
|
+
|
|
6
|
+
The pubsec and pubdeploy the launchers run are the repository's own pins:
|
|
7
|
+
deploy.providers' entry, else the one its launcher already fetches, else
|
|
8
|
+
(a launcher sync writes for the first time) the latest release
|
|
9
|
+
(releases.py).
|
|
10
|
+
|
|
11
|
+
pubrules.json is changed in place, the entry added and nothing else, so its
|
|
12
|
+
layout stays the repository's.
|
|
13
|
+
"""
|
|
14
|
+
|
|
15
|
+
from __future__ import annotations
|
|
16
|
+
|
|
17
|
+
import json
|
|
18
|
+
import re
|
|
19
|
+
from pathlib import Path
|
|
20
|
+
from typing import Any
|
|
21
|
+
|
|
22
|
+
WHEEL = re.compile(r"https://github\.com/(?P<repo>[A-Za-z0-9-]+/[A-Za-z0-9_.-]+)/releases/download/[^/\s]+/"
|
|
23
|
+
r"(?P<name>[A-Za-z0-9_.]+)-(?P<version>[^-/\s]+)-py3-none-any\.whl")
|
|
24
|
+
PUBSEC = "pubsec"
|
|
25
|
+
PUBDEPLOY = "pubdeploy"
|
|
26
|
+
# Why each of the two is listed, for the line sync --check prints.
|
|
27
|
+
WHY = {PUBSEC: "the types that hold a secret's value are pubsec's",
|
|
28
|
+
PUBDEPLOY: "the deploy walk, and the check sync runs of the block, are pubdeploy's"}
|
|
29
|
+
# The wheel a launcher fetches and runs, as the managed launchers write it.
|
|
30
|
+
FETCHED = re.compile(r"^fetch (?P<wheel>\S+) (?P<sha256>[0-9a-f]{64}) providers deploy\.providers$", re.M)
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
def listed(config: dict[str, Any], name: str) -> dict[str, str] | None:
|
|
34
|
+
"""deploy.providers' entry for `name`, if it lists one."""
|
|
35
|
+
deploy = config.get("deploy")
|
|
36
|
+
given = deploy.get("providers") if isinstance(deploy, dict) else None
|
|
37
|
+
return next((p for p in given or [] if isinstance(p, dict) and p.get("distribution") == name), None)
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
def sound(given: dict[str, str] | None) -> bool:
|
|
41
|
+
return given is not None and isinstance(given.get("wheel"), str) and isinstance(given.get("sha256"), str) \
|
|
42
|
+
and bool(given["sha256"]) and WHEEL.fullmatch(given["wheel"]) is not None
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
def in_launcher(root: Path | None, name: str) -> dict[str, str] | None:
|
|
46
|
+
"""The wheel the repository's ./`name` fetches, as sync last wrote it."""
|
|
47
|
+
path = root / name if root is not None else None
|
|
48
|
+
if path is None or not path.is_file():
|
|
49
|
+
return None
|
|
50
|
+
m = FETCHED.search(path.read_text(encoding="utf-8"))
|
|
51
|
+
return {"distribution": name, "wheel": m["wheel"], "sha256": m["sha256"]} if m else None
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
def pinned(root: Path | None, config: dict[str, Any], name: str) -> dict[str, str]:
|
|
55
|
+
"""The `name` the repository pins: deploy.providers', else its launcher's,
|
|
56
|
+
else the latest release."""
|
|
57
|
+
given = listed(config, name)
|
|
58
|
+
if sound(given):
|
|
59
|
+
return given
|
|
60
|
+
found = in_launcher(root, name)
|
|
61
|
+
if found is not None:
|
|
62
|
+
return found
|
|
63
|
+
from .releases import wheel # noqa: PLC0415
|
|
64
|
+
return wheel(name)
|
|
65
|
+
|
|
66
|
+
|
|
67
|
+
def ensure(root: Path, config: dict[str, Any], check: bool) -> tuple[list[str], bool]:
|
|
68
|
+
"""A repository with a deploy block lists pubsec and pubdeploy in
|
|
69
|
+
deploy.providers. The pins are the repository's own (`pubrules bump`
|
|
70
|
+
moves them, through pubdeploy's hook): `sync` adds one where none is
|
|
71
|
+
listed, the one its launcher runs or else the latest release, changing
|
|
72
|
+
nothing else of pubrules.json; `--check` fails instead. Lines to print,
|
|
73
|
+
and whether that fails."""
|
|
74
|
+
if not isinstance(config.get("deploy"), dict):
|
|
75
|
+
return [], False
|
|
76
|
+
out, failed = [], False
|
|
77
|
+
for name in (PUBSEC, PUBDEPLOY):
|
|
78
|
+
lines, bad = ensure_one(root, config, name, check)
|
|
79
|
+
out += lines
|
|
80
|
+
failed = failed or bad
|
|
81
|
+
if lines and not check:
|
|
82
|
+
config = json.loads((root / "pubrules.json").read_text(encoding="utf-8"))
|
|
83
|
+
return out, failed
|
|
84
|
+
|
|
85
|
+
|
|
86
|
+
def ensure_one(root: Path, config: dict[str, Any], name: str, check: bool) -> tuple[list[str], bool]:
|
|
87
|
+
given = listed(config, name)
|
|
88
|
+
if sound(given):
|
|
89
|
+
return [], False
|
|
90
|
+
said = f"does not list {name}" if given is None else f"lists {name} without a release wheel and its sha256"
|
|
91
|
+
if check:
|
|
92
|
+
return [f"pubrules: deploy.providers {said}: {WHY[name]}; run `pubrules sync`, which lists "
|
|
93
|
+
f"the one ./{name} runs, or its latest release"], True
|
|
94
|
+
want = pinned(root, {}, name)
|
|
95
|
+
path = root / "pubrules.json"
|
|
96
|
+
text = path.read_text(encoding="utf-8")
|
|
97
|
+
if given is not None: # a hand-written entry, in another shape: written whole
|
|
98
|
+
data = json.loads(text)
|
|
99
|
+
data["deploy"]["providers"] = [want if p == given else p for p in data["deploy"]["providers"]]
|
|
100
|
+
text = json.dumps(data, indent=2) + "\n"
|
|
101
|
+
else:
|
|
102
|
+
text = inserted(text, json.dumps(want), want)
|
|
103
|
+
path.write_text(text, encoding="utf-8")
|
|
104
|
+
return [f"updated pubrules.json (deploy.providers {name}: {WHEEL.fullmatch(want['wheel'])['version']})"], False
|
|
105
|
+
|
|
106
|
+
|
|
107
|
+
def inserted(text: str, entry: str, want: dict[str, str]) -> str:
|
|
108
|
+
"""pubrules.json with the entry added to the deploy block's providers, the
|
|
109
|
+
rest of the text as it was. A text edit is kept only when the file then
|
|
110
|
+
reads as the same JSON with that one entry more; otherwise it is written
|
|
111
|
+
whole."""
|
|
112
|
+
expected = json.loads(text)
|
|
113
|
+
expected["deploy"]["providers"] = [want, *expected["deploy"].get("providers", [])]
|
|
114
|
+
candidates = []
|
|
115
|
+
deploy = re.search(r'"deploy"\s*:\s*\{(?P<space>\s*)', text)
|
|
116
|
+
if deploy:
|
|
117
|
+
listed = re.compile(r'"providers"\s*:\s*\[').search(text, deploy.end())
|
|
118
|
+
if listed:
|
|
119
|
+
rest = text[listed.end():]
|
|
120
|
+
candidates.append(text[:listed.end()] + entry + ("" if rest.lstrip().startswith("]") else ", ") + rest)
|
|
121
|
+
space = deploy["space"] if "\n" in deploy["space"] else " "
|
|
122
|
+
candidates.append(text[:deploy.end()] + f'"providers": [{entry}],{space}' + text[deploy.end():])
|
|
123
|
+
for candidate in candidates:
|
|
124
|
+
try:
|
|
125
|
+
if json.loads(candidate) == expected:
|
|
126
|
+
return candidate
|
|
127
|
+
except ValueError:
|
|
128
|
+
continue
|
|
129
|
+
return json.dumps(expected, indent=2) + "\n"
|
pubrules/refs.py
ADDED
|
@@ -0,0 +1,213 @@
|
|
|
1
|
+
"""How a workflow pins the reusable workflows pubrules writes calls to: by
|
|
2
|
+
the full commit of the release's tag, with the version in a comment,
|
|
3
|
+
|
|
4
|
+
uses: <owner>/pubrules/.github/workflows/job-rust-test.yml@<commit> # v0.15.2
|
|
5
|
+
|
|
6
|
+
so the pin cannot move with the tag (SonarQube Cloud's S7637) and still
|
|
7
|
+
reads as a version. Every reader here also takes the tag alone, `@v0.15.2`,
|
|
8
|
+
as workflows written before pinned.
|
|
9
|
+
|
|
10
|
+
pubrules' own commit is in its wheel: the release CI writes the tag's
|
|
11
|
+
commit into the package, as `release-commit`, before it builds the wheel,
|
|
12
|
+
so `pubrules sync` needs no network for it. A source tree has none, and pins
|
|
13
|
+
by tag. pubrel's is the one a workflow already pins at that version, or
|
|
14
|
+
else read once from pubrel's tags with `git ls-remote`; when that cannot be
|
|
15
|
+
read either, the pin is by tag, with a warning.
|
|
16
|
+
|
|
17
|
+
Whose pubrules and pubrel these are is OWNER, set here and nowhere else; a
|
|
18
|
+
repository's pubrules.json can name another as `workflows_owner`. Every
|
|
19
|
+
writer pins to that owner, and every reader takes any owner, so a
|
|
20
|
+
workflow still pinned to an earlier one reads as before.
|
|
21
|
+
"""
|
|
22
|
+
|
|
23
|
+
from __future__ import annotations
|
|
24
|
+
|
|
25
|
+
import functools
|
|
26
|
+
import json
|
|
27
|
+
import os
|
|
28
|
+
import re
|
|
29
|
+
import subprocess
|
|
30
|
+
import sys
|
|
31
|
+
from collections.abc import Callable
|
|
32
|
+
from importlib import resources
|
|
33
|
+
from pathlib import Path
|
|
34
|
+
from typing import Any
|
|
35
|
+
|
|
36
|
+
# The owner of pubrules' and pubrel's repositories, whose workflows a
|
|
37
|
+
# repository calls, since their transfer from schryer.
|
|
38
|
+
OWNER = "Sustenaut"
|
|
39
|
+
OWNER_NAME = re.compile(r"[A-Za-z0-9](?:[A-Za-z0-9-]{0,37}[A-Za-z0-9])?")
|
|
40
|
+
# Any owner, as readers take it, before `/pubrules/` or `/pubrel/`.
|
|
41
|
+
ANY_OWNER = r"(?P<owner>[A-Za-z0-9][A-Za-z0-9-]*)/"
|
|
42
|
+
|
|
43
|
+
COMMIT = re.compile(r"[0-9a-f]{40}")
|
|
44
|
+
# What follows `@` up to the version: `v`, or a commit and `# v`.
|
|
45
|
+
AT = r"(?:v|(?P<commit>[0-9a-f]{40})[ \t]+#[ \t]*v)"
|
|
46
|
+
VERSION = r"(?P<version>[0-9][^\s'\"]*)"
|
|
47
|
+
|
|
48
|
+
RELEASE_PIN = re.compile(ANY_OWNER + r"(?P<path>pubrel/\.github/workflows/release\.yml@)"
|
|
49
|
+
+ AT + VERSION)
|
|
50
|
+
# Any use of pubrules at a release: one of its actions (actions/<name>, such
|
|
51
|
+
# as cache) or of its reusable workflows (.github/workflows/<name>,
|
|
52
|
+
# such as service-ci.yml or job-sonar.yml), by commit with the version in a
|
|
53
|
+
# comment, or by tag. It runs that pubrules' code, so it moves with the
|
|
54
|
+
# pubrules the lock pins, written by sync or by hand.
|
|
55
|
+
PUBRULES_PIN = re.compile(ANY_OWNER + r"(?P<path>pubrules/(?P<name>actions/[\w.-]+|"
|
|
56
|
+
r"\.github/workflows/[\w.-]+\.ya?ml)@)" + AT + VERSION)
|
|
57
|
+
# Any use of pubrules' actions or reusable workflows, versioned or not.
|
|
58
|
+
PUBRULES_USE = re.compile(r"[A-Za-z0-9][A-Za-z0-9-]*/pubrules/(?:actions/[\w.-]+|"
|
|
59
|
+
r"\.github/workflows/[\w.-]+\.ya?ml)@[^\s'\"]+")
|
|
60
|
+
|
|
61
|
+
|
|
62
|
+
# What this pubrules ships for other repositories to use by commit: its
|
|
63
|
+
# actions (actions/<name>), its reusable workflows (.github/workflows/<name>)
|
|
64
|
+
# and its Terraform modules (terraform/<name>). The installed wheel carries
|
|
65
|
+
# none of these files, so the names are listed here; a test holds the list
|
|
66
|
+
# to the repository's tree. A pin of a name not listed is to a pubrules that
|
|
67
|
+
# lacks it, which `pins` never moves to and `sync --check` refuses.
|
|
68
|
+
SHIPPED = {
|
|
69
|
+
"actions": ("apt-install", "cache", "pr-label", "retry", "terraform-init"),
|
|
70
|
+
"workflows": (
|
|
71
|
+
"job-node-audit.yml", "job-node-browser.yml", "job-node-test.yml", "job-osv.yml",
|
|
72
|
+
"job-python-audit.yml", "job-python-browser.yml", "job-python-functional.yml",
|
|
73
|
+
"job-review.yml", "job-rust-audit.yml", "job-rust-cross.yml", "job-rust-functional.yml",
|
|
74
|
+
"job-rust-fuzz.yml", "job-rust-gates.yml", "job-rust-msrv.yml", "job-rust-test.yml",
|
|
75
|
+
"job-rust-toolchains.yml", "job-service-checks.yml", "job-service-terraform.yml",
|
|
76
|
+
"job-service-tests.yml", "job-sonar.yml", "job-tested.yml", "service-needs.yml", "node-ci.yml",
|
|
77
|
+
"python-ci.yml", "rust-ci.yml", "service-ci.yml",
|
|
78
|
+
),
|
|
79
|
+
"terraform": (),
|
|
80
|
+
}
|
|
81
|
+
|
|
82
|
+
# A Terraform module source in another repository at a release: a
|
|
83
|
+
# `source` naming github.com, the owner and the repository, then `//` and
|
|
84
|
+
# the path, with `?ref=`: by tag (`?ref=vX.Y.Z`), or by commit with the
|
|
85
|
+
# version in a comment after the closing quote.
|
|
86
|
+
TF_SOURCE = re.compile(
|
|
87
|
+
r'(?P<lead>source\s*=\s*)"github\.com/' + ANY_OWNER + r'(?P<repo>[A-Za-z0-9_.-]+)//(?P<path>[^"?\s]+)'
|
|
88
|
+
r'\?ref=(?:v(?P<tag>[0-9][^"\s]*)|(?P<commit>[0-9a-f]{40}))"(?:[ \t]*#[ \t]*v(?P<version>[0-9][^\s]*))?')
|
|
89
|
+
|
|
90
|
+
|
|
91
|
+
def shipped(name: str) -> bool:
|
|
92
|
+
"""Whether a pubrules pin's name (actions/<a> or .github/workflows/<w>) is one this pubrules ships."""
|
|
93
|
+
kind, _, base = name.rpartition("/")
|
|
94
|
+
return base in SHIPPED["actions" if kind == "actions" else "workflows"]
|
|
95
|
+
|
|
96
|
+
|
|
97
|
+
def tf_files(root: Path) -> list[Path]:
|
|
98
|
+
"""The repository's own .tf files, outside Terraform's working directories."""
|
|
99
|
+
return sorted(p for p in root.rglob("*.tf")
|
|
100
|
+
if not any(part.startswith(".") for part in p.relative_to(root).parts[:-1]))
|
|
101
|
+
|
|
102
|
+
|
|
103
|
+
def stale_pins(root: Path, version: str, commit: str | None) -> tuple[list[str], list[str]]:
|
|
104
|
+
"""(the uses of pubrules' actions and reusable workflows in the
|
|
105
|
+
repository's workflows that pin another pubrules than `version`, or
|
|
106
|
+
another commit than `commit`; those that name no version, which nothing
|
|
107
|
+
here moves)."""
|
|
108
|
+
stale, unversioned = [], []
|
|
109
|
+
for path in sorted((root / ".github" / "workflows").glob("*.y*ml")):
|
|
110
|
+
text, rel = path.read_text(encoding="utf-8"), path.relative_to(root).as_posix()
|
|
111
|
+
pinned = {m.start() for m in PUBRULES_PIN.finditer(text)}
|
|
112
|
+
for m in PUBRULES_PIN.finditer(text):
|
|
113
|
+
name = m["name"].rsplit("/", 1)[-1]
|
|
114
|
+
if not shipped(m["name"]):
|
|
115
|
+
stale.append(f"{rel}: pubrules {version} has no {m['name']}; it moved out of "
|
|
116
|
+
f"pubrules (pubrules' README says where), so pin it from there")
|
|
117
|
+
elif m["version"] != version:
|
|
118
|
+
stale.append(f"{rel}: {name}@v{m['version']} is not pubrules {version}, which "
|
|
119
|
+
f"tests/requirements.lock pins; run `pubrules bump`, or pin it "
|
|
120
|
+
f"@{ref(version, commit)}")
|
|
121
|
+
elif commit and m["commit"] and m["commit"] != commit:
|
|
122
|
+
stale.append(f"{rel}: {name}@{m['commit']} is not the commit of pubrules "
|
|
123
|
+
f"{version}'s tag; pin it @{ref(version, commit)}")
|
|
124
|
+
unversioned += [f"{rel}: {m[0]}" for m in PUBRULES_USE.finditer(text)
|
|
125
|
+
if m.start() not in pinned]
|
|
126
|
+
for path in tf_files(root):
|
|
127
|
+
rel = path.relative_to(root).as_posix()
|
|
128
|
+
for m in TF_SOURCE.finditer(path.read_text(encoding="utf-8")):
|
|
129
|
+
module = m["path"].split("/")
|
|
130
|
+
if m["repo"].lower() == "pubrules" and not (
|
|
131
|
+
len(module) == 2 and module[0] == "terraform" and module[1] in SHIPPED["terraform"]):
|
|
132
|
+
stale.append(f"{rel}: pubrules {version} has no {m['path']}; it moved out of pubrules "
|
|
133
|
+
f"(pubrules' README says where), so take the module from there")
|
|
134
|
+
return stale, unversioned
|
|
135
|
+
|
|
136
|
+
|
|
137
|
+
def owner(config: dict[str, Any] | None) -> str:
|
|
138
|
+
"""The owner pubrules.json's `workflows_owner` names, or OWNER."""
|
|
139
|
+
given = (config or {}).get("workflows_owner", OWNER)
|
|
140
|
+
if not isinstance(given, str) or not OWNER_NAME.fullmatch(given):
|
|
141
|
+
sys.exit(f"pubrules: pubrules.json's workflows_owner must be a GitHub owner, such as "
|
|
142
|
+
f'"{OWNER}", not {given!r}')
|
|
143
|
+
return given
|
|
144
|
+
|
|
145
|
+
|
|
146
|
+
def owner_of(root: Path) -> str:
|
|
147
|
+
"""The owner for the repository at `root`, from its pubrules.json."""
|
|
148
|
+
path = root / "pubrules.json"
|
|
149
|
+
return owner(json.loads(path.read_text(encoding="utf-8")) if path.is_file() else None)
|
|
150
|
+
|
|
151
|
+
|
|
152
|
+
def ref(version: str, commit: str | None) -> str:
|
|
153
|
+
"""What follows `@` in a pin of `version`: its commit, with the version
|
|
154
|
+
in a comment, or the tag when the commit is unknown."""
|
|
155
|
+
return f"{commit} # v{version}" if commit else f"v{version}"
|
|
156
|
+
|
|
157
|
+
|
|
158
|
+
def repin(pattern: re.Pattern[str], text: str, owner: str, version: str,
|
|
159
|
+
commit: str | None, keep: Callable[[re.Match[str]], bool] | None = None) -> str:
|
|
160
|
+
"""`text` with every pin `pattern` matches (its `path`, after the
|
|
161
|
+
owner, through `@`) moved to `owner`'s, at `version`; one `keep` says
|
|
162
|
+
stays as it is."""
|
|
163
|
+
return pattern.sub(lambda m: m.group(0) if keep and keep(m)
|
|
164
|
+
else f"{owner}/{m['path']}{ref(version, commit)}", text)
|
|
165
|
+
|
|
166
|
+
|
|
167
|
+
def release_commit() -> str | None:
|
|
168
|
+
"""The commit of this pubrules' release tag, from the wheel; None from a
|
|
169
|
+
source tree. PUBRULES_RELEASE_COMMIT, which tests set, stands in for it."""
|
|
170
|
+
given = os.environ.get("PUBRULES_RELEASE_COMMIT")
|
|
171
|
+
if given is None:
|
|
172
|
+
try:
|
|
173
|
+
given = (resources.files("pubrules") / "release-commit").read_text(encoding="utf-8")
|
|
174
|
+
except FileNotFoundError:
|
|
175
|
+
return None
|
|
176
|
+
given = given.strip()
|
|
177
|
+
if not given:
|
|
178
|
+
return None
|
|
179
|
+
if not COMMIT.fullmatch(given):
|
|
180
|
+
sys.exit(f"pubrules: its release commit must be 40 hexadecimal digits, not {given!r}")
|
|
181
|
+
return given
|
|
182
|
+
|
|
183
|
+
|
|
184
|
+
def pubrel_commit(version: str, workflows: str, owner: str) -> str | None:
|
|
185
|
+
"""The commit of pubrel's tag v`version`: already pinned in
|
|
186
|
+
`workflows`, or read from the tags of `owner`'s pubrel
|
|
187
|
+
(PUBRULES_PUBREL_REMOTE, which tests set, in place of it). A transfer
|
|
188
|
+
keeps the commits, so one pinned under another owner serves."""
|
|
189
|
+
for m in RELEASE_PIN.finditer(workflows):
|
|
190
|
+
if m["commit"] and m["version"] == version:
|
|
191
|
+
return m["commit"]
|
|
192
|
+
remote = os.environ.get("PUBRULES_PUBREL_REMOTE", f"https://github.com/{owner}/pubrel")
|
|
193
|
+
return tagged(remote, f"v{version}")
|
|
194
|
+
|
|
195
|
+
|
|
196
|
+
@functools.cache
|
|
197
|
+
def tagged(remote: str, tag: str) -> str | None:
|
|
198
|
+
"""The commit `tag` names in `remote`, peeled; None, with a warning,
|
|
199
|
+
when it cannot be read."""
|
|
200
|
+
try:
|
|
201
|
+
out = subprocess.run(["git", "ls-remote", "--tags", remote, f"refs/tags/{tag}",
|
|
202
|
+
f"refs/tags/{tag}^{{}}"], capture_output=True, text=True,
|
|
203
|
+
timeout=60, check=False)
|
|
204
|
+
except (OSError, subprocess.TimeoutExpired):
|
|
205
|
+
out = None
|
|
206
|
+
found = dict(reversed(line.split("\t", 1)) for line in out.stdout.splitlines()
|
|
207
|
+
if "\t" in line) if out and out.returncode == 0 else {}
|
|
208
|
+
commit = found.get(f"refs/tags/{tag}^{{}}") or found.get(f"refs/tags/{tag}")
|
|
209
|
+
if commit and COMMIT.fullmatch(commit):
|
|
210
|
+
return commit
|
|
211
|
+
print(f"pubrules: could not read the commit of {tag} from {remote}; pinning it by tag",
|
|
212
|
+
file=sys.stderr)
|
|
213
|
+
return None
|
pubrules/release-commit
ADDED
|
@@ -0,0 +1 @@
|
|
|
1
|
+
cc8eaf673afb86eb78fa9808e5028ae7f6fa187c
|
pubrules/releases.py
ADDED
|
@@ -0,0 +1,107 @@
|
|
|
1
|
+
"""The releases of the packages a repository's pins name, where it pins none yet.
|
|
2
|
+
|
|
3
|
+
pubrules holds no other package's version. A repository pins each one
|
|
4
|
+
itself: pubrel in ci.release.pubrel (or the release workflow sync last
|
|
5
|
+
wrote), pubsec and pubdeploy in deploy.providers (or the launcher sync last
|
|
6
|
+
wrote), pubtest in tests/requirements.txt. The nightly moves each pin to the
|
|
7
|
+
package's latest release. Only a pin a repository does not have yet, as when
|
|
8
|
+
`pubrules new` or `init` starts one, or sync writes a launcher for the first
|
|
9
|
+
time, is read here: from the release-state file PUBRULES_RELEASES names, when
|
|
10
|
+
its `releases` block lists the package, otherwise from the package's latest
|
|
11
|
+
GitHub release.
|
|
12
|
+
|
|
13
|
+
{"releases": {"pubrel": "0.8.0", "pubtest": "0.2.2",
|
|
14
|
+
"pubsec": {"version": "0.3.0", "sha256": "..."}}}
|
|
15
|
+
|
|
16
|
+
A version alone, or with the sha256 of its wheel (without it, the wheel is
|
|
17
|
+
fetched and hashed) and the wheel's file name, where its distribution is
|
|
18
|
+
named otherwise than its repository (pubtest's is pubtestrules):
|
|
19
|
+
|
|
20
|
+
{"releases": {"pubtest": {"version": "0.3.1",
|
|
21
|
+
"wheel": "pubtestrules-0.3.1-py3-none-any.whl"}}}
|
|
22
|
+
|
|
23
|
+
A release the release state does not list says itself what its wheel is called.
|
|
24
|
+
"""
|
|
25
|
+
|
|
26
|
+
from __future__ import annotations
|
|
27
|
+
|
|
28
|
+
import json
|
|
29
|
+
import os
|
|
30
|
+
import re
|
|
31
|
+
import sys
|
|
32
|
+
from pathlib import Path
|
|
33
|
+
|
|
34
|
+
from .refs import OWNER
|
|
35
|
+
|
|
36
|
+
# The variable naming the release-state file, and the block it is read from.
|
|
37
|
+
STATE = "PUBRULES_RELEASES"
|
|
38
|
+
BLOCK = "releases"
|
|
39
|
+
# The packages a pin of which pubrules writes, where a repository has none.
|
|
40
|
+
PACKAGES = ("pubrel", "pubsec", "pubdeploy", "pubtest")
|
|
41
|
+
# A release's wheel. Its file is named by the distribution, which may differ
|
|
42
|
+
# from the repository (pubtest's is pubtestrules): wheel_url asks the release.
|
|
43
|
+
WHEEL = "https://github.com/{owner}/{name}/releases/download/v{version}/{file}"
|
|
44
|
+
RELEASE = re.compile(r"v?(?P<version>[0-9]+\.[0-9]+\.[0-9]+)")
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
def stated(name: str) -> dict[str, str] | None:
|
|
48
|
+
"""`name`'s entry in the release-state file, as {"version", "sha256"?}, or None."""
|
|
49
|
+
path = os.environ.get(STATE)
|
|
50
|
+
if not path:
|
|
51
|
+
return None
|
|
52
|
+
try:
|
|
53
|
+
data = json.loads(Path(path).read_text(encoding="utf-8"))
|
|
54
|
+
except (OSError, ValueError) as e:
|
|
55
|
+
sys.exit(f"pubrules: {STATE} names {path}, which cannot be read as JSON: {e}")
|
|
56
|
+
block = data.get(BLOCK) if isinstance(data, dict) else None
|
|
57
|
+
entry = block.get(name) if isinstance(block, dict) else None
|
|
58
|
+
if entry is None:
|
|
59
|
+
return None
|
|
60
|
+
if isinstance(entry, str):
|
|
61
|
+
entry = {"version": entry}
|
|
62
|
+
if (not isinstance(entry, dict) or not isinstance(entry.get("version"), str)
|
|
63
|
+
or not RELEASE.fullmatch(entry["version"])
|
|
64
|
+
or not isinstance(entry.get("sha256", ""), str)
|
|
65
|
+
or not isinstance(entry.get("wheel", ""), str)):
|
|
66
|
+
sys.exit(f'pubrules: {path}\'s {BLOCK}.{name} must be a version, such as "1.2.3", '
|
|
67
|
+
f'or {{"version": "1.2.3", "sha256": "...", "wheel": "..."}}')
|
|
68
|
+
return {**entry, "version": RELEASE.fullmatch(entry["version"])["version"]}
|
|
69
|
+
|
|
70
|
+
|
|
71
|
+
def version(name: str, owner: str = OWNER) -> str:
|
|
72
|
+
"""The release of `name` to pin: the release state's, else its latest GitHub release."""
|
|
73
|
+
entry = stated(name)
|
|
74
|
+
if entry is not None:
|
|
75
|
+
return entry["version"]
|
|
76
|
+
from .nightly import latest_release # noqa: PLC0415
|
|
77
|
+
tag = latest_release(f"{owner}/{name}")
|
|
78
|
+
m = RELEASE.fullmatch(tag or "")
|
|
79
|
+
if not m:
|
|
80
|
+
sys.exit(f"pubrules: no release of {name} to pin: neither {STATE}'s {BLOCK} block nor "
|
|
81
|
+
f"{owner}/{name}'s latest GitHub release names one")
|
|
82
|
+
return m["version"]
|
|
83
|
+
|
|
84
|
+
|
|
85
|
+
def wheel_url(name: str, version: str, owner: str = OWNER) -> str:
|
|
86
|
+
"""The URL of release `version` of `owner/name`'s wheel. Its file: the
|
|
87
|
+
release state's, where it lists that release with its wheel, else the
|
|
88
|
+
one the release carries; else (neither says) `name-version-py3-none-any.whl`."""
|
|
89
|
+
entry = stated(name) or {}
|
|
90
|
+
file = entry.get("wheel") if entry.get("version") == version else None
|
|
91
|
+
if not file:
|
|
92
|
+
from .nightly import release_wheel # noqa: PLC0415
|
|
93
|
+
file = release_wheel(f"{owner}/{name}", version)
|
|
94
|
+
file = file or f"{name}-{version}-py3-none-any.whl"
|
|
95
|
+
return WHEEL.format(owner=owner, name=name, version=version, file=file)
|
|
96
|
+
|
|
97
|
+
|
|
98
|
+
def wheel(name: str, owner: str = OWNER) -> dict[str, str]:
|
|
99
|
+
"""`name`'s release wheel to pin, as deploy.providers lists one."""
|
|
100
|
+
entry = stated(name) or {}
|
|
101
|
+
at = entry.get("version") or version(name, owner)
|
|
102
|
+
url = wheel_url(name, at, owner)
|
|
103
|
+
sha = entry.get("sha256")
|
|
104
|
+
if not sha:
|
|
105
|
+
from .cli import sha256_of # noqa: PLC0415
|
|
106
|
+
sha = sha256_of(url)
|
|
107
|
+
return {"distribution": name, "wheel": url, "sha256": sha}
|
pubrules/repos.py
ADDED
|
@@ -0,0 +1,67 @@
|
|
|
1
|
+
"""The repos file: which repositories pubrules acts on, from outside pubrules.
|
|
2
|
+
|
|
3
|
+
{"repositories": [
|
|
4
|
+
{"repo": "example-org/some-tool", "protect": true, "govern": false, "project": "tools"}
|
|
5
|
+
]}
|
|
6
|
+
|
|
7
|
+
`protect` says it should carry pubrules' rulesets, `govern` that a nightly
|
|
8
|
+
keeps it current, and `project` (optional) names a group on the board. How it
|
|
9
|
+
is protected stays in its own pubrules.json's protect block. `pubrules board`,
|
|
10
|
+
`pubrules protect` and `pubrules freshness` take one with `--repos FILE`, and the
|
|
11
|
+
reusable board.yml and nightly.yml take the same thing as a JSON input. pubrules
|
|
12
|
+
doesn't care who writes it.
|
|
13
|
+
"""
|
|
14
|
+
|
|
15
|
+
from __future__ import annotations
|
|
16
|
+
|
|
17
|
+
import json
|
|
18
|
+
import re
|
|
19
|
+
import sys
|
|
20
|
+
from pathlib import Path
|
|
21
|
+
from typing import Any
|
|
22
|
+
|
|
23
|
+
REPO = re.compile(r"[A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+")
|
|
24
|
+
KEYS = {"repo", "protect", "govern", "project"}
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
def problem(entry: Any) -> str | None:
|
|
28
|
+
"""What is wrong with one entry, or None."""
|
|
29
|
+
if not isinstance(entry, dict) or not REPO.fullmatch(str(entry.get("repo", ""))):
|
|
30
|
+
return f"{json.dumps(entry)} needs a repo, OWNER/NAME"
|
|
31
|
+
name = entry["repo"]
|
|
32
|
+
if entry.keys() - KEYS:
|
|
33
|
+
return f"{name}: {', '.join(sorted(entry.keys() - KEYS))} is not a setting; it takes " \
|
|
34
|
+
f"protect, govern and project"
|
|
35
|
+
for key in ("protect", "govern"):
|
|
36
|
+
if not isinstance(entry.get(key, False), bool):
|
|
37
|
+
return f"{name}: \"{key}\" must be true or false, not {json.dumps(entry[key])}"
|
|
38
|
+
if not isinstance(entry.get("project", ""), str):
|
|
39
|
+
return f"{name}: \"project\" must be a name, not {json.dumps(entry['project'])}"
|
|
40
|
+
return None
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
def entries(text: str | bytes, where: str) -> list[dict]:
|
|
44
|
+
"""The entries of a repos file's contents, with `protect` and `govern`
|
|
45
|
+
always there, or a ValueError saying what is wrong."""
|
|
46
|
+
try:
|
|
47
|
+
listed = json.loads(text)["repositories"]
|
|
48
|
+
except (ValueError, KeyError, TypeError) as e:
|
|
49
|
+
raise ValueError(f"{where} needs a \"repositories\" list ({e})") from e
|
|
50
|
+
if not isinstance(listed, list):
|
|
51
|
+
raise ValueError(f"{where}'s \"repositories\" is not a list")
|
|
52
|
+
for entry in listed:
|
|
53
|
+
if (bad := problem(entry)) is not None:
|
|
54
|
+
raise ValueError(f"{where}: {bad}")
|
|
55
|
+
names = [e["repo"] for e in listed]
|
|
56
|
+
twice = sorted({n for n in names if names.count(n) > 1})
|
|
57
|
+
if twice:
|
|
58
|
+
raise ValueError(f"{where} lists {', '.join(twice)} more than once")
|
|
59
|
+
return [{"protect": False, "govern": False, **e} for e in listed]
|
|
60
|
+
|
|
61
|
+
|
|
62
|
+
def read(path: Path, command: str) -> list[dict]:
|
|
63
|
+
"""The entries of the repos file at `path`, or stop with what is wrong."""
|
|
64
|
+
try:
|
|
65
|
+
return entries(path.read_text(encoding="utf-8"), str(path))
|
|
66
|
+
except (OSError, ValueError) as e:
|
|
67
|
+
sys.exit(f"pubrules {command}: {e}")
|
pubrules/ruleset.json
ADDED
|
@@ -0,0 +1,43 @@
|
|
|
1
|
+
{
|
|
2
|
+
"name": "main changes by pull request, after CI",
|
|
3
|
+
"target": "branch",
|
|
4
|
+
"enforcement": "active",
|
|
5
|
+
"conditions": { "ref_name": { "exclude": [], "include": ["~DEFAULT_BRANCH"] } },
|
|
6
|
+
"rules": [
|
|
7
|
+
{ "type": "deletion" },
|
|
8
|
+
{ "type": "non_fast_forward" },
|
|
9
|
+
{
|
|
10
|
+
"type": "pull_request",
|
|
11
|
+
"parameters": {
|
|
12
|
+
"required_approving_review_count": 0,
|
|
13
|
+
"dismiss_stale_reviews_on_push": false,
|
|
14
|
+
"required_reviewers": [],
|
|
15
|
+
"require_code_owner_review": false,
|
|
16
|
+
"require_last_push_approval": false,
|
|
17
|
+
"required_review_thread_resolution": false,
|
|
18
|
+
"allowed_merge_methods": ["merge", "squash", "rebase"]
|
|
19
|
+
}
|
|
20
|
+
},
|
|
21
|
+
{
|
|
22
|
+
"type": "required_status_checks",
|
|
23
|
+
"parameters": {
|
|
24
|
+
"strict_required_status_checks_policy": false,
|
|
25
|
+
"do_not_enforce_on_create": false,
|
|
26
|
+
"required_status_checks": []
|
|
27
|
+
}
|
|
28
|
+
},
|
|
29
|
+
{
|
|
30
|
+
"type": "merge_queue",
|
|
31
|
+
"parameters": {
|
|
32
|
+
"merge_method": "MERGE",
|
|
33
|
+
"grouping_strategy": "ALLGREEN",
|
|
34
|
+
"max_entries_to_build": 5,
|
|
35
|
+
"min_entries_to_merge": 1,
|
|
36
|
+
"max_entries_to_merge": 5,
|
|
37
|
+
"min_entries_to_merge_wait_minutes": 5,
|
|
38
|
+
"check_response_timeout_minutes": 60
|
|
39
|
+
}
|
|
40
|
+
}
|
|
41
|
+
],
|
|
42
|
+
"bypass_actors": []
|
|
43
|
+
}
|