pubfleet 0.2.1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- pubfleet/__init__.py +23 -0
- pubfleet/board/__init__.py +113 -0
- pubfleet/board/collect.py +255 -0
- pubfleet/board/connectors/__init__.py +7 -0
- pubfleet/board/connectors/actions.py +65 -0
- pubfleet/board/connectors/base.py +193 -0
- pubfleet/board/connectors/github.py +144 -0
- pubfleet/board/connectors/live.py +17 -0
- pubfleet/board/connectors/repos.py +23 -0
- pubfleet/board/connectors/scorecard.py +24 -0
- pubfleet/board/connectors/sonarcloud.py +52 -0
- pubfleet/board/page.py +125 -0
- pubfleet/board/rules.json +37 -0
- pubfleet/board/rules.py +337 -0
- pubfleet/board_publish.py +134 -0
- pubfleet/cli.py +112 -0
- pubfleet/discover.py +110 -0
- pubfleet/freshness.py +88 -0
- pubfleet/state.py +198 -0
- pubfleet/variables.py +200 -0
- pubfleet-0.2.1.dist-info/METADATA +314 -0
- pubfleet-0.2.1.dist-info/RECORD +26 -0
- pubfleet-0.2.1.dist-info/WHEEL +5 -0
- pubfleet-0.2.1.dist-info/entry_points.txt +2 -0
- pubfleet-0.2.1.dist-info/licenses/LICENSE +202 -0
- pubfleet-0.2.1.dist-info/top_level.txt +1 -0
pubfleet/discover.py
ADDED
|
@@ -0,0 +1,110 @@
|
|
|
1
|
+
"""`pubfleet state --discover [FILE] [--body PATH]`: add the owner's repositories
|
|
2
|
+
a state file does not list yet.
|
|
3
|
+
|
|
4
|
+
It lists the owner's repositories, leaving out archived ones and forks, and gives
|
|
5
|
+
each one the file lacks an entry: on the board, protected when its own
|
|
6
|
+
pubrules.json has a protect block (`state --check` refuses protect without one), and
|
|
7
|
+
not governed, so its nightly waits until someone turns it on. No project and no
|
|
8
|
+
deploy. It rewrites the file as state files are kept (two-space JSON, repositories
|
|
9
|
+
in name order), writes a pull request's description to --body when given (what it
|
|
10
|
+
added, why protect is on or off, and the repositories the file lists that are
|
|
11
|
+
archived or gone, which it never removes), and prints how many it added.
|
|
12
|
+
|
|
13
|
+
`listing` and `entry` are the pieces a workflow that makes a repository reuses to
|
|
14
|
+
add the one it made.
|
|
15
|
+
"""
|
|
16
|
+
|
|
17
|
+
from __future__ import annotations
|
|
18
|
+
|
|
19
|
+
import json
|
|
20
|
+
import sys
|
|
21
|
+
from pathlib import Path
|
|
22
|
+
|
|
23
|
+
from pubrules.protect import Api, ApiError, gh, remote_config
|
|
24
|
+
|
|
25
|
+
from .state import shape
|
|
26
|
+
|
|
27
|
+
PAGE = 100
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
def listing(owner: str, api: Api = gh) -> list[dict]:
|
|
31
|
+
"""Every repository of the owner, archived ones and forks included, each
|
|
32
|
+
as GitHub's repository object."""
|
|
33
|
+
out: list[dict] = []
|
|
34
|
+
page = 1
|
|
35
|
+
while True:
|
|
36
|
+
batch = api("GET", f"orgs/{owner}/repos?type=all&per_page={PAGE}&page={page}", None) or []
|
|
37
|
+
out += batch
|
|
38
|
+
if len(batch) < PAGE:
|
|
39
|
+
return out
|
|
40
|
+
page += 1
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
def entry(owner: str, name: str, api: Api = gh) -> dict:
|
|
44
|
+
"""A new repository's entry: on the board, protected only where its own
|
|
45
|
+
pubrules.json says how, not governed."""
|
|
46
|
+
try:
|
|
47
|
+
config = remote_config(api, f"{owner}/{name}")
|
|
48
|
+
except ValueError:
|
|
49
|
+
config = None
|
|
50
|
+
return {"board": True, "protect": isinstance(config, dict) and "protect" in config, "govern": False}
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
def discover(state: dict, listed: list[dict], entry_for) -> tuple[dict, list[str], list[str]]:
|
|
54
|
+
"""`state` with an entry for each live repository it lacks, the names added,
|
|
55
|
+
and the names it lists that are archived or no longer there."""
|
|
56
|
+
live = {r["name"] for r in listed if not r.get("archived") and not r.get("fork")}
|
|
57
|
+
known = {r["name"] for r in listed}
|
|
58
|
+
archived = {r["name"] for r in listed if r.get("archived")}
|
|
59
|
+
repos = dict(state["repos"])
|
|
60
|
+
added = sorted(name for name in live if name not in repos)
|
|
61
|
+
for name in added:
|
|
62
|
+
repos[name] = entry_for(name)
|
|
63
|
+
gone = sorted(n for n in state["repos"] if n in archived or n not in known)
|
|
64
|
+
return {**state, "repos": dict(sorted(repos.items()))}, added, gone
|
|
65
|
+
|
|
66
|
+
|
|
67
|
+
def body(state: dict, added: list[str], gone: list[str], listed: list[dict]) -> str:
|
|
68
|
+
"""The description of the pull request that adds them."""
|
|
69
|
+
visibility = {r["name"]: r.get("visibility", "unknown") for r in listed}
|
|
70
|
+
lines = ["Repositories `state.json` did not list yet, found by `pubfleet state --discover`. "
|
|
71
|
+
"Each is on the board and not governed; protect is on where its own "
|
|
72
|
+
"`pubrules.json` has a protect block.", "",
|
|
73
|
+
"| Repository | Visibility | protect |", "|---|---|---|"]
|
|
74
|
+
for name in added:
|
|
75
|
+
why = ("on: its pubrules.json has a protect block" if state["repos"][name]["protect"]
|
|
76
|
+
else "off: no protect block in its pubrules.json")
|
|
77
|
+
lines.append(f"| {name} | {visibility.get(name, 'unknown')} | {why} |")
|
|
78
|
+
lines += ["", "Merging puts them on the board and, where protect is on, brings their "
|
|
79
|
+
"rulesets in line with pubrules' (`state / apply`). Set `govern`, `project` or "
|
|
80
|
+
"`deploy` here first if one should have them."]
|
|
81
|
+
if gone:
|
|
82
|
+
lines += ["", "Listed, but archived or no longer there (never removed by this): "
|
|
83
|
+
+ ", ".join(f"`{n}`" for n in gone) + "."]
|
|
84
|
+
return "\n".join(lines) + "\n"
|
|
85
|
+
|
|
86
|
+
|
|
87
|
+
def run(path: Path, body_path: Path | None = None, api: Api = gh) -> int:
|
|
88
|
+
"""Add the missing repositories to the state file at `path`."""
|
|
89
|
+
try:
|
|
90
|
+
state = json.loads(path.read_text(encoding="utf-8"))
|
|
91
|
+
except (OSError, ValueError) as e:
|
|
92
|
+
print(f"pubfleet: {path} can't be read ({e})", file=sys.stderr)
|
|
93
|
+
return 2
|
|
94
|
+
if problems := shape(state):
|
|
95
|
+
print(f"pubfleet: {path}: {'; '.join(problems)}", file=sys.stderr)
|
|
96
|
+
return 2
|
|
97
|
+
owner = state["owner"]
|
|
98
|
+
try:
|
|
99
|
+
listed = listing(owner, api)
|
|
100
|
+
except ApiError as e:
|
|
101
|
+
print(f"pubfleet: GitHub would not list {owner}'s repositories ({e})", file=sys.stderr)
|
|
102
|
+
return 2
|
|
103
|
+
new, added, gone = discover(state, listed, lambda name: entry(owner, name, api))
|
|
104
|
+
if added:
|
|
105
|
+
path.write_text(json.dumps(new, indent=2, ensure_ascii=False) + "\n", encoding="utf-8")
|
|
106
|
+
if body_path is not None and added:
|
|
107
|
+
body_path.write_text(body(new, added, gone, listed), encoding="utf-8")
|
|
108
|
+
print(len(added))
|
|
109
|
+
return 0
|
|
110
|
+
|
pubfleet/freshness.py
ADDED
|
@@ -0,0 +1,88 @@
|
|
|
1
|
+
"""`pubfleet freshness`: keep every pub* repository on the latest pubrules, so
|
|
2
|
+
none sits on an old standard unnoticed.
|
|
3
|
+
|
|
4
|
+
`pubfleet freshness DIR...` reports, for each checked-out repository, the
|
|
5
|
+
pubrules its lock pins, whether it is the installed one, whether it conforms to
|
|
6
|
+
that pubrules' standard, and its open exceptions: one Markdown table, also
|
|
7
|
+
appended to the GitHub step summary when there is one. It reads; it changes
|
|
8
|
+
nothing. `pubfleet freshness --repos FILE [DIR]` reports instead the
|
|
9
|
+
repositories a repos file marks to govern, each checked out at DIR/<name>.
|
|
10
|
+
The installed pubrules is the latest, in the nightly that runs this; moving a
|
|
11
|
+
repository to it is `pubrules bump`'s, and the nightly's.
|
|
12
|
+
"""
|
|
13
|
+
|
|
14
|
+
from __future__ import annotations
|
|
15
|
+
|
|
16
|
+
import json
|
|
17
|
+
import os
|
|
18
|
+
import sys
|
|
19
|
+
from pathlib import Path
|
|
20
|
+
|
|
21
|
+
from pubrules import __version__
|
|
22
|
+
from pubrules.conform import run as conform
|
|
23
|
+
from pubrules.freshness import pinned
|
|
24
|
+
from pubrules.repos import read
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
def row(repo: Path) -> dict:
|
|
28
|
+
config_path = repo / "pubrules.json"
|
|
29
|
+
out = {"repo": repo.name, "pinned": pinned(repo) or "-", "latest": __version__}
|
|
30
|
+
if not config_path.is_file():
|
|
31
|
+
return {**out, "conforms": "no pubrules.json", "exceptions": []}
|
|
32
|
+
config = json.loads(config_path.read_text(encoding="utf-8"))
|
|
33
|
+
if config.get("kind") != "rust":
|
|
34
|
+
return {**out, "conforms": "not Rust", "exceptions": []}
|
|
35
|
+
try:
|
|
36
|
+
lines, problems = conform(repo, config)
|
|
37
|
+
except Exception as e: # noqa: BLE001 - report every repository, whatever fails
|
|
38
|
+
return {**out, "conforms": f"could not check: {e}", "exceptions": []}
|
|
39
|
+
exceptions = [line.removeprefix("open exception: ") for line in lines
|
|
40
|
+
if line.startswith("open exception: ")]
|
|
41
|
+
# The workflow pin is checked against the pinned pubrules by `sync --check`
|
|
42
|
+
# in the repository itself; here every rule is judged by the latest.
|
|
43
|
+
problems = [p for p in problems if not p.startswith(".github/workflows/")]
|
|
44
|
+
verdict = "yes" if not problems else f"no: {len(problems)} problem(s): " + "; ".join(problems)
|
|
45
|
+
return {**out, "conforms": verdict, "exceptions": exceptions}
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
def table(rows: list[dict]) -> str:
|
|
49
|
+
lines = [f"Every pub* repository against pubrules {__version__}.", "",
|
|
50
|
+
"| Repository | Pins pubrules | Current | Conforms to the latest standard | Open exceptions |",
|
|
51
|
+
"|---|---|---|---|---|"]
|
|
52
|
+
for r in rows:
|
|
53
|
+
current = "yes" if r["pinned"] == r["latest"] else "**no**"
|
|
54
|
+
exceptions = "<br>".join(r["exceptions"]) or "-"
|
|
55
|
+
lines.append(f"| {r['repo']} | {r['pinned']} | {current} | {r['conforms']} | {exceptions} |")
|
|
56
|
+
return "\n".join(lines) + "\n"
|
|
57
|
+
|
|
58
|
+
|
|
59
|
+
def governed_rows(repos_file: Path, where: Path) -> list[dict]:
|
|
60
|
+
"""A row for each repository the repos file governs, checked out or not."""
|
|
61
|
+
rows = []
|
|
62
|
+
for entry in read(repos_file, "freshness"):
|
|
63
|
+
if not entry["govern"]:
|
|
64
|
+
continue
|
|
65
|
+
name = entry["repo"].split("/", 1)[1]
|
|
66
|
+
checkout = where / name
|
|
67
|
+
rows.append(row(checkout.resolve()) if checkout.is_dir() else
|
|
68
|
+
{"repo": name, "pinned": "-", "latest": __version__,
|
|
69
|
+
"conforms": "not checked out", "exceptions": []})
|
|
70
|
+
return rows
|
|
71
|
+
|
|
72
|
+
|
|
73
|
+
def freshness(dirs: list[Path], as_json: bool, repos_file: Path | None = None) -> int:
|
|
74
|
+
if repos_file:
|
|
75
|
+
if len(dirs) > 1:
|
|
76
|
+
sys.exit("pubfleet freshness: --repos takes one directory, where the checkouts are")
|
|
77
|
+
rows = governed_rows(repos_file, dirs[0] if dirs else Path("."))
|
|
78
|
+
elif dirs:
|
|
79
|
+
rows = [row(d.resolve()) for d in dirs]
|
|
80
|
+
else:
|
|
81
|
+
sys.exit("pubfleet freshness: name the checkouts, or pass --repos FILE")
|
|
82
|
+
text = json.dumps(rows, indent=2) + "\n" if as_json else table(rows)
|
|
83
|
+
print(text, end="")
|
|
84
|
+
summary = os.environ.get("GITHUB_STEP_SUMMARY")
|
|
85
|
+
if summary and not as_json:
|
|
86
|
+
with open(summary, "a", encoding="utf-8") as out:
|
|
87
|
+
out.write(text)
|
|
88
|
+
return 0
|
pubfleet/state.py
ADDED
|
@@ -0,0 +1,198 @@
|
|
|
1
|
+
"""`pubfleet state --check`: is a state repository's state.json sound?
|
|
2
|
+
`pubfleet state --repos-for ROLE`: the repos file for one of its roles.
|
|
3
|
+
|
|
4
|
+
{"owner": "OWNER",
|
|
5
|
+
"repos": {"NAME": {"project": "some-group", "board": true, "protect": true, "govern": false,
|
|
6
|
+
"deploy": true, "variables": {"SOME_SETTING": "a value"}}}}
|
|
7
|
+
|
|
8
|
+
One file per GitHub owner says which of its repositories are graded, protected
|
|
9
|
+
and governed. This checks its shape, that each repository exists under the
|
|
10
|
+
owner, and that each one with `protect` has a protect block in its pubrules.json,
|
|
11
|
+
since that block is what says how, that each one with `deploy` (optional, false
|
|
12
|
+
when left out: the convergence engine walks the repositories that have it) has a
|
|
13
|
+
deploy block there, for the same reason, and that the repository has reported every check
|
|
14
|
+
the block requires, since a ruleset requiring one it never reports stops anything
|
|
15
|
+
merging. A repository's `variables` (optional) are the values of its GitHub variables,
|
|
16
|
+
which `pubfleet variables` keeps them to: each a string or JSON, and each one its own
|
|
17
|
+
pubrules.json declares in its variables block. That last check is the best it can be: it looks at the default branch's newest
|
|
18
|
+
commit and a few recent pull requests, and says nothing when it can't read them. It is the state repository's one required
|
|
19
|
+
check, so it builds nothing and takes seconds. It reads GitHub through `gh`,
|
|
20
|
+
with a token that can read every repository the file names. The one exception is
|
|
21
|
+
the repository the check runs in (`GITHUB_REPOSITORY`, which Actions sets) when the
|
|
22
|
+
file names it: its pubrules.json is the one beside the state file, the change under
|
|
23
|
+
review, so a pull request that moves its own block is checked against itself.
|
|
24
|
+
"""
|
|
25
|
+
|
|
26
|
+
from __future__ import annotations
|
|
27
|
+
|
|
28
|
+
import base64
|
|
29
|
+
import json
|
|
30
|
+
import os
|
|
31
|
+
import re
|
|
32
|
+
import sys
|
|
33
|
+
from pathlib import Path
|
|
34
|
+
from typing import Any
|
|
35
|
+
|
|
36
|
+
from pubrules.protect import Api, ApiError, gh, remote, remote_config
|
|
37
|
+
|
|
38
|
+
from . import variables
|
|
39
|
+
|
|
40
|
+
OWNER = re.compile(r"[A-Za-z0-9][A-Za-z0-9-]*")
|
|
41
|
+
OWN = "pubrules.json"
|
|
42
|
+
NAME = re.compile(r"[A-Za-z0-9_.-]+")
|
|
43
|
+
ROLES = ("board", "protect", "govern")
|
|
44
|
+
# Roles a repository may leave out, as false.
|
|
45
|
+
OPTIONAL = ("deploy",)
|
|
46
|
+
RECENT_PULLS = 5
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
def shape(state: Any) -> list[str]:
|
|
50
|
+
"""What is wrong with the file's shape, one phrase each."""
|
|
51
|
+
if not isinstance(state, dict):
|
|
52
|
+
return [f"it is {json.dumps(state)}, not an object"]
|
|
53
|
+
out = [f"{key} is not a setting; it takes owner and repos"
|
|
54
|
+
for key in sorted(state.keys() - {"owner", "repos"})]
|
|
55
|
+
if not OWNER.fullmatch(str(state.get("owner", ""))) or not isinstance(state.get("owner"), str):
|
|
56
|
+
out.append('"owner" must be a GitHub owner')
|
|
57
|
+
repos = state.get("repos")
|
|
58
|
+
if not isinstance(repos, dict):
|
|
59
|
+
return [*out, '"repos" must be an object, from each repository\'s name to its roles']
|
|
60
|
+
for name, roles in repos.items():
|
|
61
|
+
if not NAME.fullmatch(name):
|
|
62
|
+
out.append(f"{name} is not a repository's name")
|
|
63
|
+
if not isinstance(roles, dict):
|
|
64
|
+
out.append(f"{name}: its roles must be an object, not {json.dumps(roles)}")
|
|
65
|
+
continue
|
|
66
|
+
for key in sorted(roles.keys() - {"project", variables.BLOCK, *ROLES, *OPTIONAL}):
|
|
67
|
+
out.append(f"{name}: {key} is not a setting; it takes project, board, protect, govern, "
|
|
68
|
+
f"deploy and variables")
|
|
69
|
+
if variables.BLOCK in roles:
|
|
70
|
+
out += [f"{name}: {p}" for p in variables.value_problems(roles[variables.BLOCK])]
|
|
71
|
+
for role in ROLES:
|
|
72
|
+
if not isinstance(roles.get(role), bool):
|
|
73
|
+
out.append(f"{name}: \"{role}\" must be true or false")
|
|
74
|
+
for role in OPTIONAL:
|
|
75
|
+
if not isinstance(roles.get(role, False), bool):
|
|
76
|
+
out.append(f"{name}: \"{role}\" must be true or false")
|
|
77
|
+
if not isinstance(roles.get("project", ""), str) or roles.get("project") == "":
|
|
78
|
+
out.append(f"{name}: \"project\" must be a name")
|
|
79
|
+
return out
|
|
80
|
+
|
|
81
|
+
|
|
82
|
+
def reported(api: Api, slug: str, branch: str) -> set[str] | None:
|
|
83
|
+
"""Every check `slug` has reported lately on `branch` and its recent pull
|
|
84
|
+
requests, or None if they can't be read."""
|
|
85
|
+
try:
|
|
86
|
+
pulls = api("GET", f"repos/{slug}/pulls?state=closed&per_page={RECENT_PULLS}", None) or []
|
|
87
|
+
names: set[str] = set()
|
|
88
|
+
for ref in [branch, *(p["head"]["sha"] for p in pulls)]:
|
|
89
|
+
runs = api("GET", f"repos/{slug}/commits/{ref}/check-runs?per_page=100", None) or {}
|
|
90
|
+
names |= {run["name"] for run in runs.get("check_runs", [])}
|
|
91
|
+
statuses = api("GET", f"repos/{slug}/commits/{ref}/status", None) or {}
|
|
92
|
+
names |= {status["context"] for status in statuses.get("statuses", [])}
|
|
93
|
+
except (ApiError, KeyError, TypeError):
|
|
94
|
+
return None
|
|
95
|
+
return names
|
|
96
|
+
|
|
97
|
+
|
|
98
|
+
def on_github(api: Api, owner: str, repos: dict) -> list[str]:
|
|
99
|
+
"""What GitHub says is wrong: a repository missing, moved or unreadable,
|
|
100
|
+
one to protect that has no sound protect block, or one given variables its
|
|
101
|
+
pubrules.json doesn't declare."""
|
|
102
|
+
out = []
|
|
103
|
+
for name, roles in repos.items():
|
|
104
|
+
slug = f"{owner}/{name}"
|
|
105
|
+
try:
|
|
106
|
+
found = api("GET", f"repos/{slug}", None) or {}
|
|
107
|
+
except ApiError as e:
|
|
108
|
+
out.append(f"{slug} does not exist" if e.status == 404 else f"{slug}: {e}")
|
|
109
|
+
continue
|
|
110
|
+
# GitHub answers for a renamed or transferred repository's old name.
|
|
111
|
+
if found.get("full_name", slug).lower() != slug.lower():
|
|
112
|
+
out.append(f"{slug} is now {found['full_name']}")
|
|
113
|
+
continue
|
|
114
|
+
if roles["protect"]:
|
|
115
|
+
try:
|
|
116
|
+
entry = remote(api, slug)
|
|
117
|
+
except ValueError as e:
|
|
118
|
+
out.append(f"{slug} has \"protect\" on, but {e}")
|
|
119
|
+
continue
|
|
120
|
+
seen = reported(api, slug, found.get("default_branch", "main"))
|
|
121
|
+
for check in [] if seen is None else entry["checks"]:
|
|
122
|
+
if check not in seen:
|
|
123
|
+
out.append(f"{slug} requires the check \"{check}\", which it hasn't reported "
|
|
124
|
+
f"lately, so nothing could merge there")
|
|
125
|
+
if roles.get("deploy"):
|
|
126
|
+
try:
|
|
127
|
+
config = remote_config(api, slug)
|
|
128
|
+
except ValueError as e:
|
|
129
|
+
out.append(f"{slug} has \"deploy\" on, but {e}")
|
|
130
|
+
else:
|
|
131
|
+
if not isinstance(config, dict) or not isinstance(config.get("deploy"), dict):
|
|
132
|
+
out.append(f"{slug} has \"deploy\" on, but its pubrules.json has no deploy block")
|
|
133
|
+
if values := roles.get(variables.BLOCK):
|
|
134
|
+
try:
|
|
135
|
+
block = variables.declared(api, slug)
|
|
136
|
+
except ValueError as e:
|
|
137
|
+
out.append(f"{slug} has variables in the state, but {e}")
|
|
138
|
+
continue
|
|
139
|
+
if names := variables.undeclared(block, values):
|
|
140
|
+
out.append(f"{slug}: its pubrules.json doesn't declare {', '.join(names)}, "
|
|
141
|
+
f"which the state gives it")
|
|
142
|
+
return out
|
|
143
|
+
|
|
144
|
+
|
|
145
|
+
def repos_file(state: dict, role: str) -> dict:
|
|
146
|
+
"""The repos file pubrules' other commands read: the repositories with `role` on, in the
|
|
147
|
+
file's order. Nothing else reads a state file's roles."""
|
|
148
|
+
entries = []
|
|
149
|
+
for name, roles in state["repos"].items():
|
|
150
|
+
if roles.get(role, False):
|
|
151
|
+
entry = {"repo": f"{state['owner']}/{name}", "protect": roles["protect"],
|
|
152
|
+
"govern": roles["govern"]}
|
|
153
|
+
if roles.get("project"):
|
|
154
|
+
entry["project"] = roles["project"]
|
|
155
|
+
entries.append(entry)
|
|
156
|
+
return {"repositories": entries}
|
|
157
|
+
|
|
158
|
+
|
|
159
|
+
def repos_for(path: Path, role: str) -> int:
|
|
160
|
+
try:
|
|
161
|
+
state = json.loads(path.read_text(encoding="utf-8"))
|
|
162
|
+
except (OSError, ValueError) as e:
|
|
163
|
+
sys.exit(f"pubfleet state: {path}: {e}")
|
|
164
|
+
if bad := shape(state):
|
|
165
|
+
sys.exit(f"pubfleet state: {path}: {'; '.join(bad)}")
|
|
166
|
+
print(json.dumps(repos_file(state, role), separators=(",", ":")))
|
|
167
|
+
return 0
|
|
168
|
+
|
|
169
|
+
|
|
170
|
+
def here(api: Api, path: Path) -> Api:
|
|
171
|
+
"""`api`, but answering the checked-out repository's own pubrules.json from the copy
|
|
172
|
+
beside `path`, when Actions names that repository and the copy is there."""
|
|
173
|
+
slug = os.environ.get("GITHUB_REPOSITORY", "")
|
|
174
|
+
own = path.parent / OWN
|
|
175
|
+
if not slug or not own.is_file():
|
|
176
|
+
return api
|
|
177
|
+
wanted = f"repos/{slug}/contents/{OWN}".lower()
|
|
178
|
+
|
|
179
|
+
def answer(method: str, url: str, body: object) -> object:
|
|
180
|
+
if method == "GET" and url.lower() == wanted:
|
|
181
|
+
return {"type": "file", "encoding": "base64",
|
|
182
|
+
"content": base64.b64encode(own.read_bytes()).decode("ascii")}
|
|
183
|
+
return api(method, url, body)
|
|
184
|
+
return answer
|
|
185
|
+
|
|
186
|
+
|
|
187
|
+
def check(path: Path, api: Api = gh) -> int:
|
|
188
|
+
try:
|
|
189
|
+
state = json.loads(path.read_text(encoding="utf-8"))
|
|
190
|
+
except (OSError, ValueError) as e:
|
|
191
|
+
sys.exit(f"pubfleet state: {path}: {e}")
|
|
192
|
+
bad = shape(state) or on_github(here(api, path), state["owner"], state["repos"])
|
|
193
|
+
for line in bad:
|
|
194
|
+
print(f"{path}: {line}")
|
|
195
|
+
if bad:
|
|
196
|
+
return 1
|
|
197
|
+
print(f"{path}: {len(state['repos'])} repositories of {state['owner']}, all sound")
|
|
198
|
+
return 0
|
pubfleet/variables.py
ADDED
|
@@ -0,0 +1,200 @@
|
|
|
1
|
+
"""`pubfleet variables --check | --apply`: keep repositories' GitHub variables to a state file.
|
|
2
|
+
|
|
3
|
+
A repository's pubrules.json names the variables it reads, never their values, and
|
|
4
|
+
may name the workflow that takes them in, to run once one changes:
|
|
5
|
+
|
|
6
|
+
"variables": {"names": ["SOME_SETTING", "SOME_MAP"], "deploy": "deploy.yml"}
|
|
7
|
+
|
|
8
|
+
Their values are in the owner's state file, under each repository, so a change to one
|
|
9
|
+
is a pull request there, with its history:
|
|
10
|
+
|
|
11
|
+
"some-repo": {"board": true, "protect": true, "govern": false,
|
|
12
|
+
"variables": {"SOME_SETTING": "a string", "SOME_MAP": {"a": "b"}}}
|
|
13
|
+
|
|
14
|
+
A string goes to GitHub as it is. Anything else is JSON, kept structured in the state
|
|
15
|
+
so a diff shows the entry that changed, and sent compact, in the file's order. A JSON
|
|
16
|
+
value matches what GitHub holds when that parses to the same thing, however spaced.
|
|
17
|
+
|
|
18
|
+
`--check` changes nothing: it prints how each repository's variables differ from the
|
|
19
|
+
state, and fails when one does. `--apply` sets those that differ, and then runs the
|
|
20
|
+
repository's `deploy` workflow on its default branch, once, if it set any. Neither
|
|
21
|
+
touches a variable the state leaves out, nor removes one: dropping a variable is a
|
|
22
|
+
deliberate step by hand. Each acts on every repository in the state with variables,
|
|
23
|
+
or those named (NAME or OWNER/NAME), or those a repos file lists. It reads through
|
|
24
|
+
`gh`, with a token that can read the repositories' pubrules.json and variables, and,
|
|
25
|
+
for --apply, write their variables and run their workflows.
|
|
26
|
+
"""
|
|
27
|
+
|
|
28
|
+
from __future__ import annotations
|
|
29
|
+
|
|
30
|
+
import difflib
|
|
31
|
+
import json
|
|
32
|
+
import sys
|
|
33
|
+
from pathlib import Path
|
|
34
|
+
from typing import Any
|
|
35
|
+
|
|
36
|
+
from pubrules.protect import Api, ApiError, gh, gh_set, remote_config
|
|
37
|
+
from pubrules.repos import read
|
|
38
|
+
from pubrules.variables import BLOCK, block_problems, name_problem
|
|
39
|
+
|
|
40
|
+
PAGE = 30
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
def value_problems(values: Any) -> list[str]:
|
|
44
|
+
"""What is wrong with one repository's variables in a state file: one phrase each."""
|
|
45
|
+
if not isinstance(values, dict):
|
|
46
|
+
return ["its variables must be an object, from each name to its value"]
|
|
47
|
+
out = [p for n in values if (p := name_problem(n))]
|
|
48
|
+
for name, value in values.items():
|
|
49
|
+
if value is None or value == "":
|
|
50
|
+
out.append(f"{name} has no value; GitHub keeps no empty variable")
|
|
51
|
+
return out
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
def serialise(value: Any) -> str:
|
|
55
|
+
"""What GitHub holds for a value from the state."""
|
|
56
|
+
if isinstance(value, str):
|
|
57
|
+
return value
|
|
58
|
+
return json.dumps(value, ensure_ascii=False, separators=(",", ":"))
|
|
59
|
+
|
|
60
|
+
|
|
61
|
+
def canonical(value: Any) -> str:
|
|
62
|
+
return json.dumps(value, ensure_ascii=False, sort_keys=True, separators=(",", ":"))
|
|
63
|
+
|
|
64
|
+
|
|
65
|
+
def same(want: Any, have: str) -> bool:
|
|
66
|
+
if isinstance(want, str):
|
|
67
|
+
return want == have
|
|
68
|
+
try:
|
|
69
|
+
return canonical(json.loads(have)) == canonical(want)
|
|
70
|
+
except ValueError:
|
|
71
|
+
return False
|
|
72
|
+
|
|
73
|
+
|
|
74
|
+
def pretty(value: Any) -> list[str]:
|
|
75
|
+
return json.dumps(value, ensure_ascii=False, indent=2).splitlines()
|
|
76
|
+
|
|
77
|
+
|
|
78
|
+
def difference(name: str, want: Any, have: str | None) -> list[str]:
|
|
79
|
+
"""How one variable differs, as lines: a diff of the entries for JSON."""
|
|
80
|
+
if have is None:
|
|
81
|
+
return [f" {name} is not set; the state has:", *(f" + {line}" for line in
|
|
82
|
+
pretty(want) if not isinstance(want, str)),
|
|
83
|
+
*([f" + {want}"] if isinstance(want, str) else [])]
|
|
84
|
+
if isinstance(want, str):
|
|
85
|
+
return [f" {name}:", f" - {have}", f" + {want}"]
|
|
86
|
+
try:
|
|
87
|
+
before = pretty(json.loads(have))
|
|
88
|
+
except ValueError:
|
|
89
|
+
before = [have]
|
|
90
|
+
lines = difflib.unified_diff(before, pretty(want), "on GitHub", "in the state", lineterm="", n=0)
|
|
91
|
+
return [f" {name}:", *(f" {line}" for line in lines)]
|
|
92
|
+
|
|
93
|
+
|
|
94
|
+
def live(api: Api, slug: str) -> dict[str, str]:
|
|
95
|
+
"""Every variable `slug` holds, by name."""
|
|
96
|
+
out: dict[str, str] = {}
|
|
97
|
+
page = 1
|
|
98
|
+
while True:
|
|
99
|
+
found = api("GET", f"repos/{slug}/actions/variables?per_page={PAGE}&page={page}", None) or {}
|
|
100
|
+
batch = found.get("variables", [])
|
|
101
|
+
out |= {v["name"]: v["value"] for v in batch}
|
|
102
|
+
if len(batch) < PAGE:
|
|
103
|
+
return out
|
|
104
|
+
page += 1
|
|
105
|
+
|
|
106
|
+
|
|
107
|
+
def declared(api: Api, slug: str) -> dict:
|
|
108
|
+
"""`slug`'s variables block, from the pubrules.json on its default branch."""
|
|
109
|
+
config = remote_config(api, slug)
|
|
110
|
+
if not isinstance(config, dict) or BLOCK not in config:
|
|
111
|
+
raise ValueError(f"its pubrules.json declares no variables; add "
|
|
112
|
+
f'"{BLOCK}": {{"names": [...]}} to it')
|
|
113
|
+
if bad := block_problems(config[BLOCK]):
|
|
114
|
+
raise ValueError(f"its pubrules.json's {BLOCK} block: {'; '.join(bad)}")
|
|
115
|
+
return config[BLOCK]
|
|
116
|
+
|
|
117
|
+
|
|
118
|
+
def undeclared(block: dict, values: dict) -> list[str]:
|
|
119
|
+
return [n for n in values if n not in block["names"]]
|
|
120
|
+
|
|
121
|
+
|
|
122
|
+
def chosen(state: dict, names: list[str], repos_file: Path | None) -> list[str]:
|
|
123
|
+
"""The repositories to act on, by name within the state's owner."""
|
|
124
|
+
owner = state["owner"]
|
|
125
|
+
if repos_file:
|
|
126
|
+
names = [e["repo"] for e in read(repos_file, "variables")]
|
|
127
|
+
if not names:
|
|
128
|
+
return [n for n, roles in state["repos"].items() if roles.get(BLOCK)]
|
|
129
|
+
out = []
|
|
130
|
+
for name in names:
|
|
131
|
+
own, _, short = name.rpartition("/")
|
|
132
|
+
if own and own.lower() != owner.lower():
|
|
133
|
+
sys.exit(f"pubfleet variables: {name} is not one of {owner}'s, as the state is")
|
|
134
|
+
if short not in state["repos"]:
|
|
135
|
+
sys.exit(f"pubfleet variables: the state names no repository {short}")
|
|
136
|
+
if state["repos"][short].get(BLOCK):
|
|
137
|
+
out.append(short)
|
|
138
|
+
elif not repos_file:
|
|
139
|
+
sys.exit(f"pubfleet variables: the state gives {short} no variables")
|
|
140
|
+
return out
|
|
141
|
+
|
|
142
|
+
|
|
143
|
+
def one(api: Api, slug: str, values: dict, apply: bool, setter) -> tuple[list[str], bool, bool]:
|
|
144
|
+
"""Lines for one repository, whether it differs, and whether something failed."""
|
|
145
|
+
try:
|
|
146
|
+
block = declared(api, slug)
|
|
147
|
+
except ValueError as e:
|
|
148
|
+
return [f"{slug}: {e}"], False, True
|
|
149
|
+
if names := undeclared(block, values):
|
|
150
|
+
return [f"{slug}: its pubrules.json doesn't declare {', '.join(names)}, "
|
|
151
|
+
f"which the state gives it"], False, True
|
|
152
|
+
try:
|
|
153
|
+
have = live(api, slug)
|
|
154
|
+
except ApiError as e:
|
|
155
|
+
return [f"{slug}: its variables can't be read ({e})"], False, True
|
|
156
|
+
differs = [n for n, v in values.items() if n not in have or not same(v, have[n])]
|
|
157
|
+
if not differs:
|
|
158
|
+
return [f"{slug}: {len(values)} variables as the state says"], False, False
|
|
159
|
+
lines = [f"{slug}: {len(differs)} of {len(values)} variables differ from the state"]
|
|
160
|
+
for name in differs:
|
|
161
|
+
lines += difference(name, values[name], have.get(name))
|
|
162
|
+
if not apply:
|
|
163
|
+
return lines, True, False
|
|
164
|
+
failed = False
|
|
165
|
+
for name in differs:
|
|
166
|
+
if (refused := setter("variable", name, slug, serialise(values[name]))) is not None:
|
|
167
|
+
lines.append(f" FAILED {name}: {refused}")
|
|
168
|
+
failed = True
|
|
169
|
+
else:
|
|
170
|
+
lines.append(f" set {name}")
|
|
171
|
+
if workflow := block.get("deploy"):
|
|
172
|
+
try:
|
|
173
|
+
branch = (api("GET", f"repos/{slug}", None) or {}).get("default_branch", "main")
|
|
174
|
+
api("POST", f"repos/{slug}/actions/workflows/{workflow}/dispatches", {"ref": branch})
|
|
175
|
+
lines.append(f" ran {workflow} on {branch}")
|
|
176
|
+
except ApiError as e:
|
|
177
|
+
lines.append(f" FAILED to run {workflow}: {e}")
|
|
178
|
+
failed = True
|
|
179
|
+
return lines, True, failed
|
|
180
|
+
|
|
181
|
+
|
|
182
|
+
def run(path: Path, names: list[str], apply: bool, repos_file: Path | None = None,
|
|
183
|
+
api: Api = gh, setter=gh_set) -> int:
|
|
184
|
+
from .state import shape # noqa: PLC0415 - state reads this module's value_problems
|
|
185
|
+
try:
|
|
186
|
+
state = json.loads(path.read_text(encoding="utf-8"))
|
|
187
|
+
except (OSError, ValueError) as e:
|
|
188
|
+
sys.exit(f"pubfleet variables: {path}: {e}")
|
|
189
|
+
if bad := shape(state):
|
|
190
|
+
sys.exit(f"pubfleet variables: {path}: {'; '.join(bad)}")
|
|
191
|
+
repos = chosen(state, names, repos_file)
|
|
192
|
+
if not repos:
|
|
193
|
+
print(f"{path}: no repository of {state['owner']} has variables")
|
|
194
|
+
return 0
|
|
195
|
+
differs = failed = False
|
|
196
|
+
for name in repos:
|
|
197
|
+
lines, d, f = one(api, f"{state['owner']}/{name}", state["repos"][name][BLOCK], apply, setter)
|
|
198
|
+
print("\n".join(lines))
|
|
199
|
+
differs, failed = differs or d, failed or f
|
|
200
|
+
return 1 if failed or (differs and not apply) else 0
|