pubfleet 0.2.1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- pubfleet/__init__.py +23 -0
- pubfleet/board/__init__.py +113 -0
- pubfleet/board/collect.py +255 -0
- pubfleet/board/connectors/__init__.py +7 -0
- pubfleet/board/connectors/actions.py +65 -0
- pubfleet/board/connectors/base.py +193 -0
- pubfleet/board/connectors/github.py +144 -0
- pubfleet/board/connectors/live.py +17 -0
- pubfleet/board/connectors/repos.py +23 -0
- pubfleet/board/connectors/scorecard.py +24 -0
- pubfleet/board/connectors/sonarcloud.py +52 -0
- pubfleet/board/page.py +125 -0
- pubfleet/board/rules.json +37 -0
- pubfleet/board/rules.py +337 -0
- pubfleet/board_publish.py +134 -0
- pubfleet/cli.py +112 -0
- pubfleet/discover.py +110 -0
- pubfleet/freshness.py +88 -0
- pubfleet/state.py +198 -0
- pubfleet/variables.py +200 -0
- pubfleet-0.2.1.dist-info/METADATA +314 -0
- pubfleet-0.2.1.dist-info/RECORD +26 -0
- pubfleet-0.2.1.dist-info/WHEEL +5 -0
- pubfleet-0.2.1.dist-info/entry_points.txt +2 -0
- pubfleet-0.2.1.dist-info/licenses/LICENSE +202 -0
- pubfleet-0.2.1.dist-info/top_level.txt +1 -0
pubfleet/board/rules.py
ADDED
|
@@ -0,0 +1,337 @@
|
|
|
1
|
+
"""Grades from facts: rules.json applied to what the connectors read.
|
|
2
|
+
|
|
3
|
+
Nothing here reads a source. `grade` takes one repository's facts, each
|
|
4
|
+
`{"value": ..., "sources": [call ids], "note": ...}`, and returns its three
|
|
5
|
+
grades, each a start and a list of steps, so the page can show how every
|
|
6
|
+
letter was reached and which call each step rests on.
|
|
7
|
+
|
|
8
|
+
The facts, by name:
|
|
9
|
+
|
|
10
|
+
- `visibility`: "public" or "private"; `kind`: "package" or "service"
|
|
11
|
+
- `integration`: `{"pubrules": bool, "pubrel": bool}`
|
|
12
|
+
- `pins` and `latest`: `{"pubrules": [versions] or "self", "pubrel": ...}`,
|
|
13
|
+
and the latest release of each, `{"pubrules": "0.14.2", "pubrel": ...}`
|
|
14
|
+
- `released`: the newest release (a package) or production deploy (a
|
|
15
|
+
service), `{"at": ISO time, "tag": ...}`, or None
|
|
16
|
+
- `issues`, `dependabot_alerts`, `code_scanning_alerts`,
|
|
17
|
+
`secret_scanning_alerts`: counts
|
|
18
|
+
- `secret_protection`: `{"secret_scanning": status, "secret_scanning_push_protection":
|
|
19
|
+
status}`, or None where the token can't read them
|
|
20
|
+
- `ci`: main's last CI run, `{"workflow", "conclusion", "at", "jobs":
|
|
21
|
+
[{"name", "conclusion"}]}`, or None
|
|
22
|
+
- `nightly`: `{"governed": bool, "last": run or None}`
|
|
23
|
+
- `live`: what pubrules.json's live block declares, `{"health": URL,
|
|
24
|
+
"test": workflow, "deploy": workflow}`; `health`: `{"answers": bool,
|
|
25
|
+
"status": int}`; `live_test`: the test workflow's last run, or None
|
|
26
|
+
- `protection`: `{"differences": [text], "limit": bool}`
|
|
27
|
+
- `sonar`: `{"reliability", "maintainability", "security"}` letters, or
|
|
28
|
+
None; `scorecard`: `{"score", "failing"}`, or None
|
|
29
|
+
|
|
30
|
+
A fact that could not be read has the value None and a note. Where that
|
|
31
|
+
leaves a notch's question open (issues, CI, protection), the notch is
|
|
32
|
+
taken: a board that cannot see does not grade green. Alerts and Scorecard
|
|
33
|
+
that are switched off or absent are listed with no notch.
|
|
34
|
+
"""
|
|
35
|
+
|
|
36
|
+
from __future__ import annotations
|
|
37
|
+
|
|
38
|
+
import json
|
|
39
|
+
from collections.abc import Callable
|
|
40
|
+
from datetime import datetime, timedelta
|
|
41
|
+
from importlib import resources
|
|
42
|
+
from pathlib import Path
|
|
43
|
+
from typing import Any
|
|
44
|
+
|
|
45
|
+
LETTERS = "ABCDE"
|
|
46
|
+
CATEGORIES = ("quality", "security", "delivery")
|
|
47
|
+
# The job conclusions that mean a tool found something.
|
|
48
|
+
FAILED = {"failure", "timed_out", "startup_failure"}
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
def load(path: Path | None = None) -> dict:
|
|
52
|
+
"""rules.json: the file given, or the one pubfleet ships."""
|
|
53
|
+
text = path.read_text(encoding="utf-8") if path else \
|
|
54
|
+
resources.files("pubfleet.board").joinpath("rules.json").read_text(encoding="utf-8")
|
|
55
|
+
rules = json.loads(text)
|
|
56
|
+
unknown = [d["notch"] for d in rules.get("delivery", []) if d.get("notch") not in DELIVERY]
|
|
57
|
+
if unknown:
|
|
58
|
+
raise ValueError(f"no delivery notch {', '.join(map(str, unknown))}; "
|
|
59
|
+
f"the notches are {', '.join(DELIVERY)}")
|
|
60
|
+
for tool in rules.get("tools", []):
|
|
61
|
+
if tool.get("category") not in ("quality", "security"):
|
|
62
|
+
raise ValueError(f"tool {tool.get('tool')}: category must be quality or security")
|
|
63
|
+
return rules
|
|
64
|
+
|
|
65
|
+
|
|
66
|
+
def when(text: str | None) -> datetime | None:
|
|
67
|
+
return datetime.fromisoformat(text.replace("Z", "+00:00")) if text else None
|
|
68
|
+
|
|
69
|
+
|
|
70
|
+
class Facts:
|
|
71
|
+
"""One repository's facts, read by name."""
|
|
72
|
+
|
|
73
|
+
def __init__(self, facts: dict[str, Any]):
|
|
74
|
+
self.facts = facts
|
|
75
|
+
|
|
76
|
+
def value(self, name: str) -> Any:
|
|
77
|
+
return (self.facts.get(name) or {}).get("value")
|
|
78
|
+
|
|
79
|
+
def sources(self, *names: str) -> list[str]:
|
|
80
|
+
out: list[str] = []
|
|
81
|
+
for name in names:
|
|
82
|
+
out += [s for s in (self.facts.get(name) or {}).get("sources", []) if s not in out]
|
|
83
|
+
return out
|
|
84
|
+
|
|
85
|
+
def note(self, name: str) -> str | None:
|
|
86
|
+
return (self.facts.get(name) or {}).get("note")
|
|
87
|
+
|
|
88
|
+
def known(self, name: str) -> bool:
|
|
89
|
+
return name in self.facts and self.value(name) is not None
|
|
90
|
+
|
|
91
|
+
|
|
92
|
+
def step(text: str, delta: int, facts: Facts, *names: str) -> dict:
|
|
93
|
+
return {"text": text, "delta": delta, "facts": list(names), "sources": facts.sources(*names)}
|
|
94
|
+
|
|
95
|
+
|
|
96
|
+
def letter(start: str, steps: list[dict]) -> str:
|
|
97
|
+
down = -sum(s["delta"] for s in steps)
|
|
98
|
+
return LETTERS[min(LETTERS.index(start) + down, len(LETTERS) - 1)]
|
|
99
|
+
|
|
100
|
+
|
|
101
|
+
def worse(*letters: str) -> str:
|
|
102
|
+
return max(letters, key=LETTERS.index)
|
|
103
|
+
|
|
104
|
+
|
|
105
|
+
def short(job: str) -> str:
|
|
106
|
+
""""ci / test (ubuntu-latest)" -> "test"."""
|
|
107
|
+
name = job.rsplit(" / ", 1)[-1]
|
|
108
|
+
if name.endswith(")") and " (" in name:
|
|
109
|
+
name = name[:name.rindex(" (")]
|
|
110
|
+
return name.strip()
|
|
111
|
+
|
|
112
|
+
|
|
113
|
+
# Sonar's starting letter, per category, from its ratings.
|
|
114
|
+
def sonar_start(f: Facts, category: str, rules: dict) -> tuple[dict, list[dict]]:
|
|
115
|
+
ratings = f.value("sonar")
|
|
116
|
+
if not ratings:
|
|
117
|
+
return ({"letter": "A", "text": "Start at A", "sources": []},
|
|
118
|
+
[step(f"{rules.get('sonar', 'No SonarQube Cloud analysis')}"
|
|
119
|
+
f"{': ' + f.note('sonar') if f.note('sonar') else ''}", -1, f, "sonar")])
|
|
120
|
+
if category == "quality":
|
|
121
|
+
start = worse(ratings["reliability"], ratings["maintainability"])
|
|
122
|
+
text = (f"SonarQube Cloud: reliability {ratings['reliability']}, "
|
|
123
|
+
f"maintainability {ratings['maintainability']}")
|
|
124
|
+
else:
|
|
125
|
+
start = ratings["security"]
|
|
126
|
+
text = f"SonarQube Cloud: security {start}"
|
|
127
|
+
return {"letter": start, "text": text, "sources": f.sources("sonar")}, []
|
|
128
|
+
|
|
129
|
+
|
|
130
|
+
def tool_steps(f: Facts, category: str, rules: dict) -> list[dict]:
|
|
131
|
+
out = []
|
|
132
|
+
ci = f.value("ci")
|
|
133
|
+
jobs = (ci or {}).get("jobs") or []
|
|
134
|
+
for tool in rules.get("tools", []):
|
|
135
|
+
if tool["category"] != category:
|
|
136
|
+
continue
|
|
137
|
+
name = tool["tool"]
|
|
138
|
+
if "jobs" in tool:
|
|
139
|
+
failed = sorted({j["name"] for j in jobs
|
|
140
|
+
if short(j["name"]) in tool["jobs"] and j.get("conclusion") in FAILED})
|
|
141
|
+
if failed:
|
|
142
|
+
out.append(step(f"{name} has findings on main: {', '.join(failed)} failed",
|
|
143
|
+
-1, f, "ci"))
|
|
144
|
+
elif "alerts" in tool:
|
|
145
|
+
fact = tool["alerts"]
|
|
146
|
+
count = f.value(fact)
|
|
147
|
+
if count is None:
|
|
148
|
+
if fact in f.facts:
|
|
149
|
+
out.append(step(f"{name} alerts not read: {f.note(fact) or 'unknown'}",
|
|
150
|
+
0, f, fact))
|
|
151
|
+
elif count:
|
|
152
|
+
out.append(step(f"{count} open {name} alert{'s' if count != 1 else ''}",
|
|
153
|
+
-1, f, fact))
|
|
154
|
+
elif "settings" in tool:
|
|
155
|
+
fact = tool["settings"]
|
|
156
|
+
if fact not in f.facts:
|
|
157
|
+
continue
|
|
158
|
+
found = f.value(fact)
|
|
159
|
+
if f.value("visibility") == "private":
|
|
160
|
+
out.append(step(f"{name}: not available on a private repository without GitHub "
|
|
161
|
+
f"Advanced Security", 0, f, fact))
|
|
162
|
+
elif found is None:
|
|
163
|
+
out.append(step(f"{name} not read: this token can't see it", 0, f, fact))
|
|
164
|
+
else:
|
|
165
|
+
off = [label for key, label in tool["labels"].items() if found.get(key) != "enabled"]
|
|
166
|
+
if off:
|
|
167
|
+
out.append(step(f"{' and '.join(off)} off", -1, f, fact))
|
|
168
|
+
elif "scorecard_below" in tool:
|
|
169
|
+
if f.value("visibility") != "public":
|
|
170
|
+
continue
|
|
171
|
+
card = f.value("scorecard")
|
|
172
|
+
if not card or card.get("score") is None:
|
|
173
|
+
if "scorecard" in f.facts:
|
|
174
|
+
out.append(step(f"{name}: {f.note('scorecard') or 'no score'}", 0, f,
|
|
175
|
+
"scorecard"))
|
|
176
|
+
elif card["score"] < tool["scorecard_below"]:
|
|
177
|
+
failing = card.get("failing") or []
|
|
178
|
+
extra = f"; {', '.join(failing[:4])}{' …' if len(failing) > 4 else ''}" \
|
|
179
|
+
if failing else ""
|
|
180
|
+
out.append(step(f"{name} {card['score']}, below {tool['scorecard_below']}{extra}",
|
|
181
|
+
-1, f, "scorecard"))
|
|
182
|
+
return out
|
|
183
|
+
|
|
184
|
+
|
|
185
|
+
# Delivery notches: each takes the facts, its rules.json entry and the
|
|
186
|
+
# build's time, and returns its steps (none when there is no gap).
|
|
187
|
+
Notch = Callable[[Facts, dict, datetime], list[dict]]
|
|
188
|
+
|
|
189
|
+
|
|
190
|
+
def pins(f: Facts, _: dict, __: datetime) -> list[dict]:
|
|
191
|
+
pinned, latest = f.value("pins") or {}, f.value("latest") or {}
|
|
192
|
+
behind = []
|
|
193
|
+
for tool in ("pubrules", "pubrel"):
|
|
194
|
+
versions = pinned.get(tool)
|
|
195
|
+
if versions == "self" or not versions:
|
|
196
|
+
continue
|
|
197
|
+
if not latest.get(tool):
|
|
198
|
+
return [step(f"Latest {tool} release not read", -1, f, "latest")]
|
|
199
|
+
old = [v for v in versions if v != latest[tool]]
|
|
200
|
+
if old:
|
|
201
|
+
behind.append(f"{tool} {', '.join(old)}, latest {latest[tool]}")
|
|
202
|
+
return [step("; ".join(behind), -1, f, "pins", "latest")] if behind else []
|
|
203
|
+
|
|
204
|
+
|
|
205
|
+
def integration(f: Facts, _: dict, __: datetime) -> list[dict]:
|
|
206
|
+
have = f.value("integration") or {}
|
|
207
|
+
missing = ["pubrules"] if not have.get("pubrules") else []
|
|
208
|
+
if f.value("kind") != "service" and not have.get("pubrel"):
|
|
209
|
+
missing.append("pubrel")
|
|
210
|
+
if not missing:
|
|
211
|
+
return []
|
|
212
|
+
return [step(f"Not integrated with {' and '.join(missing)}", -1, f, "integration")]
|
|
213
|
+
|
|
214
|
+
|
|
215
|
+
def observability(f: Facts, _: dict, __: datetime) -> list[dict]:
|
|
216
|
+
declared = f.value("live") or {}
|
|
217
|
+
if not declared.get("health"):
|
|
218
|
+
return [step("No health URL in pubrules.json's live block", -1, f, "live")]
|
|
219
|
+
health = f.value("health") or {}
|
|
220
|
+
if health.get("answers"):
|
|
221
|
+
return []
|
|
222
|
+
status = health.get("status")
|
|
223
|
+
why = f"answered {status}" if status else (f.note("health") or "did not answer")
|
|
224
|
+
return [step(f"{declared['health']} {why}", -1, f, "health")]
|
|
225
|
+
|
|
226
|
+
|
|
227
|
+
def live_testing(f: Facts, entry: dict, now: datetime) -> list[dict]:
|
|
228
|
+
declared = f.value("live") or {}
|
|
229
|
+
if not declared.get("test"):
|
|
230
|
+
return [step("No live-test workflow in pubrules.json's live block", -1, f, "live")]
|
|
231
|
+
last = f.value("live_test")
|
|
232
|
+
hours = entry.get("hours", 24)
|
|
233
|
+
if not last:
|
|
234
|
+
why = f": {f.note('live_test')}" if f.note("live_test") else ""
|
|
235
|
+
return [step(f"{declared['test']} has not run{why}", -1, f, "live_test")]
|
|
236
|
+
if last.get("conclusion") != "success":
|
|
237
|
+
return [step(f"{declared['test']}'s last run is {last.get('conclusion')}", -1, f, "live_test")]
|
|
238
|
+
if now - when(last["at"]) > timedelta(hours=hours):
|
|
239
|
+
return [step(f"{declared['test']} last passed {last['at'][:16].replace('T', ' ')}, "
|
|
240
|
+
f"over {hours} h ago", -1, f, "live_test")]
|
|
241
|
+
return []
|
|
242
|
+
|
|
243
|
+
|
|
244
|
+
def release(f: Facts, entry: dict, now: datetime) -> list[dict]:
|
|
245
|
+
days = entry.get("days", 61)
|
|
246
|
+
last = f.value("released")
|
|
247
|
+
what = "production deploy" if f.value("kind") == "service" else "release"
|
|
248
|
+
if not last:
|
|
249
|
+
return [step(f"No {what}{': ' + f.note('released') if f.note('released') else ''}",
|
|
250
|
+
-1, f, "released")]
|
|
251
|
+
if now - when(last["at"]) > timedelta(days=days):
|
|
252
|
+
label = last.get("tag") or what
|
|
253
|
+
return [step(f"Last {what} {label} on {last['at'][:10]}, over {days} days ago",
|
|
254
|
+
-1, f, "released")]
|
|
255
|
+
return []
|
|
256
|
+
|
|
257
|
+
|
|
258
|
+
def issues(f: Facts, _: dict, __: datetime) -> list[dict]:
|
|
259
|
+
if not f.known("issues"):
|
|
260
|
+
return [step(f"Open issues not read: {f.note('issues') or 'unknown'}", -1, f, "issues")]
|
|
261
|
+
count = f.value("issues")
|
|
262
|
+
return [step(f"{count} open issue{'s' if count != 1 else ''}", -1, f, "issues")] if count else []
|
|
263
|
+
|
|
264
|
+
|
|
265
|
+
def nightly(f: Facts, entry: dict, now: datetime) -> list[dict]:
|
|
266
|
+
found = f.value("nightly") or {}
|
|
267
|
+
if found.get("governed"):
|
|
268
|
+
return []
|
|
269
|
+
last = found.get("last")
|
|
270
|
+
days = entry.get("days", 2)
|
|
271
|
+
if last and now - when(last["at"]) <= timedelta(days=days):
|
|
272
|
+
return []
|
|
273
|
+
return [step(f"Not governed, and no nightly run in {days} days", -1, f, "nightly")]
|
|
274
|
+
|
|
275
|
+
|
|
276
|
+
def protection(f: Facts, _: dict, __: datetime) -> list[dict]:
|
|
277
|
+
if not f.known("protection"):
|
|
278
|
+
return [step(f"Protection not read: {f.note('protection') or 'unknown'}", -1, f,
|
|
279
|
+
"protection")]
|
|
280
|
+
found = f.value("protection")
|
|
281
|
+
differences = found.get("differences") or []
|
|
282
|
+
if not differences:
|
|
283
|
+
return []
|
|
284
|
+
if found.get("limit"):
|
|
285
|
+
return [step(f"No ruleset: {differences[0]} (a GitHub plan limit)", -1, f, "protection")]
|
|
286
|
+
more = f" (and {len(differences) - 1} more)" if len(differences) > 1 else ""
|
|
287
|
+
return [step(f"Protection differs from pubrules': {differences[0]}{more}", -1, f, "protection")]
|
|
288
|
+
|
|
289
|
+
|
|
290
|
+
def ci(f: Facts, _: dict, __: datetime) -> list[dict]:
|
|
291
|
+
run = f.value("ci")
|
|
292
|
+
if not run:
|
|
293
|
+
return [step(f"No CI run on main{': ' + f.note('ci') if f.note('ci') else ''}", -1, f, "ci")]
|
|
294
|
+
if run.get("conclusion") != "success":
|
|
295
|
+
return [step(f"Main's last CI run ({run.get('workflow')}) is {run.get('conclusion')}",
|
|
296
|
+
-1, f, "ci")]
|
|
297
|
+
return []
|
|
298
|
+
|
|
299
|
+
|
|
300
|
+
DELIVERY: dict[str, Notch] = {
|
|
301
|
+
"pins": pins, "integration": integration, "observability": observability,
|
|
302
|
+
"live-testing": live_testing, "release": release, "issues": issues, "nightly": nightly,
|
|
303
|
+
"protection": protection, "ci": ci,
|
|
304
|
+
}
|
|
305
|
+
|
|
306
|
+
|
|
307
|
+
def applies(entry: dict, kind: str | None) -> bool:
|
|
308
|
+
scope = entry.get("applies", "all")
|
|
309
|
+
return scope == "all" or (scope == "service") == (kind == "service")
|
|
310
|
+
|
|
311
|
+
|
|
312
|
+
def grade(facts: dict[str, Any], rules: dict, now: datetime) -> dict:
|
|
313
|
+
"""One repository's grades: each a start, its steps and its letter;
|
|
314
|
+
the overall letter, the worst of them; and how many notches it took."""
|
|
315
|
+
f = Facts(facts)
|
|
316
|
+
out: dict[str, Any] = {}
|
|
317
|
+
for category in ("quality", "security"):
|
|
318
|
+
start, steps = sonar_start(f, category, rules)
|
|
319
|
+
steps += tool_steps(f, category, rules)
|
|
320
|
+
out[category] = {"start": start, "steps": steps}
|
|
321
|
+
steps = []
|
|
322
|
+
for entry in rules.get("delivery", []):
|
|
323
|
+
if applies(entry, f.value("kind")):
|
|
324
|
+
steps += DELIVERY[entry["notch"]](f, entry, now)
|
|
325
|
+
out["delivery"] = {"start": {"letter": "A", "text": "Start at A", "sources": []},
|
|
326
|
+
"steps": steps}
|
|
327
|
+
for category in CATEGORIES:
|
|
328
|
+
g = out[category]
|
|
329
|
+
g["letter"] = letter(g["start"]["letter"], g["steps"])
|
|
330
|
+
g["notches"] = -sum(s["delta"] for s in g["steps"])
|
|
331
|
+
return {"grades": out, "overall": worse(*(out[c]["letter"] for c in CATEGORIES)),
|
|
332
|
+
"notches": sum(out[c]["notches"] for c in CATEGORIES)}
|
|
333
|
+
|
|
334
|
+
|
|
335
|
+
def order(rows: list[dict]) -> list[dict]:
|
|
336
|
+
"""Worst first: by overall letter, then most notches, then name."""
|
|
337
|
+
return sorted(rows, key=lambda r: (-LETTERS.index(r["overall"]), -r["notches"], r["repo"]))
|
|
@@ -0,0 +1,134 @@
|
|
|
1
|
+
"""`pubfleet board-publish`: keep a new board in storage, if it says anything new, and point
|
|
2
|
+
its head at it.
|
|
3
|
+
|
|
4
|
+
pubfleet board-publish --url https://storage.example.org --tenant some-tenant [--file site/board.json]
|
|
5
|
+
|
|
6
|
+
A tenant's `board` head names its latest snapshot. A board differs from the last only if
|
|
7
|
+
a fact or a grade does: when it was read and the calls that read it don't count. So an
|
|
8
|
+
unchanged run uploads nothing, and every snapshot kept is a change.
|
|
9
|
+
|
|
10
|
+
It signs in with the GitHub Actions job's OpenID Connect token, for the audience that is
|
|
11
|
+
the storage's host name, so the job holds no secret. It prints no repository's name, since
|
|
12
|
+
the logs of a public repository's job are public, and a state may name private ones.
|
|
13
|
+
"""
|
|
14
|
+
|
|
15
|
+
from __future__ import annotations
|
|
16
|
+
|
|
17
|
+
import json
|
|
18
|
+
import os
|
|
19
|
+
import re
|
|
20
|
+
import sys
|
|
21
|
+
import urllib.error
|
|
22
|
+
import urllib.parse
|
|
23
|
+
import urllib.request
|
|
24
|
+
import uuid
|
|
25
|
+
from pathlib import Path
|
|
26
|
+
from typing import Any
|
|
27
|
+
|
|
28
|
+
# Cloudflare refuses Python's default User-Agent (error 1010).
|
|
29
|
+
AGENT = "pubfleet-board-publish"
|
|
30
|
+
TENANT = re.compile(r"[a-z0-9][a-z0-9-]*")
|
|
31
|
+
# What changes every time a board is read, and says nothing about the repositories.
|
|
32
|
+
READING = {"built", "calls", "summary", "sources"}
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
def facts_and_grades(value: Any) -> Any:
|
|
36
|
+
"""A board without what only says when and how it was read."""
|
|
37
|
+
if isinstance(value, dict):
|
|
38
|
+
return {k: facts_and_grades(v) for k, v in value.items() if k not in READING}
|
|
39
|
+
if isinstance(value, list):
|
|
40
|
+
return [facts_and_grades(v) for v in value]
|
|
41
|
+
return value
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
def changed(new: bytes, last: bytes | None) -> bool:
|
|
45
|
+
if last is None:
|
|
46
|
+
return True
|
|
47
|
+
return facts_and_grades(json.loads(new)) != facts_and_grades(json.loads(last))
|
|
48
|
+
|
|
49
|
+
|
|
50
|
+
class Refused(Exception):
|
|
51
|
+
"""Storage's answer to a request it didn't carry out."""
|
|
52
|
+
|
|
53
|
+
def __init__(self, status: int, body: bytes):
|
|
54
|
+
self.status = status
|
|
55
|
+
super().__init__(f"storage said {status}: {body.decode('utf-8', 'replace')[:300]}")
|
|
56
|
+
|
|
57
|
+
|
|
58
|
+
def oidc_token(audience: str) -> str:
|
|
59
|
+
url = os.environ["ACTIONS_ID_TOKEN_REQUEST_URL"]
|
|
60
|
+
separator = "&" if "?" in url else "?"
|
|
61
|
+
request = urllib.request.Request(f"{url}{separator}audience={urllib.parse.quote(audience)}", headers={
|
|
62
|
+
"Authorization": f"Bearer {os.environ['ACTIONS_ID_TOKEN_REQUEST_TOKEN']}"})
|
|
63
|
+
with urllib.request.urlopen(request, timeout=30) as response: # noqa: S310 - GitHub's own address
|
|
64
|
+
return json.load(response)["value"]
|
|
65
|
+
|
|
66
|
+
|
|
67
|
+
class Storage:
|
|
68
|
+
def __init__(self, base: str, tenant: str, token: str):
|
|
69
|
+
self.base, self.tenant, self.token = base.rstrip("/"), tenant, token
|
|
70
|
+
|
|
71
|
+
def _send(self, method: str, path: str, data: bytes | None = None,
|
|
72
|
+
headers: dict | None = None) -> tuple[int, bytes, dict]:
|
|
73
|
+
request = urllib.request.Request(
|
|
74
|
+
f"{self.base}/{self.tenant}{path}", data=data, method=method,
|
|
75
|
+
headers={"Authorization": f"Bearer {self.token}", "User-Agent": AGENT, **(headers or {})})
|
|
76
|
+
try:
|
|
77
|
+
with urllib.request.urlopen(request, timeout=60) as response: # noqa: S310 - the caller's --url
|
|
78
|
+
return response.status, response.read(), dict(response.headers)
|
|
79
|
+
except urllib.error.HTTPError as e:
|
|
80
|
+
return e.code, e.read(), dict(e.headers)
|
|
81
|
+
|
|
82
|
+
def latest(self) -> tuple[str | None, bytes | None]:
|
|
83
|
+
"""The hash the head points at and its file, or None for both before its first move."""
|
|
84
|
+
status, body, headers = self._send("GET", "/head/board")
|
|
85
|
+
if status == 404:
|
|
86
|
+
return None, None
|
|
87
|
+
if status != 200:
|
|
88
|
+
raise Refused(status, body)
|
|
89
|
+
return headers["ETag"].strip('"'), body
|
|
90
|
+
|
|
91
|
+
def upload(self, board: bytes) -> str:
|
|
92
|
+
boundary = uuid.uuid4().hex
|
|
93
|
+
form = (f'--{boundary}\r\nContent-Disposition: form-data; name="file"; filename="board.json"\r\n'
|
|
94
|
+
f"Content-Type: application/json\r\n\r\n").encode() + board + f"\r\n--{boundary}--\r\n".encode()
|
|
95
|
+
status, body, _ = self._send("POST", "/files", form,
|
|
96
|
+
{"Content-Type": f"multipart/form-data; boundary={boundary}"})
|
|
97
|
+
if status != 200:
|
|
98
|
+
raise Refused(status, body)
|
|
99
|
+
return json.loads(body)["id"]
|
|
100
|
+
|
|
101
|
+
def move(self, hash_: str, current: str | None) -> None:
|
|
102
|
+
condition = {"If-Match": f'"{current}"'} if current else {"If-None-Match": "*"}
|
|
103
|
+
status, body, _ = self._send("PUT", "/head/board", json.dumps({"hash": hash_}).encode(),
|
|
104
|
+
{"Content-Type": "application/json", **condition})
|
|
105
|
+
if status != 200:
|
|
106
|
+
raise Refused(status, body)
|
|
107
|
+
|
|
108
|
+
|
|
109
|
+
def run(url: str, tenant: str, file: Path) -> int:
|
|
110
|
+
parsed = urllib.parse.urlsplit(url)
|
|
111
|
+
if parsed.scheme not in ("http", "https") or not parsed.hostname:
|
|
112
|
+
sys.exit(f"pubfleet board-publish: --url must be an address, not {url}")
|
|
113
|
+
if not TENANT.fullmatch(tenant):
|
|
114
|
+
sys.exit(f"pubfleet board-publish: --tenant must be lower case letters, digits and -, not {tenant}")
|
|
115
|
+
try:
|
|
116
|
+
board = file.read_bytes()
|
|
117
|
+
except OSError as e:
|
|
118
|
+
sys.exit(f"pubfleet board-publish: {file}: {e.strerror}")
|
|
119
|
+
for name in ("ACTIONS_ID_TOKEN_REQUEST_URL", "ACTIONS_ID_TOKEN_REQUEST_TOKEN"):
|
|
120
|
+
if not os.environ.get(name):
|
|
121
|
+
sys.exit(f"pubfleet board-publish: no {name}: the job needs the permission id-token: write")
|
|
122
|
+
storage = Storage(url, tenant, oidc_token(parsed.hostname))
|
|
123
|
+
try:
|
|
124
|
+
current, last = storage.latest()
|
|
125
|
+
if not changed(board, last):
|
|
126
|
+
print(f"{tenant}: nothing changed since {current}; nothing uploaded")
|
|
127
|
+
return 0
|
|
128
|
+
new = storage.upload(board)
|
|
129
|
+
storage.move(new, current)
|
|
130
|
+
except Refused as e:
|
|
131
|
+
print(f"{tenant}: {e}", file=sys.stderr)
|
|
132
|
+
return 1
|
|
133
|
+
print(f"{tenant}: the board head now points at {new}")
|
|
134
|
+
return 0
|
pubfleet/cli.py
ADDED
|
@@ -0,0 +1,112 @@
|
|
|
1
|
+
"""pubfleet: what a control plane runs across many repositories.
|
|
2
|
+
|
|
3
|
+
pubfleet board --out DIR --repos FILE [--rules FILE] [--replay FILE] [--record FILE]
|
|
4
|
+
pubfleet board-publish --url URL --tenant TENANT [--file FILE]
|
|
5
|
+
pubfleet state --check | --repos-for board|govern|deploy | --discover [--body PATH] [FILE]
|
|
6
|
+
pubfleet variables --check|--apply [--state FILE] [--repos FILE | REPO...]
|
|
7
|
+
pubfleet freshness [--json] (DIR... | --repos FILE [DIR])
|
|
8
|
+
|
|
9
|
+
Each reads GitHub through `gh`, with a token the command's help names, and
|
|
10
|
+
judges by the pubrules installed beside it.
|
|
11
|
+
"""
|
|
12
|
+
|
|
13
|
+
from __future__ import annotations
|
|
14
|
+
|
|
15
|
+
import argparse
|
|
16
|
+
import sys
|
|
17
|
+
from pathlib import Path
|
|
18
|
+
|
|
19
|
+
from pubrules.paths import confined
|
|
20
|
+
|
|
21
|
+
from . import __version__
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
def parser() -> argparse.ArgumentParser:
|
|
25
|
+
parser = argparse.ArgumentParser(prog="pubfleet", description=__doc__.split("\n")[0])
|
|
26
|
+
parser.add_argument("--version", action="version", version=f"pubfleet {__version__}")
|
|
27
|
+
commands = parser.add_subparsers(dest="command", required=True)
|
|
28
|
+
p = commands.add_parser("board", help="grade every listed repository A to E into "
|
|
29
|
+
"board.json and a static index.html")
|
|
30
|
+
p.add_argument("--out", type=Path, required=True, help="the directory to write them to")
|
|
31
|
+
p.add_argument("--repos", type=Path, metavar="FILE", required=True,
|
|
32
|
+
help="the repos file: grade what it names")
|
|
33
|
+
p.add_argument("--rules", type=Path, help="other rules than the ones pubfleet ships")
|
|
34
|
+
p.add_argument("--replay", type=Path,
|
|
35
|
+
help="read recorded responses from this file instead of the network")
|
|
36
|
+
p.add_argument("--record", type=Path,
|
|
37
|
+
help="also keep every response in this file, for --replay")
|
|
38
|
+
p = commands.add_parser("board-publish", help="keep a new board.json in storage, if a fact or "
|
|
39
|
+
"a grade changed, from a GitHub Actions job")
|
|
40
|
+
p.add_argument("--url", required=True, help="the storage service's address")
|
|
41
|
+
p.add_argument("--tenant", required=True, help="the storage tenant whose board head it moves")
|
|
42
|
+
p.add_argument("--file", type=Path, default=Path("site/board.json"), metavar="FILE",
|
|
43
|
+
help="the board (default: site/board.json)")
|
|
44
|
+
p = commands.add_parser("state", help="check a state repository's state.json, or make a "
|
|
45
|
+
"repos file from it")
|
|
46
|
+
what = p.add_mutually_exclusive_group(required=True)
|
|
47
|
+
what.add_argument("--check", action="store_true",
|
|
48
|
+
help="check its shape, that each repository exists, and each protected one's block")
|
|
49
|
+
what.add_argument("--repos-for", choices=("board", "govern", "deploy"), metavar="ROLE",
|
|
50
|
+
help="print the repos file for a role, board or govern")
|
|
51
|
+
what.add_argument("--discover", action="store_true",
|
|
52
|
+
help="add the owner's repositories the file lacks (on the board, protect "
|
|
53
|
+
"where their pubrules.json says how, not governed), and print how many")
|
|
54
|
+
p.add_argument("--body", type=Path, metavar="PATH",
|
|
55
|
+
help="with --discover: write a pull request's description there")
|
|
56
|
+
p.add_argument("file", nargs="?", type=Path, default=Path("state.json"), metavar="FILE")
|
|
57
|
+
p = commands.add_parser("variables", help="keep repositories' GitHub variables to the values "
|
|
58
|
+
"a state file gives them")
|
|
59
|
+
what = p.add_mutually_exclusive_group(required=True)
|
|
60
|
+
what.add_argument("--check", action="store_true",
|
|
61
|
+
help="change nothing: print how each differs, and fail when one does")
|
|
62
|
+
what.add_argument("--apply", action="store_true",
|
|
63
|
+
help="set those that differ, then run each changed repository's deploy workflow")
|
|
64
|
+
p.add_argument("--state", type=Path, default=Path("state.json"), metavar="FILE",
|
|
65
|
+
help="the state file (default: state.json)")
|
|
66
|
+
p.add_argument("--repos", dest="repos_file", type=Path, metavar="FILE",
|
|
67
|
+
help="the repos file: act on those it lists that the state gives variables")
|
|
68
|
+
p.add_argument("repos", nargs="*", metavar="REPO",
|
|
69
|
+
help="NAME or OWNER/NAME; by default every repository the state gives variables")
|
|
70
|
+
p = commands.add_parser("freshness",
|
|
71
|
+
help="report which pinned pubrules and standard each repository meets")
|
|
72
|
+
p.add_argument("--json", action="store_true", help="JSON rows instead of a Markdown table")
|
|
73
|
+
p.add_argument("--repos", type=Path, metavar="FILE",
|
|
74
|
+
help="report the repositories this repos file governs, checked out in DIR")
|
|
75
|
+
p.add_argument("dirs", nargs="*", metavar="DIR", type=Path)
|
|
76
|
+
return parser
|
|
77
|
+
|
|
78
|
+
|
|
79
|
+
def main(argv: list[str] | None = None) -> int: # noqa: PLR0911 - one branch per command
|
|
80
|
+
args = parser().parse_args(argv)
|
|
81
|
+
if args.command == "freshness":
|
|
82
|
+
from .freshness import freshness # noqa: PLC0415
|
|
83
|
+
repos_file = confined(args.repos, "the --repos file") if args.repos else None
|
|
84
|
+
return freshness(args.dirs, args.json, repos_file)
|
|
85
|
+
if args.command == "variables":
|
|
86
|
+
from .variables import run as variables # noqa: PLC0415
|
|
87
|
+
if args.repos_file and args.repos:
|
|
88
|
+
sys.exit("pubfleet variables: --repos takes the place of naming them")
|
|
89
|
+
repos_file = confined(args.repos_file, "the --repos file") if args.repos_file else None
|
|
90
|
+
return variables(confined(args.state, "the state file"), args.repos, args.apply, repos_file)
|
|
91
|
+
if args.command == "state":
|
|
92
|
+
from .state import check, repos_for # noqa: PLC0415
|
|
93
|
+
if args.repos_for:
|
|
94
|
+
return repos_for(confined(args.file, "the state file"), args.repos_for)
|
|
95
|
+
if args.discover:
|
|
96
|
+
from .discover import run as discover # noqa: PLC0415
|
|
97
|
+
return discover(confined(args.file, "the state file"),
|
|
98
|
+
confined(args.body, "the body file") if args.body else None)
|
|
99
|
+
return check(confined(args.file, "the state file"))
|
|
100
|
+
if args.command == "board-publish":
|
|
101
|
+
from .board_publish import run as publish # noqa: PLC0415
|
|
102
|
+
return publish(args.url, args.tenant, confined(args.file, "the --file file"))
|
|
103
|
+
from .board import run as board # noqa: PLC0415
|
|
104
|
+
files = {name: confined(path, f"the --{name} file") if path else None
|
|
105
|
+
for name, path in (("rules", args.rules), ("replay", args.replay),
|
|
106
|
+
("record", args.record), ("repos", args.repos))}
|
|
107
|
+
return board(confined(args.out, "the --out directory"),
|
|
108
|
+
files["rules"], files["replay"], files["record"], files["repos"])
|
|
109
|
+
|
|
110
|
+
|
|
111
|
+
if __name__ == "__main__":
|
|
112
|
+
sys.exit(main())
|