pubfleet 0.2.1__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
pubfleet/__init__.py ADDED
@@ -0,0 +1,23 @@
1
+ """pubfleet: what a control plane runs across many repositories: grade
2
+ them (board), publish the grades (board-publish), check and read the owner's
3
+ state file (state), keep their GitHub variables (variables), and report
4
+ which pubrules each pins (freshness). Its reusable workflows grade them
5
+ (board.yml) and open their nightly pull requests (nightly.yml, which runs
6
+ `pubrules nightly` in each checkout).
7
+
8
+ It uses pubrules as a library, for the standard it grades against and the
9
+ GitHub calls pubrules makes; pubrules never imports it.
10
+ """
11
+
12
+ from __future__ import annotations
13
+
14
+
15
+ def _version() -> str:
16
+ from importlib.metadata import PackageNotFoundError, version # noqa: PLC0415
17
+ try:
18
+ return version("pubfleet")
19
+ except PackageNotFoundError:
20
+ return "0+source"
21
+
22
+
23
+ __version__ = _version()
@@ -0,0 +1,113 @@
1
+ """`pubfleet board`: grade every repository a repos file names, A to E, for
2
+ quality, security and delivery, into board.json and a static index.html.
3
+
4
+ It reads the repositories from the repos file, reads each from GitHub, GitHub Actions, SonarQube Cloud, OpenSSF Scorecard and
5
+ the services' health URLs through one connector per source
6
+ (connectors/), turns what they read into facts (collect.py), and grades
7
+ the facts by rules.json (rules.py). It writes nothing anywhere else and
8
+ keeps nothing between builds: board.json holds every call made, and links
9
+ each fact and each notch to the calls it rests on.
10
+
11
+ GITHUB_TOKEN (or GH_TOKEN) is the read-only token it reads GitHub with;
12
+ SONAR_TOKEN, when set, reads private SonarQube Cloud projects.
13
+ """
14
+
15
+ from __future__ import annotations
16
+
17
+ import json
18
+ import os
19
+ from collections import Counter
20
+ from datetime import datetime
21
+ from pathlib import Path
22
+
23
+ from pubrules import __version__ as pubrules_version
24
+
25
+ from .. import __version__
26
+ from . import page, rules as grading
27
+ from .collect import Connectors, Gatherer, inventory
28
+ from .connectors.actions import Actions
29
+ from .connectors.base import Ledger, Network, Recording, Replay, Transport, now, stamp
30
+ from .connectors.github import GitHub
31
+ from .connectors.live import Live
32
+ from .connectors.repos import Repos
33
+ from .connectors.scorecard import Scorecard
34
+ from .connectors.sonarcloud import SonarCloud
35
+
36
+
37
+ def connectors(ledger: Ledger, transport: Transport) -> Connectors:
38
+ token = os.environ.get("GITHUB_TOKEN") or os.environ.get("GH_TOKEN") or None
39
+ return Connectors(
40
+ repos=Repos(ledger, transport), github=GitHub(ledger, transport, token),
41
+ actions=Actions(ledger, transport, token),
42
+ sonarcloud=SonarCloud(ledger, transport, os.environ.get("SONAR_TOKEN") or None),
43
+ scorecard=Scorecard(ledger, transport), live=Live(ledger, transport))
44
+
45
+
46
+ def released(row_facts: dict, kind: str) -> str:
47
+ value = row_facts.get("released", {}).get("value")
48
+ if not value:
49
+ return "-"
50
+ return value.get("tag") or (f"deployed {value['at'][:10]}" if kind == "service" else value["at"][:10])
51
+
52
+
53
+ def pinned(row_facts: dict) -> str:
54
+ pins = (row_facts.get("pins", {}).get("value") or {}).get("pubrules")
55
+ if pins == "self":
56
+ return "itself"
57
+ return ", ".join(pins) if pins else "-"
58
+
59
+
60
+ def summary(rows: list[dict], built: datetime) -> str:
61
+ counts = Counter(r["overall"] for r in rows)
62
+ parts = ", ".join(f"{counts[g]} {g}" for g in grading.LETTERS if counts[g])
63
+ return (f"{len(rows)} repo{'s' if len(rows) != 1 else ''}: {parts or 'none'}. "
64
+ f"Built {built.strftime('%Y-%m-%d %H:%M')} UTC.")
65
+
66
+
67
+ def build(repos: Path, rules: dict, transport: Transport, built: datetime) -> dict:
68
+ """The board: every repository's facts and grades, and every call."""
69
+ ledger = Ledger(clock=now if isinstance(transport, (Network, Recording)) else (lambda: built))
70
+ c = connectors(ledger, transport)
71
+ items, problems = inventory(c, repos)
72
+ gatherer = Gatherer(c, rules)
73
+ rows = []
74
+ for item in items:
75
+ facts = {name: fact.as_json() for name, fact in gatherer.gather(item).items()}
76
+ graded = grading.grade(facts, rules, built)
77
+ kind = facts["kind"]["value"]
78
+ rows.append({"repo": item["repo"], "visibility": facts["visibility"]["value"] or "unknown",
79
+ "kind": kind, "released": released(facts, kind), "pubrules": pinned(facts),
80
+ **({"project": item["project"]} if item.get("project") else {}),
81
+ **graded, "facts": facts})
82
+ rows = grading.order(rows)
83
+ return {"built": stamp(built), "pubrules": pubrules_version, "pubfleet": __version__, "rules": rules.get("name", "unnamed"),
84
+ "summary": summary(rows, built), "problems": problems, "repos": rows,
85
+ "calls": ledger.calls}
86
+
87
+
88
+ def run(out: Path, rules_path: Path | None, replay: Path | None, record: Path | None,
89
+ repos: Path) -> int:
90
+ try:
91
+ rules = grading.load(rules_path)
92
+ except (OSError, ValueError) as e:
93
+ raise SystemExit(f"pubfleet board: the rules: {e}") from e
94
+ if replay:
95
+ transport: Transport = Replay(replay)
96
+ built = transport.recorded_at
97
+ else:
98
+ transport = Recording(Network()) if record else Network()
99
+ built = now()
100
+ board = build(repos, rules, transport, built)
101
+ if record and isinstance(transport, Recording):
102
+ transport.save(record, built)
103
+ out.mkdir(parents=True, exist_ok=True)
104
+ (out / "board.json").write_text(json.dumps(board, indent=1) + "\n", encoding="utf-8")
105
+ (out / "index.html").write_text(page.render(board), encoding="utf-8")
106
+ print(board["summary"])
107
+ for problem in board["problems"]:
108
+ print(f"pubfleet board: {problem}")
109
+ for row in board["repos"]:
110
+ grades = " ".join(f"{k[0].upper()}{row['grades'][k]['letter']}"
111
+ for k in grading.CATEGORIES)
112
+ print(f"{row['overall']} {row['repo']} ({grades}, {row['notches']} notches)")
113
+ return 1 if board["problems"] and not board["repos"] else 0
@@ -0,0 +1,255 @@
1
+ """Facts about each repository, from the connectors' readings.
2
+
3
+ This is where readings become the facts rules.py grades: a lock's pin
4
+ becomes a version, rulesets become their differences from pubrules'. Every
5
+ read goes through a connector, and every fact keeps the ids of the calls
6
+ it rests on.
7
+ """
8
+
9
+ from __future__ import annotations
10
+
11
+ import json
12
+ import re
13
+ from dataclasses import dataclass
14
+ from pathlib import Path
15
+ from typing import Any
16
+
17
+ from pubrules.protect import (differences as ruleset_differences, entry, on_default_branch, on_release_tags,
18
+ ruleset, tag_ruleset)
19
+ from pubrules.freshness import PIN
20
+ from pubrules.nightly import PUBREL_VERSION
21
+ from pubrules.refs import ANY_OWNER, AT, RELEASE_PIN, VERSION
22
+
23
+ from .connectors.actions import Actions
24
+ from .connectors.base import Fact
25
+ from .connectors.github import GitHub
26
+ from .connectors.live import Live
27
+ from .connectors.repos import Repos
28
+ from .connectors.scorecard import Scorecard
29
+ from .connectors.sonarcloud import SonarCloud
30
+
31
+ # Any pubrules reusable workflow, the generated job-*.yml included, by
32
+ # commit or by tag, under any owner.
33
+ PUBRULES_PIN = re.compile(ANY_OWNER + r"pubrules/\.github/workflows/[\w.-]+\.ya?ml@" + AT + VERSION)
34
+ LIVE_KEYS = {"health", "test", "deploy"}
35
+ WORKFLOW_NAME = re.compile(r"[\w.-]+\.ya?ml")
36
+
37
+
38
+ @dataclass
39
+ class Connectors:
40
+ repos: Repos
41
+ github: GitHub
42
+ actions: Actions
43
+ sonarcloud: SonarCloud
44
+ scorecard: Scorecard
45
+ live: Live
46
+
47
+
48
+ def merged(*facts: Fact, value: Any, note: str | None = None) -> Fact:
49
+ sources: list[str] = []
50
+ for fact in facts:
51
+ sources += [s for s in fact.sources if s not in sources]
52
+ return Fact(value, tuple(sources), note)
53
+
54
+
55
+ def inventory(c: Connectors, repos: Path) -> tuple[list[dict], list[str]]:
56
+ """The repositories to grade, in the file's order, each with its roles; and any
57
+ problem reading the file."""
58
+ read = c.repos.listed(repos)
59
+ return [{"repo": e["repo"], "governed": e["govern"], "protect": e["protect"],
60
+ "project": e.get("project"), "sources": list(read.sources)}
61
+ for e in read.value or []], \
62
+ ([] if read.value is not None else [f"{repos}: {read.note}"])
63
+
64
+
65
+ class Gatherer:
66
+ """Reads each repository's facts. The latest pubrules and pubrel are
67
+ read once for the whole board."""
68
+
69
+ def __init__(self, c: Connectors, rules: dict):
70
+ self.c = c
71
+ self.rules = rules
72
+ self.tools = rules.get("repositories", {})
73
+ self._latest: Fact | None = None
74
+
75
+ def latest(self) -> Fact:
76
+ if self._latest is None:
77
+ reads = {tool: self.c.github.latest_release(repo) for tool, repo in self.tools.items()}
78
+ value = {tool: (r.value or {}).get("tag", "").removeprefix("v") or None
79
+ for tool, r in reads.items()}
80
+ notes = [f"{tool}: {r.note}" for tool, r in reads.items() if r.value is None]
81
+ self._latest = merged(*reads.values(), value=value, note="; ".join(notes) or None)
82
+ return self._latest
83
+
84
+ def gather(self, item: dict) -> dict[str, Fact]:
85
+ repo, gh, actions = item["repo"], self.c.github, self.c.actions
86
+ facts: dict[str, Fact] = {}
87
+ about = gh.repository(repo)
88
+ facts["visibility"] = Fact((about.value or {}).get("visibility"), about.sources, about.note)
89
+ branch = (about.value or {}).get("default_branch", "main")
90
+
91
+ raw = gh.pubrules_json(repo)
92
+ config, config_note = None, raw.note
93
+ if raw.value is not None:
94
+ try:
95
+ config = json.loads(raw.value)
96
+ except ValueError as e:
97
+ config_note = f"pubrules.json is not JSON ({e})"
98
+ if not isinstance(config, dict):
99
+ config = None
100
+ kind = "service" if config and config.get("kind") == "service" else "package"
101
+ facts["kind"] = Fact(kind, raw.sources, config_note)
102
+
103
+ release_json = gh.release_json(repo)
104
+ lock = gh.requirements_lock(repo)
105
+ names = gh.workflow_names(repo)
106
+ texts = [gh.workflow(repo, n) for n in names.value or [] if WORKFLOW_NAME.fullmatch(n)]
107
+ facts["pins"], facts["integration"] = self.pins(repo, config, raw, release_json, lock,
108
+ names, texts)
109
+ facts["latest"] = self.latest()
110
+
111
+ live, live_note = self.live_block(config)
112
+ facts["live"] = Fact(live, raw.sources, live_note)
113
+ workflows = set(names.value or [])
114
+ facts["released"] = self.released(repo, kind, live, workflows, names)
115
+ facts["issues"] = gh.open_issues(repo)
116
+ facts["dependabot_alerts"] = gh.dependabot_alerts(repo)
117
+ facts["code_scanning_alerts"] = gh.code_scanning_alerts(repo)
118
+ facts["secret_scanning_alerts"] = gh.secret_scanning_alerts(repo)
119
+ facts["secret_protection"] = Fact((about.value or {}).get("security"), about.sources,
120
+ about.note)
121
+ facts["ci"] = self.ci(repo, branch)
122
+ facts["nightly"] = self.nightly(repo, item, workflows, names)
123
+ if kind == "service":
124
+ facts["health"] = (self.c.live.health(live["health"]) if live.get("health")
125
+ else Fact(None, (), "no health URL declared"))
126
+ facts["live_test"] = self.last_run(
127
+ actions.live_test_runs(repo, live["test"]) if live.get("test") else
128
+ Fact(None, (), "no live-test workflow declared"))
129
+ facts["protection"] = self.protection(repo, item, config, raw, facts["visibility"])
130
+ facts["sonar"] = self.c.sonarcloud.ratings(repo)
131
+ if facts["visibility"].value == "public":
132
+ facts["scorecard"] = self.c.scorecard.score(repo)
133
+ return facts
134
+
135
+ def pins(self, repo, config, raw, release_json, lock, names, texts) -> tuple[Fact, Fact]:
136
+ """The pubrules and pubrel versions the repository pins, and whether
137
+ it is integrated with each."""
138
+ workflows = "\n".join(t.value for t in texts if t.value)
139
+ pubrules = sorted(set(PIN.findall(lock.value or ""))
140
+ | {m["version"] for m in PUBRULES_PIN.finditer(workflows)})
141
+ pubrel = {m["version"] for m in RELEASE_PIN.finditer(workflows)}
142
+ pubrel |= {m["version"] for m in PUBREL_VERSION.finditer(workflows)}
143
+ declared = ((config or {}).get("ci") or {}).get("release") if config else None
144
+ if isinstance(declared, dict) and isinstance(declared.get("pubrel"), str):
145
+ pubrel.add(declared["pubrel"].removeprefix("v"))
146
+ pinned: dict[str, Any] = {"pubrules": pubrules, "pubrel": sorted(pubrel)}
147
+ for tool, home in self.tools.items():
148
+ if home == repo:
149
+ pinned[tool] = "self"
150
+ integrated = {
151
+ "pubrules": bool(pinned["pubrules"]) and (config is not None or pinned["pubrules"] == "self"),
152
+ "pubrel": bool(pinned["pubrel"]) and (release_json.value is not None
153
+ or pinned["pubrel"] == "self"),
154
+ }
155
+ read = merged(raw, lock, names, *texts, value=None)
156
+ return (Fact(pinned, read.sources),
157
+ Fact(integrated, merged(read, release_json, value=None).sources))
158
+
159
+ @staticmethod
160
+ def live_block(config: dict | None) -> tuple[dict, str | None]:
161
+ """pubrules.json's live block, checked; empty, with why, when unsound."""
162
+ block = (config or {}).get("live")
163
+ if block is None:
164
+ return {}, None
165
+ if not isinstance(block, dict):
166
+ return {}, "pubrules.json's live block is not an object"
167
+ problems = [f"{k} is not a live setting" for k in sorted(block.keys() - LIVE_KEYS)]
168
+ health = block.get("health")
169
+ if health is not None and not (isinstance(health, str) and health.startswith("https://")):
170
+ problems.append("health must be an https:// URL")
171
+ for key in ("test", "deploy"):
172
+ if key in block and not (isinstance(block[key], str)
173
+ and WORKFLOW_NAME.fullmatch(block[key])):
174
+ problems.append(f"{key} must be a workflow file name, such as live.yml")
175
+ if problems:
176
+ return {}, f"pubrules.json's live block: {'; '.join(problems)}"
177
+ return dict(block), None
178
+
179
+ def released(self, repo, kind, live, workflows, names) -> Fact:
180
+ if kind != "service":
181
+ return self.c.github.latest_release(repo)
182
+ deploy = live.get("deploy") or self.rules.get("workflows", {}).get("deploy", "deploy.yml")
183
+ if deploy not in workflows:
184
+ return Fact(None, names.sources, f"no {deploy}")
185
+ runs = self.c.actions.deploy_runs(repo, deploy)
186
+ good = [r for r in runs.value or [] if r["conclusion"] == "success"]
187
+ if runs.value is None:
188
+ return runs
189
+ if not good:
190
+ return Fact(None, runs.sources, f"no successful {deploy} run")
191
+ return Fact({"at": good[0]["at"], "url": good[0]["url"]}, runs.sources)
192
+
193
+ @staticmethod
194
+ def last_run(runs: Fact) -> Fact:
195
+ if not runs.value:
196
+ return Fact(None, runs.sources, runs.note or "no completed run")
197
+ last = runs.value[0]
198
+ return Fact({"conclusion": last["conclusion"], "at": last["at"], "url": last["url"]},
199
+ runs.sources)
200
+
201
+ def ci(self, repo: str, branch: str) -> Fact:
202
+ runs = self.c.actions.main_runs(repo, branch)
203
+ if runs.value is None:
204
+ return runs
205
+ wanted = self.rules.get("workflows", {}).get("ci", ["ci.yml", "main.yml"])
206
+ found = [r for r in runs.value if r["workflow"] in wanted]
207
+ if not found:
208
+ return Fact(None, runs.sources, f"no completed {' or '.join(wanted)} run on {branch}")
209
+ last = found[0]
210
+ jobs = self.c.actions.jobs(repo, last["id"])
211
+ return merged(runs, jobs, value={**{k: last[k] for k in ("workflow", "conclusion", "at", "url")},
212
+ "jobs": jobs.value or []}, note=jobs.note)
213
+
214
+ def nightly(self, repo, item, workflows, names) -> Fact:
215
+ workflow = self.rules.get("workflows", {}).get("nightly", "nightly.yml")
216
+ if workflow not in workflows:
217
+ return Fact({"governed": item["governed"], "last": None},
218
+ (*item["sources"], *names.sources))
219
+ runs = self.c.actions.nightly_runs(repo, workflow)
220
+ last = self.last_run(runs)
221
+ return Fact({"governed": item["governed"], "last": last.value},
222
+ (*item["sources"], *runs.sources), runs.note)
223
+
224
+ def protection(self, repo, item, config, raw, visibility) -> Fact:
225
+ """How the repository's rulesets differ from what pubrules wants of it:
226
+ not at all, where its repos file entry doesn't ask for any."""
227
+ if not item.get("protect", True):
228
+ return Fact({"differences": [], "limit": False}, item["sources"])
229
+ try:
230
+ wanted = entry(repo, config, "its pubrules.json")
231
+ except ValueError as e:
232
+ return Fact({"differences": [str(e)], "limit": False}, raw.sources)
233
+ found = self.c.github.rulesets(repo)
234
+ sources = (*raw.sources, *found.sources)
235
+ if found.value is None:
236
+ if visibility.value == "private":
237
+ return Fact({"differences": [found.note or "rulesets not readable"], "limit": True},
238
+ sources)
239
+ return Fact(None, sources, found.note)
240
+ differences = []
241
+ targets = [("branch", ruleset(wanted["checks"], wanted.get("queue", True)),
242
+ "its default branch")]
243
+ if wanted.get("tags", True):
244
+ targets.append(("tag", tag_ruleset(wanted.get("release_app")), "its release tags"))
245
+ for target, want, where in targets:
246
+ test = on_default_branch if target == "branch" else on_release_tags
247
+ have = [r for r in found.value if r.get("target") == target and test(r)]
248
+ if not have:
249
+ differences.append(f"no ruleset on {where}")
250
+ continue
251
+ named = [r for r in have if r.get("name") == want["name"]]
252
+ chosen = (named or have)[0]
253
+ differences += ruleset_differences(want, chosen)
254
+ differences += [f"also on {where}: \"{r.get('name')}\"" for r in have if r is not chosen]
255
+ return Fact({"differences": differences, "limit": False}, sources)
@@ -0,0 +1,7 @@
1
+ """One connector per data source, and the only code that reads one.
2
+
3
+ repos (a repos file), github (the REST API), actions
4
+ (the Actions API), sonarcloud, scorecard (OpenSSF) and live (the
5
+ services' own health URLs). Each is a class with one named method per
6
+ query; base.py has what they share.
7
+ """
@@ -0,0 +1,65 @@
1
+ """The actions connector: the GitHub Actions API, for what CI did. The
2
+ runs and job results on main, a repository's own nightly runs, and the
3
+ live-test and deploy runs a service declares.
4
+
5
+ It shares the github connector's token, but answers a different
6
+ question: not what a repository holds, but how its workflows went.
7
+ """
8
+
9
+ from __future__ import annotations
10
+
11
+ from .base import Connector, Fact, Ledger, Transport
12
+ from .github import API, headers
13
+
14
+
15
+ def run(item: dict) -> dict:
16
+ return {"id": item.get("id"), "workflow": (item.get("path") or "").rsplit("/", 1)[-1],
17
+ "event": item.get("event"), "status": item.get("status"),
18
+ "conclusion": item.get("conclusion"), "at": item.get("updated_at") or item.get("created_at"),
19
+ "url": item.get("html_url")}
20
+
21
+
22
+ class Actions(Connector):
23
+ name = "actions"
24
+
25
+ def __init__(self, ledger: Ledger, transport: Transport, token: str | None = None):
26
+ super().__init__(ledger, transport, headers(token))
27
+
28
+ def main_runs(self, repo: str, branch: str = "main") -> Fact:
29
+ """The latest completed runs pushes to the default branch started, newest first."""
30
+ response, call = self.get(
31
+ "main_runs",
32
+ f"{API}/repos/{repo}/actions/runs?branch={branch}&event=push&status=completed&per_page=30")
33
+ if not response.ok:
34
+ return Fact(None, (call,), response.message())
35
+ return Fact([run(r) for r in (response.json() or {}).get("workflow_runs", [])], (call,))
36
+
37
+ def jobs(self, repo: str, run_id: int) -> Fact:
38
+ """Each job of a run, by name, with its conclusion."""
39
+ response, call = self.get("jobs",
40
+ f"{API}/repos/{repo}/actions/runs/{run_id}/jobs?per_page=100")
41
+ if not response.ok:
42
+ return Fact(None, (call,), response.message())
43
+ return Fact([{"name": j.get("name"), "conclusion": j.get("conclusion")}
44
+ for j in (response.json() or {}).get("jobs", [])], (call,))
45
+
46
+ def _workflow_runs(self, method: str, repo: str, workflow: str) -> Fact:
47
+ response, call = self.get(
48
+ method, f"{API}/repos/{repo}/actions/workflows/{workflow}/runs?status=completed&per_page=10")
49
+ if response.status == 404:
50
+ return Fact([], (call,), f"no workflow {workflow}")
51
+ if not response.ok:
52
+ return Fact(None, (call,), response.message())
53
+ return Fact([run(r) for r in (response.json() or {}).get("workflow_runs", [])], (call,))
54
+
55
+ def nightly_runs(self, repo: str, workflow: str = "nightly.yml") -> Fact:
56
+ """The repository's own nightly workflow's latest completed runs."""
57
+ return self._workflow_runs("nightly_runs", repo, workflow)
58
+
59
+ def live_test_runs(self, repo: str, workflow: str) -> Fact:
60
+ """The latest completed runs of the workflow that tests the live service."""
61
+ return self._workflow_runs("live_test_runs", repo, workflow)
62
+
63
+ def deploy_runs(self, repo: str, workflow: str) -> Fact:
64
+ """The latest completed runs of the workflow that deploys to production."""
65
+ return self._workflow_runs("deploy_runs", repo, workflow)
@@ -0,0 +1,193 @@
1
+ """What every connector shares: the calls it makes, and the facts that
2
+ come of them.
3
+
4
+ A connector reads its source through a `Transport` and logs each call in
5
+ the `Ledger`: its connector, method, endpoint, time and outcome, under an
6
+ id such as `c7`. What the connector returns is a `Fact`, a value with the
7
+ ids of the calls it came from, so board.json can say where every value on
8
+ the page was read.
9
+
10
+ `Network` is the network, and the only transport that reaches it; nothing
11
+ outside the connectors package imports it. `Replay`
12
+ answers from a file of recorded responses instead (the local lists
13
+ excepted), for the tests and for
14
+ re-rendering a board without asking anyone; `Recording` passes calls to
15
+ another transport and keeps what came back, for such a file. A token is
16
+ sent as a header and never recorded.
17
+ """
18
+
19
+ from __future__ import annotations
20
+
21
+ import json
22
+ import urllib.error
23
+ import urllib.request
24
+ from dataclasses import dataclass, field
25
+ from datetime import UTC, datetime
26
+ from pathlib import Path
27
+ from typing import Any
28
+
29
+ TIMEOUT = 20
30
+
31
+
32
+ @dataclass(frozen=True)
33
+ class Response:
34
+ """What came back: a status and a body, or the error that stopped it."""
35
+
36
+ status: int | None
37
+ body: bytes = b""
38
+ error: str | None = None
39
+
40
+ @property
41
+ def ok(self) -> bool:
42
+ return self.status is not None and 200 <= self.status < 300
43
+
44
+ def json(self) -> Any:
45
+ return json.loads(self.body) if self.body.strip() else None
46
+
47
+ def message(self) -> str:
48
+ """GitHub's or SonarCloud's own words for a refusal, else the status."""
49
+ if self.error:
50
+ return self.error
51
+ try:
52
+ data = self.json()
53
+ except ValueError:
54
+ data = None
55
+ if isinstance(data, dict):
56
+ if isinstance(data.get("message"), str):
57
+ return f"{data['message']} (HTTP {self.status})"
58
+ errors = data.get("errors")
59
+ if isinstance(errors, list) and errors and isinstance(errors[0], dict):
60
+ return f"{errors[0].get('msg') or errors[0].get('message')} (HTTP {self.status})"
61
+ return f"HTTP {self.status}"
62
+
63
+
64
+ @dataclass(frozen=True)
65
+ class Fact:
66
+ """A value, the calls it was read from, and a note when it could not
67
+ be read in full (the value is then None or partial)."""
68
+
69
+ value: Any
70
+ sources: tuple[str, ...] = ()
71
+ note: str | None = None
72
+
73
+ def as_json(self) -> dict:
74
+ out: dict[str, Any] = {"value": self.value, "sources": list(self.sources)}
75
+ if self.note:
76
+ out["note"] = self.note
77
+ return out
78
+
79
+
80
+ def now() -> datetime:
81
+ return datetime.now(UTC).replace(microsecond=0)
82
+
83
+
84
+ def stamp(at: datetime) -> str:
85
+ return at.astimezone(UTC).strftime("%Y-%m-%dT%H:%M:%SZ")
86
+
87
+
88
+ @dataclass
89
+ class Ledger:
90
+ """The calls of one board build, in order."""
91
+
92
+ clock: Any = now
93
+ calls: list[dict] = field(default_factory=list)
94
+
95
+ def log(self, connector: str, method: str, endpoint: str, response: Response) -> str:
96
+ call = {"id": f"c{len(self.calls) + 1}", "connector": connector, "method": method,
97
+ "endpoint": endpoint, "at": stamp(self.clock()), "status": response.status}
98
+ if response.error:
99
+ call["error"] = response.error
100
+ self.calls.append(call)
101
+ return call["id"]
102
+
103
+
104
+ class Transport:
105
+ """Fetches one URL: a GET, or a file read for `file:` endpoints."""
106
+
107
+ def fetch(self, url: str, headers: dict[str, str]) -> Response:
108
+ raise NotImplementedError
109
+
110
+
111
+ class Network(Transport):
112
+ def fetch(self, url: str, headers: dict[str, str]) -> Response:
113
+ if url.startswith("file:"):
114
+ path = Path(url.removeprefix("file:"))
115
+ try:
116
+ return Response(200, path.read_bytes())
117
+ except OSError as e:
118
+ return Response(None, error=f"cannot read {path}: {e.strerror}")
119
+ if not url.startswith("https://"):
120
+ return Response(None, error=f"{url} is not https")
121
+ request = urllib.request.Request(url, headers=headers)
122
+ try:
123
+ with urllib.request.urlopen(request, timeout=TIMEOUT) as answer: # noqa: S310 - https only, above
124
+ return Response(answer.status, answer.read())
125
+ except urllib.error.HTTPError as e:
126
+ return Response(e.code, e.read())
127
+ except (urllib.error.URLError, TimeoutError, OSError) as e:
128
+ reason = getattr(e, "reason", e)
129
+ return Response(None, error=f"no answer: {reason}")
130
+
131
+
132
+ class Replay(Transport):
133
+ """Recorded responses, by URL; a URL not recorded has no answer."""
134
+
135
+ def __init__(self, path: Path):
136
+ data = json.loads(path.read_text(encoding="utf-8"))
137
+ self.recorded_at = datetime.fromisoformat(data["recorded"].replace("Z", "+00:00"))
138
+ self.responses = data["responses"]
139
+
140
+ def fetch(self, url: str, headers: dict[str, str]) -> Response:
141
+ if url.startswith("file:"): # the lists, which are read where they are
142
+ return Network().fetch(url, headers)
143
+ found = self.responses.get(url)
144
+ if found is None:
145
+ return Response(None, error="not recorded")
146
+ if "error" in found:
147
+ return Response(None, error=found["error"])
148
+ body = found["json"] if "json" in found else found.get("text", "")
149
+ raw = body if isinstance(body, str) else json.dumps(body)
150
+ return Response(found["status"], raw.encode("utf-8"))
151
+
152
+
153
+ class Recording(Transport):
154
+ """`inner`'s answers, kept by URL for a replay file."""
155
+
156
+ def __init__(self, inner: Transport):
157
+ self.inner = inner
158
+ self.responses: dict[str, dict] = {}
159
+
160
+ def fetch(self, url: str, headers: dict[str, str]) -> Response:
161
+ response = self.inner.fetch(url, headers)
162
+ if response.error:
163
+ self.responses[url] = {"error": response.error}
164
+ else:
165
+ try:
166
+ kept: dict[str, Any] = {"status": response.status, "json": response.json()}
167
+ except ValueError:
168
+ kept = {"status": response.status,
169
+ "text": response.body.decode("utf-8", "replace")}
170
+ self.responses[url] = kept
171
+ return response
172
+
173
+ def save(self, path: Path, recorded: datetime) -> None:
174
+ path.write_text(json.dumps({"recorded": stamp(recorded), "responses": self.responses},
175
+ indent=1, sort_keys=True) + "\n", encoding="utf-8")
176
+
177
+
178
+ class Connector:
179
+ """One data source. Each public method is one named query, and the
180
+ only way the board reads that source."""
181
+
182
+ name = ""
183
+
184
+ def __init__(self, ledger: Ledger, transport: Transport, headers: dict[str, str] | None = None):
185
+ self.ledger = ledger
186
+ self.transport = transport
187
+ self.headers = headers or {}
188
+
189
+ def get(self, method: str, url: str, endpoint: str | None = None) -> tuple[Response, str]:
190
+ """GET `url` for the query `method`; the response and its call id.
191
+ The call is logged under `endpoint`, by default the URL."""
192
+ response = self.transport.fetch(url, self.headers)
193
+ return response, self.ledger.log(self.name, method, endpoint or url, response)