pubdeploy 0.8.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- pubdeploy/__init__.py +532 -0
- pubdeploy/channels.py +188 -0
- pubdeploy/cli.py +120 -0
- pubdeploy/config.py +140 -0
- pubdeploy/contract.py +87 -0
- pubdeploy/github.py +61 -0
- pubdeploy/hooks.py +82 -0
- pubdeploy/implied.py +179 -0
- pubdeploy/journal.json +207 -0
- pubdeploy/k8s.py +472 -0
- pubdeploy/machine.py +363 -0
- pubdeploy/model.py +229 -0
- pubdeploy/names.py +25 -0
- pubdeploy/order.py +101 -0
- pubdeploy/providers.py +75 -0
- pubdeploy/registry.py +96 -0
- pubdeploy/release-commit +1 -0
- pubdeploy/schema.json +127 -0
- pubdeploy/schema.py +238 -0
- pubdeploy/store.py +127 -0
- pubdeploy/testing/__init__.py +94 -0
- pubdeploy/testing/fixtures/deploy-app/.github/workflows/deploy.yml +11 -0
- pubdeploy/testing/fixtures/deploy-app/k8s/admin/network-policy.yaml +11 -0
- pubdeploy/testing/fixtures/deploy-app/k8s/app.yaml +63 -0
- pubdeploy/testing/fixtures/deploy-app/k8s/kustomization.yaml +6 -0
- pubdeploy/testing/fixtures/deploy-app/pubrules.json +27 -0
- pubdeploy/testing/fixtures/deploy-app/rendered.yaml +86 -0
- pubdeploy/testing/fixtures/deploy-app/zones.json +8 -0
- pubdeploy/testing/plugin.py +26 -0
- pubdeploy/testing/steps.py +1124 -0
- pubdeploy/testing/world.py +569 -0
- pubdeploy/types/__init__.py +181 -0
- pubdeploy/types/auth.py +199 -0
- pubdeploy/types/github.py +110 -0
- pubdeploy/types/k8s.py +609 -0
- pubdeploy/types/net.py +237 -0
- pubdeploy/types/rollout.py +186 -0
- pubdeploy/units.py +177 -0
- pubdeploy/walk.py +325 -0
- pubdeploy-0.8.0.dist-info/METADATA +520 -0
- pubdeploy-0.8.0.dist-info/RECORD +45 -0
- pubdeploy-0.8.0.dist-info/WHEEL +5 -0
- pubdeploy-0.8.0.dist-info/entry_points.txt +11 -0
- pubdeploy-0.8.0.dist-info/licenses/LICENSE +202 -0
- pubdeploy-0.8.0.dist-info/top_level.txt +1 -0
pubdeploy/channels.py
ADDED
|
@@ -0,0 +1,188 @@
|
|
|
1
|
+
"""How a walk reaches the cluster and the server: kubectl, and a command on the server.
|
|
2
|
+
|
|
3
|
+
admin --server HOST, or SUSTENAUT_SERVER, and no --kubectl: kubectl is
|
|
4
|
+
`ssh root@HOST k3s kubectl`, and a command on the server `ssh
|
|
5
|
+
root@HOST`. Everything a fulfil does it may do. Every command
|
|
6
|
+
goes over one SSH connection the walk keeps (ssh's ControlMaster,
|
|
7
|
+
its socket in a directory only this user may enter), opened
|
|
8
|
+
before the first check: a new connection over the tailnet takes
|
|
9
|
+
about a second, longer than some checks' budgets.
|
|
10
|
+
deploy --kubectl naming the server's read-only verb, `ssh ... deploy@HOST
|
|
11
|
+
UNIT kubectl-read` ({unit} in it is each unit's namespace): reads
|
|
12
|
+
only, so every write fulfil is unknown, and nothing runs on the server.
|
|
13
|
+
plain --kubectl CMD, or kubectl: what the kubeconfig may do. Nothing runs
|
|
14
|
+
on the server.
|
|
15
|
+
plane PUBRULES_PLANE set: in-cluster kubectl. Nothing runs on the server yet.
|
|
16
|
+
|
|
17
|
+
A command's arguments reach the server as one quoted line, as ssh sends
|
|
18
|
+
them, and its standard input carries what is secret: a Secret manifest, a
|
|
19
|
+
script. Neither side writes it to a file, and no argv holds a value.
|
|
20
|
+
|
|
21
|
+
The deploy channel speaks the verb's protocol: each call is a line, a JSON
|
|
22
|
+
array of kubectl's arguments, and each answer a line, {"code", "stdout",
|
|
23
|
+
"stderr"}, over one SSH connection the walk keeps. The verb takes `get`,
|
|
24
|
+
`logs --tail=N`, `rollout status` and `secret-keys NAME`, in the unit's own
|
|
25
|
+
namespace, so the client moves a leading `-n NS` out of the arguments.
|
|
26
|
+
"""
|
|
27
|
+
|
|
28
|
+
from __future__ import annotations
|
|
29
|
+
|
|
30
|
+
import atexit
|
|
31
|
+
import json
|
|
32
|
+
import shlex
|
|
33
|
+
import shutil
|
|
34
|
+
import subprocess
|
|
35
|
+
import tempfile
|
|
36
|
+
import threading
|
|
37
|
+
from collections.abc import Mapping
|
|
38
|
+
from dataclasses import dataclass
|
|
39
|
+
from typing import Any
|
|
40
|
+
|
|
41
|
+
from .model import Kubectl, Ran, Server, no_server
|
|
42
|
+
|
|
43
|
+
TIMEOUT = 30
|
|
44
|
+
VERB = "kubectl-read"
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
def run(argv: list[str], input: bytes | None = None, timeout: float = TIMEOUT) -> Ran: # noqa: A002
|
|
48
|
+
"""A command, `input` on its standard input; its output as text."""
|
|
49
|
+
try:
|
|
50
|
+
done = subprocess.run(argv, input=input if input is not None else b"", capture_output=True,
|
|
51
|
+
timeout=timeout, check=False)
|
|
52
|
+
except (OSError, subprocess.TimeoutExpired) as e:
|
|
53
|
+
return Ran(127, "", f"{argv[0] if argv else 'a command'} did not run ({type(e).__name__})")
|
|
54
|
+
return Ran(done.returncode, done.stdout.decode(errors="replace"), done.stderr.decode(errors="replace"))
|
|
55
|
+
|
|
56
|
+
|
|
57
|
+
def plain(command: str) -> Kubectl:
|
|
58
|
+
"""kubectl as `command` gives it, on this machine."""
|
|
59
|
+
base = shlex.split(command)
|
|
60
|
+
|
|
61
|
+
def kubectl(args: list[str], input: bytes | None = None) -> Ran: # noqa: A002
|
|
62
|
+
return run([*base, *args], input)
|
|
63
|
+
return kubectl
|
|
64
|
+
|
|
65
|
+
|
|
66
|
+
class Connection:
|
|
67
|
+
"""root@HOST over one SSH connection: the first command opens it, and
|
|
68
|
+
each after it starts in about 0.1 s. The socket is in a directory of its
|
|
69
|
+
own, made 0700, and the connection is closed when the walk ends. The
|
|
70
|
+
path stays short, as a socket's must."""
|
|
71
|
+
|
|
72
|
+
def __init__(self, host: str) -> None:
|
|
73
|
+
self.host = host
|
|
74
|
+
self.dir: str | None = None
|
|
75
|
+
|
|
76
|
+
def options(self) -> list[str]:
|
|
77
|
+
if self.dir is None:
|
|
78
|
+
# A socket's path has at most 104 bytes: "s" in mkdtemp's directory, even
|
|
79
|
+
# under macOS's long $TMPDIR, is about 70.
|
|
80
|
+
self.dir = tempfile.mkdtemp(prefix="pubdeploy-ssh-")
|
|
81
|
+
atexit.register(self.close)
|
|
82
|
+
return ["-o", "ControlMaster=auto", "-o", f"ControlPath={self.dir}/s", "-o", "ControlPersist=60"]
|
|
83
|
+
|
|
84
|
+
def argv(self) -> list[str]:
|
|
85
|
+
return ["ssh", "-o", "BatchMode=yes", *self.options(), f"root@{self.host}"]
|
|
86
|
+
|
|
87
|
+
def open(self) -> Ran:
|
|
88
|
+
"""The connection, opened now, outside any check's budget."""
|
|
89
|
+
return run([*self.argv(), "true"])
|
|
90
|
+
|
|
91
|
+
def close(self) -> None:
|
|
92
|
+
if self.dir is None:
|
|
93
|
+
return
|
|
94
|
+
run(["ssh", "-o", f"ControlPath={self.dir}/s", "-O", "exit", f"root@{self.host}"], timeout=5)
|
|
95
|
+
shutil.rmtree(self.dir, ignore_errors=True)
|
|
96
|
+
self.dir = None
|
|
97
|
+
|
|
98
|
+
|
|
99
|
+
def admin_server(connection: Connection) -> Server:
|
|
100
|
+
"""A command on the server, as root, its arguments one quoted line."""
|
|
101
|
+
def server(args: list[str], input: bytes | None = None) -> Ran: # noqa: A002
|
|
102
|
+
return run([*connection.argv(), shlex.join(args)], input, timeout=120)
|
|
103
|
+
return server
|
|
104
|
+
|
|
105
|
+
|
|
106
|
+
def admin_kubectl(connection: Connection) -> Kubectl:
|
|
107
|
+
on = admin_server(connection)
|
|
108
|
+
|
|
109
|
+
def kubectl(args: list[str], input: bytes | None = None) -> Ran: # noqa: A002
|
|
110
|
+
return on(["k3s", "kubectl", *args], input)
|
|
111
|
+
return kubectl
|
|
112
|
+
|
|
113
|
+
|
|
114
|
+
class DeployChannel:
|
|
115
|
+
"""The server's read-only verb, over one SSH connection, a call at a time."""
|
|
116
|
+
|
|
117
|
+
def __init__(self, argv: list[str]) -> None:
|
|
118
|
+
self.argv = argv
|
|
119
|
+
self.process: subprocess.Popen[bytes] | None = None
|
|
120
|
+
self.lock = threading.Lock()
|
|
121
|
+
|
|
122
|
+
def __call__(self, args: list[str], input: bytes | None = None) -> Ran: # noqa: A002
|
|
123
|
+
if input is not None:
|
|
124
|
+
return Ran(2, "", f"{VERB} reads only: it takes no input")
|
|
125
|
+
if args[:1] == ["-n"]:
|
|
126
|
+
args = args[2:] # the verb runs in the unit's namespace
|
|
127
|
+
if not self.lock.acquire(timeout=TIMEOUT):
|
|
128
|
+
return Ran(124, "", f"{VERB}: the channel is busy with a call that was left")
|
|
129
|
+
try:
|
|
130
|
+
return self.call(args)
|
|
131
|
+
finally:
|
|
132
|
+
self.lock.release()
|
|
133
|
+
|
|
134
|
+
def call(self, args: list[str]) -> Ran:
|
|
135
|
+
try:
|
|
136
|
+
if self.process is None or self.process.poll() is not None:
|
|
137
|
+
self.process = subprocess.Popen(self.argv, stdin=subprocess.PIPE, stdout=subprocess.PIPE,
|
|
138
|
+
stderr=subprocess.PIPE)
|
|
139
|
+
assert self.process.stdin and self.process.stdout
|
|
140
|
+
self.process.stdin.write((json.dumps(args) + "\n").encode())
|
|
141
|
+
self.process.stdin.flush()
|
|
142
|
+
line = self.process.stdout.readline()
|
|
143
|
+
except (OSError, ValueError) as e:
|
|
144
|
+
return Ran(127, "", f"{VERB} did not run ({type(e).__name__})")
|
|
145
|
+
if not line:
|
|
146
|
+
code = self.process.wait(timeout=TIMEOUT)
|
|
147
|
+
said = self.process.stderr.read().decode(errors="replace") if self.process.stderr else ""
|
|
148
|
+
self.process = None
|
|
149
|
+
return Ran(code or 255, "", said.strip().splitlines()[-1] if said.strip() else f"{VERB} closed")
|
|
150
|
+
try:
|
|
151
|
+
answer: Any = json.loads(line)
|
|
152
|
+
return Ran(int(answer["code"]), str(answer.get("stdout", "")), str(answer.get("stderr", "")))
|
|
153
|
+
except (ValueError, KeyError, TypeError):
|
|
154
|
+
return Ran(1, "", f"{VERB}'s answer is not one line of JSON")
|
|
155
|
+
|
|
156
|
+
def close(self) -> None:
|
|
157
|
+
if self.process is not None and self.process.poll() is None:
|
|
158
|
+
if self.process.stdin:
|
|
159
|
+
self.process.stdin.close()
|
|
160
|
+
try:
|
|
161
|
+
self.process.wait(timeout=5)
|
|
162
|
+
except subprocess.TimeoutExpired:
|
|
163
|
+
self.process.kill()
|
|
164
|
+
|
|
165
|
+
|
|
166
|
+
@dataclass
|
|
167
|
+
class Channels:
|
|
168
|
+
name: str # admin, deploy, plain or plane
|
|
169
|
+
kubectl: Kubectl
|
|
170
|
+
server: Server
|
|
171
|
+
connection: Connection | None = None # the admin channel's
|
|
172
|
+
|
|
173
|
+
|
|
174
|
+
def channels(kubectl_command: str | None, server_host: str | None, env: Mapping[str, str],
|
|
175
|
+
unit: str = "") -> Channels:
|
|
176
|
+
"""The walk's channels, from --kubectl, --server and the environment, for a unit's namespace."""
|
|
177
|
+
if env.get("PUBRULES_PLANE"):
|
|
178
|
+
return Channels("plane", plain("kubectl"), no_server)
|
|
179
|
+
if kubectl_command:
|
|
180
|
+
argv = shlex.split(kubectl_command.replace("{unit}", unit))
|
|
181
|
+
if argv and argv[-1] == VERB:
|
|
182
|
+
return Channels("deploy", DeployChannel(argv), no_server)
|
|
183
|
+
return Channels("plain", plain(shlex.join(argv)), no_server)
|
|
184
|
+
host = server_host or env.get("SUSTENAUT_SERVER")
|
|
185
|
+
if host:
|
|
186
|
+
connection = Connection(host)
|
|
187
|
+
return Channels("admin", admin_kubectl(connection), admin_server(connection), connection)
|
|
188
|
+
return Channels("plain", plain("kubectl"), no_server)
|
pubdeploy/cli.py
ADDED
|
@@ -0,0 +1,120 @@
|
|
|
1
|
+
"""pubdeploy: what a service needs before it deploys, whether it has it,
|
|
2
|
+
and making it; and the checks a deploy to Kubernetes runs.
|
|
3
|
+
|
|
4
|
+
pubdeploy walk [--fulfil] [--yes] [--unit NAME] [--kubectl CMD] [--server HOST]
|
|
5
|
+
[--zones PATH]... [--journal URL] [--json] [--offline] [--namespace NS]
|
|
6
|
+
[--tolerate-unknown]
|
|
7
|
+
pubdeploy walk ... --need ID [--need ID]... [--budget-seconds S]
|
|
8
|
+
only those needs: what a live run walks
|
|
9
|
+
pubdeploy check ... walk without --fulfil
|
|
10
|
+
pubdeploy machine [--unit U] [--mode check|fulfil] [--modes ID=MODE,...]
|
|
11
|
+
the unit's needs as a state machine (XState JSON)
|
|
12
|
+
pubdeploy types every need type, and its keys
|
|
13
|
+
pubdeploy k8s preflight|diagnose before an apply, and after a failed rollout
|
|
14
|
+
pubdeploy sync-check [--check] what `pubrules sync` runs of pubdeploy's
|
|
15
|
+
pubdeploy bump what `pubrules bump` runs of pubdeploy's
|
|
16
|
+
|
|
17
|
+
A repository runs its pinned pubdeploy as ./pubdeploy, which `pubrules sync`
|
|
18
|
+
writes beside ./pubrules where pubrules.json has a deploy block, with each
|
|
19
|
+
deploy.providers wheel, so the need types other packages offer (pubsec's,
|
|
20
|
+
pg-backup's) load. It runs in the repository's root.
|
|
21
|
+
"""
|
|
22
|
+
|
|
23
|
+
from __future__ import annotations
|
|
24
|
+
|
|
25
|
+
import argparse
|
|
26
|
+
import sys
|
|
27
|
+
from pathlib import Path
|
|
28
|
+
|
|
29
|
+
from . import __version__
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
def main(argv: list[str] | None = None) -> int:
|
|
33
|
+
given = sys.argv[1:] if argv is None else argv
|
|
34
|
+
if given[:1] == ["k8s"]:
|
|
35
|
+
# Its own parser, in a file a server can run without the rest.
|
|
36
|
+
from .k8s import main as k8s # noqa: PLC0415
|
|
37
|
+
return k8s(given[1:])
|
|
38
|
+
parser = argparse.ArgumentParser(prog="pubdeploy", description=__doc__.split("\n")[0])
|
|
39
|
+
parser.add_argument("--version", action="version", version=f"pubdeploy {__version__}")
|
|
40
|
+
commands = parser.add_subparsers(dest="command", required=True)
|
|
41
|
+
for verb, said in (("walk", "check every need, in order, print each one's state, and with --fulfil make "
|
|
42
|
+
"what is missing; exit 1 when one is not ready"),
|
|
43
|
+
("check", "walk without --fulfil: check every need, changing nothing")):
|
|
44
|
+
p = commands.add_parser(verb, help=said)
|
|
45
|
+
if verb == "walk":
|
|
46
|
+
p.add_argument("--fulfil", action="store_true",
|
|
47
|
+
help="after the checks, make each auto need that is missing, once the plan is confirmed")
|
|
48
|
+
p.add_argument("--yes", action="store_true", help="confirm the plan without asking (no write token "
|
|
49
|
+
"is asked for without a terminal)")
|
|
50
|
+
p.add_argument("--tolerate-unknown", action="store_true",
|
|
51
|
+
help="exit 0 unless a need is missing, drift or blocked: an unknown one fails nothing")
|
|
52
|
+
p.add_argument("--unit", metavar="NAME", help="walk one unit of the deploy block; every unit by default")
|
|
53
|
+
p.add_argument("--offline", action="store_true",
|
|
54
|
+
help="check only what needs no network: the manifests against the needs, and the files")
|
|
55
|
+
p.add_argument("--json", action="store_true", help="print the walk as a deploy-journal/2 document")
|
|
56
|
+
p.add_argument("--kubectl", metavar="CMD",
|
|
57
|
+
help="the kubectl command, such as 'k3s kubectl --kubeconfig FILE', or the server's "
|
|
58
|
+
"read-only verb, 'ssh -i KEY deploy@HOST {unit} kubectl-read'")
|
|
59
|
+
p.add_argument("--server", metavar="HOST", help="the admin channel: kubectl and commands over ssh root@HOST "
|
|
60
|
+
"(default SUSTENAUT_SERVER, without --kubectl)")
|
|
61
|
+
p.add_argument("--zones", type=Path, metavar="PATH", action="append",
|
|
62
|
+
help="a pubrules.json, here or in a checkout beside this one, whose hosting block "
|
|
63
|
+
"keeps the zone of a unit's host; one per owner whose zones the hosts are in")
|
|
64
|
+
p.add_argument("--journal", metavar="URL", help="also write the journal to storage (or PUBRULES_JOURNAL_URL)")
|
|
65
|
+
p.add_argument("--namespace", metavar="NS", help="another namespace than the unit's")
|
|
66
|
+
p.add_argument("--need", metavar="ID", action="append", dest="needs",
|
|
67
|
+
help="walk only this need (repeat it): the needs it waits for are checked, never "
|
|
68
|
+
"fulfilled, and when one of those it waits for directly is not ready nothing is "
|
|
69
|
+
"done (exit 2)")
|
|
70
|
+
p.add_argument("--budget-seconds", type=float, metavar="S",
|
|
71
|
+
help="the most a check or a fulfil may take, below its type's budget")
|
|
72
|
+
commands.add_parser("types", help="every need type pubdeploy and the installed packages offer, and its keys")
|
|
73
|
+
commands.add_parser("k8s", help="check the Secrets rendered manifests read before a deploy, or show "
|
|
74
|
+
"why a rollout failed (pubdeploy k8s --help)")
|
|
75
|
+
p = commands.add_parser("machine", help="the unit's needs as a state machine (XState JSON), which the "
|
|
76
|
+
"convergence engine runs live")
|
|
77
|
+
p.add_argument("--unit", metavar="NAME", help="the unit; needed when the block has several")
|
|
78
|
+
p.add_argument("--mode", choices=("check", "fulfil"), help="every need in this mode")
|
|
79
|
+
p.add_argument("--modes", metavar="ID=MODE,...", default="", help="a need's own mode, over --mode")
|
|
80
|
+
p = commands.add_parser("sync-check", help="the deploy block's offline check: what pubrules sync runs")
|
|
81
|
+
p.add_argument("--check", action="store_true", help="fail when the deploy block is wrong")
|
|
82
|
+
commands.add_parser("bump", help="move deploy.providers and the uses of pubdeploy's workflows to the "
|
|
83
|
+
"latest releases: what pubrules bump runs")
|
|
84
|
+
args = parser.parse_args(given)
|
|
85
|
+
root = Path.cwd()
|
|
86
|
+
if args.command == "types":
|
|
87
|
+
from . import run_types # noqa: PLC0415
|
|
88
|
+
return run_types()
|
|
89
|
+
if args.command == "sync-check":
|
|
90
|
+
from .config import read_config # noqa: PLC0415
|
|
91
|
+
from .hooks import check # noqa: PLC0415
|
|
92
|
+
lines, failed = check(root, read_config(root), args.check)
|
|
93
|
+
for line in lines:
|
|
94
|
+
print(line, file=sys.stderr if line.startswith("pubdeploy:") else sys.stdout)
|
|
95
|
+
return 1 if failed else 0
|
|
96
|
+
if args.command == "machine":
|
|
97
|
+
from . import run_machine # noqa: PLC0415
|
|
98
|
+
modes = dict(pair.split("=", 1) for pair in args.modes.split(",") if "=" in pair)
|
|
99
|
+
return run_machine(root, args.unit, args.mode, modes)
|
|
100
|
+
if args.command == "bump":
|
|
101
|
+
from .hooks import bump # noqa: PLC0415
|
|
102
|
+
for line in bump(root):
|
|
103
|
+
print(line)
|
|
104
|
+
return 0
|
|
105
|
+
from . import run_walk # noqa: PLC0415
|
|
106
|
+
from .config import confined # noqa: PLC0415
|
|
107
|
+
# --zones is another repository's pubrules.json, so it may be in a
|
|
108
|
+
# checkout beside this one: a pubrules.json in the parent directory's tree.
|
|
109
|
+
zones = [confined(z, "the --zones file", also=(root.parent,)) for z in args.zones or []] or None
|
|
110
|
+
for z, given in zip(zones or [], args.zones or []):
|
|
111
|
+
if z.name != "pubrules.json":
|
|
112
|
+
sys.exit(f"pubdeploy: the --zones file ({given}) must be a repository's pubrules.json")
|
|
113
|
+
return run_walk(root, args.offline, args.json, args.kubectl, zones, args.namespace,
|
|
114
|
+
fulfil=getattr(args, "fulfil", False), unit=args.unit, server=args.server,
|
|
115
|
+
yes=getattr(args, "yes", False), journal_url=args.journal,
|
|
116
|
+
tolerate_unknown=args.tolerate_unknown, only=args.needs, budget_seconds=args.budget_seconds)
|
|
117
|
+
|
|
118
|
+
|
|
119
|
+
if __name__ == "__main__":
|
|
120
|
+
sys.exit(main())
|
pubdeploy/config.py
ADDED
|
@@ -0,0 +1,140 @@
|
|
|
1
|
+
"""What pubdeploy reads of a repository's pubrules.json beside the deploy
|
|
2
|
+
block: the file itself, whose owner's pubdeploy a workflow calls, and the
|
|
3
|
+
`github_app` and `secrets` blocks, which imply the github.* needs. pubrules
|
|
4
|
+
validates those blocks in `pubrules sync`; this is the same reading, kept here
|
|
5
|
+
so pubdeploy imports nothing of pubrules's."""
|
|
6
|
+
|
|
7
|
+
from __future__ import annotations
|
|
8
|
+
|
|
9
|
+
import json
|
|
10
|
+
import os
|
|
11
|
+
import re
|
|
12
|
+
import sys
|
|
13
|
+
from importlib import resources
|
|
14
|
+
from pathlib import Path
|
|
15
|
+
from typing import Any
|
|
16
|
+
|
|
17
|
+
CONFIG = "pubrules.json"
|
|
18
|
+
# The owner of pubdeploy's repository, whose workflows a repository calls;
|
|
19
|
+
# pubrules.json's `workflows_owner` names another.
|
|
20
|
+
OWNER = "Sustenaut"
|
|
21
|
+
OWNER_NAME = re.compile(r"[A-Za-z0-9](?:[A-Za-z0-9-]{0,37}[A-Za-z0-9])?")
|
|
22
|
+
COMMIT = re.compile(r"[0-9a-f]{40}")
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
def read_config(root: Path) -> dict[str, Any]:
|
|
26
|
+
path = root / CONFIG
|
|
27
|
+
if not path.exists():
|
|
28
|
+
sys.exit(f"pubdeploy: no {CONFIG} here; pubdeploy walks the deploy block of a repository's {CONFIG}")
|
|
29
|
+
config = json.loads(path.read_text(encoding="utf-8"))
|
|
30
|
+
if not isinstance(config, dict):
|
|
31
|
+
sys.exit(f"pubdeploy: {CONFIG} must be an object")
|
|
32
|
+
return config
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
# A copy of pubrules's refs.owner.
|
|
36
|
+
def owner(config: dict[str, Any] | None) -> str:
|
|
37
|
+
"""The owner pubrules.json's `workflows_owner` names, or OWNER."""
|
|
38
|
+
given = (config or {}).get("workflows_owner", OWNER)
|
|
39
|
+
if not isinstance(given, str) or not OWNER_NAME.fullmatch(given):
|
|
40
|
+
sys.exit(f"pubdeploy: pubrules.json's workflows_owner must be a GitHub owner, such as "
|
|
41
|
+
f'"{OWNER}", not {given!r}')
|
|
42
|
+
return given
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
# A copy of pubrules's refs.ref.
|
|
46
|
+
def ref(version: str, commit: str | None) -> str:
|
|
47
|
+
"""What follows `@` in a pin of `version`: its commit, with the version
|
|
48
|
+
in a comment, or the tag when the commit is unknown."""
|
|
49
|
+
return f"{commit} # v{version}" if commit else f"v{version}"
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
# After pubrules's refs.release_commit.
|
|
53
|
+
def release_commit() -> str | None:
|
|
54
|
+
"""The commit of this pubdeploy's release tag, which wheel.yml writes into
|
|
55
|
+
the wheel as `release-commit`; None from a source tree.
|
|
56
|
+
PUBDEPLOY_RELEASE_COMMIT, which tests set, stands in for it."""
|
|
57
|
+
given = os.environ.get("PUBDEPLOY_RELEASE_COMMIT")
|
|
58
|
+
if given is None:
|
|
59
|
+
try:
|
|
60
|
+
given = (resources.files("pubdeploy") / "release-commit").read_text(encoding="utf-8")
|
|
61
|
+
except FileNotFoundError:
|
|
62
|
+
return None
|
|
63
|
+
given = given.strip()
|
|
64
|
+
if not given:
|
|
65
|
+
return None
|
|
66
|
+
if not COMMIT.fullmatch(given):
|
|
67
|
+
sys.exit(f"pubdeploy: its release commit must be 40 hexadecimal digits, not {given!r}")
|
|
68
|
+
return given
|
|
69
|
+
|
|
70
|
+
|
|
71
|
+
# The github_app and secrets blocks: after pubrules's secrets_config (plan, config_problems).
|
|
72
|
+
APP = "github_app"
|
|
73
|
+
OTHERS = "secrets"
|
|
74
|
+
KINDS = {"secret", "variable"}
|
|
75
|
+
SLOTS = {"id": "variable", "installation_id": "variable", "key": "secret"}
|
|
76
|
+
FROM = re.compile(r"[A-Za-z0-9][A-Za-z0-9_.-]*")
|
|
77
|
+
TO = re.compile(r"[A-Z_][A-Z0-9_]*")
|
|
78
|
+
ENVIRONMENT = re.compile(r"[A-Za-z0-9][A-Za-z0-9_.-]*")
|
|
79
|
+
|
|
80
|
+
|
|
81
|
+
def _app_sound(block: Any) -> bool:
|
|
82
|
+
if not isinstance(block, dict) or set(block) != set(SLOTS):
|
|
83
|
+
return False
|
|
84
|
+
for given in block.values():
|
|
85
|
+
if not isinstance(given, dict) or set(given) != {"from", "to"}:
|
|
86
|
+
return False
|
|
87
|
+
if not isinstance(given["from"], str) or not FROM.fullmatch(given["from"]):
|
|
88
|
+
return False
|
|
89
|
+
if not isinstance(given["to"], str) or not TO.fullmatch(given["to"]) or given["to"].startswith("GITHUB_"):
|
|
90
|
+
return False
|
|
91
|
+
return True
|
|
92
|
+
|
|
93
|
+
|
|
94
|
+
def _entry_sound(entry: Any) -> bool:
|
|
95
|
+
if not isinstance(entry, dict) or set(entry) - {"environment"} != {"from", "to", "kind"}:
|
|
96
|
+
return False
|
|
97
|
+
return (isinstance(entry["from"], str) and bool(FROM.fullmatch(entry["from"]))
|
|
98
|
+
and isinstance(entry["to"], str) and bool(TO.fullmatch(entry["to"])) and not entry["to"].startswith("GITHUB_")
|
|
99
|
+
and entry["kind"] in KINDS
|
|
100
|
+
and ("environment" not in entry or (isinstance(entry["environment"], str)
|
|
101
|
+
and bool(ENVIRONMENT.fullmatch(entry["environment"])))))
|
|
102
|
+
|
|
103
|
+
|
|
104
|
+
def secrets_sound(config: dict[str, Any]) -> bool:
|
|
105
|
+
"""Whether both blocks read as `pubrules sync` takes them, no name used twice.
|
|
106
|
+
pubrules says what is wrong with them; an unsound block implies no need."""
|
|
107
|
+
if APP in config:
|
|
108
|
+
block = config[APP]
|
|
109
|
+
apps = block if isinstance(block, list) else [block]
|
|
110
|
+
if not apps or not all(_app_sound(a) for a in apps):
|
|
111
|
+
return False
|
|
112
|
+
if OTHERS in config:
|
|
113
|
+
if not isinstance(config[OTHERS], list) or not all(_entry_sound(e) for e in config[OTHERS]):
|
|
114
|
+
return False
|
|
115
|
+
names = [(item["kind"], item["to"], item.get("environment", "")) for item in plan(config)]
|
|
116
|
+
return len(names) == len(set(names))
|
|
117
|
+
|
|
118
|
+
|
|
119
|
+
def plan(config: dict[str, Any]) -> list[dict[str, Any]]:
|
|
120
|
+
"""Everything the two blocks name: from, to, kind, and which app slot, if any."""
|
|
121
|
+
block = config.get(APP, [])
|
|
122
|
+
apps = block if isinstance(block, list) else [block]
|
|
123
|
+
out = [{"from": app[slot]["from"], "to": app[slot]["to"], "kind": kind, "check": slot}
|
|
124
|
+
for app in apps for slot, kind in SLOTS.items()]
|
|
125
|
+
return out + [{**entry, "check": None} for entry in config.get(OTHERS, [])]
|
|
126
|
+
|
|
127
|
+
|
|
128
|
+
# A copy of pubrules's paths.confined.
|
|
129
|
+
def confined(path: Path | str, what: str, also: tuple[Path, ...] = ()) -> Path:
|
|
130
|
+
"""`path`, resolved, when it lies in the working directory, a temporary
|
|
131
|
+
directory, or one of `also`; otherwise pubdeploy stops, naming `what`."""
|
|
132
|
+
import tempfile # noqa: PLC0415
|
|
133
|
+
bases = [Path.cwd(), Path(tempfile.gettempdir())]
|
|
134
|
+
if os.environ.get("RUNNER_TEMP"):
|
|
135
|
+
bases.append(Path(os.environ["RUNNER_TEMP"]))
|
|
136
|
+
resolved = Path(path).resolve()
|
|
137
|
+
if any(resolved.is_relative_to(base.resolve()) for base in [*bases, *also]):
|
|
138
|
+
return resolved
|
|
139
|
+
where = " or ".join(["the working directory", "a temporary directory", *(str(a) for a in also)])
|
|
140
|
+
sys.exit(f"pubdeploy: {what} ({path}) must be in {where}")
|
pubdeploy/contract.py
ADDED
|
@@ -0,0 +1,87 @@
|
|
|
1
|
+
"""The provider contract: what a package offering need types or walk hooks
|
|
2
|
+
may rely on, and the one module of pubdeploy, beside pubdeploy.testing, that
|
|
3
|
+
a provider or its tests may import.
|
|
4
|
+
|
|
5
|
+
A provider offers need types under the entry-point group `sustenaut.needs`,
|
|
6
|
+
each an instance, and hooks under `sustenaut.walk`. Its code imports nothing
|
|
7
|
+
of pubdeploy: any object with a result's names serves as a Result. Its tests
|
|
8
|
+
may build Need, Result and Ran from here, and check its types with
|
|
9
|
+
`problems` and its hooks with `hook_problems`, which say what `./pubdeploy`
|
|
10
|
+
says when it leaves one out.
|
|
11
|
+
|
|
12
|
+
The name sets below are the contract. A provider detects what it needs by
|
|
13
|
+
inclusion (the Context names its code reads are a subset of CONTEXT), not by
|
|
14
|
+
a version: a change that drops or renames a name is a breaking change, said
|
|
15
|
+
in pubdeploy's changelog, and README's "Provider contract" lists the same
|
|
16
|
+
names, which a scenario keeps equal to these.
|
|
17
|
+
"""
|
|
18
|
+
|
|
19
|
+
from __future__ import annotations
|
|
20
|
+
|
|
21
|
+
import inspect
|
|
22
|
+
import re
|
|
23
|
+
from typing import Any
|
|
24
|
+
|
|
25
|
+
from .model import GRADES, STATES, Need, Ran, Result
|
|
26
|
+
|
|
27
|
+
__all__ = ["CONTEXT", "GRADES", "GROUPS", "HOOK", "LONGEST", "LONGEST_FULFIL", "NEED", "NEEDS", "RESULT",
|
|
28
|
+
"RESULT_OPTIONAL", "STATES", "TYPE", "TYPE_OPTIONAL", "WALK", "Need", "Ran", "Result", "hook_problems",
|
|
29
|
+
"problems"]
|
|
30
|
+
|
|
31
|
+
NEEDS, WALK = "sustenaut.needs", "sustenaut.walk"
|
|
32
|
+
GROUPS = (NEEDS, WALK)
|
|
33
|
+
|
|
34
|
+
# What a check or a fulfil may read of the walk's Context.
|
|
35
|
+
CONTEXT = frozenset({"root", "config", "block", "namespace", "unit", "channel", "kubectl", "server", "gh", "send",
|
|
36
|
+
"bitwarden_writer", "http_get", "env", "now", "rendered", "offline", "slug", "zones", "resolve",
|
|
37
|
+
"values", "types", "needs", "results", "sleep", "dispatched"})
|
|
38
|
+
NEED = frozenset({"id", "type", "name", "params", "implied_by", "after"})
|
|
39
|
+
RESULT = frozenset({"state", "summary", "fix", "details"})
|
|
40
|
+
RESULT_OPTIONAL = frozenset({"fixable", "step"})
|
|
41
|
+
TYPE = frozenset({"type", "schema", "grade", "budget", "offline", "default_name", "depends_on", "check", "fulfil"})
|
|
42
|
+
TYPE_OPTIONAL = frozenset({"fulfil_budget", "singleton", "grade_for", "requires", "action", "value", "secret_keys",
|
|
43
|
+
"secret_shape", "made_from", "serial", "stop", "stop_warning"})
|
|
44
|
+
# A sustenaut.walk hook's parameters: prepare(ctx, plan, terminal), returning a line to print or None.
|
|
45
|
+
HOOK = ("ctx", "plan", "terminal")
|
|
46
|
+
|
|
47
|
+
NAME = re.compile(r"[a-z][a-z0-9-]*(\.[a-z][a-z0-9-]*)+")
|
|
48
|
+
LONGEST = 10.0 # seconds a check may take, at most
|
|
49
|
+
LONGEST_FULFIL = 900.0 # seconds a fulfil may take, at most
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
def problems(kind: Any) -> list[str]:
|
|
53
|
+
"""What keeps `kind` from being a need type, each as `./pubdeploy` says it."""
|
|
54
|
+
out = []
|
|
55
|
+
name = getattr(kind, "type", None)
|
|
56
|
+
if not isinstance(name, str) or not NAME.fullmatch(name):
|
|
57
|
+
out.append("its type must be dotted lower-case words, such as example.thing")
|
|
58
|
+
schema = getattr(kind, "schema", None)
|
|
59
|
+
if not isinstance(schema, dict) or schema.get("type") != "object" or schema.get("additionalProperties") is not False:
|
|
60
|
+
out.append("its schema must be an object's, with additionalProperties false")
|
|
61
|
+
budget = getattr(kind, "budget", None)
|
|
62
|
+
if isinstance(budget, bool) or not isinstance(budget, int | float) or not 0 < budget <= LONGEST:
|
|
63
|
+
out.append(f"its budget must be more than 0 and at most {LONGEST:g} seconds")
|
|
64
|
+
fulfil = getattr(kind, "fulfil_budget", 1.0)
|
|
65
|
+
if isinstance(fulfil, bool) or not isinstance(fulfil, int | float) or not 0 < fulfil <= LONGEST_FULFIL:
|
|
66
|
+
out.append(f"its fulfil_budget must be more than 0 and at most {LONGEST_FULFIL:g} seconds")
|
|
67
|
+
if getattr(kind, "grade", None) not in GRADES:
|
|
68
|
+
out.append(f"its grade must be one of {', '.join(GRADES)}")
|
|
69
|
+
if not isinstance(getattr(kind, "offline", None), bool):
|
|
70
|
+
out.append("its offline must be true or false")
|
|
71
|
+
for method in ("default_name", "depends_on", "check", "fulfil"):
|
|
72
|
+
if not callable(getattr(kind, method, None)):
|
|
73
|
+
out.append(f"it has no {method}")
|
|
74
|
+
return out
|
|
75
|
+
|
|
76
|
+
|
|
77
|
+
def hook_problems(hook: Any) -> list[str]:
|
|
78
|
+
"""What keeps `hook` from being a sustenaut.walk hook."""
|
|
79
|
+
if not callable(hook):
|
|
80
|
+
return ["it is not a function"]
|
|
81
|
+
try:
|
|
82
|
+
inspect.signature(hook).bind(*HOOK)
|
|
83
|
+
except TypeError:
|
|
84
|
+
return [f"it must take ({', '.join(HOOK)})"]
|
|
85
|
+
except ValueError: # a builtin without a signature: called as it is
|
|
86
|
+
return []
|
|
87
|
+
return []
|
pubdeploy/github.py
ADDED
|
@@ -0,0 +1,61 @@
|
|
|
1
|
+
"""GitHub through `gh`: one API call, and setting a secret or variable with
|
|
2
|
+
its value on standard input. The walk's ctx.gh and ctx.send, with which the
|
|
3
|
+
github.* types (pubsec's among them) read and set what GitHub holds."""
|
|
4
|
+
|
|
5
|
+
from __future__ import annotations
|
|
6
|
+
|
|
7
|
+
import json
|
|
8
|
+
import re
|
|
9
|
+
import subprocess
|
|
10
|
+
import sys
|
|
11
|
+
from typing import Any
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
# A copy of pubrules's protect.gh.
|
|
15
|
+
def gh(method: str, path: str, body: Any = None) -> Any:
|
|
16
|
+
"""One GitHub API call through `gh api`; its JSON, or None when empty."""
|
|
17
|
+
argv = ["gh", "api", "-X", method, "-H", "Accept: application/vnd.github+json", path]
|
|
18
|
+
if body is not None:
|
|
19
|
+
argv += ["--input", "-"]
|
|
20
|
+
proc = subprocess.run(argv, input=None if body is None else json.dumps(body),
|
|
21
|
+
capture_output=True, text=True, check=False)
|
|
22
|
+
if proc.returncode != 0:
|
|
23
|
+
raise ApiError(f"gh api -X {method} {path}: {proc.stderr.strip()}", proc.stdout)
|
|
24
|
+
return json.loads(proc.stdout) if proc.stdout.strip() else None
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
# A copy of pubrules's protect.ApiError; pubsec's types read its .status.
|
|
28
|
+
class ApiError(Exception):
|
|
29
|
+
"""A refused call: gh's message, then the reasons GitHub's response gives."""
|
|
30
|
+
|
|
31
|
+
def __init__(self, message: str, response: str = ""):
|
|
32
|
+
m = re.search(r"\(HTTP (\d+)\)", message)
|
|
33
|
+
self.status = int(m.group(1)) if m else None
|
|
34
|
+
try:
|
|
35
|
+
errors = json.loads(response).get("errors") or []
|
|
36
|
+
except (ValueError, AttributeError):
|
|
37
|
+
errors = []
|
|
38
|
+
reasons = [e if isinstance(e, str) else e.get("message") or json.dumps(e) for e in errors]
|
|
39
|
+
super().__init__("; ".join([message, *reasons]))
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
# A copy of pubrules's protect.gh_set.
|
|
43
|
+
def gh_set(kind: str, name: str, repo: str, value: str, environment: str | None = None) -> str | None:
|
|
44
|
+
"""Set one secret or variable in `repo` through gh, the value on its standard
|
|
45
|
+
input; what gh said if it refused, else None."""
|
|
46
|
+
done = subprocess.run(["gh", kind, "set", name, "-R", repo,
|
|
47
|
+
*(["--env", environment] if environment else [])],
|
|
48
|
+
input=value, text=True, capture_output=True, check=False)
|
|
49
|
+
return done.stderr.strip() if done.returncode != 0 else None
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
# A copy of pubrules's protect.origin.
|
|
53
|
+
def origin() -> str:
|
|
54
|
+
"""OWNER/NAME of the GitHub repository `origin` points at."""
|
|
55
|
+
proc = subprocess.run(["git", "remote", "get-url", "origin"],
|
|
56
|
+
capture_output=True, text=True, check=False)
|
|
57
|
+
m = re.search(r"github\.com[:/]([A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+?)(?:\.git)?/?$",
|
|
58
|
+
proc.stdout.strip())
|
|
59
|
+
if not m:
|
|
60
|
+
sys.exit("pubdeploy: origin is not a GitHub repository")
|
|
61
|
+
return m.group(1)
|