pubdeploy 0.8.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (45) hide show
  1. pubdeploy/__init__.py +532 -0
  2. pubdeploy/channels.py +188 -0
  3. pubdeploy/cli.py +120 -0
  4. pubdeploy/config.py +140 -0
  5. pubdeploy/contract.py +87 -0
  6. pubdeploy/github.py +61 -0
  7. pubdeploy/hooks.py +82 -0
  8. pubdeploy/implied.py +179 -0
  9. pubdeploy/journal.json +207 -0
  10. pubdeploy/k8s.py +472 -0
  11. pubdeploy/machine.py +363 -0
  12. pubdeploy/model.py +229 -0
  13. pubdeploy/names.py +25 -0
  14. pubdeploy/order.py +101 -0
  15. pubdeploy/providers.py +75 -0
  16. pubdeploy/registry.py +96 -0
  17. pubdeploy/release-commit +1 -0
  18. pubdeploy/schema.json +127 -0
  19. pubdeploy/schema.py +238 -0
  20. pubdeploy/store.py +127 -0
  21. pubdeploy/testing/__init__.py +94 -0
  22. pubdeploy/testing/fixtures/deploy-app/.github/workflows/deploy.yml +11 -0
  23. pubdeploy/testing/fixtures/deploy-app/k8s/admin/network-policy.yaml +11 -0
  24. pubdeploy/testing/fixtures/deploy-app/k8s/app.yaml +63 -0
  25. pubdeploy/testing/fixtures/deploy-app/k8s/kustomization.yaml +6 -0
  26. pubdeploy/testing/fixtures/deploy-app/pubrules.json +27 -0
  27. pubdeploy/testing/fixtures/deploy-app/rendered.yaml +86 -0
  28. pubdeploy/testing/fixtures/deploy-app/zones.json +8 -0
  29. pubdeploy/testing/plugin.py +26 -0
  30. pubdeploy/testing/steps.py +1124 -0
  31. pubdeploy/testing/world.py +569 -0
  32. pubdeploy/types/__init__.py +181 -0
  33. pubdeploy/types/auth.py +199 -0
  34. pubdeploy/types/github.py +110 -0
  35. pubdeploy/types/k8s.py +609 -0
  36. pubdeploy/types/net.py +237 -0
  37. pubdeploy/types/rollout.py +186 -0
  38. pubdeploy/units.py +177 -0
  39. pubdeploy/walk.py +325 -0
  40. pubdeploy-0.8.0.dist-info/METADATA +520 -0
  41. pubdeploy-0.8.0.dist-info/RECORD +45 -0
  42. pubdeploy-0.8.0.dist-info/WHEEL +5 -0
  43. pubdeploy-0.8.0.dist-info/entry_points.txt +11 -0
  44. pubdeploy-0.8.0.dist-info/licenses/LICENSE +202 -0
  45. pubdeploy-0.8.0.dist-info/top_level.txt +1 -0
pubdeploy/store.py ADDED
@@ -0,0 +1,127 @@
1
+ """Writing a walk's journal to storage: `walk --journal URL` (or PUBRULES_JOURNAL_URL).
2
+
3
+ The deploy-journal/2 document is uploaded as a file of the tenant `plane`,
4
+ then the head OWNER.REPO.UNIT moved to it, If-Match the head's
5
+ ETag, so two walks never lose one's journal: a move another walk won is
6
+ tried once more, on the head it left. The token is PUBRULES_STORAGE_TOKEN; in
7
+ GitHub Actions, without it, the job's OIDC token for the storage's audience
8
+ (PUBRULES_STORAGE_AUDIENCE, else the URL's host), as board-publish signs in.
9
+ A laptop has neither, and writes no journal. A journal that could not be
10
+ written is said, in one line, and never fails the walk.
11
+ """
12
+
13
+ from __future__ import annotations
14
+
15
+ import json
16
+ import os
17
+ import re
18
+ import urllib.error
19
+ import urllib.parse
20
+ import urllib.request
21
+ import uuid
22
+ from collections.abc import Mapping
23
+ from typing import Any
24
+
25
+ TENANT = "plane"
26
+ # Storage's head names are flat and lower case: [a-z0-9][a-z0-9._-], at most 64.
27
+ FLAT = re.compile(r"[^a-z0-9._-]")
28
+ # Cloudflare refuses Python's default User-Agent (error 1010).
29
+ AGENT = "pubdeploy-journal"
30
+
31
+
32
+ # A copy of pubrules's board_publish.Refused.
33
+ class Refused(Exception):
34
+ """Storage's answer to a request it didn't carry out."""
35
+
36
+ def __init__(self, status: int, body: bytes):
37
+ self.status = status
38
+ super().__init__(f"storage said {status}: {body.decode('utf-8', 'replace')[:300]}")
39
+
40
+
41
+ # A copy of pubrules's board_publish.oidc_token.
42
+ def oidc_token(audience: str) -> str:
43
+ url = os.environ["ACTIONS_ID_TOKEN_REQUEST_URL"]
44
+ separator = "&" if "?" in url else "?"
45
+ request = urllib.request.Request(f"{url}{separator}audience={urllib.parse.quote(audience)}", headers={
46
+ "Authorization": f"Bearer {os.environ['ACTIONS_ID_TOKEN_REQUEST_TOKEN']}"})
47
+ with urllib.request.urlopen(request, timeout=30) as response: # noqa: S310 - GitHub's own address
48
+ return json.load(response)["value"]
49
+
50
+
51
+ class Heads:
52
+ """A storage tenant's files and heads, a head by its path: after pubrules's
53
+ board_publish.Storage."""
54
+
55
+ def __init__(self, base: str, tenant: str, token: str):
56
+ self.base, self.tenant, self.token = base.rstrip("/"), tenant, token
57
+
58
+ def _send(self, method: str, path: str, data: bytes | None = None,
59
+ headers: dict | None = None) -> tuple[int, bytes, dict]:
60
+ request = urllib.request.Request(
61
+ f"{self.base}/{self.tenant}{path}", data=data, method=method,
62
+ headers={"Authorization": f"Bearer {self.token}", "User-Agent": AGENT, **(headers or {})})
63
+ try:
64
+ with urllib.request.urlopen(request, timeout=60) as response: # noqa: S310 - the caller's --journal
65
+ return response.status, response.read(), dict(response.headers)
66
+ except urllib.error.HTTPError as e:
67
+ return e.code, e.read(), dict(e.headers)
68
+
69
+ def upload(self, document: bytes) -> str:
70
+ boundary = uuid.uuid4().hex
71
+ form = (f'--{boundary}\r\nContent-Disposition: form-data; name="file"; filename="journal.json"\r\n'
72
+ f"Content-Type: application/json\r\n\r\n").encode() + document + f"\r\n--{boundary}--\r\n".encode()
73
+ status, body, _ = self._send("POST", "/files", form,
74
+ {"Content-Type": f"multipart/form-data; boundary={boundary}"})
75
+ if status != 200:
76
+ raise Refused(status, body)
77
+ return json.loads(body)["id"]
78
+
79
+ def head(self, path: str) -> str | None:
80
+ status, body, headers = self._send("GET", f"/head/{path}")
81
+ if status == 404:
82
+ return None
83
+ if status != 200:
84
+ raise Refused(status, body)
85
+ return headers.get("ETag", "").strip('"') or None
86
+
87
+ def point(self, path: str, hash_: str, current: str | None) -> int:
88
+ condition = {"If-Match": f'"{current}"'} if current else {"If-None-Match": "*"}
89
+ status, body, _ = self._send("PUT", f"/head/{path}", json.dumps({"hash": hash_}).encode(),
90
+ {"Content-Type": "application/json", **condition})
91
+ if status not in (200, 412):
92
+ raise Refused(status, body)
93
+ return status
94
+
95
+
96
+ def token(url: str, env: Mapping[str, str]) -> str | None:
97
+ if env.get("PUBRULES_STORAGE_TOKEN"):
98
+ return env["PUBRULES_STORAGE_TOKEN"]
99
+ if env.get("ACTIONS_ID_TOKEN_REQUEST_URL") and env.get("ACTIONS_ID_TOKEN_REQUEST_TOKEN"):
100
+ return oidc_token(env.get("PUBRULES_STORAGE_AUDIENCE") or urllib.parse.urlsplit(url).hostname or "")
101
+ return None
102
+
103
+
104
+ def head_name(repository: str, unit: str) -> str:
105
+ """The unit's head: `example-org.example.api` for example-org/example's unit api."""
106
+ return FLAT.sub("-", f"{repository.replace('/', '.')}.{unit}".lower())[:64]
107
+
108
+
109
+ def write(url: str, document: dict[str, Any], env: Mapping[str, str], storage: Any = None) -> str:
110
+ """Write the journal; one line saying what was done, or why not."""
111
+ service = document["service"]
112
+ path = head_name(service["repository"] or "unknown/unknown", service["unit"])
113
+ try:
114
+ if storage is None:
115
+ given = token(url, env)
116
+ if given is None:
117
+ return "deploy: no journal written: no storage token here (PUBRULES_STORAGE_TOKEN, or a GitHub Actions job's)"
118
+ storage = Heads(url, TENANT, given)
119
+ new = storage.upload(json.dumps(document, indent=2).encode())
120
+ for _ in range(2):
121
+ if storage.point(path, new, storage.head(path)) == 200:
122
+ return f"deploy: journal {new} written, the head {path} moved to it"
123
+ return f"deploy: journal {new} written, but the head {path} moved under it twice: it was left"
124
+ except Refused as e:
125
+ return f"deploy: no journal written: storage said {e}"
126
+ except (OSError, ValueError, KeyError) as e:
127
+ return f"deploy: no journal written: {type(e).__name__}"
@@ -0,0 +1,94 @@
1
+ """pubdeploy's deploy-walk test harness, for any suite that walks a service
2
+ with pubdeploy: a pytest plugin, its fakes, and the sample service they walk.
3
+ Nothing here runs in a deploy; it ships in the wheel so a suite loads it
4
+ rather than keep a copy.
5
+
6
+ Loading it. The `pytest11` entry point `pubdeploy.testing` (`plugin.py`)
7
+ is always there, and loads nothing until a suite asks, so a repository with
8
+ pubdeploy among its test tools gets none of these steps unasked. A suite
9
+ asks either way:
10
+
11
+ - `pubdeploy_testing = true` in its pytest.ini (or `[tool.pytest.ini_options]`);
12
+ - `pytest_plugins = ["pubdeploy.testing.steps"]` in its top conftest.py.
13
+
14
+ Its steps are pubtest's voice (pubtest's GRAMMAR.md), and need pubtest's
15
+ plugin, which every suite on pubtest loads. A step that runs pubrules
16
+ (`the example service's managed files are in step`) needs pubrules
17
+ installed. The world patches only pubdeploy: a suite whose installed packages
18
+ the walk also loads (pubsec's types) patches them in its own World's `patch`,
19
+ and a suite that walks with a stand-in provider instead returns its entry
20
+ points from `providers()`.
21
+
22
+ What it gives (`steps.py`):
23
+
24
+ - the fixture `world`: a `World` (`world.py`) around a copy of the sample
25
+ service, with every fake installed, whose teardown fails the scenario
26
+ when any output, journal, exception or argv holds a value (the canary);
27
+ - the fixture `deploy_world_class`: the class `world` makes, `World`; a
28
+ suite overrides it to extend the world;
29
+ - the fixture `offered`: entry points of the `sustenaut.needs` group made up
30
+ by a scenario;
31
+ - the deploy steps: the block, packages offering need types, `When I run
32
+ pubdeploy "ARG" ...`, the plan, the walk's lines, the journal, the
33
+ cluster, the manifests, GitHub, HTTP, Bitwarden, budgets, fulfilling,
34
+ units, storage;
35
+ - `Given a file "P" copied from the sample "SRC"`: pubtest's `copied from`
36
+ for the sample service, which is outside the suite's repository: SRC is
37
+ `deploy-app`, or a path in it (`deploy-app/zones.json`).
38
+
39
+ `world` is not autouse. A test module whose every scenario should run in a
40
+ world, and under its canary, says so with
41
+ `pytestmark = pytest.mark.usefixtures("world")`.
42
+
43
+ Extending the world. Subclass `World`, and name the subclass in the suite's
44
+ conftest.py:
45
+
46
+ from pubdeploy.testing.world import World
47
+
48
+ class PubsecWorld(World):
49
+ secrets_project = "PUBRULES_SECRETS_PROJECT"
50
+
51
+ def patch(self, monkeypatch):
52
+ super().patch(monkeypatch)
53
+ monkeypatch.setattr("pubsec.needs.gcp.gcloud", self.run_gcloud) # its own fake
54
+
55
+ @pytest.fixture
56
+ def deploy_world_class():
57
+ return PubsecWorld
58
+
59
+ The hooks are `secrets_project` (the variable ./with-secrets names its
60
+ project in), `sample()` (the service copied), `__init__` (more state),
61
+ `providers()` (a `Providers` of the `sustenaut.needs` entry points and
62
+ `sustenaut.walk` hooks the walk loads in place of the installed ones, or None,
63
+ the default, for the installed), `patch(monkeypatch)` (what the walk reaches; a
64
+ subclass patches its own packages after calling it), `context(...)` (the walk's
65
+ Context) and `leaked()` (what the canary reads). A step of the suite's own with
66
+ a plugin step's text takes precedence over it.
67
+
68
+ `walk_needs(world, needs=None, types=None, fulfil=())` walks needs in the
69
+ world's context, as a provider's own scenarios do with its types, and gives
70
+ the deploy-journal/2 document as data. With these and `pubdeploy.contract`, a
71
+ provider's suite imports nothing else of pubdeploy.
72
+
73
+ `fixture_path(name)` is where a sample is, for a suite's own steps.
74
+ """
75
+
76
+ from __future__ import annotations
77
+
78
+ from pathlib import Path
79
+
80
+ __all__ = ["FIXTURES", "fixture_path"]
81
+
82
+ # The samples, as package data: deploy-app, the service the world walks.
83
+ FIXTURES = Path(__file__).resolve().parent / "fixtures"
84
+
85
+
86
+ def fixture_path(name: str = "deploy-app") -> Path:
87
+ """The sample `name`, or a path in one (`deploy-app/zones.json`). A name
88
+ that leads out of the samples, or is not there, is refused."""
89
+ path = (FIXTURES / name).resolve()
90
+ if Path(name).is_absolute() or not path.is_relative_to(FIXTURES):
91
+ raise ValueError(f"{name} is not one of pubdeploy's samples")
92
+ if not path.exists():
93
+ raise FileNotFoundError(f"pubdeploy has no sample {name}")
94
+ return path
@@ -0,0 +1,11 @@
1
+ # The example service's deploy, as a consumer of k8s-deploy.yml writes it.
2
+ name: deploy
3
+ on:
4
+ workflow_run: { workflows: [main], types: [completed], branches: [main] }
5
+ permissions:
6
+ contents: read
7
+ jobs:
8
+ deploy:
9
+ runs-on: ubuntu-latest
10
+ steps:
11
+ - run: echo "deploys k8s when DEPLOY_EXAMPLE is on"
@@ -0,0 +1,11 @@
1
+ # Applied by an admin, not by the deploy: what may reach the app.
2
+ apiVersion: networking.k8s.io/v1
3
+ kind: NetworkPolicy
4
+ metadata:
5
+ name: example-app
6
+ namespace: example
7
+ spec:
8
+ podSelector: { matchLabels: { app: example } }
9
+ ingress:
10
+ - from:
11
+ - podSelector: { matchLabels: { app: proxy } }
@@ -0,0 +1,63 @@
1
+ # Four Secrets: example-app-env whole (envFrom), example-files' token (a
2
+ # volume and an env), example-page-secrets' cookie secret, and example-extra,
3
+ # which is optional.
4
+ apiVersion: apps/v1
5
+ kind: Deployment
6
+ metadata:
7
+ name: app
8
+ spec:
9
+ selector: { matchLabels: { app: example } }
10
+ template:
11
+ metadata: { labels: { app: example } }
12
+ spec:
13
+ containers:
14
+ - name: app
15
+ image: ghcr.io/example/example-app
16
+ envFrom:
17
+ - secretRef: { name: example-app-env }
18
+ env:
19
+ - name: EXTRA
20
+ valueFrom: { secretKeyRef: { name: example-extra, key: x, optional: true } }
21
+ volumeMounts:
22
+ - { name: files, mountPath: /files }
23
+ volumes:
24
+ - name: files
25
+ secret:
26
+ secretName: example-files
27
+ items:
28
+ - { key: token, path: token }
29
+ ---
30
+ apiVersion: apps/v1
31
+ kind: Deployment
32
+ metadata:
33
+ name: proxy
34
+ spec:
35
+ selector: { matchLabels: { app: proxy } }
36
+ template:
37
+ metadata: { labels: { app: proxy } }
38
+ spec:
39
+ containers:
40
+ - name: oauth2-proxy
41
+ image: quay.io/oauth2-proxy/oauth2-proxy:v7.12.0
42
+ args: ["--provider=oidc"]
43
+ env:
44
+ - name: OAUTH2_PROXY_COOKIE_SECRET
45
+ valueFrom: { secretKeyRef: { name: example-page-secrets, key: cookie-secret } }
46
+ ---
47
+ apiVersion: batch/v1
48
+ kind: CronJob
49
+ metadata:
50
+ name: worker
51
+ spec:
52
+ schedule: "0 * * * *"
53
+ jobTemplate:
54
+ spec:
55
+ template:
56
+ spec:
57
+ restartPolicy: Never
58
+ containers:
59
+ - name: worker
60
+ image: ghcr.io/example/example-app
61
+ env:
62
+ - name: TOKEN
63
+ valueFrom: { secretKeyRef: { name: example-files, key: token } }
@@ -0,0 +1,6 @@
1
+ # The example service: an app, the proxy in front of it, and a worker.
2
+ apiVersion: kustomize.config.k8s.io/v1beta1
3
+ kind: Kustomization
4
+ namespace: example
5
+ resources:
6
+ - app.yaml
@@ -0,0 +1,27 @@
1
+ {
2
+ "kind": "service",
3
+ "ci": {},
4
+ "secrets": [
5
+ {"from": "example-token", "to": "EXAMPLE_TOKEN", "kind": "secret"},
6
+ {"from": "example-user", "to": "EXAMPLE_USER", "kind": "variable"},
7
+ {"from": "example-deploy-key", "to": "SUSTENAUT_DEPLOY_SSH_KEY", "kind": "secret", "environment": "server"}
8
+ ],
9
+ "variables": {"names": ["EXAMPLE_SETTING"], "deploy": "deploy.yml"},
10
+ "deploy": {
11
+ "namespace": "example",
12
+ "host": "example.example.eu",
13
+ "workflow": "deploy.yml",
14
+ "gate": "DEPLOY_EXAMPLE",
15
+ "overlay": "k8s",
16
+ "needs": [
17
+ {"type": "bitwarden.generated", "name": "example-cookie-secret", "shape": "oauth2-proxy-cookie",
18
+ "secret": {"name": "example-page-secrets", "key": "cookie-secret"}},
19
+ {"type": "bitwarden.secret", "name": "example-files-token",
20
+ "secret": {"name": "example-files", "key": "token"}},
21
+ {"type": "k8s.secret", "name": "example-app-env"},
22
+ {"type": "k8s.manifests", "path": "k8s/admin/network-policy.yaml"},
23
+ {"type": "upstream", "name": "storage", "health": "https://storage.example.eu/healthz"},
24
+ {"type": "service.health", "path": "/healthz", "job": "cronjob/worker", "after": ["upstream:storage"]}
25
+ ]
26
+ }
27
+ }
@@ -0,0 +1,86 @@
1
+ # What `kustomize build k8s` prints for this overlay; the tests read this, and run no kustomize.
2
+ apiVersion: apps/v1
3
+ kind: Deployment
4
+ metadata:
5
+ name: app
6
+ namespace: example
7
+ spec:
8
+ selector:
9
+ matchLabels:
10
+ app: example
11
+ template:
12
+ metadata:
13
+ labels:
14
+ app: example
15
+ spec:
16
+ containers:
17
+ - env:
18
+ - name: EXTRA
19
+ valueFrom:
20
+ secretKeyRef:
21
+ key: x
22
+ name: example-extra
23
+ optional: true
24
+ envFrom:
25
+ - secretRef:
26
+ name: example-app-env
27
+ image: ghcr.io/example/example-app
28
+ name: app
29
+ volumeMounts:
30
+ - mountPath: /files
31
+ name: files
32
+ volumes:
33
+ - name: files
34
+ secret:
35
+ items:
36
+ - key: token
37
+ path: token
38
+ secretName: example-files
39
+ ---
40
+ apiVersion: apps/v1
41
+ kind: Deployment
42
+ metadata:
43
+ name: proxy
44
+ namespace: example
45
+ spec:
46
+ selector:
47
+ matchLabels:
48
+ app: proxy
49
+ template:
50
+ metadata:
51
+ labels:
52
+ app: proxy
53
+ spec:
54
+ containers:
55
+ - args:
56
+ - --provider=oidc
57
+ env:
58
+ - name: OAUTH2_PROXY_COOKIE_SECRET
59
+ valueFrom:
60
+ secretKeyRef:
61
+ key: cookie-secret
62
+ name: example-page-secrets
63
+ image: quay.io/oauth2-proxy/oauth2-proxy:v7.12.0
64
+ name: oauth2-proxy
65
+ ---
66
+ apiVersion: batch/v1
67
+ kind: CronJob
68
+ metadata:
69
+ name: worker
70
+ namespace: example
71
+ spec:
72
+ jobTemplate:
73
+ spec:
74
+ template:
75
+ spec:
76
+ containers:
77
+ - env:
78
+ - name: TOKEN
79
+ valueFrom:
80
+ secretKeyRef:
81
+ key: token
82
+ name: example-files
83
+ image: ghcr.io/example/example-app
84
+ name: worker
85
+ restartPolicy: Never
86
+ schedule: 0 * * * *
@@ -0,0 +1,8 @@
1
+ {
2
+ "kind": "utility",
3
+ "hosting": {
4
+ "zones": {
5
+ "example.eu": {"dns": ["dns"], "sites": {"example": {"provider": "server", "kind": "server", "server": "web"}}}
6
+ }
7
+ }
8
+ }
@@ -0,0 +1,26 @@
1
+ """The `pytest11` entry point `pubdeploy.testing`: it loads the deploy-walk
2
+ steps and fixtures (`pubdeploy.testing.steps`) into a suite whose ini sets
3
+ `pubdeploy_testing = true`, and nothing into any other. pubdeploy is
4
+ installed in the test environment of every repository with a deploy block,
5
+ whose suites must not get these steps unasked."""
6
+
7
+ from __future__ import annotations
8
+
9
+ import pytest
10
+
11
+ STEPS = "pubdeploy.testing.steps"
12
+
13
+ # Installed as a plugin's modules are, an editable install's included, so a
14
+ # failed assert in a step shows its values.
15
+ pytest.register_assert_rewrite(STEPS, "pubdeploy.testing.world")
16
+
17
+
18
+ def pytest_addoption(parser: pytest.Parser) -> None:
19
+ parser.addini("pubdeploy_testing", "load pubdeploy's deploy-walk steps and fixtures (pubdeploy.testing.steps)",
20
+ type="bool", default=False)
21
+
22
+
23
+ def pytest_configure(config: pytest.Config) -> None:
24
+ if config.getini("pubdeploy_testing"):
25
+ # Loaded once, however the suite asked: pytest_plugins in a conftest registers the same name.
26
+ config.pluginmanager.import_plugin(STEPS)