shipmill 0.31.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- shipmill/__init__.py +4 -0
- shipmill/__main__.py +6 -0
- shipmill/agents.py +66 -0
- shipmill/app.py +571 -0
- shipmill/app_create.py +451 -0
- shipmill/app_install.py +180 -0
- shipmill/autonomy.py +176 -0
- shipmill/changelog.py +259 -0
- shipmill/cli.py +784 -0
- shipmill/config.py +106 -0
- shipmill/doctor.py +390 -0
- shipmill/environments.py +106 -0
- shipmill/errors.py +3 -0
- shipmill/gate.py +998 -0
- shipmill/github.py +367 -0
- shipmill/gitrepo.py +125 -0
- shipmill/init.py +299 -0
- shipmill/land.py +254 -0
- shipmill/lanes.py +15 -0
- shipmill/launchd.py +140 -0
- shipmill/notify.py +74 -0
- shipmill/operate.py +857 -0
- shipmill/planner.py +340 -0
- shipmill/policy.py +261 -0
- shipmill/propose.py +158 -0
- shipmill/py.typed +0 -0
- shipmill/roadmap.py +30 -0
- shipmill/schedule.py +85 -0
- shipmill/skills/github-issue-resolve/SKILL.md +131 -0
- shipmill/skills/github-issue-triage/SKILL.md +135 -0
- shipmill/skills/github-issue-triage/agents/openai.yaml +4 -0
- shipmill/skills/github-issue-triage/references/comments.md +112 -0
- shipmill/skills/github-issue-triage/references/design-gate.md +59 -0
- shipmill/skills/github-issue-triage/references/implementer-brief.md +34 -0
- shipmill/skills/github-issue-triage/references/needs-decision.md +64 -0
- shipmill/skills/github-issue-triage/references/spec-gate.md +100 -0
- shipmill/skills/github-issue-triage/references/triage-rubric.md +65 -0
- shipmill/skills/github-issue-triage/scripts/decisions.py +230 -0
- shipmill/skills/github-issue-triage/scripts/specs.py +810 -0
- shipmill/skills/github-issue-triage/scripts/triage_state.py +834 -0
- shipmill/skills/github-pr-triage/SKILL.md +120 -0
- shipmill/skills/github-pr-triage/agents/openai.yaml +4 -0
- shipmill/skills/github-pr-triage/references/ci-failures.md +52 -0
- shipmill/skills/github-pr-triage/references/landing.md +147 -0
- shipmill/skills/github-pr-triage/references/reviewer-brief.md +50 -0
- shipmill/skills/github-pr-triage/scripts/changelog_guard.py +347 -0
- shipmill/skills/github-pr-triage/scripts/failed_tests.py +97 -0
- shipmill/skills/github-pr-triage/scripts/landed.py +76 -0
- shipmill/skills/github-pr-triage/scripts/pr_gate.py +104 -0
- shipmill/skills/github-pr-triage/scripts/release_ready.py +169 -0
- shipmill/skills/github-pr-triage/scripts/shipped.py +107 -0
- shipmill/skills/github-ship-watch/SKILL.md +171 -0
- shipmill/skills/github-ship-watch/agents/openai.yaml +4 -0
- shipmill/skills/github-ship-watch/scripts/fleet.py +417 -0
- shipmill/skills/github-ship-watch/scripts/metrics.py +833 -0
- shipmill/skills/github-ship-watch/scripts/retro.py +231 -0
- shipmill/skills/github-ship-watch/scripts/watch_state.py +1478 -0
- shipmill/skills/product-intake/SKILL.md +170 -0
- shipmill/skills/product-intake/agents/openai.yaml +4 -0
- shipmill/skills/product-intake/references/milestone-proposal.md +40 -0
- shipmill/skills/product-intake/references/opportunity.md +66 -0
- shipmill/skills/product-intake/scripts/intake_state.py +581 -0
- shipmill/skills/product-intake/scripts/roadmap_state.py +345 -0
- shipmill/skills/shipmill-setup/SKILL.md +492 -0
- shipmill/skills/shipmill-setup/scripts/setup_state.py +343 -0
- shipmill/stamp.py +167 -0
- shipmill/status.py +664 -0
- shipmill/version.py +125 -0
- shipmill/worktrees.py +420 -0
- shipmill-0.31.0.dist-info/METADATA +473 -0
- shipmill-0.31.0.dist-info/RECORD +74 -0
- shipmill-0.31.0.dist-info/WHEEL +4 -0
- shipmill-0.31.0.dist-info/entry_points.txt +2 -0
- shipmill-0.31.0.dist-info/licenses/LICENSE +21 -0
shipmill/__init__.py
ADDED
shipmill/__main__.py
ADDED
shipmill/agents.py
ADDED
|
@@ -0,0 +1,66 @@
|
|
|
1
|
+
"""The [agents] section of the config: what `shipmill gate` starts a Claude Code session
|
|
2
|
+
for, and with which prompt. The prompt carries the scope the user grants ("merge when
|
|
3
|
+
green" or not), so it is a reviewed, committed decision like [autonomy]. Where the session
|
|
4
|
+
runs (which machine, which scheduler) is not here: that belongs to the host"""
|
|
5
|
+
|
|
6
|
+
from dataclasses import dataclass
|
|
7
|
+
from enum import StrEnum
|
|
8
|
+
from pathlib import Path
|
|
9
|
+
|
|
10
|
+
from shipmill.config import CONFIG_PATH, Table, config_path, read
|
|
11
|
+
from shipmill.errors import ReleaseError
|
|
12
|
+
|
|
13
|
+
_MAX_HOURS = 7 * 24 # a week
|
|
14
|
+
_REMIND_HOURS = 4
|
|
15
|
+
_MAX_WAIT_MINUTES = 15
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
class Mode(StrEnum):
|
|
19
|
+
"""How a gate session meets a decision for the user (spec 005)"""
|
|
20
|
+
|
|
21
|
+
INTERACTIVE = "interactive" # `claude --bg`: it asks you and waits
|
|
22
|
+
HEADLESS = "headless" # `claude -p`: it asks on GitHub and ends (D-17)
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
@dataclass(frozen=True, slots=True)
|
|
26
|
+
class AgentsConfig:
|
|
27
|
+
prompt: str # the session's prompt; {repo} becomes owner/name
|
|
28
|
+
prs: bool # open pull requests count as work, for a prompt that lands them
|
|
29
|
+
retry_hours: int # unchanged findings start a new session after this
|
|
30
|
+
notify: bool = True # a desktop notification when a session waits on you
|
|
31
|
+
remind_hours: int = _REMIND_HOURS # repeat it while the session still waits
|
|
32
|
+
max_wait_minutes: int = _MAX_WAIT_MINUTES # stop a session that waited this long; 0: never
|
|
33
|
+
app_id: int | None = None # sessions write as this GitHub App (spec 004, D-14); None: the host's gh login
|
|
34
|
+
mode: Mode = Mode.INTERACTIVE # headless: `claude -p` (spec 005); either mode asks via needs-decision (D-21)
|
|
35
|
+
|
|
36
|
+
@classmethod
|
|
37
|
+
def parse(cls, table: Table) -> AgentsConfig:
|
|
38
|
+
table.allow("prompt", "prs", "retry_hours", "notify", "remind_hours", "max_wait_minutes", "app_id", "mode")
|
|
39
|
+
prompt = table.string("prompt").strip()
|
|
40
|
+
if not prompt:
|
|
41
|
+
raise ReleaseError(f"{table.where}: prompt must not be empty")
|
|
42
|
+
retry = table.integer("retry_hours", default=24, low=1, high=_MAX_HOURS)
|
|
43
|
+
remind = table.integer("remind_hours", default=_REMIND_HOURS, low=1, high=_MAX_HOURS)
|
|
44
|
+
max_wait = table.integer("max_wait_minutes", default=_MAX_WAIT_MINUTES, low=0, high=_MAX_HOURS * 60)
|
|
45
|
+
assert retry is not None and remind is not None and max_wait is not None # defaults were given
|
|
46
|
+
return cls(
|
|
47
|
+
prompt=prompt,
|
|
48
|
+
prs=table.boolean("prs", default=False),
|
|
49
|
+
retry_hours=retry,
|
|
50
|
+
notify=table.boolean("notify", default=True),
|
|
51
|
+
remind_hours=remind,
|
|
52
|
+
max_wait_minutes=max_wait,
|
|
53
|
+
app_id=table.integer("app_id", default=None, low=1, high=None),
|
|
54
|
+
mode=table.enum("mode", Mode, default=Mode.INTERACTIVE),
|
|
55
|
+
)
|
|
56
|
+
|
|
57
|
+
@classmethod
|
|
58
|
+
def load(cls, root: Path) -> AgentsConfig:
|
|
59
|
+
"""The [agents] section alone: a repo may use the gate without the release keys"""
|
|
60
|
+
if not (root / CONFIG_PATH).is_file():
|
|
61
|
+
raise ReleaseError(f"no {CONFIG_PATH} in {root}; add an [agents] section with a prompt to use the gate")
|
|
62
|
+
path = config_path(root)
|
|
63
|
+
raw = read(path)
|
|
64
|
+
if "agents" not in raw:
|
|
65
|
+
raise ReleaseError(f"{path} has no [agents] section; add one with a prompt to use the gate")
|
|
66
|
+
return cls.parse(Table(raw, path.name).table("agents"))
|
shipmill/app.py
ADDED
|
@@ -0,0 +1,571 @@
|
|
|
1
|
+
"""The GitHub App a gated session writes as (spec 004, D-14): its private key on the host,
|
|
2
|
+
the JWT signed with it, and the checks that the App is installed on the gate's repo with
|
|
3
|
+
every permission a session needs.
|
|
4
|
+
|
|
5
|
+
The key is a host secret: shipmill reads its mode, hands its path to `openssl`, and never
|
|
6
|
+
reads its bytes, so they can't reach output. The package keeps no dependencies: the JWT is
|
|
7
|
+
signed by `openssl dgst -sha256 -sign <key>`, and GitHub is asked over urllib.
|
|
8
|
+
|
|
9
|
+
A session never holds a token (spec 004, Tokens): its `gh` and git credential helpers mint
|
|
10
|
+
one, limited to the gate's repo, at each call, or reuse the one cached in `app-token.json`
|
|
11
|
+
(mode 0600) while it has at least 10 minutes left. A token reaches stdout only as
|
|
12
|
+
`shipmill app-token`'s answer, which the helpers read; never an error message, a process's
|
|
13
|
+
arguments, or a helper's file.
|
|
14
|
+
"""
|
|
15
|
+
|
|
16
|
+
import base64
|
|
17
|
+
import datetime as dt
|
|
18
|
+
import enum
|
|
19
|
+
import http.client
|
|
20
|
+
import json
|
|
21
|
+
import os
|
|
22
|
+
import secrets
|
|
23
|
+
import shlex
|
|
24
|
+
import shutil
|
|
25
|
+
import stat
|
|
26
|
+
import subprocess
|
|
27
|
+
import urllib.error
|
|
28
|
+
import urllib.parse
|
|
29
|
+
import urllib.request
|
|
30
|
+
from collections.abc import Callable, Mapping
|
|
31
|
+
from dataclasses import dataclass, field
|
|
32
|
+
from pathlib import Path
|
|
33
|
+
from typing import Any, Protocol
|
|
34
|
+
|
|
35
|
+
from shipmill.errors import ReleaseError
|
|
36
|
+
|
|
37
|
+
API = "https://api.github.com"
|
|
38
|
+
TIMEOUT = 30.0 # seconds a GitHub API call may take
|
|
39
|
+
ISSUED_BEFORE = dt.timedelta(seconds=60) # iat: allows for a host clock ahead of GitHub's
|
|
40
|
+
VALID_FOR = dt.timedelta(minutes=9) # exp: GitHub refuses a JWT valid longer than 10 minutes
|
|
41
|
+
_BODY_LIMIT = 1 << 20
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
class Access(enum.StrEnum):
|
|
45
|
+
READ = "read"
|
|
46
|
+
WRITE = "write"
|
|
47
|
+
|
|
48
|
+
def covers(self, need: Access) -> bool:
|
|
49
|
+
"""write covers read"""
|
|
50
|
+
return self is Access.WRITE or need is Access.READ
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
@dataclass(frozen=True, slots=True)
|
|
54
|
+
class Permission:
|
|
55
|
+
key: str # the name GitHub's API gives it
|
|
56
|
+
label: str # the name GitHub's settings page gives it
|
|
57
|
+
access: Access
|
|
58
|
+
|
|
59
|
+
|
|
60
|
+
# The spec's table: what a session needs on the repo, and nothing more
|
|
61
|
+
PERMISSIONS = (
|
|
62
|
+
Permission("contents", "Contents", Access.WRITE),
|
|
63
|
+
Permission("pull_requests", "Pull requests", Access.WRITE),
|
|
64
|
+
Permission("issues", "Issues", Access.WRITE),
|
|
65
|
+
Permission("actions", "Actions", Access.WRITE),
|
|
66
|
+
Permission("workflows", "Workflows", Access.WRITE),
|
|
67
|
+
Permission("checks", "Checks", Access.READ),
|
|
68
|
+
Permission("statuses", "Commit statuses", Access.READ),
|
|
69
|
+
Permission("discussions", "Discussions", Access.READ),
|
|
70
|
+
Permission("metadata", "Metadata", Access.READ),
|
|
71
|
+
)
|
|
72
|
+
|
|
73
|
+
|
|
74
|
+
def default_key(app_id: int, home: Path) -> Path:
|
|
75
|
+
"""Where the key lives when `--app-key` doesn't say"""
|
|
76
|
+
return home / ".config" / "shipmill" / f"app-{app_id}.pem"
|
|
77
|
+
|
|
78
|
+
|
|
79
|
+
def check_key(path: Path) -> Path:
|
|
80
|
+
"""The key's path once it is a file only its owner can read; its bytes are never read here"""
|
|
81
|
+
if not path.is_file():
|
|
82
|
+
raise ReleaseError(f"no App private key at {path}; download it from the App's settings to that path")
|
|
83
|
+
if path.stat().st_mode & (stat.S_IRGRP | stat.S_IROTH):
|
|
84
|
+
raise ReleaseError(f"the App private key {path} is readable by group or others; run `chmod 600 {path}`")
|
|
85
|
+
return path
|
|
86
|
+
|
|
87
|
+
|
|
88
|
+
class Signer(Protocol):
|
|
89
|
+
def sign(self, key: Path, data: bytes) -> bytes:
|
|
90
|
+
"""data's RSA SHA-256 signature with the key"""
|
|
91
|
+
...
|
|
92
|
+
|
|
93
|
+
|
|
94
|
+
class Openssl:
|
|
95
|
+
def __init__(self, command: str = "openssl") -> None:
|
|
96
|
+
self.command = command
|
|
97
|
+
|
|
98
|
+
def sign(self, key: Path, data: bytes) -> bytes:
|
|
99
|
+
binary = shutil.which(self.command)
|
|
100
|
+
if binary is None:
|
|
101
|
+
raise ReleaseError(f"{self.command} not found; the App's JWT is signed with `openssl dgst -sha256 -sign`")
|
|
102
|
+
proc = subprocess.run(
|
|
103
|
+
[binary, "dgst", "-sha256", "-sign", str(key)], input=data, capture_output=True, check=False
|
|
104
|
+
)
|
|
105
|
+
if proc.returncode != 0 or not proc.stdout:
|
|
106
|
+
detail = proc.stderr.decode(errors="replace").strip()[:300]
|
|
107
|
+
raise ReleaseError(f"openssl could not sign with the App private key {key}: {detail}")
|
|
108
|
+
return proc.stdout
|
|
109
|
+
|
|
110
|
+
|
|
111
|
+
def _b64(data: bytes) -> str:
|
|
112
|
+
return base64.urlsafe_b64encode(data).rstrip(b"=").decode("ascii")
|
|
113
|
+
|
|
114
|
+
|
|
115
|
+
def jwt(app_id: int, key: Path, now: dt.datetime, signer: Signer) -> str:
|
|
116
|
+
"""An RS256 App JWT, issued 60 seconds before now and valid until 9 minutes after it"""
|
|
117
|
+
header = {"alg": "RS256", "typ": "JWT"}
|
|
118
|
+
claims = {
|
|
119
|
+
"iat": int((now - ISSUED_BEFORE).timestamp()),
|
|
120
|
+
"exp": int((now + VALID_FOR).timestamp()),
|
|
121
|
+
"iss": app_id,
|
|
122
|
+
}
|
|
123
|
+
signing = f"{_b64(json.dumps(header).encode())}.{_b64(json.dumps(claims).encode())}"
|
|
124
|
+
return f"{signing}.{_b64(signer.sign(key, signing.encode('ascii')))}"
|
|
125
|
+
|
|
126
|
+
|
|
127
|
+
@dataclass(frozen=True, slots=True)
|
|
128
|
+
class Answer:
|
|
129
|
+
status: int
|
|
130
|
+
body: str
|
|
131
|
+
|
|
132
|
+
|
|
133
|
+
class Api(Protocol):
|
|
134
|
+
def get(self, path: str, token: str | None) -> Answer:
|
|
135
|
+
"""GET api.github.com's path with the token as a bearer, or anonymously with None; an
|
|
136
|
+
error status is an Answer"""
|
|
137
|
+
...
|
|
138
|
+
|
|
139
|
+
def post(self, path: str, token: str, body: Mapping[str, object]) -> Answer:
|
|
140
|
+
"""POST body as JSON to api.github.com's path with the token as a bearer; an error
|
|
141
|
+
status is an Answer"""
|
|
142
|
+
...
|
|
143
|
+
|
|
144
|
+
|
|
145
|
+
class UrllibApi:
|
|
146
|
+
def get(self, path: str, token: str | None) -> Answer:
|
|
147
|
+
return self._send("GET", path, token, None)
|
|
148
|
+
|
|
149
|
+
def post(self, path: str, token: str, body: Mapping[str, object]) -> Answer:
|
|
150
|
+
return self._send("POST", path, token, json.dumps(body).encode())
|
|
151
|
+
|
|
152
|
+
def _send(self, method: str, path: str, token: str | None, data: bytes | None) -> Answer:
|
|
153
|
+
headers = {
|
|
154
|
+
"Accept": "application/vnd.github+json",
|
|
155
|
+
"User-Agent": "shipmill-gate",
|
|
156
|
+
"X-GitHub-Api-Version": "2022-11-28",
|
|
157
|
+
}
|
|
158
|
+
if token is not None:
|
|
159
|
+
headers["Authorization"] = f"Bearer {token}"
|
|
160
|
+
if data is not None:
|
|
161
|
+
headers["Content-Type"] = "application/json"
|
|
162
|
+
request = urllib.request.Request(API + path, data=data, headers=headers, method=method)
|
|
163
|
+
try:
|
|
164
|
+
with urllib.request.urlopen(request, timeout=TIMEOUT) as answer:
|
|
165
|
+
return Answer(int(answer.status), answer.read(_BODY_LIMIT).decode(errors="replace"))
|
|
166
|
+
except urllib.error.HTTPError as exc: # 4xx and 5xx: an answer to report
|
|
167
|
+
body = exc.read(_BODY_LIMIT) if exc.fp is not None else b""
|
|
168
|
+
return Answer(exc.code, body.decode(errors="replace"))
|
|
169
|
+
except (OSError, http.client.HTTPException) as exc: # URLError and timeouts are OSErrors
|
|
170
|
+
reason = getattr(exc, "reason", exc)
|
|
171
|
+
raise ReleaseError(f"GitHub's API is unreachable for {method} {path}: {reason}") from None
|
|
172
|
+
|
|
173
|
+
|
|
174
|
+
def _json(answer: Answer, path: str, method: str = "GET") -> dict[str, Any]:
|
|
175
|
+
"""The answer's JSON object; the message never quotes the body, which may hold a token"""
|
|
176
|
+
try:
|
|
177
|
+
data = json.loads(answer.body)
|
|
178
|
+
except json.JSONDecodeError:
|
|
179
|
+
raise ReleaseError(f"{method} {path} answered {answer.status} without JSON") from None
|
|
180
|
+
if not isinstance(data, dict):
|
|
181
|
+
raise ReleaseError(f"{method} {path} answered {answer.status} with JSON that is not an object")
|
|
182
|
+
return data
|
|
183
|
+
|
|
184
|
+
|
|
185
|
+
def _ok(answer: Answer, path: str, app_id: int, method: str = "GET", status: int = 200) -> dict[str, Any]:
|
|
186
|
+
"""The JSON of an answer with the expected status; an error answer's body, which is only
|
|
187
|
+
ever GitHub's message, is quoted"""
|
|
188
|
+
if answer.status == status:
|
|
189
|
+
return _json(answer, path, method)
|
|
190
|
+
message = answer.body.strip()[:300]
|
|
191
|
+
if answer.status == 401:
|
|
192
|
+
raise ReleaseError(
|
|
193
|
+
f"GitHub refused the JWT for app_id {app_id} ({method} {path}: 401 {message}); "
|
|
194
|
+
f"check that the private key is App {app_id}'s and the host's clock is right"
|
|
195
|
+
)
|
|
196
|
+
raise ReleaseError(f"{method} {path} for app_id {app_id} answered {answer.status}: {message}")
|
|
197
|
+
|
|
198
|
+
|
|
199
|
+
@dataclass(frozen=True, slots=True)
|
|
200
|
+
class Installation:
|
|
201
|
+
"""The App as GitHub knows it, installed on the gate's repo"""
|
|
202
|
+
|
|
203
|
+
slug: str
|
|
204
|
+
id: int
|
|
205
|
+
permissions: Mapping[str, Access]
|
|
206
|
+
|
|
207
|
+
@property
|
|
208
|
+
def bot(self) -> str:
|
|
209
|
+
return f"{self.slug}[bot]"
|
|
210
|
+
|
|
211
|
+
|
|
212
|
+
def missing(granted: Mapping[str, Access]) -> list[Permission]:
|
|
213
|
+
"""The table's permissions the installation lacks, or grants only read where write is needed"""
|
|
214
|
+
return [p for p in PERMISSIONS if (have := granted.get(p.key)) is None or not have.covers(p.access)]
|
|
215
|
+
|
|
216
|
+
|
|
217
|
+
_ACCESS = {"read": Access.READ, "write": Access.WRITE, "admin": Access.WRITE} # admin covers write
|
|
218
|
+
|
|
219
|
+
|
|
220
|
+
def _permissions(raw: object, path: str) -> dict[str, Access]:
|
|
221
|
+
if not isinstance(raw, dict):
|
|
222
|
+
raise ReleaseError(f"GET {path}: permissions is not an object")
|
|
223
|
+
granted = {}
|
|
224
|
+
for key, value in raw.items():
|
|
225
|
+
if not isinstance(key, str) or not isinstance(value, str):
|
|
226
|
+
raise ReleaseError(f"GET {path}: permission {key!r} has access {value!r}, not a string")
|
|
227
|
+
if value not in _ACCESS:
|
|
228
|
+
raise ReleaseError(f"GET {path}: permission {key} has unknown access {value!r}")
|
|
229
|
+
granted[key] = _ACCESS[value]
|
|
230
|
+
return granted
|
|
231
|
+
|
|
232
|
+
|
|
233
|
+
def installation(app_id: int, repo: str, token: str, api: Api) -> Installation:
|
|
234
|
+
"""Spec 004, At launch step 3: the App's slug, then its installation on repo, which must
|
|
235
|
+
grant every permission in the table"""
|
|
236
|
+
app = _ok(api.get("/app", token), "/app", app_id)
|
|
237
|
+
slug = app.get("slug")
|
|
238
|
+
if not isinstance(slug, str) or not slug:
|
|
239
|
+
raise ReleaseError(f"GET /app for app_id {app_id} gave no slug")
|
|
240
|
+
path = f"/repos/{repo}/installation"
|
|
241
|
+
answer = api.get(path, token)
|
|
242
|
+
if answer.status == 404:
|
|
243
|
+
raise ReleaseError(f"app {slug} is not installed on {repo}")
|
|
244
|
+
found = _ok(answer, path, app_id)
|
|
245
|
+
number = found.get("id")
|
|
246
|
+
if not isinstance(number, int) or isinstance(number, bool):
|
|
247
|
+
raise ReleaseError(f"GET {path} gave no installation id")
|
|
248
|
+
granted = _permissions(found.get("permissions"), path)
|
|
249
|
+
if lacking := missing(granted):
|
|
250
|
+
names = ", ".join(f"{p.label}: {p.access}" for p in lacking)
|
|
251
|
+
raise ReleaseError(
|
|
252
|
+
f"app {slug} on {repo} lacks these permissions: {names}; grant them in the App's settings, "
|
|
253
|
+
"then accept them on the installation"
|
|
254
|
+
)
|
|
255
|
+
return Installation(slug, number, granted)
|
|
256
|
+
|
|
257
|
+
|
|
258
|
+
NOREPLY = "users.noreply.github.com"
|
|
259
|
+
|
|
260
|
+
|
|
261
|
+
@dataclass(frozen=True, slots=True)
|
|
262
|
+
class Identity:
|
|
263
|
+
"""Who a gated session writes as: the App's bot account, and the key its helpers mint with"""
|
|
264
|
+
|
|
265
|
+
app_id: int
|
|
266
|
+
slug: str
|
|
267
|
+
bot_id: int # the bot account's user id, not the App's id
|
|
268
|
+
key: Path
|
|
269
|
+
|
|
270
|
+
@property
|
|
271
|
+
def login(self) -> str:
|
|
272
|
+
return f"{self.slug}[bot]"
|
|
273
|
+
|
|
274
|
+
@property
|
|
275
|
+
def email(self) -> str:
|
|
276
|
+
"""The bot's noreply address, which GitHub links its commits to"""
|
|
277
|
+
return f"{self.bot_id}+{self.login}@{NOREPLY}"
|
|
278
|
+
|
|
279
|
+
|
|
280
|
+
def bot_id(slug: str, app_id: int, api: Api) -> int:
|
|
281
|
+
"""Spec 004, At launch step 4: the user id of the App's bot account. A public read, made
|
|
282
|
+
without the JWT, which GitHub accepts only on the App's own endpoints"""
|
|
283
|
+
login = f"{slug}[bot]"
|
|
284
|
+
path = f"/users/{urllib.parse.quote(login)}"
|
|
285
|
+
answer = api.get(path, None)
|
|
286
|
+
if answer.status == 404:
|
|
287
|
+
raise ReleaseError(f"GitHub has no bot account {login} for app_id {app_id}")
|
|
288
|
+
found = _ok(answer, path, app_id)
|
|
289
|
+
number = found.get("id")
|
|
290
|
+
if not isinstance(number, int) or isinstance(number, bool) or number < 1:
|
|
291
|
+
raise ReleaseError(f"GET {path} gave no user id")
|
|
292
|
+
if found.get("login") != login:
|
|
293
|
+
raise ReleaseError(f"GET {path} answered for {found.get('login')!r}, not {login}")
|
|
294
|
+
return number
|
|
295
|
+
|
|
296
|
+
|
|
297
|
+
AppCheck = Callable[[int, dt.datetime], Identity]
|
|
298
|
+
|
|
299
|
+
|
|
300
|
+
def app_check(repo: str, key: Path | None, home: Path, signer: Signer, api: Api) -> AppCheck:
|
|
301
|
+
"""The gate's App check for repo, spec 004's At launch steps 1 to 4: the key (`--app-key`,
|
|
302
|
+
else the default path for the App's id), the JWT, the installation, then the bot account.
|
|
303
|
+
Any failure raises ReleaseError (exit 2). It writes nothing, so a dry run runs it too"""
|
|
304
|
+
|
|
305
|
+
def check(app_id: int, now: dt.datetime) -> Identity:
|
|
306
|
+
path = check_key(key if key is not None else default_key(app_id, home))
|
|
307
|
+
found = installation(app_id, repo, jwt(app_id, path, now, signer), api)
|
|
308
|
+
return Identity(app_id, found.slug, bot_id(found.slug, app_id, api), path)
|
|
309
|
+
|
|
310
|
+
return check
|
|
311
|
+
|
|
312
|
+
|
|
313
|
+
CACHE = "app-token.json" # in the gate's state folder, `$(git rev-parse --git-common-dir)/shipmill`
|
|
314
|
+
REUSE_FOR = dt.timedelta(minutes=10) # a cached token is reused while it has at least this long left
|
|
315
|
+
|
|
316
|
+
|
|
317
|
+
def repo_name(repo: str) -> str:
|
|
318
|
+
"""The name in owner/name, which the token is limited to"""
|
|
319
|
+
owner, slash, name = repo.partition("/")
|
|
320
|
+
if not owner or not slash or not name or "/" in name:
|
|
321
|
+
raise ReleaseError(f"the repo is owner/name, such as romamo/demo; got {repo!r}")
|
|
322
|
+
return name
|
|
323
|
+
|
|
324
|
+
|
|
325
|
+
@dataclass(frozen=True, slots=True)
|
|
326
|
+
class Token:
|
|
327
|
+
"""An installation token for one repository; repr leaves the value out"""
|
|
328
|
+
|
|
329
|
+
value: str = field(repr=False)
|
|
330
|
+
expires: dt.datetime
|
|
331
|
+
repo: str
|
|
332
|
+
app_id: int
|
|
333
|
+
|
|
334
|
+
def reusable(self, repo: str, app_id: int, now: dt.datetime) -> bool:
|
|
335
|
+
return self.repo.lower() == repo.lower() and self.app_id == app_id and self.expires - now >= REUSE_FOR
|
|
336
|
+
|
|
337
|
+
|
|
338
|
+
def _malformed(path: Path, reason: str) -> ReleaseError:
|
|
339
|
+
return ReleaseError(f"the App token cache {path} is malformed ({reason}); delete it to mint a new token")
|
|
340
|
+
|
|
341
|
+
|
|
342
|
+
def _expiry(value: object) -> dt.datetime | None:
|
|
343
|
+
if not isinstance(value, str):
|
|
344
|
+
return None
|
|
345
|
+
try:
|
|
346
|
+
moment = dt.datetime.fromisoformat(value)
|
|
347
|
+
except ValueError:
|
|
348
|
+
return None
|
|
349
|
+
return moment if moment.tzinfo is not None else None
|
|
350
|
+
|
|
351
|
+
|
|
352
|
+
def load_token(path: Path) -> Token | None:
|
|
353
|
+
"""The cached token, or None when there is no cache; a malformed one raises naming it.
|
|
354
|
+
No message quotes the file, which holds the token"""
|
|
355
|
+
if not path.exists():
|
|
356
|
+
return None
|
|
357
|
+
try:
|
|
358
|
+
data = json.loads(path.read_bytes())
|
|
359
|
+
except json.JSONDecodeError, UnicodeDecodeError:
|
|
360
|
+
raise _malformed(path, "not JSON") from None
|
|
361
|
+
if not isinstance(data, dict):
|
|
362
|
+
raise _malformed(path, "not a JSON object")
|
|
363
|
+
value, repo, app_id = data.get("token"), data.get("repository"), data.get("app_id")
|
|
364
|
+
if not isinstance(value, str) or not value:
|
|
365
|
+
raise _malformed(path, "token is not a non-empty string")
|
|
366
|
+
if not isinstance(repo, str) or not repo:
|
|
367
|
+
raise _malformed(path, "repository is not a non-empty string")
|
|
368
|
+
if not isinstance(app_id, int) or isinstance(app_id, bool):
|
|
369
|
+
raise _malformed(path, "app_id is not an integer")
|
|
370
|
+
expires = _expiry(data.get("expires_at"))
|
|
371
|
+
if expires is None:
|
|
372
|
+
raise _malformed(path, "expires_at is not an ISO time with an offset")
|
|
373
|
+
return Token(value, expires, repo, app_id)
|
|
374
|
+
|
|
375
|
+
|
|
376
|
+
def _write_atomic(path: Path, data: bytes, mode: int) -> None:
|
|
377
|
+
"""Write path through a sibling created with mode (never wider, so no chmod follows) and
|
|
378
|
+
renamed over it, so a reader sees the old file or the new one, never part of one"""
|
|
379
|
+
path.parent.mkdir(parents=True, exist_ok=True)
|
|
380
|
+
temporary = path.with_name(f".{path.name}.{os.getpid()}.{secrets.token_hex(4)}")
|
|
381
|
+
descriptor = os.open(temporary, os.O_WRONLY | os.O_CREAT | os.O_EXCL, mode)
|
|
382
|
+
done = False
|
|
383
|
+
try:
|
|
384
|
+
with os.fdopen(descriptor, "wb") as out:
|
|
385
|
+
out.write(data)
|
|
386
|
+
out.flush()
|
|
387
|
+
os.fsync(out.fileno())
|
|
388
|
+
os.replace(temporary, path)
|
|
389
|
+
done = True
|
|
390
|
+
finally:
|
|
391
|
+
if not done:
|
|
392
|
+
temporary.unlink(missing_ok=True)
|
|
393
|
+
|
|
394
|
+
|
|
395
|
+
def save_token(path: Path, token: Token) -> None:
|
|
396
|
+
"""Cache the token with mode 0600; two helpers minting at once each write a whole, valid
|
|
397
|
+
token, and the later rename wins"""
|
|
398
|
+
record = {
|
|
399
|
+
"app_id": token.app_id,
|
|
400
|
+
"repository": token.repo,
|
|
401
|
+
"expires_at": token.expires.isoformat(),
|
|
402
|
+
"token": token.value,
|
|
403
|
+
}
|
|
404
|
+
_write_atomic(path, (json.dumps(record, indent=2) + "\n").encode(), 0o600)
|
|
405
|
+
|
|
406
|
+
|
|
407
|
+
def _scope(data: Mapping[str, object], repo: str, path: str) -> None:
|
|
408
|
+
"""D-14: the token must be limited to the gate's repo, as asked"""
|
|
409
|
+
repos = data.get("repositories")
|
|
410
|
+
if not isinstance(repos, list) or len(repos) != 1:
|
|
411
|
+
raise ReleaseError(f"POST {path} gave a token not limited to {repo}; refusing it")
|
|
412
|
+
only = repos[0]
|
|
413
|
+
full = only.get("full_name") if isinstance(only, dict) else None
|
|
414
|
+
if not isinstance(full, str) or full.lower() != repo.lower():
|
|
415
|
+
raise ReleaseError(f"POST {path} gave a token for another repository than {repo}; refusing it")
|
|
416
|
+
|
|
417
|
+
|
|
418
|
+
def mint(app_id: int, repo: str, key: Path, now: dt.datetime, signer: Signer, api: Api) -> Token:
|
|
419
|
+
"""Spec 004, Tokens: an installation token limited to repo and the table's permissions"""
|
|
420
|
+
name = repo_name(repo)
|
|
421
|
+
bearer = jwt(app_id, check_key(key), now, signer)
|
|
422
|
+
found = installation(app_id, repo, bearer, api)
|
|
423
|
+
path = f"/app/installations/{found.id}/access_tokens"
|
|
424
|
+
body = {"repositories": [name], "permissions": {p.key: str(p.access) for p in PERMISSIONS}}
|
|
425
|
+
data = _ok(api.post(path, bearer, body), path, app_id, "POST", 201)
|
|
426
|
+
value = data.get("token")
|
|
427
|
+
if not isinstance(value, str) or not value or any(c.isspace() for c in value):
|
|
428
|
+
raise ReleaseError(f"POST {path} gave no token")
|
|
429
|
+
expires = _expiry(data.get("expires_at"))
|
|
430
|
+
if expires is None:
|
|
431
|
+
raise ReleaseError(f"POST {path} gave no expires_at with an offset")
|
|
432
|
+
_scope(data, repo, path)
|
|
433
|
+
return Token(value, expires, repo, app_id)
|
|
434
|
+
|
|
435
|
+
|
|
436
|
+
def app_token(cache: Path, repo: str, app_id: int, key: Path, now: dt.datetime, signer: Signer, api: Api) -> str:
|
|
437
|
+
"""The cached token while it has at least 10 minutes left by this host's clock, else a new
|
|
438
|
+
one, cached. GitHub's expiry is an hour out, so the margin also absorbs a host clock up to
|
|
439
|
+
10 minutes behind GitHub's"""
|
|
440
|
+
repo_name(repo)
|
|
441
|
+
cached = load_token(cache)
|
|
442
|
+
if cached is not None and cached.reusable(repo, app_id, now):
|
|
443
|
+
return cached.value
|
|
444
|
+
fresh = mint(app_id, repo, key, now, signer, api)
|
|
445
|
+
save_token(cache, fresh)
|
|
446
|
+
return fresh.value
|
|
447
|
+
|
|
448
|
+
|
|
449
|
+
CREDENTIAL_HOST = "github.com"
|
|
450
|
+
|
|
451
|
+
|
|
452
|
+
def credential(operation: str, request: str, token: Callable[[], str]) -> str:
|
|
453
|
+
"""git-credential(1)'s helper protocol. get answers an https request for github.com with
|
|
454
|
+
the App's token as x-access-token, and anything else with nothing, so git asks its next
|
|
455
|
+
helper; store, erase, and any other operation are ignored, as the protocol asks"""
|
|
456
|
+
if operation != "get":
|
|
457
|
+
return ""
|
|
458
|
+
asked: dict[str, str] = {}
|
|
459
|
+
for line in request.splitlines():
|
|
460
|
+
if not line:
|
|
461
|
+
break
|
|
462
|
+
key, equals, value = line.partition("=")
|
|
463
|
+
if not equals or not key:
|
|
464
|
+
raise ReleaseError("git sent a credential request line that is not key=value")
|
|
465
|
+
asked[key] = value
|
|
466
|
+
if asked.get("protocol") != "https" or asked.get("host") != CREDENTIAL_HOST:
|
|
467
|
+
return ""
|
|
468
|
+
return f"username=x-access-token\npassword={token()}\n"
|
|
469
|
+
|
|
470
|
+
|
|
471
|
+
HELPERS = "bin" # in the gate's state folder, next to the token cache
|
|
472
|
+
GH_HELPER = "gh"
|
|
473
|
+
CREDENTIAL_HELPER = "git-credential-shipmill"
|
|
474
|
+
|
|
475
|
+
|
|
476
|
+
@dataclass(frozen=True, slots=True)
|
|
477
|
+
class Helpers:
|
|
478
|
+
folder: Path # goes first on a session's PATH
|
|
479
|
+
gh: Path
|
|
480
|
+
git_credential: Path
|
|
481
|
+
real_gh: Path # the gh the gh helper runs
|
|
482
|
+
|
|
483
|
+
|
|
484
|
+
def resolve_gh(folder: Path, path: str) -> Path:
|
|
485
|
+
"""The gh on path, skipping the helpers' folder, so the gh helper never runs itself"""
|
|
486
|
+
mine = folder.resolve()
|
|
487
|
+
rest = [entry for entry in path.split(os.pathsep) if entry and Path(entry).resolve() != mine]
|
|
488
|
+
found = shutil.which("gh", path=os.pathsep.join(rest))
|
|
489
|
+
if found is None:
|
|
490
|
+
raise ReleaseError("gh not found on PATH; the session's gh helper runs it with the App's token")
|
|
491
|
+
return Path(found).resolve()
|
|
492
|
+
|
|
493
|
+
|
|
494
|
+
def write_helpers(folder: Path, python: Path, checkout: Path, repo: str, app_id: int, key: Path, path: str) -> Helpers:
|
|
495
|
+
"""Spec 004, Tokens: write the session's gh and git credential helpers, mode 0700. Each
|
|
496
|
+
runs `<python> -m shipmill app-token` with the App's id, key, repo, and checkout as
|
|
497
|
+
arguments; neither holds a token"""
|
|
498
|
+
repo_name(repo)
|
|
499
|
+
real_gh = resolve_gh(folder, path)
|
|
500
|
+
command = shlex.join(
|
|
501
|
+
[
|
|
502
|
+
str(python),
|
|
503
|
+
"-m",
|
|
504
|
+
"shipmill",
|
|
505
|
+
"--repo",
|
|
506
|
+
str(checkout.resolve()),
|
|
507
|
+
"app-token",
|
|
508
|
+
repo,
|
|
509
|
+
"--app-id",
|
|
510
|
+
str(app_id),
|
|
511
|
+
"--app-key",
|
|
512
|
+
str(key.resolve()),
|
|
513
|
+
]
|
|
514
|
+
)
|
|
515
|
+
header = "#!/bin/sh\n# Written by shipmill gate (spec 004). It holds no token: each call mints or reuses one.\n"
|
|
516
|
+
gh = (
|
|
517
|
+
header
|
|
518
|
+
+ f"token=$({command}) || exit $?\n"
|
|
519
|
+
+ "GH_TOKEN=$token\nexport GH_TOKEN\nunset token\n"
|
|
520
|
+
+ f'exec {shlex.quote(str(real_gh))} "$@"\n'
|
|
521
|
+
)
|
|
522
|
+
credential_helper = header + f'exec {command} --git-credential "$1"\n'
|
|
523
|
+
helpers = Helpers(folder, folder / GH_HELPER, folder / CREDENTIAL_HELPER, real_gh)
|
|
524
|
+
_write_atomic(helpers.gh, gh.encode(), 0o700)
|
|
525
|
+
_write_atomic(helpers.git_credential, credential_helper.encode(), 0o700)
|
|
526
|
+
return helpers
|
|
527
|
+
|
|
528
|
+
|
|
529
|
+
CREDENTIAL_KEY = "credential.https://github.com.helper"
|
|
530
|
+
INSTEAD_OF = "url.https://github.com/.insteadOf"
|
|
531
|
+
SSH_REMOTES = ("git@github.com:", "ssh://git@github.com/") # sent over https, so a push goes through the App
|
|
532
|
+
|
|
533
|
+
|
|
534
|
+
def _helper_value(helper: Path) -> str:
|
|
535
|
+
"""credential.helper's value for an absolute path: git runs it through the shell, so a
|
|
536
|
+
path that needs quoting goes in as a `!` shell command"""
|
|
537
|
+
quoted = shlex.quote(str(helper))
|
|
538
|
+
return quoted if quoted == str(helper) else f"!{quoted}"
|
|
539
|
+
|
|
540
|
+
|
|
541
|
+
def session_env(identity: Identity, helpers: Helpers, path: str) -> dict[str, str]:
|
|
542
|
+
"""Spec 004, At launch step 5: the env a session's `--settings` carries. The helpers'
|
|
543
|
+
folder goes first on PATH, the git author and committer are the App's bot, and git's
|
|
544
|
+
config, through GIT_CONFIG_*, drops the host's credential helpers for github.com, adds
|
|
545
|
+
the App's, and sends SSH remotes over https. It holds no token"""
|
|
546
|
+
config = [
|
|
547
|
+
(CREDENTIAL_KEY, ""),
|
|
548
|
+
(CREDENTIAL_KEY, _helper_value(helpers.git_credential)),
|
|
549
|
+
*((INSTEAD_OF, remote) for remote in SSH_REMOTES),
|
|
550
|
+
]
|
|
551
|
+
env = {
|
|
552
|
+
"PATH": os.pathsep.join([str(helpers.folder), path]) if path else str(helpers.folder),
|
|
553
|
+
"GIT_AUTHOR_NAME": identity.login,
|
|
554
|
+
"GIT_AUTHOR_EMAIL": identity.email,
|
|
555
|
+
"GIT_COMMITTER_NAME": identity.login,
|
|
556
|
+
"GIT_COMMITTER_EMAIL": identity.email,
|
|
557
|
+
"GIT_CONFIG_COUNT": str(len(config)),
|
|
558
|
+
}
|
|
559
|
+
for index, (key, value) in enumerate(config):
|
|
560
|
+
env[f"GIT_CONFIG_KEY_{index}"] = key
|
|
561
|
+
env[f"GIT_CONFIG_VALUE_{index}"] = value
|
|
562
|
+
return env
|
|
563
|
+
|
|
564
|
+
|
|
565
|
+
def prepare_session(
|
|
566
|
+
identity: Identity, folder: Path, python: Path, checkout: Path, repo: str, path: str
|
|
567
|
+
) -> dict[str, str]:
|
|
568
|
+
"""Write the session's helpers to folder, then give the env that puts them to work.
|
|
569
|
+
python is the gate's own interpreter, so the helpers run the same shipmill"""
|
|
570
|
+
helpers = write_helpers(folder.resolve(), python, checkout, repo, identity.app_id, identity.key, path)
|
|
571
|
+
return session_env(identity, helpers, path)
|