imbi-plugin-github 2.9.0__py3-none-any.whl → 2.9.2__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- imbi_plugin_github/_app_auth.py +195 -0
- imbi_plugin_github/commits.py +243 -16
- {imbi_plugin_github-2.9.0.dist-info → imbi_plugin_github-2.9.2.dist-info}/METADATA +47 -2
- {imbi_plugin_github-2.9.0.dist-info → imbi_plugin_github-2.9.2.dist-info}/RECORD +7 -6
- {imbi_plugin_github-2.9.0.dist-info → imbi_plugin_github-2.9.2.dist-info}/WHEEL +0 -0
- {imbi_plugin_github-2.9.0.dist-info → imbi_plugin_github-2.9.2.dist-info}/entry_points.txt +0 -0
- {imbi_plugin_github-2.9.0.dist-info → imbi_plugin_github-2.9.2.dist-info}/licenses/LICENSE +0 -0
|
@@ -0,0 +1,195 @@
|
|
|
1
|
+
"""GitHub App installation-token minting for webhook actions.
|
|
2
|
+
|
|
3
|
+
The commit-sync webhook plugin has no acting user, so when it is
|
|
4
|
+
configured with GitHub App credentials (``app_id`` + ``private_key``) it
|
|
5
|
+
mints a short-lived *installation* access token per call instead of
|
|
6
|
+
carrying a static PAT. Tokens are cached process-wide until shortly
|
|
7
|
+
before they expire, so a busy org makes one token-exchange round-trip
|
|
8
|
+
per hour per ``(app, installation, host)`` rather than one per webhook
|
|
9
|
+
delivery.
|
|
10
|
+
|
|
11
|
+
All three GitHub flavors work unchanged: the caller resolves the API
|
|
12
|
+
base via :func:`imbi_plugin_github._hosts.host_to_api_base` and passes
|
|
13
|
+
it in, so the JWT exchange hits ``api.github.com``,
|
|
14
|
+
``api.<tenant>.ghe.com``, or ``<ghes>/api/v3`` as appropriate.
|
|
15
|
+
"""
|
|
16
|
+
|
|
17
|
+
from __future__ import annotations
|
|
18
|
+
|
|
19
|
+
import base64
|
|
20
|
+
import binascii
|
|
21
|
+
import datetime
|
|
22
|
+
import logging
|
|
23
|
+
import time
|
|
24
|
+
import typing
|
|
25
|
+
|
|
26
|
+
import httpx
|
|
27
|
+
import jwt
|
|
28
|
+
|
|
29
|
+
from imbi_plugin_github.deployment import (
|
|
30
|
+
_auth_headers, # pyright: ignore[reportPrivateUsage]
|
|
31
|
+
_raise_on_401, # pyright: ignore[reportPrivateUsage]
|
|
32
|
+
)
|
|
33
|
+
|
|
34
|
+
LOGGER = logging.getLogger(__name__)
|
|
35
|
+
|
|
36
|
+
_HTTP_TIMEOUT_SECONDS = 10.0
|
|
37
|
+
# GitHub rejects an App JWT whose ``exp`` is more than 10 minutes out;
|
|
38
|
+
# sign for 9 to leave room for clock skew between us and GitHub.
|
|
39
|
+
_JWT_TTL_SECONDS = 540
|
|
40
|
+
# Re-mint an installation token this many seconds before it actually
|
|
41
|
+
# expires so an in-flight request never races the expiry boundary.
|
|
42
|
+
_TOKEN_REFRESH_MARGIN_SECONDS = 300.0
|
|
43
|
+
# Installation tokens last an hour; assume ~55 minutes when GitHub omits
|
|
44
|
+
# (or we can't parse) the ``expires_at`` field.
|
|
45
|
+
_DEFAULT_TOKEN_TTL_SECONDS = 3300.0
|
|
46
|
+
|
|
47
|
+
# Process-wide caches. Token cache values are ``(token, deadline)`` where
|
|
48
|
+
# ``deadline`` is a ``time.monotonic()`` instant; the installation cache
|
|
49
|
+
# avoids re-discovering the installation id on every delivery.
|
|
50
|
+
_TOKEN_CACHE: dict[tuple[str, str, str], tuple[str, float]] = {}
|
|
51
|
+
_INSTALL_CACHE: dict[tuple[str, str, str, str], str] = {}
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
def reset_cache() -> None:
|
|
55
|
+
"""Clear the process-wide token / installation caches (tests)."""
|
|
56
|
+
_TOKEN_CACHE.clear()
|
|
57
|
+
_INSTALL_CACHE.clear()
|
|
58
|
+
|
|
59
|
+
|
|
60
|
+
def _load_private_key(raw: str) -> str:
|
|
61
|
+
"""Return a PEM private key from raw PEM or a base64-encoded PEM.
|
|
62
|
+
|
|
63
|
+
Operators may paste the key GitHub generated directly, or a
|
|
64
|
+
single-line base64 encoding of it (handy where the config UI lacks a
|
|
65
|
+
multi-line field). Raises ``ValueError`` for anything else.
|
|
66
|
+
"""
|
|
67
|
+
value = raw.strip()
|
|
68
|
+
if '-----BEGIN' in value:
|
|
69
|
+
return value
|
|
70
|
+
try:
|
|
71
|
+
decoded = base64.b64decode(value, validate=True).decode('utf-8')
|
|
72
|
+
except (binascii.Error, ValueError, UnicodeDecodeError) as exc:
|
|
73
|
+
raise ValueError(
|
|
74
|
+
'github-commit-sync private_key is neither a PEM nor a '
|
|
75
|
+
'base64-encoded PEM'
|
|
76
|
+
) from exc
|
|
77
|
+
if '-----BEGIN' not in decoded:
|
|
78
|
+
raise ValueError(
|
|
79
|
+
'github-commit-sync private_key decoded but is not a PEM'
|
|
80
|
+
)
|
|
81
|
+
return decoded
|
|
82
|
+
|
|
83
|
+
|
|
84
|
+
def _app_jwt(app_id: str, private_key: str) -> str:
|
|
85
|
+
now = int(time.time())
|
|
86
|
+
return jwt.encode(
|
|
87
|
+
{'iat': now - 60, 'exp': now + _JWT_TTL_SECONDS, 'iss': app_id},
|
|
88
|
+
_load_private_key(private_key),
|
|
89
|
+
algorithm='RS256',
|
|
90
|
+
)
|
|
91
|
+
|
|
92
|
+
|
|
93
|
+
def _token_deadline(expires_at: object) -> float:
|
|
94
|
+
"""Map GitHub's ISO ``expires_at`` to a monotonic cache deadline."""
|
|
95
|
+
now = time.monotonic()
|
|
96
|
+
if not isinstance(expires_at, str):
|
|
97
|
+
return now + _DEFAULT_TOKEN_TTL_SECONDS
|
|
98
|
+
try:
|
|
99
|
+
exp = datetime.datetime.fromisoformat(expires_at)
|
|
100
|
+
except ValueError:
|
|
101
|
+
return now + _DEFAULT_TOKEN_TTL_SECONDS
|
|
102
|
+
remaining = (exp - datetime.datetime.now(datetime.UTC)).total_seconds()
|
|
103
|
+
return now + max(0.0, remaining - _TOKEN_REFRESH_MARGIN_SECONDS)
|
|
104
|
+
|
|
105
|
+
|
|
106
|
+
def _cached_token(key: tuple[str, str, str]) -> str | None:
|
|
107
|
+
entry = _TOKEN_CACHE.get(key)
|
|
108
|
+
if entry is not None and entry[1] > time.monotonic():
|
|
109
|
+
return entry[0]
|
|
110
|
+
return None
|
|
111
|
+
|
|
112
|
+
|
|
113
|
+
async def _discover_installation_id(
|
|
114
|
+
client: httpx.AsyncClient, owner: str, repo: str
|
|
115
|
+
) -> str:
|
|
116
|
+
resp = await client.get(f'/repos/{owner}/{repo}/installation')
|
|
117
|
+
resp.raise_for_status()
|
|
118
|
+
data = typing.cast('dict[str, typing.Any]', resp.json())
|
|
119
|
+
install_id = data.get('id')
|
|
120
|
+
if install_id is None:
|
|
121
|
+
raise ValueError(
|
|
122
|
+
f'no GitHub App installation found for {owner}/{repo}'
|
|
123
|
+
)
|
|
124
|
+
return str(install_id)
|
|
125
|
+
|
|
126
|
+
|
|
127
|
+
async def _mint(
|
|
128
|
+
client: httpx.AsyncClient, installation_id: str
|
|
129
|
+
) -> tuple[str, object]:
|
|
130
|
+
resp = await client.post(
|
|
131
|
+
f'/app/installations/{installation_id}/access_tokens'
|
|
132
|
+
)
|
|
133
|
+
resp.raise_for_status()
|
|
134
|
+
data = typing.cast('dict[str, typing.Any]', resp.json())
|
|
135
|
+
return str(data['token']), data.get('expires_at')
|
|
136
|
+
|
|
137
|
+
|
|
138
|
+
async def installation_token(
|
|
139
|
+
*,
|
|
140
|
+
base: str,
|
|
141
|
+
app_id: str,
|
|
142
|
+
private_key: str,
|
|
143
|
+
installation_id: str | None,
|
|
144
|
+
owner: str,
|
|
145
|
+
repo: str,
|
|
146
|
+
) -> str:
|
|
147
|
+
"""Return a valid installation token, minting/caching as needed.
|
|
148
|
+
|
|
149
|
+
``installation_id`` may be ``None``, in which case the installation
|
|
150
|
+
is discovered from the target repo (and cached). The resulting
|
|
151
|
+
token is cached until shortly before it expires.
|
|
152
|
+
"""
|
|
153
|
+
install = installation_id
|
|
154
|
+
if install is not None and (
|
|
155
|
+
cached := _cached_token((app_id, install, base))
|
|
156
|
+
):
|
|
157
|
+
return cached
|
|
158
|
+
if install is None:
|
|
159
|
+
install = _INSTALL_CACHE.get((app_id, base, owner, repo))
|
|
160
|
+
if install is not None and (
|
|
161
|
+
cached := _cached_token((app_id, install, base))
|
|
162
|
+
):
|
|
163
|
+
return cached
|
|
164
|
+
|
|
165
|
+
install_was_cached = install is not None and installation_id is None
|
|
166
|
+
app_token = _app_jwt(app_id, private_key)
|
|
167
|
+
async with httpx.AsyncClient(
|
|
168
|
+
base_url=base,
|
|
169
|
+
headers=_auth_headers(app_token),
|
|
170
|
+
timeout=_HTTP_TIMEOUT_SECONDS,
|
|
171
|
+
event_hooks={'response': [_raise_on_401]},
|
|
172
|
+
) as client:
|
|
173
|
+
if install is None:
|
|
174
|
+
install = await _discover_installation_id(client, owner, repo)
|
|
175
|
+
_INSTALL_CACHE[(app_id, base, owner, repo)] = install
|
|
176
|
+
if cached := _cached_token((app_id, install, base)):
|
|
177
|
+
return cached
|
|
178
|
+
try:
|
|
179
|
+
token, expires_at = await _mint(client, install)
|
|
180
|
+
except httpx.HTTPStatusError as exc:
|
|
181
|
+
# A 404 (or 401, surfaced as PluginAuthenticationFailed by the
|
|
182
|
+
# response hook) against a *cached* installation id means the
|
|
183
|
+
# app was uninstalled/reinstalled or transferred. Evict the
|
|
184
|
+
# stale id and rediscover once before giving up.
|
|
185
|
+
if not install_was_cached or exc.response.status_code != 404:
|
|
186
|
+
raise
|
|
187
|
+
_INSTALL_CACHE.pop((app_id, base, owner, repo), None)
|
|
188
|
+
install = await _discover_installation_id(client, owner, repo)
|
|
189
|
+
_INSTALL_CACHE[(app_id, base, owner, repo)] = install
|
|
190
|
+
token, expires_at = await _mint(client, install)
|
|
191
|
+
_TOKEN_CACHE[(app_id, install, base)] = (
|
|
192
|
+
token,
|
|
193
|
+
_token_deadline(expires_at),
|
|
194
|
+
)
|
|
195
|
+
return token
|
imbi_plugin_github/commits.py
CHANGED
|
@@ -18,6 +18,12 @@ order:
|
|
|
18
18
|
4. the push payload's ``repository.url`` (already the flavor-correct API
|
|
19
19
|
URL) as a last resort.
|
|
20
20
|
|
|
21
|
+
The same plugin also exposes :meth:`GitHubCommitSyncPlugin.sync_all_history`
|
|
22
|
+
for an on-demand, host-invoked backfill: there is no push payload, so the
|
|
23
|
+
GitHub host is read from ``ctx.service_plugins`` and ``(owner, repo)`` from
|
|
24
|
+
the project links; it walks the full default-branch history and the
|
|
25
|
+
complete tag list rather than a single push delta.
|
|
26
|
+
|
|
21
27
|
Commit / tag rows are written to the shared ClickHouse ``commits`` /
|
|
22
28
|
``tags`` tables via :func:`imbi_common.clickhouse.insert`. Writes are
|
|
23
29
|
best-effort: a storage failure is logged and swallowed so an analytics
|
|
@@ -47,11 +53,13 @@ from imbi_common.plugins.base import (
|
|
|
47
53
|
WebhookActionPlugin,
|
|
48
54
|
)
|
|
49
55
|
|
|
56
|
+
from imbi_plugin_github import _app_auth
|
|
50
57
|
from imbi_plugin_github._hosts import (
|
|
51
58
|
host_to_api_base,
|
|
52
59
|
normalize_host,
|
|
53
60
|
require_ghec_tenant_host,
|
|
54
61
|
)
|
|
62
|
+
from imbi_plugin_github._repos import resolve_owner_repo
|
|
55
63
|
from imbi_plugin_github.deployment import (
|
|
56
64
|
_auth_headers, # pyright: ignore[reportPrivateUsage]
|
|
57
65
|
_next_page_url, # pyright: ignore[reportPrivateUsage]
|
|
@@ -65,21 +73,51 @@ LOGGER = logging.getLogger(__name__)
|
|
|
65
73
|
|
|
66
74
|
_HTTP_TIMEOUT_SECONDS = 10.0
|
|
67
75
|
_ZERO_SHA = '0' * 40
|
|
76
|
+
# This plugin's own slug; skipped when reading the GitHub host/flavor
|
|
77
|
+
# from connected ``service_plugins`` so the commit-sync entry can't
|
|
78
|
+
# masquerade as a github.com host on a GHEC/GHES service.
|
|
79
|
+
_SELF_SLUG = 'github-commit-sync'
|
|
68
80
|
# GitHub's compare endpoint caps ``commits[]`` at 250 per page and
|
|
69
81
|
# paginates the rest; bound the walk so a pathological single push
|
|
70
82
|
# (force-push of thousands of commits) can't pin us on one endpoint.
|
|
71
83
|
# 250 * 20 = 5000 commits is far more than any realistic push.
|
|
72
84
|
_MAX_COMPARE_PAGES = 20
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
85
|
+
# On-demand full-history sync walks ``GET /commits`` from the default
|
|
86
|
+
# branch head. 100 per page * 100 pages = 10k commits caps a one-shot
|
|
87
|
+
# backfill so a very deep repo can't pin the worker indefinitely; the
|
|
88
|
+
# walk logs a truncation warning when it hits the cap.
|
|
89
|
+
_MAX_HISTORY_PAGES = 100
|
|
90
|
+
|
|
91
|
+
|
|
92
|
+
async def _resolve_bearer(
|
|
93
|
+
credentials: dict[str, str], base: str, owner: str, repo: str
|
|
94
|
+
) -> str:
|
|
95
|
+
"""Resolve the Bearer token used for the repo's GitHub API calls.
|
|
96
|
+
|
|
97
|
+
Prefers an explicit PAT (``access_token``/``token``). Otherwise
|
|
98
|
+
mints a short-lived GitHub App installation token from ``app_id`` +
|
|
99
|
+
``private_key`` (with an optional ``installation_id`` that skips
|
|
100
|
+
per-repo installation discovery). Tokens are cached process-wide by
|
|
101
|
+
:mod:`imbi_plugin_github._app_auth`.
|
|
102
|
+
"""
|
|
76
103
|
token = credentials.get('access_token') or credentials.get('token')
|
|
77
|
-
if
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
104
|
+
if token:
|
|
105
|
+
return token
|
|
106
|
+
app_id = credentials.get('app_id')
|
|
107
|
+
private_key = credentials.get('private_key')
|
|
108
|
+
if app_id and private_key:
|
|
109
|
+
return await _app_auth.installation_token(
|
|
110
|
+
base=base,
|
|
111
|
+
app_id=app_id,
|
|
112
|
+
private_key=private_key,
|
|
113
|
+
installation_id=credentials.get('installation_id') or None,
|
|
114
|
+
owner=owner,
|
|
115
|
+
repo=repo,
|
|
81
116
|
)
|
|
82
|
-
|
|
117
|
+
raise ValueError(
|
|
118
|
+
'github-commit-sync requires either an access_token (PAT) or '
|
|
119
|
+
'app_id + private_key (GitHub App) credentials'
|
|
120
|
+
)
|
|
83
121
|
|
|
84
122
|
|
|
85
123
|
def _resolve(pointer: jsonpointer.JsonPointer, payload: object) -> object:
|
|
@@ -150,6 +188,11 @@ def _resolve_api_base(
|
|
|
150
188
|
if explicit:
|
|
151
189
|
return explicit.rstrip('/')
|
|
152
190
|
for plugin in ctx.service_plugins:
|
|
191
|
+
# Skip our own entry: its slug starts with "github" but carries
|
|
192
|
+
# no host option, so _github_plugin_host would mis-resolve it to
|
|
193
|
+
# github.com on a GHEC/GHES service.
|
|
194
|
+
if plugin.slug == _SELF_SLUG:
|
|
195
|
+
continue
|
|
153
196
|
host = _github_plugin_host(plugin)
|
|
154
197
|
if host:
|
|
155
198
|
return host_to_api_base(host)
|
|
@@ -324,6 +367,93 @@ async def _fetch_recent_commits(
|
|
|
324
367
|
return typing.cast('list[dict[str, typing.Any]]', resp.json())
|
|
325
368
|
|
|
326
369
|
|
|
370
|
+
def _resolve_host_for_context(ctx: PluginContext) -> str | None:
|
|
371
|
+
"""Resolve the GitHub web host for an on-demand sync (no payload).
|
|
372
|
+
|
|
373
|
+
Walks the connected GitHub plugins on ``ctx.service_plugins`` and
|
|
374
|
+
returns the first usable host (github.com, a GHEC tenant, or a GHES
|
|
375
|
+
appliance), skipping this plugin's own entry so a commit-sync row on a
|
|
376
|
+
GHEC/GHES service can't be read as github.com. Unlike the webhook
|
|
377
|
+
path there is no push payload to fall back to, so the absence of a
|
|
378
|
+
connected GitHub plugin yields ``None``.
|
|
379
|
+
"""
|
|
380
|
+
for plugin in ctx.service_plugins:
|
|
381
|
+
if plugin.slug == _SELF_SLUG:
|
|
382
|
+
continue
|
|
383
|
+
host = _github_plugin_host(plugin)
|
|
384
|
+
if host:
|
|
385
|
+
return host
|
|
386
|
+
return None
|
|
387
|
+
|
|
388
|
+
|
|
389
|
+
async def _fetch_default_branch(client: httpx.AsyncClient) -> str:
|
|
390
|
+
"""Return the repo's default branch name (``main`` when unknown)."""
|
|
391
|
+
# httpx normalises ``base_url`` with a trailing slash; GHEC's gateway
|
|
392
|
+
# 404s on ``/repos/<o>/<r>/`` so request the absolute URL with the
|
|
393
|
+
# trailing slash stripped, matching the deployment plugin.
|
|
394
|
+
url = str(client.base_url).rstrip('/')
|
|
395
|
+
resp = await client.get(url)
|
|
396
|
+
resp.raise_for_status()
|
|
397
|
+
meta = typing.cast('dict[str, typing.Any]', resp.json())
|
|
398
|
+
return str(meta.get('default_branch') or 'main')
|
|
399
|
+
|
|
400
|
+
|
|
401
|
+
async def _fetch_all_commits(
|
|
402
|
+
client: httpx.AsyncClient, branch: str
|
|
403
|
+
) -> list[dict[str, typing.Any]]:
|
|
404
|
+
"""Walk every commit reachable from ``branch`` via Link pagination.
|
|
405
|
+
|
|
406
|
+
Capped at ``_MAX_HISTORY_PAGES`` (logged on truncation) so a very deep
|
|
407
|
+
repo can't pin a one-shot backfill indefinitely.
|
|
408
|
+
"""
|
|
409
|
+
params: dict[str, str] = {'sha': branch, 'per_page': '100'}
|
|
410
|
+
out: list[dict[str, typing.Any]] = []
|
|
411
|
+
for page in range(1, _MAX_HISTORY_PAGES + 1):
|
|
412
|
+
resp = await client.get('/commits', params=params)
|
|
413
|
+
resp.raise_for_status()
|
|
414
|
+
out.extend(typing.cast('list[dict[str, typing.Any]]', resp.json()))
|
|
415
|
+
next_url = _next_page_url(resp.headers.get('link'))
|
|
416
|
+
if next_url is None:
|
|
417
|
+
return out
|
|
418
|
+
next_page = _query_param(next_url, 'page')
|
|
419
|
+
if next_page is None:
|
|
420
|
+
return out
|
|
421
|
+
params['page'] = next_page
|
|
422
|
+
if page == _MAX_HISTORY_PAGES:
|
|
423
|
+
LOGGER.warning(
|
|
424
|
+
'github-commit-sync truncated history at %d pages (%d '
|
|
425
|
+
'commits); older commits will not be recorded',
|
|
426
|
+
_MAX_HISTORY_PAGES,
|
|
427
|
+
len(out),
|
|
428
|
+
)
|
|
429
|
+
return out
|
|
430
|
+
|
|
431
|
+
|
|
432
|
+
async def _insert_best_effort(
|
|
433
|
+
table: str, records: list[pydantic.BaseModel], project_id: str
|
|
434
|
+
) -> int:
|
|
435
|
+
"""Insert ``records`` into ``table``; return rows written (0 on fail).
|
|
436
|
+
|
|
437
|
+
``imbi_common.clickhouse.insert`` rejects an empty list, so an empty
|
|
438
|
+
set short-circuits to 0. A storage failure is logged and swallowed,
|
|
439
|
+
mirroring the webhook actions -- an analytics hiccup must not fail the
|
|
440
|
+
sync.
|
|
441
|
+
"""
|
|
442
|
+
if not records:
|
|
443
|
+
return 0
|
|
444
|
+
try:
|
|
445
|
+
await clickhouse.insert(table, records)
|
|
446
|
+
except Exception:
|
|
447
|
+
LOGGER.exception(
|
|
448
|
+
'github-commit-sync: failed to record %d %s rows for project %s',
|
|
449
|
+
len(records),
|
|
450
|
+
table,
|
|
451
|
+
project_id,
|
|
452
|
+
)
|
|
453
|
+
return 0
|
|
454
|
+
return len(records)
|
|
455
|
+
|
|
456
|
+
|
|
327
457
|
class SyncCommitsConfig(pydantic.BaseModel):
|
|
328
458
|
"""``WebhookRule.handler_config`` for ``sync_commits``."""
|
|
329
459
|
|
|
@@ -394,7 +524,8 @@ async def sync_commits(
|
|
|
394
524
|
ref = _branch_short_name(ref_raw if isinstance(ref_raw, str) else '')
|
|
395
525
|
before = _resolve(action_config.before_selector, payload)
|
|
396
526
|
pushed_at = datetime.datetime.now(datetime.UTC)
|
|
397
|
-
|
|
527
|
+
token = await _resolve_bearer(credentials, base, owner, repo)
|
|
528
|
+
async with _client(base, owner, repo, token) as client:
|
|
398
529
|
if isinstance(before, str) and before and before != _ZERO_SHA:
|
|
399
530
|
raw = await _fetch_compare_commits(client, before, after)
|
|
400
531
|
else:
|
|
@@ -510,7 +641,8 @@ async def sync_tags(
|
|
|
510
641
|
if resolved is None:
|
|
511
642
|
return
|
|
512
643
|
owner, repo, base = resolved
|
|
513
|
-
|
|
644
|
+
token = await _resolve_bearer(credentials, base, owner, repo)
|
|
645
|
+
async with _client(base, owner, repo, token) as client:
|
|
514
646
|
annotated = await _annotated_tag(client, after)
|
|
515
647
|
records: list[pydantic.BaseModel] = [
|
|
516
648
|
_tag_record(
|
|
@@ -566,9 +698,15 @@ sync_tags_descriptor = ActionDescriptor(
|
|
|
566
698
|
class GitHubCommitSyncPlugin(WebhookActionPlugin):
|
|
567
699
|
"""Webhook-action plugin syncing GitHub commit / tag history.
|
|
568
700
|
|
|
569
|
-
|
|
570
|
-
|
|
571
|
-
|
|
701
|
+
Carries its own service credential -- it is *not* folded into the
|
|
702
|
+
identity / deployment / lifecycle plugins, which run as the acting
|
|
703
|
+
user. Two mutually exclusive auth modes are supported (resolved by
|
|
704
|
+
:func:`_resolve_bearer`):
|
|
705
|
+
|
|
706
|
+
* **PAT** -- a static ``access_token``.
|
|
707
|
+
* **GitHub App** -- ``app_id`` + ``private_key`` (raw or base64 PEM),
|
|
708
|
+
with an optional ``installation_id``; the plugin mints a
|
|
709
|
+
short-lived installation token per call and caches it.
|
|
572
710
|
"""
|
|
573
711
|
|
|
574
712
|
manifest = PluginManifest(
|
|
@@ -582,12 +720,101 @@ class GitHubCommitSyncPlugin(WebhookActionPlugin):
|
|
|
582
720
|
credentials=[
|
|
583
721
|
CredentialField(
|
|
584
722
|
name='access_token',
|
|
585
|
-
label='GitHub Token',
|
|
586
|
-
description=
|
|
587
|
-
|
|
723
|
+
label='GitHub Token (PAT)',
|
|
724
|
+
description=(
|
|
725
|
+
'Static personal/service access token. Use this *or* '
|
|
726
|
+
'the GitHub App fields below.'
|
|
727
|
+
),
|
|
728
|
+
required=False,
|
|
729
|
+
),
|
|
730
|
+
CredentialField(
|
|
731
|
+
name='app_id',
|
|
732
|
+
label='GitHub App ID',
|
|
733
|
+
description=(
|
|
734
|
+
'GitHub App identifier; with a private key the plugin '
|
|
735
|
+
'mints short-lived installation tokens.'
|
|
736
|
+
),
|
|
737
|
+
required=False,
|
|
738
|
+
),
|
|
739
|
+
CredentialField(
|
|
740
|
+
name='private_key',
|
|
741
|
+
label='GitHub App Private Key',
|
|
742
|
+
description=(
|
|
743
|
+
'App private key, raw PEM or base64-encoded PEM.'
|
|
744
|
+
),
|
|
745
|
+
required=False,
|
|
746
|
+
),
|
|
747
|
+
CredentialField(
|
|
748
|
+
name='installation_id',
|
|
749
|
+
label='GitHub App Installation ID',
|
|
750
|
+
description=(
|
|
751
|
+
'Optional. When unset, the installation is discovered '
|
|
752
|
+
'from the pushed repository.'
|
|
753
|
+
),
|
|
754
|
+
required=False,
|
|
755
|
+
),
|
|
588
756
|
],
|
|
589
757
|
)
|
|
590
758
|
|
|
591
759
|
@classmethod
|
|
592
760
|
def actions(cls) -> list[ActionDescriptor]:
|
|
593
761
|
return [sync_commits_descriptor, sync_tags_descriptor]
|
|
762
|
+
|
|
763
|
+
async def sync_all_history(
|
|
764
|
+
self,
|
|
765
|
+
*,
|
|
766
|
+
ctx: PluginContext,
|
|
767
|
+
credentials: dict[str, str],
|
|
768
|
+
) -> tuple[int, int]:
|
|
769
|
+
"""Record the project's full default-branch history and all tags.
|
|
770
|
+
|
|
771
|
+
Host-invoked (no webhook payload): the host instantiates the
|
|
772
|
+
plugin, builds a :class:`PluginContext` carrying the project's
|
|
773
|
+
links and the connected ``service_plugins``, resolves this
|
|
774
|
+
plugin's service ``credentials``, and awaits this method. The
|
|
775
|
+
GitHub host/flavor is read from ``service_plugins``, the
|
|
776
|
+
``(owner, repo)`` from the project links, and the bearer token
|
|
777
|
+
from the same PAT-or-App resolution the webhook actions use.
|
|
778
|
+
|
|
779
|
+
Walks every commit reachable from the default branch head plus the
|
|
780
|
+
repo's complete (lightweight) tag list, maps them onto
|
|
781
|
+
``CommitRecord`` / ``TagRecord``, and upserts into the ClickHouse
|
|
782
|
+
``commits`` / ``tags`` tables. ``ReplacingMergeTree`` dedupes
|
|
783
|
+
against rows the webhook already recorded, so re-running is safe.
|
|
784
|
+
|
|
785
|
+
Returns ``(commits_recorded, tags_recorded)``. Raises
|
|
786
|
+
:class:`ValueError` only when the host or repository can't be
|
|
787
|
+
resolved; ClickHouse failures are swallowed (the count reflects
|
|
788
|
+
what was written).
|
|
789
|
+
"""
|
|
790
|
+
host = _resolve_host_for_context(ctx)
|
|
791
|
+
if host is None:
|
|
792
|
+
raise ValueError(
|
|
793
|
+
'github-commit-sync could not resolve a GitHub host for an '
|
|
794
|
+
'on-demand sync: connect a GitHub plugin to the service'
|
|
795
|
+
)
|
|
796
|
+
base = host_to_api_base(host)
|
|
797
|
+
owner, repo = resolve_owner_repo(ctx, host, 'github-commit-sync')
|
|
798
|
+
pushed_at = datetime.datetime.now(datetime.UTC)
|
|
799
|
+
token = await _resolve_bearer(credentials, base, owner, repo)
|
|
800
|
+
async with _client(base, owner, repo, token) as client:
|
|
801
|
+
branch = await _fetch_default_branch(client)
|
|
802
|
+
raw_commits = await _fetch_all_commits(client, branch)
|
|
803
|
+
tags = await _reconcile_tags(client, ctx.project_id)
|
|
804
|
+
commit_records: list[pydantic.BaseModel] = [
|
|
805
|
+
_commit_record(
|
|
806
|
+
item,
|
|
807
|
+
project_id=ctx.project_id,
|
|
808
|
+
ref=branch,
|
|
809
|
+
pushed_at=pushed_at,
|
|
810
|
+
)
|
|
811
|
+
for item in raw_commits
|
|
812
|
+
if item.get('sha')
|
|
813
|
+
]
|
|
814
|
+
commits_recorded = await _insert_best_effort(
|
|
815
|
+
'commits', commit_records, ctx.project_id
|
|
816
|
+
)
|
|
817
|
+
tags_recorded = await _insert_best_effort(
|
|
818
|
+
'tags', list(tags), ctx.project_id
|
|
819
|
+
)
|
|
820
|
+
return commits_recorded, tags_recorded
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: imbi-plugin-github
|
|
3
|
-
Version: 2.9.
|
|
3
|
+
Version: 2.9.2
|
|
4
4
|
Summary: GitHub identity plugin for Imbi (github.com / GHEC / GHES)
|
|
5
5
|
Author-email: "Gavin M. Roy" <gavinr@aweber.com>
|
|
6
6
|
License: BSD-3-Clause
|
|
@@ -13,8 +13,9 @@ Classifier: Programming Language :: Python :: 3
|
|
|
13
13
|
Classifier: Programming Language :: Python :: 3.14
|
|
14
14
|
Requires-Python: >=3.14
|
|
15
15
|
Requires-Dist: httpx>=0.27
|
|
16
|
-
Requires-Dist: imbi-common[databases]==2.9.
|
|
16
|
+
Requires-Dist: imbi-common[databases]==2.9.2
|
|
17
17
|
Requires-Dist: pydantic>=2
|
|
18
|
+
Requires-Dist: pyjwt[crypto]>=2.8
|
|
18
19
|
Description-Content-Type: text/markdown
|
|
19
20
|
|
|
20
21
|
# imbi-plugin-github
|
|
@@ -30,6 +31,7 @@ projects to the right backend.
|
|
|
30
31
|
| Identity | `github`, `github-enterprise-cloud`, `github-enterprise-server` |
|
|
31
32
|
| Deployment | `github-deployment`, `github-deployment-ec`, `github-deployment-es` |
|
|
32
33
|
| Lifecycle | `github-lifecycle`, `github-lifecycle-ec`, `github-lifecycle-es` |
|
|
34
|
+
| Webhook | `github-commit-sync` |
|
|
33
35
|
|
|
34
36
|
### Identity
|
|
35
37
|
|
|
@@ -63,6 +65,37 @@ org.
|
|
|
63
65
|
Archiving requires admin scope on the repo; transferring additionally
|
|
64
66
|
requires admin permission on the target organization.
|
|
65
67
|
|
|
68
|
+
### Webhook (commit / tag sync)
|
|
69
|
+
|
|
70
|
+
A single `github-commit-sync` webhook-action plugin exposes two actions
|
|
71
|
+
the gateway dispatches on `push` deliveries:
|
|
72
|
+
|
|
73
|
+
| Action | Handler | Records into ClickHouse |
|
|
74
|
+
| -------------- | -------------------------------- | ----------------------- |
|
|
75
|
+
| `sync_commits` | `github-commit-sync#sync_commits`| `commits` |
|
|
76
|
+
| `sync_tags` | `github-commit-sync#sync_tags` | `tags` |
|
|
77
|
+
|
|
78
|
+
`sync_commits` fetches the full set of commits in a push via the compare
|
|
79
|
+
API (paginated, so it isn't capped by the 20-commit inline payload limit);
|
|
80
|
+
`sync_tags` records the pushed tag and, with `reconcile_all`, the repo's
|
|
81
|
+
full tag list. Branch/tag gating is the rule's CEL `filter_expression`
|
|
82
|
+
(e.g. `ref == "refs/heads/main"`, `ref.startsWith("refs/tags/")`). The API
|
|
83
|
+
flavor (github.com / GHEC / GHES) is resolved at runtime — explicit
|
|
84
|
+
`api_base_url`, else a connected GitHub plugin on the same service, else
|
|
85
|
+
the service endpoint, else the payload's `repository.url`.
|
|
86
|
+
|
|
87
|
+
Unlike identity/deployment/lifecycle (which act as the OAuth user),
|
|
88
|
+
commit-sync runs without an actor and authenticates with a **service**
|
|
89
|
+
credential in one of two modes, resolved per call:
|
|
90
|
+
|
|
91
|
+
- **PAT** — a static `access_token`.
|
|
92
|
+
- **GitHub App** — `app_id` + `private_key`; the plugin signs an App JWT
|
|
93
|
+
and mints a short-lived **installation token** (cached process-wide
|
|
94
|
+
until shortly before it expires), so no static, expiring token is
|
|
95
|
+
stored. `installation_id` is optional — when unset it is discovered
|
|
96
|
+
from the pushed repository (`GET /repos/{owner}/{repo}/installation`).
|
|
97
|
+
The App needs **Contents: Read-only**.
|
|
98
|
+
|
|
66
99
|
## Manifest options (identity)
|
|
67
100
|
|
|
68
101
|
| Option | Required | Description |
|
|
@@ -77,6 +110,18 @@ requires admin permission on the target organization.
|
|
|
77
110
|
| `client_id` | yes |
|
|
78
111
|
| `client_secret` | yes |
|
|
79
112
|
|
|
113
|
+
## Credentials (commit-sync)
|
|
114
|
+
|
|
115
|
+
Provide **either** the PAT field **or** the GitHub App fields (all
|
|
116
|
+
individually optional; validated per call):
|
|
117
|
+
|
|
118
|
+
| Field | Mode | Description |
|
|
119
|
+
| ----------------- | ---- | ------------------------------------------------------ |
|
|
120
|
+
| `access_token` | PAT | Static personal/service token. |
|
|
121
|
+
| `app_id` | App | GitHub App identifier. |
|
|
122
|
+
| `private_key` | App | App private key — raw PEM or base64-encoded PEM. |
|
|
123
|
+
| `installation_id` | App | Optional; discovered from the repo when unset. |
|
|
124
|
+
|
|
80
125
|
## License
|
|
81
126
|
|
|
82
127
|
BSD-3-Clause.
|
|
@@ -1,12 +1,13 @@
|
|
|
1
1
|
imbi_plugin_github/__init__.py,sha256=8ZVhm8XpGqgYRUtpHl0uuZTT3gmMyHNhKd9k6qoatB4,298
|
|
2
|
+
imbi_plugin_github/_app_auth.py,sha256=LnLrvqmu2UC27uHHJfIzFPNDh33RtgeOX1RXceX9Pj0,7000
|
|
2
3
|
imbi_plugin_github/_hosts.py,sha256=jQ1Z6yEpjAa4b7LocLfEelnBcRYYQLNLBNCdWZtAXUc,2258
|
|
3
4
|
imbi_plugin_github/_repos.py,sha256=agp4ZgRuqbrGBRMFeHomnUf1HvvaNvapBCWHQebdviQ,4559
|
|
4
|
-
imbi_plugin_github/commits.py,sha256=
|
|
5
|
+
imbi_plugin_github/commits.py,sha256=okoJd6GM_YlkqyJb3mHlxz58JU7Ar8ve8bQcDkTrkRI,29939
|
|
5
6
|
imbi_plugin_github/deployment.py,sha256=V6YlZX24m33BPJWkC-vCZZLSsH71etkl0xkhkDsKASg,47914
|
|
6
7
|
imbi_plugin_github/identity.py,sha256=RSs1cLQpZmfyQ_0KQkohDIZEBYVBnre6RJIt166EtdQ,15525
|
|
7
8
|
imbi_plugin_github/lifecycle.py,sha256=48mLKjFPG7BOanpA7jWnmbWOdsDLh1F39g2AoxDLL7M,33211
|
|
8
|
-
imbi_plugin_github-2.9.
|
|
9
|
-
imbi_plugin_github-2.9.
|
|
10
|
-
imbi_plugin_github-2.9.
|
|
11
|
-
imbi_plugin_github-2.9.
|
|
12
|
-
imbi_plugin_github-2.9.
|
|
9
|
+
imbi_plugin_github-2.9.2.dist-info/METADATA,sha256=ACIhGSJ7VSQL2O-v0SmxRAqfov95uaEvWYZsEdjTICc,5809
|
|
10
|
+
imbi_plugin_github-2.9.2.dist-info/WHEEL,sha256=QccIxa26bgl1E6uMy58deGWi-0aeIkkangHcxk2kWfw,87
|
|
11
|
+
imbi_plugin_github-2.9.2.dist-info/entry_points.txt,sha256=RavN_FYCeS97iauHgCG3tHFdVULbEMXmQfvgCgLeGkI,805
|
|
12
|
+
imbi_plugin_github-2.9.2.dist-info/licenses/LICENSE,sha256=el9B20zOqEcv_mRrHgbdkVyxSC6I8np7am0D6mO2A9Y,1491
|
|
13
|
+
imbi_plugin_github-2.9.2.dist-info/RECORD,,
|
|
File without changes
|
|
File without changes
|
|
File without changes
|