imbi-plugin-github 2.10.0__py3-none-any.whl → 2.11.2__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- imbi_plugin_github/commits.py +329 -43
- imbi_plugin_github/deployment.py +7 -0
- {imbi_plugin_github-2.10.0.dist-info → imbi_plugin_github-2.11.2.dist-info}/METADATA +2 -2
- {imbi_plugin_github-2.10.0.dist-info → imbi_plugin_github-2.11.2.dist-info}/RECORD +7 -7
- {imbi_plugin_github-2.10.0.dist-info → imbi_plugin_github-2.11.2.dist-info}/WHEEL +1 -1
- {imbi_plugin_github-2.10.0.dist-info → imbi_plugin_github-2.11.2.dist-info}/entry_points.txt +0 -0
- {imbi_plugin_github-2.10.0.dist-info → imbi_plugin_github-2.11.2.dist-info}/licenses/LICENSE +0 -0
imbi_plugin_github/commits.py
CHANGED
|
@@ -50,12 +50,14 @@ from imbi_common.json_pointer import JsonPointer
|
|
|
50
50
|
from imbi_common.models import CommitRecord, TagRecord
|
|
51
51
|
from imbi_common.plugins.base import (
|
|
52
52
|
ActionDescriptor,
|
|
53
|
+
CheckStatus,
|
|
53
54
|
CredentialField,
|
|
54
55
|
PluginContext,
|
|
55
56
|
PluginManifest,
|
|
56
57
|
ServicePlugin,
|
|
57
58
|
WebhookActionPlugin,
|
|
58
59
|
)
|
|
60
|
+
from imbi_common.plugins.errors import PluginRateLimited
|
|
59
61
|
|
|
60
62
|
from imbi_plugin_github import _app_auth
|
|
61
63
|
from imbi_plugin_github._hosts import (
|
|
@@ -66,10 +68,13 @@ from imbi_plugin_github._hosts import (
|
|
|
66
68
|
from imbi_plugin_github._repos import resolve_owner_repo
|
|
67
69
|
from imbi_plugin_github.deployment import (
|
|
68
70
|
_auth_headers, # pyright: ignore[reportPrivateUsage]
|
|
71
|
+
_check_runs_to_status, # pyright: ignore[reportPrivateUsage]
|
|
72
|
+
_checks_disabled, # pyright: ignore[reportPrivateUsage]
|
|
69
73
|
_next_page_url, # pyright: ignore[reportPrivateUsage]
|
|
70
74
|
_parse_iso, # pyright: ignore[reportPrivateUsage]
|
|
71
75
|
_query_param, # pyright: ignore[reportPrivateUsage]
|
|
72
76
|
_raise_on_401, # pyright: ignore[reportPrivateUsage]
|
|
77
|
+
_record_checks_disabled, # pyright: ignore[reportPrivateUsage]
|
|
73
78
|
_short_sha, # pyright: ignore[reportPrivateUsage]
|
|
74
79
|
)
|
|
75
80
|
|
|
@@ -91,6 +96,11 @@ _MAX_COMPARE_PAGES = 20
|
|
|
91
96
|
# backfill so a very deep repo can't pin the worker indefinitely; the
|
|
92
97
|
# walk logs a truncation warning when it hits the cap.
|
|
93
98
|
_MAX_HISTORY_PAGES = 100
|
|
99
|
+
# CI status is hydrated with one ``/check-runs`` call per commit, which is
|
|
100
|
+
# prohibitively expensive across a full-history backfill. Bound it to the
|
|
101
|
+
# most-recent commits (the only ones whose CI is still meaningful and
|
|
102
|
+
# unexpired); older commits keep the ``'unknown'`` default.
|
|
103
|
+
_BACKFILL_CI_LIMIT = 25
|
|
94
104
|
|
|
95
105
|
# Per-context ceilings on how long a *single* rate-limit pause may block
|
|
96
106
|
# before the sync gives up best-effort (logged). The webhook actions
|
|
@@ -140,6 +150,12 @@ def _throttle_wait(response: httpx.Response) -> float | None:
|
|
|
140
150
|
* a 403/429 that exhausted the quota but gives no reset -> the
|
|
141
151
|
conservative ``_SECONDARY_LIMIT_WAIT_SECONDS`` floor.
|
|
142
152
|
|
|
153
|
+
GHES/GHEC sometimes return a secondary (abuse) limit as a 403/429 with
|
|
154
|
+
*none* of the standard headers -- only the documented body message
|
|
155
|
+
("You have exceeded a secondary rate limit ..."); that phrase is
|
|
156
|
+
matched as a last resort so such a response pauses on the floor rather
|
|
157
|
+
than being mistaken for a hard failure.
|
|
158
|
+
|
|
143
159
|
Returns ``None`` for anything that is *not* a throttle signal --
|
|
144
160
|
including a non-rate-limit 403 (e.g. insufficient scope), which must
|
|
145
161
|
fall through to the caller's normal error handling rather than be
|
|
@@ -159,9 +175,27 @@ def _throttle_wait(response: httpx.Response) -> float | None:
|
|
|
159
175
|
return max(0.0, reset - time.time()) + _RESET_BUFFER_SECONDS
|
|
160
176
|
if response.status_code in (403, 429):
|
|
161
177
|
return _SECONDARY_LIMIT_WAIT_SECONDS
|
|
178
|
+
if response.status_code in (403, 429) and _is_secondary_limit_body(
|
|
179
|
+
response
|
|
180
|
+
):
|
|
181
|
+
return _SECONDARY_LIMIT_WAIT_SECONDS
|
|
162
182
|
return None
|
|
163
183
|
|
|
164
184
|
|
|
185
|
+
def _is_secondary_limit_body(response: httpx.Response) -> bool:
|
|
186
|
+
"""True if a 403/429 body carries GitHub's secondary-limit message.
|
|
187
|
+
|
|
188
|
+
Matched conservatively on the documented phrase so a genuine
|
|
189
|
+
permission 403 (which the caller must surface) is never mistaken for
|
|
190
|
+
a throttle.
|
|
191
|
+
"""
|
|
192
|
+
try:
|
|
193
|
+
body = response.text
|
|
194
|
+
except (UnicodeDecodeError, httpx.ResponseNotRead):
|
|
195
|
+
return False
|
|
196
|
+
return 'secondary rate limit' in body.lower()
|
|
197
|
+
|
|
198
|
+
|
|
165
199
|
async def _request(
|
|
166
200
|
client: httpx.AsyncClient,
|
|
167
201
|
method: str,
|
|
@@ -176,11 +210,13 @@ async def _request(
|
|
|
176
210
|
for the interval GitHub states in its headers, then retries -- up to
|
|
177
211
|
``_MAX_THROTTLE_RETRIES`` times and never longer than *max_wait* on a
|
|
178
212
|
single pause. When the stated wait exceeds *max_wait* (or the retries
|
|
179
|
-
are exhausted)
|
|
180
|
-
|
|
181
|
-
|
|
182
|
-
|
|
183
|
-
|
|
213
|
+
are exhausted) :class:`PluginRateLimited` is raised carrying the epoch
|
|
214
|
+
at which GitHub will resume, so the host can pause the work and keep it
|
|
215
|
+
queued rather than fail it; on the webhook path the caller swallows it
|
|
216
|
+
(a later push re-syncs the gap). After a *successful* response that
|
|
217
|
+
exhausted the remaining quota, pauses pre-emptively (subject to the
|
|
218
|
+
same cap) so the next call doesn't spend a round-trip on a guaranteed
|
|
219
|
+
rejection.
|
|
184
220
|
"""
|
|
185
221
|
attempts = 0
|
|
186
222
|
while True:
|
|
@@ -204,13 +240,19 @@ async def _request(
|
|
|
204
240
|
if wait > max_wait or attempts >= _MAX_THROTTLE_RETRIES:
|
|
205
241
|
LOGGER.warning(
|
|
206
242
|
'github-commit-sync rate-limited on %s (wait %.0fs, cap '
|
|
207
|
-
'%.0fs, attempt %d);
|
|
243
|
+
'%.0fs, attempt %d); pausing job until GitHub resumes',
|
|
208
244
|
url,
|
|
209
245
|
wait,
|
|
210
246
|
max_wait,
|
|
211
247
|
attempts + 1,
|
|
212
248
|
)
|
|
213
|
-
|
|
249
|
+
raise PluginRateLimited(
|
|
250
|
+
retry_at=time.time() + wait,
|
|
251
|
+
message=(
|
|
252
|
+
f'github-commit-sync rate-limited on {url}; '
|
|
253
|
+
f'resume in {wait:.0f}s'
|
|
254
|
+
),
|
|
255
|
+
)
|
|
214
256
|
attempts += 1
|
|
215
257
|
LOGGER.warning(
|
|
216
258
|
'github-commit-sync rate-limited on %s; pausing %.0fs then '
|
|
@@ -493,6 +535,7 @@ def _commit_record(
|
|
|
493
535
|
ref: str,
|
|
494
536
|
pushed_at: datetime.datetime,
|
|
495
537
|
author_user: str = '',
|
|
538
|
+
ci_status: CheckStatus = 'unknown',
|
|
496
539
|
) -> CommitRecord:
|
|
497
540
|
"""Map a GitHub commit object onto a :class:`CommitRecord`.
|
|
498
541
|
|
|
@@ -500,7 +543,8 @@ def _commit_record(
|
|
|
500
543
|
email + linked login + committer), which the UI-facing
|
|
501
544
|
``Commit``/``_commit_from_payload`` mapping drops, so it maps the raw
|
|
502
545
|
item directly while reusing the low-level ``_short_sha`` / ``_parse_iso``
|
|
503
|
-
helpers.
|
|
546
|
+
helpers. ``ci_status`` is supplied by the caller (hydrated from
|
|
547
|
+
``/check-runs``); it defaults to ``'unknown'`` when not resolved.
|
|
504
548
|
"""
|
|
505
549
|
sha = str(item.get('sha') or '')
|
|
506
550
|
commit_meta: dict[str, typing.Any] = item.get('commit') or {}
|
|
@@ -519,6 +563,7 @@ def _commit_record(
|
|
|
519
563
|
author_login=str(gh_author.get('login') or ''),
|
|
520
564
|
author_user=author_user,
|
|
521
565
|
committer_name=str(committer_meta.get('name') or ''),
|
|
566
|
+
ci_status=ci_status,
|
|
522
567
|
authored_at=authored_at or pushed_at,
|
|
523
568
|
committed_at=_parse_iso(committer_meta.get('date')),
|
|
524
569
|
url=str(item.get('html_url') or ''),
|
|
@@ -598,6 +643,92 @@ async def _fetch_recent_commits(
|
|
|
598
643
|
return typing.cast('list[dict[str, typing.Any]]', resp.json())
|
|
599
644
|
|
|
600
645
|
|
|
646
|
+
async def _ci_status(
|
|
647
|
+
client: httpx.AsyncClient,
|
|
648
|
+
sha: str,
|
|
649
|
+
*,
|
|
650
|
+
credentials: dict[str, str],
|
|
651
|
+
base: str,
|
|
652
|
+
owner: str,
|
|
653
|
+
repo: str,
|
|
654
|
+
max_wait: float,
|
|
655
|
+
) -> CheckStatus:
|
|
656
|
+
"""Roll up ``/commits/{sha}/check-runs`` into a ci_status string.
|
|
657
|
+
|
|
658
|
+
Reuses the deployment plugin's ``/check-runs`` mapping and its 403
|
|
659
|
+
"checks disabled" cache so a token without the scope spends one 403
|
|
660
|
+
rather than one per commit. Degrades to ``'unknown'`` on a 403, any
|
|
661
|
+
non-200, a parse error, a network error, or a rate-limit pause -- CI
|
|
662
|
+
status is best-effort metadata and must never fail the sync, mirroring
|
|
663
|
+
the swallow-and-continue contract the rest of this module follows.
|
|
664
|
+
"""
|
|
665
|
+
if _checks_disabled(credentials, base, owner, repo):
|
|
666
|
+
return 'unknown'
|
|
667
|
+
try:
|
|
668
|
+
resp = await _request(
|
|
669
|
+
client, 'GET', f'/commits/{sha}/check-runs', max_wait=max_wait
|
|
670
|
+
)
|
|
671
|
+
if resp.status_code == 403:
|
|
672
|
+
_record_checks_disabled(credentials, base, owner, repo)
|
|
673
|
+
return 'unknown'
|
|
674
|
+
if resp.status_code != 200:
|
|
675
|
+
return 'unknown'
|
|
676
|
+
payload = typing.cast('dict[str, typing.Any]', resp.json())
|
|
677
|
+
except Exception: # noqa: BLE001 - CI status is best-effort metadata
|
|
678
|
+
return 'unknown'
|
|
679
|
+
return _check_runs_to_status(payload)
|
|
680
|
+
|
|
681
|
+
|
|
682
|
+
async def _hydrate_ci(
|
|
683
|
+
client: httpx.AsyncClient,
|
|
684
|
+
shas: list[str],
|
|
685
|
+
*,
|
|
686
|
+
credentials: dict[str, str],
|
|
687
|
+
base: str,
|
|
688
|
+
owner: str,
|
|
689
|
+
repo: str,
|
|
690
|
+
max_wait: float,
|
|
691
|
+
) -> dict[str, CheckStatus]:
|
|
692
|
+
"""Resolve ``ci_status`` for ``shas`` -> ``{sha: status}``.
|
|
693
|
+
|
|
694
|
+
Probes the head commit first so a 403 (missing scope / Actions
|
|
695
|
+
disabled) populates the cache and short-circuits the rest, mirroring
|
|
696
|
+
the deployment plugin's commit picker. Shas absent from the returned
|
|
697
|
+
map fall back to ``'unknown'`` at the call site.
|
|
698
|
+
"""
|
|
699
|
+
out: dict[str, CheckStatus] = {}
|
|
700
|
+
if not shas or _checks_disabled(credentials, base, owner, repo):
|
|
701
|
+
return out
|
|
702
|
+
head, *tail = shas
|
|
703
|
+
out[head] = await _ci_status(
|
|
704
|
+
client,
|
|
705
|
+
head,
|
|
706
|
+
credentials=credentials,
|
|
707
|
+
base=base,
|
|
708
|
+
owner=owner,
|
|
709
|
+
repo=repo,
|
|
710
|
+
max_wait=max_wait,
|
|
711
|
+
)
|
|
712
|
+
if not tail or _checks_disabled(credentials, base, owner, repo):
|
|
713
|
+
return out
|
|
714
|
+
results = await asyncio.gather(
|
|
715
|
+
*(
|
|
716
|
+
_ci_status(
|
|
717
|
+
client,
|
|
718
|
+
sha,
|
|
719
|
+
credentials=credentials,
|
|
720
|
+
base=base,
|
|
721
|
+
owner=owner,
|
|
722
|
+
repo=repo,
|
|
723
|
+
max_wait=max_wait,
|
|
724
|
+
)
|
|
725
|
+
for sha in tail
|
|
726
|
+
)
|
|
727
|
+
)
|
|
728
|
+
out.update(dict(zip(tail, results, strict=True)))
|
|
729
|
+
return out
|
|
730
|
+
|
|
731
|
+
|
|
601
732
|
def _resolve_host_for_context(ctx: PluginContext) -> str | None:
|
|
602
733
|
"""Resolve the GitHub web host for an on-demand sync (no payload).
|
|
603
734
|
|
|
@@ -772,24 +903,43 @@ async def sync_commits(
|
|
|
772
903
|
before = _resolve(action_config.before_selector, event)
|
|
773
904
|
pushed_at = datetime.datetime.now(datetime.UTC)
|
|
774
905
|
token = await _resolve_bearer(credentials, base, owner, repo)
|
|
775
|
-
|
|
776
|
-
|
|
777
|
-
|
|
778
|
-
|
|
779
|
-
)
|
|
780
|
-
else:
|
|
781
|
-
last = await _last_known_sha(ctx.project_id)
|
|
782
|
-
if last:
|
|
906
|
+
ci_by_sha: dict[str, CheckStatus] = {}
|
|
907
|
+
try:
|
|
908
|
+
async with _client(base, owner, repo, token) as client:
|
|
909
|
+
if isinstance(before, str) and before and before != _ZERO_SHA:
|
|
783
910
|
raw = await _fetch_compare_commits(
|
|
784
|
-
client,
|
|
911
|
+
client, before, after, max_wait=_WEBHOOK_MAX_WAIT_SECONDS
|
|
785
912
|
)
|
|
786
913
|
else:
|
|
787
|
-
|
|
788
|
-
|
|
789
|
-
|
|
790
|
-
|
|
791
|
-
|
|
792
|
-
|
|
914
|
+
last = await _last_known_sha(ctx.project_id)
|
|
915
|
+
if last:
|
|
916
|
+
raw = await _fetch_compare_commits(
|
|
917
|
+
client, last, after, max_wait=_WEBHOOK_MAX_WAIT_SECONDS
|
|
918
|
+
)
|
|
919
|
+
else:
|
|
920
|
+
raw = await _fetch_recent_commits(
|
|
921
|
+
client,
|
|
922
|
+
after,
|
|
923
|
+
action_config.initial_limit,
|
|
924
|
+
max_wait=_WEBHOOK_MAX_WAIT_SECONDS,
|
|
925
|
+
)
|
|
926
|
+
ci_by_sha = await _hydrate_ci(
|
|
927
|
+
client,
|
|
928
|
+
[str(i['sha']) for i in raw if i.get('sha')],
|
|
929
|
+
credentials=credentials,
|
|
930
|
+
base=base,
|
|
931
|
+
owner=owner,
|
|
932
|
+
repo=repo,
|
|
933
|
+
max_wait=_WEBHOOK_MAX_WAIT_SECONDS,
|
|
934
|
+
)
|
|
935
|
+
except PluginRateLimited as exc:
|
|
936
|
+
LOGGER.warning(
|
|
937
|
+
'github-commit-sync: rate-limited syncing commits for project '
|
|
938
|
+
'%s; skipping this push (a later push re-syncs the gap): %s',
|
|
939
|
+
ctx.project_id,
|
|
940
|
+
exc,
|
|
941
|
+
)
|
|
942
|
+
return
|
|
793
943
|
user_map = await _resolve_author_users(
|
|
794
944
|
raw, ctx.resolve_user_by_identity, base
|
|
795
945
|
)
|
|
@@ -800,6 +950,7 @@ async def sync_commits(
|
|
|
800
950
|
ref=ref,
|
|
801
951
|
pushed_at=pushed_at,
|
|
802
952
|
author_user=_author_user(item, user_map),
|
|
953
|
+
ci_status=ci_by_sha.get(str(item['sha']), 'unknown'),
|
|
803
954
|
)
|
|
804
955
|
for item in raw
|
|
805
956
|
if item.get('sha')
|
|
@@ -826,6 +977,73 @@ async def _annotated_tag(
|
|
|
826
977
|
return typing.cast('dict[str, typing.Any]', resp.json())
|
|
827
978
|
|
|
828
979
|
|
|
980
|
+
async def _commit_date(
|
|
981
|
+
client: httpx.AsyncClient, sha: str, *, max_wait: float
|
|
982
|
+
) -> datetime.datetime | None:
|
|
983
|
+
"""Committer date of commit *sha*, ``None`` when unavailable.
|
|
984
|
+
|
|
985
|
+
Lightweight tags carry no tagger date, so their target commit's
|
|
986
|
+
date stands in for ``tagged_at`` — otherwise the ClickHouse row
|
|
987
|
+
falls back to ``recorded_at`` (sync time) and every release in the
|
|
988
|
+
UI reads "just now" after a full reconcile.
|
|
989
|
+
"""
|
|
990
|
+
resp = await _request(
|
|
991
|
+
client, 'GET', f'/git/commits/{sha}', max_wait=max_wait
|
|
992
|
+
)
|
|
993
|
+
if resp.status_code != 200:
|
|
994
|
+
return None
|
|
995
|
+
data = typing.cast('dict[str, typing.Any]', resp.json())
|
|
996
|
+
committer: dict[str, typing.Any] = data.get('committer') or {}
|
|
997
|
+
return _parse_iso(committer.get('date'))
|
|
998
|
+
|
|
999
|
+
|
|
1000
|
+
async def _release_published_for_tag(
|
|
1001
|
+
client: httpx.AsyncClient, name: str, *, max_wait: float
|
|
1002
|
+
) -> datetime.datetime | None:
|
|
1003
|
+
"""Published date of the GitHub release for tag *name*, if any."""
|
|
1004
|
+
resp = await _request(
|
|
1005
|
+
client,
|
|
1006
|
+
'GET',
|
|
1007
|
+
f'/releases/tags/{urllib.parse.quote(name, safe="")}',
|
|
1008
|
+
max_wait=max_wait,
|
|
1009
|
+
)
|
|
1010
|
+
if resp.status_code != 200:
|
|
1011
|
+
return None
|
|
1012
|
+
data = typing.cast('dict[str, typing.Any]', resp.json())
|
|
1013
|
+
return _parse_iso(data.get('published_at'))
|
|
1014
|
+
|
|
1015
|
+
|
|
1016
|
+
async def _release_published_map(
|
|
1017
|
+
client: httpx.AsyncClient, *, max_wait: float
|
|
1018
|
+
) -> dict[str, datetime.datetime]:
|
|
1019
|
+
"""Map tag name -> GitHub release published date for the repo.
|
|
1020
|
+
|
|
1021
|
+
One paginated ``/releases`` sweep instead of a per-tag lookup;
|
|
1022
|
+
drafts (``published_at`` null) are skipped.
|
|
1023
|
+
"""
|
|
1024
|
+
out: dict[str, datetime.datetime] = {}
|
|
1025
|
+
params: dict[str, str] = {'per_page': '100'}
|
|
1026
|
+
while True:
|
|
1027
|
+
resp = await _request(
|
|
1028
|
+
client, 'GET', '/releases', params=params, max_wait=max_wait
|
|
1029
|
+
)
|
|
1030
|
+
if resp.status_code != 200:
|
|
1031
|
+
return out
|
|
1032
|
+
rows = typing.cast('list[dict[str, typing.Any]]', resp.json())
|
|
1033
|
+
for row in rows:
|
|
1034
|
+
tag = row.get('tag_name')
|
|
1035
|
+
published = _parse_iso(row.get('published_at'))
|
|
1036
|
+
if tag and published:
|
|
1037
|
+
out[str(tag)] = published
|
|
1038
|
+
next_url = _next_page_url(resp.headers.get('link'))
|
|
1039
|
+
if next_url is None:
|
|
1040
|
+
return out
|
|
1041
|
+
next_page = _query_param(next_url, 'page')
|
|
1042
|
+
if next_page is None:
|
|
1043
|
+
return out
|
|
1044
|
+
params['page'] = next_page
|
|
1045
|
+
|
|
1046
|
+
|
|
829
1047
|
def _web_base(api_base: str) -> str:
|
|
830
1048
|
"""Map a REST API base to the web host (inverse of the routing table).
|
|
831
1049
|
|
|
@@ -857,9 +1075,24 @@ def _tag_record(
|
|
|
857
1075
|
sha: str,
|
|
858
1076
|
annotated: dict[str, typing.Any] | None = None,
|
|
859
1077
|
url: str = '',
|
|
1078
|
+
published_at: datetime.datetime | None = None,
|
|
1079
|
+
fallback_tagged_at: datetime.datetime | None = None,
|
|
860
1080
|
) -> TagRecord:
|
|
1081
|
+
"""Build the ClickHouse row for one tag.
|
|
1082
|
+
|
|
1083
|
+
``tagged_at`` preference: the GitHub release's published date
|
|
1084
|
+
(*published_at*), then the annotated tag's tagger date, then the
|
|
1085
|
+
target commit's committer date (*fallback_tagged_at*, lightweight
|
|
1086
|
+
tags only).
|
|
1087
|
+
"""
|
|
861
1088
|
if annotated is None:
|
|
862
|
-
return TagRecord(
|
|
1089
|
+
return TagRecord(
|
|
1090
|
+
project_id=project_id,
|
|
1091
|
+
name=name,
|
|
1092
|
+
sha=sha,
|
|
1093
|
+
url=url,
|
|
1094
|
+
tagged_at=published_at or fallback_tagged_at,
|
|
1095
|
+
)
|
|
863
1096
|
tagger: dict[str, typing.Any] = annotated.get('tagger') or {}
|
|
864
1097
|
return TagRecord(
|
|
865
1098
|
project_id=project_id,
|
|
@@ -869,7 +1102,7 @@ def _tag_record(
|
|
|
869
1102
|
message=str(annotated.get('message') or ''),
|
|
870
1103
|
tagger_name=str(tagger.get('name') or ''),
|
|
871
1104
|
tagger_email=str(tagger.get('email') or ''),
|
|
872
|
-
tagged_at=_parse_iso(tagger.get('date')),
|
|
1105
|
+
tagged_at=published_at or _parse_iso(tagger.get('date')),
|
|
873
1106
|
)
|
|
874
1107
|
|
|
875
1108
|
|
|
@@ -886,6 +1119,8 @@ async def _reconcile_tags(
|
|
|
886
1119
|
out: list[TagRecord] = []
|
|
887
1120
|
prefix = 'refs/tags/'
|
|
888
1121
|
params: dict[str, str] = {'per_page': '100'}
|
|
1122
|
+
# Lazily fetched on the first tag so tag-less repos cost nothing.
|
|
1123
|
+
released: dict[str, datetime.datetime] | None = None
|
|
889
1124
|
while True:
|
|
890
1125
|
resp = await _request(
|
|
891
1126
|
client,
|
|
@@ -903,6 +1138,11 @@ async def _reconcile_tags(
|
|
|
903
1138
|
if not ref.startswith(prefix) or not sha:
|
|
904
1139
|
continue
|
|
905
1140
|
name = ref[len(prefix) :]
|
|
1141
|
+
if released is None:
|
|
1142
|
+
released = await _release_published_map(
|
|
1143
|
+
client, max_wait=max_wait
|
|
1144
|
+
)
|
|
1145
|
+
published = released.get(name)
|
|
906
1146
|
annotated = (
|
|
907
1147
|
await _annotated_tag(client, sha, max_wait=max_wait)
|
|
908
1148
|
if obj.get('type') == 'tag'
|
|
@@ -915,6 +1155,12 @@ async def _reconcile_tags(
|
|
|
915
1155
|
sha=sha,
|
|
916
1156
|
annotated=annotated,
|
|
917
1157
|
url=_tag_web_url(client, name),
|
|
1158
|
+
published_at=published,
|
|
1159
|
+
fallback_tagged_at=(
|
|
1160
|
+
await _commit_date(client, sha, max_wait=max_wait)
|
|
1161
|
+
if annotated is None and published is None
|
|
1162
|
+
else None
|
|
1163
|
+
),
|
|
918
1164
|
)
|
|
919
1165
|
)
|
|
920
1166
|
next_url = _next_page_url(resp.headers.get('link'))
|
|
@@ -954,25 +1200,45 @@ async def sync_tags(
|
|
|
954
1200
|
return
|
|
955
1201
|
owner, repo, base = resolved
|
|
956
1202
|
token = await _resolve_bearer(credentials, base, owner, repo)
|
|
957
|
-
|
|
958
|
-
|
|
959
|
-
|
|
960
|
-
|
|
961
|
-
records: list[pydantic.BaseModel] = [
|
|
962
|
-
_tag_record(
|
|
963
|
-
project_id=ctx.project_id,
|
|
964
|
-
name=name,
|
|
965
|
-
sha=after,
|
|
966
|
-
annotated=annotated,
|
|
967
|
-
url=_tag_web_url(client, name),
|
|
1203
|
+
try:
|
|
1204
|
+
async with _client(base, owner, repo, token) as client:
|
|
1205
|
+
annotated = await _annotated_tag(
|
|
1206
|
+
client, after, max_wait=_WEBHOOK_MAX_WAIT_SECONDS
|
|
968
1207
|
)
|
|
969
|
-
|
|
970
|
-
|
|
971
|
-
extra = await _reconcile_tags(
|
|
972
|
-
client, ctx.project_id, max_wait=_WEBHOOK_MAX_WAIT_SECONDS
|
|
1208
|
+
published = await _release_published_for_tag(
|
|
1209
|
+
client, name, max_wait=_WEBHOOK_MAX_WAIT_SECONDS
|
|
973
1210
|
)
|
|
974
|
-
|
|
975
|
-
|
|
1211
|
+
records: list[pydantic.BaseModel] = [
|
|
1212
|
+
_tag_record(
|
|
1213
|
+
project_id=ctx.project_id,
|
|
1214
|
+
name=name,
|
|
1215
|
+
sha=after,
|
|
1216
|
+
annotated=annotated,
|
|
1217
|
+
url=_tag_web_url(client, name),
|
|
1218
|
+
published_at=published,
|
|
1219
|
+
fallback_tagged_at=(
|
|
1220
|
+
await _commit_date(
|
|
1221
|
+
client, after, max_wait=_WEBHOOK_MAX_WAIT_SECONDS
|
|
1222
|
+
)
|
|
1223
|
+
if annotated is None and published is None
|
|
1224
|
+
else None
|
|
1225
|
+
),
|
|
1226
|
+
)
|
|
1227
|
+
]
|
|
1228
|
+
if action_config.reconcile_all:
|
|
1229
|
+
extra = await _reconcile_tags(
|
|
1230
|
+
client, ctx.project_id, max_wait=_WEBHOOK_MAX_WAIT_SECONDS
|
|
1231
|
+
)
|
|
1232
|
+
seen = {name}
|
|
1233
|
+
records.extend(r for r in extra if r.name not in seen)
|
|
1234
|
+
except PluginRateLimited as exc:
|
|
1235
|
+
LOGGER.warning(
|
|
1236
|
+
'github-commit-sync: rate-limited syncing tags for project '
|
|
1237
|
+
'%s; skipping this push (a later push re-syncs the gap): %s',
|
|
1238
|
+
ctx.project_id,
|
|
1239
|
+
exc,
|
|
1240
|
+
)
|
|
1241
|
+
return
|
|
976
1242
|
try:
|
|
977
1243
|
await clickhouse.insert('tags', records)
|
|
978
1244
|
except Exception:
|
|
@@ -1102,7 +1368,10 @@ class GitHubCommitSyncPlugin(WebhookActionPlugin):
|
|
|
1102
1368
|
Returns ``(commits_recorded, tags_recorded)``. Raises
|
|
1103
1369
|
:class:`ValueError` only when the host or repository can't be
|
|
1104
1370
|
resolved; ClickHouse failures are swallowed (the count reflects
|
|
1105
|
-
what was written).
|
|
1371
|
+
what was written). Propagates :class:`PluginRateLimited` when a
|
|
1372
|
+
GitHub rate-limit reset is further out than
|
|
1373
|
+
``_BACKFILL_MAX_WAIT_SECONDS`` so the host can pause the worker and
|
|
1374
|
+
keep the job queued until GitHub resumes rather than fail it.
|
|
1106
1375
|
"""
|
|
1107
1376
|
host = _resolve_host_for_context(ctx)
|
|
1108
1377
|
if host is None:
|
|
@@ -1124,6 +1393,22 @@ class GitHubCommitSyncPlugin(WebhookActionPlugin):
|
|
|
1124
1393
|
tags = await _reconcile_tags(
|
|
1125
1394
|
client, ctx.project_id, max_wait=_BACKFILL_MAX_WAIT_SECONDS
|
|
1126
1395
|
)
|
|
1396
|
+
# CI status only for the most-recent commits: one /check-runs
|
|
1397
|
+
# call per commit is too costly across full history, and old
|
|
1398
|
+
# commits' CI is no longer meaningful.
|
|
1399
|
+
ci_by_sha = await _hydrate_ci(
|
|
1400
|
+
client,
|
|
1401
|
+
[
|
|
1402
|
+
str(i['sha'])
|
|
1403
|
+
for i in raw_commits[:_BACKFILL_CI_LIMIT]
|
|
1404
|
+
if i.get('sha')
|
|
1405
|
+
],
|
|
1406
|
+
credentials=credentials,
|
|
1407
|
+
base=base,
|
|
1408
|
+
owner=owner,
|
|
1409
|
+
repo=repo,
|
|
1410
|
+
max_wait=_BACKFILL_MAX_WAIT_SECONDS,
|
|
1411
|
+
)
|
|
1127
1412
|
user_map = await _resolve_author_users(
|
|
1128
1413
|
raw_commits, ctx.resolve_user_by_identity, base
|
|
1129
1414
|
)
|
|
@@ -1134,6 +1419,7 @@ class GitHubCommitSyncPlugin(WebhookActionPlugin):
|
|
|
1134
1419
|
ref=branch,
|
|
1135
1420
|
pushed_at=pushed_at,
|
|
1136
1421
|
author_user=_author_user(item, user_map),
|
|
1422
|
+
ci_status=ci_by_sha.get(str(item['sha']), 'unknown'),
|
|
1137
1423
|
)
|
|
1138
1424
|
for item in raw_commits
|
|
1139
1425
|
if item.get('sha')
|
imbi_plugin_github/deployment.py
CHANGED
|
@@ -1035,12 +1035,18 @@ class _DeploymentBase(DeploymentPlugin):
|
|
|
1035
1035
|
description = deployment.get('description')
|
|
1036
1036
|
deployment_url = deployment.get('url') or deployment.get('html_url')
|
|
1037
1037
|
creator_login: str | None = None
|
|
1038
|
+
creator_subject: str | None = None
|
|
1038
1039
|
creator_raw = deployment.get('creator')
|
|
1039
1040
|
if isinstance(creator_raw, dict):
|
|
1040
1041
|
creator_dict = typing.cast(dict[str, typing.Any], creator_raw)
|
|
1041
1042
|
login = creator_dict.get('login')
|
|
1042
1043
|
if isinstance(login, str) and login:
|
|
1043
1044
|
creator_login = login
|
|
1045
|
+
# GitHub's numeric user id is the stable identity subject the
|
|
1046
|
+
# host resolves to an Imbi user (logins can be renamed).
|
|
1047
|
+
creator_id = creator_dict.get('id')
|
|
1048
|
+
if isinstance(creator_id, int):
|
|
1049
|
+
creator_subject = str(creator_id)
|
|
1044
1050
|
return RemoteDeployment(
|
|
1045
1051
|
environment=environment,
|
|
1046
1052
|
sha=str(sha),
|
|
@@ -1052,6 +1058,7 @@ class _DeploymentBase(DeploymentPlugin):
|
|
|
1052
1058
|
deployment_url=str(deployment_url) if deployment_url else None,
|
|
1053
1059
|
description=str(description) if description else None,
|
|
1054
1060
|
creator=creator_login,
|
|
1061
|
+
creator_subject=creator_subject,
|
|
1055
1062
|
)
|
|
1056
1063
|
|
|
1057
1064
|
async def _latest_status(
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: imbi-plugin-github
|
|
3
|
-
Version: 2.
|
|
3
|
+
Version: 2.11.2
|
|
4
4
|
Summary: GitHub identity plugin for Imbi (github.com / GHEC / GHES)
|
|
5
5
|
Author-email: "Gavin M. Roy" <gavinr@aweber.com>
|
|
6
6
|
License: BSD-3-Clause
|
|
@@ -13,7 +13,7 @@ Classifier: Programming Language :: Python :: 3
|
|
|
13
13
|
Classifier: Programming Language :: Python :: 3.14
|
|
14
14
|
Requires-Python: >=3.14
|
|
15
15
|
Requires-Dist: httpx>=0.27
|
|
16
|
-
Requires-Dist: imbi-common[databases]==2.
|
|
16
|
+
Requires-Dist: imbi-common[databases]==2.11.2
|
|
17
17
|
Requires-Dist: pydantic>=2
|
|
18
18
|
Requires-Dist: pyjwt[crypto]>=2.8
|
|
19
19
|
Description-Content-Type: text/markdown
|
|
@@ -2,12 +2,12 @@ imbi_plugin_github/__init__.py,sha256=8ZVhm8XpGqgYRUtpHl0uuZTT3gmMyHNhKd9k6qoatB
|
|
|
2
2
|
imbi_plugin_github/_app_auth.py,sha256=LnLrvqmu2UC27uHHJfIzFPNDh33RtgeOX1RXceX9Pj0,7000
|
|
3
3
|
imbi_plugin_github/_hosts.py,sha256=jQ1Z6yEpjAa4b7LocLfEelnBcRYYQLNLBNCdWZtAXUc,2258
|
|
4
4
|
imbi_plugin_github/_repos.py,sha256=agp4ZgRuqbrGBRMFeHomnUf1HvvaNvapBCWHQebdviQ,4559
|
|
5
|
-
imbi_plugin_github/commits.py,sha256=
|
|
6
|
-
imbi_plugin_github/deployment.py,sha256=
|
|
5
|
+
imbi_plugin_github/commits.py,sha256=Pp4-LHnNpDrhsDbJSmQQO7qLgWiaBpOBPQJKTkTcn5w,53014
|
|
6
|
+
imbi_plugin_github/deployment.py,sha256=d82G7zGcN-A_1J19tUZuzVzK3AOfvIbowtGJ6zLEeck,48287
|
|
7
7
|
imbi_plugin_github/identity.py,sha256=RSs1cLQpZmfyQ_0KQkohDIZEBYVBnre6RJIt166EtdQ,15525
|
|
8
8
|
imbi_plugin_github/lifecycle.py,sha256=48mLKjFPG7BOanpA7jWnmbWOdsDLh1F39g2AoxDLL7M,33211
|
|
9
|
-
imbi_plugin_github-2.
|
|
10
|
-
imbi_plugin_github-2.
|
|
11
|
-
imbi_plugin_github-2.
|
|
12
|
-
imbi_plugin_github-2.
|
|
13
|
-
imbi_plugin_github-2.
|
|
9
|
+
imbi_plugin_github-2.11.2.dist-info/METADATA,sha256=Z15EFgty_LerbZ2KfYsmAMXJKyWcM64pKMkO9igS71c,5811
|
|
10
|
+
imbi_plugin_github-2.11.2.dist-info/WHEEL,sha256=mffPy8wBnZQn2VnJUU5jE99KsxaSfiyMHV9Yt0aLVxs,87
|
|
11
|
+
imbi_plugin_github-2.11.2.dist-info/entry_points.txt,sha256=RavN_FYCeS97iauHgCG3tHFdVULbEMXmQfvgCgLeGkI,805
|
|
12
|
+
imbi_plugin_github-2.11.2.dist-info/licenses/LICENSE,sha256=el9B20zOqEcv_mRrHgbdkVyxSC6I8np7am0D6mO2A9Y,1491
|
|
13
|
+
imbi_plugin_github-2.11.2.dist-info/RECORD,,
|
{imbi_plugin_github-2.10.0.dist-info → imbi_plugin_github-2.11.2.dist-info}/entry_points.txt
RENAMED
|
File without changes
|
{imbi_plugin_github-2.10.0.dist-info → imbi_plugin_github-2.11.2.dist-info}/licenses/LICENSE
RENAMED
|
File without changes
|