lupaxa-github-intel 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- lupaxa/github_intel/__init__.py +13 -0
- lupaxa/github_intel/__main__.py +8 -0
- lupaxa/github_intel/app.py +493 -0
- lupaxa/github_intel/checks/__init__.py +1 -0
- lupaxa/github_intel/checks/base.py +17 -0
- lupaxa/github_intel/checks/releases.py +59 -0
- lupaxa/github_intel/checks/workflows.py +34 -0
- lupaxa/github_intel/cli.py +699 -0
- lupaxa/github_intel/config/__init__.py +1 -0
- lupaxa/github_intel/config/defaults.py +150 -0
- lupaxa/github_intel/config/editor.py +316 -0
- lupaxa/github_intel/config/loader.py +412 -0
- lupaxa/github_intel/config/models.py +310 -0
- lupaxa/github_intel/config/paths.py +84 -0
- lupaxa/github_intel/config/profiles.py +143 -0
- lupaxa/github_intel/constants.py +54 -0
- lupaxa/github_intel/dates.py +179 -0
- lupaxa/github_intel/errors.py +72 -0
- lupaxa/github_intel/github/__init__.py +1 -0
- lupaxa/github_intel/github/auth.py +40 -0
- lupaxa/github_intel/github/client.py +239 -0
- lupaxa/github_intel/github/rate_limit.py +107 -0
- lupaxa/github_intel/layout.py +141 -0
- lupaxa/github_intel/logging_setup.py +51 -0
- lupaxa/github_intel/present.py +361 -0
- lupaxa/github_intel/reports/__init__.py +1 -0
- lupaxa/github_intel/reports/render.py +113 -0
- lupaxa/github_intel/storage/__init__.py +1 -0
- lupaxa/github_intel/storage/database.py +419 -0
- lupaxa/github_intel/sync/__init__.py +1 -0
- lupaxa/github_intel/sync/engine.py +671 -0
- lupaxa/github_intel/sync/summary.py +308 -0
- lupaxa/github_intel/tui/__init__.py +1 -0
- lupaxa/github_intel/tui/application.py +487 -0
- lupaxa/github_intel/views.py +896 -0
- lupaxa_github_intel-0.1.0.dist-info/METADATA +180 -0
- lupaxa_github_intel-0.1.0.dist-info/RECORD +41 -0
- lupaxa_github_intel-0.1.0.dist-info/WHEEL +5 -0
- lupaxa_github_intel-0.1.0.dist-info/entry_points.txt +2 -0
- lupaxa_github_intel-0.1.0.dist-info/licenses/LICENCE +21 -0
- lupaxa_github_intel-0.1.0.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,179 @@
|
|
|
1
|
+
"""Date patterns used by tables and reports."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from datetime import UTC, datetime
|
|
6
|
+
import re
|
|
7
|
+
|
|
8
|
+
__all__ = [
|
|
9
|
+
"DEFAULT_DATE_FORMAT",
|
|
10
|
+
"format_date",
|
|
11
|
+
"format_timestamp",
|
|
12
|
+
"normalize_instant",
|
|
13
|
+
"require_date_format",
|
|
14
|
+
]
|
|
15
|
+
|
|
16
|
+
DEFAULT_DATE_FORMAT = "DD/MM/YYYY HH:mm UTC"
|
|
17
|
+
_MESSAGE = (
|
|
18
|
+
"display.date_format must contain YYYY, MM, and DD once each. "
|
|
19
|
+
"HH and mm are optional and must appear together. "
|
|
20
|
+
"ss and UTC are optional with the time."
|
|
21
|
+
)
|
|
22
|
+
_TOKEN = re.compile(r"YYYY|MM|DD|HH|SS|UTC", re.IGNORECASE)
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
def require_date_format(value: str) -> str:
|
|
26
|
+
"""
|
|
27
|
+
Accept a date and time pattern.
|
|
28
|
+
|
|
29
|
+
Month is the first ``MM``. Leave out ``HH`` and ``mm`` for a date with no
|
|
30
|
+
clock. When they are present, minute is the second ``MM``. ``ss`` and ``UTC``
|
|
31
|
+
may appear once with the clock.
|
|
32
|
+
|
|
33
|
+
Args:
|
|
34
|
+
value: Configured pattern, such as ``DD/MM/YYYY HH:mm UTC``.
|
|
35
|
+
|
|
36
|
+
Returns:
|
|
37
|
+
The pattern with date and hour tokens in upper case and ``mm`` lower.
|
|
38
|
+
|
|
39
|
+
Raises:
|
|
40
|
+
ValueError: A required token is missing or repeated, or other letters remain.
|
|
41
|
+
|
|
42
|
+
"""
|
|
43
|
+
|
|
44
|
+
text = value.strip()
|
|
45
|
+
pieces: list[str] = []
|
|
46
|
+
cursor = 0
|
|
47
|
+
counts = {"YYYY": 0, "MM": 0, "DD": 0, "HH": 0, "mm": 0, "ss": 0, "UTC": 0}
|
|
48
|
+
month_seen = False
|
|
49
|
+
for match in _TOKEN.finditer(text):
|
|
50
|
+
gap = text[cursor : match.start()]
|
|
51
|
+
if any(character.isalnum() for character in gap):
|
|
52
|
+
raise ValueError(_MESSAGE)
|
|
53
|
+
pieces.append(gap)
|
|
54
|
+
raw = match.group(0).upper()
|
|
55
|
+
if raw == "MM":
|
|
56
|
+
name = "MM" if not month_seen else "mm"
|
|
57
|
+
month_seen = True
|
|
58
|
+
elif raw == "SS":
|
|
59
|
+
name = "ss"
|
|
60
|
+
else:
|
|
61
|
+
name = raw
|
|
62
|
+
counts[name] += 1
|
|
63
|
+
pieces.append(name)
|
|
64
|
+
cursor = match.end()
|
|
65
|
+
tail = text[cursor:]
|
|
66
|
+
if any(character.isalnum() for character in tail):
|
|
67
|
+
raise ValueError(_MESSAGE)
|
|
68
|
+
pieces.append(tail)
|
|
69
|
+
if counts["YYYY"] != 1 or counts["MM"] != 1 or counts["DD"] != 1:
|
|
70
|
+
raise ValueError(_MESSAGE)
|
|
71
|
+
clock = counts["HH"] + counts["mm"]
|
|
72
|
+
if clock not in {0, 2} or counts["HH"] > 1 or counts["mm"] > 1:
|
|
73
|
+
raise ValueError(_MESSAGE)
|
|
74
|
+
if counts["ss"] > 1 or counts["UTC"] > 1:
|
|
75
|
+
raise ValueError(_MESSAGE)
|
|
76
|
+
if clock == 0 and (counts["ss"] or counts["UTC"]):
|
|
77
|
+
raise ValueError(_MESSAGE)
|
|
78
|
+
return "".join(pieces)
|
|
79
|
+
|
|
80
|
+
|
|
81
|
+
def normalize_instant(value: object) -> str:
|
|
82
|
+
"""
|
|
83
|
+
Keep a GitHub timestamp for later formatting.
|
|
84
|
+
|
|
85
|
+
Args:
|
|
86
|
+
value: ISO text from GitHub, or anything else.
|
|
87
|
+
|
|
88
|
+
Returns:
|
|
89
|
+
``YYYY-MM-DDTHH:MM:SSZ`` when a time is present, ``YYYY-MM-DD`` when only
|
|
90
|
+
a calendar day is present, or an empty string.
|
|
91
|
+
|
|
92
|
+
"""
|
|
93
|
+
|
|
94
|
+
if not isinstance(value, str):
|
|
95
|
+
return ""
|
|
96
|
+
parsed = _parse_instant(value.strip())
|
|
97
|
+
if parsed is None:
|
|
98
|
+
return ""
|
|
99
|
+
moment, has_time = parsed
|
|
100
|
+
if not has_time:
|
|
101
|
+
return moment.strftime("%Y-%m-%d")
|
|
102
|
+
return moment.strftime("%Y-%m-%dT%H:%M:%SZ")
|
|
103
|
+
|
|
104
|
+
|
|
105
|
+
def format_date(value: str, pattern: str) -> str:
|
|
106
|
+
"""
|
|
107
|
+
Format an ISO instant with the configured pattern.
|
|
108
|
+
|
|
109
|
+
Args:
|
|
110
|
+
value: ``YYYY-MM-DD`` or ``YYYY-MM-DDTHH:MM:SSZ`` text, or empty.
|
|
111
|
+
pattern: Pattern accepted by :func:`require_date_format`.
|
|
112
|
+
|
|
113
|
+
Returns:
|
|
114
|
+
The formatted date and time. Missing clock fields become ``--``. An
|
|
115
|
+
empty ``value`` stays empty.
|
|
116
|
+
|
|
117
|
+
"""
|
|
118
|
+
|
|
119
|
+
if value == "":
|
|
120
|
+
return ""
|
|
121
|
+
parsed = _parse_instant(value)
|
|
122
|
+
if parsed is None:
|
|
123
|
+
return value
|
|
124
|
+
moment, has_time = parsed
|
|
125
|
+
hour = moment.strftime("%H") if has_time else "--"
|
|
126
|
+
minute = moment.strftime("%M") if has_time else "--"
|
|
127
|
+
second = moment.strftime("%S") if has_time else "--"
|
|
128
|
+
return (
|
|
129
|
+
pattern.replace("YYYY", moment.strftime("%Y"))
|
|
130
|
+
.replace("MM", moment.strftime("%m"))
|
|
131
|
+
.replace("DD", moment.strftime("%d"))
|
|
132
|
+
.replace("HH", hour)
|
|
133
|
+
.replace("mm", minute)
|
|
134
|
+
.replace("ss", second)
|
|
135
|
+
.replace("UTC", "UTC")
|
|
136
|
+
)
|
|
137
|
+
|
|
138
|
+
|
|
139
|
+
def format_timestamp(value: float | None, pattern: str) -> str:
|
|
140
|
+
"""
|
|
141
|
+
Format a UTC instant using the configured pattern.
|
|
142
|
+
|
|
143
|
+
Args:
|
|
144
|
+
value: Unix timestamp, or None.
|
|
145
|
+
pattern: Pattern accepted by :func:`require_date_format`.
|
|
146
|
+
|
|
147
|
+
Returns:
|
|
148
|
+
Text shaped by ``pattern``, or an empty string.
|
|
149
|
+
|
|
150
|
+
"""
|
|
151
|
+
|
|
152
|
+
if value is None:
|
|
153
|
+
return ""
|
|
154
|
+
moment = datetime.fromtimestamp(value, UTC)
|
|
155
|
+
return format_date(moment.strftime("%Y-%m-%dT%H:%M:%SZ"), pattern)
|
|
156
|
+
|
|
157
|
+
|
|
158
|
+
def _parse_instant(value: str) -> tuple[datetime, bool] | None:
|
|
159
|
+
if len(value) < 10 or value[4] != "-" or value[7] != "-":
|
|
160
|
+
return None
|
|
161
|
+
if len(value) == 10:
|
|
162
|
+
try:
|
|
163
|
+
moment = datetime(
|
|
164
|
+
int(value[0:4]),
|
|
165
|
+
int(value[5:7]),
|
|
166
|
+
int(value[8:10]),
|
|
167
|
+
tzinfo=UTC,
|
|
168
|
+
)
|
|
169
|
+
except ValueError:
|
|
170
|
+
return None
|
|
171
|
+
return moment, False
|
|
172
|
+
text = value[:-1] + "+00:00" if value.endswith("Z") else value
|
|
173
|
+
try:
|
|
174
|
+
moment = datetime.fromisoformat(text)
|
|
175
|
+
except ValueError:
|
|
176
|
+
return None
|
|
177
|
+
if moment.tzinfo is None:
|
|
178
|
+
moment = moment.replace(tzinfo=UTC)
|
|
179
|
+
return moment.astimezone(UTC), True
|
|
@@ -0,0 +1,72 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Application errors and the exit codes they map to.
|
|
3
|
+
"""
|
|
4
|
+
|
|
5
|
+
from __future__ import annotations
|
|
6
|
+
|
|
7
|
+
from lupaxa.github_intel.constants import (
|
|
8
|
+
EXIT_AUTH,
|
|
9
|
+
EXIT_CHECKS,
|
|
10
|
+
EXIT_CONFIG,
|
|
11
|
+
EXIT_ERROR,
|
|
12
|
+
EXIT_NOT_FOUND,
|
|
13
|
+
EXIT_SYNC,
|
|
14
|
+
)
|
|
15
|
+
|
|
16
|
+
__all__ = [
|
|
17
|
+
"AuthError",
|
|
18
|
+
"ChecksFailed",
|
|
19
|
+
"ConfigError",
|
|
20
|
+
"IntelError",
|
|
21
|
+
"NotFound",
|
|
22
|
+
"SyncIncomplete",
|
|
23
|
+
"SyncTooSoon",
|
|
24
|
+
]
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
class IntelError(Exception):
|
|
28
|
+
"""
|
|
29
|
+
Base error for failures reported to the operator.
|
|
30
|
+
|
|
31
|
+
Attributes:
|
|
32
|
+
exit_code: Process exit code for this failure.
|
|
33
|
+
|
|
34
|
+
"""
|
|
35
|
+
|
|
36
|
+
exit_code = EXIT_ERROR
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
class ConfigError(IntelError):
|
|
40
|
+
"""The configuration or command arguments are invalid."""
|
|
41
|
+
|
|
42
|
+
exit_code = EXIT_CONFIG
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
class AuthError(IntelError):
|
|
46
|
+
"""GitHub rejected the credentials used for the API."""
|
|
47
|
+
|
|
48
|
+
exit_code = EXIT_AUTH
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
class SyncIncomplete(IntelError):
|
|
52
|
+
"""Synchronisation finished with failures or was cancelled."""
|
|
53
|
+
|
|
54
|
+
exit_code = EXIT_SYNC
|
|
55
|
+
|
|
56
|
+
|
|
57
|
+
class SyncTooSoon(IntelError):
|
|
58
|
+
"""Another synchronisation was requested inside ``sync.interval``."""
|
|
59
|
+
|
|
60
|
+
exit_code = EXIT_CONFIG
|
|
61
|
+
|
|
62
|
+
|
|
63
|
+
class NotFound(IntelError):
|
|
64
|
+
"""The named repository is not in the configuration."""
|
|
65
|
+
|
|
66
|
+
exit_code = EXIT_NOT_FOUND
|
|
67
|
+
|
|
68
|
+
|
|
69
|
+
class ChecksFailed(IntelError):
|
|
70
|
+
"""An explicit failure-on-check run found failing checks."""
|
|
71
|
+
|
|
72
|
+
exit_code = EXIT_CHECKS
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
"""GitHub REST access."""
|
|
@@ -0,0 +1,40 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Resolve a GitHub API token from the existing GitHub CLI login.
|
|
3
|
+
|
|
4
|
+
github-repo-sync authenticates with SSH and stores no token. The REST API
|
|
5
|
+
cannot use an SSH key, so the token is read from ``gh auth token`` for that
|
|
6
|
+
same GitHub account. The value is never written to config, cache, or logs.
|
|
7
|
+
"""
|
|
8
|
+
|
|
9
|
+
from __future__ import annotations
|
|
10
|
+
|
|
11
|
+
import subprocess
|
|
12
|
+
|
|
13
|
+
__all__ = [
|
|
14
|
+
"github_cli_token",
|
|
15
|
+
]
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
def github_cli_token() -> str | None:
|
|
19
|
+
"""
|
|
20
|
+
Return the current ``gh`` token, or None when gh is not logged in.
|
|
21
|
+
|
|
22
|
+
Returns:
|
|
23
|
+
Token text, or None when the GitHub CLI cannot provide one.
|
|
24
|
+
|
|
25
|
+
"""
|
|
26
|
+
|
|
27
|
+
try:
|
|
28
|
+
completed = subprocess.run( # noqa: S603
|
|
29
|
+
["gh", "auth", "token"], # noqa: S607
|
|
30
|
+
check=False,
|
|
31
|
+
capture_output=True,
|
|
32
|
+
text=True,
|
|
33
|
+
timeout=5,
|
|
34
|
+
)
|
|
35
|
+
except (OSError, subprocess.TimeoutExpired):
|
|
36
|
+
return None
|
|
37
|
+
if completed.returncode != 0:
|
|
38
|
+
return None
|
|
39
|
+
token = completed.stdout.strip()
|
|
40
|
+
return token or None
|
|
@@ -0,0 +1,239 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Bounded asynchronous GitHub REST client.
|
|
3
|
+
|
|
4
|
+
The semaphore limits in-flight HTTP calls. Retries cover timeouts and
|
|
5
|
+
transient server errors. Permanent errors are returned to the caller.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
from __future__ import annotations
|
|
9
|
+
|
|
10
|
+
import asyncio
|
|
11
|
+
from collections.abc import Callable
|
|
12
|
+
from dataclasses import dataclass
|
|
13
|
+
import random
|
|
14
|
+
from typing import Any
|
|
15
|
+
|
|
16
|
+
import httpx
|
|
17
|
+
|
|
18
|
+
from lupaxa.github_intel.constants import API_VERSION, USER_AGENT
|
|
19
|
+
from lupaxa.github_intel.github.rate_limit import RateLimitController
|
|
20
|
+
|
|
21
|
+
__all__ = [
|
|
22
|
+
"FetchResult",
|
|
23
|
+
"GitHubClient",
|
|
24
|
+
]
|
|
25
|
+
|
|
26
|
+
_RETRYABLE = {500, 502, 503, 504}
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
@dataclass(frozen=True)
|
|
30
|
+
class FetchResult:
|
|
31
|
+
"""One completed GET, including a 304 that carries no new body."""
|
|
32
|
+
|
|
33
|
+
status: int
|
|
34
|
+
body: Any
|
|
35
|
+
etag: str | None
|
|
36
|
+
error: str | None
|
|
37
|
+
not_modified: bool
|
|
38
|
+
link: str | None = None
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
class GitHubClient:
|
|
42
|
+
"""httpx client with shared rate limiting and bounded concurrency."""
|
|
43
|
+
|
|
44
|
+
def __init__(
|
|
45
|
+
self,
|
|
46
|
+
*,
|
|
47
|
+
api_url: str,
|
|
48
|
+
token: str | None,
|
|
49
|
+
timeout: float,
|
|
50
|
+
retries: int,
|
|
51
|
+
concurrency: int,
|
|
52
|
+
transport: httpx.AsyncBaseTransport | None = None,
|
|
53
|
+
limits: RateLimitController | None = None,
|
|
54
|
+
) -> None:
|
|
55
|
+
"""
|
|
56
|
+
Open a client. Call :meth:`aclose` when the sync run finishes.
|
|
57
|
+
|
|
58
|
+
Args:
|
|
59
|
+
api_url: GitHub API origin.
|
|
60
|
+
token: Bearer token, or None for unauthenticated access.
|
|
61
|
+
timeout: Per-request timeout in seconds.
|
|
62
|
+
retries: Extra attempts after a transient failure.
|
|
63
|
+
concurrency: Maximum in-flight HTTP calls.
|
|
64
|
+
transport: Optional httpx transport for tests.
|
|
65
|
+
limits: Shared rate-limit controller.
|
|
66
|
+
|
|
67
|
+
"""
|
|
68
|
+
|
|
69
|
+
headers = {
|
|
70
|
+
"Accept": "application/vnd.github+json",
|
|
71
|
+
"User-Agent": USER_AGENT,
|
|
72
|
+
"X-GitHub-Api-Version": API_VERSION,
|
|
73
|
+
}
|
|
74
|
+
if token:
|
|
75
|
+
headers["Authorization"] = f"Bearer {token}"
|
|
76
|
+
self._retries = retries
|
|
77
|
+
self.concurrency = concurrency
|
|
78
|
+
self._semaphore = asyncio.Semaphore(concurrency)
|
|
79
|
+
self._limits = limits or RateLimitController()
|
|
80
|
+
self._http = httpx.AsyncClient(
|
|
81
|
+
base_url=api_url,
|
|
82
|
+
headers=headers,
|
|
83
|
+
timeout=timeout,
|
|
84
|
+
transport=transport,
|
|
85
|
+
follow_redirects=True,
|
|
86
|
+
)
|
|
87
|
+
self.token = token
|
|
88
|
+
|
|
89
|
+
async def aclose(self) -> None:
|
|
90
|
+
"""Close the underlying HTTP client."""
|
|
91
|
+
|
|
92
|
+
await self._http.aclose()
|
|
93
|
+
|
|
94
|
+
@property
|
|
95
|
+
def rate_pauses(self) -> int:
|
|
96
|
+
"""
|
|
97
|
+
Return how many times workers waited for a GitHub rate limit.
|
|
98
|
+
|
|
99
|
+
Returns:
|
|
100
|
+
Pause count for this client.
|
|
101
|
+
|
|
102
|
+
"""
|
|
103
|
+
|
|
104
|
+
return self._limits.pauses
|
|
105
|
+
|
|
106
|
+
def watch_rate_limit(self, listener: Callable[[float], None] | None) -> None:
|
|
107
|
+
"""
|
|
108
|
+
Receive the remaining rate-limit wait while requests are paused.
|
|
109
|
+
|
|
110
|
+
Args:
|
|
111
|
+
listener: Called with seconds remaining, then 0 when the wait ends.
|
|
112
|
+
It must not await.
|
|
113
|
+
|
|
114
|
+
"""
|
|
115
|
+
|
|
116
|
+
self._limits.set_listener(listener)
|
|
117
|
+
|
|
118
|
+
async def get(
|
|
119
|
+
self,
|
|
120
|
+
path: str,
|
|
121
|
+
*,
|
|
122
|
+
etag: str | None = None,
|
|
123
|
+
params: dict[str, str] | None = None,
|
|
124
|
+
) -> FetchResult:
|
|
125
|
+
"""
|
|
126
|
+
GET one API path.
|
|
127
|
+
|
|
128
|
+
Args:
|
|
129
|
+
path: Path under the API origin.
|
|
130
|
+
etag: Previous ETag, sent as ``If-None-Match``.
|
|
131
|
+
params: Query string.
|
|
132
|
+
|
|
133
|
+
Returns:
|
|
134
|
+
Status, parsed JSON when present, and the new ETag.
|
|
135
|
+
|
|
136
|
+
"""
|
|
137
|
+
|
|
138
|
+
headers = {"If-None-Match": etag} if etag else {}
|
|
139
|
+
response = await self._request(path, headers, params)
|
|
140
|
+
if response is None:
|
|
141
|
+
return FetchResult(0, None, etag, "Network request failed.", False)
|
|
142
|
+
new_etag = response.headers.get("etag") or etag
|
|
143
|
+
self._limits.observe(
|
|
144
|
+
response.status_code,
|
|
145
|
+
{k.lower(): v for k, v in response.headers.items()},
|
|
146
|
+
)
|
|
147
|
+
if response.status_code == 304:
|
|
148
|
+
return FetchResult(304, None, new_etag, None, True)
|
|
149
|
+
if response.status_code >= 400:
|
|
150
|
+
return FetchResult(
|
|
151
|
+
response.status_code,
|
|
152
|
+
None,
|
|
153
|
+
new_etag,
|
|
154
|
+
_error_text(response),
|
|
155
|
+
False,
|
|
156
|
+
)
|
|
157
|
+
return FetchResult(
|
|
158
|
+
response.status_code,
|
|
159
|
+
_json_body(response),
|
|
160
|
+
new_etag,
|
|
161
|
+
None,
|
|
162
|
+
False,
|
|
163
|
+
response.headers.get("link"),
|
|
164
|
+
)
|
|
165
|
+
|
|
166
|
+
async def _request(
|
|
167
|
+
self,
|
|
168
|
+
path: str,
|
|
169
|
+
headers: dict[str, str],
|
|
170
|
+
params: dict[str, str] | None,
|
|
171
|
+
) -> httpx.Response | None:
|
|
172
|
+
attempt = 0
|
|
173
|
+
while True:
|
|
174
|
+
try:
|
|
175
|
+
async with self._semaphore:
|
|
176
|
+
await self._limits.wait()
|
|
177
|
+
response = await self._http.get(
|
|
178
|
+
path,
|
|
179
|
+
headers=headers,
|
|
180
|
+
params=params,
|
|
181
|
+
)
|
|
182
|
+
except httpx.HTTPError:
|
|
183
|
+
if attempt >= self._retries:
|
|
184
|
+
return None
|
|
185
|
+
await asyncio.sleep(_backoff(attempt))
|
|
186
|
+
attempt += 1
|
|
187
|
+
continue
|
|
188
|
+
if response.status_code in _RETRYABLE and attempt < self._retries:
|
|
189
|
+
await asyncio.sleep(_backoff(attempt))
|
|
190
|
+
attempt += 1
|
|
191
|
+
continue
|
|
192
|
+
if _rate_limited(response) and attempt < self._retries:
|
|
193
|
+
self._limits.observe(
|
|
194
|
+
response.status_code,
|
|
195
|
+
{key.lower(): value for key, value in response.headers.items()},
|
|
196
|
+
)
|
|
197
|
+
await self._limits.wait()
|
|
198
|
+
attempt += 1
|
|
199
|
+
continue
|
|
200
|
+
return response
|
|
201
|
+
|
|
202
|
+
|
|
203
|
+
def _rate_limited(response: httpx.Response) -> bool:
|
|
204
|
+
if response.status_code == 429:
|
|
205
|
+
return True
|
|
206
|
+
if response.status_code != 403:
|
|
207
|
+
return False
|
|
208
|
+
remaining = response.headers.get("x-ratelimit-remaining")
|
|
209
|
+
if remaining == "0":
|
|
210
|
+
return True
|
|
211
|
+
return "rate limit" in response.text.casefold()
|
|
212
|
+
|
|
213
|
+
|
|
214
|
+
def _json_body(response: httpx.Response) -> object:
|
|
215
|
+
try:
|
|
216
|
+
return response.json()
|
|
217
|
+
except ValueError:
|
|
218
|
+
return None
|
|
219
|
+
|
|
220
|
+
|
|
221
|
+
def _error_text(response: httpx.Response) -> str:
|
|
222
|
+
try:
|
|
223
|
+
payload = response.json()
|
|
224
|
+
except ValueError:
|
|
225
|
+
payload = None
|
|
226
|
+
if isinstance(payload, dict) and isinstance(payload.get("message"), str):
|
|
227
|
+
message = payload["message"]
|
|
228
|
+
else:
|
|
229
|
+
message = response.text[:300]
|
|
230
|
+
if response.status_code == 404:
|
|
231
|
+
return "GitHub returned 404. The repository may be missing or not visible."
|
|
232
|
+
if response.status_code == 401:
|
|
233
|
+
return "GitHub rejected the credentials."
|
|
234
|
+
return f"GitHub returned {response.status_code}: {message}"
|
|
235
|
+
|
|
236
|
+
|
|
237
|
+
def _backoff(attempt: int) -> float:
|
|
238
|
+
delay: float = 2.0**attempt
|
|
239
|
+
return delay + random.uniform(0, 0.3) # noqa: S311
|
|
@@ -0,0 +1,107 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Shared GitHub primary and secondary rate-limit pauses.
|
|
3
|
+
|
|
4
|
+
Every concurrent request waits on the same controller.
|
|
5
|
+
"""
|
|
6
|
+
|
|
7
|
+
from __future__ import annotations
|
|
8
|
+
|
|
9
|
+
import asyncio
|
|
10
|
+
from collections.abc import Callable
|
|
11
|
+
import time
|
|
12
|
+
|
|
13
|
+
__all__ = [
|
|
14
|
+
"RateLimitController",
|
|
15
|
+
]
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
class RateLimitController:
|
|
19
|
+
"""Track remaining quota and pause every worker when GitHub asks."""
|
|
20
|
+
|
|
21
|
+
def __init__(
|
|
22
|
+
self,
|
|
23
|
+
clock: Callable[[], float] | None = None,
|
|
24
|
+
sleeper: Callable[[float], asyncio.Future[None]] | None = None,
|
|
25
|
+
) -> None:
|
|
26
|
+
"""
|
|
27
|
+
Create a controller.
|
|
28
|
+
|
|
29
|
+
Args:
|
|
30
|
+
clock: Monotonic clock, replaced in tests.
|
|
31
|
+
sleeper: Sleep coroutine, replaced in tests.
|
|
32
|
+
|
|
33
|
+
"""
|
|
34
|
+
|
|
35
|
+
self._clock = clock or time.monotonic
|
|
36
|
+
self._sleep = sleeper or asyncio.sleep
|
|
37
|
+
self._pause_until = 0.0
|
|
38
|
+
self._lock = asyncio.Lock()
|
|
39
|
+
self._on_wait: Callable[[float], None] | None = None
|
|
40
|
+
self.pauses = 0
|
|
41
|
+
|
|
42
|
+
def observe(
|
|
43
|
+
self,
|
|
44
|
+
status: int,
|
|
45
|
+
headers: dict[str, str],
|
|
46
|
+
now: float | None = None,
|
|
47
|
+
) -> None:
|
|
48
|
+
"""
|
|
49
|
+
Record rate-limit headers from one response.
|
|
50
|
+
|
|
51
|
+
Args:
|
|
52
|
+
status: HTTP status.
|
|
53
|
+
headers: Response headers with lower-case names.
|
|
54
|
+
now: Epoch time used to interpret ``X-RateLimit-Reset``.
|
|
55
|
+
|
|
56
|
+
"""
|
|
57
|
+
|
|
58
|
+
retry_after = _seconds(headers.get("retry-after"))
|
|
59
|
+
remaining = _seconds(headers.get("x-ratelimit-remaining"))
|
|
60
|
+
reset = _seconds(headers.get("x-ratelimit-reset"))
|
|
61
|
+
moment = time.time() if now is None else now
|
|
62
|
+
delay = 0.0
|
|
63
|
+
if retry_after is not None and status in {403, 429}:
|
|
64
|
+
delay = retry_after
|
|
65
|
+
elif remaining == 0 and reset is not None:
|
|
66
|
+
delay = max(0.0, reset - moment)
|
|
67
|
+
if delay > 0:
|
|
68
|
+
self._pause_until = max(self._pause_until, self._clock() + delay)
|
|
69
|
+
|
|
70
|
+
def set_listener(self, listener: Callable[[float], None] | None) -> None:
|
|
71
|
+
"""
|
|
72
|
+
Report the remaining wait, in seconds, while a pause is in force.
|
|
73
|
+
|
|
74
|
+
Args:
|
|
75
|
+
listener: Called with the remaining delay, then with 0 when it ends.
|
|
76
|
+
It must not await.
|
|
77
|
+
|
|
78
|
+
"""
|
|
79
|
+
|
|
80
|
+
self._on_wait = listener
|
|
81
|
+
|
|
82
|
+
async def wait(self) -> None:
|
|
83
|
+
"""Sleep when a previous response asked the client to slow down."""
|
|
84
|
+
|
|
85
|
+
announced = False
|
|
86
|
+
while True:
|
|
87
|
+
async with self._lock:
|
|
88
|
+
delay = self._pause_until - self._clock()
|
|
89
|
+
if delay > 0 and not announced:
|
|
90
|
+
self.pauses += 1
|
|
91
|
+
announced = True
|
|
92
|
+
if delay <= 0:
|
|
93
|
+
if announced and self._on_wait is not None:
|
|
94
|
+
self._on_wait(0.0)
|
|
95
|
+
return
|
|
96
|
+
if self._on_wait is not None:
|
|
97
|
+
self._on_wait(delay)
|
|
98
|
+
await self._sleep(min(1.0, delay))
|
|
99
|
+
|
|
100
|
+
|
|
101
|
+
def _seconds(value: str | None) -> float | None:
|
|
102
|
+
if value is None or value.strip() == "":
|
|
103
|
+
return None
|
|
104
|
+
try:
|
|
105
|
+
return float(value)
|
|
106
|
+
except ValueError:
|
|
107
|
+
return None
|