lupaxa-github-extractor 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- lupaxa/github_extractor/__init__.py +98 -0
- lupaxa/github_extractor/__main__.py +10 -0
- lupaxa/github_extractor/cli.py +383 -0
- lupaxa/github_extractor/client.py +390 -0
- lupaxa/github_extractor/constants.py +10 -0
- lupaxa/github_extractor/convert.py +182 -0
- lupaxa/github_extractor/errors.py +76 -0
- lupaxa/github_extractor/filters.py +47 -0
- lupaxa/github_extractor/http.py +289 -0
- lupaxa/github_extractor/models.py +362 -0
- lupaxa/github_extractor/pool.py +34 -0
- lupaxa/github_extractor/resources.py +66 -0
- lupaxa/github_extractor/version.py +10 -0
- lupaxa_github_extractor-0.1.0.dist-info/METADATA +175 -0
- lupaxa_github_extractor-0.1.0.dist-info/RECORD +19 -0
- lupaxa_github_extractor-0.1.0.dist-info/WHEEL +5 -0
- lupaxa_github_extractor-0.1.0.dist-info/entry_points.txt +2 -0
- lupaxa_github_extractor-0.1.0.dist-info/licenses/LICENCE +21 -0
- lupaxa_github_extractor-0.1.0.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,390 @@
|
|
|
1
|
+
"""High-level GitHub extraction client."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from collections.abc import Mapping, Sequence
|
|
6
|
+
from functools import partial
|
|
7
|
+
from typing import Any, cast
|
|
8
|
+
from urllib.parse import quote
|
|
9
|
+
|
|
10
|
+
import requests
|
|
11
|
+
|
|
12
|
+
from .constants import API_ROOT, DEFAULT_MAX_WORKERS, DEFAULT_TIMEOUT
|
|
13
|
+
from .convert import organisation_from, repository_from, token_from, user_from
|
|
14
|
+
from .errors import (
|
|
15
|
+
MissingOrgNameError,
|
|
16
|
+
MissingTokenError,
|
|
17
|
+
NotFoundError,
|
|
18
|
+
ResourceStatusError,
|
|
19
|
+
)
|
|
20
|
+
from .filters import filter_repositories, selected
|
|
21
|
+
from .http import GitHubHttp, _normalise_api_root, _same_api_host
|
|
22
|
+
from .models import Organisation, Repository, TokenInformation, User
|
|
23
|
+
from .pool import run_workers
|
|
24
|
+
from .resources import ORGANISATION_RESOURCES, REPOSITORY_RESOURCES, USER_RESOURCES, normalize_resources, repository_addons, user_resource_path
|
|
25
|
+
|
|
26
|
+
_DEFAULT_QUERIES: dict[str, dict[str, str]] = {
|
|
27
|
+
"issues": {"state": "all"},
|
|
28
|
+
"pulls": {"state": "all"},
|
|
29
|
+
"notifications": {"all": "true"},
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
def _normalize_params(params: object | None, allowed: set[str]) -> dict[str, dict[str, str]]:
|
|
34
|
+
"""Return caller queries, or reject a params key this call will not fetch."""
|
|
35
|
+
if params is None:
|
|
36
|
+
return {}
|
|
37
|
+
if isinstance(params, str) or not isinstance(params, Mapping):
|
|
38
|
+
raise ValueError("params must be a mapping of resource names")
|
|
39
|
+
chosen: dict[str, dict[str, str]] = {}
|
|
40
|
+
for name, query in params.items():
|
|
41
|
+
if name not in allowed:
|
|
42
|
+
raise ValueError(f"unknown resource: {name}")
|
|
43
|
+
if isinstance(query, str) or not isinstance(query, Mapping):
|
|
44
|
+
raise ValueError("params must be a mapping of resource names")
|
|
45
|
+
chosen[name] = {str(key): str(value) for key, value in query.items()}
|
|
46
|
+
return chosen
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
def _query_for(name: str, params: Mapping[str, Mapping[str, str]]) -> dict[str, str]:
|
|
50
|
+
"""Return the default query for a resource, overridden by caller pairs."""
|
|
51
|
+
query = dict(_DEFAULT_QUERIES.get(name, {}))
|
|
52
|
+
query.update(params.get(name, {}))
|
|
53
|
+
return query
|
|
54
|
+
|
|
55
|
+
|
|
56
|
+
def _api_target(path: str, api_root: str) -> str:
|
|
57
|
+
"""Return a path or absolute URL on the API root, or reject it."""
|
|
58
|
+
target = path.strip()
|
|
59
|
+
if target.startswith("/") and not target.startswith("//"):
|
|
60
|
+
return target
|
|
61
|
+
if _same_api_host(target, api_root):
|
|
62
|
+
return target
|
|
63
|
+
raise ValueError("path must be a GitHub API path")
|
|
64
|
+
|
|
65
|
+
|
|
66
|
+
class GitHubExtractor:
|
|
67
|
+
"""Extract authenticated GitHub account data."""
|
|
68
|
+
|
|
69
|
+
# fmt: off
|
|
70
|
+
def __init__(self, token: str, timeout: float = DEFAULT_TIMEOUT, max_workers: int = DEFAULT_MAX_WORKERS, *, session: requests.Session | None = None, max_pages: int | None = None, api_root: str = API_ROOT) -> None: # noqa: E501
|
|
71
|
+
# fmt: on
|
|
72
|
+
"""Validate client settings and prepare authenticated HTTP access."""
|
|
73
|
+
if not token.strip():
|
|
74
|
+
raise MissingTokenError
|
|
75
|
+
if not timeout > 0:
|
|
76
|
+
raise ValueError("timeout must be greater than zero")
|
|
77
|
+
if type(max_workers) is not int or max_workers <= 0:
|
|
78
|
+
raise ValueError("max_workers must be a positive integer")
|
|
79
|
+
if max_pages is not None and (type(max_pages) is not int or max_pages <= 0):
|
|
80
|
+
raise ValueError("max_pages must be a positive integer")
|
|
81
|
+
self._max_workers = max_workers
|
|
82
|
+
self._api_root = _normalise_api_root(api_root)
|
|
83
|
+
self._http = GitHubHttp(token, timeout, session, max_pages=max_pages, api_root=self._api_root)
|
|
84
|
+
|
|
85
|
+
def get(self, path: str) -> Any: # noqa: ANN401
|
|
86
|
+
"""Return JSON from one read-only GitHub API path."""
|
|
87
|
+
return self._http.get_json(_api_target(path, self._api_root))
|
|
88
|
+
|
|
89
|
+
def token_information(self) -> TokenInformation:
|
|
90
|
+
"""Return token scope and rate-limit information from GitHub headers."""
|
|
91
|
+
headers = self._http.get_response("/user").headers
|
|
92
|
+
return token_from(
|
|
93
|
+
{
|
|
94
|
+
"scopes": headers.get("X-OAuth-Scopes", ""),
|
|
95
|
+
"rate_limit_limit": headers.get("X-RateLimit-Limit"),
|
|
96
|
+
"rate_limit_remaining": headers.get("X-RateLimit-Remaining"),
|
|
97
|
+
"rate_limit_used": headers.get("X-RateLimit-Used"),
|
|
98
|
+
"rate_limit_reset": headers.get("X-RateLimit-Reset"),
|
|
99
|
+
}
|
|
100
|
+
)
|
|
101
|
+
|
|
102
|
+
# fmt: off
|
|
103
|
+
def organisations(self, *, include: Sequence[str] = (), ignore: Sequence[str] = (), slugs: bool = False, resources: Sequence[str] = (), params: Mapping[str, Mapping[str, str]] | None = None) -> list[Organisation] | list[str]: # noqa: E501
|
|
104
|
+
# fmt: on
|
|
105
|
+
"""Return filtered organisations with optional related data."""
|
|
106
|
+
resource_names = normalize_resources(resources, ORGANISATION_RESOURCES)
|
|
107
|
+
normalized_params = _normalize_params(params, set(resource_names))
|
|
108
|
+
response = cast(list[dict[str, Any]], self._http.get_json("/user/orgs"))
|
|
109
|
+
organisations = [dict(row) for row in response if selected(cast(str, row["login"]), include, ignore)]
|
|
110
|
+
organisations.sort(key=lambda row: cast(str, row["login"]).casefold())
|
|
111
|
+
if slugs:
|
|
112
|
+
return [cast(str, row["login"]) for row in organisations]
|
|
113
|
+
self._attach_organisation_resources(organisations, resource_names, normalized_params)
|
|
114
|
+
return [organisation_from(row) for row in organisations]
|
|
115
|
+
|
|
116
|
+
# fmt: off
|
|
117
|
+
def organizations(self, *, include: Sequence[str] = (), ignore: Sequence[str] = (), slugs: bool = False, resources: Sequence[str] = (), params: Mapping[str, Mapping[str, str]] | None = None) -> list[Organisation] | list[str]: # noqa: E501
|
|
118
|
+
# fmt: on
|
|
119
|
+
"""Return organisations using the US-spelling alias."""
|
|
120
|
+
return self.organisations(include=include, ignore=ignore, slugs=slugs, resources=resources, params=params)
|
|
121
|
+
|
|
122
|
+
# fmt: off
|
|
123
|
+
def user_repositories(self, *, username: str | None = None, include: Sequence[str] = (), ignore: Sequence[str] = (), include_names: Sequence[str] = (), ignore_names: Sequence[str] = (), skip_private: bool = False, slugs: bool = False, resources: Sequence[str] = (), params: Mapping[str, Mapping[str, str]] | None = None) -> list[Repository] | list[str]: # noqa: E501
|
|
124
|
+
# fmt: on
|
|
125
|
+
"""Return repositories owned by the requested or authenticated user."""
|
|
126
|
+
resource_names = normalize_resources(resources, REPOSITORY_RESOURCES)
|
|
127
|
+
normalized_params = _normalize_params(params, set(resource_names))
|
|
128
|
+
addons = repository_addons(resource_names)
|
|
129
|
+
login = username.strip() if username is not None else ""
|
|
130
|
+
if login:
|
|
131
|
+
path = f"/users/{_encode_segment(login)}/repos"
|
|
132
|
+
else:
|
|
133
|
+
user = cast(dict[str, Any], self._http.get_json("/user"))
|
|
134
|
+
login = cast(str, user["login"])
|
|
135
|
+
path = "/user/repos"
|
|
136
|
+
rows = cast(list[dict[str, Any]], self._http.get_json(path))
|
|
137
|
+
owned = [row for row in rows if _repository_owner(row).casefold() == login.casefold()]
|
|
138
|
+
return self._repository_result(
|
|
139
|
+
owned,
|
|
140
|
+
include=include,
|
|
141
|
+
ignore=ignore,
|
|
142
|
+
include_names=include_names,
|
|
143
|
+
ignore_names=ignore_names,
|
|
144
|
+
skip_private=skip_private,
|
|
145
|
+
slugs=slugs,
|
|
146
|
+
addons=addons,
|
|
147
|
+
params=normalized_params,
|
|
148
|
+
)
|
|
149
|
+
|
|
150
|
+
# fmt: off
|
|
151
|
+
def repositories_by_org(self, org: str, *, include: Sequence[str] = (), ignore: Sequence[str] = (), include_names: Sequence[str] = (), ignore_names: Sequence[str] = (), skip_private: bool = False, slugs: bool = False, resources: Sequence[str] = (), params: Mapping[str, Mapping[str, str]] | None = None) -> list[Repository] | list[str]: # noqa: E501
|
|
152
|
+
# fmt: on
|
|
153
|
+
"""Return filtered repositories for one organisation."""
|
|
154
|
+
resource_names = normalize_resources(resources, REPOSITORY_RESOURCES)
|
|
155
|
+
normalized_params = _normalize_params(params, set(resource_names))
|
|
156
|
+
addons = repository_addons(resource_names)
|
|
157
|
+
login = org.strip()
|
|
158
|
+
if not login:
|
|
159
|
+
raise MissingOrgNameError
|
|
160
|
+
rows = cast(
|
|
161
|
+
list[dict[str, Any]],
|
|
162
|
+
self._http.get_json(f"/orgs/{_encode_segment(login)}/repos"),
|
|
163
|
+
)
|
|
164
|
+
return self._repository_result(
|
|
165
|
+
rows,
|
|
166
|
+
include=include,
|
|
167
|
+
ignore=ignore,
|
|
168
|
+
include_names=include_names,
|
|
169
|
+
ignore_names=ignore_names,
|
|
170
|
+
skip_private=skip_private,
|
|
171
|
+
slugs=slugs,
|
|
172
|
+
addons=addons,
|
|
173
|
+
params=normalized_params,
|
|
174
|
+
)
|
|
175
|
+
|
|
176
|
+
# fmt: off
|
|
177
|
+
def visible_repositories(self, *, include: Sequence[str] = (), ignore: Sequence[str] = (), include_names: Sequence[str] = (), ignore_names: Sequence[str] = (), skip_private: bool = False, slugs: bool = False, resources: Sequence[str] = (), params: Mapping[str, Mapping[str, str]] | None = None) -> list[Repository] | list[str]: # noqa: E501
|
|
178
|
+
# fmt: on
|
|
179
|
+
"""Return every repository visible to the authenticated token."""
|
|
180
|
+
resource_names = normalize_resources(resources, REPOSITORY_RESOURCES)
|
|
181
|
+
normalized_params = _normalize_params(params, set(resource_names))
|
|
182
|
+
addons = repository_addons(resource_names)
|
|
183
|
+
rows = cast(
|
|
184
|
+
list[dict[str, Any]],
|
|
185
|
+
self._http.get_json("/user/repos"),
|
|
186
|
+
)
|
|
187
|
+
return self._repository_result(
|
|
188
|
+
rows,
|
|
189
|
+
include=include,
|
|
190
|
+
ignore=ignore,
|
|
191
|
+
include_names=include_names,
|
|
192
|
+
ignore_names=ignore_names,
|
|
193
|
+
skip_private=skip_private,
|
|
194
|
+
slugs=slugs,
|
|
195
|
+
addons=addons,
|
|
196
|
+
params=normalized_params,
|
|
197
|
+
)
|
|
198
|
+
|
|
199
|
+
# fmt: off
|
|
200
|
+
def organisation_repositories(self, *, include_orgs: Sequence[str] = (), ignore_orgs: Sequence[str] = (), include: Sequence[str] = (), ignore: Sequence[str] = (), include_names: Sequence[str] = (), ignore_names: Sequence[str] = (), skip_private: bool = False, slugs: bool = False, resources: Sequence[str] = (), params: Mapping[str, Mapping[str, str]] | None = None) -> list[Repository] | list[str]: # noqa: E501
|
|
201
|
+
# fmt: on
|
|
202
|
+
"""Return filtered repositories from each selected organisation."""
|
|
203
|
+
resource_names = normalize_resources(resources, REPOSITORY_RESOURCES)
|
|
204
|
+
normalized_params = _normalize_params(params, set(resource_names))
|
|
205
|
+
addons = repository_addons(resource_names)
|
|
206
|
+
organisations = cast(
|
|
207
|
+
list[dict[str, Any]],
|
|
208
|
+
self._http.get_json("/user/orgs"),
|
|
209
|
+
)
|
|
210
|
+
rows: list[dict[str, Any]] = []
|
|
211
|
+
for organisation in organisations:
|
|
212
|
+
login = cast(str, organisation["login"])
|
|
213
|
+
if selected(login, include_orgs, ignore_orgs):
|
|
214
|
+
rows.extend(
|
|
215
|
+
cast(
|
|
216
|
+
list[dict[str, Any]],
|
|
217
|
+
self._http.get_json(f"/orgs/{_encode_segment(login)}/repos"),
|
|
218
|
+
)
|
|
219
|
+
)
|
|
220
|
+
filtered = filter_repositories(
|
|
221
|
+
rows,
|
|
222
|
+
include=include,
|
|
223
|
+
ignore=ignore,
|
|
224
|
+
include_names=include_names,
|
|
225
|
+
ignore_names=ignore_names,
|
|
226
|
+
skip_private=skip_private,
|
|
227
|
+
)
|
|
228
|
+
unique = list({cast(str, row["full_name"]): row for row in reversed(filtered)}.values())
|
|
229
|
+
self._attach_repository_addons(
|
|
230
|
+
unique,
|
|
231
|
+
addons,
|
|
232
|
+
slugs=slugs,
|
|
233
|
+
params=normalized_params,
|
|
234
|
+
)
|
|
235
|
+
return _format_repositories(unique, slugs=slugs)
|
|
236
|
+
|
|
237
|
+
def repository(self, full_name: str, *, resources: Sequence[str] = (), params: Mapping[str, Mapping[str, str]] | None = None) -> Repository:
|
|
238
|
+
"""Return one repository and the extras that were requested."""
|
|
239
|
+
names = normalize_resources(resources, REPOSITORY_RESOURCES)
|
|
240
|
+
queries = _normalize_params(params, set(names))
|
|
241
|
+
owner, separator, name = full_name.strip().partition("/")
|
|
242
|
+
if separator != "/" or not owner or not name:
|
|
243
|
+
raise ValueError("full_name must be owner/name")
|
|
244
|
+
row = dict(cast(dict[str, Any], self._http.get_json(f"/repos/{_encode_full_name(f'{owner}/{name}')}")))
|
|
245
|
+
self._attach_repository_addons([row], repository_addons(names), slugs=False, params=queries)
|
|
246
|
+
return repository_from(row)
|
|
247
|
+
|
|
248
|
+
# fmt: off
|
|
249
|
+
def _repository_result(self, rows: Sequence[dict[str, Any]], *, include: Sequence[str], ignore: Sequence[str], include_names: Sequence[str], ignore_names: Sequence[str], skip_private: bool, slugs: bool, addons: list[tuple[str, str, str | None]], params: dict[str, dict[str, str]]) -> list[Repository] | list[str]: # noqa: E501
|
|
250
|
+
# fmt: on
|
|
251
|
+
"""Filter repositories, attach requested add-ons, and format them."""
|
|
252
|
+
filtered = filter_repositories(
|
|
253
|
+
rows,
|
|
254
|
+
include=include,
|
|
255
|
+
ignore=ignore,
|
|
256
|
+
include_names=include_names,
|
|
257
|
+
ignore_names=ignore_names,
|
|
258
|
+
skip_private=skip_private,
|
|
259
|
+
)
|
|
260
|
+
self._attach_repository_addons(filtered, addons, slugs=slugs, params=params)
|
|
261
|
+
return _format_repositories(filtered, slugs=slugs)
|
|
262
|
+
|
|
263
|
+
def _attach_repository_addons(
|
|
264
|
+
self,
|
|
265
|
+
repositories: list[dict[str, Any]],
|
|
266
|
+
addons: list[tuple[str, str, str | None]],
|
|
267
|
+
*,
|
|
268
|
+
slugs: bool,
|
|
269
|
+
params: dict[str, dict[str, str]],
|
|
270
|
+
) -> None:
|
|
271
|
+
"""Attach requested add-ons concurrently to repository objects."""
|
|
272
|
+
if not addons or slugs:
|
|
273
|
+
return
|
|
274
|
+
run_workers(repositories, partial(self._add_repository_addons, addons=addons, params=params), self._max_workers)
|
|
275
|
+
|
|
276
|
+
def _add_repository_addons(self, repository: dict[str, Any], addons: list[tuple[str, str, str | None]], params: dict[str, dict[str, str]]) -> None:
|
|
277
|
+
"""Fetch enabled add-ons for one repository in flag order."""
|
|
278
|
+
full_name = cast(str, repository["full_name"])
|
|
279
|
+
encoded_full_name = _encode_full_name(full_name)
|
|
280
|
+
for stored_key, suffix, merge_key in addons:
|
|
281
|
+
path = f"/repos/{encoded_full_name}/{suffix}"
|
|
282
|
+
try:
|
|
283
|
+
payload = self._get_addon(path, merge_key, params=_query_for(stored_key, params))
|
|
284
|
+
if merge_key is not None and isinstance(payload, dict):
|
|
285
|
+
payload = payload.get(merge_key, [])
|
|
286
|
+
if stored_key == "contents" and isinstance(payload, dict):
|
|
287
|
+
payload = [payload]
|
|
288
|
+
repository[stored_key] = payload
|
|
289
|
+
except NotFoundError as error:
|
|
290
|
+
_record_resource_error(repository, stored_key, 404, str(error))
|
|
291
|
+
except ResourceStatusError as error:
|
|
292
|
+
_record_resource_error(repository, stored_key, error.status, str(error))
|
|
293
|
+
|
|
294
|
+
def _get_addon(self, path: str, merge_key: str | None, *, params: dict[str, str] | None = None) -> Any: # noqa: ANN401
|
|
295
|
+
"""Fetch one repository add-on with its required paging mode."""
|
|
296
|
+
if merge_key is None:
|
|
297
|
+
return self._http.get_json(path, params=params)
|
|
298
|
+
return self._http.get_json(path, params=params, merge_key=merge_key)
|
|
299
|
+
|
|
300
|
+
def _attach_organisation_resources(self, organisations: list[dict[str, Any]], names: Sequence[str], params: dict[str, dict[str, str]]) -> None:
|
|
301
|
+
"""Attach requested resources concurrently to organisation objects."""
|
|
302
|
+
if not names:
|
|
303
|
+
return
|
|
304
|
+
run_workers(organisations, partial(self._add_organisation_resources, names=names, params=params), self._max_workers)
|
|
305
|
+
|
|
306
|
+
def _add_organisation_resources(self, organisation: dict[str, Any], names: Sequence[str], params: dict[str, dict[str, str]]) -> None:
|
|
307
|
+
"""Store one organisation's requested resources or record a 404, 409, or 410."""
|
|
308
|
+
login = _encode_segment(cast(str, organisation["login"]))
|
|
309
|
+
for name in names:
|
|
310
|
+
suffix = "audit-log" if name == "audit_log" else name
|
|
311
|
+
try:
|
|
312
|
+
organisation[name] = self._get_addon(
|
|
313
|
+
f"/orgs/{login}/{suffix}",
|
|
314
|
+
None,
|
|
315
|
+
params=_query_for(name, params),
|
|
316
|
+
)
|
|
317
|
+
except NotFoundError as error:
|
|
318
|
+
_record_resource_error(organisation, name, 404, str(error))
|
|
319
|
+
except ResourceStatusError as error:
|
|
320
|
+
_record_resource_error(organisation, name, error.status, str(error))
|
|
321
|
+
|
|
322
|
+
# fmt: off
|
|
323
|
+
def authenticated_user(self, *, resources: Sequence[str] = (), repository_resources: Sequence[str] = (), params: Mapping[str, Mapping[str, str]] | None = None) -> User: # noqa: E501
|
|
324
|
+
# fmt: on
|
|
325
|
+
"""Return the authenticated user with selected account and repository data."""
|
|
326
|
+
if isinstance(repository_resources, str):
|
|
327
|
+
raise ValueError("repository_resources must be a list of names")
|
|
328
|
+
repo_names = normalize_resources(repository_resources, REPOSITORY_RESOURCES)
|
|
329
|
+
user_names = normalize_resources(resources, USER_RESOURCES)
|
|
330
|
+
if repo_names and not {"repos", "starred", "subscriptions"}.intersection(user_names):
|
|
331
|
+
raise ValueError("repository_resources requires repos, starred, or subscriptions")
|
|
332
|
+
queries = _normalize_params(params, set(user_names).union(repo_names))
|
|
333
|
+
user = dict(cast(dict[str, Any], self._http.get_json("/user")))
|
|
334
|
+
addons = repository_addons(repo_names)
|
|
335
|
+
for name in user_names:
|
|
336
|
+
self._add_user_extra(user, name, user_resource_path(name), repo_addons=addons, params=queries)
|
|
337
|
+
return user_from(user)
|
|
338
|
+
|
|
339
|
+
def _add_user_extra(
|
|
340
|
+
self,
|
|
341
|
+
user: dict[str, Any],
|
|
342
|
+
stored_key: str,
|
|
343
|
+
path: str,
|
|
344
|
+
*,
|
|
345
|
+
repo_addons: list[tuple[str, str, str | None]],
|
|
346
|
+
params: dict[str, dict[str, str]],
|
|
347
|
+
) -> None:
|
|
348
|
+
"""Store one requested user add-on, or an empty list when it fails."""
|
|
349
|
+
try:
|
|
350
|
+
user[stored_key] = self._http.get_json(path, params=_query_for(stored_key, params))
|
|
351
|
+
if repo_addons and stored_key in {"repos", "starred", "subscriptions"}:
|
|
352
|
+
rows = user[stored_key]
|
|
353
|
+
if isinstance(rows, list):
|
|
354
|
+
self._attach_repository_addons(rows, repo_addons, slugs=False, params=params)
|
|
355
|
+
except NotFoundError as error:
|
|
356
|
+
_record_resource_error(user, stored_key, 404, str(error))
|
|
357
|
+
except ResourceStatusError as error:
|
|
358
|
+
_record_resource_error(user, stored_key, error.status, str(error))
|
|
359
|
+
|
|
360
|
+
|
|
361
|
+
def _record_resource_error(target: dict[str, Any], name: str, status: int, message: str) -> None:
|
|
362
|
+
"""Record a failed extra and keep that name as an empty list."""
|
|
363
|
+
errors = cast(list[dict[str, Any]], target.setdefault("errors", []))
|
|
364
|
+
errors.append({"addon": name, "status": status, "message": message})
|
|
365
|
+
target[name] = []
|
|
366
|
+
|
|
367
|
+
|
|
368
|
+
def _format_repositories(rows: list[dict[str, Any]], *, slugs: bool) -> list[Repository] | list[str]:
|
|
369
|
+
"""Sort repository rows and return objects or full-name slugs."""
|
|
370
|
+
rows.sort(key=lambda row: cast(str, row["full_name"]).casefold())
|
|
371
|
+
if slugs:
|
|
372
|
+
return [cast(str, row["full_name"]) for row in rows]
|
|
373
|
+
return [repository_from(row) for row in rows]
|
|
374
|
+
|
|
375
|
+
|
|
376
|
+
def _repository_owner(row: dict[str, Any]) -> str:
|
|
377
|
+
"""Return a repository owner's login."""
|
|
378
|
+
owner = cast(dict[str, Any], row.get("owner", {}))
|
|
379
|
+
return cast(str, owner.get("login", ""))
|
|
380
|
+
|
|
381
|
+
|
|
382
|
+
def _encode_segment(segment: str) -> str:
|
|
383
|
+
"""Percent-encode one dynamic GitHub URL path segment."""
|
|
384
|
+
return quote(segment, safe="")
|
|
385
|
+
|
|
386
|
+
|
|
387
|
+
def _encode_full_name(full_name: str) -> str:
|
|
388
|
+
"""Encode a repository owner and name while retaining their separator."""
|
|
389
|
+
owner, repository = full_name.split("/", 1)
|
|
390
|
+
return f"{_encode_segment(owner)}/{_encode_segment(repository)}"
|
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
"""Values shared by the GitHub extractor."""
|
|
2
|
+
|
|
3
|
+
API_ROOT = "https://api.github.com"
|
|
4
|
+
API_ACCEPT = "application/vnd.github+json"
|
|
5
|
+
API_VERSION = "2022-11-28"
|
|
6
|
+
USER_AGENT = "lupaxa-github-extractor"
|
|
7
|
+
PER_PAGE = "100"
|
|
8
|
+
DEFAULT_TIMEOUT = 10
|
|
9
|
+
DEFAULT_MAX_WORKERS = 8
|
|
10
|
+
GH_AUTH_TIMEOUT = 10
|
|
@@ -0,0 +1,182 @@
|
|
|
1
|
+
"""Build models from GitHub JSON and write them back."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from typing import Any
|
|
6
|
+
|
|
7
|
+
from .models import (
|
|
8
|
+
AuditEvent,
|
|
9
|
+
Branch,
|
|
10
|
+
Collaborator,
|
|
11
|
+
Commit,
|
|
12
|
+
Content,
|
|
13
|
+
Contributor,
|
|
14
|
+
Deployment,
|
|
15
|
+
Email,
|
|
16
|
+
Environment,
|
|
17
|
+
Event,
|
|
18
|
+
Fork,
|
|
19
|
+
GitHubObject,
|
|
20
|
+
GpgKey,
|
|
21
|
+
Hook,
|
|
22
|
+
Issue,
|
|
23
|
+
Label,
|
|
24
|
+
Milestone,
|
|
25
|
+
Notification,
|
|
26
|
+
Organisation,
|
|
27
|
+
PullRequest,
|
|
28
|
+
Release,
|
|
29
|
+
Repository,
|
|
30
|
+
ResourceError,
|
|
31
|
+
SshKey,
|
|
32
|
+
Stargazer,
|
|
33
|
+
Subscriber,
|
|
34
|
+
Tag,
|
|
35
|
+
Team,
|
|
36
|
+
TokenInformation,
|
|
37
|
+
Topics,
|
|
38
|
+
User,
|
|
39
|
+
Workflow,
|
|
40
|
+
WorkflowRun,
|
|
41
|
+
)
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
def _errors_from(rows: list[dict[str, Any]]) -> list[ResourceError]:
|
|
45
|
+
"""Build recorded resource errors."""
|
|
46
|
+
return [ResourceError(addon=row["addon"], status=row["status"], message=row["message"]) for row in rows]
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
def _leaf_models(rows: list[dict[str, Any]], model: type[GitHubObject]) -> list[GitHubObject]:
|
|
50
|
+
"""Build leaf models from JSON rows."""
|
|
51
|
+
return [model(raw=dict(row)) for row in rows]
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
def repository_from(row: dict[str, Any]) -> Repository:
|
|
55
|
+
"""Build a repository and its requested attachments."""
|
|
56
|
+
raw = dict(row)
|
|
57
|
+
failed = {row["addon"] for row in raw.get("errors", [])}
|
|
58
|
+
errors = _errors_from(raw.pop("errors", []))
|
|
59
|
+
attachments: dict[str, Any] = {}
|
|
60
|
+
list_attachments = {
|
|
61
|
+
"branches",
|
|
62
|
+
"contributors",
|
|
63
|
+
"releases",
|
|
64
|
+
"tags",
|
|
65
|
+
"commits",
|
|
66
|
+
"issues",
|
|
67
|
+
"pulls",
|
|
68
|
+
"milestones",
|
|
69
|
+
"forks",
|
|
70
|
+
"stargazers",
|
|
71
|
+
"subscribers",
|
|
72
|
+
"collaborators",
|
|
73
|
+
"labels",
|
|
74
|
+
"workflow_runs",
|
|
75
|
+
"deployments",
|
|
76
|
+
"environments",
|
|
77
|
+
"contents",
|
|
78
|
+
}
|
|
79
|
+
for name in Repository.attached:
|
|
80
|
+
if name not in raw:
|
|
81
|
+
continue
|
|
82
|
+
value = raw[name]
|
|
83
|
+
is_attachment = (
|
|
84
|
+
value is None
|
|
85
|
+
or (name in list_attachments and isinstance(value, list))
|
|
86
|
+
or (name in {"languages", "topics"} and isinstance(value, dict))
|
|
87
|
+
or (name in {"languages", "topics"} and name in failed and value == [])
|
|
88
|
+
or (name == "workflows" and (isinstance(value, list) or (isinstance(value, dict) and isinstance(value.get("workflows"), list))))
|
|
89
|
+
)
|
|
90
|
+
if is_attachment:
|
|
91
|
+
attachments[name] = raw.pop(name)
|
|
92
|
+
model = Repository(raw=raw, errors=errors)
|
|
93
|
+
leaf_types: dict[str, type[GitHubObject]] = {
|
|
94
|
+
"branches": Branch,
|
|
95
|
+
"contributors": Contributor,
|
|
96
|
+
"releases": Release,
|
|
97
|
+
"tags": Tag,
|
|
98
|
+
"commits": Commit,
|
|
99
|
+
"issues": Issue,
|
|
100
|
+
"pulls": PullRequest,
|
|
101
|
+
"milestones": Milestone,
|
|
102
|
+
"forks": Fork,
|
|
103
|
+
"stargazers": Stargazer,
|
|
104
|
+
"subscribers": Subscriber,
|
|
105
|
+
"collaborators": Collaborator,
|
|
106
|
+
"labels": Label,
|
|
107
|
+
"workflow_runs": WorkflowRun,
|
|
108
|
+
"deployments": Deployment,
|
|
109
|
+
"environments": Environment,
|
|
110
|
+
"contents": Content,
|
|
111
|
+
}
|
|
112
|
+
for name, value in attachments.items():
|
|
113
|
+
if value is None or name == "languages" or (name == "topics" and value == []):
|
|
114
|
+
setattr(model, name, value)
|
|
115
|
+
elif name == "topics":
|
|
116
|
+
model.topics = Topics(raw=dict(value))
|
|
117
|
+
elif name == "workflows":
|
|
118
|
+
rows = value["workflows"] if isinstance(value, dict) else value
|
|
119
|
+
model.workflows = [Workflow(raw=dict(item)) for item in rows]
|
|
120
|
+
else:
|
|
121
|
+
setattr(model, name, _leaf_models(value, leaf_types[name]))
|
|
122
|
+
return model
|
|
123
|
+
|
|
124
|
+
|
|
125
|
+
def organisation_from(row: dict[str, Any]) -> Organisation:
|
|
126
|
+
"""Build an organisation and its requested attachments."""
|
|
127
|
+
raw = dict(row)
|
|
128
|
+
errors = _errors_from(raw.pop("errors", []))
|
|
129
|
+
attachments = {name: raw.pop(name) for name in Organisation.attached if name in raw and (raw[name] is None or isinstance(raw[name], list))}
|
|
130
|
+
model = Organisation(raw=raw, errors=errors)
|
|
131
|
+
leaf_types: dict[str, type[GitHubObject]] = {"teams": Team, "events": Event, "hooks": Hook, "audit_log": AuditEvent}
|
|
132
|
+
for name, value in attachments.items():
|
|
133
|
+
if value is None:
|
|
134
|
+
setattr(model, name, None)
|
|
135
|
+
elif name == "members":
|
|
136
|
+
model.members = [user_from(item) for item in value]
|
|
137
|
+
else:
|
|
138
|
+
setattr(model, name, _leaf_models(value, leaf_types[name]))
|
|
139
|
+
return model
|
|
140
|
+
|
|
141
|
+
|
|
142
|
+
def user_from(row: dict[str, Any]) -> User:
|
|
143
|
+
"""Build a user and its requested attachments."""
|
|
144
|
+
raw = dict(row)
|
|
145
|
+
errors = _errors_from(raw.pop("errors", []))
|
|
146
|
+
attachments = {name: raw.pop(name) for name in User.attached if name in raw and (raw[name] is None or isinstance(raw[name], list))}
|
|
147
|
+
model = User(raw=raw, errors=errors)
|
|
148
|
+
leaf_types: dict[str, type[GitHubObject]] = {"emails": Email, "gpg_keys": GpgKey, "ssh_keys": SshKey, "notifications": Notification}
|
|
149
|
+
for name, value in attachments.items():
|
|
150
|
+
if value is None:
|
|
151
|
+
setattr(model, name, None)
|
|
152
|
+
elif name in {"followers", "following"}:
|
|
153
|
+
setattr(model, name, [user_from(item) for item in value])
|
|
154
|
+
elif name in {"repos", "starred", "subscriptions"}:
|
|
155
|
+
setattr(model, name, [repository_from(item) for item in value])
|
|
156
|
+
else:
|
|
157
|
+
setattr(model, name, _leaf_models(value, leaf_types[name]))
|
|
158
|
+
return model
|
|
159
|
+
|
|
160
|
+
|
|
161
|
+
def token_from(row: dict[str, Any]) -> TokenInformation:
|
|
162
|
+
"""Build token information from one JSON object."""
|
|
163
|
+
return TokenInformation(raw=dict(row))
|
|
164
|
+
|
|
165
|
+
|
|
166
|
+
def to_jsonable(value: Any) -> Any: # noqa: ANN401
|
|
167
|
+
"""Return GitHub JSON for a model, list, error, or plain value."""
|
|
168
|
+
if isinstance(value, ResourceError):
|
|
169
|
+
return {"addon": value.addon, "status": value.status, "message": value.message}
|
|
170
|
+
if isinstance(value, GitHubObject):
|
|
171
|
+
payload = dict(value.raw)
|
|
172
|
+
for name in getattr(value, "attached", ()):
|
|
173
|
+
attached = getattr(value, name)
|
|
174
|
+
if attached is not None:
|
|
175
|
+
payload[name] = to_jsonable(attached)
|
|
176
|
+
errors = getattr(value, "errors", [])
|
|
177
|
+
if errors:
|
|
178
|
+
payload["errors"] = to_jsonable(errors)
|
|
179
|
+
return payload
|
|
180
|
+
if isinstance(value, list):
|
|
181
|
+
return [to_jsonable(item) for item in value]
|
|
182
|
+
return value
|
|
@@ -0,0 +1,76 @@
|
|
|
1
|
+
"""Domain errors raised by the GitHub extractor."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
|
|
6
|
+
class MissingTokenError(Exception):
|
|
7
|
+
"""Indicate that no GitHub token was supplied."""
|
|
8
|
+
|
|
9
|
+
def __init__(self, message: str = "A token is required") -> None:
|
|
10
|
+
"""Initialise the error with a safe message."""
|
|
11
|
+
super().__init__(message)
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
class MissingOrgNameError(Exception):
|
|
15
|
+
"""Indicate that no organisation name was supplied."""
|
|
16
|
+
|
|
17
|
+
def __init__(self, message: str = "An organisation name is required") -> None:
|
|
18
|
+
"""Initialise the error with a safe message."""
|
|
19
|
+
super().__init__(message)
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
class AuthenticationError(Exception):
|
|
23
|
+
"""Indicate that GitHub rejected authentication."""
|
|
24
|
+
|
|
25
|
+
def __init__(self, message: str = "Authentication failed") -> None:
|
|
26
|
+
"""Initialise the error with a safe message."""
|
|
27
|
+
super().__init__(message)
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
class RateLimitExceededError(Exception):
|
|
31
|
+
"""Indicate that GitHub has rate limited the request."""
|
|
32
|
+
|
|
33
|
+
def __init__(self, message: str = "Rate limit exceeded") -> None:
|
|
34
|
+
"""Initialise the error with a safe message."""
|
|
35
|
+
super().__init__(message)
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
class ForbiddenError(Exception):
|
|
39
|
+
"""Indicate that GitHub denied access to a resource."""
|
|
40
|
+
|
|
41
|
+
def __init__(self, message: str = "Forbidden") -> None:
|
|
42
|
+
"""Initialise the error with a safe message."""
|
|
43
|
+
super().__init__(message)
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
class NotFoundError(Exception):
|
|
47
|
+
"""Indicate that a GitHub resource was not found."""
|
|
48
|
+
|
|
49
|
+
def __init__(self, message: str = "Not found") -> None:
|
|
50
|
+
"""Initialise the error with a safe message."""
|
|
51
|
+
super().__init__(message)
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
class RequestTimeoutError(Exception):
|
|
55
|
+
"""Indicate that a GitHub request timed out."""
|
|
56
|
+
|
|
57
|
+
def __init__(self, message: str = "Request timed out") -> None:
|
|
58
|
+
"""Initialise the error with a safe message."""
|
|
59
|
+
super().__init__(message)
|
|
60
|
+
|
|
61
|
+
|
|
62
|
+
class RequestError(Exception):
|
|
63
|
+
"""Indicate that a GitHub request failed."""
|
|
64
|
+
|
|
65
|
+
def __init__(self, message: str = "Request failed") -> None:
|
|
66
|
+
"""Initialise the error with a safe message."""
|
|
67
|
+
super().__init__(message)
|
|
68
|
+
|
|
69
|
+
|
|
70
|
+
class ResourceStatusError(RequestError):
|
|
71
|
+
"""Indicate a resource status that a resource walk may record."""
|
|
72
|
+
|
|
73
|
+
def __init__(self, status: int, message: str) -> None:
|
|
74
|
+
"""Initialise the error with its HTTP status and safe message."""
|
|
75
|
+
super().__init__(message)
|
|
76
|
+
self.status = status
|