lupaxa-github-intel 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- lupaxa/github_intel/__init__.py +13 -0
- lupaxa/github_intel/__main__.py +8 -0
- lupaxa/github_intel/app.py +493 -0
- lupaxa/github_intel/checks/__init__.py +1 -0
- lupaxa/github_intel/checks/base.py +17 -0
- lupaxa/github_intel/checks/releases.py +59 -0
- lupaxa/github_intel/checks/workflows.py +34 -0
- lupaxa/github_intel/cli.py +699 -0
- lupaxa/github_intel/config/__init__.py +1 -0
- lupaxa/github_intel/config/defaults.py +150 -0
- lupaxa/github_intel/config/editor.py +316 -0
- lupaxa/github_intel/config/loader.py +412 -0
- lupaxa/github_intel/config/models.py +310 -0
- lupaxa/github_intel/config/paths.py +84 -0
- lupaxa/github_intel/config/profiles.py +143 -0
- lupaxa/github_intel/constants.py +54 -0
- lupaxa/github_intel/dates.py +179 -0
- lupaxa/github_intel/errors.py +72 -0
- lupaxa/github_intel/github/__init__.py +1 -0
- lupaxa/github_intel/github/auth.py +40 -0
- lupaxa/github_intel/github/client.py +239 -0
- lupaxa/github_intel/github/rate_limit.py +107 -0
- lupaxa/github_intel/layout.py +141 -0
- lupaxa/github_intel/logging_setup.py +51 -0
- lupaxa/github_intel/present.py +361 -0
- lupaxa/github_intel/reports/__init__.py +1 -0
- lupaxa/github_intel/reports/render.py +113 -0
- lupaxa/github_intel/storage/__init__.py +1 -0
- lupaxa/github_intel/storage/database.py +419 -0
- lupaxa/github_intel/sync/__init__.py +1 -0
- lupaxa/github_intel/sync/engine.py +671 -0
- lupaxa/github_intel/sync/summary.py +308 -0
- lupaxa/github_intel/tui/__init__.py +1 -0
- lupaxa/github_intel/tui/application.py +487 -0
- lupaxa/github_intel/views.py +896 -0
- lupaxa_github_intel-0.1.0.dist-info/METADATA +180 -0
- lupaxa_github_intel-0.1.0.dist-info/RECORD +41 -0
- lupaxa_github_intel-0.1.0.dist-info/WHEEL +5 -0
- lupaxa_github_intel-0.1.0.dist-info/entry_points.txt +2 -0
- lupaxa_github_intel-0.1.0.dist-info/licenses/LICENCE +21 -0
- lupaxa_github_intel-0.1.0.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,671 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Synchronise enabled repositories.
|
|
3
|
+
|
|
4
|
+
One repository failure does not cancel the others. Repositories are visited
|
|
5
|
+
in the same order as the default list: organisation, then name, ignoring
|
|
6
|
+
case. Up to the client's concurrency run at once. Completed rows are stored
|
|
7
|
+
as they finish.
|
|
8
|
+
"""
|
|
9
|
+
|
|
10
|
+
from __future__ import annotations
|
|
11
|
+
|
|
12
|
+
import asyncio
|
|
13
|
+
from collections.abc import Callable
|
|
14
|
+
from dataclasses import dataclass, field
|
|
15
|
+
import time
|
|
16
|
+
from typing import Any
|
|
17
|
+
from urllib.parse import quote
|
|
18
|
+
|
|
19
|
+
from lupaxa.github_intel.checks.base import CheckResult
|
|
20
|
+
from lupaxa.github_intel.checks.releases import evaluate_releases
|
|
21
|
+
from lupaxa.github_intel.checks.workflows import evaluate_workflows
|
|
22
|
+
from lupaxa.github_intel.config.loader import LoadedConfig
|
|
23
|
+
from lupaxa.github_intel.config.models import RepositorySpec
|
|
24
|
+
from lupaxa.github_intel.config.profiles import check_defaults, resolve_profile
|
|
25
|
+
from lupaxa.github_intel.github.client import FetchResult, GitHubClient
|
|
26
|
+
from lupaxa.github_intel.storage.database import Database, Document, StoredCheck
|
|
27
|
+
from lupaxa.github_intel.sync.summary import (
|
|
28
|
+
ahead_count,
|
|
29
|
+
codeql_reading,
|
|
30
|
+
collection_count,
|
|
31
|
+
dependabot_reading,
|
|
32
|
+
issue_count,
|
|
33
|
+
latest_published,
|
|
34
|
+
latest_tag,
|
|
35
|
+
workflow_definitions,
|
|
36
|
+
)
|
|
37
|
+
|
|
38
|
+
__all__ = [
|
|
39
|
+
"SyncReport",
|
|
40
|
+
"synchronise",
|
|
41
|
+
]
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
@dataclass
|
|
45
|
+
class SyncReport:
|
|
46
|
+
"""Repository outcomes and HTTP counters for one run."""
|
|
47
|
+
|
|
48
|
+
repositories: int = 0
|
|
49
|
+
successful: int = 0
|
|
50
|
+
failed: int = 0
|
|
51
|
+
skipped: int = 0
|
|
52
|
+
cancelled: bool = False
|
|
53
|
+
auth_failed: bool = False
|
|
54
|
+
requests: int = 0
|
|
55
|
+
cache_hits: int = 0
|
|
56
|
+
not_modified: int = 0
|
|
57
|
+
elapsed: float = 0.0
|
|
58
|
+
failures: list[str] = field(default_factory=list)
|
|
59
|
+
check_failures: int = 0
|
|
60
|
+
latest: str = ""
|
|
61
|
+
rate_pauses: int = 0
|
|
62
|
+
rate_wait: float = 0.0
|
|
63
|
+
on_progress: Callable[[SyncReport], None] | None = field(default=None, repr=False)
|
|
64
|
+
|
|
65
|
+
|
|
66
|
+
async def synchronise(
|
|
67
|
+
loaded: LoadedConfig,
|
|
68
|
+
database: Database,
|
|
69
|
+
client: GitHubClient,
|
|
70
|
+
*,
|
|
71
|
+
force: bool,
|
|
72
|
+
selected: tuple[RepositorySpec, ...],
|
|
73
|
+
on_progress: Callable[[SyncReport], None] | None = None,
|
|
74
|
+
) -> SyncReport:
|
|
75
|
+
"""
|
|
76
|
+
Refresh stale categories for the selected repositories.
|
|
77
|
+
|
|
78
|
+
Args:
|
|
79
|
+
loaded: Validated configuration.
|
|
80
|
+
database: Cache.
|
|
81
|
+
client: GitHub client.
|
|
82
|
+
force: Ignore freshness windows.
|
|
83
|
+
selected: Enabled repositories to visit. Order in this tuple is ignored.
|
|
84
|
+
on_progress: Called as counters change. It must not await.
|
|
85
|
+
|
|
86
|
+
Returns:
|
|
87
|
+
Run statistics. Cancelled runs keep rows already written.
|
|
88
|
+
|
|
89
|
+
"""
|
|
90
|
+
|
|
91
|
+
ordered = tuple(sorted(selected, key=lambda item: item.repo.casefold()))
|
|
92
|
+
report = SyncReport(repositories=len(ordered))
|
|
93
|
+
report.on_progress = on_progress
|
|
94
|
+
|
|
95
|
+
def waiting(seconds: float) -> None:
|
|
96
|
+
report.rate_wait = seconds
|
|
97
|
+
report.rate_pauses = client.rate_pauses
|
|
98
|
+
_notify(report)
|
|
99
|
+
|
|
100
|
+
client.watch_rate_limit(waiting)
|
|
101
|
+
_notify(report)
|
|
102
|
+
lock = asyncio.Lock()
|
|
103
|
+
pending = list(ordered)
|
|
104
|
+
|
|
105
|
+
async def worker() -> None:
|
|
106
|
+
while True:
|
|
107
|
+
async with lock:
|
|
108
|
+
if not pending:
|
|
109
|
+
return
|
|
110
|
+
repository = pending.pop(0)
|
|
111
|
+
report.latest = repository.repo
|
|
112
|
+
_notify(report)
|
|
113
|
+
await _sync_one(
|
|
114
|
+
loaded,
|
|
115
|
+
database,
|
|
116
|
+
client,
|
|
117
|
+
repository,
|
|
118
|
+
force,
|
|
119
|
+
report,
|
|
120
|
+
lock,
|
|
121
|
+
)
|
|
122
|
+
|
|
123
|
+
width = min(client.concurrency, len(ordered))
|
|
124
|
+
tasks = [asyncio.create_task(worker()) for _ in range(width)]
|
|
125
|
+
try:
|
|
126
|
+
await asyncio.gather(*tasks)
|
|
127
|
+
except asyncio.CancelledError:
|
|
128
|
+
report.cancelled = True
|
|
129
|
+
current = asyncio.current_task()
|
|
130
|
+
if current is not None:
|
|
131
|
+
while current.uncancel():
|
|
132
|
+
pass
|
|
133
|
+
for task in tasks:
|
|
134
|
+
task.cancel()
|
|
135
|
+
await asyncio.gather(*tasks, return_exceptions=True)
|
|
136
|
+
return report
|
|
137
|
+
|
|
138
|
+
|
|
139
|
+
async def _sync_one(
|
|
140
|
+
loaded: LoadedConfig,
|
|
141
|
+
database: Database,
|
|
142
|
+
client: GitHubClient,
|
|
143
|
+
repository: RepositorySpec,
|
|
144
|
+
force: bool,
|
|
145
|
+
report: SyncReport,
|
|
146
|
+
lock: asyncio.Lock,
|
|
147
|
+
) -> None:
|
|
148
|
+
try:
|
|
149
|
+
checks = await _refresh(
|
|
150
|
+
loaded,
|
|
151
|
+
database,
|
|
152
|
+
client,
|
|
153
|
+
repository,
|
|
154
|
+
force,
|
|
155
|
+
report,
|
|
156
|
+
lock,
|
|
157
|
+
)
|
|
158
|
+
await _store_checks(database, repository.repo, checks, report, lock)
|
|
159
|
+
except asyncio.CancelledError:
|
|
160
|
+
raise
|
|
161
|
+
except Exception as exc:
|
|
162
|
+
async with lock:
|
|
163
|
+
report.failed += 1
|
|
164
|
+
report.failures.append(f"{repository.repo}: {exc}")
|
|
165
|
+
_notify(report)
|
|
166
|
+
|
|
167
|
+
|
|
168
|
+
async def _refresh(
|
|
169
|
+
loaded: LoadedConfig,
|
|
170
|
+
database: Database,
|
|
171
|
+
client: GitHubClient,
|
|
172
|
+
repository: RepositorySpec,
|
|
173
|
+
force: bool,
|
|
174
|
+
report: SyncReport,
|
|
175
|
+
lock: asyncio.Lock,
|
|
176
|
+
) -> list[CheckResult]:
|
|
177
|
+
resolved = resolve_profile(
|
|
178
|
+
loaded.profiles,
|
|
179
|
+
repository.profile or loaded.config.defaults.profile,
|
|
180
|
+
repository.checks,
|
|
181
|
+
**check_defaults(loaded.config.defaults),
|
|
182
|
+
)
|
|
183
|
+
metadata = await _category(
|
|
184
|
+
database,
|
|
185
|
+
client,
|
|
186
|
+
repository.repo,
|
|
187
|
+
"metadata",
|
|
188
|
+
f"/repos/{repository.repo}",
|
|
189
|
+
loaded.config.sync.cache.metadata_ttl,
|
|
190
|
+
force,
|
|
191
|
+
loaded.config.sync.cache.enabled,
|
|
192
|
+
report,
|
|
193
|
+
lock,
|
|
194
|
+
)
|
|
195
|
+
meta = _mapping(metadata)
|
|
196
|
+
checks: list[CheckResult] = []
|
|
197
|
+
if resolved.enabled["releases"]:
|
|
198
|
+
checks.append(
|
|
199
|
+
await _releases(
|
|
200
|
+
loaded, database, client, repository, meta, force, report, lock
|
|
201
|
+
),
|
|
202
|
+
)
|
|
203
|
+
workflow_states: list[dict[str, str]] | None = None
|
|
204
|
+
workflow_headline = "No workflows"
|
|
205
|
+
if resolved.enabled["workflows"]:
|
|
206
|
+
workflow_states = await _workflow_inventory(
|
|
207
|
+
client,
|
|
208
|
+
repository.repo,
|
|
209
|
+
report,
|
|
210
|
+
lock,
|
|
211
|
+
)
|
|
212
|
+
result = evaluate_workflows(
|
|
213
|
+
workflow_states,
|
|
214
|
+
None if workflow_states is not None else "Could not list workflows.",
|
|
215
|
+
)
|
|
216
|
+
checks.append(result)
|
|
217
|
+
workflow_headline = result.status
|
|
218
|
+
await _summary(
|
|
219
|
+
loaded,
|
|
220
|
+
database,
|
|
221
|
+
client,
|
|
222
|
+
repository,
|
|
223
|
+
meta,
|
|
224
|
+
metadata,
|
|
225
|
+
force,
|
|
226
|
+
report,
|
|
227
|
+
lock,
|
|
228
|
+
workflow_headline,
|
|
229
|
+
workflow_states,
|
|
230
|
+
resolved.enabled["pull_requests"],
|
|
231
|
+
resolved.enabled["quality"],
|
|
232
|
+
resolved.enabled["issues"],
|
|
233
|
+
)
|
|
234
|
+
return checks
|
|
235
|
+
|
|
236
|
+
|
|
237
|
+
async def _releases(
|
|
238
|
+
loaded: LoadedConfig,
|
|
239
|
+
database: Database,
|
|
240
|
+
client: GitHubClient,
|
|
241
|
+
repository: RepositorySpec,
|
|
242
|
+
meta: dict[str, Any],
|
|
243
|
+
force: bool,
|
|
244
|
+
report: SyncReport,
|
|
245
|
+
lock: asyncio.Lock,
|
|
246
|
+
) -> CheckResult:
|
|
247
|
+
repo = repository.repo
|
|
248
|
+
ttl = loaded.config.sync.cache.releases_ttl
|
|
249
|
+
enabled = loaded.config.sync.cache.enabled
|
|
250
|
+
releases = await _category(
|
|
251
|
+
database,
|
|
252
|
+
client,
|
|
253
|
+
repo,
|
|
254
|
+
"releases",
|
|
255
|
+
f"/repos/{repo}/releases",
|
|
256
|
+
ttl,
|
|
257
|
+
force,
|
|
258
|
+
enabled,
|
|
259
|
+
report,
|
|
260
|
+
lock,
|
|
261
|
+
params={"per_page": "20"},
|
|
262
|
+
)
|
|
263
|
+
branch = str(meta.get("default_branch") or "HEAD")
|
|
264
|
+
published = [
|
|
265
|
+
item
|
|
266
|
+
for item in _list(releases)
|
|
267
|
+
if isinstance(
|
|
268
|
+
item,
|
|
269
|
+
dict,
|
|
270
|
+
)
|
|
271
|
+
and not item.get("draft")
|
|
272
|
+
]
|
|
273
|
+
tag = str(published[0].get("tag_name")) if published else ""
|
|
274
|
+
compare: Document | None = None
|
|
275
|
+
if tag:
|
|
276
|
+
quoted = quote(tag, safe="")
|
|
277
|
+
compare = await _category(
|
|
278
|
+
database,
|
|
279
|
+
client,
|
|
280
|
+
repo,
|
|
281
|
+
f"compare:{tag}",
|
|
282
|
+
f"/repos/{repo}/compare/{quoted}...{quote(branch, safe='')}",
|
|
283
|
+
ttl,
|
|
284
|
+
force,
|
|
285
|
+
enabled,
|
|
286
|
+
report,
|
|
287
|
+
lock,
|
|
288
|
+
)
|
|
289
|
+
return evaluate_releases(
|
|
290
|
+
_list(releases) if releases and releases.error is None else None,
|
|
291
|
+
compare_status=_compare_status(compare),
|
|
292
|
+
ahead_by=_ahead_by(compare),
|
|
293
|
+
compare_error=None
|
|
294
|
+
if compare is None or compare.error is None
|
|
295
|
+
else compare.error,
|
|
296
|
+
fetch_error=None if releases is None else releases.error,
|
|
297
|
+
)
|
|
298
|
+
|
|
299
|
+
|
|
300
|
+
async def _summary(
|
|
301
|
+
loaded: LoadedConfig,
|
|
302
|
+
database: Database,
|
|
303
|
+
client: GitHubClient,
|
|
304
|
+
repository: RepositorySpec,
|
|
305
|
+
meta: dict[str, Any],
|
|
306
|
+
metadata: Document | None,
|
|
307
|
+
force: bool,
|
|
308
|
+
report: SyncReport,
|
|
309
|
+
lock: asyncio.Lock,
|
|
310
|
+
workflow: str,
|
|
311
|
+
workflows: list[dict[str, str]] | None,
|
|
312
|
+
fetch_pulls: bool,
|
|
313
|
+
fetch_quality: bool,
|
|
314
|
+
count_issues: bool,
|
|
315
|
+
) -> None:
|
|
316
|
+
repo = repository.repo
|
|
317
|
+
current = await database.get(repo, "summary")
|
|
318
|
+
fresh = _fresh(
|
|
319
|
+
current,
|
|
320
|
+
loaded.config.sync.cache.issues_ttl,
|
|
321
|
+
force,
|
|
322
|
+
loaded.config.sync.cache.enabled,
|
|
323
|
+
)
|
|
324
|
+
if fresh:
|
|
325
|
+
async with lock:
|
|
326
|
+
report.cache_hits += 1
|
|
327
|
+
_notify(report)
|
|
328
|
+
return
|
|
329
|
+
if metadata is not None and metadata.error and "404" in metadata.error:
|
|
330
|
+
await _save_summary(database, repo, _empty_summary())
|
|
331
|
+
return
|
|
332
|
+
pulls = None
|
|
333
|
+
if fetch_pulls:
|
|
334
|
+
pulls = await _counted(
|
|
335
|
+
client,
|
|
336
|
+
report,
|
|
337
|
+
lock,
|
|
338
|
+
f"/repos/{repo}/pulls",
|
|
339
|
+
{"state": "open", "per_page": "1"},
|
|
340
|
+
)
|
|
341
|
+
alerts = None
|
|
342
|
+
if fetch_quality:
|
|
343
|
+
alerts = await _counted(
|
|
344
|
+
client,
|
|
345
|
+
report,
|
|
346
|
+
lock,
|
|
347
|
+
f"/repos/{repo}/code-scanning/alerts",
|
|
348
|
+
{"tool_name": "CodeQL", "state": "open", "per_page": "1"},
|
|
349
|
+
)
|
|
350
|
+
dependencies = await _counted(
|
|
351
|
+
client,
|
|
352
|
+
report,
|
|
353
|
+
lock,
|
|
354
|
+
f"/repos/{repo}/dependabot/alerts",
|
|
355
|
+
{"state": "open", "per_page": "1"},
|
|
356
|
+
)
|
|
357
|
+
open_pulls = (
|
|
358
|
+
None
|
|
359
|
+
if pulls is None
|
|
360
|
+
else collection_count(
|
|
361
|
+
body=pulls.body,
|
|
362
|
+
link=pulls.link,
|
|
363
|
+
failed=pulls.error is not None,
|
|
364
|
+
)
|
|
365
|
+
)
|
|
366
|
+
if alerts is None:
|
|
367
|
+
codeql, codeql_reason = None, ""
|
|
368
|
+
else:
|
|
369
|
+
codeql, codeql_reason = codeql_reading(
|
|
370
|
+
status=alerts.status,
|
|
371
|
+
error=alerts.error,
|
|
372
|
+
body=alerts.body,
|
|
373
|
+
link=alerts.link,
|
|
374
|
+
)
|
|
375
|
+
dependency_count, dependency_reason = dependabot_reading(
|
|
376
|
+
status=dependencies.status,
|
|
377
|
+
error=dependencies.error,
|
|
378
|
+
body=dependencies.body,
|
|
379
|
+
link=dependencies.link,
|
|
380
|
+
)
|
|
381
|
+
release_body = await _release_body(
|
|
382
|
+
database,
|
|
383
|
+
client,
|
|
384
|
+
repo,
|
|
385
|
+
report,
|
|
386
|
+
lock,
|
|
387
|
+
)
|
|
388
|
+
version = latest_tag(release_body)
|
|
389
|
+
branch = str(meta.get("default_branch") or "HEAD")
|
|
390
|
+
ahead = await _ahead(
|
|
391
|
+
database,
|
|
392
|
+
client,
|
|
393
|
+
repo,
|
|
394
|
+
version,
|
|
395
|
+
branch,
|
|
396
|
+
report,
|
|
397
|
+
lock,
|
|
398
|
+
)
|
|
399
|
+
await _save_summary(
|
|
400
|
+
database,
|
|
401
|
+
repo,
|
|
402
|
+
{
|
|
403
|
+
"version": version,
|
|
404
|
+
"open_pulls": open_pulls,
|
|
405
|
+
"open_issues": (
|
|
406
|
+
issue_count(meta.get("open_issues_count")) if count_issues else None
|
|
407
|
+
),
|
|
408
|
+
"codeql": codeql if fetch_quality else None,
|
|
409
|
+
"codeql_reason": codeql_reason if fetch_quality else "",
|
|
410
|
+
"release_date": latest_published(release_body),
|
|
411
|
+
"ahead": ahead,
|
|
412
|
+
"workflow": workflow,
|
|
413
|
+
"workflows": workflows,
|
|
414
|
+
"dependencies": dependency_count,
|
|
415
|
+
"dependencies_reason": dependency_reason,
|
|
416
|
+
},
|
|
417
|
+
)
|
|
418
|
+
|
|
419
|
+
|
|
420
|
+
async def _counted(
|
|
421
|
+
client: GitHubClient,
|
|
422
|
+
report: SyncReport,
|
|
423
|
+
lock: asyncio.Lock,
|
|
424
|
+
path: str,
|
|
425
|
+
params: dict[str, str],
|
|
426
|
+
) -> FetchResult:
|
|
427
|
+
result = await client.get(path, params=params)
|
|
428
|
+
async with lock:
|
|
429
|
+
report.requests += 1
|
|
430
|
+
report.rate_pauses = client.rate_pauses
|
|
431
|
+
if result.status == 401:
|
|
432
|
+
report.auth_failed = True
|
|
433
|
+
_notify(report)
|
|
434
|
+
return result
|
|
435
|
+
|
|
436
|
+
|
|
437
|
+
async def _release_body(
|
|
438
|
+
database: Database,
|
|
439
|
+
client: GitHubClient,
|
|
440
|
+
repo: str,
|
|
441
|
+
report: SyncReport,
|
|
442
|
+
lock: asyncio.Lock,
|
|
443
|
+
) -> object:
|
|
444
|
+
releases = await database.get(repo, "releases")
|
|
445
|
+
if releases is not None and releases.error is None:
|
|
446
|
+
return releases.json_body()
|
|
447
|
+
fetched = await _counted(
|
|
448
|
+
client,
|
|
449
|
+
report,
|
|
450
|
+
lock,
|
|
451
|
+
f"/repos/{repo}/releases",
|
|
452
|
+
{"per_page": "20"},
|
|
453
|
+
)
|
|
454
|
+
if fetched.error is not None:
|
|
455
|
+
return []
|
|
456
|
+
return fetched.body
|
|
457
|
+
|
|
458
|
+
|
|
459
|
+
async def _ahead(
|
|
460
|
+
database: Database,
|
|
461
|
+
client: GitHubClient,
|
|
462
|
+
repo: str,
|
|
463
|
+
tag: str,
|
|
464
|
+
branch: str,
|
|
465
|
+
report: SyncReport,
|
|
466
|
+
lock: asyncio.Lock,
|
|
467
|
+
) -> int | None:
|
|
468
|
+
if not tag:
|
|
469
|
+
return None
|
|
470
|
+
compare = await database.get(repo, f"compare:{tag}")
|
|
471
|
+
if compare is not None and compare.error is None:
|
|
472
|
+
return ahead_count(compare.json_body())
|
|
473
|
+
quoted = quote(tag, safe="")
|
|
474
|
+
head = quote(branch, safe="")
|
|
475
|
+
fetched = await _counted(
|
|
476
|
+
client,
|
|
477
|
+
report,
|
|
478
|
+
lock,
|
|
479
|
+
f"/repos/{repo}/compare/{quoted}...{head}",
|
|
480
|
+
{},
|
|
481
|
+
)
|
|
482
|
+
if fetched.error is not None:
|
|
483
|
+
return None
|
|
484
|
+
return ahead_count(fetched.body)
|
|
485
|
+
|
|
486
|
+
|
|
487
|
+
async def _workflow_inventory(
|
|
488
|
+
client: GitHubClient,
|
|
489
|
+
repo: str,
|
|
490
|
+
report: SyncReport,
|
|
491
|
+
lock: asyncio.Lock,
|
|
492
|
+
) -> list[dict[str, str]] | None:
|
|
493
|
+
fetched = await _counted(
|
|
494
|
+
client,
|
|
495
|
+
report,
|
|
496
|
+
lock,
|
|
497
|
+
f"/repos/{repo}/actions/workflows",
|
|
498
|
+
{"per_page": "100"},
|
|
499
|
+
)
|
|
500
|
+
if fetched.error is not None:
|
|
501
|
+
return None
|
|
502
|
+
return [
|
|
503
|
+
{"name": name, "status": state}
|
|
504
|
+
for name, state in workflow_definitions(_page_items(fetched.body, "workflows"))
|
|
505
|
+
]
|
|
506
|
+
|
|
507
|
+
|
|
508
|
+
def _page_items(body: object, key: str) -> list[object]:
|
|
509
|
+
if not isinstance(body, dict):
|
|
510
|
+
return []
|
|
511
|
+
batch = body.get(key)
|
|
512
|
+
if not isinstance(batch, list):
|
|
513
|
+
return []
|
|
514
|
+
return batch
|
|
515
|
+
|
|
516
|
+
|
|
517
|
+
def _empty_summary() -> dict[str, object]:
|
|
518
|
+
return {
|
|
519
|
+
"version": "",
|
|
520
|
+
"open_pulls": None,
|
|
521
|
+
"open_issues": None,
|
|
522
|
+
"codeql": None,
|
|
523
|
+
"codeql_reason": "",
|
|
524
|
+
"release_date": "",
|
|
525
|
+
"ahead": None,
|
|
526
|
+
"workflow": "No workflows",
|
|
527
|
+
"workflows": [],
|
|
528
|
+
"dependencies": None,
|
|
529
|
+
"dependencies_reason": "",
|
|
530
|
+
}
|
|
531
|
+
|
|
532
|
+
|
|
533
|
+
async def _save_summary(
|
|
534
|
+
database: Database,
|
|
535
|
+
repo: str,
|
|
536
|
+
body: dict[str, object],
|
|
537
|
+
) -> None:
|
|
538
|
+
await database.save(
|
|
539
|
+
repo,
|
|
540
|
+
"summary",
|
|
541
|
+
body=body,
|
|
542
|
+
etag=None,
|
|
543
|
+
error=None,
|
|
544
|
+
keep_body=False,
|
|
545
|
+
)
|
|
546
|
+
|
|
547
|
+
|
|
548
|
+
async def _category(
|
|
549
|
+
database: Database,
|
|
550
|
+
client: GitHubClient,
|
|
551
|
+
repo: str,
|
|
552
|
+
category: str,
|
|
553
|
+
path: str,
|
|
554
|
+
ttl: int,
|
|
555
|
+
force: bool,
|
|
556
|
+
cache_enabled: bool,
|
|
557
|
+
report: SyncReport,
|
|
558
|
+
lock: asyncio.Lock,
|
|
559
|
+
params: dict[str, str] | None = None,
|
|
560
|
+
) -> Document | None:
|
|
561
|
+
current = await database.get(repo, category)
|
|
562
|
+
if _fresh(current, ttl, force, cache_enabled):
|
|
563
|
+
async with lock:
|
|
564
|
+
report.cache_hits += 1
|
|
565
|
+
_notify(report)
|
|
566
|
+
return current
|
|
567
|
+
result = await client.get(
|
|
568
|
+
path,
|
|
569
|
+
etag=None if current is None else current.etag,
|
|
570
|
+
params=params,
|
|
571
|
+
)
|
|
572
|
+
async with lock:
|
|
573
|
+
report.requests += 1
|
|
574
|
+
report.rate_pauses = client.rate_pauses
|
|
575
|
+
if result.status == 401:
|
|
576
|
+
report.auth_failed = True
|
|
577
|
+
if result.not_modified:
|
|
578
|
+
report.not_modified += 1
|
|
579
|
+
_notify(report)
|
|
580
|
+
await _store_fetch(database, repo, category, current, result)
|
|
581
|
+
return await database.get(repo, category)
|
|
582
|
+
|
|
583
|
+
|
|
584
|
+
async def _store_fetch(
|
|
585
|
+
database: Database,
|
|
586
|
+
repo: str,
|
|
587
|
+
category: str,
|
|
588
|
+
current: Document | None,
|
|
589
|
+
result: FetchResult,
|
|
590
|
+
) -> None:
|
|
591
|
+
if result.not_modified and current is not None:
|
|
592
|
+
await database.save(
|
|
593
|
+
repo,
|
|
594
|
+
category,
|
|
595
|
+
etag=result.etag,
|
|
596
|
+
error=None,
|
|
597
|
+
keep_body=True,
|
|
598
|
+
)
|
|
599
|
+
return
|
|
600
|
+
if result.error:
|
|
601
|
+
await database.save(
|
|
602
|
+
repo,
|
|
603
|
+
category,
|
|
604
|
+
body=None if current is None else current.json_body(),
|
|
605
|
+
etag=result.etag,
|
|
606
|
+
error=result.error,
|
|
607
|
+
keep_body=False,
|
|
608
|
+
)
|
|
609
|
+
return
|
|
610
|
+
await database.save(
|
|
611
|
+
repo,
|
|
612
|
+
category,
|
|
613
|
+
body=result.body,
|
|
614
|
+
etag=result.etag,
|
|
615
|
+
error=None,
|
|
616
|
+
keep_body=False,
|
|
617
|
+
)
|
|
618
|
+
|
|
619
|
+
|
|
620
|
+
async def _store_checks(
|
|
621
|
+
database: Database,
|
|
622
|
+
repo: str,
|
|
623
|
+
checks: list[CheckResult],
|
|
624
|
+
report: SyncReport,
|
|
625
|
+
lock: asyncio.Lock,
|
|
626
|
+
) -> None:
|
|
627
|
+
moment = time.time()
|
|
628
|
+
rows = [
|
|
629
|
+
StoredCheck(repo, item.name, item.status, item.summary, item.failure, moment)
|
|
630
|
+
for item in checks
|
|
631
|
+
]
|
|
632
|
+
await database.save_checks(repo, rows)
|
|
633
|
+
async with lock:
|
|
634
|
+
report.successful += 1
|
|
635
|
+
report.check_failures += sum(1 for item in checks if item.failure)
|
|
636
|
+
_notify(report)
|
|
637
|
+
|
|
638
|
+
|
|
639
|
+
def _notify(report: SyncReport) -> None:
|
|
640
|
+
if report.on_progress is not None:
|
|
641
|
+
report.on_progress(report)
|
|
642
|
+
|
|
643
|
+
|
|
644
|
+
def _fresh(document: Document | None, ttl: int, force: bool, enabled: bool) -> bool:
|
|
645
|
+
if force or not enabled or document is None or document.fetched_at is None:
|
|
646
|
+
return False
|
|
647
|
+
return time.time() - document.fetched_at < ttl
|
|
648
|
+
|
|
649
|
+
|
|
650
|
+
def _mapping(document: Document | None) -> dict[str, Any]:
|
|
651
|
+
if document is None or document.error:
|
|
652
|
+
return {}
|
|
653
|
+
body = document.json_body()
|
|
654
|
+
return body if isinstance(body, dict) else {}
|
|
655
|
+
|
|
656
|
+
|
|
657
|
+
def _list(document: Document | None) -> list[Any]:
|
|
658
|
+
if document is None or document.error:
|
|
659
|
+
return []
|
|
660
|
+
body = document.json_body()
|
|
661
|
+
return body if isinstance(body, list) else []
|
|
662
|
+
|
|
663
|
+
|
|
664
|
+
def _compare_status(document: Document | None) -> str | None:
|
|
665
|
+
status = _mapping(document).get("status")
|
|
666
|
+
return status if isinstance(status, str) else None
|
|
667
|
+
|
|
668
|
+
|
|
669
|
+
def _ahead_by(document: Document | None) -> int | None:
|
|
670
|
+
value = _mapping(document).get("ahead_by")
|
|
671
|
+
return value if isinstance(value, int) else None
|