lupaxa-github-intel 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (41) hide show
  1. lupaxa/github_intel/__init__.py +13 -0
  2. lupaxa/github_intel/__main__.py +8 -0
  3. lupaxa/github_intel/app.py +493 -0
  4. lupaxa/github_intel/checks/__init__.py +1 -0
  5. lupaxa/github_intel/checks/base.py +17 -0
  6. lupaxa/github_intel/checks/releases.py +59 -0
  7. lupaxa/github_intel/checks/workflows.py +34 -0
  8. lupaxa/github_intel/cli.py +699 -0
  9. lupaxa/github_intel/config/__init__.py +1 -0
  10. lupaxa/github_intel/config/defaults.py +150 -0
  11. lupaxa/github_intel/config/editor.py +316 -0
  12. lupaxa/github_intel/config/loader.py +412 -0
  13. lupaxa/github_intel/config/models.py +310 -0
  14. lupaxa/github_intel/config/paths.py +84 -0
  15. lupaxa/github_intel/config/profiles.py +143 -0
  16. lupaxa/github_intel/constants.py +54 -0
  17. lupaxa/github_intel/dates.py +179 -0
  18. lupaxa/github_intel/errors.py +72 -0
  19. lupaxa/github_intel/github/__init__.py +1 -0
  20. lupaxa/github_intel/github/auth.py +40 -0
  21. lupaxa/github_intel/github/client.py +239 -0
  22. lupaxa/github_intel/github/rate_limit.py +107 -0
  23. lupaxa/github_intel/layout.py +141 -0
  24. lupaxa/github_intel/logging_setup.py +51 -0
  25. lupaxa/github_intel/present.py +361 -0
  26. lupaxa/github_intel/reports/__init__.py +1 -0
  27. lupaxa/github_intel/reports/render.py +113 -0
  28. lupaxa/github_intel/storage/__init__.py +1 -0
  29. lupaxa/github_intel/storage/database.py +419 -0
  30. lupaxa/github_intel/sync/__init__.py +1 -0
  31. lupaxa/github_intel/sync/engine.py +671 -0
  32. lupaxa/github_intel/sync/summary.py +308 -0
  33. lupaxa/github_intel/tui/__init__.py +1 -0
  34. lupaxa/github_intel/tui/application.py +487 -0
  35. lupaxa/github_intel/views.py +896 -0
  36. lupaxa_github_intel-0.1.0.dist-info/METADATA +180 -0
  37. lupaxa_github_intel-0.1.0.dist-info/RECORD +41 -0
  38. lupaxa_github_intel-0.1.0.dist-info/WHEEL +5 -0
  39. lupaxa_github_intel-0.1.0.dist-info/entry_points.txt +2 -0
  40. lupaxa_github_intel-0.1.0.dist-info/licenses/LICENCE +21 -0
  41. lupaxa_github_intel-0.1.0.dist-info/top_level.txt +1 -0
@@ -0,0 +1,671 @@
1
+ """
2
+ Synchronise enabled repositories.
3
+
4
+ One repository failure does not cancel the others. Repositories are visited
5
+ in the same order as the default list: organisation, then name, ignoring
6
+ case. Up to the client's concurrency run at once. Completed rows are stored
7
+ as they finish.
8
+ """
9
+
10
+ from __future__ import annotations
11
+
12
+ import asyncio
13
+ from collections.abc import Callable
14
+ from dataclasses import dataclass, field
15
+ import time
16
+ from typing import Any
17
+ from urllib.parse import quote
18
+
19
+ from lupaxa.github_intel.checks.base import CheckResult
20
+ from lupaxa.github_intel.checks.releases import evaluate_releases
21
+ from lupaxa.github_intel.checks.workflows import evaluate_workflows
22
+ from lupaxa.github_intel.config.loader import LoadedConfig
23
+ from lupaxa.github_intel.config.models import RepositorySpec
24
+ from lupaxa.github_intel.config.profiles import check_defaults, resolve_profile
25
+ from lupaxa.github_intel.github.client import FetchResult, GitHubClient
26
+ from lupaxa.github_intel.storage.database import Database, Document, StoredCheck
27
+ from lupaxa.github_intel.sync.summary import (
28
+ ahead_count,
29
+ codeql_reading,
30
+ collection_count,
31
+ dependabot_reading,
32
+ issue_count,
33
+ latest_published,
34
+ latest_tag,
35
+ workflow_definitions,
36
+ )
37
+
38
+ __all__ = [
39
+ "SyncReport",
40
+ "synchronise",
41
+ ]
42
+
43
+
44
+ @dataclass
45
+ class SyncReport:
46
+ """Repository outcomes and HTTP counters for one run."""
47
+
48
+ repositories: int = 0
49
+ successful: int = 0
50
+ failed: int = 0
51
+ skipped: int = 0
52
+ cancelled: bool = False
53
+ auth_failed: bool = False
54
+ requests: int = 0
55
+ cache_hits: int = 0
56
+ not_modified: int = 0
57
+ elapsed: float = 0.0
58
+ failures: list[str] = field(default_factory=list)
59
+ check_failures: int = 0
60
+ latest: str = ""
61
+ rate_pauses: int = 0
62
+ rate_wait: float = 0.0
63
+ on_progress: Callable[[SyncReport], None] | None = field(default=None, repr=False)
64
+
65
+
66
+ async def synchronise(
67
+ loaded: LoadedConfig,
68
+ database: Database,
69
+ client: GitHubClient,
70
+ *,
71
+ force: bool,
72
+ selected: tuple[RepositorySpec, ...],
73
+ on_progress: Callable[[SyncReport], None] | None = None,
74
+ ) -> SyncReport:
75
+ """
76
+ Refresh stale categories for the selected repositories.
77
+
78
+ Args:
79
+ loaded: Validated configuration.
80
+ database: Cache.
81
+ client: GitHub client.
82
+ force: Ignore freshness windows.
83
+ selected: Enabled repositories to visit. Order in this tuple is ignored.
84
+ on_progress: Called as counters change. It must not await.
85
+
86
+ Returns:
87
+ Run statistics. Cancelled runs keep rows already written.
88
+
89
+ """
90
+
91
+ ordered = tuple(sorted(selected, key=lambda item: item.repo.casefold()))
92
+ report = SyncReport(repositories=len(ordered))
93
+ report.on_progress = on_progress
94
+
95
+ def waiting(seconds: float) -> None:
96
+ report.rate_wait = seconds
97
+ report.rate_pauses = client.rate_pauses
98
+ _notify(report)
99
+
100
+ client.watch_rate_limit(waiting)
101
+ _notify(report)
102
+ lock = asyncio.Lock()
103
+ pending = list(ordered)
104
+
105
+ async def worker() -> None:
106
+ while True:
107
+ async with lock:
108
+ if not pending:
109
+ return
110
+ repository = pending.pop(0)
111
+ report.latest = repository.repo
112
+ _notify(report)
113
+ await _sync_one(
114
+ loaded,
115
+ database,
116
+ client,
117
+ repository,
118
+ force,
119
+ report,
120
+ lock,
121
+ )
122
+
123
+ width = min(client.concurrency, len(ordered))
124
+ tasks = [asyncio.create_task(worker()) for _ in range(width)]
125
+ try:
126
+ await asyncio.gather(*tasks)
127
+ except asyncio.CancelledError:
128
+ report.cancelled = True
129
+ current = asyncio.current_task()
130
+ if current is not None:
131
+ while current.uncancel():
132
+ pass
133
+ for task in tasks:
134
+ task.cancel()
135
+ await asyncio.gather(*tasks, return_exceptions=True)
136
+ return report
137
+
138
+
139
+ async def _sync_one(
140
+ loaded: LoadedConfig,
141
+ database: Database,
142
+ client: GitHubClient,
143
+ repository: RepositorySpec,
144
+ force: bool,
145
+ report: SyncReport,
146
+ lock: asyncio.Lock,
147
+ ) -> None:
148
+ try:
149
+ checks = await _refresh(
150
+ loaded,
151
+ database,
152
+ client,
153
+ repository,
154
+ force,
155
+ report,
156
+ lock,
157
+ )
158
+ await _store_checks(database, repository.repo, checks, report, lock)
159
+ except asyncio.CancelledError:
160
+ raise
161
+ except Exception as exc:
162
+ async with lock:
163
+ report.failed += 1
164
+ report.failures.append(f"{repository.repo}: {exc}")
165
+ _notify(report)
166
+
167
+
168
+ async def _refresh(
169
+ loaded: LoadedConfig,
170
+ database: Database,
171
+ client: GitHubClient,
172
+ repository: RepositorySpec,
173
+ force: bool,
174
+ report: SyncReport,
175
+ lock: asyncio.Lock,
176
+ ) -> list[CheckResult]:
177
+ resolved = resolve_profile(
178
+ loaded.profiles,
179
+ repository.profile or loaded.config.defaults.profile,
180
+ repository.checks,
181
+ **check_defaults(loaded.config.defaults),
182
+ )
183
+ metadata = await _category(
184
+ database,
185
+ client,
186
+ repository.repo,
187
+ "metadata",
188
+ f"/repos/{repository.repo}",
189
+ loaded.config.sync.cache.metadata_ttl,
190
+ force,
191
+ loaded.config.sync.cache.enabled,
192
+ report,
193
+ lock,
194
+ )
195
+ meta = _mapping(metadata)
196
+ checks: list[CheckResult] = []
197
+ if resolved.enabled["releases"]:
198
+ checks.append(
199
+ await _releases(
200
+ loaded, database, client, repository, meta, force, report, lock
201
+ ),
202
+ )
203
+ workflow_states: list[dict[str, str]] | None = None
204
+ workflow_headline = "No workflows"
205
+ if resolved.enabled["workflows"]:
206
+ workflow_states = await _workflow_inventory(
207
+ client,
208
+ repository.repo,
209
+ report,
210
+ lock,
211
+ )
212
+ result = evaluate_workflows(
213
+ workflow_states,
214
+ None if workflow_states is not None else "Could not list workflows.",
215
+ )
216
+ checks.append(result)
217
+ workflow_headline = result.status
218
+ await _summary(
219
+ loaded,
220
+ database,
221
+ client,
222
+ repository,
223
+ meta,
224
+ metadata,
225
+ force,
226
+ report,
227
+ lock,
228
+ workflow_headline,
229
+ workflow_states,
230
+ resolved.enabled["pull_requests"],
231
+ resolved.enabled["quality"],
232
+ resolved.enabled["issues"],
233
+ )
234
+ return checks
235
+
236
+
237
+ async def _releases(
238
+ loaded: LoadedConfig,
239
+ database: Database,
240
+ client: GitHubClient,
241
+ repository: RepositorySpec,
242
+ meta: dict[str, Any],
243
+ force: bool,
244
+ report: SyncReport,
245
+ lock: asyncio.Lock,
246
+ ) -> CheckResult:
247
+ repo = repository.repo
248
+ ttl = loaded.config.sync.cache.releases_ttl
249
+ enabled = loaded.config.sync.cache.enabled
250
+ releases = await _category(
251
+ database,
252
+ client,
253
+ repo,
254
+ "releases",
255
+ f"/repos/{repo}/releases",
256
+ ttl,
257
+ force,
258
+ enabled,
259
+ report,
260
+ lock,
261
+ params={"per_page": "20"},
262
+ )
263
+ branch = str(meta.get("default_branch") or "HEAD")
264
+ published = [
265
+ item
266
+ for item in _list(releases)
267
+ if isinstance(
268
+ item,
269
+ dict,
270
+ )
271
+ and not item.get("draft")
272
+ ]
273
+ tag = str(published[0].get("tag_name")) if published else ""
274
+ compare: Document | None = None
275
+ if tag:
276
+ quoted = quote(tag, safe="")
277
+ compare = await _category(
278
+ database,
279
+ client,
280
+ repo,
281
+ f"compare:{tag}",
282
+ f"/repos/{repo}/compare/{quoted}...{quote(branch, safe='')}",
283
+ ttl,
284
+ force,
285
+ enabled,
286
+ report,
287
+ lock,
288
+ )
289
+ return evaluate_releases(
290
+ _list(releases) if releases and releases.error is None else None,
291
+ compare_status=_compare_status(compare),
292
+ ahead_by=_ahead_by(compare),
293
+ compare_error=None
294
+ if compare is None or compare.error is None
295
+ else compare.error,
296
+ fetch_error=None if releases is None else releases.error,
297
+ )
298
+
299
+
300
+ async def _summary(
301
+ loaded: LoadedConfig,
302
+ database: Database,
303
+ client: GitHubClient,
304
+ repository: RepositorySpec,
305
+ meta: dict[str, Any],
306
+ metadata: Document | None,
307
+ force: bool,
308
+ report: SyncReport,
309
+ lock: asyncio.Lock,
310
+ workflow: str,
311
+ workflows: list[dict[str, str]] | None,
312
+ fetch_pulls: bool,
313
+ fetch_quality: bool,
314
+ count_issues: bool,
315
+ ) -> None:
316
+ repo = repository.repo
317
+ current = await database.get(repo, "summary")
318
+ fresh = _fresh(
319
+ current,
320
+ loaded.config.sync.cache.issues_ttl,
321
+ force,
322
+ loaded.config.sync.cache.enabled,
323
+ )
324
+ if fresh:
325
+ async with lock:
326
+ report.cache_hits += 1
327
+ _notify(report)
328
+ return
329
+ if metadata is not None and metadata.error and "404" in metadata.error:
330
+ await _save_summary(database, repo, _empty_summary())
331
+ return
332
+ pulls = None
333
+ if fetch_pulls:
334
+ pulls = await _counted(
335
+ client,
336
+ report,
337
+ lock,
338
+ f"/repos/{repo}/pulls",
339
+ {"state": "open", "per_page": "1"},
340
+ )
341
+ alerts = None
342
+ if fetch_quality:
343
+ alerts = await _counted(
344
+ client,
345
+ report,
346
+ lock,
347
+ f"/repos/{repo}/code-scanning/alerts",
348
+ {"tool_name": "CodeQL", "state": "open", "per_page": "1"},
349
+ )
350
+ dependencies = await _counted(
351
+ client,
352
+ report,
353
+ lock,
354
+ f"/repos/{repo}/dependabot/alerts",
355
+ {"state": "open", "per_page": "1"},
356
+ )
357
+ open_pulls = (
358
+ None
359
+ if pulls is None
360
+ else collection_count(
361
+ body=pulls.body,
362
+ link=pulls.link,
363
+ failed=pulls.error is not None,
364
+ )
365
+ )
366
+ if alerts is None:
367
+ codeql, codeql_reason = None, ""
368
+ else:
369
+ codeql, codeql_reason = codeql_reading(
370
+ status=alerts.status,
371
+ error=alerts.error,
372
+ body=alerts.body,
373
+ link=alerts.link,
374
+ )
375
+ dependency_count, dependency_reason = dependabot_reading(
376
+ status=dependencies.status,
377
+ error=dependencies.error,
378
+ body=dependencies.body,
379
+ link=dependencies.link,
380
+ )
381
+ release_body = await _release_body(
382
+ database,
383
+ client,
384
+ repo,
385
+ report,
386
+ lock,
387
+ )
388
+ version = latest_tag(release_body)
389
+ branch = str(meta.get("default_branch") or "HEAD")
390
+ ahead = await _ahead(
391
+ database,
392
+ client,
393
+ repo,
394
+ version,
395
+ branch,
396
+ report,
397
+ lock,
398
+ )
399
+ await _save_summary(
400
+ database,
401
+ repo,
402
+ {
403
+ "version": version,
404
+ "open_pulls": open_pulls,
405
+ "open_issues": (
406
+ issue_count(meta.get("open_issues_count")) if count_issues else None
407
+ ),
408
+ "codeql": codeql if fetch_quality else None,
409
+ "codeql_reason": codeql_reason if fetch_quality else "",
410
+ "release_date": latest_published(release_body),
411
+ "ahead": ahead,
412
+ "workflow": workflow,
413
+ "workflows": workflows,
414
+ "dependencies": dependency_count,
415
+ "dependencies_reason": dependency_reason,
416
+ },
417
+ )
418
+
419
+
420
+ async def _counted(
421
+ client: GitHubClient,
422
+ report: SyncReport,
423
+ lock: asyncio.Lock,
424
+ path: str,
425
+ params: dict[str, str],
426
+ ) -> FetchResult:
427
+ result = await client.get(path, params=params)
428
+ async with lock:
429
+ report.requests += 1
430
+ report.rate_pauses = client.rate_pauses
431
+ if result.status == 401:
432
+ report.auth_failed = True
433
+ _notify(report)
434
+ return result
435
+
436
+
437
+ async def _release_body(
438
+ database: Database,
439
+ client: GitHubClient,
440
+ repo: str,
441
+ report: SyncReport,
442
+ lock: asyncio.Lock,
443
+ ) -> object:
444
+ releases = await database.get(repo, "releases")
445
+ if releases is not None and releases.error is None:
446
+ return releases.json_body()
447
+ fetched = await _counted(
448
+ client,
449
+ report,
450
+ lock,
451
+ f"/repos/{repo}/releases",
452
+ {"per_page": "20"},
453
+ )
454
+ if fetched.error is not None:
455
+ return []
456
+ return fetched.body
457
+
458
+
459
+ async def _ahead(
460
+ database: Database,
461
+ client: GitHubClient,
462
+ repo: str,
463
+ tag: str,
464
+ branch: str,
465
+ report: SyncReport,
466
+ lock: asyncio.Lock,
467
+ ) -> int | None:
468
+ if not tag:
469
+ return None
470
+ compare = await database.get(repo, f"compare:{tag}")
471
+ if compare is not None and compare.error is None:
472
+ return ahead_count(compare.json_body())
473
+ quoted = quote(tag, safe="")
474
+ head = quote(branch, safe="")
475
+ fetched = await _counted(
476
+ client,
477
+ report,
478
+ lock,
479
+ f"/repos/{repo}/compare/{quoted}...{head}",
480
+ {},
481
+ )
482
+ if fetched.error is not None:
483
+ return None
484
+ return ahead_count(fetched.body)
485
+
486
+
487
+ async def _workflow_inventory(
488
+ client: GitHubClient,
489
+ repo: str,
490
+ report: SyncReport,
491
+ lock: asyncio.Lock,
492
+ ) -> list[dict[str, str]] | None:
493
+ fetched = await _counted(
494
+ client,
495
+ report,
496
+ lock,
497
+ f"/repos/{repo}/actions/workflows",
498
+ {"per_page": "100"},
499
+ )
500
+ if fetched.error is not None:
501
+ return None
502
+ return [
503
+ {"name": name, "status": state}
504
+ for name, state in workflow_definitions(_page_items(fetched.body, "workflows"))
505
+ ]
506
+
507
+
508
+ def _page_items(body: object, key: str) -> list[object]:
509
+ if not isinstance(body, dict):
510
+ return []
511
+ batch = body.get(key)
512
+ if not isinstance(batch, list):
513
+ return []
514
+ return batch
515
+
516
+
517
+ def _empty_summary() -> dict[str, object]:
518
+ return {
519
+ "version": "",
520
+ "open_pulls": None,
521
+ "open_issues": None,
522
+ "codeql": None,
523
+ "codeql_reason": "",
524
+ "release_date": "",
525
+ "ahead": None,
526
+ "workflow": "No workflows",
527
+ "workflows": [],
528
+ "dependencies": None,
529
+ "dependencies_reason": "",
530
+ }
531
+
532
+
533
+ async def _save_summary(
534
+ database: Database,
535
+ repo: str,
536
+ body: dict[str, object],
537
+ ) -> None:
538
+ await database.save(
539
+ repo,
540
+ "summary",
541
+ body=body,
542
+ etag=None,
543
+ error=None,
544
+ keep_body=False,
545
+ )
546
+
547
+
548
+ async def _category(
549
+ database: Database,
550
+ client: GitHubClient,
551
+ repo: str,
552
+ category: str,
553
+ path: str,
554
+ ttl: int,
555
+ force: bool,
556
+ cache_enabled: bool,
557
+ report: SyncReport,
558
+ lock: asyncio.Lock,
559
+ params: dict[str, str] | None = None,
560
+ ) -> Document | None:
561
+ current = await database.get(repo, category)
562
+ if _fresh(current, ttl, force, cache_enabled):
563
+ async with lock:
564
+ report.cache_hits += 1
565
+ _notify(report)
566
+ return current
567
+ result = await client.get(
568
+ path,
569
+ etag=None if current is None else current.etag,
570
+ params=params,
571
+ )
572
+ async with lock:
573
+ report.requests += 1
574
+ report.rate_pauses = client.rate_pauses
575
+ if result.status == 401:
576
+ report.auth_failed = True
577
+ if result.not_modified:
578
+ report.not_modified += 1
579
+ _notify(report)
580
+ await _store_fetch(database, repo, category, current, result)
581
+ return await database.get(repo, category)
582
+
583
+
584
+ async def _store_fetch(
585
+ database: Database,
586
+ repo: str,
587
+ category: str,
588
+ current: Document | None,
589
+ result: FetchResult,
590
+ ) -> None:
591
+ if result.not_modified and current is not None:
592
+ await database.save(
593
+ repo,
594
+ category,
595
+ etag=result.etag,
596
+ error=None,
597
+ keep_body=True,
598
+ )
599
+ return
600
+ if result.error:
601
+ await database.save(
602
+ repo,
603
+ category,
604
+ body=None if current is None else current.json_body(),
605
+ etag=result.etag,
606
+ error=result.error,
607
+ keep_body=False,
608
+ )
609
+ return
610
+ await database.save(
611
+ repo,
612
+ category,
613
+ body=result.body,
614
+ etag=result.etag,
615
+ error=None,
616
+ keep_body=False,
617
+ )
618
+
619
+
620
+ async def _store_checks(
621
+ database: Database,
622
+ repo: str,
623
+ checks: list[CheckResult],
624
+ report: SyncReport,
625
+ lock: asyncio.Lock,
626
+ ) -> None:
627
+ moment = time.time()
628
+ rows = [
629
+ StoredCheck(repo, item.name, item.status, item.summary, item.failure, moment)
630
+ for item in checks
631
+ ]
632
+ await database.save_checks(repo, rows)
633
+ async with lock:
634
+ report.successful += 1
635
+ report.check_failures += sum(1 for item in checks if item.failure)
636
+ _notify(report)
637
+
638
+
639
+ def _notify(report: SyncReport) -> None:
640
+ if report.on_progress is not None:
641
+ report.on_progress(report)
642
+
643
+
644
+ def _fresh(document: Document | None, ttl: int, force: bool, enabled: bool) -> bool:
645
+ if force or not enabled or document is None or document.fetched_at is None:
646
+ return False
647
+ return time.time() - document.fetched_at < ttl
648
+
649
+
650
+ def _mapping(document: Document | None) -> dict[str, Any]:
651
+ if document is None or document.error:
652
+ return {}
653
+ body = document.json_body()
654
+ return body if isinstance(body, dict) else {}
655
+
656
+
657
+ def _list(document: Document | None) -> list[Any]:
658
+ if document is None or document.error:
659
+ return []
660
+ body = document.json_body()
661
+ return body if isinstance(body, list) else []
662
+
663
+
664
+ def _compare_status(document: Document | None) -> str | None:
665
+ status = _mapping(document).get("status")
666
+ return status if isinstance(status, str) else None
667
+
668
+
669
+ def _ahead_by(document: Document | None) -> int | None:
670
+ value = _mapping(document).get("ahead_by")
671
+ return value if isinstance(value, int) else None