lupaxa-github-intel 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- lupaxa/github_intel/__init__.py +13 -0
- lupaxa/github_intel/__main__.py +8 -0
- lupaxa/github_intel/app.py +493 -0
- lupaxa/github_intel/checks/__init__.py +1 -0
- lupaxa/github_intel/checks/base.py +17 -0
- lupaxa/github_intel/checks/releases.py +59 -0
- lupaxa/github_intel/checks/workflows.py +34 -0
- lupaxa/github_intel/cli.py +699 -0
- lupaxa/github_intel/config/__init__.py +1 -0
- lupaxa/github_intel/config/defaults.py +150 -0
- lupaxa/github_intel/config/editor.py +316 -0
- lupaxa/github_intel/config/loader.py +412 -0
- lupaxa/github_intel/config/models.py +310 -0
- lupaxa/github_intel/config/paths.py +84 -0
- lupaxa/github_intel/config/profiles.py +143 -0
- lupaxa/github_intel/constants.py +54 -0
- lupaxa/github_intel/dates.py +179 -0
- lupaxa/github_intel/errors.py +72 -0
- lupaxa/github_intel/github/__init__.py +1 -0
- lupaxa/github_intel/github/auth.py +40 -0
- lupaxa/github_intel/github/client.py +239 -0
- lupaxa/github_intel/github/rate_limit.py +107 -0
- lupaxa/github_intel/layout.py +141 -0
- lupaxa/github_intel/logging_setup.py +51 -0
- lupaxa/github_intel/present.py +361 -0
- lupaxa/github_intel/reports/__init__.py +1 -0
- lupaxa/github_intel/reports/render.py +113 -0
- lupaxa/github_intel/storage/__init__.py +1 -0
- lupaxa/github_intel/storage/database.py +419 -0
- lupaxa/github_intel/sync/__init__.py +1 -0
- lupaxa/github_intel/sync/engine.py +671 -0
- lupaxa/github_intel/sync/summary.py +308 -0
- lupaxa/github_intel/tui/__init__.py +1 -0
- lupaxa/github_intel/tui/application.py +487 -0
- lupaxa/github_intel/views.py +896 -0
- lupaxa_github_intel-0.1.0.dist-info/METADATA +180 -0
- lupaxa_github_intel-0.1.0.dist-info/RECORD +41 -0
- lupaxa_github_intel-0.1.0.dist-info/WHEEL +5 -0
- lupaxa_github_intel-0.1.0.dist-info/entry_points.txt +2 -0
- lupaxa_github_intel-0.1.0.dist-info/licenses/LICENCE +21 -0
- lupaxa_github_intel-0.1.0.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,412 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Load YAML or JSON configuration and apply environment overrides.
|
|
3
|
+
|
|
4
|
+
CLI code applies its own overrides after this loader. Environment values
|
|
5
|
+
win over the file. Invalid environment values are errors.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
from __future__ import annotations
|
|
9
|
+
|
|
10
|
+
from dataclasses import dataclass
|
|
11
|
+
import json
|
|
12
|
+
import os
|
|
13
|
+
from pathlib import Path
|
|
14
|
+
from typing import Any, TypeVar, cast
|
|
15
|
+
|
|
16
|
+
from pydantic import ValidationError
|
|
17
|
+
from ruamel.yaml import YAML
|
|
18
|
+
from ruamel.yaml.constructor import DuplicateKeyError
|
|
19
|
+
|
|
20
|
+
from lupaxa.github_intel.config.models import (
|
|
21
|
+
AppConfig,
|
|
22
|
+
ProfileSpec,
|
|
23
|
+
RepositorySpec,
|
|
24
|
+
)
|
|
25
|
+
from lupaxa.github_intel.config.paths import default_config_file
|
|
26
|
+
from lupaxa.github_intel.constants import UNGROUPED
|
|
27
|
+
from lupaxa.github_intel.errors import ConfigError
|
|
28
|
+
|
|
29
|
+
__all__ = [
|
|
30
|
+
"LoadedConfig",
|
|
31
|
+
"load_config",
|
|
32
|
+
"repository_key",
|
|
33
|
+
"validate_repository_file",
|
|
34
|
+
]
|
|
35
|
+
|
|
36
|
+
ModelT = TypeVar("ModelT", AppConfig, ProfileSpec, RepositorySpec)
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
@dataclass(frozen=True)
|
|
40
|
+
class LoadedConfig:
|
|
41
|
+
"""A validated configuration and the repository records it names."""
|
|
42
|
+
|
|
43
|
+
path: Path
|
|
44
|
+
config: AppConfig
|
|
45
|
+
profiles: dict[str, ProfileSpec]
|
|
46
|
+
repositories: tuple[RepositorySpec, ...]
|
|
47
|
+
profiles_path: Path
|
|
48
|
+
repositories_path: Path
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
def load_config(path: Path | None = None) -> LoadedConfig:
|
|
52
|
+
"""
|
|
53
|
+
Load the main file plus the profile and repository files it names.
|
|
54
|
+
|
|
55
|
+
Args:
|
|
56
|
+
path: Main configuration file. Defaults to the XDG config file.
|
|
57
|
+
|
|
58
|
+
Returns:
|
|
59
|
+
Validated configuration.
|
|
60
|
+
|
|
61
|
+
Raises:
|
|
62
|
+
ConfigError: A file is missing, unreadable, or invalid.
|
|
63
|
+
|
|
64
|
+
"""
|
|
65
|
+
|
|
66
|
+
config_path = _main_path(path)
|
|
67
|
+
document = _read_document(config_path)
|
|
68
|
+
config = _parse_model(AppConfig, document, str(config_path))
|
|
69
|
+
config = _apply_environment(config)
|
|
70
|
+
root = config_path.parent
|
|
71
|
+
profiles_path = _resolve_file(root, config.files.profiles)
|
|
72
|
+
repositories_path = _resolve_file(root, config.files.repositories)
|
|
73
|
+
profiles = _load_profiles(profiles_path)
|
|
74
|
+
repositories = _load_repositories(repositories_path, config.defaults.profile)
|
|
75
|
+
_require_known_profiles(profiles, repositories, config.defaults.profile)
|
|
76
|
+
return LoadedConfig(
|
|
77
|
+
path=config_path,
|
|
78
|
+
config=config,
|
|
79
|
+
profiles=profiles,
|
|
80
|
+
repositories=repositories,
|
|
81
|
+
profiles_path=profiles_path,
|
|
82
|
+
repositories_path=repositories_path,
|
|
83
|
+
)
|
|
84
|
+
|
|
85
|
+
|
|
86
|
+
def validate_repository_file(
|
|
87
|
+
path: Path,
|
|
88
|
+
default_profile: str,
|
|
89
|
+
profiles: dict[str, ProfileSpec],
|
|
90
|
+
) -> tuple[RepositorySpec, ...]:
|
|
91
|
+
"""
|
|
92
|
+
Validate a repository file against the already loaded profiles.
|
|
93
|
+
|
|
94
|
+
Args:
|
|
95
|
+
path: Repository YAML or JSON file.
|
|
96
|
+
default_profile: Profile used when an entry omits one.
|
|
97
|
+
profiles: Known profiles.
|
|
98
|
+
|
|
99
|
+
Returns:
|
|
100
|
+
Parsed repositories.
|
|
101
|
+
|
|
102
|
+
Raises:
|
|
103
|
+
ConfigError: The file or a profile reference is invalid.
|
|
104
|
+
|
|
105
|
+
"""
|
|
106
|
+
|
|
107
|
+
repositories = _load_repositories(path, default_profile)
|
|
108
|
+
_require_known_profiles(profiles, repositories, default_profile)
|
|
109
|
+
return repositories
|
|
110
|
+
|
|
111
|
+
|
|
112
|
+
def repository_key(name: str) -> str:
|
|
113
|
+
"""
|
|
114
|
+
Return the case-insensitive identity of an ``owner/name`` string.
|
|
115
|
+
|
|
116
|
+
Args:
|
|
117
|
+
name: Repository identifier.
|
|
118
|
+
|
|
119
|
+
Returns:
|
|
120
|
+
Folded identifier used to detect duplicates.
|
|
121
|
+
|
|
122
|
+
"""
|
|
123
|
+
|
|
124
|
+
owner, _, repo = name.partition("/")
|
|
125
|
+
return f"{owner.casefold()}/{repo.casefold()}"
|
|
126
|
+
|
|
127
|
+
|
|
128
|
+
def _main_path(path: Path | None) -> Path:
|
|
129
|
+
if path is not None:
|
|
130
|
+
return path.expanduser()
|
|
131
|
+
env = os.environ.get("GITHUB_INTEL_CONFIG")
|
|
132
|
+
if env is not None and env.strip() != "":
|
|
133
|
+
return Path(env).expanduser()
|
|
134
|
+
return default_config_file()
|
|
135
|
+
|
|
136
|
+
|
|
137
|
+
def _resolve_file(root: Path, reference: str) -> Path:
|
|
138
|
+
candidate = Path(reference).expanduser()
|
|
139
|
+
if not candidate.is_absolute():
|
|
140
|
+
candidate = root / candidate
|
|
141
|
+
return candidate
|
|
142
|
+
|
|
143
|
+
|
|
144
|
+
def _read_document(path: Path) -> dict[str, Any]:
|
|
145
|
+
if not path.is_file():
|
|
146
|
+
raise ConfigError(f"Configuration file not found: {path}")
|
|
147
|
+
suffix = _format_suffix(path)
|
|
148
|
+
if suffix in {".yml", ".yaml"}:
|
|
149
|
+
return _read_yaml(path)
|
|
150
|
+
if suffix == ".json":
|
|
151
|
+
return _read_json(path)
|
|
152
|
+
raise ConfigError(f"Unsupported configuration file: {path.name}")
|
|
153
|
+
|
|
154
|
+
|
|
155
|
+
def _read_yaml(path: Path) -> dict[str, Any]:
|
|
156
|
+
yaml = YAML(typ="safe")
|
|
157
|
+
yaml.allow_duplicate_keys = False
|
|
158
|
+
try:
|
|
159
|
+
loaded = yaml.load(path.read_text(encoding="utf-8"))
|
|
160
|
+
except DuplicateKeyError as exc:
|
|
161
|
+
raise ConfigError(f"Duplicate key in {path}: {exc}") from exc
|
|
162
|
+
except Exception as exc:
|
|
163
|
+
raise ConfigError(f"Invalid YAML in {path}: {exc}") from exc
|
|
164
|
+
return _as_mapping(loaded, path)
|
|
165
|
+
|
|
166
|
+
|
|
167
|
+
def _read_json(path: Path) -> dict[str, Any]:
|
|
168
|
+
def pairs(values: list[tuple[str, Any]]) -> dict[str, Any]:
|
|
169
|
+
result: dict[str, Any] = {}
|
|
170
|
+
for key, value in values:
|
|
171
|
+
if key in result:
|
|
172
|
+
raise ConfigError(f"Duplicate key in {path}: {key}")
|
|
173
|
+
result[key] = value
|
|
174
|
+
return result
|
|
175
|
+
|
|
176
|
+
try:
|
|
177
|
+
loaded = json.loads(path.read_text(encoding="utf-8"), object_pairs_hook=pairs)
|
|
178
|
+
except ConfigError:
|
|
179
|
+
raise
|
|
180
|
+
except json.JSONDecodeError as exc:
|
|
181
|
+
raise ConfigError(f"Invalid JSON in {path}: {exc}") from exc
|
|
182
|
+
return _as_mapping(loaded, path)
|
|
183
|
+
|
|
184
|
+
|
|
185
|
+
def _format_suffix(path: Path) -> str:
|
|
186
|
+
name = path.name[: -len(".part")] if path.name.endswith(".part") else path.name
|
|
187
|
+
return Path(name).suffix.casefold()
|
|
188
|
+
|
|
189
|
+
|
|
190
|
+
def _as_mapping(loaded: object, path: Path) -> dict[str, Any]:
|
|
191
|
+
if not isinstance(loaded, dict):
|
|
192
|
+
raise ConfigError(f"{path} must contain an object.")
|
|
193
|
+
return cast("dict[str, Any]", loaded)
|
|
194
|
+
|
|
195
|
+
|
|
196
|
+
def _parse_model(
|
|
197
|
+
model: type[ModelT],
|
|
198
|
+
document: dict[str, Any],
|
|
199
|
+
label: str,
|
|
200
|
+
) -> ModelT:
|
|
201
|
+
try:
|
|
202
|
+
return model.model_validate(document)
|
|
203
|
+
except ValidationError as exc:
|
|
204
|
+
raise ConfigError(_validation_message(label, exc)) from exc
|
|
205
|
+
|
|
206
|
+
|
|
207
|
+
def _validation_message(label: str, exc: ValidationError) -> str:
|
|
208
|
+
lines = [f"Invalid configuration in {label}:"]
|
|
209
|
+
for error in exc.errors():
|
|
210
|
+
location = ".".join(str(part) for part in error["loc"]) or "(root)"
|
|
211
|
+
lines.append(f" {location}: {error['msg']}")
|
|
212
|
+
return "\n".join(lines)
|
|
213
|
+
|
|
214
|
+
|
|
215
|
+
def _apply_environment(config: AppConfig) -> AppConfig:
|
|
216
|
+
updates: dict[str, Any] = {}
|
|
217
|
+
concurrency = _env_int("GITHUB_INTEL_CONCURRENCY", minimum=1)
|
|
218
|
+
if concurrency is not None:
|
|
219
|
+
updates["sync"] = config.sync.model_copy(update={"concurrency": concurrency})
|
|
220
|
+
api_url = os.environ.get("GITHUB_INTEL_API_URL")
|
|
221
|
+
if api_url is not None and api_url.strip() != "":
|
|
222
|
+
try:
|
|
223
|
+
updates["github"] = type(config.github).model_validate(
|
|
224
|
+
{"api_url": api_url.strip()},
|
|
225
|
+
)
|
|
226
|
+
except ValidationError as exc:
|
|
227
|
+
raise ConfigError(_validation_message("GITHUB_INTEL_API_URL", exc)) from exc
|
|
228
|
+
level = os.environ.get("GITHUB_INTEL_LOG_LEVEL")
|
|
229
|
+
if level is not None and level.strip() != "":
|
|
230
|
+
try:
|
|
231
|
+
updates["logging"] = type(config.logging).model_validate({"level": level})
|
|
232
|
+
except ValidationError as exc:
|
|
233
|
+
raise ConfigError(
|
|
234
|
+
_validation_message("GITHUB_INTEL_LOG_LEVEL", exc),
|
|
235
|
+
) from exc
|
|
236
|
+
if not updates:
|
|
237
|
+
return config
|
|
238
|
+
return config.model_copy(update=updates)
|
|
239
|
+
|
|
240
|
+
|
|
241
|
+
def _env_int(name: str, *, minimum: int) -> int | None:
|
|
242
|
+
raw = os.environ.get(name)
|
|
243
|
+
if raw is None or raw.strip() == "":
|
|
244
|
+
return None
|
|
245
|
+
if not raw.strip().isdigit() or int(raw.strip()) < minimum:
|
|
246
|
+
raise ConfigError(f"{name} must be an integer >= {minimum}.")
|
|
247
|
+
return int(raw.strip())
|
|
248
|
+
|
|
249
|
+
|
|
250
|
+
def _load_profiles(path: Path) -> dict[str, ProfileSpec]:
|
|
251
|
+
document = _read_document(path)
|
|
252
|
+
raw = document.get("profiles")
|
|
253
|
+
if not isinstance(raw, dict):
|
|
254
|
+
raise ConfigError(f"{path} must contain a profiles object.")
|
|
255
|
+
profiles: dict[str, ProfileSpec] = {}
|
|
256
|
+
for name, body in raw.items():
|
|
257
|
+
if not isinstance(name, str) or name.strip() == "":
|
|
258
|
+
raise ConfigError(f"Profile names in {path} must be strings.")
|
|
259
|
+
if not isinstance(body, dict):
|
|
260
|
+
raise ConfigError(f"Profile {name} must be an object.")
|
|
261
|
+
profiles[name] = _parse_model(ProfileSpec, body, f"{path} profile {name}")
|
|
262
|
+
return profiles
|
|
263
|
+
|
|
264
|
+
|
|
265
|
+
_ORG_KEYS = {"name", "group", "profile", "repositories"}
|
|
266
|
+
_REPO_KEYS = {"name", "group", "profile", "tags", "enabled", "checks"}
|
|
267
|
+
|
|
268
|
+
|
|
269
|
+
def _load_repositories(path: Path, default_profile: str) -> tuple[RepositorySpec, ...]:
|
|
270
|
+
document = _read_document(path)
|
|
271
|
+
if "repositories" in document:
|
|
272
|
+
raise ConfigError(
|
|
273
|
+
f"{path} lists owners under organisations, each with its repositories.",
|
|
274
|
+
)
|
|
275
|
+
raw = document.get("organisations", [])
|
|
276
|
+
if not isinstance(raw, list):
|
|
277
|
+
raise ConfigError(f"{path} organisations must be a list.")
|
|
278
|
+
repositories: list[RepositorySpec] = []
|
|
279
|
+
seen: dict[str, str] = {}
|
|
280
|
+
owners: set[str] = set()
|
|
281
|
+
for index, item in enumerate(raw):
|
|
282
|
+
for spec in _one_organisation(item, path, index, default_profile, owners):
|
|
283
|
+
key = repository_key(spec.repo)
|
|
284
|
+
if key in seen:
|
|
285
|
+
raise ConfigError(
|
|
286
|
+
f"Duplicate repository {spec.repo} (already listed as {seen[key]})."
|
|
287
|
+
)
|
|
288
|
+
seen[key] = spec.repo
|
|
289
|
+
repositories.append(spec)
|
|
290
|
+
return tuple(repositories)
|
|
291
|
+
|
|
292
|
+
|
|
293
|
+
def _one_organisation(
|
|
294
|
+
item: object,
|
|
295
|
+
path: Path,
|
|
296
|
+
index: int,
|
|
297
|
+
default_profile: str,
|
|
298
|
+
owners: set[str],
|
|
299
|
+
) -> list[RepositorySpec]:
|
|
300
|
+
label = f"{path} organisations[{index}]"
|
|
301
|
+
if not isinstance(item, dict):
|
|
302
|
+
raise ConfigError(f"{label} must be an object.")
|
|
303
|
+
unknown = sorted(set(item) - _ORG_KEYS)
|
|
304
|
+
if unknown:
|
|
305
|
+
raise ConfigError(f"{label} has unknown keys: {', '.join(unknown)}.")
|
|
306
|
+
raw_name = item.get("name")
|
|
307
|
+
if not isinstance(raw_name, str):
|
|
308
|
+
raise ConfigError(f"{label} name must be a GitHub organisation or user.")
|
|
309
|
+
owner = _owner_name(raw_name, label)
|
|
310
|
+
if owner.casefold() in owners:
|
|
311
|
+
raise ConfigError(f"Duplicate owner {owner}.")
|
|
312
|
+
owners.add(owner.casefold())
|
|
313
|
+
listed = item.get("repositories", [])
|
|
314
|
+
if not isinstance(listed, list):
|
|
315
|
+
raise ConfigError(f"{label} repositories must be a list.")
|
|
316
|
+
group = None
|
|
317
|
+
if "group" in item:
|
|
318
|
+
group = _optional_text(item.get("group"), f"{label} group")
|
|
319
|
+
profile = None
|
|
320
|
+
if "profile" in item:
|
|
321
|
+
profile = _optional_text(item.get("profile"), f"{label} profile")
|
|
322
|
+
specs: list[RepositorySpec] = []
|
|
323
|
+
for repo_index, entry in enumerate(listed):
|
|
324
|
+
specs.append(
|
|
325
|
+
_named_repository(
|
|
326
|
+
entry,
|
|
327
|
+
path,
|
|
328
|
+
index,
|
|
329
|
+
repo_index,
|
|
330
|
+
owner,
|
|
331
|
+
group,
|
|
332
|
+
profile,
|
|
333
|
+
default_profile,
|
|
334
|
+
),
|
|
335
|
+
)
|
|
336
|
+
return specs
|
|
337
|
+
|
|
338
|
+
|
|
339
|
+
def _named_repository(
|
|
340
|
+
item: object,
|
|
341
|
+
path: Path,
|
|
342
|
+
org_index: int,
|
|
343
|
+
index: int,
|
|
344
|
+
owner: str,
|
|
345
|
+
org_group: str | None,
|
|
346
|
+
org_profile: str | None,
|
|
347
|
+
default_profile: str,
|
|
348
|
+
) -> RepositorySpec:
|
|
349
|
+
label = f"{path} organisations[{org_index}].repositories[{index}]"
|
|
350
|
+
if isinstance(item, str):
|
|
351
|
+
return RepositorySpec(
|
|
352
|
+
repo=_canonical(owner, item, label),
|
|
353
|
+
group=org_group or UNGROUPED,
|
|
354
|
+
profile=org_profile or default_profile,
|
|
355
|
+
)
|
|
356
|
+
if not isinstance(item, dict):
|
|
357
|
+
raise ConfigError(f"{label} must be a repository name or an object.")
|
|
358
|
+
unknown = sorted(set(item) - _REPO_KEYS)
|
|
359
|
+
if unknown:
|
|
360
|
+
raise ConfigError(f"{label} has unknown keys: {', '.join(unknown)}.")
|
|
361
|
+
raw_name = item.get("name")
|
|
362
|
+
if not isinstance(raw_name, str):
|
|
363
|
+
raise ConfigError(f"{label} name must be a repository name.")
|
|
364
|
+
body: dict[str, Any] = {
|
|
365
|
+
"repo": _canonical(owner, raw_name, label),
|
|
366
|
+
"group": _chosen_text(item, "group", org_group) or UNGROUPED,
|
|
367
|
+
"profile": _chosen_text(item, "profile", org_profile) or default_profile,
|
|
368
|
+
}
|
|
369
|
+
for key in ("tags", "enabled", "checks"):
|
|
370
|
+
if key in item:
|
|
371
|
+
body[key] = item[key]
|
|
372
|
+
return _parse_model(RepositorySpec, body, label)
|
|
373
|
+
|
|
374
|
+
|
|
375
|
+
def _owner_name(value: str, label: str) -> str:
|
|
376
|
+
canonical = _canonical(value, "name", label)
|
|
377
|
+
owner, _, _name = canonical.partition("/")
|
|
378
|
+
return owner
|
|
379
|
+
|
|
380
|
+
|
|
381
|
+
def _canonical(owner: str, name: str, label: str) -> str:
|
|
382
|
+
try:
|
|
383
|
+
return RepositorySpec(repo=f"{owner}/{name}").repo
|
|
384
|
+
except ValidationError as exc:
|
|
385
|
+
raise ConfigError(f"{label} must be a repository name.") from exc
|
|
386
|
+
|
|
387
|
+
|
|
388
|
+
def _optional_text(value: object, label: str) -> str:
|
|
389
|
+
if not isinstance(value, str) or value.strip() == "":
|
|
390
|
+
raise ConfigError(f"{label} must be text.")
|
|
391
|
+
return value.strip()
|
|
392
|
+
|
|
393
|
+
|
|
394
|
+
def _chosen_text(item: dict[str, Any], key: str, inherited: str | None) -> str | None:
|
|
395
|
+
if key not in item:
|
|
396
|
+
return inherited
|
|
397
|
+
return _optional_text(item[key], key)
|
|
398
|
+
|
|
399
|
+
|
|
400
|
+
def _require_known_profiles(
|
|
401
|
+
profiles: dict[str, ProfileSpec],
|
|
402
|
+
repositories: tuple[RepositorySpec, ...],
|
|
403
|
+
default_profile: str,
|
|
404
|
+
) -> None:
|
|
405
|
+
if default_profile not in profiles:
|
|
406
|
+
raise ConfigError(
|
|
407
|
+
f"defaults.profile references unknown profile {default_profile}.",
|
|
408
|
+
)
|
|
409
|
+
for repository in repositories:
|
|
410
|
+
name = repository.profile or default_profile
|
|
411
|
+
if name not in profiles:
|
|
412
|
+
raise ConfigError(f"{repository.repo} references unknown profile {name}.")
|