romledger 0.2.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- romledger/__init__.py +4 -0
- romledger/canary.py +368 -0
- romledger/cli.py +320 -0
- romledger/config.py +336 -0
- romledger/context.py +20 -0
- romledger/datindex.py +73 -0
- romledger/dats.py +82 -0
- romledger/fetch.py +123 -0
- romledger/http.py +113 -0
- romledger/igir.py +145 -0
- romledger/importer.py +138 -0
- romledger/links.py +19 -0
- romledger/lookup.py +54 -0
- romledger/platforms.toml +165 -0
- romledger/registry.py +61 -0
- romledger/scan.py +91 -0
- romledger/sources/__init__.py +0 -0
- romledger/sources/base.py +134 -0
- romledger/sources/coolrom.py +137 -0
- romledger/sources/errors.py +17 -0
- romledger/sources/internet_archive.py +56 -0
- romledger/state.py +237 -0
- romledger/titles.py +19 -0
- romledger/web/__init__.py +0 -0
- romledger/web/actions.py +58 -0
- romledger/web/api.py +108 -0
- romledger/web/app.py +122 -0
- romledger/web/auth.py +78 -0
- romledger/web/deps.py +51 -0
- romledger/web/jobs.py +133 -0
- romledger/web/pages.py +126 -0
- romledger/web/render.py +33 -0
- romledger/web/static/app.css +860 -0
- romledger/web/static/app.js +119 -0
- romledger/web/templates/_parts.html +27 -0
- romledger/web/templates/activity.html +33 -0
- romledger/web/templates/base.html +51 -0
- romledger/web/templates/error.html +16 -0
- romledger/web/templates/inbox.html +63 -0
- romledger/web/templates/overview.html +49 -0
- romledger/web/templates/platform.html +70 -0
- romledger/web/upload.py +151 -0
- romledger/web/views.py +170 -0
- romledger-0.2.0.dist-info/METADATA +119 -0
- romledger-0.2.0.dist-info/RECORD +48 -0
- romledger-0.2.0.dist-info/WHEEL +4 -0
- romledger-0.2.0.dist-info/entry_points.txt +2 -0
- romledger-0.2.0.dist-info/licenses/LICENSE +21 -0
romledger/__init__.py
ADDED
romledger/canary.py
ADDED
|
@@ -0,0 +1,368 @@
|
|
|
1
|
+
import re
|
|
2
|
+
from collections.abc import Callable, Mapping, Sequence
|
|
3
|
+
from dataclasses import dataclass
|
|
4
|
+
from functools import partial
|
|
5
|
+
|
|
6
|
+
import httpx
|
|
7
|
+
|
|
8
|
+
from romledger import dats, http, igir, registry
|
|
9
|
+
from romledger.sources import coolrom, internet_archive
|
|
10
|
+
from romledger.sources.base import Site
|
|
11
|
+
from romledger.sources.errors import FetchError, SourceBlocked, SourceChanged
|
|
12
|
+
|
|
13
|
+
COOLROM_PLATFORM = "atari7800"
|
|
14
|
+
VIMM_URL = "https://vimm.net"
|
|
15
|
+
VIMM_HOSTS = ("vimm.net",)
|
|
16
|
+
VIMM_SYSTEM = "NES"
|
|
17
|
+
ARCHIVE_ITEM = "No-Intro_NES"
|
|
18
|
+
IGIR_RELEASE_URL = "https://api.github.com/repos/emmercm/igir/releases/latest"
|
|
19
|
+
COOLROM_PLATFORM_LINK = re.compile(r'href="/roms/([a-z0-9]+)/"')
|
|
20
|
+
VIMM_PLATFORM_LINK = re.compile(r'href="/vault/([A-Za-z0-9-]+)"')
|
|
21
|
+
VIMM_ROW_LINK = re.compile(r'href="/vault/\d+"')
|
|
22
|
+
CHALLENGE_MARKERS = ("just a moment", "checking if you are human")
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
@dataclass(frozen=True)
|
|
26
|
+
class Finding:
|
|
27
|
+
key: str
|
|
28
|
+
title: str
|
|
29
|
+
detail: str
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
@dataclass(frozen=True)
|
|
33
|
+
class Report:
|
|
34
|
+
findings: list[Finding]
|
|
35
|
+
notes: list[str]
|
|
36
|
+
inconclusive: list[str]
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
class Inconclusive(Exception):
|
|
40
|
+
pass
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
class Skipped(Exception):
|
|
44
|
+
pass
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
class Drift(Exception):
|
|
48
|
+
pass
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
Check = Callable[[], list[Finding]]
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
@dataclass(frozen=True)
|
|
55
|
+
class Step:
|
|
56
|
+
check: Check
|
|
57
|
+
keys: tuple[str, ...]
|
|
58
|
+
|
|
59
|
+
|
|
60
|
+
def run(
|
|
61
|
+
client: httpx.Client,
|
|
62
|
+
throttle: http.Throttle,
|
|
63
|
+
entries: Mapping[str, registry.Entry],
|
|
64
|
+
ignored: Mapping[str, frozenset[str]],
|
|
65
|
+
pinned_igir: str = igir.IGIR_VERSION,
|
|
66
|
+
token: str | None = None,
|
|
67
|
+
) -> Report:
|
|
68
|
+
return _merge(
|
|
69
|
+
[
|
|
70
|
+
_run_steps("coolrom", _coolrom_steps(client, throttle, entries, ignored["coolrom"])),
|
|
71
|
+
_run_steps("vimm", _vimm_steps(client, throttle, entries, ignored["vimm"])),
|
|
72
|
+
_run_steps("internet-archive", _archive_steps(client, throttle)),
|
|
73
|
+
_run_steps("dat", _dat_steps(client, throttle, entries)),
|
|
74
|
+
_run_steps("igir", _igir_steps(client, throttle, pinned_igir, token)),
|
|
75
|
+
]
|
|
76
|
+
)
|
|
77
|
+
|
|
78
|
+
|
|
79
|
+
def _merge(reports: Sequence[Report]) -> Report:
|
|
80
|
+
return Report(
|
|
81
|
+
[finding for report in reports for finding in report.findings],
|
|
82
|
+
[note for report in reports for note in report.notes],
|
|
83
|
+
[prefix for report in reports for prefix in report.inconclusive],
|
|
84
|
+
)
|
|
85
|
+
|
|
86
|
+
|
|
87
|
+
def _run_steps(group: str, steps: Sequence[Step]) -> Report:
|
|
88
|
+
report = Report([], [], [])
|
|
89
|
+
for index, step in enumerate(steps):
|
|
90
|
+
try:
|
|
91
|
+
report.findings.extend(step.check())
|
|
92
|
+
except Skipped:
|
|
93
|
+
report.inconclusive.extend(step.keys)
|
|
94
|
+
except Inconclusive as note:
|
|
95
|
+
report.notes.append(str(note))
|
|
96
|
+
report.inconclusive.append(group)
|
|
97
|
+
report.inconclusive.extend(key for later in steps[index:] for key in later.keys)
|
|
98
|
+
break
|
|
99
|
+
return report
|
|
100
|
+
|
|
101
|
+
|
|
102
|
+
def _step(keys: tuple[str, ...], title: str, check: Check) -> Step:
|
|
103
|
+
def guarded() -> list[Finding]:
|
|
104
|
+
try:
|
|
105
|
+
return check()
|
|
106
|
+
except Drift as error:
|
|
107
|
+
detail = str(error)
|
|
108
|
+
except SourceChanged as error:
|
|
109
|
+
detail = f"{error.url}: {error.what}"
|
|
110
|
+
return [Finding(keys[0], title, detail)]
|
|
111
|
+
|
|
112
|
+
return Step(guarded, keys)
|
|
113
|
+
|
|
114
|
+
|
|
115
|
+
def _coolrom_steps(
|
|
116
|
+
client: httpx.Client,
|
|
117
|
+
throttle: http.Throttle,
|
|
118
|
+
entries: Mapping[str, registry.Entry],
|
|
119
|
+
ignored: frozenset[str],
|
|
120
|
+
) -> list[Step]:
|
|
121
|
+
site = Site(coolrom.BASE_URL, coolrom.HOSTS, coolrom.MIN_GAP, client, throttle)
|
|
122
|
+
first_rows: list[coolrom.Row] = []
|
|
123
|
+
return [
|
|
124
|
+
_step(
|
|
125
|
+
(
|
|
126
|
+
"coolrom:contract:platform-links",
|
|
127
|
+
"coolrom:missing-platform",
|
|
128
|
+
"coolrom:new-platform",
|
|
129
|
+
),
|
|
130
|
+
"CoolROM platform list changed",
|
|
131
|
+
partial(_coolrom_platforms, site, entries, ignored),
|
|
132
|
+
),
|
|
133
|
+
_step(
|
|
134
|
+
("coolrom:contract:section",),
|
|
135
|
+
"CoolROM section page changed",
|
|
136
|
+
partial(_coolrom_section, site, first_rows),
|
|
137
|
+
),
|
|
138
|
+
_step(
|
|
139
|
+
("coolrom:contract:download-link",),
|
|
140
|
+
"CoolROM download page changed",
|
|
141
|
+
partial(_coolrom_download, site, first_rows),
|
|
142
|
+
),
|
|
143
|
+
]
|
|
144
|
+
|
|
145
|
+
|
|
146
|
+
def _coolrom_platforms(
|
|
147
|
+
site: Site, entries: Mapping[str, registry.Entry], ignored: frozenset[str]
|
|
148
|
+
) -> list[Finding]:
|
|
149
|
+
url = f"{coolrom.BASE_URL}/roms/"
|
|
150
|
+
found = set(COOLROM_PLATFORM_LINK.findall(_page(site, url).text))
|
|
151
|
+
mapped = {entry.coolrom for entry in entries.values() if entry.coolrom}
|
|
152
|
+
return _platform_findings("coolrom", url, found, mapped, ignored)
|
|
153
|
+
|
|
154
|
+
|
|
155
|
+
def _coolrom_section(site: Site, first_rows: list[coolrom.Row]) -> list[Finding]:
|
|
156
|
+
url = f"{coolrom.BASE_URL}/roms/{COOLROM_PLATFORM}/x/"
|
|
157
|
+
rows = coolrom.parse_section(_page(site, url).text, COOLROM_PLATFORM, url)
|
|
158
|
+
if not rows:
|
|
159
|
+
raise Drift(f"{url}: no game rows found")
|
|
160
|
+
first_rows.append(rows[0])
|
|
161
|
+
return []
|
|
162
|
+
|
|
163
|
+
|
|
164
|
+
def _coolrom_download(site: Site, first_rows: list[coolrom.Row]) -> list[Finding]:
|
|
165
|
+
if not first_rows:
|
|
166
|
+
raise Skipped
|
|
167
|
+
url = f"{coolrom.BASE_URL}/dlpop.php?id={first_rows[0].id}"
|
|
168
|
+
coolrom.parse_signed_link(_page(site, url).text, url)
|
|
169
|
+
return []
|
|
170
|
+
|
|
171
|
+
|
|
172
|
+
def _vimm_steps(
|
|
173
|
+
client: httpx.Client,
|
|
174
|
+
throttle: http.Throttle,
|
|
175
|
+
entries: Mapping[str, registry.Entry],
|
|
176
|
+
ignored: frozenset[str],
|
|
177
|
+
) -> list[Step]:
|
|
178
|
+
site = Site(VIMM_URL, VIMM_HOSTS, http.DEFAULT_GAP, client, throttle)
|
|
179
|
+
return [
|
|
180
|
+
_step(
|
|
181
|
+
("vimm:contract:platform-links", "vimm:missing-platform", "vimm:new-platform"),
|
|
182
|
+
"Vimm's Lair platform list changed",
|
|
183
|
+
partial(_vimm_platforms, site, entries, ignored),
|
|
184
|
+
),
|
|
185
|
+
_step(
|
|
186
|
+
("vimm:contract:list",),
|
|
187
|
+
"Vimm's Lair list page changed",
|
|
188
|
+
partial(_vimm_list, site),
|
|
189
|
+
),
|
|
190
|
+
]
|
|
191
|
+
|
|
192
|
+
|
|
193
|
+
def _vimm_platforms(
|
|
194
|
+
site: Site, entries: Mapping[str, registry.Entry], ignored: frozenset[str]
|
|
195
|
+
) -> list[Finding]:
|
|
196
|
+
url = f"{VIMM_URL}/vault/"
|
|
197
|
+
codes = VIMM_PLATFORM_LINK.findall(_page(site, url).text)
|
|
198
|
+
found = {code for code in codes if not code.isdigit()}
|
|
199
|
+
mapped = {entry.vimm for entry in entries.values() if entry.vimm}
|
|
200
|
+
return _platform_findings("vimm", url, found, mapped, ignored)
|
|
201
|
+
|
|
202
|
+
|
|
203
|
+
def _vimm_list(site: Site) -> list[Finding]:
|
|
204
|
+
url = f"{VIMM_URL}/vault/?p=list&system={VIMM_SYSTEM}§ion=x"
|
|
205
|
+
if not VIMM_ROW_LINK.search(_page(site, url).text):
|
|
206
|
+
raise Drift(f"{url}: no game row links found")
|
|
207
|
+
return []
|
|
208
|
+
|
|
209
|
+
|
|
210
|
+
def _platform_findings(
|
|
211
|
+
site: str, url: str, found: set[str], mapped: set[str], ignored: frozenset[str]
|
|
212
|
+
) -> list[Finding]:
|
|
213
|
+
if not found:
|
|
214
|
+
raise Drift(f"{url}: no platform links found")
|
|
215
|
+
findings = [
|
|
216
|
+
Finding(
|
|
217
|
+
f"{site}:missing-platform:{code}",
|
|
218
|
+
f"{site} no longer lists a registry platform: {code}",
|
|
219
|
+
f"{url} has no link for {code}. Correct the {site} code in "
|
|
220
|
+
"src/romledger/platforms.toml.",
|
|
221
|
+
)
|
|
222
|
+
for code in sorted(mapped - found)
|
|
223
|
+
]
|
|
224
|
+
findings.extend(
|
|
225
|
+
Finding(
|
|
226
|
+
f"{site}:new-platform:{code}",
|
|
227
|
+
f"{site} lists a platform missing from the registry: {code}",
|
|
228
|
+
f'{url} links {code}. Add `{site} = "{code}"` to its platform in '
|
|
229
|
+
f"src/romledger/platforms.toml, or add it to [ignore] {site}.",
|
|
230
|
+
)
|
|
231
|
+
for code in sorted(found - mapped - ignored)
|
|
232
|
+
)
|
|
233
|
+
return findings
|
|
234
|
+
|
|
235
|
+
|
|
236
|
+
def _archive_steps(client: httpx.Client, throttle: http.Throttle) -> list[Step]:
|
|
237
|
+
site = Site(
|
|
238
|
+
internet_archive.BASE_URL, internet_archive.HOSTS, http.DEFAULT_GAP, client, throttle
|
|
239
|
+
)
|
|
240
|
+
return [
|
|
241
|
+
_step(
|
|
242
|
+
("internet-archive:contract:metadata",),
|
|
243
|
+
"Internet Archive metadata changed",
|
|
244
|
+
partial(_archive_metadata, site),
|
|
245
|
+
)
|
|
246
|
+
]
|
|
247
|
+
|
|
248
|
+
|
|
249
|
+
def _archive_metadata(site: Site) -> list[Finding]:
|
|
250
|
+
url = f"{internet_archive.BASE_URL}/metadata/{ARCHIVE_ITEM}"
|
|
251
|
+
files = internet_archive.index(_page(site, url), url, ARCHIVE_ITEM)
|
|
252
|
+
if not any(isinstance(entry.get("md5"), str) for entry in files.values()):
|
|
253
|
+
raise Drift(f"{url}: no file entry has an md5")
|
|
254
|
+
return []
|
|
255
|
+
|
|
256
|
+
|
|
257
|
+
def _dat_steps(
|
|
258
|
+
client: httpx.Client, throttle: http.Throttle, entries: Mapping[str, registry.Entry]
|
|
259
|
+
) -> list[Step]:
|
|
260
|
+
return [
|
|
261
|
+
Step(partial(_dat_exists, client, throttle, entry), (f"dat:missing:{entry.slug}",))
|
|
262
|
+
for entry in entries.values()
|
|
263
|
+
]
|
|
264
|
+
|
|
265
|
+
|
|
266
|
+
def _dat_exists(
|
|
267
|
+
client: httpx.Client, throttle: http.Throttle, entry: registry.Entry
|
|
268
|
+
) -> list[Finding]:
|
|
269
|
+
url = dats.url(entry.dat)
|
|
270
|
+
status = _send(client, throttle, "HEAD", url).status_code
|
|
271
|
+
if status == httpx.codes.OK:
|
|
272
|
+
return []
|
|
273
|
+
if status != httpx.codes.NOT_FOUND:
|
|
274
|
+
raise Inconclusive(f"{url}: HTTP {status}")
|
|
275
|
+
return [
|
|
276
|
+
Finding(
|
|
277
|
+
f"dat:missing:{entry.slug}",
|
|
278
|
+
f"libretro-database has no DAT for {entry.slug}",
|
|
279
|
+
f"{url}: HTTP 404. Correct dat for {entry.slug} in src/romledger/platforms.toml.",
|
|
280
|
+
)
|
|
281
|
+
]
|
|
282
|
+
|
|
283
|
+
|
|
284
|
+
def _igir_steps(
|
|
285
|
+
client: httpx.Client, throttle: http.Throttle, pinned: str, token: str | None
|
|
286
|
+
) -> list[Step]:
|
|
287
|
+
return [
|
|
288
|
+
_step(
|
|
289
|
+
("igir:contract:release", "igir:update"),
|
|
290
|
+
"igir release lookup changed",
|
|
291
|
+
partial(_igir_update, client, throttle, pinned, token),
|
|
292
|
+
)
|
|
293
|
+
]
|
|
294
|
+
|
|
295
|
+
|
|
296
|
+
def _igir_update(
|
|
297
|
+
client: httpx.Client, throttle: http.Throttle, pinned: str, token: str | None
|
|
298
|
+
) -> list[Finding]:
|
|
299
|
+
headers = {"Accept": "application/vnd.github+json"}
|
|
300
|
+
if token:
|
|
301
|
+
headers["Authorization"] = f"Bearer {token}"
|
|
302
|
+
response = _send(client, throttle, "GET", IGIR_RELEASE_URL, headers, follow_redirects=False)
|
|
303
|
+
if response.status_code != httpx.codes.OK:
|
|
304
|
+
raise Inconclusive(f"{IGIR_RELEASE_URL}: HTTP {response.status_code}")
|
|
305
|
+
tag = _release_tag(response)
|
|
306
|
+
if tag.removeprefix("v") == pinned:
|
|
307
|
+
return []
|
|
308
|
+
return [
|
|
309
|
+
Finding(
|
|
310
|
+
f"igir:update:{tag}",
|
|
311
|
+
f"igir {tag} is out; romledger pins {pinned}",
|
|
312
|
+
f"Run `uv run pytest -m igir --no-cov`, then set IGIR_VERSION to "
|
|
313
|
+
f"{tag.removeprefix('v')} in src/romledger/igir.py.",
|
|
314
|
+
)
|
|
315
|
+
]
|
|
316
|
+
|
|
317
|
+
|
|
318
|
+
def _release_tag(response: httpx.Response) -> str:
|
|
319
|
+
try:
|
|
320
|
+
tag = response.json()["tag_name"]
|
|
321
|
+
except (ValueError, KeyError, TypeError) as error:
|
|
322
|
+
raise Drift(f"{IGIR_RELEASE_URL}: response has no tag_name") from error
|
|
323
|
+
if not isinstance(tag, str) or not tag:
|
|
324
|
+
raise Drift(f"{IGIR_RELEASE_URL}: tag_name is not a string")
|
|
325
|
+
return tag
|
|
326
|
+
|
|
327
|
+
|
|
328
|
+
def _page(site: Site, url: str) -> httpx.Response:
|
|
329
|
+
try:
|
|
330
|
+
response = site.get(url, expect=(httpx.codes.OK, httpx.codes.NOT_FOUND))
|
|
331
|
+
except (FetchError, SourceBlocked) as error:
|
|
332
|
+
raise Inconclusive(str(error)) from error
|
|
333
|
+
except httpx.HTTPError as error:
|
|
334
|
+
raise Inconclusive(_unreachable(url, error)) from error
|
|
335
|
+
if response.status_code == httpx.codes.NOT_FOUND:
|
|
336
|
+
raise Drift(f"{url}: HTTP 404")
|
|
337
|
+
if _is_challenge(response):
|
|
338
|
+
raise Inconclusive(f"{url}: bot challenge page")
|
|
339
|
+
return response
|
|
340
|
+
|
|
341
|
+
|
|
342
|
+
def _send(
|
|
343
|
+
client: httpx.Client,
|
|
344
|
+
throttle: http.Throttle,
|
|
345
|
+
method: str,
|
|
346
|
+
url: str,
|
|
347
|
+
headers: Mapping[str, str] | None = None,
|
|
348
|
+
*,
|
|
349
|
+
follow_redirects: bool = True,
|
|
350
|
+
) -> httpx.Response:
|
|
351
|
+
throttle.wait(url)
|
|
352
|
+
try:
|
|
353
|
+
return client.request(method, url, headers=headers, follow_redirects=follow_redirects)
|
|
354
|
+
except httpx.HTTPError as error:
|
|
355
|
+
raise Inconclusive(_unreachable(url, error)) from error
|
|
356
|
+
|
|
357
|
+
|
|
358
|
+
def _unreachable(url: str, error: httpx.HTTPError) -> str:
|
|
359
|
+
return f"{url}: request failed ({str(error) or type(error).__name__})"
|
|
360
|
+
|
|
361
|
+
|
|
362
|
+
def _is_challenge(response: httpx.Response) -> bool:
|
|
363
|
+
if response.headers.get("cf-mitigated") == "challenge":
|
|
364
|
+
return True
|
|
365
|
+
if "text/html" not in response.headers.get("content-type", ""):
|
|
366
|
+
return False
|
|
367
|
+
text = response.text.lower()
|
|
368
|
+
return any(marker in text for marker in CHALLENGE_MARKERS)
|
romledger/cli.py
ADDED
|
@@ -0,0 +1,320 @@
|
|
|
1
|
+
import argparse
|
|
2
|
+
import dataclasses
|
|
3
|
+
import functools
|
|
4
|
+
import json
|
|
5
|
+
import os
|
|
6
|
+
import sys
|
|
7
|
+
import webbrowser
|
|
8
|
+
from collections.abc import Callable, Sequence
|
|
9
|
+
|
|
10
|
+
import uvicorn
|
|
11
|
+
|
|
12
|
+
from romledger import (
|
|
13
|
+
__version__,
|
|
14
|
+
canary,
|
|
15
|
+
config,
|
|
16
|
+
datindex,
|
|
17
|
+
dats,
|
|
18
|
+
fetch,
|
|
19
|
+
http,
|
|
20
|
+
igir,
|
|
21
|
+
importer,
|
|
22
|
+
links,
|
|
23
|
+
registry,
|
|
24
|
+
scan,
|
|
25
|
+
state,
|
|
26
|
+
)
|
|
27
|
+
from romledger.context import Context, load_context
|
|
28
|
+
from romledger.web import auth
|
|
29
|
+
from romledger.web.app import create_app
|
|
30
|
+
|
|
31
|
+
DEFAULT_HOST = "127.0.0.1"
|
|
32
|
+
WILDCARD_HOSTS = ("0.0.0.0", "::") # noqa: S104
|
|
33
|
+
DEFAULT_PORT = 8730
|
|
34
|
+
STATUS_COLUMNS = ("wanted", "have", "inbox", "missing", "unmatched", "done")
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
def build_parser() -> argparse.ArgumentParser:
|
|
38
|
+
parser = argparse.ArgumentParser(
|
|
39
|
+
prog="romledger",
|
|
40
|
+
description="Track which retro games you have and which are missing.",
|
|
41
|
+
)
|
|
42
|
+
parser.add_argument("--version", action="version", version=f"%(prog)s {__version__}")
|
|
43
|
+
parser.add_argument(
|
|
44
|
+
"--config",
|
|
45
|
+
metavar="FILE",
|
|
46
|
+
help="config file (default: $ROMLEDGER_CONFIG, else "
|
|
47
|
+
"$XDG_CONFIG_HOME/romledger/config.toml, i.e. ~/.config/romledger/config.toml)",
|
|
48
|
+
)
|
|
49
|
+
commands = parser.add_subparsers(dest="command", required=True, metavar="command")
|
|
50
|
+
commands.add_parser("config", help="validate the config file and print a summary")
|
|
51
|
+
dats_parser = commands.add_parser("dats", help="manage platform DATs")
|
|
52
|
+
dats_commands = dats_parser.add_subparsers(dest="dats_command", required=True, metavar="action")
|
|
53
|
+
update = dats_commands.add_parser("update", help="download the latest DATs")
|
|
54
|
+
update.set_defaults(command="dats update")
|
|
55
|
+
update.add_argument("platforms", nargs="*", metavar="PLATFORM", help="default: all configured")
|
|
56
|
+
index = dats_commands.add_parser("index", help="rebuild the DAT hash index")
|
|
57
|
+
index.set_defaults(command="dats index")
|
|
58
|
+
index.add_argument("platforms", nargs="*", metavar="PLATFORM", help="default: all configured")
|
|
59
|
+
scan_parser = commands.add_parser("scan", help="match library and inbox files to DATs")
|
|
60
|
+
scan_parser.add_argument(
|
|
61
|
+
"platforms", nargs="*", metavar="PLATFORM", help="default: all configured"
|
|
62
|
+
)
|
|
63
|
+
import_parser = commands.add_parser("import", help="file inbox games into the library")
|
|
64
|
+
import_parser.add_argument(
|
|
65
|
+
"--dry-run", action="store_true", help="show what would happen; move nothing"
|
|
66
|
+
)
|
|
67
|
+
commands.add_parser("status", help="show have/missing counts per platform")
|
|
68
|
+
fetch_parser = commands.add_parser(
|
|
69
|
+
"fetch", help="download missing games from configured sources"
|
|
70
|
+
)
|
|
71
|
+
fetch_parser.add_argument("platform", metavar="PLATFORM")
|
|
72
|
+
fetch_parser.add_argument(
|
|
73
|
+
"--limit", type=positive_int, metavar="N", help="fetch at most N games"
|
|
74
|
+
)
|
|
75
|
+
fetch_parser.add_argument(
|
|
76
|
+
"--dry-run", action="store_true", help="show matches; download nothing"
|
|
77
|
+
)
|
|
78
|
+
fetch_parser.add_argument(
|
|
79
|
+
"--no-import", action="store_true", help="leave the downloads in the inbox"
|
|
80
|
+
)
|
|
81
|
+
fetch_parser.add_argument("--source", metavar="NAME", help="use only this configured source")
|
|
82
|
+
canary_parser = commands.add_parser(
|
|
83
|
+
"canary", help="check that the supported sites and upstream pins still match"
|
|
84
|
+
)
|
|
85
|
+
canary_parser.add_argument(
|
|
86
|
+
"--json", action="store_true", help="print findings as JSON; notes go to stderr"
|
|
87
|
+
)
|
|
88
|
+
missing = commands.add_parser("missing", help="list the games a platform is missing")
|
|
89
|
+
missing.add_argument("platform", metavar="PLATFORM")
|
|
90
|
+
missing.add_argument("--links", action="store_true", help="append a Vimm's Lair search URL")
|
|
91
|
+
serve_parser = commands.add_parser("serve", help="run the local dashboard and API")
|
|
92
|
+
serve_parser.add_argument(
|
|
93
|
+
"--host", default=DEFAULT_HOST, help=f"address to bind (default: {DEFAULT_HOST})"
|
|
94
|
+
)
|
|
95
|
+
serve_parser.add_argument(
|
|
96
|
+
"--port", type=port_number, default=DEFAULT_PORT, help=f"port (default: {DEFAULT_PORT})"
|
|
97
|
+
)
|
|
98
|
+
serve_parser.add_argument("--open", action="store_true", help="open the dashboard in a browser")
|
|
99
|
+
return parser
|
|
100
|
+
|
|
101
|
+
|
|
102
|
+
def port_number(text: str) -> int:
|
|
103
|
+
if not text.isdigit() or not 1 <= int(text) <= 65535:
|
|
104
|
+
raise argparse.ArgumentTypeError(f"{text!r} is not a port between 1 and 65535")
|
|
105
|
+
return int(text)
|
|
106
|
+
|
|
107
|
+
|
|
108
|
+
def positive_int(text: str) -> int:
|
|
109
|
+
if not text.isdigit() or int(text) < 1:
|
|
110
|
+
raise argparse.ArgumentTypeError(f"{text!r} is not a positive integer")
|
|
111
|
+
return int(text)
|
|
112
|
+
|
|
113
|
+
|
|
114
|
+
def selected(context: Context, names: Sequence[str]) -> list[str]:
|
|
115
|
+
configured = list(context.config.platforms)
|
|
116
|
+
for name in names:
|
|
117
|
+
if name not in context.config.platforms:
|
|
118
|
+
raise config.ConfigError(
|
|
119
|
+
f"unknown platform {name!r}. Configured platforms: "
|
|
120
|
+
f"{', '.join(configured) or 'none'}."
|
|
121
|
+
)
|
|
122
|
+
return list(dict.fromkeys(names)) or configured
|
|
123
|
+
|
|
124
|
+
|
|
125
|
+
def show_config(args: argparse.Namespace) -> int:
|
|
126
|
+
context = load_context(args.config)
|
|
127
|
+
print(f"config: {context.path}")
|
|
128
|
+
print(f"data: {context.data_dir}")
|
|
129
|
+
print(f"roms: {context.config.library.roms}")
|
|
130
|
+
for inbox in context.config.library.inboxes:
|
|
131
|
+
print(f"inbox: {inbox}")
|
|
132
|
+
for slug, platform in context.config.platforms.items():
|
|
133
|
+
print(f"{slug}: {platform.dat}")
|
|
134
|
+
print(f" regions: {', '.join(platform.regions)}")
|
|
135
|
+
print(f" languages: {', '.join(platform.languages)}")
|
|
136
|
+
if platform.sources:
|
|
137
|
+
print(f" sources: {', '.join(platform.sources)}")
|
|
138
|
+
if platform.archive_item:
|
|
139
|
+
print(f" archive_item: {platform.archive_item}")
|
|
140
|
+
if context.config.network.proxy:
|
|
141
|
+
print(f"proxy: {http.redact(context.config.network.proxy)}")
|
|
142
|
+
for argv in context.config.hooks.after_import:
|
|
143
|
+
print(f"hook: after_import {' '.join(argv)}")
|
|
144
|
+
return 0
|
|
145
|
+
|
|
146
|
+
|
|
147
|
+
def update_dats(args: argparse.Namespace) -> int:
|
|
148
|
+
context = load_context(args.config)
|
|
149
|
+
slugs = selected(context, args.platforms)
|
|
150
|
+
wanted = {slug: context.config.platforms[slug].dat for slug in slugs}
|
|
151
|
+
with http.client(context.config.network.proxy) as client:
|
|
152
|
+
results = dats.update(context.data_dir, wanted, client)
|
|
153
|
+
for slug, result in results.items():
|
|
154
|
+
print(f"{slug} {result.version} ({'updated' if result.changed else 'unchanged'})")
|
|
155
|
+
return 0
|
|
156
|
+
|
|
157
|
+
|
|
158
|
+
def scan_platforms(args: argparse.Namespace) -> int:
|
|
159
|
+
context = load_context(args.config)
|
|
160
|
+
slugs = selected(context, args.platforms)
|
|
161
|
+
scan.require_dats(context.config, slugs, context.data_dir)
|
|
162
|
+
scan.scan_each(slugs, lambda slug: scan.scan_platform(context.config, slug, context.data_dir))
|
|
163
|
+
return 0
|
|
164
|
+
|
|
165
|
+
|
|
166
|
+
def import_games(args: argparse.Namespace) -> int:
|
|
167
|
+
return run_import(load_context(args.config), args.dry_run)
|
|
168
|
+
|
|
169
|
+
|
|
170
|
+
def run_import(context: Context, dry_run: bool) -> int:
|
|
171
|
+
result = importer.import_inboxes(context.config, context.data_dir, dry_run)
|
|
172
|
+
return importer.print_result(result, dry_run=dry_run)
|
|
173
|
+
|
|
174
|
+
|
|
175
|
+
def fetch_games(args: argparse.Namespace) -> int:
|
|
176
|
+
context = load_context(args.config)
|
|
177
|
+
(slug,) = selected(context, [args.platform])
|
|
178
|
+
if not context.config.platforms[slug].sources:
|
|
179
|
+
raise config.ConfigError(f"{slug} has no sources configured; set platforms.{slug}.sources")
|
|
180
|
+
with http.client(context.config.network.proxy) as client:
|
|
181
|
+
throttle = http.Throttle()
|
|
182
|
+
result = fetch.fetch_platform(
|
|
183
|
+
context.config,
|
|
184
|
+
slug,
|
|
185
|
+
context.data_dir,
|
|
186
|
+
args.limit,
|
|
187
|
+
args.dry_run,
|
|
188
|
+
functools.partial(fetch.build_sources, client=client, throttle=throttle),
|
|
189
|
+
client,
|
|
190
|
+
throttle,
|
|
191
|
+
args.source,
|
|
192
|
+
)
|
|
193
|
+
status = 1 if result.failed else 0
|
|
194
|
+
if result.fetched and not args.dry_run and not args.no_import:
|
|
195
|
+
status = max(status, run_import(context, dry_run=False))
|
|
196
|
+
return status
|
|
197
|
+
|
|
198
|
+
|
|
199
|
+
def show_status(args: argparse.Namespace) -> int:
|
|
200
|
+
context = load_context(args.config)
|
|
201
|
+
with state.State(context.data_dir / "state.db") as database:
|
|
202
|
+
counts = database.counts()
|
|
203
|
+
rows = [status_row(slug, counts.get(slug)) for slug in context.config.platforms]
|
|
204
|
+
for line in format_table(("platform", *STATUS_COLUMNS), rows):
|
|
205
|
+
print(line)
|
|
206
|
+
if any(slug not in counts for slug in context.config.platforms):
|
|
207
|
+
print(scan.SCAN_HINT)
|
|
208
|
+
return 0
|
|
209
|
+
|
|
210
|
+
|
|
211
|
+
def status_row(slug: str, counts: state.Counts | None) -> tuple[str, ...]:
|
|
212
|
+
if counts is None:
|
|
213
|
+
return (slug, *("-" for _ in STATUS_COLUMNS))
|
|
214
|
+
wanted = counts.have + counts.inbox + counts.missing
|
|
215
|
+
done = f"{counts.have * 100 // wanted}%" if wanted else "-"
|
|
216
|
+
values = (wanted, counts.have, counts.inbox, counts.missing, counts.unmatched)
|
|
217
|
+
return (slug, *(str(value) for value in values), done)
|
|
218
|
+
|
|
219
|
+
|
|
220
|
+
def format_table(header: tuple[str, ...], rows: list[tuple[str, ...]]) -> list[str]:
|
|
221
|
+
widths = [max(len(cells[i]) for cells in (header, *rows)) for i in range(len(header))]
|
|
222
|
+
return [
|
|
223
|
+
" ".join(
|
|
224
|
+
cell.ljust(width) if i == 0 else cell.rjust(width)
|
|
225
|
+
for i, (cell, width) in enumerate(zip(cells, widths, strict=True))
|
|
226
|
+
).rstrip()
|
|
227
|
+
for cells in (header, *rows)
|
|
228
|
+
]
|
|
229
|
+
|
|
230
|
+
|
|
231
|
+
def show_missing(args: argparse.Namespace) -> int:
|
|
232
|
+
context = load_context(args.config)
|
|
233
|
+
(slug,) = selected(context, [args.platform])
|
|
234
|
+
games = scan.missing_games(context.data_dir, slug)
|
|
235
|
+
vimm = context.platforms[slug].vimm if slug in context.platforms else None
|
|
236
|
+
for game in games:
|
|
237
|
+
link = links.vimm_search(vimm, game) if args.links and vimm else None
|
|
238
|
+
print(f"{game}\t{link}" if link else game)
|
|
239
|
+
return 0
|
|
240
|
+
|
|
241
|
+
|
|
242
|
+
def index_dats(args: argparse.Namespace) -> int:
|
|
243
|
+
context = load_context(args.config)
|
|
244
|
+
slugs = selected(context, args.platforms)
|
|
245
|
+
scan.require_dats(context.config, slugs, context.data_dir)
|
|
246
|
+
with state.State(context.data_dir / "state.db") as database:
|
|
247
|
+
for slug in slugs:
|
|
248
|
+
dat = dats.dat_path(context.data_dir, context.config.platforms[slug].dat)
|
|
249
|
+
rebuilt = datindex.ensure(database, slug, dat)
|
|
250
|
+
print(f"{slug} {'indexed' if rebuilt else 'unchanged'}")
|
|
251
|
+
return 0
|
|
252
|
+
|
|
253
|
+
|
|
254
|
+
def serve(args: argparse.Namespace) -> int:
|
|
255
|
+
context = load_context(args.config)
|
|
256
|
+
token = auth.load_token(context.data_dir)
|
|
257
|
+
app = create_app(lambda: load_context(args.config), token)
|
|
258
|
+
link_host = "localhost" if args.host in WILDCARD_HOSTS else args.host
|
|
259
|
+
url = f"http://{link_host}:{args.port}/?token={token}"
|
|
260
|
+
print(f"romledger dashboard: {url}", flush=True)
|
|
261
|
+
if args.open:
|
|
262
|
+
webbrowser.open(url)
|
|
263
|
+
uvicorn.run(app, host=args.host, port=args.port, access_log=False)
|
|
264
|
+
return 0
|
|
265
|
+
|
|
266
|
+
|
|
267
|
+
def run_canary(args: argparse.Namespace) -> int:
|
|
268
|
+
with http.client() as client:
|
|
269
|
+
report = canary.run(
|
|
270
|
+
client,
|
|
271
|
+
http.Throttle(),
|
|
272
|
+
registry.load(),
|
|
273
|
+
registry.ignored(),
|
|
274
|
+
token=os.environ.get("GITHUB_TOKEN"),
|
|
275
|
+
)
|
|
276
|
+
if args.json:
|
|
277
|
+
payload = {
|
|
278
|
+
"findings": [dataclasses.asdict(finding) for finding in report.findings],
|
|
279
|
+
"inconclusive": report.inconclusive,
|
|
280
|
+
}
|
|
281
|
+
print(json.dumps(payload, indent=2))
|
|
282
|
+
else:
|
|
283
|
+
for finding in report.findings:
|
|
284
|
+
print(f"finding {finding.key}: {finding.title}")
|
|
285
|
+
diagnostics = sys.stderr if args.json else sys.stdout
|
|
286
|
+
for note in report.notes:
|
|
287
|
+
print(f"note: {note}", file=diagnostics)
|
|
288
|
+
if not args.json:
|
|
289
|
+
for prefix in report.inconclusive:
|
|
290
|
+
print(f"unverified {prefix}")
|
|
291
|
+
return 1 if report.findings else 0
|
|
292
|
+
|
|
293
|
+
|
|
294
|
+
HANDLERS: dict[str, Callable[[argparse.Namespace], int]] = {
|
|
295
|
+
"config": show_config,
|
|
296
|
+
"dats update": update_dats,
|
|
297
|
+
"dats index": index_dats,
|
|
298
|
+
"scan": scan_platforms,
|
|
299
|
+
"import": import_games,
|
|
300
|
+
"fetch": fetch_games,
|
|
301
|
+
"status": show_status,
|
|
302
|
+
"missing": show_missing,
|
|
303
|
+
"canary": run_canary,
|
|
304
|
+
"serve": serve,
|
|
305
|
+
}
|
|
306
|
+
|
|
307
|
+
|
|
308
|
+
def main(argv: list[str] | None = None) -> int:
|
|
309
|
+
args = build_parser().parse_args(argv)
|
|
310
|
+
try:
|
|
311
|
+
return HANDLERS[args.command](args)
|
|
312
|
+
except (config.ConfigError, dats.DatError, igir.ToolError) as error:
|
|
313
|
+
print(f"romledger: error: {error}", file=sys.stderr)
|
|
314
|
+
return 2
|
|
315
|
+
except BrokenPipeError:
|
|
316
|
+
os.dup2(os.open(os.devnull, os.O_WRONLY), sys.stdout.fileno())
|
|
317
|
+
return 0
|
|
318
|
+
except OSError as error:
|
|
319
|
+
print(f"romledger: error: {error}", file=sys.stderr)
|
|
320
|
+
return 2
|