flacfetch 0.35.0__tar.gz → 0.36.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {flacfetch-0.35.0/flacfetch.egg-info → flacfetch-0.36.0}/PKG-INFO +1 -1
- {flacfetch-0.35.0 → flacfetch-0.36.0}/flacfetch/core/manager.py +7 -0
- {flacfetch-0.35.0 → flacfetch-0.36.0}/flacfetch/providers/gazelle.py +199 -10
- {flacfetch-0.35.0 → flacfetch-0.36.0}/flacfetch/providers/youtube.py +5 -3
- {flacfetch-0.35.0 → flacfetch-0.36.0/flacfetch.egg-info}/PKG-INFO +1 -1
- {flacfetch-0.35.0 → flacfetch-0.36.0}/flacfetch.egg-info/SOURCES.txt +2 -1
- {flacfetch-0.35.0 → flacfetch-0.36.0}/pyproject.toml +1 -1
- {flacfetch-0.35.0 → flacfetch-0.36.0}/tests/test_gazelle.py +249 -0
- {flacfetch-0.35.0 → flacfetch-0.36.0}/tests/test_manager.py +25 -0
- flacfetch-0.36.0/tests/test_youtube_provider.py +32 -0
- {flacfetch-0.35.0 → flacfetch-0.36.0}/ARCHITECTURE.md +0 -0
- {flacfetch-0.35.0 → flacfetch-0.36.0}/LICENSE +0 -0
- {flacfetch-0.35.0 → flacfetch-0.36.0}/MANIFEST.in +0 -0
- {flacfetch-0.35.0 → flacfetch-0.36.0}/PLAN.md +0 -0
- {flacfetch-0.35.0 → flacfetch-0.36.0}/README.md +0 -0
- {flacfetch-0.35.0 → flacfetch-0.36.0}/flacfetch/__init__.py +0 -0
- {flacfetch-0.35.0 → flacfetch-0.36.0}/flacfetch/api/__init__.py +0 -0
- {flacfetch-0.35.0 → flacfetch-0.36.0}/flacfetch/api/auth.py +0 -0
- {flacfetch-0.35.0 → flacfetch-0.36.0}/flacfetch/api/main.py +0 -0
- {flacfetch-0.35.0 → flacfetch-0.36.0}/flacfetch/api/models.py +0 -0
- {flacfetch-0.35.0 → flacfetch-0.36.0}/flacfetch/api/routes/__init__.py +0 -0
- {flacfetch-0.35.0 → flacfetch-0.36.0}/flacfetch/api/routes/cache.py +0 -0
- {flacfetch-0.35.0 → flacfetch-0.36.0}/flacfetch/api/routes/config.py +0 -0
- {flacfetch-0.35.0 → flacfetch-0.36.0}/flacfetch/api/routes/credentials.py +0 -0
- {flacfetch-0.35.0 → flacfetch-0.36.0}/flacfetch/api/routes/download.py +0 -0
- {flacfetch-0.35.0 → flacfetch-0.36.0}/flacfetch/api/routes/health.py +0 -0
- {flacfetch-0.35.0 → flacfetch-0.36.0}/flacfetch/api/routes/search.py +0 -0
- {flacfetch-0.35.0 → flacfetch-0.36.0}/flacfetch/api/routes/spotify.py +0 -0
- {flacfetch-0.35.0 → flacfetch-0.36.0}/flacfetch/api/routes/torrents.py +0 -0
- {flacfetch-0.35.0 → flacfetch-0.36.0}/flacfetch/api/services/__init__.py +0 -0
- {flacfetch-0.35.0 → flacfetch-0.36.0}/flacfetch/api/services/credential_check.py +0 -0
- {flacfetch-0.35.0 → flacfetch-0.36.0}/flacfetch/api/services/disk_manager.py +0 -0
- {flacfetch-0.35.0 → flacfetch-0.36.0}/flacfetch/api/services/download_manager.py +0 -0
- {flacfetch-0.35.0 → flacfetch-0.36.0}/flacfetch/api/services/health_check.py +0 -0
- {flacfetch-0.35.0 → flacfetch-0.36.0}/flacfetch/api/services/search_cache.py +0 -0
- {flacfetch-0.35.0 → flacfetch-0.36.0}/flacfetch/api/services/secret_manager_utils.py +0 -0
- {flacfetch-0.35.0 → flacfetch-0.36.0}/flacfetch/api/services/spotify_budget.py +0 -0
- {flacfetch-0.35.0 → flacfetch-0.36.0}/flacfetch/core/album_match.py +0 -0
- {flacfetch-0.35.0 → flacfetch-0.36.0}/flacfetch/core/categorize.py +0 -0
- {flacfetch-0.35.0 → flacfetch-0.36.0}/flacfetch/core/config.py +0 -0
- {flacfetch-0.35.0 → flacfetch-0.36.0}/flacfetch/core/interfaces.py +0 -0
- {flacfetch-0.35.0 → flacfetch-0.36.0}/flacfetch/core/log.py +0 -0
- {flacfetch-0.35.0 → flacfetch-0.36.0}/flacfetch/core/matching.py +0 -0
- {flacfetch-0.35.0 → flacfetch-0.36.0}/flacfetch/core/models.py +0 -0
- {flacfetch-0.35.0 → flacfetch-0.36.0}/flacfetch/credential_keeper/__init__.py +0 -0
- {flacfetch-0.35.0 → flacfetch-0.36.0}/flacfetch/credential_keeper/__main__.py +0 -0
- {flacfetch-0.35.0 → flacfetch-0.36.0}/flacfetch/credential_keeper/browser.py +0 -0
- {flacfetch-0.35.0 → flacfetch-0.36.0}/flacfetch/credential_keeper/google_login.py +0 -0
- {flacfetch-0.35.0 → flacfetch-0.36.0}/flacfetch/credential_keeper/keeper.py +0 -0
- {flacfetch-0.35.0 → flacfetch-0.36.0}/flacfetch/credential_keeper/librespot.py +0 -0
- {flacfetch-0.35.0 → flacfetch-0.36.0}/flacfetch/credential_keeper/probe.py +0 -0
- {flacfetch-0.35.0 → flacfetch-0.36.0}/flacfetch/credential_keeper/spotify.py +0 -0
- {flacfetch-0.35.0 → flacfetch-0.36.0}/flacfetch/credential_keeper/youtube.py +0 -0
- {flacfetch-0.35.0 → flacfetch-0.36.0}/flacfetch/downloaders/spotify.py +0 -0
- {flacfetch-0.35.0 → flacfetch-0.36.0}/flacfetch/downloaders/torrent.py +0 -0
- {flacfetch-0.35.0 → flacfetch-0.36.0}/flacfetch/downloaders/torrent_coordinator.py +0 -0
- {flacfetch-0.35.0 → flacfetch-0.36.0}/flacfetch/downloaders/youtube.py +0 -0
- {flacfetch-0.35.0 → flacfetch-0.36.0}/flacfetch/interface/cli.py +0 -0
- {flacfetch-0.35.0 → flacfetch-0.36.0}/flacfetch/interface/cli_remote.py +0 -0
- {flacfetch-0.35.0 → flacfetch-0.36.0}/flacfetch/providers/ops.py +0 -0
- {flacfetch-0.35.0 → flacfetch-0.36.0}/flacfetch/providers/red.py +0 -0
- {flacfetch-0.35.0 → flacfetch-0.36.0}/flacfetch/providers/spotify.py +0 -0
- {flacfetch-0.35.0 → flacfetch-0.36.0}/flacfetch.egg-info/dependency_links.txt +0 -0
- {flacfetch-0.35.0 → flacfetch-0.36.0}/flacfetch.egg-info/entry_points.txt +0 -0
- {flacfetch-0.35.0 → flacfetch-0.36.0}/flacfetch.egg-info/requires.txt +0 -0
- {flacfetch-0.35.0 → flacfetch-0.36.0}/flacfetch.egg-info/top_level.txt +0 -0
- {flacfetch-0.35.0 → flacfetch-0.36.0}/requirements.txt +0 -0
- {flacfetch-0.35.0 → flacfetch-0.36.0}/setup.cfg +0 -0
- {flacfetch-0.35.0 → flacfetch-0.36.0}/setup.py +0 -0
- {flacfetch-0.35.0 → flacfetch-0.36.0}/tests/_route_paths.py +0 -0
- {flacfetch-0.35.0 → flacfetch-0.36.0}/tests/test_album_search.py +0 -0
- {flacfetch-0.35.0 → flacfetch-0.36.0}/tests/test_api_auth.py +0 -0
- {flacfetch-0.35.0 → flacfetch-0.36.0}/tests/test_api_config.py +0 -0
- {flacfetch-0.35.0 → flacfetch-0.36.0}/tests/test_api_deep_health.py +0 -0
- {flacfetch-0.35.0 → flacfetch-0.36.0}/tests/test_api_health_ytdlp.py +0 -0
- {flacfetch-0.35.0 → flacfetch-0.36.0}/tests/test_api_integration.py +0 -0
- {flacfetch-0.35.0 → flacfetch-0.36.0}/tests/test_api_models.py +0 -0
- {flacfetch-0.35.0 → flacfetch-0.36.0}/tests/test_api_routes.py +0 -0
- {flacfetch-0.35.0 → flacfetch-0.36.0}/tests/test_api_services.py +0 -0
- {flacfetch-0.35.0 → flacfetch-0.36.0}/tests/test_cache_api.py +0 -0
- {flacfetch-0.35.0 → flacfetch-0.36.0}/tests/test_cli_cookies.py +0 -0
- {flacfetch-0.35.0 → flacfetch-0.36.0}/tests/test_cli_remote.py +0 -0
- {flacfetch-0.35.0 → flacfetch-0.36.0}/tests/test_core.py +0 -0
- {flacfetch-0.35.0 → flacfetch-0.36.0}/tests/test_credential_keeper.py +0 -0
- {flacfetch-0.35.0 → flacfetch-0.36.0}/tests/test_keeper_probe.py +0 -0
- {flacfetch-0.35.0 → flacfetch-0.36.0}/tests/test_logging.py +0 -0
- {flacfetch-0.35.0 → flacfetch-0.36.0}/tests/test_matching.py +0 -0
- {flacfetch-0.35.0 → flacfetch-0.36.0}/tests/test_models.py +0 -0
- {flacfetch-0.35.0 → flacfetch-0.36.0}/tests/test_ops.py +0 -0
- {flacfetch-0.35.0 → flacfetch-0.36.0}/tests/test_provider_errors.py +0 -0
- {flacfetch-0.35.0 → flacfetch-0.36.0}/tests/test_red.py +0 -0
- {flacfetch-0.35.0 → flacfetch-0.36.0}/tests/test_search_api.py +0 -0
- {flacfetch-0.35.0 → flacfetch-0.36.0}/tests/test_spotify.py +0 -0
- {flacfetch-0.35.0 → flacfetch-0.36.0}/tests/test_spotify_budget.py +0 -0
- {flacfetch-0.35.0 → flacfetch-0.36.0}/tests/test_torrent_coordinator.py +0 -0
- {flacfetch-0.35.0 → flacfetch-0.36.0}/tests/test_torrent_downloader.py +0 -0
- {flacfetch-0.35.0 → flacfetch-0.36.0}/tests/test_torrent_shared_album.py +0 -0
- {flacfetch-0.35.0 → flacfetch-0.36.0}/tests/test_transcode_hf_noise.py +0 -0
- {flacfetch-0.35.0 → flacfetch-0.36.0}/tests/test_youtube.py +0 -0
- {flacfetch-0.35.0 → flacfetch-0.36.0}/tests/test_youtube_cookies.py +0 -0
|
@@ -373,6 +373,10 @@ class FetchManager:
|
|
|
373
373
|
# If download_url is http..., TorrentDownloader will fail.
|
|
374
374
|
if release.download_url and release.download_url.startswith("http"):
|
|
375
375
|
msg = "Failed to download torrent file from provider. Cannot proceed with download."
|
|
376
|
+
torrent_id = torrent_id_from_url(release.download_url) if torrent_id_from_url else None
|
|
377
|
+
reason = getattr(provider, "fetch_errors", {}).get(torrent_id)
|
|
378
|
+
if reason:
|
|
379
|
+
msg = f"{msg} {reason}"
|
|
376
380
|
logger.error(msg)
|
|
377
381
|
raise ValueError(msg)
|
|
378
382
|
|
|
@@ -520,6 +524,9 @@ class FetchManager:
|
|
|
520
524
|
|
|
521
525
|
if not artifact:
|
|
522
526
|
msg = f"Failed to fetch torrent file for {source_name} ID: {source_id}"
|
|
527
|
+
reason = getattr(provider, "fetch_errors", {}).get(source_id)
|
|
528
|
+
if reason:
|
|
529
|
+
msg = f"{msg}: {reason}"
|
|
523
530
|
logger.error(msg)
|
|
524
531
|
raise ValueError(msg)
|
|
525
532
|
|
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
"""Base provider for Gazelle-based private music trackers (RED, OPS, etc.)."""
|
|
2
2
|
|
|
3
3
|
import html
|
|
4
|
+
import json
|
|
4
5
|
import threading
|
|
5
6
|
import time
|
|
6
7
|
from abc import abstractmethod
|
|
@@ -9,6 +10,7 @@ from typing import Any, Optional
|
|
|
9
10
|
from urllib.parse import parse_qs, urlparse
|
|
10
11
|
|
|
11
12
|
import requests
|
|
13
|
+
from urllib3.exceptions import NewConnectionError
|
|
12
14
|
|
|
13
15
|
from ..core.album_match import AlbumCandidate, AlbumTrack, match_album_tracks, parse_file_list
|
|
14
16
|
from ..core.interfaces import Provider
|
|
@@ -55,6 +57,24 @@ class GazelleProvider(Provider):
|
|
|
55
57
|
# After a token spend fails for a torrent, don't retry it for this long.
|
|
56
58
|
FL_TOKEN_FAILURE_BACKOFF_SECONDS = 600
|
|
57
59
|
|
|
60
|
+
# RED lets a user fetch each .torrent file through the API only four times,
|
|
61
|
+
# ever ("You have already downloaded this torrent file four times"); after
|
|
62
|
+
# that it must come from a browser. Every request that may have counted is
|
|
63
|
+
# recorded per torrent in an on-disk ledger, and we stop at this many so one
|
|
64
|
+
# download is always left for a manual browser recovery.
|
|
65
|
+
TORRENT_DOWNLOAD_BUDGET = 3
|
|
66
|
+
|
|
67
|
+
# A request the tracker never answered (read timeout, dropped connection,
|
|
68
|
+
# 5xx) may still have counted server-side: on 2026-10-23 RED stalled and
|
|
69
|
+
# three timed-out fetches used up a torrent's limit with nothing cached.
|
|
70
|
+
# After one, don't ask the tracker for that torrent again for this long.
|
|
71
|
+
UNANSWERED_COOLDOWN_SECONDS = 300
|
|
72
|
+
|
|
73
|
+
# (connect, read) timeouts for .torrent downloads. The file is a few KB, so
|
|
74
|
+
# a slow response is the tracker stalling; waiting it out is far cheaper
|
|
75
|
+
# than a retry that spends another of the four downloads.
|
|
76
|
+
DOWNLOAD_TIMEOUT = (10, 60)
|
|
77
|
+
|
|
58
78
|
def __init__(self, api_key: str, base_url: str, cache_subdir: str, use_fl_token: bool = False):
|
|
59
79
|
"""Initialize the Gazelle provider.
|
|
60
80
|
|
|
@@ -90,6 +110,12 @@ class GazelleProvider(Provider):
|
|
|
90
110
|
# ONE tracker fetch: the first caches the .torrent, the rest hit the cache.
|
|
91
111
|
self._fetch_locks: dict[str, threading.Lock] = {}
|
|
92
112
|
self._fetch_locks_guard = threading.Lock()
|
|
113
|
+
# Per-torrent download ledgers (see TORRENT_DOWNLOAD_BUDGET) kept in
|
|
114
|
+
# memory: when there is no cache dir, or the last disk write failed (an
|
|
115
|
+
# unrecorded request must never let retries reach the tracker again).
|
|
116
|
+
self._download_ledgers: dict[str, dict] = {}
|
|
117
|
+
# {torrent_id: reason} for the last failed fetch, surfaced in job errors.
|
|
118
|
+
self.fetch_errors: dict[str, str] = {}
|
|
93
119
|
self.session = requests.Session()
|
|
94
120
|
self.session.headers.update({"Authorization": self.api_key})
|
|
95
121
|
self.search_limit = 10
|
|
@@ -259,9 +285,13 @@ class GazelleProvider(Provider):
|
|
|
259
285
|
def _fetch_torrent_from_url(self, url: str, torrent_id: Optional[str] = None, max_retries: int = 3) -> Optional[bytes]:
|
|
260
286
|
"""Fetch a torrent file from URL and cache it.
|
|
261
287
|
|
|
288
|
+
Every request goes through the per-torrent download ledger: nothing is
|
|
289
|
+
sent once the torrent's budget is used up or while it is cooling down
|
|
290
|
+
after an unanswered request (see TORRENT_DOWNLOAD_BUDGET).
|
|
291
|
+
|
|
262
292
|
Args:
|
|
263
293
|
url: Download URL for the torrent file
|
|
264
|
-
torrent_id: Optional torrent ID for caching
|
|
294
|
+
torrent_id: Optional torrent ID for caching (parsed from the URL if omitted)
|
|
265
295
|
max_retries: Maximum number of retries on rate limiting (default 3)
|
|
266
296
|
|
|
267
297
|
Returns:
|
|
@@ -271,8 +301,17 @@ class GazelleProvider(Provider):
|
|
|
271
301
|
if url.startswith("/") or url.startswith("file://"):
|
|
272
302
|
return None
|
|
273
303
|
|
|
304
|
+
torrent_id = torrent_id or self.torrent_id_from_url(url)
|
|
305
|
+
kind = "token" if "usetoken=1" in url else "normal"
|
|
306
|
+
|
|
274
307
|
retries = 0
|
|
275
308
|
while retries <= max_retries:
|
|
309
|
+
blocked = self._download_blocked_reason(torrent_id)
|
|
310
|
+
if blocked:
|
|
311
|
+
logger.error(f"{self.name}: not requesting .torrent {torrent_id}: {blocked}")
|
|
312
|
+
self.fetch_errors[torrent_id] = blocked
|
|
313
|
+
return None
|
|
314
|
+
|
|
276
315
|
try:
|
|
277
316
|
logger.info(f"Fetching artifact from {url}")
|
|
278
317
|
|
|
@@ -283,7 +322,7 @@ class GazelleProvider(Provider):
|
|
|
283
322
|
logger.debug(f"Downloading artifact from: {url}")
|
|
284
323
|
logger.debug(f"Request Headers: {req_headers}")
|
|
285
324
|
|
|
286
|
-
resp = self.session.get(url, timeout=
|
|
325
|
+
resp = self.session.get(url, timeout=self.DOWNLOAD_TIMEOUT, allow_redirects=False)
|
|
287
326
|
logger.debug(f"Response Status: {resp.status_code}")
|
|
288
327
|
logger.debug(f"Response Headers: {dict(resp.headers)}")
|
|
289
328
|
|
|
@@ -291,12 +330,10 @@ class GazelleProvider(Provider):
|
|
|
291
330
|
try:
|
|
292
331
|
content = resp.json()
|
|
293
332
|
logger.debug(f"Response Body (JSON): {content}")
|
|
294
|
-
if
|
|
295
|
-
|
|
296
|
-
|
|
297
|
-
|
|
298
|
-
logger.info(f" {self.cache_dir}/{torrent_id}.torrent")
|
|
299
|
-
except:
|
|
333
|
+
if self._is_limit_error(content):
|
|
334
|
+
self._handle_limit_error(torrent_id, content)
|
|
335
|
+
return None
|
|
336
|
+
except Exception:
|
|
300
337
|
logger.debug(f"Response Body (Text/Raw): {resp.text[:500]}")
|
|
301
338
|
|
|
302
339
|
if resp.status_code in (301, 302, 303, 307, 308):
|
|
@@ -305,7 +342,7 @@ class GazelleProvider(Provider):
|
|
|
305
342
|
if redirect_url:
|
|
306
343
|
if redirect_url.startswith("/"):
|
|
307
344
|
redirect_url = self.base_url + redirect_url
|
|
308
|
-
resp = self.session.get(redirect_url, timeout=
|
|
345
|
+
resp = self.session.get(redirect_url, timeout=self.DOWNLOAD_TIMEOUT)
|
|
309
346
|
|
|
310
347
|
if resp.status_code == 200:
|
|
311
348
|
# The download endpoint returns raw bencoded .torrent bytes on
|
|
@@ -316,13 +353,20 @@ class GazelleProvider(Provider):
|
|
|
316
353
|
content_type = resp.headers.get("Content-Type", "").lower()
|
|
317
354
|
is_torrent = resp.content[:1] == b"d" and "json" not in content_type
|
|
318
355
|
if not is_torrent:
|
|
356
|
+
self._record_download_request(torrent_id, kind, "refused")
|
|
319
357
|
try:
|
|
320
358
|
err = resp.json()
|
|
359
|
+
if self._is_limit_error(err):
|
|
360
|
+
self._handle_limit_error(torrent_id, err)
|
|
361
|
+
return None
|
|
321
362
|
logger.warning(f"{self.name} download returned an error instead of a torrent: {err.get('error', err)}")
|
|
363
|
+
if torrent_id:
|
|
364
|
+
self.fetch_errors[torrent_id] = f"{self.name} refused the .torrent download: {err.get('error', err)}"
|
|
322
365
|
except ValueError:
|
|
323
366
|
logger.warning(f"{self.name} download returned non-torrent content ({len(resp.content)} bytes): {resp.content[:100]}")
|
|
324
367
|
return None
|
|
325
368
|
|
|
369
|
+
self._record_download_request(torrent_id, kind, "ok")
|
|
326
370
|
logger.debug(f"Artifact fetched successfully ({len(resp.content)} bytes)")
|
|
327
371
|
if len(resp.content) < 1000:
|
|
328
372
|
logger.warning(f"Artifact seems too small ({len(resp.content)} bytes). Content sample: {resp.content[:100]}")
|
|
@@ -339,6 +383,7 @@ class GazelleProvider(Provider):
|
|
|
339
383
|
|
|
340
384
|
return resp.content
|
|
341
385
|
elif resp.status_code == 429:
|
|
386
|
+
self._record_download_request(torrent_id, kind, "refused")
|
|
342
387
|
retries += 1
|
|
343
388
|
if retries <= max_retries:
|
|
344
389
|
logger.warning(f"Rate limited while fetching artifact. Retry {retries}/{max_retries} in 2s...")
|
|
@@ -348,14 +393,152 @@ class GazelleProvider(Provider):
|
|
|
348
393
|
logger.error(f"Rate limit exceeded after {max_retries} retries")
|
|
349
394
|
return None
|
|
350
395
|
else:
|
|
396
|
+
# A 4xx is an explicit refusal (nothing served). A 5xx may have
|
|
397
|
+
# been generated and counted before the server failed.
|
|
398
|
+
outcome = "unanswered" if resp.status_code >= 500 else "refused"
|
|
399
|
+
self._record_download_request(torrent_id, kind, outcome)
|
|
351
400
|
logger.error(f"Failed to fetch artifact: Status {resp.status_code}")
|
|
401
|
+
if outcome == "unanswered":
|
|
402
|
+
self.fetch_errors[torrent_id] = f"{self.name} returned HTTP {resp.status_code} for the .torrent download"
|
|
403
|
+
return None
|
|
404
|
+
except requests.exceptions.RequestException as e:
|
|
405
|
+
logger.error(f"Error fetching artifact: {e}")
|
|
406
|
+
if self._request_never_sent(e):
|
|
407
|
+
# DNS / refused / TLS / proxy failure: the tracker never saw it.
|
|
408
|
+
self.fetch_errors[torrent_id] = f"Could not connect to {self.name} ({type(e).__name__})"
|
|
352
409
|
return None
|
|
410
|
+
# Read timeout / dropped connection: the tracker may have counted it.
|
|
411
|
+
self._record_download_request(torrent_id, kind, "unanswered")
|
|
412
|
+
self.fetch_errors[torrent_id] = f"{self.name} did not answer the .torrent download ({type(e).__name__})"
|
|
413
|
+
return None
|
|
353
414
|
except Exception as e:
|
|
415
|
+
# Anything else may have happened after the tracker served the
|
|
416
|
+
# file, so count it rather than risk an unrecorded download.
|
|
417
|
+
self._record_download_request(torrent_id, kind, "unanswered")
|
|
354
418
|
logger.error(f"Error fetching artifact: {e}")
|
|
419
|
+
self.fetch_errors[torrent_id] = f"{self.name} .torrent download failed ({type(e).__name__})"
|
|
355
420
|
return None
|
|
356
421
|
|
|
357
422
|
return None
|
|
358
423
|
|
|
424
|
+
@staticmethod
|
|
425
|
+
def _request_never_sent(error: Exception) -> bool:
|
|
426
|
+
"""True when a requests error happened before the request reached the tracker."""
|
|
427
|
+
if isinstance(error, (requests.exceptions.ConnectTimeout, requests.exceptions.SSLError, requests.exceptions.ProxyError)):
|
|
428
|
+
return True
|
|
429
|
+
if isinstance(error, requests.exceptions.ConnectionError):
|
|
430
|
+
# Wraps urllib3's MaxRetryError; NewConnectionError (incl. DNS
|
|
431
|
+
# NameResolutionError) means no connection was ever made.
|
|
432
|
+
reason = getattr(error.args[0], "reason", None) if error.args else None
|
|
433
|
+
return isinstance(reason, NewConnectionError)
|
|
434
|
+
return False
|
|
435
|
+
|
|
436
|
+
@staticmethod
|
|
437
|
+
def _is_limit_error(body) -> bool:
|
|
438
|
+
return isinstance(body, dict) and body.get("status") == "failure" and "already downloaded" in str(body.get("error", ""))
|
|
439
|
+
|
|
440
|
+
def _handle_limit_error(self, torrent_id: Optional[str], body: dict) -> None:
|
|
441
|
+
"""The tracker says this torrent's download limit is used up."""
|
|
442
|
+
logger.error(f"{self.name} Limit Reached: {body.get('error')}")
|
|
443
|
+
if torrent_id:
|
|
444
|
+
self._mark_download_limit_reached(torrent_id)
|
|
445
|
+
self.fetch_errors[torrent_id] = self._limit_reached_message(torrent_id)
|
|
446
|
+
|
|
447
|
+
# ---- per-torrent download ledger --------------------------------------
|
|
448
|
+
|
|
449
|
+
def _download_ledger_path(self, torrent_id: str) -> Optional[Path]:
|
|
450
|
+
if not self.cache_dir or not torrent_id:
|
|
451
|
+
return None
|
|
452
|
+
return self.cache_dir / f"{torrent_id}.downloads.json"
|
|
453
|
+
|
|
454
|
+
def _load_download_ledger(self, torrent_id: str) -> dict:
|
|
455
|
+
"""The torrent's ledger: ``{"requests": [{t, kind, outcome}], "limit_reached": bool}``.
|
|
456
|
+
|
|
457
|
+
Re-read from disk each time (when there is a cache dir) so an operator can
|
|
458
|
+
inspect or reset a torrent's ledger file without restarting the service.
|
|
459
|
+
"""
|
|
460
|
+
path = self._download_ledger_path(torrent_id)
|
|
461
|
+
if not path:
|
|
462
|
+
return self._download_ledgers.setdefault(torrent_id, {"requests": [], "limit_reached": False})
|
|
463
|
+
if torrent_id in self._download_ledgers:
|
|
464
|
+
return self._download_ledgers[torrent_id]
|
|
465
|
+
if not path.exists():
|
|
466
|
+
return {"requests": [], "limit_reached": False}
|
|
467
|
+
try:
|
|
468
|
+
ledger = json.loads(path.read_text())
|
|
469
|
+
if not (isinstance(ledger, dict) and isinstance(ledger.get("requests"), list)
|
|
470
|
+
and all(isinstance(r, dict) for r in ledger["requests"])):
|
|
471
|
+
raise ValueError("unexpected ledger shape")
|
|
472
|
+
return ledger
|
|
473
|
+
except (OSError, ValueError) as e:
|
|
474
|
+
# An unreadable ledger must not unlock fresh downloads.
|
|
475
|
+
logger.warning(f"{self.name}: unreadable download ledger for {torrent_id} ({e}); treating as used up")
|
|
476
|
+
return {"requests": [], "limit_reached": True}
|
|
477
|
+
|
|
478
|
+
def _record_download_request(self, torrent_id: Optional[str], kind: str, outcome: str) -> None:
|
|
479
|
+
"""Record one download-endpoint request.
|
|
480
|
+
|
|
481
|
+
``outcome`` is ``ok`` (torrent served), ``unanswered`` (timeout, dropped
|
|
482
|
+
connection, 5xx: may have counted) or ``refused`` (an explicit error or
|
|
483
|
+
429: nothing served, not counted).
|
|
484
|
+
"""
|
|
485
|
+
if not torrent_id:
|
|
486
|
+
return
|
|
487
|
+
ledger = self._load_download_ledger(torrent_id)
|
|
488
|
+
ledger["requests"].append({"t": time.time(), "kind": kind, "outcome": outcome})
|
|
489
|
+
self._save_download_ledger(torrent_id, ledger)
|
|
490
|
+
|
|
491
|
+
def _mark_download_limit_reached(self, torrent_id: Optional[str]) -> None:
|
|
492
|
+
if not torrent_id:
|
|
493
|
+
return
|
|
494
|
+
ledger = self._load_download_ledger(torrent_id)
|
|
495
|
+
ledger["limit_reached"] = True
|
|
496
|
+
self._save_download_ledger(torrent_id, ledger)
|
|
497
|
+
|
|
498
|
+
def _save_download_ledger(self, torrent_id: str, ledger: dict) -> None:
|
|
499
|
+
path = self._download_ledger_path(torrent_id)
|
|
500
|
+
if not path:
|
|
501
|
+
return
|
|
502
|
+
tmp = path.with_name(path.name + ".tmp")
|
|
503
|
+
try:
|
|
504
|
+
tmp.write_text(json.dumps(ledger))
|
|
505
|
+
tmp.replace(path)
|
|
506
|
+
self._download_ledgers.pop(torrent_id, None)
|
|
507
|
+
except OSError as e:
|
|
508
|
+
logger.warning(f"{self.name}: could not write download ledger for {torrent_id}: {e}; keeping it in memory")
|
|
509
|
+
self._download_ledgers[torrent_id] = ledger
|
|
510
|
+
|
|
511
|
+
def downloads_used(self, torrent_id: str) -> int:
|
|
512
|
+
"""Requests for this torrent that may have counted against the tracker's limit."""
|
|
513
|
+
ledger = self._load_download_ledger(torrent_id)
|
|
514
|
+
return sum(1 for r in ledger["requests"] if r.get("outcome") in ("ok", "unanswered"))
|
|
515
|
+
|
|
516
|
+
def _download_blocked_reason(self, torrent_id: Optional[str]) -> Optional[str]:
|
|
517
|
+
"""Why the tracker must not be asked for this torrent right now, or None."""
|
|
518
|
+
if not torrent_id:
|
|
519
|
+
return None
|
|
520
|
+
ledger = self._load_download_ledger(torrent_id)
|
|
521
|
+
if ledger.get("limit_reached") or self.downloads_used(torrent_id) >= self.TORRENT_DOWNLOAD_BUDGET:
|
|
522
|
+
return self._limit_reached_message(torrent_id)
|
|
523
|
+
unanswered = [r.get("t", 0) for r in ledger["requests"] if r.get("outcome") == "unanswered"]
|
|
524
|
+
if unanswered:
|
|
525
|
+
wait = unanswered[-1] + self.UNANSWERED_COOLDOWN_SECONDS - time.time()
|
|
526
|
+
if wait > 0:
|
|
527
|
+
return (
|
|
528
|
+
f"{self.name} did not answer the last .torrent request for {torrent_id}; "
|
|
529
|
+
f"not asking again for {int(wait) + 1}s so a stalled tracker can't use up "
|
|
530
|
+
f"its download limit. Retry in a few minutes."
|
|
531
|
+
)
|
|
532
|
+
return None
|
|
533
|
+
|
|
534
|
+
def _limit_reached_message(self, torrent_id: str) -> str:
|
|
535
|
+
path = self.cache_dir / f"{torrent_id}.torrent" if self.cache_dir else f"<cache>/{torrent_id}.torrent"
|
|
536
|
+
return (
|
|
537
|
+
f"{self.name} .torrent download budget used up for torrent {torrent_id} "
|
|
538
|
+
f"({self.downloads_used(torrent_id)} recorded requests). Download the .torrent "
|
|
539
|
+
f"in a browser and save it as {path}, then retry."
|
|
540
|
+
)
|
|
541
|
+
|
|
359
542
|
def _token_marker_path(self, torrent_id: str) -> Optional[Path]:
|
|
360
543
|
"""Path of the local marker recording a token spend for a torrent."""
|
|
361
544
|
if not self.cache_dir or not torrent_id:
|
|
@@ -549,15 +732,22 @@ class GazelleProvider(Provider):
|
|
|
549
732
|
# each fetched the .torrent -- counting against the tracker's per-torrent
|
|
550
733
|
# download limit (RED: four). Siblings now wait and take the cached copy.
|
|
551
734
|
with self._fetch_lock(source_id):
|
|
735
|
+
self.fetch_errors.pop(source_id, None)
|
|
736
|
+
cached = self.get_cached_artifact(source_id)
|
|
737
|
+
|
|
552
738
|
# If we should spend a token and haven't already tokened this torrent
|
|
553
739
|
# within the freeleech window, register the token FIRST -- before the
|
|
554
740
|
# cache short-circuit. A cached .torrent file does NOT make a download
|
|
555
741
|
# freeleech; the token must be registered server-side via usetoken=1,
|
|
556
742
|
# otherwise the (re-)download of the torrent data counts against ratio.
|
|
743
|
+
# A successful token spend is itself a .torrent download, though, so
|
|
744
|
+
# with a cached copy in hand only spend one while it leaves room in the
|
|
745
|
+
# download budget: running out would cost far more than one torrent's ratio.
|
|
557
746
|
if (
|
|
558
747
|
use_token
|
|
559
748
|
and not self._token_recently_spent(source_id)
|
|
560
749
|
and not self._token_recently_failed(source_id)
|
|
750
|
+
and not (cached and self.downloads_used(source_id) + 1 >= self.TORRENT_DOWNLOAD_BUDGET)
|
|
561
751
|
):
|
|
562
752
|
with self._token_lock:
|
|
563
753
|
content = self._spend_token(url, source_id)
|
|
@@ -570,7 +760,6 @@ class GazelleProvider(Provider):
|
|
|
570
760
|
|
|
571
761
|
# Serve the cached .torrent if present (token, if any, is already
|
|
572
762
|
# registered above; a cached torrent is fine for the actual data fetch).
|
|
573
|
-
cached = self.get_cached_artifact(source_id)
|
|
574
763
|
if cached:
|
|
575
764
|
return cached
|
|
576
765
|
|
|
@@ -29,9 +29,11 @@ class YoutubeProvider(Provider):
|
|
|
29
29
|
return "YouTube"
|
|
30
30
|
|
|
31
31
|
def search(self, query: TrackQuery) -> list[Release]:
|
|
32
|
-
# Search for 5 results
|
|
33
|
-
#
|
|
34
|
-
|
|
32
|
+
# Search for 5 results. Keep the query plain: a " topic" suffix (meant to
|
|
33
|
+
# surface auto-generated Topic channel audio) pushed the actual song out of
|
|
34
|
+
# the top 5 entirely for lesser-known tracks (Playa Fly - N God We Trust).
|
|
35
|
+
# select_best() still prefers Topic/official uploads when they do appear.
|
|
36
|
+
search_query = f"ytsearch5:{query.artist} {query.title}"
|
|
35
37
|
|
|
36
38
|
# Get base options (includes cookies if available)
|
|
37
39
|
ydl_opts = get_ytdlp_base_opts(self.cookies_file)
|
|
@@ -831,3 +831,252 @@ class TestConcurrentTorrentFetch:
|
|
|
831
831
|
p = self._provider(tmp_path, use_fl_token=False)
|
|
832
832
|
assert p._fetch_lock("1") is p._fetch_lock("1")
|
|
833
833
|
assert p._fetch_lock("1") is not p._fetch_lock("2")
|
|
834
|
+
|
|
835
|
+
|
|
836
|
+
class TestDownloadBudget:
|
|
837
|
+
"""The per-torrent ledger keeps RED's four-download limit out of reach.
|
|
838
|
+
|
|
839
|
+
Incident 2026-10-23 (RED 2319570): RED stalled, three fetches timed out at
|
|
840
|
+
10s with nothing cached, and they counted server-side anyway; the limit
|
|
841
|
+
was gone before a retry could succeed.
|
|
842
|
+
"""
|
|
843
|
+
|
|
844
|
+
TORRENT = b"d8:announce" + b"x" * 2000
|
|
845
|
+
|
|
846
|
+
def _provider(self, tmp_path, use_fl_token=False):
|
|
847
|
+
p = ConcreteGazelleProvider(
|
|
848
|
+
api_key="test",
|
|
849
|
+
base_url="https://test.example",
|
|
850
|
+
cache_subdir="test",
|
|
851
|
+
use_fl_token=use_fl_token,
|
|
852
|
+
)
|
|
853
|
+
p.cache_dir = tmp_path
|
|
854
|
+
return p
|
|
855
|
+
|
|
856
|
+
def _resp(self, status=200, content=None, content_type="application/x-bittorrent", json_body=None):
|
|
857
|
+
resp = MagicMock()
|
|
858
|
+
resp.status_code = status
|
|
859
|
+
resp.headers = {"Content-Type": content_type}
|
|
860
|
+
resp.content = self.TORRENT if content is None else content
|
|
861
|
+
if json_body is not None:
|
|
862
|
+
resp.json.return_value = json_body
|
|
863
|
+
else:
|
|
864
|
+
resp.json.side_effect = ValueError("not json")
|
|
865
|
+
resp.text = ""
|
|
866
|
+
return resp
|
|
867
|
+
|
|
868
|
+
def _read_timeout(self):
|
|
869
|
+
import requests
|
|
870
|
+
return requests.exceptions.ReadTimeout("Read timed out. (read timeout=60)")
|
|
871
|
+
|
|
872
|
+
def test_successful_fetch_recorded_and_cached(self, tmp_path):
|
|
873
|
+
p = self._provider(tmp_path)
|
|
874
|
+
p.session.get = MagicMock(return_value=self._resp())
|
|
875
|
+
|
|
876
|
+
assert p.fetch_artifact_by_id("42") == self.TORRENT
|
|
877
|
+
assert p.downloads_used("42") == 1
|
|
878
|
+
assert (tmp_path / "42.torrent").read_bytes() == self.TORRENT
|
|
879
|
+
assert p.session.get.call_args.kwargs["timeout"] == p.DOWNLOAD_TIMEOUT
|
|
880
|
+
|
|
881
|
+
def test_timeout_counts_and_blocks_retries_during_cooldown(self, tmp_path):
|
|
882
|
+
p = self._provider(tmp_path)
|
|
883
|
+
p.session.get = MagicMock(side_effect=self._read_timeout())
|
|
884
|
+
|
|
885
|
+
assert p.fetch_artifact_by_id("42") is None
|
|
886
|
+
assert p.downloads_used("42") == 1
|
|
887
|
+
# gen's auto-retries 40s later must not touch the tracker.
|
|
888
|
+
assert p.fetch_artifact_by_id("42") is None
|
|
889
|
+
assert p.fetch_artifact_by_id("42") is None
|
|
890
|
+
assert p.session.get.call_count == 1
|
|
891
|
+
assert "did not answer" in p.fetch_errors["42"]
|
|
892
|
+
|
|
893
|
+
def test_retry_allowed_after_cooldown(self, tmp_path):
|
|
894
|
+
p = self._provider(tmp_path)
|
|
895
|
+
p.session.get = MagicMock(side_effect=[self._read_timeout(), self._resp()])
|
|
896
|
+
p.fetch_artifact_by_id("42")
|
|
897
|
+
|
|
898
|
+
ledger = p._load_download_ledger("42")
|
|
899
|
+
ledger["requests"][-1]["t"] -= p.UNANSWERED_COOLDOWN_SECONDS + 1
|
|
900
|
+
p._save_download_ledger("42", ledger)
|
|
901
|
+
assert p.fetch_artifact_by_id("42") == self.TORRENT
|
|
902
|
+
assert p.session.get.call_count == 2
|
|
903
|
+
assert "42" not in p.fetch_errors
|
|
904
|
+
|
|
905
|
+
def test_incident_replay_one_tracker_request(self, tmp_path):
|
|
906
|
+
"""Token refused, then RED stalls: only ONE countable request goes out."""
|
|
907
|
+
p = self._provider(tmp_path, use_fl_token=True)
|
|
908
|
+
refused = self._resp(400, content=b"{}", content_type="application/json", json_body={"status": "failure"})
|
|
909
|
+
p.session.get = MagicMock(side_effect=[refused, self._read_timeout()])
|
|
910
|
+
|
|
911
|
+
for _ in range(3): # first attempt + gen's two auto-retries
|
|
912
|
+
assert p.fetch_artifact_by_id("2319570") is None
|
|
913
|
+
|
|
914
|
+
urls = [c.args[0] for c in p.session.get.call_args_list]
|
|
915
|
+
assert len(urls) == 2
|
|
916
|
+
assert "usetoken=1" in urls[0] and "usetoken" not in urls[1]
|
|
917
|
+
assert p.downloads_used("2319570") == 1
|
|
918
|
+
|
|
919
|
+
def test_budget_stops_before_tracker_limit(self, tmp_path):
|
|
920
|
+
p = self._provider(tmp_path)
|
|
921
|
+
p.session.get = MagicMock(return_value=self._resp())
|
|
922
|
+
for _ in range(p.TORRENT_DOWNLOAD_BUDGET):
|
|
923
|
+
assert p.fetch_artifact_by_id("42") == self.TORRENT
|
|
924
|
+
(tmp_path / "42.torrent").unlink() # cache lost each time
|
|
925
|
+
|
|
926
|
+
assert p.fetch_artifact_by_id("42") is None
|
|
927
|
+
assert p.session.get.call_count == p.TORRENT_DOWNLOAD_BUDGET
|
|
928
|
+
assert "budget used up" in p.fetch_errors["42"]
|
|
929
|
+
assert str(tmp_path / "42.torrent") in p.fetch_errors["42"]
|
|
930
|
+
|
|
931
|
+
def test_ledger_survives_restart(self, tmp_path):
|
|
932
|
+
p = self._provider(tmp_path)
|
|
933
|
+
p.session.get = MagicMock(side_effect=self._read_timeout())
|
|
934
|
+
p.fetch_artifact_by_id("42")
|
|
935
|
+
|
|
936
|
+
restarted = self._provider(tmp_path)
|
|
937
|
+
restarted.session.get = MagicMock(side_effect=AssertionError("network"))
|
|
938
|
+
assert restarted.fetch_artifact_by_id("42") is None
|
|
939
|
+
assert restarted.downloads_used("42") == 1
|
|
940
|
+
|
|
941
|
+
def test_tracker_limit_error_blocks_until_torrent_supplied(self, tmp_path):
|
|
942
|
+
p = self._provider(tmp_path)
|
|
943
|
+
body = {"status": "failure", "error": "You have already downloaded this torrent file four times."}
|
|
944
|
+
p.session.get = MagicMock(return_value=self._resp(403, content=b"{}", content_type="application/json", json_body=body))
|
|
945
|
+
|
|
946
|
+
assert p.fetch_artifact_by_id("42") is None
|
|
947
|
+
assert p.fetch_artifact_by_id("42") is None
|
|
948
|
+
assert p.session.get.call_count == 1
|
|
949
|
+
assert self._provider(tmp_path)._download_blocked_reason("42")
|
|
950
|
+
|
|
951
|
+
# A browser-downloaded .torrent dropped into the cache is served.
|
|
952
|
+
(tmp_path / "42.torrent").write_bytes(self.TORRENT)
|
|
953
|
+
assert p.fetch_artifact_by_id("42") == self.TORRENT
|
|
954
|
+
assert p.session.get.call_count == 1
|
|
955
|
+
|
|
956
|
+
def test_tracker_limit_error_with_http_200(self, tmp_path):
|
|
957
|
+
p = self._provider(tmp_path)
|
|
958
|
+
body = {"status": "failure", "error": "You have already downloaded this torrent file four times."}
|
|
959
|
+
p.session.get = MagicMock(return_value=self._resp(200, content=b"{}", content_type="application/json", json_body=body))
|
|
960
|
+
|
|
961
|
+
assert p.fetch_artifact_by_id("42") is None
|
|
962
|
+
assert p.fetch_artifact_by_id("42") is None
|
|
963
|
+
assert p.session.get.call_count == 1
|
|
964
|
+
assert "budget used up" in p.fetch_errors["42"]
|
|
965
|
+
|
|
966
|
+
def test_json_refusal_reason_surfaced(self, tmp_path):
|
|
967
|
+
p = self._provider(tmp_path)
|
|
968
|
+
body = {"status": "failure", "error": "bad id"}
|
|
969
|
+
p.session.get = MagicMock(return_value=self._resp(200, content=b"{}", content_type="application/json", json_body=body))
|
|
970
|
+
assert p.fetch_artifact_by_id("42") is None
|
|
971
|
+
assert "bad id" in p.fetch_errors["42"]
|
|
972
|
+
|
|
973
|
+
def test_explicit_refusals_not_counted(self, tmp_path):
|
|
974
|
+
p = self._provider(tmp_path)
|
|
975
|
+
p.session.get = MagicMock(return_value=self._resp(400, content=b"{}", content_type="application/json", json_body={}))
|
|
976
|
+
for _ in range(5):
|
|
977
|
+
p.fetch_artifact_by_id("42")
|
|
978
|
+
assert p.downloads_used("42") == 0
|
|
979
|
+
assert p.session.get.call_count == 5
|
|
980
|
+
|
|
981
|
+
def test_server_error_counts_with_cooldown(self, tmp_path):
|
|
982
|
+
p = self._provider(tmp_path)
|
|
983
|
+
p.session.get = MagicMock(return_value=self._resp(502, content=b"bad gateway"))
|
|
984
|
+
assert p.fetch_artifact_by_id("42") is None
|
|
985
|
+
assert p.fetch_artifact_by_id("42") is None
|
|
986
|
+
assert p.session.get.call_count == 1
|
|
987
|
+
assert p.downloads_used("42") == 1
|
|
988
|
+
|
|
989
|
+
def test_connect_timeout_not_counted(self, tmp_path):
|
|
990
|
+
import requests
|
|
991
|
+
p = self._provider(tmp_path)
|
|
992
|
+
p.session.get = MagicMock(side_effect=[requests.exceptions.ConnectTimeout("no route"), self._resp()])
|
|
993
|
+
assert p.fetch_artifact_by_id("42") is None
|
|
994
|
+
assert p.fetch_artifact_by_id("42") == self.TORRENT
|
|
995
|
+
assert p.downloads_used("42") == 1
|
|
996
|
+
|
|
997
|
+
@pytest.mark.parametrize("make_error", [
|
|
998
|
+
lambda: __import__("requests").exceptions.SSLError("handshake failed"),
|
|
999
|
+
lambda: __import__("requests").exceptions.ConnectionError(
|
|
1000
|
+
__import__("urllib3").exceptions.MaxRetryError(
|
|
1001
|
+
None, "/", __import__("urllib3").exceptions.NewConnectionError(None, "refused"))),
|
|
1002
|
+
])
|
|
1003
|
+
def test_errors_before_sending_not_counted(self, tmp_path, make_error):
|
|
1004
|
+
p = self._provider(tmp_path)
|
|
1005
|
+
p.session.get = MagicMock(side_effect=[make_error(), self._resp()])
|
|
1006
|
+
assert p.fetch_artifact_by_id("42") is None
|
|
1007
|
+
assert "Could not connect" in p.fetch_errors["42"]
|
|
1008
|
+
assert p.fetch_artifact_by_id("42") == self.TORRENT # no cooldown
|
|
1009
|
+
assert p.downloads_used("42") == 1
|
|
1010
|
+
|
|
1011
|
+
def test_dropped_connection_after_send_counted(self, tmp_path):
|
|
1012
|
+
import requests
|
|
1013
|
+
p = self._provider(tmp_path)
|
|
1014
|
+
p.session.get = MagicMock(side_effect=requests.exceptions.ConnectionError("Connection aborted."))
|
|
1015
|
+
assert p.fetch_artifact_by_id("42") is None
|
|
1016
|
+
assert p.downloads_used("42") == 1
|
|
1017
|
+
|
|
1018
|
+
@pytest.mark.parametrize("content", ["{}", "[]", '{"requests": [1]}', '{"limit_reached": false}'])
|
|
1019
|
+
def test_wrong_shaped_ledger_blocks(self, tmp_path, content):
|
|
1020
|
+
(tmp_path / "42.downloads.json").write_text(content)
|
|
1021
|
+
p = self._provider(tmp_path)
|
|
1022
|
+
p.session.get = MagicMock(side_effect=AssertionError("network"))
|
|
1023
|
+
assert p.fetch_artifact_by_id("42") is None
|
|
1024
|
+
assert "budget used up" in p.fetch_errors["42"]
|
|
1025
|
+
|
|
1026
|
+
def test_failed_ledger_write_kept_in_memory(self, tmp_path, monkeypatch):
|
|
1027
|
+
from pathlib import Path
|
|
1028
|
+
p = self._provider(tmp_path)
|
|
1029
|
+
p.session.get = MagicMock(side_effect=self._read_timeout())
|
|
1030
|
+
monkeypatch.setattr(Path, "write_text", MagicMock(side_effect=OSError("disk full")))
|
|
1031
|
+
|
|
1032
|
+
p.fetch_artifact_by_id("42")
|
|
1033
|
+
assert not (tmp_path / "42.downloads.json").exists()
|
|
1034
|
+
assert p.fetch_artifact_by_id("42") is None
|
|
1035
|
+
assert p.session.get.call_count == 1
|
|
1036
|
+
assert p.downloads_used("42") == 1
|
|
1037
|
+
|
|
1038
|
+
def test_unreadable_ledger_blocks(self, tmp_path):
|
|
1039
|
+
(tmp_path / "42.downloads.json").write_text("{not json")
|
|
1040
|
+
p = self._provider(tmp_path)
|
|
1041
|
+
p.session.get = MagicMock(side_effect=AssertionError("network"))
|
|
1042
|
+
assert p.fetch_artifact_by_id("42") is None
|
|
1043
|
+
|
|
1044
|
+
def test_token_skipped_on_cached_torrent_when_budget_tight(self, tmp_path):
|
|
1045
|
+
p = self._provider(tmp_path, use_fl_token=True)
|
|
1046
|
+
(tmp_path / "42.torrent").write_bytes(self.TORRENT)
|
|
1047
|
+
for _ in range(p.TORRENT_DOWNLOAD_BUDGET - 1):
|
|
1048
|
+
p._record_download_request("42", "normal", "ok")
|
|
1049
|
+
p.session.get = MagicMock(side_effect=AssertionError("network"))
|
|
1050
|
+
|
|
1051
|
+
assert p.fetch_artifact_by_id("42") == self.TORRENT
|
|
1052
|
+
|
|
1053
|
+
def test_token_still_spent_on_cached_torrent_with_room(self, tmp_path):
|
|
1054
|
+
p = self._provider(tmp_path, use_fl_token=True)
|
|
1055
|
+
(tmp_path / "42.torrent").write_bytes(self.TORRENT)
|
|
1056
|
+
p.session.get = MagicMock(return_value=self._resp())
|
|
1057
|
+
|
|
1058
|
+
assert p.fetch_artifact_by_id("42") == self.TORRENT
|
|
1059
|
+
assert "usetoken=1" in p.session.get.call_args.args[0]
|
|
1060
|
+
assert p.downloads_used("42") == 1
|
|
1061
|
+
|
|
1062
|
+
def test_ledger_without_cache_dir_kept_in_memory(self, tmp_path):
|
|
1063
|
+
p = self._provider(tmp_path)
|
|
1064
|
+
p.cache_dir = None
|
|
1065
|
+
p.session.get = MagicMock(side_effect=self._read_timeout())
|
|
1066
|
+
p.fetch_artifact_by_id("42")
|
|
1067
|
+
assert p.fetch_artifact_by_id("42") is None
|
|
1068
|
+
assert p.session.get.call_count == 1
|
|
1069
|
+
|
|
1070
|
+
def test_deleting_ledger_file_resets_without_restart(self, tmp_path):
|
|
1071
|
+
p = self._provider(tmp_path)
|
|
1072
|
+
p.session.get = MagicMock(side_effect=[self._read_timeout(), self._resp()])
|
|
1073
|
+
p.fetch_artifact_by_id("42")
|
|
1074
|
+
(tmp_path / "42.downloads.json").unlink()
|
|
1075
|
+
assert p.fetch_artifact_by_id("42") == self.TORRENT
|
|
1076
|
+
|
|
1077
|
+
def test_token_spend_counts_against_budget(self, tmp_path):
|
|
1078
|
+
p = self._provider(tmp_path, use_fl_token=True)
|
|
1079
|
+
p.session.get = MagicMock(return_value=self._resp())
|
|
1080
|
+
p.fetch_artifact_by_id("42")
|
|
1081
|
+
ledger = p._load_download_ledger("42")
|
|
1082
|
+
assert [r["kind"] for r in ledger["requests"]] == ["token"]
|
|
@@ -589,6 +589,31 @@ class TestDownloadById:
|
|
|
589
589
|
assert release.source_name == "RED"
|
|
590
590
|
assert release.target_file == "01 - Track.flac"
|
|
591
591
|
|
|
592
|
+
def test_download_by_id_red_failure_includes_provider_reason(self):
|
|
593
|
+
"""The provider's reason (e.g. budget used up) reaches the job error."""
|
|
594
|
+
mgr = FetchManager()
|
|
595
|
+
mock_dl = Mock(spec=Downloader)
|
|
596
|
+
|
|
597
|
+
class MockREDProvider(Provider):
|
|
598
|
+
fetch_errors = {"12345": "RED .torrent download budget used up for torrent 12345"}
|
|
599
|
+
|
|
600
|
+
@property
|
|
601
|
+
def name(self):
|
|
602
|
+
return "RED"
|
|
603
|
+
|
|
604
|
+
def search(self, q):
|
|
605
|
+
return []
|
|
606
|
+
|
|
607
|
+
def fetch_artifact_by_id(self, torrent_id):
|
|
608
|
+
return None
|
|
609
|
+
|
|
610
|
+
mgr.add_provider(MockREDProvider())
|
|
611
|
+
mgr.register_downloader("RED", mock_dl)
|
|
612
|
+
|
|
613
|
+
with pytest.raises(ValueError, match="Failed to fetch torrent file for RED ID: 12345: RED .torrent download budget used up"):
|
|
614
|
+
mgr.download_by_id(source_name="RED", source_id="12345", output_path="/output")
|
|
615
|
+
mock_dl.download.assert_not_called()
|
|
616
|
+
|
|
592
617
|
def test_download_by_id_ops_torrent(self):
|
|
593
618
|
"""Test download_by_id for OPS creates Release with correct quality"""
|
|
594
619
|
mgr = FetchManager()
|
|
@@ -0,0 +1,32 @@
|
|
|
1
|
+
"""Tests for the YouTube search provider's query construction."""
|
|
2
|
+
from contextlib import contextmanager
|
|
3
|
+
from unittest.mock import MagicMock, patch
|
|
4
|
+
|
|
5
|
+
from flacfetch.core.models import TrackQuery
|
|
6
|
+
from flacfetch.providers.youtube import YoutubeProvider
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
def _search_and_capture_query(artist: str, title: str) -> str:
|
|
10
|
+
ydl = MagicMock()
|
|
11
|
+
ydl.extract_info.return_value = {"entries": []}
|
|
12
|
+
ydl_cm = MagicMock()
|
|
13
|
+
ydl_cm.__enter__.return_value = ydl
|
|
14
|
+
|
|
15
|
+
@contextmanager
|
|
16
|
+
def fake_isolated(opts):
|
|
17
|
+
yield opts
|
|
18
|
+
|
|
19
|
+
with patch("flacfetch.providers.youtube.get_ytdlp_base_opts", return_value={}), \
|
|
20
|
+
patch("flacfetch.providers.youtube.ytdlp_opts_isolated", fake_isolated), \
|
|
21
|
+
patch("flacfetch.providers.youtube.yt_dlp.YoutubeDL", return_value=ydl_cm):
|
|
22
|
+
YoutubeProvider().search(TrackQuery(artist=artist, title=title))
|
|
23
|
+
|
|
24
|
+
return ydl.extract_info.call_args.args[0]
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
def test_search_query_is_plain_artist_title():
|
|
28
|
+
"""A ' topic' suffix pushed the actual song out of the results
|
|
29
|
+
(Playa Fly - N God We Trust), so the query must be just artist + title."""
|
|
30
|
+
query = _search_and_capture_query("Playa Fly", "N God We Trust")
|
|
31
|
+
assert query == "ytsearch5:Playa Fly N God We Trust"
|
|
32
|
+
assert "topic" not in query.lower()
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|