flacfetch 0.35.1__tar.gz → 0.36.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (100) hide show
  1. {flacfetch-0.35.1/flacfetch.egg-info → flacfetch-0.36.0}/PKG-INFO +1 -1
  2. {flacfetch-0.35.1 → flacfetch-0.36.0}/flacfetch/core/manager.py +7 -0
  3. {flacfetch-0.35.1 → flacfetch-0.36.0}/flacfetch/providers/gazelle.py +199 -10
  4. {flacfetch-0.35.1 → flacfetch-0.36.0/flacfetch.egg-info}/PKG-INFO +1 -1
  5. {flacfetch-0.35.1 → flacfetch-0.36.0}/pyproject.toml +1 -1
  6. {flacfetch-0.35.1 → flacfetch-0.36.0}/tests/test_gazelle.py +249 -0
  7. {flacfetch-0.35.1 → flacfetch-0.36.0}/tests/test_manager.py +25 -0
  8. {flacfetch-0.35.1 → flacfetch-0.36.0}/ARCHITECTURE.md +0 -0
  9. {flacfetch-0.35.1 → flacfetch-0.36.0}/LICENSE +0 -0
  10. {flacfetch-0.35.1 → flacfetch-0.36.0}/MANIFEST.in +0 -0
  11. {flacfetch-0.35.1 → flacfetch-0.36.0}/PLAN.md +0 -0
  12. {flacfetch-0.35.1 → flacfetch-0.36.0}/README.md +0 -0
  13. {flacfetch-0.35.1 → flacfetch-0.36.0}/flacfetch/__init__.py +0 -0
  14. {flacfetch-0.35.1 → flacfetch-0.36.0}/flacfetch/api/__init__.py +0 -0
  15. {flacfetch-0.35.1 → flacfetch-0.36.0}/flacfetch/api/auth.py +0 -0
  16. {flacfetch-0.35.1 → flacfetch-0.36.0}/flacfetch/api/main.py +0 -0
  17. {flacfetch-0.35.1 → flacfetch-0.36.0}/flacfetch/api/models.py +0 -0
  18. {flacfetch-0.35.1 → flacfetch-0.36.0}/flacfetch/api/routes/__init__.py +0 -0
  19. {flacfetch-0.35.1 → flacfetch-0.36.0}/flacfetch/api/routes/cache.py +0 -0
  20. {flacfetch-0.35.1 → flacfetch-0.36.0}/flacfetch/api/routes/config.py +0 -0
  21. {flacfetch-0.35.1 → flacfetch-0.36.0}/flacfetch/api/routes/credentials.py +0 -0
  22. {flacfetch-0.35.1 → flacfetch-0.36.0}/flacfetch/api/routes/download.py +0 -0
  23. {flacfetch-0.35.1 → flacfetch-0.36.0}/flacfetch/api/routes/health.py +0 -0
  24. {flacfetch-0.35.1 → flacfetch-0.36.0}/flacfetch/api/routes/search.py +0 -0
  25. {flacfetch-0.35.1 → flacfetch-0.36.0}/flacfetch/api/routes/spotify.py +0 -0
  26. {flacfetch-0.35.1 → flacfetch-0.36.0}/flacfetch/api/routes/torrents.py +0 -0
  27. {flacfetch-0.35.1 → flacfetch-0.36.0}/flacfetch/api/services/__init__.py +0 -0
  28. {flacfetch-0.35.1 → flacfetch-0.36.0}/flacfetch/api/services/credential_check.py +0 -0
  29. {flacfetch-0.35.1 → flacfetch-0.36.0}/flacfetch/api/services/disk_manager.py +0 -0
  30. {flacfetch-0.35.1 → flacfetch-0.36.0}/flacfetch/api/services/download_manager.py +0 -0
  31. {flacfetch-0.35.1 → flacfetch-0.36.0}/flacfetch/api/services/health_check.py +0 -0
  32. {flacfetch-0.35.1 → flacfetch-0.36.0}/flacfetch/api/services/search_cache.py +0 -0
  33. {flacfetch-0.35.1 → flacfetch-0.36.0}/flacfetch/api/services/secret_manager_utils.py +0 -0
  34. {flacfetch-0.35.1 → flacfetch-0.36.0}/flacfetch/api/services/spotify_budget.py +0 -0
  35. {flacfetch-0.35.1 → flacfetch-0.36.0}/flacfetch/core/album_match.py +0 -0
  36. {flacfetch-0.35.1 → flacfetch-0.36.0}/flacfetch/core/categorize.py +0 -0
  37. {flacfetch-0.35.1 → flacfetch-0.36.0}/flacfetch/core/config.py +0 -0
  38. {flacfetch-0.35.1 → flacfetch-0.36.0}/flacfetch/core/interfaces.py +0 -0
  39. {flacfetch-0.35.1 → flacfetch-0.36.0}/flacfetch/core/log.py +0 -0
  40. {flacfetch-0.35.1 → flacfetch-0.36.0}/flacfetch/core/matching.py +0 -0
  41. {flacfetch-0.35.1 → flacfetch-0.36.0}/flacfetch/core/models.py +0 -0
  42. {flacfetch-0.35.1 → flacfetch-0.36.0}/flacfetch/credential_keeper/__init__.py +0 -0
  43. {flacfetch-0.35.1 → flacfetch-0.36.0}/flacfetch/credential_keeper/__main__.py +0 -0
  44. {flacfetch-0.35.1 → flacfetch-0.36.0}/flacfetch/credential_keeper/browser.py +0 -0
  45. {flacfetch-0.35.1 → flacfetch-0.36.0}/flacfetch/credential_keeper/google_login.py +0 -0
  46. {flacfetch-0.35.1 → flacfetch-0.36.0}/flacfetch/credential_keeper/keeper.py +0 -0
  47. {flacfetch-0.35.1 → flacfetch-0.36.0}/flacfetch/credential_keeper/librespot.py +0 -0
  48. {flacfetch-0.35.1 → flacfetch-0.36.0}/flacfetch/credential_keeper/probe.py +0 -0
  49. {flacfetch-0.35.1 → flacfetch-0.36.0}/flacfetch/credential_keeper/spotify.py +0 -0
  50. {flacfetch-0.35.1 → flacfetch-0.36.0}/flacfetch/credential_keeper/youtube.py +0 -0
  51. {flacfetch-0.35.1 → flacfetch-0.36.0}/flacfetch/downloaders/spotify.py +0 -0
  52. {flacfetch-0.35.1 → flacfetch-0.36.0}/flacfetch/downloaders/torrent.py +0 -0
  53. {flacfetch-0.35.1 → flacfetch-0.36.0}/flacfetch/downloaders/torrent_coordinator.py +0 -0
  54. {flacfetch-0.35.1 → flacfetch-0.36.0}/flacfetch/downloaders/youtube.py +0 -0
  55. {flacfetch-0.35.1 → flacfetch-0.36.0}/flacfetch/interface/cli.py +0 -0
  56. {flacfetch-0.35.1 → flacfetch-0.36.0}/flacfetch/interface/cli_remote.py +0 -0
  57. {flacfetch-0.35.1 → flacfetch-0.36.0}/flacfetch/providers/ops.py +0 -0
  58. {flacfetch-0.35.1 → flacfetch-0.36.0}/flacfetch/providers/red.py +0 -0
  59. {flacfetch-0.35.1 → flacfetch-0.36.0}/flacfetch/providers/spotify.py +0 -0
  60. {flacfetch-0.35.1 → flacfetch-0.36.0}/flacfetch/providers/youtube.py +0 -0
  61. {flacfetch-0.35.1 → flacfetch-0.36.0}/flacfetch.egg-info/SOURCES.txt +0 -0
  62. {flacfetch-0.35.1 → flacfetch-0.36.0}/flacfetch.egg-info/dependency_links.txt +0 -0
  63. {flacfetch-0.35.1 → flacfetch-0.36.0}/flacfetch.egg-info/entry_points.txt +0 -0
  64. {flacfetch-0.35.1 → flacfetch-0.36.0}/flacfetch.egg-info/requires.txt +0 -0
  65. {flacfetch-0.35.1 → flacfetch-0.36.0}/flacfetch.egg-info/top_level.txt +0 -0
  66. {flacfetch-0.35.1 → flacfetch-0.36.0}/requirements.txt +0 -0
  67. {flacfetch-0.35.1 → flacfetch-0.36.0}/setup.cfg +0 -0
  68. {flacfetch-0.35.1 → flacfetch-0.36.0}/setup.py +0 -0
  69. {flacfetch-0.35.1 → flacfetch-0.36.0}/tests/_route_paths.py +0 -0
  70. {flacfetch-0.35.1 → flacfetch-0.36.0}/tests/test_album_search.py +0 -0
  71. {flacfetch-0.35.1 → flacfetch-0.36.0}/tests/test_api_auth.py +0 -0
  72. {flacfetch-0.35.1 → flacfetch-0.36.0}/tests/test_api_config.py +0 -0
  73. {flacfetch-0.35.1 → flacfetch-0.36.0}/tests/test_api_deep_health.py +0 -0
  74. {flacfetch-0.35.1 → flacfetch-0.36.0}/tests/test_api_health_ytdlp.py +0 -0
  75. {flacfetch-0.35.1 → flacfetch-0.36.0}/tests/test_api_integration.py +0 -0
  76. {flacfetch-0.35.1 → flacfetch-0.36.0}/tests/test_api_models.py +0 -0
  77. {flacfetch-0.35.1 → flacfetch-0.36.0}/tests/test_api_routes.py +0 -0
  78. {flacfetch-0.35.1 → flacfetch-0.36.0}/tests/test_api_services.py +0 -0
  79. {flacfetch-0.35.1 → flacfetch-0.36.0}/tests/test_cache_api.py +0 -0
  80. {flacfetch-0.35.1 → flacfetch-0.36.0}/tests/test_cli_cookies.py +0 -0
  81. {flacfetch-0.35.1 → flacfetch-0.36.0}/tests/test_cli_remote.py +0 -0
  82. {flacfetch-0.35.1 → flacfetch-0.36.0}/tests/test_core.py +0 -0
  83. {flacfetch-0.35.1 → flacfetch-0.36.0}/tests/test_credential_keeper.py +0 -0
  84. {flacfetch-0.35.1 → flacfetch-0.36.0}/tests/test_keeper_probe.py +0 -0
  85. {flacfetch-0.35.1 → flacfetch-0.36.0}/tests/test_logging.py +0 -0
  86. {flacfetch-0.35.1 → flacfetch-0.36.0}/tests/test_matching.py +0 -0
  87. {flacfetch-0.35.1 → flacfetch-0.36.0}/tests/test_models.py +0 -0
  88. {flacfetch-0.35.1 → flacfetch-0.36.0}/tests/test_ops.py +0 -0
  89. {flacfetch-0.35.1 → flacfetch-0.36.0}/tests/test_provider_errors.py +0 -0
  90. {flacfetch-0.35.1 → flacfetch-0.36.0}/tests/test_red.py +0 -0
  91. {flacfetch-0.35.1 → flacfetch-0.36.0}/tests/test_search_api.py +0 -0
  92. {flacfetch-0.35.1 → flacfetch-0.36.0}/tests/test_spotify.py +0 -0
  93. {flacfetch-0.35.1 → flacfetch-0.36.0}/tests/test_spotify_budget.py +0 -0
  94. {flacfetch-0.35.1 → flacfetch-0.36.0}/tests/test_torrent_coordinator.py +0 -0
  95. {flacfetch-0.35.1 → flacfetch-0.36.0}/tests/test_torrent_downloader.py +0 -0
  96. {flacfetch-0.35.1 → flacfetch-0.36.0}/tests/test_torrent_shared_album.py +0 -0
  97. {flacfetch-0.35.1 → flacfetch-0.36.0}/tests/test_transcode_hf_noise.py +0 -0
  98. {flacfetch-0.35.1 → flacfetch-0.36.0}/tests/test_youtube.py +0 -0
  99. {flacfetch-0.35.1 → flacfetch-0.36.0}/tests/test_youtube_cookies.py +0 -0
  100. {flacfetch-0.35.1 → flacfetch-0.36.0}/tests/test_youtube_provider.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: flacfetch
3
- Version: 0.35.1
3
+ Version: 0.36.0
4
4
  Summary: Search and download high-quality audio from multiple sources
5
5
  Home-page: https://github.com/yourusername/flacfetch
6
6
  Author: Your Name
@@ -373,6 +373,10 @@ class FetchManager:
373
373
  # If download_url is http..., TorrentDownloader will fail.
374
374
  if release.download_url and release.download_url.startswith("http"):
375
375
  msg = "Failed to download torrent file from provider. Cannot proceed with download."
376
+ torrent_id = torrent_id_from_url(release.download_url) if torrent_id_from_url else None
377
+ reason = getattr(provider, "fetch_errors", {}).get(torrent_id)
378
+ if reason:
379
+ msg = f"{msg} {reason}"
376
380
  logger.error(msg)
377
381
  raise ValueError(msg)
378
382
 
@@ -520,6 +524,9 @@ class FetchManager:
520
524
 
521
525
  if not artifact:
522
526
  msg = f"Failed to fetch torrent file for {source_name} ID: {source_id}"
527
+ reason = getattr(provider, "fetch_errors", {}).get(source_id)
528
+ if reason:
529
+ msg = f"{msg}: {reason}"
523
530
  logger.error(msg)
524
531
  raise ValueError(msg)
525
532
 
@@ -1,6 +1,7 @@
1
1
  """Base provider for Gazelle-based private music trackers (RED, OPS, etc.)."""
2
2
 
3
3
  import html
4
+ import json
4
5
  import threading
5
6
  import time
6
7
  from abc import abstractmethod
@@ -9,6 +10,7 @@ from typing import Any, Optional
9
10
  from urllib.parse import parse_qs, urlparse
10
11
 
11
12
  import requests
13
+ from urllib3.exceptions import NewConnectionError
12
14
 
13
15
  from ..core.album_match import AlbumCandidate, AlbumTrack, match_album_tracks, parse_file_list
14
16
  from ..core.interfaces import Provider
@@ -55,6 +57,24 @@ class GazelleProvider(Provider):
55
57
  # After a token spend fails for a torrent, don't retry it for this long.
56
58
  FL_TOKEN_FAILURE_BACKOFF_SECONDS = 600
57
59
 
60
+ # RED lets a user fetch each .torrent file through the API only four times,
61
+ # ever ("You have already downloaded this torrent file four times"); after
62
+ # that it must come from a browser. Every request that may have counted is
63
+ # recorded per torrent in an on-disk ledger, and we stop at this many so one
64
+ # download is always left for a manual browser recovery.
65
+ TORRENT_DOWNLOAD_BUDGET = 3
66
+
67
+ # A request the tracker never answered (read timeout, dropped connection,
68
+ # 5xx) may still have counted server-side: on 2026-10-23 RED stalled and
69
+ # three timed-out fetches used up a torrent's limit with nothing cached.
70
+ # After one, don't ask the tracker for that torrent again for this long.
71
+ UNANSWERED_COOLDOWN_SECONDS = 300
72
+
73
+ # (connect, read) timeouts for .torrent downloads. The file is a few KB, so
74
+ # a slow response is the tracker stalling; waiting it out is far cheaper
75
+ # than a retry that spends another of the four downloads.
76
+ DOWNLOAD_TIMEOUT = (10, 60)
77
+
58
78
  def __init__(self, api_key: str, base_url: str, cache_subdir: str, use_fl_token: bool = False):
59
79
  """Initialize the Gazelle provider.
60
80
 
@@ -90,6 +110,12 @@ class GazelleProvider(Provider):
90
110
  # ONE tracker fetch: the first caches the .torrent, the rest hit the cache.
91
111
  self._fetch_locks: dict[str, threading.Lock] = {}
92
112
  self._fetch_locks_guard = threading.Lock()
113
+ # Per-torrent download ledgers (see TORRENT_DOWNLOAD_BUDGET) kept in
114
+ # memory: when there is no cache dir, or the last disk write failed (an
115
+ # unrecorded request must never let retries reach the tracker again).
116
+ self._download_ledgers: dict[str, dict] = {}
117
+ # {torrent_id: reason} for the last failed fetch, surfaced in job errors.
118
+ self.fetch_errors: dict[str, str] = {}
93
119
  self.session = requests.Session()
94
120
  self.session.headers.update({"Authorization": self.api_key})
95
121
  self.search_limit = 10
@@ -259,9 +285,13 @@ class GazelleProvider(Provider):
259
285
  def _fetch_torrent_from_url(self, url: str, torrent_id: Optional[str] = None, max_retries: int = 3) -> Optional[bytes]:
260
286
  """Fetch a torrent file from URL and cache it.
261
287
 
288
+ Every request goes through the per-torrent download ledger: nothing is
289
+ sent once the torrent's budget is used up or while it is cooling down
290
+ after an unanswered request (see TORRENT_DOWNLOAD_BUDGET).
291
+
262
292
  Args:
263
293
  url: Download URL for the torrent file
264
- torrent_id: Optional torrent ID for caching
294
+ torrent_id: Optional torrent ID for caching (parsed from the URL if omitted)
265
295
  max_retries: Maximum number of retries on rate limiting (default 3)
266
296
 
267
297
  Returns:
@@ -271,8 +301,17 @@ class GazelleProvider(Provider):
271
301
  if url.startswith("/") or url.startswith("file://"):
272
302
  return None
273
303
 
304
+ torrent_id = torrent_id or self.torrent_id_from_url(url)
305
+ kind = "token" if "usetoken=1" in url else "normal"
306
+
274
307
  retries = 0
275
308
  while retries <= max_retries:
309
+ blocked = self._download_blocked_reason(torrent_id)
310
+ if blocked:
311
+ logger.error(f"{self.name}: not requesting .torrent {torrent_id}: {blocked}")
312
+ self.fetch_errors[torrent_id] = blocked
313
+ return None
314
+
276
315
  try:
277
316
  logger.info(f"Fetching artifact from {url}")
278
317
 
@@ -283,7 +322,7 @@ class GazelleProvider(Provider):
283
322
  logger.debug(f"Downloading artifact from: {url}")
284
323
  logger.debug(f"Request Headers: {req_headers}")
285
324
 
286
- resp = self.session.get(url, timeout=10, allow_redirects=False)
325
+ resp = self.session.get(url, timeout=self.DOWNLOAD_TIMEOUT, allow_redirects=False)
287
326
  logger.debug(f"Response Status: {resp.status_code}")
288
327
  logger.debug(f"Response Headers: {dict(resp.headers)}")
289
328
 
@@ -291,12 +330,10 @@ class GazelleProvider(Provider):
291
330
  try:
292
331
  content = resp.json()
293
332
  logger.debug(f"Response Body (JSON): {content}")
294
- if content.get("status") == "failure" and "already downloaded" in content.get("error", ""):
295
- logger.error(f"{self.name} Limit Reached: {content.get('error')}")
296
- logger.info("Tip: You can download the .torrent file manually from the website and place it in the cache directory:")
297
- if self.cache_dir and torrent_id:
298
- logger.info(f" {self.cache_dir}/{torrent_id}.torrent")
299
- except:
333
+ if self._is_limit_error(content):
334
+ self._handle_limit_error(torrent_id, content)
335
+ return None
336
+ except Exception:
300
337
  logger.debug(f"Response Body (Text/Raw): {resp.text[:500]}")
301
338
 
302
339
  if resp.status_code in (301, 302, 303, 307, 308):
@@ -305,7 +342,7 @@ class GazelleProvider(Provider):
305
342
  if redirect_url:
306
343
  if redirect_url.startswith("/"):
307
344
  redirect_url = self.base_url + redirect_url
308
- resp = self.session.get(redirect_url, timeout=10)
345
+ resp = self.session.get(redirect_url, timeout=self.DOWNLOAD_TIMEOUT)
309
346
 
310
347
  if resp.status_code == 200:
311
348
  # The download endpoint returns raw bencoded .torrent bytes on
@@ -316,13 +353,20 @@ class GazelleProvider(Provider):
316
353
  content_type = resp.headers.get("Content-Type", "").lower()
317
354
  is_torrent = resp.content[:1] == b"d" and "json" not in content_type
318
355
  if not is_torrent:
356
+ self._record_download_request(torrent_id, kind, "refused")
319
357
  try:
320
358
  err = resp.json()
359
+ if self._is_limit_error(err):
360
+ self._handle_limit_error(torrent_id, err)
361
+ return None
321
362
  logger.warning(f"{self.name} download returned an error instead of a torrent: {err.get('error', err)}")
363
+ if torrent_id:
364
+ self.fetch_errors[torrent_id] = f"{self.name} refused the .torrent download: {err.get('error', err)}"
322
365
  except ValueError:
323
366
  logger.warning(f"{self.name} download returned non-torrent content ({len(resp.content)} bytes): {resp.content[:100]}")
324
367
  return None
325
368
 
369
+ self._record_download_request(torrent_id, kind, "ok")
326
370
  logger.debug(f"Artifact fetched successfully ({len(resp.content)} bytes)")
327
371
  if len(resp.content) < 1000:
328
372
  logger.warning(f"Artifact seems too small ({len(resp.content)} bytes). Content sample: {resp.content[:100]}")
@@ -339,6 +383,7 @@ class GazelleProvider(Provider):
339
383
 
340
384
  return resp.content
341
385
  elif resp.status_code == 429:
386
+ self._record_download_request(torrent_id, kind, "refused")
342
387
  retries += 1
343
388
  if retries <= max_retries:
344
389
  logger.warning(f"Rate limited while fetching artifact. Retry {retries}/{max_retries} in 2s...")
@@ -348,14 +393,152 @@ class GazelleProvider(Provider):
348
393
  logger.error(f"Rate limit exceeded after {max_retries} retries")
349
394
  return None
350
395
  else:
396
+ # A 4xx is an explicit refusal (nothing served). A 5xx may have
397
+ # been generated and counted before the server failed.
398
+ outcome = "unanswered" if resp.status_code >= 500 else "refused"
399
+ self._record_download_request(torrent_id, kind, outcome)
351
400
  logger.error(f"Failed to fetch artifact: Status {resp.status_code}")
401
+ if outcome == "unanswered":
402
+ self.fetch_errors[torrent_id] = f"{self.name} returned HTTP {resp.status_code} for the .torrent download"
403
+ return None
404
+ except requests.exceptions.RequestException as e:
405
+ logger.error(f"Error fetching artifact: {e}")
406
+ if self._request_never_sent(e):
407
+ # DNS / refused / TLS / proxy failure: the tracker never saw it.
408
+ self.fetch_errors[torrent_id] = f"Could not connect to {self.name} ({type(e).__name__})"
352
409
  return None
410
+ # Read timeout / dropped connection: the tracker may have counted it.
411
+ self._record_download_request(torrent_id, kind, "unanswered")
412
+ self.fetch_errors[torrent_id] = f"{self.name} did not answer the .torrent download ({type(e).__name__})"
413
+ return None
353
414
  except Exception as e:
415
+ # Anything else may have happened after the tracker served the
416
+ # file, so count it rather than risk an unrecorded download.
417
+ self._record_download_request(torrent_id, kind, "unanswered")
354
418
  logger.error(f"Error fetching artifact: {e}")
419
+ self.fetch_errors[torrent_id] = f"{self.name} .torrent download failed ({type(e).__name__})"
355
420
  return None
356
421
 
357
422
  return None
358
423
 
424
+ @staticmethod
425
+ def _request_never_sent(error: Exception) -> bool:
426
+ """True when a requests error happened before the request reached the tracker."""
427
+ if isinstance(error, (requests.exceptions.ConnectTimeout, requests.exceptions.SSLError, requests.exceptions.ProxyError)):
428
+ return True
429
+ if isinstance(error, requests.exceptions.ConnectionError):
430
+ # Wraps urllib3's MaxRetryError; NewConnectionError (incl. DNS
431
+ # NameResolutionError) means no connection was ever made.
432
+ reason = getattr(error.args[0], "reason", None) if error.args else None
433
+ return isinstance(reason, NewConnectionError)
434
+ return False
435
+
436
+ @staticmethod
437
+ def _is_limit_error(body) -> bool:
438
+ return isinstance(body, dict) and body.get("status") == "failure" and "already downloaded" in str(body.get("error", ""))
439
+
440
+ def _handle_limit_error(self, torrent_id: Optional[str], body: dict) -> None:
441
+ """The tracker says this torrent's download limit is used up."""
442
+ logger.error(f"{self.name} Limit Reached: {body.get('error')}")
443
+ if torrent_id:
444
+ self._mark_download_limit_reached(torrent_id)
445
+ self.fetch_errors[torrent_id] = self._limit_reached_message(torrent_id)
446
+
447
+ # ---- per-torrent download ledger --------------------------------------
448
+
449
+ def _download_ledger_path(self, torrent_id: str) -> Optional[Path]:
450
+ if not self.cache_dir or not torrent_id:
451
+ return None
452
+ return self.cache_dir / f"{torrent_id}.downloads.json"
453
+
454
+ def _load_download_ledger(self, torrent_id: str) -> dict:
455
+ """The torrent's ledger: ``{"requests": [{t, kind, outcome}], "limit_reached": bool}``.
456
+
457
+ Re-read from disk each time (when there is a cache dir) so an operator can
458
+ inspect or reset a torrent's ledger file without restarting the service.
459
+ """
460
+ path = self._download_ledger_path(torrent_id)
461
+ if not path:
462
+ return self._download_ledgers.setdefault(torrent_id, {"requests": [], "limit_reached": False})
463
+ if torrent_id in self._download_ledgers:
464
+ return self._download_ledgers[torrent_id]
465
+ if not path.exists():
466
+ return {"requests": [], "limit_reached": False}
467
+ try:
468
+ ledger = json.loads(path.read_text())
469
+ if not (isinstance(ledger, dict) and isinstance(ledger.get("requests"), list)
470
+ and all(isinstance(r, dict) for r in ledger["requests"])):
471
+ raise ValueError("unexpected ledger shape")
472
+ return ledger
473
+ except (OSError, ValueError) as e:
474
+ # An unreadable ledger must not unlock fresh downloads.
475
+ logger.warning(f"{self.name}: unreadable download ledger for {torrent_id} ({e}); treating as used up")
476
+ return {"requests": [], "limit_reached": True}
477
+
478
+ def _record_download_request(self, torrent_id: Optional[str], kind: str, outcome: str) -> None:
479
+ """Record one download-endpoint request.
480
+
481
+ ``outcome`` is ``ok`` (torrent served), ``unanswered`` (timeout, dropped
482
+ connection, 5xx: may have counted) or ``refused`` (an explicit error or
483
+ 429: nothing served, not counted).
484
+ """
485
+ if not torrent_id:
486
+ return
487
+ ledger = self._load_download_ledger(torrent_id)
488
+ ledger["requests"].append({"t": time.time(), "kind": kind, "outcome": outcome})
489
+ self._save_download_ledger(torrent_id, ledger)
490
+
491
+ def _mark_download_limit_reached(self, torrent_id: Optional[str]) -> None:
492
+ if not torrent_id:
493
+ return
494
+ ledger = self._load_download_ledger(torrent_id)
495
+ ledger["limit_reached"] = True
496
+ self._save_download_ledger(torrent_id, ledger)
497
+
498
+ def _save_download_ledger(self, torrent_id: str, ledger: dict) -> None:
499
+ path = self._download_ledger_path(torrent_id)
500
+ if not path:
501
+ return
502
+ tmp = path.with_name(path.name + ".tmp")
503
+ try:
504
+ tmp.write_text(json.dumps(ledger))
505
+ tmp.replace(path)
506
+ self._download_ledgers.pop(torrent_id, None)
507
+ except OSError as e:
508
+ logger.warning(f"{self.name}: could not write download ledger for {torrent_id}: {e}; keeping it in memory")
509
+ self._download_ledgers[torrent_id] = ledger
510
+
511
+ def downloads_used(self, torrent_id: str) -> int:
512
+ """Requests for this torrent that may have counted against the tracker's limit."""
513
+ ledger = self._load_download_ledger(torrent_id)
514
+ return sum(1 for r in ledger["requests"] if r.get("outcome") in ("ok", "unanswered"))
515
+
516
+ def _download_blocked_reason(self, torrent_id: Optional[str]) -> Optional[str]:
517
+ """Why the tracker must not be asked for this torrent right now, or None."""
518
+ if not torrent_id:
519
+ return None
520
+ ledger = self._load_download_ledger(torrent_id)
521
+ if ledger.get("limit_reached") or self.downloads_used(torrent_id) >= self.TORRENT_DOWNLOAD_BUDGET:
522
+ return self._limit_reached_message(torrent_id)
523
+ unanswered = [r.get("t", 0) for r in ledger["requests"] if r.get("outcome") == "unanswered"]
524
+ if unanswered:
525
+ wait = unanswered[-1] + self.UNANSWERED_COOLDOWN_SECONDS - time.time()
526
+ if wait > 0:
527
+ return (
528
+ f"{self.name} did not answer the last .torrent request for {torrent_id}; "
529
+ f"not asking again for {int(wait) + 1}s so a stalled tracker can't use up "
530
+ f"its download limit. Retry in a few minutes."
531
+ )
532
+ return None
533
+
534
+ def _limit_reached_message(self, torrent_id: str) -> str:
535
+ path = self.cache_dir / f"{torrent_id}.torrent" if self.cache_dir else f"<cache>/{torrent_id}.torrent"
536
+ return (
537
+ f"{self.name} .torrent download budget used up for torrent {torrent_id} "
538
+ f"({self.downloads_used(torrent_id)} recorded requests). Download the .torrent "
539
+ f"in a browser and save it as {path}, then retry."
540
+ )
541
+
359
542
  def _token_marker_path(self, torrent_id: str) -> Optional[Path]:
360
543
  """Path of the local marker recording a token spend for a torrent."""
361
544
  if not self.cache_dir or not torrent_id:
@@ -549,15 +732,22 @@ class GazelleProvider(Provider):
549
732
  # each fetched the .torrent -- counting against the tracker's per-torrent
550
733
  # download limit (RED: four). Siblings now wait and take the cached copy.
551
734
  with self._fetch_lock(source_id):
735
+ self.fetch_errors.pop(source_id, None)
736
+ cached = self.get_cached_artifact(source_id)
737
+
552
738
  # If we should spend a token and haven't already tokened this torrent
553
739
  # within the freeleech window, register the token FIRST -- before the
554
740
  # cache short-circuit. A cached .torrent file does NOT make a download
555
741
  # freeleech; the token must be registered server-side via usetoken=1,
556
742
  # otherwise the (re-)download of the torrent data counts against ratio.
743
+ # A successful token spend is itself a .torrent download, though, so
744
+ # with a cached copy in hand only spend one while it leaves room in the
745
+ # download budget: running out would cost far more than one torrent's ratio.
557
746
  if (
558
747
  use_token
559
748
  and not self._token_recently_spent(source_id)
560
749
  and not self._token_recently_failed(source_id)
750
+ and not (cached and self.downloads_used(source_id) + 1 >= self.TORRENT_DOWNLOAD_BUDGET)
561
751
  ):
562
752
  with self._token_lock:
563
753
  content = self._spend_token(url, source_id)
@@ -570,7 +760,6 @@ class GazelleProvider(Provider):
570
760
 
571
761
  # Serve the cached .torrent if present (token, if any, is already
572
762
  # registered above; a cached torrent is fine for the actual data fetch).
573
- cached = self.get_cached_artifact(source_id)
574
763
  if cached:
575
764
  return cached
576
765
 
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: flacfetch
3
- Version: 0.35.1
3
+ Version: 0.36.0
4
4
  Summary: Search and download high-quality audio from multiple sources
5
5
  Home-page: https://github.com/yourusername/flacfetch
6
6
  Author: Your Name
@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
4
4
 
5
5
  [project]
6
6
  name = "flacfetch"
7
- version = "0.35.1"
7
+ version = "0.36.0"
8
8
  description = "Search and download high-quality audio from multiple sources"
9
9
  readme = "README.md"
10
10
  requires-python = ">=3.10,<4.0"
@@ -831,3 +831,252 @@ class TestConcurrentTorrentFetch:
831
831
  p = self._provider(tmp_path, use_fl_token=False)
832
832
  assert p._fetch_lock("1") is p._fetch_lock("1")
833
833
  assert p._fetch_lock("1") is not p._fetch_lock("2")
834
+
835
+
836
+ class TestDownloadBudget:
837
+ """The per-torrent ledger keeps RED's four-download limit out of reach.
838
+
839
+ Incident 2026-10-23 (RED 2319570): RED stalled, three fetches timed out at
840
+ 10s with nothing cached, and they counted server-side anyway; the limit
841
+ was gone before a retry could succeed.
842
+ """
843
+
844
+ TORRENT = b"d8:announce" + b"x" * 2000
845
+
846
+ def _provider(self, tmp_path, use_fl_token=False):
847
+ p = ConcreteGazelleProvider(
848
+ api_key="test",
849
+ base_url="https://test.example",
850
+ cache_subdir="test",
851
+ use_fl_token=use_fl_token,
852
+ )
853
+ p.cache_dir = tmp_path
854
+ return p
855
+
856
+ def _resp(self, status=200, content=None, content_type="application/x-bittorrent", json_body=None):
857
+ resp = MagicMock()
858
+ resp.status_code = status
859
+ resp.headers = {"Content-Type": content_type}
860
+ resp.content = self.TORRENT if content is None else content
861
+ if json_body is not None:
862
+ resp.json.return_value = json_body
863
+ else:
864
+ resp.json.side_effect = ValueError("not json")
865
+ resp.text = ""
866
+ return resp
867
+
868
+ def _read_timeout(self):
869
+ import requests
870
+ return requests.exceptions.ReadTimeout("Read timed out. (read timeout=60)")
871
+
872
+ def test_successful_fetch_recorded_and_cached(self, tmp_path):
873
+ p = self._provider(tmp_path)
874
+ p.session.get = MagicMock(return_value=self._resp())
875
+
876
+ assert p.fetch_artifact_by_id("42") == self.TORRENT
877
+ assert p.downloads_used("42") == 1
878
+ assert (tmp_path / "42.torrent").read_bytes() == self.TORRENT
879
+ assert p.session.get.call_args.kwargs["timeout"] == p.DOWNLOAD_TIMEOUT
880
+
881
+ def test_timeout_counts_and_blocks_retries_during_cooldown(self, tmp_path):
882
+ p = self._provider(tmp_path)
883
+ p.session.get = MagicMock(side_effect=self._read_timeout())
884
+
885
+ assert p.fetch_artifact_by_id("42") is None
886
+ assert p.downloads_used("42") == 1
887
+ # gen's auto-retries 40s later must not touch the tracker.
888
+ assert p.fetch_artifact_by_id("42") is None
889
+ assert p.fetch_artifact_by_id("42") is None
890
+ assert p.session.get.call_count == 1
891
+ assert "did not answer" in p.fetch_errors["42"]
892
+
893
+ def test_retry_allowed_after_cooldown(self, tmp_path):
894
+ p = self._provider(tmp_path)
895
+ p.session.get = MagicMock(side_effect=[self._read_timeout(), self._resp()])
896
+ p.fetch_artifact_by_id("42")
897
+
898
+ ledger = p._load_download_ledger("42")
899
+ ledger["requests"][-1]["t"] -= p.UNANSWERED_COOLDOWN_SECONDS + 1
900
+ p._save_download_ledger("42", ledger)
901
+ assert p.fetch_artifact_by_id("42") == self.TORRENT
902
+ assert p.session.get.call_count == 2
903
+ assert "42" not in p.fetch_errors
904
+
905
+ def test_incident_replay_one_tracker_request(self, tmp_path):
906
+ """Token refused, then RED stalls: only ONE countable request goes out."""
907
+ p = self._provider(tmp_path, use_fl_token=True)
908
+ refused = self._resp(400, content=b"{}", content_type="application/json", json_body={"status": "failure"})
909
+ p.session.get = MagicMock(side_effect=[refused, self._read_timeout()])
910
+
911
+ for _ in range(3): # first attempt + gen's two auto-retries
912
+ assert p.fetch_artifact_by_id("2319570") is None
913
+
914
+ urls = [c.args[0] for c in p.session.get.call_args_list]
915
+ assert len(urls) == 2
916
+ assert "usetoken=1" in urls[0] and "usetoken" not in urls[1]
917
+ assert p.downloads_used("2319570") == 1
918
+
919
+ def test_budget_stops_before_tracker_limit(self, tmp_path):
920
+ p = self._provider(tmp_path)
921
+ p.session.get = MagicMock(return_value=self._resp())
922
+ for _ in range(p.TORRENT_DOWNLOAD_BUDGET):
923
+ assert p.fetch_artifact_by_id("42") == self.TORRENT
924
+ (tmp_path / "42.torrent").unlink() # cache lost each time
925
+
926
+ assert p.fetch_artifact_by_id("42") is None
927
+ assert p.session.get.call_count == p.TORRENT_DOWNLOAD_BUDGET
928
+ assert "budget used up" in p.fetch_errors["42"]
929
+ assert str(tmp_path / "42.torrent") in p.fetch_errors["42"]
930
+
931
+ def test_ledger_survives_restart(self, tmp_path):
932
+ p = self._provider(tmp_path)
933
+ p.session.get = MagicMock(side_effect=self._read_timeout())
934
+ p.fetch_artifact_by_id("42")
935
+
936
+ restarted = self._provider(tmp_path)
937
+ restarted.session.get = MagicMock(side_effect=AssertionError("network"))
938
+ assert restarted.fetch_artifact_by_id("42") is None
939
+ assert restarted.downloads_used("42") == 1
940
+
941
+ def test_tracker_limit_error_blocks_until_torrent_supplied(self, tmp_path):
942
+ p = self._provider(tmp_path)
943
+ body = {"status": "failure", "error": "You have already downloaded this torrent file four times."}
944
+ p.session.get = MagicMock(return_value=self._resp(403, content=b"{}", content_type="application/json", json_body=body))
945
+
946
+ assert p.fetch_artifact_by_id("42") is None
947
+ assert p.fetch_artifact_by_id("42") is None
948
+ assert p.session.get.call_count == 1
949
+ assert self._provider(tmp_path)._download_blocked_reason("42")
950
+
951
+ # A browser-downloaded .torrent dropped into the cache is served.
952
+ (tmp_path / "42.torrent").write_bytes(self.TORRENT)
953
+ assert p.fetch_artifact_by_id("42") == self.TORRENT
954
+ assert p.session.get.call_count == 1
955
+
956
+ def test_tracker_limit_error_with_http_200(self, tmp_path):
957
+ p = self._provider(tmp_path)
958
+ body = {"status": "failure", "error": "You have already downloaded this torrent file four times."}
959
+ p.session.get = MagicMock(return_value=self._resp(200, content=b"{}", content_type="application/json", json_body=body))
960
+
961
+ assert p.fetch_artifact_by_id("42") is None
962
+ assert p.fetch_artifact_by_id("42") is None
963
+ assert p.session.get.call_count == 1
964
+ assert "budget used up" in p.fetch_errors["42"]
965
+
966
+ def test_json_refusal_reason_surfaced(self, tmp_path):
967
+ p = self._provider(tmp_path)
968
+ body = {"status": "failure", "error": "bad id"}
969
+ p.session.get = MagicMock(return_value=self._resp(200, content=b"{}", content_type="application/json", json_body=body))
970
+ assert p.fetch_artifact_by_id("42") is None
971
+ assert "bad id" in p.fetch_errors["42"]
972
+
973
+ def test_explicit_refusals_not_counted(self, tmp_path):
974
+ p = self._provider(tmp_path)
975
+ p.session.get = MagicMock(return_value=self._resp(400, content=b"{}", content_type="application/json", json_body={}))
976
+ for _ in range(5):
977
+ p.fetch_artifact_by_id("42")
978
+ assert p.downloads_used("42") == 0
979
+ assert p.session.get.call_count == 5
980
+
981
+ def test_server_error_counts_with_cooldown(self, tmp_path):
982
+ p = self._provider(tmp_path)
983
+ p.session.get = MagicMock(return_value=self._resp(502, content=b"bad gateway"))
984
+ assert p.fetch_artifact_by_id("42") is None
985
+ assert p.fetch_artifact_by_id("42") is None
986
+ assert p.session.get.call_count == 1
987
+ assert p.downloads_used("42") == 1
988
+
989
+ def test_connect_timeout_not_counted(self, tmp_path):
990
+ import requests
991
+ p = self._provider(tmp_path)
992
+ p.session.get = MagicMock(side_effect=[requests.exceptions.ConnectTimeout("no route"), self._resp()])
993
+ assert p.fetch_artifact_by_id("42") is None
994
+ assert p.fetch_artifact_by_id("42") == self.TORRENT
995
+ assert p.downloads_used("42") == 1
996
+
997
+ @pytest.mark.parametrize("make_error", [
998
+ lambda: __import__("requests").exceptions.SSLError("handshake failed"),
999
+ lambda: __import__("requests").exceptions.ConnectionError(
1000
+ __import__("urllib3").exceptions.MaxRetryError(
1001
+ None, "/", __import__("urllib3").exceptions.NewConnectionError(None, "refused"))),
1002
+ ])
1003
+ def test_errors_before_sending_not_counted(self, tmp_path, make_error):
1004
+ p = self._provider(tmp_path)
1005
+ p.session.get = MagicMock(side_effect=[make_error(), self._resp()])
1006
+ assert p.fetch_artifact_by_id("42") is None
1007
+ assert "Could not connect" in p.fetch_errors["42"]
1008
+ assert p.fetch_artifact_by_id("42") == self.TORRENT # no cooldown
1009
+ assert p.downloads_used("42") == 1
1010
+
1011
+ def test_dropped_connection_after_send_counted(self, tmp_path):
1012
+ import requests
1013
+ p = self._provider(tmp_path)
1014
+ p.session.get = MagicMock(side_effect=requests.exceptions.ConnectionError("Connection aborted."))
1015
+ assert p.fetch_artifact_by_id("42") is None
1016
+ assert p.downloads_used("42") == 1
1017
+
1018
+ @pytest.mark.parametrize("content", ["{}", "[]", '{"requests": [1]}', '{"limit_reached": false}'])
1019
+ def test_wrong_shaped_ledger_blocks(self, tmp_path, content):
1020
+ (tmp_path / "42.downloads.json").write_text(content)
1021
+ p = self._provider(tmp_path)
1022
+ p.session.get = MagicMock(side_effect=AssertionError("network"))
1023
+ assert p.fetch_artifact_by_id("42") is None
1024
+ assert "budget used up" in p.fetch_errors["42"]
1025
+
1026
+ def test_failed_ledger_write_kept_in_memory(self, tmp_path, monkeypatch):
1027
+ from pathlib import Path
1028
+ p = self._provider(tmp_path)
1029
+ p.session.get = MagicMock(side_effect=self._read_timeout())
1030
+ monkeypatch.setattr(Path, "write_text", MagicMock(side_effect=OSError("disk full")))
1031
+
1032
+ p.fetch_artifact_by_id("42")
1033
+ assert not (tmp_path / "42.downloads.json").exists()
1034
+ assert p.fetch_artifact_by_id("42") is None
1035
+ assert p.session.get.call_count == 1
1036
+ assert p.downloads_used("42") == 1
1037
+
1038
+ def test_unreadable_ledger_blocks(self, tmp_path):
1039
+ (tmp_path / "42.downloads.json").write_text("{not json")
1040
+ p = self._provider(tmp_path)
1041
+ p.session.get = MagicMock(side_effect=AssertionError("network"))
1042
+ assert p.fetch_artifact_by_id("42") is None
1043
+
1044
+ def test_token_skipped_on_cached_torrent_when_budget_tight(self, tmp_path):
1045
+ p = self._provider(tmp_path, use_fl_token=True)
1046
+ (tmp_path / "42.torrent").write_bytes(self.TORRENT)
1047
+ for _ in range(p.TORRENT_DOWNLOAD_BUDGET - 1):
1048
+ p._record_download_request("42", "normal", "ok")
1049
+ p.session.get = MagicMock(side_effect=AssertionError("network"))
1050
+
1051
+ assert p.fetch_artifact_by_id("42") == self.TORRENT
1052
+
1053
+ def test_token_still_spent_on_cached_torrent_with_room(self, tmp_path):
1054
+ p = self._provider(tmp_path, use_fl_token=True)
1055
+ (tmp_path / "42.torrent").write_bytes(self.TORRENT)
1056
+ p.session.get = MagicMock(return_value=self._resp())
1057
+
1058
+ assert p.fetch_artifact_by_id("42") == self.TORRENT
1059
+ assert "usetoken=1" in p.session.get.call_args.args[0]
1060
+ assert p.downloads_used("42") == 1
1061
+
1062
+ def test_ledger_without_cache_dir_kept_in_memory(self, tmp_path):
1063
+ p = self._provider(tmp_path)
1064
+ p.cache_dir = None
1065
+ p.session.get = MagicMock(side_effect=self._read_timeout())
1066
+ p.fetch_artifact_by_id("42")
1067
+ assert p.fetch_artifact_by_id("42") is None
1068
+ assert p.session.get.call_count == 1
1069
+
1070
+ def test_deleting_ledger_file_resets_without_restart(self, tmp_path):
1071
+ p = self._provider(tmp_path)
1072
+ p.session.get = MagicMock(side_effect=[self._read_timeout(), self._resp()])
1073
+ p.fetch_artifact_by_id("42")
1074
+ (tmp_path / "42.downloads.json").unlink()
1075
+ assert p.fetch_artifact_by_id("42") == self.TORRENT
1076
+
1077
+ def test_token_spend_counts_against_budget(self, tmp_path):
1078
+ p = self._provider(tmp_path, use_fl_token=True)
1079
+ p.session.get = MagicMock(return_value=self._resp())
1080
+ p.fetch_artifact_by_id("42")
1081
+ ledger = p._load_download_ledger("42")
1082
+ assert [r["kind"] for r in ledger["requests"]] == ["token"]
@@ -589,6 +589,31 @@ class TestDownloadById:
589
589
  assert release.source_name == "RED"
590
590
  assert release.target_file == "01 - Track.flac"
591
591
 
592
+ def test_download_by_id_red_failure_includes_provider_reason(self):
593
+ """The provider's reason (e.g. budget used up) reaches the job error."""
594
+ mgr = FetchManager()
595
+ mock_dl = Mock(spec=Downloader)
596
+
597
+ class MockREDProvider(Provider):
598
+ fetch_errors = {"12345": "RED .torrent download budget used up for torrent 12345"}
599
+
600
+ @property
601
+ def name(self):
602
+ return "RED"
603
+
604
+ def search(self, q):
605
+ return []
606
+
607
+ def fetch_artifact_by_id(self, torrent_id):
608
+ return None
609
+
610
+ mgr.add_provider(MockREDProvider())
611
+ mgr.register_downloader("RED", mock_dl)
612
+
613
+ with pytest.raises(ValueError, match="Failed to fetch torrent file for RED ID: 12345: RED .torrent download budget used up"):
614
+ mgr.download_by_id(source_name="RED", source_id="12345", output_path="/output")
615
+ mock_dl.download.assert_not_called()
616
+
592
617
  def test_download_by_id_ops_torrent(self):
593
618
  """Test download_by_id for OPS creates Release with correct quality"""
594
619
  mgr = FetchManager()
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes